diff --git a/.gitattributes b/.gitattributes index 5df39ea16d3119275c097c86e393e7dcc7bd9713..d591d9a618456e08a288fbdf1487cc9761e40982 100644 --- a/.gitattributes +++ b/.gitattributes @@ -1049,3 +1049,1031 @@ EmptyFix_InstructBlip_just_segment_pretrain_from_stage1_multiobjects_many_object EmptyFix_InstructBlip_just_segment_pretrain_from_stage1_multiobjects_many_objects_segmentation_bbox_filtered_objects/005-GPT-XL/checkpoints/eval_step_90000/batch_2_cfg_7.5_topk_16384.jpg filter=lfs diff=lfs merge=lfs -text EmptyFix_InstructBlip_just_segment_pretrain_from_stage1_multiobjects_many_objects_segmentation_bbox_filtered_objects/wandb/run-20250423_172209-0k7f5yzn/run-0k7f5yzn.wandb filter=lfs diff=lfs merge=lfs -text EmptyFix_InstructBlip_just_segment_pretrain_from_stage1_multiobjects_many_objects_segmentation_bbox_filtered_objects/wandb/run-20250424_143123-yx27lyn0/run-yx27lyn0.wandb filter=lfs diff=lfs merge=lfs -text +CKPTS/Afteri2i_100kckpt_LLAVAT5_empty_Fixed_just_segment_pretrain_stage3_dreambench_recap_400k_t2i_400k_flux_200k_midjourney_150k_recovery_150k_3e4_epoch2_half_rotated/004-GPT-XL/checkpoints/eval_step_2000/batch_0_cfg_7.5_topk_16384.jpg filter=lfs diff=lfs merge=lfs -text +CKPTS/Afteri2i_100kckpt_LLAVAT5_empty_Fixed_just_segment_pretrain_stage3_dreambench_recap_400k_t2i_400k_flux_200k_midjourney_150k_recovery_150k_3e4_epoch2_half_rotated/004-GPT-XL/checkpoints/eval_step_2000/batch_1_cfg_7.5_topk_16384.jpg filter=lfs diff=lfs merge=lfs -text +CKPTS/Afteri2i_100kckpt_LLAVAT5_empty_Fixed_just_segment_pretrain_stage3_dreambench_recap_400k_t2i_400k_flux_200k_midjourney_150k_recovery_150k_3e4_epoch2_half_rotated/004-GPT-XL/checkpoints/eval_step_2000/batch_2_cfg_7.5_topk_16384.jpg filter=lfs diff=lfs merge=lfs -text +CKPTS/Afteri2i_100kckpt_LLAVAT5_empty_Fixed_just_segment_pretrain_stage3_dreambench_recap_400k_t2i_400k_flux_200k_midjourney_150k_recovery_150k_3e4_epoch2_half_rotated/004-GPT-XL/checkpoints/eval_step_2000/batch_3_cfg_7.5_topk_16384.jpg filter=lfs diff=lfs merge=lfs -text +CKPTS/Afteri2i_100kckpt_LLAVAT5_empty_Fixed_just_segment_pretrain_stage3_dreambench_recap_400k_t2i_400k_flux_200k_midjourney_150k_recovery_150k_3e4_epoch2_half_rotated/004-GPT-XL/checkpoints/eval_step_2000/batch_4_cfg_7.5_topk_16384.jpg filter=lfs diff=lfs merge=lfs -text +CKPTS/Afteri2i_100kckpt_LLAVAT5_empty_Fixed_just_segment_pretrain_stage3_dreambench_recap_400k_t2i_400k_flux_200k_midjourney_150k_recovery_150k_3e4_epoch2_half_rotated/004-GPT-XL/checkpoints/eval_step_2000/batch_5_cfg_7.5_topk_16384.jpg filter=lfs diff=lfs merge=lfs -text +CKPTS/Afteri2i_100kckpt_LLAVAT5_empty_Fixed_just_segment_pretrain_stage3_dreambench_recap_400k_t2i_400k_flux_200k_midjourney_150k_recovery_150k_3e4_epoch2_half_rotated/004-GPT-XL/checkpoints/eval_step_2000/batch_6_cfg_7.5_topk_16384.jpg filter=lfs diff=lfs merge=lfs -text +CKPTS/Afteri2i_100kckpt_LLAVAT5_empty_Fixed_just_segment_pretrain_stage3_dreambench_recap_400k_t2i_400k_flux_200k_midjourney_150k_recovery_150k_3e4_epoch2_half_rotated/004-GPT-XL/checkpoints/eval_step_2000/batch_7_cfg_7.5_topk_16384.jpg filter=lfs diff=lfs merge=lfs -text +CKPTS/Afteri2i_100kckpt_LLAVAT5_empty_Fixed_just_segment_pretrain_stage3_dreambench_recap_400k_t2i_400k_flux_200k_midjourney_150k_recovery_150k_3e4_epoch2_half_rotated/004-GPT-XL/checkpoints/eval_step_2000/batch_8_cfg_7.5_topk_16384.jpg filter=lfs diff=lfs merge=lfs -text +CKPTS/Afteri2i_100kckpt_LLAVAT5_empty_Fixed_just_segment_pretrain_stage3_dreambench_recap_400k_t2i_400k_flux_200k_midjourney_150k_recovery_150k_3e4_epoch2_half_rotated/005-GPT-XL/checkpoints/eval_step_10000/batch_0_cfg_7.5_topk_16384.jpg filter=lfs diff=lfs merge=lfs -text +CKPTS/Afteri2i_100kckpt_LLAVAT5_empty_Fixed_just_segment_pretrain_stage3_dreambench_recap_400k_t2i_400k_flux_200k_midjourney_150k_recovery_150k_3e4_epoch2_half_rotated/005-GPT-XL/checkpoints/eval_step_10000/batch_1_cfg_7.5_topk_16384.jpg filter=lfs diff=lfs merge=lfs -text +CKPTS/Afteri2i_100kckpt_LLAVAT5_empty_Fixed_just_segment_pretrain_stage3_dreambench_recap_400k_t2i_400k_flux_200k_midjourney_150k_recovery_150k_3e4_epoch2_half_rotated/005-GPT-XL/checkpoints/eval_step_10000/batch_2_cfg_7.5_topk_16384.jpg filter=lfs diff=lfs merge=lfs -text +CKPTS/Afteri2i_100kckpt_LLAVAT5_empty_Fixed_just_segment_pretrain_stage3_dreambench_recap_400k_t2i_400k_flux_200k_midjourney_150k_recovery_150k_3e4_epoch2_half_rotated/005-GPT-XL/checkpoints/eval_step_10000/batch_3_cfg_7.5_topk_16384.jpg filter=lfs diff=lfs merge=lfs -text +CKPTS/Afteri2i_100kckpt_LLAVAT5_empty_Fixed_just_segment_pretrain_stage3_dreambench_recap_400k_t2i_400k_flux_200k_midjourney_150k_recovery_150k_3e4_epoch2_half_rotated/005-GPT-XL/checkpoints/eval_step_12000/batch_0_cfg_7.5_topk_16384.jpg filter=lfs diff=lfs merge=lfs -text +CKPTS/Afteri2i_100kckpt_LLAVAT5_empty_Fixed_just_segment_pretrain_stage3_dreambench_recap_400k_t2i_400k_flux_200k_midjourney_150k_recovery_150k_3e4_epoch2_half_rotated/005-GPT-XL/checkpoints/eval_step_12000/batch_1_cfg_7.5_topk_16384.jpg filter=lfs diff=lfs merge=lfs -text +CKPTS/Afteri2i_100kckpt_LLAVAT5_empty_Fixed_just_segment_pretrain_stage3_dreambench_recap_400k_t2i_400k_flux_200k_midjourney_150k_recovery_150k_3e4_epoch2_half_rotated/005-GPT-XL/checkpoints/eval_step_12000/batch_2_cfg_7.5_topk_16384.jpg filter=lfs diff=lfs merge=lfs -text +CKPTS/Afteri2i_100kckpt_LLAVAT5_empty_Fixed_just_segment_pretrain_stage3_dreambench_recap_400k_t2i_400k_flux_200k_midjourney_150k_recovery_150k_3e4_epoch2_half_rotated/005-GPT-XL/checkpoints/eval_step_12000/batch_3_cfg_7.5_topk_16384.jpg filter=lfs diff=lfs merge=lfs -text +CKPTS/Afteri2i_100kckpt_LLAVAT5_empty_Fixed_just_segment_pretrain_stage3_dreambench_recap_400k_t2i_400k_flux_200k_midjourney_150k_recovery_150k_3e4_epoch2_half_rotated/005-GPT-XL/checkpoints/eval_step_14000/batch_0_cfg_7.5_topk_16384.jpg filter=lfs diff=lfs merge=lfs -text +CKPTS/Afteri2i_100kckpt_LLAVAT5_empty_Fixed_just_segment_pretrain_stage3_dreambench_recap_400k_t2i_400k_flux_200k_midjourney_150k_recovery_150k_3e4_epoch2_half_rotated/005-GPT-XL/checkpoints/eval_step_14000/batch_1_cfg_7.5_topk_16384.jpg filter=lfs diff=lfs merge=lfs -text +CKPTS/Afteri2i_100kckpt_LLAVAT5_empty_Fixed_just_segment_pretrain_stage3_dreambench_recap_400k_t2i_400k_flux_200k_midjourney_150k_recovery_150k_3e4_epoch2_half_rotated/005-GPT-XL/checkpoints/eval_step_14000/batch_2_cfg_7.5_topk_16384.jpg filter=lfs diff=lfs merge=lfs -text +CKPTS/Afteri2i_100kckpt_LLAVAT5_empty_Fixed_just_segment_pretrain_stage3_dreambench_recap_400k_t2i_400k_flux_200k_midjourney_150k_recovery_150k_3e4_epoch2_half_rotated/005-GPT-XL/checkpoints/eval_step_14000/batch_3_cfg_7.5_topk_16384.jpg filter=lfs diff=lfs merge=lfs -text +CKPTS/Afteri2i_100kckpt_LLAVAT5_empty_Fixed_just_segment_pretrain_stage3_dreambench_recap_400k_t2i_400k_flux_200k_midjourney_150k_recovery_150k_3e4_epoch2_half_rotated/005-GPT-XL/checkpoints/eval_step_16000/batch_0_cfg_7.5_topk_16384.jpg filter=lfs diff=lfs merge=lfs -text +CKPTS/Afteri2i_100kckpt_LLAVAT5_empty_Fixed_just_segment_pretrain_stage3_dreambench_recap_400k_t2i_400k_flux_200k_midjourney_150k_recovery_150k_3e4_epoch2_half_rotated/005-GPT-XL/checkpoints/eval_step_16000/batch_1_cfg_7.5_topk_16384.jpg filter=lfs diff=lfs merge=lfs -text +CKPTS/Afteri2i_100kckpt_LLAVAT5_empty_Fixed_just_segment_pretrain_stage3_dreambench_recap_400k_t2i_400k_flux_200k_midjourney_150k_recovery_150k_3e4_epoch2_half_rotated/005-GPT-XL/checkpoints/eval_step_16000/batch_2_cfg_7.5_topk_16384.jpg filter=lfs diff=lfs merge=lfs -text +CKPTS/Afteri2i_100kckpt_LLAVAT5_empty_Fixed_just_segment_pretrain_stage3_dreambench_recap_400k_t2i_400k_flux_200k_midjourney_150k_recovery_150k_3e4_epoch2_half_rotated/005-GPT-XL/checkpoints/eval_step_16000/batch_3_cfg_7.5_topk_16384.jpg filter=lfs diff=lfs merge=lfs -text +CKPTS/Afteri2i_100kckpt_LLAVAT5_empty_Fixed_just_segment_pretrain_stage3_dreambench_recap_400k_t2i_400k_flux_200k_midjourney_150k_recovery_150k_3e4_epoch2_half_rotated/005-GPT-XL/checkpoints/eval_step_18000/batch_0_cfg_7.5_topk_16384.jpg filter=lfs diff=lfs merge=lfs -text +CKPTS/Afteri2i_100kckpt_LLAVAT5_empty_Fixed_just_segment_pretrain_stage3_dreambench_recap_400k_t2i_400k_flux_200k_midjourney_150k_recovery_150k_3e4_epoch2_half_rotated/005-GPT-XL/checkpoints/eval_step_18000/batch_1_cfg_7.5_topk_16384.jpg filter=lfs diff=lfs merge=lfs -text +CKPTS/Afteri2i_100kckpt_LLAVAT5_empty_Fixed_just_segment_pretrain_stage3_dreambench_recap_400k_t2i_400k_flux_200k_midjourney_150k_recovery_150k_3e4_epoch2_half_rotated/005-GPT-XL/checkpoints/eval_step_18000/batch_2_cfg_7.5_topk_16384.jpg filter=lfs diff=lfs merge=lfs -text +CKPTS/Afteri2i_100kckpt_LLAVAT5_empty_Fixed_just_segment_pretrain_stage3_dreambench_recap_400k_t2i_400k_flux_200k_midjourney_150k_recovery_150k_3e4_epoch2_half_rotated/005-GPT-XL/checkpoints/eval_step_18000/batch_3_cfg_7.5_topk_16384.jpg filter=lfs diff=lfs merge=lfs -text +CKPTS/Afteri2i_100kckpt_LLAVAT5_empty_Fixed_just_segment_pretrain_stage3_dreambench_recap_400k_t2i_400k_flux_200k_midjourney_150k_recovery_150k_3e4_epoch2_half_rotated/005-GPT-XL/checkpoints/eval_step_2000/batch_0_cfg_7.5_topk_16384.jpg filter=lfs diff=lfs merge=lfs -text +CKPTS/Afteri2i_100kckpt_LLAVAT5_empty_Fixed_just_segment_pretrain_stage3_dreambench_recap_400k_t2i_400k_flux_200k_midjourney_150k_recovery_150k_3e4_epoch2_half_rotated/005-GPT-XL/checkpoints/eval_step_2000/batch_1_cfg_7.5_topk_16384.jpg filter=lfs diff=lfs merge=lfs -text +CKPTS/Afteri2i_100kckpt_LLAVAT5_empty_Fixed_just_segment_pretrain_stage3_dreambench_recap_400k_t2i_400k_flux_200k_midjourney_150k_recovery_150k_3e4_epoch2_half_rotated/005-GPT-XL/checkpoints/eval_step_2000/batch_2_cfg_7.5_topk_16384.jpg filter=lfs diff=lfs merge=lfs -text +CKPTS/Afteri2i_100kckpt_LLAVAT5_empty_Fixed_just_segment_pretrain_stage3_dreambench_recap_400k_t2i_400k_flux_200k_midjourney_150k_recovery_150k_3e4_epoch2_half_rotated/005-GPT-XL/checkpoints/eval_step_2000/batch_3_cfg_7.5_topk_16384.jpg filter=lfs diff=lfs merge=lfs -text +CKPTS/Afteri2i_100kckpt_LLAVAT5_empty_Fixed_just_segment_pretrain_stage3_dreambench_recap_400k_t2i_400k_flux_200k_midjourney_150k_recovery_150k_3e4_epoch2_half_rotated/005-GPT-XL/checkpoints/eval_step_20000/batch_0_cfg_7.5_topk_16384.jpg filter=lfs diff=lfs merge=lfs -text +CKPTS/Afteri2i_100kckpt_LLAVAT5_empty_Fixed_just_segment_pretrain_stage3_dreambench_recap_400k_t2i_400k_flux_200k_midjourney_150k_recovery_150k_3e4_epoch2_half_rotated/005-GPT-XL/checkpoints/eval_step_20000/batch_1_cfg_7.5_topk_16384.jpg filter=lfs diff=lfs merge=lfs -text +CKPTS/Afteri2i_100kckpt_LLAVAT5_empty_Fixed_just_segment_pretrain_stage3_dreambench_recap_400k_t2i_400k_flux_200k_midjourney_150k_recovery_150k_3e4_epoch2_half_rotated/005-GPT-XL/checkpoints/eval_step_20000/batch_2_cfg_7.5_topk_16384.jpg filter=lfs diff=lfs merge=lfs -text +CKPTS/Afteri2i_100kckpt_LLAVAT5_empty_Fixed_just_segment_pretrain_stage3_dreambench_recap_400k_t2i_400k_flux_200k_midjourney_150k_recovery_150k_3e4_epoch2_half_rotated/005-GPT-XL/checkpoints/eval_step_20000/batch_3_cfg_7.5_topk_16384.jpg filter=lfs diff=lfs merge=lfs -text +CKPTS/Afteri2i_100kckpt_LLAVAT5_empty_Fixed_just_segment_pretrain_stage3_dreambench_recap_400k_t2i_400k_flux_200k_midjourney_150k_recovery_150k_3e4_epoch2_half_rotated/005-GPT-XL/checkpoints/eval_step_22000/batch_0_cfg_7.5_topk_16384.jpg filter=lfs diff=lfs merge=lfs -text +CKPTS/Afteri2i_100kckpt_LLAVAT5_empty_Fixed_just_segment_pretrain_stage3_dreambench_recap_400k_t2i_400k_flux_200k_midjourney_150k_recovery_150k_3e4_epoch2_half_rotated/005-GPT-XL/checkpoints/eval_step_22000/batch_1_cfg_7.5_topk_16384.jpg filter=lfs diff=lfs merge=lfs -text +CKPTS/Afteri2i_100kckpt_LLAVAT5_empty_Fixed_just_segment_pretrain_stage3_dreambench_recap_400k_t2i_400k_flux_200k_midjourney_150k_recovery_150k_3e4_epoch2_half_rotated/005-GPT-XL/checkpoints/eval_step_22000/batch_2_cfg_7.5_topk_16384.jpg filter=lfs diff=lfs merge=lfs -text +CKPTS/Afteri2i_100kckpt_LLAVAT5_empty_Fixed_just_segment_pretrain_stage3_dreambench_recap_400k_t2i_400k_flux_200k_midjourney_150k_recovery_150k_3e4_epoch2_half_rotated/005-GPT-XL/checkpoints/eval_step_22000/batch_3_cfg_7.5_topk_16384.jpg filter=lfs diff=lfs merge=lfs -text +CKPTS/Afteri2i_100kckpt_LLAVAT5_empty_Fixed_just_segment_pretrain_stage3_dreambench_recap_400k_t2i_400k_flux_200k_midjourney_150k_recovery_150k_3e4_epoch2_half_rotated/005-GPT-XL/checkpoints/eval_step_24000/batch_0_cfg_7.5_topk_16384.jpg filter=lfs diff=lfs merge=lfs -text +CKPTS/Afteri2i_100kckpt_LLAVAT5_empty_Fixed_just_segment_pretrain_stage3_dreambench_recap_400k_t2i_400k_flux_200k_midjourney_150k_recovery_150k_3e4_epoch2_half_rotated/005-GPT-XL/checkpoints/eval_step_24000/batch_1_cfg_7.5_topk_16384.jpg filter=lfs diff=lfs merge=lfs -text +CKPTS/Afteri2i_100kckpt_LLAVAT5_empty_Fixed_just_segment_pretrain_stage3_dreambench_recap_400k_t2i_400k_flux_200k_midjourney_150k_recovery_150k_3e4_epoch2_half_rotated/005-GPT-XL/checkpoints/eval_step_24000/batch_2_cfg_7.5_topk_16384.jpg filter=lfs diff=lfs merge=lfs -text +CKPTS/Afteri2i_100kckpt_LLAVAT5_empty_Fixed_just_segment_pretrain_stage3_dreambench_recap_400k_t2i_400k_flux_200k_midjourney_150k_recovery_150k_3e4_epoch2_half_rotated/005-GPT-XL/checkpoints/eval_step_24000/batch_3_cfg_7.5_topk_16384.jpg filter=lfs diff=lfs merge=lfs -text +CKPTS/Afteri2i_100kckpt_LLAVAT5_empty_Fixed_just_segment_pretrain_stage3_dreambench_recap_400k_t2i_400k_flux_200k_midjourney_150k_recovery_150k_3e4_epoch2_half_rotated/005-GPT-XL/checkpoints/eval_step_26000/batch_0_cfg_7.5_topk_16384.jpg filter=lfs diff=lfs merge=lfs -text +CKPTS/Afteri2i_100kckpt_LLAVAT5_empty_Fixed_just_segment_pretrain_stage3_dreambench_recap_400k_t2i_400k_flux_200k_midjourney_150k_recovery_150k_3e4_epoch2_half_rotated/005-GPT-XL/checkpoints/eval_step_26000/batch_1_cfg_7.5_topk_16384.jpg filter=lfs diff=lfs merge=lfs -text +CKPTS/Afteri2i_100kckpt_LLAVAT5_empty_Fixed_just_segment_pretrain_stage3_dreambench_recap_400k_t2i_400k_flux_200k_midjourney_150k_recovery_150k_3e4_epoch2_half_rotated/005-GPT-XL/checkpoints/eval_step_26000/batch_2_cfg_7.5_topk_16384.jpg filter=lfs diff=lfs merge=lfs -text +CKPTS/Afteri2i_100kckpt_LLAVAT5_empty_Fixed_just_segment_pretrain_stage3_dreambench_recap_400k_t2i_400k_flux_200k_midjourney_150k_recovery_150k_3e4_epoch2_half_rotated/005-GPT-XL/checkpoints/eval_step_26000/batch_3_cfg_7.5_topk_16384.jpg filter=lfs diff=lfs merge=lfs -text +CKPTS/Afteri2i_100kckpt_LLAVAT5_empty_Fixed_just_segment_pretrain_stage3_dreambench_recap_400k_t2i_400k_flux_200k_midjourney_150k_recovery_150k_3e4_epoch2_half_rotated/005-GPT-XL/checkpoints/eval_step_28000/batch_0_cfg_7.5_topk_16384.jpg filter=lfs diff=lfs merge=lfs -text +CKPTS/Afteri2i_100kckpt_LLAVAT5_empty_Fixed_just_segment_pretrain_stage3_dreambench_recap_400k_t2i_400k_flux_200k_midjourney_150k_recovery_150k_3e4_epoch2_half_rotated/005-GPT-XL/checkpoints/eval_step_28000/batch_1_cfg_7.5_topk_16384.jpg filter=lfs diff=lfs merge=lfs -text +CKPTS/Afteri2i_100kckpt_LLAVAT5_empty_Fixed_just_segment_pretrain_stage3_dreambench_recap_400k_t2i_400k_flux_200k_midjourney_150k_recovery_150k_3e4_epoch2_half_rotated/005-GPT-XL/checkpoints/eval_step_28000/batch_2_cfg_7.5_topk_16384.jpg filter=lfs diff=lfs merge=lfs -text +CKPTS/Afteri2i_100kckpt_LLAVAT5_empty_Fixed_just_segment_pretrain_stage3_dreambench_recap_400k_t2i_400k_flux_200k_midjourney_150k_recovery_150k_3e4_epoch2_half_rotated/005-GPT-XL/checkpoints/eval_step_28000/batch_3_cfg_7.5_topk_16384.jpg filter=lfs diff=lfs merge=lfs -text +CKPTS/Afteri2i_100kckpt_LLAVAT5_empty_Fixed_just_segment_pretrain_stage3_dreambench_recap_400k_t2i_400k_flux_200k_midjourney_150k_recovery_150k_3e4_epoch2_half_rotated/005-GPT-XL/checkpoints/eval_step_30000/batch_0_cfg_7.5_topk_16384.jpg filter=lfs diff=lfs merge=lfs -text +CKPTS/Afteri2i_100kckpt_LLAVAT5_empty_Fixed_just_segment_pretrain_stage3_dreambench_recap_400k_t2i_400k_flux_200k_midjourney_150k_recovery_150k_3e4_epoch2_half_rotated/005-GPT-XL/checkpoints/eval_step_30000/batch_1_cfg_7.5_topk_16384.jpg filter=lfs diff=lfs merge=lfs -text +CKPTS/Afteri2i_100kckpt_LLAVAT5_empty_Fixed_just_segment_pretrain_stage3_dreambench_recap_400k_t2i_400k_flux_200k_midjourney_150k_recovery_150k_3e4_epoch2_half_rotated/005-GPT-XL/checkpoints/eval_step_30000/batch_2_cfg_7.5_topk_16384.jpg filter=lfs diff=lfs merge=lfs -text +CKPTS/Afteri2i_100kckpt_LLAVAT5_empty_Fixed_just_segment_pretrain_stage3_dreambench_recap_400k_t2i_400k_flux_200k_midjourney_150k_recovery_150k_3e4_epoch2_half_rotated/005-GPT-XL/checkpoints/eval_step_30000/batch_3_cfg_7.5_topk_16384.jpg filter=lfs diff=lfs merge=lfs -text +CKPTS/Afteri2i_100kckpt_LLAVAT5_empty_Fixed_just_segment_pretrain_stage3_dreambench_recap_400k_t2i_400k_flux_200k_midjourney_150k_recovery_150k_3e4_epoch2_half_rotated/005-GPT-XL/checkpoints/eval_step_32000/batch_0_cfg_7.5_topk_16384.jpg filter=lfs diff=lfs merge=lfs -text +CKPTS/Afteri2i_100kckpt_LLAVAT5_empty_Fixed_just_segment_pretrain_stage3_dreambench_recap_400k_t2i_400k_flux_200k_midjourney_150k_recovery_150k_3e4_epoch2_half_rotated/005-GPT-XL/checkpoints/eval_step_32000/batch_1_cfg_7.5_topk_16384.jpg filter=lfs diff=lfs merge=lfs -text +CKPTS/Afteri2i_100kckpt_LLAVAT5_empty_Fixed_just_segment_pretrain_stage3_dreambench_recap_400k_t2i_400k_flux_200k_midjourney_150k_recovery_150k_3e4_epoch2_half_rotated/005-GPT-XL/checkpoints/eval_step_32000/batch_2_cfg_7.5_topk_16384.jpg filter=lfs diff=lfs merge=lfs -text +CKPTS/Afteri2i_100kckpt_LLAVAT5_empty_Fixed_just_segment_pretrain_stage3_dreambench_recap_400k_t2i_400k_flux_200k_midjourney_150k_recovery_150k_3e4_epoch2_half_rotated/005-GPT-XL/checkpoints/eval_step_32000/batch_3_cfg_7.5_topk_16384.jpg filter=lfs diff=lfs merge=lfs -text +CKPTS/Afteri2i_100kckpt_LLAVAT5_empty_Fixed_just_segment_pretrain_stage3_dreambench_recap_400k_t2i_400k_flux_200k_midjourney_150k_recovery_150k_3e4_epoch2_half_rotated/005-GPT-XL/checkpoints/eval_step_34000/batch_0_cfg_7.5_topk_16384.jpg filter=lfs diff=lfs merge=lfs -text +CKPTS/Afteri2i_100kckpt_LLAVAT5_empty_Fixed_just_segment_pretrain_stage3_dreambench_recap_400k_t2i_400k_flux_200k_midjourney_150k_recovery_150k_3e4_epoch2_half_rotated/005-GPT-XL/checkpoints/eval_step_34000/batch_1_cfg_7.5_topk_16384.jpg filter=lfs diff=lfs merge=lfs -text +CKPTS/Afteri2i_100kckpt_LLAVAT5_empty_Fixed_just_segment_pretrain_stage3_dreambench_recap_400k_t2i_400k_flux_200k_midjourney_150k_recovery_150k_3e4_epoch2_half_rotated/005-GPT-XL/checkpoints/eval_step_34000/batch_2_cfg_7.5_topk_16384.jpg filter=lfs diff=lfs merge=lfs -text +CKPTS/Afteri2i_100kckpt_LLAVAT5_empty_Fixed_just_segment_pretrain_stage3_dreambench_recap_400k_t2i_400k_flux_200k_midjourney_150k_recovery_150k_3e4_epoch2_half_rotated/005-GPT-XL/checkpoints/eval_step_34000/batch_3_cfg_7.5_topk_16384.jpg filter=lfs diff=lfs merge=lfs -text +CKPTS/Afteri2i_100kckpt_LLAVAT5_empty_Fixed_just_segment_pretrain_stage3_dreambench_recap_400k_t2i_400k_flux_200k_midjourney_150k_recovery_150k_3e4_epoch2_half_rotated/005-GPT-XL/checkpoints/eval_step_36000/batch_0_cfg_7.5_topk_16384.jpg filter=lfs diff=lfs merge=lfs -text +CKPTS/Afteri2i_100kckpt_LLAVAT5_empty_Fixed_just_segment_pretrain_stage3_dreambench_recap_400k_t2i_400k_flux_200k_midjourney_150k_recovery_150k_3e4_epoch2_half_rotated/005-GPT-XL/checkpoints/eval_step_36000/batch_1_cfg_7.5_topk_16384.jpg filter=lfs diff=lfs merge=lfs -text +CKPTS/Afteri2i_100kckpt_LLAVAT5_empty_Fixed_just_segment_pretrain_stage3_dreambench_recap_400k_t2i_400k_flux_200k_midjourney_150k_recovery_150k_3e4_epoch2_half_rotated/005-GPT-XL/checkpoints/eval_step_36000/batch_2_cfg_7.5_topk_16384.jpg filter=lfs diff=lfs merge=lfs -text +CKPTS/Afteri2i_100kckpt_LLAVAT5_empty_Fixed_just_segment_pretrain_stage3_dreambench_recap_400k_t2i_400k_flux_200k_midjourney_150k_recovery_150k_3e4_epoch2_half_rotated/005-GPT-XL/checkpoints/eval_step_36000/batch_3_cfg_7.5_topk_16384.jpg filter=lfs diff=lfs merge=lfs -text +CKPTS/Afteri2i_100kckpt_LLAVAT5_empty_Fixed_just_segment_pretrain_stage3_dreambench_recap_400k_t2i_400k_flux_200k_midjourney_150k_recovery_150k_3e4_epoch2_half_rotated/005-GPT-XL/checkpoints/eval_step_38000/batch_0_cfg_7.5_topk_16384.jpg filter=lfs diff=lfs merge=lfs -text +CKPTS/Afteri2i_100kckpt_LLAVAT5_empty_Fixed_just_segment_pretrain_stage3_dreambench_recap_400k_t2i_400k_flux_200k_midjourney_150k_recovery_150k_3e4_epoch2_half_rotated/005-GPT-XL/checkpoints/eval_step_38000/batch_1_cfg_7.5_topk_16384.jpg filter=lfs diff=lfs merge=lfs -text +CKPTS/Afteri2i_100kckpt_LLAVAT5_empty_Fixed_just_segment_pretrain_stage3_dreambench_recap_400k_t2i_400k_flux_200k_midjourney_150k_recovery_150k_3e4_epoch2_half_rotated/005-GPT-XL/checkpoints/eval_step_38000/batch_2_cfg_7.5_topk_16384.jpg filter=lfs diff=lfs merge=lfs -text +CKPTS/Afteri2i_100kckpt_LLAVAT5_empty_Fixed_just_segment_pretrain_stage3_dreambench_recap_400k_t2i_400k_flux_200k_midjourney_150k_recovery_150k_3e4_epoch2_half_rotated/005-GPT-XL/checkpoints/eval_step_38000/batch_3_cfg_7.5_topk_16384.jpg filter=lfs diff=lfs merge=lfs -text +CKPTS/Afteri2i_100kckpt_LLAVAT5_empty_Fixed_just_segment_pretrain_stage3_dreambench_recap_400k_t2i_400k_flux_200k_midjourney_150k_recovery_150k_3e4_epoch2_half_rotated/005-GPT-XL/checkpoints/eval_step_4000/batch_0_cfg_7.5_topk_16384.jpg filter=lfs diff=lfs merge=lfs -text +CKPTS/Afteri2i_100kckpt_LLAVAT5_empty_Fixed_just_segment_pretrain_stage3_dreambench_recap_400k_t2i_400k_flux_200k_midjourney_150k_recovery_150k_3e4_epoch2_half_rotated/005-GPT-XL/checkpoints/eval_step_4000/batch_1_cfg_7.5_topk_16384.jpg filter=lfs diff=lfs merge=lfs -text +CKPTS/Afteri2i_100kckpt_LLAVAT5_empty_Fixed_just_segment_pretrain_stage3_dreambench_recap_400k_t2i_400k_flux_200k_midjourney_150k_recovery_150k_3e4_epoch2_half_rotated/005-GPT-XL/checkpoints/eval_step_4000/batch_2_cfg_7.5_topk_16384.jpg filter=lfs diff=lfs merge=lfs -text +CKPTS/Afteri2i_100kckpt_LLAVAT5_empty_Fixed_just_segment_pretrain_stage3_dreambench_recap_400k_t2i_400k_flux_200k_midjourney_150k_recovery_150k_3e4_epoch2_half_rotated/005-GPT-XL/checkpoints/eval_step_4000/batch_3_cfg_7.5_topk_16384.jpg filter=lfs diff=lfs merge=lfs -text +CKPTS/Afteri2i_100kckpt_LLAVAT5_empty_Fixed_just_segment_pretrain_stage3_dreambench_recap_400k_t2i_400k_flux_200k_midjourney_150k_recovery_150k_3e4_epoch2_half_rotated/005-GPT-XL/checkpoints/eval_step_40000/batch_0_cfg_7.5_topk_16384.jpg filter=lfs diff=lfs merge=lfs -text +CKPTS/Afteri2i_100kckpt_LLAVAT5_empty_Fixed_just_segment_pretrain_stage3_dreambench_recap_400k_t2i_400k_flux_200k_midjourney_150k_recovery_150k_3e4_epoch2_half_rotated/005-GPT-XL/checkpoints/eval_step_40000/batch_1_cfg_7.5_topk_16384.jpg filter=lfs diff=lfs merge=lfs -text +CKPTS/Afteri2i_100kckpt_LLAVAT5_empty_Fixed_just_segment_pretrain_stage3_dreambench_recap_400k_t2i_400k_flux_200k_midjourney_150k_recovery_150k_3e4_epoch2_half_rotated/005-GPT-XL/checkpoints/eval_step_40000/batch_2_cfg_7.5_topk_16384.jpg filter=lfs diff=lfs merge=lfs -text +CKPTS/Afteri2i_100kckpt_LLAVAT5_empty_Fixed_just_segment_pretrain_stage3_dreambench_recap_400k_t2i_400k_flux_200k_midjourney_150k_recovery_150k_3e4_epoch2_half_rotated/005-GPT-XL/checkpoints/eval_step_40000/batch_3_cfg_7.5_topk_16384.jpg filter=lfs diff=lfs merge=lfs -text +CKPTS/Afteri2i_100kckpt_LLAVAT5_empty_Fixed_just_segment_pretrain_stage3_dreambench_recap_400k_t2i_400k_flux_200k_midjourney_150k_recovery_150k_3e4_epoch2_half_rotated/005-GPT-XL/checkpoints/eval_step_42000/batch_0_cfg_7.5_topk_16384.jpg filter=lfs diff=lfs merge=lfs -text +CKPTS/Afteri2i_100kckpt_LLAVAT5_empty_Fixed_just_segment_pretrain_stage3_dreambench_recap_400k_t2i_400k_flux_200k_midjourney_150k_recovery_150k_3e4_epoch2_half_rotated/005-GPT-XL/checkpoints/eval_step_42000/batch_1_cfg_7.5_topk_16384.jpg filter=lfs diff=lfs merge=lfs -text +CKPTS/Afteri2i_100kckpt_LLAVAT5_empty_Fixed_just_segment_pretrain_stage3_dreambench_recap_400k_t2i_400k_flux_200k_midjourney_150k_recovery_150k_3e4_epoch2_half_rotated/005-GPT-XL/checkpoints/eval_step_42000/batch_2_cfg_7.5_topk_16384.jpg filter=lfs diff=lfs merge=lfs -text +CKPTS/Afteri2i_100kckpt_LLAVAT5_empty_Fixed_just_segment_pretrain_stage3_dreambench_recap_400k_t2i_400k_flux_200k_midjourney_150k_recovery_150k_3e4_epoch2_half_rotated/005-GPT-XL/checkpoints/eval_step_42000/batch_3_cfg_7.5_topk_16384.jpg filter=lfs diff=lfs merge=lfs -text +CKPTS/Afteri2i_100kckpt_LLAVAT5_empty_Fixed_just_segment_pretrain_stage3_dreambench_recap_400k_t2i_400k_flux_200k_midjourney_150k_recovery_150k_3e4_epoch2_half_rotated/005-GPT-XL/checkpoints/eval_step_44000/batch_0_cfg_7.5_topk_16384.jpg filter=lfs diff=lfs merge=lfs -text +CKPTS/Afteri2i_100kckpt_LLAVAT5_empty_Fixed_just_segment_pretrain_stage3_dreambench_recap_400k_t2i_400k_flux_200k_midjourney_150k_recovery_150k_3e4_epoch2_half_rotated/005-GPT-XL/checkpoints/eval_step_44000/batch_1_cfg_7.5_topk_16384.jpg filter=lfs diff=lfs merge=lfs -text +CKPTS/Afteri2i_100kckpt_LLAVAT5_empty_Fixed_just_segment_pretrain_stage3_dreambench_recap_400k_t2i_400k_flux_200k_midjourney_150k_recovery_150k_3e4_epoch2_half_rotated/005-GPT-XL/checkpoints/eval_step_44000/batch_2_cfg_7.5_topk_16384.jpg filter=lfs diff=lfs merge=lfs -text +CKPTS/Afteri2i_100kckpt_LLAVAT5_empty_Fixed_just_segment_pretrain_stage3_dreambench_recap_400k_t2i_400k_flux_200k_midjourney_150k_recovery_150k_3e4_epoch2_half_rotated/005-GPT-XL/checkpoints/eval_step_44000/batch_3_cfg_7.5_topk_16384.jpg filter=lfs diff=lfs merge=lfs -text +CKPTS/Afteri2i_100kckpt_LLAVAT5_empty_Fixed_just_segment_pretrain_stage3_dreambench_recap_400k_t2i_400k_flux_200k_midjourney_150k_recovery_150k_3e4_epoch2_half_rotated/005-GPT-XL/checkpoints/eval_step_46000/batch_0_cfg_7.5_topk_16384.jpg filter=lfs diff=lfs merge=lfs -text +CKPTS/Afteri2i_100kckpt_LLAVAT5_empty_Fixed_just_segment_pretrain_stage3_dreambench_recap_400k_t2i_400k_flux_200k_midjourney_150k_recovery_150k_3e4_epoch2_half_rotated/005-GPT-XL/checkpoints/eval_step_46000/batch_1_cfg_7.5_topk_16384.jpg filter=lfs diff=lfs merge=lfs -text +CKPTS/Afteri2i_100kckpt_LLAVAT5_empty_Fixed_just_segment_pretrain_stage3_dreambench_recap_400k_t2i_400k_flux_200k_midjourney_150k_recovery_150k_3e4_epoch2_half_rotated/005-GPT-XL/checkpoints/eval_step_46000/batch_2_cfg_7.5_topk_16384.jpg filter=lfs diff=lfs merge=lfs -text +CKPTS/Afteri2i_100kckpt_LLAVAT5_empty_Fixed_just_segment_pretrain_stage3_dreambench_recap_400k_t2i_400k_flux_200k_midjourney_150k_recovery_150k_3e4_epoch2_half_rotated/005-GPT-XL/checkpoints/eval_step_46000/batch_3_cfg_7.5_topk_16384.jpg filter=lfs diff=lfs merge=lfs -text +CKPTS/Afteri2i_100kckpt_LLAVAT5_empty_Fixed_just_segment_pretrain_stage3_dreambench_recap_400k_t2i_400k_flux_200k_midjourney_150k_recovery_150k_3e4_epoch2_half_rotated/005-GPT-XL/checkpoints/eval_step_6000/batch_0_cfg_7.5_topk_16384.jpg filter=lfs diff=lfs merge=lfs -text +CKPTS/Afteri2i_100kckpt_LLAVAT5_empty_Fixed_just_segment_pretrain_stage3_dreambench_recap_400k_t2i_400k_flux_200k_midjourney_150k_recovery_150k_3e4_epoch2_half_rotated/005-GPT-XL/checkpoints/eval_step_6000/batch_1_cfg_7.5_topk_16384.jpg filter=lfs diff=lfs merge=lfs -text +CKPTS/Afteri2i_100kckpt_LLAVAT5_empty_Fixed_just_segment_pretrain_stage3_dreambench_recap_400k_t2i_400k_flux_200k_midjourney_150k_recovery_150k_3e4_epoch2_half_rotated/005-GPT-XL/checkpoints/eval_step_6000/batch_2_cfg_7.5_topk_16384.jpg filter=lfs diff=lfs merge=lfs -text +CKPTS/Afteri2i_100kckpt_LLAVAT5_empty_Fixed_just_segment_pretrain_stage3_dreambench_recap_400k_t2i_400k_flux_200k_midjourney_150k_recovery_150k_3e4_epoch2_half_rotated/005-GPT-XL/checkpoints/eval_step_6000/batch_3_cfg_7.5_topk_16384.jpg filter=lfs diff=lfs merge=lfs -text +CKPTS/Afteri2i_100kckpt_LLAVAT5_empty_Fixed_just_segment_pretrain_stage3_dreambench_recap_400k_t2i_400k_flux_200k_midjourney_150k_recovery_150k_3e4_epoch2_half_rotated/005-GPT-XL/checkpoints/eval_step_8000/batch_0_cfg_7.5_topk_16384.jpg filter=lfs diff=lfs merge=lfs -text +CKPTS/Afteri2i_100kckpt_LLAVAT5_empty_Fixed_just_segment_pretrain_stage3_dreambench_recap_400k_t2i_400k_flux_200k_midjourney_150k_recovery_150k_3e4_epoch2_half_rotated/005-GPT-XL/checkpoints/eval_step_8000/batch_1_cfg_7.5_topk_16384.jpg filter=lfs diff=lfs merge=lfs -text +CKPTS/Afteri2i_100kckpt_LLAVAT5_empty_Fixed_just_segment_pretrain_stage3_dreambench_recap_400k_t2i_400k_flux_200k_midjourney_150k_recovery_150k_3e4_epoch2_half_rotated/005-GPT-XL/checkpoints/eval_step_8000/batch_2_cfg_7.5_topk_16384.jpg filter=lfs diff=lfs merge=lfs -text +CKPTS/Afteri2i_100kckpt_LLAVAT5_empty_Fixed_just_segment_pretrain_stage3_dreambench_recap_400k_t2i_400k_flux_200k_midjourney_150k_recovery_150k_3e4_epoch2_half_rotated/005-GPT-XL/checkpoints/eval_step_8000/batch_3_cfg_7.5_topk_16384.jpg filter=lfs diff=lfs merge=lfs -text +CKPTS/Afteri2i_100kckpt_LLAVAT5_empty_Fixed_just_segment_pretrain_stage3_dreambench_recap_400k_t2i_400k_flux_200k_midjourney_150k_recovery_150k_3e4_epoch2_half_rotated/wandb/run-20250423_111340-sphrjdl0/run-sphrjdl0.wandb filter=lfs diff=lfs merge=lfs -text +CKPTS/Afteri2i_100kckpt_LLAVAT5_empty_Fixed_just_segment_pretrain_stage3_dreambench_recap_400k_t2i_400k_flux_200k_midjourney_150k_recovery_150k_3e4_epoch2_half_rotated/wandb/run-20250423_122459-k5belznu/run-k5belznu.wandb filter=lfs diff=lfs merge=lfs -text +CKPTS/Afteri2i_100kckpt_LLAVAT5_empty_Fixed_just_segment_pretrain_stage3_dreambench_recap_400k_t2i_400k_flux_200k_midjourney_150k_recovery_150k_3e4_epoch2_no_rotated/000-GPT-XL/checkpoints/eval_step_10000/batch_0_cfg_7.5_topk_16384.jpg filter=lfs diff=lfs merge=lfs -text +CKPTS/Afteri2i_100kckpt_LLAVAT5_empty_Fixed_just_segment_pretrain_stage3_dreambench_recap_400k_t2i_400k_flux_200k_midjourney_150k_recovery_150k_3e4_epoch2_no_rotated/000-GPT-XL/checkpoints/eval_step_10000/batch_1_cfg_7.5_topk_16384.jpg filter=lfs diff=lfs merge=lfs -text +CKPTS/Afteri2i_100kckpt_LLAVAT5_empty_Fixed_just_segment_pretrain_stage3_dreambench_recap_400k_t2i_400k_flux_200k_midjourney_150k_recovery_150k_3e4_epoch2_no_rotated/000-GPT-XL/checkpoints/eval_step_10000/batch_2_cfg_7.5_topk_16384.jpg filter=lfs diff=lfs merge=lfs -text +CKPTS/Afteri2i_100kckpt_LLAVAT5_empty_Fixed_just_segment_pretrain_stage3_dreambench_recap_400k_t2i_400k_flux_200k_midjourney_150k_recovery_150k_3e4_epoch2_no_rotated/000-GPT-XL/checkpoints/eval_step_10000/batch_3_cfg_7.5_topk_16384.jpg filter=lfs diff=lfs merge=lfs -text +CKPTS/Afteri2i_100kckpt_LLAVAT5_empty_Fixed_just_segment_pretrain_stage3_dreambench_recap_400k_t2i_400k_flux_200k_midjourney_150k_recovery_150k_3e4_epoch2_no_rotated/000-GPT-XL/checkpoints/eval_step_12000/batch_0_cfg_7.5_topk_16384.jpg filter=lfs diff=lfs merge=lfs -text +CKPTS/Afteri2i_100kckpt_LLAVAT5_empty_Fixed_just_segment_pretrain_stage3_dreambench_recap_400k_t2i_400k_flux_200k_midjourney_150k_recovery_150k_3e4_epoch2_no_rotated/000-GPT-XL/checkpoints/eval_step_12000/batch_1_cfg_7.5_topk_16384.jpg filter=lfs diff=lfs merge=lfs -text +CKPTS/Afteri2i_100kckpt_LLAVAT5_empty_Fixed_just_segment_pretrain_stage3_dreambench_recap_400k_t2i_400k_flux_200k_midjourney_150k_recovery_150k_3e4_epoch2_no_rotated/000-GPT-XL/checkpoints/eval_step_12000/batch_2_cfg_7.5_topk_16384.jpg filter=lfs diff=lfs merge=lfs -text +CKPTS/Afteri2i_100kckpt_LLAVAT5_empty_Fixed_just_segment_pretrain_stage3_dreambench_recap_400k_t2i_400k_flux_200k_midjourney_150k_recovery_150k_3e4_epoch2_no_rotated/000-GPT-XL/checkpoints/eval_step_12000/batch_3_cfg_7.5_topk_16384.jpg filter=lfs diff=lfs merge=lfs -text +CKPTS/Afteri2i_100kckpt_LLAVAT5_empty_Fixed_just_segment_pretrain_stage3_dreambench_recap_400k_t2i_400k_flux_200k_midjourney_150k_recovery_150k_3e4_epoch2_no_rotated/000-GPT-XL/checkpoints/eval_step_14000/batch_0_cfg_7.5_topk_16384.jpg filter=lfs diff=lfs merge=lfs -text +CKPTS/Afteri2i_100kckpt_LLAVAT5_empty_Fixed_just_segment_pretrain_stage3_dreambench_recap_400k_t2i_400k_flux_200k_midjourney_150k_recovery_150k_3e4_epoch2_no_rotated/000-GPT-XL/checkpoints/eval_step_14000/batch_1_cfg_7.5_topk_16384.jpg filter=lfs diff=lfs merge=lfs -text +CKPTS/Afteri2i_100kckpt_LLAVAT5_empty_Fixed_just_segment_pretrain_stage3_dreambench_recap_400k_t2i_400k_flux_200k_midjourney_150k_recovery_150k_3e4_epoch2_no_rotated/000-GPT-XL/checkpoints/eval_step_14000/batch_2_cfg_7.5_topk_16384.jpg filter=lfs diff=lfs merge=lfs -text +CKPTS/Afteri2i_100kckpt_LLAVAT5_empty_Fixed_just_segment_pretrain_stage3_dreambench_recap_400k_t2i_400k_flux_200k_midjourney_150k_recovery_150k_3e4_epoch2_no_rotated/000-GPT-XL/checkpoints/eval_step_14000/batch_3_cfg_7.5_topk_16384.jpg filter=lfs diff=lfs merge=lfs -text +CKPTS/Afteri2i_100kckpt_LLAVAT5_empty_Fixed_just_segment_pretrain_stage3_dreambench_recap_400k_t2i_400k_flux_200k_midjourney_150k_recovery_150k_3e4_epoch2_no_rotated/000-GPT-XL/checkpoints/eval_step_16000/batch_0_cfg_7.5_topk_16384.jpg filter=lfs diff=lfs merge=lfs -text +CKPTS/Afteri2i_100kckpt_LLAVAT5_empty_Fixed_just_segment_pretrain_stage3_dreambench_recap_400k_t2i_400k_flux_200k_midjourney_150k_recovery_150k_3e4_epoch2_no_rotated/000-GPT-XL/checkpoints/eval_step_16000/batch_1_cfg_7.5_topk_16384.jpg filter=lfs diff=lfs merge=lfs -text +CKPTS/Afteri2i_100kckpt_LLAVAT5_empty_Fixed_just_segment_pretrain_stage3_dreambench_recap_400k_t2i_400k_flux_200k_midjourney_150k_recovery_150k_3e4_epoch2_no_rotated/000-GPT-XL/checkpoints/eval_step_16000/batch_2_cfg_7.5_topk_16384.jpg filter=lfs diff=lfs merge=lfs -text +CKPTS/Afteri2i_100kckpt_LLAVAT5_empty_Fixed_just_segment_pretrain_stage3_dreambench_recap_400k_t2i_400k_flux_200k_midjourney_150k_recovery_150k_3e4_epoch2_no_rotated/000-GPT-XL/checkpoints/eval_step_16000/batch_3_cfg_7.5_topk_16384.jpg filter=lfs diff=lfs merge=lfs -text +CKPTS/Afteri2i_100kckpt_LLAVAT5_empty_Fixed_just_segment_pretrain_stage3_dreambench_recap_400k_t2i_400k_flux_200k_midjourney_150k_recovery_150k_3e4_epoch2_no_rotated/000-GPT-XL/checkpoints/eval_step_18000/batch_0_cfg_7.5_topk_16384.jpg filter=lfs diff=lfs merge=lfs -text +CKPTS/Afteri2i_100kckpt_LLAVAT5_empty_Fixed_just_segment_pretrain_stage3_dreambench_recap_400k_t2i_400k_flux_200k_midjourney_150k_recovery_150k_3e4_epoch2_no_rotated/000-GPT-XL/checkpoints/eval_step_18000/batch_1_cfg_7.5_topk_16384.jpg filter=lfs diff=lfs merge=lfs -text +CKPTS/Afteri2i_100kckpt_LLAVAT5_empty_Fixed_just_segment_pretrain_stage3_dreambench_recap_400k_t2i_400k_flux_200k_midjourney_150k_recovery_150k_3e4_epoch2_no_rotated/000-GPT-XL/checkpoints/eval_step_18000/batch_2_cfg_7.5_topk_16384.jpg filter=lfs diff=lfs merge=lfs -text +CKPTS/Afteri2i_100kckpt_LLAVAT5_empty_Fixed_just_segment_pretrain_stage3_dreambench_recap_400k_t2i_400k_flux_200k_midjourney_150k_recovery_150k_3e4_epoch2_no_rotated/000-GPT-XL/checkpoints/eval_step_18000/batch_3_cfg_7.5_topk_16384.jpg filter=lfs diff=lfs merge=lfs -text +CKPTS/Afteri2i_100kckpt_LLAVAT5_empty_Fixed_just_segment_pretrain_stage3_dreambench_recap_400k_t2i_400k_flux_200k_midjourney_150k_recovery_150k_3e4_epoch2_no_rotated/000-GPT-XL/checkpoints/eval_step_2000/batch_0_cfg_7.5_topk_16384.jpg filter=lfs diff=lfs merge=lfs -text +CKPTS/Afteri2i_100kckpt_LLAVAT5_empty_Fixed_just_segment_pretrain_stage3_dreambench_recap_400k_t2i_400k_flux_200k_midjourney_150k_recovery_150k_3e4_epoch2_no_rotated/000-GPT-XL/checkpoints/eval_step_2000/batch_1_cfg_7.5_topk_16384.jpg filter=lfs diff=lfs merge=lfs -text +CKPTS/Afteri2i_100kckpt_LLAVAT5_empty_Fixed_just_segment_pretrain_stage3_dreambench_recap_400k_t2i_400k_flux_200k_midjourney_150k_recovery_150k_3e4_epoch2_no_rotated/000-GPT-XL/checkpoints/eval_step_2000/batch_2_cfg_7.5_topk_16384.jpg filter=lfs diff=lfs merge=lfs -text +CKPTS/Afteri2i_100kckpt_LLAVAT5_empty_Fixed_just_segment_pretrain_stage3_dreambench_recap_400k_t2i_400k_flux_200k_midjourney_150k_recovery_150k_3e4_epoch2_no_rotated/000-GPT-XL/checkpoints/eval_step_2000/batch_3_cfg_7.5_topk_16384.jpg filter=lfs diff=lfs merge=lfs -text +CKPTS/Afteri2i_100kckpt_LLAVAT5_empty_Fixed_just_segment_pretrain_stage3_dreambench_recap_400k_t2i_400k_flux_200k_midjourney_150k_recovery_150k_3e4_epoch2_no_rotated/000-GPT-XL/checkpoints/eval_step_20000/batch_0_cfg_7.5_topk_16384.jpg filter=lfs diff=lfs merge=lfs -text +CKPTS/Afteri2i_100kckpt_LLAVAT5_empty_Fixed_just_segment_pretrain_stage3_dreambench_recap_400k_t2i_400k_flux_200k_midjourney_150k_recovery_150k_3e4_epoch2_no_rotated/000-GPT-XL/checkpoints/eval_step_20000/batch_1_cfg_7.5_topk_16384.jpg filter=lfs diff=lfs merge=lfs -text +CKPTS/Afteri2i_100kckpt_LLAVAT5_empty_Fixed_just_segment_pretrain_stage3_dreambench_recap_400k_t2i_400k_flux_200k_midjourney_150k_recovery_150k_3e4_epoch2_no_rotated/000-GPT-XL/checkpoints/eval_step_20000/batch_2_cfg_7.5_topk_16384.jpg filter=lfs diff=lfs merge=lfs -text +CKPTS/Afteri2i_100kckpt_LLAVAT5_empty_Fixed_just_segment_pretrain_stage3_dreambench_recap_400k_t2i_400k_flux_200k_midjourney_150k_recovery_150k_3e4_epoch2_no_rotated/000-GPT-XL/checkpoints/eval_step_20000/batch_3_cfg_7.5_topk_16384.jpg filter=lfs diff=lfs merge=lfs -text +CKPTS/Afteri2i_100kckpt_LLAVAT5_empty_Fixed_just_segment_pretrain_stage3_dreambench_recap_400k_t2i_400k_flux_200k_midjourney_150k_recovery_150k_3e4_epoch2_no_rotated/000-GPT-XL/checkpoints/eval_step_22000/batch_0_cfg_7.5_topk_16384.jpg filter=lfs diff=lfs merge=lfs -text +CKPTS/Afteri2i_100kckpt_LLAVAT5_empty_Fixed_just_segment_pretrain_stage3_dreambench_recap_400k_t2i_400k_flux_200k_midjourney_150k_recovery_150k_3e4_epoch2_no_rotated/000-GPT-XL/checkpoints/eval_step_22000/batch_1_cfg_7.5_topk_16384.jpg filter=lfs diff=lfs merge=lfs -text +CKPTS/Afteri2i_100kckpt_LLAVAT5_empty_Fixed_just_segment_pretrain_stage3_dreambench_recap_400k_t2i_400k_flux_200k_midjourney_150k_recovery_150k_3e4_epoch2_no_rotated/000-GPT-XL/checkpoints/eval_step_22000/batch_2_cfg_7.5_topk_16384.jpg filter=lfs diff=lfs merge=lfs -text +CKPTS/Afteri2i_100kckpt_LLAVAT5_empty_Fixed_just_segment_pretrain_stage3_dreambench_recap_400k_t2i_400k_flux_200k_midjourney_150k_recovery_150k_3e4_epoch2_no_rotated/000-GPT-XL/checkpoints/eval_step_22000/batch_3_cfg_7.5_topk_16384.jpg filter=lfs diff=lfs merge=lfs -text +CKPTS/Afteri2i_100kckpt_LLAVAT5_empty_Fixed_just_segment_pretrain_stage3_dreambench_recap_400k_t2i_400k_flux_200k_midjourney_150k_recovery_150k_3e4_epoch2_no_rotated/000-GPT-XL/checkpoints/eval_step_24000/batch_0_cfg_7.5_topk_16384.jpg filter=lfs diff=lfs merge=lfs -text +CKPTS/Afteri2i_100kckpt_LLAVAT5_empty_Fixed_just_segment_pretrain_stage3_dreambench_recap_400k_t2i_400k_flux_200k_midjourney_150k_recovery_150k_3e4_epoch2_no_rotated/000-GPT-XL/checkpoints/eval_step_24000/batch_1_cfg_7.5_topk_16384.jpg filter=lfs diff=lfs merge=lfs -text +CKPTS/Afteri2i_100kckpt_LLAVAT5_empty_Fixed_just_segment_pretrain_stage3_dreambench_recap_400k_t2i_400k_flux_200k_midjourney_150k_recovery_150k_3e4_epoch2_no_rotated/000-GPT-XL/checkpoints/eval_step_24000/batch_2_cfg_7.5_topk_16384.jpg filter=lfs diff=lfs merge=lfs -text +CKPTS/Afteri2i_100kckpt_LLAVAT5_empty_Fixed_just_segment_pretrain_stage3_dreambench_recap_400k_t2i_400k_flux_200k_midjourney_150k_recovery_150k_3e4_epoch2_no_rotated/000-GPT-XL/checkpoints/eval_step_24000/batch_3_cfg_7.5_topk_16384.jpg filter=lfs diff=lfs merge=lfs -text +CKPTS/Afteri2i_100kckpt_LLAVAT5_empty_Fixed_just_segment_pretrain_stage3_dreambench_recap_400k_t2i_400k_flux_200k_midjourney_150k_recovery_150k_3e4_epoch2_no_rotated/000-GPT-XL/checkpoints/eval_step_26000/batch_0_cfg_7.5_topk_16384.jpg filter=lfs diff=lfs merge=lfs -text +CKPTS/Afteri2i_100kckpt_LLAVAT5_empty_Fixed_just_segment_pretrain_stage3_dreambench_recap_400k_t2i_400k_flux_200k_midjourney_150k_recovery_150k_3e4_epoch2_no_rotated/000-GPT-XL/checkpoints/eval_step_26000/batch_1_cfg_7.5_topk_16384.jpg filter=lfs diff=lfs merge=lfs -text +CKPTS/Afteri2i_100kckpt_LLAVAT5_empty_Fixed_just_segment_pretrain_stage3_dreambench_recap_400k_t2i_400k_flux_200k_midjourney_150k_recovery_150k_3e4_epoch2_no_rotated/000-GPT-XL/checkpoints/eval_step_26000/batch_2_cfg_7.5_topk_16384.jpg filter=lfs diff=lfs merge=lfs -text +CKPTS/Afteri2i_100kckpt_LLAVAT5_empty_Fixed_just_segment_pretrain_stage3_dreambench_recap_400k_t2i_400k_flux_200k_midjourney_150k_recovery_150k_3e4_epoch2_no_rotated/000-GPT-XL/checkpoints/eval_step_26000/batch_3_cfg_7.5_topk_16384.jpg filter=lfs diff=lfs merge=lfs -text +CKPTS/Afteri2i_100kckpt_LLAVAT5_empty_Fixed_just_segment_pretrain_stage3_dreambench_recap_400k_t2i_400k_flux_200k_midjourney_150k_recovery_150k_3e4_epoch2_no_rotated/000-GPT-XL/checkpoints/eval_step_28000/batch_0_cfg_7.5_topk_16384.jpg filter=lfs diff=lfs merge=lfs -text +CKPTS/Afteri2i_100kckpt_LLAVAT5_empty_Fixed_just_segment_pretrain_stage3_dreambench_recap_400k_t2i_400k_flux_200k_midjourney_150k_recovery_150k_3e4_epoch2_no_rotated/000-GPT-XL/checkpoints/eval_step_28000/batch_1_cfg_7.5_topk_16384.jpg filter=lfs diff=lfs merge=lfs -text +CKPTS/Afteri2i_100kckpt_LLAVAT5_empty_Fixed_just_segment_pretrain_stage3_dreambench_recap_400k_t2i_400k_flux_200k_midjourney_150k_recovery_150k_3e4_epoch2_no_rotated/000-GPT-XL/checkpoints/eval_step_28000/batch_2_cfg_7.5_topk_16384.jpg filter=lfs diff=lfs merge=lfs -text +CKPTS/Afteri2i_100kckpt_LLAVAT5_empty_Fixed_just_segment_pretrain_stage3_dreambench_recap_400k_t2i_400k_flux_200k_midjourney_150k_recovery_150k_3e4_epoch2_no_rotated/000-GPT-XL/checkpoints/eval_step_28000/batch_3_cfg_7.5_topk_16384.jpg filter=lfs diff=lfs merge=lfs -text +CKPTS/Afteri2i_100kckpt_LLAVAT5_empty_Fixed_just_segment_pretrain_stage3_dreambench_recap_400k_t2i_400k_flux_200k_midjourney_150k_recovery_150k_3e4_epoch2_no_rotated/000-GPT-XL/checkpoints/eval_step_30000/batch_0_cfg_7.5_topk_16384.jpg filter=lfs diff=lfs merge=lfs -text +CKPTS/Afteri2i_100kckpt_LLAVAT5_empty_Fixed_just_segment_pretrain_stage3_dreambench_recap_400k_t2i_400k_flux_200k_midjourney_150k_recovery_150k_3e4_epoch2_no_rotated/000-GPT-XL/checkpoints/eval_step_30000/batch_1_cfg_7.5_topk_16384.jpg filter=lfs diff=lfs merge=lfs -text +CKPTS/Afteri2i_100kckpt_LLAVAT5_empty_Fixed_just_segment_pretrain_stage3_dreambench_recap_400k_t2i_400k_flux_200k_midjourney_150k_recovery_150k_3e4_epoch2_no_rotated/000-GPT-XL/checkpoints/eval_step_30000/batch_2_cfg_7.5_topk_16384.jpg filter=lfs diff=lfs merge=lfs -text +CKPTS/Afteri2i_100kckpt_LLAVAT5_empty_Fixed_just_segment_pretrain_stage3_dreambench_recap_400k_t2i_400k_flux_200k_midjourney_150k_recovery_150k_3e4_epoch2_no_rotated/000-GPT-XL/checkpoints/eval_step_30000/batch_3_cfg_7.5_topk_16384.jpg filter=lfs diff=lfs merge=lfs -text +CKPTS/Afteri2i_100kckpt_LLAVAT5_empty_Fixed_just_segment_pretrain_stage3_dreambench_recap_400k_t2i_400k_flux_200k_midjourney_150k_recovery_150k_3e4_epoch2_no_rotated/000-GPT-XL/checkpoints/eval_step_32000/batch_0_cfg_7.5_topk_16384.jpg filter=lfs diff=lfs merge=lfs -text +CKPTS/Afteri2i_100kckpt_LLAVAT5_empty_Fixed_just_segment_pretrain_stage3_dreambench_recap_400k_t2i_400k_flux_200k_midjourney_150k_recovery_150k_3e4_epoch2_no_rotated/000-GPT-XL/checkpoints/eval_step_32000/batch_1_cfg_7.5_topk_16384.jpg filter=lfs diff=lfs merge=lfs -text +CKPTS/Afteri2i_100kckpt_LLAVAT5_empty_Fixed_just_segment_pretrain_stage3_dreambench_recap_400k_t2i_400k_flux_200k_midjourney_150k_recovery_150k_3e4_epoch2_no_rotated/000-GPT-XL/checkpoints/eval_step_32000/batch_2_cfg_7.5_topk_16384.jpg filter=lfs diff=lfs merge=lfs -text +CKPTS/Afteri2i_100kckpt_LLAVAT5_empty_Fixed_just_segment_pretrain_stage3_dreambench_recap_400k_t2i_400k_flux_200k_midjourney_150k_recovery_150k_3e4_epoch2_no_rotated/000-GPT-XL/checkpoints/eval_step_32000/batch_3_cfg_7.5_topk_16384.jpg filter=lfs diff=lfs merge=lfs -text +CKPTS/Afteri2i_100kckpt_LLAVAT5_empty_Fixed_just_segment_pretrain_stage3_dreambench_recap_400k_t2i_400k_flux_200k_midjourney_150k_recovery_150k_3e4_epoch2_no_rotated/000-GPT-XL/checkpoints/eval_step_34000/batch_0_cfg_7.5_topk_16384.jpg filter=lfs diff=lfs merge=lfs -text +CKPTS/Afteri2i_100kckpt_LLAVAT5_empty_Fixed_just_segment_pretrain_stage3_dreambench_recap_400k_t2i_400k_flux_200k_midjourney_150k_recovery_150k_3e4_epoch2_no_rotated/000-GPT-XL/checkpoints/eval_step_34000/batch_1_cfg_7.5_topk_16384.jpg filter=lfs diff=lfs merge=lfs -text +CKPTS/Afteri2i_100kckpt_LLAVAT5_empty_Fixed_just_segment_pretrain_stage3_dreambench_recap_400k_t2i_400k_flux_200k_midjourney_150k_recovery_150k_3e4_epoch2_no_rotated/000-GPT-XL/checkpoints/eval_step_34000/batch_2_cfg_7.5_topk_16384.jpg filter=lfs diff=lfs merge=lfs -text +CKPTS/Afteri2i_100kckpt_LLAVAT5_empty_Fixed_just_segment_pretrain_stage3_dreambench_recap_400k_t2i_400k_flux_200k_midjourney_150k_recovery_150k_3e4_epoch2_no_rotated/000-GPT-XL/checkpoints/eval_step_34000/batch_3_cfg_7.5_topk_16384.jpg filter=lfs diff=lfs merge=lfs -text +CKPTS/Afteri2i_100kckpt_LLAVAT5_empty_Fixed_just_segment_pretrain_stage3_dreambench_recap_400k_t2i_400k_flux_200k_midjourney_150k_recovery_150k_3e4_epoch2_no_rotated/000-GPT-XL/checkpoints/eval_step_36000/batch_0_cfg_7.5_topk_16384.jpg filter=lfs diff=lfs merge=lfs -text +CKPTS/Afteri2i_100kckpt_LLAVAT5_empty_Fixed_just_segment_pretrain_stage3_dreambench_recap_400k_t2i_400k_flux_200k_midjourney_150k_recovery_150k_3e4_epoch2_no_rotated/000-GPT-XL/checkpoints/eval_step_36000/batch_1_cfg_7.5_topk_16384.jpg filter=lfs diff=lfs merge=lfs -text +CKPTS/Afteri2i_100kckpt_LLAVAT5_empty_Fixed_just_segment_pretrain_stage3_dreambench_recap_400k_t2i_400k_flux_200k_midjourney_150k_recovery_150k_3e4_epoch2_no_rotated/000-GPT-XL/checkpoints/eval_step_36000/batch_2_cfg_7.5_topk_16384.jpg filter=lfs diff=lfs merge=lfs -text +CKPTS/Afteri2i_100kckpt_LLAVAT5_empty_Fixed_just_segment_pretrain_stage3_dreambench_recap_400k_t2i_400k_flux_200k_midjourney_150k_recovery_150k_3e4_epoch2_no_rotated/000-GPT-XL/checkpoints/eval_step_36000/batch_3_cfg_7.5_topk_16384.jpg filter=lfs diff=lfs merge=lfs -text +CKPTS/Afteri2i_100kckpt_LLAVAT5_empty_Fixed_just_segment_pretrain_stage3_dreambench_recap_400k_t2i_400k_flux_200k_midjourney_150k_recovery_150k_3e4_epoch2_no_rotated/000-GPT-XL/checkpoints/eval_step_38000/batch_0_cfg_7.5_topk_16384.jpg filter=lfs diff=lfs merge=lfs -text +CKPTS/Afteri2i_100kckpt_LLAVAT5_empty_Fixed_just_segment_pretrain_stage3_dreambench_recap_400k_t2i_400k_flux_200k_midjourney_150k_recovery_150k_3e4_epoch2_no_rotated/000-GPT-XL/checkpoints/eval_step_38000/batch_1_cfg_7.5_topk_16384.jpg filter=lfs diff=lfs merge=lfs -text +CKPTS/Afteri2i_100kckpt_LLAVAT5_empty_Fixed_just_segment_pretrain_stage3_dreambench_recap_400k_t2i_400k_flux_200k_midjourney_150k_recovery_150k_3e4_epoch2_no_rotated/000-GPT-XL/checkpoints/eval_step_38000/batch_2_cfg_7.5_topk_16384.jpg filter=lfs diff=lfs merge=lfs -text +CKPTS/Afteri2i_100kckpt_LLAVAT5_empty_Fixed_just_segment_pretrain_stage3_dreambench_recap_400k_t2i_400k_flux_200k_midjourney_150k_recovery_150k_3e4_epoch2_no_rotated/000-GPT-XL/checkpoints/eval_step_38000/batch_3_cfg_7.5_topk_16384.jpg filter=lfs diff=lfs merge=lfs -text +CKPTS/Afteri2i_100kckpt_LLAVAT5_empty_Fixed_just_segment_pretrain_stage3_dreambench_recap_400k_t2i_400k_flux_200k_midjourney_150k_recovery_150k_3e4_epoch2_no_rotated/000-GPT-XL/checkpoints/eval_step_4000/batch_0_cfg_7.5_topk_16384.jpg filter=lfs diff=lfs merge=lfs -text +CKPTS/Afteri2i_100kckpt_LLAVAT5_empty_Fixed_just_segment_pretrain_stage3_dreambench_recap_400k_t2i_400k_flux_200k_midjourney_150k_recovery_150k_3e4_epoch2_no_rotated/000-GPT-XL/checkpoints/eval_step_4000/batch_1_cfg_7.5_topk_16384.jpg filter=lfs diff=lfs merge=lfs -text +CKPTS/Afteri2i_100kckpt_LLAVAT5_empty_Fixed_just_segment_pretrain_stage3_dreambench_recap_400k_t2i_400k_flux_200k_midjourney_150k_recovery_150k_3e4_epoch2_no_rotated/000-GPT-XL/checkpoints/eval_step_4000/batch_2_cfg_7.5_topk_16384.jpg filter=lfs diff=lfs merge=lfs -text +CKPTS/Afteri2i_100kckpt_LLAVAT5_empty_Fixed_just_segment_pretrain_stage3_dreambench_recap_400k_t2i_400k_flux_200k_midjourney_150k_recovery_150k_3e4_epoch2_no_rotated/000-GPT-XL/checkpoints/eval_step_4000/batch_3_cfg_7.5_topk_16384.jpg filter=lfs diff=lfs merge=lfs -text +CKPTS/Afteri2i_100kckpt_LLAVAT5_empty_Fixed_just_segment_pretrain_stage3_dreambench_recap_400k_t2i_400k_flux_200k_midjourney_150k_recovery_150k_3e4_epoch2_no_rotated/000-GPT-XL/checkpoints/eval_step_40000/batch_0_cfg_7.5_topk_16384.jpg filter=lfs diff=lfs merge=lfs -text +CKPTS/Afteri2i_100kckpt_LLAVAT5_empty_Fixed_just_segment_pretrain_stage3_dreambench_recap_400k_t2i_400k_flux_200k_midjourney_150k_recovery_150k_3e4_epoch2_no_rotated/000-GPT-XL/checkpoints/eval_step_40000/batch_1_cfg_7.5_topk_16384.jpg filter=lfs diff=lfs merge=lfs -text +CKPTS/Afteri2i_100kckpt_LLAVAT5_empty_Fixed_just_segment_pretrain_stage3_dreambench_recap_400k_t2i_400k_flux_200k_midjourney_150k_recovery_150k_3e4_epoch2_no_rotated/000-GPT-XL/checkpoints/eval_step_40000/batch_2_cfg_7.5_topk_16384.jpg filter=lfs diff=lfs merge=lfs -text +CKPTS/Afteri2i_100kckpt_LLAVAT5_empty_Fixed_just_segment_pretrain_stage3_dreambench_recap_400k_t2i_400k_flux_200k_midjourney_150k_recovery_150k_3e4_epoch2_no_rotated/000-GPT-XL/checkpoints/eval_step_40000/batch_3_cfg_7.5_topk_16384.jpg filter=lfs diff=lfs merge=lfs -text +CKPTS/Afteri2i_100kckpt_LLAVAT5_empty_Fixed_just_segment_pretrain_stage3_dreambench_recap_400k_t2i_400k_flux_200k_midjourney_150k_recovery_150k_3e4_epoch2_no_rotated/000-GPT-XL/checkpoints/eval_step_42000/batch_0_cfg_7.5_topk_16384.jpg filter=lfs diff=lfs merge=lfs -text +CKPTS/Afteri2i_100kckpt_LLAVAT5_empty_Fixed_just_segment_pretrain_stage3_dreambench_recap_400k_t2i_400k_flux_200k_midjourney_150k_recovery_150k_3e4_epoch2_no_rotated/000-GPT-XL/checkpoints/eval_step_42000/batch_1_cfg_7.5_topk_16384.jpg filter=lfs diff=lfs merge=lfs -text +CKPTS/Afteri2i_100kckpt_LLAVAT5_empty_Fixed_just_segment_pretrain_stage3_dreambench_recap_400k_t2i_400k_flux_200k_midjourney_150k_recovery_150k_3e4_epoch2_no_rotated/000-GPT-XL/checkpoints/eval_step_42000/batch_2_cfg_7.5_topk_16384.jpg filter=lfs diff=lfs merge=lfs -text +CKPTS/Afteri2i_100kckpt_LLAVAT5_empty_Fixed_just_segment_pretrain_stage3_dreambench_recap_400k_t2i_400k_flux_200k_midjourney_150k_recovery_150k_3e4_epoch2_no_rotated/000-GPT-XL/checkpoints/eval_step_42000/batch_3_cfg_7.5_topk_16384.jpg filter=lfs diff=lfs merge=lfs -text +CKPTS/Afteri2i_100kckpt_LLAVAT5_empty_Fixed_just_segment_pretrain_stage3_dreambench_recap_400k_t2i_400k_flux_200k_midjourney_150k_recovery_150k_3e4_epoch2_no_rotated/000-GPT-XL/checkpoints/eval_step_44000/batch_0_cfg_7.5_topk_16384.jpg filter=lfs diff=lfs merge=lfs -text +CKPTS/Afteri2i_100kckpt_LLAVAT5_empty_Fixed_just_segment_pretrain_stage3_dreambench_recap_400k_t2i_400k_flux_200k_midjourney_150k_recovery_150k_3e4_epoch2_no_rotated/000-GPT-XL/checkpoints/eval_step_44000/batch_1_cfg_7.5_topk_16384.jpg filter=lfs diff=lfs merge=lfs -text +CKPTS/Afteri2i_100kckpt_LLAVAT5_empty_Fixed_just_segment_pretrain_stage3_dreambench_recap_400k_t2i_400k_flux_200k_midjourney_150k_recovery_150k_3e4_epoch2_no_rotated/000-GPT-XL/checkpoints/eval_step_44000/batch_2_cfg_7.5_topk_16384.jpg filter=lfs diff=lfs merge=lfs -text +CKPTS/Afteri2i_100kckpt_LLAVAT5_empty_Fixed_just_segment_pretrain_stage3_dreambench_recap_400k_t2i_400k_flux_200k_midjourney_150k_recovery_150k_3e4_epoch2_no_rotated/000-GPT-XL/checkpoints/eval_step_44000/batch_3_cfg_7.5_topk_16384.jpg filter=lfs diff=lfs merge=lfs -text +CKPTS/Afteri2i_100kckpt_LLAVAT5_empty_Fixed_just_segment_pretrain_stage3_dreambench_recap_400k_t2i_400k_flux_200k_midjourney_150k_recovery_150k_3e4_epoch2_no_rotated/000-GPT-XL/checkpoints/eval_step_46000/batch_0_cfg_7.5_topk_16384.jpg filter=lfs diff=lfs merge=lfs -text +CKPTS/Afteri2i_100kckpt_LLAVAT5_empty_Fixed_just_segment_pretrain_stage3_dreambench_recap_400k_t2i_400k_flux_200k_midjourney_150k_recovery_150k_3e4_epoch2_no_rotated/000-GPT-XL/checkpoints/eval_step_46000/batch_1_cfg_7.5_topk_16384.jpg filter=lfs diff=lfs merge=lfs -text +CKPTS/Afteri2i_100kckpt_LLAVAT5_empty_Fixed_just_segment_pretrain_stage3_dreambench_recap_400k_t2i_400k_flux_200k_midjourney_150k_recovery_150k_3e4_epoch2_no_rotated/000-GPT-XL/checkpoints/eval_step_46000/batch_2_cfg_7.5_topk_16384.jpg filter=lfs diff=lfs merge=lfs -text +CKPTS/Afteri2i_100kckpt_LLAVAT5_empty_Fixed_just_segment_pretrain_stage3_dreambench_recap_400k_t2i_400k_flux_200k_midjourney_150k_recovery_150k_3e4_epoch2_no_rotated/000-GPT-XL/checkpoints/eval_step_46000/batch_3_cfg_7.5_topk_16384.jpg filter=lfs diff=lfs merge=lfs -text +CKPTS/Afteri2i_100kckpt_LLAVAT5_empty_Fixed_just_segment_pretrain_stage3_dreambench_recap_400k_t2i_400k_flux_200k_midjourney_150k_recovery_150k_3e4_epoch2_no_rotated/000-GPT-XL/checkpoints/eval_step_6000/batch_0_cfg_7.5_topk_16384.jpg filter=lfs diff=lfs merge=lfs -text +CKPTS/Afteri2i_100kckpt_LLAVAT5_empty_Fixed_just_segment_pretrain_stage3_dreambench_recap_400k_t2i_400k_flux_200k_midjourney_150k_recovery_150k_3e4_epoch2_no_rotated/000-GPT-XL/checkpoints/eval_step_6000/batch_1_cfg_7.5_topk_16384.jpg filter=lfs diff=lfs merge=lfs -text +CKPTS/Afteri2i_100kckpt_LLAVAT5_empty_Fixed_just_segment_pretrain_stage3_dreambench_recap_400k_t2i_400k_flux_200k_midjourney_150k_recovery_150k_3e4_epoch2_no_rotated/000-GPT-XL/checkpoints/eval_step_6000/batch_2_cfg_7.5_topk_16384.jpg filter=lfs diff=lfs merge=lfs -text +CKPTS/Afteri2i_100kckpt_LLAVAT5_empty_Fixed_just_segment_pretrain_stage3_dreambench_recap_400k_t2i_400k_flux_200k_midjourney_150k_recovery_150k_3e4_epoch2_no_rotated/000-GPT-XL/checkpoints/eval_step_6000/batch_3_cfg_7.5_topk_16384.jpg filter=lfs diff=lfs merge=lfs -text +CKPTS/Afteri2i_100kckpt_LLAVAT5_empty_Fixed_just_segment_pretrain_stage3_dreambench_recap_400k_t2i_400k_flux_200k_midjourney_150k_recovery_150k_3e4_epoch2_no_rotated/000-GPT-XL/checkpoints/eval_step_8000/batch_0_cfg_7.5_topk_16384.jpg filter=lfs diff=lfs merge=lfs -text +CKPTS/Afteri2i_100kckpt_LLAVAT5_empty_Fixed_just_segment_pretrain_stage3_dreambench_recap_400k_t2i_400k_flux_200k_midjourney_150k_recovery_150k_3e4_epoch2_no_rotated/000-GPT-XL/checkpoints/eval_step_8000/batch_1_cfg_7.5_topk_16384.jpg filter=lfs diff=lfs merge=lfs -text +CKPTS/Afteri2i_100kckpt_LLAVAT5_empty_Fixed_just_segment_pretrain_stage3_dreambench_recap_400k_t2i_400k_flux_200k_midjourney_150k_recovery_150k_3e4_epoch2_no_rotated/000-GPT-XL/checkpoints/eval_step_8000/batch_2_cfg_7.5_topk_16384.jpg filter=lfs diff=lfs merge=lfs -text +CKPTS/Afteri2i_100kckpt_LLAVAT5_empty_Fixed_just_segment_pretrain_stage3_dreambench_recap_400k_t2i_400k_flux_200k_midjourney_150k_recovery_150k_3e4_epoch2_no_rotated/000-GPT-XL/checkpoints/eval_step_8000/batch_3_cfg_7.5_topk_16384.jpg filter=lfs diff=lfs merge=lfs -text +CKPTS/Afteri2i_100kckpt_LLAVAT5_empty_Fixed_just_segment_pretrain_stage3_dreambench_recap_400k_t2i_400k_flux_200k_midjourney_150k_recovery_150k_3e4_epoch2_no_rotated/002-GPT-XL/checkpoints/eval_step_48000/batch_0_cfg_7.5_topk_16384.jpg filter=lfs diff=lfs merge=lfs -text +CKPTS/Afteri2i_100kckpt_LLAVAT5_empty_Fixed_just_segment_pretrain_stage3_dreambench_recap_400k_t2i_400k_flux_200k_midjourney_150k_recovery_150k_3e4_epoch2_no_rotated/002-GPT-XL/checkpoints/eval_step_48000/batch_1_cfg_7.5_topk_16384.jpg filter=lfs diff=lfs merge=lfs -text +CKPTS/Afteri2i_100kckpt_LLAVAT5_empty_Fixed_just_segment_pretrain_stage3_dreambench_recap_400k_t2i_400k_flux_200k_midjourney_150k_recovery_150k_3e4_epoch2_no_rotated/002-GPT-XL/checkpoints/eval_step_48000/batch_2_cfg_7.5_topk_16384.jpg filter=lfs diff=lfs merge=lfs -text +CKPTS/Afteri2i_100kckpt_LLAVAT5_empty_Fixed_just_segment_pretrain_stage3_dreambench_recap_400k_t2i_400k_flux_200k_midjourney_150k_recovery_150k_3e4_epoch2_no_rotated/002-GPT-XL/checkpoints/eval_step_48000/batch_3_cfg_7.5_topk_16384.jpg filter=lfs diff=lfs merge=lfs -text +CKPTS/Afteri2i_100kckpt_LLAVAT5_empty_Fixed_just_segment_pretrain_stage3_dreambench_recap_400k_t2i_400k_flux_200k_midjourney_150k_recovery_150k_3e4_epoch2_no_rotated/002-GPT-XL/checkpoints/eval_step_50000/batch_0_cfg_7.5_topk_16384.jpg filter=lfs diff=lfs merge=lfs -text +CKPTS/Afteri2i_100kckpt_LLAVAT5_empty_Fixed_just_segment_pretrain_stage3_dreambench_recap_400k_t2i_400k_flux_200k_midjourney_150k_recovery_150k_3e4_epoch2_no_rotated/002-GPT-XL/checkpoints/eval_step_50000/batch_1_cfg_7.5_topk_16384.jpg filter=lfs diff=lfs merge=lfs -text +CKPTS/Afteri2i_100kckpt_LLAVAT5_empty_Fixed_just_segment_pretrain_stage3_dreambench_recap_400k_t2i_400k_flux_200k_midjourney_150k_recovery_150k_3e4_epoch2_no_rotated/002-GPT-XL/checkpoints/eval_step_50000/batch_2_cfg_7.5_topk_16384.jpg filter=lfs diff=lfs merge=lfs -text +CKPTS/Afteri2i_100kckpt_LLAVAT5_empty_Fixed_just_segment_pretrain_stage3_dreambench_recap_400k_t2i_400k_flux_200k_midjourney_150k_recovery_150k_3e4_epoch2_no_rotated/002-GPT-XL/checkpoints/eval_step_50000/batch_3_cfg_7.5_topk_16384.jpg filter=lfs diff=lfs merge=lfs -text +CKPTS/Afteri2i_100kckpt_LLAVAT5_empty_Fixed_just_segment_pretrain_stage3_dreambench_recap_400k_t2i_400k_flux_200k_midjourney_150k_recovery_150k_3e4_epoch2_no_rotated/002-GPT-XL/checkpoints/eval_step_52000/batch_0_cfg_7.5_topk_16384.jpg filter=lfs diff=lfs merge=lfs -text +CKPTS/Afteri2i_100kckpt_LLAVAT5_empty_Fixed_just_segment_pretrain_stage3_dreambench_recap_400k_t2i_400k_flux_200k_midjourney_150k_recovery_150k_3e4_epoch2_no_rotated/002-GPT-XL/checkpoints/eval_step_52000/batch_1_cfg_7.5_topk_16384.jpg filter=lfs diff=lfs merge=lfs -text +CKPTS/Afteri2i_100kckpt_LLAVAT5_empty_Fixed_just_segment_pretrain_stage3_dreambench_recap_400k_t2i_400k_flux_200k_midjourney_150k_recovery_150k_3e4_epoch2_no_rotated/002-GPT-XL/checkpoints/eval_step_52000/batch_2_cfg_7.5_topk_16384.jpg filter=lfs diff=lfs merge=lfs -text +CKPTS/Afteri2i_100kckpt_LLAVAT5_empty_Fixed_just_segment_pretrain_stage3_dreambench_recap_400k_t2i_400k_flux_200k_midjourney_150k_recovery_150k_3e4_epoch2_no_rotated/002-GPT-XL/checkpoints/eval_step_52000/batch_3_cfg_7.5_topk_16384.jpg filter=lfs diff=lfs merge=lfs -text +CKPTS/Afteri2i_100kckpt_LLAVAT5_empty_Fixed_just_segment_pretrain_stage3_dreambench_recap_400k_t2i_400k_flux_200k_midjourney_150k_recovery_150k_3e4_epoch2_no_rotated/002-GPT-XL/checkpoints/eval_step_54000/batch_0_cfg_7.5_topk_16384.jpg filter=lfs diff=lfs merge=lfs -text +CKPTS/Afteri2i_100kckpt_LLAVAT5_empty_Fixed_just_segment_pretrain_stage3_dreambench_recap_400k_t2i_400k_flux_200k_midjourney_150k_recovery_150k_3e4_epoch2_no_rotated/002-GPT-XL/checkpoints/eval_step_54000/batch_1_cfg_7.5_topk_16384.jpg filter=lfs diff=lfs merge=lfs -text +CKPTS/Afteri2i_100kckpt_LLAVAT5_empty_Fixed_just_segment_pretrain_stage3_dreambench_recap_400k_t2i_400k_flux_200k_midjourney_150k_recovery_150k_3e4_epoch2_no_rotated/002-GPT-XL/checkpoints/eval_step_54000/batch_2_cfg_7.5_topk_16384.jpg filter=lfs diff=lfs merge=lfs -text +CKPTS/Afteri2i_100kckpt_LLAVAT5_empty_Fixed_just_segment_pretrain_stage3_dreambench_recap_400k_t2i_400k_flux_200k_midjourney_150k_recovery_150k_3e4_epoch2_no_rotated/002-GPT-XL/checkpoints/eval_step_54000/batch_3_cfg_7.5_topk_16384.jpg filter=lfs diff=lfs merge=lfs -text +CKPTS/Afteri2i_100kckpt_LLAVAT5_empty_Fixed_just_segment_pretrain_stage3_dreambench_recap_400k_t2i_400k_flux_200k_midjourney_150k_recovery_150k_3e4_epoch2_no_rotated/002-GPT-XL/checkpoints/eval_step_56000/batch_0_cfg_7.5_topk_16384.jpg filter=lfs diff=lfs merge=lfs -text +CKPTS/Afteri2i_100kckpt_LLAVAT5_empty_Fixed_just_segment_pretrain_stage3_dreambench_recap_400k_t2i_400k_flux_200k_midjourney_150k_recovery_150k_3e4_epoch2_no_rotated/002-GPT-XL/checkpoints/eval_step_56000/batch_1_cfg_7.5_topk_16384.jpg filter=lfs diff=lfs merge=lfs -text +CKPTS/Afteri2i_100kckpt_LLAVAT5_empty_Fixed_just_segment_pretrain_stage3_dreambench_recap_400k_t2i_400k_flux_200k_midjourney_150k_recovery_150k_3e4_epoch2_no_rotated/002-GPT-XL/checkpoints/eval_step_56000/batch_2_cfg_7.5_topk_16384.jpg filter=lfs diff=lfs merge=lfs -text +CKPTS/Afteri2i_100kckpt_LLAVAT5_empty_Fixed_just_segment_pretrain_stage3_dreambench_recap_400k_t2i_400k_flux_200k_midjourney_150k_recovery_150k_3e4_epoch2_no_rotated/002-GPT-XL/checkpoints/eval_step_56000/batch_3_cfg_7.5_topk_16384.jpg filter=lfs diff=lfs merge=lfs -text +CKPTS/Afteri2i_100kckpt_LLAVAT5_empty_Fixed_just_segment_pretrain_stage3_dreambench_recap_400k_t2i_400k_flux_200k_midjourney_150k_recovery_150k_3e4_epoch2_no_rotated/002-GPT-XL/checkpoints/eval_step_58000/batch_0_cfg_7.5_topk_16384.jpg filter=lfs diff=lfs merge=lfs -text +CKPTS/Afteri2i_100kckpt_LLAVAT5_empty_Fixed_just_segment_pretrain_stage3_dreambench_recap_400k_t2i_400k_flux_200k_midjourney_150k_recovery_150k_3e4_epoch2_no_rotated/002-GPT-XL/checkpoints/eval_step_58000/batch_1_cfg_7.5_topk_16384.jpg filter=lfs diff=lfs merge=lfs -text +CKPTS/Afteri2i_100kckpt_LLAVAT5_empty_Fixed_just_segment_pretrain_stage3_dreambench_recap_400k_t2i_400k_flux_200k_midjourney_150k_recovery_150k_3e4_epoch2_no_rotated/002-GPT-XL/checkpoints/eval_step_58000/batch_2_cfg_7.5_topk_16384.jpg filter=lfs diff=lfs merge=lfs -text +CKPTS/Afteri2i_100kckpt_LLAVAT5_empty_Fixed_just_segment_pretrain_stage3_dreambench_recap_400k_t2i_400k_flux_200k_midjourney_150k_recovery_150k_3e4_epoch2_no_rotated/002-GPT-XL/checkpoints/eval_step_58000/batch_3_cfg_7.5_topk_16384.jpg filter=lfs diff=lfs merge=lfs -text +CKPTS/Afteri2i_100kckpt_LLAVAT5_empty_Fixed_just_segment_pretrain_stage3_dreambench_recap_400k_t2i_400k_flux_200k_midjourney_150k_recovery_150k_3e4_epoch2_no_rotated/002-GPT-XL/checkpoints/eval_step_60000/batch_0_cfg_7.5_topk_16384.jpg filter=lfs diff=lfs merge=lfs -text +CKPTS/Afteri2i_100kckpt_LLAVAT5_empty_Fixed_just_segment_pretrain_stage3_dreambench_recap_400k_t2i_400k_flux_200k_midjourney_150k_recovery_150k_3e4_epoch2_no_rotated/002-GPT-XL/checkpoints/eval_step_60000/batch_1_cfg_7.5_topk_16384.jpg filter=lfs diff=lfs merge=lfs -text +CKPTS/Afteri2i_100kckpt_LLAVAT5_empty_Fixed_just_segment_pretrain_stage3_dreambench_recap_400k_t2i_400k_flux_200k_midjourney_150k_recovery_150k_3e4_epoch2_no_rotated/002-GPT-XL/checkpoints/eval_step_60000/batch_2_cfg_7.5_topk_16384.jpg filter=lfs diff=lfs merge=lfs -text +CKPTS/Afteri2i_100kckpt_LLAVAT5_empty_Fixed_just_segment_pretrain_stage3_dreambench_recap_400k_t2i_400k_flux_200k_midjourney_150k_recovery_150k_3e4_epoch2_no_rotated/002-GPT-XL/checkpoints/eval_step_60000/batch_3_cfg_7.5_topk_16384.jpg filter=lfs diff=lfs merge=lfs -text +CKPTS/Afteri2i_100kckpt_LLAVAT5_empty_Fixed_just_segment_pretrain_stage3_dreambench_recap_400k_t2i_400k_flux_200k_midjourney_150k_recovery_150k_3e4_epoch2_no_rotated/002-GPT-XL/checkpoints/eval_step_62000/batch_0_cfg_7.5_topk_16384.jpg filter=lfs diff=lfs merge=lfs -text +CKPTS/Afteri2i_100kckpt_LLAVAT5_empty_Fixed_just_segment_pretrain_stage3_dreambench_recap_400k_t2i_400k_flux_200k_midjourney_150k_recovery_150k_3e4_epoch2_no_rotated/002-GPT-XL/checkpoints/eval_step_62000/batch_1_cfg_7.5_topk_16384.jpg filter=lfs diff=lfs merge=lfs -text +CKPTS/Afteri2i_100kckpt_LLAVAT5_empty_Fixed_just_segment_pretrain_stage3_dreambench_recap_400k_t2i_400k_flux_200k_midjourney_150k_recovery_150k_3e4_epoch2_no_rotated/002-GPT-XL/checkpoints/eval_step_62000/batch_2_cfg_7.5_topk_16384.jpg filter=lfs diff=lfs merge=lfs -text +CKPTS/Afteri2i_100kckpt_LLAVAT5_empty_Fixed_just_segment_pretrain_stage3_dreambench_recap_400k_t2i_400k_flux_200k_midjourney_150k_recovery_150k_3e4_epoch2_no_rotated/002-GPT-XL/checkpoints/eval_step_62000/batch_3_cfg_7.5_topk_16384.jpg filter=lfs diff=lfs merge=lfs -text +CKPTS/Afteri2i_100kckpt_LLAVAT5_empty_Fixed_just_segment_pretrain_stage3_dreambench_recap_400k_t2i_400k_flux_200k_midjourney_150k_recovery_150k_3e4_epoch2_no_rotated/002-GPT-XL/checkpoints/eval_step_64000/batch_0_cfg_7.5_topk_16384.jpg filter=lfs diff=lfs merge=lfs -text +CKPTS/Afteri2i_100kckpt_LLAVAT5_empty_Fixed_just_segment_pretrain_stage3_dreambench_recap_400k_t2i_400k_flux_200k_midjourney_150k_recovery_150k_3e4_epoch2_no_rotated/002-GPT-XL/checkpoints/eval_step_64000/batch_1_cfg_7.5_topk_16384.jpg filter=lfs diff=lfs merge=lfs -text +CKPTS/Afteri2i_100kckpt_LLAVAT5_empty_Fixed_just_segment_pretrain_stage3_dreambench_recap_400k_t2i_400k_flux_200k_midjourney_150k_recovery_150k_3e4_epoch2_no_rotated/002-GPT-XL/checkpoints/eval_step_64000/batch_2_cfg_7.5_topk_16384.jpg filter=lfs diff=lfs merge=lfs -text +CKPTS/Afteri2i_100kckpt_LLAVAT5_empty_Fixed_just_segment_pretrain_stage3_dreambench_recap_400k_t2i_400k_flux_200k_midjourney_150k_recovery_150k_3e4_epoch2_no_rotated/002-GPT-XL/checkpoints/eval_step_64000/batch_3_cfg_7.5_topk_16384.jpg filter=lfs diff=lfs merge=lfs -text +CKPTS/Afteri2i_100kckpt_LLAVAT5_empty_Fixed_just_segment_pretrain_stage3_dreambench_recap_400k_t2i_400k_flux_200k_midjourney_150k_recovery_150k_3e4_epoch2_no_rotated/002-GPT-XL/checkpoints/eval_step_66000/batch_0_cfg_7.5_topk_16384.jpg filter=lfs diff=lfs merge=lfs -text +CKPTS/Afteri2i_100kckpt_LLAVAT5_empty_Fixed_just_segment_pretrain_stage3_dreambench_recap_400k_t2i_400k_flux_200k_midjourney_150k_recovery_150k_3e4_epoch2_no_rotated/002-GPT-XL/checkpoints/eval_step_66000/batch_1_cfg_7.5_topk_16384.jpg filter=lfs diff=lfs merge=lfs -text +CKPTS/Afteri2i_100kckpt_LLAVAT5_empty_Fixed_just_segment_pretrain_stage3_dreambench_recap_400k_t2i_400k_flux_200k_midjourney_150k_recovery_150k_3e4_epoch2_no_rotated/002-GPT-XL/checkpoints/eval_step_66000/batch_2_cfg_7.5_topk_16384.jpg filter=lfs diff=lfs merge=lfs -text +CKPTS/Afteri2i_100kckpt_LLAVAT5_empty_Fixed_just_segment_pretrain_stage3_dreambench_recap_400k_t2i_400k_flux_200k_midjourney_150k_recovery_150k_3e4_epoch2_no_rotated/002-GPT-XL/checkpoints/eval_step_66000/batch_3_cfg_7.5_topk_16384.jpg filter=lfs diff=lfs merge=lfs -text +CKPTS/Afteri2i_100kckpt_LLAVAT5_empty_Fixed_just_segment_pretrain_stage3_dreambench_recap_400k_t2i_400k_flux_200k_midjourney_150k_recovery_150k_3e4_epoch2_no_rotated/002-GPT-XL/checkpoints/eval_step_68000/batch_0_cfg_7.5_topk_16384.jpg filter=lfs diff=lfs merge=lfs -text +CKPTS/Afteri2i_100kckpt_LLAVAT5_empty_Fixed_just_segment_pretrain_stage3_dreambench_recap_400k_t2i_400k_flux_200k_midjourney_150k_recovery_150k_3e4_epoch2_no_rotated/002-GPT-XL/checkpoints/eval_step_68000/batch_1_cfg_7.5_topk_16384.jpg filter=lfs diff=lfs merge=lfs -text +CKPTS/Afteri2i_100kckpt_LLAVAT5_empty_Fixed_just_segment_pretrain_stage3_dreambench_recap_400k_t2i_400k_flux_200k_midjourney_150k_recovery_150k_3e4_epoch2_no_rotated/002-GPT-XL/checkpoints/eval_step_68000/batch_2_cfg_7.5_topk_16384.jpg filter=lfs diff=lfs merge=lfs -text +CKPTS/Afteri2i_100kckpt_LLAVAT5_empty_Fixed_just_segment_pretrain_stage3_dreambench_recap_400k_t2i_400k_flux_200k_midjourney_150k_recovery_150k_3e4_epoch2_no_rotated/002-GPT-XL/checkpoints/eval_step_68000/batch_3_cfg_7.5_topk_16384.jpg filter=lfs diff=lfs merge=lfs -text +CKPTS/Afteri2i_100kckpt_LLAVAT5_empty_Fixed_just_segment_pretrain_stage3_dreambench_recap_400k_t2i_400k_flux_200k_midjourney_150k_recovery_150k_3e4_epoch2_no_rotated/002-GPT-XL/checkpoints/eval_step_70000/batch_0_cfg_7.5_topk_16384.jpg filter=lfs diff=lfs merge=lfs -text +CKPTS/Afteri2i_100kckpt_LLAVAT5_empty_Fixed_just_segment_pretrain_stage3_dreambench_recap_400k_t2i_400k_flux_200k_midjourney_150k_recovery_150k_3e4_epoch2_no_rotated/002-GPT-XL/checkpoints/eval_step_70000/batch_1_cfg_7.5_topk_16384.jpg filter=lfs diff=lfs merge=lfs -text +CKPTS/Afteri2i_100kckpt_LLAVAT5_empty_Fixed_just_segment_pretrain_stage3_dreambench_recap_400k_t2i_400k_flux_200k_midjourney_150k_recovery_150k_3e4_epoch2_no_rotated/002-GPT-XL/checkpoints/eval_step_70000/batch_2_cfg_7.5_topk_16384.jpg filter=lfs diff=lfs merge=lfs -text +CKPTS/Afteri2i_100kckpt_LLAVAT5_empty_Fixed_just_segment_pretrain_stage3_dreambench_recap_400k_t2i_400k_flux_200k_midjourney_150k_recovery_150k_3e4_epoch2_no_rotated/002-GPT-XL/checkpoints/eval_step_70000/batch_3_cfg_7.5_topk_16384.jpg filter=lfs diff=lfs merge=lfs -text +CKPTS/Afteri2i_100kckpt_LLAVAT5_empty_Fixed_just_segment_pretrain_stage3_dreambench_recap_400k_t2i_400k_flux_200k_midjourney_150k_recovery_150k_3e4_epoch2_no_rotated/003-GPT-XL/checkpoints/eval_step_72000/batch_0_cfg_7.5_topk_16384.jpg filter=lfs diff=lfs merge=lfs -text +CKPTS/Afteri2i_100kckpt_LLAVAT5_empty_Fixed_just_segment_pretrain_stage3_dreambench_recap_400k_t2i_400k_flux_200k_midjourney_150k_recovery_150k_3e4_epoch2_no_rotated/003-GPT-XL/checkpoints/eval_step_72000/batch_1_cfg_7.5_topk_16384.jpg filter=lfs diff=lfs merge=lfs -text +CKPTS/Afteri2i_100kckpt_LLAVAT5_empty_Fixed_just_segment_pretrain_stage3_dreambench_recap_400k_t2i_400k_flux_200k_midjourney_150k_recovery_150k_3e4_epoch2_no_rotated/003-GPT-XL/checkpoints/eval_step_72000/batch_2_cfg_7.5_topk_16384.jpg filter=lfs diff=lfs merge=lfs -text +CKPTS/Afteri2i_100kckpt_LLAVAT5_empty_Fixed_just_segment_pretrain_stage3_dreambench_recap_400k_t2i_400k_flux_200k_midjourney_150k_recovery_150k_3e4_epoch2_no_rotated/003-GPT-XL/checkpoints/eval_step_72000/batch_3_cfg_7.5_topk_16384.jpg filter=lfs diff=lfs merge=lfs -text +CKPTS/Afteri2i_100kckpt_LLAVAT5_empty_Fixed_just_segment_pretrain_stage3_dreambench_recap_400k_t2i_400k_flux_200k_midjourney_150k_recovery_150k_3e4_epoch2_no_rotated/003-GPT-XL/checkpoints/eval_step_74000/batch_0_cfg_7.5_topk_16384.jpg filter=lfs diff=lfs merge=lfs -text +CKPTS/Afteri2i_100kckpt_LLAVAT5_empty_Fixed_just_segment_pretrain_stage3_dreambench_recap_400k_t2i_400k_flux_200k_midjourney_150k_recovery_150k_3e4_epoch2_no_rotated/003-GPT-XL/checkpoints/eval_step_74000/batch_1_cfg_7.5_topk_16384.jpg filter=lfs diff=lfs merge=lfs -text +CKPTS/Afteri2i_100kckpt_LLAVAT5_empty_Fixed_just_segment_pretrain_stage3_dreambench_recap_400k_t2i_400k_flux_200k_midjourney_150k_recovery_150k_3e4_epoch2_no_rotated/003-GPT-XL/checkpoints/eval_step_74000/batch_2_cfg_7.5_topk_16384.jpg filter=lfs diff=lfs merge=lfs -text +CKPTS/Afteri2i_100kckpt_LLAVAT5_empty_Fixed_just_segment_pretrain_stage3_dreambench_recap_400k_t2i_400k_flux_200k_midjourney_150k_recovery_150k_3e4_epoch2_no_rotated/003-GPT-XL/checkpoints/eval_step_74000/batch_3_cfg_7.5_topk_16384.jpg filter=lfs diff=lfs merge=lfs -text +CKPTS/Afteri2i_100kckpt_LLAVAT5_empty_Fixed_just_segment_pretrain_stage3_dreambench_recap_400k_t2i_400k_flux_200k_midjourney_150k_recovery_150k_3e4_epoch2_no_rotated/003-GPT-XL/checkpoints/eval_step_76000/batch_0_cfg_7.5_topk_16384.jpg filter=lfs diff=lfs merge=lfs -text +CKPTS/Afteri2i_100kckpt_LLAVAT5_empty_Fixed_just_segment_pretrain_stage3_dreambench_recap_400k_t2i_400k_flux_200k_midjourney_150k_recovery_150k_3e4_epoch2_no_rotated/003-GPT-XL/checkpoints/eval_step_76000/batch_1_cfg_7.5_topk_16384.jpg filter=lfs diff=lfs merge=lfs -text +CKPTS/Afteri2i_100kckpt_LLAVAT5_empty_Fixed_just_segment_pretrain_stage3_dreambench_recap_400k_t2i_400k_flux_200k_midjourney_150k_recovery_150k_3e4_epoch2_no_rotated/003-GPT-XL/checkpoints/eval_step_76000/batch_2_cfg_7.5_topk_16384.jpg filter=lfs diff=lfs merge=lfs -text +CKPTS/Afteri2i_100kckpt_LLAVAT5_empty_Fixed_just_segment_pretrain_stage3_dreambench_recap_400k_t2i_400k_flux_200k_midjourney_150k_recovery_150k_3e4_epoch2_no_rotated/003-GPT-XL/checkpoints/eval_step_76000/batch_3_cfg_7.5_topk_16384.jpg filter=lfs diff=lfs merge=lfs -text +CKPTS/Afteri2i_100kckpt_LLAVAT5_empty_Fixed_just_segment_pretrain_stage3_dreambench_recap_400k_t2i_400k_flux_200k_midjourney_150k_recovery_150k_3e4_epoch2_no_rotated/003-GPT-XL/checkpoints/eval_step_78000/batch_0_cfg_7.5_topk_16384.jpg filter=lfs diff=lfs merge=lfs -text +CKPTS/Afteri2i_100kckpt_LLAVAT5_empty_Fixed_just_segment_pretrain_stage3_dreambench_recap_400k_t2i_400k_flux_200k_midjourney_150k_recovery_150k_3e4_epoch2_no_rotated/003-GPT-XL/checkpoints/eval_step_78000/batch_1_cfg_7.5_topk_16384.jpg filter=lfs diff=lfs merge=lfs -text +CKPTS/Afteri2i_100kckpt_LLAVAT5_empty_Fixed_just_segment_pretrain_stage3_dreambench_recap_400k_t2i_400k_flux_200k_midjourney_150k_recovery_150k_3e4_epoch2_no_rotated/003-GPT-XL/checkpoints/eval_step_78000/batch_2_cfg_7.5_topk_16384.jpg filter=lfs diff=lfs merge=lfs -text +CKPTS/Afteri2i_100kckpt_LLAVAT5_empty_Fixed_just_segment_pretrain_stage3_dreambench_recap_400k_t2i_400k_flux_200k_midjourney_150k_recovery_150k_3e4_epoch2_no_rotated/003-GPT-XL/checkpoints/eval_step_78000/batch_3_cfg_7.5_topk_16384.jpg filter=lfs diff=lfs merge=lfs -text +CKPTS/Afteri2i_100kckpt_LLAVAT5_empty_Fixed_just_segment_pretrain_stage3_dreambench_recap_400k_t2i_400k_flux_200k_midjourney_150k_recovery_150k_3e4_epoch2_no_rotated/003-GPT-XL/checkpoints/eval_step_80000/batch_0_cfg_7.5_topk_16384.jpg filter=lfs diff=lfs merge=lfs -text +CKPTS/Afteri2i_100kckpt_LLAVAT5_empty_Fixed_just_segment_pretrain_stage3_dreambench_recap_400k_t2i_400k_flux_200k_midjourney_150k_recovery_150k_3e4_epoch2_no_rotated/003-GPT-XL/checkpoints/eval_step_80000/batch_1_cfg_7.5_topk_16384.jpg filter=lfs diff=lfs merge=lfs -text +CKPTS/Afteri2i_100kckpt_LLAVAT5_empty_Fixed_just_segment_pretrain_stage3_dreambench_recap_400k_t2i_400k_flux_200k_midjourney_150k_recovery_150k_3e4_epoch2_no_rotated/003-GPT-XL/checkpoints/eval_step_80000/batch_2_cfg_7.5_topk_16384.jpg filter=lfs diff=lfs merge=lfs -text +CKPTS/Afteri2i_100kckpt_LLAVAT5_empty_Fixed_just_segment_pretrain_stage3_dreambench_recap_400k_t2i_400k_flux_200k_midjourney_150k_recovery_150k_3e4_epoch2_no_rotated/003-GPT-XL/checkpoints/eval_step_80000/batch_3_cfg_7.5_topk_16384.jpg filter=lfs diff=lfs merge=lfs -text +CKPTS/Afteri2i_100kckpt_LLAVAT5_empty_Fixed_just_segment_pretrain_stage3_dreambench_recap_400k_t2i_400k_flux_200k_midjourney_150k_recovery_150k_3e4_epoch2_no_rotated/003-GPT-XL/checkpoints/eval_step_82000/batch_0_cfg_7.5_topk_16384.jpg filter=lfs diff=lfs merge=lfs -text +CKPTS/Afteri2i_100kckpt_LLAVAT5_empty_Fixed_just_segment_pretrain_stage3_dreambench_recap_400k_t2i_400k_flux_200k_midjourney_150k_recovery_150k_3e4_epoch2_no_rotated/003-GPT-XL/checkpoints/eval_step_82000/batch_1_cfg_7.5_topk_16384.jpg filter=lfs diff=lfs merge=lfs -text +CKPTS/Afteri2i_100kckpt_LLAVAT5_empty_Fixed_just_segment_pretrain_stage3_dreambench_recap_400k_t2i_400k_flux_200k_midjourney_150k_recovery_150k_3e4_epoch2_no_rotated/003-GPT-XL/checkpoints/eval_step_82000/batch_2_cfg_7.5_topk_16384.jpg filter=lfs diff=lfs merge=lfs -text +CKPTS/Afteri2i_100kckpt_LLAVAT5_empty_Fixed_just_segment_pretrain_stage3_dreambench_recap_400k_t2i_400k_flux_200k_midjourney_150k_recovery_150k_3e4_epoch2_no_rotated/003-GPT-XL/checkpoints/eval_step_82000/batch_3_cfg_7.5_topk_16384.jpg filter=lfs diff=lfs merge=lfs -text +CKPTS/Afteri2i_100kckpt_LLAVAT5_empty_Fixed_just_segment_pretrain_stage3_dreambench_recap_400k_t2i_400k_flux_200k_midjourney_150k_recovery_150k_3e4_epoch2_no_rotated/003-GPT-XL/checkpoints/eval_step_84000/batch_0_cfg_7.5_topk_16384.jpg filter=lfs diff=lfs merge=lfs -text +CKPTS/Afteri2i_100kckpt_LLAVAT5_empty_Fixed_just_segment_pretrain_stage3_dreambench_recap_400k_t2i_400k_flux_200k_midjourney_150k_recovery_150k_3e4_epoch2_no_rotated/003-GPT-XL/checkpoints/eval_step_84000/batch_1_cfg_7.5_topk_16384.jpg filter=lfs diff=lfs merge=lfs -text +CKPTS/Afteri2i_100kckpt_LLAVAT5_empty_Fixed_just_segment_pretrain_stage3_dreambench_recap_400k_t2i_400k_flux_200k_midjourney_150k_recovery_150k_3e4_epoch2_no_rotated/003-GPT-XL/checkpoints/eval_step_84000/batch_2_cfg_7.5_topk_16384.jpg filter=lfs diff=lfs merge=lfs -text +CKPTS/Afteri2i_100kckpt_LLAVAT5_empty_Fixed_just_segment_pretrain_stage3_dreambench_recap_400k_t2i_400k_flux_200k_midjourney_150k_recovery_150k_3e4_epoch2_no_rotated/003-GPT-XL/checkpoints/eval_step_84000/batch_3_cfg_7.5_topk_16384.jpg filter=lfs diff=lfs merge=lfs -text +CKPTS/Afteri2i_100kckpt_LLAVAT5_empty_Fixed_just_segment_pretrain_stage3_dreambench_recap_400k_t2i_400k_flux_200k_midjourney_150k_recovery_150k_3e4_epoch2_no_rotated/003-GPT-XL/checkpoints/eval_step_86000/batch_0_cfg_7.5_topk_16384.jpg filter=lfs diff=lfs merge=lfs -text +CKPTS/Afteri2i_100kckpt_LLAVAT5_empty_Fixed_just_segment_pretrain_stage3_dreambench_recap_400k_t2i_400k_flux_200k_midjourney_150k_recovery_150k_3e4_epoch2_no_rotated/003-GPT-XL/checkpoints/eval_step_86000/batch_1_cfg_7.5_topk_16384.jpg filter=lfs diff=lfs merge=lfs -text +CKPTS/Afteri2i_100kckpt_LLAVAT5_empty_Fixed_just_segment_pretrain_stage3_dreambench_recap_400k_t2i_400k_flux_200k_midjourney_150k_recovery_150k_3e4_epoch2_no_rotated/003-GPT-XL/checkpoints/eval_step_86000/batch_2_cfg_7.5_topk_16384.jpg filter=lfs diff=lfs merge=lfs -text +CKPTS/Afteri2i_100kckpt_LLAVAT5_empty_Fixed_just_segment_pretrain_stage3_dreambench_recap_400k_t2i_400k_flux_200k_midjourney_150k_recovery_150k_3e4_epoch2_no_rotated/003-GPT-XL/checkpoints/eval_step_86000/batch_3_cfg_7.5_topk_16384.jpg filter=lfs diff=lfs merge=lfs -text +CKPTS/Afteri2i_100kckpt_LLAVAT5_empty_Fixed_just_segment_pretrain_stage3_dreambench_recap_400k_t2i_400k_flux_200k_midjourney_150k_recovery_150k_3e4_epoch2_no_rotated/003-GPT-XL/checkpoints/eval_step_88000/batch_0_cfg_7.5_topk_16384.jpg filter=lfs diff=lfs merge=lfs -text +CKPTS/Afteri2i_100kckpt_LLAVAT5_empty_Fixed_just_segment_pretrain_stage3_dreambench_recap_400k_t2i_400k_flux_200k_midjourney_150k_recovery_150k_3e4_epoch2_no_rotated/003-GPT-XL/checkpoints/eval_step_88000/batch_1_cfg_7.5_topk_16384.jpg filter=lfs diff=lfs merge=lfs -text +CKPTS/Afteri2i_100kckpt_LLAVAT5_empty_Fixed_just_segment_pretrain_stage3_dreambench_recap_400k_t2i_400k_flux_200k_midjourney_150k_recovery_150k_3e4_epoch2_no_rotated/003-GPT-XL/checkpoints/eval_step_88000/batch_2_cfg_7.5_topk_16384.jpg filter=lfs diff=lfs merge=lfs -text +CKPTS/Afteri2i_100kckpt_LLAVAT5_empty_Fixed_just_segment_pretrain_stage3_dreambench_recap_400k_t2i_400k_flux_200k_midjourney_150k_recovery_150k_3e4_epoch2_no_rotated/003-GPT-XL/checkpoints/eval_step_88000/batch_3_cfg_7.5_topk_16384.jpg filter=lfs diff=lfs merge=lfs -text +CKPTS/Afteri2i_100kckpt_LLAVAT5_empty_Fixed_just_segment_pretrain_stage3_dreambench_recap_400k_t2i_400k_flux_200k_midjourney_150k_recovery_150k_3e4_epoch2_no_rotated/003-GPT-XL/checkpoints/eval_step_90000/batch_0_cfg_7.5_topk_16384.jpg filter=lfs diff=lfs merge=lfs -text +CKPTS/Afteri2i_100kckpt_LLAVAT5_empty_Fixed_just_segment_pretrain_stage3_dreambench_recap_400k_t2i_400k_flux_200k_midjourney_150k_recovery_150k_3e4_epoch2_no_rotated/003-GPT-XL/checkpoints/eval_step_90000/batch_1_cfg_7.5_topk_16384.jpg filter=lfs diff=lfs merge=lfs -text +CKPTS/Afteri2i_100kckpt_LLAVAT5_empty_Fixed_just_segment_pretrain_stage3_dreambench_recap_400k_t2i_400k_flux_200k_midjourney_150k_recovery_150k_3e4_epoch2_no_rotated/003-GPT-XL/checkpoints/eval_step_90000/batch_2_cfg_7.5_topk_16384.jpg filter=lfs diff=lfs merge=lfs -text +CKPTS/Afteri2i_100kckpt_LLAVAT5_empty_Fixed_just_segment_pretrain_stage3_dreambench_recap_400k_t2i_400k_flux_200k_midjourney_150k_recovery_150k_3e4_epoch2_no_rotated/003-GPT-XL/checkpoints/eval_step_90000/batch_3_cfg_7.5_topk_16384.jpg filter=lfs diff=lfs merge=lfs -text +CKPTS/Afteri2i_100kckpt_LLAVAT5_empty_Fixed_just_segment_pretrain_stage3_dreambench_recap_400k_t2i_400k_flux_200k_midjourney_150k_recovery_150k_3e4_epoch2_no_rotated/003-GPT-XL/checkpoints/eval_step_92000/batch_0_cfg_7.5_topk_16384.jpg filter=lfs diff=lfs merge=lfs -text +CKPTS/Afteri2i_100kckpt_LLAVAT5_empty_Fixed_just_segment_pretrain_stage3_dreambench_recap_400k_t2i_400k_flux_200k_midjourney_150k_recovery_150k_3e4_epoch2_no_rotated/003-GPT-XL/checkpoints/eval_step_92000/batch_1_cfg_7.5_topk_16384.jpg filter=lfs diff=lfs merge=lfs -text +CKPTS/Afteri2i_100kckpt_LLAVAT5_empty_Fixed_just_segment_pretrain_stage3_dreambench_recap_400k_t2i_400k_flux_200k_midjourney_150k_recovery_150k_3e4_epoch2_no_rotated/003-GPT-XL/checkpoints/eval_step_92000/batch_2_cfg_7.5_topk_16384.jpg filter=lfs diff=lfs merge=lfs -text +CKPTS/Afteri2i_100kckpt_LLAVAT5_empty_Fixed_just_segment_pretrain_stage3_dreambench_recap_400k_t2i_400k_flux_200k_midjourney_150k_recovery_150k_3e4_epoch2_no_rotated/003-GPT-XL/checkpoints/eval_step_92000/batch_3_cfg_7.5_topk_16384.jpg filter=lfs diff=lfs merge=lfs -text +CKPTS/Afteri2i_100kckpt_LLAVAT5_empty_Fixed_just_segment_pretrain_stage3_dreambench_recap_400k_t2i_400k_flux_200k_midjourney_150k_recovery_150k_3e4_epoch2_no_rotated/wandb/run-20250424_021033-arc66v3w/run-arc66v3w.wandb filter=lfs diff=lfs merge=lfs -text +CKPTS/Afteri2i_100kckpt_LLAVAT5_empty_Fixed_just_segment_pretrain_stage3_dreambench_recap_400k_t2i_400k_flux_200k_midjourney_150k_recovery_150k_3e4_epoch2_no_rotated/wandb/run-20250425_060734-amr5cnvk/run-amr5cnvk.wandb filter=lfs diff=lfs merge=lfs -text +CKPTS/Afteri2i_100kckpt_LLAVAT5_empty_Fixed_just_segment_pretrain_stage3_dreambench_recap_400k_t2i_400k_flux_200k_midjourney_150k_recovery_150k_3e4_epoch2_no_rotated/wandb/run-20250426_080403-2j21z9h0/run-2j21z9h0.wandb filter=lfs diff=lfs merge=lfs -text +CKPTS/BeforeFix_Blip_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_objects/002-GPT-XL/checkpoints/eval_step_10000/batch_0_cfg_7.5_topk_16384.jpg filter=lfs diff=lfs merge=lfs -text +CKPTS/BeforeFix_Blip_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_objects/002-GPT-XL/checkpoints/eval_step_10000/batch_1_cfg_7.5_topk_16384.jpg filter=lfs diff=lfs merge=lfs -text +CKPTS/BeforeFix_Blip_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_objects/002-GPT-XL/checkpoints/eval_step_10000/batch_2_cfg_7.5_topk_16384.jpg filter=lfs diff=lfs merge=lfs -text +CKPTS/BeforeFix_Blip_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_objects/002-GPT-XL/checkpoints/eval_step_12000/batch_0_cfg_7.5_topk_16384.jpg filter=lfs diff=lfs merge=lfs -text +CKPTS/BeforeFix_Blip_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_objects/002-GPT-XL/checkpoints/eval_step_12000/batch_1_cfg_7.5_topk_16384.jpg filter=lfs diff=lfs merge=lfs -text +CKPTS/BeforeFix_Blip_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_objects/002-GPT-XL/checkpoints/eval_step_12000/batch_2_cfg_7.5_topk_16384.jpg filter=lfs diff=lfs merge=lfs -text +CKPTS/BeforeFix_Blip_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_objects/002-GPT-XL/checkpoints/eval_step_14000/batch_0_cfg_7.5_topk_16384.jpg filter=lfs diff=lfs merge=lfs -text +CKPTS/BeforeFix_Blip_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_objects/002-GPT-XL/checkpoints/eval_step_14000/batch_1_cfg_7.5_topk_16384.jpg filter=lfs diff=lfs merge=lfs -text +CKPTS/BeforeFix_Blip_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_objects/002-GPT-XL/checkpoints/eval_step_14000/batch_2_cfg_7.5_topk_16384.jpg filter=lfs diff=lfs merge=lfs -text +CKPTS/BeforeFix_Blip_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_objects/002-GPT-XL/checkpoints/eval_step_16000/batch_0_cfg_7.5_topk_16384.jpg filter=lfs diff=lfs merge=lfs -text +CKPTS/BeforeFix_Blip_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_objects/002-GPT-XL/checkpoints/eval_step_16000/batch_1_cfg_7.5_topk_16384.jpg filter=lfs diff=lfs merge=lfs -text +CKPTS/BeforeFix_Blip_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_objects/002-GPT-XL/checkpoints/eval_step_16000/batch_2_cfg_7.5_topk_16384.jpg filter=lfs diff=lfs merge=lfs -text +CKPTS/BeforeFix_Blip_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_objects/002-GPT-XL/checkpoints/eval_step_18000/batch_0_cfg_7.5_topk_16384.jpg filter=lfs diff=lfs merge=lfs -text +CKPTS/BeforeFix_Blip_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_objects/002-GPT-XL/checkpoints/eval_step_18000/batch_1_cfg_7.5_topk_16384.jpg filter=lfs diff=lfs merge=lfs -text +CKPTS/BeforeFix_Blip_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_objects/002-GPT-XL/checkpoints/eval_step_18000/batch_2_cfg_7.5_topk_16384.jpg filter=lfs diff=lfs merge=lfs -text +CKPTS/BeforeFix_Blip_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_objects/002-GPT-XL/checkpoints/eval_step_2000/batch_0_cfg_7.5_topk_16384.jpg filter=lfs diff=lfs merge=lfs -text +CKPTS/BeforeFix_Blip_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_objects/002-GPT-XL/checkpoints/eval_step_2000/batch_1_cfg_7.5_topk_16384.jpg filter=lfs diff=lfs merge=lfs -text +CKPTS/BeforeFix_Blip_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_objects/002-GPT-XL/checkpoints/eval_step_2000/batch_2_cfg_7.5_topk_16384.jpg filter=lfs diff=lfs merge=lfs -text +CKPTS/BeforeFix_Blip_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_objects/002-GPT-XL/checkpoints/eval_step_20000/batch_0_cfg_7.5_topk_16384.jpg filter=lfs diff=lfs merge=lfs -text +CKPTS/BeforeFix_Blip_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_objects/002-GPT-XL/checkpoints/eval_step_20000/batch_1_cfg_7.5_topk_16384.jpg filter=lfs diff=lfs merge=lfs -text +CKPTS/BeforeFix_Blip_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_objects/002-GPT-XL/checkpoints/eval_step_20000/batch_2_cfg_7.5_topk_16384.jpg filter=lfs diff=lfs merge=lfs -text +CKPTS/BeforeFix_Blip_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_objects/002-GPT-XL/checkpoints/eval_step_22000/batch_0_cfg_7.5_topk_16384.jpg filter=lfs diff=lfs merge=lfs -text +CKPTS/BeforeFix_Blip_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_objects/002-GPT-XL/checkpoints/eval_step_22000/batch_1_cfg_7.5_topk_16384.jpg filter=lfs diff=lfs merge=lfs -text +CKPTS/BeforeFix_Blip_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_objects/002-GPT-XL/checkpoints/eval_step_22000/batch_2_cfg_7.5_topk_16384.jpg filter=lfs diff=lfs merge=lfs -text +CKPTS/BeforeFix_Blip_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_objects/002-GPT-XL/checkpoints/eval_step_24000/batch_0_cfg_7.5_topk_16384.jpg filter=lfs diff=lfs merge=lfs -text +CKPTS/BeforeFix_Blip_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_objects/002-GPT-XL/checkpoints/eval_step_24000/batch_1_cfg_7.5_topk_16384.jpg filter=lfs diff=lfs merge=lfs -text +CKPTS/BeforeFix_Blip_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_objects/002-GPT-XL/checkpoints/eval_step_24000/batch_2_cfg_7.5_topk_16384.jpg filter=lfs diff=lfs merge=lfs -text +CKPTS/BeforeFix_Blip_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_objects/002-GPT-XL/checkpoints/eval_step_26000/batch_0_cfg_7.5_topk_16384.jpg filter=lfs diff=lfs merge=lfs -text +CKPTS/BeforeFix_Blip_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_objects/002-GPT-XL/checkpoints/eval_step_26000/batch_1_cfg_7.5_topk_16384.jpg filter=lfs diff=lfs merge=lfs -text +CKPTS/BeforeFix_Blip_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_objects/002-GPT-XL/checkpoints/eval_step_26000/batch_2_cfg_7.5_topk_16384.jpg filter=lfs diff=lfs merge=lfs -text +CKPTS/BeforeFix_Blip_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_objects/002-GPT-XL/checkpoints/eval_step_28000/batch_0_cfg_7.5_topk_16384.jpg filter=lfs diff=lfs merge=lfs -text +CKPTS/BeforeFix_Blip_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_objects/002-GPT-XL/checkpoints/eval_step_28000/batch_1_cfg_7.5_topk_16384.jpg filter=lfs diff=lfs merge=lfs -text +CKPTS/BeforeFix_Blip_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_objects/002-GPT-XL/checkpoints/eval_step_28000/batch_2_cfg_7.5_topk_16384.jpg filter=lfs diff=lfs merge=lfs -text +CKPTS/BeforeFix_Blip_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_objects/002-GPT-XL/checkpoints/eval_step_30000/batch_0_cfg_7.5_topk_16384.jpg filter=lfs diff=lfs merge=lfs -text +CKPTS/BeforeFix_Blip_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_objects/002-GPT-XL/checkpoints/eval_step_30000/batch_1_cfg_7.5_topk_16384.jpg filter=lfs diff=lfs merge=lfs -text +CKPTS/BeforeFix_Blip_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_objects/002-GPT-XL/checkpoints/eval_step_30000/batch_2_cfg_7.5_topk_16384.jpg filter=lfs diff=lfs merge=lfs -text +CKPTS/BeforeFix_Blip_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_objects/002-GPT-XL/checkpoints/eval_step_32000/batch_0_cfg_7.5_topk_16384.jpg filter=lfs diff=lfs merge=lfs -text +CKPTS/BeforeFix_Blip_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_objects/002-GPT-XL/checkpoints/eval_step_32000/batch_1_cfg_7.5_topk_16384.jpg filter=lfs diff=lfs merge=lfs -text +CKPTS/BeforeFix_Blip_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_objects/002-GPT-XL/checkpoints/eval_step_32000/batch_2_cfg_7.5_topk_16384.jpg filter=lfs diff=lfs merge=lfs -text +CKPTS/BeforeFix_Blip_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_objects/002-GPT-XL/checkpoints/eval_step_34000/batch_0_cfg_7.5_topk_16384.jpg filter=lfs diff=lfs merge=lfs -text +CKPTS/BeforeFix_Blip_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_objects/002-GPT-XL/checkpoints/eval_step_34000/batch_1_cfg_7.5_topk_16384.jpg filter=lfs diff=lfs merge=lfs -text +CKPTS/BeforeFix_Blip_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_objects/002-GPT-XL/checkpoints/eval_step_34000/batch_2_cfg_7.5_topk_16384.jpg filter=lfs diff=lfs merge=lfs -text +CKPTS/BeforeFix_Blip_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_objects/002-GPT-XL/checkpoints/eval_step_36000/batch_0_cfg_7.5_topk_16384.jpg filter=lfs diff=lfs merge=lfs -text +CKPTS/BeforeFix_Blip_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_objects/002-GPT-XL/checkpoints/eval_step_36000/batch_1_cfg_7.5_topk_16384.jpg filter=lfs diff=lfs merge=lfs -text +CKPTS/BeforeFix_Blip_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_objects/002-GPT-XL/checkpoints/eval_step_36000/batch_2_cfg_7.5_topk_16384.jpg filter=lfs diff=lfs merge=lfs -text +CKPTS/BeforeFix_Blip_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_objects/002-GPT-XL/checkpoints/eval_step_38000/batch_0_cfg_7.5_topk_16384.jpg filter=lfs diff=lfs merge=lfs -text +CKPTS/BeforeFix_Blip_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_objects/002-GPT-XL/checkpoints/eval_step_38000/batch_1_cfg_7.5_topk_16384.jpg filter=lfs diff=lfs merge=lfs -text +CKPTS/BeforeFix_Blip_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_objects/002-GPT-XL/checkpoints/eval_step_38000/batch_2_cfg_7.5_topk_16384.jpg filter=lfs diff=lfs merge=lfs -text +CKPTS/BeforeFix_Blip_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_objects/002-GPT-XL/checkpoints/eval_step_4000/batch_0_cfg_7.5_topk_16384.jpg filter=lfs diff=lfs merge=lfs -text +CKPTS/BeforeFix_Blip_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_objects/002-GPT-XL/checkpoints/eval_step_4000/batch_1_cfg_7.5_topk_16384.jpg filter=lfs diff=lfs merge=lfs -text +CKPTS/BeforeFix_Blip_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_objects/002-GPT-XL/checkpoints/eval_step_4000/batch_2_cfg_7.5_topk_16384.jpg filter=lfs diff=lfs merge=lfs -text +CKPTS/BeforeFix_Blip_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_objects/002-GPT-XL/checkpoints/eval_step_40000/batch_0_cfg_7.5_topk_16384.jpg filter=lfs diff=lfs merge=lfs -text +CKPTS/BeforeFix_Blip_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_objects/002-GPT-XL/checkpoints/eval_step_40000/batch_1_cfg_7.5_topk_16384.jpg filter=lfs diff=lfs merge=lfs -text +CKPTS/BeforeFix_Blip_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_objects/002-GPT-XL/checkpoints/eval_step_40000/batch_2_cfg_7.5_topk_16384.jpg filter=lfs diff=lfs merge=lfs -text +CKPTS/BeforeFix_Blip_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_objects/002-GPT-XL/checkpoints/eval_step_42000/batch_0_cfg_7.5_topk_16384.jpg filter=lfs diff=lfs merge=lfs -text +CKPTS/BeforeFix_Blip_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_objects/002-GPT-XL/checkpoints/eval_step_42000/batch_1_cfg_7.5_topk_16384.jpg filter=lfs diff=lfs merge=lfs -text +CKPTS/BeforeFix_Blip_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_objects/002-GPT-XL/checkpoints/eval_step_42000/batch_2_cfg_7.5_topk_16384.jpg filter=lfs diff=lfs merge=lfs -text +CKPTS/BeforeFix_Blip_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_objects/002-GPT-XL/checkpoints/eval_step_44000/batch_0_cfg_7.5_topk_16384.jpg filter=lfs diff=lfs merge=lfs -text +CKPTS/BeforeFix_Blip_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_objects/002-GPT-XL/checkpoints/eval_step_44000/batch_1_cfg_7.5_topk_16384.jpg filter=lfs diff=lfs merge=lfs -text +CKPTS/BeforeFix_Blip_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_objects/002-GPT-XL/checkpoints/eval_step_44000/batch_2_cfg_7.5_topk_16384.jpg filter=lfs diff=lfs merge=lfs -text +CKPTS/BeforeFix_Blip_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_objects/002-GPT-XL/checkpoints/eval_step_46000/batch_0_cfg_7.5_topk_16384.jpg filter=lfs diff=lfs merge=lfs -text +CKPTS/BeforeFix_Blip_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_objects/002-GPT-XL/checkpoints/eval_step_46000/batch_1_cfg_7.5_topk_16384.jpg filter=lfs diff=lfs merge=lfs -text +CKPTS/BeforeFix_Blip_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_objects/002-GPT-XL/checkpoints/eval_step_46000/batch_2_cfg_7.5_topk_16384.jpg filter=lfs diff=lfs merge=lfs -text +CKPTS/BeforeFix_Blip_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_objects/002-GPT-XL/checkpoints/eval_step_48000/batch_0_cfg_7.5_topk_16384.jpg filter=lfs diff=lfs merge=lfs -text +CKPTS/BeforeFix_Blip_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_objects/002-GPT-XL/checkpoints/eval_step_48000/batch_1_cfg_7.5_topk_16384.jpg filter=lfs diff=lfs merge=lfs -text +CKPTS/BeforeFix_Blip_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_objects/002-GPT-XL/checkpoints/eval_step_48000/batch_2_cfg_7.5_topk_16384.jpg filter=lfs diff=lfs merge=lfs -text +CKPTS/BeforeFix_Blip_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_objects/002-GPT-XL/checkpoints/eval_step_50000/batch_0_cfg_7.5_topk_16384.jpg filter=lfs diff=lfs merge=lfs -text +CKPTS/BeforeFix_Blip_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_objects/002-GPT-XL/checkpoints/eval_step_50000/batch_1_cfg_7.5_topk_16384.jpg filter=lfs diff=lfs merge=lfs -text +CKPTS/BeforeFix_Blip_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_objects/002-GPT-XL/checkpoints/eval_step_50000/batch_2_cfg_7.5_topk_16384.jpg filter=lfs diff=lfs merge=lfs -text +CKPTS/BeforeFix_Blip_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_objects/002-GPT-XL/checkpoints/eval_step_52000/batch_0_cfg_7.5_topk_16384.jpg filter=lfs diff=lfs merge=lfs -text +CKPTS/BeforeFix_Blip_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_objects/002-GPT-XL/checkpoints/eval_step_52000/batch_1_cfg_7.5_topk_16384.jpg filter=lfs diff=lfs merge=lfs -text +CKPTS/BeforeFix_Blip_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_objects/002-GPT-XL/checkpoints/eval_step_52000/batch_2_cfg_7.5_topk_16384.jpg filter=lfs diff=lfs merge=lfs -text +CKPTS/BeforeFix_Blip_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_objects/002-GPT-XL/checkpoints/eval_step_54000/batch_0_cfg_7.5_topk_16384.jpg filter=lfs diff=lfs merge=lfs -text +CKPTS/BeforeFix_Blip_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_objects/002-GPT-XL/checkpoints/eval_step_54000/batch_1_cfg_7.5_topk_16384.jpg filter=lfs diff=lfs merge=lfs -text +CKPTS/BeforeFix_Blip_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_objects/002-GPT-XL/checkpoints/eval_step_54000/batch_2_cfg_7.5_topk_16384.jpg filter=lfs diff=lfs merge=lfs -text +CKPTS/BeforeFix_Blip_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_objects/002-GPT-XL/checkpoints/eval_step_56000/batch_0_cfg_7.5_topk_16384.jpg filter=lfs diff=lfs merge=lfs -text +CKPTS/BeforeFix_Blip_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_objects/002-GPT-XL/checkpoints/eval_step_56000/batch_1_cfg_7.5_topk_16384.jpg filter=lfs diff=lfs merge=lfs -text +CKPTS/BeforeFix_Blip_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_objects/002-GPT-XL/checkpoints/eval_step_56000/batch_2_cfg_7.5_topk_16384.jpg filter=lfs diff=lfs merge=lfs -text +CKPTS/BeforeFix_Blip_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_objects/002-GPT-XL/checkpoints/eval_step_58000/batch_0_cfg_7.5_topk_16384.jpg filter=lfs diff=lfs merge=lfs -text +CKPTS/BeforeFix_Blip_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_objects/002-GPT-XL/checkpoints/eval_step_58000/batch_1_cfg_7.5_topk_16384.jpg filter=lfs diff=lfs merge=lfs -text +CKPTS/BeforeFix_Blip_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_objects/002-GPT-XL/checkpoints/eval_step_58000/batch_2_cfg_7.5_topk_16384.jpg filter=lfs diff=lfs merge=lfs -text +CKPTS/BeforeFix_Blip_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_objects/002-GPT-XL/checkpoints/eval_step_6000/batch_0_cfg_7.5_topk_16384.jpg filter=lfs diff=lfs merge=lfs -text +CKPTS/BeforeFix_Blip_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_objects/002-GPT-XL/checkpoints/eval_step_6000/batch_1_cfg_7.5_topk_16384.jpg filter=lfs diff=lfs merge=lfs -text +CKPTS/BeforeFix_Blip_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_objects/002-GPT-XL/checkpoints/eval_step_6000/batch_2_cfg_7.5_topk_16384.jpg filter=lfs diff=lfs merge=lfs -text +CKPTS/BeforeFix_Blip_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_objects/002-GPT-XL/checkpoints/eval_step_60000/batch_0_cfg_7.5_topk_16384.jpg filter=lfs diff=lfs merge=lfs -text +CKPTS/BeforeFix_Blip_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_objects/002-GPT-XL/checkpoints/eval_step_60000/batch_1_cfg_7.5_topk_16384.jpg filter=lfs diff=lfs merge=lfs -text +CKPTS/BeforeFix_Blip_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_objects/002-GPT-XL/checkpoints/eval_step_60000/batch_2_cfg_7.5_topk_16384.jpg filter=lfs diff=lfs merge=lfs -text +CKPTS/BeforeFix_Blip_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_objects/002-GPT-XL/checkpoints/eval_step_62000/batch_0_cfg_7.5_topk_16384.jpg filter=lfs diff=lfs merge=lfs -text +CKPTS/BeforeFix_Blip_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_objects/002-GPT-XL/checkpoints/eval_step_62000/batch_1_cfg_7.5_topk_16384.jpg filter=lfs diff=lfs merge=lfs -text +CKPTS/BeforeFix_Blip_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_objects/002-GPT-XL/checkpoints/eval_step_62000/batch_2_cfg_7.5_topk_16384.jpg filter=lfs diff=lfs merge=lfs -text +CKPTS/BeforeFix_Blip_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_objects/002-GPT-XL/checkpoints/eval_step_64000/batch_0_cfg_7.5_topk_16384.jpg filter=lfs diff=lfs merge=lfs -text +CKPTS/BeforeFix_Blip_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_objects/002-GPT-XL/checkpoints/eval_step_64000/batch_1_cfg_7.5_topk_16384.jpg filter=lfs diff=lfs merge=lfs -text +CKPTS/BeforeFix_Blip_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_objects/002-GPT-XL/checkpoints/eval_step_64000/batch_2_cfg_7.5_topk_16384.jpg filter=lfs diff=lfs merge=lfs -text +CKPTS/BeforeFix_Blip_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_objects/002-GPT-XL/checkpoints/eval_step_66000/batch_0_cfg_7.5_topk_16384.jpg filter=lfs diff=lfs merge=lfs -text +CKPTS/BeforeFix_Blip_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_objects/002-GPT-XL/checkpoints/eval_step_66000/batch_1_cfg_7.5_topk_16384.jpg filter=lfs diff=lfs merge=lfs -text +CKPTS/BeforeFix_Blip_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_objects/002-GPT-XL/checkpoints/eval_step_66000/batch_2_cfg_7.5_topk_16384.jpg filter=lfs diff=lfs merge=lfs -text +CKPTS/BeforeFix_Blip_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_objects/002-GPT-XL/checkpoints/eval_step_68000/batch_0_cfg_7.5_topk_16384.jpg filter=lfs diff=lfs merge=lfs -text +CKPTS/BeforeFix_Blip_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_objects/002-GPT-XL/checkpoints/eval_step_68000/batch_1_cfg_7.5_topk_16384.jpg filter=lfs diff=lfs merge=lfs -text +CKPTS/BeforeFix_Blip_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_objects/002-GPT-XL/checkpoints/eval_step_68000/batch_2_cfg_7.5_topk_16384.jpg filter=lfs diff=lfs merge=lfs -text +CKPTS/BeforeFix_Blip_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_objects/002-GPT-XL/checkpoints/eval_step_70000/batch_0_cfg_7.5_topk_16384.jpg filter=lfs diff=lfs merge=lfs -text +CKPTS/BeforeFix_Blip_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_objects/002-GPT-XL/checkpoints/eval_step_70000/batch_1_cfg_7.5_topk_16384.jpg filter=lfs diff=lfs merge=lfs -text +CKPTS/BeforeFix_Blip_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_objects/002-GPT-XL/checkpoints/eval_step_70000/batch_2_cfg_7.5_topk_16384.jpg filter=lfs diff=lfs merge=lfs -text +CKPTS/BeforeFix_Blip_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_objects/002-GPT-XL/checkpoints/eval_step_72000/batch_0_cfg_7.5_topk_16384.jpg filter=lfs diff=lfs merge=lfs -text +CKPTS/BeforeFix_Blip_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_objects/002-GPT-XL/checkpoints/eval_step_72000/batch_1_cfg_7.5_topk_16384.jpg filter=lfs diff=lfs merge=lfs -text +CKPTS/BeforeFix_Blip_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_objects/002-GPT-XL/checkpoints/eval_step_72000/batch_2_cfg_7.5_topk_16384.jpg filter=lfs diff=lfs merge=lfs -text +CKPTS/BeforeFix_Blip_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_objects/002-GPT-XL/checkpoints/eval_step_74000/batch_0_cfg_7.5_topk_16384.jpg filter=lfs diff=lfs merge=lfs -text +CKPTS/BeforeFix_Blip_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_objects/002-GPT-XL/checkpoints/eval_step_74000/batch_1_cfg_7.5_topk_16384.jpg filter=lfs diff=lfs merge=lfs -text +CKPTS/BeforeFix_Blip_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_objects/002-GPT-XL/checkpoints/eval_step_74000/batch_2_cfg_7.5_topk_16384.jpg filter=lfs diff=lfs merge=lfs -text +CKPTS/BeforeFix_Blip_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_objects/002-GPT-XL/checkpoints/eval_step_76000/batch_0_cfg_7.5_topk_16384.jpg filter=lfs diff=lfs merge=lfs -text +CKPTS/BeforeFix_Blip_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_objects/002-GPT-XL/checkpoints/eval_step_76000/batch_1_cfg_7.5_topk_16384.jpg filter=lfs diff=lfs merge=lfs -text +CKPTS/BeforeFix_Blip_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_objects/002-GPT-XL/checkpoints/eval_step_76000/batch_2_cfg_7.5_topk_16384.jpg filter=lfs diff=lfs merge=lfs -text +CKPTS/BeforeFix_Blip_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_objects/002-GPT-XL/checkpoints/eval_step_78000/batch_0_cfg_7.5_topk_16384.jpg filter=lfs diff=lfs merge=lfs -text +CKPTS/BeforeFix_Blip_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_objects/002-GPT-XL/checkpoints/eval_step_78000/batch_1_cfg_7.5_topk_16384.jpg filter=lfs diff=lfs merge=lfs -text +CKPTS/BeforeFix_Blip_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_objects/002-GPT-XL/checkpoints/eval_step_78000/batch_2_cfg_7.5_topk_16384.jpg filter=lfs diff=lfs merge=lfs -text +CKPTS/BeforeFix_Blip_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_objects/002-GPT-XL/checkpoints/eval_step_8000/batch_0_cfg_7.5_topk_16384.jpg filter=lfs diff=lfs merge=lfs -text +CKPTS/BeforeFix_Blip_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_objects/002-GPT-XL/checkpoints/eval_step_8000/batch_1_cfg_7.5_topk_16384.jpg filter=lfs diff=lfs merge=lfs -text +CKPTS/BeforeFix_Blip_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_objects/002-GPT-XL/checkpoints/eval_step_8000/batch_2_cfg_7.5_topk_16384.jpg filter=lfs diff=lfs merge=lfs -text +CKPTS/BeforeFix_Blip_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_objects/002-GPT-XL/checkpoints/eval_step_80000/batch_0_cfg_7.5_topk_16384.jpg filter=lfs diff=lfs merge=lfs -text +CKPTS/BeforeFix_Blip_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_objects/002-GPT-XL/checkpoints/eval_step_80000/batch_1_cfg_7.5_topk_16384.jpg filter=lfs diff=lfs merge=lfs -text +CKPTS/BeforeFix_Blip_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_objects/002-GPT-XL/checkpoints/eval_step_80000/batch_2_cfg_7.5_topk_16384.jpg filter=lfs diff=lfs merge=lfs -text +CKPTS/BeforeFix_Blip_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_objects/wandb/run-20250422_014729-ni4jp6ul/run-ni4jp6ul.wandb filter=lfs diff=lfs merge=lfs -text +CKPTS/EmptyFix_InstructBlip_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_objects/002-GPT-XL/checkpoints/eval_step_10000/batch_0_cfg_7.5_topk_16384.jpg filter=lfs diff=lfs merge=lfs -text +CKPTS/EmptyFix_InstructBlip_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_objects/002-GPT-XL/checkpoints/eval_step_10000/batch_1_cfg_7.5_topk_16384.jpg filter=lfs diff=lfs merge=lfs -text +CKPTS/EmptyFix_InstructBlip_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_objects/002-GPT-XL/checkpoints/eval_step_10000/batch_2_cfg_7.5_topk_16384.jpg filter=lfs diff=lfs merge=lfs -text +CKPTS/EmptyFix_InstructBlip_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_objects/002-GPT-XL/checkpoints/eval_step_12000/batch_0_cfg_7.5_topk_16384.jpg filter=lfs diff=lfs merge=lfs -text +CKPTS/EmptyFix_InstructBlip_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_objects/002-GPT-XL/checkpoints/eval_step_12000/batch_1_cfg_7.5_topk_16384.jpg filter=lfs diff=lfs merge=lfs -text +CKPTS/EmptyFix_InstructBlip_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_objects/002-GPT-XL/checkpoints/eval_step_12000/batch_2_cfg_7.5_topk_16384.jpg filter=lfs diff=lfs merge=lfs -text +CKPTS/EmptyFix_InstructBlip_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_objects/002-GPT-XL/checkpoints/eval_step_14000/batch_0_cfg_7.5_topk_16384.jpg filter=lfs diff=lfs merge=lfs -text +CKPTS/EmptyFix_InstructBlip_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_objects/002-GPT-XL/checkpoints/eval_step_14000/batch_1_cfg_7.5_topk_16384.jpg filter=lfs diff=lfs merge=lfs -text +CKPTS/EmptyFix_InstructBlip_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_objects/002-GPT-XL/checkpoints/eval_step_14000/batch_2_cfg_7.5_topk_16384.jpg filter=lfs diff=lfs merge=lfs -text +CKPTS/EmptyFix_InstructBlip_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_objects/002-GPT-XL/checkpoints/eval_step_16000/batch_0_cfg_7.5_topk_16384.jpg filter=lfs diff=lfs merge=lfs -text +CKPTS/EmptyFix_InstructBlip_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_objects/002-GPT-XL/checkpoints/eval_step_16000/batch_1_cfg_7.5_topk_16384.jpg filter=lfs diff=lfs merge=lfs -text +CKPTS/EmptyFix_InstructBlip_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_objects/002-GPT-XL/checkpoints/eval_step_16000/batch_2_cfg_7.5_topk_16384.jpg filter=lfs diff=lfs merge=lfs -text +CKPTS/EmptyFix_InstructBlip_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_objects/002-GPT-XL/checkpoints/eval_step_18000/batch_0_cfg_7.5_topk_16384.jpg filter=lfs diff=lfs merge=lfs -text +CKPTS/EmptyFix_InstructBlip_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_objects/002-GPT-XL/checkpoints/eval_step_18000/batch_1_cfg_7.5_topk_16384.jpg filter=lfs diff=lfs merge=lfs -text +CKPTS/EmptyFix_InstructBlip_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_objects/002-GPT-XL/checkpoints/eval_step_18000/batch_2_cfg_7.5_topk_16384.jpg filter=lfs diff=lfs merge=lfs -text +CKPTS/EmptyFix_InstructBlip_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_objects/002-GPT-XL/checkpoints/eval_step_2000/batch_0_cfg_7.5_topk_16384.jpg filter=lfs diff=lfs merge=lfs -text +CKPTS/EmptyFix_InstructBlip_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_objects/002-GPT-XL/checkpoints/eval_step_2000/batch_1_cfg_7.5_topk_16384.jpg filter=lfs diff=lfs merge=lfs -text +CKPTS/EmptyFix_InstructBlip_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_objects/002-GPT-XL/checkpoints/eval_step_2000/batch_2_cfg_7.5_topk_16384.jpg filter=lfs diff=lfs merge=lfs -text +CKPTS/EmptyFix_InstructBlip_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_objects/002-GPT-XL/checkpoints/eval_step_20000/batch_0_cfg_7.5_topk_16384.jpg filter=lfs diff=lfs merge=lfs -text +CKPTS/EmptyFix_InstructBlip_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_objects/002-GPT-XL/checkpoints/eval_step_20000/batch_1_cfg_7.5_topk_16384.jpg filter=lfs diff=lfs merge=lfs -text +CKPTS/EmptyFix_InstructBlip_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_objects/002-GPT-XL/checkpoints/eval_step_20000/batch_2_cfg_7.5_topk_16384.jpg filter=lfs diff=lfs merge=lfs -text +CKPTS/EmptyFix_InstructBlip_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_objects/002-GPT-XL/checkpoints/eval_step_22000/batch_0_cfg_7.5_topk_16384.jpg filter=lfs diff=lfs merge=lfs -text +CKPTS/EmptyFix_InstructBlip_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_objects/002-GPT-XL/checkpoints/eval_step_22000/batch_1_cfg_7.5_topk_16384.jpg filter=lfs diff=lfs merge=lfs -text +CKPTS/EmptyFix_InstructBlip_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_objects/002-GPT-XL/checkpoints/eval_step_22000/batch_2_cfg_7.5_topk_16384.jpg filter=lfs diff=lfs merge=lfs -text +CKPTS/EmptyFix_InstructBlip_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_objects/002-GPT-XL/checkpoints/eval_step_24000/batch_0_cfg_7.5_topk_16384.jpg filter=lfs diff=lfs merge=lfs -text +CKPTS/EmptyFix_InstructBlip_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_objects/002-GPT-XL/checkpoints/eval_step_24000/batch_1_cfg_7.5_topk_16384.jpg filter=lfs diff=lfs merge=lfs -text +CKPTS/EmptyFix_InstructBlip_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_objects/002-GPT-XL/checkpoints/eval_step_24000/batch_2_cfg_7.5_topk_16384.jpg filter=lfs diff=lfs merge=lfs -text +CKPTS/EmptyFix_InstructBlip_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_objects/002-GPT-XL/checkpoints/eval_step_26000/batch_0_cfg_7.5_topk_16384.jpg filter=lfs diff=lfs merge=lfs -text +CKPTS/EmptyFix_InstructBlip_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_objects/002-GPT-XL/checkpoints/eval_step_26000/batch_1_cfg_7.5_topk_16384.jpg filter=lfs diff=lfs merge=lfs -text +CKPTS/EmptyFix_InstructBlip_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_objects/002-GPT-XL/checkpoints/eval_step_26000/batch_2_cfg_7.5_topk_16384.jpg filter=lfs diff=lfs merge=lfs -text +CKPTS/EmptyFix_InstructBlip_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_objects/002-GPT-XL/checkpoints/eval_step_28000/batch_0_cfg_7.5_topk_16384.jpg filter=lfs diff=lfs merge=lfs -text +CKPTS/EmptyFix_InstructBlip_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_objects/002-GPT-XL/checkpoints/eval_step_28000/batch_1_cfg_7.5_topk_16384.jpg filter=lfs diff=lfs merge=lfs -text +CKPTS/EmptyFix_InstructBlip_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_objects/002-GPT-XL/checkpoints/eval_step_28000/batch_2_cfg_7.5_topk_16384.jpg filter=lfs diff=lfs merge=lfs -text +CKPTS/EmptyFix_InstructBlip_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_objects/002-GPT-XL/checkpoints/eval_step_30000/batch_0_cfg_7.5_topk_16384.jpg filter=lfs diff=lfs merge=lfs -text +CKPTS/EmptyFix_InstructBlip_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_objects/002-GPT-XL/checkpoints/eval_step_30000/batch_1_cfg_7.5_topk_16384.jpg filter=lfs diff=lfs merge=lfs -text +CKPTS/EmptyFix_InstructBlip_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_objects/002-GPT-XL/checkpoints/eval_step_30000/batch_2_cfg_7.5_topk_16384.jpg filter=lfs diff=lfs merge=lfs -text +CKPTS/EmptyFix_InstructBlip_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_objects/002-GPT-XL/checkpoints/eval_step_32000/batch_0_cfg_7.5_topk_16384.jpg filter=lfs diff=lfs merge=lfs -text +CKPTS/EmptyFix_InstructBlip_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_objects/002-GPT-XL/checkpoints/eval_step_32000/batch_1_cfg_7.5_topk_16384.jpg filter=lfs diff=lfs merge=lfs -text +CKPTS/EmptyFix_InstructBlip_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_objects/002-GPT-XL/checkpoints/eval_step_32000/batch_2_cfg_7.5_topk_16384.jpg filter=lfs diff=lfs merge=lfs -text +CKPTS/EmptyFix_InstructBlip_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_objects/002-GPT-XL/checkpoints/eval_step_34000/batch_0_cfg_7.5_topk_16384.jpg filter=lfs diff=lfs merge=lfs -text +CKPTS/EmptyFix_InstructBlip_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_objects/002-GPT-XL/checkpoints/eval_step_34000/batch_1_cfg_7.5_topk_16384.jpg filter=lfs diff=lfs merge=lfs -text +CKPTS/EmptyFix_InstructBlip_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_objects/002-GPT-XL/checkpoints/eval_step_34000/batch_2_cfg_7.5_topk_16384.jpg filter=lfs diff=lfs merge=lfs -text +CKPTS/EmptyFix_InstructBlip_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_objects/002-GPT-XL/checkpoints/eval_step_36000/batch_0_cfg_7.5_topk_16384.jpg filter=lfs diff=lfs merge=lfs -text +CKPTS/EmptyFix_InstructBlip_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_objects/002-GPT-XL/checkpoints/eval_step_36000/batch_1_cfg_7.5_topk_16384.jpg filter=lfs diff=lfs merge=lfs -text +CKPTS/EmptyFix_InstructBlip_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_objects/002-GPT-XL/checkpoints/eval_step_36000/batch_2_cfg_7.5_topk_16384.jpg filter=lfs diff=lfs merge=lfs -text +CKPTS/EmptyFix_InstructBlip_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_objects/002-GPT-XL/checkpoints/eval_step_38000/batch_0_cfg_7.5_topk_16384.jpg filter=lfs diff=lfs merge=lfs -text +CKPTS/EmptyFix_InstructBlip_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_objects/002-GPT-XL/checkpoints/eval_step_38000/batch_1_cfg_7.5_topk_16384.jpg filter=lfs diff=lfs merge=lfs -text +CKPTS/EmptyFix_InstructBlip_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_objects/002-GPT-XL/checkpoints/eval_step_38000/batch_2_cfg_7.5_topk_16384.jpg filter=lfs diff=lfs merge=lfs -text +CKPTS/EmptyFix_InstructBlip_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_objects/002-GPT-XL/checkpoints/eval_step_4000/batch_0_cfg_7.5_topk_16384.jpg filter=lfs diff=lfs merge=lfs -text +CKPTS/EmptyFix_InstructBlip_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_objects/002-GPT-XL/checkpoints/eval_step_4000/batch_1_cfg_7.5_topk_16384.jpg filter=lfs diff=lfs merge=lfs -text +CKPTS/EmptyFix_InstructBlip_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_objects/002-GPT-XL/checkpoints/eval_step_4000/batch_2_cfg_7.5_topk_16384.jpg filter=lfs diff=lfs merge=lfs -text +CKPTS/EmptyFix_InstructBlip_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_objects/002-GPT-XL/checkpoints/eval_step_40000/batch_0_cfg_7.5_topk_16384.jpg filter=lfs diff=lfs merge=lfs -text +CKPTS/EmptyFix_InstructBlip_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_objects/002-GPT-XL/checkpoints/eval_step_40000/batch_1_cfg_7.5_topk_16384.jpg filter=lfs diff=lfs merge=lfs -text +CKPTS/EmptyFix_InstructBlip_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_objects/002-GPT-XL/checkpoints/eval_step_40000/batch_2_cfg_7.5_topk_16384.jpg filter=lfs diff=lfs merge=lfs -text +CKPTS/EmptyFix_InstructBlip_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_objects/002-GPT-XL/checkpoints/eval_step_42000/batch_0_cfg_7.5_topk_16384.jpg filter=lfs diff=lfs merge=lfs -text +CKPTS/EmptyFix_InstructBlip_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_objects/002-GPT-XL/checkpoints/eval_step_42000/batch_1_cfg_7.5_topk_16384.jpg filter=lfs diff=lfs merge=lfs -text +CKPTS/EmptyFix_InstructBlip_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_objects/002-GPT-XL/checkpoints/eval_step_42000/batch_2_cfg_7.5_topk_16384.jpg filter=lfs diff=lfs merge=lfs -text +CKPTS/EmptyFix_InstructBlip_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_objects/002-GPT-XL/checkpoints/eval_step_44000/batch_0_cfg_7.5_topk_16384.jpg filter=lfs diff=lfs merge=lfs -text +CKPTS/EmptyFix_InstructBlip_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_objects/002-GPT-XL/checkpoints/eval_step_44000/batch_1_cfg_7.5_topk_16384.jpg filter=lfs diff=lfs merge=lfs -text +CKPTS/EmptyFix_InstructBlip_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_objects/002-GPT-XL/checkpoints/eval_step_44000/batch_2_cfg_7.5_topk_16384.jpg filter=lfs diff=lfs merge=lfs -text +CKPTS/EmptyFix_InstructBlip_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_objects/002-GPT-XL/checkpoints/eval_step_46000/batch_0_cfg_7.5_topk_16384.jpg filter=lfs diff=lfs merge=lfs -text +CKPTS/EmptyFix_InstructBlip_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_objects/002-GPT-XL/checkpoints/eval_step_46000/batch_1_cfg_7.5_topk_16384.jpg filter=lfs diff=lfs merge=lfs -text +CKPTS/EmptyFix_InstructBlip_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_objects/002-GPT-XL/checkpoints/eval_step_46000/batch_2_cfg_7.5_topk_16384.jpg filter=lfs diff=lfs merge=lfs -text +CKPTS/EmptyFix_InstructBlip_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_objects/002-GPT-XL/checkpoints/eval_step_48000/batch_0_cfg_7.5_topk_16384.jpg filter=lfs diff=lfs merge=lfs -text +CKPTS/EmptyFix_InstructBlip_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_objects/002-GPT-XL/checkpoints/eval_step_48000/batch_1_cfg_7.5_topk_16384.jpg filter=lfs diff=lfs merge=lfs -text +CKPTS/EmptyFix_InstructBlip_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_objects/002-GPT-XL/checkpoints/eval_step_48000/batch_2_cfg_7.5_topk_16384.jpg filter=lfs diff=lfs merge=lfs -text +CKPTS/EmptyFix_InstructBlip_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_objects/002-GPT-XL/checkpoints/eval_step_50000/batch_0_cfg_7.5_topk_16384.jpg filter=lfs diff=lfs merge=lfs -text +CKPTS/EmptyFix_InstructBlip_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_objects/002-GPT-XL/checkpoints/eval_step_50000/batch_1_cfg_7.5_topk_16384.jpg filter=lfs diff=lfs merge=lfs -text +CKPTS/EmptyFix_InstructBlip_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_objects/002-GPT-XL/checkpoints/eval_step_50000/batch_2_cfg_7.5_topk_16384.jpg filter=lfs diff=lfs merge=lfs -text +CKPTS/EmptyFix_InstructBlip_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_objects/002-GPT-XL/checkpoints/eval_step_52000/batch_0_cfg_7.5_topk_16384.jpg filter=lfs diff=lfs merge=lfs -text +CKPTS/EmptyFix_InstructBlip_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_objects/002-GPT-XL/checkpoints/eval_step_52000/batch_1_cfg_7.5_topk_16384.jpg filter=lfs diff=lfs merge=lfs -text +CKPTS/EmptyFix_InstructBlip_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_objects/002-GPT-XL/checkpoints/eval_step_52000/batch_2_cfg_7.5_topk_16384.jpg filter=lfs diff=lfs merge=lfs -text +CKPTS/EmptyFix_InstructBlip_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_objects/002-GPT-XL/checkpoints/eval_step_54000/batch_0_cfg_7.5_topk_16384.jpg filter=lfs diff=lfs merge=lfs -text +CKPTS/EmptyFix_InstructBlip_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_objects/002-GPT-XL/checkpoints/eval_step_54000/batch_1_cfg_7.5_topk_16384.jpg filter=lfs diff=lfs merge=lfs -text +CKPTS/EmptyFix_InstructBlip_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_objects/002-GPT-XL/checkpoints/eval_step_54000/batch_2_cfg_7.5_topk_16384.jpg filter=lfs diff=lfs merge=lfs -text +CKPTS/EmptyFix_InstructBlip_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_objects/002-GPT-XL/checkpoints/eval_step_56000/batch_0_cfg_7.5_topk_16384.jpg filter=lfs diff=lfs merge=lfs -text +CKPTS/EmptyFix_InstructBlip_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_objects/002-GPT-XL/checkpoints/eval_step_56000/batch_1_cfg_7.5_topk_16384.jpg filter=lfs diff=lfs merge=lfs -text +CKPTS/EmptyFix_InstructBlip_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_objects/002-GPT-XL/checkpoints/eval_step_56000/batch_2_cfg_7.5_topk_16384.jpg filter=lfs diff=lfs merge=lfs -text +CKPTS/EmptyFix_InstructBlip_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_objects/002-GPT-XL/checkpoints/eval_step_58000/batch_0_cfg_7.5_topk_16384.jpg filter=lfs diff=lfs merge=lfs -text +CKPTS/EmptyFix_InstructBlip_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_objects/002-GPT-XL/checkpoints/eval_step_58000/batch_1_cfg_7.5_topk_16384.jpg filter=lfs diff=lfs merge=lfs -text +CKPTS/EmptyFix_InstructBlip_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_objects/002-GPT-XL/checkpoints/eval_step_58000/batch_2_cfg_7.5_topk_16384.jpg filter=lfs diff=lfs merge=lfs -text +CKPTS/EmptyFix_InstructBlip_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_objects/002-GPT-XL/checkpoints/eval_step_6000/batch_0_cfg_7.5_topk_16384.jpg filter=lfs diff=lfs merge=lfs -text +CKPTS/EmptyFix_InstructBlip_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_objects/002-GPT-XL/checkpoints/eval_step_6000/batch_1_cfg_7.5_topk_16384.jpg filter=lfs diff=lfs merge=lfs -text +CKPTS/EmptyFix_InstructBlip_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_objects/002-GPT-XL/checkpoints/eval_step_6000/batch_2_cfg_7.5_topk_16384.jpg filter=lfs diff=lfs merge=lfs -text +CKPTS/EmptyFix_InstructBlip_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_objects/002-GPT-XL/checkpoints/eval_step_60000/batch_0_cfg_7.5_topk_16384.jpg filter=lfs diff=lfs merge=lfs -text +CKPTS/EmptyFix_InstructBlip_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_objects/002-GPT-XL/checkpoints/eval_step_60000/batch_1_cfg_7.5_topk_16384.jpg filter=lfs diff=lfs merge=lfs -text +CKPTS/EmptyFix_InstructBlip_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_objects/002-GPT-XL/checkpoints/eval_step_60000/batch_2_cfg_7.5_topk_16384.jpg filter=lfs diff=lfs merge=lfs -text +CKPTS/EmptyFix_InstructBlip_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_objects/002-GPT-XL/checkpoints/eval_step_8000/batch_0_cfg_7.5_topk_16384.jpg filter=lfs diff=lfs merge=lfs -text +CKPTS/EmptyFix_InstructBlip_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_objects/002-GPT-XL/checkpoints/eval_step_8000/batch_1_cfg_7.5_topk_16384.jpg filter=lfs diff=lfs merge=lfs -text +CKPTS/EmptyFix_InstructBlip_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_objects/002-GPT-XL/checkpoints/eval_step_8000/batch_2_cfg_7.5_topk_16384.jpg filter=lfs diff=lfs merge=lfs -text +CKPTS/EmptyFix_InstructBlip_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_objects/wandb/run-20250420_165633-wzvwagyn/run-wzvwagyn.wandb filter=lfs diff=lfs merge=lfs -text +CKPTS/X2I_700k_3mask_200k_LLava_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_filtered_objects_3mask_888_context/003-GPT-XL/checkpoints/eval_step_10000/batch_0_cfg_7.5_topk_16384.jpg filter=lfs diff=lfs merge=lfs -text +CKPTS/X2I_700k_3mask_200k_LLava_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_filtered_objects_3mask_888_context/003-GPT-XL/checkpoints/eval_step_10000/batch_1_cfg_7.5_topk_16384.jpg filter=lfs diff=lfs merge=lfs -text +CKPTS/X2I_700k_3mask_200k_LLava_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_filtered_objects_3mask_888_context/003-GPT-XL/checkpoints/eval_step_10000/batch_2_cfg_7.5_topk_16384.jpg filter=lfs diff=lfs merge=lfs -text +CKPTS/X2I_700k_3mask_200k_LLava_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_filtered_objects_3mask_888_context/003-GPT-XL/checkpoints/eval_step_10000/batch_3_cfg_7.5_topk_16384.jpg filter=lfs diff=lfs merge=lfs -text +CKPTS/X2I_700k_3mask_200k_LLava_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_filtered_objects_3mask_888_context/003-GPT-XL/checkpoints/eval_step_10000/batch_4_cfg_7.5_topk_16384.jpg filter=lfs diff=lfs merge=lfs -text +CKPTS/X2I_700k_3mask_200k_LLava_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_filtered_objects_3mask_888_context/003-GPT-XL/checkpoints/eval_step_12000/batch_0_cfg_7.5_topk_16384.jpg filter=lfs diff=lfs merge=lfs -text +CKPTS/X2I_700k_3mask_200k_LLava_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_filtered_objects_3mask_888_context/003-GPT-XL/checkpoints/eval_step_12000/batch_1_cfg_7.5_topk_16384.jpg filter=lfs diff=lfs merge=lfs -text +CKPTS/X2I_700k_3mask_200k_LLava_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_filtered_objects_3mask_888_context/003-GPT-XL/checkpoints/eval_step_12000/batch_2_cfg_7.5_topk_16384.jpg filter=lfs diff=lfs merge=lfs -text +CKPTS/X2I_700k_3mask_200k_LLava_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_filtered_objects_3mask_888_context/003-GPT-XL/checkpoints/eval_step_12000/batch_3_cfg_7.5_topk_16384.jpg filter=lfs diff=lfs merge=lfs -text +CKPTS/X2I_700k_3mask_200k_LLava_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_filtered_objects_3mask_888_context/003-GPT-XL/checkpoints/eval_step_12000/batch_4_cfg_7.5_topk_16384.jpg filter=lfs diff=lfs merge=lfs -text +CKPTS/X2I_700k_3mask_200k_LLava_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_filtered_objects_3mask_888_context/003-GPT-XL/checkpoints/eval_step_14000/batch_0_cfg_7.5_topk_16384.jpg filter=lfs diff=lfs merge=lfs -text +CKPTS/X2I_700k_3mask_200k_LLava_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_filtered_objects_3mask_888_context/003-GPT-XL/checkpoints/eval_step_14000/batch_1_cfg_7.5_topk_16384.jpg filter=lfs diff=lfs merge=lfs -text +CKPTS/X2I_700k_3mask_200k_LLava_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_filtered_objects_3mask_888_context/003-GPT-XL/checkpoints/eval_step_14000/batch_2_cfg_7.5_topk_16384.jpg filter=lfs diff=lfs merge=lfs -text +CKPTS/X2I_700k_3mask_200k_LLava_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_filtered_objects_3mask_888_context/003-GPT-XL/checkpoints/eval_step_14000/batch_3_cfg_7.5_topk_16384.jpg filter=lfs diff=lfs merge=lfs -text +CKPTS/X2I_700k_3mask_200k_LLava_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_filtered_objects_3mask_888_context/003-GPT-XL/checkpoints/eval_step_14000/batch_4_cfg_7.5_topk_16384.jpg filter=lfs diff=lfs merge=lfs -text +CKPTS/X2I_700k_3mask_200k_LLava_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_filtered_objects_3mask_888_context/003-GPT-XL/checkpoints/eval_step_16000/batch_0_cfg_7.5_topk_16384.jpg filter=lfs diff=lfs merge=lfs -text +CKPTS/X2I_700k_3mask_200k_LLava_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_filtered_objects_3mask_888_context/003-GPT-XL/checkpoints/eval_step_16000/batch_1_cfg_7.5_topk_16384.jpg filter=lfs diff=lfs merge=lfs -text +CKPTS/X2I_700k_3mask_200k_LLava_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_filtered_objects_3mask_888_context/003-GPT-XL/checkpoints/eval_step_16000/batch_2_cfg_7.5_topk_16384.jpg filter=lfs diff=lfs merge=lfs -text +CKPTS/X2I_700k_3mask_200k_LLava_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_filtered_objects_3mask_888_context/003-GPT-XL/checkpoints/eval_step_16000/batch_3_cfg_7.5_topk_16384.jpg filter=lfs diff=lfs merge=lfs -text +CKPTS/X2I_700k_3mask_200k_LLava_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_filtered_objects_3mask_888_context/003-GPT-XL/checkpoints/eval_step_16000/batch_4_cfg_7.5_topk_16384.jpg filter=lfs diff=lfs merge=lfs -text +CKPTS/X2I_700k_3mask_200k_LLava_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_filtered_objects_3mask_888_context/003-GPT-XL/checkpoints/eval_step_18000/batch_0_cfg_7.5_topk_16384.jpg filter=lfs diff=lfs merge=lfs -text +CKPTS/X2I_700k_3mask_200k_LLava_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_filtered_objects_3mask_888_context/003-GPT-XL/checkpoints/eval_step_18000/batch_1_cfg_7.5_topk_16384.jpg filter=lfs diff=lfs merge=lfs -text +CKPTS/X2I_700k_3mask_200k_LLava_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_filtered_objects_3mask_888_context/003-GPT-XL/checkpoints/eval_step_18000/batch_2_cfg_7.5_topk_16384.jpg filter=lfs diff=lfs merge=lfs -text +CKPTS/X2I_700k_3mask_200k_LLava_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_filtered_objects_3mask_888_context/003-GPT-XL/checkpoints/eval_step_18000/batch_3_cfg_7.5_topk_16384.jpg filter=lfs diff=lfs merge=lfs -text +CKPTS/X2I_700k_3mask_200k_LLava_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_filtered_objects_3mask_888_context/003-GPT-XL/checkpoints/eval_step_18000/batch_4_cfg_7.5_topk_16384.jpg filter=lfs diff=lfs merge=lfs -text +CKPTS/X2I_700k_3mask_200k_LLava_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_filtered_objects_3mask_888_context/003-GPT-XL/checkpoints/eval_step_2000/batch_0_cfg_7.5_topk_16384.jpg filter=lfs diff=lfs merge=lfs -text +CKPTS/X2I_700k_3mask_200k_LLava_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_filtered_objects_3mask_888_context/003-GPT-XL/checkpoints/eval_step_2000/batch_1_cfg_7.5_topk_16384.jpg filter=lfs diff=lfs merge=lfs -text +CKPTS/X2I_700k_3mask_200k_LLava_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_filtered_objects_3mask_888_context/003-GPT-XL/checkpoints/eval_step_2000/batch_2_cfg_7.5_topk_16384.jpg filter=lfs diff=lfs merge=lfs -text +CKPTS/X2I_700k_3mask_200k_LLava_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_filtered_objects_3mask_888_context/003-GPT-XL/checkpoints/eval_step_2000/batch_3_cfg_7.5_topk_16384.jpg filter=lfs diff=lfs merge=lfs -text +CKPTS/X2I_700k_3mask_200k_LLava_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_filtered_objects_3mask_888_context/003-GPT-XL/checkpoints/eval_step_2000/batch_4_cfg_7.5_topk_16384.jpg filter=lfs diff=lfs merge=lfs -text +CKPTS/X2I_700k_3mask_200k_LLava_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_filtered_objects_3mask_888_context/003-GPT-XL/checkpoints/eval_step_20000/batch_0_cfg_7.5_topk_16384.jpg filter=lfs diff=lfs merge=lfs -text +CKPTS/X2I_700k_3mask_200k_LLava_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_filtered_objects_3mask_888_context/003-GPT-XL/checkpoints/eval_step_20000/batch_1_cfg_7.5_topk_16384.jpg filter=lfs diff=lfs merge=lfs -text +CKPTS/X2I_700k_3mask_200k_LLava_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_filtered_objects_3mask_888_context/003-GPT-XL/checkpoints/eval_step_20000/batch_2_cfg_7.5_topk_16384.jpg filter=lfs diff=lfs merge=lfs -text +CKPTS/X2I_700k_3mask_200k_LLava_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_filtered_objects_3mask_888_context/003-GPT-XL/checkpoints/eval_step_20000/batch_3_cfg_7.5_topk_16384.jpg filter=lfs diff=lfs merge=lfs -text +CKPTS/X2I_700k_3mask_200k_LLava_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_filtered_objects_3mask_888_context/003-GPT-XL/checkpoints/eval_step_20000/batch_4_cfg_7.5_topk_16384.jpg filter=lfs diff=lfs merge=lfs -text +CKPTS/X2I_700k_3mask_200k_LLava_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_filtered_objects_3mask_888_context/003-GPT-XL/checkpoints/eval_step_22000/batch_0_cfg_7.5_topk_16384.jpg filter=lfs diff=lfs merge=lfs -text +CKPTS/X2I_700k_3mask_200k_LLava_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_filtered_objects_3mask_888_context/003-GPT-XL/checkpoints/eval_step_22000/batch_1_cfg_7.5_topk_16384.jpg filter=lfs diff=lfs merge=lfs -text +CKPTS/X2I_700k_3mask_200k_LLava_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_filtered_objects_3mask_888_context/003-GPT-XL/checkpoints/eval_step_22000/batch_2_cfg_7.5_topk_16384.jpg filter=lfs diff=lfs merge=lfs -text +CKPTS/X2I_700k_3mask_200k_LLava_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_filtered_objects_3mask_888_context/003-GPT-XL/checkpoints/eval_step_22000/batch_3_cfg_7.5_topk_16384.jpg filter=lfs diff=lfs merge=lfs -text +CKPTS/X2I_700k_3mask_200k_LLava_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_filtered_objects_3mask_888_context/003-GPT-XL/checkpoints/eval_step_22000/batch_4_cfg_7.5_topk_16384.jpg filter=lfs diff=lfs merge=lfs -text +CKPTS/X2I_700k_3mask_200k_LLava_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_filtered_objects_3mask_888_context/003-GPT-XL/checkpoints/eval_step_24000/batch_0_cfg_7.5_topk_16384.jpg filter=lfs diff=lfs merge=lfs -text +CKPTS/X2I_700k_3mask_200k_LLava_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_filtered_objects_3mask_888_context/003-GPT-XL/checkpoints/eval_step_24000/batch_1_cfg_7.5_topk_16384.jpg filter=lfs diff=lfs merge=lfs -text +CKPTS/X2I_700k_3mask_200k_LLava_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_filtered_objects_3mask_888_context/003-GPT-XL/checkpoints/eval_step_24000/batch_2_cfg_7.5_topk_16384.jpg filter=lfs diff=lfs merge=lfs -text +CKPTS/X2I_700k_3mask_200k_LLava_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_filtered_objects_3mask_888_context/003-GPT-XL/checkpoints/eval_step_24000/batch_3_cfg_7.5_topk_16384.jpg filter=lfs diff=lfs merge=lfs -text +CKPTS/X2I_700k_3mask_200k_LLava_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_filtered_objects_3mask_888_context/003-GPT-XL/checkpoints/eval_step_24000/batch_4_cfg_7.5_topk_16384.jpg filter=lfs diff=lfs merge=lfs -text +CKPTS/X2I_700k_3mask_200k_LLava_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_filtered_objects_3mask_888_context/003-GPT-XL/checkpoints/eval_step_26000/batch_0_cfg_7.5_topk_16384.jpg filter=lfs diff=lfs merge=lfs -text +CKPTS/X2I_700k_3mask_200k_LLava_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_filtered_objects_3mask_888_context/003-GPT-XL/checkpoints/eval_step_26000/batch_1_cfg_7.5_topk_16384.jpg filter=lfs diff=lfs merge=lfs -text +CKPTS/X2I_700k_3mask_200k_LLava_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_filtered_objects_3mask_888_context/003-GPT-XL/checkpoints/eval_step_26000/batch_2_cfg_7.5_topk_16384.jpg filter=lfs diff=lfs merge=lfs -text +CKPTS/X2I_700k_3mask_200k_LLava_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_filtered_objects_3mask_888_context/003-GPT-XL/checkpoints/eval_step_26000/batch_3_cfg_7.5_topk_16384.jpg filter=lfs diff=lfs merge=lfs -text +CKPTS/X2I_700k_3mask_200k_LLava_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_filtered_objects_3mask_888_context/003-GPT-XL/checkpoints/eval_step_26000/batch_4_cfg_7.5_topk_16384.jpg filter=lfs diff=lfs merge=lfs -text +CKPTS/X2I_700k_3mask_200k_LLava_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_filtered_objects_3mask_888_context/003-GPT-XL/checkpoints/eval_step_28000/batch_0_cfg_7.5_topk_16384.jpg filter=lfs diff=lfs merge=lfs -text +CKPTS/X2I_700k_3mask_200k_LLava_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_filtered_objects_3mask_888_context/003-GPT-XL/checkpoints/eval_step_28000/batch_1_cfg_7.5_topk_16384.jpg filter=lfs diff=lfs merge=lfs -text +CKPTS/X2I_700k_3mask_200k_LLava_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_filtered_objects_3mask_888_context/003-GPT-XL/checkpoints/eval_step_28000/batch_2_cfg_7.5_topk_16384.jpg filter=lfs diff=lfs merge=lfs -text +CKPTS/X2I_700k_3mask_200k_LLava_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_filtered_objects_3mask_888_context/003-GPT-XL/checkpoints/eval_step_28000/batch_3_cfg_7.5_topk_16384.jpg filter=lfs diff=lfs merge=lfs -text +CKPTS/X2I_700k_3mask_200k_LLava_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_filtered_objects_3mask_888_context/003-GPT-XL/checkpoints/eval_step_28000/batch_4_cfg_7.5_topk_16384.jpg filter=lfs diff=lfs merge=lfs -text +CKPTS/X2I_700k_3mask_200k_LLava_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_filtered_objects_3mask_888_context/003-GPT-XL/checkpoints/eval_step_30000/batch_0_cfg_7.5_topk_16384.jpg filter=lfs diff=lfs merge=lfs -text +CKPTS/X2I_700k_3mask_200k_LLava_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_filtered_objects_3mask_888_context/003-GPT-XL/checkpoints/eval_step_30000/batch_1_cfg_7.5_topk_16384.jpg filter=lfs diff=lfs merge=lfs -text +CKPTS/X2I_700k_3mask_200k_LLava_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_filtered_objects_3mask_888_context/003-GPT-XL/checkpoints/eval_step_30000/batch_2_cfg_7.5_topk_16384.jpg filter=lfs diff=lfs merge=lfs -text +CKPTS/X2I_700k_3mask_200k_LLava_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_filtered_objects_3mask_888_context/003-GPT-XL/checkpoints/eval_step_30000/batch_3_cfg_7.5_topk_16384.jpg filter=lfs diff=lfs merge=lfs -text +CKPTS/X2I_700k_3mask_200k_LLava_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_filtered_objects_3mask_888_context/003-GPT-XL/checkpoints/eval_step_30000/batch_4_cfg_7.5_topk_16384.jpg filter=lfs diff=lfs merge=lfs -text +CKPTS/X2I_700k_3mask_200k_LLava_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_filtered_objects_3mask_888_context/003-GPT-XL/checkpoints/eval_step_32000/batch_0_cfg_7.5_topk_16384.jpg filter=lfs diff=lfs merge=lfs -text +CKPTS/X2I_700k_3mask_200k_LLava_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_filtered_objects_3mask_888_context/003-GPT-XL/checkpoints/eval_step_32000/batch_1_cfg_7.5_topk_16384.jpg filter=lfs diff=lfs merge=lfs -text +CKPTS/X2I_700k_3mask_200k_LLava_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_filtered_objects_3mask_888_context/003-GPT-XL/checkpoints/eval_step_32000/batch_2_cfg_7.5_topk_16384.jpg filter=lfs diff=lfs merge=lfs -text +CKPTS/X2I_700k_3mask_200k_LLava_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_filtered_objects_3mask_888_context/003-GPT-XL/checkpoints/eval_step_32000/batch_3_cfg_7.5_topk_16384.jpg filter=lfs diff=lfs merge=lfs -text +CKPTS/X2I_700k_3mask_200k_LLava_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_filtered_objects_3mask_888_context/003-GPT-XL/checkpoints/eval_step_32000/batch_4_cfg_7.5_topk_16384.jpg filter=lfs diff=lfs merge=lfs -text +CKPTS/X2I_700k_3mask_200k_LLava_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_filtered_objects_3mask_888_context/003-GPT-XL/checkpoints/eval_step_34000/batch_0_cfg_7.5_topk_16384.jpg filter=lfs diff=lfs merge=lfs -text +CKPTS/X2I_700k_3mask_200k_LLava_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_filtered_objects_3mask_888_context/003-GPT-XL/checkpoints/eval_step_34000/batch_1_cfg_7.5_topk_16384.jpg filter=lfs diff=lfs merge=lfs -text +CKPTS/X2I_700k_3mask_200k_LLava_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_filtered_objects_3mask_888_context/003-GPT-XL/checkpoints/eval_step_34000/batch_2_cfg_7.5_topk_16384.jpg filter=lfs diff=lfs merge=lfs -text +CKPTS/X2I_700k_3mask_200k_LLava_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_filtered_objects_3mask_888_context/003-GPT-XL/checkpoints/eval_step_4000/batch_0_cfg_7.5_topk_16384.jpg filter=lfs diff=lfs merge=lfs -text +CKPTS/X2I_700k_3mask_200k_LLava_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_filtered_objects_3mask_888_context/003-GPT-XL/checkpoints/eval_step_4000/batch_1_cfg_7.5_topk_16384.jpg filter=lfs diff=lfs merge=lfs -text +CKPTS/X2I_700k_3mask_200k_LLava_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_filtered_objects_3mask_888_context/003-GPT-XL/checkpoints/eval_step_4000/batch_2_cfg_7.5_topk_16384.jpg filter=lfs diff=lfs merge=lfs -text +CKPTS/X2I_700k_3mask_200k_LLava_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_filtered_objects_3mask_888_context/003-GPT-XL/checkpoints/eval_step_4000/batch_3_cfg_7.5_topk_16384.jpg filter=lfs diff=lfs merge=lfs -text +CKPTS/X2I_700k_3mask_200k_LLava_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_filtered_objects_3mask_888_context/003-GPT-XL/checkpoints/eval_step_4000/batch_4_cfg_7.5_topk_16384.jpg filter=lfs diff=lfs merge=lfs -text +CKPTS/X2I_700k_3mask_200k_LLava_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_filtered_objects_3mask_888_context/003-GPT-XL/checkpoints/eval_step_6000/batch_0_cfg_7.5_topk_16384.jpg filter=lfs diff=lfs merge=lfs -text +CKPTS/X2I_700k_3mask_200k_LLava_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_filtered_objects_3mask_888_context/003-GPT-XL/checkpoints/eval_step_6000/batch_1_cfg_7.5_topk_16384.jpg filter=lfs diff=lfs merge=lfs -text +CKPTS/X2I_700k_3mask_200k_LLava_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_filtered_objects_3mask_888_context/003-GPT-XL/checkpoints/eval_step_6000/batch_2_cfg_7.5_topk_16384.jpg filter=lfs diff=lfs merge=lfs -text +CKPTS/X2I_700k_3mask_200k_LLava_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_filtered_objects_3mask_888_context/003-GPT-XL/checkpoints/eval_step_6000/batch_3_cfg_7.5_topk_16384.jpg filter=lfs diff=lfs merge=lfs -text +CKPTS/X2I_700k_3mask_200k_LLava_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_filtered_objects_3mask_888_context/003-GPT-XL/checkpoints/eval_step_6000/batch_4_cfg_7.5_topk_16384.jpg filter=lfs diff=lfs merge=lfs -text +CKPTS/X2I_700k_3mask_200k_LLava_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_filtered_objects_3mask_888_context/003-GPT-XL/checkpoints/eval_step_8000/batch_0_cfg_7.5_topk_16384.jpg filter=lfs diff=lfs merge=lfs -text +CKPTS/X2I_700k_3mask_200k_LLava_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_filtered_objects_3mask_888_context/003-GPT-XL/checkpoints/eval_step_8000/batch_1_cfg_7.5_topk_16384.jpg filter=lfs diff=lfs merge=lfs -text +CKPTS/X2I_700k_3mask_200k_LLava_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_filtered_objects_3mask_888_context/003-GPT-XL/checkpoints/eval_step_8000/batch_2_cfg_7.5_topk_16384.jpg filter=lfs diff=lfs merge=lfs -text +CKPTS/X2I_700k_3mask_200k_LLava_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_filtered_objects_3mask_888_context/003-GPT-XL/checkpoints/eval_step_8000/batch_3_cfg_7.5_topk_16384.jpg filter=lfs diff=lfs merge=lfs -text +CKPTS/X2I_700k_3mask_200k_LLava_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_filtered_objects_3mask_888_context/003-GPT-XL/checkpoints/eval_step_8000/batch_4_cfg_7.5_topk_16384.jpg filter=lfs diff=lfs merge=lfs -text +CKPTS/X2I_700k_3mask_200k_LLava_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_filtered_objects_3mask_888_context/wandb/run-20250429_030813-6rsqv32i/run-6rsqv32i.wandb filter=lfs diff=lfs merge=lfs -text +CKPTS/X2I_700k_3mask_200k_LLava_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_filtered_objects_3mask_888_context/wandb/run-20250429_032803-lbha1z77/run-lbha1z77.wandb filter=lfs diff=lfs merge=lfs -text +CKPTS/X2I_UltraEdiit_fourmask_LLava_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_filtered_objects_4mask_1280_context/003-GPT-XL/checkpoints/eval_step_10000/batch_0_cfg_7.5_topk_16384.jpg filter=lfs diff=lfs merge=lfs -text +CKPTS/X2I_UltraEdiit_fourmask_LLava_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_filtered_objects_4mask_1280_context/003-GPT-XL/checkpoints/eval_step_10000/batch_1_cfg_7.5_topk_16384.jpg filter=lfs diff=lfs merge=lfs -text +CKPTS/X2I_UltraEdiit_fourmask_LLava_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_filtered_objects_4mask_1280_context/003-GPT-XL/checkpoints/eval_step_10000/batch_2_cfg_7.5_topk_16384.jpg filter=lfs diff=lfs merge=lfs -text +CKPTS/X2I_UltraEdiit_fourmask_LLava_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_filtered_objects_4mask_1280_context/003-GPT-XL/checkpoints/eval_step_10000/batch_3_cfg_7.5_topk_16384.jpg filter=lfs diff=lfs merge=lfs -text +CKPTS/X2I_UltraEdiit_fourmask_LLava_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_filtered_objects_4mask_1280_context/003-GPT-XL/checkpoints/eval_step_10000/batch_4_cfg_7.5_topk_16384.jpg filter=lfs diff=lfs merge=lfs -text +CKPTS/X2I_UltraEdiit_fourmask_LLava_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_filtered_objects_4mask_1280_context/003-GPT-XL/checkpoints/eval_step_100000/batch_0_cfg_7.5_topk_16384.jpg filter=lfs diff=lfs merge=lfs -text +CKPTS/X2I_UltraEdiit_fourmask_LLava_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_filtered_objects_4mask_1280_context/003-GPT-XL/checkpoints/eval_step_100000/batch_1_cfg_7.5_topk_16384.jpg filter=lfs diff=lfs merge=lfs -text +CKPTS/X2I_UltraEdiit_fourmask_LLava_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_filtered_objects_4mask_1280_context/003-GPT-XL/checkpoints/eval_step_100000/batch_2_cfg_7.5_topk_16384.jpg filter=lfs diff=lfs merge=lfs -text +CKPTS/X2I_UltraEdiit_fourmask_LLava_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_filtered_objects_4mask_1280_context/003-GPT-XL/checkpoints/eval_step_100000/batch_3_cfg_7.5_topk_16384.jpg filter=lfs diff=lfs merge=lfs -text +CKPTS/X2I_UltraEdiit_fourmask_LLava_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_filtered_objects_4mask_1280_context/003-GPT-XL/checkpoints/eval_step_100000/batch_4_cfg_7.5_topk_16384.jpg filter=lfs diff=lfs merge=lfs -text +CKPTS/X2I_UltraEdiit_fourmask_LLava_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_filtered_objects_4mask_1280_context/003-GPT-XL/checkpoints/eval_step_102000/batch_0_cfg_7.5_topk_16384.jpg filter=lfs diff=lfs merge=lfs -text +CKPTS/X2I_UltraEdiit_fourmask_LLava_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_filtered_objects_4mask_1280_context/003-GPT-XL/checkpoints/eval_step_102000/batch_1_cfg_7.5_topk_16384.jpg filter=lfs diff=lfs merge=lfs -text +CKPTS/X2I_UltraEdiit_fourmask_LLava_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_filtered_objects_4mask_1280_context/003-GPT-XL/checkpoints/eval_step_102000/batch_2_cfg_7.5_topk_16384.jpg filter=lfs diff=lfs merge=lfs -text +CKPTS/X2I_UltraEdiit_fourmask_LLava_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_filtered_objects_4mask_1280_context/003-GPT-XL/checkpoints/eval_step_102000/batch_3_cfg_7.5_topk_16384.jpg filter=lfs diff=lfs merge=lfs -text +CKPTS/X2I_UltraEdiit_fourmask_LLava_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_filtered_objects_4mask_1280_context/003-GPT-XL/checkpoints/eval_step_102000/batch_4_cfg_7.5_topk_16384.jpg filter=lfs diff=lfs merge=lfs -text +CKPTS/X2I_UltraEdiit_fourmask_LLava_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_filtered_objects_4mask_1280_context/003-GPT-XL/checkpoints/eval_step_104000/batch_0_cfg_7.5_topk_16384.jpg filter=lfs diff=lfs merge=lfs -text +CKPTS/X2I_UltraEdiit_fourmask_LLava_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_filtered_objects_4mask_1280_context/003-GPT-XL/checkpoints/eval_step_104000/batch_1_cfg_7.5_topk_16384.jpg filter=lfs diff=lfs merge=lfs -text +CKPTS/X2I_UltraEdiit_fourmask_LLava_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_filtered_objects_4mask_1280_context/003-GPT-XL/checkpoints/eval_step_104000/batch_2_cfg_7.5_topk_16384.jpg filter=lfs diff=lfs merge=lfs -text +CKPTS/X2I_UltraEdiit_fourmask_LLava_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_filtered_objects_4mask_1280_context/003-GPT-XL/checkpoints/eval_step_104000/batch_3_cfg_7.5_topk_16384.jpg filter=lfs diff=lfs merge=lfs -text +CKPTS/X2I_UltraEdiit_fourmask_LLava_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_filtered_objects_4mask_1280_context/003-GPT-XL/checkpoints/eval_step_104000/batch_4_cfg_7.5_topk_16384.jpg filter=lfs diff=lfs merge=lfs -text +CKPTS/X2I_UltraEdiit_fourmask_LLava_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_filtered_objects_4mask_1280_context/003-GPT-XL/checkpoints/eval_step_106000/batch_0_cfg_7.5_topk_16384.jpg filter=lfs diff=lfs merge=lfs -text +CKPTS/X2I_UltraEdiit_fourmask_LLava_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_filtered_objects_4mask_1280_context/003-GPT-XL/checkpoints/eval_step_106000/batch_1_cfg_7.5_topk_16384.jpg filter=lfs diff=lfs merge=lfs -text +CKPTS/X2I_UltraEdiit_fourmask_LLava_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_filtered_objects_4mask_1280_context/003-GPT-XL/checkpoints/eval_step_106000/batch_2_cfg_7.5_topk_16384.jpg filter=lfs diff=lfs merge=lfs -text +CKPTS/X2I_UltraEdiit_fourmask_LLava_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_filtered_objects_4mask_1280_context/003-GPT-XL/checkpoints/eval_step_106000/batch_3_cfg_7.5_topk_16384.jpg filter=lfs diff=lfs merge=lfs -text +CKPTS/X2I_UltraEdiit_fourmask_LLava_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_filtered_objects_4mask_1280_context/003-GPT-XL/checkpoints/eval_step_106000/batch_4_cfg_7.5_topk_16384.jpg filter=lfs diff=lfs merge=lfs -text +CKPTS/X2I_UltraEdiit_fourmask_LLava_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_filtered_objects_4mask_1280_context/003-GPT-XL/checkpoints/eval_step_108000/batch_0_cfg_7.5_topk_16384.jpg filter=lfs diff=lfs merge=lfs -text +CKPTS/X2I_UltraEdiit_fourmask_LLava_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_filtered_objects_4mask_1280_context/003-GPT-XL/checkpoints/eval_step_108000/batch_1_cfg_7.5_topk_16384.jpg filter=lfs diff=lfs merge=lfs -text +CKPTS/X2I_UltraEdiit_fourmask_LLava_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_filtered_objects_4mask_1280_context/003-GPT-XL/checkpoints/eval_step_108000/batch_2_cfg_7.5_topk_16384.jpg filter=lfs diff=lfs merge=lfs -text +CKPTS/X2I_UltraEdiit_fourmask_LLava_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_filtered_objects_4mask_1280_context/003-GPT-XL/checkpoints/eval_step_108000/batch_3_cfg_7.5_topk_16384.jpg filter=lfs diff=lfs merge=lfs -text +CKPTS/X2I_UltraEdiit_fourmask_LLava_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_filtered_objects_4mask_1280_context/003-GPT-XL/checkpoints/eval_step_108000/batch_4_cfg_7.5_topk_16384.jpg filter=lfs diff=lfs merge=lfs -text +CKPTS/X2I_UltraEdiit_fourmask_LLava_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_filtered_objects_4mask_1280_context/003-GPT-XL/checkpoints/eval_step_110000/batch_0_cfg_7.5_topk_16384.jpg filter=lfs diff=lfs merge=lfs -text +CKPTS/X2I_UltraEdiit_fourmask_LLava_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_filtered_objects_4mask_1280_context/003-GPT-XL/checkpoints/eval_step_110000/batch_1_cfg_7.5_topk_16384.jpg filter=lfs diff=lfs merge=lfs -text +CKPTS/X2I_UltraEdiit_fourmask_LLava_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_filtered_objects_4mask_1280_context/003-GPT-XL/checkpoints/eval_step_110000/batch_2_cfg_7.5_topk_16384.jpg filter=lfs diff=lfs merge=lfs -text +CKPTS/X2I_UltraEdiit_fourmask_LLava_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_filtered_objects_4mask_1280_context/003-GPT-XL/checkpoints/eval_step_110000/batch_3_cfg_7.5_topk_16384.jpg filter=lfs diff=lfs merge=lfs -text +CKPTS/X2I_UltraEdiit_fourmask_LLava_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_filtered_objects_4mask_1280_context/003-GPT-XL/checkpoints/eval_step_110000/batch_4_cfg_7.5_topk_16384.jpg filter=lfs diff=lfs merge=lfs -text +CKPTS/X2I_UltraEdiit_fourmask_LLava_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_filtered_objects_4mask_1280_context/003-GPT-XL/checkpoints/eval_step_12000/batch_0_cfg_7.5_topk_16384.jpg filter=lfs diff=lfs merge=lfs -text +CKPTS/X2I_UltraEdiit_fourmask_LLava_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_filtered_objects_4mask_1280_context/003-GPT-XL/checkpoints/eval_step_12000/batch_1_cfg_7.5_topk_16384.jpg filter=lfs diff=lfs merge=lfs -text +CKPTS/X2I_UltraEdiit_fourmask_LLava_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_filtered_objects_4mask_1280_context/003-GPT-XL/checkpoints/eval_step_12000/batch_2_cfg_7.5_topk_16384.jpg filter=lfs diff=lfs merge=lfs -text +CKPTS/X2I_UltraEdiit_fourmask_LLava_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_filtered_objects_4mask_1280_context/003-GPT-XL/checkpoints/eval_step_12000/batch_3_cfg_7.5_topk_16384.jpg filter=lfs diff=lfs merge=lfs -text +CKPTS/X2I_UltraEdiit_fourmask_LLava_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_filtered_objects_4mask_1280_context/003-GPT-XL/checkpoints/eval_step_12000/batch_4_cfg_7.5_topk_16384.jpg filter=lfs diff=lfs merge=lfs -text +CKPTS/X2I_UltraEdiit_fourmask_LLava_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_filtered_objects_4mask_1280_context/003-GPT-XL/checkpoints/eval_step_14000/batch_0_cfg_7.5_topk_16384.jpg filter=lfs diff=lfs merge=lfs -text +CKPTS/X2I_UltraEdiit_fourmask_LLava_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_filtered_objects_4mask_1280_context/003-GPT-XL/checkpoints/eval_step_14000/batch_1_cfg_7.5_topk_16384.jpg filter=lfs diff=lfs merge=lfs -text +CKPTS/X2I_UltraEdiit_fourmask_LLava_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_filtered_objects_4mask_1280_context/003-GPT-XL/checkpoints/eval_step_14000/batch_2_cfg_7.5_topk_16384.jpg filter=lfs diff=lfs merge=lfs -text +CKPTS/X2I_UltraEdiit_fourmask_LLava_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_filtered_objects_4mask_1280_context/003-GPT-XL/checkpoints/eval_step_14000/batch_3_cfg_7.5_topk_16384.jpg filter=lfs diff=lfs merge=lfs -text +CKPTS/X2I_UltraEdiit_fourmask_LLava_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_filtered_objects_4mask_1280_context/003-GPT-XL/checkpoints/eval_step_14000/batch_4_cfg_7.5_topk_16384.jpg filter=lfs diff=lfs merge=lfs -text +CKPTS/X2I_UltraEdiit_fourmask_LLava_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_filtered_objects_4mask_1280_context/003-GPT-XL/checkpoints/eval_step_16000/batch_0_cfg_7.5_topk_16384.jpg filter=lfs diff=lfs merge=lfs -text +CKPTS/X2I_UltraEdiit_fourmask_LLava_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_filtered_objects_4mask_1280_context/003-GPT-XL/checkpoints/eval_step_16000/batch_1_cfg_7.5_topk_16384.jpg filter=lfs diff=lfs merge=lfs -text +CKPTS/X2I_UltraEdiit_fourmask_LLava_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_filtered_objects_4mask_1280_context/003-GPT-XL/checkpoints/eval_step_16000/batch_2_cfg_7.5_topk_16384.jpg filter=lfs diff=lfs merge=lfs -text +CKPTS/X2I_UltraEdiit_fourmask_LLava_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_filtered_objects_4mask_1280_context/003-GPT-XL/checkpoints/eval_step_16000/batch_3_cfg_7.5_topk_16384.jpg filter=lfs diff=lfs merge=lfs -text +CKPTS/X2I_UltraEdiit_fourmask_LLava_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_filtered_objects_4mask_1280_context/003-GPT-XL/checkpoints/eval_step_16000/batch_4_cfg_7.5_topk_16384.jpg filter=lfs diff=lfs merge=lfs -text +CKPTS/X2I_UltraEdiit_fourmask_LLava_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_filtered_objects_4mask_1280_context/003-GPT-XL/checkpoints/eval_step_18000/batch_0_cfg_7.5_topk_16384.jpg filter=lfs diff=lfs merge=lfs -text +CKPTS/X2I_UltraEdiit_fourmask_LLava_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_filtered_objects_4mask_1280_context/003-GPT-XL/checkpoints/eval_step_18000/batch_1_cfg_7.5_topk_16384.jpg filter=lfs diff=lfs merge=lfs -text +CKPTS/X2I_UltraEdiit_fourmask_LLava_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_filtered_objects_4mask_1280_context/003-GPT-XL/checkpoints/eval_step_18000/batch_2_cfg_7.5_topk_16384.jpg filter=lfs diff=lfs merge=lfs -text +CKPTS/X2I_UltraEdiit_fourmask_LLava_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_filtered_objects_4mask_1280_context/003-GPT-XL/checkpoints/eval_step_18000/batch_3_cfg_7.5_topk_16384.jpg filter=lfs diff=lfs merge=lfs -text +CKPTS/X2I_UltraEdiit_fourmask_LLava_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_filtered_objects_4mask_1280_context/003-GPT-XL/checkpoints/eval_step_18000/batch_4_cfg_7.5_topk_16384.jpg filter=lfs diff=lfs merge=lfs -text +CKPTS/X2I_UltraEdiit_fourmask_LLava_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_filtered_objects_4mask_1280_context/003-GPT-XL/checkpoints/eval_step_2000/batch_0_cfg_7.5_topk_16384.jpg filter=lfs diff=lfs merge=lfs -text +CKPTS/X2I_UltraEdiit_fourmask_LLava_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_filtered_objects_4mask_1280_context/003-GPT-XL/checkpoints/eval_step_2000/batch_1_cfg_7.5_topk_16384.jpg filter=lfs diff=lfs merge=lfs -text +CKPTS/X2I_UltraEdiit_fourmask_LLava_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_filtered_objects_4mask_1280_context/003-GPT-XL/checkpoints/eval_step_2000/batch_2_cfg_7.5_topk_16384.jpg filter=lfs diff=lfs merge=lfs -text +CKPTS/X2I_UltraEdiit_fourmask_LLava_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_filtered_objects_4mask_1280_context/003-GPT-XL/checkpoints/eval_step_2000/batch_3_cfg_7.5_topk_16384.jpg filter=lfs diff=lfs merge=lfs -text +CKPTS/X2I_UltraEdiit_fourmask_LLava_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_filtered_objects_4mask_1280_context/003-GPT-XL/checkpoints/eval_step_2000/batch_4_cfg_7.5_topk_16384.jpg filter=lfs diff=lfs merge=lfs -text +CKPTS/X2I_UltraEdiit_fourmask_LLava_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_filtered_objects_4mask_1280_context/003-GPT-XL/checkpoints/eval_step_20000/batch_0_cfg_7.5_topk_16384.jpg filter=lfs diff=lfs merge=lfs -text +CKPTS/X2I_UltraEdiit_fourmask_LLava_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_filtered_objects_4mask_1280_context/003-GPT-XL/checkpoints/eval_step_20000/batch_1_cfg_7.5_topk_16384.jpg filter=lfs diff=lfs merge=lfs -text +CKPTS/X2I_UltraEdiit_fourmask_LLava_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_filtered_objects_4mask_1280_context/003-GPT-XL/checkpoints/eval_step_20000/batch_2_cfg_7.5_topk_16384.jpg filter=lfs diff=lfs merge=lfs -text +CKPTS/X2I_UltraEdiit_fourmask_LLava_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_filtered_objects_4mask_1280_context/003-GPT-XL/checkpoints/eval_step_20000/batch_3_cfg_7.5_topk_16384.jpg filter=lfs diff=lfs merge=lfs -text +CKPTS/X2I_UltraEdiit_fourmask_LLava_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_filtered_objects_4mask_1280_context/003-GPT-XL/checkpoints/eval_step_20000/batch_4_cfg_7.5_topk_16384.jpg filter=lfs diff=lfs merge=lfs -text +CKPTS/X2I_UltraEdiit_fourmask_LLava_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_filtered_objects_4mask_1280_context/003-GPT-XL/checkpoints/eval_step_22000/batch_0_cfg_7.5_topk_16384.jpg filter=lfs diff=lfs merge=lfs -text +CKPTS/X2I_UltraEdiit_fourmask_LLava_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_filtered_objects_4mask_1280_context/003-GPT-XL/checkpoints/eval_step_22000/batch_1_cfg_7.5_topk_16384.jpg filter=lfs diff=lfs merge=lfs -text +CKPTS/X2I_UltraEdiit_fourmask_LLava_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_filtered_objects_4mask_1280_context/003-GPT-XL/checkpoints/eval_step_22000/batch_2_cfg_7.5_topk_16384.jpg filter=lfs diff=lfs merge=lfs -text +CKPTS/X2I_UltraEdiit_fourmask_LLava_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_filtered_objects_4mask_1280_context/003-GPT-XL/checkpoints/eval_step_22000/batch_3_cfg_7.5_topk_16384.jpg filter=lfs diff=lfs merge=lfs -text +CKPTS/X2I_UltraEdiit_fourmask_LLava_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_filtered_objects_4mask_1280_context/003-GPT-XL/checkpoints/eval_step_22000/batch_4_cfg_7.5_topk_16384.jpg filter=lfs diff=lfs merge=lfs -text +CKPTS/X2I_UltraEdiit_fourmask_LLava_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_filtered_objects_4mask_1280_context/003-GPT-XL/checkpoints/eval_step_24000/batch_0_cfg_7.5_topk_16384.jpg filter=lfs diff=lfs merge=lfs -text +CKPTS/X2I_UltraEdiit_fourmask_LLava_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_filtered_objects_4mask_1280_context/003-GPT-XL/checkpoints/eval_step_24000/batch_1_cfg_7.5_topk_16384.jpg filter=lfs diff=lfs merge=lfs -text +CKPTS/X2I_UltraEdiit_fourmask_LLava_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_filtered_objects_4mask_1280_context/003-GPT-XL/checkpoints/eval_step_24000/batch_2_cfg_7.5_topk_16384.jpg filter=lfs diff=lfs merge=lfs -text +CKPTS/X2I_UltraEdiit_fourmask_LLava_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_filtered_objects_4mask_1280_context/003-GPT-XL/checkpoints/eval_step_24000/batch_3_cfg_7.5_topk_16384.jpg filter=lfs diff=lfs merge=lfs -text +CKPTS/X2I_UltraEdiit_fourmask_LLava_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_filtered_objects_4mask_1280_context/003-GPT-XL/checkpoints/eval_step_24000/batch_4_cfg_7.5_topk_16384.jpg filter=lfs diff=lfs merge=lfs -text +CKPTS/X2I_UltraEdiit_fourmask_LLava_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_filtered_objects_4mask_1280_context/003-GPT-XL/checkpoints/eval_step_26000/batch_0_cfg_7.5_topk_16384.jpg filter=lfs diff=lfs merge=lfs -text +CKPTS/X2I_UltraEdiit_fourmask_LLava_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_filtered_objects_4mask_1280_context/003-GPT-XL/checkpoints/eval_step_26000/batch_1_cfg_7.5_topk_16384.jpg filter=lfs diff=lfs merge=lfs -text +CKPTS/X2I_UltraEdiit_fourmask_LLava_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_filtered_objects_4mask_1280_context/003-GPT-XL/checkpoints/eval_step_26000/batch_2_cfg_7.5_topk_16384.jpg filter=lfs diff=lfs merge=lfs -text +CKPTS/X2I_UltraEdiit_fourmask_LLava_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_filtered_objects_4mask_1280_context/003-GPT-XL/checkpoints/eval_step_26000/batch_3_cfg_7.5_topk_16384.jpg filter=lfs diff=lfs merge=lfs -text +CKPTS/X2I_UltraEdiit_fourmask_LLava_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_filtered_objects_4mask_1280_context/003-GPT-XL/checkpoints/eval_step_26000/batch_4_cfg_7.5_topk_16384.jpg filter=lfs diff=lfs merge=lfs -text +CKPTS/X2I_UltraEdiit_fourmask_LLava_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_filtered_objects_4mask_1280_context/003-GPT-XL/checkpoints/eval_step_28000/batch_0_cfg_7.5_topk_16384.jpg filter=lfs diff=lfs merge=lfs -text +CKPTS/X2I_UltraEdiit_fourmask_LLava_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_filtered_objects_4mask_1280_context/003-GPT-XL/checkpoints/eval_step_28000/batch_1_cfg_7.5_topk_16384.jpg filter=lfs diff=lfs merge=lfs -text +CKPTS/X2I_UltraEdiit_fourmask_LLava_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_filtered_objects_4mask_1280_context/003-GPT-XL/checkpoints/eval_step_28000/batch_2_cfg_7.5_topk_16384.jpg filter=lfs diff=lfs merge=lfs -text +CKPTS/X2I_UltraEdiit_fourmask_LLava_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_filtered_objects_4mask_1280_context/003-GPT-XL/checkpoints/eval_step_28000/batch_3_cfg_7.5_topk_16384.jpg filter=lfs diff=lfs merge=lfs -text +CKPTS/X2I_UltraEdiit_fourmask_LLava_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_filtered_objects_4mask_1280_context/003-GPT-XL/checkpoints/eval_step_28000/batch_4_cfg_7.5_topk_16384.jpg filter=lfs diff=lfs merge=lfs -text +CKPTS/X2I_UltraEdiit_fourmask_LLava_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_filtered_objects_4mask_1280_context/003-GPT-XL/checkpoints/eval_step_30000/batch_0_cfg_7.5_topk_16384.jpg filter=lfs diff=lfs merge=lfs -text +CKPTS/X2I_UltraEdiit_fourmask_LLava_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_filtered_objects_4mask_1280_context/003-GPT-XL/checkpoints/eval_step_30000/batch_1_cfg_7.5_topk_16384.jpg filter=lfs diff=lfs merge=lfs -text +CKPTS/X2I_UltraEdiit_fourmask_LLava_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_filtered_objects_4mask_1280_context/003-GPT-XL/checkpoints/eval_step_30000/batch_2_cfg_7.5_topk_16384.jpg filter=lfs diff=lfs merge=lfs -text +CKPTS/X2I_UltraEdiit_fourmask_LLava_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_filtered_objects_4mask_1280_context/003-GPT-XL/checkpoints/eval_step_30000/batch_3_cfg_7.5_topk_16384.jpg filter=lfs diff=lfs merge=lfs -text +CKPTS/X2I_UltraEdiit_fourmask_LLava_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_filtered_objects_4mask_1280_context/003-GPT-XL/checkpoints/eval_step_30000/batch_4_cfg_7.5_topk_16384.jpg filter=lfs diff=lfs merge=lfs -text +CKPTS/X2I_UltraEdiit_fourmask_LLava_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_filtered_objects_4mask_1280_context/003-GPT-XL/checkpoints/eval_step_32000/batch_0_cfg_7.5_topk_16384.jpg filter=lfs diff=lfs merge=lfs -text +CKPTS/X2I_UltraEdiit_fourmask_LLava_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_filtered_objects_4mask_1280_context/003-GPT-XL/checkpoints/eval_step_32000/batch_1_cfg_7.5_topk_16384.jpg filter=lfs diff=lfs merge=lfs -text +CKPTS/X2I_UltraEdiit_fourmask_LLava_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_filtered_objects_4mask_1280_context/003-GPT-XL/checkpoints/eval_step_32000/batch_2_cfg_7.5_topk_16384.jpg filter=lfs diff=lfs merge=lfs -text +CKPTS/X2I_UltraEdiit_fourmask_LLava_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_filtered_objects_4mask_1280_context/003-GPT-XL/checkpoints/eval_step_32000/batch_3_cfg_7.5_topk_16384.jpg filter=lfs diff=lfs merge=lfs -text +CKPTS/X2I_UltraEdiit_fourmask_LLava_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_filtered_objects_4mask_1280_context/003-GPT-XL/checkpoints/eval_step_32000/batch_4_cfg_7.5_topk_16384.jpg filter=lfs diff=lfs merge=lfs -text +CKPTS/X2I_UltraEdiit_fourmask_LLava_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_filtered_objects_4mask_1280_context/003-GPT-XL/checkpoints/eval_step_34000/batch_0_cfg_7.5_topk_16384.jpg filter=lfs diff=lfs merge=lfs -text +CKPTS/X2I_UltraEdiit_fourmask_LLava_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_filtered_objects_4mask_1280_context/003-GPT-XL/checkpoints/eval_step_34000/batch_1_cfg_7.5_topk_16384.jpg filter=lfs diff=lfs merge=lfs -text +CKPTS/X2I_UltraEdiit_fourmask_LLava_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_filtered_objects_4mask_1280_context/003-GPT-XL/checkpoints/eval_step_34000/batch_2_cfg_7.5_topk_16384.jpg filter=lfs diff=lfs merge=lfs -text +CKPTS/X2I_UltraEdiit_fourmask_LLava_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_filtered_objects_4mask_1280_context/003-GPT-XL/checkpoints/eval_step_34000/batch_3_cfg_7.5_topk_16384.jpg filter=lfs diff=lfs merge=lfs -text +CKPTS/X2I_UltraEdiit_fourmask_LLava_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_filtered_objects_4mask_1280_context/003-GPT-XL/checkpoints/eval_step_34000/batch_4_cfg_7.5_topk_16384.jpg filter=lfs diff=lfs merge=lfs -text +CKPTS/X2I_UltraEdiit_fourmask_LLava_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_filtered_objects_4mask_1280_context/003-GPT-XL/checkpoints/eval_step_36000/batch_0_cfg_7.5_topk_16384.jpg filter=lfs diff=lfs merge=lfs -text +CKPTS/X2I_UltraEdiit_fourmask_LLava_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_filtered_objects_4mask_1280_context/003-GPT-XL/checkpoints/eval_step_36000/batch_1_cfg_7.5_topk_16384.jpg filter=lfs diff=lfs merge=lfs -text +CKPTS/X2I_UltraEdiit_fourmask_LLava_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_filtered_objects_4mask_1280_context/003-GPT-XL/checkpoints/eval_step_36000/batch_2_cfg_7.5_topk_16384.jpg filter=lfs diff=lfs merge=lfs -text +CKPTS/X2I_UltraEdiit_fourmask_LLava_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_filtered_objects_4mask_1280_context/003-GPT-XL/checkpoints/eval_step_36000/batch_3_cfg_7.5_topk_16384.jpg filter=lfs diff=lfs merge=lfs -text +CKPTS/X2I_UltraEdiit_fourmask_LLava_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_filtered_objects_4mask_1280_context/003-GPT-XL/checkpoints/eval_step_36000/batch_4_cfg_7.5_topk_16384.jpg filter=lfs diff=lfs merge=lfs -text +CKPTS/X2I_UltraEdiit_fourmask_LLava_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_filtered_objects_4mask_1280_context/003-GPT-XL/checkpoints/eval_step_38000/batch_0_cfg_7.5_topk_16384.jpg filter=lfs diff=lfs merge=lfs -text +CKPTS/X2I_UltraEdiit_fourmask_LLava_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_filtered_objects_4mask_1280_context/003-GPT-XL/checkpoints/eval_step_38000/batch_1_cfg_7.5_topk_16384.jpg filter=lfs diff=lfs merge=lfs -text +CKPTS/X2I_UltraEdiit_fourmask_LLava_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_filtered_objects_4mask_1280_context/003-GPT-XL/checkpoints/eval_step_38000/batch_2_cfg_7.5_topk_16384.jpg filter=lfs diff=lfs merge=lfs -text +CKPTS/X2I_UltraEdiit_fourmask_LLava_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_filtered_objects_4mask_1280_context/003-GPT-XL/checkpoints/eval_step_38000/batch_3_cfg_7.5_topk_16384.jpg filter=lfs diff=lfs merge=lfs -text +CKPTS/X2I_UltraEdiit_fourmask_LLava_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_filtered_objects_4mask_1280_context/003-GPT-XL/checkpoints/eval_step_38000/batch_4_cfg_7.5_topk_16384.jpg filter=lfs diff=lfs merge=lfs -text +CKPTS/X2I_UltraEdiit_fourmask_LLava_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_filtered_objects_4mask_1280_context/003-GPT-XL/checkpoints/eval_step_4000/batch_0_cfg_7.5_topk_16384.jpg filter=lfs diff=lfs merge=lfs -text +CKPTS/X2I_UltraEdiit_fourmask_LLava_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_filtered_objects_4mask_1280_context/003-GPT-XL/checkpoints/eval_step_4000/batch_1_cfg_7.5_topk_16384.jpg filter=lfs diff=lfs merge=lfs -text +CKPTS/X2I_UltraEdiit_fourmask_LLava_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_filtered_objects_4mask_1280_context/003-GPT-XL/checkpoints/eval_step_4000/batch_2_cfg_7.5_topk_16384.jpg filter=lfs diff=lfs merge=lfs -text +CKPTS/X2I_UltraEdiit_fourmask_LLava_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_filtered_objects_4mask_1280_context/003-GPT-XL/checkpoints/eval_step_4000/batch_3_cfg_7.5_topk_16384.jpg filter=lfs diff=lfs merge=lfs -text +CKPTS/X2I_UltraEdiit_fourmask_LLava_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_filtered_objects_4mask_1280_context/003-GPT-XL/checkpoints/eval_step_4000/batch_4_cfg_7.5_topk_16384.jpg filter=lfs diff=lfs merge=lfs -text +CKPTS/X2I_UltraEdiit_fourmask_LLava_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_filtered_objects_4mask_1280_context/003-GPT-XL/checkpoints/eval_step_40000/batch_0_cfg_7.5_topk_16384.jpg filter=lfs diff=lfs merge=lfs -text +CKPTS/X2I_UltraEdiit_fourmask_LLava_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_filtered_objects_4mask_1280_context/003-GPT-XL/checkpoints/eval_step_40000/batch_1_cfg_7.5_topk_16384.jpg filter=lfs diff=lfs merge=lfs -text +CKPTS/X2I_UltraEdiit_fourmask_LLava_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_filtered_objects_4mask_1280_context/003-GPT-XL/checkpoints/eval_step_40000/batch_2_cfg_7.5_topk_16384.jpg filter=lfs diff=lfs merge=lfs -text +CKPTS/X2I_UltraEdiit_fourmask_LLava_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_filtered_objects_4mask_1280_context/003-GPT-XL/checkpoints/eval_step_40000/batch_3_cfg_7.5_topk_16384.jpg filter=lfs diff=lfs merge=lfs -text +CKPTS/X2I_UltraEdiit_fourmask_LLava_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_filtered_objects_4mask_1280_context/003-GPT-XL/checkpoints/eval_step_40000/batch_4_cfg_7.5_topk_16384.jpg filter=lfs diff=lfs merge=lfs -text +CKPTS/X2I_UltraEdiit_fourmask_LLava_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_filtered_objects_4mask_1280_context/003-GPT-XL/checkpoints/eval_step_42000/batch_0_cfg_7.5_topk_16384.jpg filter=lfs diff=lfs merge=lfs -text +CKPTS/X2I_UltraEdiit_fourmask_LLava_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_filtered_objects_4mask_1280_context/003-GPT-XL/checkpoints/eval_step_42000/batch_1_cfg_7.5_topk_16384.jpg filter=lfs diff=lfs merge=lfs -text +CKPTS/X2I_UltraEdiit_fourmask_LLava_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_filtered_objects_4mask_1280_context/003-GPT-XL/checkpoints/eval_step_42000/batch_2_cfg_7.5_topk_16384.jpg filter=lfs diff=lfs merge=lfs -text +CKPTS/X2I_UltraEdiit_fourmask_LLava_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_filtered_objects_4mask_1280_context/003-GPT-XL/checkpoints/eval_step_42000/batch_3_cfg_7.5_topk_16384.jpg filter=lfs diff=lfs merge=lfs -text +CKPTS/X2I_UltraEdiit_fourmask_LLava_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_filtered_objects_4mask_1280_context/003-GPT-XL/checkpoints/eval_step_42000/batch_4_cfg_7.5_topk_16384.jpg filter=lfs diff=lfs merge=lfs -text +CKPTS/X2I_UltraEdiit_fourmask_LLava_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_filtered_objects_4mask_1280_context/003-GPT-XL/checkpoints/eval_step_44000/batch_0_cfg_7.5_topk_16384.jpg filter=lfs diff=lfs merge=lfs -text +CKPTS/X2I_UltraEdiit_fourmask_LLava_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_filtered_objects_4mask_1280_context/003-GPT-XL/checkpoints/eval_step_44000/batch_1_cfg_7.5_topk_16384.jpg filter=lfs diff=lfs merge=lfs -text +CKPTS/X2I_UltraEdiit_fourmask_LLava_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_filtered_objects_4mask_1280_context/003-GPT-XL/checkpoints/eval_step_44000/batch_2_cfg_7.5_topk_16384.jpg filter=lfs diff=lfs merge=lfs -text +CKPTS/X2I_UltraEdiit_fourmask_LLava_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_filtered_objects_4mask_1280_context/003-GPT-XL/checkpoints/eval_step_44000/batch_3_cfg_7.5_topk_16384.jpg filter=lfs diff=lfs merge=lfs -text +CKPTS/X2I_UltraEdiit_fourmask_LLava_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_filtered_objects_4mask_1280_context/003-GPT-XL/checkpoints/eval_step_44000/batch_4_cfg_7.5_topk_16384.jpg filter=lfs diff=lfs merge=lfs -text +CKPTS/X2I_UltraEdiit_fourmask_LLava_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_filtered_objects_4mask_1280_context/003-GPT-XL/checkpoints/eval_step_46000/batch_0_cfg_7.5_topk_16384.jpg filter=lfs diff=lfs merge=lfs -text +CKPTS/X2I_UltraEdiit_fourmask_LLava_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_filtered_objects_4mask_1280_context/003-GPT-XL/checkpoints/eval_step_46000/batch_1_cfg_7.5_topk_16384.jpg filter=lfs diff=lfs merge=lfs -text +CKPTS/X2I_UltraEdiit_fourmask_LLava_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_filtered_objects_4mask_1280_context/003-GPT-XL/checkpoints/eval_step_46000/batch_2_cfg_7.5_topk_16384.jpg filter=lfs diff=lfs merge=lfs -text +CKPTS/X2I_UltraEdiit_fourmask_LLava_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_filtered_objects_4mask_1280_context/003-GPT-XL/checkpoints/eval_step_46000/batch_3_cfg_7.5_topk_16384.jpg filter=lfs diff=lfs merge=lfs -text +CKPTS/X2I_UltraEdiit_fourmask_LLava_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_filtered_objects_4mask_1280_context/003-GPT-XL/checkpoints/eval_step_46000/batch_4_cfg_7.5_topk_16384.jpg filter=lfs diff=lfs merge=lfs -text +CKPTS/X2I_UltraEdiit_fourmask_LLava_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_filtered_objects_4mask_1280_context/003-GPT-XL/checkpoints/eval_step_48000/batch_0_cfg_7.5_topk_16384.jpg filter=lfs diff=lfs merge=lfs -text +CKPTS/X2I_UltraEdiit_fourmask_LLava_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_filtered_objects_4mask_1280_context/003-GPT-XL/checkpoints/eval_step_48000/batch_1_cfg_7.5_topk_16384.jpg filter=lfs diff=lfs merge=lfs -text +CKPTS/X2I_UltraEdiit_fourmask_LLava_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_filtered_objects_4mask_1280_context/003-GPT-XL/checkpoints/eval_step_48000/batch_2_cfg_7.5_topk_16384.jpg filter=lfs diff=lfs merge=lfs -text +CKPTS/X2I_UltraEdiit_fourmask_LLava_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_filtered_objects_4mask_1280_context/003-GPT-XL/checkpoints/eval_step_48000/batch_3_cfg_7.5_topk_16384.jpg filter=lfs diff=lfs merge=lfs -text +CKPTS/X2I_UltraEdiit_fourmask_LLava_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_filtered_objects_4mask_1280_context/003-GPT-XL/checkpoints/eval_step_48000/batch_4_cfg_7.5_topk_16384.jpg filter=lfs diff=lfs merge=lfs -text +CKPTS/X2I_UltraEdiit_fourmask_LLava_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_filtered_objects_4mask_1280_context/003-GPT-XL/checkpoints/eval_step_50000/batch_0_cfg_7.5_topk_16384.jpg filter=lfs diff=lfs merge=lfs -text +CKPTS/X2I_UltraEdiit_fourmask_LLava_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_filtered_objects_4mask_1280_context/003-GPT-XL/checkpoints/eval_step_50000/batch_1_cfg_7.5_topk_16384.jpg filter=lfs diff=lfs merge=lfs -text +CKPTS/X2I_UltraEdiit_fourmask_LLava_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_filtered_objects_4mask_1280_context/003-GPT-XL/checkpoints/eval_step_50000/batch_2_cfg_7.5_topk_16384.jpg filter=lfs diff=lfs merge=lfs -text +CKPTS/X2I_UltraEdiit_fourmask_LLava_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_filtered_objects_4mask_1280_context/003-GPT-XL/checkpoints/eval_step_50000/batch_3_cfg_7.5_topk_16384.jpg filter=lfs diff=lfs merge=lfs -text +CKPTS/X2I_UltraEdiit_fourmask_LLava_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_filtered_objects_4mask_1280_context/003-GPT-XL/checkpoints/eval_step_50000/batch_4_cfg_7.5_topk_16384.jpg filter=lfs diff=lfs merge=lfs -text +CKPTS/X2I_UltraEdiit_fourmask_LLava_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_filtered_objects_4mask_1280_context/003-GPT-XL/checkpoints/eval_step_52000/batch_0_cfg_7.5_topk_16384.jpg filter=lfs diff=lfs merge=lfs -text +CKPTS/X2I_UltraEdiit_fourmask_LLava_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_filtered_objects_4mask_1280_context/003-GPT-XL/checkpoints/eval_step_52000/batch_1_cfg_7.5_topk_16384.jpg filter=lfs diff=lfs merge=lfs -text +CKPTS/X2I_UltraEdiit_fourmask_LLava_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_filtered_objects_4mask_1280_context/003-GPT-XL/checkpoints/eval_step_52000/batch_2_cfg_7.5_topk_16384.jpg filter=lfs diff=lfs merge=lfs -text +CKPTS/X2I_UltraEdiit_fourmask_LLava_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_filtered_objects_4mask_1280_context/003-GPT-XL/checkpoints/eval_step_52000/batch_3_cfg_7.5_topk_16384.jpg filter=lfs diff=lfs merge=lfs -text +CKPTS/X2I_UltraEdiit_fourmask_LLava_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_filtered_objects_4mask_1280_context/003-GPT-XL/checkpoints/eval_step_52000/batch_4_cfg_7.5_topk_16384.jpg filter=lfs diff=lfs merge=lfs -text +CKPTS/X2I_UltraEdiit_fourmask_LLava_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_filtered_objects_4mask_1280_context/003-GPT-XL/checkpoints/eval_step_54000/batch_0_cfg_7.5_topk_16384.jpg filter=lfs diff=lfs merge=lfs -text +CKPTS/X2I_UltraEdiit_fourmask_LLava_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_filtered_objects_4mask_1280_context/003-GPT-XL/checkpoints/eval_step_54000/batch_1_cfg_7.5_topk_16384.jpg filter=lfs diff=lfs merge=lfs -text +CKPTS/X2I_UltraEdiit_fourmask_LLava_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_filtered_objects_4mask_1280_context/003-GPT-XL/checkpoints/eval_step_54000/batch_2_cfg_7.5_topk_16384.jpg filter=lfs diff=lfs merge=lfs -text +CKPTS/X2I_UltraEdiit_fourmask_LLava_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_filtered_objects_4mask_1280_context/003-GPT-XL/checkpoints/eval_step_54000/batch_3_cfg_7.5_topk_16384.jpg filter=lfs diff=lfs merge=lfs -text +CKPTS/X2I_UltraEdiit_fourmask_LLava_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_filtered_objects_4mask_1280_context/003-GPT-XL/checkpoints/eval_step_54000/batch_4_cfg_7.5_topk_16384.jpg filter=lfs diff=lfs merge=lfs -text +CKPTS/X2I_UltraEdiit_fourmask_LLava_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_filtered_objects_4mask_1280_context/003-GPT-XL/checkpoints/eval_step_56000/batch_0_cfg_7.5_topk_16384.jpg filter=lfs diff=lfs merge=lfs -text +CKPTS/X2I_UltraEdiit_fourmask_LLava_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_filtered_objects_4mask_1280_context/003-GPT-XL/checkpoints/eval_step_56000/batch_1_cfg_7.5_topk_16384.jpg filter=lfs diff=lfs merge=lfs -text +CKPTS/X2I_UltraEdiit_fourmask_LLava_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_filtered_objects_4mask_1280_context/003-GPT-XL/checkpoints/eval_step_56000/batch_2_cfg_7.5_topk_16384.jpg filter=lfs diff=lfs merge=lfs -text +CKPTS/X2I_UltraEdiit_fourmask_LLava_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_filtered_objects_4mask_1280_context/003-GPT-XL/checkpoints/eval_step_56000/batch_3_cfg_7.5_topk_16384.jpg filter=lfs diff=lfs merge=lfs -text +CKPTS/X2I_UltraEdiit_fourmask_LLava_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_filtered_objects_4mask_1280_context/003-GPT-XL/checkpoints/eval_step_56000/batch_4_cfg_7.5_topk_16384.jpg filter=lfs diff=lfs merge=lfs -text +CKPTS/X2I_UltraEdiit_fourmask_LLava_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_filtered_objects_4mask_1280_context/003-GPT-XL/checkpoints/eval_step_58000/batch_0_cfg_7.5_topk_16384.jpg filter=lfs diff=lfs merge=lfs -text +CKPTS/X2I_UltraEdiit_fourmask_LLava_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_filtered_objects_4mask_1280_context/003-GPT-XL/checkpoints/eval_step_58000/batch_1_cfg_7.5_topk_16384.jpg filter=lfs diff=lfs merge=lfs -text +CKPTS/X2I_UltraEdiit_fourmask_LLava_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_filtered_objects_4mask_1280_context/003-GPT-XL/checkpoints/eval_step_58000/batch_2_cfg_7.5_topk_16384.jpg filter=lfs diff=lfs merge=lfs -text +CKPTS/X2I_UltraEdiit_fourmask_LLava_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_filtered_objects_4mask_1280_context/003-GPT-XL/checkpoints/eval_step_58000/batch_3_cfg_7.5_topk_16384.jpg filter=lfs diff=lfs merge=lfs -text +CKPTS/X2I_UltraEdiit_fourmask_LLava_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_filtered_objects_4mask_1280_context/003-GPT-XL/checkpoints/eval_step_58000/batch_4_cfg_7.5_topk_16384.jpg filter=lfs diff=lfs merge=lfs -text +CKPTS/X2I_UltraEdiit_fourmask_LLava_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_filtered_objects_4mask_1280_context/003-GPT-XL/checkpoints/eval_step_6000/batch_0_cfg_7.5_topk_16384.jpg filter=lfs diff=lfs merge=lfs -text +CKPTS/X2I_UltraEdiit_fourmask_LLava_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_filtered_objects_4mask_1280_context/003-GPT-XL/checkpoints/eval_step_6000/batch_1_cfg_7.5_topk_16384.jpg filter=lfs diff=lfs merge=lfs -text +CKPTS/X2I_UltraEdiit_fourmask_LLava_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_filtered_objects_4mask_1280_context/003-GPT-XL/checkpoints/eval_step_6000/batch_2_cfg_7.5_topk_16384.jpg filter=lfs diff=lfs merge=lfs -text +CKPTS/X2I_UltraEdiit_fourmask_LLava_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_filtered_objects_4mask_1280_context/003-GPT-XL/checkpoints/eval_step_6000/batch_3_cfg_7.5_topk_16384.jpg filter=lfs diff=lfs merge=lfs -text +CKPTS/X2I_UltraEdiit_fourmask_LLava_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_filtered_objects_4mask_1280_context/003-GPT-XL/checkpoints/eval_step_6000/batch_4_cfg_7.5_topk_16384.jpg filter=lfs diff=lfs merge=lfs -text +CKPTS/X2I_UltraEdiit_fourmask_LLava_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_filtered_objects_4mask_1280_context/003-GPT-XL/checkpoints/eval_step_60000/batch_0_cfg_7.5_topk_16384.jpg filter=lfs diff=lfs merge=lfs -text +CKPTS/X2I_UltraEdiit_fourmask_LLava_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_filtered_objects_4mask_1280_context/003-GPT-XL/checkpoints/eval_step_60000/batch_1_cfg_7.5_topk_16384.jpg filter=lfs diff=lfs merge=lfs -text +CKPTS/X2I_UltraEdiit_fourmask_LLava_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_filtered_objects_4mask_1280_context/003-GPT-XL/checkpoints/eval_step_60000/batch_2_cfg_7.5_topk_16384.jpg filter=lfs diff=lfs merge=lfs -text +CKPTS/X2I_UltraEdiit_fourmask_LLava_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_filtered_objects_4mask_1280_context/003-GPT-XL/checkpoints/eval_step_60000/batch_3_cfg_7.5_topk_16384.jpg filter=lfs diff=lfs merge=lfs -text +CKPTS/X2I_UltraEdiit_fourmask_LLava_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_filtered_objects_4mask_1280_context/003-GPT-XL/checkpoints/eval_step_60000/batch_4_cfg_7.5_topk_16384.jpg filter=lfs diff=lfs merge=lfs -text +CKPTS/X2I_UltraEdiit_fourmask_LLava_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_filtered_objects_4mask_1280_context/003-GPT-XL/checkpoints/eval_step_62000/batch_0_cfg_7.5_topk_16384.jpg filter=lfs diff=lfs merge=lfs -text +CKPTS/X2I_UltraEdiit_fourmask_LLava_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_filtered_objects_4mask_1280_context/003-GPT-XL/checkpoints/eval_step_62000/batch_1_cfg_7.5_topk_16384.jpg filter=lfs diff=lfs merge=lfs -text +CKPTS/X2I_UltraEdiit_fourmask_LLava_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_filtered_objects_4mask_1280_context/003-GPT-XL/checkpoints/eval_step_62000/batch_2_cfg_7.5_topk_16384.jpg filter=lfs diff=lfs merge=lfs -text +CKPTS/X2I_UltraEdiit_fourmask_LLava_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_filtered_objects_4mask_1280_context/003-GPT-XL/checkpoints/eval_step_62000/batch_3_cfg_7.5_topk_16384.jpg filter=lfs diff=lfs merge=lfs -text +CKPTS/X2I_UltraEdiit_fourmask_LLava_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_filtered_objects_4mask_1280_context/003-GPT-XL/checkpoints/eval_step_62000/batch_4_cfg_7.5_topk_16384.jpg filter=lfs diff=lfs merge=lfs -text +CKPTS/X2I_UltraEdiit_fourmask_LLava_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_filtered_objects_4mask_1280_context/003-GPT-XL/checkpoints/eval_step_64000/batch_0_cfg_7.5_topk_16384.jpg filter=lfs diff=lfs merge=lfs -text +CKPTS/X2I_UltraEdiit_fourmask_LLava_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_filtered_objects_4mask_1280_context/003-GPT-XL/checkpoints/eval_step_64000/batch_1_cfg_7.5_topk_16384.jpg filter=lfs diff=lfs merge=lfs -text +CKPTS/X2I_UltraEdiit_fourmask_LLava_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_filtered_objects_4mask_1280_context/003-GPT-XL/checkpoints/eval_step_64000/batch_2_cfg_7.5_topk_16384.jpg filter=lfs diff=lfs merge=lfs -text +CKPTS/X2I_UltraEdiit_fourmask_LLava_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_filtered_objects_4mask_1280_context/003-GPT-XL/checkpoints/eval_step_64000/batch_3_cfg_7.5_topk_16384.jpg filter=lfs diff=lfs merge=lfs -text +CKPTS/X2I_UltraEdiit_fourmask_LLava_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_filtered_objects_4mask_1280_context/003-GPT-XL/checkpoints/eval_step_64000/batch_4_cfg_7.5_topk_16384.jpg filter=lfs diff=lfs merge=lfs -text +CKPTS/X2I_UltraEdiit_fourmask_LLava_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_filtered_objects_4mask_1280_context/003-GPT-XL/checkpoints/eval_step_66000/batch_0_cfg_7.5_topk_16384.jpg filter=lfs diff=lfs merge=lfs -text +CKPTS/X2I_UltraEdiit_fourmask_LLava_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_filtered_objects_4mask_1280_context/003-GPT-XL/checkpoints/eval_step_66000/batch_1_cfg_7.5_topk_16384.jpg filter=lfs diff=lfs merge=lfs -text +CKPTS/X2I_UltraEdiit_fourmask_LLava_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_filtered_objects_4mask_1280_context/003-GPT-XL/checkpoints/eval_step_66000/batch_2_cfg_7.5_topk_16384.jpg filter=lfs diff=lfs merge=lfs -text +CKPTS/X2I_UltraEdiit_fourmask_LLava_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_filtered_objects_4mask_1280_context/003-GPT-XL/checkpoints/eval_step_66000/batch_3_cfg_7.5_topk_16384.jpg filter=lfs diff=lfs merge=lfs -text +CKPTS/X2I_UltraEdiit_fourmask_LLava_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_filtered_objects_4mask_1280_context/003-GPT-XL/checkpoints/eval_step_66000/batch_4_cfg_7.5_topk_16384.jpg filter=lfs diff=lfs merge=lfs -text +CKPTS/X2I_UltraEdiit_fourmask_LLava_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_filtered_objects_4mask_1280_context/003-GPT-XL/checkpoints/eval_step_68000/batch_0_cfg_7.5_topk_16384.jpg filter=lfs diff=lfs merge=lfs -text +CKPTS/X2I_UltraEdiit_fourmask_LLava_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_filtered_objects_4mask_1280_context/003-GPT-XL/checkpoints/eval_step_68000/batch_1_cfg_7.5_topk_16384.jpg filter=lfs diff=lfs merge=lfs -text +CKPTS/X2I_UltraEdiit_fourmask_LLava_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_filtered_objects_4mask_1280_context/003-GPT-XL/checkpoints/eval_step_68000/batch_2_cfg_7.5_topk_16384.jpg filter=lfs diff=lfs merge=lfs -text +CKPTS/X2I_UltraEdiit_fourmask_LLava_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_filtered_objects_4mask_1280_context/003-GPT-XL/checkpoints/eval_step_68000/batch_3_cfg_7.5_topk_16384.jpg filter=lfs diff=lfs merge=lfs -text +CKPTS/X2I_UltraEdiit_fourmask_LLava_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_filtered_objects_4mask_1280_context/003-GPT-XL/checkpoints/eval_step_68000/batch_4_cfg_7.5_topk_16384.jpg filter=lfs diff=lfs merge=lfs -text +CKPTS/X2I_UltraEdiit_fourmask_LLava_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_filtered_objects_4mask_1280_context/003-GPT-XL/checkpoints/eval_step_70000/batch_0_cfg_7.5_topk_16384.jpg filter=lfs diff=lfs merge=lfs -text +CKPTS/X2I_UltraEdiit_fourmask_LLava_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_filtered_objects_4mask_1280_context/003-GPT-XL/checkpoints/eval_step_70000/batch_1_cfg_7.5_topk_16384.jpg filter=lfs diff=lfs merge=lfs -text +CKPTS/X2I_UltraEdiit_fourmask_LLava_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_filtered_objects_4mask_1280_context/003-GPT-XL/checkpoints/eval_step_70000/batch_2_cfg_7.5_topk_16384.jpg filter=lfs diff=lfs merge=lfs -text +CKPTS/X2I_UltraEdiit_fourmask_LLava_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_filtered_objects_4mask_1280_context/003-GPT-XL/checkpoints/eval_step_70000/batch_3_cfg_7.5_topk_16384.jpg filter=lfs diff=lfs merge=lfs -text +CKPTS/X2I_UltraEdiit_fourmask_LLava_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_filtered_objects_4mask_1280_context/003-GPT-XL/checkpoints/eval_step_70000/batch_4_cfg_7.5_topk_16384.jpg filter=lfs diff=lfs merge=lfs -text +CKPTS/X2I_UltraEdiit_fourmask_LLava_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_filtered_objects_4mask_1280_context/003-GPT-XL/checkpoints/eval_step_72000/batch_0_cfg_7.5_topk_16384.jpg filter=lfs diff=lfs merge=lfs -text +CKPTS/X2I_UltraEdiit_fourmask_LLava_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_filtered_objects_4mask_1280_context/003-GPT-XL/checkpoints/eval_step_72000/batch_1_cfg_7.5_topk_16384.jpg filter=lfs diff=lfs merge=lfs -text +CKPTS/X2I_UltraEdiit_fourmask_LLava_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_filtered_objects_4mask_1280_context/003-GPT-XL/checkpoints/eval_step_72000/batch_2_cfg_7.5_topk_16384.jpg filter=lfs diff=lfs merge=lfs -text +CKPTS/X2I_UltraEdiit_fourmask_LLava_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_filtered_objects_4mask_1280_context/003-GPT-XL/checkpoints/eval_step_72000/batch_3_cfg_7.5_topk_16384.jpg filter=lfs diff=lfs merge=lfs -text +CKPTS/X2I_UltraEdiit_fourmask_LLava_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_filtered_objects_4mask_1280_context/003-GPT-XL/checkpoints/eval_step_72000/batch_4_cfg_7.5_topk_16384.jpg filter=lfs diff=lfs merge=lfs -text +CKPTS/X2I_UltraEdiit_fourmask_LLava_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_filtered_objects_4mask_1280_context/003-GPT-XL/checkpoints/eval_step_74000/batch_0_cfg_7.5_topk_16384.jpg filter=lfs diff=lfs merge=lfs -text +CKPTS/X2I_UltraEdiit_fourmask_LLava_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_filtered_objects_4mask_1280_context/003-GPT-XL/checkpoints/eval_step_74000/batch_1_cfg_7.5_topk_16384.jpg filter=lfs diff=lfs merge=lfs -text +CKPTS/X2I_UltraEdiit_fourmask_LLava_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_filtered_objects_4mask_1280_context/003-GPT-XL/checkpoints/eval_step_74000/batch_2_cfg_7.5_topk_16384.jpg filter=lfs diff=lfs merge=lfs -text +CKPTS/X2I_UltraEdiit_fourmask_LLava_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_filtered_objects_4mask_1280_context/003-GPT-XL/checkpoints/eval_step_74000/batch_3_cfg_7.5_topk_16384.jpg filter=lfs diff=lfs merge=lfs -text +CKPTS/X2I_UltraEdiit_fourmask_LLava_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_filtered_objects_4mask_1280_context/003-GPT-XL/checkpoints/eval_step_74000/batch_4_cfg_7.5_topk_16384.jpg filter=lfs diff=lfs merge=lfs -text +CKPTS/X2I_UltraEdiit_fourmask_LLava_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_filtered_objects_4mask_1280_context/003-GPT-XL/checkpoints/eval_step_76000/batch_0_cfg_7.5_topk_16384.jpg filter=lfs diff=lfs merge=lfs -text +CKPTS/X2I_UltraEdiit_fourmask_LLava_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_filtered_objects_4mask_1280_context/003-GPT-XL/checkpoints/eval_step_76000/batch_1_cfg_7.5_topk_16384.jpg filter=lfs diff=lfs merge=lfs -text +CKPTS/X2I_UltraEdiit_fourmask_LLava_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_filtered_objects_4mask_1280_context/003-GPT-XL/checkpoints/eval_step_76000/batch_2_cfg_7.5_topk_16384.jpg filter=lfs diff=lfs merge=lfs -text +CKPTS/X2I_UltraEdiit_fourmask_LLava_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_filtered_objects_4mask_1280_context/003-GPT-XL/checkpoints/eval_step_76000/batch_3_cfg_7.5_topk_16384.jpg filter=lfs diff=lfs merge=lfs -text +CKPTS/X2I_UltraEdiit_fourmask_LLava_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_filtered_objects_4mask_1280_context/003-GPT-XL/checkpoints/eval_step_76000/batch_4_cfg_7.5_topk_16384.jpg filter=lfs diff=lfs merge=lfs -text +CKPTS/X2I_UltraEdiit_fourmask_LLava_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_filtered_objects_4mask_1280_context/003-GPT-XL/checkpoints/eval_step_78000/batch_0_cfg_7.5_topk_16384.jpg filter=lfs diff=lfs merge=lfs -text +CKPTS/X2I_UltraEdiit_fourmask_LLava_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_filtered_objects_4mask_1280_context/003-GPT-XL/checkpoints/eval_step_78000/batch_1_cfg_7.5_topk_16384.jpg filter=lfs diff=lfs merge=lfs -text +CKPTS/X2I_UltraEdiit_fourmask_LLava_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_filtered_objects_4mask_1280_context/003-GPT-XL/checkpoints/eval_step_78000/batch_2_cfg_7.5_topk_16384.jpg filter=lfs diff=lfs merge=lfs -text +CKPTS/X2I_UltraEdiit_fourmask_LLava_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_filtered_objects_4mask_1280_context/003-GPT-XL/checkpoints/eval_step_78000/batch_3_cfg_7.5_topk_16384.jpg filter=lfs diff=lfs merge=lfs -text +CKPTS/X2I_UltraEdiit_fourmask_LLava_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_filtered_objects_4mask_1280_context/003-GPT-XL/checkpoints/eval_step_78000/batch_4_cfg_7.5_topk_16384.jpg filter=lfs diff=lfs merge=lfs -text +CKPTS/X2I_UltraEdiit_fourmask_LLava_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_filtered_objects_4mask_1280_context/003-GPT-XL/checkpoints/eval_step_8000/batch_0_cfg_7.5_topk_16384.jpg filter=lfs diff=lfs merge=lfs -text +CKPTS/X2I_UltraEdiit_fourmask_LLava_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_filtered_objects_4mask_1280_context/003-GPT-XL/checkpoints/eval_step_8000/batch_1_cfg_7.5_topk_16384.jpg filter=lfs diff=lfs merge=lfs -text +CKPTS/X2I_UltraEdiit_fourmask_LLava_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_filtered_objects_4mask_1280_context/003-GPT-XL/checkpoints/eval_step_8000/batch_2_cfg_7.5_topk_16384.jpg filter=lfs diff=lfs merge=lfs -text +CKPTS/X2I_UltraEdiit_fourmask_LLava_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_filtered_objects_4mask_1280_context/003-GPT-XL/checkpoints/eval_step_8000/batch_3_cfg_7.5_topk_16384.jpg filter=lfs diff=lfs merge=lfs -text +CKPTS/X2I_UltraEdiit_fourmask_LLava_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_filtered_objects_4mask_1280_context/003-GPT-XL/checkpoints/eval_step_8000/batch_4_cfg_7.5_topk_16384.jpg filter=lfs diff=lfs merge=lfs -text +CKPTS/X2I_UltraEdiit_fourmask_LLava_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_filtered_objects_4mask_1280_context/003-GPT-XL/checkpoints/eval_step_80000/batch_0_cfg_7.5_topk_16384.jpg filter=lfs diff=lfs merge=lfs -text +CKPTS/X2I_UltraEdiit_fourmask_LLava_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_filtered_objects_4mask_1280_context/003-GPT-XL/checkpoints/eval_step_80000/batch_1_cfg_7.5_topk_16384.jpg filter=lfs diff=lfs merge=lfs -text +CKPTS/X2I_UltraEdiit_fourmask_LLava_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_filtered_objects_4mask_1280_context/003-GPT-XL/checkpoints/eval_step_80000/batch_2_cfg_7.5_topk_16384.jpg filter=lfs diff=lfs merge=lfs -text +CKPTS/X2I_UltraEdiit_fourmask_LLava_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_filtered_objects_4mask_1280_context/003-GPT-XL/checkpoints/eval_step_80000/batch_3_cfg_7.5_topk_16384.jpg filter=lfs diff=lfs merge=lfs -text +CKPTS/X2I_UltraEdiit_fourmask_LLava_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_filtered_objects_4mask_1280_context/003-GPT-XL/checkpoints/eval_step_80000/batch_4_cfg_7.5_topk_16384.jpg filter=lfs diff=lfs merge=lfs -text +CKPTS/X2I_UltraEdiit_fourmask_LLava_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_filtered_objects_4mask_1280_context/003-GPT-XL/checkpoints/eval_step_82000/batch_0_cfg_7.5_topk_16384.jpg filter=lfs diff=lfs merge=lfs -text +CKPTS/X2I_UltraEdiit_fourmask_LLava_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_filtered_objects_4mask_1280_context/003-GPT-XL/checkpoints/eval_step_82000/batch_1_cfg_7.5_topk_16384.jpg filter=lfs diff=lfs merge=lfs -text +CKPTS/X2I_UltraEdiit_fourmask_LLava_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_filtered_objects_4mask_1280_context/003-GPT-XL/checkpoints/eval_step_82000/batch_2_cfg_7.5_topk_16384.jpg filter=lfs diff=lfs merge=lfs -text +CKPTS/X2I_UltraEdiit_fourmask_LLava_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_filtered_objects_4mask_1280_context/003-GPT-XL/checkpoints/eval_step_82000/batch_3_cfg_7.5_topk_16384.jpg filter=lfs diff=lfs merge=lfs -text +CKPTS/X2I_UltraEdiit_fourmask_LLava_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_filtered_objects_4mask_1280_context/003-GPT-XL/checkpoints/eval_step_82000/batch_4_cfg_7.5_topk_16384.jpg filter=lfs diff=lfs merge=lfs -text +CKPTS/X2I_UltraEdiit_fourmask_LLava_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_filtered_objects_4mask_1280_context/003-GPT-XL/checkpoints/eval_step_84000/batch_0_cfg_7.5_topk_16384.jpg filter=lfs diff=lfs merge=lfs -text +CKPTS/X2I_UltraEdiit_fourmask_LLava_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_filtered_objects_4mask_1280_context/003-GPT-XL/checkpoints/eval_step_84000/batch_1_cfg_7.5_topk_16384.jpg filter=lfs diff=lfs merge=lfs -text +CKPTS/X2I_UltraEdiit_fourmask_LLava_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_filtered_objects_4mask_1280_context/003-GPT-XL/checkpoints/eval_step_84000/batch_2_cfg_7.5_topk_16384.jpg filter=lfs diff=lfs merge=lfs -text +CKPTS/X2I_UltraEdiit_fourmask_LLava_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_filtered_objects_4mask_1280_context/003-GPT-XL/checkpoints/eval_step_84000/batch_3_cfg_7.5_topk_16384.jpg filter=lfs diff=lfs merge=lfs -text +CKPTS/X2I_UltraEdiit_fourmask_LLava_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_filtered_objects_4mask_1280_context/003-GPT-XL/checkpoints/eval_step_84000/batch_4_cfg_7.5_topk_16384.jpg filter=lfs diff=lfs merge=lfs -text +CKPTS/X2I_UltraEdiit_fourmask_LLava_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_filtered_objects_4mask_1280_context/003-GPT-XL/checkpoints/eval_step_86000/batch_0_cfg_7.5_topk_16384.jpg filter=lfs diff=lfs merge=lfs -text +CKPTS/X2I_UltraEdiit_fourmask_LLava_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_filtered_objects_4mask_1280_context/003-GPT-XL/checkpoints/eval_step_86000/batch_1_cfg_7.5_topk_16384.jpg filter=lfs diff=lfs merge=lfs -text +CKPTS/X2I_UltraEdiit_fourmask_LLava_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_filtered_objects_4mask_1280_context/003-GPT-XL/checkpoints/eval_step_86000/batch_2_cfg_7.5_topk_16384.jpg filter=lfs diff=lfs merge=lfs -text +CKPTS/X2I_UltraEdiit_fourmask_LLava_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_filtered_objects_4mask_1280_context/003-GPT-XL/checkpoints/eval_step_86000/batch_3_cfg_7.5_topk_16384.jpg filter=lfs diff=lfs merge=lfs -text +CKPTS/X2I_UltraEdiit_fourmask_LLava_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_filtered_objects_4mask_1280_context/003-GPT-XL/checkpoints/eval_step_86000/batch_4_cfg_7.5_topk_16384.jpg filter=lfs diff=lfs merge=lfs -text +CKPTS/X2I_UltraEdiit_fourmask_LLava_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_filtered_objects_4mask_1280_context/003-GPT-XL/checkpoints/eval_step_88000/batch_0_cfg_7.5_topk_16384.jpg filter=lfs diff=lfs merge=lfs -text +CKPTS/X2I_UltraEdiit_fourmask_LLava_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_filtered_objects_4mask_1280_context/003-GPT-XL/checkpoints/eval_step_88000/batch_1_cfg_7.5_topk_16384.jpg filter=lfs diff=lfs merge=lfs -text +CKPTS/X2I_UltraEdiit_fourmask_LLava_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_filtered_objects_4mask_1280_context/003-GPT-XL/checkpoints/eval_step_88000/batch_2_cfg_7.5_topk_16384.jpg filter=lfs diff=lfs merge=lfs -text +CKPTS/X2I_UltraEdiit_fourmask_LLava_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_filtered_objects_4mask_1280_context/003-GPT-XL/checkpoints/eval_step_88000/batch_3_cfg_7.5_topk_16384.jpg filter=lfs diff=lfs merge=lfs -text +CKPTS/X2I_UltraEdiit_fourmask_LLava_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_filtered_objects_4mask_1280_context/003-GPT-XL/checkpoints/eval_step_88000/batch_4_cfg_7.5_topk_16384.jpg filter=lfs diff=lfs merge=lfs -text +CKPTS/X2I_UltraEdiit_fourmask_LLava_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_filtered_objects_4mask_1280_context/003-GPT-XL/checkpoints/eval_step_90000/batch_0_cfg_7.5_topk_16384.jpg filter=lfs diff=lfs merge=lfs -text +CKPTS/X2I_UltraEdiit_fourmask_LLava_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_filtered_objects_4mask_1280_context/003-GPT-XL/checkpoints/eval_step_90000/batch_1_cfg_7.5_topk_16384.jpg filter=lfs diff=lfs merge=lfs -text +CKPTS/X2I_UltraEdiit_fourmask_LLava_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_filtered_objects_4mask_1280_context/003-GPT-XL/checkpoints/eval_step_90000/batch_2_cfg_7.5_topk_16384.jpg filter=lfs diff=lfs merge=lfs -text +CKPTS/X2I_UltraEdiit_fourmask_LLava_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_filtered_objects_4mask_1280_context/003-GPT-XL/checkpoints/eval_step_90000/batch_3_cfg_7.5_topk_16384.jpg filter=lfs diff=lfs merge=lfs -text +CKPTS/X2I_UltraEdiit_fourmask_LLava_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_filtered_objects_4mask_1280_context/003-GPT-XL/checkpoints/eval_step_90000/batch_4_cfg_7.5_topk_16384.jpg filter=lfs diff=lfs merge=lfs -text +CKPTS/X2I_UltraEdiit_fourmask_LLava_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_filtered_objects_4mask_1280_context/003-GPT-XL/checkpoints/eval_step_92000/batch_0_cfg_7.5_topk_16384.jpg filter=lfs diff=lfs merge=lfs -text +CKPTS/X2I_UltraEdiit_fourmask_LLava_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_filtered_objects_4mask_1280_context/003-GPT-XL/checkpoints/eval_step_92000/batch_1_cfg_7.5_topk_16384.jpg filter=lfs diff=lfs merge=lfs -text +CKPTS/X2I_UltraEdiit_fourmask_LLava_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_filtered_objects_4mask_1280_context/003-GPT-XL/checkpoints/eval_step_92000/batch_2_cfg_7.5_topk_16384.jpg filter=lfs diff=lfs merge=lfs -text +CKPTS/X2I_UltraEdiit_fourmask_LLava_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_filtered_objects_4mask_1280_context/003-GPT-XL/checkpoints/eval_step_92000/batch_3_cfg_7.5_topk_16384.jpg filter=lfs diff=lfs merge=lfs -text +CKPTS/X2I_UltraEdiit_fourmask_LLava_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_filtered_objects_4mask_1280_context/003-GPT-XL/checkpoints/eval_step_92000/batch_4_cfg_7.5_topk_16384.jpg filter=lfs diff=lfs merge=lfs -text +CKPTS/X2I_UltraEdiit_fourmask_LLava_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_filtered_objects_4mask_1280_context/003-GPT-XL/checkpoints/eval_step_94000/batch_0_cfg_7.5_topk_16384.jpg filter=lfs diff=lfs merge=lfs -text +CKPTS/X2I_UltraEdiit_fourmask_LLava_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_filtered_objects_4mask_1280_context/003-GPT-XL/checkpoints/eval_step_94000/batch_1_cfg_7.5_topk_16384.jpg filter=lfs diff=lfs merge=lfs -text +CKPTS/X2I_UltraEdiit_fourmask_LLava_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_filtered_objects_4mask_1280_context/003-GPT-XL/checkpoints/eval_step_94000/batch_2_cfg_7.5_topk_16384.jpg filter=lfs diff=lfs merge=lfs -text +CKPTS/X2I_UltraEdiit_fourmask_LLava_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_filtered_objects_4mask_1280_context/003-GPT-XL/checkpoints/eval_step_94000/batch_3_cfg_7.5_topk_16384.jpg filter=lfs diff=lfs merge=lfs -text +CKPTS/X2I_UltraEdiit_fourmask_LLava_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_filtered_objects_4mask_1280_context/003-GPT-XL/checkpoints/eval_step_94000/batch_4_cfg_7.5_topk_16384.jpg filter=lfs diff=lfs merge=lfs -text +CKPTS/X2I_UltraEdiit_fourmask_LLava_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_filtered_objects_4mask_1280_context/003-GPT-XL/checkpoints/eval_step_96000/batch_0_cfg_7.5_topk_16384.jpg filter=lfs diff=lfs merge=lfs -text +CKPTS/X2I_UltraEdiit_fourmask_LLava_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_filtered_objects_4mask_1280_context/003-GPT-XL/checkpoints/eval_step_96000/batch_1_cfg_7.5_topk_16384.jpg filter=lfs diff=lfs merge=lfs -text +CKPTS/X2I_UltraEdiit_fourmask_LLava_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_filtered_objects_4mask_1280_context/003-GPT-XL/checkpoints/eval_step_96000/batch_2_cfg_7.5_topk_16384.jpg filter=lfs diff=lfs merge=lfs -text +CKPTS/X2I_UltraEdiit_fourmask_LLava_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_filtered_objects_4mask_1280_context/003-GPT-XL/checkpoints/eval_step_96000/batch_3_cfg_7.5_topk_16384.jpg filter=lfs diff=lfs merge=lfs -text +CKPTS/X2I_UltraEdiit_fourmask_LLava_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_filtered_objects_4mask_1280_context/003-GPT-XL/checkpoints/eval_step_96000/batch_4_cfg_7.5_topk_16384.jpg filter=lfs diff=lfs merge=lfs -text +CKPTS/X2I_UltraEdiit_fourmask_LLava_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_filtered_objects_4mask_1280_context/003-GPT-XL/checkpoints/eval_step_98000/batch_0_cfg_7.5_topk_16384.jpg filter=lfs diff=lfs merge=lfs -text +CKPTS/X2I_UltraEdiit_fourmask_LLava_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_filtered_objects_4mask_1280_context/003-GPT-XL/checkpoints/eval_step_98000/batch_1_cfg_7.5_topk_16384.jpg filter=lfs diff=lfs merge=lfs -text +CKPTS/X2I_UltraEdiit_fourmask_LLava_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_filtered_objects_4mask_1280_context/003-GPT-XL/checkpoints/eval_step_98000/batch_2_cfg_7.5_topk_16384.jpg filter=lfs diff=lfs merge=lfs -text +CKPTS/X2I_UltraEdiit_fourmask_LLava_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_filtered_objects_4mask_1280_context/003-GPT-XL/checkpoints/eval_step_98000/batch_3_cfg_7.5_topk_16384.jpg filter=lfs diff=lfs merge=lfs -text +CKPTS/X2I_UltraEdiit_fourmask_LLava_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_filtered_objects_4mask_1280_context/003-GPT-XL/checkpoints/eval_step_98000/batch_4_cfg_7.5_topk_16384.jpg filter=lfs diff=lfs merge=lfs -text +CKPTS/X2I_UltraEdiit_fourmask_LLava_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_filtered_objects_4mask_1280_context/wandb/run-20250426_205319-m68pf6i5/run-m68pf6i5.wandb filter=lfs diff=lfs merge=lfs -text +CKPTS/X2I_UltraEdiit_fourmask_LLava_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_filtered_objects_4mask_1280_context/wandb/run-20250426_210323-vtrxdt92/run-vtrxdt92.wandb filter=lfs diff=lfs merge=lfs -text +CKPTS/X2I_UltraEdiit_fourmask_LLava_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_filtered_objects_4mask_1280_context_after_stage1/000-GPT-XL/checkpoints/eval_step_2000/batch_0_cfg_7.5_topk_16384.jpg filter=lfs diff=lfs merge=lfs -text +CKPTS/X2I_UltraEdiit_fourmask_LLava_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_filtered_objects_4mask_1280_context_after_stage1/000-GPT-XL/checkpoints/eval_step_2000/batch_1_cfg_7.5_topk_16384.jpg filter=lfs diff=lfs merge=lfs -text +CKPTS/X2I_UltraEdiit_fourmask_LLava_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_filtered_objects_4mask_1280_context_after_stage1/000-GPT-XL/checkpoints/eval_step_2000/batch_2_cfg_7.5_topk_16384.jpg filter=lfs diff=lfs merge=lfs -text +CKPTS/X2I_UltraEdiit_fourmask_LLava_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_filtered_objects_4mask_1280_context_after_stage1/000-GPT-XL/checkpoints/eval_step_2000/batch_3_cfg_7.5_topk_16384.jpg filter=lfs diff=lfs merge=lfs -text +CKPTS/X2I_UltraEdiit_fourmask_LLava_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_filtered_objects_4mask_1280_context_after_stage1/000-GPT-XL/checkpoints/eval_step_2000/batch_4_cfg_7.5_topk_16384.jpg filter=lfs diff=lfs merge=lfs -text +CKPTS/X2I_UltraEdiit_fourmask_LLava_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_filtered_objects_4mask_1280_context_after_stage1/000-GPT-XL/checkpoints/eval_step_2000/batch_5_cfg_7.5_topk_16384.jpg filter=lfs diff=lfs merge=lfs -text +CKPTS/X2I_UltraEdiit_fourmask_LLava_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_filtered_objects_4mask_1280_context_after_stage1/000-GPT-XL/checkpoints/eval_step_4000/batch_0_cfg_7.5_topk_16384.jpg filter=lfs diff=lfs merge=lfs -text +CKPTS/X2I_UltraEdiit_fourmask_LLava_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_filtered_objects_4mask_1280_context_after_stage1/000-GPT-XL/checkpoints/eval_step_4000/batch_1_cfg_7.5_topk_16384.jpg filter=lfs diff=lfs merge=lfs -text +CKPTS/X2I_UltraEdiit_fourmask_LLava_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_filtered_objects_4mask_1280_context_after_stage1/000-GPT-XL/checkpoints/eval_step_4000/batch_2_cfg_7.5_topk_16384.jpg filter=lfs diff=lfs merge=lfs -text +CKPTS/X2I_UltraEdiit_fourmask_LLava_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_filtered_objects_4mask_1280_context_after_stage1/000-GPT-XL/checkpoints/eval_step_4000/batch_3_cfg_7.5_topk_16384.jpg filter=lfs diff=lfs merge=lfs -text +CKPTS/X2I_UltraEdiit_fourmask_LLava_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_filtered_objects_4mask_1280_context_after_stage1/000-GPT-XL/checkpoints/eval_step_4000/batch_4_cfg_7.5_topk_16384.jpg filter=lfs diff=lfs merge=lfs -text +CKPTS/X2I_UltraEdiit_fourmask_LLava_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_filtered_objects_4mask_1280_context_after_stage1/000-GPT-XL/checkpoints/eval_step_4000/batch_5_cfg_7.5_topk_16384.jpg filter=lfs diff=lfs merge=lfs -text +CKPTS/X2I_UltraEdiit_fourmask_LLava_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_filtered_objects_4mask_1280_context_after_stage1/wandb/run-20250428_085127-jzkufte5/run-jzkufte5.wandb filter=lfs diff=lfs merge=lfs -text +CKPTS/X2I_UltraEdiit_fourmask_LLava_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_filtered_objects_4mask_1280_context_after_stage1_5e4/000-GPT-XL/checkpoints/eval_step_10000/batch_0_cfg_7.5_topk_16384.jpg filter=lfs diff=lfs merge=lfs -text +CKPTS/X2I_UltraEdiit_fourmask_LLava_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_filtered_objects_4mask_1280_context_after_stage1_5e4/000-GPT-XL/checkpoints/eval_step_10000/batch_1_cfg_7.5_topk_16384.jpg filter=lfs diff=lfs merge=lfs -text +CKPTS/X2I_UltraEdiit_fourmask_LLava_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_filtered_objects_4mask_1280_context_after_stage1_5e4/000-GPT-XL/checkpoints/eval_step_10000/batch_2_cfg_7.5_topk_16384.jpg filter=lfs diff=lfs merge=lfs -text +CKPTS/X2I_UltraEdiit_fourmask_LLava_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_filtered_objects_4mask_1280_context_after_stage1_5e4/000-GPT-XL/checkpoints/eval_step_10000/batch_3_cfg_7.5_topk_16384.jpg filter=lfs diff=lfs merge=lfs -text +CKPTS/X2I_UltraEdiit_fourmask_LLava_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_filtered_objects_4mask_1280_context_after_stage1_5e4/000-GPT-XL/checkpoints/eval_step_10000/batch_4_cfg_7.5_topk_16384.jpg filter=lfs diff=lfs merge=lfs -text +CKPTS/X2I_UltraEdiit_fourmask_LLava_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_filtered_objects_4mask_1280_context_after_stage1_5e4/000-GPT-XL/checkpoints/eval_step_10000/batch_5_cfg_7.5_topk_16384.jpg filter=lfs diff=lfs merge=lfs -text +CKPTS/X2I_UltraEdiit_fourmask_LLava_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_filtered_objects_4mask_1280_context_after_stage1_5e4/000-GPT-XL/checkpoints/eval_step_12000/batch_0_cfg_7.5_topk_16384.jpg filter=lfs diff=lfs merge=lfs -text +CKPTS/X2I_UltraEdiit_fourmask_LLava_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_filtered_objects_4mask_1280_context_after_stage1_5e4/000-GPT-XL/checkpoints/eval_step_12000/batch_1_cfg_7.5_topk_16384.jpg filter=lfs diff=lfs merge=lfs -text +CKPTS/X2I_UltraEdiit_fourmask_LLava_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_filtered_objects_4mask_1280_context_after_stage1_5e4/000-GPT-XL/checkpoints/eval_step_12000/batch_2_cfg_7.5_topk_16384.jpg filter=lfs diff=lfs merge=lfs -text +CKPTS/X2I_UltraEdiit_fourmask_LLava_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_filtered_objects_4mask_1280_context_after_stage1_5e4/000-GPT-XL/checkpoints/eval_step_12000/batch_3_cfg_7.5_topk_16384.jpg filter=lfs diff=lfs merge=lfs -text +CKPTS/X2I_UltraEdiit_fourmask_LLava_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_filtered_objects_4mask_1280_context_after_stage1_5e4/000-GPT-XL/checkpoints/eval_step_12000/batch_4_cfg_7.5_topk_16384.jpg filter=lfs diff=lfs merge=lfs -text +CKPTS/X2I_UltraEdiit_fourmask_LLava_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_filtered_objects_4mask_1280_context_after_stage1_5e4/000-GPT-XL/checkpoints/eval_step_12000/batch_5_cfg_7.5_topk_16384.jpg filter=lfs diff=lfs merge=lfs -text +CKPTS/X2I_UltraEdiit_fourmask_LLava_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_filtered_objects_4mask_1280_context_after_stage1_5e4/000-GPT-XL/checkpoints/eval_step_14000/batch_0_cfg_7.5_topk_16384.jpg filter=lfs diff=lfs merge=lfs -text +CKPTS/X2I_UltraEdiit_fourmask_LLava_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_filtered_objects_4mask_1280_context_after_stage1_5e4/000-GPT-XL/checkpoints/eval_step_14000/batch_1_cfg_7.5_topk_16384.jpg filter=lfs diff=lfs merge=lfs -text +CKPTS/X2I_UltraEdiit_fourmask_LLava_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_filtered_objects_4mask_1280_context_after_stage1_5e4/000-GPT-XL/checkpoints/eval_step_14000/batch_2_cfg_7.5_topk_16384.jpg filter=lfs diff=lfs merge=lfs -text +CKPTS/X2I_UltraEdiit_fourmask_LLava_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_filtered_objects_4mask_1280_context_after_stage1_5e4/000-GPT-XL/checkpoints/eval_step_14000/batch_3_cfg_7.5_topk_16384.jpg filter=lfs diff=lfs merge=lfs -text +CKPTS/X2I_UltraEdiit_fourmask_LLava_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_filtered_objects_4mask_1280_context_after_stage1_5e4/000-GPT-XL/checkpoints/eval_step_14000/batch_4_cfg_7.5_topk_16384.jpg filter=lfs diff=lfs merge=lfs -text +CKPTS/X2I_UltraEdiit_fourmask_LLava_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_filtered_objects_4mask_1280_context_after_stage1_5e4/000-GPT-XL/checkpoints/eval_step_14000/batch_5_cfg_7.5_topk_16384.jpg filter=lfs diff=lfs merge=lfs -text +CKPTS/X2I_UltraEdiit_fourmask_LLava_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_filtered_objects_4mask_1280_context_after_stage1_5e4/000-GPT-XL/checkpoints/eval_step_16000/batch_0_cfg_7.5_topk_16384.jpg filter=lfs diff=lfs merge=lfs -text +CKPTS/X2I_UltraEdiit_fourmask_LLava_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_filtered_objects_4mask_1280_context_after_stage1_5e4/000-GPT-XL/checkpoints/eval_step_16000/batch_1_cfg_7.5_topk_16384.jpg filter=lfs diff=lfs merge=lfs -text +CKPTS/X2I_UltraEdiit_fourmask_LLava_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_filtered_objects_4mask_1280_context_after_stage1_5e4/000-GPT-XL/checkpoints/eval_step_16000/batch_2_cfg_7.5_topk_16384.jpg filter=lfs diff=lfs merge=lfs -text +CKPTS/X2I_UltraEdiit_fourmask_LLava_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_filtered_objects_4mask_1280_context_after_stage1_5e4/000-GPT-XL/checkpoints/eval_step_16000/batch_3_cfg_7.5_topk_16384.jpg filter=lfs diff=lfs merge=lfs -text +CKPTS/X2I_UltraEdiit_fourmask_LLava_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_filtered_objects_4mask_1280_context_after_stage1_5e4/000-GPT-XL/checkpoints/eval_step_16000/batch_4_cfg_7.5_topk_16384.jpg filter=lfs diff=lfs merge=lfs -text +CKPTS/X2I_UltraEdiit_fourmask_LLava_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_filtered_objects_4mask_1280_context_after_stage1_5e4/000-GPT-XL/checkpoints/eval_step_16000/batch_5_cfg_7.5_topk_16384.jpg filter=lfs diff=lfs merge=lfs -text +CKPTS/X2I_UltraEdiit_fourmask_LLava_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_filtered_objects_4mask_1280_context_after_stage1_5e4/000-GPT-XL/checkpoints/eval_step_18000/batch_0_cfg_7.5_topk_16384.jpg filter=lfs diff=lfs merge=lfs -text +CKPTS/X2I_UltraEdiit_fourmask_LLava_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_filtered_objects_4mask_1280_context_after_stage1_5e4/000-GPT-XL/checkpoints/eval_step_18000/batch_1_cfg_7.5_topk_16384.jpg filter=lfs diff=lfs merge=lfs -text +CKPTS/X2I_UltraEdiit_fourmask_LLava_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_filtered_objects_4mask_1280_context_after_stage1_5e4/000-GPT-XL/checkpoints/eval_step_18000/batch_2_cfg_7.5_topk_16384.jpg filter=lfs diff=lfs merge=lfs -text +CKPTS/X2I_UltraEdiit_fourmask_LLava_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_filtered_objects_4mask_1280_context_after_stage1_5e4/000-GPT-XL/checkpoints/eval_step_18000/batch_3_cfg_7.5_topk_16384.jpg filter=lfs diff=lfs merge=lfs -text +CKPTS/X2I_UltraEdiit_fourmask_LLava_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_filtered_objects_4mask_1280_context_after_stage1_5e4/000-GPT-XL/checkpoints/eval_step_18000/batch_4_cfg_7.5_topk_16384.jpg filter=lfs diff=lfs merge=lfs -text +CKPTS/X2I_UltraEdiit_fourmask_LLava_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_filtered_objects_4mask_1280_context_after_stage1_5e4/000-GPT-XL/checkpoints/eval_step_18000/batch_5_cfg_7.5_topk_16384.jpg filter=lfs diff=lfs merge=lfs -text +CKPTS/X2I_UltraEdiit_fourmask_LLava_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_filtered_objects_4mask_1280_context_after_stage1_5e4/000-GPT-XL/checkpoints/eval_step_2000/batch_0_cfg_7.5_topk_16384.jpg filter=lfs diff=lfs merge=lfs -text +CKPTS/X2I_UltraEdiit_fourmask_LLava_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_filtered_objects_4mask_1280_context_after_stage1_5e4/000-GPT-XL/checkpoints/eval_step_2000/batch_1_cfg_7.5_topk_16384.jpg filter=lfs diff=lfs merge=lfs -text +CKPTS/X2I_UltraEdiit_fourmask_LLava_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_filtered_objects_4mask_1280_context_after_stage1_5e4/000-GPT-XL/checkpoints/eval_step_2000/batch_2_cfg_7.5_topk_16384.jpg filter=lfs diff=lfs merge=lfs -text +CKPTS/X2I_UltraEdiit_fourmask_LLava_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_filtered_objects_4mask_1280_context_after_stage1_5e4/000-GPT-XL/checkpoints/eval_step_2000/batch_3_cfg_7.5_topk_16384.jpg filter=lfs diff=lfs merge=lfs -text +CKPTS/X2I_UltraEdiit_fourmask_LLava_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_filtered_objects_4mask_1280_context_after_stage1_5e4/000-GPT-XL/checkpoints/eval_step_2000/batch_4_cfg_7.5_topk_16384.jpg filter=lfs diff=lfs merge=lfs -text +CKPTS/X2I_UltraEdiit_fourmask_LLava_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_filtered_objects_4mask_1280_context_after_stage1_5e4/000-GPT-XL/checkpoints/eval_step_2000/batch_5_cfg_7.5_topk_16384.jpg filter=lfs diff=lfs merge=lfs -text +CKPTS/X2I_UltraEdiit_fourmask_LLava_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_filtered_objects_4mask_1280_context_after_stage1_5e4/000-GPT-XL/checkpoints/eval_step_20000/batch_0_cfg_7.5_topk_16384.jpg filter=lfs diff=lfs merge=lfs -text +CKPTS/X2I_UltraEdiit_fourmask_LLava_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_filtered_objects_4mask_1280_context_after_stage1_5e4/000-GPT-XL/checkpoints/eval_step_20000/batch_1_cfg_7.5_topk_16384.jpg filter=lfs diff=lfs merge=lfs -text +CKPTS/X2I_UltraEdiit_fourmask_LLava_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_filtered_objects_4mask_1280_context_after_stage1_5e4/000-GPT-XL/checkpoints/eval_step_20000/batch_2_cfg_7.5_topk_16384.jpg filter=lfs diff=lfs merge=lfs -text +CKPTS/X2I_UltraEdiit_fourmask_LLava_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_filtered_objects_4mask_1280_context_after_stage1_5e4/000-GPT-XL/checkpoints/eval_step_20000/batch_3_cfg_7.5_topk_16384.jpg filter=lfs diff=lfs merge=lfs -text +CKPTS/X2I_UltraEdiit_fourmask_LLava_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_filtered_objects_4mask_1280_context_after_stage1_5e4/000-GPT-XL/checkpoints/eval_step_20000/batch_4_cfg_7.5_topk_16384.jpg filter=lfs diff=lfs merge=lfs -text +CKPTS/X2I_UltraEdiit_fourmask_LLava_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_filtered_objects_4mask_1280_context_after_stage1_5e4/000-GPT-XL/checkpoints/eval_step_20000/batch_5_cfg_7.5_topk_16384.jpg filter=lfs diff=lfs merge=lfs -text +CKPTS/X2I_UltraEdiit_fourmask_LLava_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_filtered_objects_4mask_1280_context_after_stage1_5e4/000-GPT-XL/checkpoints/eval_step_22000/batch_0_cfg_7.5_topk_16384.jpg filter=lfs diff=lfs merge=lfs -text +CKPTS/X2I_UltraEdiit_fourmask_LLava_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_filtered_objects_4mask_1280_context_after_stage1_5e4/000-GPT-XL/checkpoints/eval_step_22000/batch_1_cfg_7.5_topk_16384.jpg filter=lfs diff=lfs merge=lfs -text +CKPTS/X2I_UltraEdiit_fourmask_LLava_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_filtered_objects_4mask_1280_context_after_stage1_5e4/000-GPT-XL/checkpoints/eval_step_22000/batch_2_cfg_7.5_topk_16384.jpg filter=lfs diff=lfs merge=lfs -text +CKPTS/X2I_UltraEdiit_fourmask_LLava_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_filtered_objects_4mask_1280_context_after_stage1_5e4/000-GPT-XL/checkpoints/eval_step_22000/batch_3_cfg_7.5_topk_16384.jpg filter=lfs diff=lfs merge=lfs -text +CKPTS/X2I_UltraEdiit_fourmask_LLava_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_filtered_objects_4mask_1280_context_after_stage1_5e4/000-GPT-XL/checkpoints/eval_step_22000/batch_4_cfg_7.5_topk_16384.jpg filter=lfs diff=lfs merge=lfs -text +CKPTS/X2I_UltraEdiit_fourmask_LLava_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_filtered_objects_4mask_1280_context_after_stage1_5e4/000-GPT-XL/checkpoints/eval_step_22000/batch_5_cfg_7.5_topk_16384.jpg filter=lfs diff=lfs merge=lfs -text +CKPTS/X2I_UltraEdiit_fourmask_LLava_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_filtered_objects_4mask_1280_context_after_stage1_5e4/000-GPT-XL/checkpoints/eval_step_24000/batch_0_cfg_7.5_topk_16384.jpg filter=lfs diff=lfs merge=lfs -text +CKPTS/X2I_UltraEdiit_fourmask_LLava_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_filtered_objects_4mask_1280_context_after_stage1_5e4/000-GPT-XL/checkpoints/eval_step_24000/batch_1_cfg_7.5_topk_16384.jpg filter=lfs diff=lfs merge=lfs -text +CKPTS/X2I_UltraEdiit_fourmask_LLava_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_filtered_objects_4mask_1280_context_after_stage1_5e4/000-GPT-XL/checkpoints/eval_step_24000/batch_2_cfg_7.5_topk_16384.jpg filter=lfs diff=lfs merge=lfs -text +CKPTS/X2I_UltraEdiit_fourmask_LLava_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_filtered_objects_4mask_1280_context_after_stage1_5e4/000-GPT-XL/checkpoints/eval_step_24000/batch_3_cfg_7.5_topk_16384.jpg filter=lfs diff=lfs merge=lfs -text +CKPTS/X2I_UltraEdiit_fourmask_LLava_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_filtered_objects_4mask_1280_context_after_stage1_5e4/000-GPT-XL/checkpoints/eval_step_24000/batch_4_cfg_7.5_topk_16384.jpg filter=lfs diff=lfs merge=lfs -text +CKPTS/X2I_UltraEdiit_fourmask_LLava_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_filtered_objects_4mask_1280_context_after_stage1_5e4/000-GPT-XL/checkpoints/eval_step_24000/batch_5_cfg_7.5_topk_16384.jpg filter=lfs diff=lfs merge=lfs -text +CKPTS/X2I_UltraEdiit_fourmask_LLava_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_filtered_objects_4mask_1280_context_after_stage1_5e4/000-GPT-XL/checkpoints/eval_step_26000/batch_0_cfg_7.5_topk_16384.jpg filter=lfs diff=lfs merge=lfs -text +CKPTS/X2I_UltraEdiit_fourmask_LLava_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_filtered_objects_4mask_1280_context_after_stage1_5e4/000-GPT-XL/checkpoints/eval_step_26000/batch_1_cfg_7.5_topk_16384.jpg filter=lfs diff=lfs merge=lfs -text +CKPTS/X2I_UltraEdiit_fourmask_LLava_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_filtered_objects_4mask_1280_context_after_stage1_5e4/000-GPT-XL/checkpoints/eval_step_26000/batch_2_cfg_7.5_topk_16384.jpg filter=lfs diff=lfs merge=lfs -text +CKPTS/X2I_UltraEdiit_fourmask_LLava_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_filtered_objects_4mask_1280_context_after_stage1_5e4/000-GPT-XL/checkpoints/eval_step_26000/batch_3_cfg_7.5_topk_16384.jpg filter=lfs diff=lfs merge=lfs -text +CKPTS/X2I_UltraEdiit_fourmask_LLava_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_filtered_objects_4mask_1280_context_after_stage1_5e4/000-GPT-XL/checkpoints/eval_step_26000/batch_4_cfg_7.5_topk_16384.jpg filter=lfs diff=lfs merge=lfs -text +CKPTS/X2I_UltraEdiit_fourmask_LLava_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_filtered_objects_4mask_1280_context_after_stage1_5e4/000-GPT-XL/checkpoints/eval_step_26000/batch_5_cfg_7.5_topk_16384.jpg filter=lfs diff=lfs merge=lfs -text +CKPTS/X2I_UltraEdiit_fourmask_LLava_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_filtered_objects_4mask_1280_context_after_stage1_5e4/000-GPT-XL/checkpoints/eval_step_28000/batch_0_cfg_7.5_topk_16384.jpg filter=lfs diff=lfs merge=lfs -text +CKPTS/X2I_UltraEdiit_fourmask_LLava_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_filtered_objects_4mask_1280_context_after_stage1_5e4/000-GPT-XL/checkpoints/eval_step_28000/batch_1_cfg_7.5_topk_16384.jpg filter=lfs diff=lfs merge=lfs -text +CKPTS/X2I_UltraEdiit_fourmask_LLava_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_filtered_objects_4mask_1280_context_after_stage1_5e4/000-GPT-XL/checkpoints/eval_step_28000/batch_2_cfg_7.5_topk_16384.jpg filter=lfs diff=lfs merge=lfs -text +CKPTS/X2I_UltraEdiit_fourmask_LLava_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_filtered_objects_4mask_1280_context_after_stage1_5e4/000-GPT-XL/checkpoints/eval_step_28000/batch_3_cfg_7.5_topk_16384.jpg filter=lfs diff=lfs merge=lfs -text +CKPTS/X2I_UltraEdiit_fourmask_LLava_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_filtered_objects_4mask_1280_context_after_stage1_5e4/000-GPT-XL/checkpoints/eval_step_28000/batch_4_cfg_7.5_topk_16384.jpg filter=lfs diff=lfs merge=lfs -text +CKPTS/X2I_UltraEdiit_fourmask_LLava_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_filtered_objects_4mask_1280_context_after_stage1_5e4/000-GPT-XL/checkpoints/eval_step_28000/batch_5_cfg_7.5_topk_16384.jpg filter=lfs diff=lfs merge=lfs -text +CKPTS/X2I_UltraEdiit_fourmask_LLava_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_filtered_objects_4mask_1280_context_after_stage1_5e4/000-GPT-XL/checkpoints/eval_step_30000/batch_0_cfg_7.5_topk_16384.jpg filter=lfs diff=lfs merge=lfs -text +CKPTS/X2I_UltraEdiit_fourmask_LLava_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_filtered_objects_4mask_1280_context_after_stage1_5e4/000-GPT-XL/checkpoints/eval_step_30000/batch_1_cfg_7.5_topk_16384.jpg filter=lfs diff=lfs merge=lfs -text +CKPTS/X2I_UltraEdiit_fourmask_LLava_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_filtered_objects_4mask_1280_context_after_stage1_5e4/000-GPT-XL/checkpoints/eval_step_30000/batch_2_cfg_7.5_topk_16384.jpg filter=lfs diff=lfs merge=lfs -text +CKPTS/X2I_UltraEdiit_fourmask_LLava_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_filtered_objects_4mask_1280_context_after_stage1_5e4/000-GPT-XL/checkpoints/eval_step_30000/batch_3_cfg_7.5_topk_16384.jpg filter=lfs diff=lfs merge=lfs -text +CKPTS/X2I_UltraEdiit_fourmask_LLava_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_filtered_objects_4mask_1280_context_after_stage1_5e4/000-GPT-XL/checkpoints/eval_step_30000/batch_4_cfg_7.5_topk_16384.jpg filter=lfs diff=lfs merge=lfs -text +CKPTS/X2I_UltraEdiit_fourmask_LLava_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_filtered_objects_4mask_1280_context_after_stage1_5e4/000-GPT-XL/checkpoints/eval_step_30000/batch_5_cfg_7.5_topk_16384.jpg filter=lfs diff=lfs merge=lfs -text +CKPTS/X2I_UltraEdiit_fourmask_LLava_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_filtered_objects_4mask_1280_context_after_stage1_5e4/000-GPT-XL/checkpoints/eval_step_32000/batch_0_cfg_7.5_topk_16384.jpg filter=lfs diff=lfs merge=lfs -text +CKPTS/X2I_UltraEdiit_fourmask_LLava_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_filtered_objects_4mask_1280_context_after_stage1_5e4/000-GPT-XL/checkpoints/eval_step_32000/batch_1_cfg_7.5_topk_16384.jpg filter=lfs diff=lfs merge=lfs -text +CKPTS/X2I_UltraEdiit_fourmask_LLava_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_filtered_objects_4mask_1280_context_after_stage1_5e4/000-GPT-XL/checkpoints/eval_step_32000/batch_2_cfg_7.5_topk_16384.jpg filter=lfs diff=lfs merge=lfs -text +CKPTS/X2I_UltraEdiit_fourmask_LLava_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_filtered_objects_4mask_1280_context_after_stage1_5e4/000-GPT-XL/checkpoints/eval_step_32000/batch_3_cfg_7.5_topk_16384.jpg filter=lfs diff=lfs merge=lfs -text +CKPTS/X2I_UltraEdiit_fourmask_LLava_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_filtered_objects_4mask_1280_context_after_stage1_5e4/000-GPT-XL/checkpoints/eval_step_32000/batch_4_cfg_7.5_topk_16384.jpg filter=lfs diff=lfs merge=lfs -text +CKPTS/X2I_UltraEdiit_fourmask_LLava_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_filtered_objects_4mask_1280_context_after_stage1_5e4/000-GPT-XL/checkpoints/eval_step_32000/batch_5_cfg_7.5_topk_16384.jpg filter=lfs diff=lfs merge=lfs -text +CKPTS/X2I_UltraEdiit_fourmask_LLava_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_filtered_objects_4mask_1280_context_after_stage1_5e4/000-GPT-XL/checkpoints/eval_step_34000/batch_0_cfg_7.5_topk_16384.jpg filter=lfs diff=lfs merge=lfs -text +CKPTS/X2I_UltraEdiit_fourmask_LLava_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_filtered_objects_4mask_1280_context_after_stage1_5e4/000-GPT-XL/checkpoints/eval_step_34000/batch_1_cfg_7.5_topk_16384.jpg filter=lfs diff=lfs merge=lfs -text +CKPTS/X2I_UltraEdiit_fourmask_LLava_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_filtered_objects_4mask_1280_context_after_stage1_5e4/000-GPT-XL/checkpoints/eval_step_34000/batch_2_cfg_7.5_topk_16384.jpg filter=lfs diff=lfs merge=lfs -text +CKPTS/X2I_UltraEdiit_fourmask_LLava_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_filtered_objects_4mask_1280_context_after_stage1_5e4/000-GPT-XL/checkpoints/eval_step_34000/batch_3_cfg_7.5_topk_16384.jpg filter=lfs diff=lfs merge=lfs -text +CKPTS/X2I_UltraEdiit_fourmask_LLava_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_filtered_objects_4mask_1280_context_after_stage1_5e4/000-GPT-XL/checkpoints/eval_step_34000/batch_4_cfg_7.5_topk_16384.jpg filter=lfs diff=lfs merge=lfs -text +CKPTS/X2I_UltraEdiit_fourmask_LLava_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_filtered_objects_4mask_1280_context_after_stage1_5e4/000-GPT-XL/checkpoints/eval_step_34000/batch_5_cfg_7.5_topk_16384.jpg filter=lfs diff=lfs merge=lfs -text +CKPTS/X2I_UltraEdiit_fourmask_LLava_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_filtered_objects_4mask_1280_context_after_stage1_5e4/000-GPT-XL/checkpoints/eval_step_36000/batch_0_cfg_7.5_topk_16384.jpg filter=lfs diff=lfs merge=lfs -text +CKPTS/X2I_UltraEdiit_fourmask_LLava_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_filtered_objects_4mask_1280_context_after_stage1_5e4/000-GPT-XL/checkpoints/eval_step_36000/batch_1_cfg_7.5_topk_16384.jpg filter=lfs diff=lfs merge=lfs -text +CKPTS/X2I_UltraEdiit_fourmask_LLava_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_filtered_objects_4mask_1280_context_after_stage1_5e4/000-GPT-XL/checkpoints/eval_step_36000/batch_2_cfg_7.5_topk_16384.jpg filter=lfs diff=lfs merge=lfs -text +CKPTS/X2I_UltraEdiit_fourmask_LLava_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_filtered_objects_4mask_1280_context_after_stage1_5e4/000-GPT-XL/checkpoints/eval_step_36000/batch_3_cfg_7.5_topk_16384.jpg filter=lfs diff=lfs merge=lfs -text +CKPTS/X2I_UltraEdiit_fourmask_LLava_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_filtered_objects_4mask_1280_context_after_stage1_5e4/000-GPT-XL/checkpoints/eval_step_36000/batch_4_cfg_7.5_topk_16384.jpg filter=lfs diff=lfs merge=lfs -text +CKPTS/X2I_UltraEdiit_fourmask_LLava_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_filtered_objects_4mask_1280_context_after_stage1_5e4/000-GPT-XL/checkpoints/eval_step_36000/batch_5_cfg_7.5_topk_16384.jpg filter=lfs diff=lfs merge=lfs -text +CKPTS/X2I_UltraEdiit_fourmask_LLava_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_filtered_objects_4mask_1280_context_after_stage1_5e4/000-GPT-XL/checkpoints/eval_step_38000/batch_0_cfg_7.5_topk_16384.jpg filter=lfs diff=lfs merge=lfs -text +CKPTS/X2I_UltraEdiit_fourmask_LLava_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_filtered_objects_4mask_1280_context_after_stage1_5e4/000-GPT-XL/checkpoints/eval_step_38000/batch_1_cfg_7.5_topk_16384.jpg filter=lfs diff=lfs merge=lfs -text +CKPTS/X2I_UltraEdiit_fourmask_LLava_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_filtered_objects_4mask_1280_context_after_stage1_5e4/000-GPT-XL/checkpoints/eval_step_38000/batch_2_cfg_7.5_topk_16384.jpg filter=lfs diff=lfs merge=lfs -text +CKPTS/X2I_UltraEdiit_fourmask_LLava_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_filtered_objects_4mask_1280_context_after_stage1_5e4/000-GPT-XL/checkpoints/eval_step_38000/batch_3_cfg_7.5_topk_16384.jpg filter=lfs diff=lfs merge=lfs -text +CKPTS/X2I_UltraEdiit_fourmask_LLava_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_filtered_objects_4mask_1280_context_after_stage1_5e4/000-GPT-XL/checkpoints/eval_step_38000/batch_4_cfg_7.5_topk_16384.jpg filter=lfs diff=lfs merge=lfs -text +CKPTS/X2I_UltraEdiit_fourmask_LLava_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_filtered_objects_4mask_1280_context_after_stage1_5e4/000-GPT-XL/checkpoints/eval_step_38000/batch_5_cfg_7.5_topk_16384.jpg filter=lfs diff=lfs merge=lfs -text +CKPTS/X2I_UltraEdiit_fourmask_LLava_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_filtered_objects_4mask_1280_context_after_stage1_5e4/000-GPT-XL/checkpoints/eval_step_4000/batch_0_cfg_7.5_topk_16384.jpg filter=lfs diff=lfs merge=lfs -text +CKPTS/X2I_UltraEdiit_fourmask_LLava_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_filtered_objects_4mask_1280_context_after_stage1_5e4/000-GPT-XL/checkpoints/eval_step_4000/batch_1_cfg_7.5_topk_16384.jpg filter=lfs diff=lfs merge=lfs -text +CKPTS/X2I_UltraEdiit_fourmask_LLava_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_filtered_objects_4mask_1280_context_after_stage1_5e4/000-GPT-XL/checkpoints/eval_step_4000/batch_2_cfg_7.5_topk_16384.jpg filter=lfs diff=lfs merge=lfs -text +CKPTS/X2I_UltraEdiit_fourmask_LLava_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_filtered_objects_4mask_1280_context_after_stage1_5e4/000-GPT-XL/checkpoints/eval_step_4000/batch_3_cfg_7.5_topk_16384.jpg filter=lfs diff=lfs merge=lfs -text +CKPTS/X2I_UltraEdiit_fourmask_LLava_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_filtered_objects_4mask_1280_context_after_stage1_5e4/000-GPT-XL/checkpoints/eval_step_4000/batch_4_cfg_7.5_topk_16384.jpg filter=lfs diff=lfs merge=lfs -text +CKPTS/X2I_UltraEdiit_fourmask_LLava_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_filtered_objects_4mask_1280_context_after_stage1_5e4/000-GPT-XL/checkpoints/eval_step_4000/batch_5_cfg_7.5_topk_16384.jpg filter=lfs diff=lfs merge=lfs -text +CKPTS/X2I_UltraEdiit_fourmask_LLava_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_filtered_objects_4mask_1280_context_after_stage1_5e4/000-GPT-XL/checkpoints/eval_step_40000/batch_0_cfg_7.5_topk_16384.jpg filter=lfs diff=lfs merge=lfs -text +CKPTS/X2I_UltraEdiit_fourmask_LLava_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_filtered_objects_4mask_1280_context_after_stage1_5e4/000-GPT-XL/checkpoints/eval_step_40000/batch_1_cfg_7.5_topk_16384.jpg filter=lfs diff=lfs merge=lfs -text +CKPTS/X2I_UltraEdiit_fourmask_LLava_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_filtered_objects_4mask_1280_context_after_stage1_5e4/000-GPT-XL/checkpoints/eval_step_40000/batch_2_cfg_7.5_topk_16384.jpg filter=lfs diff=lfs merge=lfs -text +CKPTS/X2I_UltraEdiit_fourmask_LLava_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_filtered_objects_4mask_1280_context_after_stage1_5e4/000-GPT-XL/checkpoints/eval_step_40000/batch_3_cfg_7.5_topk_16384.jpg filter=lfs diff=lfs merge=lfs -text +CKPTS/X2I_UltraEdiit_fourmask_LLava_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_filtered_objects_4mask_1280_context_after_stage1_5e4/000-GPT-XL/checkpoints/eval_step_40000/batch_4_cfg_7.5_topk_16384.jpg filter=lfs diff=lfs merge=lfs -text +CKPTS/X2I_UltraEdiit_fourmask_LLava_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_filtered_objects_4mask_1280_context_after_stage1_5e4/000-GPT-XL/checkpoints/eval_step_40000/batch_5_cfg_7.5_topk_16384.jpg filter=lfs diff=lfs merge=lfs -text +CKPTS/X2I_UltraEdiit_fourmask_LLava_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_filtered_objects_4mask_1280_context_after_stage1_5e4/000-GPT-XL/checkpoints/eval_step_42000/batch_0_cfg_7.5_topk_16384.jpg filter=lfs diff=lfs merge=lfs -text +CKPTS/X2I_UltraEdiit_fourmask_LLava_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_filtered_objects_4mask_1280_context_after_stage1_5e4/000-GPT-XL/checkpoints/eval_step_42000/batch_1_cfg_7.5_topk_16384.jpg filter=lfs diff=lfs merge=lfs -text +CKPTS/X2I_UltraEdiit_fourmask_LLava_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_filtered_objects_4mask_1280_context_after_stage1_5e4/000-GPT-XL/checkpoints/eval_step_42000/batch_2_cfg_7.5_topk_16384.jpg filter=lfs diff=lfs merge=lfs -text +CKPTS/X2I_UltraEdiit_fourmask_LLava_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_filtered_objects_4mask_1280_context_after_stage1_5e4/000-GPT-XL/checkpoints/eval_step_42000/batch_3_cfg_7.5_topk_16384.jpg filter=lfs diff=lfs merge=lfs -text +CKPTS/X2I_UltraEdiit_fourmask_LLava_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_filtered_objects_4mask_1280_context_after_stage1_5e4/000-GPT-XL/checkpoints/eval_step_42000/batch_4_cfg_7.5_topk_16384.jpg filter=lfs diff=lfs merge=lfs -text +CKPTS/X2I_UltraEdiit_fourmask_LLava_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_filtered_objects_4mask_1280_context_after_stage1_5e4/000-GPT-XL/checkpoints/eval_step_42000/batch_5_cfg_7.5_topk_16384.jpg filter=lfs diff=lfs merge=lfs -text +CKPTS/X2I_UltraEdiit_fourmask_LLava_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_filtered_objects_4mask_1280_context_after_stage1_5e4/000-GPT-XL/checkpoints/eval_step_44000/batch_0_cfg_7.5_topk_16384.jpg filter=lfs diff=lfs merge=lfs -text +CKPTS/X2I_UltraEdiit_fourmask_LLava_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_filtered_objects_4mask_1280_context_after_stage1_5e4/000-GPT-XL/checkpoints/eval_step_44000/batch_1_cfg_7.5_topk_16384.jpg filter=lfs diff=lfs merge=lfs -text +CKPTS/X2I_UltraEdiit_fourmask_LLava_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_filtered_objects_4mask_1280_context_after_stage1_5e4/000-GPT-XL/checkpoints/eval_step_44000/batch_2_cfg_7.5_topk_16384.jpg filter=lfs diff=lfs merge=lfs -text +CKPTS/X2I_UltraEdiit_fourmask_LLava_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_filtered_objects_4mask_1280_context_after_stage1_5e4/000-GPT-XL/checkpoints/eval_step_44000/batch_3_cfg_7.5_topk_16384.jpg filter=lfs diff=lfs merge=lfs -text +CKPTS/X2I_UltraEdiit_fourmask_LLava_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_filtered_objects_4mask_1280_context_after_stage1_5e4/000-GPT-XL/checkpoints/eval_step_44000/batch_4_cfg_7.5_topk_16384.jpg filter=lfs diff=lfs merge=lfs -text +CKPTS/X2I_UltraEdiit_fourmask_LLava_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_filtered_objects_4mask_1280_context_after_stage1_5e4/000-GPT-XL/checkpoints/eval_step_44000/batch_5_cfg_7.5_topk_16384.jpg filter=lfs diff=lfs merge=lfs -text +CKPTS/X2I_UltraEdiit_fourmask_LLava_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_filtered_objects_4mask_1280_context_after_stage1_5e4/000-GPT-XL/checkpoints/eval_step_46000/batch_0_cfg_7.5_topk_16384.jpg filter=lfs diff=lfs merge=lfs -text +CKPTS/X2I_UltraEdiit_fourmask_LLava_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_filtered_objects_4mask_1280_context_after_stage1_5e4/000-GPT-XL/checkpoints/eval_step_46000/batch_1_cfg_7.5_topk_16384.jpg filter=lfs diff=lfs merge=lfs -text +CKPTS/X2I_UltraEdiit_fourmask_LLava_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_filtered_objects_4mask_1280_context_after_stage1_5e4/000-GPT-XL/checkpoints/eval_step_46000/batch_2_cfg_7.5_topk_16384.jpg filter=lfs diff=lfs merge=lfs -text +CKPTS/X2I_UltraEdiit_fourmask_LLava_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_filtered_objects_4mask_1280_context_after_stage1_5e4/000-GPT-XL/checkpoints/eval_step_46000/batch_3_cfg_7.5_topk_16384.jpg filter=lfs diff=lfs merge=lfs -text +CKPTS/X2I_UltraEdiit_fourmask_LLava_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_filtered_objects_4mask_1280_context_after_stage1_5e4/000-GPT-XL/checkpoints/eval_step_46000/batch_4_cfg_7.5_topk_16384.jpg filter=lfs diff=lfs merge=lfs -text +CKPTS/X2I_UltraEdiit_fourmask_LLava_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_filtered_objects_4mask_1280_context_after_stage1_5e4/000-GPT-XL/checkpoints/eval_step_46000/batch_5_cfg_7.5_topk_16384.jpg filter=lfs diff=lfs merge=lfs -text +CKPTS/X2I_UltraEdiit_fourmask_LLava_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_filtered_objects_4mask_1280_context_after_stage1_5e4/000-GPT-XL/checkpoints/eval_step_48000/batch_0_cfg_7.5_topk_16384.jpg filter=lfs diff=lfs merge=lfs -text +CKPTS/X2I_UltraEdiit_fourmask_LLava_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_filtered_objects_4mask_1280_context_after_stage1_5e4/000-GPT-XL/checkpoints/eval_step_48000/batch_1_cfg_7.5_topk_16384.jpg filter=lfs diff=lfs merge=lfs -text +CKPTS/X2I_UltraEdiit_fourmask_LLava_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_filtered_objects_4mask_1280_context_after_stage1_5e4/000-GPT-XL/checkpoints/eval_step_48000/batch_2_cfg_7.5_topk_16384.jpg filter=lfs diff=lfs merge=lfs -text +CKPTS/X2I_UltraEdiit_fourmask_LLava_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_filtered_objects_4mask_1280_context_after_stage1_5e4/000-GPT-XL/checkpoints/eval_step_48000/batch_3_cfg_7.5_topk_16384.jpg filter=lfs diff=lfs merge=lfs -text +CKPTS/X2I_UltraEdiit_fourmask_LLava_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_filtered_objects_4mask_1280_context_after_stage1_5e4/000-GPT-XL/checkpoints/eval_step_48000/batch_4_cfg_7.5_topk_16384.jpg filter=lfs diff=lfs merge=lfs -text +CKPTS/X2I_UltraEdiit_fourmask_LLava_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_filtered_objects_4mask_1280_context_after_stage1_5e4/000-GPT-XL/checkpoints/eval_step_48000/batch_5_cfg_7.5_topk_16384.jpg filter=lfs diff=lfs merge=lfs -text +CKPTS/X2I_UltraEdiit_fourmask_LLava_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_filtered_objects_4mask_1280_context_after_stage1_5e4/000-GPT-XL/checkpoints/eval_step_50000/batch_0_cfg_7.5_topk_16384.jpg filter=lfs diff=lfs merge=lfs -text +CKPTS/X2I_UltraEdiit_fourmask_LLava_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_filtered_objects_4mask_1280_context_after_stage1_5e4/000-GPT-XL/checkpoints/eval_step_50000/batch_1_cfg_7.5_topk_16384.jpg filter=lfs diff=lfs merge=lfs -text +CKPTS/X2I_UltraEdiit_fourmask_LLava_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_filtered_objects_4mask_1280_context_after_stage1_5e4/000-GPT-XL/checkpoints/eval_step_50000/batch_2_cfg_7.5_topk_16384.jpg filter=lfs diff=lfs merge=lfs -text +CKPTS/X2I_UltraEdiit_fourmask_LLava_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_filtered_objects_4mask_1280_context_after_stage1_5e4/000-GPT-XL/checkpoints/eval_step_50000/batch_3_cfg_7.5_topk_16384.jpg filter=lfs diff=lfs merge=lfs -text +CKPTS/X2I_UltraEdiit_fourmask_LLava_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_filtered_objects_4mask_1280_context_after_stage1_5e4/000-GPT-XL/checkpoints/eval_step_50000/batch_4_cfg_7.5_topk_16384.jpg filter=lfs diff=lfs merge=lfs -text +CKPTS/X2I_UltraEdiit_fourmask_LLava_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_filtered_objects_4mask_1280_context_after_stage1_5e4/000-GPT-XL/checkpoints/eval_step_50000/batch_5_cfg_7.5_topk_16384.jpg filter=lfs diff=lfs merge=lfs -text +CKPTS/X2I_UltraEdiit_fourmask_LLava_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_filtered_objects_4mask_1280_context_after_stage1_5e4/000-GPT-XL/checkpoints/eval_step_6000/batch_0_cfg_7.5_topk_16384.jpg filter=lfs diff=lfs merge=lfs -text +CKPTS/X2I_UltraEdiit_fourmask_LLava_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_filtered_objects_4mask_1280_context_after_stage1_5e4/000-GPT-XL/checkpoints/eval_step_6000/batch_1_cfg_7.5_topk_16384.jpg filter=lfs diff=lfs merge=lfs -text +CKPTS/X2I_UltraEdiit_fourmask_LLava_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_filtered_objects_4mask_1280_context_after_stage1_5e4/000-GPT-XL/checkpoints/eval_step_6000/batch_2_cfg_7.5_topk_16384.jpg filter=lfs diff=lfs merge=lfs -text +CKPTS/X2I_UltraEdiit_fourmask_LLava_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_filtered_objects_4mask_1280_context_after_stage1_5e4/000-GPT-XL/checkpoints/eval_step_6000/batch_3_cfg_7.5_topk_16384.jpg filter=lfs diff=lfs merge=lfs -text +CKPTS/X2I_UltraEdiit_fourmask_LLava_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_filtered_objects_4mask_1280_context_after_stage1_5e4/000-GPT-XL/checkpoints/eval_step_6000/batch_4_cfg_7.5_topk_16384.jpg filter=lfs diff=lfs merge=lfs -text +CKPTS/X2I_UltraEdiit_fourmask_LLava_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_filtered_objects_4mask_1280_context_after_stage1_5e4/000-GPT-XL/checkpoints/eval_step_6000/batch_5_cfg_7.5_topk_16384.jpg filter=lfs diff=lfs merge=lfs -text +CKPTS/X2I_UltraEdiit_fourmask_LLava_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_filtered_objects_4mask_1280_context_after_stage1_5e4/000-GPT-XL/checkpoints/eval_step_8000/batch_0_cfg_7.5_topk_16384.jpg filter=lfs diff=lfs merge=lfs -text +CKPTS/X2I_UltraEdiit_fourmask_LLava_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_filtered_objects_4mask_1280_context_after_stage1_5e4/000-GPT-XL/checkpoints/eval_step_8000/batch_1_cfg_7.5_topk_16384.jpg filter=lfs diff=lfs merge=lfs -text +CKPTS/X2I_UltraEdiit_fourmask_LLava_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_filtered_objects_4mask_1280_context_after_stage1_5e4/000-GPT-XL/checkpoints/eval_step_8000/batch_2_cfg_7.5_topk_16384.jpg filter=lfs diff=lfs merge=lfs -text +CKPTS/X2I_UltraEdiit_fourmask_LLava_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_filtered_objects_4mask_1280_context_after_stage1_5e4/000-GPT-XL/checkpoints/eval_step_8000/batch_3_cfg_7.5_topk_16384.jpg filter=lfs diff=lfs merge=lfs -text +CKPTS/X2I_UltraEdiit_fourmask_LLava_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_filtered_objects_4mask_1280_context_after_stage1_5e4/000-GPT-XL/checkpoints/eval_step_8000/batch_4_cfg_7.5_topk_16384.jpg filter=lfs diff=lfs merge=lfs -text +CKPTS/X2I_UltraEdiit_fourmask_LLava_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_filtered_objects_4mask_1280_context_after_stage1_5e4/000-GPT-XL/checkpoints/eval_step_8000/batch_5_cfg_7.5_topk_16384.jpg filter=lfs diff=lfs merge=lfs -text +CKPTS/X2I_UltraEdiit_fourmask_LLava_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_filtered_objects_4mask_1280_context_after_stage1_5e4/wandb/run-20250428_102501-vl9wvenp/run-vl9wvenp.wandb filter=lfs diff=lfs merge=lfs -text diff --git a/CKPTS/Afteri2i_100kckpt_LLAVAT5_empty_Fixed_just_segment_pretrain_stage3_dreambench_recap_400k_t2i_400k_flux_200k_midjourney_150k_recovery_150k_3e4_epoch2_half_rotated/000-GPT-XL/log.txt b/CKPTS/Afteri2i_100kckpt_LLAVAT5_empty_Fixed_just_segment_pretrain_stage3_dreambench_recap_400k_t2i_400k_flux_200k_midjourney_150k_recovery_150k_3e4_epoch2_half_rotated/000-GPT-XL/log.txt new file mode 100644 index 0000000000000000000000000000000000000000..54e00452a3148a9700f9dfbcc06383b625b084e6 --- /dev/null +++ b/CKPTS/Afteri2i_100kckpt_LLAVAT5_empty_Fixed_just_segment_pretrain_stage3_dreambench_recap_400k_t2i_400k_flux_200k_midjourney_150k_recovery_150k_3e4_epoch2_half_rotated/000-GPT-XL/log.txt @@ -0,0 +1,10 @@ +[2025-04-23 09:11:27] Experiment directory created at checkpoint/Afteri2i_100kckpt_LLAVAT5_empty_Fixed_just_segment_pretrain_stage3_dreambench_recap_400k_t2i_400k_flux_200k_midjourney_150k_recovery_150k_3e4_epoch2_half_rotated/000-GPT-XL +[2025-04-23 09:11:27] Namespace(data_path='/tmp/haozhezhao/MLLMG/jsonl_data/merged_train_set_set_subject_400k_recap_t2i_400k_flux_200k_midjourney_150k_recovery_150k_grounding_100fluxseg_50samseg.jsonl', cloud_save_path='/tmp/haozhezhao/checkpoint', no_local_save=False, vq_model='VQ-16', vq_ckpt='/tmp/haozhezhao/model/llamagen_t2i/vq_ds16_t2i.pt', codebook_size=16384, codebook_embed_dim=8, gpt_model='GPT-XL', gpt_ckpt='/tmp/haozhezhao/MLLMG_ckpkts/checkpoints/AfterI2I_llavat5_subject_train_all_T5_subject_t2i_ti2i_120_w_flux_segment_noimageonly_empty_fix_train_all/0100000.pt', gpt_type='t2i', vocab_size=16384, cls_token_num=512, dropout_p=0.1, token_dropout_p=0.1, drop_path=0.0, no_compile=False, results_dir='checkpoint/Afteri2i_100kckpt_LLAVAT5_empty_Fixed_just_segment_pretrain_stage3_dreambench_recap_400k_t2i_400k_flux_200k_midjourney_150k_recovery_150k_3e4_epoch2_half_rotated', dataset='ti2i', image_size=512, downsample_size=16, num_classes=1000, epochs=2, lr=0.0003, weight_decay=0.05, beta1=0.9, beta2=0.95, max_grad_norm=1.0, global_batch_size=64, global_seed=0, num_workers=8, log_every=25, ckpt_every=2000, gradient_accumulation_steps=4, mixed_precision='bf16', val_data_path='/tmp/haozhezhao/MLLMG/jsonl_data/dreambench_plus_valid.jsonl', use_vision_tower=True, model_name_or_path='/tmp/haozhezhao/model/blip2-flan-t5-xl', image_place_holder='', processor_path=None, do_eval=True, max_eval_samples=500, train_text_encoder=True, no_left_padding=False, cfg_scale=7.5, top_k=16384, temperature=0.9, top_p=1.0, eval_steps=2000, project_name='llamagen_ti2i', load_from_checkpoint='/tmp/haozhezhao/MLLMG/checkpoint/FIXed_3M_1epoch_step32000.pt', warmup=0.05, lr_decay_style='cosine', lr_decay_ratio=0.1, train_iters=500000, class_dropout_prob=0.1, with_image_only=False, image_only_rate=0.1, stage2=False, subject_driven=True, load_subject_embedding=None, reference_data_path='/tmp/haozhezhao/MLLMG/cc12m_reference_tunnel.jsonl', multimodal_encoder='llava', do_recovery=True, no_replace=False, resume=False, dreambench_eval=True, find_unused_parameters=True, load_visual_encoder=False, continue_stage1=False, replace_subject=False, train_all=True, save_total_limit=1, load_language_projection='/tmp/haozhezhao/MLLMG/llava-v1.5-flant5_fixed-pretrain/mm_projector.bin', mm_vision_tower='openai/clip-vit-large-patch14', load_fixed_llamagen=True, unfreeze_output=False, fix='gpt-empty-fix', rank=0, world_size=8, gpu=0, dist_url='env://', distributed=True, dist_backend='nccl') +[2025-04-23 09:11:27] Starting rank=0, seed=0, world_size=8. +[2025-04-23 09:11:27] text_config is None. Initializing the text config with default values (`OPTConfig`). +[2025-04-23 09:12:39] GPT Parameters: 2,310,680,832 +[2025-04-23 09:12:39] num decayed parameter tensors: 356, with 2,007,303,168 parameters +[2025-04-23 09:12:39] num non-decayed parameter tensors: 124, with 197,888 parameters +[2025-04-23 09:12:39] using fused AdamW: True +[2025-04-23 09:12:49] Dataset contains 1,313,682 images +[2025-04-23 09:12:49] Train iters 41052 , warmup 2052.6, len of loader 20526 diff --git a/CKPTS/Afteri2i_100kckpt_LLAVAT5_empty_Fixed_just_segment_pretrain_stage3_dreambench_recap_400k_t2i_400k_flux_200k_midjourney_150k_recovery_150k_3e4_epoch2_half_rotated/001-GPT-XL/log.txt b/CKPTS/Afteri2i_100kckpt_LLAVAT5_empty_Fixed_just_segment_pretrain_stage3_dreambench_recap_400k_t2i_400k_flux_200k_midjourney_150k_recovery_150k_3e4_epoch2_half_rotated/001-GPT-XL/log.txt new file mode 100644 index 0000000000000000000000000000000000000000..0925235c285c2017ff87668101fcedf55f222daf --- /dev/null +++ b/CKPTS/Afteri2i_100kckpt_LLAVAT5_empty_Fixed_just_segment_pretrain_stage3_dreambench_recap_400k_t2i_400k_flux_200k_midjourney_150k_recovery_150k_3e4_epoch2_half_rotated/001-GPT-XL/log.txt @@ -0,0 +1,18 @@ +[2025-04-23 10:50:20] Experiment directory created at checkpoint/Afteri2i_100kckpt_LLAVAT5_empty_Fixed_just_segment_pretrain_stage3_dreambench_recap_400k_t2i_400k_flux_200k_midjourney_150k_recovery_150k_3e4_epoch2_half_rotated/001-GPT-XL +[2025-04-23 10:50:20] Namespace(data_path='/tmp/haozhezhao/MLLMG/jsonl_data/merged_train_set_set_subject_400k_recap_t2i_400k_flux_200k_midjourney_150k_recovery_150k_grounding_100fluxseg_50samseg.jsonl', cloud_save_path='/tmp/haozhezhao/checkpoint', no_local_save=False, vq_model='VQ-16', vq_ckpt='/tmp/haozhezhao/model/llamagen_t2i/vq_ds16_t2i.pt', codebook_size=16384, codebook_embed_dim=8, gpt_model='GPT-XL', gpt_ckpt='/tmp/haozhezhao/MLLMG/MLLMG_ckpts/checkpoints/AfterI2I_llavat5_subject_train_all_T5_subject_t2i_ti2i_120_w_flux_segment_noimageonly_empty_fix_train_all/0100000.pt', gpt_type='t2i', vocab_size=16384, cls_token_num=512, dropout_p=0.1, token_dropout_p=0.1, drop_path=0.0, no_compile=False, results_dir='checkpoint/Afteri2i_100kckpt_LLAVAT5_empty_Fixed_just_segment_pretrain_stage3_dreambench_recap_400k_t2i_400k_flux_200k_midjourney_150k_recovery_150k_3e4_epoch2_half_rotated', dataset='ti2i', image_size=512, downsample_size=16, num_classes=1000, epochs=2, lr=0.0003, weight_decay=0.05, beta1=0.9, beta2=0.95, max_grad_norm=1.0, global_batch_size=64, global_seed=0, num_workers=8, log_every=25, ckpt_every=2000, gradient_accumulation_steps=4, mixed_precision='bf16', val_data_path='/tmp/haozhezhao/MLLMG/jsonl_data/dreambench_plus_valid.jsonl', use_vision_tower=True, model_name_or_path='/tmp/haozhezhao/model/blip2-flan-t5-xl', image_place_holder='', processor_path=None, do_eval=True, max_eval_samples=500, train_text_encoder=True, no_left_padding=False, cfg_scale=7.5, top_k=16384, temperature=0.9, top_p=1.0, eval_steps=2000, project_name='llamagen_ti2i', load_from_checkpoint='/tmp/haozhezhao/MLLMG/checkpoint/FIXed_3M_1epoch_step32000.pt', warmup=0.05, lr_decay_style='cosine', lr_decay_ratio=0.1, train_iters=500000, class_dropout_prob=0.1, with_image_only=False, image_only_rate=0.1, stage2=False, subject_driven=True, load_subject_embedding=None, reference_data_path='/tmp/haozhezhao/MLLMG/cc12m_reference_tunnel.jsonl', multimodal_encoder='llava', do_recovery=True, no_replace=False, resume=False, dreambench_eval=True, find_unused_parameters=True, load_visual_encoder=False, continue_stage1=False, replace_subject=False, train_all=True, save_total_limit=1, load_language_projection='/tmp/haozhezhao/MLLMG/llava-v1.5-flant5_fixed-pretrain/mm_projector.bin', mm_vision_tower='openai/clip-vit-large-patch14', load_fixed_llamagen=True, unfreeze_output=False, fix='gpt-empty-fix', rank=0, world_size=8, gpu=0, dist_url='env://', distributed=True, dist_backend='nccl') +[2025-04-23 10:50:20] Starting rank=0, seed=0, world_size=8. +[2025-04-23 10:50:20] text_config is None. Initializing the text config with default values (`OPTConfig`). +[2025-04-23 10:51:26] GPT Parameters: 2,310,680,832 +[2025-04-23 10:51:26] num decayed parameter tensors: 356, with 2,007,303,168 parameters +[2025-04-23 10:51:26] num non-decayed parameter tensors: 124, with 197,888 parameters +[2025-04-23 10:51:26] using fused AdamW: True +[2025-04-23 10:51:35] Dataset contains 1,313,682 images +[2025-04-23 10:51:35] Train iters 41052 , warmup 2052.6, len of loader 20526 +[2025-04-23 10:52:03] ### LOAD pretraining weights from checkpoint: /tmp/haozhezhao/MLLMG/MLLMG_ckpts/checkpoints/AfterI2I_llavat5_subject_train_all_T5_subject_t2i_ti2i_120_w_flux_segment_noimageonly_empty_fix_train_all/0100000.pt +[2025-04-23 10:52:03] Initial state: steps=0, epochs=0 +[2025-04-23 10:52:03] compiling the model... (may take several minutes) +[2025-04-23 10:52:04] freeze the vit +[2025-04-23 10:52:04] ***** total param is 2310680832 ***** +[2025-04-23 10:52:04] ***** total trained param is 2007501056 ***** +[2025-04-23 10:52:06] Training for 2 epochs... +[2025-04-23 10:52:06] Beginning epoch 0... diff --git a/CKPTS/Afteri2i_100kckpt_LLAVAT5_empty_Fixed_just_segment_pretrain_stage3_dreambench_recap_400k_t2i_400k_flux_200k_midjourney_150k_recovery_150k_3e4_epoch2_half_rotated/003-GPT-XL/log.txt b/CKPTS/Afteri2i_100kckpt_LLAVAT5_empty_Fixed_just_segment_pretrain_stage3_dreambench_recap_400k_t2i_400k_flux_200k_midjourney_150k_recovery_150k_3e4_epoch2_half_rotated/003-GPT-XL/log.txt new file mode 100644 index 0000000000000000000000000000000000000000..7b7ccde40171ccfa36097397eaed68df015b0b39 --- /dev/null +++ b/CKPTS/Afteri2i_100kckpt_LLAVAT5_empty_Fixed_just_segment_pretrain_stage3_dreambench_recap_400k_t2i_400k_flux_200k_midjourney_150k_recovery_150k_3e4_epoch2_half_rotated/003-GPT-XL/log.txt @@ -0,0 +1,4 @@ +[2025-04-23 11:01:15] Experiment directory created at checkpoint/Afteri2i_100kckpt_LLAVAT5_empty_Fixed_just_segment_pretrain_stage3_dreambench_recap_400k_t2i_400k_flux_200k_midjourney_150k_recovery_150k_3e4_epoch2_half_rotated/003-GPT-XL +[2025-04-23 11:01:15] Namespace(data_path='/tmp/haozhezhao/MLLMG/jsonl_data/merged_train_set_set_subject_400k_recap_t2i_400k_flux_200k_midjourney_150k_recovery_150k_grounding_100fluxseg_50samseg.jsonl', cloud_save_path='/tmp/haozhezhao/checkpoint', no_local_save=False, vq_model='VQ-16', vq_ckpt='/tmp/haozhezhao/model/llamagen_t2i/vq_ds16_t2i.pt', codebook_size=16384, codebook_embed_dim=8, gpt_model='GPT-XL', gpt_ckpt='/tmp/haozhezhao/MLLMG/MLLMG_ckpts/checkpoints/AfterI2I_llavat5_subject_train_all_T5_subject_t2i_ti2i_120_w_flux_segment_noimageonly_empty_fix_train_all/0100000.pt', gpt_type='t2i', vocab_size=16384, cls_token_num=512, dropout_p=0.1, token_dropout_p=0.1, drop_path=0.0, no_compile=False, results_dir='checkpoint/Afteri2i_100kckpt_LLAVAT5_empty_Fixed_just_segment_pretrain_stage3_dreambench_recap_400k_t2i_400k_flux_200k_midjourney_150k_recovery_150k_3e4_epoch2_half_rotated', dataset='ti2i', image_size=512, downsample_size=16, num_classes=1000, epochs=2, lr=0.0003, weight_decay=0.05, beta1=0.9, beta2=0.95, max_grad_norm=1.0, global_batch_size=64, global_seed=0, num_workers=8, log_every=25, ckpt_every=2000, gradient_accumulation_steps=4, mixed_precision='bf16', val_data_path='/tmp/haozhezhao/MLLMG/jsonl_data/dreambench_plus_valid.jsonl', use_vision_tower=True, model_name_or_path='/tmp/haozhezhao/model/blip2-flan-t5-xl', image_place_holder='', processor_path=None, do_eval=True, max_eval_samples=500, train_text_encoder=True, no_left_padding=False, cfg_scale=7.5, top_k=16384, temperature=0.9, top_p=1.0, eval_steps=2000, project_name='llamagen_ti2i', load_from_checkpoint='/tmp/haozhezhao/MLLMG/checkpoint/FIXed_3M_1epoch_step32000.pt', warmup=0.05, lr_decay_style='cosine', lr_decay_ratio=0.1, train_iters=500000, class_dropout_prob=0.1, with_image_only=False, image_only_rate=0.1, stage2=False, subject_driven=True, load_subject_embedding=None, reference_data_path='/tmp/haozhezhao/MLLMG/cc12m_reference_tunnel.jsonl', multimodal_encoder='llava', do_recovery=True, no_replace=False, resume=False, dreambench_eval=True, find_unused_parameters=True, load_visual_encoder=False, continue_stage1=False, replace_subject=False, train_all=True, save_total_limit=1, load_language_projection='/tmp/haozhezhao/MLLMG/llava-v1.5-flant5_fixed-pretrain/mm_projector.bin', mm_vision_tower='openai/clip-vit-large-patch14', load_fixed_llamagen=True, unfreeze_output=False, fix='gpt-empty-fix', rank=0, world_size=8, gpu=0, dist_url='env://', distributed=True, dist_backend='nccl') +[2025-04-23 11:01:15] Starting rank=0, seed=0, world_size=8. +[2025-04-23 11:01:15] text_config is None. Initializing the text config with default values (`OPTConfig`). diff --git a/CKPTS/Afteri2i_100kckpt_LLAVAT5_empty_Fixed_just_segment_pretrain_stage3_dreambench_recap_400k_t2i_400k_flux_200k_midjourney_150k_recovery_150k_3e4_epoch2_half_rotated/004-GPT-XL/checkpoints/eval_step_2000/batch_0_cfg_7.5_topk_16384.jpg b/CKPTS/Afteri2i_100kckpt_LLAVAT5_empty_Fixed_just_segment_pretrain_stage3_dreambench_recap_400k_t2i_400k_flux_200k_midjourney_150k_recovery_150k_3e4_epoch2_half_rotated/004-GPT-XL/checkpoints/eval_step_2000/batch_0_cfg_7.5_topk_16384.jpg new file mode 100644 index 0000000000000000000000000000000000000000..cba99adc4f1e3ebc11a5ff7544f68bc7ad6042d1 --- /dev/null +++ b/CKPTS/Afteri2i_100kckpt_LLAVAT5_empty_Fixed_just_segment_pretrain_stage3_dreambench_recap_400k_t2i_400k_flux_200k_midjourney_150k_recovery_150k_3e4_epoch2_half_rotated/004-GPT-XL/checkpoints/eval_step_2000/batch_0_cfg_7.5_topk_16384.jpg @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:aaf5fc9a54982643efdf91372f83a0d7f6867020fb055706d5f3fb8904d19368 +size 728957 diff --git a/CKPTS/Afteri2i_100kckpt_LLAVAT5_empty_Fixed_just_segment_pretrain_stage3_dreambench_recap_400k_t2i_400k_flux_200k_midjourney_150k_recovery_150k_3e4_epoch2_half_rotated/004-GPT-XL/checkpoints/eval_step_2000/batch_1_cfg_7.5_topk_16384.jpg b/CKPTS/Afteri2i_100kckpt_LLAVAT5_empty_Fixed_just_segment_pretrain_stage3_dreambench_recap_400k_t2i_400k_flux_200k_midjourney_150k_recovery_150k_3e4_epoch2_half_rotated/004-GPT-XL/checkpoints/eval_step_2000/batch_1_cfg_7.5_topk_16384.jpg new file mode 100644 index 0000000000000000000000000000000000000000..aa216229d2087cce5ad0f1cc2a92d6b500f51561 --- /dev/null +++ b/CKPTS/Afteri2i_100kckpt_LLAVAT5_empty_Fixed_just_segment_pretrain_stage3_dreambench_recap_400k_t2i_400k_flux_200k_midjourney_150k_recovery_150k_3e4_epoch2_half_rotated/004-GPT-XL/checkpoints/eval_step_2000/batch_1_cfg_7.5_topk_16384.jpg @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:e70b587f87f57ed75da73b110aefbf08ca8b789591465e7380520078e09b3442 +size 712384 diff --git a/CKPTS/Afteri2i_100kckpt_LLAVAT5_empty_Fixed_just_segment_pretrain_stage3_dreambench_recap_400k_t2i_400k_flux_200k_midjourney_150k_recovery_150k_3e4_epoch2_half_rotated/004-GPT-XL/checkpoints/eval_step_2000/batch_2_cfg_7.5_topk_16384.jpg b/CKPTS/Afteri2i_100kckpt_LLAVAT5_empty_Fixed_just_segment_pretrain_stage3_dreambench_recap_400k_t2i_400k_flux_200k_midjourney_150k_recovery_150k_3e4_epoch2_half_rotated/004-GPT-XL/checkpoints/eval_step_2000/batch_2_cfg_7.5_topk_16384.jpg new file mode 100644 index 0000000000000000000000000000000000000000..394d35797a803ec92ece72f3fd8129d86dc6c9b0 --- /dev/null +++ b/CKPTS/Afteri2i_100kckpt_LLAVAT5_empty_Fixed_just_segment_pretrain_stage3_dreambench_recap_400k_t2i_400k_flux_200k_midjourney_150k_recovery_150k_3e4_epoch2_half_rotated/004-GPT-XL/checkpoints/eval_step_2000/batch_2_cfg_7.5_topk_16384.jpg @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:e651d7561f4cda84ec1dad96b2581125d375741cca68b08e1aa08f0f3b1c29e9 +size 632162 diff --git a/CKPTS/Afteri2i_100kckpt_LLAVAT5_empty_Fixed_just_segment_pretrain_stage3_dreambench_recap_400k_t2i_400k_flux_200k_midjourney_150k_recovery_150k_3e4_epoch2_half_rotated/004-GPT-XL/checkpoints/eval_step_2000/batch_3_cfg_7.5_topk_16384.jpg b/CKPTS/Afteri2i_100kckpt_LLAVAT5_empty_Fixed_just_segment_pretrain_stage3_dreambench_recap_400k_t2i_400k_flux_200k_midjourney_150k_recovery_150k_3e4_epoch2_half_rotated/004-GPT-XL/checkpoints/eval_step_2000/batch_3_cfg_7.5_topk_16384.jpg new file mode 100644 index 0000000000000000000000000000000000000000..9863f2606a3f43753655d627f9fb09a986803b40 --- /dev/null +++ b/CKPTS/Afteri2i_100kckpt_LLAVAT5_empty_Fixed_just_segment_pretrain_stage3_dreambench_recap_400k_t2i_400k_flux_200k_midjourney_150k_recovery_150k_3e4_epoch2_half_rotated/004-GPT-XL/checkpoints/eval_step_2000/batch_3_cfg_7.5_topk_16384.jpg @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:ac1126f0f3a5ef4d7a7f8681c0d68159298df32927fe94bb7736f83e0f24a946 +size 927109 diff --git a/CKPTS/Afteri2i_100kckpt_LLAVAT5_empty_Fixed_just_segment_pretrain_stage3_dreambench_recap_400k_t2i_400k_flux_200k_midjourney_150k_recovery_150k_3e4_epoch2_half_rotated/004-GPT-XL/checkpoints/eval_step_2000/batch_4_cfg_7.5_topk_16384.jpg b/CKPTS/Afteri2i_100kckpt_LLAVAT5_empty_Fixed_just_segment_pretrain_stage3_dreambench_recap_400k_t2i_400k_flux_200k_midjourney_150k_recovery_150k_3e4_epoch2_half_rotated/004-GPT-XL/checkpoints/eval_step_2000/batch_4_cfg_7.5_topk_16384.jpg new file mode 100644 index 0000000000000000000000000000000000000000..179f727acb1986951bb715139892f51fbfba4622 --- /dev/null +++ b/CKPTS/Afteri2i_100kckpt_LLAVAT5_empty_Fixed_just_segment_pretrain_stage3_dreambench_recap_400k_t2i_400k_flux_200k_midjourney_150k_recovery_150k_3e4_epoch2_half_rotated/004-GPT-XL/checkpoints/eval_step_2000/batch_4_cfg_7.5_topk_16384.jpg @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:d262a639da189ce1da04c479a19e999d211657f1f5046de459b7e574d15a6c84 +size 752470 diff --git a/CKPTS/Afteri2i_100kckpt_LLAVAT5_empty_Fixed_just_segment_pretrain_stage3_dreambench_recap_400k_t2i_400k_flux_200k_midjourney_150k_recovery_150k_3e4_epoch2_half_rotated/004-GPT-XL/checkpoints/eval_step_2000/batch_5_cfg_7.5_topk_16384.jpg b/CKPTS/Afteri2i_100kckpt_LLAVAT5_empty_Fixed_just_segment_pretrain_stage3_dreambench_recap_400k_t2i_400k_flux_200k_midjourney_150k_recovery_150k_3e4_epoch2_half_rotated/004-GPT-XL/checkpoints/eval_step_2000/batch_5_cfg_7.5_topk_16384.jpg new file mode 100644 index 0000000000000000000000000000000000000000..68ccabbaec16abb08e4575f090619ddb05c1a7eb --- /dev/null +++ b/CKPTS/Afteri2i_100kckpt_LLAVAT5_empty_Fixed_just_segment_pretrain_stage3_dreambench_recap_400k_t2i_400k_flux_200k_midjourney_150k_recovery_150k_3e4_epoch2_half_rotated/004-GPT-XL/checkpoints/eval_step_2000/batch_5_cfg_7.5_topk_16384.jpg @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:ae03006ade59d93fa96e684040ee986fd3d5ffbe4c139559dcd8e0d112911de0 +size 753077 diff --git a/CKPTS/Afteri2i_100kckpt_LLAVAT5_empty_Fixed_just_segment_pretrain_stage3_dreambench_recap_400k_t2i_400k_flux_200k_midjourney_150k_recovery_150k_3e4_epoch2_half_rotated/004-GPT-XL/checkpoints/eval_step_2000/batch_6_cfg_7.5_topk_16384.jpg b/CKPTS/Afteri2i_100kckpt_LLAVAT5_empty_Fixed_just_segment_pretrain_stage3_dreambench_recap_400k_t2i_400k_flux_200k_midjourney_150k_recovery_150k_3e4_epoch2_half_rotated/004-GPT-XL/checkpoints/eval_step_2000/batch_6_cfg_7.5_topk_16384.jpg new file mode 100644 index 0000000000000000000000000000000000000000..46d5460ffaed02dc0a466fcd15aec4da17a711cd --- /dev/null +++ b/CKPTS/Afteri2i_100kckpt_LLAVAT5_empty_Fixed_just_segment_pretrain_stage3_dreambench_recap_400k_t2i_400k_flux_200k_midjourney_150k_recovery_150k_3e4_epoch2_half_rotated/004-GPT-XL/checkpoints/eval_step_2000/batch_6_cfg_7.5_topk_16384.jpg @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:9c4d9e2077c85adfc3dd377fd3358766fec93289eeceb9be75f0019518f9656d +size 682786 diff --git a/CKPTS/Afteri2i_100kckpt_LLAVAT5_empty_Fixed_just_segment_pretrain_stage3_dreambench_recap_400k_t2i_400k_flux_200k_midjourney_150k_recovery_150k_3e4_epoch2_half_rotated/004-GPT-XL/checkpoints/eval_step_2000/batch_7_cfg_7.5_topk_16384.jpg b/CKPTS/Afteri2i_100kckpt_LLAVAT5_empty_Fixed_just_segment_pretrain_stage3_dreambench_recap_400k_t2i_400k_flux_200k_midjourney_150k_recovery_150k_3e4_epoch2_half_rotated/004-GPT-XL/checkpoints/eval_step_2000/batch_7_cfg_7.5_topk_16384.jpg new file mode 100644 index 0000000000000000000000000000000000000000..54767b884c72a84af9bd4ee2a918861bf3ef30c6 --- /dev/null +++ b/CKPTS/Afteri2i_100kckpt_LLAVAT5_empty_Fixed_just_segment_pretrain_stage3_dreambench_recap_400k_t2i_400k_flux_200k_midjourney_150k_recovery_150k_3e4_epoch2_half_rotated/004-GPT-XL/checkpoints/eval_step_2000/batch_7_cfg_7.5_topk_16384.jpg @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:c15e6a1fff3d66ce68c271afd82badad97f7f56fcc9843d512662995c9cfe137 +size 673567 diff --git a/CKPTS/Afteri2i_100kckpt_LLAVAT5_empty_Fixed_just_segment_pretrain_stage3_dreambench_recap_400k_t2i_400k_flux_200k_midjourney_150k_recovery_150k_3e4_epoch2_half_rotated/004-GPT-XL/checkpoints/eval_step_2000/batch_8_cfg_7.5_topk_16384.jpg b/CKPTS/Afteri2i_100kckpt_LLAVAT5_empty_Fixed_just_segment_pretrain_stage3_dreambench_recap_400k_t2i_400k_flux_200k_midjourney_150k_recovery_150k_3e4_epoch2_half_rotated/004-GPT-XL/checkpoints/eval_step_2000/batch_8_cfg_7.5_topk_16384.jpg new file mode 100644 index 0000000000000000000000000000000000000000..09b24583aad336d496c0903afcbf0685a9a66924 --- /dev/null +++ b/CKPTS/Afteri2i_100kckpt_LLAVAT5_empty_Fixed_just_segment_pretrain_stage3_dreambench_recap_400k_t2i_400k_flux_200k_midjourney_150k_recovery_150k_3e4_epoch2_half_rotated/004-GPT-XL/checkpoints/eval_step_2000/batch_8_cfg_7.5_topk_16384.jpg @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:41e338529665ada5f70987a9ff14b1b4a7e3dccafa258c960f80344b34e54c34 +size 754776 diff --git a/CKPTS/Afteri2i_100kckpt_LLAVAT5_empty_Fixed_just_segment_pretrain_stage3_dreambench_recap_400k_t2i_400k_flux_200k_midjourney_150k_recovery_150k_3e4_epoch2_half_rotated/004-GPT-XL/log.txt b/CKPTS/Afteri2i_100kckpt_LLAVAT5_empty_Fixed_just_segment_pretrain_stage3_dreambench_recap_400k_t2i_400k_flux_200k_midjourney_150k_recovery_150k_3e4_epoch2_half_rotated/004-GPT-XL/log.txt new file mode 100644 index 0000000000000000000000000000000000000000..62373123cf316af626f8c08b47fdd0ab5bd34b82 --- /dev/null +++ b/CKPTS/Afteri2i_100kckpt_LLAVAT5_empty_Fixed_just_segment_pretrain_stage3_dreambench_recap_400k_t2i_400k_flux_200k_midjourney_150k_recovery_150k_3e4_epoch2_half_rotated/004-GPT-XL/log.txt @@ -0,0 +1,99 @@ +[2025-04-23 11:12:04] Experiment directory created at checkpoint/Afteri2i_100kckpt_LLAVAT5_empty_Fixed_just_segment_pretrain_stage3_dreambench_recap_400k_t2i_400k_flux_200k_midjourney_150k_recovery_150k_3e4_epoch2_half_rotated/004-GPT-XL +[2025-04-23 11:12:04] Namespace(data_path='/tmp/haozhezhao/MLLMG/jsonl_data/merged_train_set_set_subject_400k_recap_t2i_400k_flux_200k_midjourney_150k_recovery_150k_grounding_100fluxseg_50samseg.jsonl', cloud_save_path='/tmp/haozhezhao/checkpoint', no_local_save=False, vq_model='VQ-16', vq_ckpt='/tmp/haozhezhao/model/llamagen_t2i/vq_ds16_t2i.pt', codebook_size=16384, codebook_embed_dim=8, gpt_model='GPT-XL', gpt_ckpt='/tmp/haozhezhao/MLLMG/MLLMG_ckpts/checkpoints/AfterI2I_llavat5_subject_train_all_T5_subject_t2i_ti2i_120_w_flux_segment_noimageonly_empty_fix_train_all/0100000.pt', gpt_type='t2i', vocab_size=16384, cls_token_num=512, dropout_p=0.1, token_dropout_p=0.1, drop_path=0.0, no_compile=False, results_dir='checkpoint/Afteri2i_100kckpt_LLAVAT5_empty_Fixed_just_segment_pretrain_stage3_dreambench_recap_400k_t2i_400k_flux_200k_midjourney_150k_recovery_150k_3e4_epoch2_half_rotated', dataset='ti2i', image_size=512, downsample_size=16, num_classes=1000, epochs=2, lr=0.0003, weight_decay=0.05, beta1=0.9, beta2=0.95, max_grad_norm=1.0, global_batch_size=56, global_seed=0, num_workers=8, log_every=25, ckpt_every=2000, gradient_accumulation_steps=4, mixed_precision='bf16', val_data_path='/tmp/haozhezhao/MLLMG/jsonl_data/dreambench_plus_valid.jsonl', use_vision_tower=True, model_name_or_path='/tmp/haozhezhao/model/blip2-flan-t5-xl', image_place_holder='', processor_path=None, do_eval=True, max_eval_samples=500, train_text_encoder=True, no_left_padding=False, cfg_scale=7.5, top_k=16384, temperature=0.9, top_p=1.0, eval_steps=2000, project_name='llamagen_ti2i', load_from_checkpoint='/tmp/haozhezhao/MLLMG/checkpoint/FIXed_3M_1epoch_step32000.pt', warmup=0.05, lr_decay_style='cosine', lr_decay_ratio=0.1, train_iters=500000, class_dropout_prob=0.1, with_image_only=False, image_only_rate=0.1, stage2=False, subject_driven=True, load_subject_embedding=None, reference_data_path='/tmp/haozhezhao/MLLMG/cc12m_reference_tunnel.jsonl', multimodal_encoder='llava', do_recovery=True, no_replace=False, resume=False, dreambench_eval=True, find_unused_parameters=True, load_visual_encoder=False, continue_stage1=False, replace_subject=False, train_all=True, save_total_limit=1, load_language_projection='/tmp/haozhezhao/MLLMG/llava-v1.5-flant5_fixed-pretrain/mm_projector.bin', mm_vision_tower='openai/clip-vit-large-patch14', load_fixed_llamagen=True, unfreeze_output=False, fix='gpt-empty-fix', rank=0, world_size=8, gpu=0, dist_url='env://', distributed=True, dist_backend='nccl') +[2025-04-23 11:12:04] Starting rank=0, seed=0, world_size=8. +[2025-04-23 11:12:04] text_config is None. Initializing the text config with default values (`OPTConfig`). +[2025-04-23 11:13:13] GPT Parameters: 2,310,680,832 +[2025-04-23 11:13:13] num decayed parameter tensors: 356, with 2,007,303,168 parameters +[2025-04-23 11:13:13] num non-decayed parameter tensors: 124, with 197,888 parameters +[2025-04-23 11:13:13] using fused AdamW: True +[2025-04-23 11:13:22] Dataset contains 1,313,682 images +[2025-04-23 11:13:22] Train iters 46916 , warmup 2345.8, len of loader 23458 +[2025-04-23 11:13:38] ### LOAD pretraining weights from checkpoint: /tmp/haozhezhao/MLLMG/MLLMG_ckpts/checkpoints/AfterI2I_llavat5_subject_train_all_T5_subject_t2i_ti2i_120_w_flux_segment_noimageonly_empty_fix_train_all/0100000.pt +[2025-04-23 11:13:38] Initial state: steps=0, epochs=0 +[2025-04-23 11:13:38] compiling the model... (may take several minutes) +[2025-04-23 11:13:39] freeze the vit +[2025-04-23 11:13:39] ***** total param is 2310680832 ***** +[2025-04-23 11:13:39] ***** total trained param is 2007501056 ***** +[2025-04-23 11:13:40] Training for 2 epochs... +[2025-04-23 11:13:40] Beginning epoch 0... +[2025-04-23 11:18:40] (step=0000025) Train Loss: 6.6005, Train Steps/Sec: 0.08 +[2025-04-23 11:19:29] (step=0000050) Train Loss: 6.6556, Train Steps/Sec: 0.51 +[2025-04-23 11:21:08] (step=0000075) Train Loss: 6.4599, Train Steps/Sec: 0.25 +[2025-04-23 11:22:11] (step=0000100) Train Loss: 6.3558, Train Steps/Sec: 0.40 +[2025-04-23 11:22:33] (step=0000125) Train Loss: 6.2557, Train Steps/Sec: 1.17 +[2025-04-23 11:23:23] (step=0000150) Train Loss: 6.1054, Train Steps/Sec: 0.50 +[2025-04-23 11:24:13] (step=0000175) Train Loss: 6.1218, Train Steps/Sec: 0.50 +[2025-04-23 11:24:35] (step=0000200) Train Loss: 6.0449, Train Steps/Sec: 1.15 +[2025-04-23 11:24:56] (step=0000225) Train Loss: 5.9306, Train Steps/Sec: 1.17 +[2025-04-23 11:25:46] (step=0000250) Train Loss: 5.8833, Train Steps/Sec: 0.50 +[2025-04-23 11:27:04] (step=0000275) Train Loss: 5.9763, Train Steps/Sec: 0.32 +[2025-04-23 11:27:25] (step=0000300) Train Loss: 5.8122, Train Steps/Sec: 1.17 +[2025-04-23 11:27:47] (step=0000325) Train Loss: 5.9543, Train Steps/Sec: 1.17 +[2025-04-23 11:28:08] (step=0000350) Train Loss: 5.8445, Train Steps/Sec: 1.17 +[2025-04-23 11:28:29] (step=0000375) Train Loss: 5.8658, Train Steps/Sec: 1.17 +[2025-04-23 11:29:20] (step=0000400) Train Loss: 5.7741, Train Steps/Sec: 0.49 +[2025-04-23 11:29:41] (step=0000425) Train Loss: 5.8472, Train Steps/Sec: 1.17 +[2025-04-23 11:30:03] (step=0000450) Train Loss: 5.8283, Train Steps/Sec: 1.17 +[2025-04-23 11:30:25] (step=0000475) Train Loss: 5.8059, Train Steps/Sec: 1.12 +[2025-04-23 11:30:46] (step=0000500) Train Loss: 5.8193, Train Steps/Sec: 1.17 +[2025-04-23 11:31:08] (step=0000525) Train Loss: 5.8235, Train Steps/Sec: 1.17 +[2025-04-23 11:31:29] (step=0000550) Train Loss: 5.8375, Train Steps/Sec: 1.17 +[2025-04-23 11:31:50] (step=0000575) Train Loss: 5.8427, Train Steps/Sec: 1.17 +[2025-04-23 11:32:12] (step=0000600) Train Loss: 5.7903, Train Steps/Sec: 1.16 +[2025-04-23 11:32:33] (step=0000625) Train Loss: 5.8660, Train Steps/Sec: 1.17 +[2025-04-23 11:32:55] (step=0000650) Train Loss: 5.7774, Train Steps/Sec: 1.17 +[2025-04-23 11:33:16] (step=0000675) Train Loss: 5.7929, Train Steps/Sec: 1.17 +[2025-04-23 11:33:37] (step=0000700) Train Loss: 5.9227, Train Steps/Sec: 1.17 +[2025-04-23 11:33:59] (step=0000725) Train Loss: 5.7613, Train Steps/Sec: 1.17 +[2025-04-23 11:34:21] (step=0000750) Train Loss: 5.8464, Train Steps/Sec: 1.11 +[2025-04-23 11:34:42] (step=0000775) Train Loss: 5.8326, Train Steps/Sec: 1.17 +[2025-04-23 11:35:05] (step=0000800) Train Loss: 5.7977, Train Steps/Sec: 1.11 +[2025-04-23 11:35:26] (step=0000825) Train Loss: 5.7656, Train Steps/Sec: 1.17 +[2025-04-23 11:35:48] (step=0000850) Train Loss: 5.7243, Train Steps/Sec: 1.17 +[2025-04-23 11:36:11] (step=0000875) Train Loss: 5.7829, Train Steps/Sec: 1.07 +[2025-04-23 11:36:34] (step=0000900) Train Loss: 5.7715, Train Steps/Sec: 1.10 +[2025-04-23 11:36:55] (step=0000925) Train Loss: 5.8266, Train Steps/Sec: 1.17 +[2025-04-23 11:37:16] (step=0000950) Train Loss: 5.8739, Train Steps/Sec: 1.17 +[2025-04-23 11:37:37] (step=0000975) Train Loss: 5.7752, Train Steps/Sec: 1.17 +[2025-04-23 11:37:59] (step=0001000) Train Loss: 5.7676, Train Steps/Sec: 1.17 +[2025-04-23 11:38:22] (step=0001025) Train Loss: 5.7233, Train Steps/Sec: 1.06 +[2025-04-23 11:38:44] (step=0001050) Train Loss: 5.7606, Train Steps/Sec: 1.17 +[2025-04-23 11:39:05] (step=0001075) Train Loss: 5.7211, Train Steps/Sec: 1.17 +[2025-04-23 11:39:26] (step=0001100) Train Loss: 5.7764, Train Steps/Sec: 1.17 +[2025-04-23 11:39:48] (step=0001125) Train Loss: 5.7317, Train Steps/Sec: 1.18 +[2025-04-23 11:40:10] (step=0001150) Train Loss: 5.7573, Train Steps/Sec: 1.12 +[2025-04-23 11:40:31] (step=0001175) Train Loss: 5.7894, Train Steps/Sec: 1.17 +[2025-04-23 11:40:53] (step=0001200) Train Loss: 5.7181, Train Steps/Sec: 1.17 +[2025-04-23 11:41:14] (step=0001225) Train Loss: 5.7548, Train Steps/Sec: 1.17 +[2025-04-23 11:41:35] (step=0001250) Train Loss: 5.6894, Train Steps/Sec: 1.17 +[2025-04-23 11:41:57] (step=0001275) Train Loss: 5.7133, Train Steps/Sec: 1.17 +[2025-04-23 11:42:18] (step=0001300) Train Loss: 5.7372, Train Steps/Sec: 1.17 +[2025-04-23 11:42:39] (step=0001325) Train Loss: 5.8133, Train Steps/Sec: 1.17 +[2025-04-23 11:43:01] (step=0001350) Train Loss: 5.7414, Train Steps/Sec: 1.17 +[2025-04-23 11:43:22] (step=0001375) Train Loss: 5.7914, Train Steps/Sec: 1.17 +[2025-04-23 11:43:44] (step=0001400) Train Loss: 5.6661, Train Steps/Sec: 1.17 +[2025-04-23 11:44:05] (step=0001425) Train Loss: 5.7186, Train Steps/Sec: 1.17 +[2025-04-23 11:44:26] (step=0001450) Train Loss: 5.8405, Train Steps/Sec: 1.17 +[2025-04-23 11:44:48] (step=0001475) Train Loss: 5.7429, Train Steps/Sec: 1.17 +[2025-04-23 11:45:09] (step=0001500) Train Loss: 5.6897, Train Steps/Sec: 1.17 +[2025-04-23 11:45:30] (step=0001525) Train Loss: 5.7404, Train Steps/Sec: 1.17 +[2025-04-23 11:45:53] (step=0001550) Train Loss: 5.7304, Train Steps/Sec: 1.12 +[2025-04-23 11:46:14] (step=0001575) Train Loss: 5.7393, Train Steps/Sec: 1.17 +[2025-04-23 11:46:35] (step=0001600) Train Loss: 5.7214, Train Steps/Sec: 1.17 +[2025-04-23 11:46:58] (step=0001625) Train Loss: 5.7278, Train Steps/Sec: 1.12 +[2025-04-23 11:47:21] (step=0001650) Train Loss: 5.7655, Train Steps/Sec: 1.08 +[2025-04-23 11:47:42] (step=0001675) Train Loss: 5.7132, Train Steps/Sec: 1.17 +[2025-04-23 11:48:05] (step=0001700) Train Loss: 5.8445, Train Steps/Sec: 1.11 +[2025-04-23 11:48:26] (step=0001725) Train Loss: 5.7540, Train Steps/Sec: 1.17 +[2025-04-23 11:48:48] (step=0001750) Train Loss: 5.6806, Train Steps/Sec: 1.17 +[2025-04-23 11:49:10] (step=0001775) Train Loss: 5.7114, Train Steps/Sec: 1.11 +[2025-04-23 11:49:32] (step=0001800) Train Loss: 5.7751, Train Steps/Sec: 1.17 +[2025-04-23 11:49:53] (step=0001825) Train Loss: 5.7199, Train Steps/Sec: 1.17 +[2025-04-23 11:50:16] (step=0001850) Train Loss: 5.7680, Train Steps/Sec: 1.07 +[2025-04-23 11:50:38] (step=0001875) Train Loss: 5.7145, Train Steps/Sec: 1.17 +[2025-04-23 11:50:59] (step=0001900) Train Loss: 5.6735, Train Steps/Sec: 1.17 +[2025-04-23 11:51:20] (step=0001925) Train Loss: 5.8258, Train Steps/Sec: 1.17 +[2025-04-23 11:51:42] (step=0001950) Train Loss: 5.7326, Train Steps/Sec: 1.17 +[2025-04-23 11:52:03] (step=0001975) Train Loss: 5.6651, Train Steps/Sec: 1.17 +[2025-04-23 11:52:25] (step=0002000) Train Loss: 5.6799, Train Steps/Sec: 1.17 +[2025-04-23 11:52:25] text_config is None. Initializing the text config with default values (`OPTConfig`). diff --git a/CKPTS/Afteri2i_100kckpt_LLAVAT5_empty_Fixed_just_segment_pretrain_stage3_dreambench_recap_400k_t2i_400k_flux_200k_midjourney_150k_recovery_150k_3e4_epoch2_half_rotated/005-GPT-XL/checkpoints/0046000.pt b/CKPTS/Afteri2i_100kckpt_LLAVAT5_empty_Fixed_just_segment_pretrain_stage3_dreambench_recap_400k_t2i_400k_flux_200k_midjourney_150k_recovery_150k_3e4_epoch2_half_rotated/005-GPT-XL/checkpoints/0046000.pt new file mode 100644 index 0000000000000000000000000000000000000000..a0737f0f6ec339cdaa87373fb8ad3b6ae41585da --- /dev/null +++ b/CKPTS/Afteri2i_100kckpt_LLAVAT5_empty_Fixed_just_segment_pretrain_stage3_dreambench_recap_400k_t2i_400k_flux_200k_midjourney_150k_recovery_150k_3e4_epoch2_half_rotated/005-GPT-XL/checkpoints/0046000.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:1d8a592e16377c249d4988c125dbe55d5d04d491fd305c8184ebb29e85d2eafc +size 17322479662 diff --git a/CKPTS/Afteri2i_100kckpt_LLAVAT5_empty_Fixed_just_segment_pretrain_stage3_dreambench_recap_400k_t2i_400k_flux_200k_midjourney_150k_recovery_150k_3e4_epoch2_half_rotated/005-GPT-XL/checkpoints/eval_step_10000/batch_0_cfg_7.5_topk_16384.jpg b/CKPTS/Afteri2i_100kckpt_LLAVAT5_empty_Fixed_just_segment_pretrain_stage3_dreambench_recap_400k_t2i_400k_flux_200k_midjourney_150k_recovery_150k_3e4_epoch2_half_rotated/005-GPT-XL/checkpoints/eval_step_10000/batch_0_cfg_7.5_topk_16384.jpg new file mode 100644 index 0000000000000000000000000000000000000000..10f2d437d316d0f93a265d33961229e3f7d8b918 --- /dev/null +++ b/CKPTS/Afteri2i_100kckpt_LLAVAT5_empty_Fixed_just_segment_pretrain_stage3_dreambench_recap_400k_t2i_400k_flux_200k_midjourney_150k_recovery_150k_3e4_epoch2_half_rotated/005-GPT-XL/checkpoints/eval_step_10000/batch_0_cfg_7.5_topk_16384.jpg @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:8b6a49fad270ba3bc87e580a39e79f8afb4bf236680161ea5d5b36b86944a506 +size 745850 diff --git a/CKPTS/Afteri2i_100kckpt_LLAVAT5_empty_Fixed_just_segment_pretrain_stage3_dreambench_recap_400k_t2i_400k_flux_200k_midjourney_150k_recovery_150k_3e4_epoch2_half_rotated/005-GPT-XL/checkpoints/eval_step_10000/batch_1_cfg_7.5_topk_16384.jpg b/CKPTS/Afteri2i_100kckpt_LLAVAT5_empty_Fixed_just_segment_pretrain_stage3_dreambench_recap_400k_t2i_400k_flux_200k_midjourney_150k_recovery_150k_3e4_epoch2_half_rotated/005-GPT-XL/checkpoints/eval_step_10000/batch_1_cfg_7.5_topk_16384.jpg new file mode 100644 index 0000000000000000000000000000000000000000..8c7e38b60ebe07fd6138b08d8001e6b05eb8c7e7 --- /dev/null +++ b/CKPTS/Afteri2i_100kckpt_LLAVAT5_empty_Fixed_just_segment_pretrain_stage3_dreambench_recap_400k_t2i_400k_flux_200k_midjourney_150k_recovery_150k_3e4_epoch2_half_rotated/005-GPT-XL/checkpoints/eval_step_10000/batch_1_cfg_7.5_topk_16384.jpg @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:bc5020d0b2124296502f38606504f5c5413427a70625fb8ebd35ed77fd9c2997 +size 706761 diff --git a/CKPTS/Afteri2i_100kckpt_LLAVAT5_empty_Fixed_just_segment_pretrain_stage3_dreambench_recap_400k_t2i_400k_flux_200k_midjourney_150k_recovery_150k_3e4_epoch2_half_rotated/005-GPT-XL/checkpoints/eval_step_10000/batch_2_cfg_7.5_topk_16384.jpg b/CKPTS/Afteri2i_100kckpt_LLAVAT5_empty_Fixed_just_segment_pretrain_stage3_dreambench_recap_400k_t2i_400k_flux_200k_midjourney_150k_recovery_150k_3e4_epoch2_half_rotated/005-GPT-XL/checkpoints/eval_step_10000/batch_2_cfg_7.5_topk_16384.jpg new file mode 100644 index 0000000000000000000000000000000000000000..cbef60ce75a1136726f02f0d5236c7da9b6a4fa5 --- /dev/null +++ b/CKPTS/Afteri2i_100kckpt_LLAVAT5_empty_Fixed_just_segment_pretrain_stage3_dreambench_recap_400k_t2i_400k_flux_200k_midjourney_150k_recovery_150k_3e4_epoch2_half_rotated/005-GPT-XL/checkpoints/eval_step_10000/batch_2_cfg_7.5_topk_16384.jpg @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:1fc60b1b446cb1a6e04f450293e0618bb6b7e55cde6e71ea0cf16a2540182fc8 +size 722719 diff --git a/CKPTS/Afteri2i_100kckpt_LLAVAT5_empty_Fixed_just_segment_pretrain_stage3_dreambench_recap_400k_t2i_400k_flux_200k_midjourney_150k_recovery_150k_3e4_epoch2_half_rotated/005-GPT-XL/checkpoints/eval_step_10000/batch_3_cfg_7.5_topk_16384.jpg b/CKPTS/Afteri2i_100kckpt_LLAVAT5_empty_Fixed_just_segment_pretrain_stage3_dreambench_recap_400k_t2i_400k_flux_200k_midjourney_150k_recovery_150k_3e4_epoch2_half_rotated/005-GPT-XL/checkpoints/eval_step_10000/batch_3_cfg_7.5_topk_16384.jpg new file mode 100644 index 0000000000000000000000000000000000000000..c736e2bc66c03daa2a99661a3e8f31e77993df4c --- /dev/null +++ b/CKPTS/Afteri2i_100kckpt_LLAVAT5_empty_Fixed_just_segment_pretrain_stage3_dreambench_recap_400k_t2i_400k_flux_200k_midjourney_150k_recovery_150k_3e4_epoch2_half_rotated/005-GPT-XL/checkpoints/eval_step_10000/batch_3_cfg_7.5_topk_16384.jpg @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:2b9f6d76d5a795f73e765edef4a9d66803f83222b55f57640c8528559fcde404 +size 855858 diff --git a/CKPTS/Afteri2i_100kckpt_LLAVAT5_empty_Fixed_just_segment_pretrain_stage3_dreambench_recap_400k_t2i_400k_flux_200k_midjourney_150k_recovery_150k_3e4_epoch2_half_rotated/005-GPT-XL/checkpoints/eval_step_12000/batch_0_cfg_7.5_topk_16384.jpg b/CKPTS/Afteri2i_100kckpt_LLAVAT5_empty_Fixed_just_segment_pretrain_stage3_dreambench_recap_400k_t2i_400k_flux_200k_midjourney_150k_recovery_150k_3e4_epoch2_half_rotated/005-GPT-XL/checkpoints/eval_step_12000/batch_0_cfg_7.5_topk_16384.jpg new file mode 100644 index 0000000000000000000000000000000000000000..be922adf3cf174112652df1b59ce2ddc08785764 --- /dev/null +++ b/CKPTS/Afteri2i_100kckpt_LLAVAT5_empty_Fixed_just_segment_pretrain_stage3_dreambench_recap_400k_t2i_400k_flux_200k_midjourney_150k_recovery_150k_3e4_epoch2_half_rotated/005-GPT-XL/checkpoints/eval_step_12000/batch_0_cfg_7.5_topk_16384.jpg @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:43d8eeb60e1162827b1c9f298b53900bee68c68d8c3a7624110c66f334cd95c7 +size 754193 diff --git a/CKPTS/Afteri2i_100kckpt_LLAVAT5_empty_Fixed_just_segment_pretrain_stage3_dreambench_recap_400k_t2i_400k_flux_200k_midjourney_150k_recovery_150k_3e4_epoch2_half_rotated/005-GPT-XL/checkpoints/eval_step_12000/batch_1_cfg_7.5_topk_16384.jpg b/CKPTS/Afteri2i_100kckpt_LLAVAT5_empty_Fixed_just_segment_pretrain_stage3_dreambench_recap_400k_t2i_400k_flux_200k_midjourney_150k_recovery_150k_3e4_epoch2_half_rotated/005-GPT-XL/checkpoints/eval_step_12000/batch_1_cfg_7.5_topk_16384.jpg new file mode 100644 index 0000000000000000000000000000000000000000..b1ebfc8f0fcfd5d401223f3e4cbb147fcb820ad6 --- /dev/null +++ b/CKPTS/Afteri2i_100kckpt_LLAVAT5_empty_Fixed_just_segment_pretrain_stage3_dreambench_recap_400k_t2i_400k_flux_200k_midjourney_150k_recovery_150k_3e4_epoch2_half_rotated/005-GPT-XL/checkpoints/eval_step_12000/batch_1_cfg_7.5_topk_16384.jpg @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:2800ced603f7025663dc26286cc9c1f6f2675751cd1145f7a9331c50858e9dd0 +size 702659 diff --git a/CKPTS/Afteri2i_100kckpt_LLAVAT5_empty_Fixed_just_segment_pretrain_stage3_dreambench_recap_400k_t2i_400k_flux_200k_midjourney_150k_recovery_150k_3e4_epoch2_half_rotated/005-GPT-XL/checkpoints/eval_step_12000/batch_2_cfg_7.5_topk_16384.jpg b/CKPTS/Afteri2i_100kckpt_LLAVAT5_empty_Fixed_just_segment_pretrain_stage3_dreambench_recap_400k_t2i_400k_flux_200k_midjourney_150k_recovery_150k_3e4_epoch2_half_rotated/005-GPT-XL/checkpoints/eval_step_12000/batch_2_cfg_7.5_topk_16384.jpg new file mode 100644 index 0000000000000000000000000000000000000000..c23a9f56a8bfc234a287ec1c0513137692798fd6 --- /dev/null +++ b/CKPTS/Afteri2i_100kckpt_LLAVAT5_empty_Fixed_just_segment_pretrain_stage3_dreambench_recap_400k_t2i_400k_flux_200k_midjourney_150k_recovery_150k_3e4_epoch2_half_rotated/005-GPT-XL/checkpoints/eval_step_12000/batch_2_cfg_7.5_topk_16384.jpg @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:9c28237ce5314e76994a808e0bf874db3dba782f120e37a3c162cd66aa5f36fc +size 723760 diff --git a/CKPTS/Afteri2i_100kckpt_LLAVAT5_empty_Fixed_just_segment_pretrain_stage3_dreambench_recap_400k_t2i_400k_flux_200k_midjourney_150k_recovery_150k_3e4_epoch2_half_rotated/005-GPT-XL/checkpoints/eval_step_12000/batch_3_cfg_7.5_topk_16384.jpg b/CKPTS/Afteri2i_100kckpt_LLAVAT5_empty_Fixed_just_segment_pretrain_stage3_dreambench_recap_400k_t2i_400k_flux_200k_midjourney_150k_recovery_150k_3e4_epoch2_half_rotated/005-GPT-XL/checkpoints/eval_step_12000/batch_3_cfg_7.5_topk_16384.jpg new file mode 100644 index 0000000000000000000000000000000000000000..94be16c798bfd52ec34b8a0262462ba4d3bbb923 --- /dev/null +++ b/CKPTS/Afteri2i_100kckpt_LLAVAT5_empty_Fixed_just_segment_pretrain_stage3_dreambench_recap_400k_t2i_400k_flux_200k_midjourney_150k_recovery_150k_3e4_epoch2_half_rotated/005-GPT-XL/checkpoints/eval_step_12000/batch_3_cfg_7.5_topk_16384.jpg @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:0f6302fcdb24ab5d2ad2674b6efd2a0ba34ee8693b7f3f3b75ed2d2e4fc22777 +size 854294 diff --git a/CKPTS/Afteri2i_100kckpt_LLAVAT5_empty_Fixed_just_segment_pretrain_stage3_dreambench_recap_400k_t2i_400k_flux_200k_midjourney_150k_recovery_150k_3e4_epoch2_half_rotated/005-GPT-XL/checkpoints/eval_step_14000/batch_0_cfg_7.5_topk_16384.jpg b/CKPTS/Afteri2i_100kckpt_LLAVAT5_empty_Fixed_just_segment_pretrain_stage3_dreambench_recap_400k_t2i_400k_flux_200k_midjourney_150k_recovery_150k_3e4_epoch2_half_rotated/005-GPT-XL/checkpoints/eval_step_14000/batch_0_cfg_7.5_topk_16384.jpg new file mode 100644 index 0000000000000000000000000000000000000000..cfc7e87a1dd97e2c0a4c751983c89ccccc355d5a --- /dev/null +++ b/CKPTS/Afteri2i_100kckpt_LLAVAT5_empty_Fixed_just_segment_pretrain_stage3_dreambench_recap_400k_t2i_400k_flux_200k_midjourney_150k_recovery_150k_3e4_epoch2_half_rotated/005-GPT-XL/checkpoints/eval_step_14000/batch_0_cfg_7.5_topk_16384.jpg @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:b18969f2471626aaaf5ad6da3ebfbed1a636babaf6aaec6359061fb15a5491a3 +size 730055 diff --git a/CKPTS/Afteri2i_100kckpt_LLAVAT5_empty_Fixed_just_segment_pretrain_stage3_dreambench_recap_400k_t2i_400k_flux_200k_midjourney_150k_recovery_150k_3e4_epoch2_half_rotated/005-GPT-XL/checkpoints/eval_step_14000/batch_1_cfg_7.5_topk_16384.jpg b/CKPTS/Afteri2i_100kckpt_LLAVAT5_empty_Fixed_just_segment_pretrain_stage3_dreambench_recap_400k_t2i_400k_flux_200k_midjourney_150k_recovery_150k_3e4_epoch2_half_rotated/005-GPT-XL/checkpoints/eval_step_14000/batch_1_cfg_7.5_topk_16384.jpg new file mode 100644 index 0000000000000000000000000000000000000000..36837c20d05710215a1a5ee78ac9d080433bdd31 --- /dev/null +++ b/CKPTS/Afteri2i_100kckpt_LLAVAT5_empty_Fixed_just_segment_pretrain_stage3_dreambench_recap_400k_t2i_400k_flux_200k_midjourney_150k_recovery_150k_3e4_epoch2_half_rotated/005-GPT-XL/checkpoints/eval_step_14000/batch_1_cfg_7.5_topk_16384.jpg @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:7c00c97a83959a99495b601991694c2e74ca41248585ea5971b4c2c797c3003f +size 715541 diff --git a/CKPTS/Afteri2i_100kckpt_LLAVAT5_empty_Fixed_just_segment_pretrain_stage3_dreambench_recap_400k_t2i_400k_flux_200k_midjourney_150k_recovery_150k_3e4_epoch2_half_rotated/005-GPT-XL/checkpoints/eval_step_14000/batch_2_cfg_7.5_topk_16384.jpg b/CKPTS/Afteri2i_100kckpt_LLAVAT5_empty_Fixed_just_segment_pretrain_stage3_dreambench_recap_400k_t2i_400k_flux_200k_midjourney_150k_recovery_150k_3e4_epoch2_half_rotated/005-GPT-XL/checkpoints/eval_step_14000/batch_2_cfg_7.5_topk_16384.jpg new file mode 100644 index 0000000000000000000000000000000000000000..07ff2fb86aa8fca4d5a590bda9eb84e8ccf21155 --- /dev/null +++ b/CKPTS/Afteri2i_100kckpt_LLAVAT5_empty_Fixed_just_segment_pretrain_stage3_dreambench_recap_400k_t2i_400k_flux_200k_midjourney_150k_recovery_150k_3e4_epoch2_half_rotated/005-GPT-XL/checkpoints/eval_step_14000/batch_2_cfg_7.5_topk_16384.jpg @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:41cabf50d7b9832f2a2686662855dc1b1dc0ad97b2c3f60bb3431bda6a8a84ca +size 713225 diff --git a/CKPTS/Afteri2i_100kckpt_LLAVAT5_empty_Fixed_just_segment_pretrain_stage3_dreambench_recap_400k_t2i_400k_flux_200k_midjourney_150k_recovery_150k_3e4_epoch2_half_rotated/005-GPT-XL/checkpoints/eval_step_14000/batch_3_cfg_7.5_topk_16384.jpg b/CKPTS/Afteri2i_100kckpt_LLAVAT5_empty_Fixed_just_segment_pretrain_stage3_dreambench_recap_400k_t2i_400k_flux_200k_midjourney_150k_recovery_150k_3e4_epoch2_half_rotated/005-GPT-XL/checkpoints/eval_step_14000/batch_3_cfg_7.5_topk_16384.jpg new file mode 100644 index 0000000000000000000000000000000000000000..4144f7de8a310bada7223e483e225fca3d2b7e60 --- /dev/null +++ b/CKPTS/Afteri2i_100kckpt_LLAVAT5_empty_Fixed_just_segment_pretrain_stage3_dreambench_recap_400k_t2i_400k_flux_200k_midjourney_150k_recovery_150k_3e4_epoch2_half_rotated/005-GPT-XL/checkpoints/eval_step_14000/batch_3_cfg_7.5_topk_16384.jpg @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:341e0637416a26ed1999b13d435e507e7717bee1c8aafdeb4965158e20b05ed3 +size 879113 diff --git a/CKPTS/Afteri2i_100kckpt_LLAVAT5_empty_Fixed_just_segment_pretrain_stage3_dreambench_recap_400k_t2i_400k_flux_200k_midjourney_150k_recovery_150k_3e4_epoch2_half_rotated/005-GPT-XL/checkpoints/eval_step_16000/batch_0_cfg_7.5_topk_16384.jpg b/CKPTS/Afteri2i_100kckpt_LLAVAT5_empty_Fixed_just_segment_pretrain_stage3_dreambench_recap_400k_t2i_400k_flux_200k_midjourney_150k_recovery_150k_3e4_epoch2_half_rotated/005-GPT-XL/checkpoints/eval_step_16000/batch_0_cfg_7.5_topk_16384.jpg new file mode 100644 index 0000000000000000000000000000000000000000..d331aa7aa4151f981f76313d47cfde0cc09969a6 --- /dev/null +++ b/CKPTS/Afteri2i_100kckpt_LLAVAT5_empty_Fixed_just_segment_pretrain_stage3_dreambench_recap_400k_t2i_400k_flux_200k_midjourney_150k_recovery_150k_3e4_epoch2_half_rotated/005-GPT-XL/checkpoints/eval_step_16000/batch_0_cfg_7.5_topk_16384.jpg @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:ff4d19dcb0b0b89de87ebba2c9584884f3dace4f9888e24497f852e8199d3652 +size 758918 diff --git a/CKPTS/Afteri2i_100kckpt_LLAVAT5_empty_Fixed_just_segment_pretrain_stage3_dreambench_recap_400k_t2i_400k_flux_200k_midjourney_150k_recovery_150k_3e4_epoch2_half_rotated/005-GPT-XL/checkpoints/eval_step_16000/batch_1_cfg_7.5_topk_16384.jpg b/CKPTS/Afteri2i_100kckpt_LLAVAT5_empty_Fixed_just_segment_pretrain_stage3_dreambench_recap_400k_t2i_400k_flux_200k_midjourney_150k_recovery_150k_3e4_epoch2_half_rotated/005-GPT-XL/checkpoints/eval_step_16000/batch_1_cfg_7.5_topk_16384.jpg new file mode 100644 index 0000000000000000000000000000000000000000..a261a80130bce013dc1fe0a1ea96402db3014a37 --- /dev/null +++ b/CKPTS/Afteri2i_100kckpt_LLAVAT5_empty_Fixed_just_segment_pretrain_stage3_dreambench_recap_400k_t2i_400k_flux_200k_midjourney_150k_recovery_150k_3e4_epoch2_half_rotated/005-GPT-XL/checkpoints/eval_step_16000/batch_1_cfg_7.5_topk_16384.jpg @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:3863808c89b73f7ed31c319f6b01bae8f691aeee6050b6152a4e3c3098b95404 +size 700555 diff --git a/CKPTS/Afteri2i_100kckpt_LLAVAT5_empty_Fixed_just_segment_pretrain_stage3_dreambench_recap_400k_t2i_400k_flux_200k_midjourney_150k_recovery_150k_3e4_epoch2_half_rotated/005-GPT-XL/checkpoints/eval_step_16000/batch_2_cfg_7.5_topk_16384.jpg b/CKPTS/Afteri2i_100kckpt_LLAVAT5_empty_Fixed_just_segment_pretrain_stage3_dreambench_recap_400k_t2i_400k_flux_200k_midjourney_150k_recovery_150k_3e4_epoch2_half_rotated/005-GPT-XL/checkpoints/eval_step_16000/batch_2_cfg_7.5_topk_16384.jpg new file mode 100644 index 0000000000000000000000000000000000000000..c1e75b22c0802da9061f5d39cd086c076d51b7e5 --- /dev/null +++ b/CKPTS/Afteri2i_100kckpt_LLAVAT5_empty_Fixed_just_segment_pretrain_stage3_dreambench_recap_400k_t2i_400k_flux_200k_midjourney_150k_recovery_150k_3e4_epoch2_half_rotated/005-GPT-XL/checkpoints/eval_step_16000/batch_2_cfg_7.5_topk_16384.jpg @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:f6513da77785889ba60c0145e68969a2c893bb959c1d80bce754178bd22afaf1 +size 735630 diff --git a/CKPTS/Afteri2i_100kckpt_LLAVAT5_empty_Fixed_just_segment_pretrain_stage3_dreambench_recap_400k_t2i_400k_flux_200k_midjourney_150k_recovery_150k_3e4_epoch2_half_rotated/005-GPT-XL/checkpoints/eval_step_16000/batch_3_cfg_7.5_topk_16384.jpg b/CKPTS/Afteri2i_100kckpt_LLAVAT5_empty_Fixed_just_segment_pretrain_stage3_dreambench_recap_400k_t2i_400k_flux_200k_midjourney_150k_recovery_150k_3e4_epoch2_half_rotated/005-GPT-XL/checkpoints/eval_step_16000/batch_3_cfg_7.5_topk_16384.jpg new file mode 100644 index 0000000000000000000000000000000000000000..2ac0e46113155b9962cd8e94405fc36d048edabe --- /dev/null +++ b/CKPTS/Afteri2i_100kckpt_LLAVAT5_empty_Fixed_just_segment_pretrain_stage3_dreambench_recap_400k_t2i_400k_flux_200k_midjourney_150k_recovery_150k_3e4_epoch2_half_rotated/005-GPT-XL/checkpoints/eval_step_16000/batch_3_cfg_7.5_topk_16384.jpg @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:a3c9a6015b8731bff8f227807d860d09eac65af9ce04c539155ba1912f74d582 +size 855637 diff --git a/CKPTS/Afteri2i_100kckpt_LLAVAT5_empty_Fixed_just_segment_pretrain_stage3_dreambench_recap_400k_t2i_400k_flux_200k_midjourney_150k_recovery_150k_3e4_epoch2_half_rotated/005-GPT-XL/checkpoints/eval_step_18000/batch_0_cfg_7.5_topk_16384.jpg b/CKPTS/Afteri2i_100kckpt_LLAVAT5_empty_Fixed_just_segment_pretrain_stage3_dreambench_recap_400k_t2i_400k_flux_200k_midjourney_150k_recovery_150k_3e4_epoch2_half_rotated/005-GPT-XL/checkpoints/eval_step_18000/batch_0_cfg_7.5_topk_16384.jpg new file mode 100644 index 0000000000000000000000000000000000000000..e1ebccf67bb0966a2d530af9bc8e9342de39e970 --- /dev/null +++ b/CKPTS/Afteri2i_100kckpt_LLAVAT5_empty_Fixed_just_segment_pretrain_stage3_dreambench_recap_400k_t2i_400k_flux_200k_midjourney_150k_recovery_150k_3e4_epoch2_half_rotated/005-GPT-XL/checkpoints/eval_step_18000/batch_0_cfg_7.5_topk_16384.jpg @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:4b4f65f5d81711fc46301a5ca9ed10ff7e404869c7288187886bbb9e8ab4b8a9 +size 745425 diff --git a/CKPTS/Afteri2i_100kckpt_LLAVAT5_empty_Fixed_just_segment_pretrain_stage3_dreambench_recap_400k_t2i_400k_flux_200k_midjourney_150k_recovery_150k_3e4_epoch2_half_rotated/005-GPT-XL/checkpoints/eval_step_18000/batch_1_cfg_7.5_topk_16384.jpg b/CKPTS/Afteri2i_100kckpt_LLAVAT5_empty_Fixed_just_segment_pretrain_stage3_dreambench_recap_400k_t2i_400k_flux_200k_midjourney_150k_recovery_150k_3e4_epoch2_half_rotated/005-GPT-XL/checkpoints/eval_step_18000/batch_1_cfg_7.5_topk_16384.jpg new file mode 100644 index 0000000000000000000000000000000000000000..16906dc53913b53524a9b6881494800e668a656c --- /dev/null +++ b/CKPTS/Afteri2i_100kckpt_LLAVAT5_empty_Fixed_just_segment_pretrain_stage3_dreambench_recap_400k_t2i_400k_flux_200k_midjourney_150k_recovery_150k_3e4_epoch2_half_rotated/005-GPT-XL/checkpoints/eval_step_18000/batch_1_cfg_7.5_topk_16384.jpg @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:cdede1a0e9f16936ba9a366755f8dfe5ac119b100f1bd66e17529a52b94c680d +size 707992 diff --git a/CKPTS/Afteri2i_100kckpt_LLAVAT5_empty_Fixed_just_segment_pretrain_stage3_dreambench_recap_400k_t2i_400k_flux_200k_midjourney_150k_recovery_150k_3e4_epoch2_half_rotated/005-GPT-XL/checkpoints/eval_step_18000/batch_2_cfg_7.5_topk_16384.jpg b/CKPTS/Afteri2i_100kckpt_LLAVAT5_empty_Fixed_just_segment_pretrain_stage3_dreambench_recap_400k_t2i_400k_flux_200k_midjourney_150k_recovery_150k_3e4_epoch2_half_rotated/005-GPT-XL/checkpoints/eval_step_18000/batch_2_cfg_7.5_topk_16384.jpg new file mode 100644 index 0000000000000000000000000000000000000000..84995452ee8d7c1edc666c2c6856e9832346c223 --- /dev/null +++ b/CKPTS/Afteri2i_100kckpt_LLAVAT5_empty_Fixed_just_segment_pretrain_stage3_dreambench_recap_400k_t2i_400k_flux_200k_midjourney_150k_recovery_150k_3e4_epoch2_half_rotated/005-GPT-XL/checkpoints/eval_step_18000/batch_2_cfg_7.5_topk_16384.jpg @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:07f70a96d1b8544b54fa46c5321be2ffdd50c7b6411daf7eb02699bc87d8bc4c +size 756648 diff --git a/CKPTS/Afteri2i_100kckpt_LLAVAT5_empty_Fixed_just_segment_pretrain_stage3_dreambench_recap_400k_t2i_400k_flux_200k_midjourney_150k_recovery_150k_3e4_epoch2_half_rotated/005-GPT-XL/checkpoints/eval_step_18000/batch_3_cfg_7.5_topk_16384.jpg b/CKPTS/Afteri2i_100kckpt_LLAVAT5_empty_Fixed_just_segment_pretrain_stage3_dreambench_recap_400k_t2i_400k_flux_200k_midjourney_150k_recovery_150k_3e4_epoch2_half_rotated/005-GPT-XL/checkpoints/eval_step_18000/batch_3_cfg_7.5_topk_16384.jpg new file mode 100644 index 0000000000000000000000000000000000000000..c6139e0c9a1c0d95ea94f2a3f27eab061f1476b9 --- /dev/null +++ b/CKPTS/Afteri2i_100kckpt_LLAVAT5_empty_Fixed_just_segment_pretrain_stage3_dreambench_recap_400k_t2i_400k_flux_200k_midjourney_150k_recovery_150k_3e4_epoch2_half_rotated/005-GPT-XL/checkpoints/eval_step_18000/batch_3_cfg_7.5_topk_16384.jpg @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:5a06417f175ccf3f26a78db64463e2d9501a95546557b50331875d1c0d73ed76 +size 871042 diff --git a/CKPTS/Afteri2i_100kckpt_LLAVAT5_empty_Fixed_just_segment_pretrain_stage3_dreambench_recap_400k_t2i_400k_flux_200k_midjourney_150k_recovery_150k_3e4_epoch2_half_rotated/005-GPT-XL/checkpoints/eval_step_2000/batch_0_cfg_7.5_topk_16384.jpg b/CKPTS/Afteri2i_100kckpt_LLAVAT5_empty_Fixed_just_segment_pretrain_stage3_dreambench_recap_400k_t2i_400k_flux_200k_midjourney_150k_recovery_150k_3e4_epoch2_half_rotated/005-GPT-XL/checkpoints/eval_step_2000/batch_0_cfg_7.5_topk_16384.jpg new file mode 100644 index 0000000000000000000000000000000000000000..3bcf1dade356ede219478aaebbb11812c04b53af --- /dev/null +++ b/CKPTS/Afteri2i_100kckpt_LLAVAT5_empty_Fixed_just_segment_pretrain_stage3_dreambench_recap_400k_t2i_400k_flux_200k_midjourney_150k_recovery_150k_3e4_epoch2_half_rotated/005-GPT-XL/checkpoints/eval_step_2000/batch_0_cfg_7.5_topk_16384.jpg @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:f2462b604cf119f28cfd5f9f2ee6ba65bb902d1ae0f3744618b283f4a77fc0d6 +size 786136 diff --git a/CKPTS/Afteri2i_100kckpt_LLAVAT5_empty_Fixed_just_segment_pretrain_stage3_dreambench_recap_400k_t2i_400k_flux_200k_midjourney_150k_recovery_150k_3e4_epoch2_half_rotated/005-GPT-XL/checkpoints/eval_step_2000/batch_1_cfg_7.5_topk_16384.jpg b/CKPTS/Afteri2i_100kckpt_LLAVAT5_empty_Fixed_just_segment_pretrain_stage3_dreambench_recap_400k_t2i_400k_flux_200k_midjourney_150k_recovery_150k_3e4_epoch2_half_rotated/005-GPT-XL/checkpoints/eval_step_2000/batch_1_cfg_7.5_topk_16384.jpg new file mode 100644 index 0000000000000000000000000000000000000000..cc1043572213dc789333f1db871390b2190d5943 --- /dev/null +++ b/CKPTS/Afteri2i_100kckpt_LLAVAT5_empty_Fixed_just_segment_pretrain_stage3_dreambench_recap_400k_t2i_400k_flux_200k_midjourney_150k_recovery_150k_3e4_epoch2_half_rotated/005-GPT-XL/checkpoints/eval_step_2000/batch_1_cfg_7.5_topk_16384.jpg @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:b3e3d9a67c23566414d2f1c0625f320b3c59874c0ce09c3cfafab47530fffac2 +size 707222 diff --git a/CKPTS/Afteri2i_100kckpt_LLAVAT5_empty_Fixed_just_segment_pretrain_stage3_dreambench_recap_400k_t2i_400k_flux_200k_midjourney_150k_recovery_150k_3e4_epoch2_half_rotated/005-GPT-XL/checkpoints/eval_step_2000/batch_2_cfg_7.5_topk_16384.jpg b/CKPTS/Afteri2i_100kckpt_LLAVAT5_empty_Fixed_just_segment_pretrain_stage3_dreambench_recap_400k_t2i_400k_flux_200k_midjourney_150k_recovery_150k_3e4_epoch2_half_rotated/005-GPT-XL/checkpoints/eval_step_2000/batch_2_cfg_7.5_topk_16384.jpg new file mode 100644 index 0000000000000000000000000000000000000000..55b483cba54c5fe73cd346ae7e46cff50d9241bb --- /dev/null +++ b/CKPTS/Afteri2i_100kckpt_LLAVAT5_empty_Fixed_just_segment_pretrain_stage3_dreambench_recap_400k_t2i_400k_flux_200k_midjourney_150k_recovery_150k_3e4_epoch2_half_rotated/005-GPT-XL/checkpoints/eval_step_2000/batch_2_cfg_7.5_topk_16384.jpg @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:05974ebd36c343484fa35cda09eebc15534df05b5e3f53682213f8cd8b33404f +size 648665 diff --git a/CKPTS/Afteri2i_100kckpt_LLAVAT5_empty_Fixed_just_segment_pretrain_stage3_dreambench_recap_400k_t2i_400k_flux_200k_midjourney_150k_recovery_150k_3e4_epoch2_half_rotated/005-GPT-XL/checkpoints/eval_step_2000/batch_3_cfg_7.5_topk_16384.jpg b/CKPTS/Afteri2i_100kckpt_LLAVAT5_empty_Fixed_just_segment_pretrain_stage3_dreambench_recap_400k_t2i_400k_flux_200k_midjourney_150k_recovery_150k_3e4_epoch2_half_rotated/005-GPT-XL/checkpoints/eval_step_2000/batch_3_cfg_7.5_topk_16384.jpg new file mode 100644 index 0000000000000000000000000000000000000000..36589ec9da1f49510208efec08813a45a6a5c52a --- /dev/null +++ b/CKPTS/Afteri2i_100kckpt_LLAVAT5_empty_Fixed_just_segment_pretrain_stage3_dreambench_recap_400k_t2i_400k_flux_200k_midjourney_150k_recovery_150k_3e4_epoch2_half_rotated/005-GPT-XL/checkpoints/eval_step_2000/batch_3_cfg_7.5_topk_16384.jpg @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:154fa170ea37a686b6c7ddaeea7143d4f33815c8822ef6dff5af68b9f2cd4026 +size 783177 diff --git a/CKPTS/Afteri2i_100kckpt_LLAVAT5_empty_Fixed_just_segment_pretrain_stage3_dreambench_recap_400k_t2i_400k_flux_200k_midjourney_150k_recovery_150k_3e4_epoch2_half_rotated/005-GPT-XL/checkpoints/eval_step_20000/batch_0_cfg_7.5_topk_16384.jpg b/CKPTS/Afteri2i_100kckpt_LLAVAT5_empty_Fixed_just_segment_pretrain_stage3_dreambench_recap_400k_t2i_400k_flux_200k_midjourney_150k_recovery_150k_3e4_epoch2_half_rotated/005-GPT-XL/checkpoints/eval_step_20000/batch_0_cfg_7.5_topk_16384.jpg new file mode 100644 index 0000000000000000000000000000000000000000..6902ec8d0bc6008095671d0897b51980ff69abe5 --- /dev/null +++ b/CKPTS/Afteri2i_100kckpt_LLAVAT5_empty_Fixed_just_segment_pretrain_stage3_dreambench_recap_400k_t2i_400k_flux_200k_midjourney_150k_recovery_150k_3e4_epoch2_half_rotated/005-GPT-XL/checkpoints/eval_step_20000/batch_0_cfg_7.5_topk_16384.jpg @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:4ebe6a371377f359a16895a71d12dc4051c2c955cdbc49619aa1837af128b7e2 +size 739398 diff --git a/CKPTS/Afteri2i_100kckpt_LLAVAT5_empty_Fixed_just_segment_pretrain_stage3_dreambench_recap_400k_t2i_400k_flux_200k_midjourney_150k_recovery_150k_3e4_epoch2_half_rotated/005-GPT-XL/checkpoints/eval_step_20000/batch_1_cfg_7.5_topk_16384.jpg b/CKPTS/Afteri2i_100kckpt_LLAVAT5_empty_Fixed_just_segment_pretrain_stage3_dreambench_recap_400k_t2i_400k_flux_200k_midjourney_150k_recovery_150k_3e4_epoch2_half_rotated/005-GPT-XL/checkpoints/eval_step_20000/batch_1_cfg_7.5_topk_16384.jpg new file mode 100644 index 0000000000000000000000000000000000000000..9957fa56103230873f1cdb95248619af0e9bb5e0 --- /dev/null +++ b/CKPTS/Afteri2i_100kckpt_LLAVAT5_empty_Fixed_just_segment_pretrain_stage3_dreambench_recap_400k_t2i_400k_flux_200k_midjourney_150k_recovery_150k_3e4_epoch2_half_rotated/005-GPT-XL/checkpoints/eval_step_20000/batch_1_cfg_7.5_topk_16384.jpg @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:5caf60867ed22f81254153f9471f69921a72733b88b5ca8cb04128c23be728a3 +size 719076 diff --git a/CKPTS/Afteri2i_100kckpt_LLAVAT5_empty_Fixed_just_segment_pretrain_stage3_dreambench_recap_400k_t2i_400k_flux_200k_midjourney_150k_recovery_150k_3e4_epoch2_half_rotated/005-GPT-XL/checkpoints/eval_step_20000/batch_2_cfg_7.5_topk_16384.jpg b/CKPTS/Afteri2i_100kckpt_LLAVAT5_empty_Fixed_just_segment_pretrain_stage3_dreambench_recap_400k_t2i_400k_flux_200k_midjourney_150k_recovery_150k_3e4_epoch2_half_rotated/005-GPT-XL/checkpoints/eval_step_20000/batch_2_cfg_7.5_topk_16384.jpg new file mode 100644 index 0000000000000000000000000000000000000000..d84fbfbcb26652cb1f0064ac7450e579dc988bb8 --- /dev/null +++ b/CKPTS/Afteri2i_100kckpt_LLAVAT5_empty_Fixed_just_segment_pretrain_stage3_dreambench_recap_400k_t2i_400k_flux_200k_midjourney_150k_recovery_150k_3e4_epoch2_half_rotated/005-GPT-XL/checkpoints/eval_step_20000/batch_2_cfg_7.5_topk_16384.jpg @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:1d122e9b0f49b13b38ba55ecc5526622271400641bc74abec28ba3173a371633 +size 705699 diff --git a/CKPTS/Afteri2i_100kckpt_LLAVAT5_empty_Fixed_just_segment_pretrain_stage3_dreambench_recap_400k_t2i_400k_flux_200k_midjourney_150k_recovery_150k_3e4_epoch2_half_rotated/005-GPT-XL/checkpoints/eval_step_20000/batch_3_cfg_7.5_topk_16384.jpg b/CKPTS/Afteri2i_100kckpt_LLAVAT5_empty_Fixed_just_segment_pretrain_stage3_dreambench_recap_400k_t2i_400k_flux_200k_midjourney_150k_recovery_150k_3e4_epoch2_half_rotated/005-GPT-XL/checkpoints/eval_step_20000/batch_3_cfg_7.5_topk_16384.jpg new file mode 100644 index 0000000000000000000000000000000000000000..861f7a5a7f92bfcd3030f9429d94fb0445595011 --- /dev/null +++ b/CKPTS/Afteri2i_100kckpt_LLAVAT5_empty_Fixed_just_segment_pretrain_stage3_dreambench_recap_400k_t2i_400k_flux_200k_midjourney_150k_recovery_150k_3e4_epoch2_half_rotated/005-GPT-XL/checkpoints/eval_step_20000/batch_3_cfg_7.5_topk_16384.jpg @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:b07f07e782284bd82c474f4805efb901b00e21469784cd5e642b529948a64e1c +size 861849 diff --git a/CKPTS/Afteri2i_100kckpt_LLAVAT5_empty_Fixed_just_segment_pretrain_stage3_dreambench_recap_400k_t2i_400k_flux_200k_midjourney_150k_recovery_150k_3e4_epoch2_half_rotated/005-GPT-XL/checkpoints/eval_step_22000/batch_0_cfg_7.5_topk_16384.jpg b/CKPTS/Afteri2i_100kckpt_LLAVAT5_empty_Fixed_just_segment_pretrain_stage3_dreambench_recap_400k_t2i_400k_flux_200k_midjourney_150k_recovery_150k_3e4_epoch2_half_rotated/005-GPT-XL/checkpoints/eval_step_22000/batch_0_cfg_7.5_topk_16384.jpg new file mode 100644 index 0000000000000000000000000000000000000000..3a6ef68fab3a4092bbd6547f5f6f88f79f1729fd --- /dev/null +++ b/CKPTS/Afteri2i_100kckpt_LLAVAT5_empty_Fixed_just_segment_pretrain_stage3_dreambench_recap_400k_t2i_400k_flux_200k_midjourney_150k_recovery_150k_3e4_epoch2_half_rotated/005-GPT-XL/checkpoints/eval_step_22000/batch_0_cfg_7.5_topk_16384.jpg @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:24c6f028d5ae9ee2a08124eefbb90ae2a1a0deed8984ab144fb20309dc961fd1 +size 729147 diff --git a/CKPTS/Afteri2i_100kckpt_LLAVAT5_empty_Fixed_just_segment_pretrain_stage3_dreambench_recap_400k_t2i_400k_flux_200k_midjourney_150k_recovery_150k_3e4_epoch2_half_rotated/005-GPT-XL/checkpoints/eval_step_22000/batch_1_cfg_7.5_topk_16384.jpg b/CKPTS/Afteri2i_100kckpt_LLAVAT5_empty_Fixed_just_segment_pretrain_stage3_dreambench_recap_400k_t2i_400k_flux_200k_midjourney_150k_recovery_150k_3e4_epoch2_half_rotated/005-GPT-XL/checkpoints/eval_step_22000/batch_1_cfg_7.5_topk_16384.jpg new file mode 100644 index 0000000000000000000000000000000000000000..192ab2b9c103d55cde68783db9650665668b7716 --- /dev/null +++ b/CKPTS/Afteri2i_100kckpt_LLAVAT5_empty_Fixed_just_segment_pretrain_stage3_dreambench_recap_400k_t2i_400k_flux_200k_midjourney_150k_recovery_150k_3e4_epoch2_half_rotated/005-GPT-XL/checkpoints/eval_step_22000/batch_1_cfg_7.5_topk_16384.jpg @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:1cabb87aec1f60eae3632d50c68ba503c017970580e941a78bdd16bd52475ac5 +size 688894 diff --git a/CKPTS/Afteri2i_100kckpt_LLAVAT5_empty_Fixed_just_segment_pretrain_stage3_dreambench_recap_400k_t2i_400k_flux_200k_midjourney_150k_recovery_150k_3e4_epoch2_half_rotated/005-GPT-XL/checkpoints/eval_step_22000/batch_2_cfg_7.5_topk_16384.jpg b/CKPTS/Afteri2i_100kckpt_LLAVAT5_empty_Fixed_just_segment_pretrain_stage3_dreambench_recap_400k_t2i_400k_flux_200k_midjourney_150k_recovery_150k_3e4_epoch2_half_rotated/005-GPT-XL/checkpoints/eval_step_22000/batch_2_cfg_7.5_topk_16384.jpg new file mode 100644 index 0000000000000000000000000000000000000000..938e8e4769cafdc29ef4cf7eeaf3aeb3437cbf1b --- /dev/null +++ b/CKPTS/Afteri2i_100kckpt_LLAVAT5_empty_Fixed_just_segment_pretrain_stage3_dreambench_recap_400k_t2i_400k_flux_200k_midjourney_150k_recovery_150k_3e4_epoch2_half_rotated/005-GPT-XL/checkpoints/eval_step_22000/batch_2_cfg_7.5_topk_16384.jpg @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:a302685a58fcce7d92ba1b8f0e507d976410fe41af71d5675b9096e886243b18 +size 712779 diff --git a/CKPTS/Afteri2i_100kckpt_LLAVAT5_empty_Fixed_just_segment_pretrain_stage3_dreambench_recap_400k_t2i_400k_flux_200k_midjourney_150k_recovery_150k_3e4_epoch2_half_rotated/005-GPT-XL/checkpoints/eval_step_22000/batch_3_cfg_7.5_topk_16384.jpg b/CKPTS/Afteri2i_100kckpt_LLAVAT5_empty_Fixed_just_segment_pretrain_stage3_dreambench_recap_400k_t2i_400k_flux_200k_midjourney_150k_recovery_150k_3e4_epoch2_half_rotated/005-GPT-XL/checkpoints/eval_step_22000/batch_3_cfg_7.5_topk_16384.jpg new file mode 100644 index 0000000000000000000000000000000000000000..06aab6a3393301ffa9f5b04f8537680621509f76 --- /dev/null +++ b/CKPTS/Afteri2i_100kckpt_LLAVAT5_empty_Fixed_just_segment_pretrain_stage3_dreambench_recap_400k_t2i_400k_flux_200k_midjourney_150k_recovery_150k_3e4_epoch2_half_rotated/005-GPT-XL/checkpoints/eval_step_22000/batch_3_cfg_7.5_topk_16384.jpg @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:5a98023753832f679d5dc8b3e0df0b4b7a2ecc31d869578c01ec8ba89337a247 +size 844261 diff --git a/CKPTS/Afteri2i_100kckpt_LLAVAT5_empty_Fixed_just_segment_pretrain_stage3_dreambench_recap_400k_t2i_400k_flux_200k_midjourney_150k_recovery_150k_3e4_epoch2_half_rotated/005-GPT-XL/checkpoints/eval_step_24000/batch_0_cfg_7.5_topk_16384.jpg b/CKPTS/Afteri2i_100kckpt_LLAVAT5_empty_Fixed_just_segment_pretrain_stage3_dreambench_recap_400k_t2i_400k_flux_200k_midjourney_150k_recovery_150k_3e4_epoch2_half_rotated/005-GPT-XL/checkpoints/eval_step_24000/batch_0_cfg_7.5_topk_16384.jpg new file mode 100644 index 0000000000000000000000000000000000000000..d5a763257106e18dd7c76cea93ece79229ff1c12 --- /dev/null +++ b/CKPTS/Afteri2i_100kckpt_LLAVAT5_empty_Fixed_just_segment_pretrain_stage3_dreambench_recap_400k_t2i_400k_flux_200k_midjourney_150k_recovery_150k_3e4_epoch2_half_rotated/005-GPT-XL/checkpoints/eval_step_24000/batch_0_cfg_7.5_topk_16384.jpg @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:0eec9a7a6d3c3b41a0f58144362de2104553a2eed85d649ca752e215e031baa6 +size 748233 diff --git a/CKPTS/Afteri2i_100kckpt_LLAVAT5_empty_Fixed_just_segment_pretrain_stage3_dreambench_recap_400k_t2i_400k_flux_200k_midjourney_150k_recovery_150k_3e4_epoch2_half_rotated/005-GPT-XL/checkpoints/eval_step_24000/batch_1_cfg_7.5_topk_16384.jpg b/CKPTS/Afteri2i_100kckpt_LLAVAT5_empty_Fixed_just_segment_pretrain_stage3_dreambench_recap_400k_t2i_400k_flux_200k_midjourney_150k_recovery_150k_3e4_epoch2_half_rotated/005-GPT-XL/checkpoints/eval_step_24000/batch_1_cfg_7.5_topk_16384.jpg new file mode 100644 index 0000000000000000000000000000000000000000..c7c56392a5db3fb0ccd7f24435b324551635ac50 --- /dev/null +++ b/CKPTS/Afteri2i_100kckpt_LLAVAT5_empty_Fixed_just_segment_pretrain_stage3_dreambench_recap_400k_t2i_400k_flux_200k_midjourney_150k_recovery_150k_3e4_epoch2_half_rotated/005-GPT-XL/checkpoints/eval_step_24000/batch_1_cfg_7.5_topk_16384.jpg @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:15c4233c29f1c6c831495d39380b111a5076812bded69b24cfb9059406585db6 +size 710573 diff --git a/CKPTS/Afteri2i_100kckpt_LLAVAT5_empty_Fixed_just_segment_pretrain_stage3_dreambench_recap_400k_t2i_400k_flux_200k_midjourney_150k_recovery_150k_3e4_epoch2_half_rotated/005-GPT-XL/checkpoints/eval_step_24000/batch_2_cfg_7.5_topk_16384.jpg b/CKPTS/Afteri2i_100kckpt_LLAVAT5_empty_Fixed_just_segment_pretrain_stage3_dreambench_recap_400k_t2i_400k_flux_200k_midjourney_150k_recovery_150k_3e4_epoch2_half_rotated/005-GPT-XL/checkpoints/eval_step_24000/batch_2_cfg_7.5_topk_16384.jpg new file mode 100644 index 0000000000000000000000000000000000000000..158b222adda3ac0c413c523e59e0b73fbf159d8a --- /dev/null +++ b/CKPTS/Afteri2i_100kckpt_LLAVAT5_empty_Fixed_just_segment_pretrain_stage3_dreambench_recap_400k_t2i_400k_flux_200k_midjourney_150k_recovery_150k_3e4_epoch2_half_rotated/005-GPT-XL/checkpoints/eval_step_24000/batch_2_cfg_7.5_topk_16384.jpg @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:5eb29c0ae823f4246625adb079ab80abacf1c3f487e6384c576250cb73c0459a +size 720261 diff --git a/CKPTS/Afteri2i_100kckpt_LLAVAT5_empty_Fixed_just_segment_pretrain_stage3_dreambench_recap_400k_t2i_400k_flux_200k_midjourney_150k_recovery_150k_3e4_epoch2_half_rotated/005-GPT-XL/checkpoints/eval_step_24000/batch_3_cfg_7.5_topk_16384.jpg b/CKPTS/Afteri2i_100kckpt_LLAVAT5_empty_Fixed_just_segment_pretrain_stage3_dreambench_recap_400k_t2i_400k_flux_200k_midjourney_150k_recovery_150k_3e4_epoch2_half_rotated/005-GPT-XL/checkpoints/eval_step_24000/batch_3_cfg_7.5_topk_16384.jpg new file mode 100644 index 0000000000000000000000000000000000000000..d8df84c82bcf9b405259d18aaf7bf0868efe9e0d --- /dev/null +++ b/CKPTS/Afteri2i_100kckpt_LLAVAT5_empty_Fixed_just_segment_pretrain_stage3_dreambench_recap_400k_t2i_400k_flux_200k_midjourney_150k_recovery_150k_3e4_epoch2_half_rotated/005-GPT-XL/checkpoints/eval_step_24000/batch_3_cfg_7.5_topk_16384.jpg @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:2aa71663a949c5d66e7ed9e3b196e6dc7da119d0816d8df32552771cdd1a2515 +size 860785 diff --git a/CKPTS/Afteri2i_100kckpt_LLAVAT5_empty_Fixed_just_segment_pretrain_stage3_dreambench_recap_400k_t2i_400k_flux_200k_midjourney_150k_recovery_150k_3e4_epoch2_half_rotated/005-GPT-XL/checkpoints/eval_step_26000/batch_0_cfg_7.5_topk_16384.jpg b/CKPTS/Afteri2i_100kckpt_LLAVAT5_empty_Fixed_just_segment_pretrain_stage3_dreambench_recap_400k_t2i_400k_flux_200k_midjourney_150k_recovery_150k_3e4_epoch2_half_rotated/005-GPT-XL/checkpoints/eval_step_26000/batch_0_cfg_7.5_topk_16384.jpg new file mode 100644 index 0000000000000000000000000000000000000000..0dd395f9bb9c9659ac286b6bd3b7bb3f6edaca80 --- /dev/null +++ b/CKPTS/Afteri2i_100kckpt_LLAVAT5_empty_Fixed_just_segment_pretrain_stage3_dreambench_recap_400k_t2i_400k_flux_200k_midjourney_150k_recovery_150k_3e4_epoch2_half_rotated/005-GPT-XL/checkpoints/eval_step_26000/batch_0_cfg_7.5_topk_16384.jpg @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:e6e1f8e384e148203aa39cb13843f7a872c1ec9d56e80d13d15b937aa8b313fb +size 732315 diff --git a/CKPTS/Afteri2i_100kckpt_LLAVAT5_empty_Fixed_just_segment_pretrain_stage3_dreambench_recap_400k_t2i_400k_flux_200k_midjourney_150k_recovery_150k_3e4_epoch2_half_rotated/005-GPT-XL/checkpoints/eval_step_26000/batch_1_cfg_7.5_topk_16384.jpg b/CKPTS/Afteri2i_100kckpt_LLAVAT5_empty_Fixed_just_segment_pretrain_stage3_dreambench_recap_400k_t2i_400k_flux_200k_midjourney_150k_recovery_150k_3e4_epoch2_half_rotated/005-GPT-XL/checkpoints/eval_step_26000/batch_1_cfg_7.5_topk_16384.jpg new file mode 100644 index 0000000000000000000000000000000000000000..b4e63236a4a7f364a05b7463577470dd664d0f2d --- /dev/null +++ b/CKPTS/Afteri2i_100kckpt_LLAVAT5_empty_Fixed_just_segment_pretrain_stage3_dreambench_recap_400k_t2i_400k_flux_200k_midjourney_150k_recovery_150k_3e4_epoch2_half_rotated/005-GPT-XL/checkpoints/eval_step_26000/batch_1_cfg_7.5_topk_16384.jpg @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:dc76edf3ae6d5b3284c244bc9755834c5d4c1fdf910487ff12abde1b0801b649 +size 686124 diff --git a/CKPTS/Afteri2i_100kckpt_LLAVAT5_empty_Fixed_just_segment_pretrain_stage3_dreambench_recap_400k_t2i_400k_flux_200k_midjourney_150k_recovery_150k_3e4_epoch2_half_rotated/005-GPT-XL/checkpoints/eval_step_26000/batch_2_cfg_7.5_topk_16384.jpg b/CKPTS/Afteri2i_100kckpt_LLAVAT5_empty_Fixed_just_segment_pretrain_stage3_dreambench_recap_400k_t2i_400k_flux_200k_midjourney_150k_recovery_150k_3e4_epoch2_half_rotated/005-GPT-XL/checkpoints/eval_step_26000/batch_2_cfg_7.5_topk_16384.jpg new file mode 100644 index 0000000000000000000000000000000000000000..208f7463ddf87a0e5d646283c728597a19a739bf --- /dev/null +++ b/CKPTS/Afteri2i_100kckpt_LLAVAT5_empty_Fixed_just_segment_pretrain_stage3_dreambench_recap_400k_t2i_400k_flux_200k_midjourney_150k_recovery_150k_3e4_epoch2_half_rotated/005-GPT-XL/checkpoints/eval_step_26000/batch_2_cfg_7.5_topk_16384.jpg @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:4bd3fd317b2a4461787bb403d830f20ec59724e98c8ae88d499c4c935c8851be +size 744710 diff --git a/CKPTS/Afteri2i_100kckpt_LLAVAT5_empty_Fixed_just_segment_pretrain_stage3_dreambench_recap_400k_t2i_400k_flux_200k_midjourney_150k_recovery_150k_3e4_epoch2_half_rotated/005-GPT-XL/checkpoints/eval_step_26000/batch_3_cfg_7.5_topk_16384.jpg b/CKPTS/Afteri2i_100kckpt_LLAVAT5_empty_Fixed_just_segment_pretrain_stage3_dreambench_recap_400k_t2i_400k_flux_200k_midjourney_150k_recovery_150k_3e4_epoch2_half_rotated/005-GPT-XL/checkpoints/eval_step_26000/batch_3_cfg_7.5_topk_16384.jpg new file mode 100644 index 0000000000000000000000000000000000000000..b261da93ac9f5acfaf3ed58ccc9f608d4e3aefb6 --- /dev/null +++ b/CKPTS/Afteri2i_100kckpt_LLAVAT5_empty_Fixed_just_segment_pretrain_stage3_dreambench_recap_400k_t2i_400k_flux_200k_midjourney_150k_recovery_150k_3e4_epoch2_half_rotated/005-GPT-XL/checkpoints/eval_step_26000/batch_3_cfg_7.5_topk_16384.jpg @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:ba7e8f0e331247c1946c76d3c0e2723715406f52a73a34a969f477771a5f6ad0 +size 868780 diff --git a/CKPTS/Afteri2i_100kckpt_LLAVAT5_empty_Fixed_just_segment_pretrain_stage3_dreambench_recap_400k_t2i_400k_flux_200k_midjourney_150k_recovery_150k_3e4_epoch2_half_rotated/005-GPT-XL/checkpoints/eval_step_28000/batch_0_cfg_7.5_topk_16384.jpg b/CKPTS/Afteri2i_100kckpt_LLAVAT5_empty_Fixed_just_segment_pretrain_stage3_dreambench_recap_400k_t2i_400k_flux_200k_midjourney_150k_recovery_150k_3e4_epoch2_half_rotated/005-GPT-XL/checkpoints/eval_step_28000/batch_0_cfg_7.5_topk_16384.jpg new file mode 100644 index 0000000000000000000000000000000000000000..cef374f2223df49e4fcffec84f37155aa92c277f --- /dev/null +++ b/CKPTS/Afteri2i_100kckpt_LLAVAT5_empty_Fixed_just_segment_pretrain_stage3_dreambench_recap_400k_t2i_400k_flux_200k_midjourney_150k_recovery_150k_3e4_epoch2_half_rotated/005-GPT-XL/checkpoints/eval_step_28000/batch_0_cfg_7.5_topk_16384.jpg @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:82f02eb3f7dc27361e988711fa65ef7ed8934083d9254183c97e03620a4e2418 +size 718092 diff --git a/CKPTS/Afteri2i_100kckpt_LLAVAT5_empty_Fixed_just_segment_pretrain_stage3_dreambench_recap_400k_t2i_400k_flux_200k_midjourney_150k_recovery_150k_3e4_epoch2_half_rotated/005-GPT-XL/checkpoints/eval_step_28000/batch_1_cfg_7.5_topk_16384.jpg b/CKPTS/Afteri2i_100kckpt_LLAVAT5_empty_Fixed_just_segment_pretrain_stage3_dreambench_recap_400k_t2i_400k_flux_200k_midjourney_150k_recovery_150k_3e4_epoch2_half_rotated/005-GPT-XL/checkpoints/eval_step_28000/batch_1_cfg_7.5_topk_16384.jpg new file mode 100644 index 0000000000000000000000000000000000000000..6248e527c2a9e53ecc5d7b685c515f6cdef3dfb2 --- /dev/null +++ b/CKPTS/Afteri2i_100kckpt_LLAVAT5_empty_Fixed_just_segment_pretrain_stage3_dreambench_recap_400k_t2i_400k_flux_200k_midjourney_150k_recovery_150k_3e4_epoch2_half_rotated/005-GPT-XL/checkpoints/eval_step_28000/batch_1_cfg_7.5_topk_16384.jpg @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:7aa0ea768eaeef3679ab2c23cdea868e632aa20c7688c31fd87cdefa9117b509 +size 705074 diff --git a/CKPTS/Afteri2i_100kckpt_LLAVAT5_empty_Fixed_just_segment_pretrain_stage3_dreambench_recap_400k_t2i_400k_flux_200k_midjourney_150k_recovery_150k_3e4_epoch2_half_rotated/005-GPT-XL/checkpoints/eval_step_28000/batch_2_cfg_7.5_topk_16384.jpg b/CKPTS/Afteri2i_100kckpt_LLAVAT5_empty_Fixed_just_segment_pretrain_stage3_dreambench_recap_400k_t2i_400k_flux_200k_midjourney_150k_recovery_150k_3e4_epoch2_half_rotated/005-GPT-XL/checkpoints/eval_step_28000/batch_2_cfg_7.5_topk_16384.jpg new file mode 100644 index 0000000000000000000000000000000000000000..e2a32925afe071842b17d37c18ef3adb9087de2c --- /dev/null +++ b/CKPTS/Afteri2i_100kckpt_LLAVAT5_empty_Fixed_just_segment_pretrain_stage3_dreambench_recap_400k_t2i_400k_flux_200k_midjourney_150k_recovery_150k_3e4_epoch2_half_rotated/005-GPT-XL/checkpoints/eval_step_28000/batch_2_cfg_7.5_topk_16384.jpg @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:16b49127e4826aa1dcca4d6014c4c700fff188ebe4bf9b6c2c30b81048808cec +size 732702 diff --git a/CKPTS/Afteri2i_100kckpt_LLAVAT5_empty_Fixed_just_segment_pretrain_stage3_dreambench_recap_400k_t2i_400k_flux_200k_midjourney_150k_recovery_150k_3e4_epoch2_half_rotated/005-GPT-XL/checkpoints/eval_step_28000/batch_3_cfg_7.5_topk_16384.jpg b/CKPTS/Afteri2i_100kckpt_LLAVAT5_empty_Fixed_just_segment_pretrain_stage3_dreambench_recap_400k_t2i_400k_flux_200k_midjourney_150k_recovery_150k_3e4_epoch2_half_rotated/005-GPT-XL/checkpoints/eval_step_28000/batch_3_cfg_7.5_topk_16384.jpg new file mode 100644 index 0000000000000000000000000000000000000000..3fceafe86f558f02dbf861b437ea8adcadccf68b --- /dev/null +++ b/CKPTS/Afteri2i_100kckpt_LLAVAT5_empty_Fixed_just_segment_pretrain_stage3_dreambench_recap_400k_t2i_400k_flux_200k_midjourney_150k_recovery_150k_3e4_epoch2_half_rotated/005-GPT-XL/checkpoints/eval_step_28000/batch_3_cfg_7.5_topk_16384.jpg @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:4f4e62e065875219392c9d8a89e934470b9e45a35dcc97bcd778c607231e707f +size 895186 diff --git a/CKPTS/Afteri2i_100kckpt_LLAVAT5_empty_Fixed_just_segment_pretrain_stage3_dreambench_recap_400k_t2i_400k_flux_200k_midjourney_150k_recovery_150k_3e4_epoch2_half_rotated/005-GPT-XL/checkpoints/eval_step_30000/batch_0_cfg_7.5_topk_16384.jpg b/CKPTS/Afteri2i_100kckpt_LLAVAT5_empty_Fixed_just_segment_pretrain_stage3_dreambench_recap_400k_t2i_400k_flux_200k_midjourney_150k_recovery_150k_3e4_epoch2_half_rotated/005-GPT-XL/checkpoints/eval_step_30000/batch_0_cfg_7.5_topk_16384.jpg new file mode 100644 index 0000000000000000000000000000000000000000..769981aa5259bf46ac2352fbc46acee5a66672da --- /dev/null +++ b/CKPTS/Afteri2i_100kckpt_LLAVAT5_empty_Fixed_just_segment_pretrain_stage3_dreambench_recap_400k_t2i_400k_flux_200k_midjourney_150k_recovery_150k_3e4_epoch2_half_rotated/005-GPT-XL/checkpoints/eval_step_30000/batch_0_cfg_7.5_topk_16384.jpg @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:bd55667fa0e30042e5e4425b6b757a61e3c355971eb47a7d8e179112bd148040 +size 732569 diff --git a/CKPTS/Afteri2i_100kckpt_LLAVAT5_empty_Fixed_just_segment_pretrain_stage3_dreambench_recap_400k_t2i_400k_flux_200k_midjourney_150k_recovery_150k_3e4_epoch2_half_rotated/005-GPT-XL/checkpoints/eval_step_30000/batch_1_cfg_7.5_topk_16384.jpg b/CKPTS/Afteri2i_100kckpt_LLAVAT5_empty_Fixed_just_segment_pretrain_stage3_dreambench_recap_400k_t2i_400k_flux_200k_midjourney_150k_recovery_150k_3e4_epoch2_half_rotated/005-GPT-XL/checkpoints/eval_step_30000/batch_1_cfg_7.5_topk_16384.jpg new file mode 100644 index 0000000000000000000000000000000000000000..4512b94ffc22c882b7ddc1c4db0ba165a63b7cc6 --- /dev/null +++ b/CKPTS/Afteri2i_100kckpt_LLAVAT5_empty_Fixed_just_segment_pretrain_stage3_dreambench_recap_400k_t2i_400k_flux_200k_midjourney_150k_recovery_150k_3e4_epoch2_half_rotated/005-GPT-XL/checkpoints/eval_step_30000/batch_1_cfg_7.5_topk_16384.jpg @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:ae32f42eedd8bc329eae466c987f41c9885023c193bc129f5b11261d207275bd +size 714820 diff --git a/CKPTS/Afteri2i_100kckpt_LLAVAT5_empty_Fixed_just_segment_pretrain_stage3_dreambench_recap_400k_t2i_400k_flux_200k_midjourney_150k_recovery_150k_3e4_epoch2_half_rotated/005-GPT-XL/checkpoints/eval_step_30000/batch_2_cfg_7.5_topk_16384.jpg b/CKPTS/Afteri2i_100kckpt_LLAVAT5_empty_Fixed_just_segment_pretrain_stage3_dreambench_recap_400k_t2i_400k_flux_200k_midjourney_150k_recovery_150k_3e4_epoch2_half_rotated/005-GPT-XL/checkpoints/eval_step_30000/batch_2_cfg_7.5_topk_16384.jpg new file mode 100644 index 0000000000000000000000000000000000000000..14b519e0bd15405b0ef404fee7272fd9ed0b817e --- /dev/null +++ b/CKPTS/Afteri2i_100kckpt_LLAVAT5_empty_Fixed_just_segment_pretrain_stage3_dreambench_recap_400k_t2i_400k_flux_200k_midjourney_150k_recovery_150k_3e4_epoch2_half_rotated/005-GPT-XL/checkpoints/eval_step_30000/batch_2_cfg_7.5_topk_16384.jpg @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:41747af61f066278668cee11bb1c92a158af62cb8f21f43e99c0937fd74e336e +size 704057 diff --git a/CKPTS/Afteri2i_100kckpt_LLAVAT5_empty_Fixed_just_segment_pretrain_stage3_dreambench_recap_400k_t2i_400k_flux_200k_midjourney_150k_recovery_150k_3e4_epoch2_half_rotated/005-GPT-XL/checkpoints/eval_step_30000/batch_3_cfg_7.5_topk_16384.jpg b/CKPTS/Afteri2i_100kckpt_LLAVAT5_empty_Fixed_just_segment_pretrain_stage3_dreambench_recap_400k_t2i_400k_flux_200k_midjourney_150k_recovery_150k_3e4_epoch2_half_rotated/005-GPT-XL/checkpoints/eval_step_30000/batch_3_cfg_7.5_topk_16384.jpg new file mode 100644 index 0000000000000000000000000000000000000000..bb1463733ddc9ea139a18ffb47400e71c7152e23 --- /dev/null +++ b/CKPTS/Afteri2i_100kckpt_LLAVAT5_empty_Fixed_just_segment_pretrain_stage3_dreambench_recap_400k_t2i_400k_flux_200k_midjourney_150k_recovery_150k_3e4_epoch2_half_rotated/005-GPT-XL/checkpoints/eval_step_30000/batch_3_cfg_7.5_topk_16384.jpg @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:afa5d719ac2d881daeb64590d9590e9d2e8bf5a85d1e3786861cbd9c51e2214e +size 880140 diff --git a/CKPTS/Afteri2i_100kckpt_LLAVAT5_empty_Fixed_just_segment_pretrain_stage3_dreambench_recap_400k_t2i_400k_flux_200k_midjourney_150k_recovery_150k_3e4_epoch2_half_rotated/005-GPT-XL/checkpoints/eval_step_32000/batch_0_cfg_7.5_topk_16384.jpg b/CKPTS/Afteri2i_100kckpt_LLAVAT5_empty_Fixed_just_segment_pretrain_stage3_dreambench_recap_400k_t2i_400k_flux_200k_midjourney_150k_recovery_150k_3e4_epoch2_half_rotated/005-GPT-XL/checkpoints/eval_step_32000/batch_0_cfg_7.5_topk_16384.jpg new file mode 100644 index 0000000000000000000000000000000000000000..64095276df05bd2d4979b7027c37bb335cc8a8df --- /dev/null +++ b/CKPTS/Afteri2i_100kckpt_LLAVAT5_empty_Fixed_just_segment_pretrain_stage3_dreambench_recap_400k_t2i_400k_flux_200k_midjourney_150k_recovery_150k_3e4_epoch2_half_rotated/005-GPT-XL/checkpoints/eval_step_32000/batch_0_cfg_7.5_topk_16384.jpg @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:0d1b3d6254e6f71ca2a4dd449ed1badbfb7f74d312f718134196ed43b8c68541 +size 738840 diff --git a/CKPTS/Afteri2i_100kckpt_LLAVAT5_empty_Fixed_just_segment_pretrain_stage3_dreambench_recap_400k_t2i_400k_flux_200k_midjourney_150k_recovery_150k_3e4_epoch2_half_rotated/005-GPT-XL/checkpoints/eval_step_32000/batch_1_cfg_7.5_topk_16384.jpg b/CKPTS/Afteri2i_100kckpt_LLAVAT5_empty_Fixed_just_segment_pretrain_stage3_dreambench_recap_400k_t2i_400k_flux_200k_midjourney_150k_recovery_150k_3e4_epoch2_half_rotated/005-GPT-XL/checkpoints/eval_step_32000/batch_1_cfg_7.5_topk_16384.jpg new file mode 100644 index 0000000000000000000000000000000000000000..ca7a8d32b1c0981737f087f059788219164872d3 --- /dev/null +++ b/CKPTS/Afteri2i_100kckpt_LLAVAT5_empty_Fixed_just_segment_pretrain_stage3_dreambench_recap_400k_t2i_400k_flux_200k_midjourney_150k_recovery_150k_3e4_epoch2_half_rotated/005-GPT-XL/checkpoints/eval_step_32000/batch_1_cfg_7.5_topk_16384.jpg @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:e32adf35875b16b068852db3bb0856bd40988ce2d2cef4a893312274e1601afe +size 693955 diff --git a/CKPTS/Afteri2i_100kckpt_LLAVAT5_empty_Fixed_just_segment_pretrain_stage3_dreambench_recap_400k_t2i_400k_flux_200k_midjourney_150k_recovery_150k_3e4_epoch2_half_rotated/005-GPT-XL/checkpoints/eval_step_32000/batch_2_cfg_7.5_topk_16384.jpg b/CKPTS/Afteri2i_100kckpt_LLAVAT5_empty_Fixed_just_segment_pretrain_stage3_dreambench_recap_400k_t2i_400k_flux_200k_midjourney_150k_recovery_150k_3e4_epoch2_half_rotated/005-GPT-XL/checkpoints/eval_step_32000/batch_2_cfg_7.5_topk_16384.jpg new file mode 100644 index 0000000000000000000000000000000000000000..858d427f5b26bf304183770afca1505b4697e5d6 --- /dev/null +++ b/CKPTS/Afteri2i_100kckpt_LLAVAT5_empty_Fixed_just_segment_pretrain_stage3_dreambench_recap_400k_t2i_400k_flux_200k_midjourney_150k_recovery_150k_3e4_epoch2_half_rotated/005-GPT-XL/checkpoints/eval_step_32000/batch_2_cfg_7.5_topk_16384.jpg @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:2d18b321f01414c29bbcf9f16171e702ffd5d85f82001f54ae25ba70f7b4bfde +size 714313 diff --git a/CKPTS/Afteri2i_100kckpt_LLAVAT5_empty_Fixed_just_segment_pretrain_stage3_dreambench_recap_400k_t2i_400k_flux_200k_midjourney_150k_recovery_150k_3e4_epoch2_half_rotated/005-GPT-XL/checkpoints/eval_step_32000/batch_3_cfg_7.5_topk_16384.jpg b/CKPTS/Afteri2i_100kckpt_LLAVAT5_empty_Fixed_just_segment_pretrain_stage3_dreambench_recap_400k_t2i_400k_flux_200k_midjourney_150k_recovery_150k_3e4_epoch2_half_rotated/005-GPT-XL/checkpoints/eval_step_32000/batch_3_cfg_7.5_topk_16384.jpg new file mode 100644 index 0000000000000000000000000000000000000000..f7c66bc42fee5ab9dc5aaf3e175516be060a4f60 --- /dev/null +++ b/CKPTS/Afteri2i_100kckpt_LLAVAT5_empty_Fixed_just_segment_pretrain_stage3_dreambench_recap_400k_t2i_400k_flux_200k_midjourney_150k_recovery_150k_3e4_epoch2_half_rotated/005-GPT-XL/checkpoints/eval_step_32000/batch_3_cfg_7.5_topk_16384.jpg @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:c024d7ad83a08e791d84de1e6eeb866fe9a74aed004658be1aa10a7af67e0cfe +size 840364 diff --git a/CKPTS/Afteri2i_100kckpt_LLAVAT5_empty_Fixed_just_segment_pretrain_stage3_dreambench_recap_400k_t2i_400k_flux_200k_midjourney_150k_recovery_150k_3e4_epoch2_half_rotated/005-GPT-XL/checkpoints/eval_step_34000/batch_0_cfg_7.5_topk_16384.jpg b/CKPTS/Afteri2i_100kckpt_LLAVAT5_empty_Fixed_just_segment_pretrain_stage3_dreambench_recap_400k_t2i_400k_flux_200k_midjourney_150k_recovery_150k_3e4_epoch2_half_rotated/005-GPT-XL/checkpoints/eval_step_34000/batch_0_cfg_7.5_topk_16384.jpg new file mode 100644 index 0000000000000000000000000000000000000000..df8220d660aff12e11e4c0cf85d0dbe0dcb41783 --- /dev/null +++ b/CKPTS/Afteri2i_100kckpt_LLAVAT5_empty_Fixed_just_segment_pretrain_stage3_dreambench_recap_400k_t2i_400k_flux_200k_midjourney_150k_recovery_150k_3e4_epoch2_half_rotated/005-GPT-XL/checkpoints/eval_step_34000/batch_0_cfg_7.5_topk_16384.jpg @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:71c0753379d224f62152c96b255ca4e995eb65afe2b5547887e6f80ca4dd12cb +size 733019 diff --git a/CKPTS/Afteri2i_100kckpt_LLAVAT5_empty_Fixed_just_segment_pretrain_stage3_dreambench_recap_400k_t2i_400k_flux_200k_midjourney_150k_recovery_150k_3e4_epoch2_half_rotated/005-GPT-XL/checkpoints/eval_step_34000/batch_1_cfg_7.5_topk_16384.jpg b/CKPTS/Afteri2i_100kckpt_LLAVAT5_empty_Fixed_just_segment_pretrain_stage3_dreambench_recap_400k_t2i_400k_flux_200k_midjourney_150k_recovery_150k_3e4_epoch2_half_rotated/005-GPT-XL/checkpoints/eval_step_34000/batch_1_cfg_7.5_topk_16384.jpg new file mode 100644 index 0000000000000000000000000000000000000000..8bd649843ad8d066226c28919af6bb91f1668039 --- /dev/null +++ b/CKPTS/Afteri2i_100kckpt_LLAVAT5_empty_Fixed_just_segment_pretrain_stage3_dreambench_recap_400k_t2i_400k_flux_200k_midjourney_150k_recovery_150k_3e4_epoch2_half_rotated/005-GPT-XL/checkpoints/eval_step_34000/batch_1_cfg_7.5_topk_16384.jpg @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:5a7263c64eaf0ef78ef02305f89440754d21c20024bfa6ba8e23a352396e1ef4 +size 718651 diff --git a/CKPTS/Afteri2i_100kckpt_LLAVAT5_empty_Fixed_just_segment_pretrain_stage3_dreambench_recap_400k_t2i_400k_flux_200k_midjourney_150k_recovery_150k_3e4_epoch2_half_rotated/005-GPT-XL/checkpoints/eval_step_34000/batch_2_cfg_7.5_topk_16384.jpg b/CKPTS/Afteri2i_100kckpt_LLAVAT5_empty_Fixed_just_segment_pretrain_stage3_dreambench_recap_400k_t2i_400k_flux_200k_midjourney_150k_recovery_150k_3e4_epoch2_half_rotated/005-GPT-XL/checkpoints/eval_step_34000/batch_2_cfg_7.5_topk_16384.jpg new file mode 100644 index 0000000000000000000000000000000000000000..ebdd191d3f31be594419b2e90c182fe272ecb2bd --- /dev/null +++ b/CKPTS/Afteri2i_100kckpt_LLAVAT5_empty_Fixed_just_segment_pretrain_stage3_dreambench_recap_400k_t2i_400k_flux_200k_midjourney_150k_recovery_150k_3e4_epoch2_half_rotated/005-GPT-XL/checkpoints/eval_step_34000/batch_2_cfg_7.5_topk_16384.jpg @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:f6f2fff39d4e50ef3438898641de41dae6ff9448290b42f1664caca2a268b0b9 +size 712834 diff --git a/CKPTS/Afteri2i_100kckpt_LLAVAT5_empty_Fixed_just_segment_pretrain_stage3_dreambench_recap_400k_t2i_400k_flux_200k_midjourney_150k_recovery_150k_3e4_epoch2_half_rotated/005-GPT-XL/checkpoints/eval_step_34000/batch_3_cfg_7.5_topk_16384.jpg b/CKPTS/Afteri2i_100kckpt_LLAVAT5_empty_Fixed_just_segment_pretrain_stage3_dreambench_recap_400k_t2i_400k_flux_200k_midjourney_150k_recovery_150k_3e4_epoch2_half_rotated/005-GPT-XL/checkpoints/eval_step_34000/batch_3_cfg_7.5_topk_16384.jpg new file mode 100644 index 0000000000000000000000000000000000000000..08eafe933286d2160e2ab1b10ff7fb3cea5f2fc8 --- /dev/null +++ b/CKPTS/Afteri2i_100kckpt_LLAVAT5_empty_Fixed_just_segment_pretrain_stage3_dreambench_recap_400k_t2i_400k_flux_200k_midjourney_150k_recovery_150k_3e4_epoch2_half_rotated/005-GPT-XL/checkpoints/eval_step_34000/batch_3_cfg_7.5_topk_16384.jpg @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:cb5b212b49de4e3dbce5d299a9e10e291ad06c697e94a0a3d5083aa628fe0622 +size 866504 diff --git a/CKPTS/Afteri2i_100kckpt_LLAVAT5_empty_Fixed_just_segment_pretrain_stage3_dreambench_recap_400k_t2i_400k_flux_200k_midjourney_150k_recovery_150k_3e4_epoch2_half_rotated/005-GPT-XL/checkpoints/eval_step_36000/batch_0_cfg_7.5_topk_16384.jpg b/CKPTS/Afteri2i_100kckpt_LLAVAT5_empty_Fixed_just_segment_pretrain_stage3_dreambench_recap_400k_t2i_400k_flux_200k_midjourney_150k_recovery_150k_3e4_epoch2_half_rotated/005-GPT-XL/checkpoints/eval_step_36000/batch_0_cfg_7.5_topk_16384.jpg new file mode 100644 index 0000000000000000000000000000000000000000..40a13d91673820afcbccf6cf8a87ae996c2a1e84 --- /dev/null +++ b/CKPTS/Afteri2i_100kckpt_LLAVAT5_empty_Fixed_just_segment_pretrain_stage3_dreambench_recap_400k_t2i_400k_flux_200k_midjourney_150k_recovery_150k_3e4_epoch2_half_rotated/005-GPT-XL/checkpoints/eval_step_36000/batch_0_cfg_7.5_topk_16384.jpg @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:dcb13df5c44cd2401a0b2c2d2903f8d15859937d2d9d2c36af4dcd45aebc9388 +size 732380 diff --git a/CKPTS/Afteri2i_100kckpt_LLAVAT5_empty_Fixed_just_segment_pretrain_stage3_dreambench_recap_400k_t2i_400k_flux_200k_midjourney_150k_recovery_150k_3e4_epoch2_half_rotated/005-GPT-XL/checkpoints/eval_step_36000/batch_1_cfg_7.5_topk_16384.jpg b/CKPTS/Afteri2i_100kckpt_LLAVAT5_empty_Fixed_just_segment_pretrain_stage3_dreambench_recap_400k_t2i_400k_flux_200k_midjourney_150k_recovery_150k_3e4_epoch2_half_rotated/005-GPT-XL/checkpoints/eval_step_36000/batch_1_cfg_7.5_topk_16384.jpg new file mode 100644 index 0000000000000000000000000000000000000000..9aab3303933c133ea0c122750b2178f622243b6e --- /dev/null +++ b/CKPTS/Afteri2i_100kckpt_LLAVAT5_empty_Fixed_just_segment_pretrain_stage3_dreambench_recap_400k_t2i_400k_flux_200k_midjourney_150k_recovery_150k_3e4_epoch2_half_rotated/005-GPT-XL/checkpoints/eval_step_36000/batch_1_cfg_7.5_topk_16384.jpg @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:cd19c7b4a8805c9cc720296b0e6261c198d1215df208fd4d921254da7871a181 +size 723563 diff --git a/CKPTS/Afteri2i_100kckpt_LLAVAT5_empty_Fixed_just_segment_pretrain_stage3_dreambench_recap_400k_t2i_400k_flux_200k_midjourney_150k_recovery_150k_3e4_epoch2_half_rotated/005-GPT-XL/checkpoints/eval_step_36000/batch_2_cfg_7.5_topk_16384.jpg b/CKPTS/Afteri2i_100kckpt_LLAVAT5_empty_Fixed_just_segment_pretrain_stage3_dreambench_recap_400k_t2i_400k_flux_200k_midjourney_150k_recovery_150k_3e4_epoch2_half_rotated/005-GPT-XL/checkpoints/eval_step_36000/batch_2_cfg_7.5_topk_16384.jpg new file mode 100644 index 0000000000000000000000000000000000000000..8edd5bf37f432b2eeb73c4fbcf01909e6f544951 --- /dev/null +++ b/CKPTS/Afteri2i_100kckpt_LLAVAT5_empty_Fixed_just_segment_pretrain_stage3_dreambench_recap_400k_t2i_400k_flux_200k_midjourney_150k_recovery_150k_3e4_epoch2_half_rotated/005-GPT-XL/checkpoints/eval_step_36000/batch_2_cfg_7.5_topk_16384.jpg @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:4e3e8fc4c6500f05fcbe812a7834364f4006482f1daf4a99b0e56aa6c2853696 +size 715212 diff --git a/CKPTS/Afteri2i_100kckpt_LLAVAT5_empty_Fixed_just_segment_pretrain_stage3_dreambench_recap_400k_t2i_400k_flux_200k_midjourney_150k_recovery_150k_3e4_epoch2_half_rotated/005-GPT-XL/checkpoints/eval_step_36000/batch_3_cfg_7.5_topk_16384.jpg b/CKPTS/Afteri2i_100kckpt_LLAVAT5_empty_Fixed_just_segment_pretrain_stage3_dreambench_recap_400k_t2i_400k_flux_200k_midjourney_150k_recovery_150k_3e4_epoch2_half_rotated/005-GPT-XL/checkpoints/eval_step_36000/batch_3_cfg_7.5_topk_16384.jpg new file mode 100644 index 0000000000000000000000000000000000000000..2837448bbc8a4cb0f427f9b05cb70635b9b1bba8 --- /dev/null +++ b/CKPTS/Afteri2i_100kckpt_LLAVAT5_empty_Fixed_just_segment_pretrain_stage3_dreambench_recap_400k_t2i_400k_flux_200k_midjourney_150k_recovery_150k_3e4_epoch2_half_rotated/005-GPT-XL/checkpoints/eval_step_36000/batch_3_cfg_7.5_topk_16384.jpg @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:e2abfb5c17e9151a46a17c226af8b2997f575861a41ce6744492a30926ae5bfc +size 851709 diff --git a/CKPTS/Afteri2i_100kckpt_LLAVAT5_empty_Fixed_just_segment_pretrain_stage3_dreambench_recap_400k_t2i_400k_flux_200k_midjourney_150k_recovery_150k_3e4_epoch2_half_rotated/005-GPT-XL/checkpoints/eval_step_38000/batch_0_cfg_7.5_topk_16384.jpg b/CKPTS/Afteri2i_100kckpt_LLAVAT5_empty_Fixed_just_segment_pretrain_stage3_dreambench_recap_400k_t2i_400k_flux_200k_midjourney_150k_recovery_150k_3e4_epoch2_half_rotated/005-GPT-XL/checkpoints/eval_step_38000/batch_0_cfg_7.5_topk_16384.jpg new file mode 100644 index 0000000000000000000000000000000000000000..9d96380938c9c2cbe0e96e59983308785a7475e6 --- /dev/null +++ b/CKPTS/Afteri2i_100kckpt_LLAVAT5_empty_Fixed_just_segment_pretrain_stage3_dreambench_recap_400k_t2i_400k_flux_200k_midjourney_150k_recovery_150k_3e4_epoch2_half_rotated/005-GPT-XL/checkpoints/eval_step_38000/batch_0_cfg_7.5_topk_16384.jpg @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:72c066cc0bdf7c96d58841ea570f3f973a654e92ae21a0195c4de5d50427df94 +size 728539 diff --git a/CKPTS/Afteri2i_100kckpt_LLAVAT5_empty_Fixed_just_segment_pretrain_stage3_dreambench_recap_400k_t2i_400k_flux_200k_midjourney_150k_recovery_150k_3e4_epoch2_half_rotated/005-GPT-XL/checkpoints/eval_step_38000/batch_1_cfg_7.5_topk_16384.jpg b/CKPTS/Afteri2i_100kckpt_LLAVAT5_empty_Fixed_just_segment_pretrain_stage3_dreambench_recap_400k_t2i_400k_flux_200k_midjourney_150k_recovery_150k_3e4_epoch2_half_rotated/005-GPT-XL/checkpoints/eval_step_38000/batch_1_cfg_7.5_topk_16384.jpg new file mode 100644 index 0000000000000000000000000000000000000000..403eb4f148653736faa6422d830d0d830d47d707 --- /dev/null +++ b/CKPTS/Afteri2i_100kckpt_LLAVAT5_empty_Fixed_just_segment_pretrain_stage3_dreambench_recap_400k_t2i_400k_flux_200k_midjourney_150k_recovery_150k_3e4_epoch2_half_rotated/005-GPT-XL/checkpoints/eval_step_38000/batch_1_cfg_7.5_topk_16384.jpg @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:2f380e6e482f615d35de6ae2ed4631c7908d51a5f505f37b77ddf1c27a1b228c +size 683682 diff --git a/CKPTS/Afteri2i_100kckpt_LLAVAT5_empty_Fixed_just_segment_pretrain_stage3_dreambench_recap_400k_t2i_400k_flux_200k_midjourney_150k_recovery_150k_3e4_epoch2_half_rotated/005-GPT-XL/checkpoints/eval_step_38000/batch_2_cfg_7.5_topk_16384.jpg b/CKPTS/Afteri2i_100kckpt_LLAVAT5_empty_Fixed_just_segment_pretrain_stage3_dreambench_recap_400k_t2i_400k_flux_200k_midjourney_150k_recovery_150k_3e4_epoch2_half_rotated/005-GPT-XL/checkpoints/eval_step_38000/batch_2_cfg_7.5_topk_16384.jpg new file mode 100644 index 0000000000000000000000000000000000000000..3a645ae957475184730de096cc8560b79f16e944 --- /dev/null +++ b/CKPTS/Afteri2i_100kckpt_LLAVAT5_empty_Fixed_just_segment_pretrain_stage3_dreambench_recap_400k_t2i_400k_flux_200k_midjourney_150k_recovery_150k_3e4_epoch2_half_rotated/005-GPT-XL/checkpoints/eval_step_38000/batch_2_cfg_7.5_topk_16384.jpg @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:a4f758d21767b4cfe464cec1b4ce2fee1f56b3bb941551f32dcebb1a93573ab3 +size 710400 diff --git a/CKPTS/Afteri2i_100kckpt_LLAVAT5_empty_Fixed_just_segment_pretrain_stage3_dreambench_recap_400k_t2i_400k_flux_200k_midjourney_150k_recovery_150k_3e4_epoch2_half_rotated/005-GPT-XL/checkpoints/eval_step_38000/batch_3_cfg_7.5_topk_16384.jpg b/CKPTS/Afteri2i_100kckpt_LLAVAT5_empty_Fixed_just_segment_pretrain_stage3_dreambench_recap_400k_t2i_400k_flux_200k_midjourney_150k_recovery_150k_3e4_epoch2_half_rotated/005-GPT-XL/checkpoints/eval_step_38000/batch_3_cfg_7.5_topk_16384.jpg new file mode 100644 index 0000000000000000000000000000000000000000..1204af84babe7386092750b27cd8dcce678a9c09 --- /dev/null +++ b/CKPTS/Afteri2i_100kckpt_LLAVAT5_empty_Fixed_just_segment_pretrain_stage3_dreambench_recap_400k_t2i_400k_flux_200k_midjourney_150k_recovery_150k_3e4_epoch2_half_rotated/005-GPT-XL/checkpoints/eval_step_38000/batch_3_cfg_7.5_topk_16384.jpg @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:7b289631835301d18e74ce3cef3542dba06c1f1707c8ea680eeba5d9bd0791e9 +size 862741 diff --git a/CKPTS/Afteri2i_100kckpt_LLAVAT5_empty_Fixed_just_segment_pretrain_stage3_dreambench_recap_400k_t2i_400k_flux_200k_midjourney_150k_recovery_150k_3e4_epoch2_half_rotated/005-GPT-XL/checkpoints/eval_step_4000/batch_0_cfg_7.5_topk_16384.jpg b/CKPTS/Afteri2i_100kckpt_LLAVAT5_empty_Fixed_just_segment_pretrain_stage3_dreambench_recap_400k_t2i_400k_flux_200k_midjourney_150k_recovery_150k_3e4_epoch2_half_rotated/005-GPT-XL/checkpoints/eval_step_4000/batch_0_cfg_7.5_topk_16384.jpg new file mode 100644 index 0000000000000000000000000000000000000000..06326b8866f482e7a44c1f7cdef6227927ff85c3 --- /dev/null +++ b/CKPTS/Afteri2i_100kckpt_LLAVAT5_empty_Fixed_just_segment_pretrain_stage3_dreambench_recap_400k_t2i_400k_flux_200k_midjourney_150k_recovery_150k_3e4_epoch2_half_rotated/005-GPT-XL/checkpoints/eval_step_4000/batch_0_cfg_7.5_topk_16384.jpg @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:53f69426e03f325c00a6b8ea2d0389f91c1981ad71b5c36cd4746de7ce247fdf +size 727616 diff --git a/CKPTS/Afteri2i_100kckpt_LLAVAT5_empty_Fixed_just_segment_pretrain_stage3_dreambench_recap_400k_t2i_400k_flux_200k_midjourney_150k_recovery_150k_3e4_epoch2_half_rotated/005-GPT-XL/checkpoints/eval_step_4000/batch_1_cfg_7.5_topk_16384.jpg b/CKPTS/Afteri2i_100kckpt_LLAVAT5_empty_Fixed_just_segment_pretrain_stage3_dreambench_recap_400k_t2i_400k_flux_200k_midjourney_150k_recovery_150k_3e4_epoch2_half_rotated/005-GPT-XL/checkpoints/eval_step_4000/batch_1_cfg_7.5_topk_16384.jpg new file mode 100644 index 0000000000000000000000000000000000000000..7ac5661c3eb7677ca3f94e221f5c49318be935ca --- /dev/null +++ b/CKPTS/Afteri2i_100kckpt_LLAVAT5_empty_Fixed_just_segment_pretrain_stage3_dreambench_recap_400k_t2i_400k_flux_200k_midjourney_150k_recovery_150k_3e4_epoch2_half_rotated/005-GPT-XL/checkpoints/eval_step_4000/batch_1_cfg_7.5_topk_16384.jpg @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:4db80806afecd46d56d9fb50e08341a80b244330822e0ad3973c6ad998321479 +size 721445 diff --git a/CKPTS/Afteri2i_100kckpt_LLAVAT5_empty_Fixed_just_segment_pretrain_stage3_dreambench_recap_400k_t2i_400k_flux_200k_midjourney_150k_recovery_150k_3e4_epoch2_half_rotated/005-GPT-XL/checkpoints/eval_step_4000/batch_2_cfg_7.5_topk_16384.jpg b/CKPTS/Afteri2i_100kckpt_LLAVAT5_empty_Fixed_just_segment_pretrain_stage3_dreambench_recap_400k_t2i_400k_flux_200k_midjourney_150k_recovery_150k_3e4_epoch2_half_rotated/005-GPT-XL/checkpoints/eval_step_4000/batch_2_cfg_7.5_topk_16384.jpg new file mode 100644 index 0000000000000000000000000000000000000000..1b91436b7009bcf85e439306653a6f1d9a8d26e4 --- /dev/null +++ b/CKPTS/Afteri2i_100kckpt_LLAVAT5_empty_Fixed_just_segment_pretrain_stage3_dreambench_recap_400k_t2i_400k_flux_200k_midjourney_150k_recovery_150k_3e4_epoch2_half_rotated/005-GPT-XL/checkpoints/eval_step_4000/batch_2_cfg_7.5_topk_16384.jpg @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:155feac9238f2d6902abb6099797e1171207dbc2e5ae43a5e8e4d70c670f2871 +size 673846 diff --git a/CKPTS/Afteri2i_100kckpt_LLAVAT5_empty_Fixed_just_segment_pretrain_stage3_dreambench_recap_400k_t2i_400k_flux_200k_midjourney_150k_recovery_150k_3e4_epoch2_half_rotated/005-GPT-XL/checkpoints/eval_step_4000/batch_3_cfg_7.5_topk_16384.jpg b/CKPTS/Afteri2i_100kckpt_LLAVAT5_empty_Fixed_just_segment_pretrain_stage3_dreambench_recap_400k_t2i_400k_flux_200k_midjourney_150k_recovery_150k_3e4_epoch2_half_rotated/005-GPT-XL/checkpoints/eval_step_4000/batch_3_cfg_7.5_topk_16384.jpg new file mode 100644 index 0000000000000000000000000000000000000000..1c24cd2214a7f04bf0f3d1bfc8c26b3fc39b4e7f --- /dev/null +++ b/CKPTS/Afteri2i_100kckpt_LLAVAT5_empty_Fixed_just_segment_pretrain_stage3_dreambench_recap_400k_t2i_400k_flux_200k_midjourney_150k_recovery_150k_3e4_epoch2_half_rotated/005-GPT-XL/checkpoints/eval_step_4000/batch_3_cfg_7.5_topk_16384.jpg @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:c8242ebfc99af8a5dc6a4e58b3067281d87a7e8e3c08c4bc085f2f3e57c444c4 +size 836803 diff --git a/CKPTS/Afteri2i_100kckpt_LLAVAT5_empty_Fixed_just_segment_pretrain_stage3_dreambench_recap_400k_t2i_400k_flux_200k_midjourney_150k_recovery_150k_3e4_epoch2_half_rotated/005-GPT-XL/checkpoints/eval_step_40000/batch_0_cfg_7.5_topk_16384.jpg b/CKPTS/Afteri2i_100kckpt_LLAVAT5_empty_Fixed_just_segment_pretrain_stage3_dreambench_recap_400k_t2i_400k_flux_200k_midjourney_150k_recovery_150k_3e4_epoch2_half_rotated/005-GPT-XL/checkpoints/eval_step_40000/batch_0_cfg_7.5_topk_16384.jpg new file mode 100644 index 0000000000000000000000000000000000000000..c7cbf4c25f154573ee93359b8d4e4c7b5c5cd436 --- /dev/null +++ b/CKPTS/Afteri2i_100kckpt_LLAVAT5_empty_Fixed_just_segment_pretrain_stage3_dreambench_recap_400k_t2i_400k_flux_200k_midjourney_150k_recovery_150k_3e4_epoch2_half_rotated/005-GPT-XL/checkpoints/eval_step_40000/batch_0_cfg_7.5_topk_16384.jpg @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:17d55f0beefbd4e290b77c98e207c432c720861844678ccce5ceee8a24a80e06 +size 753703 diff --git a/CKPTS/Afteri2i_100kckpt_LLAVAT5_empty_Fixed_just_segment_pretrain_stage3_dreambench_recap_400k_t2i_400k_flux_200k_midjourney_150k_recovery_150k_3e4_epoch2_half_rotated/005-GPT-XL/checkpoints/eval_step_40000/batch_1_cfg_7.5_topk_16384.jpg b/CKPTS/Afteri2i_100kckpt_LLAVAT5_empty_Fixed_just_segment_pretrain_stage3_dreambench_recap_400k_t2i_400k_flux_200k_midjourney_150k_recovery_150k_3e4_epoch2_half_rotated/005-GPT-XL/checkpoints/eval_step_40000/batch_1_cfg_7.5_topk_16384.jpg new file mode 100644 index 0000000000000000000000000000000000000000..716b4f0f6e6aea38f96a1dd19e78f0e28d9cd065 --- /dev/null +++ b/CKPTS/Afteri2i_100kckpt_LLAVAT5_empty_Fixed_just_segment_pretrain_stage3_dreambench_recap_400k_t2i_400k_flux_200k_midjourney_150k_recovery_150k_3e4_epoch2_half_rotated/005-GPT-XL/checkpoints/eval_step_40000/batch_1_cfg_7.5_topk_16384.jpg @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:5e1c36690f9c9023276f537df843a1b19243c30f1d66bc2428151059b082bb0e +size 698849 diff --git a/CKPTS/Afteri2i_100kckpt_LLAVAT5_empty_Fixed_just_segment_pretrain_stage3_dreambench_recap_400k_t2i_400k_flux_200k_midjourney_150k_recovery_150k_3e4_epoch2_half_rotated/005-GPT-XL/checkpoints/eval_step_40000/batch_2_cfg_7.5_topk_16384.jpg b/CKPTS/Afteri2i_100kckpt_LLAVAT5_empty_Fixed_just_segment_pretrain_stage3_dreambench_recap_400k_t2i_400k_flux_200k_midjourney_150k_recovery_150k_3e4_epoch2_half_rotated/005-GPT-XL/checkpoints/eval_step_40000/batch_2_cfg_7.5_topk_16384.jpg new file mode 100644 index 0000000000000000000000000000000000000000..806a78f4f52d5468564af8f2eb34e5f39cbf14d3 --- /dev/null +++ b/CKPTS/Afteri2i_100kckpt_LLAVAT5_empty_Fixed_just_segment_pretrain_stage3_dreambench_recap_400k_t2i_400k_flux_200k_midjourney_150k_recovery_150k_3e4_epoch2_half_rotated/005-GPT-XL/checkpoints/eval_step_40000/batch_2_cfg_7.5_topk_16384.jpg @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:1e36772ccd0f3966a1e6430b898ea46834052a868793e7485b5e1e68ccc34480 +size 731312 diff --git a/CKPTS/Afteri2i_100kckpt_LLAVAT5_empty_Fixed_just_segment_pretrain_stage3_dreambench_recap_400k_t2i_400k_flux_200k_midjourney_150k_recovery_150k_3e4_epoch2_half_rotated/005-GPT-XL/checkpoints/eval_step_40000/batch_3_cfg_7.5_topk_16384.jpg b/CKPTS/Afteri2i_100kckpt_LLAVAT5_empty_Fixed_just_segment_pretrain_stage3_dreambench_recap_400k_t2i_400k_flux_200k_midjourney_150k_recovery_150k_3e4_epoch2_half_rotated/005-GPT-XL/checkpoints/eval_step_40000/batch_3_cfg_7.5_topk_16384.jpg new file mode 100644 index 0000000000000000000000000000000000000000..2100e9ece07459be4d46599405a8eed5abea484b --- /dev/null +++ b/CKPTS/Afteri2i_100kckpt_LLAVAT5_empty_Fixed_just_segment_pretrain_stage3_dreambench_recap_400k_t2i_400k_flux_200k_midjourney_150k_recovery_150k_3e4_epoch2_half_rotated/005-GPT-XL/checkpoints/eval_step_40000/batch_3_cfg_7.5_topk_16384.jpg @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:74fa8e757f1801d223933683742efcc579923c9d0cd3c6ace2d0972df8773415 +size 881257 diff --git a/CKPTS/Afteri2i_100kckpt_LLAVAT5_empty_Fixed_just_segment_pretrain_stage3_dreambench_recap_400k_t2i_400k_flux_200k_midjourney_150k_recovery_150k_3e4_epoch2_half_rotated/005-GPT-XL/checkpoints/eval_step_42000/batch_0_cfg_7.5_topk_16384.jpg b/CKPTS/Afteri2i_100kckpt_LLAVAT5_empty_Fixed_just_segment_pretrain_stage3_dreambench_recap_400k_t2i_400k_flux_200k_midjourney_150k_recovery_150k_3e4_epoch2_half_rotated/005-GPT-XL/checkpoints/eval_step_42000/batch_0_cfg_7.5_topk_16384.jpg new file mode 100644 index 0000000000000000000000000000000000000000..0cce763f315de57d5a6bbdec5de1296b845b1711 --- /dev/null +++ b/CKPTS/Afteri2i_100kckpt_LLAVAT5_empty_Fixed_just_segment_pretrain_stage3_dreambench_recap_400k_t2i_400k_flux_200k_midjourney_150k_recovery_150k_3e4_epoch2_half_rotated/005-GPT-XL/checkpoints/eval_step_42000/batch_0_cfg_7.5_topk_16384.jpg @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:a114f8e9aaa6c36080d8487f02a338f460ae481804137aea3ae8e79c72535b40 +size 742794 diff --git a/CKPTS/Afteri2i_100kckpt_LLAVAT5_empty_Fixed_just_segment_pretrain_stage3_dreambench_recap_400k_t2i_400k_flux_200k_midjourney_150k_recovery_150k_3e4_epoch2_half_rotated/005-GPT-XL/checkpoints/eval_step_42000/batch_1_cfg_7.5_topk_16384.jpg b/CKPTS/Afteri2i_100kckpt_LLAVAT5_empty_Fixed_just_segment_pretrain_stage3_dreambench_recap_400k_t2i_400k_flux_200k_midjourney_150k_recovery_150k_3e4_epoch2_half_rotated/005-GPT-XL/checkpoints/eval_step_42000/batch_1_cfg_7.5_topk_16384.jpg new file mode 100644 index 0000000000000000000000000000000000000000..49bbdb34bd99aad6b0b8f8fba8be5761cd0b767b --- /dev/null +++ b/CKPTS/Afteri2i_100kckpt_LLAVAT5_empty_Fixed_just_segment_pretrain_stage3_dreambench_recap_400k_t2i_400k_flux_200k_midjourney_150k_recovery_150k_3e4_epoch2_half_rotated/005-GPT-XL/checkpoints/eval_step_42000/batch_1_cfg_7.5_topk_16384.jpg @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:030326b9cf6b4df57401436b4a8652b7f888c3865c2f56f18412e3c09780780d +size 706833 diff --git a/CKPTS/Afteri2i_100kckpt_LLAVAT5_empty_Fixed_just_segment_pretrain_stage3_dreambench_recap_400k_t2i_400k_flux_200k_midjourney_150k_recovery_150k_3e4_epoch2_half_rotated/005-GPT-XL/checkpoints/eval_step_42000/batch_2_cfg_7.5_topk_16384.jpg b/CKPTS/Afteri2i_100kckpt_LLAVAT5_empty_Fixed_just_segment_pretrain_stage3_dreambench_recap_400k_t2i_400k_flux_200k_midjourney_150k_recovery_150k_3e4_epoch2_half_rotated/005-GPT-XL/checkpoints/eval_step_42000/batch_2_cfg_7.5_topk_16384.jpg new file mode 100644 index 0000000000000000000000000000000000000000..53f3621565cd23a9788eed0237fe2b6becce94f9 --- /dev/null +++ b/CKPTS/Afteri2i_100kckpt_LLAVAT5_empty_Fixed_just_segment_pretrain_stage3_dreambench_recap_400k_t2i_400k_flux_200k_midjourney_150k_recovery_150k_3e4_epoch2_half_rotated/005-GPT-XL/checkpoints/eval_step_42000/batch_2_cfg_7.5_topk_16384.jpg @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:8402762d7a22757574f0bd1954bd1dd40d334f6bfb8dea0511eaf4893b4cf2a2 +size 725682 diff --git a/CKPTS/Afteri2i_100kckpt_LLAVAT5_empty_Fixed_just_segment_pretrain_stage3_dreambench_recap_400k_t2i_400k_flux_200k_midjourney_150k_recovery_150k_3e4_epoch2_half_rotated/005-GPT-XL/checkpoints/eval_step_42000/batch_3_cfg_7.5_topk_16384.jpg b/CKPTS/Afteri2i_100kckpt_LLAVAT5_empty_Fixed_just_segment_pretrain_stage3_dreambench_recap_400k_t2i_400k_flux_200k_midjourney_150k_recovery_150k_3e4_epoch2_half_rotated/005-GPT-XL/checkpoints/eval_step_42000/batch_3_cfg_7.5_topk_16384.jpg new file mode 100644 index 0000000000000000000000000000000000000000..1bebbc18ebf1e59e1f0c37f4a1d74d048a3be900 --- /dev/null +++ b/CKPTS/Afteri2i_100kckpt_LLAVAT5_empty_Fixed_just_segment_pretrain_stage3_dreambench_recap_400k_t2i_400k_flux_200k_midjourney_150k_recovery_150k_3e4_epoch2_half_rotated/005-GPT-XL/checkpoints/eval_step_42000/batch_3_cfg_7.5_topk_16384.jpg @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:1b31c3d92892cbb3f27c57fa3bd35b04d50fd8453c9686b519f5e7c0140e3043 +size 848967 diff --git a/CKPTS/Afteri2i_100kckpt_LLAVAT5_empty_Fixed_just_segment_pretrain_stage3_dreambench_recap_400k_t2i_400k_flux_200k_midjourney_150k_recovery_150k_3e4_epoch2_half_rotated/005-GPT-XL/checkpoints/eval_step_44000/batch_0_cfg_7.5_topk_16384.jpg b/CKPTS/Afteri2i_100kckpt_LLAVAT5_empty_Fixed_just_segment_pretrain_stage3_dreambench_recap_400k_t2i_400k_flux_200k_midjourney_150k_recovery_150k_3e4_epoch2_half_rotated/005-GPT-XL/checkpoints/eval_step_44000/batch_0_cfg_7.5_topk_16384.jpg new file mode 100644 index 0000000000000000000000000000000000000000..cba4bcd089aadfd9514e5cebd70e50c79826c40b --- /dev/null +++ b/CKPTS/Afteri2i_100kckpt_LLAVAT5_empty_Fixed_just_segment_pretrain_stage3_dreambench_recap_400k_t2i_400k_flux_200k_midjourney_150k_recovery_150k_3e4_epoch2_half_rotated/005-GPT-XL/checkpoints/eval_step_44000/batch_0_cfg_7.5_topk_16384.jpg @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:780a93c24b62a7b0922b367f4cf951feb3206e0471ce8efca9ab740e378d9941 +size 730695 diff --git a/CKPTS/Afteri2i_100kckpt_LLAVAT5_empty_Fixed_just_segment_pretrain_stage3_dreambench_recap_400k_t2i_400k_flux_200k_midjourney_150k_recovery_150k_3e4_epoch2_half_rotated/005-GPT-XL/checkpoints/eval_step_44000/batch_1_cfg_7.5_topk_16384.jpg b/CKPTS/Afteri2i_100kckpt_LLAVAT5_empty_Fixed_just_segment_pretrain_stage3_dreambench_recap_400k_t2i_400k_flux_200k_midjourney_150k_recovery_150k_3e4_epoch2_half_rotated/005-GPT-XL/checkpoints/eval_step_44000/batch_1_cfg_7.5_topk_16384.jpg new file mode 100644 index 0000000000000000000000000000000000000000..62f7a4070d9f4b03c456b72aed8ffa054a98e942 --- /dev/null +++ b/CKPTS/Afteri2i_100kckpt_LLAVAT5_empty_Fixed_just_segment_pretrain_stage3_dreambench_recap_400k_t2i_400k_flux_200k_midjourney_150k_recovery_150k_3e4_epoch2_half_rotated/005-GPT-XL/checkpoints/eval_step_44000/batch_1_cfg_7.5_topk_16384.jpg @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:24bd4e9c0b534ba346aaa9fa6144e30664be07c8be4af9fdff415f8dca301964 +size 721805 diff --git a/CKPTS/Afteri2i_100kckpt_LLAVAT5_empty_Fixed_just_segment_pretrain_stage3_dreambench_recap_400k_t2i_400k_flux_200k_midjourney_150k_recovery_150k_3e4_epoch2_half_rotated/005-GPT-XL/checkpoints/eval_step_44000/batch_2_cfg_7.5_topk_16384.jpg b/CKPTS/Afteri2i_100kckpt_LLAVAT5_empty_Fixed_just_segment_pretrain_stage3_dreambench_recap_400k_t2i_400k_flux_200k_midjourney_150k_recovery_150k_3e4_epoch2_half_rotated/005-GPT-XL/checkpoints/eval_step_44000/batch_2_cfg_7.5_topk_16384.jpg new file mode 100644 index 0000000000000000000000000000000000000000..da82a46c04b80fe4085bf4beb1e75335dca4cbfd --- /dev/null +++ b/CKPTS/Afteri2i_100kckpt_LLAVAT5_empty_Fixed_just_segment_pretrain_stage3_dreambench_recap_400k_t2i_400k_flux_200k_midjourney_150k_recovery_150k_3e4_epoch2_half_rotated/005-GPT-XL/checkpoints/eval_step_44000/batch_2_cfg_7.5_topk_16384.jpg @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:c89bb4828190fd1c6a7e5f97c40833d0c3710481ec3bd446ae33498eb12172f2 +size 738886 diff --git a/CKPTS/Afteri2i_100kckpt_LLAVAT5_empty_Fixed_just_segment_pretrain_stage3_dreambench_recap_400k_t2i_400k_flux_200k_midjourney_150k_recovery_150k_3e4_epoch2_half_rotated/005-GPT-XL/checkpoints/eval_step_44000/batch_3_cfg_7.5_topk_16384.jpg b/CKPTS/Afteri2i_100kckpt_LLAVAT5_empty_Fixed_just_segment_pretrain_stage3_dreambench_recap_400k_t2i_400k_flux_200k_midjourney_150k_recovery_150k_3e4_epoch2_half_rotated/005-GPT-XL/checkpoints/eval_step_44000/batch_3_cfg_7.5_topk_16384.jpg new file mode 100644 index 0000000000000000000000000000000000000000..25dfa57afa3528d5fe112e6ad55111ccb5895d33 --- /dev/null +++ b/CKPTS/Afteri2i_100kckpt_LLAVAT5_empty_Fixed_just_segment_pretrain_stage3_dreambench_recap_400k_t2i_400k_flux_200k_midjourney_150k_recovery_150k_3e4_epoch2_half_rotated/005-GPT-XL/checkpoints/eval_step_44000/batch_3_cfg_7.5_topk_16384.jpg @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:9c140fd5115b9bd79ce2bcf49805b18ea06c0ad2aaf1ec912939ccf9a3793e75 +size 857425 diff --git a/CKPTS/Afteri2i_100kckpt_LLAVAT5_empty_Fixed_just_segment_pretrain_stage3_dreambench_recap_400k_t2i_400k_flux_200k_midjourney_150k_recovery_150k_3e4_epoch2_half_rotated/005-GPT-XL/checkpoints/eval_step_46000/batch_0_cfg_7.5_topk_16384.jpg b/CKPTS/Afteri2i_100kckpt_LLAVAT5_empty_Fixed_just_segment_pretrain_stage3_dreambench_recap_400k_t2i_400k_flux_200k_midjourney_150k_recovery_150k_3e4_epoch2_half_rotated/005-GPT-XL/checkpoints/eval_step_46000/batch_0_cfg_7.5_topk_16384.jpg new file mode 100644 index 0000000000000000000000000000000000000000..c5a97f07390936ef703f08dd314ec8f27b881e53 --- /dev/null +++ b/CKPTS/Afteri2i_100kckpt_LLAVAT5_empty_Fixed_just_segment_pretrain_stage3_dreambench_recap_400k_t2i_400k_flux_200k_midjourney_150k_recovery_150k_3e4_epoch2_half_rotated/005-GPT-XL/checkpoints/eval_step_46000/batch_0_cfg_7.5_topk_16384.jpg @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:144192d2e74ca44fcd6aec1c3c0584d8e2adb64276c7f983a2d6b61d1cf07f96 +size 749081 diff --git a/CKPTS/Afteri2i_100kckpt_LLAVAT5_empty_Fixed_just_segment_pretrain_stage3_dreambench_recap_400k_t2i_400k_flux_200k_midjourney_150k_recovery_150k_3e4_epoch2_half_rotated/005-GPT-XL/checkpoints/eval_step_46000/batch_1_cfg_7.5_topk_16384.jpg b/CKPTS/Afteri2i_100kckpt_LLAVAT5_empty_Fixed_just_segment_pretrain_stage3_dreambench_recap_400k_t2i_400k_flux_200k_midjourney_150k_recovery_150k_3e4_epoch2_half_rotated/005-GPT-XL/checkpoints/eval_step_46000/batch_1_cfg_7.5_topk_16384.jpg new file mode 100644 index 0000000000000000000000000000000000000000..8f182861647ea66da44a20126b5e7cf612a833d9 --- /dev/null +++ b/CKPTS/Afteri2i_100kckpt_LLAVAT5_empty_Fixed_just_segment_pretrain_stage3_dreambench_recap_400k_t2i_400k_flux_200k_midjourney_150k_recovery_150k_3e4_epoch2_half_rotated/005-GPT-XL/checkpoints/eval_step_46000/batch_1_cfg_7.5_topk_16384.jpg @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:b22817e7426ceff7034a19cd9a3d4b53317c710eaed0d1b185ce4e921a7f827d +size 707405 diff --git a/CKPTS/Afteri2i_100kckpt_LLAVAT5_empty_Fixed_just_segment_pretrain_stage3_dreambench_recap_400k_t2i_400k_flux_200k_midjourney_150k_recovery_150k_3e4_epoch2_half_rotated/005-GPT-XL/checkpoints/eval_step_46000/batch_2_cfg_7.5_topk_16384.jpg b/CKPTS/Afteri2i_100kckpt_LLAVAT5_empty_Fixed_just_segment_pretrain_stage3_dreambench_recap_400k_t2i_400k_flux_200k_midjourney_150k_recovery_150k_3e4_epoch2_half_rotated/005-GPT-XL/checkpoints/eval_step_46000/batch_2_cfg_7.5_topk_16384.jpg new file mode 100644 index 0000000000000000000000000000000000000000..fc5010d88166f39079a80fed17e4f0764b0e010f --- /dev/null +++ b/CKPTS/Afteri2i_100kckpt_LLAVAT5_empty_Fixed_just_segment_pretrain_stage3_dreambench_recap_400k_t2i_400k_flux_200k_midjourney_150k_recovery_150k_3e4_epoch2_half_rotated/005-GPT-XL/checkpoints/eval_step_46000/batch_2_cfg_7.5_topk_16384.jpg @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:c864bb5d2cda7f539e04907045cc9882ecb0bb0c1df037473a730e08d3d9644d +size 731104 diff --git a/CKPTS/Afteri2i_100kckpt_LLAVAT5_empty_Fixed_just_segment_pretrain_stage3_dreambench_recap_400k_t2i_400k_flux_200k_midjourney_150k_recovery_150k_3e4_epoch2_half_rotated/005-GPT-XL/checkpoints/eval_step_46000/batch_3_cfg_7.5_topk_16384.jpg b/CKPTS/Afteri2i_100kckpt_LLAVAT5_empty_Fixed_just_segment_pretrain_stage3_dreambench_recap_400k_t2i_400k_flux_200k_midjourney_150k_recovery_150k_3e4_epoch2_half_rotated/005-GPT-XL/checkpoints/eval_step_46000/batch_3_cfg_7.5_topk_16384.jpg new file mode 100644 index 0000000000000000000000000000000000000000..51666677cd18f17c10e952dae59280901c24244f --- /dev/null +++ b/CKPTS/Afteri2i_100kckpt_LLAVAT5_empty_Fixed_just_segment_pretrain_stage3_dreambench_recap_400k_t2i_400k_flux_200k_midjourney_150k_recovery_150k_3e4_epoch2_half_rotated/005-GPT-XL/checkpoints/eval_step_46000/batch_3_cfg_7.5_topk_16384.jpg @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:62f6c0bb366f3a701f0d20ef7d57b53820568fa481e248a7b6b0dc50341deb4d +size 912355 diff --git a/CKPTS/Afteri2i_100kckpt_LLAVAT5_empty_Fixed_just_segment_pretrain_stage3_dreambench_recap_400k_t2i_400k_flux_200k_midjourney_150k_recovery_150k_3e4_epoch2_half_rotated/005-GPT-XL/checkpoints/eval_step_6000/batch_0_cfg_7.5_topk_16384.jpg b/CKPTS/Afteri2i_100kckpt_LLAVAT5_empty_Fixed_just_segment_pretrain_stage3_dreambench_recap_400k_t2i_400k_flux_200k_midjourney_150k_recovery_150k_3e4_epoch2_half_rotated/005-GPT-XL/checkpoints/eval_step_6000/batch_0_cfg_7.5_topk_16384.jpg new file mode 100644 index 0000000000000000000000000000000000000000..d14af595772cfac7ae87d0ff3f0b409425ab8d25 --- /dev/null +++ b/CKPTS/Afteri2i_100kckpt_LLAVAT5_empty_Fixed_just_segment_pretrain_stage3_dreambench_recap_400k_t2i_400k_flux_200k_midjourney_150k_recovery_150k_3e4_epoch2_half_rotated/005-GPT-XL/checkpoints/eval_step_6000/batch_0_cfg_7.5_topk_16384.jpg @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:a12edee27ae2d73f978a5505d5f9adf6b805dfab930ef34b8ca62910f094173f +size 768280 diff --git a/CKPTS/Afteri2i_100kckpt_LLAVAT5_empty_Fixed_just_segment_pretrain_stage3_dreambench_recap_400k_t2i_400k_flux_200k_midjourney_150k_recovery_150k_3e4_epoch2_half_rotated/005-GPT-XL/checkpoints/eval_step_6000/batch_1_cfg_7.5_topk_16384.jpg b/CKPTS/Afteri2i_100kckpt_LLAVAT5_empty_Fixed_just_segment_pretrain_stage3_dreambench_recap_400k_t2i_400k_flux_200k_midjourney_150k_recovery_150k_3e4_epoch2_half_rotated/005-GPT-XL/checkpoints/eval_step_6000/batch_1_cfg_7.5_topk_16384.jpg new file mode 100644 index 0000000000000000000000000000000000000000..c1f115c265aac6cd373582f58963666fbf878f30 --- /dev/null +++ b/CKPTS/Afteri2i_100kckpt_LLAVAT5_empty_Fixed_just_segment_pretrain_stage3_dreambench_recap_400k_t2i_400k_flux_200k_midjourney_150k_recovery_150k_3e4_epoch2_half_rotated/005-GPT-XL/checkpoints/eval_step_6000/batch_1_cfg_7.5_topk_16384.jpg @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:2024791ba40b1b6264aa825300c60ceca7708f4754d92bca6a08af4c1965d1d0 +size 702001 diff --git a/CKPTS/Afteri2i_100kckpt_LLAVAT5_empty_Fixed_just_segment_pretrain_stage3_dreambench_recap_400k_t2i_400k_flux_200k_midjourney_150k_recovery_150k_3e4_epoch2_half_rotated/005-GPT-XL/checkpoints/eval_step_6000/batch_2_cfg_7.5_topk_16384.jpg b/CKPTS/Afteri2i_100kckpt_LLAVAT5_empty_Fixed_just_segment_pretrain_stage3_dreambench_recap_400k_t2i_400k_flux_200k_midjourney_150k_recovery_150k_3e4_epoch2_half_rotated/005-GPT-XL/checkpoints/eval_step_6000/batch_2_cfg_7.5_topk_16384.jpg new file mode 100644 index 0000000000000000000000000000000000000000..4b452232134fc10f99fc116cd7ba6c8af65ce4e0 --- /dev/null +++ b/CKPTS/Afteri2i_100kckpt_LLAVAT5_empty_Fixed_just_segment_pretrain_stage3_dreambench_recap_400k_t2i_400k_flux_200k_midjourney_150k_recovery_150k_3e4_epoch2_half_rotated/005-GPT-XL/checkpoints/eval_step_6000/batch_2_cfg_7.5_topk_16384.jpg @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:c3d46108343d5a7b530cc1b7c4b0fb39f0c8f3ba5c658bab037fa06860b0442f +size 738265 diff --git a/CKPTS/Afteri2i_100kckpt_LLAVAT5_empty_Fixed_just_segment_pretrain_stage3_dreambench_recap_400k_t2i_400k_flux_200k_midjourney_150k_recovery_150k_3e4_epoch2_half_rotated/005-GPT-XL/checkpoints/eval_step_6000/batch_3_cfg_7.5_topk_16384.jpg b/CKPTS/Afteri2i_100kckpt_LLAVAT5_empty_Fixed_just_segment_pretrain_stage3_dreambench_recap_400k_t2i_400k_flux_200k_midjourney_150k_recovery_150k_3e4_epoch2_half_rotated/005-GPT-XL/checkpoints/eval_step_6000/batch_3_cfg_7.5_topk_16384.jpg new file mode 100644 index 0000000000000000000000000000000000000000..2a4614a06350d69c3a2c5369d4a762718c6faa34 --- /dev/null +++ b/CKPTS/Afteri2i_100kckpt_LLAVAT5_empty_Fixed_just_segment_pretrain_stage3_dreambench_recap_400k_t2i_400k_flux_200k_midjourney_150k_recovery_150k_3e4_epoch2_half_rotated/005-GPT-XL/checkpoints/eval_step_6000/batch_3_cfg_7.5_topk_16384.jpg @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:1b88c599b5b4ad8dc728a7b1abbbb093bfb97be61b1f8da15e73c15681ee976b +size 883375 diff --git a/CKPTS/Afteri2i_100kckpt_LLAVAT5_empty_Fixed_just_segment_pretrain_stage3_dreambench_recap_400k_t2i_400k_flux_200k_midjourney_150k_recovery_150k_3e4_epoch2_half_rotated/005-GPT-XL/checkpoints/eval_step_8000/batch_0_cfg_7.5_topk_16384.jpg b/CKPTS/Afteri2i_100kckpt_LLAVAT5_empty_Fixed_just_segment_pretrain_stage3_dreambench_recap_400k_t2i_400k_flux_200k_midjourney_150k_recovery_150k_3e4_epoch2_half_rotated/005-GPT-XL/checkpoints/eval_step_8000/batch_0_cfg_7.5_topk_16384.jpg new file mode 100644 index 0000000000000000000000000000000000000000..84d771791ee980da70a3836473534e75b68425c2 --- /dev/null +++ b/CKPTS/Afteri2i_100kckpt_LLAVAT5_empty_Fixed_just_segment_pretrain_stage3_dreambench_recap_400k_t2i_400k_flux_200k_midjourney_150k_recovery_150k_3e4_epoch2_half_rotated/005-GPT-XL/checkpoints/eval_step_8000/batch_0_cfg_7.5_topk_16384.jpg @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:5e0711967034268fb6e591c84143e0d800c80a5b059199ecd7456709f4030130 +size 760563 diff --git a/CKPTS/Afteri2i_100kckpt_LLAVAT5_empty_Fixed_just_segment_pretrain_stage3_dreambench_recap_400k_t2i_400k_flux_200k_midjourney_150k_recovery_150k_3e4_epoch2_half_rotated/005-GPT-XL/checkpoints/eval_step_8000/batch_1_cfg_7.5_topk_16384.jpg b/CKPTS/Afteri2i_100kckpt_LLAVAT5_empty_Fixed_just_segment_pretrain_stage3_dreambench_recap_400k_t2i_400k_flux_200k_midjourney_150k_recovery_150k_3e4_epoch2_half_rotated/005-GPT-XL/checkpoints/eval_step_8000/batch_1_cfg_7.5_topk_16384.jpg new file mode 100644 index 0000000000000000000000000000000000000000..191b9df2d787e2b4041cc2284d24714eca17f105 --- /dev/null +++ b/CKPTS/Afteri2i_100kckpt_LLAVAT5_empty_Fixed_just_segment_pretrain_stage3_dreambench_recap_400k_t2i_400k_flux_200k_midjourney_150k_recovery_150k_3e4_epoch2_half_rotated/005-GPT-XL/checkpoints/eval_step_8000/batch_1_cfg_7.5_topk_16384.jpg @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:4ea6950337013eaec4c29412dc3bb9eceaa33891ff8c934140dabc80d6d3c699 +size 708325 diff --git a/CKPTS/Afteri2i_100kckpt_LLAVAT5_empty_Fixed_just_segment_pretrain_stage3_dreambench_recap_400k_t2i_400k_flux_200k_midjourney_150k_recovery_150k_3e4_epoch2_half_rotated/005-GPT-XL/checkpoints/eval_step_8000/batch_2_cfg_7.5_topk_16384.jpg b/CKPTS/Afteri2i_100kckpt_LLAVAT5_empty_Fixed_just_segment_pretrain_stage3_dreambench_recap_400k_t2i_400k_flux_200k_midjourney_150k_recovery_150k_3e4_epoch2_half_rotated/005-GPT-XL/checkpoints/eval_step_8000/batch_2_cfg_7.5_topk_16384.jpg new file mode 100644 index 0000000000000000000000000000000000000000..7d531ce9304c7e149bec349df0ba3d11721fbec2 --- /dev/null +++ b/CKPTS/Afteri2i_100kckpt_LLAVAT5_empty_Fixed_just_segment_pretrain_stage3_dreambench_recap_400k_t2i_400k_flux_200k_midjourney_150k_recovery_150k_3e4_epoch2_half_rotated/005-GPT-XL/checkpoints/eval_step_8000/batch_2_cfg_7.5_topk_16384.jpg @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:e6e133fbc2ca1b67f4d806f1ce79ee691b68f1334576855b50a1cb36f90ed19d +size 734487 diff --git a/CKPTS/Afteri2i_100kckpt_LLAVAT5_empty_Fixed_just_segment_pretrain_stage3_dreambench_recap_400k_t2i_400k_flux_200k_midjourney_150k_recovery_150k_3e4_epoch2_half_rotated/005-GPT-XL/checkpoints/eval_step_8000/batch_3_cfg_7.5_topk_16384.jpg b/CKPTS/Afteri2i_100kckpt_LLAVAT5_empty_Fixed_just_segment_pretrain_stage3_dreambench_recap_400k_t2i_400k_flux_200k_midjourney_150k_recovery_150k_3e4_epoch2_half_rotated/005-GPT-XL/checkpoints/eval_step_8000/batch_3_cfg_7.5_topk_16384.jpg new file mode 100644 index 0000000000000000000000000000000000000000..5d66b3516efec6c41b026046b7bb5157c6b9bf05 --- /dev/null +++ b/CKPTS/Afteri2i_100kckpt_LLAVAT5_empty_Fixed_just_segment_pretrain_stage3_dreambench_recap_400k_t2i_400k_flux_200k_midjourney_150k_recovery_150k_3e4_epoch2_half_rotated/005-GPT-XL/checkpoints/eval_step_8000/batch_3_cfg_7.5_topk_16384.jpg @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:69770bbf0abaa855d7fda0b8a0011100baa977f54e645623305c8ce93bd0c011 +size 841817 diff --git a/CKPTS/Afteri2i_100kckpt_LLAVAT5_empty_Fixed_just_segment_pretrain_stage3_dreambench_recap_400k_t2i_400k_flux_200k_midjourney_150k_recovery_150k_3e4_epoch2_half_rotated/005-GPT-XL/log.txt b/CKPTS/Afteri2i_100kckpt_LLAVAT5_empty_Fixed_just_segment_pretrain_stage3_dreambench_recap_400k_t2i_400k_flux_200k_midjourney_150k_recovery_150k_3e4_epoch2_half_rotated/005-GPT-XL/log.txt new file mode 100644 index 0000000000000000000000000000000000000000..6edc12b241d3bc7b146c700eceed26ba49a16539 --- /dev/null +++ b/CKPTS/Afteri2i_100kckpt_LLAVAT5_empty_Fixed_just_segment_pretrain_stage3_dreambench_recap_400k_t2i_400k_flux_200k_midjourney_150k_recovery_150k_3e4_epoch2_half_rotated/005-GPT-XL/log.txt @@ -0,0 +1,1987 @@ +[2025-04-23 12:23:28] Experiment directory created at checkpoint/Afteri2i_100kckpt_LLAVAT5_empty_Fixed_just_segment_pretrain_stage3_dreambench_recap_400k_t2i_400k_flux_200k_midjourney_150k_recovery_150k_3e4_epoch2_half_rotated/005-GPT-XL +[2025-04-23 12:23:28] Namespace(data_path='/tmp/haozhezhao/MLLMG/jsonl_data/merged_train_set_set_subject_400k_recap_t2i_400k_flux_200k_midjourney_150k_recovery_150k_grounding_100fluxseg_50samseg.jsonl', cloud_save_path='/tmp/haozhezhao/checkpoint', no_local_save=False, vq_model='VQ-16', vq_ckpt='/tmp/haozhezhao/model/llamagen_t2i/vq_ds16_t2i.pt', codebook_size=16384, codebook_embed_dim=8, gpt_model='GPT-XL', gpt_ckpt='/tmp/haozhezhao/MLLMG/MLLMG_ckpts/checkpoints/AfterI2I_llavat5_subject_train_all_T5_subject_t2i_ti2i_120_w_flux_segment_noimageonly_empty_fix_train_all/0100000.pt', gpt_type='t2i', vocab_size=16384, cls_token_num=512, dropout_p=0.1, token_dropout_p=0.1, drop_path=0.0, no_compile=False, results_dir='checkpoint/Afteri2i_100kckpt_LLAVAT5_empty_Fixed_just_segment_pretrain_stage3_dreambench_recap_400k_t2i_400k_flux_200k_midjourney_150k_recovery_150k_3e4_epoch2_half_rotated', dataset='ti2i', image_size=512, downsample_size=16, num_classes=1000, epochs=2, lr=0.0003, weight_decay=0.05, beta1=0.9, beta2=0.95, max_grad_norm=1.0, global_batch_size=56, global_seed=0, num_workers=8, log_every=25, ckpt_every=2000, gradient_accumulation_steps=4, mixed_precision='bf16', val_data_path='/tmp/haozhezhao/MLLMG/jsonl_data/dreambench_plus_valid.jsonl', use_vision_tower=True, model_name_or_path='/tmp/haozhezhao/model/blip2-flan-t5-xl', image_place_holder='', processor_path=None, do_eval=True, max_eval_samples=250, train_text_encoder=True, no_left_padding=False, cfg_scale=7.5, top_k=16384, temperature=0.9, top_p=1.0, eval_steps=2000, project_name='llamagen_ti2i', load_from_checkpoint='/tmp/haozhezhao/MLLMG/checkpoint/FIXed_3M_1epoch_step32000.pt', warmup=0.05, lr_decay_style='cosine', lr_decay_ratio=0.1, train_iters=500000, class_dropout_prob=0.1, with_image_only=False, image_only_rate=0.1, stage2=False, subject_driven=True, load_subject_embedding=None, reference_data_path='/tmp/haozhezhao/MLLMG/cc12m_reference_tunnel.jsonl', multimodal_encoder='llava', do_recovery=True, no_replace=False, resume=False, dreambench_eval=True, find_unused_parameters=True, load_visual_encoder=False, continue_stage1=False, replace_subject=False, train_all=True, save_total_limit=1, load_language_projection='/tmp/haozhezhao/MLLMG/llava-v1.5-flant5_fixed-pretrain/mm_projector.bin', mm_vision_tower='openai/clip-vit-large-patch14', load_fixed_llamagen=True, unfreeze_output=False, fix='gpt-empty-fix', rank=0, world_size=8, gpu=0, dist_url='env://', distributed=True, dist_backend='nccl') +[2025-04-23 12:23:28] Starting rank=0, seed=0, world_size=8. +[2025-04-23 12:23:28] text_config is None. Initializing the text config with default values (`OPTConfig`). +[2025-04-23 12:24:34] GPT Parameters: 2,310,680,832 +[2025-04-23 12:24:34] num decayed parameter tensors: 356, with 2,007,303,168 parameters +[2025-04-23 12:24:34] num non-decayed parameter tensors: 124, with 197,888 parameters +[2025-04-23 12:24:34] using fused AdamW: True +[2025-04-23 12:24:42] Dataset contains 1,313,682 images +[2025-04-23 12:24:42] Train iters 46916 , warmup 2345.8, len of loader 23458 +[2025-04-23 12:24:58] ### LOAD pretraining weights from checkpoint: /tmp/haozhezhao/MLLMG/MLLMG_ckpts/checkpoints/AfterI2I_llavat5_subject_train_all_T5_subject_t2i_ti2i_120_w_flux_segment_noimageonly_empty_fix_train_all/0100000.pt +[2025-04-23 12:24:58] Initial state: steps=0, epochs=0 +[2025-04-23 12:24:58] compiling the model... (may take several minutes) +[2025-04-23 12:24:58] freeze the vit +[2025-04-23 12:24:58] ***** total param is 2310680832 ***** +[2025-04-23 12:24:58] ***** total trained param is 2007501056 ***** +[2025-04-23 12:25:00] Training for 2 epochs... +[2025-04-23 12:25:00] Beginning epoch 0... +[2025-04-23 12:29:29] (step=0000025) Train Loss: 6.6005, Train Steps/Sec: 0.09 +[2025-04-23 12:30:16] (step=0000050) Train Loss: 6.6555, Train Steps/Sec: 0.53 +[2025-04-23 12:31:45] (step=0000075) Train Loss: 6.4598, Train Steps/Sec: 0.28 +[2025-04-23 12:32:39] (step=0000100) Train Loss: 6.3559, Train Steps/Sec: 0.46 +[2025-04-23 12:33:00] (step=0000125) Train Loss: 6.2558, Train Steps/Sec: 1.18 +[2025-04-23 12:33:48] (step=0000150) Train Loss: 6.1055, Train Steps/Sec: 0.52 +[2025-04-23 12:34:35] (step=0000175) Train Loss: 6.1220, Train Steps/Sec: 0.53 +[2025-04-23 12:34:58] (step=0000200) Train Loss: 6.0450, Train Steps/Sec: 1.12 +[2025-04-23 12:35:19] (step=0000225) Train Loss: 5.9307, Train Steps/Sec: 1.18 +[2025-04-23 12:36:06] (step=0000250) Train Loss: 5.8834, Train Steps/Sec: 0.53 +[2025-04-23 12:37:18] (step=0000275) Train Loss: 5.9763, Train Steps/Sec: 0.34 +[2025-04-23 12:37:40] (step=0000300) Train Loss: 5.8123, Train Steps/Sec: 1.17 +[2025-04-23 12:38:01] (step=0000325) Train Loss: 5.9544, Train Steps/Sec: 1.18 +[2025-04-23 12:38:22] (step=0000350) Train Loss: 5.8446, Train Steps/Sec: 1.17 +[2025-04-23 12:38:43] (step=0000375) Train Loss: 5.8658, Train Steps/Sec: 1.18 +[2025-04-23 12:39:31] (step=0000400) Train Loss: 5.7741, Train Steps/Sec: 0.52 +[2025-04-23 12:39:54] (step=0000425) Train Loss: 5.8472, Train Steps/Sec: 1.12 +[2025-04-23 12:40:16] (step=0000450) Train Loss: 5.8283, Train Steps/Sec: 1.11 +[2025-04-23 12:40:37] (step=0000475) Train Loss: 5.8059, Train Steps/Sec: 1.18 +[2025-04-23 12:40:59] (step=0000500) Train Loss: 5.8193, Train Steps/Sec: 1.17 +[2025-04-23 12:41:20] (step=0000525) Train Loss: 5.8235, Train Steps/Sec: 1.17 +[2025-04-23 12:41:41] (step=0000550) Train Loss: 5.8375, Train Steps/Sec: 1.17 +[2025-04-23 12:42:04] (step=0000575) Train Loss: 5.8428, Train Steps/Sec: 1.11 +[2025-04-23 12:42:28] (step=0000600) Train Loss: 5.7903, Train Steps/Sec: 1.03 +[2025-04-23 12:42:50] (step=0000625) Train Loss: 5.8660, Train Steps/Sec: 1.13 +[2025-04-23 12:43:12] (step=0000650) Train Loss: 5.7774, Train Steps/Sec: 1.17 +[2025-04-23 12:43:34] (step=0000675) Train Loss: 5.7929, Train Steps/Sec: 1.13 +[2025-04-23 12:43:55] (step=0000700) Train Loss: 5.9228, Train Steps/Sec: 1.17 +[2025-04-23 12:44:17] (step=0000725) Train Loss: 5.7613, Train Steps/Sec: 1.16 +[2025-04-23 12:44:38] (step=0000750) Train Loss: 5.8464, Train Steps/Sec: 1.17 +[2025-04-23 12:44:59] (step=0000775) Train Loss: 5.8326, Train Steps/Sec: 1.17 +[2025-04-23 12:45:21] (step=0000800) Train Loss: 5.7977, Train Steps/Sec: 1.16 +[2025-04-23 12:45:42] (step=0000825) Train Loss: 5.7656, Train Steps/Sec: 1.17 +[2025-04-23 12:46:04] (step=0000850) Train Loss: 5.7243, Train Steps/Sec: 1.17 +[2025-04-23 12:46:25] (step=0000875) Train Loss: 5.7828, Train Steps/Sec: 1.17 +[2025-04-23 12:46:46] (step=0000900) Train Loss: 5.7714, Train Steps/Sec: 1.17 +[2025-04-23 12:47:08] (step=0000925) Train Loss: 5.8266, Train Steps/Sec: 1.17 +[2025-04-23 12:47:29] (step=0000950) Train Loss: 5.8739, Train Steps/Sec: 1.17 +[2025-04-23 12:47:50] (step=0000975) Train Loss: 5.7751, Train Steps/Sec: 1.17 +[2025-04-23 12:48:12] (step=0001000) Train Loss: 5.7676, Train Steps/Sec: 1.17 +[2025-04-23 12:48:33] (step=0001025) Train Loss: 5.7234, Train Steps/Sec: 1.17 +[2025-04-23 12:48:55] (step=0001050) Train Loss: 5.7606, Train Steps/Sec: 1.17 +[2025-04-23 12:49:16] (step=0001075) Train Loss: 5.7210, Train Steps/Sec: 1.17 +[2025-04-23 12:49:37] (step=0001100) Train Loss: 5.7764, Train Steps/Sec: 1.17 +[2025-04-23 12:49:59] (step=0001125) Train Loss: 5.7319, Train Steps/Sec: 1.17 +[2025-04-23 12:50:20] (step=0001150) Train Loss: 5.7573, Train Steps/Sec: 1.17 +[2025-04-23 12:50:42] (step=0001175) Train Loss: 5.7895, Train Steps/Sec: 1.12 +[2025-04-23 12:51:05] (step=0001200) Train Loss: 5.7181, Train Steps/Sec: 1.11 +[2025-04-23 12:51:27] (step=0001225) Train Loss: 5.7546, Train Steps/Sec: 1.13 +[2025-04-23 12:51:49] (step=0001250) Train Loss: 5.6895, Train Steps/Sec: 1.12 +[2025-04-23 12:52:12] (step=0001275) Train Loss: 5.7132, Train Steps/Sec: 1.09 +[2025-04-23 12:52:34] (step=0001300) Train Loss: 5.7371, Train Steps/Sec: 1.17 +[2025-04-23 12:52:55] (step=0001325) Train Loss: 5.8133, Train Steps/Sec: 1.18 +[2025-04-23 12:53:17] (step=0001350) Train Loss: 5.7413, Train Steps/Sec: 1.12 +[2025-04-23 12:53:38] (step=0001375) Train Loss: 5.7914, Train Steps/Sec: 1.17 +[2025-04-23 12:54:00] (step=0001400) Train Loss: 5.6662, Train Steps/Sec: 1.16 +[2025-04-23 12:54:21] (step=0001425) Train Loss: 5.7186, Train Steps/Sec: 1.17 +[2025-04-23 12:54:43] (step=0001450) Train Loss: 5.8405, Train Steps/Sec: 1.17 +[2025-04-23 12:55:04] (step=0001475) Train Loss: 5.7428, Train Steps/Sec: 1.17 +[2025-04-23 12:55:25] (step=0001500) Train Loss: 5.6896, Train Steps/Sec: 1.17 +[2025-04-23 12:55:47] (step=0001525) Train Loss: 5.7403, Train Steps/Sec: 1.17 +[2025-04-23 12:56:08] (step=0001550) Train Loss: 5.7303, Train Steps/Sec: 1.17 +[2025-04-23 12:56:29] (step=0001575) Train Loss: 5.7393, Train Steps/Sec: 1.17 +[2025-04-23 12:56:51] (step=0001600) Train Loss: 5.7214, Train Steps/Sec: 1.17 +[2025-04-23 12:57:12] (step=0001625) Train Loss: 5.7278, Train Steps/Sec: 1.18 +[2025-04-23 12:57:33] (step=0001650) Train Loss: 5.7655, Train Steps/Sec: 1.17 +[2025-04-23 12:57:55] (step=0001675) Train Loss: 5.7133, Train Steps/Sec: 1.17 +[2025-04-23 12:58:16] (step=0001700) Train Loss: 5.8444, Train Steps/Sec: 1.17 +[2025-04-23 12:58:38] (step=0001725) Train Loss: 5.7540, Train Steps/Sec: 1.17 +[2025-04-23 12:58:59] (step=0001750) Train Loss: 5.6806, Train Steps/Sec: 1.17 +[2025-04-23 12:59:20] (step=0001775) Train Loss: 5.7114, Train Steps/Sec: 1.17 +[2025-04-23 12:59:42] (step=0001800) Train Loss: 5.7752, Train Steps/Sec: 1.17 +[2025-04-23 13:00:03] (step=0001825) Train Loss: 5.7199, Train Steps/Sec: 1.17 +[2025-04-23 13:00:24] (step=0001850) Train Loss: 5.7680, Train Steps/Sec: 1.17 +[2025-04-23 13:00:46] (step=0001875) Train Loss: 5.7145, Train Steps/Sec: 1.17 +[2025-04-23 13:01:10] (step=0001900) Train Loss: 5.6734, Train Steps/Sec: 1.03 +[2025-04-23 13:01:34] (step=0001925) Train Loss: 5.8257, Train Steps/Sec: 1.05 +[2025-04-23 13:01:56] (step=0001950) Train Loss: 5.7327, Train Steps/Sec: 1.13 +[2025-04-23 13:02:17] (step=0001975) Train Loss: 5.6651, Train Steps/Sec: 1.17 +[2025-04-23 13:02:39] (step=0002000) Train Loss: 5.6799, Train Steps/Sec: 1.12 +[2025-04-23 13:02:39] text_config is None. Initializing the text config with default values (`OPTConfig`). +[2025-04-23 13:07:44] Finish Eval in 2000 steps... +[2025-04-23 13:08:03] Saved checkpoint to checkpoint/Afteri2i_100kckpt_LLAVAT5_empty_Fixed_just_segment_pretrain_stage3_dreambench_recap_400k_t2i_400k_flux_200k_midjourney_150k_recovery_150k_3e4_epoch2_half_rotated/005-GPT-XL/checkpoints/0002000.pt +[2025-04-23 13:08:24] (step=0002025) Train Loss: 5.7603, Train Steps/Sec: 0.07 +[2025-04-23 13:08:45] (step=0002050) Train Loss: 5.6321, Train Steps/Sec: 1.18 +[2025-04-23 13:09:07] (step=0002075) Train Loss: 5.6664, Train Steps/Sec: 1.17 +[2025-04-23 13:09:28] (step=0002100) Train Loss: 5.6938, Train Steps/Sec: 1.17 +[2025-04-23 13:09:50] (step=0002125) Train Loss: 5.6041, Train Steps/Sec: 1.17 +[2025-04-23 13:10:11] (step=0002150) Train Loss: 5.7848, Train Steps/Sec: 1.17 +[2025-04-23 13:10:32] (step=0002175) Train Loss: 5.5715, Train Steps/Sec: 1.17 +[2025-04-23 13:10:54] (step=0002200) Train Loss: 5.7407, Train Steps/Sec: 1.17 +[2025-04-23 13:11:15] (step=0002225) Train Loss: 5.6500, Train Steps/Sec: 1.17 +[2025-04-23 13:11:37] (step=0002250) Train Loss: 5.7571, Train Steps/Sec: 1.17 +[2025-04-23 13:11:58] (step=0002275) Train Loss: 5.7398, Train Steps/Sec: 1.17 +[2025-04-23 13:12:19] (step=0002300) Train Loss: 5.6682, Train Steps/Sec: 1.17 +[2025-04-23 13:12:41] (step=0002325) Train Loss: 5.6743, Train Steps/Sec: 1.17 +[2025-04-23 13:13:02] (step=0002350) Train Loss: 5.7383, Train Steps/Sec: 1.17 +[2025-04-23 13:13:23] (step=0002375) Train Loss: 5.6478, Train Steps/Sec: 1.17 +[2025-04-23 13:13:45] (step=0002400) Train Loss: 5.6100, Train Steps/Sec: 1.17 +[2025-04-23 13:14:06] (step=0002425) Train Loss: 5.7386, Train Steps/Sec: 1.18 +[2025-04-23 13:14:28] (step=0002450) Train Loss: 5.7221, Train Steps/Sec: 1.17 +[2025-04-23 13:14:49] (step=0002475) Train Loss: 5.7493, Train Steps/Sec: 1.17 +[2025-04-23 13:15:10] (step=0002500) Train Loss: 5.7347, Train Steps/Sec: 1.16 +[2025-04-23 13:15:32] (step=0002525) Train Loss: 5.6865, Train Steps/Sec: 1.17 +[2025-04-23 13:15:54] (step=0002550) Train Loss: 5.7746, Train Steps/Sec: 1.12 +[2025-04-23 13:16:17] (step=0002575) Train Loss: 5.6799, Train Steps/Sec: 1.08 +[2025-04-23 13:16:39] (step=0002600) Train Loss: 5.6503, Train Steps/Sec: 1.13 +[2025-04-23 13:17:02] (step=0002625) Train Loss: 5.6986, Train Steps/Sec: 1.08 +[2025-04-23 13:17:24] (step=0002650) Train Loss: 5.5998, Train Steps/Sec: 1.13 +[2025-04-23 13:17:47] (step=0002675) Train Loss: 5.6865, Train Steps/Sec: 1.12 +[2025-04-23 13:18:08] (step=0002700) Train Loss: 5.6617, Train Steps/Sec: 1.17 +[2025-04-23 13:18:29] (step=0002725) Train Loss: 5.6786, Train Steps/Sec: 1.17 +[2025-04-23 13:18:51] (step=0002750) Train Loss: 5.7432, Train Steps/Sec: 1.17 +[2025-04-23 13:19:12] (step=0002775) Train Loss: 5.6411, Train Steps/Sec: 1.17 +[2025-04-23 13:19:33] (step=0002800) Train Loss: 5.6815, Train Steps/Sec: 1.17 +[2025-04-23 13:19:55] (step=0002825) Train Loss: 5.7673, Train Steps/Sec: 1.17 +[2025-04-23 13:20:16] (step=0002850) Train Loss: 5.6858, Train Steps/Sec: 1.17 +[2025-04-23 13:20:37] (step=0002875) Train Loss: 5.7520, Train Steps/Sec: 1.17 +[2025-04-23 13:20:59] (step=0002900) Train Loss: 5.6676, Train Steps/Sec: 1.17 +[2025-04-23 13:21:20] (step=0002925) Train Loss: 5.6467, Train Steps/Sec: 1.17 +[2025-04-23 13:21:41] (step=0002950) Train Loss: 5.5843, Train Steps/Sec: 1.18 +[2025-04-23 13:22:03] (step=0002975) Train Loss: 5.6466, Train Steps/Sec: 1.17 +[2025-04-23 13:22:24] (step=0003000) Train Loss: 5.6850, Train Steps/Sec: 1.17 +[2025-04-23 13:22:46] (step=0003025) Train Loss: 5.6788, Train Steps/Sec: 1.17 +[2025-04-23 13:23:07] (step=0003050) Train Loss: 5.7179, Train Steps/Sec: 1.17 +[2025-04-23 13:23:28] (step=0003075) Train Loss: 5.6049, Train Steps/Sec: 1.17 +[2025-04-23 13:23:50] (step=0003100) Train Loss: 5.8088, Train Steps/Sec: 1.17 +[2025-04-23 13:24:11] (step=0003125) Train Loss: 5.7588, Train Steps/Sec: 1.17 +[2025-04-23 13:24:32] (step=0003150) Train Loss: 5.7036, Train Steps/Sec: 1.17 +[2025-04-23 13:24:54] (step=0003175) Train Loss: 5.6707, Train Steps/Sec: 1.17 +[2025-04-23 13:25:16] (step=0003200) Train Loss: 5.6567, Train Steps/Sec: 1.12 +[2025-04-23 13:25:38] (step=0003225) Train Loss: 5.7040, Train Steps/Sec: 1.13 +[2025-04-23 13:26:00] (step=0003250) Train Loss: 5.6184, Train Steps/Sec: 1.13 +[2025-04-23 13:26:22] (step=0003275) Train Loss: 5.6360, Train Steps/Sec: 1.13 +[2025-04-23 13:26:45] (step=0003300) Train Loss: 5.5691, Train Steps/Sec: 1.12 +[2025-04-23 13:27:07] (step=0003325) Train Loss: 5.5636, Train Steps/Sec: 1.13 +[2025-04-23 13:27:30] (step=0003350) Train Loss: 5.7092, Train Steps/Sec: 1.07 +[2025-04-23 13:27:52] (step=0003375) Train Loss: 5.6541, Train Steps/Sec: 1.15 +[2025-04-23 13:28:13] (step=0003400) Train Loss: 5.6858, Train Steps/Sec: 1.17 +[2025-04-23 13:28:34] (step=0003425) Train Loss: 5.6972, Train Steps/Sec: 1.17 +[2025-04-23 13:28:56] (step=0003450) Train Loss: 5.7138, Train Steps/Sec: 1.17 +[2025-04-23 13:29:17] (step=0003475) Train Loss: 5.6577, Train Steps/Sec: 1.17 +[2025-04-23 13:29:39] (step=0003500) Train Loss: 5.6438, Train Steps/Sec: 1.17 +[2025-04-23 13:30:00] (step=0003525) Train Loss: 5.6804, Train Steps/Sec: 1.17 +[2025-04-23 13:30:21] (step=0003550) Train Loss: 5.7214, Train Steps/Sec: 1.18 +[2025-04-23 13:30:42] (step=0003575) Train Loss: 5.7003, Train Steps/Sec: 1.17 +[2025-04-23 13:31:04] (step=0003600) Train Loss: 5.6853, Train Steps/Sec: 1.17 +[2025-04-23 13:31:25] (step=0003625) Train Loss: 5.6664, Train Steps/Sec: 1.17 +[2025-04-23 13:31:47] (step=0003650) Train Loss: 5.5661, Train Steps/Sec: 1.16 +[2025-04-23 13:32:08] (step=0003675) Train Loss: 5.6536, Train Steps/Sec: 1.17 +[2025-04-23 13:32:30] (step=0003700) Train Loss: 5.7186, Train Steps/Sec: 1.16 +[2025-04-23 13:32:51] (step=0003725) Train Loss: 5.6504, Train Steps/Sec: 1.17 +[2025-04-23 13:33:12] (step=0003750) Train Loss: 5.6724, Train Steps/Sec: 1.17 +[2025-04-23 13:33:34] (step=0003775) Train Loss: 5.6309, Train Steps/Sec: 1.17 +[2025-04-23 13:33:55] (step=0003800) Train Loss: 5.6176, Train Steps/Sec: 1.17 +[2025-04-23 13:34:16] (step=0003825) Train Loss: 5.7505, Train Steps/Sec: 1.17 +[2025-04-23 13:34:38] (step=0003850) Train Loss: 5.6001, Train Steps/Sec: 1.17 +[2025-04-23 13:35:00] (step=0003875) Train Loss: 5.6056, Train Steps/Sec: 1.13 +[2025-04-23 13:35:23] (step=0003900) Train Loss: 5.6593, Train Steps/Sec: 1.08 +[2025-04-23 13:35:46] (step=0003925) Train Loss: 5.6551, Train Steps/Sec: 1.11 +[2025-04-23 13:36:07] (step=0003950) Train Loss: 5.7095, Train Steps/Sec: 1.17 +[2025-04-23 13:36:30] (step=0003975) Train Loss: 5.6192, Train Steps/Sec: 1.08 +[2025-04-23 13:36:51] (step=0004000) Train Loss: 5.6475, Train Steps/Sec: 1.17 +[2025-04-23 13:36:52] text_config is None. Initializing the text config with default values (`OPTConfig`). +[2025-04-23 13:42:01] Finish Eval in 4000 steps... +[2025-04-23 13:42:19] Saved checkpoint to checkpoint/Afteri2i_100kckpt_LLAVAT5_empty_Fixed_just_segment_pretrain_stage3_dreambench_recap_400k_t2i_400k_flux_200k_midjourney_150k_recovery_150k_3e4_epoch2_half_rotated/005-GPT-XL/checkpoints/0004000.pt +[2025-04-23 13:42:21] Removed old checkpoint: checkpoint/Afteri2i_100kckpt_LLAVAT5_empty_Fixed_just_segment_pretrain_stage3_dreambench_recap_400k_t2i_400k_flux_200k_midjourney_150k_recovery_150k_3e4_epoch2_half_rotated/005-GPT-XL/checkpoints/0002000.pt +[2025-04-23 13:42:42] (step=0004025) Train Loss: 5.6919, Train Steps/Sec: 0.07 +[2025-04-23 13:43:03] (step=0004050) Train Loss: 5.6571, Train Steps/Sec: 1.17 +[2025-04-23 13:43:27] (step=0004075) Train Loss: 5.6853, Train Steps/Sec: 1.06 +[2025-04-23 13:43:49] (step=0004100) Train Loss: 5.5925, Train Steps/Sec: 1.17 +[2025-04-23 13:44:10] (step=0004125) Train Loss: 5.5697, Train Steps/Sec: 1.17 +[2025-04-23 13:44:31] (step=0004150) Train Loss: 5.6736, Train Steps/Sec: 1.17 +[2025-04-23 13:44:53] (step=0004175) Train Loss: 5.6693, Train Steps/Sec: 1.17 +[2025-04-23 13:45:14] (step=0004200) Train Loss: 5.6101, Train Steps/Sec: 1.16 +[2025-04-23 13:45:35] (step=0004225) Train Loss: 5.6586, Train Steps/Sec: 1.17 +[2025-04-23 13:45:57] (step=0004250) Train Loss: 5.5988, Train Steps/Sec: 1.17 +[2025-04-23 13:46:18] (step=0004275) Train Loss: 5.6851, Train Steps/Sec: 1.17 +[2025-04-23 13:46:40] (step=0004300) Train Loss: 5.6974, Train Steps/Sec: 1.17 +[2025-04-23 13:47:01] (step=0004325) Train Loss: 5.6270, Train Steps/Sec: 1.17 +[2025-04-23 13:47:22] (step=0004350) Train Loss: 5.6210, Train Steps/Sec: 1.17 +[2025-04-23 13:47:43] (step=0004375) Train Loss: 5.6534, Train Steps/Sec: 1.17 +[2025-04-23 13:48:05] (step=0004400) Train Loss: 5.5567, Train Steps/Sec: 1.17 +[2025-04-23 13:48:26] (step=0004425) Train Loss: 5.6524, Train Steps/Sec: 1.17 +[2025-04-23 13:48:48] (step=0004450) Train Loss: 5.6659, Train Steps/Sec: 1.17 +[2025-04-23 13:49:09] (step=0004475) Train Loss: 5.6258, Train Steps/Sec: 1.17 +[2025-04-23 13:49:30] (step=0004500) Train Loss: 5.5931, Train Steps/Sec: 1.17 +[2025-04-23 13:49:52] (step=0004525) Train Loss: 5.7624, Train Steps/Sec: 1.13 +[2025-04-23 13:50:15] (step=0004550) Train Loss: 5.6551, Train Steps/Sec: 1.13 +[2025-04-23 13:50:38] (step=0004575) Train Loss: 5.5787, Train Steps/Sec: 1.09 +[2025-04-23 13:50:59] (step=0004600) Train Loss: 5.6529, Train Steps/Sec: 1.17 +[2025-04-23 13:51:20] (step=0004625) Train Loss: 5.6690, Train Steps/Sec: 1.17 +[2025-04-23 13:51:42] (step=0004650) Train Loss: 5.6965, Train Steps/Sec: 1.13 +[2025-04-23 13:52:05] (step=0004675) Train Loss: 5.6870, Train Steps/Sec: 1.13 +[2025-04-23 13:52:26] (step=0004700) Train Loss: 5.5647, Train Steps/Sec: 1.16 +[2025-04-23 13:52:48] (step=0004725) Train Loss: 5.6140, Train Steps/Sec: 1.17 +[2025-04-23 13:53:09] (step=0004750) Train Loss: 5.6157, Train Steps/Sec: 1.17 +[2025-04-23 13:53:30] (step=0004775) Train Loss: 5.6005, Train Steps/Sec: 1.17 +[2025-04-23 13:53:53] (step=0004800) Train Loss: 5.5967, Train Steps/Sec: 1.07 +[2025-04-23 13:54:15] (step=0004825) Train Loss: 5.5716, Train Steps/Sec: 1.17 +[2025-04-23 13:54:36] (step=0004850) Train Loss: 5.5744, Train Steps/Sec: 1.17 +[2025-04-23 13:54:57] (step=0004875) Train Loss: 5.6118, Train Steps/Sec: 1.17 +[2025-04-23 13:55:19] (step=0004900) Train Loss: 5.5919, Train Steps/Sec: 1.17 +[2025-04-23 13:55:40] (step=0004925) Train Loss: 5.6854, Train Steps/Sec: 1.17 +[2025-04-23 13:56:01] (step=0004950) Train Loss: 5.6321, Train Steps/Sec: 1.17 +[2025-04-23 13:56:23] (step=0004975) Train Loss: 5.6389, Train Steps/Sec: 1.17 +[2025-04-23 13:56:44] (step=0005000) Train Loss: 5.7056, Train Steps/Sec: 1.17 +[2025-04-23 13:57:06] (step=0005025) Train Loss: 5.5637, Train Steps/Sec: 1.17 +[2025-04-23 13:57:27] (step=0005050) Train Loss: 5.5964, Train Steps/Sec: 1.17 +[2025-04-23 13:57:48] (step=0005075) Train Loss: 5.6212, Train Steps/Sec: 1.17 +[2025-04-23 13:58:10] (step=0005100) Train Loss: 5.6681, Train Steps/Sec: 1.17 +[2025-04-23 13:58:31] (step=0005125) Train Loss: 5.6060, Train Steps/Sec: 1.17 +[2025-04-23 13:58:53] (step=0005150) Train Loss: 5.5869, Train Steps/Sec: 1.17 +[2025-04-23 13:59:15] (step=0005175) Train Loss: 5.6572, Train Steps/Sec: 1.13 +[2025-04-23 13:59:37] (step=0005200) Train Loss: 5.6732, Train Steps/Sec: 1.12 +[2025-04-23 13:59:59] (step=0005225) Train Loss: 5.5777, Train Steps/Sec: 1.13 +[2025-04-23 14:00:21] (step=0005250) Train Loss: 5.6421, Train Steps/Sec: 1.13 +[2025-04-23 14:00:43] (step=0005275) Train Loss: 5.6117, Train Steps/Sec: 1.17 +[2025-04-23 14:01:05] (step=0005300) Train Loss: 5.5680, Train Steps/Sec: 1.12 +[2025-04-23 14:01:27] (step=0005325) Train Loss: 5.6310, Train Steps/Sec: 1.13 +[2025-04-23 14:01:48] (step=0005350) Train Loss: 5.6257, Train Steps/Sec: 1.17 +[2025-04-23 14:02:10] (step=0005375) Train Loss: 5.6289, Train Steps/Sec: 1.17 +[2025-04-23 14:02:31] (step=0005400) Train Loss: 5.6249, Train Steps/Sec: 1.17 +[2025-04-23 14:02:53] (step=0005425) Train Loss: 5.6360, Train Steps/Sec: 1.17 +[2025-04-23 14:03:14] (step=0005450) Train Loss: 5.7072, Train Steps/Sec: 1.17 +[2025-04-23 14:03:35] (step=0005475) Train Loss: 5.5918, Train Steps/Sec: 1.17 +[2025-04-23 14:03:57] (step=0005500) Train Loss: 5.6338, Train Steps/Sec: 1.17 +[2025-04-23 14:04:20] (step=0005525) Train Loss: 5.5733, Train Steps/Sec: 1.07 +[2025-04-23 14:04:41] (step=0005550) Train Loss: 5.6391, Train Steps/Sec: 1.17 +[2025-04-23 14:05:02] (step=0005575) Train Loss: 5.5895, Train Steps/Sec: 1.17 +[2025-04-23 14:05:24] (step=0005600) Train Loss: 5.6949, Train Steps/Sec: 1.17 +[2025-04-23 14:05:45] (step=0005625) Train Loss: 5.5547, Train Steps/Sec: 1.17 +[2025-04-23 14:06:07] (step=0005650) Train Loss: 5.6769, Train Steps/Sec: 1.17 +[2025-04-23 14:06:28] (step=0005675) Train Loss: 5.6761, Train Steps/Sec: 1.17 +[2025-04-23 14:06:49] (step=0005700) Train Loss: 5.6436, Train Steps/Sec: 1.17 +[2025-04-23 14:07:11] (step=0005725) Train Loss: 5.6285, Train Steps/Sec: 1.17 +[2025-04-23 14:07:32] (step=0005750) Train Loss: 5.5609, Train Steps/Sec: 1.17 +[2025-04-23 14:07:53] (step=0005775) Train Loss: 5.6778, Train Steps/Sec: 1.17 +[2025-04-23 14:08:15] (step=0005800) Train Loss: 5.5576, Train Steps/Sec: 1.17 +[2025-04-23 14:08:36] (step=0005825) Train Loss: 5.6316, Train Steps/Sec: 1.17 +[2025-04-23 14:08:58] (step=0005850) Train Loss: 5.6223, Train Steps/Sec: 1.13 +[2025-04-23 14:09:21] (step=0005875) Train Loss: 5.5707, Train Steps/Sec: 1.09 +[2025-04-23 14:09:43] (step=0005900) Train Loss: 5.5906, Train Steps/Sec: 1.13 +[2025-04-23 14:10:05] (step=0005925) Train Loss: 5.5430, Train Steps/Sec: 1.17 +[2025-04-23 14:10:27] (step=0005950) Train Loss: 5.6428, Train Steps/Sec: 1.12 +[2025-04-23 14:10:49] (step=0005975) Train Loss: 5.6063, Train Steps/Sec: 1.13 +[2025-04-23 14:11:11] (step=0006000) Train Loss: 5.6270, Train Steps/Sec: 1.17 +[2025-04-23 14:11:11] text_config is None. Initializing the text config with default values (`OPTConfig`). +[2025-04-23 14:16:18] Finish Eval in 6000 steps... +[2025-04-23 14:16:37] Saved checkpoint to checkpoint/Afteri2i_100kckpt_LLAVAT5_empty_Fixed_just_segment_pretrain_stage3_dreambench_recap_400k_t2i_400k_flux_200k_midjourney_150k_recovery_150k_3e4_epoch2_half_rotated/005-GPT-XL/checkpoints/0006000.pt +[2025-04-23 14:16:39] Removed old checkpoint: checkpoint/Afteri2i_100kckpt_LLAVAT5_empty_Fixed_just_segment_pretrain_stage3_dreambench_recap_400k_t2i_400k_flux_200k_midjourney_150k_recovery_150k_3e4_epoch2_half_rotated/005-GPT-XL/checkpoints/0004000.pt +[2025-04-23 14:17:00] (step=0006025) Train Loss: 5.6388, Train Steps/Sec: 0.07 +[2025-04-23 14:17:22] (step=0006050) Train Loss: 5.7000, Train Steps/Sec: 1.17 +[2025-04-23 14:17:43] (step=0006075) Train Loss: 5.5189, Train Steps/Sec: 1.17 +[2025-04-23 14:18:04] (step=0006100) Train Loss: 5.6494, Train Steps/Sec: 1.17 +[2025-04-23 14:18:26] (step=0006125) Train Loss: 5.5979, Train Steps/Sec: 1.17 +[2025-04-23 14:18:47] (step=0006150) Train Loss: 5.6135, Train Steps/Sec: 1.17 +[2025-04-23 14:19:08] (step=0006175) Train Loss: 5.6027, Train Steps/Sec: 1.17 +[2025-04-23 14:19:30] (step=0006200) Train Loss: 5.6541, Train Steps/Sec: 1.17 +[2025-04-23 14:19:51] (step=0006225) Train Loss: 5.6020, Train Steps/Sec: 1.17 +[2025-04-23 14:20:14] (step=0006250) Train Loss: 5.6513, Train Steps/Sec: 1.08 +[2025-04-23 14:20:35] (step=0006275) Train Loss: 5.5224, Train Steps/Sec: 1.17 +[2025-04-23 14:20:57] (step=0006300) Train Loss: 5.5502, Train Steps/Sec: 1.17 +[2025-04-23 14:21:18] (step=0006325) Train Loss: 5.6070, Train Steps/Sec: 1.17 +[2025-04-23 14:21:40] (step=0006350) Train Loss: 5.5896, Train Steps/Sec: 1.16 +[2025-04-23 14:22:01] (step=0006375) Train Loss: 5.6756, Train Steps/Sec: 1.16 +[2025-04-23 14:22:23] (step=0006400) Train Loss: 5.5540, Train Steps/Sec: 1.16 +[2025-04-23 14:22:44] (step=0006425) Train Loss: 5.6123, Train Steps/Sec: 1.17 +[2025-04-23 14:23:05] (step=0006450) Train Loss: 5.5652, Train Steps/Sec: 1.17 +[2025-04-23 14:23:27] (step=0006475) Train Loss: 5.5895, Train Steps/Sec: 1.17 +[2025-04-23 14:23:49] (step=0006500) Train Loss: 5.5579, Train Steps/Sec: 1.12 +[2025-04-23 14:24:11] (step=0006525) Train Loss: 5.6102, Train Steps/Sec: 1.13 +[2025-04-23 14:24:34] (step=0006550) Train Loss: 5.6376, Train Steps/Sec: 1.09 +[2025-04-23 14:24:56] (step=0006575) Train Loss: 5.6739, Train Steps/Sec: 1.17 +[2025-04-23 14:25:17] (step=0006600) Train Loss: 5.5637, Train Steps/Sec: 1.16 +[2025-04-23 14:25:40] (step=0006625) Train Loss: 5.5159, Train Steps/Sec: 1.12 +[2025-04-23 14:26:02] (step=0006650) Train Loss: 5.5283, Train Steps/Sec: 1.12 +[2025-04-23 14:26:23] (step=0006675) Train Loss: 5.6168, Train Steps/Sec: 1.17 +[2025-04-23 14:26:44] (step=0006700) Train Loss: 5.5888, Train Steps/Sec: 1.17 +[2025-04-23 14:27:06] (step=0006725) Train Loss: 5.5776, Train Steps/Sec: 1.17 +[2025-04-23 14:27:27] (step=0006750) Train Loss: 5.5456, Train Steps/Sec: 1.17 +[2025-04-23 14:27:48] (step=0006775) Train Loss: 5.6196, Train Steps/Sec: 1.17 +[2025-04-23 14:28:10] (step=0006800) Train Loss: 5.6310, Train Steps/Sec: 1.17 +[2025-04-23 14:28:31] (step=0006825) Train Loss: 5.6441, Train Steps/Sec: 1.17 +[2025-04-23 14:28:52] (step=0006850) Train Loss: 5.6383, Train Steps/Sec: 1.17 +[2025-04-23 14:29:14] (step=0006875) Train Loss: 5.5304, Train Steps/Sec: 1.17 +[2025-04-23 14:29:35] (step=0006900) Train Loss: 5.6645, Train Steps/Sec: 1.17 +[2025-04-23 14:29:57] (step=0006925) Train Loss: 5.5917, Train Steps/Sec: 1.17 +[2025-04-23 14:30:18] (step=0006950) Train Loss: 5.5810, Train Steps/Sec: 1.17 +[2025-04-23 14:30:41] (step=0006975) Train Loss: 5.5698, Train Steps/Sec: 1.08 +[2025-04-23 14:31:03] (step=0007000) Train Loss: 5.6576, Train Steps/Sec: 1.17 +[2025-04-23 14:31:24] (step=0007025) Train Loss: 5.5626, Train Steps/Sec: 1.18 +[2025-04-23 14:31:45] (step=0007050) Train Loss: 5.5361, Train Steps/Sec: 1.16 +[2025-04-23 14:32:07] (step=0007075) Train Loss: 5.4693, Train Steps/Sec: 1.17 +[2025-04-23 14:32:28] (step=0007100) Train Loss: 5.6563, Train Steps/Sec: 1.17 +[2025-04-23 14:32:49] (step=0007125) Train Loss: 5.6235, Train Steps/Sec: 1.17 +[2025-04-23 14:33:12] (step=0007150) Train Loss: 5.6044, Train Steps/Sec: 1.13 +[2025-04-23 14:33:34] (step=0007175) Train Loss: 5.5612, Train Steps/Sec: 1.13 +[2025-04-23 14:33:56] (step=0007200) Train Loss: 5.5341, Train Steps/Sec: 1.12 +[2025-04-23 14:34:18] (step=0007225) Train Loss: 5.5611, Train Steps/Sec: 1.13 +[2025-04-23 14:34:39] (step=0007250) Train Loss: 5.6122, Train Steps/Sec: 1.17 +[2025-04-23 14:35:02] (step=0007275) Train Loss: 5.5743, Train Steps/Sec: 1.13 +[2025-04-23 14:35:23] (step=0007300) Train Loss: 5.6194, Train Steps/Sec: 1.17 +[2025-04-23 14:35:45] (step=0007325) Train Loss: 5.6055, Train Steps/Sec: 1.13 +[2025-04-23 14:36:06] (step=0007350) Train Loss: 5.5483, Train Steps/Sec: 1.17 +[2025-04-23 14:36:28] (step=0007375) Train Loss: 5.6296, Train Steps/Sec: 1.17 +[2025-04-23 14:36:49] (step=0007400) Train Loss: 5.5149, Train Steps/Sec: 1.17 +[2025-04-23 14:37:10] (step=0007425) Train Loss: 5.6453, Train Steps/Sec: 1.17 +[2025-04-23 14:37:32] (step=0007450) Train Loss: 5.5708, Train Steps/Sec: 1.17 +[2025-04-23 14:37:53] (step=0007475) Train Loss: 5.6707, Train Steps/Sec: 1.17 +[2025-04-23 14:38:14] (step=0007500) Train Loss: 5.6034, Train Steps/Sec: 1.17 +[2025-04-23 14:38:36] (step=0007525) Train Loss: 5.5281, Train Steps/Sec: 1.18 +[2025-04-23 14:38:57] (step=0007550) Train Loss: 5.5392, Train Steps/Sec: 1.18 +[2025-04-23 14:39:18] (step=0007575) Train Loss: 5.6421, Train Steps/Sec: 1.17 +[2025-04-23 14:39:40] (step=0007600) Train Loss: 5.5481, Train Steps/Sec: 1.17 +[2025-04-23 14:40:01] (step=0007625) Train Loss: 5.6720, Train Steps/Sec: 1.17 +[2025-04-23 14:40:22] (step=0007650) Train Loss: 5.5935, Train Steps/Sec: 1.18 +[2025-04-23 14:40:44] (step=0007675) Train Loss: 5.5662, Train Steps/Sec: 1.17 +[2025-04-23 14:41:07] (step=0007700) Train Loss: 5.4702, Train Steps/Sec: 1.08 +[2025-04-23 14:41:28] (step=0007725) Train Loss: 5.6161, Train Steps/Sec: 1.17 +[2025-04-23 14:41:49] (step=0007750) Train Loss: 5.5227, Train Steps/Sec: 1.18 +[2025-04-23 14:42:11] (step=0007775) Train Loss: 5.5485, Train Steps/Sec: 1.17 +[2025-04-23 14:42:32] (step=0007800) Train Loss: 5.6116, Train Steps/Sec: 1.17 +[2025-04-23 14:42:55] (step=0007825) Train Loss: 5.5707, Train Steps/Sec: 1.09 +[2025-04-23 14:43:17] (step=0007850) Train Loss: 5.6047, Train Steps/Sec: 1.13 +[2025-04-23 14:43:39] (step=0007875) Train Loss: 5.5710, Train Steps/Sec: 1.13 +[2025-04-23 14:44:01] (step=0007900) Train Loss: 5.6073, Train Steps/Sec: 1.17 +[2025-04-23 14:44:22] (step=0007925) Train Loss: 5.5761, Train Steps/Sec: 1.17 +[2025-04-23 14:44:44] (step=0007950) Train Loss: 5.6180, Train Steps/Sec: 1.13 +[2025-04-23 14:45:06] (step=0007975) Train Loss: 5.5812, Train Steps/Sec: 1.13 +[2025-04-23 14:45:28] (step=0008000) Train Loss: 5.6081, Train Steps/Sec: 1.17 +[2025-04-23 14:45:28] text_config is None. Initializing the text config with default values (`OPTConfig`). +[2025-04-23 14:50:35] Finish Eval in 8000 steps... +[2025-04-23 14:50:54] Saved checkpoint to checkpoint/Afteri2i_100kckpt_LLAVAT5_empty_Fixed_just_segment_pretrain_stage3_dreambench_recap_400k_t2i_400k_flux_200k_midjourney_150k_recovery_150k_3e4_epoch2_half_rotated/005-GPT-XL/checkpoints/0008000.pt +[2025-04-23 14:50:56] Removed old checkpoint: checkpoint/Afteri2i_100kckpt_LLAVAT5_empty_Fixed_just_segment_pretrain_stage3_dreambench_recap_400k_t2i_400k_flux_200k_midjourney_150k_recovery_150k_3e4_epoch2_half_rotated/005-GPT-XL/checkpoints/0006000.pt +[2025-04-23 14:51:18] (step=0008025) Train Loss: 5.5404, Train Steps/Sec: 0.07 +[2025-04-23 14:51:39] (step=0008050) Train Loss: 5.7199, Train Steps/Sec: 1.17 +[2025-04-23 14:52:00] (step=0008075) Train Loss: 5.6386, Train Steps/Sec: 1.18 +[2025-04-23 14:52:22] (step=0008100) Train Loss: 5.7066, Train Steps/Sec: 1.17 +[2025-04-23 14:52:43] (step=0008125) Train Loss: 5.5921, Train Steps/Sec: 1.17 +[2025-04-23 14:53:04] (step=0008150) Train Loss: 5.5478, Train Steps/Sec: 1.17 +[2025-04-23 14:53:25] (step=0008175) Train Loss: 5.5166, Train Steps/Sec: 1.18 +[2025-04-23 14:53:47] (step=0008200) Train Loss: 5.5654, Train Steps/Sec: 1.17 +[2025-04-23 14:54:08] (step=0008225) Train Loss: 5.5901, Train Steps/Sec: 1.18 +[2025-04-23 14:54:29] (step=0008250) Train Loss: 5.6473, Train Steps/Sec: 1.18 +[2025-04-23 14:54:51] (step=0008275) Train Loss: 5.5355, Train Steps/Sec: 1.17 +[2025-04-23 14:55:12] (step=0008300) Train Loss: 5.6486, Train Steps/Sec: 1.17 +[2025-04-23 14:55:33] (step=0008325) Train Loss: 5.6025, Train Steps/Sec: 1.18 +[2025-04-23 14:55:55] (step=0008350) Train Loss: 5.6015, Train Steps/Sec: 1.17 +[2025-04-23 14:56:16] (step=0008375) Train Loss: 5.5669, Train Steps/Sec: 1.18 +[2025-04-23 14:56:37] (step=0008400) Train Loss: 5.5281, Train Steps/Sec: 1.17 +[2025-04-23 14:57:00] (step=0008425) Train Loss: 5.5368, Train Steps/Sec: 1.08 +[2025-04-23 14:57:22] (step=0008450) Train Loss: 5.6245, Train Steps/Sec: 1.17 +[2025-04-23 14:57:44] (step=0008475) Train Loss: 5.6405, Train Steps/Sec: 1.13 +[2025-04-23 14:58:06] (step=0008500) Train Loss: 5.5176, Train Steps/Sec: 1.13 +[2025-04-23 14:58:28] (step=0008525) Train Loss: 5.6100, Train Steps/Sec: 1.13 +[2025-04-23 14:58:50] (step=0008550) Train Loss: 5.6509, Train Steps/Sec: 1.14 +[2025-04-23 14:59:12] (step=0008575) Train Loss: 5.5722, Train Steps/Sec: 1.18 +[2025-04-23 14:59:34] (step=0008600) Train Loss: 5.5628, Train Steps/Sec: 1.12 +[2025-04-23 14:59:55] (step=0008625) Train Loss: 5.5778, Train Steps/Sec: 1.17 +[2025-04-23 15:00:16] (step=0008650) Train Loss: 5.5785, Train Steps/Sec: 1.17 +[2025-04-23 15:00:39] (step=0008675) Train Loss: 5.5469, Train Steps/Sec: 1.12 +[2025-04-23 15:01:00] (step=0008700) Train Loss: 5.5830, Train Steps/Sec: 1.17 +[2025-04-23 15:01:21] (step=0008725) Train Loss: 5.5970, Train Steps/Sec: 1.17 +[2025-04-23 15:01:43] (step=0008750) Train Loss: 5.6109, Train Steps/Sec: 1.18 +[2025-04-23 15:02:04] (step=0008775) Train Loss: 5.6733, Train Steps/Sec: 1.17 +[2025-04-23 15:02:25] (step=0008800) Train Loss: 5.5923, Train Steps/Sec: 1.17 +[2025-04-23 15:02:47] (step=0008825) Train Loss: 5.5614, Train Steps/Sec: 1.17 +[2025-04-23 15:03:08] (step=0008850) Train Loss: 5.6193, Train Steps/Sec: 1.17 +[2025-04-23 15:03:29] (step=0008875) Train Loss: 5.5727, Train Steps/Sec: 1.18 +[2025-04-23 15:03:51] (step=0008900) Train Loss: 5.6106, Train Steps/Sec: 1.17 +[2025-04-23 15:04:12] (step=0008925) Train Loss: 5.7130, Train Steps/Sec: 1.17 +[2025-04-23 15:04:33] (step=0008950) Train Loss: 5.4615, Train Steps/Sec: 1.17 +[2025-04-23 15:04:55] (step=0008975) Train Loss: 5.5942, Train Steps/Sec: 1.18 +[2025-04-23 15:05:16] (step=0009000) Train Loss: 5.5529, Train Steps/Sec: 1.17 +[2025-04-23 15:05:37] (step=0009025) Train Loss: 5.5656, Train Steps/Sec: 1.17 +[2025-04-23 15:05:59] (step=0009050) Train Loss: 5.4979, Train Steps/Sec: 1.17 +[2025-04-23 15:06:20] (step=0009075) Train Loss: 5.6053, Train Steps/Sec: 1.18 +[2025-04-23 15:06:41] (step=0009100) Train Loss: 5.5881, Train Steps/Sec: 1.17 +[2025-04-23 15:07:05] (step=0009125) Train Loss: 5.5723, Train Steps/Sec: 1.08 +[2025-04-23 15:07:28] (step=0009150) Train Loss: 5.6203, Train Steps/Sec: 1.09 +[2025-04-23 15:07:50] (step=0009175) Train Loss: 5.5956, Train Steps/Sec: 1.13 +[2025-04-23 15:08:12] (step=0009200) Train Loss: 5.6060, Train Steps/Sec: 1.13 +[2025-04-23 15:08:33] (step=0009225) Train Loss: 5.6353, Train Steps/Sec: 1.17 +[2025-04-23 15:08:56] (step=0009250) Train Loss: 5.4870, Train Steps/Sec: 1.10 +[2025-04-23 15:09:17] (step=0009275) Train Loss: 5.6289, Train Steps/Sec: 1.17 +[2025-04-23 15:09:39] (step=0009300) Train Loss: 5.6067, Train Steps/Sec: 1.16 +[2025-04-23 15:10:01] (step=0009325) Train Loss: 5.5867, Train Steps/Sec: 1.11 +[2025-04-23 15:10:22] (step=0009350) Train Loss: 5.5610, Train Steps/Sec: 1.17 +[2025-04-23 15:10:44] (step=0009375) Train Loss: 5.5917, Train Steps/Sec: 1.17 +[2025-04-23 15:11:05] (step=0009400) Train Loss: 5.5593, Train Steps/Sec: 1.17 +[2025-04-23 15:11:26] (step=0009425) Train Loss: 5.5825, Train Steps/Sec: 1.17 +[2025-04-23 15:11:48] (step=0009450) Train Loss: 5.5783, Train Steps/Sec: 1.18 +[2025-04-23 15:12:09] (step=0009475) Train Loss: 5.5665, Train Steps/Sec: 1.18 +[2025-04-23 15:12:30] (step=0009500) Train Loss: 5.6472, Train Steps/Sec: 1.17 +[2025-04-23 15:12:52] (step=0009525) Train Loss: 5.5373, Train Steps/Sec: 1.18 +[2025-04-23 15:13:13] (step=0009550) Train Loss: 5.5705, Train Steps/Sec: 1.17 +[2025-04-23 15:13:34] (step=0009575) Train Loss: 5.5873, Train Steps/Sec: 1.17 +[2025-04-23 15:13:56] (step=0009600) Train Loss: 5.5547, Train Steps/Sec: 1.17 +[2025-04-23 15:14:17] (step=0009625) Train Loss: 5.6206, Train Steps/Sec: 1.17 +[2025-04-23 15:14:38] (step=0009650) Train Loss: 5.5514, Train Steps/Sec: 1.18 +[2025-04-23 15:14:59] (step=0009675) Train Loss: 5.5911, Train Steps/Sec: 1.18 +[2025-04-23 15:15:21] (step=0009700) Train Loss: 5.5386, Train Steps/Sec: 1.17 +[2025-04-23 15:15:42] (step=0009725) Train Loss: 5.5181, Train Steps/Sec: 1.18 +[2025-04-23 15:16:03] (step=0009750) Train Loss: 5.5701, Train Steps/Sec: 1.17 +[2025-04-23 15:16:25] (step=0009775) Train Loss: 5.5818, Train Steps/Sec: 1.13 +[2025-04-23 15:16:48] (step=0009800) Train Loss: 5.5788, Train Steps/Sec: 1.13 +[2025-04-23 15:17:09] (step=0009825) Train Loss: 5.5230, Train Steps/Sec: 1.17 +[2025-04-23 15:17:33] (step=0009850) Train Loss: 5.5792, Train Steps/Sec: 1.05 +[2025-04-23 15:17:55] (step=0009875) Train Loss: 5.5720, Train Steps/Sec: 1.13 +[2025-04-23 15:18:16] (step=0009900) Train Loss: 5.5957, Train Steps/Sec: 1.17 +[2025-04-23 15:18:39] (step=0009925) Train Loss: 5.5793, Train Steps/Sec: 1.12 +[2025-04-23 15:19:00] (step=0009950) Train Loss: 5.5788, Train Steps/Sec: 1.17 +[2025-04-23 15:19:22] (step=0009975) Train Loss: 5.5354, Train Steps/Sec: 1.13 +[2025-04-23 15:19:44] (step=0010000) Train Loss: 5.5674, Train Steps/Sec: 1.17 +[2025-04-23 15:19:44] text_config is None. Initializing the text config with default values (`OPTConfig`). +[2025-04-23 15:24:51] Finish Eval in 10000 steps... +[2025-04-23 15:25:10] Saved checkpoint to checkpoint/Afteri2i_100kckpt_LLAVAT5_empty_Fixed_just_segment_pretrain_stage3_dreambench_recap_400k_t2i_400k_flux_200k_midjourney_150k_recovery_150k_3e4_epoch2_half_rotated/005-GPT-XL/checkpoints/0010000.pt +[2025-04-23 15:25:12] Removed old checkpoint: checkpoint/Afteri2i_100kckpt_LLAVAT5_empty_Fixed_just_segment_pretrain_stage3_dreambench_recap_400k_t2i_400k_flux_200k_midjourney_150k_recovery_150k_3e4_epoch2_half_rotated/005-GPT-XL/checkpoints/0008000.pt +[2025-04-23 15:25:34] (step=0010025) Train Loss: 5.6156, Train Steps/Sec: 0.07 +[2025-04-23 15:25:55] (step=0010050) Train Loss: 5.5371, Train Steps/Sec: 1.17 +[2025-04-23 15:26:16] (step=0010075) Train Loss: 5.5876, Train Steps/Sec: 1.17 +[2025-04-23 15:26:37] (step=0010100) Train Loss: 5.4552, Train Steps/Sec: 1.17 +[2025-04-23 15:26:59] (step=0010125) Train Loss: 5.6099, Train Steps/Sec: 1.17 +[2025-04-23 15:27:20] (step=0010150) Train Loss: 5.4981, Train Steps/Sec: 1.18 +[2025-04-23 15:27:41] (step=0010175) Train Loss: 5.5953, Train Steps/Sec: 1.18 +[2025-04-23 15:28:03] (step=0010200) Train Loss: 5.5519, Train Steps/Sec: 1.16 +[2025-04-23 15:28:24] (step=0010225) Train Loss: 5.6715, Train Steps/Sec: 1.17 +[2025-04-23 15:28:45] (step=0010250) Train Loss: 5.5475, Train Steps/Sec: 1.18 +[2025-04-23 15:29:07] (step=0010275) Train Loss: 5.5861, Train Steps/Sec: 1.17 +[2025-04-23 15:29:28] (step=0010300) Train Loss: 5.5589, Train Steps/Sec: 1.17 +[2025-04-23 15:29:49] (step=0010325) Train Loss: 5.5434, Train Steps/Sec: 1.18 +[2025-04-23 15:30:11] (step=0010350) Train Loss: 5.4776, Train Steps/Sec: 1.17 +[2025-04-23 15:30:32] (step=0010375) Train Loss: 5.5061, Train Steps/Sec: 1.17 +[2025-04-23 15:30:53] (step=0010400) Train Loss: 5.5098, Train Steps/Sec: 1.17 +[2025-04-23 15:31:15] (step=0010425) Train Loss: 5.5151, Train Steps/Sec: 1.18 +[2025-04-23 15:31:37] (step=0010450) Train Loss: 5.5400, Train Steps/Sec: 1.13 +[2025-04-23 15:31:59] (step=0010475) Train Loss: 5.5182, Train Steps/Sec: 1.13 +[2025-04-23 15:32:21] (step=0010500) Train Loss: 5.5829, Train Steps/Sec: 1.12 +[2025-04-23 15:32:43] (step=0010525) Train Loss: 5.5006, Train Steps/Sec: 1.13 +[2025-04-23 15:33:05] (step=0010550) Train Loss: 5.5134, Train Steps/Sec: 1.18 +[2025-04-23 15:33:28] (step=0010575) Train Loss: 5.5385, Train Steps/Sec: 1.08 +[2025-04-23 15:33:50] (step=0010600) Train Loss: 5.5723, Train Steps/Sec: 1.17 +[2025-04-23 15:34:11] (step=0010625) Train Loss: 5.5182, Train Steps/Sec: 1.17 +[2025-04-23 15:34:34] (step=0010650) Train Loss: 5.5741, Train Steps/Sec: 1.13 +[2025-04-23 15:34:55] (step=0010675) Train Loss: 5.4932, Train Steps/Sec: 1.17 +[2025-04-23 15:35:16] (step=0010700) Train Loss: 5.5962, Train Steps/Sec: 1.17 +[2025-04-23 15:35:38] (step=0010725) Train Loss: 5.4978, Train Steps/Sec: 1.17 +[2025-04-23 15:35:59] (step=0010750) Train Loss: 5.6340, Train Steps/Sec: 1.17 +[2025-04-23 15:36:21] (step=0010775) Train Loss: 5.5669, Train Steps/Sec: 1.16 +[2025-04-23 15:36:42] (step=0010800) Train Loss: 5.4868, Train Steps/Sec: 1.16 +[2025-04-23 15:37:03] (step=0010825) Train Loss: 5.4880, Train Steps/Sec: 1.18 +[2025-04-23 15:37:25] (step=0010850) Train Loss: 5.6324, Train Steps/Sec: 1.17 +[2025-04-23 15:37:46] (step=0010875) Train Loss: 5.6262, Train Steps/Sec: 1.17 +[2025-04-23 15:38:08] (step=0010900) Train Loss: 5.6832, Train Steps/Sec: 1.17 +[2025-04-23 15:38:29] (step=0010925) Train Loss: 5.5074, Train Steps/Sec: 1.18 +[2025-04-23 15:38:50] (step=0010950) Train Loss: 5.4988, Train Steps/Sec: 1.18 +[2025-04-23 15:39:11] (step=0010975) Train Loss: 5.6623, Train Steps/Sec: 1.17 +[2025-04-23 15:39:33] (step=0011000) Train Loss: 5.5117, Train Steps/Sec: 1.17 +[2025-04-23 15:39:54] (step=0011025) Train Loss: 5.5253, Train Steps/Sec: 1.17 +[2025-04-23 15:40:15] (step=0011050) Train Loss: 5.5331, Train Steps/Sec: 1.17 +[2025-04-23 15:40:37] (step=0011075) Train Loss: 5.5381, Train Steps/Sec: 1.16 +[2025-04-23 15:40:59] (step=0011100) Train Loss: 5.5380, Train Steps/Sec: 1.11 +[2025-04-23 15:41:22] (step=0011125) Train Loss: 5.5613, Train Steps/Sec: 1.12 +[2025-04-23 15:41:44] (step=0011150) Train Loss: 5.5830, Train Steps/Sec: 1.12 +[2025-04-23 15:42:06] (step=0011175) Train Loss: 5.5523, Train Steps/Sec: 1.13 +[2025-04-23 15:42:28] (step=0011200) Train Loss: 5.4977, Train Steps/Sec: 1.17 +[2025-04-23 15:42:50] (step=0011225) Train Loss: 5.5470, Train Steps/Sec: 1.12 +[2025-04-23 15:43:11] (step=0011250) Train Loss: 5.5740, Train Steps/Sec: 1.17 +[2025-04-23 15:43:33] (step=0011275) Train Loss: 5.5944, Train Steps/Sec: 1.17 +[2025-04-23 15:43:55] (step=0011300) Train Loss: 5.6211, Train Steps/Sec: 1.12 +[2025-04-23 15:44:18] (step=0011325) Train Loss: 5.5454, Train Steps/Sec: 1.10 +[2025-04-23 15:44:40] (step=0011350) Train Loss: 5.4876, Train Steps/Sec: 1.13 +[2025-04-23 15:45:01] (step=0011375) Train Loss: 5.5263, Train Steps/Sec: 1.17 +[2025-04-23 15:45:23] (step=0011400) Train Loss: 5.5661, Train Steps/Sec: 1.16 +[2025-04-23 15:45:44] (step=0011425) Train Loss: 5.5351, Train Steps/Sec: 1.17 +[2025-04-23 15:46:06] (step=0011450) Train Loss: 5.4415, Train Steps/Sec: 1.17 +[2025-04-23 15:46:27] (step=0011475) Train Loss: 5.5481, Train Steps/Sec: 1.17 +[2025-04-23 15:46:49] (step=0011500) Train Loss: 5.5826, Train Steps/Sec: 1.16 +[2025-04-23 15:47:10] (step=0011525) Train Loss: 5.4938, Train Steps/Sec: 1.16 +[2025-04-23 15:47:31] (step=0011550) Train Loss: 5.5277, Train Steps/Sec: 1.17 +[2025-04-23 15:47:53] (step=0011575) Train Loss: 5.5737, Train Steps/Sec: 1.17 +[2025-04-23 15:48:14] (step=0011600) Train Loss: 5.5385, Train Steps/Sec: 1.17 +[2025-04-23 15:48:36] (step=0011625) Train Loss: 5.5714, Train Steps/Sec: 1.16 +[2025-04-23 15:48:57] (step=0011650) Train Loss: 5.5084, Train Steps/Sec: 1.17 +[2025-04-23 15:49:19] (step=0011675) Train Loss: 5.5478, Train Steps/Sec: 1.17 +[2025-04-23 15:49:40] (step=0011700) Train Loss: 5.4984, Train Steps/Sec: 1.16 +[2025-04-23 15:50:02] (step=0011725) Train Loss: 5.5137, Train Steps/Sec: 1.17 +[2025-04-23 15:50:23] (step=0011750) Train Loss: 5.5878, Train Steps/Sec: 1.17 +[2025-04-23 15:50:46] (step=0011775) Train Loss: 5.6074, Train Steps/Sec: 1.09 +[2025-04-23 15:51:07] (step=0011800) Train Loss: 5.5734, Train Steps/Sec: 1.17 +[2025-04-23 15:51:30] (step=0011825) Train Loss: 5.5841, Train Steps/Sec: 1.09 +[2025-04-23 15:51:52] (step=0011850) Train Loss: 5.5362, Train Steps/Sec: 1.17 +[2025-04-23 15:52:13] (step=0011875) Train Loss: 5.5609, Train Steps/Sec: 1.17 +[2025-04-23 15:52:35] (step=0011900) Train Loss: 5.6409, Train Steps/Sec: 1.12 +[2025-04-23 15:52:57] (step=0011925) Train Loss: 5.4469, Train Steps/Sec: 1.17 +[2025-04-23 15:53:18] (step=0011950) Train Loss: 5.5627, Train Steps/Sec: 1.17 +[2025-04-23 15:53:41] (step=0011975) Train Loss: 5.5937, Train Steps/Sec: 1.10 +[2025-04-23 15:54:02] (step=0012000) Train Loss: 5.5848, Train Steps/Sec: 1.16 +[2025-04-23 15:54:02] text_config is None. Initializing the text config with default values (`OPTConfig`). +[2025-04-23 15:59:08] Finish Eval in 12000 steps... +[2025-04-23 15:59:27] Saved checkpoint to checkpoint/Afteri2i_100kckpt_LLAVAT5_empty_Fixed_just_segment_pretrain_stage3_dreambench_recap_400k_t2i_400k_flux_200k_midjourney_150k_recovery_150k_3e4_epoch2_half_rotated/005-GPT-XL/checkpoints/0012000.pt +[2025-04-23 15:59:29] Removed old checkpoint: checkpoint/Afteri2i_100kckpt_LLAVAT5_empty_Fixed_just_segment_pretrain_stage3_dreambench_recap_400k_t2i_400k_flux_200k_midjourney_150k_recovery_150k_3e4_epoch2_half_rotated/005-GPT-XL/checkpoints/0010000.pt +[2025-04-23 15:59:51] (step=0012025) Train Loss: 5.6000, Train Steps/Sec: 0.07 +[2025-04-23 16:00:13] (step=0012050) Train Loss: 5.5448, Train Steps/Sec: 1.18 +[2025-04-23 16:00:35] (step=0012075) Train Loss: 5.5937, Train Steps/Sec: 1.13 +[2025-04-23 16:00:56] (step=0012100) Train Loss: 5.5361, Train Steps/Sec: 1.17 +[2025-04-23 16:01:17] (step=0012125) Train Loss: 5.5431, Train Steps/Sec: 1.17 +[2025-04-23 16:01:39] (step=0012150) Train Loss: 5.5239, Train Steps/Sec: 1.18 +[2025-04-23 16:02:00] (step=0012175) Train Loss: 5.5776, Train Steps/Sec: 1.17 +[2025-04-23 16:02:21] (step=0012200) Train Loss: 5.5693, Train Steps/Sec: 1.17 +[2025-04-23 16:02:43] (step=0012225) Train Loss: 5.6415, Train Steps/Sec: 1.18 +[2025-04-23 16:03:04] (step=0012250) Train Loss: 5.5763, Train Steps/Sec: 1.17 +[2025-04-23 16:03:25] (step=0012275) Train Loss: 5.4533, Train Steps/Sec: 1.18 +[2025-04-23 16:03:46] (step=0012300) Train Loss: 5.5006, Train Steps/Sec: 1.17 +[2025-04-23 16:04:08] (step=0012325) Train Loss: 5.5328, Train Steps/Sec: 1.18 +[2025-04-23 16:04:29] (step=0012350) Train Loss: 5.5630, Train Steps/Sec: 1.17 +[2025-04-23 16:04:50] (step=0012375) Train Loss: 5.6340, Train Steps/Sec: 1.17 +[2025-04-23 16:05:12] (step=0012400) Train Loss: 5.5472, Train Steps/Sec: 1.17 +[2025-04-23 16:05:35] (step=0012425) Train Loss: 5.5494, Train Steps/Sec: 1.09 +[2025-04-23 16:05:56] (step=0012450) Train Loss: 5.5384, Train Steps/Sec: 1.17 +[2025-04-23 16:06:18] (step=0012475) Train Loss: 5.5601, Train Steps/Sec: 1.13 +[2025-04-23 16:06:40] (step=0012500) Train Loss: 5.5386, Train Steps/Sec: 1.13 +[2025-04-23 16:07:01] (step=0012525) Train Loss: 5.5353, Train Steps/Sec: 1.18 +[2025-04-23 16:07:24] (step=0012550) Train Loss: 5.4409, Train Steps/Sec: 1.13 +[2025-04-23 16:07:45] (step=0012575) Train Loss: 5.5736, Train Steps/Sec: 1.17 +[2025-04-23 16:08:06] (step=0012600) Train Loss: 5.5057, Train Steps/Sec: 1.17 +[2025-04-23 16:08:28] (step=0012625) Train Loss: 5.5021, Train Steps/Sec: 1.17 +[2025-04-23 16:08:49] (step=0012650) Train Loss: 5.5416, Train Steps/Sec: 1.17 +[2025-04-23 16:09:11] (step=0012675) Train Loss: 5.4875, Train Steps/Sec: 1.13 +[2025-04-23 16:09:33] (step=0012700) Train Loss: 5.5195, Train Steps/Sec: 1.17 +[2025-04-23 16:09:54] (step=0012725) Train Loss: 5.4831, Train Steps/Sec: 1.18 +[2025-04-23 16:10:16] (step=0012750) Train Loss: 5.5845, Train Steps/Sec: 1.12 +[2025-04-23 16:10:37] (step=0012775) Train Loss: 5.5646, Train Steps/Sec: 1.17 +[2025-04-23 16:11:00] (step=0012800) Train Loss: 5.5868, Train Steps/Sec: 1.12 +[2025-04-23 16:11:21] (step=0012825) Train Loss: 5.5849, Train Steps/Sec: 1.18 +[2025-04-23 16:11:42] (step=0012850) Train Loss: 5.5360, Train Steps/Sec: 1.18 +[2025-04-23 16:12:03] (step=0012875) Train Loss: 5.5797, Train Steps/Sec: 1.18 +[2025-04-23 16:12:25] (step=0012900) Train Loss: 5.5852, Train Steps/Sec: 1.17 +[2025-04-23 16:12:46] (step=0012925) Train Loss: 5.6204, Train Steps/Sec: 1.17 +[2025-04-23 16:13:07] (step=0012950) Train Loss: 5.4621, Train Steps/Sec: 1.18 +[2025-04-23 16:13:29] (step=0012975) Train Loss: 5.4966, Train Steps/Sec: 1.17 +[2025-04-23 16:13:50] (step=0013000) Train Loss: 5.4166, Train Steps/Sec: 1.17 +[2025-04-23 16:14:11] (step=0013025) Train Loss: 5.5993, Train Steps/Sec: 1.17 +[2025-04-23 16:14:33] (step=0013050) Train Loss: 5.4689, Train Steps/Sec: 1.18 +[2025-04-23 16:14:55] (step=0013075) Train Loss: 5.5226, Train Steps/Sec: 1.13 +[2025-04-23 16:15:17] (step=0013100) Train Loss: 5.5382, Train Steps/Sec: 1.13 +[2025-04-23 16:15:39] (step=0013125) Train Loss: 5.4808, Train Steps/Sec: 1.13 +[2025-04-23 16:16:01] (step=0013150) Train Loss: 5.4702, Train Steps/Sec: 1.14 +[2025-04-23 16:16:22] (step=0013175) Train Loss: 5.5122, Train Steps/Sec: 1.17 +[2025-04-23 16:16:44] (step=0013200) Train Loss: 5.5480, Train Steps/Sec: 1.17 +[2025-04-23 16:17:06] (step=0013225) Train Loss: 5.5017, Train Steps/Sec: 1.13 +[2025-04-23 16:17:27] (step=0013250) Train Loss: 5.5094, Train Steps/Sec: 1.17 +[2025-04-23 16:17:48] (step=0013275) Train Loss: 5.5620, Train Steps/Sec: 1.17 +[2025-04-23 16:18:10] (step=0013300) Train Loss: 5.5395, Train Steps/Sec: 1.17 +[2025-04-23 16:18:32] (step=0013325) Train Loss: 5.6032, Train Steps/Sec: 1.13 +[2025-04-23 16:18:53] (step=0013350) Train Loss: 5.5252, Train Steps/Sec: 1.17 +[2025-04-23 16:19:15] (step=0013375) Train Loss: 5.6318, Train Steps/Sec: 1.18 +[2025-04-23 16:19:36] (step=0013400) Train Loss: 5.5852, Train Steps/Sec: 1.17 +[2025-04-23 16:19:57] (step=0013425) Train Loss: 5.5445, Train Steps/Sec: 1.18 +[2025-04-23 16:20:19] (step=0013450) Train Loss: 5.4993, Train Steps/Sec: 1.17 +[2025-04-23 16:20:41] (step=0013475) Train Loss: 5.4998, Train Steps/Sec: 1.12 +[2025-04-23 16:21:02] (step=0013500) Train Loss: 5.5100, Train Steps/Sec: 1.17 +[2025-04-23 16:21:24] (step=0013525) Train Loss: 5.4805, Train Steps/Sec: 1.12 +[2025-04-23 16:21:46] (step=0013550) Train Loss: 5.5203, Train Steps/Sec: 1.17 +[2025-04-23 16:22:07] (step=0013575) Train Loss: 5.4657, Train Steps/Sec: 1.18 +[2025-04-23 16:22:28] (step=0013600) Train Loss: 5.5671, Train Steps/Sec: 1.17 +[2025-04-23 16:22:50] (step=0013625) Train Loss: 5.5200, Train Steps/Sec: 1.17 +[2025-04-23 16:23:11] (step=0013650) Train Loss: 5.5218, Train Steps/Sec: 1.17 +[2025-04-23 16:23:32] (step=0013675) Train Loss: 5.4432, Train Steps/Sec: 1.17 +[2025-04-23 16:23:54] (step=0013700) Train Loss: 5.5052, Train Steps/Sec: 1.17 +[2025-04-23 16:24:15] (step=0013725) Train Loss: 5.5429, Train Steps/Sec: 1.17 +[2025-04-23 16:24:38] (step=0013750) Train Loss: 5.4848, Train Steps/Sec: 1.09 +[2025-04-23 16:25:00] (step=0013775) Train Loss: 5.5653, Train Steps/Sec: 1.13 +[2025-04-23 16:25:21] (step=0013800) Train Loss: 5.6074, Train Steps/Sec: 1.17 +[2025-04-23 16:25:43] (step=0013825) Train Loss: 5.4420, Train Steps/Sec: 1.13 +[2025-04-23 16:26:05] (step=0013850) Train Loss: 5.5832, Train Steps/Sec: 1.17 +[2025-04-23 16:26:27] (step=0013875) Train Loss: 5.5569, Train Steps/Sec: 1.13 +[2025-04-23 16:26:48] (step=0013900) Train Loss: 5.5526, Train Steps/Sec: 1.17 +[2025-04-23 16:27:09] (step=0013925) Train Loss: 5.4985, Train Steps/Sec: 1.18 +[2025-04-23 16:27:31] (step=0013950) Train Loss: 5.5561, Train Steps/Sec: 1.17 +[2025-04-23 16:27:53] (step=0013975) Train Loss: 5.5362, Train Steps/Sec: 1.13 +[2025-04-23 16:28:14] (step=0014000) Train Loss: 5.5002, Train Steps/Sec: 1.17 +[2025-04-23 16:28:14] text_config is None. Initializing the text config with default values (`OPTConfig`). +[2025-04-23 16:33:20] Finish Eval in 14000 steps... +[2025-04-23 16:33:39] Saved checkpoint to checkpoint/Afteri2i_100kckpt_LLAVAT5_empty_Fixed_just_segment_pretrain_stage3_dreambench_recap_400k_t2i_400k_flux_200k_midjourney_150k_recovery_150k_3e4_epoch2_half_rotated/005-GPT-XL/checkpoints/0014000.pt +[2025-04-23 16:33:41] Removed old checkpoint: checkpoint/Afteri2i_100kckpt_LLAVAT5_empty_Fixed_just_segment_pretrain_stage3_dreambench_recap_400k_t2i_400k_flux_200k_midjourney_150k_recovery_150k_3e4_epoch2_half_rotated/005-GPT-XL/checkpoints/0012000.pt +[2025-04-23 16:34:02] (step=0014025) Train Loss: 5.4944, Train Steps/Sec: 0.07 +[2025-04-23 16:34:23] (step=0014050) Train Loss: 5.5202, Train Steps/Sec: 1.17 +[2025-04-23 16:34:45] (step=0014075) Train Loss: 5.5473, Train Steps/Sec: 1.18 +[2025-04-23 16:35:06] (step=0014100) Train Loss: 5.5673, Train Steps/Sec: 1.16 +[2025-04-23 16:35:27] (step=0014125) Train Loss: 5.5259, Train Steps/Sec: 1.18 +[2025-04-23 16:35:49] (step=0014150) Train Loss: 5.5159, Train Steps/Sec: 1.17 +[2025-04-23 16:36:10] (step=0014175) Train Loss: 5.5393, Train Steps/Sec: 1.18 +[2025-04-23 16:36:32] (step=0014200) Train Loss: 5.4852, Train Steps/Sec: 1.12 +[2025-04-23 16:36:54] (step=0014225) Train Loss: 5.5351, Train Steps/Sec: 1.17 +[2025-04-23 16:37:16] (step=0014250) Train Loss: 5.5801, Train Steps/Sec: 1.12 +[2025-04-23 16:37:37] (step=0014275) Train Loss: 5.4511, Train Steps/Sec: 1.17 +[2025-04-23 16:37:59] (step=0014300) Train Loss: 5.4504, Train Steps/Sec: 1.17 +[2025-04-23 16:38:20] (step=0014325) Train Loss: 5.4499, Train Steps/Sec: 1.17 +[2025-04-23 16:38:41] (step=0014350) Train Loss: 5.3827, Train Steps/Sec: 1.17 +[2025-04-23 16:39:03] (step=0014375) Train Loss: 5.4427, Train Steps/Sec: 1.18 +[2025-04-23 16:39:26] (step=0014400) Train Loss: 5.5057, Train Steps/Sec: 1.13 +[2025-04-23 16:39:47] (step=0014425) Train Loss: 5.5141, Train Steps/Sec: 1.17 +[2025-04-23 16:40:09] (step=0014450) Train Loss: 5.4834, Train Steps/Sec: 1.13 +[2025-04-23 16:40:31] (step=0014475) Train Loss: 5.5216, Train Steps/Sec: 1.14 +[2025-04-23 16:40:52] (step=0014500) Train Loss: 5.5890, Train Steps/Sec: 1.17 +[2025-04-23 16:41:15] (step=0014525) Train Loss: 5.5878, Train Steps/Sec: 1.13 +[2025-04-23 16:41:36] (step=0014550) Train Loss: 5.5000, Train Steps/Sec: 1.17 +[2025-04-23 16:41:57] (step=0014575) Train Loss: 5.4697, Train Steps/Sec: 1.17 +[2025-04-23 16:42:19] (step=0014600) Train Loss: 5.4923, Train Steps/Sec: 1.17 +[2025-04-23 16:42:40] (step=0014625) Train Loss: 5.4902, Train Steps/Sec: 1.17 +[2025-04-23 16:43:02] (step=0014650) Train Loss: 5.4978, Train Steps/Sec: 1.13 +[2025-04-23 16:43:23] (step=0014675) Train Loss: 5.5541, Train Steps/Sec: 1.18 +[2025-04-23 16:43:45] (step=0014700) Train Loss: 5.4956, Train Steps/Sec: 1.17 +[2025-04-23 16:44:06] (step=0014725) Train Loss: 5.5468, Train Steps/Sec: 1.17 +[2025-04-23 16:44:27] (step=0014750) Train Loss: 5.5262, Train Steps/Sec: 1.17 +[2025-04-23 16:44:49] (step=0014775) Train Loss: 5.4890, Train Steps/Sec: 1.17 +[2025-04-23 16:45:10] (step=0014800) Train Loss: 5.5048, Train Steps/Sec: 1.17 +[2025-04-23 16:45:31] (step=0014825) Train Loss: 5.5443, Train Steps/Sec: 1.18 +[2025-04-23 16:45:53] (step=0014850) Train Loss: 5.5363, Train Steps/Sec: 1.17 +[2025-04-23 16:46:14] (step=0014875) Train Loss: 5.5088, Train Steps/Sec: 1.18 +[2025-04-23 16:46:35] (step=0014900) Train Loss: 5.5094, Train Steps/Sec: 1.17 +[2025-04-23 16:46:58] (step=0014925) Train Loss: 5.5696, Train Steps/Sec: 1.12 +[2025-04-23 16:47:19] (step=0014950) Train Loss: 5.5767, Train Steps/Sec: 1.18 +[2025-04-23 16:47:41] (step=0014975) Train Loss: 5.5766, Train Steps/Sec: 1.13 +[2025-04-23 16:48:02] (step=0015000) Train Loss: 5.4737, Train Steps/Sec: 1.17 +[2025-04-23 16:48:24] (step=0015025) Train Loss: 5.5230, Train Steps/Sec: 1.18 +[2025-04-23 16:48:45] (step=0015050) Train Loss: 5.6084, Train Steps/Sec: 1.17 +[2025-04-23 16:49:08] (step=0015075) Train Loss: 5.5118, Train Steps/Sec: 1.09 +[2025-04-23 16:49:30] (step=0015100) Train Loss: 5.4299, Train Steps/Sec: 1.12 +[2025-04-23 16:49:52] (step=0015125) Train Loss: 5.4758, Train Steps/Sec: 1.14 +[2025-04-23 16:50:13] (step=0015150) Train Loss: 5.5543, Train Steps/Sec: 1.18 +[2025-04-23 16:50:35] (step=0015175) Train Loss: 5.5298, Train Steps/Sec: 1.18 +[2025-04-23 16:50:57] (step=0015200) Train Loss: 5.5201, Train Steps/Sec: 1.13 +[2025-04-23 16:51:18] (step=0015225) Train Loss: 5.5137, Train Steps/Sec: 1.17 +[2025-04-23 16:51:39] (step=0015250) Train Loss: 5.4890, Train Steps/Sec: 1.17 +[2025-04-23 16:52:01] (step=0015275) Train Loss: 5.6258, Train Steps/Sec: 1.17 +[2025-04-23 16:52:22] (step=0015300) Train Loss: 5.4413, Train Steps/Sec: 1.17 +[2025-04-23 16:52:44] (step=0015325) Train Loss: 5.4613, Train Steps/Sec: 1.12 +[2025-04-23 16:53:05] (step=0015350) Train Loss: 5.5241, Train Steps/Sec: 1.18 +[2025-04-23 16:53:27] (step=0015375) Train Loss: 5.5844, Train Steps/Sec: 1.17 +[2025-04-23 16:53:48] (step=0015400) Train Loss: 5.5655, Train Steps/Sec: 1.17 +[2025-04-23 16:54:09] (step=0015425) Train Loss: 5.4655, Train Steps/Sec: 1.18 +[2025-04-23 16:54:31] (step=0015450) Train Loss: 5.5500, Train Steps/Sec: 1.18 +[2025-04-23 16:54:52] (step=0015475) Train Loss: 5.5030, Train Steps/Sec: 1.18 +[2025-04-23 16:55:13] (step=0015500) Train Loss: 5.5372, Train Steps/Sec: 1.17 +[2025-04-23 16:55:35] (step=0015525) Train Loss: 5.6197, Train Steps/Sec: 1.18 +[2025-04-23 16:55:56] (step=0015550) Train Loss: 5.5320, Train Steps/Sec: 1.17 +[2025-04-23 16:56:17] (step=0015575) Train Loss: 5.5126, Train Steps/Sec: 1.18 +[2025-04-23 16:56:39] (step=0015600) Train Loss: 5.4923, Train Steps/Sec: 1.17 +[2025-04-23 16:57:00] (step=0015625) Train Loss: 5.5476, Train Steps/Sec: 1.18 +[2025-04-23 16:57:22] (step=0015650) Train Loss: 5.5669, Train Steps/Sec: 1.12 +[2025-04-23 16:57:43] (step=0015675) Train Loss: 5.4960, Train Steps/Sec: 1.17 +[2025-04-23 16:58:06] (step=0015700) Train Loss: 5.5474, Train Steps/Sec: 1.12 +[2025-04-23 16:58:29] (step=0015725) Train Loss: 5.4340, Train Steps/Sec: 1.09 +[2025-04-23 16:58:51] (step=0015750) Train Loss: 5.4586, Train Steps/Sec: 1.13 +[2025-04-23 16:59:13] (step=0015775) Train Loss: 5.5310, Train Steps/Sec: 1.14 +[2025-04-23 16:59:34] (step=0015800) Train Loss: 5.5939, Train Steps/Sec: 1.17 +[2025-04-23 16:59:56] (step=0015825) Train Loss: 5.5842, Train Steps/Sec: 1.17 +[2025-04-23 17:00:18] (step=0015850) Train Loss: 5.4688, Train Steps/Sec: 1.14 +[2025-04-23 17:00:39] (step=0015875) Train Loss: 5.5409, Train Steps/Sec: 1.17 +[2025-04-23 17:01:00] (step=0015900) Train Loss: 5.4748, Train Steps/Sec: 1.17 +[2025-04-23 17:01:22] (step=0015925) Train Loss: 5.5507, Train Steps/Sec: 1.17 +[2025-04-23 17:01:43] (step=0015950) Train Loss: 5.5252, Train Steps/Sec: 1.18 +[2025-04-23 17:02:05] (step=0015975) Train Loss: 5.5258, Train Steps/Sec: 1.13 +[2025-04-23 17:02:26] (step=0016000) Train Loss: 5.5051, Train Steps/Sec: 1.17 +[2025-04-23 17:02:26] text_config is None. Initializing the text config with default values (`OPTConfig`). +[2025-04-23 17:07:30] Finish Eval in 16000 steps... +[2025-04-23 17:07:48] Saved checkpoint to checkpoint/Afteri2i_100kckpt_LLAVAT5_empty_Fixed_just_segment_pretrain_stage3_dreambench_recap_400k_t2i_400k_flux_200k_midjourney_150k_recovery_150k_3e4_epoch2_half_rotated/005-GPT-XL/checkpoints/0016000.pt +[2025-04-23 17:07:50] Removed old checkpoint: checkpoint/Afteri2i_100kckpt_LLAVAT5_empty_Fixed_just_segment_pretrain_stage3_dreambench_recap_400k_t2i_400k_flux_200k_midjourney_150k_recovery_150k_3e4_epoch2_half_rotated/005-GPT-XL/checkpoints/0014000.pt +[2025-04-23 17:08:12] (step=0016025) Train Loss: 5.5073, Train Steps/Sec: 0.07 +[2025-04-23 17:08:33] (step=0016050) Train Loss: 5.4484, Train Steps/Sec: 1.18 +[2025-04-23 17:08:54] (step=0016075) Train Loss: 5.4636, Train Steps/Sec: 1.17 +[2025-04-23 17:09:16] (step=0016100) Train Loss: 5.5710, Train Steps/Sec: 1.17 +[2025-04-23 17:09:37] (step=0016125) Train Loss: 5.5144, Train Steps/Sec: 1.17 +[2025-04-23 17:09:58] (step=0016150) Train Loss: 5.4943, Train Steps/Sec: 1.17 +[2025-04-23 17:10:19] (step=0016175) Train Loss: 5.3842, Train Steps/Sec: 1.18 +[2025-04-23 17:10:41] (step=0016200) Train Loss: 5.5843, Train Steps/Sec: 1.17 +[2025-04-23 17:11:02] (step=0016225) Train Loss: 5.4896, Train Steps/Sec: 1.18 +[2025-04-23 17:11:23] (step=0016250) Train Loss: 5.4983, Train Steps/Sec: 1.17 +[2025-04-23 17:11:45] (step=0016275) Train Loss: 5.4915, Train Steps/Sec: 1.18 +[2025-04-23 17:12:06] (step=0016300) Train Loss: 5.4555, Train Steps/Sec: 1.17 +[2025-04-23 17:12:27] (step=0016325) Train Loss: 5.4928, Train Steps/Sec: 1.18 +[2025-04-23 17:12:50] (step=0016350) Train Loss: 5.4474, Train Steps/Sec: 1.12 +[2025-04-23 17:13:12] (step=0016375) Train Loss: 5.5091, Train Steps/Sec: 1.09 +[2025-04-23 17:13:34] (step=0016400) Train Loss: 5.5999, Train Steps/Sec: 1.17 +[2025-04-23 17:13:57] (step=0016425) Train Loss: 5.4645, Train Steps/Sec: 1.08 +[2025-04-23 17:14:19] (step=0016450) Train Loss: 5.5456, Train Steps/Sec: 1.13 +[2025-04-23 17:14:40] (step=0016475) Train Loss: 5.4847, Train Steps/Sec: 1.17 +[2025-04-23 17:15:02] (step=0016500) Train Loss: 5.5084, Train Steps/Sec: 1.17 +[2025-04-23 17:15:24] (step=0016525) Train Loss: 5.4697, Train Steps/Sec: 1.13 +[2025-04-23 17:15:45] (step=0016550) Train Loss: 5.5254, Train Steps/Sec: 1.18 +[2025-04-23 17:16:06] (step=0016575) Train Loss: 5.4840, Train Steps/Sec: 1.17 +[2025-04-23 17:16:28] (step=0016600) Train Loss: 5.5310, Train Steps/Sec: 1.17 +[2025-04-23 17:16:49] (step=0016625) Train Loss: 5.4647, Train Steps/Sec: 1.17 +[2025-04-23 17:17:10] (step=0016650) Train Loss: 5.5448, Train Steps/Sec: 1.17 +[2025-04-23 17:17:32] (step=0016675) Train Loss: 5.4914, Train Steps/Sec: 1.13 +[2025-04-23 17:17:54] (step=0016700) Train Loss: 5.5227, Train Steps/Sec: 1.17 +[2025-04-23 17:18:15] (step=0016725) Train Loss: 5.4889, Train Steps/Sec: 1.17 +[2025-04-23 17:18:37] (step=0016750) Train Loss: 5.4594, Train Steps/Sec: 1.17 +[2025-04-23 17:18:58] (step=0016775) Train Loss: 5.4908, Train Steps/Sec: 1.18 +[2025-04-23 17:19:19] (step=0016800) Train Loss: 5.5653, Train Steps/Sec: 1.17 +[2025-04-23 17:19:40] (step=0016825) Train Loss: 5.5185, Train Steps/Sec: 1.18 +[2025-04-23 17:20:02] (step=0016850) Train Loss: 5.4538, Train Steps/Sec: 1.18 +[2025-04-23 17:20:23] (step=0016875) Train Loss: 5.4837, Train Steps/Sec: 1.17 +[2025-04-23 17:20:44] (step=0016900) Train Loss: 5.6197, Train Steps/Sec: 1.17 +[2025-04-23 17:21:06] (step=0016925) Train Loss: 5.5145, Train Steps/Sec: 1.18 +[2025-04-23 17:21:27] (step=0016950) Train Loss: 5.4973, Train Steps/Sec: 1.18 +[2025-04-23 17:21:48] (step=0016975) Train Loss: 5.5083, Train Steps/Sec: 1.18 +[2025-04-23 17:22:10] (step=0017000) Train Loss: 5.5124, Train Steps/Sec: 1.17 +[2025-04-23 17:22:31] (step=0017025) Train Loss: 5.4440, Train Steps/Sec: 1.18 +[2025-04-23 17:22:54] (step=0017050) Train Loss: 5.5187, Train Steps/Sec: 1.09 +[2025-04-23 17:23:17] (step=0017075) Train Loss: 5.5263, Train Steps/Sec: 1.08 +[2025-04-23 17:23:39] (step=0017100) Train Loss: 5.5183, Train Steps/Sec: 1.13 +[2025-04-23 17:24:00] (step=0017125) Train Loss: 5.4882, Train Steps/Sec: 1.18 +[2025-04-23 17:24:23] (step=0017150) Train Loss: 5.6059, Train Steps/Sec: 1.13 +[2025-04-23 17:24:45] (step=0017175) Train Loss: 5.4793, Train Steps/Sec: 1.14 +[2025-04-23 17:25:06] (step=0017200) Train Loss: 5.4168, Train Steps/Sec: 1.16 +[2025-04-23 17:25:27] (step=0017225) Train Loss: 5.5064, Train Steps/Sec: 1.17 +[2025-04-23 17:25:49] (step=0017250) Train Loss: 5.5083, Train Steps/Sec: 1.17 +[2025-04-23 17:26:10] (step=0017275) Train Loss: 5.5279, Train Steps/Sec: 1.17 +[2025-04-23 17:26:31] (step=0017300) Train Loss: 5.5151, Train Steps/Sec: 1.17 +[2025-04-23 17:26:54] (step=0017325) Train Loss: 5.5180, Train Steps/Sec: 1.13 +[2025-04-23 17:27:15] (step=0017350) Train Loss: 5.4521, Train Steps/Sec: 1.17 +[2025-04-23 17:27:36] (step=0017375) Train Loss: 5.5857, Train Steps/Sec: 1.18 +[2025-04-23 17:27:58] (step=0017400) Train Loss: 5.4588, Train Steps/Sec: 1.17 +[2025-04-23 17:28:19] (step=0017425) Train Loss: 5.4749, Train Steps/Sec: 1.18 +[2025-04-23 17:28:40] (step=0017450) Train Loss: 5.5341, Train Steps/Sec: 1.18 +[2025-04-23 17:29:01] (step=0017475) Train Loss: 5.5657, Train Steps/Sec: 1.18 +[2025-04-23 17:29:23] (step=0017500) Train Loss: 5.5260, Train Steps/Sec: 1.17 +[2025-04-23 17:29:44] (step=0017525) Train Loss: 5.5095, Train Steps/Sec: 1.17 +[2025-04-23 17:30:05] (step=0017550) Train Loss: 5.5808, Train Steps/Sec: 1.18 +[2025-04-23 17:30:27] (step=0017575) Train Loss: 5.4506, Train Steps/Sec: 1.17 +[2025-04-23 17:30:48] (step=0017600) Train Loss: 5.5192, Train Steps/Sec: 1.17 +[2025-04-23 17:31:09] (step=0017625) Train Loss: 5.4927, Train Steps/Sec: 1.18 +[2025-04-23 17:31:31] (step=0017650) Train Loss: 5.5119, Train Steps/Sec: 1.17 +[2025-04-23 17:31:52] (step=0017675) Train Loss: 5.4708, Train Steps/Sec: 1.17 +[2025-04-23 17:32:15] (step=0017700) Train Loss: 5.4813, Train Steps/Sec: 1.09 +[2025-04-23 17:32:36] (step=0017725) Train Loss: 5.5356, Train Steps/Sec: 1.17 +[2025-04-23 17:32:58] (step=0017750) Train Loss: 5.4586, Train Steps/Sec: 1.13 +[2025-04-23 17:33:20] (step=0017775) Train Loss: 5.4935, Train Steps/Sec: 1.13 +[2025-04-23 17:33:43] (step=0017800) Train Loss: 5.5303, Train Steps/Sec: 1.12 +[2025-04-23 17:34:05] (step=0017825) Train Loss: 5.5399, Train Steps/Sec: 1.14 +[2025-04-23 17:34:26] (step=0017850) Train Loss: 5.4776, Train Steps/Sec: 1.18 +[2025-04-23 17:34:48] (step=0017875) Train Loss: 5.4696, Train Steps/Sec: 1.13 +[2025-04-23 17:35:10] (step=0017900) Train Loss: 5.4982, Train Steps/Sec: 1.17 +[2025-04-23 17:35:31] (step=0017925) Train Loss: 5.5126, Train Steps/Sec: 1.18 +[2025-04-23 17:35:52] (step=0017950) Train Loss: 5.4615, Train Steps/Sec: 1.17 +[2025-04-23 17:36:14] (step=0017975) Train Loss: 5.4229, Train Steps/Sec: 1.13 +[2025-04-23 17:36:36] (step=0018000) Train Loss: 5.4664, Train Steps/Sec: 1.17 +[2025-04-23 17:36:36] text_config is None. Initializing the text config with default values (`OPTConfig`). +[2025-04-23 17:41:42] Finish Eval in 18000 steps... +[2025-04-23 17:42:01] Saved checkpoint to checkpoint/Afteri2i_100kckpt_LLAVAT5_empty_Fixed_just_segment_pretrain_stage3_dreambench_recap_400k_t2i_400k_flux_200k_midjourney_150k_recovery_150k_3e4_epoch2_half_rotated/005-GPT-XL/checkpoints/0018000.pt +[2025-04-23 17:42:03] Removed old checkpoint: checkpoint/Afteri2i_100kckpt_LLAVAT5_empty_Fixed_just_segment_pretrain_stage3_dreambench_recap_400k_t2i_400k_flux_200k_midjourney_150k_recovery_150k_3e4_epoch2_half_rotated/005-GPT-XL/checkpoints/0016000.pt +[2025-04-23 17:42:24] (step=0018025) Train Loss: 5.4380, Train Steps/Sec: 0.07 +[2025-04-23 17:42:46] (step=0018050) Train Loss: 5.4936, Train Steps/Sec: 1.17 +[2025-04-23 17:43:07] (step=0018075) Train Loss: 5.4542, Train Steps/Sec: 1.17 +[2025-04-23 17:43:28] (step=0018100) Train Loss: 5.5214, Train Steps/Sec: 1.17 +[2025-04-23 17:43:50] (step=0018125) Train Loss: 5.4827, Train Steps/Sec: 1.18 +[2025-04-23 17:44:11] (step=0018150) Train Loss: 5.5650, Train Steps/Sec: 1.17 +[2025-04-23 17:44:32] (step=0018175) Train Loss: 5.3987, Train Steps/Sec: 1.17 +[2025-04-23 17:44:54] (step=0018200) Train Loss: 5.5915, Train Steps/Sec: 1.17 +[2025-04-23 17:45:15] (step=0018225) Train Loss: 5.5382, Train Steps/Sec: 1.17 +[2025-04-23 17:45:36] (step=0018250) Train Loss: 5.5963, Train Steps/Sec: 1.18 +[2025-04-23 17:45:57] (step=0018275) Train Loss: 5.3962, Train Steps/Sec: 1.17 +[2025-04-23 17:46:19] (step=0018300) Train Loss: 5.5199, Train Steps/Sec: 1.17 +[2025-04-23 17:46:40] (step=0018325) Train Loss: 5.4581, Train Steps/Sec: 1.18 +[2025-04-23 17:47:03] (step=0018350) Train Loss: 5.4808, Train Steps/Sec: 1.09 +[2025-04-23 17:47:24] (step=0018375) Train Loss: 5.4966, Train Steps/Sec: 1.18 +[2025-04-23 17:47:47] (step=0018400) Train Loss: 5.5382, Train Steps/Sec: 1.12 +[2025-04-23 17:48:09] (step=0018425) Train Loss: 5.4815, Train Steps/Sec: 1.14 +[2025-04-23 17:48:30] (step=0018450) Train Loss: 5.3649, Train Steps/Sec: 1.18 +[2025-04-23 17:48:51] (step=0018475) Train Loss: 5.4459, Train Steps/Sec: 1.17 +[2025-04-23 17:49:13] (step=0018500) Train Loss: 5.4935, Train Steps/Sec: 1.13 +[2025-04-23 17:49:36] (step=0018525) Train Loss: 5.4489, Train Steps/Sec: 1.12 +[2025-04-23 17:49:57] (step=0018550) Train Loss: 5.4854, Train Steps/Sec: 1.17 +[2025-04-23 17:50:18] (step=0018575) Train Loss: 5.4201, Train Steps/Sec: 1.17 +[2025-04-23 17:50:41] (step=0018600) Train Loss: 5.5395, Train Steps/Sec: 1.12 +[2025-04-23 17:51:02] (step=0018625) Train Loss: 5.5183, Train Steps/Sec: 1.18 +[2025-04-23 17:51:24] (step=0018650) Train Loss: 5.4414, Train Steps/Sec: 1.13 +[2025-04-23 17:51:45] (step=0018675) Train Loss: 5.3476, Train Steps/Sec: 1.17 +[2025-04-23 17:52:07] (step=0018700) Train Loss: 5.4791, Train Steps/Sec: 1.17 +[2025-04-23 17:52:28] (step=0018725) Train Loss: 5.4835, Train Steps/Sec: 1.17 +[2025-04-23 17:52:49] (step=0018750) Train Loss: 5.5659, Train Steps/Sec: 1.17 +[2025-04-23 17:53:11] (step=0018775) Train Loss: 5.5081, Train Steps/Sec: 1.17 +[2025-04-23 17:53:32] (step=0018800) Train Loss: 5.5219, Train Steps/Sec: 1.17 +[2025-04-23 17:53:53] (step=0018825) Train Loss: 5.5392, Train Steps/Sec: 1.18 +[2025-04-23 17:54:15] (step=0018850) Train Loss: 5.4428, Train Steps/Sec: 1.18 +[2025-04-23 17:54:36] (step=0018875) Train Loss: 5.4854, Train Steps/Sec: 1.18 +[2025-04-23 17:54:57] (step=0018900) Train Loss: 5.4445, Train Steps/Sec: 1.17 +[2025-04-23 17:55:19] (step=0018925) Train Loss: 5.4381, Train Steps/Sec: 1.18 +[2025-04-23 17:55:40] (step=0018950) Train Loss: 5.4748, Train Steps/Sec: 1.17 +[2025-04-23 17:56:01] (step=0018975) Train Loss: 5.5270, Train Steps/Sec: 1.17 +[2025-04-23 17:56:22] (step=0019000) Train Loss: 5.4199, Train Steps/Sec: 1.17 +[2025-04-23 17:56:45] (step=0019025) Train Loss: 5.5101, Train Steps/Sec: 1.09 +[2025-04-23 17:57:08] (step=0019050) Train Loss: 5.5405, Train Steps/Sec: 1.13 +[2025-04-23 17:57:30] (step=0019075) Train Loss: 5.5772, Train Steps/Sec: 1.13 +[2025-04-23 17:57:51] (step=0019100) Train Loss: 5.5151, Train Steps/Sec: 1.17 +[2025-04-23 17:58:12] (step=0019125) Train Loss: 5.4811, Train Steps/Sec: 1.17 +[2025-04-23 17:58:34] (step=0019150) Train Loss: 5.4930, Train Steps/Sec: 1.13 +[2025-04-23 17:58:56] (step=0019175) Train Loss: 5.5418, Train Steps/Sec: 1.17 +[2025-04-23 17:59:17] (step=0019200) Train Loss: 5.4810, Train Steps/Sec: 1.17 +[2025-04-23 17:59:38] (step=0019225) Train Loss: 5.4984, Train Steps/Sec: 1.17 +[2025-04-23 18:00:01] (step=0019250) Train Loss: 5.4334, Train Steps/Sec: 1.12 +[2025-04-23 18:00:22] (step=0019275) Train Loss: 5.4749, Train Steps/Sec: 1.18 +[2025-04-23 18:00:43] (step=0019300) Train Loss: 5.4974, Train Steps/Sec: 1.17 +[2025-04-23 18:01:06] (step=0019325) Train Loss: 5.4649, Train Steps/Sec: 1.08 +[2025-04-23 18:01:28] (step=0019350) Train Loss: 5.4927, Train Steps/Sec: 1.17 +[2025-04-23 18:01:49] (step=0019375) Train Loss: 5.4684, Train Steps/Sec: 1.18 +[2025-04-23 18:02:10] (step=0019400) Train Loss: 5.4934, Train Steps/Sec: 1.17 +[2025-04-23 18:02:32] (step=0019425) Train Loss: 5.5249, Train Steps/Sec: 1.18 +[2025-04-23 18:02:53] (step=0019450) Train Loss: 5.5430, Train Steps/Sec: 1.17 +[2025-04-23 18:03:14] (step=0019475) Train Loss: 5.4646, Train Steps/Sec: 1.17 +[2025-04-23 18:03:36] (step=0019500) Train Loss: 5.4541, Train Steps/Sec: 1.17 +[2025-04-23 18:03:57] (step=0019525) Train Loss: 5.5072, Train Steps/Sec: 1.18 +[2025-04-23 18:04:18] (step=0019550) Train Loss: 5.4671, Train Steps/Sec: 1.17 +[2025-04-23 18:04:39] (step=0019575) Train Loss: 5.5037, Train Steps/Sec: 1.17 +[2025-04-23 18:05:01] (step=0019600) Train Loss: 5.4647, Train Steps/Sec: 1.17 +[2025-04-23 18:05:22] (step=0019625) Train Loss: 5.5186, Train Steps/Sec: 1.18 +[2025-04-23 18:05:43] (step=0019650) Train Loss: 5.5793, Train Steps/Sec: 1.18 +[2025-04-23 18:06:06] (step=0019675) Train Loss: 5.4259, Train Steps/Sec: 1.09 +[2025-04-23 18:06:28] (step=0019700) Train Loss: 5.5110, Train Steps/Sec: 1.17 +[2025-04-23 18:06:50] (step=0019725) Train Loss: 5.4663, Train Steps/Sec: 1.13 +[2025-04-23 18:07:12] (step=0019750) Train Loss: 5.4128, Train Steps/Sec: 1.14 +[2025-04-23 18:07:33] (step=0019775) Train Loss: 5.5712, Train Steps/Sec: 1.17 +[2025-04-23 18:07:55] (step=0019800) Train Loss: 5.4101, Train Steps/Sec: 1.13 +[2025-04-23 18:08:17] (step=0019825) Train Loss: 5.5493, Train Steps/Sec: 1.17 +[2025-04-23 18:08:38] (step=0019850) Train Loss: 5.5030, Train Steps/Sec: 1.17 +[2025-04-23 18:08:59] (step=0019875) Train Loss: 5.4184, Train Steps/Sec: 1.17 +[2025-04-23 18:09:21] (step=0019900) Train Loss: 5.4748, Train Steps/Sec: 1.17 +[2025-04-23 18:09:42] (step=0019925) Train Loss: 5.5153, Train Steps/Sec: 1.18 +[2025-04-23 18:10:03] (step=0019950) Train Loss: 5.5157, Train Steps/Sec: 1.17 +[2025-04-23 18:10:26] (step=0019975) Train Loss: 5.5104, Train Steps/Sec: 1.08 +[2025-04-23 18:10:48] (step=0020000) Train Loss: 5.3854, Train Steps/Sec: 1.17 +[2025-04-23 18:10:48] text_config is None. Initializing the text config with default values (`OPTConfig`). +[2025-04-23 18:15:52] Finish Eval in 20000 steps... +[2025-04-23 18:16:10] Saved checkpoint to checkpoint/Afteri2i_100kckpt_LLAVAT5_empty_Fixed_just_segment_pretrain_stage3_dreambench_recap_400k_t2i_400k_flux_200k_midjourney_150k_recovery_150k_3e4_epoch2_half_rotated/005-GPT-XL/checkpoints/0020000.pt +[2025-04-23 18:16:12] Removed old checkpoint: checkpoint/Afteri2i_100kckpt_LLAVAT5_empty_Fixed_just_segment_pretrain_stage3_dreambench_recap_400k_t2i_400k_flux_200k_midjourney_150k_recovery_150k_3e4_epoch2_half_rotated/005-GPT-XL/checkpoints/0018000.pt +[2025-04-23 18:16:33] (step=0020025) Train Loss: 5.5148, Train Steps/Sec: 0.07 +[2025-04-23 18:16:54] (step=0020050) Train Loss: 5.5381, Train Steps/Sec: 1.18 +[2025-04-23 18:17:17] (step=0020075) Train Loss: 5.4998, Train Steps/Sec: 1.13 +[2025-04-23 18:17:38] (step=0020100) Train Loss: 5.4355, Train Steps/Sec: 1.17 +[2025-04-23 18:17:59] (step=0020125) Train Loss: 5.4976, Train Steps/Sec: 1.17 +[2025-04-23 18:18:21] (step=0020150) Train Loss: 5.3721, Train Steps/Sec: 1.17 +[2025-04-23 18:18:42] (step=0020175) Train Loss: 5.4104, Train Steps/Sec: 1.18 +[2025-04-23 18:19:03] (step=0020200) Train Loss: 5.5523, Train Steps/Sec: 1.17 +[2025-04-23 18:19:24] (step=0020225) Train Loss: 5.4362, Train Steps/Sec: 1.18 +[2025-04-23 18:19:46] (step=0020250) Train Loss: 5.3952, Train Steps/Sec: 1.18 +[2025-04-23 18:20:07] (step=0020275) Train Loss: 5.5176, Train Steps/Sec: 1.18 +[2025-04-23 18:20:28] (step=0020300) Train Loss: 5.4308, Train Steps/Sec: 1.17 +[2025-04-23 18:20:50] (step=0020325) Train Loss: 5.4276, Train Steps/Sec: 1.13 +[2025-04-23 18:21:13] (step=0020350) Train Loss: 5.5634, Train Steps/Sec: 1.13 +[2025-04-23 18:21:35] (step=0020375) Train Loss: 5.4865, Train Steps/Sec: 1.13 +[2025-04-23 18:21:57] (step=0020400) Train Loss: 5.4668, Train Steps/Sec: 1.13 +[2025-04-23 18:22:18] (step=0020425) Train Loss: 5.5218, Train Steps/Sec: 1.17 +[2025-04-23 18:22:40] (step=0020450) Train Loss: 5.5229, Train Steps/Sec: 1.18 +[2025-04-23 18:23:02] (step=0020475) Train Loss: 5.5165, Train Steps/Sec: 1.13 +[2025-04-23 18:23:23] (step=0020500) Train Loss: 5.4731, Train Steps/Sec: 1.17 +[2025-04-23 18:23:44] (step=0020525) Train Loss: 5.4485, Train Steps/Sec: 1.17 +[2025-04-23 18:24:06] (step=0020550) Train Loss: 5.5158, Train Steps/Sec: 1.17 +[2025-04-23 18:24:27] (step=0020575) Train Loss: 5.5741, Train Steps/Sec: 1.17 +[2025-04-23 18:24:48] (step=0020600) Train Loss: 5.4617, Train Steps/Sec: 1.17 +[2025-04-23 18:25:10] (step=0020625) Train Loss: 5.3876, Train Steps/Sec: 1.17 +[2025-04-23 18:25:31] (step=0020650) Train Loss: 5.4394, Train Steps/Sec: 1.17 +[2025-04-23 18:25:53] (step=0020675) Train Loss: 5.4998, Train Steps/Sec: 1.13 +[2025-04-23 18:26:16] (step=0020700) Train Loss: 5.4903, Train Steps/Sec: 1.12 +[2025-04-23 18:26:37] (step=0020725) Train Loss: 5.4613, Train Steps/Sec: 1.18 +[2025-04-23 18:26:58] (step=0020750) Train Loss: 5.4745, Train Steps/Sec: 1.18 +[2025-04-23 18:27:19] (step=0020775) Train Loss: 5.5120, Train Steps/Sec: 1.17 +[2025-04-23 18:27:42] (step=0020800) Train Loss: 5.5091, Train Steps/Sec: 1.12 +[2025-04-23 18:28:03] (step=0020825) Train Loss: 5.5535, Train Steps/Sec: 1.18 +[2025-04-23 18:28:24] (step=0020850) Train Loss: 5.4445, Train Steps/Sec: 1.17 +[2025-04-23 18:28:45] (step=0020875) Train Loss: 5.5180, Train Steps/Sec: 1.18 +[2025-04-23 18:29:07] (step=0020900) Train Loss: 5.5108, Train Steps/Sec: 1.17 +[2025-04-23 18:29:28] (step=0020925) Train Loss: 5.4860, Train Steps/Sec: 1.18 +[2025-04-23 18:29:49] (step=0020950) Train Loss: 5.5766, Train Steps/Sec: 1.18 +[2025-04-23 18:30:11] (step=0020975) Train Loss: 5.4739, Train Steps/Sec: 1.13 +[2025-04-23 18:30:34] (step=0021000) Train Loss: 5.4721, Train Steps/Sec: 1.13 +[2025-04-23 18:30:56] (step=0021025) Train Loss: 5.4161, Train Steps/Sec: 1.12 +[2025-04-23 18:31:18] (step=0021050) Train Loss: 5.5278, Train Steps/Sec: 1.13 +[2025-04-23 18:31:39] (step=0021075) Train Loss: 5.4090, Train Steps/Sec: 1.18 +[2025-04-23 18:32:01] (step=0021100) Train Loss: 5.4616, Train Steps/Sec: 1.17 +[2025-04-23 18:32:23] (step=0021125) Train Loss: 5.4332, Train Steps/Sec: 1.14 +[2025-04-23 18:32:44] (step=0021150) Train Loss: 5.4067, Train Steps/Sec: 1.17 +[2025-04-23 18:33:05] (step=0021175) Train Loss: 5.4750, Train Steps/Sec: 1.17 +[2025-04-23 18:33:27] (step=0021200) Train Loss: 5.5103, Train Steps/Sec: 1.17 +[2025-04-23 18:33:48] (step=0021225) Train Loss: 5.5817, Train Steps/Sec: 1.18 +[2025-04-23 18:34:09] (step=0021250) Train Loss: 5.4150, Train Steps/Sec: 1.17 +[2025-04-23 18:34:31] (step=0021275) Train Loss: 5.4253, Train Steps/Sec: 1.17 +[2025-04-23 18:34:52] (step=0021300) Train Loss: 5.4396, Train Steps/Sec: 1.17 +[2025-04-23 18:35:14] (step=0021325) Train Loss: 5.5064, Train Steps/Sec: 1.13 +[2025-04-23 18:35:35] (step=0021350) Train Loss: 5.5072, Train Steps/Sec: 1.18 +[2025-04-23 18:35:57] (step=0021375) Train Loss: 5.4767, Train Steps/Sec: 1.18 +[2025-04-23 18:36:18] (step=0021400) Train Loss: 5.4136, Train Steps/Sec: 1.17 +[2025-04-23 18:36:40] (step=0021425) Train Loss: 5.4315, Train Steps/Sec: 1.13 +[2025-04-23 18:37:01] (step=0021450) Train Loss: 5.5147, Train Steps/Sec: 1.18 +[2025-04-23 18:37:23] (step=0021475) Train Loss: 5.4653, Train Steps/Sec: 1.18 +[2025-04-23 18:37:44] (step=0021500) Train Loss: 5.4821, Train Steps/Sec: 1.17 +[2025-04-23 18:38:06] (step=0021525) Train Loss: 5.4439, Train Steps/Sec: 1.13 +[2025-04-23 18:38:27] (step=0021550) Train Loss: 5.5194, Train Steps/Sec: 1.18 +[2025-04-23 18:38:49] (step=0021575) Train Loss: 5.4654, Train Steps/Sec: 1.17 +[2025-04-23 18:39:10] (step=0021600) Train Loss: 5.5274, Train Steps/Sec: 1.17 +[2025-04-23 18:39:31] (step=0021625) Train Loss: 5.4841, Train Steps/Sec: 1.18 +[2025-04-23 18:39:54] (step=0021650) Train Loss: 5.4985, Train Steps/Sec: 1.09 +[2025-04-23 18:40:15] (step=0021675) Train Loss: 5.4527, Train Steps/Sec: 1.18 +[2025-04-23 18:40:38] (step=0021700) Train Loss: 5.5049, Train Steps/Sec: 1.12 +[2025-04-23 18:41:00] (step=0021725) Train Loss: 5.4813, Train Steps/Sec: 1.13 +[2025-04-23 18:41:21] (step=0021750) Train Loss: 5.4673, Train Steps/Sec: 1.18 +[2025-04-23 18:41:43] (step=0021775) Train Loss: 5.5992, Train Steps/Sec: 1.14 +[2025-04-23 18:42:05] (step=0021800) Train Loss: 5.4940, Train Steps/Sec: 1.17 +[2025-04-23 18:42:26] (step=0021825) Train Loss: 5.4572, Train Steps/Sec: 1.18 +[2025-04-23 18:42:47] (step=0021850) Train Loss: 5.4755, Train Steps/Sec: 1.18 +[2025-04-23 18:43:08] (step=0021875) Train Loss: 5.4363, Train Steps/Sec: 1.17 +[2025-04-23 18:43:30] (step=0021900) Train Loss: 5.4521, Train Steps/Sec: 1.17 +[2025-04-23 18:43:51] (step=0021925) Train Loss: 5.5288, Train Steps/Sec: 1.18 +[2025-04-23 18:44:12] (step=0021950) Train Loss: 5.4991, Train Steps/Sec: 1.18 +[2025-04-23 18:44:34] (step=0021975) Train Loss: 5.4827, Train Steps/Sec: 1.13 +[2025-04-23 18:44:56] (step=0022000) Train Loss: 5.4695, Train Steps/Sec: 1.17 +[2025-04-23 18:44:56] text_config is None. Initializing the text config with default values (`OPTConfig`). +[2025-04-23 18:50:00] Finish Eval in 22000 steps... +[2025-04-23 18:50:18] Saved checkpoint to checkpoint/Afteri2i_100kckpt_LLAVAT5_empty_Fixed_just_segment_pretrain_stage3_dreambench_recap_400k_t2i_400k_flux_200k_midjourney_150k_recovery_150k_3e4_epoch2_half_rotated/005-GPT-XL/checkpoints/0022000.pt +[2025-04-23 18:50:20] Removed old checkpoint: checkpoint/Afteri2i_100kckpt_LLAVAT5_empty_Fixed_just_segment_pretrain_stage3_dreambench_recap_400k_t2i_400k_flux_200k_midjourney_150k_recovery_150k_3e4_epoch2_half_rotated/005-GPT-XL/checkpoints/0020000.pt +[2025-04-23 18:50:41] (step=0022025) Train Loss: 5.4840, Train Steps/Sec: 0.07 +[2025-04-23 18:51:03] (step=0022050) Train Loss: 5.4934, Train Steps/Sec: 1.17 +[2025-04-23 18:51:24] (step=0022075) Train Loss: 5.4617, Train Steps/Sec: 1.18 +[2025-04-23 18:51:45] (step=0022100) Train Loss: 5.5155, Train Steps/Sec: 1.17 +[2025-04-23 18:52:07] (step=0022125) Train Loss: 5.5130, Train Steps/Sec: 1.18 +[2025-04-23 18:52:29] (step=0022150) Train Loss: 5.4719, Train Steps/Sec: 1.13 +[2025-04-23 18:52:50] (step=0022175) Train Loss: 5.5081, Train Steps/Sec: 1.18 +[2025-04-23 18:53:11] (step=0022200) Train Loss: 5.5269, Train Steps/Sec: 1.17 +[2025-04-23 18:53:33] (step=0022225) Train Loss: 5.5090, Train Steps/Sec: 1.18 +[2025-04-23 18:53:55] (step=0022250) Train Loss: 5.4276, Train Steps/Sec: 1.13 +[2025-04-23 18:54:16] (step=0022275) Train Loss: 5.4669, Train Steps/Sec: 1.18 +[2025-04-23 18:54:38] (step=0022300) Train Loss: 5.5430, Train Steps/Sec: 1.13 +[2025-04-23 18:55:00] (step=0022325) Train Loss: 5.5019, Train Steps/Sec: 1.12 +[2025-04-23 18:55:22] (step=0022350) Train Loss: 5.4668, Train Steps/Sec: 1.13 +[2025-04-23 18:55:45] (step=0022375) Train Loss: 5.5145, Train Steps/Sec: 1.13 +[2025-04-23 18:56:06] (step=0022400) Train Loss: 5.5035, Train Steps/Sec: 1.17 +[2025-04-23 18:56:27] (step=0022425) Train Loss: 5.4885, Train Steps/Sec: 1.18 +[2025-04-23 18:56:49] (step=0022450) Train Loss: 5.5200, Train Steps/Sec: 1.14 +[2025-04-23 18:57:10] (step=0022475) Train Loss: 5.5014, Train Steps/Sec: 1.18 +[2025-04-23 18:57:32] (step=0022500) Train Loss: 5.5108, Train Steps/Sec: 1.17 +[2025-04-23 18:57:53] (step=0022525) Train Loss: 5.4219, Train Steps/Sec: 1.18 +[2025-04-23 18:58:14] (step=0022550) Train Loss: 5.4895, Train Steps/Sec: 1.18 +[2025-04-23 18:58:35] (step=0022575) Train Loss: 5.4960, Train Steps/Sec: 1.18 +[2025-04-23 18:58:57] (step=0022600) Train Loss: 5.5473, Train Steps/Sec: 1.17 +[2025-04-23 18:59:18] (step=0022625) Train Loss: 5.4619, Train Steps/Sec: 1.18 +[2025-04-23 18:59:39] (step=0022650) Train Loss: 5.3881, Train Steps/Sec: 1.18 +[2025-04-23 19:00:01] (step=0022675) Train Loss: 5.5086, Train Steps/Sec: 1.13 +[2025-04-23 19:00:23] (step=0022700) Train Loss: 5.4392, Train Steps/Sec: 1.17 +[2025-04-23 19:00:44] (step=0022725) Train Loss: 5.5065, Train Steps/Sec: 1.18 +[2025-04-23 19:01:05] (step=0022750) Train Loss: 5.5123, Train Steps/Sec: 1.18 +[2025-04-23 19:01:27] (step=0022775) Train Loss: 5.5216, Train Steps/Sec: 1.18 +[2025-04-23 19:01:48] (step=0022800) Train Loss: 5.5073, Train Steps/Sec: 1.17 +[2025-04-23 19:02:09] (step=0022825) Train Loss: 5.4362, Train Steps/Sec: 1.18 +[2025-04-23 19:02:30] (step=0022850) Train Loss: 5.4648, Train Steps/Sec: 1.18 +[2025-04-23 19:02:53] (step=0022875) Train Loss: 5.3853, Train Steps/Sec: 1.13 +[2025-04-23 19:03:14] (step=0022900) Train Loss: 5.5334, Train Steps/Sec: 1.17 +[2025-04-23 19:03:35] (step=0022925) Train Loss: 5.3682, Train Steps/Sec: 1.17 +[2025-04-23 19:03:57] (step=0022950) Train Loss: 5.4453, Train Steps/Sec: 1.13 +[2025-04-23 19:04:20] (step=0022975) Train Loss: 5.4295, Train Steps/Sec: 1.09 +[2025-04-23 19:04:42] (step=0023000) Train Loss: 5.4528, Train Steps/Sec: 1.17 +[2025-04-23 19:05:05] (step=0023025) Train Loss: 5.5646, Train Steps/Sec: 1.09 +[2025-04-23 19:05:26] (step=0023050) Train Loss: 5.5188, Train Steps/Sec: 1.18 +[2025-04-23 19:05:47] (step=0023075) Train Loss: 5.4824, Train Steps/Sec: 1.18 +[2025-04-23 19:06:09] (step=0023100) Train Loss: 5.5094, Train Steps/Sec: 1.13 +[2025-04-23 19:06:31] (step=0023125) Train Loss: 5.4275, Train Steps/Sec: 1.18 +[2025-04-23 19:06:52] (step=0023150) Train Loss: 5.5436, Train Steps/Sec: 1.18 +[2025-04-23 19:07:13] (step=0023175) Train Loss: 5.5329, Train Steps/Sec: 1.18 +[2025-04-23 19:07:34] (step=0023200) Train Loss: 5.4307, Train Steps/Sec: 1.17 +[2025-04-23 19:07:56] (step=0023225) Train Loss: 5.5028, Train Steps/Sec: 1.18 +[2025-04-23 19:08:17] (step=0023250) Train Loss: 5.4717, Train Steps/Sec: 1.18 +[2025-04-23 19:08:38] (step=0023275) Train Loss: 5.4428, Train Steps/Sec: 1.18 +[2025-04-23 19:08:59] (step=0023300) Train Loss: 5.5185, Train Steps/Sec: 1.17 +[2025-04-23 19:09:22] (step=0023325) Train Loss: 5.4731, Train Steps/Sec: 1.13 +[2025-04-23 19:09:43] (step=0023350) Train Loss: 5.4346, Train Steps/Sec: 1.17 +[2025-04-23 19:10:04] (step=0023375) Train Loss: 5.4568, Train Steps/Sec: 1.18 +[2025-04-23 19:10:26] (step=0023400) Train Loss: 5.4664, Train Steps/Sec: 1.17 +[2025-04-23 19:10:47] (step=0023425) Train Loss: 5.4016, Train Steps/Sec: 1.17 +[2025-04-23 19:11:08] (step=0023450) Train Loss: 5.4485, Train Steps/Sec: 1.18 +[2025-04-23 19:11:16] Beginning epoch 1... +[2025-04-23 19:11:34] (step=0023475) Train Loss: 5.4174, Train Steps/Sec: 0.98 +[2025-04-23 19:11:55] (step=0023500) Train Loss: 5.3740, Train Steps/Sec: 1.16 +[2025-04-23 19:12:17] (step=0023525) Train Loss: 5.4172, Train Steps/Sec: 1.17 +[2025-04-23 19:12:38] (step=0023550) Train Loss: 5.3438, Train Steps/Sec: 1.18 +[2025-04-23 19:12:59] (step=0023575) Train Loss: 5.4250, Train Steps/Sec: 1.17 +[2025-04-23 19:13:22] (step=0023600) Train Loss: 5.4292, Train Steps/Sec: 1.11 +[2025-04-23 19:13:45] (step=0023625) Train Loss: 5.5044, Train Steps/Sec: 1.07 +[2025-04-23 19:14:06] (step=0023650) Train Loss: 5.3244, Train Steps/Sec: 1.17 +[2025-04-23 19:14:29] (step=0023675) Train Loss: 5.4025, Train Steps/Sec: 1.12 +[2025-04-23 19:14:52] (step=0023700) Train Loss: 5.3761, Train Steps/Sec: 1.06 +[2025-04-23 19:15:14] (step=0023725) Train Loss: 5.4133, Train Steps/Sec: 1.17 +[2025-04-23 19:15:36] (step=0023750) Train Loss: 5.3435, Train Steps/Sec: 1.12 +[2025-04-23 19:15:57] (step=0023775) Train Loss: 5.4328, Train Steps/Sec: 1.17 +[2025-04-23 19:16:19] (step=0023800) Train Loss: 5.4057, Train Steps/Sec: 1.17 +[2025-04-23 19:16:40] (step=0023825) Train Loss: 5.4560, Train Steps/Sec: 1.17 +[2025-04-23 19:17:01] (step=0023850) Train Loss: 5.3452, Train Steps/Sec: 1.18 +[2025-04-23 19:17:23] (step=0023875) Train Loss: 5.3795, Train Steps/Sec: 1.17 +[2025-04-23 19:17:44] (step=0023900) Train Loss: 5.3339, Train Steps/Sec: 1.16 +[2025-04-23 19:18:05] (step=0023925) Train Loss: 5.4558, Train Steps/Sec: 1.17 +[2025-04-23 19:18:27] (step=0023950) Train Loss: 5.3740, Train Steps/Sec: 1.17 +[2025-04-23 19:18:49] (step=0023975) Train Loss: 5.3980, Train Steps/Sec: 1.12 +[2025-04-23 19:19:10] (step=0024000) Train Loss: 5.4179, Train Steps/Sec: 1.17 +[2025-04-23 19:19:10] text_config is None. Initializing the text config with default values (`OPTConfig`). +[2025-04-23 19:24:18] Finish Eval in 24000 steps... +[2025-04-23 19:24:37] Saved checkpoint to checkpoint/Afteri2i_100kckpt_LLAVAT5_empty_Fixed_just_segment_pretrain_stage3_dreambench_recap_400k_t2i_400k_flux_200k_midjourney_150k_recovery_150k_3e4_epoch2_half_rotated/005-GPT-XL/checkpoints/0024000.pt +[2025-04-23 19:24:39] Removed old checkpoint: checkpoint/Afteri2i_100kckpt_LLAVAT5_empty_Fixed_just_segment_pretrain_stage3_dreambench_recap_400k_t2i_400k_flux_200k_midjourney_150k_recovery_150k_3e4_epoch2_half_rotated/005-GPT-XL/checkpoints/0022000.pt +[2025-04-23 19:25:00] (step=0024025) Train Loss: 5.3983, Train Steps/Sec: 0.07 +[2025-04-23 19:25:21] (step=0024050) Train Loss: 5.3482, Train Steps/Sec: 1.18 +[2025-04-23 19:25:43] (step=0024075) Train Loss: 5.3873, Train Steps/Sec: 1.18 +[2025-04-23 19:26:04] (step=0024100) Train Loss: 5.3387, Train Steps/Sec: 1.16 +[2025-04-23 19:26:25] (step=0024125) Train Loss: 5.3996, Train Steps/Sec: 1.18 +[2025-04-23 19:26:47] (step=0024150) Train Loss: 5.3409, Train Steps/Sec: 1.17 +[2025-04-23 19:27:08] (step=0024175) Train Loss: 5.4733, Train Steps/Sec: 1.17 +[2025-04-23 19:27:29] (step=0024200) Train Loss: 5.3975, Train Steps/Sec: 1.17 +[2025-04-23 19:27:51] (step=0024225) Train Loss: 5.4183, Train Steps/Sec: 1.17 +[2025-04-23 19:28:12] (step=0024250) Train Loss: 5.3978, Train Steps/Sec: 1.17 +[2025-04-23 19:28:34] (step=0024275) Train Loss: 5.3387, Train Steps/Sec: 1.13 +[2025-04-23 19:28:56] (step=0024300) Train Loss: 5.4247, Train Steps/Sec: 1.12 +[2025-04-23 19:29:20] (step=0024325) Train Loss: 5.4288, Train Steps/Sec: 1.06 +[2025-04-23 19:29:42] (step=0024350) Train Loss: 5.4009, Train Steps/Sec: 1.13 +[2025-04-23 19:30:03] (step=0024375) Train Loss: 5.3887, Train Steps/Sec: 1.17 +[2025-04-23 19:30:25] (step=0024400) Train Loss: 5.3427, Train Steps/Sec: 1.17 +[2025-04-23 19:30:48] (step=0024425) Train Loss: 5.3827, Train Steps/Sec: 1.08 +[2025-04-23 19:31:09] (step=0024450) Train Loss: 5.4196, Train Steps/Sec: 1.17 +[2025-04-23 19:31:31] (step=0024475) Train Loss: 5.4046, Train Steps/Sec: 1.17 +[2025-04-23 19:31:52] (step=0024500) Train Loss: 5.3989, Train Steps/Sec: 1.17 +[2025-04-23 19:32:13] (step=0024525) Train Loss: 5.4122, Train Steps/Sec: 1.17 +[2025-04-23 19:32:35] (step=0024550) Train Loss: 5.4750, Train Steps/Sec: 1.17 +[2025-04-23 19:32:56] (step=0024575) Train Loss: 5.3793, Train Steps/Sec: 1.17 +[2025-04-23 19:33:17] (step=0024600) Train Loss: 5.3428, Train Steps/Sec: 1.17 +[2025-04-23 19:33:38] (step=0024625) Train Loss: 5.3406, Train Steps/Sec: 1.18 +[2025-04-23 19:34:01] (step=0024650) Train Loss: 5.3333, Train Steps/Sec: 1.13 +[2025-04-23 19:34:22] (step=0024675) Train Loss: 5.3736, Train Steps/Sec: 1.18 +[2025-04-23 19:34:43] (step=0024700) Train Loss: 5.3979, Train Steps/Sec: 1.17 +[2025-04-23 19:35:05] (step=0024725) Train Loss: 5.3841, Train Steps/Sec: 1.17 +[2025-04-23 19:35:26] (step=0024750) Train Loss: 5.4464, Train Steps/Sec: 1.17 +[2025-04-23 19:35:47] (step=0024775) Train Loss: 5.3683, Train Steps/Sec: 1.17 +[2025-04-23 19:36:09] (step=0024800) Train Loss: 5.4800, Train Steps/Sec: 1.17 +[2025-04-23 19:36:30] (step=0024825) Train Loss: 5.4058, Train Steps/Sec: 1.17 +[2025-04-23 19:36:51] (step=0024850) Train Loss: 5.3878, Train Steps/Sec: 1.17 +[2025-04-23 19:37:13] (step=0024875) Train Loss: 5.4238, Train Steps/Sec: 1.17 +[2025-04-23 19:37:34] (step=0024900) Train Loss: 5.3983, Train Steps/Sec: 1.17 +[2025-04-23 19:37:56] (step=0024925) Train Loss: 5.4086, Train Steps/Sec: 1.13 +[2025-04-23 19:38:18] (step=0024950) Train Loss: 5.4784, Train Steps/Sec: 1.13 +[2025-04-23 19:38:40] (step=0024975) Train Loss: 5.3757, Train Steps/Sec: 1.17 +[2025-04-23 19:39:02] (step=0025000) Train Loss: 5.4066, Train Steps/Sec: 1.13 +[2025-04-23 19:39:24] (step=0025025) Train Loss: 5.4579, Train Steps/Sec: 1.13 +[2025-04-23 19:39:46] (step=0025050) Train Loss: 5.3656, Train Steps/Sec: 1.12 +[2025-04-23 19:40:08] (step=0025075) Train Loss: 5.4259, Train Steps/Sec: 1.13 +[2025-04-23 19:40:30] (step=0025100) Train Loss: 5.3540, Train Steps/Sec: 1.17 +[2025-04-23 19:40:51] (step=0025125) Train Loss: 5.3896, Train Steps/Sec: 1.17 +[2025-04-23 19:41:13] (step=0025150) Train Loss: 5.4091, Train Steps/Sec: 1.12 +[2025-04-23 19:41:35] (step=0025175) Train Loss: 5.3545, Train Steps/Sec: 1.17 +[2025-04-23 19:41:56] (step=0025200) Train Loss: 5.4128, Train Steps/Sec: 1.17 +[2025-04-23 19:42:17] (step=0025225) Train Loss: 5.3742, Train Steps/Sec: 1.17 +[2025-04-23 19:42:39] (step=0025250) Train Loss: 5.3809, Train Steps/Sec: 1.18 +[2025-04-23 19:43:00] (step=0025275) Train Loss: 5.3863, Train Steps/Sec: 1.17 +[2025-04-23 19:43:21] (step=0025300) Train Loss: 5.4147, Train Steps/Sec: 1.17 +[2025-04-23 19:43:43] (step=0025325) Train Loss: 5.3152, Train Steps/Sec: 1.13 +[2025-04-23 19:44:05] (step=0025350) Train Loss: 5.4485, Train Steps/Sec: 1.17 +[2025-04-23 19:44:26] (step=0025375) Train Loss: 5.3609, Train Steps/Sec: 1.17 +[2025-04-23 19:44:47] (step=0025400) Train Loss: 5.4033, Train Steps/Sec: 1.17 +[2025-04-23 19:45:09] (step=0025425) Train Loss: 5.4181, Train Steps/Sec: 1.17 +[2025-04-23 19:45:30] (step=0025450) Train Loss: 5.4180, Train Steps/Sec: 1.17 +[2025-04-23 19:45:51] (step=0025475) Train Loss: 5.3186, Train Steps/Sec: 1.17 +[2025-04-23 19:46:13] (step=0025500) Train Loss: 5.4063, Train Steps/Sec: 1.17 +[2025-04-23 19:46:34] (step=0025525) Train Loss: 5.4142, Train Steps/Sec: 1.17 +[2025-04-23 19:46:55] (step=0025550) Train Loss: 5.3491, Train Steps/Sec: 1.17 +[2025-04-23 19:47:18] (step=0025575) Train Loss: 5.4041, Train Steps/Sec: 1.13 +[2025-04-23 19:47:40] (step=0025600) Train Loss: 5.3443, Train Steps/Sec: 1.13 +[2025-04-23 19:48:01] (step=0025625) Train Loss: 5.4332, Train Steps/Sec: 1.18 +[2025-04-23 19:48:23] (step=0025650) Train Loss: 5.4041, Train Steps/Sec: 1.13 +[2025-04-23 19:48:45] (step=0025675) Train Loss: 5.3836, Train Steps/Sec: 1.13 +[2025-04-23 19:49:07] (step=0025700) Train Loss: 5.4514, Train Steps/Sec: 1.17 +[2025-04-23 19:49:28] (step=0025725) Train Loss: 5.3894, Train Steps/Sec: 1.17 +[2025-04-23 19:49:50] (step=0025750) Train Loss: 5.3401, Train Steps/Sec: 1.13 +[2025-04-23 19:50:12] (step=0025775) Train Loss: 5.4372, Train Steps/Sec: 1.12 +[2025-04-23 19:50:34] (step=0025800) Train Loss: 5.3973, Train Steps/Sec: 1.17 +[2025-04-23 19:50:55] (step=0025825) Train Loss: 5.3967, Train Steps/Sec: 1.17 +[2025-04-23 19:51:16] (step=0025850) Train Loss: 5.3730, Train Steps/Sec: 1.17 +[2025-04-23 19:51:39] (step=0025875) Train Loss: 5.4737, Train Steps/Sec: 1.12 +[2025-04-23 19:52:00] (step=0025900) Train Loss: 5.4434, Train Steps/Sec: 1.17 +[2025-04-23 19:52:21] (step=0025925) Train Loss: 5.3944, Train Steps/Sec: 1.17 +[2025-04-23 19:52:43] (step=0025950) Train Loss: 5.3865, Train Steps/Sec: 1.17 +[2025-04-23 19:53:05] (step=0025975) Train Loss: 5.3950, Train Steps/Sec: 1.13 +[2025-04-23 19:53:26] (step=0026000) Train Loss: 5.3654, Train Steps/Sec: 1.17 +[2025-04-23 19:53:26] text_config is None. Initializing the text config with default values (`OPTConfig`). +[2025-04-23 19:58:33] Finish Eval in 26000 steps... +[2025-04-23 19:58:52] Saved checkpoint to checkpoint/Afteri2i_100kckpt_LLAVAT5_empty_Fixed_just_segment_pretrain_stage3_dreambench_recap_400k_t2i_400k_flux_200k_midjourney_150k_recovery_150k_3e4_epoch2_half_rotated/005-GPT-XL/checkpoints/0026000.pt +[2025-04-23 19:58:54] Removed old checkpoint: checkpoint/Afteri2i_100kckpt_LLAVAT5_empty_Fixed_just_segment_pretrain_stage3_dreambench_recap_400k_t2i_400k_flux_200k_midjourney_150k_recovery_150k_3e4_epoch2_half_rotated/005-GPT-XL/checkpoints/0024000.pt +[2025-04-23 19:59:16] (step=0026025) Train Loss: 5.4119, Train Steps/Sec: 0.07 +[2025-04-23 19:59:37] (step=0026050) Train Loss: 5.4172, Train Steps/Sec: 1.18 +[2025-04-23 19:59:58] (step=0026075) Train Loss: 5.4157, Train Steps/Sec: 1.18 +[2025-04-23 20:00:20] (step=0026100) Train Loss: 5.3657, Train Steps/Sec: 1.17 +[2025-04-23 20:00:41] (step=0026125) Train Loss: 5.3820, Train Steps/Sec: 1.17 +[2025-04-23 20:01:02] (step=0026150) Train Loss: 5.5298, Train Steps/Sec: 1.18 +[2025-04-23 20:01:24] (step=0026175) Train Loss: 5.3919, Train Steps/Sec: 1.17 +[2025-04-23 20:01:45] (step=0026200) Train Loss: 5.3736, Train Steps/Sec: 1.17 +[2025-04-23 20:02:06] (step=0026225) Train Loss: 5.4061, Train Steps/Sec: 1.17 +[2025-04-23 20:02:28] (step=0026250) Train Loss: 5.4054, Train Steps/Sec: 1.13 +[2025-04-23 20:02:50] (step=0026275) Train Loss: 5.4132, Train Steps/Sec: 1.13 +[2025-04-23 20:03:13] (step=0026300) Train Loss: 5.3719, Train Steps/Sec: 1.13 +[2025-04-23 20:03:35] (step=0026325) Train Loss: 5.4100, Train Steps/Sec: 1.13 +[2025-04-23 20:03:56] (step=0026350) Train Loss: 5.4325, Train Steps/Sec: 1.18 +[2025-04-23 20:04:17] (step=0026375) Train Loss: 5.3857, Train Steps/Sec: 1.17 +[2025-04-23 20:04:40] (step=0026400) Train Loss: 5.4480, Train Steps/Sec: 1.12 +[2025-04-23 20:05:01] (step=0026425) Train Loss: 5.4550, Train Steps/Sec: 1.18 +[2025-04-23 20:05:22] (step=0026450) Train Loss: 5.3832, Train Steps/Sec: 1.17 +[2025-04-23 20:05:44] (step=0026475) Train Loss: 5.3467, Train Steps/Sec: 1.17 +[2025-04-23 20:06:06] (step=0026500) Train Loss: 5.4369, Train Steps/Sec: 1.12 +[2025-04-23 20:06:27] (step=0026525) Train Loss: 5.4723, Train Steps/Sec: 1.17 +[2025-04-23 20:06:49] (step=0026550) Train Loss: 5.4152, Train Steps/Sec: 1.18 +[2025-04-23 20:07:10] (step=0026575) Train Loss: 5.3301, Train Steps/Sec: 1.17 +[2025-04-23 20:07:32] (step=0026600) Train Loss: 5.5070, Train Steps/Sec: 1.12 +[2025-04-23 20:07:54] (step=0026625) Train Loss: 5.4112, Train Steps/Sec: 1.18 +[2025-04-23 20:08:15] (step=0026650) Train Loss: 5.4180, Train Steps/Sec: 1.18 +[2025-04-23 20:08:37] (step=0026675) Train Loss: 5.4051, Train Steps/Sec: 1.13 +[2025-04-23 20:08:58] (step=0026700) Train Loss: 5.4687, Train Steps/Sec: 1.17 +[2025-04-23 20:09:20] (step=0026725) Train Loss: 5.3545, Train Steps/Sec: 1.17 +[2025-04-23 20:09:41] (step=0026750) Train Loss: 5.3963, Train Steps/Sec: 1.17 +[2025-04-23 20:10:02] (step=0026775) Train Loss: 5.4072, Train Steps/Sec: 1.17 +[2025-04-23 20:10:24] (step=0026800) Train Loss: 5.3293, Train Steps/Sec: 1.17 +[2025-04-23 20:10:45] (step=0026825) Train Loss: 5.4767, Train Steps/Sec: 1.18 +[2025-04-23 20:11:06] (step=0026850) Train Loss: 5.3216, Train Steps/Sec: 1.17 +[2025-04-23 20:11:28] (step=0026875) Train Loss: 5.3796, Train Steps/Sec: 1.17 +[2025-04-23 20:11:50] (step=0026900) Train Loss: 5.3617, Train Steps/Sec: 1.12 +[2025-04-23 20:12:12] (step=0026925) Train Loss: 5.4032, Train Steps/Sec: 1.13 +[2025-04-23 20:12:34] (step=0026950) Train Loss: 5.3905, Train Steps/Sec: 1.13 +[2025-04-23 20:12:56] (step=0026975) Train Loss: 5.4509, Train Steps/Sec: 1.17 +[2025-04-23 20:13:18] (step=0027000) Train Loss: 5.3270, Train Steps/Sec: 1.13 +[2025-04-23 20:13:39] (step=0027025) Train Loss: 5.4122, Train Steps/Sec: 1.18 +[2025-04-23 20:14:01] (step=0027050) Train Loss: 5.3150, Train Steps/Sec: 1.13 +[2025-04-23 20:14:22] (step=0027075) Train Loss: 5.3616, Train Steps/Sec: 1.17 +[2025-04-23 20:14:44] (step=0027100) Train Loss: 5.4413, Train Steps/Sec: 1.17 +[2025-04-23 20:15:05] (step=0027125) Train Loss: 5.3499, Train Steps/Sec: 1.17 +[2025-04-23 20:15:26] (step=0027150) Train Loss: 5.5160, Train Steps/Sec: 1.17 +[2025-04-23 20:15:48] (step=0027175) Train Loss: 5.4341, Train Steps/Sec: 1.17 +[2025-04-23 20:16:10] (step=0027200) Train Loss: 5.3179, Train Steps/Sec: 1.12 +[2025-04-23 20:16:31] (step=0027225) Train Loss: 5.4310, Train Steps/Sec: 1.17 +[2025-04-23 20:16:53] (step=0027250) Train Loss: 5.3930, Train Steps/Sec: 1.17 +[2025-04-23 20:17:14] (step=0027275) Train Loss: 5.3980, Train Steps/Sec: 1.17 +[2025-04-23 20:17:35] (step=0027300) Train Loss: 5.4707, Train Steps/Sec: 1.17 +[2025-04-23 20:17:58] (step=0027325) Train Loss: 5.4527, Train Steps/Sec: 1.08 +[2025-04-23 20:18:20] (step=0027350) Train Loss: 5.5001, Train Steps/Sec: 1.17 +[2025-04-23 20:18:41] (step=0027375) Train Loss: 5.3263, Train Steps/Sec: 1.17 +[2025-04-23 20:19:02] (step=0027400) Train Loss: 5.4937, Train Steps/Sec: 1.17 +[2025-04-23 20:19:24] (step=0027425) Train Loss: 5.3632, Train Steps/Sec: 1.18 +[2025-04-23 20:19:45] (step=0027450) Train Loss: 5.3613, Train Steps/Sec: 1.17 +[2025-04-23 20:20:06] (step=0027475) Train Loss: 5.3969, Train Steps/Sec: 1.17 +[2025-04-23 20:20:28] (step=0027500) Train Loss: 5.3874, Train Steps/Sec: 1.17 +[2025-04-23 20:20:49] (step=0027525) Train Loss: 5.3163, Train Steps/Sec: 1.17 +[2025-04-23 20:21:10] (step=0027550) Train Loss: 5.3904, Train Steps/Sec: 1.17 +[2025-04-23 20:21:32] (step=0027575) Train Loss: 5.3699, Train Steps/Sec: 1.13 +[2025-04-23 20:21:55] (step=0027600) Train Loss: 5.4471, Train Steps/Sec: 1.13 +[2025-04-23 20:22:17] (step=0027625) Train Loss: 5.3816, Train Steps/Sec: 1.13 +[2025-04-23 20:22:39] (step=0027650) Train Loss: 5.4404, Train Steps/Sec: 1.13 +[2025-04-23 20:23:00] (step=0027675) Train Loss: 5.4443, Train Steps/Sec: 1.17 +[2025-04-23 20:23:21] (step=0027700) Train Loss: 5.3018, Train Steps/Sec: 1.17 +[2025-04-23 20:23:44] (step=0027725) Train Loss: 5.3741, Train Steps/Sec: 1.13 +[2025-04-23 20:24:05] (step=0027750) Train Loss: 5.4134, Train Steps/Sec: 1.17 +[2025-04-23 20:24:26] (step=0027775) Train Loss: 5.4315, Train Steps/Sec: 1.17 +[2025-04-23 20:24:48] (step=0027800) Train Loss: 5.3562, Train Steps/Sec: 1.17 +[2025-04-23 20:25:09] (step=0027825) Train Loss: 5.3671, Train Steps/Sec: 1.18 +[2025-04-23 20:25:30] (step=0027850) Train Loss: 5.4406, Train Steps/Sec: 1.18 +[2025-04-23 20:25:51] (step=0027875) Train Loss: 5.4056, Train Steps/Sec: 1.17 +[2025-04-23 20:26:13] (step=0027900) Train Loss: 5.4272, Train Steps/Sec: 1.17 +[2025-04-23 20:26:35] (step=0027925) Train Loss: 5.4302, Train Steps/Sec: 1.12 +[2025-04-23 20:26:56] (step=0027950) Train Loss: 5.3244, Train Steps/Sec: 1.18 +[2025-04-23 20:27:19] (step=0027975) Train Loss: 5.4145, Train Steps/Sec: 1.13 +[2025-04-23 20:27:40] (step=0028000) Train Loss: 5.3865, Train Steps/Sec: 1.17 +[2025-04-23 20:27:40] text_config is None. Initializing the text config with default values (`OPTConfig`). +[2025-04-23 20:32:47] Finish Eval in 28000 steps... +[2025-04-23 20:33:07] Saved checkpoint to checkpoint/Afteri2i_100kckpt_LLAVAT5_empty_Fixed_just_segment_pretrain_stage3_dreambench_recap_400k_t2i_400k_flux_200k_midjourney_150k_recovery_150k_3e4_epoch2_half_rotated/005-GPT-XL/checkpoints/0028000.pt +[2025-04-23 20:33:09] Removed old checkpoint: checkpoint/Afteri2i_100kckpt_LLAVAT5_empty_Fixed_just_segment_pretrain_stage3_dreambench_recap_400k_t2i_400k_flux_200k_midjourney_150k_recovery_150k_3e4_epoch2_half_rotated/005-GPT-XL/checkpoints/0026000.pt +[2025-04-23 20:33:30] (step=0028025) Train Loss: 5.4250, Train Steps/Sec: 0.07 +[2025-04-23 20:33:52] (step=0028050) Train Loss: 5.3656, Train Steps/Sec: 1.13 +[2025-04-23 20:34:13] (step=0028075) Train Loss: 5.3309, Train Steps/Sec: 1.18 +[2025-04-23 20:34:35] (step=0028100) Train Loss: 5.4065, Train Steps/Sec: 1.17 +[2025-04-23 20:34:56] (step=0028125) Train Loss: 5.4231, Train Steps/Sec: 1.18 +[2025-04-23 20:35:17] (step=0028150) Train Loss: 5.4082, Train Steps/Sec: 1.18 +[2025-04-23 20:35:39] (step=0028175) Train Loss: 5.4198, Train Steps/Sec: 1.18 +[2025-04-23 20:36:00] (step=0028200) Train Loss: 5.3020, Train Steps/Sec: 1.17 +[2025-04-23 20:36:22] (step=0028225) Train Loss: 5.3681, Train Steps/Sec: 1.13 +[2025-04-23 20:36:44] (step=0028250) Train Loss: 5.4337, Train Steps/Sec: 1.13 +[2025-04-23 20:37:06] (step=0028275) Train Loss: 5.4037, Train Steps/Sec: 1.13 +[2025-04-23 20:37:28] (step=0028300) Train Loss: 5.3903, Train Steps/Sec: 1.13 +[2025-04-23 20:37:50] (step=0028325) Train Loss: 5.4367, Train Steps/Sec: 1.17 +[2025-04-23 20:38:11] (step=0028350) Train Loss: 5.4005, Train Steps/Sec: 1.18 +[2025-04-23 20:38:33] (step=0028375) Train Loss: 5.3581, Train Steps/Sec: 1.13 +[2025-04-23 20:38:55] (step=0028400) Train Loss: 5.3840, Train Steps/Sec: 1.17 +[2025-04-23 20:39:16] (step=0028425) Train Loss: 5.4066, Train Steps/Sec: 1.17 +[2025-04-23 20:39:37] (step=0028450) Train Loss: 5.3970, Train Steps/Sec: 1.18 +[2025-04-23 20:39:58] (step=0028475) Train Loss: 5.4128, Train Steps/Sec: 1.17 +[2025-04-23 20:40:20] (step=0028500) Train Loss: 5.4726, Train Steps/Sec: 1.17 +[2025-04-23 20:40:41] (step=0028525) Train Loss: 5.4221, Train Steps/Sec: 1.17 +[2025-04-23 20:41:02] (step=0028550) Train Loss: 5.3688, Train Steps/Sec: 1.17 +[2025-04-23 20:41:24] (step=0028575) Train Loss: 5.4715, Train Steps/Sec: 1.17 +[2025-04-23 20:41:45] (step=0028600) Train Loss: 5.4031, Train Steps/Sec: 1.16 +[2025-04-23 20:42:07] (step=0028625) Train Loss: 5.4668, Train Steps/Sec: 1.17 +[2025-04-23 20:42:29] (step=0028650) Train Loss: 5.3720, Train Steps/Sec: 1.12 +[2025-04-23 20:42:51] (step=0028675) Train Loss: 5.4440, Train Steps/Sec: 1.13 +[2025-04-23 20:43:13] (step=0028700) Train Loss: 5.3702, Train Steps/Sec: 1.17 +[2025-04-23 20:43:34] (step=0028725) Train Loss: 5.3752, Train Steps/Sec: 1.18 +[2025-04-23 20:43:55] (step=0028750) Train Loss: 5.4572, Train Steps/Sec: 1.17 +[2025-04-23 20:44:17] (step=0028775) Train Loss: 5.3379, Train Steps/Sec: 1.12 +[2025-04-23 20:44:39] (step=0028800) Train Loss: 5.2850, Train Steps/Sec: 1.17 +[2025-04-23 20:45:00] (step=0028825) Train Loss: 5.3998, Train Steps/Sec: 1.18 +[2025-04-23 20:45:21] (step=0028850) Train Loss: 5.3774, Train Steps/Sec: 1.17 +[2025-04-23 20:45:44] (step=0028875) Train Loss: 5.3579, Train Steps/Sec: 1.13 +[2025-04-23 20:46:06] (step=0028900) Train Loss: 5.3886, Train Steps/Sec: 1.12 +[2025-04-23 20:46:28] (step=0028925) Train Loss: 5.4147, Train Steps/Sec: 1.13 +[2025-04-23 20:46:49] (step=0028950) Train Loss: 5.4051, Train Steps/Sec: 1.17 +[2025-04-23 20:47:11] (step=0028975) Train Loss: 5.3700, Train Steps/Sec: 1.13 +[2025-04-23 20:47:33] (step=0029000) Train Loss: 5.3681, Train Steps/Sec: 1.17 +[2025-04-23 20:47:55] (step=0029025) Train Loss: 5.3941, Train Steps/Sec: 1.13 +[2025-04-23 20:48:16] (step=0029050) Train Loss: 5.3807, Train Steps/Sec: 1.18 +[2025-04-23 20:48:38] (step=0029075) Train Loss: 5.3420, Train Steps/Sec: 1.17 +[2025-04-23 20:48:59] (step=0029100) Train Loss: 5.3802, Train Steps/Sec: 1.17 +[2025-04-23 20:49:20] (step=0029125) Train Loss: 5.3791, Train Steps/Sec: 1.18 +[2025-04-23 20:49:41] (step=0029150) Train Loss: 5.4458, Train Steps/Sec: 1.17 +[2025-04-23 20:50:03] (step=0029175) Train Loss: 5.4045, Train Steps/Sec: 1.17 +[2025-04-23 20:50:24] (step=0029200) Train Loss: 5.3776, Train Steps/Sec: 1.17 +[2025-04-23 20:50:45] (step=0029225) Train Loss: 5.4324, Train Steps/Sec: 1.18 +[2025-04-23 20:51:07] (step=0029250) Train Loss: 5.4198, Train Steps/Sec: 1.18 +[2025-04-23 20:51:28] (step=0029275) Train Loss: 5.4694, Train Steps/Sec: 1.17 +[2025-04-23 20:51:49] (step=0029300) Train Loss: 5.4150, Train Steps/Sec: 1.17 +[2025-04-23 20:52:11] (step=0029325) Train Loss: 5.4157, Train Steps/Sec: 1.13 +[2025-04-23 20:52:33] (step=0029350) Train Loss: 5.4133, Train Steps/Sec: 1.17 +[2025-04-23 20:52:55] (step=0029375) Train Loss: 5.4303, Train Steps/Sec: 1.13 +[2025-04-23 20:53:16] (step=0029400) Train Loss: 5.4748, Train Steps/Sec: 1.17 +[2025-04-23 20:53:38] (step=0029425) Train Loss: 5.4604, Train Steps/Sec: 1.17 +[2025-04-23 20:53:59] (step=0029450) Train Loss: 5.3906, Train Steps/Sec: 1.17 +[2025-04-23 20:54:20] (step=0029475) Train Loss: 5.3669, Train Steps/Sec: 1.17 +[2025-04-23 20:54:43] (step=0029500) Train Loss: 5.3674, Train Steps/Sec: 1.12 +[2025-04-23 20:55:05] (step=0029525) Train Loss: 5.3818, Train Steps/Sec: 1.13 +[2025-04-23 20:55:26] (step=0029550) Train Loss: 5.3160, Train Steps/Sec: 1.17 +[2025-04-23 20:55:48] (step=0029575) Train Loss: 5.3156, Train Steps/Sec: 1.13 +[2025-04-23 20:56:10] (step=0029600) Train Loss: 5.3053, Train Steps/Sec: 1.13 +[2025-04-23 20:56:32] (step=0029625) Train Loss: 5.3710, Train Steps/Sec: 1.13 +[2025-04-23 20:56:54] (step=0029650) Train Loss: 5.4710, Train Steps/Sec: 1.17 +[2025-04-23 20:57:15] (step=0029675) Train Loss: 5.4087, Train Steps/Sec: 1.17 +[2025-04-23 20:57:37] (step=0029700) Train Loss: 5.3816, Train Steps/Sec: 1.13 +[2025-04-23 20:57:59] (step=0029725) Train Loss: 5.3148, Train Steps/Sec: 1.17 +[2025-04-23 20:58:20] (step=0029750) Train Loss: 5.4222, Train Steps/Sec: 1.18 +[2025-04-23 20:58:41] (step=0029775) Train Loss: 5.4219, Train Steps/Sec: 1.18 +[2025-04-23 20:59:03] (step=0029800) Train Loss: 5.3816, Train Steps/Sec: 1.17 +[2025-04-23 20:59:24] (step=0029825) Train Loss: 5.3279, Train Steps/Sec: 1.18 +[2025-04-23 20:59:45] (step=0029850) Train Loss: 5.3966, Train Steps/Sec: 1.17 +[2025-04-23 21:00:06] (step=0029875) Train Loss: 5.3477, Train Steps/Sec: 1.17 +[2025-04-23 21:00:28] (step=0029900) Train Loss: 5.3581, Train Steps/Sec: 1.17 +[2025-04-23 21:00:49] (step=0029925) Train Loss: 5.3700, Train Steps/Sec: 1.17 +[2025-04-23 21:01:11] (step=0029950) Train Loss: 5.4150, Train Steps/Sec: 1.17 +[2025-04-23 21:01:33] (step=0029975) Train Loss: 5.3065, Train Steps/Sec: 1.13 +[2025-04-23 21:01:54] (step=0030000) Train Loss: 5.4136, Train Steps/Sec: 1.17 +[2025-04-23 21:01:54] text_config is None. Initializing the text config with default values (`OPTConfig`). +[2025-04-23 21:07:00] Finish Eval in 30000 steps... +[2025-04-23 21:07:18] Saved checkpoint to checkpoint/Afteri2i_100kckpt_LLAVAT5_empty_Fixed_just_segment_pretrain_stage3_dreambench_recap_400k_t2i_400k_flux_200k_midjourney_150k_recovery_150k_3e4_epoch2_half_rotated/005-GPT-XL/checkpoints/0030000.pt +[2025-04-23 21:07:20] Removed old checkpoint: checkpoint/Afteri2i_100kckpt_LLAVAT5_empty_Fixed_just_segment_pretrain_stage3_dreambench_recap_400k_t2i_400k_flux_200k_midjourney_150k_recovery_150k_3e4_epoch2_half_rotated/005-GPT-XL/checkpoints/0028000.pt +[2025-04-23 21:07:41] (step=0030025) Train Loss: 5.4096, Train Steps/Sec: 0.07 +[2025-04-23 21:08:03] (step=0030050) Train Loss: 5.4076, Train Steps/Sec: 1.18 +[2025-04-23 21:08:24] (step=0030075) Train Loss: 5.3457, Train Steps/Sec: 1.17 +[2025-04-23 21:08:46] (step=0030100) Train Loss: 5.3870, Train Steps/Sec: 1.12 +[2025-04-23 21:09:08] (step=0030125) Train Loss: 5.4202, Train Steps/Sec: 1.17 +[2025-04-23 21:09:29] (step=0030150) Train Loss: 5.3886, Train Steps/Sec: 1.18 +[2025-04-23 21:09:50] (step=0030175) Train Loss: 5.3820, Train Steps/Sec: 1.17 +[2025-04-23 21:10:13] (step=0030200) Train Loss: 5.4192, Train Steps/Sec: 1.13 +[2025-04-23 21:10:36] (step=0030225) Train Loss: 5.4410, Train Steps/Sec: 1.08 +[2025-04-23 21:10:58] (step=0030250) Train Loss: 5.3164, Train Steps/Sec: 1.13 +[2025-04-23 21:11:20] (step=0030275) Train Loss: 5.3762, Train Steps/Sec: 1.13 +[2025-04-23 21:11:41] (step=0030300) Train Loss: 5.4445, Train Steps/Sec: 1.17 +[2025-04-23 21:12:03] (step=0030325) Train Loss: 5.3777, Train Steps/Sec: 1.17 +[2025-04-23 21:12:25] (step=0030350) Train Loss: 5.3615, Train Steps/Sec: 1.13 +[2025-04-23 21:12:46] (step=0030375) Train Loss: 5.3787, Train Steps/Sec: 1.17 +[2025-04-23 21:13:07] (step=0030400) Train Loss: 5.4333, Train Steps/Sec: 1.17 +[2025-04-23 21:13:29] (step=0030425) Train Loss: 5.3668, Train Steps/Sec: 1.17 +[2025-04-23 21:13:50] (step=0030450) Train Loss: 5.4656, Train Steps/Sec: 1.18 +[2025-04-23 21:14:11] (step=0030475) Train Loss: 5.4235, Train Steps/Sec: 1.17 +[2025-04-23 21:14:33] (step=0030500) Train Loss: 5.4513, Train Steps/Sec: 1.17 +[2025-04-23 21:14:54] (step=0030525) Train Loss: 5.4046, Train Steps/Sec: 1.17 +[2025-04-23 21:15:16] (step=0030550) Train Loss: 5.3750, Train Steps/Sec: 1.17 +[2025-04-23 21:15:37] (step=0030575) Train Loss: 5.3534, Train Steps/Sec: 1.17 +[2025-04-23 21:15:58] (step=0030600) Train Loss: 5.4231, Train Steps/Sec: 1.17 +[2025-04-23 21:16:19] (step=0030625) Train Loss: 5.3057, Train Steps/Sec: 1.17 +[2025-04-23 21:16:41] (step=0030650) Train Loss: 5.3776, Train Steps/Sec: 1.17 +[2025-04-23 21:17:03] (step=0030675) Train Loss: 5.3863, Train Steps/Sec: 1.13 +[2025-04-23 21:17:24] (step=0030700) Train Loss: 5.4075, Train Steps/Sec: 1.17 +[2025-04-23 21:17:46] (step=0030725) Train Loss: 5.3710, Train Steps/Sec: 1.17 +[2025-04-23 21:18:07] (step=0030750) Train Loss: 5.3375, Train Steps/Sec: 1.17 +[2025-04-23 21:18:28] (step=0030775) Train Loss: 5.3561, Train Steps/Sec: 1.18 +[2025-04-23 21:18:50] (step=0030800) Train Loss: 5.3653, Train Steps/Sec: 1.17 +[2025-04-23 21:19:12] (step=0030825) Train Loss: 5.4238, Train Steps/Sec: 1.12 +[2025-04-23 21:19:34] (step=0030850) Train Loss: 5.3402, Train Steps/Sec: 1.13 +[2025-04-23 21:19:56] (step=0030875) Train Loss: 5.4087, Train Steps/Sec: 1.13 +[2025-04-23 21:20:17] (step=0030900) Train Loss: 5.3780, Train Steps/Sec: 1.17 +[2025-04-23 21:20:40] (step=0030925) Train Loss: 5.3767, Train Steps/Sec: 1.13 +[2025-04-23 21:21:02] (step=0030950) Train Loss: 5.4041, Train Steps/Sec: 1.13 +[2025-04-23 21:21:24] (step=0030975) Train Loss: 5.4028, Train Steps/Sec: 1.12 +[2025-04-23 21:21:45] (step=0031000) Train Loss: 5.3647, Train Steps/Sec: 1.17 +[2025-04-23 21:22:07] (step=0031025) Train Loss: 5.3483, Train Steps/Sec: 1.13 +[2025-04-23 21:22:29] (step=0031050) Train Loss: 5.3735, Train Steps/Sec: 1.17 +[2025-04-23 21:22:50] (step=0031075) Train Loss: 5.3554, Train Steps/Sec: 1.17 +[2025-04-23 21:23:12] (step=0031100) Train Loss: 5.4916, Train Steps/Sec: 1.17 +[2025-04-23 21:23:33] (step=0031125) Train Loss: 5.4115, Train Steps/Sec: 1.17 +[2025-04-23 21:23:54] (step=0031150) Train Loss: 5.3993, Train Steps/Sec: 1.17 +[2025-04-23 21:24:15] (step=0031175) Train Loss: 5.4073, Train Steps/Sec: 1.17 +[2025-04-23 21:24:37] (step=0031200) Train Loss: 5.4690, Train Steps/Sec: 1.16 +[2025-04-23 21:24:58] (step=0031225) Train Loss: 5.3896, Train Steps/Sec: 1.17 +[2025-04-23 21:25:20] (step=0031250) Train Loss: 5.4227, Train Steps/Sec: 1.18 +[2025-04-23 21:25:41] (step=0031275) Train Loss: 5.3561, Train Steps/Sec: 1.17 +[2025-04-23 21:26:02] (step=0031300) Train Loss: 5.4647, Train Steps/Sec: 1.17 +[2025-04-23 21:26:24] (step=0031325) Train Loss: 5.4018, Train Steps/Sec: 1.13 +[2025-04-23 21:26:46] (step=0031350) Train Loss: 5.3700, Train Steps/Sec: 1.18 +[2025-04-23 21:27:07] (step=0031375) Train Loss: 5.4462, Train Steps/Sec: 1.17 +[2025-04-23 21:27:28] (step=0031400) Train Loss: 5.3412, Train Steps/Sec: 1.17 +[2025-04-23 21:27:50] (step=0031425) Train Loss: 5.3855, Train Steps/Sec: 1.17 +[2025-04-23 21:28:11] (step=0031450) Train Loss: 5.4602, Train Steps/Sec: 1.17 +[2025-04-23 21:28:32] (step=0031475) Train Loss: 5.3626, Train Steps/Sec: 1.17 +[2025-04-23 21:28:54] (step=0031500) Train Loss: 5.2661, Train Steps/Sec: 1.13 +[2025-04-23 21:29:16] (step=0031525) Train Loss: 5.4389, Train Steps/Sec: 1.17 +[2025-04-23 21:29:39] (step=0031550) Train Loss: 5.3640, Train Steps/Sec: 1.08 +[2025-04-23 21:30:01] (step=0031575) Train Loss: 5.3886, Train Steps/Sec: 1.13 +[2025-04-23 21:30:23] (step=0031600) Train Loss: 5.4189, Train Steps/Sec: 1.13 +[2025-04-23 21:30:44] (step=0031625) Train Loss: 5.3754, Train Steps/Sec: 1.18 +[2025-04-23 21:31:06] (step=0031650) Train Loss: 5.3548, Train Steps/Sec: 1.17 +[2025-04-23 21:31:28] (step=0031675) Train Loss: 5.3486, Train Steps/Sec: 1.13 +[2025-04-23 21:31:50] (step=0031700) Train Loss: 5.4211, Train Steps/Sec: 1.12 +[2025-04-23 21:32:12] (step=0031725) Train Loss: 5.3881, Train Steps/Sec: 1.17 +[2025-04-23 21:32:33] (step=0031750) Train Loss: 5.3662, Train Steps/Sec: 1.18 +[2025-04-23 21:32:54] (step=0031775) Train Loss: 5.4181, Train Steps/Sec: 1.17 +[2025-04-23 21:33:16] (step=0031800) Train Loss: 5.4754, Train Steps/Sec: 1.17 +[2025-04-23 21:33:37] (step=0031825) Train Loss: 5.4031, Train Steps/Sec: 1.18 +[2025-04-23 21:33:58] (step=0031850) Train Loss: 5.4572, Train Steps/Sec: 1.17 +[2025-04-23 21:34:20] (step=0031875) Train Loss: 5.4129, Train Steps/Sec: 1.17 +[2025-04-23 21:34:41] (step=0031900) Train Loss: 5.3975, Train Steps/Sec: 1.17 +[2025-04-23 21:35:02] (step=0031925) Train Loss: 5.3735, Train Steps/Sec: 1.17 +[2025-04-23 21:35:24] (step=0031950) Train Loss: 5.3686, Train Steps/Sec: 1.17 +[2025-04-23 21:35:46] (step=0031975) Train Loss: 5.4216, Train Steps/Sec: 1.13 +[2025-04-23 21:36:07] (step=0032000) Train Loss: 5.3514, Train Steps/Sec: 1.17 +[2025-04-23 21:36:07] text_config is None. Initializing the text config with default values (`OPTConfig`). +[2025-04-23 21:41:15] Finish Eval in 32000 steps... +[2025-04-23 21:41:33] Saved checkpoint to checkpoint/Afteri2i_100kckpt_LLAVAT5_empty_Fixed_just_segment_pretrain_stage3_dreambench_recap_400k_t2i_400k_flux_200k_midjourney_150k_recovery_150k_3e4_epoch2_half_rotated/005-GPT-XL/checkpoints/0032000.pt +[2025-04-23 21:41:34] Removed old checkpoint: checkpoint/Afteri2i_100kckpt_LLAVAT5_empty_Fixed_just_segment_pretrain_stage3_dreambench_recap_400k_t2i_400k_flux_200k_midjourney_150k_recovery_150k_3e4_epoch2_half_rotated/005-GPT-XL/checkpoints/0030000.pt +[2025-04-23 21:41:56] (step=0032025) Train Loss: 5.4417, Train Steps/Sec: 0.07 +[2025-04-23 21:42:17] (step=0032050) Train Loss: 5.3834, Train Steps/Sec: 1.18 +[2025-04-23 21:42:38] (step=0032075) Train Loss: 5.3745, Train Steps/Sec: 1.17 +[2025-04-23 21:43:00] (step=0032100) Train Loss: 5.3968, Train Steps/Sec: 1.17 +[2025-04-23 21:43:21] (step=0032125) Train Loss: 5.3523, Train Steps/Sec: 1.17 +[2025-04-23 21:43:42] (step=0032150) Train Loss: 5.4187, Train Steps/Sec: 1.18 +[2025-04-23 21:44:04] (step=0032175) Train Loss: 5.3914, Train Steps/Sec: 1.13 +[2025-04-23 21:44:27] (step=0032200) Train Loss: 5.4057, Train Steps/Sec: 1.12 +[2025-04-23 21:44:49] (step=0032225) Train Loss: 5.4031, Train Steps/Sec: 1.13 +[2025-04-23 21:45:10] (step=0032250) Train Loss: 5.3868, Train Steps/Sec: 1.18 +[2025-04-23 21:45:33] (step=0032275) Train Loss: 5.3993, Train Steps/Sec: 1.08 +[2025-04-23 21:45:55] (step=0032300) Train Loss: 5.4455, Train Steps/Sec: 1.17 +[2025-04-23 21:46:17] (step=0032325) Train Loss: 5.3910, Train Steps/Sec: 1.13 +[2025-04-23 21:46:38] (step=0032350) Train Loss: 5.4280, Train Steps/Sec: 1.17 +[2025-04-23 21:46:59] (step=0032375) Train Loss: 5.3548, Train Steps/Sec: 1.18 +[2025-04-23 21:47:21] (step=0032400) Train Loss: 5.4170, Train Steps/Sec: 1.17 +[2025-04-23 21:47:43] (step=0032425) Train Loss: 5.3917, Train Steps/Sec: 1.12 +[2025-04-23 21:48:04] (step=0032450) Train Loss: 5.4098, Train Steps/Sec: 1.16 +[2025-04-23 21:48:26] (step=0032475) Train Loss: 5.4718, Train Steps/Sec: 1.17 +[2025-04-23 21:48:47] (step=0032500) Train Loss: 5.4001, Train Steps/Sec: 1.17 +[2025-04-23 21:49:08] (step=0032525) Train Loss: 5.4274, Train Steps/Sec: 1.18 +[2025-04-23 21:49:30] (step=0032550) Train Loss: 5.3383, Train Steps/Sec: 1.17 +[2025-04-23 21:49:51] (step=0032575) Train Loss: 5.3653, Train Steps/Sec: 1.18 +[2025-04-23 21:50:12] (step=0032600) Train Loss: 5.3794, Train Steps/Sec: 1.17 +[2025-04-23 21:50:34] (step=0032625) Train Loss: 5.3595, Train Steps/Sec: 1.18 +[2025-04-23 21:50:56] (step=0032650) Train Loss: 5.2826, Train Steps/Sec: 1.13 +[2025-04-23 21:51:17] (step=0032675) Train Loss: 5.3793, Train Steps/Sec: 1.17 +[2025-04-23 21:51:38] (step=0032700) Train Loss: 5.3620, Train Steps/Sec: 1.17 +[2025-04-23 21:52:00] (step=0032725) Train Loss: 5.3940, Train Steps/Sec: 1.18 +[2025-04-23 21:52:21] (step=0032750) Train Loss: 5.3954, Train Steps/Sec: 1.17 +[2025-04-23 21:52:42] (step=0032775) Train Loss: 5.4081, Train Steps/Sec: 1.17 +[2025-04-23 21:53:04] (step=0032800) Train Loss: 5.4412, Train Steps/Sec: 1.17 +[2025-04-23 21:53:26] (step=0032825) Train Loss: 5.3997, Train Steps/Sec: 1.13 +[2025-04-23 21:53:47] (step=0032850) Train Loss: 5.4044, Train Steps/Sec: 1.17 +[2025-04-23 21:54:09] (step=0032875) Train Loss: 5.3911, Train Steps/Sec: 1.13 +[2025-04-23 21:54:32] (step=0032900) Train Loss: 5.3904, Train Steps/Sec: 1.13 +[2025-04-23 21:54:54] (step=0032925) Train Loss: 5.3529, Train Steps/Sec: 1.13 +[2025-04-23 21:55:15] (step=0032950) Train Loss: 5.3633, Train Steps/Sec: 1.17 +[2025-04-23 21:55:36] (step=0032975) Train Loss: 5.4431, Train Steps/Sec: 1.17 +[2025-04-23 21:55:59] (step=0033000) Train Loss: 5.3853, Train Steps/Sec: 1.12 +[2025-04-23 21:56:20] (step=0033025) Train Loss: 5.4077, Train Steps/Sec: 1.17 +[2025-04-23 21:56:41] (step=0033050) Train Loss: 5.4354, Train Steps/Sec: 1.17 +[2025-04-23 21:57:03] (step=0033075) Train Loss: 5.3923, Train Steps/Sec: 1.17 +[2025-04-23 21:57:24] (step=0033100) Train Loss: 5.4223, Train Steps/Sec: 1.17 +[2025-04-23 21:57:45] (step=0033125) Train Loss: 5.3510, Train Steps/Sec: 1.18 +[2025-04-23 21:58:08] (step=0033150) Train Loss: 5.3875, Train Steps/Sec: 1.12 +[2025-04-23 21:58:29] (step=0033175) Train Loss: 5.4351, Train Steps/Sec: 1.18 +[2025-04-23 21:58:50] (step=0033200) Train Loss: 5.4083, Train Steps/Sec: 1.17 +[2025-04-23 21:59:11] (step=0033225) Train Loss: 5.3731, Train Steps/Sec: 1.17 +[2025-04-23 21:59:33] (step=0033250) Train Loss: 5.3368, Train Steps/Sec: 1.17 +[2025-04-23 21:59:54] (step=0033275) Train Loss: 5.3518, Train Steps/Sec: 1.17 +[2025-04-23 22:00:15] (step=0033300) Train Loss: 5.3395, Train Steps/Sec: 1.17 +[2025-04-23 22:00:38] (step=0033325) Train Loss: 5.4656, Train Steps/Sec: 1.13 +[2025-04-23 22:00:59] (step=0033350) Train Loss: 5.3995, Train Steps/Sec: 1.17 +[2025-04-23 22:01:20] (step=0033375) Train Loss: 5.2446, Train Steps/Sec: 1.17 +[2025-04-23 22:01:42] (step=0033400) Train Loss: 5.4042, Train Steps/Sec: 1.17 +[2025-04-23 22:02:03] (step=0033425) Train Loss: 5.4499, Train Steps/Sec: 1.18 +[2025-04-23 22:02:24] (step=0033450) Train Loss: 5.3568, Train Steps/Sec: 1.17 +[2025-04-23 22:02:46] (step=0033475) Train Loss: 5.4555, Train Steps/Sec: 1.14 +[2025-04-23 22:03:08] (step=0033500) Train Loss: 5.4428, Train Steps/Sec: 1.17 +[2025-04-23 22:03:30] (step=0033525) Train Loss: 5.3390, Train Steps/Sec: 1.13 +[2025-04-23 22:03:52] (step=0033550) Train Loss: 5.4137, Train Steps/Sec: 1.13 +[2025-04-23 22:04:14] (step=0033575) Train Loss: 5.4014, Train Steps/Sec: 1.13 +[2025-04-23 22:04:35] (step=0033600) Train Loss: 5.3448, Train Steps/Sec: 1.17 +[2025-04-23 22:04:57] (step=0033625) Train Loss: 5.3621, Train Steps/Sec: 1.18 +[2025-04-23 22:05:19] (step=0033650) Train Loss: 5.4091, Train Steps/Sec: 1.13 +[2025-04-23 22:05:40] (step=0033675) Train Loss: 5.4435, Train Steps/Sec: 1.17 +[2025-04-23 22:06:01] (step=0033700) Train Loss: 5.3973, Train Steps/Sec: 1.17 +[2025-04-23 22:06:24] (step=0033725) Train Loss: 5.3883, Train Steps/Sec: 1.12 +[2025-04-23 22:06:45] (step=0033750) Train Loss: 5.3613, Train Steps/Sec: 1.17 +[2025-04-23 22:07:06] (step=0033775) Train Loss: 5.4016, Train Steps/Sec: 1.17 +[2025-04-23 22:07:28] (step=0033800) Train Loss: 5.4425, Train Steps/Sec: 1.17 +[2025-04-23 22:07:49] (step=0033825) Train Loss: 5.3679, Train Steps/Sec: 1.17 +[2025-04-23 22:08:10] (step=0033850) Train Loss: 5.3571, Train Steps/Sec: 1.17 +[2025-04-23 22:08:33] (step=0033875) Train Loss: 5.4002, Train Steps/Sec: 1.12 +[2025-04-23 22:08:54] (step=0033900) Train Loss: 5.3632, Train Steps/Sec: 1.17 +[2025-04-23 22:09:15] (step=0033925) Train Loss: 5.4580, Train Steps/Sec: 1.17 +[2025-04-23 22:09:37] (step=0033950) Train Loss: 5.3472, Train Steps/Sec: 1.17 +[2025-04-23 22:09:59] (step=0033975) Train Loss: 5.4271, Train Steps/Sec: 1.13 +[2025-04-23 22:10:20] (step=0034000) Train Loss: 5.3878, Train Steps/Sec: 1.17 +[2025-04-23 22:10:20] text_config is None. Initializing the text config with default values (`OPTConfig`). +[2025-04-23 22:15:24] Finish Eval in 34000 steps... +[2025-04-23 22:15:42] Saved checkpoint to checkpoint/Afteri2i_100kckpt_LLAVAT5_empty_Fixed_just_segment_pretrain_stage3_dreambench_recap_400k_t2i_400k_flux_200k_midjourney_150k_recovery_150k_3e4_epoch2_half_rotated/005-GPT-XL/checkpoints/0034000.pt +[2025-04-23 22:15:44] Removed old checkpoint: checkpoint/Afteri2i_100kckpt_LLAVAT5_empty_Fixed_just_segment_pretrain_stage3_dreambench_recap_400k_t2i_400k_flux_200k_midjourney_150k_recovery_150k_3e4_epoch2_half_rotated/005-GPT-XL/checkpoints/0032000.pt +[2025-04-23 22:16:05] (step=0034025) Train Loss: 5.4371, Train Steps/Sec: 0.07 +[2025-04-23 22:16:27] (step=0034050) Train Loss: 5.2774, Train Steps/Sec: 1.18 +[2025-04-23 22:16:48] (step=0034075) Train Loss: 5.4196, Train Steps/Sec: 1.17 +[2025-04-23 22:17:09] (step=0034100) Train Loss: 5.3792, Train Steps/Sec: 1.17 +[2025-04-23 22:17:30] (step=0034125) Train Loss: 5.3469, Train Steps/Sec: 1.17 +[2025-04-23 22:17:53] (step=0034150) Train Loss: 5.3619, Train Steps/Sec: 1.13 +[2025-04-23 22:18:15] (step=0034175) Train Loss: 5.4123, Train Steps/Sec: 1.13 +[2025-04-23 22:18:36] (step=0034200) Train Loss: 5.3822, Train Steps/Sec: 1.17 +[2025-04-23 22:18:58] (step=0034225) Train Loss: 5.3243, Train Steps/Sec: 1.13 +[2025-04-23 22:19:20] (step=0034250) Train Loss: 5.3464, Train Steps/Sec: 1.13 +[2025-04-23 22:19:42] (step=0034275) Train Loss: 5.4133, Train Steps/Sec: 1.17 +[2025-04-23 22:20:04] (step=0034300) Train Loss: 5.4281, Train Steps/Sec: 1.12 +[2025-04-23 22:20:25] (step=0034325) Train Loss: 5.4190, Train Steps/Sec: 1.17 +[2025-04-23 22:20:47] (step=0034350) Train Loss: 5.4853, Train Steps/Sec: 1.17 +[2025-04-23 22:21:08] (step=0034375) Train Loss: 5.4432, Train Steps/Sec: 1.17 +[2025-04-23 22:21:29] (step=0034400) Train Loss: 5.2998, Train Steps/Sec: 1.17 +[2025-04-23 22:21:51] (step=0034425) Train Loss: 5.3369, Train Steps/Sec: 1.17 +[2025-04-23 22:22:13] (step=0034450) Train Loss: 5.3543, Train Steps/Sec: 1.12 +[2025-04-23 22:22:34] (step=0034475) Train Loss: 5.4444, Train Steps/Sec: 1.17 +[2025-04-23 22:22:56] (step=0034500) Train Loss: 5.3747, Train Steps/Sec: 1.17 +[2025-04-23 22:23:17] (step=0034525) Train Loss: 5.3406, Train Steps/Sec: 1.18 +[2025-04-23 22:23:38] (step=0034550) Train Loss: 5.3214, Train Steps/Sec: 1.17 +[2025-04-23 22:24:00] (step=0034575) Train Loss: 5.3612, Train Steps/Sec: 1.18 +[2025-04-23 22:24:22] (step=0034600) Train Loss: 5.4126, Train Steps/Sec: 1.12 +[2025-04-23 22:24:43] (step=0034625) Train Loss: 5.4218, Train Steps/Sec: 1.18 +[2025-04-23 22:25:05] (step=0034650) Train Loss: 5.4286, Train Steps/Sec: 1.13 +[2025-04-23 22:25:27] (step=0034675) Train Loss: 5.3075, Train Steps/Sec: 1.17 +[2025-04-23 22:25:48] (step=0034700) Train Loss: 5.3645, Train Steps/Sec: 1.17 +[2025-04-23 22:26:09] (step=0034725) Train Loss: 5.4036, Train Steps/Sec: 1.17 +[2025-04-23 22:26:31] (step=0034750) Train Loss: 5.3741, Train Steps/Sec: 1.17 +[2025-04-23 22:26:52] (step=0034775) Train Loss: 5.4892, Train Steps/Sec: 1.17 +[2025-04-23 22:27:14] (step=0034800) Train Loss: 5.3743, Train Steps/Sec: 1.13 +[2025-04-23 22:27:36] (step=0034825) Train Loss: 5.3553, Train Steps/Sec: 1.17 +[2025-04-23 22:27:58] (step=0034850) Train Loss: 5.4150, Train Steps/Sec: 1.13 +[2025-04-23 22:28:20] (step=0034875) Train Loss: 5.3872, Train Steps/Sec: 1.13 +[2025-04-23 22:28:42] (step=0034900) Train Loss: 5.4290, Train Steps/Sec: 1.13 +[2025-04-23 22:29:03] (step=0034925) Train Loss: 5.2310, Train Steps/Sec: 1.17 +[2025-04-23 22:29:25] (step=0034950) Train Loss: 5.3820, Train Steps/Sec: 1.17 +[2025-04-23 22:29:47] (step=0034975) Train Loss: 5.3105, Train Steps/Sec: 1.13 +[2025-04-23 22:30:08] (step=0035000) Train Loss: 5.4366, Train Steps/Sec: 1.17 +[2025-04-23 22:30:29] (step=0035025) Train Loss: 5.3499, Train Steps/Sec: 1.17 +[2025-04-23 22:30:51] (step=0035050) Train Loss: 5.4256, Train Steps/Sec: 1.17 +[2025-04-23 22:31:12] (step=0035075) Train Loss: 5.4234, Train Steps/Sec: 1.18 +[2025-04-23 22:31:33] (step=0035100) Train Loss: 5.3765, Train Steps/Sec: 1.17 +[2025-04-23 22:31:55] (step=0035125) Train Loss: 5.4680, Train Steps/Sec: 1.17 +[2025-04-23 22:32:16] (step=0035150) Train Loss: 5.4583, Train Steps/Sec: 1.17 +[2025-04-23 22:32:38] (step=0035175) Train Loss: 5.3711, Train Steps/Sec: 1.13 +[2025-04-23 22:33:00] (step=0035200) Train Loss: 5.4463, Train Steps/Sec: 1.17 +[2025-04-23 22:33:21] (step=0035225) Train Loss: 5.3599, Train Steps/Sec: 1.17 +[2025-04-23 22:33:42] (step=0035250) Train Loss: 5.3913, Train Steps/Sec: 1.17 +[2025-04-23 22:34:04] (step=0035275) Train Loss: 5.4934, Train Steps/Sec: 1.18 +[2025-04-23 22:34:25] (step=0035300) Train Loss: 5.4533, Train Steps/Sec: 1.17 +[2025-04-23 22:34:48] (step=0035325) Train Loss: 5.3313, Train Steps/Sec: 1.08 +[2025-04-23 22:35:09] (step=0035350) Train Loss: 5.4213, Train Steps/Sec: 1.17 +[2025-04-23 22:35:31] (step=0035375) Train Loss: 5.3647, Train Steps/Sec: 1.18 +[2025-04-23 22:35:52] (step=0035400) Train Loss: 5.4382, Train Steps/Sec: 1.17 +[2025-04-23 22:36:13] (step=0035425) Train Loss: 5.3685, Train Steps/Sec: 1.18 +[2025-04-23 22:36:35] (step=0035450) Train Loss: 5.3200, Train Steps/Sec: 1.13 +[2025-04-23 22:36:57] (step=0035475) Train Loss: 5.3561, Train Steps/Sec: 1.18 +[2025-04-23 22:37:19] (step=0035500) Train Loss: 5.3949, Train Steps/Sec: 1.13 +[2025-04-23 22:37:41] (step=0035525) Train Loss: 5.4336, Train Steps/Sec: 1.13 +[2025-04-23 22:38:03] (step=0035550) Train Loss: 5.3783, Train Steps/Sec: 1.13 +[2025-04-23 22:38:24] (step=0035575) Train Loss: 5.3440, Train Steps/Sec: 1.17 +[2025-04-23 22:38:46] (step=0035600) Train Loss: 5.3149, Train Steps/Sec: 1.17 +[2025-04-23 22:39:08] (step=0035625) Train Loss: 5.4315, Train Steps/Sec: 1.13 +[2025-04-23 22:39:29] (step=0035650) Train Loss: 5.4243, Train Steps/Sec: 1.18 +[2025-04-23 22:39:50] (step=0035675) Train Loss: 5.3682, Train Steps/Sec: 1.17 +[2025-04-23 22:40:12] (step=0035700) Train Loss: 5.3575, Train Steps/Sec: 1.17 +[2025-04-23 22:40:33] (step=0035725) Train Loss: 5.3390, Train Steps/Sec: 1.17 +[2025-04-23 22:40:54] (step=0035750) Train Loss: 5.4031, Train Steps/Sec: 1.18 +[2025-04-23 22:41:16] (step=0035775) Train Loss: 5.3380, Train Steps/Sec: 1.18 +[2025-04-23 22:41:37] (step=0035800) Train Loss: 5.3929, Train Steps/Sec: 1.17 +[2025-04-23 22:41:58] (step=0035825) Train Loss: 5.3560, Train Steps/Sec: 1.17 +[2025-04-23 22:42:20] (step=0035850) Train Loss: 5.3544, Train Steps/Sec: 1.17 +[2025-04-23 22:42:41] (step=0035875) Train Loss: 5.3694, Train Steps/Sec: 1.18 +[2025-04-23 22:43:03] (step=0035900) Train Loss: 5.3476, Train Steps/Sec: 1.12 +[2025-04-23 22:43:25] (step=0035925) Train Loss: 5.4198, Train Steps/Sec: 1.17 +[2025-04-23 22:43:46] (step=0035950) Train Loss: 5.3488, Train Steps/Sec: 1.18 +[2025-04-23 22:44:08] (step=0035975) Train Loss: 5.2390, Train Steps/Sec: 1.13 +[2025-04-23 22:44:29] (step=0036000) Train Loss: 5.3828, Train Steps/Sec: 1.17 +[2025-04-23 22:44:29] text_config is None. Initializing the text config with default values (`OPTConfig`). +[2025-04-23 22:49:36] Finish Eval in 36000 steps... +[2025-04-23 22:49:54] Saved checkpoint to checkpoint/Afteri2i_100kckpt_LLAVAT5_empty_Fixed_just_segment_pretrain_stage3_dreambench_recap_400k_t2i_400k_flux_200k_midjourney_150k_recovery_150k_3e4_epoch2_half_rotated/005-GPT-XL/checkpoints/0036000.pt +[2025-04-23 22:49:56] Removed old checkpoint: checkpoint/Afteri2i_100kckpt_LLAVAT5_empty_Fixed_just_segment_pretrain_stage3_dreambench_recap_400k_t2i_400k_flux_200k_midjourney_150k_recovery_150k_3e4_epoch2_half_rotated/005-GPT-XL/checkpoints/0034000.pt +[2025-04-23 22:50:17] (step=0036025) Train Loss: 5.3790, Train Steps/Sec: 0.07 +[2025-04-23 22:50:39] (step=0036050) Train Loss: 5.3539, Train Steps/Sec: 1.13 +[2025-04-23 22:51:01] (step=0036075) Train Loss: 5.3567, Train Steps/Sec: 1.18 +[2025-04-23 22:51:22] (step=0036100) Train Loss: 5.3533, Train Steps/Sec: 1.17 +[2025-04-23 22:51:44] (step=0036125) Train Loss: 5.4074, Train Steps/Sec: 1.13 +[2025-04-23 22:52:06] (step=0036150) Train Loss: 5.3441, Train Steps/Sec: 1.13 +[2025-04-23 22:52:27] (step=0036175) Train Loss: 5.3494, Train Steps/Sec: 1.18 +[2025-04-23 22:52:50] (step=0036200) Train Loss: 5.4470, Train Steps/Sec: 1.12 +[2025-04-23 22:53:12] (step=0036225) Train Loss: 5.3582, Train Steps/Sec: 1.13 +[2025-04-23 22:53:33] (step=0036250) Train Loss: 5.3350, Train Steps/Sec: 1.18 +[2025-04-23 22:53:55] (step=0036275) Train Loss: 5.4208, Train Steps/Sec: 1.13 +[2025-04-23 22:54:17] (step=0036300) Train Loss: 5.3479, Train Steps/Sec: 1.16 +[2025-04-23 22:54:38] (step=0036325) Train Loss: 5.3859, Train Steps/Sec: 1.18 +[2025-04-23 22:54:59] (step=0036350) Train Loss: 5.4346, Train Steps/Sec: 1.18 +[2025-04-23 22:55:20] (step=0036375) Train Loss: 5.3912, Train Steps/Sec: 1.18 +[2025-04-23 22:55:42] (step=0036400) Train Loss: 5.4131, Train Steps/Sec: 1.17 +[2025-04-23 22:56:03] (step=0036425) Train Loss: 5.4164, Train Steps/Sec: 1.17 +[2025-04-23 22:56:24] (step=0036450) Train Loss: 5.3990, Train Steps/Sec: 1.17 +[2025-04-23 22:56:46] (step=0036475) Train Loss: 5.3417, Train Steps/Sec: 1.17 +[2025-04-23 22:57:07] (step=0036500) Train Loss: 5.3738, Train Steps/Sec: 1.17 +[2025-04-23 22:57:28] (step=0036525) Train Loss: 5.2784, Train Steps/Sec: 1.17 +[2025-04-23 22:57:50] (step=0036550) Train Loss: 5.3725, Train Steps/Sec: 1.18 +[2025-04-23 22:58:11] (step=0036575) Train Loss: 5.3762, Train Steps/Sec: 1.17 +[2025-04-23 22:58:32] (step=0036600) Train Loss: 5.4691, Train Steps/Sec: 1.17 +[2025-04-23 22:58:55] (step=0036625) Train Loss: 5.3568, Train Steps/Sec: 1.12 +[2025-04-23 22:59:17] (step=0036650) Train Loss: 5.3300, Train Steps/Sec: 1.13 +[2025-04-23 22:59:38] (step=0036675) Train Loss: 5.3973, Train Steps/Sec: 1.17 +[2025-04-23 23:00:00] (step=0036700) Train Loss: 5.4537, Train Steps/Sec: 1.17 +[2025-04-23 23:00:21] (step=0036725) Train Loss: 5.3412, Train Steps/Sec: 1.17 +[2025-04-23 23:00:42] (step=0036750) Train Loss: 5.4091, Train Steps/Sec: 1.17 +[2025-04-23 23:01:05] (step=0036775) Train Loss: 5.3964, Train Steps/Sec: 1.08 +[2025-04-23 23:01:27] (step=0036800) Train Loss: 5.4315, Train Steps/Sec: 1.16 +[2025-04-23 23:01:49] (step=0036825) Train Loss: 5.3615, Train Steps/Sec: 1.13 +[2025-04-23 23:02:11] (step=0036850) Train Loss: 5.3504, Train Steps/Sec: 1.13 +[2025-04-23 23:02:33] (step=0036875) Train Loss: 5.3631, Train Steps/Sec: 1.13 +[2025-04-23 23:02:55] (step=0036900) Train Loss: 5.4203, Train Steps/Sec: 1.17 +[2025-04-23 23:03:16] (step=0036925) Train Loss: 5.3840, Train Steps/Sec: 1.18 +[2025-04-23 23:03:38] (step=0036950) Train Loss: 5.3981, Train Steps/Sec: 1.13 +[2025-04-23 23:03:59] (step=0036975) Train Loss: 5.3261, Train Steps/Sec: 1.17 +[2025-04-23 23:04:21] (step=0037000) Train Loss: 5.4062, Train Steps/Sec: 1.17 +[2025-04-23 23:04:42] (step=0037025) Train Loss: 5.3930, Train Steps/Sec: 1.17 +[2025-04-23 23:05:03] (step=0037050) Train Loss: 5.4281, Train Steps/Sec: 1.18 +[2025-04-23 23:05:25] (step=0037075) Train Loss: 5.3868, Train Steps/Sec: 1.17 +[2025-04-23 23:05:46] (step=0037100) Train Loss: 5.4008, Train Steps/Sec: 1.17 +[2025-04-23 23:06:07] (step=0037125) Train Loss: 5.3622, Train Steps/Sec: 1.17 +[2025-04-23 23:06:29] (step=0037150) Train Loss: 5.3278, Train Steps/Sec: 1.18 +[2025-04-23 23:06:50] (step=0037175) Train Loss: 5.4297, Train Steps/Sec: 1.17 +[2025-04-23 23:07:11] (step=0037200) Train Loss: 5.4077, Train Steps/Sec: 1.17 +[2025-04-23 23:07:33] (step=0037225) Train Loss: 5.4055, Train Steps/Sec: 1.17 +[2025-04-23 23:07:54] (step=0037250) Train Loss: 5.3539, Train Steps/Sec: 1.17 +[2025-04-23 23:08:15] (step=0037275) Train Loss: 5.3229, Train Steps/Sec: 1.17 +[2025-04-23 23:08:37] (step=0037300) Train Loss: 5.3898, Train Steps/Sec: 1.17 +[2025-04-23 23:08:59] (step=0037325) Train Loss: 5.4445, Train Steps/Sec: 1.13 +[2025-04-23 23:09:21] (step=0037350) Train Loss: 5.3467, Train Steps/Sec: 1.12 +[2025-04-23 23:09:42] (step=0037375) Train Loss: 5.3916, Train Steps/Sec: 1.17 +[2025-04-23 23:10:04] (step=0037400) Train Loss: 5.4618, Train Steps/Sec: 1.17 +[2025-04-23 23:10:26] (step=0037425) Train Loss: 5.4282, Train Steps/Sec: 1.13 +[2025-04-23 23:10:47] (step=0037450) Train Loss: 5.4454, Train Steps/Sec: 1.17 +[2025-04-23 23:11:09] (step=0037475) Train Loss: 5.3074, Train Steps/Sec: 1.13 +[2025-04-23 23:11:32] (step=0037500) Train Loss: 5.3737, Train Steps/Sec: 1.08 +[2025-04-23 23:11:54] (step=0037525) Train Loss: 5.4218, Train Steps/Sec: 1.18 +[2025-04-23 23:12:16] (step=0037550) Train Loss: 5.4343, Train Steps/Sec: 1.13 +[2025-04-23 23:12:37] (step=0037575) Train Loss: 5.3233, Train Steps/Sec: 1.18 +[2025-04-23 23:12:59] (step=0037600) Train Loss: 5.3864, Train Steps/Sec: 1.13 +[2025-04-23 23:13:20] (step=0037625) Train Loss: 5.4162, Train Steps/Sec: 1.17 +[2025-04-23 23:13:42] (step=0037650) Train Loss: 5.3804, Train Steps/Sec: 1.18 +[2025-04-23 23:14:03] (step=0037675) Train Loss: 5.3594, Train Steps/Sec: 1.17 +[2025-04-23 23:14:24] (step=0037700) Train Loss: 5.4147, Train Steps/Sec: 1.17 +[2025-04-23 23:14:46] (step=0037725) Train Loss: 5.4446, Train Steps/Sec: 1.17 +[2025-04-23 23:15:07] (step=0037750) Train Loss: 5.4378, Train Steps/Sec: 1.17 +[2025-04-23 23:15:28] (step=0037775) Train Loss: 5.4126, Train Steps/Sec: 1.17 +[2025-04-23 23:15:50] (step=0037800) Train Loss: 5.3200, Train Steps/Sec: 1.17 +[2025-04-23 23:16:11] (step=0037825) Train Loss: 5.3891, Train Steps/Sec: 1.17 +[2025-04-23 23:16:32] (step=0037850) Train Loss: 5.3157, Train Steps/Sec: 1.17 +[2025-04-23 23:16:54] (step=0037875) Train Loss: 5.3974, Train Steps/Sec: 1.17 +[2025-04-23 23:17:15] (step=0037900) Train Loss: 5.3568, Train Steps/Sec: 1.17 +[2025-04-23 23:17:36] (step=0037925) Train Loss: 5.4022, Train Steps/Sec: 1.17 +[2025-04-23 23:17:58] (step=0037950) Train Loss: 5.3916, Train Steps/Sec: 1.17 +[2025-04-23 23:18:20] (step=0037975) Train Loss: 5.3339, Train Steps/Sec: 1.13 +[2025-04-23 23:18:41] (step=0038000) Train Loss: 5.3766, Train Steps/Sec: 1.17 +[2025-04-23 23:18:41] text_config is None. Initializing the text config with default values (`OPTConfig`). +[2025-04-23 23:23:47] Finish Eval in 38000 steps... +[2025-04-23 23:24:07] Saved checkpoint to checkpoint/Afteri2i_100kckpt_LLAVAT5_empty_Fixed_just_segment_pretrain_stage3_dreambench_recap_400k_t2i_400k_flux_200k_midjourney_150k_recovery_150k_3e4_epoch2_half_rotated/005-GPT-XL/checkpoints/0038000.pt +[2025-04-23 23:24:09] Removed old checkpoint: checkpoint/Afteri2i_100kckpt_LLAVAT5_empty_Fixed_just_segment_pretrain_stage3_dreambench_recap_400k_t2i_400k_flux_200k_midjourney_150k_recovery_150k_3e4_epoch2_half_rotated/005-GPT-XL/checkpoints/0036000.pt +[2025-04-23 23:24:30] (step=0038025) Train Loss: 5.3590, Train Steps/Sec: 0.07 +[2025-04-23 23:24:51] (step=0038050) Train Loss: 5.3531, Train Steps/Sec: 1.17 +[2025-04-23 23:25:15] (step=0038075) Train Loss: 5.3644, Train Steps/Sec: 1.08 +[2025-04-23 23:25:36] (step=0038100) Train Loss: 5.3416, Train Steps/Sec: 1.17 +[2025-04-23 23:25:58] (step=0038125) Train Loss: 5.3045, Train Steps/Sec: 1.13 +[2025-04-23 23:26:19] (step=0038150) Train Loss: 5.4094, Train Steps/Sec: 1.18 +[2025-04-23 23:26:41] (step=0038175) Train Loss: 5.3995, Train Steps/Sec: 1.13 +[2025-04-23 23:27:04] (step=0038200) Train Loss: 5.3725, Train Steps/Sec: 1.13 +[2025-04-23 23:27:26] (step=0038225) Train Loss: 5.3797, Train Steps/Sec: 1.11 +[2025-04-23 23:27:47] (step=0038250) Train Loss: 5.3976, Train Steps/Sec: 1.18 +[2025-04-23 23:28:09] (step=0038275) Train Loss: 5.4130, Train Steps/Sec: 1.13 +[2025-04-23 23:28:31] (step=0038300) Train Loss: 5.3126, Train Steps/Sec: 1.17 +[2025-04-23 23:28:52] (step=0038325) Train Loss: 5.4095, Train Steps/Sec: 1.17 +[2025-04-23 23:29:13] (step=0038350) Train Loss: 5.3758, Train Steps/Sec: 1.18 +[2025-04-23 23:29:35] (step=0038375) Train Loss: 5.4124, Train Steps/Sec: 1.17 +[2025-04-23 23:29:56] (step=0038400) Train Loss: 5.4171, Train Steps/Sec: 1.17 +[2025-04-23 23:30:17] (step=0038425) Train Loss: 5.4059, Train Steps/Sec: 1.17 +[2025-04-23 23:30:39] (step=0038450) Train Loss: 5.3667, Train Steps/Sec: 1.17 +[2025-04-23 23:31:00] (step=0038475) Train Loss: 5.3690, Train Steps/Sec: 1.17 +[2025-04-23 23:31:21] (step=0038500) Train Loss: 5.3078, Train Steps/Sec: 1.17 +[2025-04-23 23:31:43] (step=0038525) Train Loss: 5.3146, Train Steps/Sec: 1.18 +[2025-04-23 23:32:04] (step=0038550) Train Loss: 5.3358, Train Steps/Sec: 1.17 +[2025-04-23 23:32:25] (step=0038575) Train Loss: 5.3623, Train Steps/Sec: 1.17 +[2025-04-23 23:32:47] (step=0038600) Train Loss: 5.3100, Train Steps/Sec: 1.17 +[2025-04-23 23:33:08] (step=0038625) Train Loss: 5.3963, Train Steps/Sec: 1.18 +[2025-04-23 23:33:30] (step=0038650) Train Loss: 5.3278, Train Steps/Sec: 1.12 +[2025-04-23 23:33:52] (step=0038675) Train Loss: 5.3523, Train Steps/Sec: 1.18 +[2025-04-23 23:34:13] (step=0038700) Train Loss: 5.3518, Train Steps/Sec: 1.17 +[2025-04-23 23:34:34] (step=0038725) Train Loss: 5.3352, Train Steps/Sec: 1.18 +[2025-04-23 23:34:56] (step=0038750) Train Loss: 5.2914, Train Steps/Sec: 1.13 +[2025-04-23 23:35:19] (step=0038775) Train Loss: 5.3661, Train Steps/Sec: 1.12 +[2025-04-23 23:35:41] (step=0038800) Train Loss: 5.2478, Train Steps/Sec: 1.12 +[2025-04-23 23:36:03] (step=0038825) Train Loss: 5.3765, Train Steps/Sec: 1.13 +[2025-04-23 23:36:25] (step=0038850) Train Loss: 5.3632, Train Steps/Sec: 1.13 +[2025-04-23 23:36:47] (step=0038875) Train Loss: 5.4172, Train Steps/Sec: 1.17 +[2025-04-23 23:37:08] (step=0038900) Train Loss: 5.4116, Train Steps/Sec: 1.17 +[2025-04-23 23:37:30] (step=0038925) Train Loss: 5.3879, Train Steps/Sec: 1.13 +[2025-04-23 23:37:52] (step=0038950) Train Loss: 5.3630, Train Steps/Sec: 1.13 +[2025-04-23 23:38:14] (step=0038975) Train Loss: 5.4462, Train Steps/Sec: 1.18 +[2025-04-23 23:38:35] (step=0039000) Train Loss: 5.5286, Train Steps/Sec: 1.17 +[2025-04-23 23:38:56] (step=0039025) Train Loss: 5.4227, Train Steps/Sec: 1.18 +[2025-04-23 23:39:18] (step=0039050) Train Loss: 5.3918, Train Steps/Sec: 1.17 +[2025-04-23 23:39:39] (step=0039075) Train Loss: 5.3715, Train Steps/Sec: 1.17 +[2025-04-23 23:40:00] (step=0039100) Train Loss: 5.3515, Train Steps/Sec: 1.17 +[2025-04-23 23:40:21] (step=0039125) Train Loss: 5.3846, Train Steps/Sec: 1.18 +[2025-04-23 23:40:43] (step=0039150) Train Loss: 5.3918, Train Steps/Sec: 1.17 +[2025-04-23 23:41:04] (step=0039175) Train Loss: 5.3484, Train Steps/Sec: 1.17 +[2025-04-23 23:41:26] (step=0039200) Train Loss: 5.3339, Train Steps/Sec: 1.17 +[2025-04-23 23:41:47] (step=0039225) Train Loss: 5.3835, Train Steps/Sec: 1.17 +[2025-04-23 23:42:08] (step=0039250) Train Loss: 5.4423, Train Steps/Sec: 1.17 +[2025-04-23 23:42:29] (step=0039275) Train Loss: 5.4178, Train Steps/Sec: 1.17 +[2025-04-23 23:42:51] (step=0039300) Train Loss: 5.3216, Train Steps/Sec: 1.17 +[2025-04-23 23:43:13] (step=0039325) Train Loss: 5.3191, Train Steps/Sec: 1.13 +[2025-04-23 23:43:34] (step=0039350) Train Loss: 5.3856, Train Steps/Sec: 1.17 +[2025-04-23 23:43:56] (step=0039375) Train Loss: 5.3545, Train Steps/Sec: 1.17 +[2025-04-23 23:44:18] (step=0039400) Train Loss: 5.4015, Train Steps/Sec: 1.13 +[2025-04-23 23:44:39] (step=0039425) Train Loss: 5.3459, Train Steps/Sec: 1.18 +[2025-04-23 23:45:01] (step=0039450) Train Loss: 5.2856, Train Steps/Sec: 1.13 +[2025-04-23 23:45:23] (step=0039475) Train Loss: 5.3826, Train Steps/Sec: 1.18 +[2025-04-23 23:45:46] (step=0039500) Train Loss: 5.3143, Train Steps/Sec: 1.08 +[2025-04-23 23:46:08] (step=0039525) Train Loss: 5.3453, Train Steps/Sec: 1.13 +[2025-04-23 23:46:29] (step=0039550) Train Loss: 5.3632, Train Steps/Sec: 1.17 +[2025-04-23 23:46:51] (step=0039575) Train Loss: 5.3582, Train Steps/Sec: 1.12 +[2025-04-23 23:47:13] (step=0039600) Train Loss: 5.4030, Train Steps/Sec: 1.17 +[2025-04-23 23:47:34] (step=0039625) Train Loss: 5.3576, Train Steps/Sec: 1.17 +[2025-04-23 23:47:55] (step=0039650) Train Loss: 5.4064, Train Steps/Sec: 1.17 +[2025-04-23 23:48:18] (step=0039675) Train Loss: 5.3791, Train Steps/Sec: 1.12 +[2025-04-23 23:48:39] (step=0039700) Train Loss: 5.3443, Train Steps/Sec: 1.17 +[2025-04-23 23:49:00] (step=0039725) Train Loss: 5.4084, Train Steps/Sec: 1.17 +[2025-04-23 23:49:22] (step=0039750) Train Loss: 5.2893, Train Steps/Sec: 1.17 +[2025-04-23 23:49:43] (step=0039775) Train Loss: 5.3140, Train Steps/Sec: 1.17 +[2025-04-23 23:50:05] (step=0039800) Train Loss: 5.3849, Train Steps/Sec: 1.17 +[2025-04-23 23:50:26] (step=0039825) Train Loss: 5.3833, Train Steps/Sec: 1.17 +[2025-04-23 23:50:47] (step=0039850) Train Loss: 5.3417, Train Steps/Sec: 1.18 +[2025-04-23 23:51:08] (step=0039875) Train Loss: 5.3438, Train Steps/Sec: 1.18 +[2025-04-23 23:51:30] (step=0039900) Train Loss: 5.3542, Train Steps/Sec: 1.17 +[2025-04-23 23:51:51] (step=0039925) Train Loss: 5.3098, Train Steps/Sec: 1.17 +[2025-04-23 23:52:12] (step=0039950) Train Loss: 5.3280, Train Steps/Sec: 1.17 +[2025-04-23 23:52:35] (step=0039975) Train Loss: 5.4016, Train Steps/Sec: 1.13 +[2025-04-23 23:52:56] (step=0040000) Train Loss: 5.3193, Train Steps/Sec: 1.17 +[2025-04-23 23:52:56] text_config is None. Initializing the text config with default values (`OPTConfig`). +[2025-04-23 23:58:01] Finish Eval in 40000 steps... +[2025-04-23 23:58:19] Saved checkpoint to checkpoint/Afteri2i_100kckpt_LLAVAT5_empty_Fixed_just_segment_pretrain_stage3_dreambench_recap_400k_t2i_400k_flux_200k_midjourney_150k_recovery_150k_3e4_epoch2_half_rotated/005-GPT-XL/checkpoints/0040000.pt +[2025-04-23 23:58:21] Removed old checkpoint: checkpoint/Afteri2i_100kckpt_LLAVAT5_empty_Fixed_just_segment_pretrain_stage3_dreambench_recap_400k_t2i_400k_flux_200k_midjourney_150k_recovery_150k_3e4_epoch2_half_rotated/005-GPT-XL/checkpoints/0038000.pt +[2025-04-23 23:58:42] (step=0040025) Train Loss: 5.4095, Train Steps/Sec: 0.07 +[2025-04-23 23:59:04] (step=0040050) Train Loss: 5.2760, Train Steps/Sec: 1.13 +[2025-04-23 23:59:25] (step=0040075) Train Loss: 5.4388, Train Steps/Sec: 1.18 +[2025-04-23 23:59:47] (step=0040100) Train Loss: 5.3207, Train Steps/Sec: 1.17 +[2025-04-24 00:00:09] (step=0040125) Train Loss: 5.3305, Train Steps/Sec: 1.13 +[2025-04-24 00:00:31] (step=0040150) Train Loss: 5.3170, Train Steps/Sec: 1.12 +[2025-04-24 00:00:53] (step=0040175) Train Loss: 5.3081, Train Steps/Sec: 1.13 +[2025-04-24 00:01:15] (step=0040200) Train Loss: 5.4608, Train Steps/Sec: 1.17 +[2025-04-24 00:01:37] (step=0040225) Train Loss: 5.2915, Train Steps/Sec: 1.12 +[2025-04-24 00:01:59] (step=0040250) Train Loss: 5.3805, Train Steps/Sec: 1.13 +[2025-04-24 00:02:20] (step=0040275) Train Loss: 5.3991, Train Steps/Sec: 1.17 +[2025-04-24 00:02:42] (step=0040300) Train Loss: 5.3909, Train Steps/Sec: 1.17 +[2025-04-24 00:03:03] (step=0040325) Train Loss: 5.3365, Train Steps/Sec: 1.17 +[2025-04-24 00:03:24] (step=0040350) Train Loss: 5.3318, Train Steps/Sec: 1.18 +[2025-04-24 00:03:46] (step=0040375) Train Loss: 5.4299, Train Steps/Sec: 1.17 +[2025-04-24 00:04:08] (step=0040400) Train Loss: 5.3854, Train Steps/Sec: 1.12 +[2025-04-24 00:04:29] (step=0040425) Train Loss: 5.3985, Train Steps/Sec: 1.18 +[2025-04-24 00:04:51] (step=0040450) Train Loss: 5.3305, Train Steps/Sec: 1.18 +[2025-04-24 00:05:12] (step=0040475) Train Loss: 5.4187, Train Steps/Sec: 1.18 +[2025-04-24 00:05:33] (step=0040500) Train Loss: 5.3614, Train Steps/Sec: 1.17 +[2025-04-24 00:05:54] (step=0040525) Train Loss: 5.3796, Train Steps/Sec: 1.18 +[2025-04-24 00:06:16] (step=0040550) Train Loss: 5.3545, Train Steps/Sec: 1.17 +[2025-04-24 00:06:37] (step=0040575) Train Loss: 5.4509, Train Steps/Sec: 1.17 +[2025-04-24 00:06:58] (step=0040600) Train Loss: 5.4385, Train Steps/Sec: 1.17 +[2025-04-24 00:07:20] (step=0040625) Train Loss: 5.3605, Train Steps/Sec: 1.17 +[2025-04-24 00:07:42] (step=0040650) Train Loss: 5.3840, Train Steps/Sec: 1.13 +[2025-04-24 00:08:03] (step=0040675) Train Loss: 5.4537, Train Steps/Sec: 1.17 +[2025-04-24 00:08:25] (step=0040700) Train Loss: 5.3196, Train Steps/Sec: 1.13 +[2025-04-24 00:08:47] (step=0040725) Train Loss: 5.3378, Train Steps/Sec: 1.17 +[2025-04-24 00:09:08] (step=0040750) Train Loss: 5.4329, Train Steps/Sec: 1.17 +[2025-04-24 00:09:30] (step=0040775) Train Loss: 5.3876, Train Steps/Sec: 1.13 +[2025-04-24 00:09:52] (step=0040800) Train Loss: 5.3579, Train Steps/Sec: 1.17 +[2025-04-24 00:10:14] (step=0040825) Train Loss: 5.3172, Train Steps/Sec: 1.09 +[2025-04-24 00:10:36] (step=0040850) Train Loss: 5.3524, Train Steps/Sec: 1.17 +[2025-04-24 00:10:57] (step=0040875) Train Loss: 5.3476, Train Steps/Sec: 1.17 +[2025-04-24 00:11:19] (step=0040900) Train Loss: 5.3729, Train Steps/Sec: 1.12 +[2025-04-24 00:11:41] (step=0040925) Train Loss: 5.4439, Train Steps/Sec: 1.17 +[2025-04-24 00:12:03] (step=0040950) Train Loss: 5.2763, Train Steps/Sec: 1.13 +[2025-04-24 00:12:24] (step=0040975) Train Loss: 5.4001, Train Steps/Sec: 1.17 +[2025-04-24 00:12:46] (step=0041000) Train Loss: 5.3769, Train Steps/Sec: 1.17 +[2025-04-24 00:13:07] (step=0041025) Train Loss: 5.4464, Train Steps/Sec: 1.18 +[2025-04-24 00:13:28] (step=0041050) Train Loss: 5.3393, Train Steps/Sec: 1.17 +[2025-04-24 00:13:49] (step=0041075) Train Loss: 5.3334, Train Steps/Sec: 1.18 +[2025-04-24 00:14:11] (step=0041100) Train Loss: 5.3468, Train Steps/Sec: 1.17 +[2025-04-24 00:14:33] (step=0041125) Train Loss: 5.3493, Train Steps/Sec: 1.13 +[2025-04-24 00:14:54] (step=0041150) Train Loss: 5.3088, Train Steps/Sec: 1.17 +[2025-04-24 00:15:16] (step=0041175) Train Loss: 5.3304, Train Steps/Sec: 1.17 +[2025-04-24 00:15:37] (step=0041200) Train Loss: 5.3949, Train Steps/Sec: 1.17 +[2025-04-24 00:15:58] (step=0041225) Train Loss: 5.3861, Train Steps/Sec: 1.17 +[2025-04-24 00:16:20] (step=0041250) Train Loss: 5.4214, Train Steps/Sec: 1.18 +[2025-04-24 00:16:41] (step=0041275) Train Loss: 5.3366, Train Steps/Sec: 1.17 +[2025-04-24 00:17:02] (step=0041300) Train Loss: 5.3624, Train Steps/Sec: 1.17 +[2025-04-24 00:17:24] (step=0041325) Train Loss: 5.4542, Train Steps/Sec: 1.13 +[2025-04-24 00:17:46] (step=0041350) Train Loss: 5.4167, Train Steps/Sec: 1.17 +[2025-04-24 00:18:08] (step=0041375) Train Loss: 5.3460, Train Steps/Sec: 1.13 +[2025-04-24 00:18:29] (step=0041400) Train Loss: 5.3523, Train Steps/Sec: 1.17 +[2025-04-24 00:18:51] (step=0041425) Train Loss: 5.4027, Train Steps/Sec: 1.13 +[2025-04-24 00:19:13] (step=0041450) Train Loss: 5.2846, Train Steps/Sec: 1.18 +[2025-04-24 00:19:35] (step=0041475) Train Loss: 5.3474, Train Steps/Sec: 1.13 +[2025-04-24 00:19:57] (step=0041500) Train Loss: 5.3155, Train Steps/Sec: 1.13 +[2025-04-24 00:20:18] (step=0041525) Train Loss: 5.3846, Train Steps/Sec: 1.18 +[2025-04-24 00:20:40] (step=0041550) Train Loss: 5.3794, Train Steps/Sec: 1.13 +[2025-04-24 00:21:02] (step=0041575) Train Loss: 5.3146, Train Steps/Sec: 1.17 +[2025-04-24 00:21:23] (step=0041600) Train Loss: 5.4006, Train Steps/Sec: 1.17 +[2025-04-24 00:21:44] (step=0041625) Train Loss: 5.3050, Train Steps/Sec: 1.18 +[2025-04-24 00:22:06] (step=0041650) Train Loss: 5.3408, Train Steps/Sec: 1.17 +[2025-04-24 00:22:28] (step=0041675) Train Loss: 5.3566, Train Steps/Sec: 1.13 +[2025-04-24 00:22:49] (step=0041700) Train Loss: 5.4591, Train Steps/Sec: 1.17 +[2025-04-24 00:23:10] (step=0041725) Train Loss: 5.3439, Train Steps/Sec: 1.18 +[2025-04-24 00:23:32] (step=0041750) Train Loss: 5.3379, Train Steps/Sec: 1.18 +[2025-04-24 00:23:53] (step=0041775) Train Loss: 5.3794, Train Steps/Sec: 1.17 +[2025-04-24 00:24:14] (step=0041800) Train Loss: 5.3415, Train Steps/Sec: 1.17 +[2025-04-24 00:24:36] (step=0041825) Train Loss: 5.3305, Train Steps/Sec: 1.17 +[2025-04-24 00:24:58] (step=0041850) Train Loss: 5.3871, Train Steps/Sec: 1.13 +[2025-04-24 00:25:19] (step=0041875) Train Loss: 5.3249, Train Steps/Sec: 1.18 +[2025-04-24 00:25:40] (step=0041900) Train Loss: 5.3297, Train Steps/Sec: 1.17 +[2025-04-24 00:26:02] (step=0041925) Train Loss: 5.3853, Train Steps/Sec: 1.17 +[2025-04-24 00:26:23] (step=0041950) Train Loss: 5.4272, Train Steps/Sec: 1.17 +[2025-04-24 00:26:45] (step=0041975) Train Loss: 5.3421, Train Steps/Sec: 1.13 +[2025-04-24 00:27:07] (step=0042000) Train Loss: 5.3171, Train Steps/Sec: 1.17 +[2025-04-24 00:27:07] text_config is None. Initializing the text config with default values (`OPTConfig`). +[2025-04-24 00:32:14] Finish Eval in 42000 steps... +[2025-04-24 00:32:32] Saved checkpoint to checkpoint/Afteri2i_100kckpt_LLAVAT5_empty_Fixed_just_segment_pretrain_stage3_dreambench_recap_400k_t2i_400k_flux_200k_midjourney_150k_recovery_150k_3e4_epoch2_half_rotated/005-GPT-XL/checkpoints/0042000.pt +[2025-04-24 00:32:34] Removed old checkpoint: checkpoint/Afteri2i_100kckpt_LLAVAT5_empty_Fixed_just_segment_pretrain_stage3_dreambench_recap_400k_t2i_400k_flux_200k_midjourney_150k_recovery_150k_3e4_epoch2_half_rotated/005-GPT-XL/checkpoints/0040000.pt +[2025-04-24 00:32:56] (step=0042025) Train Loss: 5.3470, Train Steps/Sec: 0.07 +[2025-04-24 00:33:17] (step=0042050) Train Loss: 5.4467, Train Steps/Sec: 1.18 +[2025-04-24 00:33:39] (step=0042075) Train Loss: 5.3431, Train Steps/Sec: 1.17 +[2025-04-24 00:34:01] (step=0042100) Train Loss: 5.3825, Train Steps/Sec: 1.13 +[2025-04-24 00:34:23] (step=0042125) Train Loss: 5.3305, Train Steps/Sec: 1.13 +[2025-04-24 00:34:45] (step=0042150) Train Loss: 5.3939, Train Steps/Sec: 1.13 +[2025-04-24 00:35:06] (step=0042175) Train Loss: 5.4662, Train Steps/Sec: 1.17 +[2025-04-24 00:35:28] (step=0042200) Train Loss: 5.3089, Train Steps/Sec: 1.17 +[2025-04-24 00:35:50] (step=0042225) Train Loss: 5.3543, Train Steps/Sec: 1.13 +[2025-04-24 00:36:11] (step=0042250) Train Loss: 5.4207, Train Steps/Sec: 1.17 +[2025-04-24 00:36:33] (step=0042275) Train Loss: 5.4137, Train Steps/Sec: 1.17 +[2025-04-24 00:36:54] (step=0042300) Train Loss: 5.4344, Train Steps/Sec: 1.16 +[2025-04-24 00:37:15] (step=0042325) Train Loss: 5.2703, Train Steps/Sec: 1.18 +[2025-04-24 00:37:37] (step=0042350) Train Loss: 5.3861, Train Steps/Sec: 1.17 +[2025-04-24 00:37:58] (step=0042375) Train Loss: 5.4347, Train Steps/Sec: 1.18 +[2025-04-24 00:38:20] (step=0042400) Train Loss: 5.3871, Train Steps/Sec: 1.12 +[2025-04-24 00:38:42] (step=0042425) Train Loss: 5.4073, Train Steps/Sec: 1.17 +[2025-04-24 00:39:03] (step=0042450) Train Loss: 5.3576, Train Steps/Sec: 1.17 +[2025-04-24 00:39:24] (step=0042475) Train Loss: 5.3949, Train Steps/Sec: 1.17 +[2025-04-24 00:39:46] (step=0042500) Train Loss: 5.3970, Train Steps/Sec: 1.17 +[2025-04-24 00:40:07] (step=0042525) Train Loss: 5.3570, Train Steps/Sec: 1.18 +[2025-04-24 00:40:28] (step=0042550) Train Loss: 5.3524, Train Steps/Sec: 1.17 +[2025-04-24 00:40:50] (step=0042575) Train Loss: 5.3619, Train Steps/Sec: 1.13 +[2025-04-24 00:41:12] (step=0042600) Train Loss: 5.3608, Train Steps/Sec: 1.17 +[2025-04-24 00:41:33] (step=0042625) Train Loss: 5.4109, Train Steps/Sec: 1.17 +[2025-04-24 00:41:55] (step=0042650) Train Loss: 5.3627, Train Steps/Sec: 1.13 +[2025-04-24 00:42:16] (step=0042675) Train Loss: 5.3263, Train Steps/Sec: 1.18 +[2025-04-24 00:42:39] (step=0042700) Train Loss: 5.4153, Train Steps/Sec: 1.12 +[2025-04-24 00:43:00] (step=0042725) Train Loss: 5.3874, Train Steps/Sec: 1.17 +[2025-04-24 00:43:22] (step=0042750) Train Loss: 5.3222, Train Steps/Sec: 1.12 +[2025-04-24 00:43:44] (step=0042775) Train Loss: 5.3801, Train Steps/Sec: 1.18 +[2025-04-24 00:44:07] (step=0042800) Train Loss: 5.4633, Train Steps/Sec: 1.09 +[2025-04-24 00:44:28] (step=0042825) Train Loss: 5.3519, Train Steps/Sec: 1.17 +[2025-04-24 00:44:49] (step=0042850) Train Loss: 5.4045, Train Steps/Sec: 1.18 +[2025-04-24 00:45:11] (step=0042875) Train Loss: 5.4304, Train Steps/Sec: 1.13 +[2025-04-24 00:45:33] (step=0042900) Train Loss: 5.3971, Train Steps/Sec: 1.17 +[2025-04-24 00:45:54] (step=0042925) Train Loss: 5.3408, Train Steps/Sec: 1.17 +[2025-04-24 00:46:15] (step=0042950) Train Loss: 5.3681, Train Steps/Sec: 1.17 +[2025-04-24 00:46:37] (step=0042975) Train Loss: 5.2992, Train Steps/Sec: 1.17 +[2025-04-24 00:46:58] (step=0043000) Train Loss: 5.3539, Train Steps/Sec: 1.17 +[2025-04-24 00:47:19] (step=0043025) Train Loss: 5.4103, Train Steps/Sec: 1.17 +[2025-04-24 00:47:41] (step=0043050) Train Loss: 5.3787, Train Steps/Sec: 1.17 +[2025-04-24 00:48:02] (step=0043075) Train Loss: 5.3696, Train Steps/Sec: 1.17 +[2025-04-24 00:48:23] (step=0043100) Train Loss: 5.4051, Train Steps/Sec: 1.17 +[2025-04-24 00:48:46] (step=0043125) Train Loss: 5.4003, Train Steps/Sec: 1.12 +[2025-04-24 00:49:07] (step=0043150) Train Loss: 5.4427, Train Steps/Sec: 1.18 +[2025-04-24 00:49:28] (step=0043175) Train Loss: 5.3010, Train Steps/Sec: 1.18 +[2025-04-24 00:49:49] (step=0043200) Train Loss: 5.3214, Train Steps/Sec: 1.17 +[2025-04-24 00:50:11] (step=0043225) Train Loss: 5.3099, Train Steps/Sec: 1.18 +[2025-04-24 00:50:32] (step=0043250) Train Loss: 5.3984, Train Steps/Sec: 1.17 +[2025-04-24 00:50:53] (step=0043275) Train Loss: 5.3031, Train Steps/Sec: 1.17 +[2025-04-24 00:51:16] (step=0043300) Train Loss: 5.3809, Train Steps/Sec: 1.17 +[2025-04-24 00:51:38] (step=0043325) Train Loss: 5.3812, Train Steps/Sec: 1.12 +[2025-04-24 00:52:00] (step=0043350) Train Loss: 5.4348, Train Steps/Sec: 1.13 +[2025-04-24 00:52:21] (step=0043375) Train Loss: 5.2752, Train Steps/Sec: 1.17 +[2025-04-24 00:52:44] (step=0043400) Train Loss: 5.3548, Train Steps/Sec: 1.13 +[2025-04-24 00:53:05] (step=0043425) Train Loss: 5.2957, Train Steps/Sec: 1.17 +[2025-04-24 00:53:27] (step=0043450) Train Loss: 5.4098, Train Steps/Sec: 1.13 +[2025-04-24 00:53:49] (step=0043475) Train Loss: 5.4316, Train Steps/Sec: 1.13 +[2025-04-24 00:54:11] (step=0043500) Train Loss: 5.3742, Train Steps/Sec: 1.17 +[2025-04-24 00:54:32] (step=0043525) Train Loss: 5.3088, Train Steps/Sec: 1.18 +[2025-04-24 00:54:54] (step=0043550) Train Loss: 5.3163, Train Steps/Sec: 1.13 +[2025-04-24 00:55:15] (step=0043575) Train Loss: 5.3271, Train Steps/Sec: 1.17 +[2025-04-24 00:55:37] (step=0043600) Train Loss: 5.3670, Train Steps/Sec: 1.16 +[2025-04-24 00:55:58] (step=0043625) Train Loss: 5.3834, Train Steps/Sec: 1.17 +[2025-04-24 00:56:19] (step=0043650) Train Loss: 5.3511, Train Steps/Sec: 1.17 +[2025-04-24 00:56:41] (step=0043675) Train Loss: 5.3287, Train Steps/Sec: 1.17 +[2025-04-24 00:57:02] (step=0043700) Train Loss: 5.3573, Train Steps/Sec: 1.17 +[2025-04-24 00:57:23] (step=0043725) Train Loss: 5.2892, Train Steps/Sec: 1.18 +[2025-04-24 00:57:45] (step=0043750) Train Loss: 5.4193, Train Steps/Sec: 1.17 +[2025-04-24 00:58:06] (step=0043775) Train Loss: 5.3373, Train Steps/Sec: 1.17 +[2025-04-24 00:58:27] (step=0043800) Train Loss: 5.4285, Train Steps/Sec: 1.17 +[2025-04-24 00:58:49] (step=0043825) Train Loss: 5.3405, Train Steps/Sec: 1.17 +[2025-04-24 00:59:11] (step=0043850) Train Loss: 5.3053, Train Steps/Sec: 1.12 +[2025-04-24 00:59:32] (step=0043875) Train Loss: 5.3937, Train Steps/Sec: 1.18 +[2025-04-24 00:59:54] (step=0043900) Train Loss: 5.3497, Train Steps/Sec: 1.17 +[2025-04-24 01:00:15] (step=0043925) Train Loss: 5.3421, Train Steps/Sec: 1.18 +[2025-04-24 01:00:36] (step=0043950) Train Loss: 5.3362, Train Steps/Sec: 1.18 +[2025-04-24 01:00:58] (step=0043975) Train Loss: 5.4042, Train Steps/Sec: 1.13 +[2025-04-24 01:01:20] (step=0044000) Train Loss: 5.3956, Train Steps/Sec: 1.13 +[2025-04-24 01:01:20] text_config is None. Initializing the text config with default values (`OPTConfig`). +[2025-04-24 01:06:25] Finish Eval in 44000 steps... +[2025-04-24 01:06:43] Saved checkpoint to checkpoint/Afteri2i_100kckpt_LLAVAT5_empty_Fixed_just_segment_pretrain_stage3_dreambench_recap_400k_t2i_400k_flux_200k_midjourney_150k_recovery_150k_3e4_epoch2_half_rotated/005-GPT-XL/checkpoints/0044000.pt +[2025-04-24 01:06:45] Removed old checkpoint: checkpoint/Afteri2i_100kckpt_LLAVAT5_empty_Fixed_just_segment_pretrain_stage3_dreambench_recap_400k_t2i_400k_flux_200k_midjourney_150k_recovery_150k_3e4_epoch2_half_rotated/005-GPT-XL/checkpoints/0042000.pt +[2025-04-24 01:07:07] (step=0044025) Train Loss: 5.2977, Train Steps/Sec: 0.07 +[2025-04-24 01:07:29] (step=0044050) Train Loss: 5.3354, Train Steps/Sec: 1.12 +[2025-04-24 01:07:51] (step=0044075) Train Loss: 5.3169, Train Steps/Sec: 1.13 +[2025-04-24 01:08:13] (step=0044100) Train Loss: 5.4186, Train Steps/Sec: 1.13 +[2025-04-24 01:08:35] (step=0044125) Train Loss: 5.3317, Train Steps/Sec: 1.13 +[2025-04-24 01:08:57] (step=0044150) Train Loss: 5.4193, Train Steps/Sec: 1.18 +[2025-04-24 01:09:18] (step=0044175) Train Loss: 5.4134, Train Steps/Sec: 1.17 +[2025-04-24 01:09:40] (step=0044200) Train Loss: 5.3286, Train Steps/Sec: 1.12 +[2025-04-24 01:10:01] (step=0044225) Train Loss: 5.4532, Train Steps/Sec: 1.18 +[2025-04-24 01:10:23] (step=0044250) Train Loss: 5.2563, Train Steps/Sec: 1.17 +[2025-04-24 01:10:44] (step=0044275) Train Loss: 5.4491, Train Steps/Sec: 1.17 +[2025-04-24 01:11:05] (step=0044300) Train Loss: 5.3308, Train Steps/Sec: 1.17 +[2025-04-24 01:11:27] (step=0044325) Train Loss: 5.3398, Train Steps/Sec: 1.17 +[2025-04-24 01:11:48] (step=0044350) Train Loss: 5.3435, Train Steps/Sec: 1.18 +[2025-04-24 01:12:09] (step=0044375) Train Loss: 5.3156, Train Steps/Sec: 1.17 +[2025-04-24 01:12:31] (step=0044400) Train Loss: 5.3139, Train Steps/Sec: 1.17 +[2025-04-24 01:12:52] (step=0044425) Train Loss: 5.3983, Train Steps/Sec: 1.18 +[2025-04-24 01:13:13] (step=0044450) Train Loss: 5.4016, Train Steps/Sec: 1.17 +[2025-04-24 01:13:35] (step=0044475) Train Loss: 5.2697, Train Steps/Sec: 1.17 +[2025-04-24 01:13:56] (step=0044500) Train Loss: 5.3671, Train Steps/Sec: 1.17 +[2025-04-24 01:14:17] (step=0044525) Train Loss: 5.3122, Train Steps/Sec: 1.18 +[2025-04-24 01:14:39] (step=0044550) Train Loss: 5.2865, Train Steps/Sec: 1.17 +[2025-04-24 01:15:01] (step=0044575) Train Loss: 5.3465, Train Steps/Sec: 1.12 +[2025-04-24 01:15:22] (step=0044600) Train Loss: 5.4583, Train Steps/Sec: 1.17 +[2025-04-24 01:15:44] (step=0044625) Train Loss: 5.3009, Train Steps/Sec: 1.18 +[2025-04-24 01:16:06] (step=0044650) Train Loss: 5.2811, Train Steps/Sec: 1.09 +[2025-04-24 01:16:28] (step=0044675) Train Loss: 5.3601, Train Steps/Sec: 1.17 +[2025-04-24 01:16:49] (step=0044700) Train Loss: 5.2750, Train Steps/Sec: 1.17 +[2025-04-24 01:17:11] (step=0044725) Train Loss: 5.3450, Train Steps/Sec: 1.13 +[2025-04-24 01:17:34] (step=0044750) Train Loss: 5.4142, Train Steps/Sec: 1.13 +[2025-04-24 01:17:57] (step=0044775) Train Loss: 5.4143, Train Steps/Sec: 1.09 +[2025-04-24 01:18:18] (step=0044800) Train Loss: 5.3673, Train Steps/Sec: 1.17 +[2025-04-24 01:18:39] (step=0044825) Train Loss: 5.4454, Train Steps/Sec: 1.18 +[2025-04-24 01:19:01] (step=0044850) Train Loss: 5.3339, Train Steps/Sec: 1.13 +[2025-04-24 01:19:23] (step=0044875) Train Loss: 5.3611, Train Steps/Sec: 1.18 +[2025-04-24 01:19:44] (step=0044900) Train Loss: 5.4296, Train Steps/Sec: 1.17 +[2025-04-24 01:20:05] (step=0044925) Train Loss: 5.3284, Train Steps/Sec: 1.18 +[2025-04-24 01:20:27] (step=0044950) Train Loss: 5.3542, Train Steps/Sec: 1.17 +[2025-04-24 01:20:48] (step=0044975) Train Loss: 5.4280, Train Steps/Sec: 1.17 +[2025-04-24 01:21:09] (step=0045000) Train Loss: 5.3534, Train Steps/Sec: 1.17 +[2025-04-24 01:21:31] (step=0045025) Train Loss: 5.3394, Train Steps/Sec: 1.17 +[2025-04-24 01:21:52] (step=0045050) Train Loss: 5.3765, Train Steps/Sec: 1.17 +[2025-04-24 01:22:13] (step=0045075) Train Loss: 5.2853, Train Steps/Sec: 1.17 +[2025-04-24 01:22:35] (step=0045100) Train Loss: 5.3460, Train Steps/Sec: 1.17 +[2025-04-24 01:22:56] (step=0045125) Train Loss: 5.2925, Train Steps/Sec: 1.18 +[2025-04-24 01:23:17] (step=0045150) Train Loss: 5.4047, Train Steps/Sec: 1.17 +[2025-04-24 01:23:39] (step=0045175) Train Loss: 5.3833, Train Steps/Sec: 1.17 +[2025-04-24 01:24:00] (step=0045200) Train Loss: 5.3247, Train Steps/Sec: 1.17 +[2025-04-24 01:24:21] (step=0045225) Train Loss: 5.3694, Train Steps/Sec: 1.17 +[2025-04-24 01:24:43] (step=0045250) Train Loss: 5.4376, Train Steps/Sec: 1.18 +[2025-04-24 01:25:04] (step=0045275) Train Loss: 5.2446, Train Steps/Sec: 1.18 +[2025-04-24 01:25:26] (step=0045300) Train Loss: 5.3384, Train Steps/Sec: 1.11 +[2025-04-24 01:25:49] (step=0045325) Train Loss: 5.3731, Train Steps/Sec: 1.09 +[2025-04-24 01:26:11] (step=0045350) Train Loss: 5.4790, Train Steps/Sec: 1.17 +[2025-04-24 01:26:33] (step=0045375) Train Loss: 5.3663, Train Steps/Sec: 1.13 +[2025-04-24 01:26:54] (step=0045400) Train Loss: 5.2886, Train Steps/Sec: 1.17 +[2025-04-24 01:27:16] (step=0045425) Train Loss: 5.3127, Train Steps/Sec: 1.13 +[2025-04-24 01:27:38] (step=0045450) Train Loss: 5.3928, Train Steps/Sec: 1.13 +[2025-04-24 01:28:00] (step=0045475) Train Loss: 5.4182, Train Steps/Sec: 1.17 +[2025-04-24 01:28:22] (step=0045500) Train Loss: 5.4539, Train Steps/Sec: 1.12 +[2025-04-24 01:28:44] (step=0045525) Train Loss: 5.3889, Train Steps/Sec: 1.13 +[2025-04-24 01:29:06] (step=0045550) Train Loss: 5.3678, Train Steps/Sec: 1.17 +[2025-04-24 01:29:27] (step=0045575) Train Loss: 5.3170, Train Steps/Sec: 1.17 +[2025-04-24 01:29:48] (step=0045600) Train Loss: 5.3305, Train Steps/Sec: 1.17 +[2025-04-24 01:30:10] (step=0045625) Train Loss: 5.3841, Train Steps/Sec: 1.17 +[2025-04-24 01:30:31] (step=0045650) Train Loss: 5.3210, Train Steps/Sec: 1.17 +[2025-04-24 01:30:52] (step=0045675) Train Loss: 5.4832, Train Steps/Sec: 1.17 +[2025-04-24 01:31:14] (step=0045700) Train Loss: 5.4124, Train Steps/Sec: 1.17 +[2025-04-24 01:31:35] (step=0045725) Train Loss: 5.3231, Train Steps/Sec: 1.18 +[2025-04-24 01:31:56] (step=0045750) Train Loss: 5.3632, Train Steps/Sec: 1.17 +[2025-04-24 01:32:17] (step=0045775) Train Loss: 5.3009, Train Steps/Sec: 1.17 +[2025-04-24 01:32:39] (step=0045800) Train Loss: 5.3580, Train Steps/Sec: 1.17 +[2025-04-24 01:33:00] (step=0045825) Train Loss: 5.4282, Train Steps/Sec: 1.17 +[2025-04-24 01:33:21] (step=0045850) Train Loss: 5.3535, Train Steps/Sec: 1.18 +[2025-04-24 01:33:43] (step=0045875) Train Loss: 5.3838, Train Steps/Sec: 1.18 +[2025-04-24 01:34:04] (step=0045900) Train Loss: 5.3273, Train Steps/Sec: 1.17 +[2025-04-24 01:34:25] (step=0045925) Train Loss: 5.3174, Train Steps/Sec: 1.18 +[2025-04-24 01:34:47] (step=0045950) Train Loss: 5.4085, Train Steps/Sec: 1.17 +[2025-04-24 01:35:10] (step=0045975) Train Loss: 5.3665, Train Steps/Sec: 1.09 +[2025-04-24 01:35:31] (step=0046000) Train Loss: 5.3453, Train Steps/Sec: 1.17 +[2025-04-24 01:35:31] text_config is None. Initializing the text config with default values (`OPTConfig`). +[2025-04-24 01:40:35] Finish Eval in 46000 steps... +[2025-04-24 01:40:54] Saved checkpoint to checkpoint/Afteri2i_100kckpt_LLAVAT5_empty_Fixed_just_segment_pretrain_stage3_dreambench_recap_400k_t2i_400k_flux_200k_midjourney_150k_recovery_150k_3e4_epoch2_half_rotated/005-GPT-XL/checkpoints/0046000.pt +[2025-04-24 01:40:56] Removed old checkpoint: checkpoint/Afteri2i_100kckpt_LLAVAT5_empty_Fixed_just_segment_pretrain_stage3_dreambench_recap_400k_t2i_400k_flux_200k_midjourney_150k_recovery_150k_3e4_epoch2_half_rotated/005-GPT-XL/checkpoints/0044000.pt +[2025-04-24 01:41:18] (step=0046025) Train Loss: 5.2180, Train Steps/Sec: 0.07 +[2025-04-24 01:41:40] (step=0046050) Train Loss: 5.3345, Train Steps/Sec: 1.13 +[2025-04-24 01:42:02] (step=0046075) Train Loss: 5.2980, Train Steps/Sec: 1.13 +[2025-04-24 01:42:25] (step=0046100) Train Loss: 5.3652, Train Steps/Sec: 1.13 +[2025-04-24 01:42:46] (step=0046125) Train Loss: 5.3861, Train Steps/Sec: 1.18 +[2025-04-24 01:43:07] (step=0046150) Train Loss: 5.2832, Train Steps/Sec: 1.18 +[2025-04-24 01:43:29] (step=0046175) Train Loss: 5.3837, Train Steps/Sec: 1.13 +[2025-04-24 01:43:51] (step=0046200) Train Loss: 5.3489, Train Steps/Sec: 1.17 +[2025-04-24 01:44:13] (step=0046225) Train Loss: 5.3975, Train Steps/Sec: 1.12 +[2025-04-24 01:44:34] (step=0046250) Train Loss: 5.2774, Train Steps/Sec: 1.17 +[2025-04-24 01:44:55] (step=0046275) Train Loss: 5.3690, Train Steps/Sec: 1.18 +[2025-04-24 01:45:17] (step=0046300) Train Loss: 5.2927, Train Steps/Sec: 1.17 +[2025-04-24 01:45:38] (step=0046325) Train Loss: 5.3940, Train Steps/Sec: 1.18 +[2025-04-24 01:45:59] (step=0046350) Train Loss: 5.4671, Train Steps/Sec: 1.17 +[2025-04-24 01:46:21] (step=0046375) Train Loss: 5.3494, Train Steps/Sec: 1.17 +[2025-04-24 01:46:42] (step=0046400) Train Loss: 5.3517, Train Steps/Sec: 1.17 +[2025-04-24 01:47:03] (step=0046425) Train Loss: 5.4245, Train Steps/Sec: 1.17 +[2025-04-24 01:47:25] (step=0046450) Train Loss: 5.3157, Train Steps/Sec: 1.17 +[2025-04-24 01:47:46] (step=0046475) Train Loss: 5.3498, Train Steps/Sec: 1.17 +[2025-04-24 01:48:07] (step=0046500) Train Loss: 5.2964, Train Steps/Sec: 1.17 +[2025-04-24 01:48:29] (step=0046525) Train Loss: 5.3717, Train Steps/Sec: 1.17 +[2025-04-24 01:48:50] (step=0046550) Train Loss: 5.3683, Train Steps/Sec: 1.18 +[2025-04-24 01:49:11] (step=0046575) Train Loss: 5.2902, Train Steps/Sec: 1.18 +[2025-04-24 01:49:33] (step=0046600) Train Loss: 5.3673, Train Steps/Sec: 1.16 +[2025-04-24 01:49:55] (step=0046625) Train Loss: 5.3834, Train Steps/Sec: 1.13 +[2025-04-24 01:50:17] (step=0046650) Train Loss: 5.4046, Train Steps/Sec: 1.13 +[2025-04-24 01:50:38] (step=0046675) Train Loss: 5.3893, Train Steps/Sec: 1.18 +[2025-04-24 01:51:01] (step=0046700) Train Loss: 5.3200, Train Steps/Sec: 1.12 +[2025-04-24 01:51:22] (step=0046725) Train Loss: 5.2936, Train Steps/Sec: 1.18 +[2025-04-24 01:51:45] (step=0046750) Train Loss: 5.3798, Train Steps/Sec: 1.08 +[2025-04-24 01:52:07] (step=0046775) Train Loss: 5.4106, Train Steps/Sec: 1.13 +[2025-04-24 01:52:28] (step=0046800) Train Loss: 5.2462, Train Steps/Sec: 1.17 +[2025-04-24 01:52:51] (step=0046825) Train Loss: 5.3853, Train Steps/Sec: 1.13 +[2025-04-24 01:53:12] (step=0046850) Train Loss: 5.3750, Train Steps/Sec: 1.17 +[2025-04-24 01:53:33] (step=0046875) Train Loss: 5.3776, Train Steps/Sec: 1.17 +[2025-04-24 01:53:55] (step=0046900) Train Loss: 5.4468, Train Steps/Sec: 1.17 +[2025-04-24 01:54:09] Done! diff --git a/CKPTS/Afteri2i_100kckpt_LLAVAT5_empty_Fixed_just_segment_pretrain_stage3_dreambench_recap_400k_t2i_400k_flux_200k_midjourney_150k_recovery_150k_3e4_epoch2_half_rotated/wandb/debug-internal.log b/CKPTS/Afteri2i_100kckpt_LLAVAT5_empty_Fixed_just_segment_pretrain_stage3_dreambench_recap_400k_t2i_400k_flux_200k_midjourney_150k_recovery_150k_3e4_epoch2_half_rotated/wandb/debug-internal.log new file mode 100644 index 0000000000000000000000000000000000000000..a30a26e7a381db2249e8240109339da415939267 --- /dev/null +++ b/CKPTS/Afteri2i_100kckpt_LLAVAT5_empty_Fixed_just_segment_pretrain_stage3_dreambench_recap_400k_t2i_400k_flux_200k_midjourney_150k_recovery_150k_3e4_epoch2_half_rotated/wandb/debug-internal.log @@ -0,0 +1,18 @@ +{"time":"2025-04-23T12:24:59.620484542Z","level":"INFO","msg":"stream: starting","core version":"0.19.8","symlink path":"checkpoint/Afteri2i_100kckpt_LLAVAT5_empty_Fixed_just_segment_pretrain_stage3_dreambench_recap_400k_t2i_400k_flux_200k_midjourney_150k_recovery_150k_3e4_epoch2_half_rotated/wandb/run-20250423_122459-k5belznu/logs/debug-core.log"} +{"time":"2025-04-23T12:24:59.946787591Z","level":"INFO","msg":"created new stream","id":"k5belznu"} +{"time":"2025-04-23T12:24:59.946828497Z","level":"INFO","msg":"stream: started","id":"k5belznu"} +{"time":"2025-04-23T12:24:59.946878972Z","level":"INFO","msg":"writer: Do: started","stream_id":"k5belznu"} +{"time":"2025-04-23T12:24:59.947187961Z","level":"INFO","msg":"handler: started","stream_id":"k5belznu"} +{"time":"2025-04-23T12:24:59.947210323Z","level":"INFO","msg":"sender: started","stream_id":"k5belznu"} +{"time":"2025-04-23T12:25:00.224703755Z","level":"INFO","msg":"Starting system monitor"} +{"time":"2025-04-23T12:38:21.68929416Z","level":"INFO","msg":"api: retrying HTTP error","status":500,"url":"https://api.wandb.ai/files/haozhezhao/llamagen_ti2i/k5belznu/file_stream","body":"{\"error\":\"context deadline exceeded\"}"} +{"time":"2025-04-23T19:50:15.838080604Z","level":"INFO","msg":"api: retrying HTTP error","status":502,"url":"https://api.wandb.ai/files/haozhezhao/llamagen_ti2i/k5belznu/file_stream","body":"\n\n\n502 Server Error\n\n\n

Error: Server Error

\n

The server encountered a temporary error and could not complete your request.

Please try again in 30 seconds.

\n

\n\n"} +{"time":"2025-04-23T21:12:19.07764583Z","level":"INFO","msg":"api: retrying HTTP error","status":502,"url":"https://api.wandb.ai/files/haozhezhao/llamagen_ti2i/k5belznu/file_stream","body":"\n\n\n502 Server Error\n\n\n

Error: Server Error

\n

The server encountered a temporary error and could not complete your request.

Please try again in 30 seconds.

\n

\n\n"} +{"time":"2025-04-24T01:54:12.787428225Z","level":"INFO","msg":"stream: closing","id":"k5belznu"} +{"time":"2025-04-24T01:54:12.787458982Z","level":"INFO","msg":"Stopping system monitor"} +{"time":"2025-04-24T01:54:12.788155851Z","level":"INFO","msg":"Stopped system monitor"} +{"time":"2025-04-24T01:54:13.990702152Z","level":"INFO","msg":"fileTransfer: Close: file transfer manager closed"} +{"time":"2025-04-24T01:54:14.220871084Z","level":"INFO","msg":"handler: closed","stream_id":"k5belznu"} +{"time":"2025-04-24T01:54:14.22088996Z","level":"INFO","msg":"writer: Close: closed","stream_id":"k5belznu"} +{"time":"2025-04-24T01:54:14.220940735Z","level":"INFO","msg":"sender: closed","stream_id":"k5belznu"} +{"time":"2025-04-24T01:54:14.220951605Z","level":"INFO","msg":"stream: closed","id":"k5belznu"} diff --git a/CKPTS/Afteri2i_100kckpt_LLAVAT5_empty_Fixed_just_segment_pretrain_stage3_dreambench_recap_400k_t2i_400k_flux_200k_midjourney_150k_recovery_150k_3e4_epoch2_half_rotated/wandb/debug.log b/CKPTS/Afteri2i_100kckpt_LLAVAT5_empty_Fixed_just_segment_pretrain_stage3_dreambench_recap_400k_t2i_400k_flux_200k_midjourney_150k_recovery_150k_3e4_epoch2_half_rotated/wandb/debug.log new file mode 100644 index 0000000000000000000000000000000000000000..9b182da5f456197a5acd00209f6c64b4b6ad21ad --- /dev/null +++ b/CKPTS/Afteri2i_100kckpt_LLAVAT5_empty_Fixed_just_segment_pretrain_stage3_dreambench_recap_400k_t2i_400k_flux_200k_midjourney_150k_recovery_150k_3e4_epoch2_half_rotated/wandb/debug.log @@ -0,0 +1,23 @@ +2025-04-23 12:24:59,608 INFO MainThread:3176497 [wandb_setup.py:_flush():67] Current SDK version is 0.19.8 +2025-04-23 12:24:59,608 INFO MainThread:3176497 [wandb_setup.py:_flush():67] Configure stats pid to 3176497 +2025-04-23 12:24:59,608 INFO MainThread:3176497 [wandb_setup.py:_flush():67] Loading settings from /tmp/haozhezhao/.config/wandb/settings +2025-04-23 12:24:59,608 INFO MainThread:3176497 [wandb_setup.py:_flush():67] Loading settings from /tmp/haozhezhao/MLLMG/wandb/settings +2025-04-23 12:24:59,608 INFO MainThread:3176497 [wandb_setup.py:_flush():67] Loading settings from environment variables +2025-04-23 12:24:59,608 INFO MainThread:3176497 [wandb_init.py:setup_run_log_directory():647] Logging user logs to checkpoint/Afteri2i_100kckpt_LLAVAT5_empty_Fixed_just_segment_pretrain_stage3_dreambench_recap_400k_t2i_400k_flux_200k_midjourney_150k_recovery_150k_3e4_epoch2_half_rotated/wandb/run-20250423_122459-k5belznu/logs/debug.log +2025-04-23 12:24:59,608 INFO MainThread:3176497 [wandb_init.py:setup_run_log_directory():648] Logging internal logs to checkpoint/Afteri2i_100kckpt_LLAVAT5_empty_Fixed_just_segment_pretrain_stage3_dreambench_recap_400k_t2i_400k_flux_200k_midjourney_150k_recovery_150k_3e4_epoch2_half_rotated/wandb/run-20250423_122459-k5belznu/logs/debug-internal.log +2025-04-23 12:24:59,608 INFO MainThread:3176497 [wandb_init.py:init():761] calling init triggers +2025-04-23 12:24:59,609 INFO MainThread:3176497 [wandb_init.py:init():766] wandb.init called with sweep_config: {} +config: {'data_path': '/tmp/haozhezhao/MLLMG/jsonl_data/merged_train_set_set_subject_400k_recap_t2i_400k_flux_200k_midjourney_150k_recovery_150k_grounding_100fluxseg_50samseg.jsonl', 'cloud_save_path': '/tmp/haozhezhao/checkpoint', 'no_local_save': False, 'vq_model': 'VQ-16', 'vq_ckpt': '/tmp/haozhezhao/model/llamagen_t2i/vq_ds16_t2i.pt', 'codebook_size': 16384, 'codebook_embed_dim': 8, 'gpt_model': 'GPT-XL', 'gpt_ckpt': '/tmp/haozhezhao/MLLMG/MLLMG_ckpts/checkpoints/AfterI2I_llavat5_subject_train_all_T5_subject_t2i_ti2i_120_w_flux_segment_noimageonly_empty_fix_train_all/0100000.pt', 'gpt_type': 't2i', 'vocab_size': 16384, 'cls_token_num': 512, 'dropout_p': 0.1, 'token_dropout_p': 0.1, 'drop_path': 0.0, 'no_compile': False, 'results_dir': 'checkpoint/Afteri2i_100kckpt_LLAVAT5_empty_Fixed_just_segment_pretrain_stage3_dreambench_recap_400k_t2i_400k_flux_200k_midjourney_150k_recovery_150k_3e4_epoch2_half_rotated', 'dataset': 'ti2i', 'image_size': 512, 'downsample_size': 16, 'num_classes': 1000, 'epochs': 2, 'lr': 0.0003, 'weight_decay': 0.05, 'beta1': 0.9, 'beta2': 0.95, 'max_grad_norm': 1.0, 'global_batch_size': 56, 'global_seed': 0, 'num_workers': 8, 'log_every': 25, 'ckpt_every': 2000, 'gradient_accumulation_steps': 4, 'mixed_precision': 'bf16', 'val_data_path': '/tmp/haozhezhao/MLLMG/jsonl_data/dreambench_plus_valid.jsonl', 'use_vision_tower': True, 'model_name_or_path': '/tmp/haozhezhao/model/blip2-flan-t5-xl', 'image_place_holder': '', 'processor_path': None, 'do_eval': True, 'max_eval_samples': 250, 'train_text_encoder': True, 'no_left_padding': False, 'cfg_scale': 7.5, 'top_k': 16384, 'temperature': 0.9, 'top_p': 1.0, 'eval_steps': 2000, 'project_name': 'llamagen_ti2i', 'load_from_checkpoint': '/tmp/haozhezhao/MLLMG/checkpoint/FIXed_3M_1epoch_step32000.pt', 'warmup': 0.05, 'lr_decay_style': 'cosine', 'lr_decay_ratio': 0.1, 'train_iters': 500000, 'class_dropout_prob': 0.1, 'with_image_only': False, 'image_only_rate': 0.1, 'stage2': False, 'subject_driven': True, 'load_subject_embedding': None, 'reference_data_path': '/tmp/haozhezhao/MLLMG/cc12m_reference_tunnel.jsonl', 'multimodal_encoder': 'llava', 'do_recovery': True, 'no_replace': False, 'resume': False, 'dreambench_eval': True, 'find_unused_parameters': True, 'load_visual_encoder': False, 'continue_stage1': False, 'replace_subject': False, 'train_all': True, 'save_total_limit': 1, 'load_language_projection': '/tmp/haozhezhao/MLLMG/llava-v1.5-flant5_fixed-pretrain/mm_projector.bin', 'mm_vision_tower': 'openai/clip-vit-large-patch14', 'load_fixed_llamagen': True, 'unfreeze_output': False, 'fix': 'gpt-empty-fix', 'rank': 0, 'world_size': 8, 'gpu': 0, 'dist_url': 'env://', 'distributed': True, 'dist_backend': 'nccl', '_wandb': {}} +2025-04-23 12:24:59,609 INFO MainThread:3176497 [wandb_init.py:init():784] starting backend +2025-04-23 12:24:59,609 INFO MainThread:3176497 [wandb_init.py:init():788] sending inform_init request +2025-04-23 12:24:59,614 INFO MainThread:3176497 [backend.py:_multiprocessing_setup():101] multiprocessing start_methods=fork,spawn,forkserver, using: spawn +2025-04-23 12:24:59,614 INFO MainThread:3176497 [wandb_init.py:init():798] backend started and connected +2025-04-23 12:24:59,616 INFO MainThread:3176497 [wandb_init.py:init():891] updated telemetry +2025-04-23 12:24:59,616 INFO MainThread:3176497 [wandb_init.py:init():915] communicating run to backend with 90.0 second timeout +2025-04-23 12:25:00,221 INFO MainThread:3176497 [wandb_init.py:init():990] starting run threads in backend +2025-04-23 12:25:00,347 INFO MainThread:3176497 [wandb_run.py:_console_start():2375] atexit reg +2025-04-23 12:25:00,347 INFO MainThread:3176497 [wandb_run.py:_redirect():2227] redirect: wrap_raw +2025-04-23 12:25:00,348 INFO MainThread:3176497 [wandb_run.py:_redirect():2292] Wrapping output streams. +2025-04-23 12:25:00,348 INFO MainThread:3176497 [wandb_run.py:_redirect():2315] Redirects installed. +2025-04-23 12:25:00,349 INFO MainThread:3176497 [wandb_init.py:init():1032] run started, returning control to user process +2025-04-24 01:54:12,786 INFO MsgRouterThr:3176497 [mailbox.py:close():129] Closing mailbox, abandoning 1 handles. diff --git a/CKPTS/Afteri2i_100kckpt_LLAVAT5_empty_Fixed_just_segment_pretrain_stage3_dreambench_recap_400k_t2i_400k_flux_200k_midjourney_150k_recovery_150k_3e4_epoch2_half_rotated/wandb/run-20250423_105205-kqcre01n/files/output.log b/CKPTS/Afteri2i_100kckpt_LLAVAT5_empty_Fixed_just_segment_pretrain_stage3_dreambench_recap_400k_t2i_400k_flux_200k_midjourney_150k_recovery_150k_3e4_epoch2_half_rotated/wandb/run-20250423_105205-kqcre01n/files/output.log new file mode 100644 index 0000000000000000000000000000000000000000..92e03b8773e10556479a7ebce057b11d51f3d892 --- /dev/null +++ b/CKPTS/Afteri2i_100kckpt_LLAVAT5_empty_Fixed_just_segment_pretrain_stage3_dreambench_recap_400k_t2i_400k_flux_200k_midjourney_150k_recovery_150k_3e4_epoch2_half_rotated/wandb/run-20250423_105205-kqcre01n/files/output.log @@ -0,0 +1,24 @@ +[2025-04-23 10:52:06] Training for 2 epochs... +[2025-04-23 10:52:06] Beginning epoch 0... + 0%| | 0/20526 [00:00", + "--do_eval", + "--eval_steps", + "2000", + "--max_eval_samples", + "500", + "--cfg-scale", + "7.5", + "--top-k", + "16384", + "--load_from_checkpoint", + "/tmp/haozhezhao/MLLMG/checkpoint/FIXed_3M_1epoch_step32000.pt", + "--global-batch-size", + "64", + "--num-workers", + "8", + "--warmup", + "0.05", + "--gradient-accumulation-steps", + "4", + "--train_text_encoder", + "--ckpt-every", + "2000", + "--epochs", + "2", + "--subject_driven", + "--reference_data_path", + "/tmp/haozhezhao/MLLMG/cc12m_reference_tunnel.jsonl", + "--multimodal_encoder", + "llava", + "--do_recovery", + "--find_unused_parameters", + "--cls-token-num", + "512", + "--dreambench_eval", + "--save_total_limit", + "1", + "--load_language_projection", + "/tmp/haozhezhao/MLLMG/llava-v1.5-flant5_fixed-pretrain/mm_projector.bin", + "--gpt-ckpt", + "/tmp/haozhezhao/MLLMG/MLLMG_ckpts/checkpoints/AfterI2I_llavat5_subject_train_all_T5_subject_t2i_ti2i_120_w_flux_segment_noimageonly_empty_fix_train_all/0100000.pt", + "--mm_vision_tower", + "openai/clip-vit-large-patch14", + "--train_all", + "--load_fixed_llamagen", + "--fix", + "gpt-empty-fix" + ], + "program": "/tmp/haozhezhao/MLLMG/autoregressive/train/train_t2i.py", + "codePath": "autoregressive/train/train_t2i.py", + "email": "mimazhe55360@gmail.com", + "root": "checkpoint/Afteri2i_100kckpt_LLAVAT5_empty_Fixed_just_segment_pretrain_stage3_dreambench_recap_400k_t2i_400k_flux_200k_midjourney_150k_recovery_150k_3e4_epoch2_half_rotated", + "host": "447cc403a8794092814259713c51c1df00001X", + "executable": "/tmp/haozhezhao/anaconda3/envs/nlp/bin/python", + "codePathLocal": "autoregressive/train/train_t2i.py", + "cpu_count": 96, + "cpu_count_logical": 96, + "gpu": "NVIDIA A100-SXM4-80GB", + "gpu_count": 8, + "disk": { + "/": { + "total": "133003395072", + "used": "65572716544" + } + }, + "memory": { + "total": "1902387884032" + }, + "cpu": { + "count": 96, + "countLogical": 96 + }, + "gpu_nvidia": [ + { + "name": "NVIDIA A100-SXM4-80GB", + "memoryTotal": "85899345920", + "cudaCores": 6912, + "architecture": "Ampere" + }, + { + "name": "NVIDIA A100-SXM4-80GB", + "memoryTotal": "85899345920", + "cudaCores": 6912, + "architecture": "Ampere" + }, + { + "name": "NVIDIA A100-SXM4-80GB", + "memoryTotal": "85899345920", + "cudaCores": 6912, + "architecture": "Ampere" + }, + { + "name": "NVIDIA A100-SXM4-80GB", + "memoryTotal": "85899345920", + "cudaCores": 6912, + "architecture": "Ampere" + }, + { + "name": "NVIDIA A100-SXM4-80GB", + "memoryTotal": "85899345920", + "cudaCores": 6912, + "architecture": "Ampere" + }, + { + "name": "NVIDIA A100-SXM4-80GB", + "memoryTotal": "85899345920", + "cudaCores": 6912, + "architecture": "Ampere" + }, + { + "name": "NVIDIA A100-SXM4-80GB", + "memoryTotal": "85899345920", + "cudaCores": 6912, + "architecture": "Ampere" + }, + { + "name": "NVIDIA A100-SXM4-80GB", + "memoryTotal": "85899345920", + "cudaCores": 6912, + "architecture": "Ampere" + } + ], + "cudaVersion": "12.2" +} \ No newline at end of file diff --git a/CKPTS/Afteri2i_100kckpt_LLAVAT5_empty_Fixed_just_segment_pretrain_stage3_dreambench_recap_400k_t2i_400k_flux_200k_midjourney_150k_recovery_150k_3e4_epoch2_half_rotated/wandb/run-20250423_105205-kqcre01n/logs/debug-core.log b/CKPTS/Afteri2i_100kckpt_LLAVAT5_empty_Fixed_just_segment_pretrain_stage3_dreambench_recap_400k_t2i_400k_flux_200k_midjourney_150k_recovery_150k_3e4_epoch2_half_rotated/wandb/run-20250423_105205-kqcre01n/logs/debug-core.log new file mode 100644 index 0000000000000000000000000000000000000000..186b21c58e0746d06af89f3c8e90f95e5774f4d9 --- /dev/null +++ b/CKPTS/Afteri2i_100kckpt_LLAVAT5_empty_Fixed_just_segment_pretrain_stage3_dreambench_recap_400k_t2i_400k_flux_200k_midjourney_150k_recovery_150k_3e4_epoch2_half_rotated/wandb/run-20250423_105205-kqcre01n/logs/debug-core.log @@ -0,0 +1,7 @@ +{"time":"2025-04-23T10:52:04.452234624Z","level":"INFO","msg":"main: starting server","port-filename":"/tmp/tmp1udc_6p5/port-3125442.txt","pid":3125442,"log-level":0,"disable-analytics":false,"shutdown-on-parent-exit":false} +{"time":"2025-04-23T10:52:04.454025961Z","level":"INFO","msg":"Will exit if parent process dies.","ppid":3125442} +{"time":"2025-04-23T10:52:04.453998274Z","level":"INFO","msg":"server is running","addr":{"IP":"127.0.0.1","Port":34751,"Zone":""}} +{"time":"2025-04-23T10:52:04.639847709Z","level":"INFO","msg":"connection: ManageConnectionData: new connection created","id":"127.0.0.1:37208"} +{"time":"2025-04-23T10:52:05.528908019Z","level":"INFO","msg":"handleInformInit: received","streamId":"kqcre01n","id":"127.0.0.1:37208"} +{"time":"2025-04-23T10:52:05.83979549Z","level":"INFO","msg":"handleInformInit: stream started","streamId":"kqcre01n","id":"127.0.0.1:37208"} +{"time":"2025-04-23T11:00:32.45069641Z","level":"INFO","msg":"Parent process exited, terminating service process."} diff --git a/CKPTS/Afteri2i_100kckpt_LLAVAT5_empty_Fixed_just_segment_pretrain_stage3_dreambench_recap_400k_t2i_400k_flux_200k_midjourney_150k_recovery_150k_3e4_epoch2_half_rotated/wandb/run-20250423_105205-kqcre01n/logs/debug-internal.log b/CKPTS/Afteri2i_100kckpt_LLAVAT5_empty_Fixed_just_segment_pretrain_stage3_dreambench_recap_400k_t2i_400k_flux_200k_midjourney_150k_recovery_150k_3e4_epoch2_half_rotated/wandb/run-20250423_105205-kqcre01n/logs/debug-internal.log new file mode 100644 index 0000000000000000000000000000000000000000..03fd3521d0f1b13367f90e626a755a4bb2aae339 --- /dev/null +++ b/CKPTS/Afteri2i_100kckpt_LLAVAT5_empty_Fixed_just_segment_pretrain_stage3_dreambench_recap_400k_t2i_400k_flux_200k_midjourney_150k_recovery_150k_3e4_epoch2_half_rotated/wandb/run-20250423_105205-kqcre01n/logs/debug-internal.log @@ -0,0 +1,7 @@ +{"time":"2025-04-23T10:52:05.529201659Z","level":"INFO","msg":"stream: starting","core version":"0.19.8","symlink path":"checkpoint/Afteri2i_100kckpt_LLAVAT5_empty_Fixed_just_segment_pretrain_stage3_dreambench_recap_400k_t2i_400k_flux_200k_midjourney_150k_recovery_150k_3e4_epoch2_half_rotated/wandb/run-20250423_105205-kqcre01n/logs/debug-core.log"} +{"time":"2025-04-23T10:52:05.839742401Z","level":"INFO","msg":"created new stream","id":"kqcre01n"} +{"time":"2025-04-23T10:52:05.839789719Z","level":"INFO","msg":"stream: started","id":"kqcre01n"} +{"time":"2025-04-23T10:52:05.839858859Z","level":"INFO","msg":"writer: Do: started","stream_id":"kqcre01n"} +{"time":"2025-04-23T10:52:05.839925183Z","level":"INFO","msg":"handler: started","stream_id":"kqcre01n"} +{"time":"2025-04-23T10:52:05.839863608Z","level":"INFO","msg":"sender: started","stream_id":"kqcre01n"} +{"time":"2025-04-23T10:52:06.17964599Z","level":"INFO","msg":"Starting system monitor"} diff --git a/CKPTS/Afteri2i_100kckpt_LLAVAT5_empty_Fixed_just_segment_pretrain_stage3_dreambench_recap_400k_t2i_400k_flux_200k_midjourney_150k_recovery_150k_3e4_epoch2_half_rotated/wandb/run-20250423_105205-kqcre01n/logs/debug.log b/CKPTS/Afteri2i_100kckpt_LLAVAT5_empty_Fixed_just_segment_pretrain_stage3_dreambench_recap_400k_t2i_400k_flux_200k_midjourney_150k_recovery_150k_3e4_epoch2_half_rotated/wandb/run-20250423_105205-kqcre01n/logs/debug.log new file mode 100644 index 0000000000000000000000000000000000000000..2f30e184e290dfd9e374c03d3e9e556698566029 --- /dev/null +++ b/CKPTS/Afteri2i_100kckpt_LLAVAT5_empty_Fixed_just_segment_pretrain_stage3_dreambench_recap_400k_t2i_400k_flux_200k_midjourney_150k_recovery_150k_3e4_epoch2_half_rotated/wandb/run-20250423_105205-kqcre01n/logs/debug.log @@ -0,0 +1,22 @@ +2025-04-23 10:52:05,517 INFO MainThread:3125442 [wandb_setup.py:_flush():67] Current SDK version is 0.19.8 +2025-04-23 10:52:05,517 INFO MainThread:3125442 [wandb_setup.py:_flush():67] Configure stats pid to 3125442 +2025-04-23 10:52:05,517 INFO MainThread:3125442 [wandb_setup.py:_flush():67] Loading settings from /tmp/haozhezhao/.config/wandb/settings +2025-04-23 10:52:05,517 INFO MainThread:3125442 [wandb_setup.py:_flush():67] Loading settings from /tmp/haozhezhao/MLLMG/wandb/settings +2025-04-23 10:52:05,517 INFO MainThread:3125442 [wandb_setup.py:_flush():67] Loading settings from environment variables +2025-04-23 10:52:05,517 INFO MainThread:3125442 [wandb_init.py:setup_run_log_directory():647] Logging user logs to checkpoint/Afteri2i_100kckpt_LLAVAT5_empty_Fixed_just_segment_pretrain_stage3_dreambench_recap_400k_t2i_400k_flux_200k_midjourney_150k_recovery_150k_3e4_epoch2_half_rotated/wandb/run-20250423_105205-kqcre01n/logs/debug.log +2025-04-23 10:52:05,517 INFO MainThread:3125442 [wandb_init.py:setup_run_log_directory():648] Logging internal logs to checkpoint/Afteri2i_100kckpt_LLAVAT5_empty_Fixed_just_segment_pretrain_stage3_dreambench_recap_400k_t2i_400k_flux_200k_midjourney_150k_recovery_150k_3e4_epoch2_half_rotated/wandb/run-20250423_105205-kqcre01n/logs/debug-internal.log +2025-04-23 10:52:05,517 INFO MainThread:3125442 [wandb_init.py:init():761] calling init triggers +2025-04-23 10:52:05,517 INFO MainThread:3125442 [wandb_init.py:init():766] wandb.init called with sweep_config: {} +config: {'data_path': '/tmp/haozhezhao/MLLMG/jsonl_data/merged_train_set_set_subject_400k_recap_t2i_400k_flux_200k_midjourney_150k_recovery_150k_grounding_100fluxseg_50samseg.jsonl', 'cloud_save_path': '/tmp/haozhezhao/checkpoint', 'no_local_save': False, 'vq_model': 'VQ-16', 'vq_ckpt': '/tmp/haozhezhao/model/llamagen_t2i/vq_ds16_t2i.pt', 'codebook_size': 16384, 'codebook_embed_dim': 8, 'gpt_model': 'GPT-XL', 'gpt_ckpt': '/tmp/haozhezhao/MLLMG/MLLMG_ckpts/checkpoints/AfterI2I_llavat5_subject_train_all_T5_subject_t2i_ti2i_120_w_flux_segment_noimageonly_empty_fix_train_all/0100000.pt', 'gpt_type': 't2i', 'vocab_size': 16384, 'cls_token_num': 512, 'dropout_p': 0.1, 'token_dropout_p': 0.1, 'drop_path': 0.0, 'no_compile': False, 'results_dir': 'checkpoint/Afteri2i_100kckpt_LLAVAT5_empty_Fixed_just_segment_pretrain_stage3_dreambench_recap_400k_t2i_400k_flux_200k_midjourney_150k_recovery_150k_3e4_epoch2_half_rotated', 'dataset': 'ti2i', 'image_size': 512, 'downsample_size': 16, 'num_classes': 1000, 'epochs': 2, 'lr': 0.0003, 'weight_decay': 0.05, 'beta1': 0.9, 'beta2': 0.95, 'max_grad_norm': 1.0, 'global_batch_size': 64, 'global_seed': 0, 'num_workers': 8, 'log_every': 25, 'ckpt_every': 2000, 'gradient_accumulation_steps': 4, 'mixed_precision': 'bf16', 'val_data_path': '/tmp/haozhezhao/MLLMG/jsonl_data/dreambench_plus_valid.jsonl', 'use_vision_tower': True, 'model_name_or_path': '/tmp/haozhezhao/model/blip2-flan-t5-xl', 'image_place_holder': '', 'processor_path': None, 'do_eval': True, 'max_eval_samples': 500, 'train_text_encoder': True, 'no_left_padding': False, 'cfg_scale': 7.5, 'top_k': 16384, 'temperature': 0.9, 'top_p': 1.0, 'eval_steps': 2000, 'project_name': 'llamagen_ti2i', 'load_from_checkpoint': '/tmp/haozhezhao/MLLMG/checkpoint/FIXed_3M_1epoch_step32000.pt', 'warmup': 0.05, 'lr_decay_style': 'cosine', 'lr_decay_ratio': 0.1, 'train_iters': 500000, 'class_dropout_prob': 0.1, 'with_image_only': False, 'image_only_rate': 0.1, 'stage2': False, 'subject_driven': True, 'load_subject_embedding': None, 'reference_data_path': '/tmp/haozhezhao/MLLMG/cc12m_reference_tunnel.jsonl', 'multimodal_encoder': 'llava', 'do_recovery': True, 'no_replace': False, 'resume': False, 'dreambench_eval': True, 'find_unused_parameters': True, 'load_visual_encoder': False, 'continue_stage1': False, 'replace_subject': False, 'train_all': True, 'save_total_limit': 1, 'load_language_projection': '/tmp/haozhezhao/MLLMG/llava-v1.5-flant5_fixed-pretrain/mm_projector.bin', 'mm_vision_tower': 'openai/clip-vit-large-patch14', 'load_fixed_llamagen': True, 'unfreeze_output': False, 'fix': 'gpt-empty-fix', 'rank': 0, 'world_size': 8, 'gpu': 0, 'dist_url': 'env://', 'distributed': True, 'dist_backend': 'nccl', '_wandb': {}} +2025-04-23 10:52:05,518 INFO MainThread:3125442 [wandb_init.py:init():784] starting backend +2025-04-23 10:52:05,518 INFO MainThread:3125442 [wandb_init.py:init():788] sending inform_init request +2025-04-23 10:52:05,524 INFO MainThread:3125442 [backend.py:_multiprocessing_setup():101] multiprocessing start_methods=fork,spawn,forkserver, using: spawn +2025-04-23 10:52:05,524 INFO MainThread:3125442 [wandb_init.py:init():798] backend started and connected +2025-04-23 10:52:05,526 INFO MainThread:3125442 [wandb_init.py:init():891] updated telemetry +2025-04-23 10:52:05,532 INFO MainThread:3125442 [wandb_init.py:init():915] communicating run to backend with 90.0 second timeout +2025-04-23 10:52:06,176 INFO MainThread:3125442 [wandb_init.py:init():990] starting run threads in backend +2025-04-23 10:52:06,263 INFO MainThread:3125442 [wandb_run.py:_console_start():2375] atexit reg +2025-04-23 10:52:06,264 INFO MainThread:3125442 [wandb_run.py:_redirect():2227] redirect: wrap_raw +2025-04-23 10:52:06,264 INFO MainThread:3125442 [wandb_run.py:_redirect():2292] Wrapping output streams. +2025-04-23 10:52:06,264 INFO MainThread:3125442 [wandb_run.py:_redirect():2315] Redirects installed. +2025-04-23 10:52:06,266 INFO MainThread:3125442 [wandb_init.py:init():1032] run started, returning control to user process diff --git a/CKPTS/Afteri2i_100kckpt_LLAVAT5_empty_Fixed_just_segment_pretrain_stage3_dreambench_recap_400k_t2i_400k_flux_200k_midjourney_150k_recovery_150k_3e4_epoch2_half_rotated/wandb/run-20250423_105205-kqcre01n/run-kqcre01n.wandb b/CKPTS/Afteri2i_100kckpt_LLAVAT5_empty_Fixed_just_segment_pretrain_stage3_dreambench_recap_400k_t2i_400k_flux_200k_midjourney_150k_recovery_150k_3e4_epoch2_half_rotated/wandb/run-20250423_105205-kqcre01n/run-kqcre01n.wandb new file mode 100644 index 0000000000000000000000000000000000000000..aacabadca17148cab1031dcff45db2615ec2cc3e Binary files /dev/null and b/CKPTS/Afteri2i_100kckpt_LLAVAT5_empty_Fixed_just_segment_pretrain_stage3_dreambench_recap_400k_t2i_400k_flux_200k_midjourney_150k_recovery_150k_3e4_epoch2_half_rotated/wandb/run-20250423_105205-kqcre01n/run-kqcre01n.wandb differ diff --git a/CKPTS/Afteri2i_100kckpt_LLAVAT5_empty_Fixed_just_segment_pretrain_stage3_dreambench_recap_400k_t2i_400k_flux_200k_midjourney_150k_recovery_150k_3e4_epoch2_half_rotated/wandb/run-20250423_111340-sphrjdl0/files/output.log b/CKPTS/Afteri2i_100kckpt_LLAVAT5_empty_Fixed_just_segment_pretrain_stage3_dreambench_recap_400k_t2i_400k_flux_200k_midjourney_150k_recovery_150k_3e4_epoch2_half_rotated/wandb/run-20250423_111340-sphrjdl0/files/output.log new file mode 100644 index 0000000000000000000000000000000000000000..437ccfd870e21940a06eaebc645691a30c69f0ea --- /dev/null +++ b/CKPTS/Afteri2i_100kckpt_LLAVAT5_empty_Fixed_just_segment_pretrain_stage3_dreambench_recap_400k_t2i_400k_flux_200k_midjourney_150k_recovery_150k_3e4_epoch2_half_rotated/wandb/run-20250423_111340-sphrjdl0/files/output.log @@ -0,0 +1,122 @@ +[2025-04-23 11:13:40] Training for 2 epochs... +[2025-04-23 11:13:40] Beginning epoch 0... + 0%| | 0/23458 [00:00 +tokenizer length after expend 32101 +tokenizer length before expend 32100 + /tmp/haozhezhao/anaconda3/envs/nlp/lib/python3.11/contextlib.py:105: FutureWarning: `torch.backends.cuda.sdp_kernel()` is deprecated. In the future, this context manager will be removed. Please see `torch.nn.attention.sdpa_kernel()` for the new context manager, with updated signature. + self.gen = func(*args, **kwds) | 0/9 [00:00", + "--do_eval", + "--eval_steps", + "2000", + "--max_eval_samples", + "500", + "--cfg-scale", + "7.5", + "--top-k", + "16384", + "--load_from_checkpoint", + "/tmp/haozhezhao/MLLMG/checkpoint/FIXed_3M_1epoch_step32000.pt", + "--global-batch-size", + "56", + "--num-workers", + "8", + "--warmup", + "0.05", + "--gradient-accumulation-steps", + "4", + "--train_text_encoder", + "--ckpt-every", + "2000", + "--epochs", + "2", + "--subject_driven", + "--reference_data_path", + "/tmp/haozhezhao/MLLMG/cc12m_reference_tunnel.jsonl", + "--multimodal_encoder", + "llava", + "--do_recovery", + "--find_unused_parameters", + "--cls-token-num", + "512", + "--dreambench_eval", + "--save_total_limit", + "1", + "--load_language_projection", + "/tmp/haozhezhao/MLLMG/llava-v1.5-flant5_fixed-pretrain/mm_projector.bin", + "--gpt-ckpt", + "/tmp/haozhezhao/MLLMG/MLLMG_ckpts/checkpoints/AfterI2I_llavat5_subject_train_all_T5_subject_t2i_ti2i_120_w_flux_segment_noimageonly_empty_fix_train_all/0100000.pt", + "--mm_vision_tower", + "openai/clip-vit-large-patch14", + "--train_all", + "--load_fixed_llamagen", + "--fix", + "gpt-empty-fix" + ], + "program": "/tmp/haozhezhao/MLLMG/autoregressive/train/train_t2i.py", + "codePath": "autoregressive/train/train_t2i.py", + "email": "mimazhe55360@gmail.com", + "root": "checkpoint/Afteri2i_100kckpt_LLAVAT5_empty_Fixed_just_segment_pretrain_stage3_dreambench_recap_400k_t2i_400k_flux_200k_midjourney_150k_recovery_150k_3e4_epoch2_half_rotated", + "host": "447cc403a8794092814259713c51c1df00001X", + "executable": "/tmp/haozhezhao/anaconda3/envs/nlp/bin/python", + "codePathLocal": "autoregressive/train/train_t2i.py", + "cpu_count": 96, + "cpu_count_logical": 96, + "gpu": "NVIDIA A100-SXM4-80GB", + "gpu_count": 8, + "disk": { + "/": { + "total": "133003395072", + "used": "65540562944" + } + }, + "memory": { + "total": "1902387884032" + }, + "cpu": { + "count": 96, + "countLogical": 96 + }, + "gpu_nvidia": [ + { + "name": "NVIDIA A100-SXM4-80GB", + "memoryTotal": "85899345920", + "cudaCores": 6912, + "architecture": "Ampere" + }, + { + "name": "NVIDIA A100-SXM4-80GB", + "memoryTotal": "85899345920", + "cudaCores": 6912, + "architecture": "Ampere" + }, + { + "name": "NVIDIA A100-SXM4-80GB", + "memoryTotal": "85899345920", + "cudaCores": 6912, + "architecture": "Ampere" + }, + { + "name": "NVIDIA A100-SXM4-80GB", + "memoryTotal": "85899345920", + "cudaCores": 6912, + "architecture": "Ampere" + }, + { + "name": "NVIDIA A100-SXM4-80GB", + "memoryTotal": "85899345920", + "cudaCores": 6912, + "architecture": "Ampere" + }, + { + "name": "NVIDIA A100-SXM4-80GB", + "memoryTotal": "85899345920", + "cudaCores": 6912, + "architecture": "Ampere" + }, + { + "name": "NVIDIA A100-SXM4-80GB", + "memoryTotal": "85899345920", + "cudaCores": 6912, + "architecture": "Ampere" + }, + { + "name": "NVIDIA A100-SXM4-80GB", + "memoryTotal": "85899345920", + "cudaCores": 6912, + "architecture": "Ampere" + } + ], + "cudaVersion": "12.2" +} \ No newline at end of file diff --git a/CKPTS/Afteri2i_100kckpt_LLAVAT5_empty_Fixed_just_segment_pretrain_stage3_dreambench_recap_400k_t2i_400k_flux_200k_midjourney_150k_recovery_150k_3e4_epoch2_half_rotated/wandb/run-20250423_111340-sphrjdl0/logs/debug-core.log b/CKPTS/Afteri2i_100kckpt_LLAVAT5_empty_Fixed_just_segment_pretrain_stage3_dreambench_recap_400k_t2i_400k_flux_200k_midjourney_150k_recovery_150k_3e4_epoch2_half_rotated/wandb/run-20250423_111340-sphrjdl0/logs/debug-core.log new file mode 100644 index 0000000000000000000000000000000000000000..af9b6e8bbc5802e24e5aa165ecdca7264614105f --- /dev/null +++ b/CKPTS/Afteri2i_100kckpt_LLAVAT5_empty_Fixed_just_segment_pretrain_stage3_dreambench_recap_400k_t2i_400k_flux_200k_midjourney_150k_recovery_150k_3e4_epoch2_half_rotated/wandb/run-20250423_111340-sphrjdl0/logs/debug-core.log @@ -0,0 +1,7 @@ +{"time":"2025-04-23T11:13:39.599695424Z","level":"INFO","msg":"main: starting server","port-filename":"/tmp/tmprivvgoqd/port-3152613.txt","pid":3152613,"log-level":0,"disable-analytics":false,"shutdown-on-parent-exit":false} +{"time":"2025-04-23T11:13:39.601107316Z","level":"INFO","msg":"Will exit if parent process dies.","ppid":3152613} +{"time":"2025-04-23T11:13:39.601081809Z","level":"INFO","msg":"server is running","addr":{"IP":"127.0.0.1","Port":35967,"Zone":""}} +{"time":"2025-04-23T11:13:39.785888172Z","level":"INFO","msg":"connection: ManageConnectionData: new connection created","id":"127.0.0.1:56550"} +{"time":"2025-04-23T11:13:40.099412439Z","level":"INFO","msg":"handleInformInit: received","streamId":"sphrjdl0","id":"127.0.0.1:56550"} +{"time":"2025-04-23T11:13:40.306090007Z","level":"INFO","msg":"handleInformInit: stream started","streamId":"sphrjdl0","id":"127.0.0.1:56550"} +{"time":"2025-04-23T12:02:26.298651111Z","level":"INFO","msg":"Parent process exited, terminating service process."} diff --git a/CKPTS/Afteri2i_100kckpt_LLAVAT5_empty_Fixed_just_segment_pretrain_stage3_dreambench_recap_400k_t2i_400k_flux_200k_midjourney_150k_recovery_150k_3e4_epoch2_half_rotated/wandb/run-20250423_111340-sphrjdl0/logs/debug-internal.log b/CKPTS/Afteri2i_100kckpt_LLAVAT5_empty_Fixed_just_segment_pretrain_stage3_dreambench_recap_400k_t2i_400k_flux_200k_midjourney_150k_recovery_150k_3e4_epoch2_half_rotated/wandb/run-20250423_111340-sphrjdl0/logs/debug-internal.log new file mode 100644 index 0000000000000000000000000000000000000000..bdc96dcc3d4926afba6b69cb37f48aeaecdda338 --- /dev/null +++ b/CKPTS/Afteri2i_100kckpt_LLAVAT5_empty_Fixed_just_segment_pretrain_stage3_dreambench_recap_400k_t2i_400k_flux_200k_midjourney_150k_recovery_150k_3e4_epoch2_half_rotated/wandb/run-20250423_111340-sphrjdl0/logs/debug-internal.log @@ -0,0 +1,7 @@ +{"time":"2025-04-23T11:13:40.099627972Z","level":"INFO","msg":"stream: starting","core version":"0.19.8","symlink path":"checkpoint/Afteri2i_100kckpt_LLAVAT5_empty_Fixed_just_segment_pretrain_stage3_dreambench_recap_400k_t2i_400k_flux_200k_midjourney_150k_recovery_150k_3e4_epoch2_half_rotated/wandb/run-20250423_111340-sphrjdl0/logs/debug-core.log"} +{"time":"2025-04-23T11:13:40.306051876Z","level":"INFO","msg":"created new stream","id":"sphrjdl0"} +{"time":"2025-04-23T11:13:40.306083735Z","level":"INFO","msg":"stream: started","id":"sphrjdl0"} +{"time":"2025-04-23T11:13:40.307359352Z","level":"INFO","msg":"handler: started","stream_id":"sphrjdl0"} +{"time":"2025-04-23T11:13:40.307389028Z","level":"INFO","msg":"writer: Do: started","stream_id":"sphrjdl0"} +{"time":"2025-04-23T11:13:40.307578803Z","level":"INFO","msg":"sender: started","stream_id":"sphrjdl0"} +{"time":"2025-04-23T11:13:40.59187667Z","level":"INFO","msg":"Starting system monitor"} diff --git a/CKPTS/Afteri2i_100kckpt_LLAVAT5_empty_Fixed_just_segment_pretrain_stage3_dreambench_recap_400k_t2i_400k_flux_200k_midjourney_150k_recovery_150k_3e4_epoch2_half_rotated/wandb/run-20250423_111340-sphrjdl0/logs/debug.log b/CKPTS/Afteri2i_100kckpt_LLAVAT5_empty_Fixed_just_segment_pretrain_stage3_dreambench_recap_400k_t2i_400k_flux_200k_midjourney_150k_recovery_150k_3e4_epoch2_half_rotated/wandb/run-20250423_111340-sphrjdl0/logs/debug.log new file mode 100644 index 0000000000000000000000000000000000000000..c25076963903014f4d29133e1a151d822b21b6ce --- /dev/null +++ b/CKPTS/Afteri2i_100kckpt_LLAVAT5_empty_Fixed_just_segment_pretrain_stage3_dreambench_recap_400k_t2i_400k_flux_200k_midjourney_150k_recovery_150k_3e4_epoch2_half_rotated/wandb/run-20250423_111340-sphrjdl0/logs/debug.log @@ -0,0 +1,22 @@ +2025-04-23 11:13:40,093 INFO MainThread:3152613 [wandb_setup.py:_flush():67] Current SDK version is 0.19.8 +2025-04-23 11:13:40,093 INFO MainThread:3152613 [wandb_setup.py:_flush():67] Configure stats pid to 3152613 +2025-04-23 11:13:40,093 INFO MainThread:3152613 [wandb_setup.py:_flush():67] Loading settings from /tmp/haozhezhao/.config/wandb/settings +2025-04-23 11:13:40,093 INFO MainThread:3152613 [wandb_setup.py:_flush():67] Loading settings from /tmp/haozhezhao/MLLMG/wandb/settings +2025-04-23 11:13:40,093 INFO MainThread:3152613 [wandb_setup.py:_flush():67] Loading settings from environment variables +2025-04-23 11:13:40,093 INFO MainThread:3152613 [wandb_init.py:setup_run_log_directory():647] Logging user logs to checkpoint/Afteri2i_100kckpt_LLAVAT5_empty_Fixed_just_segment_pretrain_stage3_dreambench_recap_400k_t2i_400k_flux_200k_midjourney_150k_recovery_150k_3e4_epoch2_half_rotated/wandb/run-20250423_111340-sphrjdl0/logs/debug.log +2025-04-23 11:13:40,093 INFO MainThread:3152613 [wandb_init.py:setup_run_log_directory():648] Logging internal logs to checkpoint/Afteri2i_100kckpt_LLAVAT5_empty_Fixed_just_segment_pretrain_stage3_dreambench_recap_400k_t2i_400k_flux_200k_midjourney_150k_recovery_150k_3e4_epoch2_half_rotated/wandb/run-20250423_111340-sphrjdl0/logs/debug-internal.log +2025-04-23 11:13:40,093 INFO MainThread:3152613 [wandb_init.py:init():761] calling init triggers +2025-04-23 11:13:40,093 INFO MainThread:3152613 [wandb_init.py:init():766] wandb.init called with sweep_config: {} +config: {'data_path': '/tmp/haozhezhao/MLLMG/jsonl_data/merged_train_set_set_subject_400k_recap_t2i_400k_flux_200k_midjourney_150k_recovery_150k_grounding_100fluxseg_50samseg.jsonl', 'cloud_save_path': '/tmp/haozhezhao/checkpoint', 'no_local_save': False, 'vq_model': 'VQ-16', 'vq_ckpt': '/tmp/haozhezhao/model/llamagen_t2i/vq_ds16_t2i.pt', 'codebook_size': 16384, 'codebook_embed_dim': 8, 'gpt_model': 'GPT-XL', 'gpt_ckpt': '/tmp/haozhezhao/MLLMG/MLLMG_ckpts/checkpoints/AfterI2I_llavat5_subject_train_all_T5_subject_t2i_ti2i_120_w_flux_segment_noimageonly_empty_fix_train_all/0100000.pt', 'gpt_type': 't2i', 'vocab_size': 16384, 'cls_token_num': 512, 'dropout_p': 0.1, 'token_dropout_p': 0.1, 'drop_path': 0.0, 'no_compile': False, 'results_dir': 'checkpoint/Afteri2i_100kckpt_LLAVAT5_empty_Fixed_just_segment_pretrain_stage3_dreambench_recap_400k_t2i_400k_flux_200k_midjourney_150k_recovery_150k_3e4_epoch2_half_rotated', 'dataset': 'ti2i', 'image_size': 512, 'downsample_size': 16, 'num_classes': 1000, 'epochs': 2, 'lr': 0.0003, 'weight_decay': 0.05, 'beta1': 0.9, 'beta2': 0.95, 'max_grad_norm': 1.0, 'global_batch_size': 56, 'global_seed': 0, 'num_workers': 8, 'log_every': 25, 'ckpt_every': 2000, 'gradient_accumulation_steps': 4, 'mixed_precision': 'bf16', 'val_data_path': '/tmp/haozhezhao/MLLMG/jsonl_data/dreambench_plus_valid.jsonl', 'use_vision_tower': True, 'model_name_or_path': '/tmp/haozhezhao/model/blip2-flan-t5-xl', 'image_place_holder': '', 'processor_path': None, 'do_eval': True, 'max_eval_samples': 500, 'train_text_encoder': True, 'no_left_padding': False, 'cfg_scale': 7.5, 'top_k': 16384, 'temperature': 0.9, 'top_p': 1.0, 'eval_steps': 2000, 'project_name': 'llamagen_ti2i', 'load_from_checkpoint': '/tmp/haozhezhao/MLLMG/checkpoint/FIXed_3M_1epoch_step32000.pt', 'warmup': 0.05, 'lr_decay_style': 'cosine', 'lr_decay_ratio': 0.1, 'train_iters': 500000, 'class_dropout_prob': 0.1, 'with_image_only': False, 'image_only_rate': 0.1, 'stage2': False, 'subject_driven': True, 'load_subject_embedding': None, 'reference_data_path': '/tmp/haozhezhao/MLLMG/cc12m_reference_tunnel.jsonl', 'multimodal_encoder': 'llava', 'do_recovery': True, 'no_replace': False, 'resume': False, 'dreambench_eval': True, 'find_unused_parameters': True, 'load_visual_encoder': False, 'continue_stage1': False, 'replace_subject': False, 'train_all': True, 'save_total_limit': 1, 'load_language_projection': '/tmp/haozhezhao/MLLMG/llava-v1.5-flant5_fixed-pretrain/mm_projector.bin', 'mm_vision_tower': 'openai/clip-vit-large-patch14', 'load_fixed_llamagen': True, 'unfreeze_output': False, 'fix': 'gpt-empty-fix', 'rank': 0, 'world_size': 8, 'gpu': 0, 'dist_url': 'env://', 'distributed': True, 'dist_backend': 'nccl', '_wandb': {}} +2025-04-23 11:13:40,093 INFO MainThread:3152613 [wandb_init.py:init():784] starting backend +2025-04-23 11:13:40,093 INFO MainThread:3152613 [wandb_init.py:init():788] sending inform_init request +2025-04-23 11:13:40,097 INFO MainThread:3152613 [backend.py:_multiprocessing_setup():101] multiprocessing start_methods=fork,spawn,forkserver, using: spawn +2025-04-23 11:13:40,097 INFO MainThread:3152613 [wandb_init.py:init():798] backend started and connected +2025-04-23 11:13:40,099 INFO MainThread:3152613 [wandb_init.py:init():891] updated telemetry +2025-04-23 11:13:40,100 INFO MainThread:3152613 [wandb_init.py:init():915] communicating run to backend with 90.0 second timeout +2025-04-23 11:13:40,589 INFO MainThread:3152613 [wandb_init.py:init():990] starting run threads in backend +2025-04-23 11:13:40,711 INFO MainThread:3152613 [wandb_run.py:_console_start():2375] atexit reg +2025-04-23 11:13:40,712 INFO MainThread:3152613 [wandb_run.py:_redirect():2227] redirect: wrap_raw +2025-04-23 11:13:40,712 INFO MainThread:3152613 [wandb_run.py:_redirect():2292] Wrapping output streams. +2025-04-23 11:13:40,712 INFO MainThread:3152613 [wandb_run.py:_redirect():2315] Redirects installed. +2025-04-23 11:13:40,714 INFO MainThread:3152613 [wandb_init.py:init():1032] run started, returning control to user process diff --git a/CKPTS/Afteri2i_100kckpt_LLAVAT5_empty_Fixed_just_segment_pretrain_stage3_dreambench_recap_400k_t2i_400k_flux_200k_midjourney_150k_recovery_150k_3e4_epoch2_half_rotated/wandb/run-20250423_111340-sphrjdl0/run-sphrjdl0.wandb b/CKPTS/Afteri2i_100kckpt_LLAVAT5_empty_Fixed_just_segment_pretrain_stage3_dreambench_recap_400k_t2i_400k_flux_200k_midjourney_150k_recovery_150k_3e4_epoch2_half_rotated/wandb/run-20250423_111340-sphrjdl0/run-sphrjdl0.wandb new file mode 100644 index 0000000000000000000000000000000000000000..d04264ed201b69dfba0325acdfde0dbe01166506 --- /dev/null +++ b/CKPTS/Afteri2i_100kckpt_LLAVAT5_empty_Fixed_just_segment_pretrain_stage3_dreambench_recap_400k_t2i_400k_flux_200k_midjourney_150k_recovery_150k_3e4_epoch2_half_rotated/wandb/run-20250423_111340-sphrjdl0/run-sphrjdl0.wandb @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:1357a055a17159c9a40eeaba47f49cc78c693d50dd3e97b3e0ea819755d62260 +size 1212416 diff --git a/CKPTS/Afteri2i_100kckpt_LLAVAT5_empty_Fixed_just_segment_pretrain_stage3_dreambench_recap_400k_t2i_400k_flux_200k_midjourney_150k_recovery_150k_3e4_epoch2_half_rotated/wandb/run-20250423_122459-k5belznu/files/config.yaml b/CKPTS/Afteri2i_100kckpt_LLAVAT5_empty_Fixed_just_segment_pretrain_stage3_dreambench_recap_400k_t2i_400k_flux_200k_midjourney_150k_recovery_150k_3e4_epoch2_half_rotated/wandb/run-20250423_122459-k5belznu/files/config.yaml new file mode 100644 index 0000000000000000000000000000000000000000..97dc1ed2eee3bd2ecb9bd044b4cf646ae91da697 --- /dev/null +++ b/CKPTS/Afteri2i_100kckpt_LLAVAT5_empty_Fixed_just_segment_pretrain_stage3_dreambench_recap_400k_t2i_400k_flux_200k_midjourney_150k_recovery_150k_3e4_epoch2_half_rotated/wandb/run-20250423_122459-k5belznu/files/config.yaml @@ -0,0 +1,199 @@ +_wandb: + value: + cli_version: 0.19.8 + m: [] + python_version: 3.11.11 + t: + "1": + - 1 + - 11 + - 41 + - 49 + - 51 + - 55 + "2": + - 1 + - 11 + - 41 + - 49 + - 51 + - 55 + "3": + - 13 + - 16 + - 23 + - 55 + - 61 + "4": 3.11.11 + "5": 0.19.8 + "6": 4.49.0 + "8": + - 5 + "12": 0.19.8 + "13": linux-x86_64 +beta1: + value: 0.9 +beta2: + value: 0.95 +cfg_scale: + value: 7.5 +ckpt_every: + value: 2000 +class_dropout_prob: + value: 0.1 +cloud_save_path: + value: /tmp/haozhezhao/checkpoint +cls_token_num: + value: 512 +codebook_embed_dim: + value: 8 +codebook_size: + value: 16384 +continue_stage1: + value: false +data_path: + value: /tmp/haozhezhao/MLLMG/jsonl_data/merged_train_set_set_subject_400k_recap_t2i_400k_flux_200k_midjourney_150k_recovery_150k_grounding_100fluxseg_50samseg.jsonl +dataset: + value: ti2i +dist_backend: + value: nccl +dist_url: + value: env:// +distributed: + value: true +do_eval: + value: true +do_recovery: + value: true +downsample_size: + value: 16 +dreambench_eval: + value: true +drop_path: + value: 0 +dropout_p: + value: 0.1 +epochs: + value: 2 +eval_steps: + value: 2000 +find_unused_parameters: + value: true +fix: + value: gpt-empty-fix +global_batch_size: + value: 56 +global_seed: + value: 0 +gpt_ckpt: + value: /tmp/haozhezhao/MLLMG/MLLMG_ckpts/checkpoints/AfterI2I_llavat5_subject_train_all_T5_subject_t2i_ti2i_120_w_flux_segment_noimageonly_empty_fix_train_all/0100000.pt +gpt_model: + value: GPT-XL +gpt_type: + value: t2i +gpu: + value: 0 +gradient_accumulation_steps: + value: 4 +image_only_rate: + value: 0.1 +image_place_holder: + value: +image_size: + value: 512 +load_fixed_llamagen: + value: true +load_from_checkpoint: + value: /tmp/haozhezhao/MLLMG/checkpoint/FIXed_3M_1epoch_step32000.pt +load_language_projection: + value: /tmp/haozhezhao/MLLMG/llava-v1.5-flant5_fixed-pretrain/mm_projector.bin +load_subject_embedding: + value: null +load_visual_encoder: + value: false +log_every: + value: 25 +lr: + value: 0.0003 +lr_decay_ratio: + value: 0.1 +lr_decay_style: + value: cosine +max_eval_samples: + value: 250 +max_grad_norm: + value: 1 +mixed_precision: + value: bf16 +mm_vision_tower: + value: openai/clip-vit-large-patch14 +model_name_or_path: + value: /tmp/haozhezhao/model/blip2-flan-t5-xl +multimodal_encoder: + value: llava +no_compile: + value: false +no_left_padding: + value: false +no_local_save: + value: false +no_replace: + value: false +num_classes: + value: 1000 +num_workers: + value: 8 +processor_path: + value: null +project_name: + value: llamagen_ti2i +rank: + value: 0 +reference_data_path: + value: /tmp/haozhezhao/MLLMG/cc12m_reference_tunnel.jsonl +replace_subject: + value: false +results_dir: + value: checkpoint/Afteri2i_100kckpt_LLAVAT5_empty_Fixed_just_segment_pretrain_stage3_dreambench_recap_400k_t2i_400k_flux_200k_midjourney_150k_recovery_150k_3e4_epoch2_half_rotated +resume: + value: false +save_total_limit: + value: 1 +stage2: + value: false +subject_driven: + value: true +temperature: + value: 0.9 +token_dropout_p: + value: 0.1 +top_k: + value: 16384 +top_p: + value: 1 +train_all: + value: true +train_iters: + value: 500000 +train_text_encoder: + value: true +unfreeze_output: + value: false +use_vision_tower: + value: true +val_data_path: + value: /tmp/haozhezhao/MLLMG/jsonl_data/dreambench_plus_valid.jsonl +vocab_size: + value: 16384 +vq_ckpt: + value: /tmp/haozhezhao/model/llamagen_t2i/vq_ds16_t2i.pt +vq_model: + value: VQ-16 +warmup: + value: 0.05 +weight_decay: + value: 0.05 +with_image_only: + value: false +world_size: + value: 8 diff --git a/CKPTS/Afteri2i_100kckpt_LLAVAT5_empty_Fixed_just_segment_pretrain_stage3_dreambench_recap_400k_t2i_400k_flux_200k_midjourney_150k_recovery_150k_3e4_epoch2_half_rotated/wandb/run-20250423_122459-k5belznu/files/output.log b/CKPTS/Afteri2i_100kckpt_LLAVAT5_empty_Fixed_just_segment_pretrain_stage3_dreambench_recap_400k_t2i_400k_flux_200k_midjourney_150k_recovery_150k_3e4_epoch2_half_rotated/wandb/run-20250423_122459-k5belznu/files/output.log new file mode 100644 index 0000000000000000000000000000000000000000..a252b18b1d5ca648a9bef5be8892ca0683be5561 --- /dev/null +++ b/CKPTS/Afteri2i_100kckpt_LLAVAT5_empty_Fixed_just_segment_pretrain_stage3_dreambench_recap_400k_t2i_400k_flux_200k_midjourney_150k_recovery_150k_3e4_epoch2_half_rotated/wandb/run-20250423_122459-k5belznu/files/output.log @@ -0,0 +1,2226 @@ +[2025-04-23 12:25:00] Training for 2 epochs... +[2025-04-23 12:25:00] Beginning epoch 0... + 0%| | 0/23458 [00:00 +tokenizer length after expend 32101 +tokenizer length before expend 32100 + /tmp/haozhezhao/anaconda3/envs/nlp/lib/python3.11/contextlib.py:105: FutureWarning: `torch.backends.cuda.sdp_kernel()` is deprecated. In the future, this context manager will be removed. Please see `torch.nn.attention.sdpa_kernel()` for the new context manager, with updated signature. + self.gen = func(*args, **kwds) | 0/4 [00:00 +tokenizer length after expend 32101 +tokenizer length before expend 32100 +100%|████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████| 4/4 [03:58<00:00, 59.59s/it] +[2025-04-23 13:42:01] Finish Eval in 4000 steps...██████████████████████████████████████████████████████████████████████████████████████████████████████████████████| 4/4 [03:57<00:00, 59.33s/it] +processing left-padding... +processing left-padding... +processing left-padding... +processing left-padding... +[2025-04-23 13:42:19] Saved checkpoint to checkpoint/Afteri2i_100kckpt_LLAVAT5_empty_Fixed_just_segment_pretrain_stage3_dreambench_recap_400k_t2i_400k_flux_200k_midjourney_150k_recovery_150k_3e4_epoch2_half_rotated/005-GPT-XL/checkpoints/0004000.pt +[2025-04-23 13:42:21] Removed old checkpoint: checkpoint/Afteri2i_100kckpt_LLAVAT5_empty_Fixed_just_segment_pretrain_stage3_dreambench_recap_400k_t2i_400k_flux_200k_midjourney_150k_recovery_150k_3e4_epoch2_half_rotated/005-GPT-XL/checkpoints/0002000.pt + 17%|██████████████████████████▉ | 4024/23458 [1:17:41<4:46:39, 1.13it/s][2025-04-23 13:42:42] (step=0004025) Train Loss: 5.6919, Train Steps/Sec: 0.07 + 17%|███████████████████████████ | 4049/23458 [1:18:02<4:36:44, 1.17it/s][2025-04-23 13:43:03] (step=0004050) Train Loss: 5.6571, Train Steps/Sec: 1.17 + 17%|███████████████████████████▎ | 4074/23458 [1:18:26<4:42:04, 1.15it/s][2025-04-23 13:43:27] (step=0004075) Train Loss: 5.6853, Train Steps/Sec: 1.06 + 17%|███████████████████████████▍ | 4099/23458 [1:18:47<4:31:31, 1.19it/s][2025-04-23 13:43:49] (step=0004100) Train Loss: 5.5925, Train Steps/Sec: 1.17 + 18%|███████████████████████████▌ | 4124/23458 [1:19:08<4:39:55, 1.15it/s][2025-04-23 13:44:10] (step=0004125) Train Loss: 5.5697, Train Steps/Sec: 1.17 + 18%|███████████████████████████▊ | 4149/23458 [1:19:30<4:35:05, 1.17it/s][2025-04-23 13:44:31] (step=0004150) Train Loss: 5.6736, Train Steps/Sec: 1.17 + 18%|███████████████████████████▉ | 4174/23458 [1:19:51<4:33:09, 1.18it/s][2025-04-23 13:44:53] (step=0004175) Train Loss: 5.6693, Train Steps/Sec: 1.17 + 18%|████████████████████████████ | 4199/23458 [1:20:13<4:30:17, 1.19it/s][2025-04-23 13:45:14] (step=0004200) Train Loss: 5.6101, Train Steps/Sec: 1.16 + 18%|████████████████████████████▎ | 4224/23458 [1:20:34<4:38:20, 1.15it/s][2025-04-23 13:45:35] (step=0004225) Train Loss: 5.6586, Train Steps/Sec: 1.17 + 18%|████████████████████████████▍ | 4249/23458 [1:20:55<4:33:25, 1.17it/s][2025-04-23 13:45:57] (step=0004250) Train Loss: 5.5988, Train Steps/Sec: 1.17 + 18%|████████████████████████████▌ | 4274/23458 [1:21:17<4:32:56, 1.17it/s][2025-04-23 13:46:18] (step=0004275) Train Loss: 5.6851, Train Steps/Sec: 1.17 + 18%|████████████████████████████▊ | 4299/23458 [1:21:38<4:28:46, 1.19it/s][2025-04-23 13:46:40] (step=0004300) Train Loss: 5.6974, Train Steps/Sec: 1.17 + 18%|████████████████████████████▉ | 4324/23458 [1:21:59<4:37:21, 1.15it/s][2025-04-23 13:47:01] (step=0004325) Train Loss: 5.6270, Train Steps/Sec: 1.17 + 19%|█████████████████████████████ | 4349/23458 [1:22:21<4:31:37, 1.17it/s][2025-04-23 13:47:22] (step=0004350) Train Loss: 5.6210, Train Steps/Sec: 1.17 + 19%|█████████████████████████████▎ | 4374/23458 [1:22:42<4:28:58, 1.18it/s][2025-04-23 13:47:43] (step=0004375) Train Loss: 5.6534, Train Steps/Sec: 1.17 + 19%|█████████████████████████████▍ | 4399/23458 [1:23:03<4:27:28, 1.19it/s][2025-04-23 13:48:05] (step=0004400) Train Loss: 5.5567, Train Steps/Sec: 1.17 + 19%|█████████████████████████████▌ | 4424/23458 [1:23:25<4:34:57, 1.15it/s][2025-04-23 13:48:26] (step=0004425) Train Loss: 5.6524, Train Steps/Sec: 1.17 + 19%|█████████████████████████████▊ | 4449/23458 [1:23:46<4:31:18, 1.17it/s][2025-04-23 13:48:48] (step=0004450) Train Loss: 5.6659, Train Steps/Sec: 1.17 + 19%|█████████████████████████████▉ | 4474/23458 [1:24:08<4:29:35, 1.17it/s][2025-04-23 13:49:09] (step=0004475) Train Loss: 5.6258, Train Steps/Sec: 1.17 + 19%|██████████████████████████████ | 4499/23458 [1:24:29<4:26:00, 1.19it/s][2025-04-23 13:49:30] (step=0004500) Train Loss: 5.5931, Train Steps/Sec: 1.17 + 19%|██████████████████████████████▎ | 4524/23458 [1:24:51<4:33:55, 1.15it/s][2025-04-23 13:49:52] (step=0004525) Train Loss: 5.7624, Train Steps/Sec: 1.13 + 19%|██████████████████████████████▍ | 4549/23458 [1:25:13<4:32:39, 1.16it/s][2025-04-23 13:50:15] (step=0004550) Train Loss: 5.6551, Train Steps/Sec: 1.13 + 19%|██████████████████████████████▌ | 4574/23458 [1:25:36<5:20:59, 1.02s/it][2025-04-23 13:50:38] (step=0004575) Train Loss: 5.5787, Train Steps/Sec: 1.09 + 20%|██████████████████████████████▊ | 4599/23458 [1:25:58<4:25:20, 1.18it/s][2025-04-23 13:50:59] (step=0004600) Train Loss: 5.6529, Train Steps/Sec: 1.17 + 20%|██████████████████████████████▉ | 4624/23458 [1:26:19<4:33:42, 1.15it/s][2025-04-23 13:51:20] (step=0004625) Train Loss: 5.6690, Train Steps/Sec: 1.17 + 20%|███████████████████████████████ | 4649/23458 [1:26:41<4:28:27, 1.17it/s][2025-04-23 13:51:42] (step=0004650) Train Loss: 5.6965, Train Steps/Sec: 1.13 + 20%|███████████████████████████████▎ | 4674/23458 [1:27:03<4:25:25, 1.18it/s][2025-04-23 13:52:05] (step=0004675) Train Loss: 5.6870, Train Steps/Sec: 1.13 + 20%|███████████████████████████████▍ | 4699/23458 [1:27:25<4:24:35, 1.18it/s][2025-04-23 13:52:26] (step=0004700) Train Loss: 5.5647, Train Steps/Sec: 1.16 + 20%|███████████████████████████████▌ | 4724/23458 [1:27:46<4:32:51, 1.14it/s][2025-04-23 13:52:48] (step=0004725) Train Loss: 5.6140, Train Steps/Sec: 1.17 + 20%|███████████████████████████████▊ | 4749/23458 [1:28:07<4:25:33, 1.17it/s][2025-04-23 13:53:09] (step=0004750) Train Loss: 5.6157, Train Steps/Sec: 1.17 + 20%|███████████████████████████████▉ | 4774/23458 [1:28:29<4:24:40, 1.18it/s][2025-04-23 13:53:30] (step=0004775) Train Loss: 5.6005, Train Steps/Sec: 1.17 + 20%|████████████████████████████████ | 4799/23458 [1:28:52<4:24:43, 1.17it/s][2025-04-23 13:53:53] (step=0004800) Train Loss: 5.5967, Train Steps/Sec: 1.07 + 21%|████████████████████████████████▎ | 4824/23458 [1:29:13<4:29:26, 1.15it/s][2025-04-23 13:54:15] (step=0004825) Train Loss: 5.5716, Train Steps/Sec: 1.17 + 21%|████████████████████████████████▍ | 4849/23458 [1:29:35<4:24:34, 1.17it/s][2025-04-23 13:54:36] (step=0004850) Train Loss: 5.5744, Train Steps/Sec: 1.17 + 21%|████████████████████████████████▌ | 4874/23458 [1:29:56<4:22:18, 1.18it/s][2025-04-23 13:54:57] (step=0004875) Train Loss: 5.6118, Train Steps/Sec: 1.17 + 21%|████████████████████████████████▊ | 4899/23458 [1:30:17<4:20:21, 1.19it/s][2025-04-23 13:55:19] (step=0004900) Train Loss: 5.5919, Train Steps/Sec: 1.17 + 21%|████████████████████████████████▉ | 4924/23458 [1:30:39<4:29:45, 1.15it/s][2025-04-23 13:55:40] (step=0004925) Train Loss: 5.6854, Train Steps/Sec: 1.17 + 21%|█████████████████████████████████ | 4949/23458 [1:31:00<4:22:44, 1.17it/s][2025-04-23 13:56:01] (step=0004950) Train Loss: 5.6321, Train Steps/Sec: 1.17 + 21%|█████████████████████████████████▎ | 4974/23458 [1:31:22<4:20:39, 1.18it/s][2025-04-23 13:56:23] (step=0004975) Train Loss: 5.6389, Train Steps/Sec: 1.17 + 21%|█████████████████████████████████▍ | 4999/23458 [1:31:43<4:20:40, 1.18it/s][2025-04-23 13:56:44] (step=0005000) Train Loss: 5.7056, Train Steps/Sec: 1.17 + 21%|█████████████████████████████████▌ | 5024/23458 [1:32:04<4:26:59, 1.15it/s][2025-04-23 13:57:06] (step=0005025) Train Loss: 5.5637, Train Steps/Sec: 1.17 + 22%|█████████████████████████████████▊ | 5049/23458 [1:32:26<4:22:07, 1.17it/s][2025-04-23 13:57:27] (step=0005050) Train Loss: 5.5964, Train Steps/Sec: 1.17 + 22%|█████████████████████████████████▉ | 5074/23458 [1:32:47<4:22:10, 1.17it/s][2025-04-23 13:57:48] (step=0005075) Train Loss: 5.6212, Train Steps/Sec: 1.17 + 22%|██████████████████████████████████▏ | 5099/23458 [1:33:08<4:19:34, 1.18it/s][2025-04-23 13:58:10] (step=0005100) Train Loss: 5.6681, Train Steps/Sec: 1.17 + 22%|██████████████████████████████████▎ | 5124/23458 [1:33:30<4:24:53, 1.15it/s][2025-04-23 13:58:31] (step=0005125) Train Loss: 5.6060, Train Steps/Sec: 1.17 + 22%|██████████████████████████████████▍ | 5149/23458 [1:33:51<4:21:21, 1.17it/s][2025-04-23 13:58:53] (step=0005150) Train Loss: 5.5869, Train Steps/Sec: 1.17 + 22%|██████████████████████████████████▋ | 5174/23458 [1:34:13<4:31:00, 1.12it/s][2025-04-23 13:59:15] (step=0005175) Train Loss: 5.6572, Train Steps/Sec: 1.13 + 22%|██████████████████████████████████▊ | 5199/23458 [1:34:36<4:56:39, 1.03it/s][2025-04-23 13:59:37] (step=0005200) Train Loss: 5.6732, Train Steps/Sec: 1.12 + 22%|██████████████████████████████████▉ | 5224/23458 [1:34:58<4:25:39, 1.14it/s][2025-04-23 13:59:59] (step=0005225) Train Loss: 5.5777, Train Steps/Sec: 1.13 + 22%|███████████████████████████████████▏ | 5249/23458 [1:35:20<4:19:30, 1.17it/s][2025-04-23 14:00:21] (step=0005250) Train Loss: 5.6421, Train Steps/Sec: 1.13 + 22%|███████████████████████████████████▎ | 5274/23458 [1:35:41<4:16:12, 1.18it/s][2025-04-23 14:00:43] (step=0005275) Train Loss: 5.6117, Train Steps/Sec: 1.17 + 23%|███████████████████████████████████▍ | 5299/23458 [1:36:03<4:15:34, 1.18it/s][2025-04-23 14:01:05] (step=0005300) Train Loss: 5.5680, Train Steps/Sec: 1.12 + 23%|███████████████████████████████████▋ | 5324/23458 [1:36:26<4:22:57, 1.15it/s][2025-04-23 14:01:27] (step=0005325) Train Loss: 5.6310, Train Steps/Sec: 1.13 + 23%|███████████████████████████████████▊ | 5349/23458 [1:36:47<4:18:14, 1.17it/s][2025-04-23 14:01:48] (step=0005350) Train Loss: 5.6257, Train Steps/Sec: 1.17 + 23%|███████████████████████████████████▉ | 5374/23458 [1:37:08<4:16:17, 1.18it/s][2025-04-23 14:02:10] (step=0005375) Train Loss: 5.6289, Train Steps/Sec: 1.17 + 23%|████████████████████████████████████▏ | 5399/23458 [1:37:30<4:14:37, 1.18it/s][2025-04-23 14:02:31] (step=0005400) Train Loss: 5.6249, Train Steps/Sec: 1.17 + 23%|████████████████████████████████████▎ | 5424/23458 [1:37:51<4:21:26, 1.15it/s][2025-04-23 14:02:53] (step=0005425) Train Loss: 5.6360, Train Steps/Sec: 1.17 + 23%|████████████████████████████████████▍ | 5449/23458 [1:38:13<4:16:40, 1.17it/s][2025-04-23 14:03:14] (step=0005450) Train Loss: 5.7072, Train Steps/Sec: 1.17 + 23%|████████████████████████████████████▋ | 5474/23458 [1:38:34<4:15:03, 1.18it/s][2025-04-23 14:03:35] (step=0005475) Train Loss: 5.5918, Train Steps/Sec: 1.17 + 23%|████████████████████████████████████▊ | 5499/23458 [1:38:55<4:13:21, 1.18it/s][2025-04-23 14:03:57] (step=0005500) Train Loss: 5.6338, Train Steps/Sec: 1.17 + 24%|████████████████████████████████████▉ | 5524/23458 [1:39:19<4:20:44, 1.15it/s][2025-04-23 14:04:20] (step=0005525) Train Loss: 5.5733, Train Steps/Sec: 1.07 + 24%|█████████████████████████████████████▏ | 5549/23458 [1:39:40<4:14:35, 1.17it/s][2025-04-23 14:04:41] (step=0005550) Train Loss: 5.6391, Train Steps/Sec: 1.17 + 24%|█████████████████████████████████████▎ | 5574/23458 [1:40:01<4:11:47, 1.18it/s][2025-04-23 14:05:02] (step=0005575) Train Loss: 5.5895, Train Steps/Sec: 1.17 + 24%|█████████████████████████████████████▍ | 5599/23458 [1:40:22<4:11:59, 1.18it/s][2025-04-23 14:05:24] (step=0005600) Train Loss: 5.6949, Train Steps/Sec: 1.17 + 24%|█████████████████████████████████████▋ | 5624/23458 [1:40:44<4:18:44, 1.15it/s][2025-04-23 14:05:45] (step=0005625) Train Loss: 5.5547, Train Steps/Sec: 1.17 + 24%|█████████████████████████████████████▊ | 5649/23458 [1:41:05<4:13:37, 1.17it/s][2025-04-23 14:06:07] (step=0005650) Train Loss: 5.6769, Train Steps/Sec: 1.17 + 24%|█████████████████████████████████████▉ | 5674/23458 [1:41:27<4:11:14, 1.18it/s][2025-04-23 14:06:28] (step=0005675) Train Loss: 5.6761, Train Steps/Sec: 1.17 + 24%|██████████████████████████████████████▏ | 5699/23458 [1:41:48<4:09:08, 1.19it/s][2025-04-23 14:06:49] (step=0005700) Train Loss: 5.6436, Train Steps/Sec: 1.17 + 24%|██████████████████████████████████████▎ | 5724/23458 [1:42:09<4:16:02, 1.15it/s][2025-04-23 14:07:11] (step=0005725) Train Loss: 5.6285, Train Steps/Sec: 1.17 + 25%|██████████████████████████████████████▍ | 5749/23458 [1:42:31<4:11:55, 1.17it/s][2025-04-23 14:07:32] (step=0005750) Train Loss: 5.5609, Train Steps/Sec: 1.17 + 25%|██████████████████████████████████████▋ | 5774/23458 [1:42:52<4:10:14, 1.18it/s][2025-04-23 14:07:53] (step=0005775) Train Loss: 5.6778, Train Steps/Sec: 1.17 + 25%|██████████████████████████████████████▊ | 5799/23458 [1:43:13<4:08:32, 1.18it/s][2025-04-23 14:08:15] (step=0005800) Train Loss: 5.5576, Train Steps/Sec: 1.17 + 25%|██████████████████████████████████████▉ | 5824/23458 [1:43:35<4:15:05, 1.15it/s][2025-04-23 14:08:36] (step=0005825) Train Loss: 5.6316, Train Steps/Sec: 1.17 + 25%|███████████████████████████████████████▏ | 5849/23458 [1:43:57<4:11:21, 1.17it/s][2025-04-23 14:08:58] (step=0005850) Train Loss: 5.6223, Train Steps/Sec: 1.13 + 25%|███████████████████████████████████████▎ | 5874/23458 [1:44:20<4:46:30, 1.02it/s][2025-04-23 14:09:21] (step=0005875) Train Loss: 5.5707, Train Steps/Sec: 1.09 + 25%|███████████████████████████████████████▍ | 5899/23458 [1:44:42<4:09:49, 1.17it/s][2025-04-23 14:09:43] (step=0005900) Train Loss: 5.5906, Train Steps/Sec: 1.13 + 25%|███████████████████████████████████████▋ | 5924/23458 [1:45:03<4:13:16, 1.15it/s][2025-04-23 14:10:05] (step=0005925) Train Loss: 5.5430, Train Steps/Sec: 1.17 + 25%|███████████████████████████████████████▊ | 5949/23458 [1:45:26<5:04:58, 1.05s/it][2025-04-23 14:10:27] (step=0005950) Train Loss: 5.6428, Train Steps/Sec: 1.12 + 25%|███████████████████████████████████████▉ | 5974/23458 [1:45:48<4:45:50, 1.02it/s][2025-04-23 14:10:49] (step=0005975) Train Loss: 5.6063, Train Steps/Sec: 1.13 + 26%|████████████████████████████████████████▏ | 5999/23458 [1:46:09<4:07:06, 1.18it/s][2025-04-23 14:11:11] (step=0006000) Train Loss: 5.6270, Train Steps/Sec: 1.17 +You are using a model of type blip-2 to instantiate a model of type instructblip. This is not supported for all configurations of models and can yield errors. +[2025-04-23 14:11:11] text_config is None. Initializing the text config with default values (`OPTConfig`). +model_name_or_path:/tmp/haozhezhao/model/blip2-flan-t5-xl +image_place_holder: +tokenizer length after expend 32101 +tokenizer length before expend 32100 +100%|████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████| 4/4 [03:59<00:00, 59.85s/it] +[2025-04-23 14:16:18] Finish Eval in 6000 steps...██████████████████████████████████████████████████████████████████████████████████████████████████████████████████| 4/4 [03:59<00:00, 59.42s/it] +processing left-padding... +processing left-padding... +processing left-padding... +processing left-padding... +[2025-04-23 14:16:37] Saved checkpoint to checkpoint/Afteri2i_100kckpt_LLAVAT5_empty_Fixed_just_segment_pretrain_stage3_dreambench_recap_400k_t2i_400k_flux_200k_midjourney_150k_recovery_150k_3e4_epoch2_half_rotated/005-GPT-XL/checkpoints/0006000.pt +[2025-04-23 14:16:39] Removed old checkpoint: checkpoint/Afteri2i_100kckpt_LLAVAT5_empty_Fixed_just_segment_pretrain_stage3_dreambench_recap_400k_t2i_400k_flux_200k_midjourney_150k_recovery_150k_3e4_epoch2_half_rotated/005-GPT-XL/checkpoints/0004000.pt + 26%|████████████████████████████████████████▎ | 6024/23458 [1:51:59<4:17:25, 1.13it/s][2025-04-23 14:17:00] (step=0006025) Train Loss: 5.6388, Train Steps/Sec: 0.07 + 26%|████████████████████████████████████████▍ | 6049/23458 [1:52:20<4:07:12, 1.17it/s][2025-04-23 14:17:22] (step=0006050) Train Loss: 5.7000, Train Steps/Sec: 1.17 + 26%|████████████████████████████████████████▋ | 6074/23458 [1:52:41<4:06:22, 1.18it/s][2025-04-23 14:17:43] (step=0006075) Train Loss: 5.5189, Train Steps/Sec: 1.17 + 26%|████████████████████████████████████████▊ | 6099/23458 [1:53:03<4:04:15, 1.18it/s][2025-04-23 14:18:04] (step=0006100) Train Loss: 5.6494, Train Steps/Sec: 1.17 + 26%|████████████████████████████████████████▉ | 6124/23458 [1:53:24<4:10:40, 1.15it/s][2025-04-23 14:18:26] (step=0006125) Train Loss: 5.5979, Train Steps/Sec: 1.17 + 26%|█████████████████████████████████████████▏ | 6149/23458 [1:53:45<4:06:31, 1.17it/s][2025-04-23 14:18:47] (step=0006150) Train Loss: 5.6135, Train Steps/Sec: 1.17 + 26%|█████████████████████████████████████████▎ | 6174/23458 [1:54:07<4:04:27, 1.18it/s][2025-04-23 14:19:08] (step=0006175) Train Loss: 5.6027, Train Steps/Sec: 1.17 + 26%|█████████████████████████████████████████▍ | 6199/23458 [1:54:28<4:02:42, 1.19it/s][2025-04-23 14:19:30] (step=0006200) Train Loss: 5.6541, Train Steps/Sec: 1.17 + 27%|█████████████████████████████████████████▋ | 6224/23458 [1:54:49<4:08:49, 1.15it/s][2025-04-23 14:19:51] (step=0006225) Train Loss: 5.6020, Train Steps/Sec: 1.17 + 27%|█████████████████████████████████████████▊ | 6249/23458 [1:55:13<4:06:24, 1.16it/s][2025-04-23 14:20:14] (step=0006250) Train Loss: 5.6513, Train Steps/Sec: 1.08 + 27%|█████████████████████████████████████████▉ | 6274/23458 [1:55:34<4:02:51, 1.18it/s][2025-04-23 14:20:35] (step=0006275) Train Loss: 5.5224, Train Steps/Sec: 1.17 + 27%|██████████████████████████████████████████▏ | 6299/23458 [1:55:55<4:02:14, 1.18it/s][2025-04-23 14:20:57] (step=0006300) Train Loss: 5.5502, Train Steps/Sec: 1.17 + 27%|██████████████████████████████████████████▎ | 6324/23458 [1:56:17<4:08:05, 1.15it/s][2025-04-23 14:21:18] (step=0006325) Train Loss: 5.6070, Train Steps/Sec: 1.17 + 27%|██████████████████████████████████████████▍ | 6349/23458 [1:56:38<4:09:24, 1.14it/s][2025-04-23 14:21:40] (step=0006350) Train Loss: 5.5896, Train Steps/Sec: 1.16 + 27%|██████████████████████████████████████████▋ | 6374/23458 [1:57:00<4:02:24, 1.17it/s][2025-04-23 14:22:01] (step=0006375) Train Loss: 5.6756, Train Steps/Sec: 1.16 + 27%|██████████████████████████████████████████▊ | 6399/23458 [1:57:21<4:03:07, 1.17it/s][2025-04-23 14:22:23] (step=0006400) Train Loss: 5.5540, Train Steps/Sec: 1.16 + 27%|██████████████████████████████████████████▉ | 6424/23458 [1:57:43<4:07:18, 1.15it/s][2025-04-23 14:22:44] (step=0006425) Train Loss: 5.6123, Train Steps/Sec: 1.17 + 27%|███████████████████████████████████████████▏ | 6449/23458 [1:58:04<4:02:35, 1.17it/s][2025-04-23 14:23:05] (step=0006450) Train Loss: 5.5652, Train Steps/Sec: 1.17 + 28%|███████████████████████████████████████████▎ | 6474/23458 [1:58:25<4:02:36, 1.17it/s][2025-04-23 14:23:27] (step=0006475) Train Loss: 5.5895, Train Steps/Sec: 1.17 + 28%|███████████████████████████████████████████▍ | 6499/23458 [1:58:48<3:59:08, 1.18it/s][2025-04-23 14:23:49] (step=0006500) Train Loss: 5.5579, Train Steps/Sec: 1.12 + 28%|███████████████████████████████████████████▋ | 6524/23458 [1:59:10<4:04:43, 1.15it/s][2025-04-23 14:24:11] (step=0006525) Train Loss: 5.6102, Train Steps/Sec: 1.13 + 28%|███████████████████████████████████████████▊ | 6549/23458 [1:59:33<4:24:48, 1.06it/s][2025-04-23 14:24:34] (step=0006550) Train Loss: 5.6376, Train Steps/Sec: 1.09 + 28%|███████████████████████████████████████████▉ | 6574/23458 [1:59:54<3:58:57, 1.18it/s][2025-04-23 14:24:56] (step=0006575) Train Loss: 5.6739, Train Steps/Sec: 1.17 + 28%|████████████████████████████████████████████▏ | 6599/23458 [2:00:16<4:00:34, 1.17it/s][2025-04-23 14:25:17] (step=0006600) Train Loss: 5.5637, Train Steps/Sec: 1.16 + 28%|████████████████████████████████████████████▎ | 6624/23458 [2:00:38<4:05:39, 1.14it/s][2025-04-23 14:25:40] (step=0006625) Train Loss: 5.5159, Train Steps/Sec: 1.12 + 28%|████████████████████████████████████████████▌ | 6649/23458 [2:01:00<4:35:57, 1.02it/s][2025-04-23 14:26:02] (step=0006650) Train Loss: 5.5283, Train Steps/Sec: 1.12 + 28%|████████████████████████████████████████████▋ | 6674/23458 [2:01:22<3:58:25, 1.17it/s][2025-04-23 14:26:23] (step=0006675) Train Loss: 5.6168, Train Steps/Sec: 1.17 + 29%|████████████████████████████████████████████▊ | 6699/23458 [2:01:43<3:55:25, 1.19it/s][2025-04-23 14:26:44] (step=0006700) Train Loss: 5.5888, Train Steps/Sec: 1.17 + 29%|█████████████████████████████████████████████ | 6724/23458 [2:02:04<4:03:52, 1.14it/s][2025-04-23 14:27:06] (step=0006725) Train Loss: 5.5776, Train Steps/Sec: 1.17 + 29%|█████████████████████████████████████████████▏ | 6749/23458 [2:02:26<3:57:31, 1.17it/s][2025-04-23 14:27:27] (step=0006750) Train Loss: 5.5456, Train Steps/Sec: 1.17 + 29%|█████████████████████████████████████████████▎ | 6774/23458 [2:02:47<3:56:01, 1.18it/s][2025-04-23 14:27:48] (step=0006775) Train Loss: 5.6196, Train Steps/Sec: 1.17 + 29%|█████████████████████████████████████████████▌ | 6799/23458 [2:03:08<3:53:40, 1.19it/s][2025-04-23 14:28:10] (step=0006800) Train Loss: 5.6310, Train Steps/Sec: 1.17 + 29%|█████████████████████████████████████████████▋ | 6824/23458 [2:03:30<4:00:39, 1.15it/s][2025-04-23 14:28:31] (step=0006825) Train Loss: 5.6441, Train Steps/Sec: 1.17 + 29%|█████████████████████████████████████████████▊ | 6849/23458 [2:03:51<3:57:13, 1.17it/s][2025-04-23 14:28:52] (step=0006850) Train Loss: 5.6383, Train Steps/Sec: 1.17 + 29%|██████████████████████████████████████████████ | 6874/23458 [2:04:12<3:54:17, 1.18it/s][2025-04-23 14:29:14] (step=0006875) Train Loss: 5.5304, Train Steps/Sec: 1.17 + 29%|██████████████████████████████████████████████▏ | 6899/23458 [2:04:34<3:52:58, 1.18it/s][2025-04-23 14:29:35] (step=0006900) Train Loss: 5.6645, Train Steps/Sec: 1.17 + 30%|██████████████████████████████████████████████▎ | 6924/23458 [2:04:55<4:00:36, 1.15it/s][2025-04-23 14:29:57] (step=0006925) Train Loss: 5.5917, Train Steps/Sec: 1.17 + 30%|██████████████████████████████████████████████▌ | 6949/23458 [2:05:16<3:54:26, 1.17it/s][2025-04-23 14:30:18] (step=0006950) Train Loss: 5.5810, Train Steps/Sec: 1.17 + 30%|██████████████████████████████████████████████▋ | 6974/23458 [2:05:40<3:55:09, 1.17it/s][2025-04-23 14:30:41] (step=0006975) Train Loss: 5.5698, Train Steps/Sec: 1.08 + 30%|██████████████████████████████████████████████▊ | 6999/23458 [2:06:01<3:53:33, 1.17it/s][2025-04-23 14:31:03] (step=0007000) Train Loss: 5.6576, Train Steps/Sec: 1.17 + 30%|███████████████████████████████████████████████ | 7024/23458 [2:06:22<3:57:29, 1.15it/s][2025-04-23 14:31:24] (step=0007025) Train Loss: 5.5626, Train Steps/Sec: 1.18 + 30%|███████████████████████████████████████████████▏ | 7049/23458 [2:06:44<3:55:44, 1.16it/s][2025-04-23 14:31:45] (step=0007050) Train Loss: 5.5361, Train Steps/Sec: 1.16 + 30%|███████████████████████████████████████████████▎ | 7074/23458 [2:07:05<3:51:25, 1.18it/s][2025-04-23 14:32:07] (step=0007075) Train Loss: 5.4693, Train Steps/Sec: 1.17 + 30%|███████████████████████████████████████████████▌ | 7099/23458 [2:07:27<3:50:18, 1.18it/s][2025-04-23 14:32:28] (step=0007100) Train Loss: 5.6563, Train Steps/Sec: 1.17 + 30%|███████████████████████████████████████████████▋ | 7124/23458 [2:07:48<3:57:18, 1.15it/s][2025-04-23 14:32:49] (step=0007125) Train Loss: 5.6235, Train Steps/Sec: 1.17 + 30%|███████████████████████████████████████████████▊ | 7149/23458 [2:08:10<3:59:39, 1.13it/s][2025-04-23 14:33:12] (step=0007150) Train Loss: 5.6044, Train Steps/Sec: 1.13 + 31%|████████████████████████████████████████████████ | 7174/23458 [2:08:32<3:58:59, 1.14it/s][2025-04-23 14:33:34] (step=0007175) Train Loss: 5.5612, Train Steps/Sec: 1.13 + 31%|████████████████████████████████████████████████▏ | 7199/23458 [2:08:55<3:50:49, 1.17it/s][2025-04-23 14:33:56] (step=0007200) Train Loss: 5.5341, Train Steps/Sec: 1.12 + 31%|████████████████████████████████████████████████▎ | 7224/23458 [2:09:17<3:53:48, 1.16it/s][2025-04-23 14:34:18] (step=0007225) Train Loss: 5.5611, Train Steps/Sec: 1.13 + 31%|████████████████████████████████████████████████▌ | 7249/23458 [2:09:38<3:51:09, 1.17it/s][2025-04-23 14:34:39] (step=0007250) Train Loss: 5.6122, Train Steps/Sec: 1.17 + 31%|████████████████████████████████████████████████▋ | 7274/23458 [2:10:00<4:01:36, 1.12it/s][2025-04-23 14:35:02] (step=0007275) Train Loss: 5.5743, Train Steps/Sec: 1.13 + 31%|████████████████████████████████████████████████▊ | 7299/23458 [2:10:22<3:46:27, 1.19it/s][2025-04-23 14:35:23] (step=0007300) Train Loss: 5.6194, Train Steps/Sec: 1.17 + 31%|█████████████████████████████████████████████████ | 7324/23458 [2:10:44<3:54:19, 1.15it/s][2025-04-23 14:35:45] (step=0007325) Train Loss: 5.6055, Train Steps/Sec: 1.13 + 31%|█████████████████████████████████████████████████▏ | 7349/23458 [2:11:05<3:48:59, 1.17it/s][2025-04-23 14:36:06] (step=0007350) Train Loss: 5.5483, Train Steps/Sec: 1.17 + 31%|█████████████████████████████████████████████████▎ | 7374/23458 [2:11:26<3:46:34, 1.18it/s][2025-04-23 14:36:28] (step=0007375) Train Loss: 5.6296, Train Steps/Sec: 1.17 + 32%|█████████████████████████████████████████████████▌ | 7399/23458 [2:11:48<3:45:04, 1.19it/s][2025-04-23 14:36:49] (step=0007400) Train Loss: 5.5149, Train Steps/Sec: 1.17 + 32%|█████████████████████████████████████████████████▋ | 7424/23458 [2:12:09<3:52:13, 1.15it/s][2025-04-23 14:37:10] (step=0007425) Train Loss: 5.6453, Train Steps/Sec: 1.17 + 32%|█████████████████████████████████████████████████▊ | 7449/23458 [2:12:30<3:47:04, 1.18it/s][2025-04-23 14:37:32] (step=0007450) Train Loss: 5.5708, Train Steps/Sec: 1.17 + 32%|██████████████████████████████████████████████████ | 7474/23458 [2:12:52<3:45:57, 1.18it/s][2025-04-23 14:37:53] (step=0007475) Train Loss: 5.6707, Train Steps/Sec: 1.17 + 32%|██████████████████████████████████████████████████▏ | 7499/23458 [2:13:13<3:43:45, 1.19it/s][2025-04-23 14:38:14] (step=0007500) Train Loss: 5.6034, Train Steps/Sec: 1.17 + 32%|██████████████████████████████████████████████████▎ | 7524/23458 [2:13:34<3:50:12, 1.15it/s][2025-04-23 14:38:36] (step=0007525) Train Loss: 5.5281, Train Steps/Sec: 1.18 + 32%|██████████████████████████████████████████████████▌ | 7549/23458 [2:13:56<3:45:36, 1.18it/s][2025-04-23 14:38:57] (step=0007550) Train Loss: 5.5392, Train Steps/Sec: 1.18 + 32%|██████████████████████████████████████████████████▋ | 7574/23458 [2:14:17<3:43:34, 1.18it/s][2025-04-23 14:39:18] (step=0007575) Train Loss: 5.6421, Train Steps/Sec: 1.17 + 32%|██████████████████████████████████████████████████▊ | 7599/23458 [2:14:38<3:42:23, 1.19it/s][2025-04-23 14:39:40] (step=0007600) Train Loss: 5.5481, Train Steps/Sec: 1.17 + 33%|███████████████████████████████████████████████████ | 7624/23458 [2:15:00<3:48:54, 1.15it/s][2025-04-23 14:40:01] (step=0007625) Train Loss: 5.6720, Train Steps/Sec: 1.17 + 33%|███████████████████████████████████████████████████▏ | 7649/23458 [2:15:21<3:44:24, 1.17it/s][2025-04-23 14:40:22] (step=0007650) Train Loss: 5.5935, Train Steps/Sec: 1.18 + 33%|███████████████████████████████████████████████████▎ | 7674/23458 [2:15:42<3:43:51, 1.18it/s][2025-04-23 14:40:44] (step=0007675) Train Loss: 5.5662, Train Steps/Sec: 1.17 + 33%|███████████████████████████████████████████████████▌ | 7699/23458 [2:16:05<3:45:21, 1.17it/s][2025-04-23 14:41:07] (step=0007700) Train Loss: 5.4702, Train Steps/Sec: 1.08 + 33%|███████████████████████████████████████████████████▋ | 7724/23458 [2:16:27<3:47:54, 1.15it/s][2025-04-23 14:41:28] (step=0007725) Train Loss: 5.6161, Train Steps/Sec: 1.17 + 33%|███████████████████████████████████████████████████▊ | 7749/23458 [2:16:48<3:42:43, 1.18it/s][2025-04-23 14:41:49] (step=0007750) Train Loss: 5.5227, Train Steps/Sec: 1.18 + 33%|████████████████████████████████████████████████████ | 7774/23458 [2:17:09<3:41:35, 1.18it/s][2025-04-23 14:42:11] (step=0007775) Train Loss: 5.5485, Train Steps/Sec: 1.17 + 33%|████████████████████████████████████████████████████▏ | 7799/23458 [2:17:31<3:39:14, 1.19it/s][2025-04-23 14:42:32] (step=0007800) Train Loss: 5.6116, Train Steps/Sec: 1.17 + 33%|████████████████████████████████████████████████████▎ | 7824/23458 [2:17:53<3:45:38, 1.15it/s][2025-04-23 14:42:55] (step=0007825) Train Loss: 5.5707, Train Steps/Sec: 1.09 + 33%|████████████████████████████████████████████████████▌ | 7849/23458 [2:18:15<3:41:34, 1.17it/s][2025-04-23 14:43:17] (step=0007850) Train Loss: 5.6047, Train Steps/Sec: 1.13 + 34%|████████████████████████████████████████████████████▋ | 7874/23458 [2:18:38<3:44:26, 1.16it/s][2025-04-23 14:43:39] (step=0007875) Train Loss: 5.5710, Train Steps/Sec: 1.13 + 34%|████████████████████████████████████████████████████▊ | 7899/23458 [2:18:59<3:37:38, 1.19it/s][2025-04-23 14:44:01] (step=0007900) Train Loss: 5.6073, Train Steps/Sec: 1.17 + 34%|█████████████████████████████████████████████████████ | 7924/23458 [2:19:21<3:44:37, 1.15it/s][2025-04-23 14:44:22] (step=0007925) Train Loss: 5.5761, Train Steps/Sec: 1.17 + 34%|█████████████████████████████████████████████████████▏ | 7949/23458 [2:19:43<3:40:10, 1.17it/s][2025-04-23 14:44:44] (step=0007950) Train Loss: 5.6180, Train Steps/Sec: 1.13 + 34%|█████████████████████████████████████████████████████▎ | 7974/23458 [2:20:05<3:39:29, 1.18it/s][2025-04-23 14:45:06] (step=0007975) Train Loss: 5.5812, Train Steps/Sec: 1.13 + 34%|█████████████████████████████████████████████████████▌ | 7999/23458 [2:20:26<3:36:44, 1.19it/s][2025-04-23 14:45:28] (step=0008000) Train Loss: 5.6081, Train Steps/Sec: 1.17 +You are using a model of type blip-2 to instantiate a model of type instructblip. This is not supported for all configurations of models and can yield errors. +[2025-04-23 14:45:28] text_config is None. Initializing the text config with default values (`OPTConfig`). +model_name_or_path:/tmp/haozhezhao/model/blip2-flan-t5-xl +image_place_holder: +tokenizer length after expend 32101 +tokenizer length before expend 32100 +100%|████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████| 4/4 [03:58<00:00, 59.59s/it] +[2025-04-23 14:50:35] Finish Eval in 8000 steps...██████████████████████████████████████████████████████████████████████████████████████████████████████████████████| 4/4 [03:57<00:00, 59.24s/it] +processing left-padding... +processing left-padding... +processing left-padding... +processing left-padding... +[2025-04-23 14:50:54] Saved checkpoint to checkpoint/Afteri2i_100kckpt_LLAVAT5_empty_Fixed_just_segment_pretrain_stage3_dreambench_recap_400k_t2i_400k_flux_200k_midjourney_150k_recovery_150k_3e4_epoch2_half_rotated/005-GPT-XL/checkpoints/0008000.pt +[2025-04-23 14:50:56] Removed old checkpoint: checkpoint/Afteri2i_100kckpt_LLAVAT5_empty_Fixed_just_segment_pretrain_stage3_dreambench_recap_400k_t2i_400k_flux_200k_midjourney_150k_recovery_150k_3e4_epoch2_half_rotated/005-GPT-XL/checkpoints/0006000.pt + 34%|█████████████████████████████████████████████████████▋ | 8024/23458 [2:26:16<3:48:57, 1.12it/s][2025-04-23 14:51:18] (step=0008025) Train Loss: 5.5404, Train Steps/Sec: 0.07 + 34%|█████████████████████████████████████████████████████▊ | 8049/23458 [2:26:37<3:39:18, 1.17it/s][2025-04-23 14:51:39] (step=0008050) Train Loss: 5.7199, Train Steps/Sec: 1.17 + 34%|██████████████████████████████████████████████████████ | 8074/23458 [2:26:59<3:36:43, 1.18it/s][2025-04-23 14:52:00] (step=0008075) Train Loss: 5.6386, Train Steps/Sec: 1.18 + 35%|██████████████████████████████████████████████████████▏ | 8099/23458 [2:27:20<3:35:34, 1.19it/s][2025-04-23 14:52:22] (step=0008100) Train Loss: 5.7066, Train Steps/Sec: 1.17 + 35%|██████████████████████████████████████████████████████▎ | 8124/23458 [2:27:42<3:41:39, 1.15it/s][2025-04-23 14:52:43] (step=0008125) Train Loss: 5.5921, Train Steps/Sec: 1.17 + 35%|██████████████████████████████████████████████████████▌ | 8149/23458 [2:28:03<3:38:18, 1.17it/s][2025-04-23 14:53:04] (step=0008150) Train Loss: 5.5478, Train Steps/Sec: 1.17 + 35%|██████████████████████████████████████████████████████▋ | 8174/23458 [2:28:24<3:35:19, 1.18it/s][2025-04-23 14:53:25] (step=0008175) Train Loss: 5.5166, Train Steps/Sec: 1.18 + 35%|██████████████████████████████████████████████████████▊ | 8199/23458 [2:28:45<3:34:40, 1.18it/s][2025-04-23 14:53:47] (step=0008200) Train Loss: 5.5654, Train Steps/Sec: 1.17 + 35%|███████████████████████████████████████████████████████ | 8224/23458 [2:29:07<3:39:11, 1.16it/s][2025-04-23 14:54:08] (step=0008225) Train Loss: 5.5901, Train Steps/Sec: 1.18 + 35%|███████████████████████████████████████████████████████▏ | 8249/23458 [2:29:28<3:35:50, 1.17it/s][2025-04-23 14:54:29] (step=0008250) Train Loss: 5.6473, Train Steps/Sec: 1.18 + 35%|███████████████████████████████████████████████████████▍ | 8274/23458 [2:29:49<3:34:03, 1.18it/s][2025-04-23 14:54:51] (step=0008275) Train Loss: 5.5355, Train Steps/Sec: 1.17 + 35%|███████████████████████████████████████████████████████▌ | 8299/23458 [2:30:11<3:33:36, 1.18it/s][2025-04-23 14:55:12] (step=0008300) Train Loss: 5.6486, Train Steps/Sec: 1.17 + 35%|███████████████████████████████████████████████████████▋ | 8324/23458 [2:30:32<3:38:31, 1.15it/s][2025-04-23 14:55:33] (step=0008325) Train Loss: 5.6025, Train Steps/Sec: 1.18 + 36%|███████████████████████████████████████████████████████▉ | 8349/23458 [2:30:53<3:33:46, 1.18it/s][2025-04-23 14:55:55] (step=0008350) Train Loss: 5.6015, Train Steps/Sec: 1.17 + 36%|████████████████████████████████████████████████████████ | 8374/23458 [2:31:15<3:32:58, 1.18it/s][2025-04-23 14:56:16] (step=0008375) Train Loss: 5.5669, Train Steps/Sec: 1.18 + 36%|████████████████████████████████████████████████████████▏ | 8399/23458 [2:31:36<3:31:27, 1.19it/s][2025-04-23 14:56:37] (step=0008400) Train Loss: 5.5281, Train Steps/Sec: 1.17 + 36%|████████████████████████████████████████████████████████▍ | 8424/23458 [2:31:59<3:48:10, 1.10it/s][2025-04-23 14:57:00] (step=0008425) Train Loss: 5.5368, Train Steps/Sec: 1.08 + 36%|████████████████████████████████████████████████████████▌ | 8449/23458 [2:32:20<3:33:20, 1.17it/s][2025-04-23 14:57:22] (step=0008450) Train Loss: 5.6245, Train Steps/Sec: 1.17 + 36%|████████████████████████████████████████████████████████▋ | 8474/23458 [2:32:43<3:32:15, 1.18it/s][2025-04-23 14:57:44] (step=0008475) Train Loss: 5.6405, Train Steps/Sec: 1.13 + 36%|████████████████████████████████████████████████████████▉ | 8499/23458 [2:33:05<3:30:50, 1.18it/s][2025-04-23 14:58:06] (step=0008500) Train Loss: 5.5176, Train Steps/Sec: 1.13 + 36%|█████████████████████████████████████████████████████████ | 8524/23458 [2:33:27<3:35:58, 1.15it/s][2025-04-23 14:58:28] (step=0008525) Train Loss: 5.6100, Train Steps/Sec: 1.13 + 36%|█████████████████████████████████████████████████████████▏ | 8549/23458 [2:33:49<3:31:38, 1.17it/s][2025-04-23 14:58:50] (step=0008550) Train Loss: 5.6509, Train Steps/Sec: 1.14 + 37%|█████████████████████████████████████████████████████████▍ | 8574/23458 [2:34:10<3:29:15, 1.19it/s][2025-04-23 14:59:12] (step=0008575) Train Loss: 5.5722, Train Steps/Sec: 1.18 + 37%|█████████████████████████████████████████████████████████▌ | 8599/23458 [2:34:32<3:29:10, 1.18it/s][2025-04-23 14:59:34] (step=0008600) Train Loss: 5.5628, Train Steps/Sec: 1.12 + 37%|█████████████████████████████████████████████████████████▋ | 8624/23458 [2:34:54<3:34:00, 1.16it/s][2025-04-23 14:59:55] (step=0008625) Train Loss: 5.5778, Train Steps/Sec: 1.17 + 37%|█████████████████████████████████████████████████████████▉ | 8649/23458 [2:35:15<3:31:00, 1.17it/s][2025-04-23 15:00:16] (step=0008650) Train Loss: 5.5785, Train Steps/Sec: 1.17 + 37%|██████████████████████████████████████████████████████████ | 8674/23458 [2:35:37<3:28:26, 1.18it/s][2025-04-23 15:00:39] (step=0008675) Train Loss: 5.5469, Train Steps/Sec: 1.12 + 37%|██████████████████████████████████████████████████████████▏ | 8699/23458 [2:35:59<3:26:54, 1.19it/s][2025-04-23 15:01:00] (step=0008700) Train Loss: 5.5830, Train Steps/Sec: 1.17 + 37%|██████████████████████████████████████████████████████████▍ | 8724/23458 [2:36:20<3:34:43, 1.14it/s][2025-04-23 15:01:21] (step=0008725) Train Loss: 5.5970, Train Steps/Sec: 1.17 + 37%|██████████████████████████████████████████████████████████▌ | 8749/23458 [2:36:41<3:28:56, 1.17it/s][2025-04-23 15:01:43] (step=0008750) Train Loss: 5.6109, Train Steps/Sec: 1.18 + 37%|██████████████████████████████████████████████████████████▋ | 8774/23458 [2:37:03<3:27:39, 1.18it/s][2025-04-23 15:02:04] (step=0008775) Train Loss: 5.6733, Train Steps/Sec: 1.17 + 38%|██████████████████████████████████████████████████████████▉ | 8799/23458 [2:37:24<3:25:26, 1.19it/s][2025-04-23 15:02:25] (step=0008800) Train Loss: 5.5923, Train Steps/Sec: 1.17 + 38%|███████████████████████████████████████████████████████████ | 8824/23458 [2:37:45<3:31:02, 1.16it/s][2025-04-23 15:02:47] (step=0008825) Train Loss: 5.5614, Train Steps/Sec: 1.17 + 38%|███████████████████████████████████████████████████████████▏ | 8849/23458 [2:38:07<3:27:36, 1.17it/s][2025-04-23 15:03:08] (step=0008850) Train Loss: 5.6193, Train Steps/Sec: 1.17 + 38%|███████████████████████████████████████████████████████████▍ | 8874/23458 [2:38:28<3:25:24, 1.18it/s][2025-04-23 15:03:29] (step=0008875) Train Loss: 5.5727, Train Steps/Sec: 1.18 + 38%|███████████████████████████████████████████████████████████▌ | 8899/23458 [2:38:49<3:24:00, 1.19it/s][2025-04-23 15:03:51] (step=0008900) Train Loss: 5.6106, Train Steps/Sec: 1.17 + 38%|███████████████████████████████████████████████████████████▋ | 8924/23458 [2:39:11<3:31:46, 1.14it/s][2025-04-23 15:04:12] (step=0008925) Train Loss: 5.7130, Train Steps/Sec: 1.17 + 38%|███████████████████████████████████████████████████████████▉ | 8949/23458 [2:39:32<3:29:12, 1.16it/s][2025-04-23 15:04:33] (step=0008950) Train Loss: 5.4615, Train Steps/Sec: 1.17 + 38%|████████████████████████████████████████████████████████████ | 8974/23458 [2:39:53<3:23:52, 1.18it/s][2025-04-23 15:04:55] (step=0008975) Train Loss: 5.5942, Train Steps/Sec: 1.18 + 38%|████████████████████████████████████████████████████████████▏ | 8999/23458 [2:40:15<3:23:06, 1.19it/s][2025-04-23 15:05:16] (step=0009000) Train Loss: 5.5529, Train Steps/Sec: 1.17 + 38%|████████████████████████████████████████████████████████████▍ | 9024/23458 [2:40:36<3:29:51, 1.15it/s][2025-04-23 15:05:37] (step=0009025) Train Loss: 5.5656, Train Steps/Sec: 1.17 + 39%|████████████████████████████████████████████████████████████▌ | 9049/23458 [2:40:57<3:27:29, 1.16it/s][2025-04-23 15:05:59] (step=0009050) Train Loss: 5.4979, Train Steps/Sec: 1.17 + 39%|████████████████████████████████████████████████████████████▋ | 9074/23458 [2:41:19<3:22:44, 1.18it/s][2025-04-23 15:06:20] (step=0009075) Train Loss: 5.6053, Train Steps/Sec: 1.18 + 39%|████████████████████████████████████████████████████████████▉ | 9099/23458 [2:41:40<3:20:42, 1.19it/s][2025-04-23 15:06:41] (step=0009100) Train Loss: 5.5881, Train Steps/Sec: 1.17 + 39%|█████████████████████████████████████████████████████████████ | 9124/23458 [2:42:02<3:34:21, 1.11it/s][2025-04-23 15:07:05] (step=0009125) Train Loss: 5.5723, Train Steps/Sec: 1.08 + 39%|█████████████████████████████████████████████████████████████▏ | 9149/23458 [2:42:26<3:46:32, 1.05it/s][2025-04-23 15:07:28] (step=0009150) Train Loss: 5.6203, Train Steps/Sec: 1.09 + 39%|█████████████████████████████████████████████████████████████▍ | 9174/23458 [2:42:48<3:35:10, 1.11it/s][2025-04-23 15:07:50] (step=0009175) Train Loss: 5.5956, Train Steps/Sec: 1.13 + 39%|█████████████████████████████████████████████████████████████▌ | 9199/23458 [2:43:10<3:21:50, 1.18it/s][2025-04-23 15:08:12] (step=0009200) Train Loss: 5.6060, Train Steps/Sec: 1.13 + 39%|█████████████████████████████████████████████████████████████▋ | 9224/23458 [2:43:32<3:26:11, 1.15it/s][2025-04-23 15:08:33] (step=0009225) Train Loss: 5.6353, Train Steps/Sec: 1.17 + 39%|█████████████████████████████████████████████████████████████▉ | 9249/23458 [2:43:54<4:08:38, 1.05s/it][2025-04-23 15:08:56] (step=0009250) Train Loss: 5.4870, Train Steps/Sec: 1.10 + 40%|██████████████████████████████████████████████████████████████ | 9274/23458 [2:44:16<3:20:37, 1.18it/s][2025-04-23 15:09:17] (step=0009275) Train Loss: 5.6289, Train Steps/Sec: 1.17 + 40%|██████████████████████████████████████████████████████████████▏ | 9299/23458 [2:44:37<3:18:35, 1.19it/s][2025-04-23 15:09:39] (step=0009300) Train Loss: 5.6067, Train Steps/Sec: 1.16 + 40%|██████████████████████████████████████████████████████████████▍ | 9324/23458 [2:45:00<3:25:20, 1.15it/s][2025-04-23 15:10:01] (step=0009325) Train Loss: 5.5867, Train Steps/Sec: 1.11 + 40%|██████████████████████████████████████████████████████████████▌ | 9349/23458 [2:45:21<3:20:38, 1.17it/s][2025-04-23 15:10:22] (step=0009350) Train Loss: 5.5610, Train Steps/Sec: 1.17 + 40%|██████████████████████████████████████████████████████████████▋ | 9374/23458 [2:45:42<3:19:09, 1.18it/s][2025-04-23 15:10:44] (step=0009375) Train Loss: 5.5917, Train Steps/Sec: 1.17 + 40%|██████████████████████████████████████████████████████████████▉ | 9399/23458 [2:46:04<3:16:58, 1.19it/s][2025-04-23 15:11:05] (step=0009400) Train Loss: 5.5593, Train Steps/Sec: 1.17 + 40%|███████████████████████████████████████████████████████████████ | 9424/23458 [2:46:25<3:23:12, 1.15it/s][2025-04-23 15:11:26] (step=0009425) Train Loss: 5.5825, Train Steps/Sec: 1.17 + 40%|███████████████████████████████████████████████████████████████▏ | 9449/23458 [2:46:46<3:18:33, 1.18it/s][2025-04-23 15:11:48] (step=0009450) Train Loss: 5.5783, Train Steps/Sec: 1.18 + 40%|███████████████████████████████████████████████████████████████▍ | 9474/23458 [2:47:08<3:17:34, 1.18it/s][2025-04-23 15:12:09] (step=0009475) Train Loss: 5.5665, Train Steps/Sec: 1.18 + 40%|███████████████████████████████████████████████████████████████▌ | 9499/23458 [2:47:29<3:15:12, 1.19it/s][2025-04-23 15:12:30] (step=0009500) Train Loss: 5.6472, Train Steps/Sec: 1.17 + 41%|███████████████████████████████████████████████████████████████▋ | 9524/23458 [2:47:50<3:20:59, 1.16it/s][2025-04-23 15:12:52] (step=0009525) Train Loss: 5.5373, Train Steps/Sec: 1.18 + 41%|███████████████████████████████████████████████████████████████▉ | 9549/23458 [2:48:12<3:17:43, 1.17it/s][2025-04-23 15:13:13] (step=0009550) Train Loss: 5.5705, Train Steps/Sec: 1.17 + 41%|████████████████████████████████████████████████████████████████ | 9574/23458 [2:48:33<3:15:56, 1.18it/s][2025-04-23 15:13:34] (step=0009575) Train Loss: 5.5873, Train Steps/Sec: 1.17 + 41%|████████████████████████████████████████████████████████████████▏ | 9599/23458 [2:48:54<3:13:55, 1.19it/s][2025-04-23 15:13:56] (step=0009600) Train Loss: 5.5547, Train Steps/Sec: 1.17 + 41%|████████████████████████████████████████████████████████████████▍ | 9624/23458 [2:49:16<3:20:33, 1.15it/s][2025-04-23 15:14:17] (step=0009625) Train Loss: 5.6206, Train Steps/Sec: 1.17 + 41%|████████████████████████████████████████████████████████████████▌ | 9649/23458 [2:49:37<3:16:29, 1.17it/s][2025-04-23 15:14:38] (step=0009650) Train Loss: 5.5514, Train Steps/Sec: 1.18 + 41%|████████████████████████████████████████████████████████████████▋ | 9674/23458 [2:49:58<3:14:38, 1.18it/s][2025-04-23 15:14:59] (step=0009675) Train Loss: 5.5911, Train Steps/Sec: 1.18 + 41%|████████████████████████████████████████████████████████████████▉ | 9699/23458 [2:50:19<3:13:38, 1.18it/s][2025-04-23 15:15:21] (step=0009700) Train Loss: 5.5386, Train Steps/Sec: 1.17 + 41%|█████████████████████████████████████████████████████████████████ | 9724/23458 [2:50:41<3:19:00, 1.15it/s][2025-04-23 15:15:42] (step=0009725) Train Loss: 5.5181, Train Steps/Sec: 1.18 + 42%|█████████████████████████████████████████████████████████████████▏ | 9749/23458 [2:51:02<3:15:22, 1.17it/s][2025-04-23 15:16:03] (step=0009750) Train Loss: 5.5701, Train Steps/Sec: 1.17 + 42%|█████████████████████████████████████████████████████████████████▍ | 9774/23458 [2:51:24<4:10:44, 1.10s/it][2025-04-23 15:16:25] (step=0009775) Train Loss: 5.5818, Train Steps/Sec: 1.13 + 42%|█████████████████████████████████████████████████████████████████▌ | 9799/23458 [2:51:46<3:24:02, 1.12it/s][2025-04-23 15:16:48] (step=0009800) Train Loss: 5.5788, Train Steps/Sec: 1.13 + 42%|█████████████████████████████████████████████████████████████████▊ | 9824/23458 [2:52:08<3:17:45, 1.15it/s][2025-04-23 15:17:09] (step=0009825) Train Loss: 5.5230, Train Steps/Sec: 1.17 + 42%|█████████████████████████████████████████████████████████████████▉ | 9849/23458 [2:52:32<3:42:25, 1.02it/s][2025-04-23 15:17:33] (step=0009850) Train Loss: 5.5792, Train Steps/Sec: 1.05 + 42%|██████████████████████████████████████████████████████████████████ | 9874/23458 [2:52:54<3:54:50, 1.04s/it][2025-04-23 15:17:55] (step=0009875) Train Loss: 5.5720, Train Steps/Sec: 1.13 + 42%|██████████████████████████████████████████████████████████████████▎ | 9899/23458 [2:53:15<3:10:13, 1.19it/s][2025-04-23 15:18:16] (step=0009900) Train Loss: 5.5957, Train Steps/Sec: 1.17 + 42%|██████████████████████████████████████████████████████████████████▍ | 9924/23458 [2:53:37<3:16:49, 1.15it/s][2025-04-23 15:18:39] (step=0009925) Train Loss: 5.5793, Train Steps/Sec: 1.12 + 42%|██████████████████████████████████████████████████████████████████▌ | 9949/23458 [2:53:59<3:12:40, 1.17it/s][2025-04-23 15:19:00] (step=0009950) Train Loss: 5.5788, Train Steps/Sec: 1.17 + 43%|██████████████████████████████████████████████████████████████████▊ | 9974/23458 [2:54:21<3:17:56, 1.14it/s][2025-04-23 15:19:22] (step=0009975) Train Loss: 5.5354, Train Steps/Sec: 1.13 + 43%|██████████████████████████████████████████████████████████████████▉ | 9999/23458 [2:54:42<3:09:31, 1.18it/s][2025-04-23 15:19:44] (step=0010000) Train Loss: 5.5674, Train Steps/Sec: 1.17 +You are using a model of type blip-2 to instantiate a model of type instructblip. This is not supported for all configurations of models and can yield errors. +[2025-04-23 15:19:44] text_config is None. Initializing the text config with default values (`OPTConfig`). +model_name_or_path:/tmp/haozhezhao/model/blip2-flan-t5-xl +image_place_holder: +tokenizer length after expend 32101 +tokenizer length before expend 32100 +100%|████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████| 4/4 [03:59<00:00, 59.81s/it] +[2025-04-23 15:24:51] Finish Eval in 10000 steps...█████████████████████████████████████████████████████████████████████████████████████████████████████████████████| 4/4 [03:58<00:00, 59.53s/it] +processing left-padding... +processing left-padding... +processing left-padding... +processing left-padding... +[2025-04-23 15:25:10] Saved checkpoint to checkpoint/Afteri2i_100kckpt_LLAVAT5_empty_Fixed_just_segment_pretrain_stage3_dreambench_recap_400k_t2i_400k_flux_200k_midjourney_150k_recovery_150k_3e4_epoch2_half_rotated/005-GPT-XL/checkpoints/0010000.pt +[2025-04-23 15:25:12] Removed old checkpoint: checkpoint/Afteri2i_100kckpt_LLAVAT5_empty_Fixed_just_segment_pretrain_stage3_dreambench_recap_400k_t2i_400k_flux_200k_midjourney_150k_recovery_150k_3e4_epoch2_half_rotated/005-GPT-XL/checkpoints/0008000.pt + 43%|██████████████████████████████████████████████████████████████████▋ | 10024/23458 [3:00:32<3:18:15, 1.13it/s][2025-04-23 15:25:34] (step=0010025) Train Loss: 5.6156, Train Steps/Sec: 0.07 + 43%|██████████████████████████████████████████████████████████████████▊ | 10049/23458 [3:00:53<3:11:09, 1.17it/s][2025-04-23 15:25:55] (step=0010050) Train Loss: 5.5371, Train Steps/Sec: 1.17 + 43%|██████████████████████████████████████████████████████████████████▉ | 10074/23458 [3:01:15<3:10:02, 1.17it/s][2025-04-23 15:26:16] (step=0010075) Train Loss: 5.5876, Train Steps/Sec: 1.17 + 43%|███████████████████████████████████████████████████████████████████▏ | 10099/23458 [3:01:36<3:07:16, 1.19it/s][2025-04-23 15:26:37] (step=0010100) Train Loss: 5.4552, Train Steps/Sec: 1.17 + 43%|███████████████████████████████████████████████████████████████████▎ | 10124/23458 [3:01:57<3:12:55, 1.15it/s][2025-04-23 15:26:59] (step=0010125) Train Loss: 5.6099, Train Steps/Sec: 1.17 + 43%|███████████████████████████████████████████████████████████████████▍ | 10149/23458 [3:02:19<3:08:49, 1.17it/s][2025-04-23 15:27:20] (step=0010150) Train Loss: 5.4981, Train Steps/Sec: 1.18 + 43%|███████████████████████████████████████████████████████████████████▋ | 10174/23458 [3:02:40<3:08:05, 1.18it/s][2025-04-23 15:27:41] (step=0010175) Train Loss: 5.5953, Train Steps/Sec: 1.18 + 43%|███████████████████████████████████████████████████████████████████▊ | 10199/23458 [3:03:01<3:06:19, 1.19it/s][2025-04-23 15:28:03] (step=0010200) Train Loss: 5.5519, Train Steps/Sec: 1.16 + 44%|███████████████████████████████████████████████████████████████████▉ | 10224/23458 [3:03:23<3:12:01, 1.15it/s][2025-04-23 15:28:24] (step=0010225) Train Loss: 5.6715, Train Steps/Sec: 1.17 + 44%|████████████████████████████████████████████████████████████████████▏ | 10249/23458 [3:03:44<3:07:37, 1.17it/s][2025-04-23 15:28:45] (step=0010250) Train Loss: 5.5475, Train Steps/Sec: 1.18 + 44%|████████████████████████████████████████████████████████████████████▎ | 10274/23458 [3:04:05<3:05:58, 1.18it/s][2025-04-23 15:29:07] (step=0010275) Train Loss: 5.5861, Train Steps/Sec: 1.17 + 44%|████████████████████████████████████████████████████████████████████▍ | 10299/23458 [3:04:27<3:05:27, 1.18it/s][2025-04-23 15:29:28] (step=0010300) Train Loss: 5.5589, Train Steps/Sec: 1.17 + 44%|████████████████████████████████████████████████████████████████████▋ | 10324/23458 [3:04:48<3:09:23, 1.16it/s][2025-04-23 15:29:49] (step=0010325) Train Loss: 5.5434, Train Steps/Sec: 1.18 + 44%|████████████████████████████████████████████████████████████████████▊ | 10349/23458 [3:05:09<3:06:15, 1.17it/s][2025-04-23 15:30:11] (step=0010350) Train Loss: 5.4776, Train Steps/Sec: 1.17 + 44%|████████████████████████████████████████████████████████████████████▉ | 10374/23458 [3:05:31<3:04:38, 1.18it/s][2025-04-23 15:30:32] (step=0010375) Train Loss: 5.5061, Train Steps/Sec: 1.17 + 44%|█████████████████████████████████████████████████████████████████████▏ | 10399/23458 [3:05:52<3:02:50, 1.19it/s][2025-04-23 15:30:53] (step=0010400) Train Loss: 5.5098, Train Steps/Sec: 1.17 + 44%|█████████████████████████████████████████████████████████████████████▎ | 10424/23458 [3:06:13<3:07:30, 1.16it/s][2025-04-23 15:31:15] (step=0010425) Train Loss: 5.5151, Train Steps/Sec: 1.18 + 45%|█████████████████████████████████████████████████████████████████████▍ | 10449/23458 [3:06:35<3:06:04, 1.17it/s][2025-04-23 15:31:37] (step=0010450) Train Loss: 5.5400, Train Steps/Sec: 1.13 + 45%|█████████████████████████████████████████████████████████████████████▋ | 10474/23458 [3:06:58<3:03:15, 1.18it/s][2025-04-23 15:31:59] (step=0010475) Train Loss: 5.5182, Train Steps/Sec: 1.13 + 45%|█████████████████████████████████████████████████████████████████████▊ | 10499/23458 [3:07:20<3:05:25, 1.16it/s][2025-04-23 15:32:21] (step=0010500) Train Loss: 5.5829, Train Steps/Sec: 1.12 + 45%|█████████████████████████████████████████████████████████████████████▉ | 10524/23458 [3:07:42<3:08:10, 1.15it/s][2025-04-23 15:32:43] (step=0010525) Train Loss: 5.5006, Train Steps/Sec: 1.13 + 45%|██████████████████████████████████████████████████████████████████████▏ | 10549/23458 [3:08:03<3:03:11, 1.17it/s][2025-04-23 15:33:05] (step=0010550) Train Loss: 5.5134, Train Steps/Sec: 1.18 + 45%|██████████████████████████████████████████████████████████████████████▎ | 10574/23458 [3:08:26<3:34:29, 1.00it/s][2025-04-23 15:33:28] (step=0010575) Train Loss: 5.5385, Train Steps/Sec: 1.08 + 45%|██████████████████████████████████████████████████████████████████████▍ | 10599/23458 [3:08:48<3:00:23, 1.19it/s][2025-04-23 15:33:50] (step=0010600) Train Loss: 5.5723, Train Steps/Sec: 1.17 + 45%|██████████████████████████████████████████████████████████████████████▋ | 10624/23458 [3:09:10<3:05:36, 1.15it/s][2025-04-23 15:34:11] (step=0010625) Train Loss: 5.5182, Train Steps/Sec: 1.17 + 45%|██████████████████████████████████████████████████████████████████████▊ | 10649/23458 [3:09:32<3:42:12, 1.04s/it][2025-04-23 15:34:34] (step=0010650) Train Loss: 5.5741, Train Steps/Sec: 1.13 + 46%|██████████████████████████████████████████████████████████████████████▉ | 10674/23458 [3:09:53<3:01:01, 1.18it/s][2025-04-23 15:34:55] (step=0010675) Train Loss: 5.4932, Train Steps/Sec: 1.17 + 46%|███████████████████████████████████████████████████████████████████████▏ | 10699/23458 [3:10:15<2:59:05, 1.19it/s][2025-04-23 15:35:16] (step=0010700) Train Loss: 5.5962, Train Steps/Sec: 1.17 + 46%|███████████████████████████████████████████████████████████████████████▎ | 10724/23458 [3:10:36<3:03:54, 1.15it/s][2025-04-23 15:35:38] (step=0010725) Train Loss: 5.4978, Train Steps/Sec: 1.17 + 46%|███████████████████████████████████████████████████████████████████████▍ | 10749/23458 [3:10:58<3:02:01, 1.16it/s][2025-04-23 15:35:59] (step=0010750) Train Loss: 5.6340, Train Steps/Sec: 1.17 + 46%|███████████████████████████████████████████████████████████████████████▋ | 10774/23458 [3:11:19<3:00:30, 1.17it/s][2025-04-23 15:36:21] (step=0010775) Train Loss: 5.5669, Train Steps/Sec: 1.16 + 46%|███████████████████████████████████████████████████████████████████████▊ | 10799/23458 [3:11:41<2:58:22, 1.18it/s][2025-04-23 15:36:42] (step=0010800) Train Loss: 5.4868, Train Steps/Sec: 1.16 + 46%|███████████████████████████████████████████████████████████████████████▉ | 10824/23458 [3:12:02<3:02:32, 1.15it/s][2025-04-23 15:37:03] (step=0010825) Train Loss: 5.4880, Train Steps/Sec: 1.18 + 46%|████████████████████████████████████████████████████████████████████████▏ | 10849/23458 [3:12:23<2:59:45, 1.17it/s][2025-04-23 15:37:25] (step=0010850) Train Loss: 5.6324, Train Steps/Sec: 1.17 + 46%|████████████████████████████████████████████████████████████████████████▎ | 10874/23458 [3:12:45<2:57:20, 1.18it/s][2025-04-23 15:37:46] (step=0010875) Train Loss: 5.6262, Train Steps/Sec: 1.17 + 46%|████████████████████████████████████████████████████████████████████████▍ | 10899/23458 [3:13:06<2:56:05, 1.19it/s][2025-04-23 15:38:08] (step=0010900) Train Loss: 5.6832, Train Steps/Sec: 1.17 + 47%|████████████████████████████████████████████████████████████████████████▋ | 10924/23458 [3:13:27<3:01:31, 1.15it/s][2025-04-23 15:38:29] (step=0010925) Train Loss: 5.5074, Train Steps/Sec: 1.18 + 47%|████████████████████████████████████████████████████████████████████████▊ | 10949/23458 [3:13:49<2:57:23, 1.18it/s][2025-04-23 15:38:50] (step=0010950) Train Loss: 5.4988, Train Steps/Sec: 1.18 + 47%|████████████████████████████████████████████████████████████████████████▉ | 10974/23458 [3:14:10<2:56:48, 1.18it/s][2025-04-23 15:39:11] (step=0010975) Train Loss: 5.6623, Train Steps/Sec: 1.17 + 47%|█████████████████████████████████████████████████████████████████████████▏ | 10999/23458 [3:14:31<2:54:42, 1.19it/s][2025-04-23 15:39:33] (step=0011000) Train Loss: 5.5117, Train Steps/Sec: 1.17 + 47%|█████████████████████████████████████████████████████████████████████████▎ | 11024/23458 [3:14:53<3:00:10, 1.15it/s][2025-04-23 15:39:54] (step=0011025) Train Loss: 5.5253, Train Steps/Sec: 1.17 + 47%|█████████████████████████████████████████████████████████████████████████▍ | 11049/23458 [3:15:14<2:56:23, 1.17it/s][2025-04-23 15:40:15] (step=0011050) Train Loss: 5.5331, Train Steps/Sec: 1.17 + 47%|█████████████████████████████████████████████████████████████████████████▋ | 11074/23458 [3:15:36<2:56:40, 1.17it/s][2025-04-23 15:40:37] (step=0011075) Train Loss: 5.5381, Train Steps/Sec: 1.16 + 47%|█████████████████████████████████████████████████████████████████████████▊ | 11099/23458 [3:15:58<3:13:38, 1.06it/s][2025-04-23 15:40:59] (step=0011100) Train Loss: 5.5380, Train Steps/Sec: 1.11 + 47%|█████████████████████████████████████████████████████████████████████████▉ | 11124/23458 [3:16:20<2:58:53, 1.15it/s][2025-04-23 15:41:22] (step=0011125) Train Loss: 5.5613, Train Steps/Sec: 1.12 + 48%|██████████████████████████████████████████████████████████████████████████▏ | 11149/23458 [3:16:43<3:20:24, 1.02it/s][2025-04-23 15:41:44] (step=0011150) Train Loss: 5.5830, Train Steps/Sec: 1.12 + 48%|██████████████████████████████████████████████████████████████████████████▎ | 11174/23458 [3:17:05<2:58:10, 1.15it/s][2025-04-23 15:42:06] (step=0011175) Train Loss: 5.5523, Train Steps/Sec: 1.13 + 48%|██████████████████████████████████████████████████████████████████████████▍ | 11199/23458 [3:17:26<2:52:20, 1.19it/s][2025-04-23 15:42:28] (step=0011200) Train Loss: 5.4977, Train Steps/Sec: 1.17 + 48%|██████████████████████████████████████████████████████████████████████████▋ | 11224/23458 [3:17:49<3:23:52, 1.00it/s][2025-04-23 15:42:50] (step=0011225) Train Loss: 5.5470, Train Steps/Sec: 1.12 + 48%|██████████████████████████████████████████████████████████████████████████▊ | 11249/23458 [3:18:10<2:53:45, 1.17it/s][2025-04-23 15:43:11] (step=0011250) Train Loss: 5.5740, Train Steps/Sec: 1.17 + 48%|██████████████████████████████████████████████████████████████████████████▉ | 11274/23458 [3:18:31<2:52:27, 1.18it/s][2025-04-23 15:43:33] (step=0011275) Train Loss: 5.5944, Train Steps/Sec: 1.17 + 48%|███████████████████████████████████████████████████████████████████████████▏ | 11299/23458 [3:18:53<3:04:22, 1.10it/s][2025-04-23 15:43:55] (step=0011300) Train Loss: 5.6211, Train Steps/Sec: 1.12 + 48%|███████████████████████████████████████████████████████████████████████████▎ | 11324/23458 [3:19:16<2:55:43, 1.15it/s][2025-04-23 15:44:18] (step=0011325) Train Loss: 5.5454, Train Steps/Sec: 1.10 + 48%|███████████████████████████████████████████████████████████████████████████▍ | 11349/23458 [3:19:38<2:53:02, 1.17it/s][2025-04-23 15:44:40] (step=0011350) Train Loss: 5.4876, Train Steps/Sec: 1.13 + 48%|███████████████████████████████████████████████████████████████████████████▋ | 11374/23458 [3:20:00<2:50:50, 1.18it/s][2025-04-23 15:45:01] (step=0011375) Train Loss: 5.5263, Train Steps/Sec: 1.17 + 49%|███████████████████████████████████████████████████████████████████████████▊ | 11399/23458 [3:20:21<2:50:13, 1.18it/s][2025-04-23 15:45:23] (step=0011400) Train Loss: 5.5661, Train Steps/Sec: 1.16 + 49%|███████████████████████████████████████████████████████████████████████████▉ | 11424/23458 [3:20:43<2:54:28, 1.15it/s][2025-04-23 15:45:44] (step=0011425) Train Loss: 5.5351, Train Steps/Sec: 1.17 + 49%|████████████████████████████████████████████████████████████████████████████▏ | 11449/23458 [3:21:04<2:51:54, 1.16it/s][2025-04-23 15:46:06] (step=0011450) Train Loss: 5.4415, Train Steps/Sec: 1.17 + 49%|████████████████████████████████████████████████████████████████████████████▎ | 11474/23458 [3:21:26<2:50:32, 1.17it/s][2025-04-23 15:46:27] (step=0011475) Train Loss: 5.5481, Train Steps/Sec: 1.17 + 49%|████████████████████████████████████████████████████████████████████████████▍ | 11499/23458 [3:21:47<2:50:17, 1.17it/s][2025-04-23 15:46:49] (step=0011500) Train Loss: 5.5826, Train Steps/Sec: 1.16 + 49%|████████████████████████████████████████████████████████████████████████████▋ | 11524/23458 [3:22:09<2:53:52, 1.14it/s][2025-04-23 15:47:10] (step=0011525) Train Loss: 5.4938, Train Steps/Sec: 1.16 + 49%|████████████████████████████████████████████████████████████████████████████▊ | 11549/23458 [3:22:30<2:50:25, 1.16it/s][2025-04-23 15:47:31] (step=0011550) Train Loss: 5.5277, Train Steps/Sec: 1.17 + 49%|████████████████████████████████████████████████████████████████████████████▉ | 11574/23458 [3:22:52<2:48:23, 1.18it/s][2025-04-23 15:47:53] (step=0011575) Train Loss: 5.5737, Train Steps/Sec: 1.17 + 49%|█████████████████████████████████████████████████████████████████████████████▏ | 11599/23458 [3:23:13<2:46:18, 1.19it/s][2025-04-23 15:48:14] (step=0011600) Train Loss: 5.5385, Train Steps/Sec: 1.17 + 50%|█████████████████████████████████████████████████████████████████████████████▎ | 11624/23458 [3:23:35<2:52:27, 1.14it/s][2025-04-23 15:48:36] (step=0011625) Train Loss: 5.5714, Train Steps/Sec: 1.16 + 50%|█████████████████████████████████████████████████████████████████████████████▍ | 11649/23458 [3:23:56<2:48:29, 1.17it/s][2025-04-23 15:48:57] (step=0011650) Train Loss: 5.5084, Train Steps/Sec: 1.17 + 50%|█████████████████████████████████████████████████████████████████████████████▋ | 11674/23458 [3:24:17<2:46:41, 1.18it/s][2025-04-23 15:49:19] (step=0011675) Train Loss: 5.5478, Train Steps/Sec: 1.17 + 50%|█████████████████████████████████████████████████████████████████████████████▊ | 11699/23458 [3:24:39<2:46:06, 1.18it/s][2025-04-23 15:49:40] (step=0011700) Train Loss: 5.4984, Train Steps/Sec: 1.16 + 50%|█████████████████████████████████████████████████████████████████████████████▉ | 11724/23458 [3:25:00<2:49:49, 1.15it/s][2025-04-23 15:50:02] (step=0011725) Train Loss: 5.5137, Train Steps/Sec: 1.17 + 50%|██████████████████████████████████████████████████████████████████████████████▏ | 11749/23458 [3:25:22<2:47:12, 1.17it/s][2025-04-23 15:50:23] (step=0011750) Train Loss: 5.5878, Train Steps/Sec: 1.17 + 50%|██████████████████████████████████████████████████████████████████████████████▎ | 11774/23458 [3:25:45<2:48:35, 1.16it/s][2025-04-23 15:50:46] (step=0011775) Train Loss: 5.6074, Train Steps/Sec: 1.09 + 50%|██████████████████████████████████████████████████████████████████████████████▍ | 11799/23458 [3:26:06<2:43:50, 1.19it/s][2025-04-23 15:51:07] (step=0011800) Train Loss: 5.5734, Train Steps/Sec: 1.17 + 50%|██████████████████████████████████████████████████████████████████████████████▋ | 11824/23458 [3:26:29<3:33:03, 1.10s/it][2025-04-23 15:51:30] (step=0011825) Train Loss: 5.5841, Train Steps/Sec: 1.09 + 51%|██████████████████████████████████████████████████████████████████████████████▊ | 11849/23458 [3:26:50<2:45:28, 1.17it/s][2025-04-23 15:51:52] (step=0011850) Train Loss: 5.5362, Train Steps/Sec: 1.17 + 51%|██████████████████████████████████████████████████████████████████████████████▉ | 11874/23458 [3:27:12<2:43:24, 1.18it/s][2025-04-23 15:52:13] (step=0011875) Train Loss: 5.5609, Train Steps/Sec: 1.17 + 51%|███████████████████████████████████████████████████████████████████████████████▏ | 11899/23458 [3:27:34<2:42:49, 1.18it/s][2025-04-23 15:52:35] (step=0011900) Train Loss: 5.6409, Train Steps/Sec: 1.12 + 51%|███████████████████████████████████████████████████████████████████████████████▎ | 11924/23458 [3:27:55<2:46:45, 1.15it/s][2025-04-23 15:52:57] (step=0011925) Train Loss: 5.4469, Train Steps/Sec: 1.17 + 51%|███████████████████████████████████████████████████████████████████████████████▍ | 11949/23458 [3:28:17<2:43:30, 1.17it/s][2025-04-23 15:53:18] (step=0011950) Train Loss: 5.5627, Train Steps/Sec: 1.17 + 51%|███████████████████████████████████████████████████████████████████████████████▋ | 11974/23458 [3:28:39<2:51:44, 1.11it/s][2025-04-23 15:53:41] (step=0011975) Train Loss: 5.5937, Train Steps/Sec: 1.10 + 51%|███████████████████████████████████████████████████████████████████████████████▊ | 11999/23458 [3:29:01<2:42:02, 1.18it/s][2025-04-23 15:54:02] (step=0012000) Train Loss: 5.5848, Train Steps/Sec: 1.16 +You are using a model of type blip-2 to instantiate a model of type instructblip. This is not supported for all configurations of models and can yield errors. +[2025-04-23 15:54:02] text_config is None. Initializing the text config with default values (`OPTConfig`). +model_name_or_path:/tmp/haozhezhao/model/blip2-flan-t5-xl +image_place_holder: +tokenizer length after expend 32101 +tokenizer length before expend 32100 +100%|███████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████| 4/4 [03:57<00:00, 59.34s/it] +[2025-04-23 15:59:08] Finish Eval in 12000 steps...████████████████████████████████████████████████████████████████████████████████████████████████| 4/4 [03:56<00:00, 59.05s/it] +processing left-padding... +processing left-padding... +processing left-padding... +processing left-padding... +[2025-04-23 15:59:27] Saved checkpoint to checkpoint/Afteri2i_100kckpt_LLAVAT5_empty_Fixed_just_segment_pretrain_stage3_dreambench_recap_400k_t2i_400k_flux_200k_midjourney_150k_recovery_150k_3e4_epoch2_half_rotated/005-GPT-XL/checkpoints/0012000.pt +[2025-04-23 15:59:29] Removed old checkpoint: checkpoint/Afteri2i_100kckpt_LLAVAT5_empty_Fixed_just_segment_pretrain_stage3_dreambench_recap_400k_t2i_400k_flux_200k_midjourney_150k_recovery_150k_3e4_epoch2_half_rotated/005-GPT-XL/checkpoints/0010000.pt + 51%|███████████████████████████████████████████████████████████████████████████████▉ | 12024/23458 [3:34:50<2:58:09, 1.07it/s][2025-04-23 15:59:51] (step=0012025) Train Loss: 5.6000, Train Steps/Sec: 0.07 + 51%|████████████████████████████████████████████████████████████████████████████████▏ | 12049/23458 [3:35:11<2:42:17, 1.17it/s][2025-04-23 16:00:13] (step=0012050) Train Loss: 5.5448, Train Steps/Sec: 1.18 + 51%|████████████████████████████████████████████████████████████████████████████████▎ | 12074/23458 [3:35:33<2:40:22, 1.18it/s][2025-04-23 16:00:35] (step=0012075) Train Loss: 5.5937, Train Steps/Sec: 1.13 + 52%|████████████████████████████████████████████████████████████████████████████████▍ | 12099/23458 [3:35:55<2:38:54, 1.19it/s][2025-04-23 16:00:56] (step=0012100) Train Loss: 5.5361, Train Steps/Sec: 1.17 + 52%|████████████████████████████████████████████████████████████████████████████████▋ | 12124/23458 [3:36:16<2:43:40, 1.15it/s][2025-04-23 16:01:17] (step=0012125) Train Loss: 5.5431, Train Steps/Sec: 1.17 + 52%|████████████████████████████████████████████████████████████████████████████████▊ | 12149/23458 [3:36:37<2:40:12, 1.18it/s][2025-04-23 16:01:39] (step=0012150) Train Loss: 5.5239, Train Steps/Sec: 1.18 + 52%|████████████████████████████████████████████████████████████████████████████████▉ | 12174/23458 [3:36:59<2:38:47, 1.18it/s][2025-04-23 16:02:00] (step=0012175) Train Loss: 5.5776, Train Steps/Sec: 1.17 + 52%|█████████████████████████████████████████████████████████████████████████████████▏ | 12199/23458 [3:37:20<2:37:40, 1.19it/s][2025-04-23 16:02:21] (step=0012200) Train Loss: 5.5693, Train Steps/Sec: 1.17 + 52%|█████████████████████████████████████████████████████████████████████████████████▎ | 12224/23458 [3:37:41<2:42:13, 1.15it/s][2025-04-23 16:02:43] (step=0012225) Train Loss: 5.6415, Train Steps/Sec: 1.18 + 52%|█████████████████████████████████████████████████████████████████████████████████▍ | 12249/23458 [3:38:02<2:40:10, 1.17it/s][2025-04-23 16:03:04] (step=0012250) Train Loss: 5.5763, Train Steps/Sec: 1.17 + 52%|█████████████████████████████████████████████████████████████████████████████████▌ | 12274/23458 [3:38:24<2:37:53, 1.18it/s][2025-04-23 16:03:25] (step=0012275) Train Loss: 5.4533, Train Steps/Sec: 1.18 + 52%|█████████████████████████████████████████████████████████████████████████████████▊ | 12299/23458 [3:38:45<2:36:22, 1.19it/s][2025-04-23 16:03:46] (step=0012300) Train Loss: 5.5006, Train Steps/Sec: 1.17 + 53%|█████████████████████████████████████████████████████████████████████████████████▉ | 12324/23458 [3:39:06<2:40:50, 1.15it/s][2025-04-23 16:04:08] (step=0012325) Train Loss: 5.5328, Train Steps/Sec: 1.18 + 53%|██████████████████████████████████████████████████████████████████████████████████ | 12349/23458 [3:39:28<2:37:27, 1.18it/s][2025-04-23 16:04:29] (step=0012350) Train Loss: 5.5630, Train Steps/Sec: 1.17 + 53%|██████████████████████████████████████████████████████████████████████████████████▎ | 12374/23458 [3:39:49<2:36:19, 1.18it/s][2025-04-23 16:04:50] (step=0012375) Train Loss: 5.6340, Train Steps/Sec: 1.17 + 53%|██████████████████████████████████████████████████████████████████████████████████▍ | 12399/23458 [3:40:10<2:35:01, 1.19it/s][2025-04-23 16:05:12] (step=0012400) Train Loss: 5.5472, Train Steps/Sec: 1.17 + 53%|██████████████████████████████████████████████████████████████████████████████████▌ | 12424/23458 [3:40:33<3:06:28, 1.01s/it][2025-04-23 16:05:35] (step=0012425) Train Loss: 5.5494, Train Steps/Sec: 1.09 + 53%|██████████████████████████████████████████████████████████████████████████████████▊ | 12449/23458 [3:40:55<2:36:50, 1.17it/s][2025-04-23 16:05:56] (step=0012450) Train Loss: 5.5384, Train Steps/Sec: 1.17 + 53%|██████████████████████████████████████████████████████████████████████████████████▉ | 12474/23458 [3:41:17<2:35:41, 1.18it/s][2025-04-23 16:06:18] (step=0012475) Train Loss: 5.5601, Train Steps/Sec: 1.13 + 53%|███████████████████████████████████████████████████████████████████████████████████ | 12499/23458 [3:41:39<2:33:26, 1.19it/s][2025-04-23 16:06:40] (step=0012500) Train Loss: 5.5386, Train Steps/Sec: 1.13 + 53%|███████████████████████████████████████████████████████████████████████████████████▎ | 12524/23458 [3:42:00<2:37:22, 1.16it/s][2025-04-23 16:07:01] (step=0012525) Train Loss: 5.5353, Train Steps/Sec: 1.18 + 53%|███████████████████████████████████████████████████████████████████████████████████▍ | 12549/23458 [3:42:22<2:46:41, 1.09it/s][2025-04-23 16:07:24] (step=0012550) Train Loss: 5.4409, Train Steps/Sec: 1.13 + 54%|███████████████████████████████████████████████████████████████████████████████████▌ | 12574/23458 [3:42:44<2:33:42, 1.18it/s][2025-04-23 16:07:45] (step=0012575) Train Loss: 5.5736, Train Steps/Sec: 1.17 + 54%|███████████████████████████████████████████████████████████████████████████████████▊ | 12599/23458 [3:43:05<2:32:08, 1.19it/s][2025-04-23 16:08:06] (step=0012600) Train Loss: 5.5057, Train Steps/Sec: 1.17 + 54%|███████████████████████████████████████████████████████████████████████████████████▉ | 12624/23458 [3:43:26<2:36:42, 1.15it/s][2025-04-23 16:08:28] (step=0012625) Train Loss: 5.5021, Train Steps/Sec: 1.17 + 54%|████████████████████████████████████████████████████████████████████████████████████ | 12649/23458 [3:43:48<2:34:49, 1.16it/s][2025-04-23 16:08:49] (step=0012650) Train Loss: 5.5416, Train Steps/Sec: 1.17 + 54%|████████████████████████████████████████████████████████████████████████████████████▎ | 12674/23458 [3:44:10<2:32:49, 1.18it/s][2025-04-23 16:09:11] (step=0012675) Train Loss: 5.4875, Train Steps/Sec: 1.13 + 54%|████████████████████████████████████████████████████████████████████████████████████▍ | 12699/23458 [3:44:31<2:31:16, 1.19it/s][2025-04-23 16:09:33] (step=0012700) Train Loss: 5.5195, Train Steps/Sec: 1.17 + 54%|████████████████████████████████████████████████████████████████████████████████████▌ | 12724/23458 [3:44:52<2:34:43, 1.16it/s][2025-04-23 16:09:54] (step=0012725) Train Loss: 5.4831, Train Steps/Sec: 1.18 + 54%|████████████████████████████████████████████████████████████████████████████████████▊ | 12749/23458 [3:45:15<2:32:48, 1.17it/s][2025-04-23 16:10:16] (step=0012750) Train Loss: 5.5845, Train Steps/Sec: 1.12 + 54%|████████████████████████████████████████████████████████████████████████████████████▉ | 12774/23458 [3:45:36<2:30:51, 1.18it/s][2025-04-23 16:10:37] (step=0012775) Train Loss: 5.5646, Train Steps/Sec: 1.17 + 55%|█████████████████████████████████████████████████████████████████████████████████████ | 12799/23458 [3:45:58<2:29:28, 1.19it/s][2025-04-23 16:11:00] (step=0012800) Train Loss: 5.5868, Train Steps/Sec: 1.12 + 55%|█████████████████████████████████████████████████████████████████████████████████████▎ | 12824/23458 [3:46:20<2:34:26, 1.15it/s][2025-04-23 16:11:21] (step=0012825) Train Loss: 5.5849, Train Steps/Sec: 1.18 + 55%|█████████████████████████████████████████████████████████████████████████████████████▍ | 12849/23458 [3:46:41<2:30:55, 1.17it/s][2025-04-23 16:11:42] (step=0012850) Train Loss: 5.5360, Train Steps/Sec: 1.18 + 55%|█████████████████████████████████████████████████████████████████████████████████████▌ | 12874/23458 [3:47:02<2:28:53, 1.18it/s][2025-04-23 16:12:03] (step=0012875) Train Loss: 5.5797, Train Steps/Sec: 1.18 + 55%|█████████████████████████████████████████████████████████████████████████████████████▊ | 12899/23458 [3:47:23<2:28:03, 1.19it/s][2025-04-23 16:12:25] (step=0012900) Train Loss: 5.5852, Train Steps/Sec: 1.17 + 55%|█████████████████████████████████████████████████████████████████████████████████████▉ | 12924/23458 [3:47:45<2:32:13, 1.15it/s][2025-04-23 16:12:46] (step=0012925) Train Loss: 5.6204, Train Steps/Sec: 1.17 + 55%|██████████████████████████████████████████████████████████████████████████████████████ | 12949/23458 [3:48:06<2:29:50, 1.17it/s][2025-04-23 16:13:07] (step=0012950) Train Loss: 5.4621, Train Steps/Sec: 1.18 + 55%|██████████████████████████████████████████████████████████████████████████████████████▎ | 12974/23458 [3:48:27<2:28:26, 1.18it/s][2025-04-23 16:13:29] (step=0012975) Train Loss: 5.4966, Train Steps/Sec: 1.17 + 55%|██████████████████████████████████████████████████████████████████████████████████████▍ | 12999/23458 [3:48:49<2:26:51, 1.19it/s][2025-04-23 16:13:50] (step=0013000) Train Loss: 5.4166, Train Steps/Sec: 1.17 + 56%|██████████████████████████████████████████████████████████████████████████████████████▌ | 13024/23458 [3:49:10<2:30:25, 1.16it/s][2025-04-23 16:14:11] (step=0013025) Train Loss: 5.5993, Train Steps/Sec: 1.17 + 56%|██████████████████████████████████████████████████████████████████████████████████████▊ | 13049/23458 [3:49:31<2:27:45, 1.17it/s][2025-04-23 16:14:33] (step=0013050) Train Loss: 5.4689, Train Steps/Sec: 1.18 + 56%|██████████████████████████████████████████████████████████████████████████████████████▉ | 13074/23458 [3:49:53<2:59:02, 1.03s/it][2025-04-23 16:14:55] (step=0013075) Train Loss: 5.5226, Train Steps/Sec: 1.13 + 56%|███████████████████████████████████████████████████████████████████████████████████████ | 13099/23458 [3:50:16<2:25:43, 1.18it/s][2025-04-23 16:15:17] (step=0013100) Train Loss: 5.5382, Train Steps/Sec: 1.13 + 56%|███████████████████████████████████████████████████████████████████████████████████████▎ | 13124/23458 [3:50:38<2:32:43, 1.13it/s][2025-04-23 16:15:39] (step=0013125) Train Loss: 5.4808, Train Steps/Sec: 1.13 + 56%|███████████████████████████████████████████████████████████████████████████████████████▍ | 13149/23458 [3:51:00<2:28:33, 1.16it/s][2025-04-23 16:16:01] (step=0013150) Train Loss: 5.4702, Train Steps/Sec: 1.14 + 56%|███████████████████████████████████████████████████████████████████████████████████████▌ | 13174/23458 [3:51:21<2:25:15, 1.18it/s][2025-04-23 16:16:22] (step=0013175) Train Loss: 5.5122, Train Steps/Sec: 1.17 + 56%|███████████████████████████████████████████████████████████████████████████████████████▊ | 13199/23458 [3:51:42<2:23:37, 1.19it/s][2025-04-23 16:16:44] (step=0013200) Train Loss: 5.5480, Train Steps/Sec: 1.17 + 56%|███████████████████████████████████████████████████████████████████████████████████████▉ | 13224/23458 [3:52:05<2:27:49, 1.15it/s][2025-04-23 16:17:06] (step=0013225) Train Loss: 5.5017, Train Steps/Sec: 1.13 + 56%|████████████████████████████████████████████████████████████████████████████████████████ | 13249/23458 [3:52:26<2:25:36, 1.17it/s][2025-04-23 16:17:27] (step=0013250) Train Loss: 5.5094, Train Steps/Sec: 1.17 + 57%|████████████████████████████████████████████████████████████████████████████████████████▎ | 13274/23458 [3:52:47<2:23:46, 1.18it/s][2025-04-23 16:17:48] (step=0013275) Train Loss: 5.5620, Train Steps/Sec: 1.17 + 57%|████████████████████████████████████████████████████████████████████████████████████████▍ | 13299/23458 [3:53:08<2:22:22, 1.19it/s][2025-04-23 16:18:10] (step=0013300) Train Loss: 5.5395, Train Steps/Sec: 1.17 + 57%|████████████████████████████████████████████████████████████████████████████████████████▌ | 13324/23458 [3:53:31<2:26:43, 1.15it/s][2025-04-23 16:18:32] (step=0013325) Train Loss: 5.6032, Train Steps/Sec: 1.13 + 57%|████████████████████████████████████████████████████████████████████████████████████████▊ | 13349/23458 [3:53:52<2:23:51, 1.17it/s][2025-04-23 16:18:53] (step=0013350) Train Loss: 5.5252, Train Steps/Sec: 1.17 + 57%|████████████████████████████████████████████████████████████████████████████████████████▉ | 13374/23458 [3:54:13<2:21:52, 1.18it/s][2025-04-23 16:19:15] (step=0013375) Train Loss: 5.6318, Train Steps/Sec: 1.18 + 57%|█████████████████████████████████████████████████████████████████████████████████████████ | 13399/23458 [3:54:34<2:21:40, 1.18it/s][2025-04-23 16:19:36] (step=0013400) Train Loss: 5.5852, Train Steps/Sec: 1.17 + 57%|█████████████████████████████████████████████████████████████████████████████████████████▎ | 13424/23458 [3:54:56<2:24:35, 1.16it/s][2025-04-23 16:19:57] (step=0013425) Train Loss: 5.5445, Train Steps/Sec: 1.18 + 57%|█████████████████████████████████████████████████████████████████████████████████████████▍ | 13449/23458 [3:55:17<2:22:49, 1.17it/s][2025-04-23 16:20:19] (step=0013450) Train Loss: 5.4993, Train Steps/Sec: 1.17 + 57%|█████████████████████████████████████████████████████████████████████████████████████████▌ | 13474/23458 [3:55:39<2:20:55, 1.18it/s][2025-04-23 16:20:41] (step=0013475) Train Loss: 5.4998, Train Steps/Sec: 1.12 + 58%|█████████████████████████████████████████████████████████████████████████████████████████▊ | 13499/23458 [3:56:01<2:19:39, 1.19it/s][2025-04-23 16:21:02] (step=0013500) Train Loss: 5.5100, Train Steps/Sec: 1.17 + 58%|█████████████████████████████████████████████████████████████████████████████████████████▉ | 13524/23458 [3:56:23<2:24:22, 1.15it/s][2025-04-23 16:21:24] (step=0013525) Train Loss: 5.4805, Train Steps/Sec: 1.12 + 58%|██████████████████████████████████████████████████████████████████████████████████████████ | 13549/23458 [3:56:44<2:21:00, 1.17it/s][2025-04-23 16:21:46] (step=0013550) Train Loss: 5.5203, Train Steps/Sec: 1.17 + 58%|██████████████████████████████████████████████████████████████████████████████████████████▎ | 13574/23458 [3:57:06<2:18:56, 1.19it/s][2025-04-23 16:22:07] (step=0013575) Train Loss: 5.4657, Train Steps/Sec: 1.18 + 58%|██████████████████████████████████████████████████████████████████████████████████████████▍ | 13599/23458 [3:57:27<2:17:47, 1.19it/s][2025-04-23 16:22:28] (step=0013600) Train Loss: 5.5671, Train Steps/Sec: 1.17 + 58%|██████████████████████████████████████████████████████████████████████████████████████████▌ | 13624/23458 [3:57:48<2:22:29, 1.15it/s][2025-04-23 16:22:50] (step=0013625) Train Loss: 5.5200, Train Steps/Sec: 1.17 + 58%|██████████████████████████████████████████████████████████████████████████████████████████▊ | 13649/23458 [3:58:10<2:19:18, 1.17it/s][2025-04-23 16:23:11] (step=0013650) Train Loss: 5.5218, Train Steps/Sec: 1.17 + 58%|██████████████████████████████████████████████████████████████████████████████████████████▉ | 13674/23458 [3:58:31<2:17:44, 1.18it/s][2025-04-23 16:23:32] (step=0013675) Train Loss: 5.4432, Train Steps/Sec: 1.17 + 58%|███████████████████████████████████████████████████████████████████████████████████████████ | 13699/23458 [3:58:52<2:17:37, 1.18it/s][2025-04-23 16:23:54] (step=0013700) Train Loss: 5.5052, Train Steps/Sec: 1.17 + 59%|███████████████████████████████████████████████████████████████████████████████████████████▎ | 13724/23458 [3:59:13<2:20:42, 1.15it/s][2025-04-23 16:24:15] (step=0013725) Train Loss: 5.5429, Train Steps/Sec: 1.17 + 59%|███████████████████████████████████████████████████████████████████████████████████████████▍ | 13749/23458 [3:59:36<2:19:15, 1.16it/s][2025-04-23 16:24:38] (step=0013750) Train Loss: 5.4848, Train Steps/Sec: 1.09 + 59%|███████████████████████████████████████████████████████████████████████████████████████████▌ | 13774/23458 [3:59:59<2:45:15, 1.02s/it][2025-04-23 16:25:00] (step=0013775) Train Loss: 5.5653, Train Steps/Sec: 1.13 + 59%|███████████████████████████████████████████████████████████████████████████████████████████▊ | 13799/23458 [4:00:20<2:16:20, 1.18it/s][2025-04-23 16:25:21] (step=0013800) Train Loss: 5.6074, Train Steps/Sec: 1.17 + 59%|███████████████████████████████████████████████████████████████████████████████████████████▉ | 13824/23458 [4:00:42<2:18:47, 1.16it/s][2025-04-23 16:25:43] (step=0013825) Train Loss: 5.4420, Train Steps/Sec: 1.13 + 59%|████████████████████████████████████████████████████████████████████████████████████████████ | 13849/23458 [4:01:03<2:16:40, 1.17it/s][2025-04-23 16:26:05] (step=0013850) Train Loss: 5.5832, Train Steps/Sec: 1.17 + 59%|████████████████████████████████████████████████████████████████████████████████████████████▎ | 13874/23458 [4:01:25<2:17:14, 1.16it/s][2025-04-23 16:26:27] (step=0013875) Train Loss: 5.5569, Train Steps/Sec: 1.13 + 59%|████████████████████████████████████████████████████████████████████████████████████████████▍ | 13899/23458 [4:01:47<2:14:31, 1.18it/s][2025-04-23 16:26:48] (step=0013900) Train Loss: 5.5526, Train Steps/Sec: 1.17 + 59%|████████████████████████████████████████████████████████████████████████████████████████████▌ | 13924/23458 [4:02:08<2:17:27, 1.16it/s][2025-04-23 16:27:09] (step=0013925) Train Loss: 5.4985, Train Steps/Sec: 1.18 + 59%|████████████████████████████████████████████████████████████████████████████████████████████▊ | 13949/23458 [4:02:29<2:14:57, 1.17it/s][2025-04-23 16:27:31] (step=0013950) Train Loss: 5.5561, Train Steps/Sec: 1.17 + 60%|████████████████████████████████████████████████████████████████████████████████████████████▉ | 13974/23458 [4:02:51<2:15:13, 1.17it/s][2025-04-23 16:27:53] (step=0013975) Train Loss: 5.5362, Train Steps/Sec: 1.13 + 60%|█████████████████████████████████████████████████████████████████████████████████████████████ | 13999/23458 [4:03:13<2:13:20, 1.18it/s][2025-04-23 16:28:14] (step=0014000) Train Loss: 5.5002, Train Steps/Sec: 1.17 +You are using a model of type blip-2 to instantiate a model of type instructblip. This is not supported for all configurations of models and can yield errors. +[2025-04-23 16:28:14] text_config is None. Initializing the text config with default values (`OPTConfig`). +model_name_or_path:/tmp/haozhezhao/model/blip2-flan-t5-xl +image_place_holder: +tokenizer length after expend 32101 +tokenizer length before expend 32100 +100%|███████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████| 4/4 [03:57<00:00, 59.35s/it] +[2025-04-23 16:33:20] Finish Eval in 14000 steps...████████████████████████████████████████████████████████████████████████████████████████████████| 4/4 [03:56<00:00, 59.06s/it] +processing left-padding... +processing left-padding... +processing left-padding... +processing left-padding... +[2025-04-23 16:33:39] Saved checkpoint to checkpoint/Afteri2i_100kckpt_LLAVAT5_empty_Fixed_just_segment_pretrain_stage3_dreambench_recap_400k_t2i_400k_flux_200k_midjourney_150k_recovery_150k_3e4_epoch2_half_rotated/005-GPT-XL/checkpoints/0014000.pt +[2025-04-23 16:33:41] Removed old checkpoint: checkpoint/Afteri2i_100kckpt_LLAVAT5_empty_Fixed_just_segment_pretrain_stage3_dreambench_recap_400k_t2i_400k_flux_200k_midjourney_150k_recovery_150k_3e4_epoch2_half_rotated/005-GPT-XL/checkpoints/0012000.pt + 60%|█████████████████████████████████████████████████████████████████████████████████████████████▎ | 14024/23458 [4:09:01<2:19:43, 1.13it/s][2025-04-23 16:34:02] (step=0014025) Train Loss: 5.4944, Train Steps/Sec: 0.07 + 60%|█████████████████████████████████████████████████████████████████████████████████████████████▍ | 14049/23458 [4:09:22<2:13:48, 1.17it/s][2025-04-23 16:34:23] (step=0014050) Train Loss: 5.5202, Train Steps/Sec: 1.17 + 60%|█████████████████████████████████████████████████████████████████████████████████████████████▌ | 14074/23458 [4:09:43<2:11:54, 1.19it/s][2025-04-23 16:34:45] (step=0014075) Train Loss: 5.5473, Train Steps/Sec: 1.18 + 60%|█████████████████████████████████████████████████████████████████████████████████████████████▊ | 14099/23458 [4:10:05<2:12:19, 1.18it/s][2025-04-23 16:35:06] (step=0014100) Train Loss: 5.5673, Train Steps/Sec: 1.16 + 60%|█████████████████████████████████████████████████████████████████████████████████████████████▉ | 14124/23458 [4:10:26<2:14:44, 1.15it/s][2025-04-23 16:35:27] (step=0014125) Train Loss: 5.5259, Train Steps/Sec: 1.18 + 60%|██████████████████████████████████████████████████████████████████████████████████████████████ | 14149/23458 [4:10:47<2:12:32, 1.17it/s][2025-04-23 16:35:49] (step=0014150) Train Loss: 5.5159, Train Steps/Sec: 1.17 + 60%|██████████████████████████████████████████████████████████████████████████████████████████████▎ | 14174/23458 [4:11:09<2:10:42, 1.18it/s][2025-04-23 16:36:10] (step=0014175) Train Loss: 5.5393, Train Steps/Sec: 1.18 + 61%|██████████████████████████████████████████████████████████████████████████████████████████████▍ | 14199/23458 [4:11:31<2:09:57, 1.19it/s][2025-04-23 16:36:32] (step=0014200) Train Loss: 5.4852, Train Steps/Sec: 1.12 + 61%|██████████████████████████████████████████████████████████████████████████████████████████████▌ | 14224/23458 [4:11:52<2:13:31, 1.15it/s][2025-04-23 16:36:54] (step=0014225) Train Loss: 5.5351, Train Steps/Sec: 1.17 + 61%|██████████████████████████████████████████████████████████████████████████████████████████████▊ | 14249/23458 [4:12:15<2:12:55, 1.15it/s][2025-04-23 16:37:16] (step=0014250) Train Loss: 5.5801, Train Steps/Sec: 1.12 + 61%|██████████████████████████████████████████████████████████████████████████████████████████████▉ | 14274/23458 [4:12:36<2:10:05, 1.18it/s][2025-04-23 16:37:37] (step=0014275) Train Loss: 5.4511, Train Steps/Sec: 1.17 + 61%|███████████████████████████████████████████████████████████████████████████████████████████████ | 14299/23458 [4:12:57<2:08:33, 1.19it/s][2025-04-23 16:37:59] (step=0014300) Train Loss: 5.4504, Train Steps/Sec: 1.17 + 61%|███████████████████████████████████████████████████████████████████████████████████████████████▎ | 14324/23458 [4:13:19<2:11:43, 1.16it/s][2025-04-23 16:38:20] (step=0014325) Train Loss: 5.4499, Train Steps/Sec: 1.17 + 61%|███████████████████████████████████████████████████████████████████████████████████████████████▍ | 14349/23458 [4:13:40<2:09:16, 1.17it/s][2025-04-23 16:38:41] (step=0014350) Train Loss: 5.3827, Train Steps/Sec: 1.17 + 61%|███████████████████████████████████████████████████████████████████████████████████████████████▌ | 14374/23458 [4:14:01<2:07:39, 1.19it/s][2025-04-23 16:39:03] (step=0014375) Train Loss: 5.4427, Train Steps/Sec: 1.18 + 61%|███████████████████████████████████████████████████████████████████████████████████████████████▊ | 14399/23458 [4:14:23<2:08:33, 1.17it/s][2025-04-23 16:39:26] (step=0014400) Train Loss: 5.5057, Train Steps/Sec: 1.13 + 61%|███████████████████████████████████████████████████████████████████████████████████████████████▉ | 14424/23458 [4:14:46<2:10:00, 1.16it/s][2025-04-23 16:39:47] (step=0014425) Train Loss: 5.5141, Train Steps/Sec: 1.17 + 62%|████████████████████████████████████████████████████████████████████████████████████████████████ | 14449/23458 [4:15:08<2:08:02, 1.17it/s][2025-04-23 16:40:09] (step=0014450) Train Loss: 5.4834, Train Steps/Sec: 1.13 + 62%|████████████████████████████████████████████████████████████████████████████████████████████████▎ | 14474/23458 [4:15:30<2:07:19, 1.18it/s][2025-04-23 16:40:31] (step=0014475) Train Loss: 5.5216, Train Steps/Sec: 1.14 + 62%|████████████████████████████████████████████████████████████████████████████████████████████████▍ | 14499/23458 [4:15:51<2:05:47, 1.19it/s][2025-04-23 16:40:52] (step=0014500) Train Loss: 5.5890, Train Steps/Sec: 1.17 + 62%|████████████████████████████████████████████████████████████████████████████████████████████████▌ | 14524/23458 [4:16:13<2:34:19, 1.04s/it][2025-04-23 16:41:15] (step=0014525) Train Loss: 5.5878, Train Steps/Sec: 1.13 + 62%|████████████████████████████████████████████████████████████████████████████████████████████████▊ | 14549/23458 [4:16:35<2:06:32, 1.17it/s][2025-04-23 16:41:36] (step=0014550) Train Loss: 5.5000, Train Steps/Sec: 1.17 + 62%|████████████████████████████████████████████████████████████████████████████████████████████████▉ | 14574/23458 [4:16:56<2:05:37, 1.18it/s][2025-04-23 16:41:57] (step=0014575) Train Loss: 5.4697, Train Steps/Sec: 1.17 + 62%|█████████████████████████████████████████████████████████████████████████████████████████████████ | 14599/23458 [4:17:17<2:05:24, 1.18it/s][2025-04-23 16:42:19] (step=0014600) Train Loss: 5.4923, Train Steps/Sec: 1.17 + 62%|█████████████████████████████████████████████████████████████████████████████████████████████████▎ | 14624/23458 [4:17:39<2:07:43, 1.15it/s][2025-04-23 16:42:40] (step=0014625) Train Loss: 5.4902, Train Steps/Sec: 1.17 + 62%|█████████████████████████████████████████████████████████████████████████████████████████████████▍ | 14649/23458 [4:18:01<2:32:59, 1.04s/it][2025-04-23 16:43:02] (step=0014650) Train Loss: 5.4978, Train Steps/Sec: 1.13 + 63%|█████████████████████████████████████████████████████████████████████████████████████████████████▌ | 14674/23458 [4:18:22<2:03:40, 1.18it/s][2025-04-23 16:43:23] (step=0014675) Train Loss: 5.5541, Train Steps/Sec: 1.18 + 63%|█████████████████████████████████████████████████████████████████████████████████████████████████▊ | 14699/23458 [4:18:43<2:03:45, 1.18it/s][2025-04-23 16:43:45] (step=0014700) Train Loss: 5.4956, Train Steps/Sec: 1.17 + 63%|█████████████████████████████████████████████████████████████████████████████████████████████████▉ | 14724/23458 [4:19:05<2:06:13, 1.15it/s][2025-04-23 16:44:06] (step=0014725) Train Loss: 5.5468, Train Steps/Sec: 1.17 + 63%|██████████████████████████████████████████████████████████████████████████████████████████████████ | 14749/23458 [4:19:26<2:03:42, 1.17it/s][2025-04-23 16:44:27] (step=0014750) Train Loss: 5.5262, Train Steps/Sec: 1.17 + 63%|██████████████████████████████████████████████████████████████████████████████████████████████████▏ | 14774/23458 [4:19:47<2:02:42, 1.18it/s][2025-04-23 16:44:49] (step=0014775) Train Loss: 5.4890, Train Steps/Sec: 1.17 + 63%|██████████████████████████████████████████████████████████████████████████████████████████████████▍ | 14799/23458 [4:20:09<2:01:51, 1.18it/s][2025-04-23 16:45:10] (step=0014800) Train Loss: 5.5048, Train Steps/Sec: 1.17 + 63%|██████████████████████████████████████████████████████████████████████████████████████████████████▌ | 14824/23458 [4:20:30<2:04:30, 1.16it/s][2025-04-23 16:45:31] (step=0014825) Train Loss: 5.5443, Train Steps/Sec: 1.18 + 63%|██████████████████████████████████████████████████████████████████████████████████████████████████▋ | 14849/23458 [4:20:51<2:02:32, 1.17it/s][2025-04-23 16:45:53] (step=0014850) Train Loss: 5.5363, Train Steps/Sec: 1.17 + 63%|██████████████████████████████████████████████████████████████████████████████████████████████████▉ | 14874/23458 [4:21:13<2:01:23, 1.18it/s][2025-04-23 16:46:14] (step=0014875) Train Loss: 5.5088, Train Steps/Sec: 1.18 + 64%|███████████████████████████████████████████████████████████████████████████████████████████████████ | 14899/23458 [4:21:34<2:00:08, 1.19it/s][2025-04-23 16:46:35] (step=0014900) Train Loss: 5.5094, Train Steps/Sec: 1.17 + 64%|███████████████████████████████████████████████████████████████████████████████████████████████████▏ | 14924/23458 [4:21:56<2:04:05, 1.15it/s][2025-04-23 16:46:58] (step=0014925) Train Loss: 5.5696, Train Steps/Sec: 1.12 + 64%|███████████████████████████████████████████████████████████████████████████████████████████████████▍ | 14949/23458 [4:22:17<2:00:45, 1.17it/s][2025-04-23 16:47:19] (step=0014950) Train Loss: 5.5767, Train Steps/Sec: 1.18 + 64%|███████████████████████████████████████████████████████████████████████████████████████████████████▌ | 14974/23458 [4:22:40<2:02:38, 1.15it/s][2025-04-23 16:47:41] (step=0014975) Train Loss: 5.5766, Train Steps/Sec: 1.13 + 64%|███████████████████████████████████████████████████████████████████████████████████████████████████▋ | 14999/23458 [4:23:01<1:58:11, 1.19it/s][2025-04-23 16:48:02] (step=0015000) Train Loss: 5.4737, Train Steps/Sec: 1.17 + 64%|███████████████████████████████████████████████████████████████████████████████████████████████████▉ | 15024/23458 [4:23:22<2:01:47, 1.15it/s][2025-04-23 16:48:24] (step=0015025) Train Loss: 5.5230, Train Steps/Sec: 1.18 + 64%|████████████████████████████████████████████████████████████████████████████████████████████████████ | 15049/23458 [4:23:44<2:00:02, 1.17it/s][2025-04-23 16:48:45] (step=0015050) Train Loss: 5.6084, Train Steps/Sec: 1.17 + 64%|████████████████████████████████████████████████████████████████████████████████████████████████████▏ | 15074/23458 [4:24:06<1:58:29, 1.18it/s][2025-04-23 16:49:08] (step=0015075) Train Loss: 5.5118, Train Steps/Sec: 1.09 + 64%|████████████████████████████████████████████████████████████████████████████████████████████████████▍ | 15099/23458 [4:24:29<1:59:26, 1.17it/s][2025-04-23 16:49:30] (step=0015100) Train Loss: 5.4299, Train Steps/Sec: 1.12 + 64%|████████████████████████████████████████████████████████████████████████████████████████████████████▌ | 15124/23458 [4:24:51<2:03:52, 1.12it/s][2025-04-23 16:49:52] (step=0015125) Train Loss: 5.4758, Train Steps/Sec: 1.14 + 65%|████████████████████████████████████████████████████████████████████████████████████████████████████▋ | 15149/23458 [4:25:12<1:57:59, 1.17it/s][2025-04-23 16:50:13] (step=0015150) Train Loss: 5.5543, Train Steps/Sec: 1.18 + 65%|████████████████████████████████████████████████████████████████████████████████████████████████████▉ | 15174/23458 [4:25:33<1:56:33, 1.18it/s][2025-04-23 16:50:35] (step=0015175) Train Loss: 5.5298, Train Steps/Sec: 1.18 + 65%|█████████████████████████████████████████████████████████████████████████████████████████████████████ | 15199/23458 [4:25:55<1:55:33, 1.19it/s][2025-04-23 16:50:57] (step=0015200) Train Loss: 5.5201, Train Steps/Sec: 1.13 + 65%|█████████████████████████████████████████████████████████████████████████████████████████████████████▏ | 15224/23458 [4:26:17<1:58:57, 1.15it/s][2025-04-23 16:51:18] (step=0015225) Train Loss: 5.5137, Train Steps/Sec: 1.17 + 65%|█████████████████████████████████████████████████████████████████████████████████████████████████████▍ | 15249/23458 [4:26:38<1:56:42, 1.17it/s][2025-04-23 16:51:39] (step=0015250) Train Loss: 5.4890, Train Steps/Sec: 1.17 + 65%|█████████████████████████████████████████████████████████████████████████████████████████████████████▌ | 15274/23458 [4:26:59<1:56:14, 1.17it/s][2025-04-23 16:52:01] (step=0015275) Train Loss: 5.6258, Train Steps/Sec: 1.17 + 65%|█████████████████████████████████████████████████████████████████████████████████████████████████████▋ | 15299/23458 [4:27:21<1:54:28, 1.19it/s][2025-04-23 16:52:22] (step=0015300) Train Loss: 5.4413, Train Steps/Sec: 1.17 + 65%|█████████████████████████████████████████████████████████████████████████████████████████████████████▉ | 15324/23458 [4:27:43<1:57:37, 1.15it/s][2025-04-23 16:52:44] (step=0015325) Train Loss: 5.4613, Train Steps/Sec: 1.12 + 65%|██████████████████████████████████████████████████████████████████████████████████████████████████████ | 15349/23458 [4:28:04<1:55:08, 1.17it/s][2025-04-23 16:53:05] (step=0015350) Train Loss: 5.5241, Train Steps/Sec: 1.18 + 66%|██████████████████████████████████████████████████████████████████████████████████████████████████████▏ | 15374/23458 [4:28:25<1:54:12, 1.18it/s][2025-04-23 16:53:27] (step=0015375) Train Loss: 5.5844, Train Steps/Sec: 1.17 + 66%|██████████████████████████████████████████████████████████████████████████████████████████████████████▍ | 15399/23458 [4:28:47<1:53:10, 1.19it/s][2025-04-23 16:53:48] (step=0015400) Train Loss: 5.5655, Train Steps/Sec: 1.17 + 66%|██████████████████████████████████████████████████████████████████████████████████████████████████████▌ | 15424/23458 [4:29:08<1:56:03, 1.15it/s][2025-04-23 16:54:09] (step=0015425) Train Loss: 5.4655, Train Steps/Sec: 1.18 + 66%|██████████████████████████████████████████████████████████████████████████████████████████████████████▋ | 15449/23458 [4:29:29<1:53:25, 1.18it/s][2025-04-23 16:54:31] (step=0015450) Train Loss: 5.5500, Train Steps/Sec: 1.18 + 66%|██████████████████████████████████████████████████████████████████████████████████████████████████████▉ | 15474/23458 [4:29:51<1:52:32, 1.18it/s][2025-04-23 16:54:52] (step=0015475) Train Loss: 5.5030, Train Steps/Sec: 1.18 + 66%|███████████████████████████████████████████████████████████████████████████████████████████████████████ | 15499/23458 [4:30:12<1:51:50, 1.19it/s][2025-04-23 16:55:13] (step=0015500) Train Loss: 5.5372, Train Steps/Sec: 1.17 + 66%|███████████████████████████████████████████████████████████████████████████████████████████████████████▏ | 15524/23458 [4:30:33<1:54:35, 1.15it/s][2025-04-23 16:55:35] (step=0015525) Train Loss: 5.6197, Train Steps/Sec: 1.18 + 66%|███████████████████████████████████████████████████████████████████████████████████████████████████████▍ | 15549/23458 [4:30:55<1:52:36, 1.17it/s][2025-04-23 16:55:56] (step=0015550) Train Loss: 5.5320, Train Steps/Sec: 1.17 + 66%|███████████████████████████████████████████████████████████████████████████████████████████████████████▌ | 15574/23458 [4:31:16<1:51:15, 1.18it/s][2025-04-23 16:56:17] (step=0015575) Train Loss: 5.5126, Train Steps/Sec: 1.18 + 66%|███████████████████████████████████████████████████████████████████████████████████████████████████████▋ | 15599/23458 [4:31:37<1:50:33, 1.18it/s][2025-04-23 16:56:39] (step=0015600) Train Loss: 5.4923, Train Steps/Sec: 1.17 + 67%|███████████████████████████████████████████████████████████████████████████████████████████████████████▉ | 15624/23458 [4:31:59<1:52:54, 1.16it/s][2025-04-23 16:57:00] (step=0015625) Train Loss: 5.5476, Train Steps/Sec: 1.18 + 67%|████████████████████████████████████████████████████████████████████████████████████████████████████████ | 15649/23458 [4:32:21<1:50:58, 1.17it/s][2025-04-23 16:57:22] (step=0015650) Train Loss: 5.5669, Train Steps/Sec: 1.12 + 67%|████████████████████████████████████████████████████████████████████████████████████████████████████████▏ | 15674/23458 [4:32:42<1:49:50, 1.18it/s][2025-04-23 16:57:43] (step=0015675) Train Loss: 5.4960, Train Steps/Sec: 1.17 + 67%|████████████████████████████████████████████████████████████████████████████████████████████████████████▍ | 15699/23458 [4:33:04<1:57:15, 1.10it/s][2025-04-23 16:58:06] (step=0015700) Train Loss: 5.5474, Train Steps/Sec: 1.12 + 67%|████████████████████████████████████████████████████████████████████████████████████████████████████████▌ | 15724/23458 [4:33:27<1:55:44, 1.11it/s][2025-04-23 16:58:29] (step=0015725) Train Loss: 5.4340, Train Steps/Sec: 1.09 + 67%|████████████████████████████████████████████████████████████████████████████████████████████████████████▋ | 15749/23458 [4:33:50<2:05:19, 1.03it/s][2025-04-23 16:58:51] (step=0015750) Train Loss: 5.4586, Train Steps/Sec: 1.13 + 67%|████████████████████████████████████████████████████████████████████████████████████████████████████████▉ | 15774/23458 [4:34:11<1:48:22, 1.18it/s][2025-04-23 16:59:13] (step=0015775) Train Loss: 5.5310, Train Steps/Sec: 1.14 + 67%|█████████████████████████████████████████████████████████████████████████████████████████████████████████ | 15799/23458 [4:34:33<1:47:23, 1.19it/s][2025-04-23 16:59:34] (step=0015800) Train Loss: 5.5939, Train Steps/Sec: 1.17 + 67%|█████████████████████████████████████████████████████████████████████████████████████████████████████████▏ | 15824/23458 [4:34:54<1:50:13, 1.15it/s][2025-04-23 16:59:56] (step=0015825) Train Loss: 5.5842, Train Steps/Sec: 1.17 + 68%|█████████████████████████████████████████████████████████████████████████████████████████████████████████▍ | 15849/23458 [4:35:16<1:53:04, 1.12it/s][2025-04-23 17:00:18] (step=0015850) Train Loss: 5.4688, Train Steps/Sec: 1.14 + 68%|█████████████████████████████████████████████████████████████████████████████████████████████████████████▌ | 15874/23458 [4:35:38<1:47:02, 1.18it/s][2025-04-23 17:00:39] (step=0015875) Train Loss: 5.5409, Train Steps/Sec: 1.17 + 68%|█████████████████████████████████████████████████████████████████████████████████████████████████████████▋ | 15899/23458 [4:35:59<1:46:22, 1.18it/s][2025-04-23 17:01:00] (step=0015900) Train Loss: 5.4748, Train Steps/Sec: 1.17 + 68%|█████████████████████████████████████████████████████████████████████████████████████████████████████████▉ | 15924/23458 [4:36:20<1:48:45, 1.15it/s][2025-04-23 17:01:22] (step=0015925) Train Loss: 5.5507, Train Steps/Sec: 1.17 + 68%|██████████████████████████████████████████████████████████████████████████████████████████████████████████ | 15949/23458 [4:36:42<1:46:50, 1.17it/s][2025-04-23 17:01:43] (step=0015950) Train Loss: 5.5252, Train Steps/Sec: 1.18 + 68%|██████████████████████████████████████████████████████████████████████████████████████████████████████████▏ | 15974/23458 [4:37:04<1:46:42, 1.17it/s][2025-04-23 17:02:05] (step=0015975) Train Loss: 5.5258, Train Steps/Sec: 1.13 + 68%|██████████████████████████████████████████████████████████████████████████████████████████████████████████▍ | 15999/23458 [4:37:25<1:44:48, 1.19it/s][2025-04-23 17:02:26] (step=0016000) Train Loss: 5.5051, Train Steps/Sec: 1.17 +You are using a model of type blip-2 to instantiate a model of type instructblip. This is not supported for all configurations of models and can yield errors. +[2025-04-23 17:02:26] text_config is None. Initializing the text config with default values (`OPTConfig`). +model_name_or_path:/tmp/haozhezhao/model/blip2-flan-t5-xl +image_place_holder: +tokenizer length after expend 32101 +tokenizer length before expend 32100 +100%|███████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████| 4/4 [03:56<00:00, 59.18s/it] +[2025-04-23 17:07:30] Finish Eval in 16000 steps...████████████████████████████████████████████████████████████████████████████████████████████████| 4/4 [03:56<00:00, 58.92s/it] +processing left-padding... +processing left-padding... +processing left-padding... +processing left-padding... +[2025-04-23 17:07:48] Saved checkpoint to checkpoint/Afteri2i_100kckpt_LLAVAT5_empty_Fixed_just_segment_pretrain_stage3_dreambench_recap_400k_t2i_400k_flux_200k_midjourney_150k_recovery_150k_3e4_epoch2_half_rotated/005-GPT-XL/checkpoints/0016000.pt +[2025-04-23 17:07:50] Removed old checkpoint: checkpoint/Afteri2i_100kckpt_LLAVAT5_empty_Fixed_just_segment_pretrain_stage3_dreambench_recap_400k_t2i_400k_flux_200k_midjourney_150k_recovery_150k_3e4_epoch2_half_rotated/005-GPT-XL/checkpoints/0014000.pt + 68%|██████████████████████████████████████████████████████████████████████████████████████████████████████████▌ | 16024/23458 [4:43:10<1:49:19, 1.13it/s][2025-04-23 17:08:12] (step=0016025) Train Loss: 5.5073, Train Steps/Sec: 0.07 + 68%|██████████████████████████████████████████████████████████████████████████████████████████████████████████▋ | 16049/23458 [4:43:31<1:45:11, 1.17it/s][2025-04-23 17:08:33] (step=0016050) Train Loss: 5.4484, Train Steps/Sec: 1.18 + 69%|██████████████████████████████████████████████████████████████████████████████████████████████████████████▉ | 16074/23458 [4:43:53<1:44:35, 1.18it/s][2025-04-23 17:08:54] (step=0016075) Train Loss: 5.4636, Train Steps/Sec: 1.17 + 69%|███████████████████████████████████████████████████████████████████████████████████████████████████████████ | 16099/23458 [4:44:14<1:42:58, 1.19it/s][2025-04-23 17:09:16] (step=0016100) Train Loss: 5.5710, Train Steps/Sec: 1.17 + 69%|███████████████████████████████████████████████████████████████████████████████████████████████████████████▏ | 16124/23458 [4:44:35<1:46:06, 1.15it/s][2025-04-23 17:09:37] (step=0016125) Train Loss: 5.5144, Train Steps/Sec: 1.17 + 69%|███████████████████████████████████████████████████████████████████████████████████████████████████████████▍ | 16149/23458 [4:44:57<1:43:41, 1.17it/s][2025-04-23 17:09:58] (step=0016150) Train Loss: 5.4943, Train Steps/Sec: 1.17 + 69%|███████████████████████████████████████████████████████████████████████████████████████████████████████████▌ | 16174/23458 [4:45:18<1:42:27, 1.18it/s][2025-04-23 17:10:19] (step=0016175) Train Loss: 5.3842, Train Steps/Sec: 1.18 + 69%|███████████████████████████████████████████████████████████████████████████████████████████████████████████▋ | 16199/23458 [4:45:39<1:42:31, 1.18it/s][2025-04-23 17:10:41] (step=0016200) Train Loss: 5.5843, Train Steps/Sec: 1.17 + 69%|███████████████████████████████████████████████████████████████████████████████████████████████████████████▉ | 16224/23458 [4:46:01<1:44:32, 1.15it/s][2025-04-23 17:11:02] (step=0016225) Train Loss: 5.4896, Train Steps/Sec: 1.18 + 69%|████████████████████████████████████████████████████████████████████████████████████████████████████████████ | 16249/23458 [4:46:22<1:42:29, 1.17it/s][2025-04-23 17:11:23] (step=0016250) Train Loss: 5.4983, Train Steps/Sec: 1.17 + 69%|████████████████████████████████████████████████████████████████████████████████████████████████████████████▏ | 16274/23458 [4:46:43<1:41:19, 1.18it/s][2025-04-23 17:11:45] (step=0016275) Train Loss: 5.4915, Train Steps/Sec: 1.18 + 69%|████████████████████████████████████████████████████████████████████████████████████████████████████████████▍ | 16299/23458 [4:47:05<1:40:30, 1.19it/s][2025-04-23 17:12:06] (step=0016300) Train Loss: 5.4555, Train Steps/Sec: 1.17 + 70%|████████████████████████████████████████████████████████████████████████████████████████████████████████████▌ | 16324/23458 [4:47:26<1:43:08, 1.15it/s][2025-04-23 17:12:27] (step=0016325) Train Loss: 5.4928, Train Steps/Sec: 1.18 + 70%|████████████████████████████████████████████████████████████████████████████████████████████████████████████▋ | 16349/23458 [4:47:48<2:15:15, 1.14s/it][2025-04-23 17:12:50] (step=0016350) Train Loss: 5.4474, Train Steps/Sec: 1.12 + 70%|████████████████████████████████████████████████████████████████████████████████████████████████████████████▉ | 16374/23458 [4:48:11<2:22:53, 1.21s/it][2025-04-23 17:13:12] (step=0016375) Train Loss: 5.5091, Train Steps/Sec: 1.09 + 70%|█████████████████████████████████████████████████████████████████████████████████████████████████████████████ | 16399/23458 [4:48:32<1:38:55, 1.19it/s][2025-04-23 17:13:34] (step=0016400) Train Loss: 5.5999, Train Steps/Sec: 1.17 + 70%|█████████████████████████████████████████████████████████████████████████████████████████████████████████████▏ | 16424/23458 [4:48:56<1:53:09, 1.04it/s][2025-04-23 17:13:57] (step=0016425) Train Loss: 5.4645, Train Steps/Sec: 1.08 + 70%|█████████████████████████████████████████████████████████████████████████████████████████████████████████████▍ | 16449/23458 [4:49:18<1:39:32, 1.17it/s][2025-04-23 17:14:19] (step=0016450) Train Loss: 5.5456, Train Steps/Sec: 1.13 + 70%|█████████████████████████████████████████████████████████████████████████████████████████████████████████████▌ | 16474/23458 [4:49:39<1:38:33, 1.18it/s][2025-04-23 17:14:40] (step=0016475) Train Loss: 5.4847, Train Steps/Sec: 1.17 + 70%|█████████████████████████████████████████████████████████████████████████████████████████████████████████████▋ | 16499/23458 [4:50:00<1:38:05, 1.18it/s][2025-04-23 17:15:02] (step=0016500) Train Loss: 5.5084, Train Steps/Sec: 1.17 + 70%|█████████████████████████████████████████████████████████████████████████████████████████████████████████████▉ | 16524/23458 [4:50:22<1:40:11, 1.15it/s][2025-04-23 17:15:24] (step=0016525) Train Loss: 5.4697, Train Steps/Sec: 1.13 + 71%|██████████████████████████████████████████████████████████████████████████████████████████████████████████████ | 16549/23458 [4:50:44<1:38:04, 1.17it/s][2025-04-23 17:15:45] (step=0016550) Train Loss: 5.5254, Train Steps/Sec: 1.18 + 71%|██████████████████████████████████████████████████████████████████████████████████████████████████████████████▏ | 16574/23458 [4:51:05<1:37:17, 1.18it/s][2025-04-23 17:16:06] (step=0016575) Train Loss: 5.4840, Train Steps/Sec: 1.17 + 71%|██████████████████████████████████████████████████████████████████████████████████████████████████████████████▍ | 16599/23458 [4:51:26<1:36:29, 1.18it/s][2025-04-23 17:16:28] (step=0016600) Train Loss: 5.5310, Train Steps/Sec: 1.17 + 71%|██████████████████████████████████████████████████████████████████████████████████████████████████████████████▌ | 16624/23458 [4:51:48<1:39:08, 1.15it/s][2025-04-23 17:16:49] (step=0016625) Train Loss: 5.4647, Train Steps/Sec: 1.17 + 71%|██████████████████████████████████████████████████████████████████████████████████████████████████████████████▋ | 16649/23458 [4:52:09<1:36:52, 1.17it/s][2025-04-23 17:17:10] (step=0016650) Train Loss: 5.5448, Train Steps/Sec: 1.17 + 71%|██████████████████████████████████████████████████████████████████████████████████████████████████████████████▉ | 16674/23458 [4:52:31<1:35:46, 1.18it/s][2025-04-23 17:17:32] (step=0016675) Train Loss: 5.4914, Train Steps/Sec: 1.13 + 71%|███████████████████████████████████████████████████████████████████████████████████████████████████████████████ | 16699/23458 [4:52:52<1:34:51, 1.19it/s][2025-04-23 17:17:54] (step=0016700) Train Loss: 5.5227, Train Steps/Sec: 1.17 + 71%|███████████████████████████████████████████████████████████████████████████████████████████████████████████████▏ | 16724/23458 [4:53:14<1:37:57, 1.15it/s][2025-04-23 17:18:15] (step=0016725) Train Loss: 5.4889, Train Steps/Sec: 1.17 + 71%|███████████████████████████████████████████████████████████████████████████████████████████████████████████████▍ | 16749/23458 [4:53:35<1:35:19, 1.17it/s][2025-04-23 17:18:37] (step=0016750) Train Loss: 5.4594, Train Steps/Sec: 1.17 + 72%|███████████████████████████████████████████████████████████████████████████████████████████████████████████████▌ | 16774/23458 [4:53:56<1:34:21, 1.18it/s][2025-04-23 17:18:58] (step=0016775) Train Loss: 5.4908, Train Steps/Sec: 1.18 + 72%|███████████████████████████████████████████████████████████████████████████████████████████████████████████████▋ | 16799/23458 [4:54:18<1:33:35, 1.19it/s][2025-04-23 17:19:19] (step=0016800) Train Loss: 5.5653, Train Steps/Sec: 1.17 + 72%|███████████████████████████████████████████████████████████████████████████████████████████████████████████████▉ | 16824/23458 [4:54:39<1:35:47, 1.15it/s][2025-04-23 17:19:40] (step=0016825) Train Loss: 5.5185, Train Steps/Sec: 1.18 + 72%|████████████████████████████████████████████████████████████████████████████████████████████████████████████████ | 16849/23458 [4:55:00<1:33:48, 1.17it/s][2025-04-23 17:20:02] (step=0016850) Train Loss: 5.4538, Train Steps/Sec: 1.18 + 72%|████████████████████████████████████████████████████████████████████████████████████████████████████████████████▏ | 16874/23458 [4:55:22<1:32:49, 1.18it/s][2025-04-23 17:20:23] (step=0016875) Train Loss: 5.4837, Train Steps/Sec: 1.17 + 72%|████████████████████████████████████████████████████████████████████████████████████████████████████████████████▍ | 16899/23458 [4:55:43<1:32:12, 1.19it/s][2025-04-23 17:20:44] (step=0016900) Train Loss: 5.6197, Train Steps/Sec: 1.17 + 72%|████████████████████████████████████████████████████████████████████████████████████████████████████████████████▌ | 16924/23458 [4:56:04<1:34:10, 1.16it/s][2025-04-23 17:21:06] (step=0016925) Train Loss: 5.5145, Train Steps/Sec: 1.18 + 72%|████████████████████████████████████████████████████████████████████████████████████████████████████████████████▋ | 16949/23458 [4:56:26<1:32:29, 1.17it/s][2025-04-23 17:21:27] (step=0016950) Train Loss: 5.4973, Train Steps/Sec: 1.18 + 72%|████████████████████████████████████████████████████████████████████████████████████████████████████████████████▉ | 16974/23458 [4:56:47<1:31:21, 1.18it/s][2025-04-23 17:21:48] (step=0016975) Train Loss: 5.5083, Train Steps/Sec: 1.18 + 72%|█████████████████████████████████████████████████████████████████████████████████████████████████████████████████ | 16999/23458 [4:57:08<1:30:45, 1.19it/s][2025-04-23 17:22:10] (step=0017000) Train Loss: 5.5124, Train Steps/Sec: 1.17 + 73%|█████████████████████████████████████████████████████████████████████████████████████████████████████████████████▏ | 17024/23458 [4:57:29<1:32:56, 1.15it/s][2025-04-23 17:22:31] (step=0017025) Train Loss: 5.4440, Train Steps/Sec: 1.18 + 73%|█████████████████████████████████████████████████████████████████████████████████████████████████████████████████▍ | 17049/23458 [4:57:52<1:31:12, 1.17it/s][2025-04-23 17:22:54] (step=0017050) Train Loss: 5.5187, Train Steps/Sec: 1.09 + 73%|█████████████████████████████████████████████████████████████████████████████████████████████████████████████████▌ | 17074/23458 [4:58:15<1:50:01, 1.03s/it][2025-04-23 17:23:17] (step=0017075) Train Loss: 5.5263, Train Steps/Sec: 1.08 + 73%|█████████████████████████████████████████████████████████████████████████████████████████████████████████████████▋ | 17099/23458 [4:58:38<1:37:54, 1.08it/s][2025-04-23 17:23:39] (step=0017100) Train Loss: 5.5183, Train Steps/Sec: 1.13 + 73%|█████████████████████████████████████████████████████████████████████████████████████████████████████████████████▉ | 17124/23458 [4:58:59<1:31:22, 1.16it/s][2025-04-23 17:24:00] (step=0017125) Train Loss: 5.4882, Train Steps/Sec: 1.18 + 73%|██████████████████████████████████████████████████████████████████████████████████████████████████████████████████ | 17149/23458 [4:59:21<1:58:41, 1.13s/it][2025-04-23 17:24:23] (step=0017150) Train Loss: 5.6059, Train Steps/Sec: 1.13 + 73%|██████████████████████████████████████████████████████████████████████████████████████████████████████████████████▏ | 17174/23458 [4:59:43<1:29:35, 1.17it/s][2025-04-23 17:24:45] (step=0017175) Train Loss: 5.4793, Train Steps/Sec: 1.14 + 73%|██████████████████████████████████████████████████████████████████████████████████████████████████████████████████▍ | 17199/23458 [5:00:05<1:29:05, 1.17it/s][2025-04-23 17:25:06] (step=0017200) Train Loss: 5.4168, Train Steps/Sec: 1.16 + 73%|██████████████████████████████████████████████████████████████████████████████████████████████████████████████████▌ | 17224/23458 [5:00:26<1:31:07, 1.14it/s][2025-04-23 17:25:27] (step=0017225) Train Loss: 5.5064, Train Steps/Sec: 1.17 + 74%|██████████████████████████████████████████████████████████████████████████████████████████████████████████████████▋ | 17249/23458 [5:00:47<1:28:25, 1.17it/s][2025-04-23 17:25:49] (step=0017250) Train Loss: 5.5083, Train Steps/Sec: 1.17 + 74%|██████████████████████████████████████████████████████████████████████████████████████████████████████████████████▉ | 17274/23458 [5:01:09<1:27:24, 1.18it/s][2025-04-23 17:26:10] (step=0017275) Train Loss: 5.5279, Train Steps/Sec: 1.17 + 74%|███████████████████████████████████████████████████████████████████████████████████████████████████████████████████ | 17299/23458 [5:01:30<1:26:21, 1.19it/s][2025-04-23 17:26:31] (step=0017300) Train Loss: 5.5151, Train Steps/Sec: 1.17 + 74%|███████████████████████████████████████████████████████████████████████████████████████████████████████████████████▏ | 17324/23458 [5:01:52<1:28:37, 1.15it/s][2025-04-23 17:26:54] (step=0017325) Train Loss: 5.5180, Train Steps/Sec: 1.13 + 74%|███████████████████████████████████████████████████████████████████████████████████████████████████████████████████▎ | 17349/23458 [5:02:14<1:27:02, 1.17it/s][2025-04-23 17:27:15] (step=0017350) Train Loss: 5.4521, Train Steps/Sec: 1.17 + 74%|███████████████████████████████████████████████████████████████████████████████████████████████████████████████████▌ | 17374/23458 [5:02:35<1:25:51, 1.18it/s][2025-04-23 17:27:36] (step=0017375) Train Loss: 5.5857, Train Steps/Sec: 1.18 + 74%|███████████████████████████████████████████████████████████████████████████████████████████████████████████████████▋ | 17399/23458 [5:02:56<1:25:01, 1.19it/s][2025-04-23 17:27:58] (step=0017400) Train Loss: 5.4588, Train Steps/Sec: 1.17 + 74%|███████████████████████████████████████████████████████████████████████████████████████████████████████████████████▊ | 17424/23458 [5:03:17<1:27:05, 1.15it/s][2025-04-23 17:28:19] (step=0017425) Train Loss: 5.4749, Train Steps/Sec: 1.18 + 74%|████████████████████████████████████████████████████████████████████████████████████████████████████████████████████ | 17449/23458 [5:03:39<1:25:09, 1.18it/s][2025-04-23 17:28:40] (step=0017450) Train Loss: 5.5341, Train Steps/Sec: 1.18 + 74%|████████████████████████████████████████████████████████████████████████████████████████████████████████████████████▏ | 17474/23458 [5:04:00<1:24:31, 1.18it/s][2025-04-23 17:29:01] (step=0017475) Train Loss: 5.5657, Train Steps/Sec: 1.18 + 75%|████████████████████████████████████████████████████████████████████████████████████████████████████████████████████▎ | 17499/23458 [5:04:21<1:23:31, 1.19it/s][2025-04-23 17:29:23] (step=0017500) Train Loss: 5.5260, Train Steps/Sec: 1.17 + 75%|████████████████████████████████████████████████████████████████████████████████████████████████████████████████████▌ | 17524/23458 [5:04:43<1:25:43, 1.15it/s][2025-04-23 17:29:44] (step=0017525) Train Loss: 5.5095, Train Steps/Sec: 1.17 + 75%|████████████████████████████████████████████████████████████████████████████████████████████████████████████████████▋ | 17549/23458 [5:05:04<1:23:49, 1.17it/s][2025-04-23 17:30:05] (step=0017550) Train Loss: 5.5808, Train Steps/Sec: 1.18 + 75%|████████████████████████████████████████████████████████████████████████████████████████████████████████████████████▊ | 17574/23458 [5:05:25<1:23:01, 1.18it/s][2025-04-23 17:30:27] (step=0017575) Train Loss: 5.4506, Train Steps/Sec: 1.17 + 75%|█████████████████████████████████████████████████████████████████████████████████████████████████████████████████████ | 17599/23458 [5:05:47<1:22:00, 1.19it/s][2025-04-23 17:30:48] (step=0017600) Train Loss: 5.5192, Train Steps/Sec: 1.17 + 75%|█████████████████████████████████████████████████████████████████████████████████████████████████████████████████████▏ | 17624/23458 [5:06:08<1:24:28, 1.15it/s][2025-04-23 17:31:09] (step=0017625) Train Loss: 5.4927, Train Steps/Sec: 1.18 + 75%|█████████████████████████████████████████████████████████████████████████████████████████████████████████████████████▎ | 17649/23458 [5:06:29<1:22:44, 1.17it/s][2025-04-23 17:31:31] (step=0017650) Train Loss: 5.5119, Train Steps/Sec: 1.17 + 75%|█████████████████████████████████████████████████████████████████████████████████████████████████████████████████████▌ | 17674/23458 [5:06:50<1:21:36, 1.18it/s][2025-04-23 17:31:52] (step=0017675) Train Loss: 5.4708, Train Steps/Sec: 1.17 + 75%|█████████████████████████████████████████████████████████████████████████████████████████████████████████████████████▋ | 17699/23458 [5:07:13<1:21:53, 1.17it/s][2025-04-23 17:32:15] (step=0017700) Train Loss: 5.4813, Train Steps/Sec: 1.09 + 76%|█████████████████████████████████████████████████████████████████████████████████████████████████████████████████████▊ | 17724/23458 [5:07:35<1:22:57, 1.15it/s][2025-04-23 17:32:36] (step=0017725) Train Loss: 5.5356, Train Steps/Sec: 1.17 + 76%|██████████████████████████████████████████████████████████████████████████████████████████████████████████████████████ | 17749/23458 [5:07:57<1:21:02, 1.17it/s][2025-04-23 17:32:58] (step=0017750) Train Loss: 5.4586, Train Steps/Sec: 1.13 + 76%|██████████████████████████████████████████████████████████████████████████████████████████████████████████████████████▏ | 17774/23458 [5:08:19<1:20:10, 1.18it/s][2025-04-23 17:33:20] (step=0017775) Train Loss: 5.4935, Train Steps/Sec: 1.13 + 76%|██████████████████████████████████████████████████████████████████████████████████████████████████████████████████████▎ | 17799/23458 [5:08:41<1:28:51, 1.06it/s][2025-04-23 17:33:43] (step=0017800) Train Loss: 5.5303, Train Steps/Sec: 1.12 + 76%|██████████████████████████████████████████████████████████████████████████████████████████████████████████████████████▌ | 17824/23458 [5:09:03<1:28:37, 1.06it/s][2025-04-23 17:34:05] (step=0017825) Train Loss: 5.5399, Train Steps/Sec: 1.14 + 76%|██████████████████████████████████████████████████████████████████████████████████████████████████████████████████████▋ | 17849/23458 [5:09:25<1:19:30, 1.18it/s][2025-04-23 17:34:26] (step=0017850) Train Loss: 5.4776, Train Steps/Sec: 1.18 + 76%|██████████████████████████████████████████████████████████████████████████████████████████████████████████████████████▊ | 17874/23458 [5:09:46<1:18:38, 1.18it/s][2025-04-23 17:34:48] (step=0017875) Train Loss: 5.4696, Train Steps/Sec: 1.13 + 76%|███████████████████████████████████████████████████████████████████████████████████████████████████████████████████████ | 17899/23458 [5:10:08<1:17:46, 1.19it/s][2025-04-23 17:35:10] (step=0017900) Train Loss: 5.4982, Train Steps/Sec: 1.17 + 76%|███████████████████████████████████████████████████████████████████████████████████████████████████████████████████████▏ | 17924/23458 [5:10:29<1:19:47, 1.16it/s][2025-04-23 17:35:31] (step=0017925) Train Loss: 5.5126, Train Steps/Sec: 1.18 + 77%|███████████████████████████████████████████████████████████████████████████████████████████████████████████████████████▎ | 17949/23458 [5:10:51<1:18:21, 1.17it/s][2025-04-23 17:35:52] (step=0017950) Train Loss: 5.4615, Train Steps/Sec: 1.17 + 77%|███████████████████████████████████████████████████████████████████████████████████████████████████████████████████████▌ | 17974/23458 [5:11:12<1:17:13, 1.18it/s][2025-04-23 17:36:14] (step=0017975) Train Loss: 5.4229, Train Steps/Sec: 1.13 + 77%|███████████████████████████████████████████████████████████████████████████████████████████████████████████████████████▋ | 17999/23458 [5:11:34<1:16:24, 1.19it/s][2025-04-23 17:36:36] (step=0018000) Train Loss: 5.4664, Train Steps/Sec: 1.17 +You are using a model of type blip-2 to instantiate a model of type instructblip. This is not supported for all configurations of models and can yield errors. +[2025-04-23 17:36:36] text_config is None. Initializing the text config with default values (`OPTConfig`). +model_name_or_path:/tmp/haozhezhao/model/blip2-flan-t5-xl +image_place_holder: +tokenizer length after expend 32101 +tokenizer length before expend 32100 +100%|███████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████| 4/4 [03:58<00:00, 59.54s/it] +[2025-04-23 17:41:42] Finish Eval in 18000 steps...████████████████████████████████████████████████████████████████████████████████████████████████| 4/4 [03:57<00:00, 59.26s/it] +processing left-padding... +processing left-padding... +processing left-padding... +processing left-padding... +[2025-04-23 17:42:01] Saved checkpoint to checkpoint/Afteri2i_100kckpt_LLAVAT5_empty_Fixed_just_segment_pretrain_stage3_dreambench_recap_400k_t2i_400k_flux_200k_midjourney_150k_recovery_150k_3e4_epoch2_half_rotated/005-GPT-XL/checkpoints/0018000.pt +[2025-04-23 17:42:03] Removed old checkpoint: checkpoint/Afteri2i_100kckpt_LLAVAT5_empty_Fixed_just_segment_pretrain_stage3_dreambench_recap_400k_t2i_400k_flux_200k_midjourney_150k_recovery_150k_3e4_epoch2_half_rotated/005-GPT-XL/checkpoints/0016000.pt + 77%|███████████████████████████████████████████████████████████████████████████████████████████████████████████████████████▊ | 18024/23458 [5:17:23<1:20:00, 1.13it/s][2025-04-23 17:42:24] (step=0018025) Train Loss: 5.4380, Train Steps/Sec: 0.07 + 77%|████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████ | 18049/23458 [5:17:44<1:16:42, 1.18it/s][2025-04-23 17:42:46] (step=0018050) Train Loss: 5.4936, Train Steps/Sec: 1.17 + 77%|████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████▏ | 18074/23458 [5:18:05<1:16:03, 1.18it/s][2025-04-23 17:43:07] (step=0018075) Train Loss: 5.4542, Train Steps/Sec: 1.17 + 77%|████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████▎ | 18099/23458 [5:18:27<1:15:16, 1.19it/s][2025-04-23 17:43:28] (step=0018100) Train Loss: 5.5214, Train Steps/Sec: 1.17 + 77%|████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████▌ | 18124/23458 [5:18:48<1:17:01, 1.15it/s][2025-04-23 17:43:50] (step=0018125) Train Loss: 5.4827, Train Steps/Sec: 1.18 + 77%|████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████▋ | 18149/23458 [5:19:09<1:15:19, 1.17it/s][2025-04-23 17:44:11] (step=0018150) Train Loss: 5.5650, Train Steps/Sec: 1.17 + 77%|████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████▊ | 18174/23458 [5:19:31<1:14:29, 1.18it/s][2025-04-23 17:44:32] (step=0018175) Train Loss: 5.3987, Train Steps/Sec: 1.17 + 78%|█████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████ | 18199/23458 [5:19:52<1:13:59, 1.18it/s][2025-04-23 17:44:54] (step=0018200) Train Loss: 5.5915, Train Steps/Sec: 1.17 + 78%|█████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████▏ | 18224/23458 [5:20:14<1:15:47, 1.15it/s][2025-04-23 17:45:15] (step=0018225) Train Loss: 5.5382, Train Steps/Sec: 1.17 + 78%|█████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████▎ | 18249/23458 [5:20:35<1:13:58, 1.17it/s][2025-04-23 17:45:36] (step=0018250) Train Loss: 5.5963, Train Steps/Sec: 1.18 + 78%|█████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████▌ | 18274/23458 [5:20:56<1:13:11, 1.18it/s][2025-04-23 17:45:57] (step=0018275) Train Loss: 5.3962, Train Steps/Sec: 1.17 + 78%|█████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████▋ | 18299/23458 [5:21:17<1:12:33, 1.19it/s][2025-04-23 17:46:19] (step=0018300) Train Loss: 5.5199, Train Steps/Sec: 1.17 + 78%|█████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████▊ | 18324/23458 [5:21:39<1:14:04, 1.16it/s][2025-04-23 17:46:40] (step=0018325) Train Loss: 5.4581, Train Steps/Sec: 1.18 + 78%|██████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████ | 18349/23458 [5:22:02<1:32:35, 1.09s/it][2025-04-23 17:47:03] (step=0018350) Train Loss: 5.4808, Train Steps/Sec: 1.09 + 78%|██████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████▏ | 18374/23458 [5:22:23<1:11:19, 1.19it/s][2025-04-23 17:47:24] (step=0018375) Train Loss: 5.4966, Train Steps/Sec: 1.18 + 78%|██████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████▎ | 18399/23458 [5:22:45<1:11:58, 1.17it/s][2025-04-23 17:47:47] (step=0018400) Train Loss: 5.5382, Train Steps/Sec: 1.12 + 79%|██████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████▌ | 18424/23458 [5:23:07<1:12:48, 1.15it/s][2025-04-23 17:48:09] (step=0018425) Train Loss: 5.4815, Train Steps/Sec: 1.14 + 79%|██████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████▋ | 18449/23458 [5:23:29<1:11:02, 1.18it/s][2025-04-23 17:48:30] (step=0018450) Train Loss: 5.3649, Train Steps/Sec: 1.18 + 79%|██████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████▊ | 18474/23458 [5:23:50<1:10:26, 1.18it/s][2025-04-23 17:48:51] (step=0018475) Train Loss: 5.4459, Train Steps/Sec: 1.17 + 79%|███████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████ | 18499/23458 [5:24:12<1:09:43, 1.19it/s][2025-04-23 17:49:13] (step=0018500) Train Loss: 5.4935, Train Steps/Sec: 1.13 + 79%|███████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████▏ | 18524/23458 [5:24:34<1:28:05, 1.07s/it][2025-04-23 17:49:36] (step=0018525) Train Loss: 5.4489, Train Steps/Sec: 1.12 + 79%|███████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████▎ | 18549/23458 [5:24:56<1:10:45, 1.16it/s][2025-04-23 17:49:57] (step=0018550) Train Loss: 5.4854, Train Steps/Sec: 1.17 + 79%|███████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████▌ | 18574/23458 [5:25:17<1:08:53, 1.18it/s][2025-04-23 17:50:18] (step=0018575) Train Loss: 5.4201, Train Steps/Sec: 1.17 + 79%|███████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████▋ | 18599/23458 [5:25:39<1:24:01, 1.04s/it][2025-04-23 17:50:41] (step=0018600) Train Loss: 5.5395, Train Steps/Sec: 1.12 + 79%|███████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████▊ | 18624/23458 [5:26:01<1:09:43, 1.16it/s][2025-04-23 17:51:02] (step=0018625) Train Loss: 5.5183, Train Steps/Sec: 1.18 + 79%|████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████ | 18649/23458 [5:26:23<1:18:40, 1.02it/s][2025-04-23 17:51:24] (step=0018650) Train Loss: 5.4414, Train Steps/Sec: 1.13 + 80%|████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████▏ | 18674/23458 [5:26:44<1:07:43, 1.18it/s][2025-04-23 17:51:45] (step=0018675) Train Loss: 5.3476, Train Steps/Sec: 1.17 + 80%|████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████▎ | 18699/23458 [5:27:05<1:07:00, 1.18it/s][2025-04-23 17:52:07] (step=0018700) Train Loss: 5.4791, Train Steps/Sec: 1.17 + 80%|████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████▌ | 18724/23458 [5:27:27<1:08:23, 1.15it/s][2025-04-23 17:52:28] (step=0018725) Train Loss: 5.4835, Train Steps/Sec: 1.17 + 80%|████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████▋ | 18749/23458 [5:27:48<1:06:55, 1.17it/s][2025-04-23 17:52:49] (step=0018750) Train Loss: 5.5659, Train Steps/Sec: 1.17 + 80%|████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████▊ | 18774/23458 [5:28:09<1:06:18, 1.18it/s][2025-04-23 17:53:11] (step=0018775) Train Loss: 5.5081, Train Steps/Sec: 1.17 + 80%|█████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████ | 18799/23458 [5:28:31<1:05:11, 1.19it/s][2025-04-23 17:53:32] (step=0018800) Train Loss: 5.5219, Train Steps/Sec: 1.17 + 80%|█████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████▏ | 18824/23458 [5:28:52<1:06:57, 1.15it/s][2025-04-23 17:53:53] (step=0018825) Train Loss: 5.5392, Train Steps/Sec: 1.18 + 80%|█████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████▎ | 18849/23458 [5:29:13<1:05:49, 1.17it/s][2025-04-23 17:54:15] (step=0018850) Train Loss: 5.4428, Train Steps/Sec: 1.18 + 80%|█████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████▌ | 18874/23458 [5:29:35<1:04:36, 1.18it/s][2025-04-23 17:54:36] (step=0018875) Train Loss: 5.4854, Train Steps/Sec: 1.18 + 81%|█████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████▋ | 18899/23458 [5:29:56<1:04:01, 1.19it/s][2025-04-23 17:54:57] (step=0018900) Train Loss: 5.4445, Train Steps/Sec: 1.17 + 81%|█████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████▊ | 18924/23458 [5:30:17<1:05:20, 1.16it/s][2025-04-23 17:55:19] (step=0018925) Train Loss: 5.4381, Train Steps/Sec: 1.18 + 81%|██████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████ | 18949/23458 [5:30:38<1:03:59, 1.17it/s][2025-04-23 17:55:40] (step=0018950) Train Loss: 5.4748, Train Steps/Sec: 1.17 + 81%|██████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████▏ | 18974/23458 [5:31:00<1:03:25, 1.18it/s][2025-04-23 17:56:01] (step=0018975) Train Loss: 5.5270, Train Steps/Sec: 1.17 + 81%|██████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████▎ | 18999/23458 [5:31:21<1:02:27, 1.19it/s][2025-04-23 17:56:22] (step=0019000) Train Loss: 5.4199, Train Steps/Sec: 1.17 + 81%|██████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████▌ | 19024/23458 [5:31:44<1:03:56, 1.16it/s][2025-04-23 17:56:45] (step=0019025) Train Loss: 5.5101, Train Steps/Sec: 1.09 + 81%|██████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████▋ | 19049/23458 [5:32:06<1:21:36, 1.11s/it][2025-04-23 17:57:08] (step=0019050) Train Loss: 5.5405, Train Steps/Sec: 1.13 + 81%|██████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████▊ | 19074/23458 [5:32:28<1:02:47, 1.16it/s][2025-04-23 17:57:30] (step=0019075) Train Loss: 5.5772, Train Steps/Sec: 1.13 + 81%|███████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████ | 19099/23458 [5:32:49<1:01:23, 1.18it/s][2025-04-23 17:57:51] (step=0019100) Train Loss: 5.5151, Train Steps/Sec: 1.17 + 82%|███████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████▏ | 19124/23458 [5:33:11<1:02:40, 1.15it/s][2025-04-23 17:58:12] (step=0019125) Train Loss: 5.4811, Train Steps/Sec: 1.17 + 82%|███████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████▎ | 19149/23458 [5:33:33<1:01:41, 1.16it/s][2025-04-23 17:58:34] (step=0019150) Train Loss: 5.4930, Train Steps/Sec: 1.13 + 82%|███████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████▌ | 19174/23458 [5:33:54<1:00:58, 1.17it/s][2025-04-23 17:58:56] (step=0019175) Train Loss: 5.5418, Train Steps/Sec: 1.17 + 82%|███████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████▋ | 19199/23458 [5:34:16<1:00:01, 1.18it/s][2025-04-23 17:59:17] (step=0019200) Train Loss: 5.4810, Train Steps/Sec: 1.17 + 82%|███████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████▊ | 19224/23458 [5:34:37<1:01:14, 1.15it/s][2025-04-23 17:59:38] (step=0019225) Train Loss: 5.4984, Train Steps/Sec: 1.17 + 82%|████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████ | 19249/23458 [5:34:59<1:09:53, 1.00it/s][2025-04-23 18:00:01] (step=0019250) Train Loss: 5.4334, Train Steps/Sec: 1.12 + 82%|█████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████▊ | 19274/23458 [5:35:21<59:10, 1.18it/s][2025-04-23 18:00:22] (step=0019275) Train Loss: 5.4749, Train Steps/Sec: 1.18 + 82%|█████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████▉ | 19299/23458 [5:35:42<58:16, 1.19it/s][2025-04-23 18:00:43] (step=0019300) Train Loss: 5.4974, Train Steps/Sec: 1.17 + 82%|██████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████▏ | 19324/23458 [5:36:04<59:53, 1.15it/s][2025-04-23 18:01:06] (step=0019325) Train Loss: 5.4649, Train Steps/Sec: 1.08 + 82%|██████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████▎ | 19349/23458 [5:36:26<58:26, 1.17it/s][2025-04-23 18:01:28] (step=0019350) Train Loss: 5.4927, Train Steps/Sec: 1.17 + 83%|██████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████▍ | 19374/23458 [5:36:48<57:38, 1.18it/s][2025-04-23 18:01:49] (step=0019375) Train Loss: 5.4684, Train Steps/Sec: 1.18 + 83%|██████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████▋ | 19399/23458 [5:37:09<56:53, 1.19it/s][2025-04-23 18:02:10] (step=0019400) Train Loss: 5.4934, Train Steps/Sec: 1.17 + 83%|██████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████▊ | 19424/23458 [5:37:30<58:11, 1.16it/s][2025-04-23 18:02:32] (step=0019425) Train Loss: 5.5249, Train Steps/Sec: 1.18 + 83%|██████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████▉ | 19449/23458 [5:37:52<56:53, 1.17it/s][2025-04-23 18:02:53] (step=0019450) Train Loss: 5.5430, Train Steps/Sec: 1.17 + 83%|███████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████▏ | 19474/23458 [5:38:13<56:04, 1.18it/s][2025-04-23 18:03:14] (step=0019475) Train Loss: 5.4646, Train Steps/Sec: 1.17 + 83%|███████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████▎ | 19499/23458 [5:38:34<55:26, 1.19it/s][2025-04-23 18:03:36] (step=0019500) Train Loss: 5.4541, Train Steps/Sec: 1.17 + 83%|███████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████▌ | 19524/23458 [5:38:55<56:51, 1.15it/s][2025-04-23 18:03:57] (step=0019525) Train Loss: 5.5072, Train Steps/Sec: 1.18 + 83%|███████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████▋ | 19549/23458 [5:39:17<55:45, 1.17it/s][2025-04-23 18:04:18] (step=0019550) Train Loss: 5.4671, Train Steps/Sec: 1.17 + 83%|███████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████▊ | 19574/23458 [5:39:38<54:50, 1.18it/s][2025-04-23 18:04:39] (step=0019575) Train Loss: 5.5037, Train Steps/Sec: 1.17 + 84%|████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████ | 19599/23458 [5:39:59<54:05, 1.19it/s][2025-04-23 18:05:01] (step=0019600) Train Loss: 5.4647, Train Steps/Sec: 1.17 + 84%|████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████▏ | 19624/23458 [5:40:21<55:32, 1.15it/s][2025-04-23 18:05:22] (step=0019625) Train Loss: 5.5186, Train Steps/Sec: 1.18 + 84%|████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████▎ | 19649/23458 [5:40:42<54:07, 1.17it/s][2025-04-23 18:05:43] (step=0019650) Train Loss: 5.5793, Train Steps/Sec: 1.18 + 84%|████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████▌ | 19674/23458 [5:41:05<54:39, 1.15it/s][2025-04-23 18:06:06] (step=0019675) Train Loss: 5.4259, Train Steps/Sec: 1.09 + 84%|████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████▋ | 19699/23458 [5:41:26<52:52, 1.18it/s][2025-04-23 18:06:28] (step=0019700) Train Loss: 5.5110, Train Steps/Sec: 1.17 + 84%|████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████▊ | 19724/23458 [5:41:49<53:44, 1.16it/s][2025-04-23 18:06:50] (step=0019725) Train Loss: 5.4663, Train Steps/Sec: 1.13 + 84%|█████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████ | 19749/23458 [5:42:11<52:46, 1.17it/s][2025-04-23 18:07:12] (step=0019750) Train Loss: 5.4128, Train Steps/Sec: 1.14 + 84%|█████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████▏ | 19774/23458 [5:42:32<51:48, 1.19it/s][2025-04-23 18:07:33] (step=0019775) Train Loss: 5.5712, Train Steps/Sec: 1.17 + 84%|█████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████▎ | 19799/23458 [5:42:54<53:42, 1.14it/s][2025-04-23 18:07:55] (step=0019800) Train Loss: 5.4101, Train Steps/Sec: 1.13 + 85%|█████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████▌ | 19824/23458 [5:43:15<52:33, 1.15it/s][2025-04-23 18:08:17] (step=0019825) Train Loss: 5.5493, Train Steps/Sec: 1.17 + 85%|█████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████▋ | 19849/23458 [5:43:37<51:35, 1.17it/s][2025-04-23 18:08:38] (step=0019850) Train Loss: 5.5030, Train Steps/Sec: 1.17 + 85%|█████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████▊ | 19874/23458 [5:43:58<50:34, 1.18it/s][2025-04-23 18:08:59] (step=0019875) Train Loss: 5.4184, Train Steps/Sec: 1.17 + 85%|██████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████ | 19899/23458 [5:44:19<49:57, 1.19it/s][2025-04-23 18:09:21] (step=0019900) Train Loss: 5.4748, Train Steps/Sec: 1.17 + 85%|██████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████▏ | 19924/23458 [5:44:40<50:51, 1.16it/s][2025-04-23 18:09:42] (step=0019925) Train Loss: 5.5153, Train Steps/Sec: 1.18 + 85%|██████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████▎ | 19949/23458 [5:45:02<49:45, 1.18it/s][2025-04-23 18:10:03] (step=0019950) Train Loss: 5.5157, Train Steps/Sec: 1.17 + 85%|██████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████▌ | 19974/23458 [5:45:25<57:22, 1.01it/s][2025-04-23 18:10:26] (step=0019975) Train Loss: 5.5104, Train Steps/Sec: 1.08 + 85%|██████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████▋ | 19999/23458 [5:45:46<48:28, 1.19it/s][2025-04-23 18:10:48] (step=0020000) Train Loss: 5.3854, Train Steps/Sec: 1.17 +You are using a model of type blip-2 to instantiate a model of type instructblip. This is not supported for all configurations of models and can yield errors. +[2025-04-23 18:10:48] text_config is None. Initializing the text config with default values (`OPTConfig`). +model_name_or_path:/tmp/haozhezhao/model/blip2-flan-t5-xl +image_place_holder: +tokenizer length after expend 32101 +tokenizer length before expend 32100 +100%|███████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████| 4/4 [03:55<00:00, 58.99s/it] +[2025-04-23 18:15:52] Finish Eval in 20000 steps...████████████████████████████████████████████████████████████████████████████████████████████████| 4/4 [03:55<00:00, 58.71s/it] +processing left-padding... +processing left-padding... +processing left-padding... +processing left-padding... +[2025-04-23 18:16:10] Saved checkpoint to checkpoint/Afteri2i_100kckpt_LLAVAT5_empty_Fixed_just_segment_pretrain_stage3_dreambench_recap_400k_t2i_400k_flux_200k_midjourney_150k_recovery_150k_3e4_epoch2_half_rotated/005-GPT-XL/checkpoints/0020000.pt +[2025-04-23 18:16:12] Removed old checkpoint: checkpoint/Afteri2i_100kckpt_LLAVAT5_empty_Fixed_just_segment_pretrain_stage3_dreambench_recap_400k_t2i_400k_flux_200k_midjourney_150k_recovery_150k_3e4_epoch2_half_rotated/005-GPT-XL/checkpoints/0018000.pt + 85%|██████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████▊ | 20024/23458 [5:51:32<50:37, 1.13it/s][2025-04-23 18:16:33] (step=0020025) Train Loss: 5.5148, Train Steps/Sec: 0.07 + 85%|███████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████ | 20049/23458 [5:51:53<48:28, 1.17it/s][2025-04-23 18:16:54] (step=0020050) Train Loss: 5.5381, Train Steps/Sec: 1.18 + 86%|███████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████▏ | 20074/23458 [5:52:15<47:52, 1.18it/s][2025-04-23 18:17:17] (step=0020075) Train Loss: 5.4998, Train Steps/Sec: 1.13 + 86%|███████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████▍ | 20099/23458 [5:52:36<47:19, 1.18it/s][2025-04-23 18:17:38] (step=0020100) Train Loss: 5.4355, Train Steps/Sec: 1.17 + 86%|███████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████▌ | 20124/23458 [5:52:58<48:10, 1.15it/s][2025-04-23 18:17:59] (step=0020125) Train Loss: 5.4976, Train Steps/Sec: 1.17 + 86%|███████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████▋ | 20149/23458 [5:53:19<46:53, 1.18it/s][2025-04-23 18:18:21] (step=0020150) Train Loss: 5.3721, Train Steps/Sec: 1.17 + 86%|███████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████▉ | 20174/23458 [5:53:40<46:16, 1.18it/s][2025-04-23 18:18:42] (step=0020175) Train Loss: 5.4104, Train Steps/Sec: 1.18 + 86%|████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████ | 20199/23458 [5:54:02<45:39, 1.19it/s][2025-04-23 18:19:03] (step=0020200) Train Loss: 5.5523, Train Steps/Sec: 1.17 + 86%|████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████▏ | 20224/23458 [5:54:23<46:50, 1.15it/s][2025-04-23 18:19:24] (step=0020225) Train Loss: 5.4362, Train Steps/Sec: 1.18 + 86%|████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████▍ | 20249/23458 [5:54:44<45:32, 1.17it/s][2025-04-23 18:19:46] (step=0020250) Train Loss: 5.3952, Train Steps/Sec: 1.18 + 86%|████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████▌ | 20274/23458 [5:55:06<44:48, 1.18it/s][2025-04-23 18:20:07] (step=0020275) Train Loss: 5.5176, Train Steps/Sec: 1.18 + 87%|████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████▋ | 20299/23458 [5:55:27<44:16, 1.19it/s][2025-04-23 18:20:28] (step=0020300) Train Loss: 5.4308, Train Steps/Sec: 1.17 + 87%|████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████▉ | 20324/23458 [5:55:49<45:21, 1.15it/s][2025-04-23 18:20:50] (step=0020325) Train Loss: 5.4276, Train Steps/Sec: 1.13 + 87%|█████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████ | 20349/23458 [5:56:11<44:05, 1.18it/s][2025-04-23 18:21:13] (step=0020350) Train Loss: 5.5634, Train Steps/Sec: 1.13 + 87%|█████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████▏ | 20374/23458 [5:56:33<43:48, 1.17it/s][2025-04-23 18:21:35] (step=0020375) Train Loss: 5.4865, Train Steps/Sec: 1.13 + 87%|█████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████▍ | 20399/23458 [5:56:56<43:12, 1.18it/s][2025-04-23 18:21:57] (step=0020400) Train Loss: 5.4668, Train Steps/Sec: 1.13 + 87%|█████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████▌ | 20424/23458 [5:57:17<43:51, 1.15it/s][2025-04-23 18:22:18] (step=0020425) Train Loss: 5.5218, Train Steps/Sec: 1.17 + 87%|█████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████▋ | 20449/23458 [5:57:38<42:49, 1.17it/s][2025-04-23 18:22:40] (step=0020450) Train Loss: 5.5229, Train Steps/Sec: 1.18 + 87%|█████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████▉ | 20474/23458 [5:58:00<42:44, 1.16it/s][2025-04-23 18:23:02] (step=0020475) Train Loss: 5.5165, Train Steps/Sec: 1.13 + 87%|██████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████ | 20499/23458 [5:58:22<41:35, 1.19it/s][2025-04-23 18:23:23] (step=0020500) Train Loss: 5.4731, Train Steps/Sec: 1.17 + 87%|██████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████▏ | 20524/23458 [5:58:43<42:16, 1.16it/s][2025-04-23 18:23:44] (step=0020525) Train Loss: 5.4485, Train Steps/Sec: 1.17 + 88%|██████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████▍ | 20549/23458 [5:59:04<41:17, 1.17it/s][2025-04-23 18:24:06] (step=0020550) Train Loss: 5.5158, Train Steps/Sec: 1.17 + 88%|██████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████▌ | 20574/23458 [5:59:26<40:43, 1.18it/s][2025-04-23 18:24:27] (step=0020575) Train Loss: 5.5741, Train Steps/Sec: 1.17 + 88%|██████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████▋ | 20599/23458 [5:59:47<39:59, 1.19it/s][2025-04-23 18:24:48] (step=0020600) Train Loss: 5.4617, Train Steps/Sec: 1.17 + 88%|██████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████▉ | 20624/23458 [6:00:08<41:04, 1.15it/s][2025-04-23 18:25:10] (step=0020625) Train Loss: 5.3876, Train Steps/Sec: 1.17 + 88%|███████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████ | 20649/23458 [6:00:30<39:58, 1.17it/s][2025-04-23 18:25:31] (step=0020650) Train Loss: 5.4394, Train Steps/Sec: 1.17 + 88%|███████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████▏ | 20674/23458 [6:00:52<39:15, 1.18it/s][2025-04-23 18:25:53] (step=0020675) Train Loss: 5.4998, Train Steps/Sec: 1.13 + 88%|███████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████▍ | 20699/23458 [6:01:14<39:02, 1.18it/s][2025-04-23 18:26:16] (step=0020700) Train Loss: 5.4903, Train Steps/Sec: 1.12 + 88%|███████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████▌ | 20724/23458 [6:01:35<39:34, 1.15it/s][2025-04-23 18:26:37] (step=0020725) Train Loss: 5.4613, Train Steps/Sec: 1.18 + 88%|███████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████▊ | 20749/23458 [6:01:57<38:26, 1.17it/s][2025-04-23 18:26:58] (step=0020750) Train Loss: 5.4745, Train Steps/Sec: 1.18 + 89%|███████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████▉ | 20774/23458 [6:02:18<37:52, 1.18it/s][2025-04-23 18:27:19] (step=0020775) Train Loss: 5.5120, Train Steps/Sec: 1.17 + 89%|████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████ | 20799/23458 [6:02:40<37:15, 1.19it/s][2025-04-23 18:27:42] (step=0020800) Train Loss: 5.5091, Train Steps/Sec: 1.12 + 89%|████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████▎ | 20824/23458 [6:03:02<38:04, 1.15it/s][2025-04-23 18:28:03] (step=0020825) Train Loss: 5.5535, Train Steps/Sec: 1.18 + 89%|████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████▍ | 20849/23458 [6:03:23<37:10, 1.17it/s][2025-04-23 18:28:24] (step=0020850) Train Loss: 5.4445, Train Steps/Sec: 1.17 + 89%|████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████▌ | 20874/23458 [6:03:44<36:30, 1.18it/s][2025-04-23 18:28:45] (step=0020875) Train Loss: 5.5180, Train Steps/Sec: 1.18 + 89%|████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████▊ | 20899/23458 [6:04:05<35:53, 1.19it/s][2025-04-23 18:29:07] (step=0020900) Train Loss: 5.5108, Train Steps/Sec: 1.17 + 89%|████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████▉ | 20924/23458 [6:04:27<36:44, 1.15it/s][2025-04-23 18:29:28] (step=0020925) Train Loss: 5.4860, Train Steps/Sec: 1.18 + 89%|█████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████ | 20949/23458 [6:04:48<35:31, 1.18it/s][2025-04-23 18:29:49] (step=0020950) Train Loss: 5.5766, Train Steps/Sec: 1.18 + 89%|█████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████▎ | 20974/23458 [6:05:10<42:39, 1.03s/it][2025-04-23 18:30:11] (step=0020975) Train Loss: 5.4739, Train Steps/Sec: 1.13 + 90%|█████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████▍ | 20999/23458 [6:05:32<34:37, 1.18it/s][2025-04-23 18:30:34] (step=0021000) Train Loss: 5.4721, Train Steps/Sec: 1.13 + 90%|█████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████▌ | 21024/23458 [6:05:55<42:24, 1.05s/it][2025-04-23 18:30:56] (step=0021025) Train Loss: 5.4161, Train Steps/Sec: 1.12 + 90%|█████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████▊ | 21049/23458 [6:06:17<44:24, 1.11s/it][2025-04-23 18:31:18] (step=0021050) Train Loss: 5.5278, Train Steps/Sec: 1.13 + 90%|█████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████▉ | 21074/23458 [6:06:38<33:38, 1.18it/s][2025-04-23 18:31:39] (step=0021075) Train Loss: 5.4090, Train Steps/Sec: 1.18 + 90%|██████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████ | 21099/23458 [6:06:59<33:10, 1.19it/s][2025-04-23 18:32:01] (step=0021100) Train Loss: 5.4616, Train Steps/Sec: 1.17 + 90%|██████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████▎ | 21124/23458 [6:07:21<33:46, 1.15it/s][2025-04-23 18:32:23] (step=0021125) Train Loss: 5.4332, Train Steps/Sec: 1.14 + 90%|██████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████▍ | 21149/23458 [6:07:43<32:50, 1.17it/s][2025-04-23 18:32:44] (step=0021150) Train Loss: 5.4067, Train Steps/Sec: 1.17 + 90%|██████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████▌ | 21174/23458 [6:08:04<32:16, 1.18it/s][2025-04-23 18:33:05] (step=0021175) Train Loss: 5.4750, Train Steps/Sec: 1.17 + 90%|██████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████▊ | 21199/23458 [6:08:25<31:37, 1.19it/s][2025-04-23 18:33:27] (step=0021200) Train Loss: 5.5103, Train Steps/Sec: 1.17 + 90%|██████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████▉ | 21224/23458 [6:08:47<32:14, 1.15it/s][2025-04-23 18:33:48] (step=0021225) Train Loss: 5.5817, Train Steps/Sec: 1.18 + 91%|███████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████ | 21249/23458 [6:09:08<31:22, 1.17it/s][2025-04-23 18:34:09] (step=0021250) Train Loss: 5.4150, Train Steps/Sec: 1.17 + 91%|███████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████▎ | 21274/23458 [6:09:29<30:48, 1.18it/s][2025-04-23 18:34:31] (step=0021275) Train Loss: 5.4253, Train Steps/Sec: 1.17 + 91%|███████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████▍ | 21299/23458 [6:09:50<30:17, 1.19it/s][2025-04-23 18:34:52] (step=0021300) Train Loss: 5.4396, Train Steps/Sec: 1.17 + 91%|███████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████▋ | 21324/23458 [6:10:13<30:54, 1.15it/s][2025-04-23 18:35:14] (step=0021325) Train Loss: 5.5064, Train Steps/Sec: 1.13 + 91%|███████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████▊ | 21349/23458 [6:10:34<29:58, 1.17it/s][2025-04-23 18:35:35] (step=0021350) Train Loss: 5.5072, Train Steps/Sec: 1.18 + 91%|███████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████▉ | 21374/23458 [6:10:55<29:24, 1.18it/s][2025-04-23 18:35:57] (step=0021375) Train Loss: 5.4767, Train Steps/Sec: 1.18 + 91%|████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████▏ | 21399/23458 [6:11:16<28:49, 1.19it/s][2025-04-23 18:36:18] (step=0021400) Train Loss: 5.4136, Train Steps/Sec: 1.17 + 91%|████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████▎ | 21424/23458 [6:11:39<29:22, 1.15it/s][2025-04-23 18:36:40] (step=0021425) Train Loss: 5.4315, Train Steps/Sec: 1.13 + 91%|████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████▍ | 21449/23458 [6:12:00<28:26, 1.18it/s][2025-04-23 18:37:01] (step=0021450) Train Loss: 5.5147, Train Steps/Sec: 1.18 + 92%|████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████▋ | 21474/23458 [6:12:21<27:56, 1.18it/s][2025-04-23 18:37:23] (step=0021475) Train Loss: 5.4653, Train Steps/Sec: 1.18 + 92%|████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████▊ | 21499/23458 [6:12:42<27:26, 1.19it/s][2025-04-23 18:37:44] (step=0021500) Train Loss: 5.4821, Train Steps/Sec: 1.17 + 92%|████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████▉ | 21524/23458 [6:13:05<27:50, 1.16it/s][2025-04-23 18:38:06] (step=0021525) Train Loss: 5.4439, Train Steps/Sec: 1.13 + 92%|█████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████▏ | 21549/23458 [6:13:26<27:04, 1.17it/s][2025-04-23 18:38:27] (step=0021550) Train Loss: 5.5194, Train Steps/Sec: 1.18 + 92%|█████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████▎ | 21574/23458 [6:13:47<26:34, 1.18it/s][2025-04-23 18:38:49] (step=0021575) Train Loss: 5.4654, Train Steps/Sec: 1.17 + 92%|█████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████▍ | 21599/23458 [6:14:09<26:00, 1.19it/s][2025-04-23 18:39:10] (step=0021600) Train Loss: 5.5274, Train Steps/Sec: 1.17 + 92%|█████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████▋ | 21624/23458 [6:14:30<26:28, 1.15it/s][2025-04-23 18:39:31] (step=0021625) Train Loss: 5.4841, Train Steps/Sec: 1.18 + 92%|█████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████▊ | 21649/23458 [6:14:53<33:48, 1.12s/it][2025-04-23 18:39:54] (step=0021650) Train Loss: 5.4985, Train Steps/Sec: 1.09 + 92%|█████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████▉ | 21674/23458 [6:15:14<25:11, 1.18it/s][2025-04-23 18:40:15] (step=0021675) Train Loss: 5.4527, Train Steps/Sec: 1.18 + 93%|██████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████▏ | 21699/23458 [6:15:36<24:44, 1.19it/s][2025-04-23 18:40:38] (step=0021700) Train Loss: 5.5049, Train Steps/Sec: 1.12 + 93%|██████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████▎ | 21724/23458 [6:15:59<25:06, 1.15it/s][2025-04-23 18:41:00] (step=0021725) Train Loss: 5.4813, Train Steps/Sec: 1.13 + 93%|██████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████▍ | 21749/23458 [6:16:20<24:16, 1.17it/s][2025-04-23 18:41:21] (step=0021750) Train Loss: 5.4673, Train Steps/Sec: 1.18 + 93%|██████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████▋ | 21774/23458 [6:16:42<24:25, 1.15it/s][2025-04-23 18:41:43] (step=0021775) Train Loss: 5.5992, Train Steps/Sec: 1.14 + 93%|██████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████▊ | 21799/23458 [6:17:03<23:12, 1.19it/s][2025-04-23 18:42:05] (step=0021800) Train Loss: 5.4940, Train Steps/Sec: 1.17 + 93%|██████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████▉ | 21824/23458 [6:17:24<23:46, 1.15it/s][2025-04-23 18:42:26] (step=0021825) Train Loss: 5.4572, Train Steps/Sec: 1.18 + 93%|███████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████▏ | 21849/23458 [6:17:46<22:49, 1.17it/s][2025-04-23 18:42:47] (step=0021850) Train Loss: 5.4755, Train Steps/Sec: 1.18 + 93%|███████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████▎ | 21874/23458 [6:18:07<22:21, 1.18it/s][2025-04-23 18:43:08] (step=0021875) Train Loss: 5.4363, Train Steps/Sec: 1.17 + 93%|███████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████▍ | 21899/23458 [6:18:28<21:50, 1.19it/s][2025-04-23 18:43:30] (step=0021900) Train Loss: 5.4521, Train Steps/Sec: 1.17 + 93%|███████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████▋ | 21924/23458 [6:18:50<22:08, 1.16it/s][2025-04-23 18:43:51] (step=0021925) Train Loss: 5.5288, Train Steps/Sec: 1.18 + 94%|███████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████▊ | 21949/23458 [6:19:11<21:24, 1.17it/s][2025-04-23 18:44:12] (step=0021950) Train Loss: 5.4991, Train Steps/Sec: 1.18 + 94%|████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████ | 21974/23458 [6:19:33<23:04, 1.07it/s][2025-04-23 18:44:34] (step=0021975) Train Loss: 5.4827, Train Steps/Sec: 1.13 + 94%|████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████▏ | 21999/23458 [6:19:54<20:24, 1.19it/s][2025-04-23 18:44:56] (step=0022000) Train Loss: 5.4695, Train Steps/Sec: 1.17 +You are using a model of type blip-2 to instantiate a model of type instructblip. This is not supported for all configurations of models and can yield errors. +[2025-04-23 18:44:56] text_config is None. Initializing the text config with default values (`OPTConfig`). +model_name_or_path:/tmp/haozhezhao/model/blip2-flan-t5-xl +image_place_holder: +tokenizer length after expend 32101 +tokenizer length before expend 32100 +100%|███████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████| 4/4 [03:56<00:00, 59.09s/it] +[2025-04-23 18:50:00] Finish Eval in 22000 steps...████████████████████████████████████████████████████████████████████████████████████████████████| 4/4 [03:56<00:00, 58.81s/it] +processing left-padding... +processing left-padding... +processing left-padding... +processing left-padding... +[2025-04-23 18:50:18] Saved checkpoint to checkpoint/Afteri2i_100kckpt_LLAVAT5_empty_Fixed_just_segment_pretrain_stage3_dreambench_recap_400k_t2i_400k_flux_200k_midjourney_150k_recovery_150k_3e4_epoch2_half_rotated/005-GPT-XL/checkpoints/0022000.pt +[2025-04-23 18:50:20] Removed old checkpoint: checkpoint/Afteri2i_100kckpt_LLAVAT5_empty_Fixed_just_segment_pretrain_stage3_dreambench_recap_400k_t2i_400k_flux_200k_midjourney_150k_recovery_150k_3e4_epoch2_half_rotated/005-GPT-XL/checkpoints/0020000.pt + 94%|████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████▎ | 22024/23458 [6:25:40<21:04, 1.13it/s][2025-04-23 18:50:41] (step=0022025) Train Loss: 5.4840, Train Steps/Sec: 0.07 + 94%|████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████▌ | 22049/23458 [6:26:01<20:04, 1.17it/s][2025-04-23 18:51:03] (step=0022050) Train Loss: 5.4934, Train Steps/Sec: 1.17 + 94%|████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████▋ | 22074/23458 [6:26:23<19:26, 1.19it/s][2025-04-23 18:51:24] (step=0022075) Train Loss: 5.4617, Train Steps/Sec: 1.18 + 94%|████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████▊ | 22099/23458 [6:26:44<19:01, 1.19it/s][2025-04-23 18:51:45] (step=0022100) Train Loss: 5.5155, Train Steps/Sec: 1.17 + 94%|█████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████ | 22124/23458 [6:27:05<19:15, 1.15it/s][2025-04-23 18:52:07] (step=0022125) Train Loss: 5.5130, Train Steps/Sec: 1.18 + 94%|█████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████▏ | 22149/23458 [6:27:27<18:35, 1.17it/s][2025-04-23 18:52:29] (step=0022150) Train Loss: 5.4719, Train Steps/Sec: 1.13 + 95%|█████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████▎ | 22174/23458 [6:27:49<18:07, 1.18it/s][2025-04-23 18:52:50] (step=0022175) Train Loss: 5.5081, Train Steps/Sec: 1.18 + 95%|█████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████▌ | 22199/23458 [6:28:10<17:39, 1.19it/s][2025-04-23 18:53:11] (step=0022200) Train Loss: 5.5269, Train Steps/Sec: 1.17 + 95%|█████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████▋ | 22224/23458 [6:28:31<17:46, 1.16it/s][2025-04-23 18:53:33] (step=0022225) Train Loss: 5.5090, Train Steps/Sec: 1.18 + 95%|█████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████▊ | 22249/23458 [6:28:53<17:07, 1.18it/s][2025-04-23 18:53:55] (step=0022250) Train Loss: 5.4276, Train Steps/Sec: 1.13 + 95%|██████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████ | 22274/23458 [6:29:15<16:42, 1.18it/s][2025-04-23 18:54:16] (step=0022275) Train Loss: 5.4669, Train Steps/Sec: 1.18 + 95%|██████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████▏ | 22299/23458 [6:29:37<16:12, 1.19it/s][2025-04-23 18:54:38] (step=0022300) Train Loss: 5.5430, Train Steps/Sec: 1.13 + 95%|██████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████▎ | 22324/23458 [6:29:59<16:22, 1.15it/s][2025-04-23 18:55:00] (step=0022325) Train Loss: 5.5019, Train Steps/Sec: 1.12 + 95%|██████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████▌ | 22349/23458 [6:30:21<16:33, 1.12it/s][2025-04-23 18:55:22] (step=0022350) Train Loss: 5.4668, Train Steps/Sec: 1.13 + 95%|██████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████▋ | 22374/23458 [6:30:43<16:00, 1.13it/s][2025-04-23 18:55:45] (step=0022375) Train Loss: 5.5145, Train Steps/Sec: 1.13 + 95%|██████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████▊ | 22399/23458 [6:31:04<14:48, 1.19it/s][2025-04-23 18:56:06] (step=0022400) Train Loss: 5.5035, Train Steps/Sec: 1.17 + 96%|███████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████ | 22424/23458 [6:31:26<14:55, 1.15it/s][2025-04-23 18:56:27] (step=0022425) Train Loss: 5.4885, Train Steps/Sec: 1.18 + 96%|███████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████▏ | 22449/23458 [6:31:48<14:18, 1.18it/s][2025-04-23 18:56:49] (step=0022450) Train Loss: 5.5200, Train Steps/Sec: 1.14 + 96%|███████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████▎ | 22474/23458 [6:32:09<13:51, 1.18it/s][2025-04-23 18:57:10] (step=0022475) Train Loss: 5.5014, Train Steps/Sec: 1.18 + 96%|███████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████▌ | 22499/23458 [6:32:30<13:27, 1.19it/s][2025-04-23 18:57:32] (step=0022500) Train Loss: 5.5108, Train Steps/Sec: 1.17 + 96%|███████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████▋ | 22524/23458 [6:32:52<13:28, 1.16it/s][2025-04-23 18:57:53] (step=0022525) Train Loss: 5.4219, Train Steps/Sec: 1.18 + 96%|███████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████▉ | 22549/23458 [6:33:13<12:53, 1.17it/s][2025-04-23 18:58:14] (step=0022550) Train Loss: 5.4895, Train Steps/Sec: 1.18 + 96%|████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████ | 22574/23458 [6:33:34<12:26, 1.18it/s][2025-04-23 18:58:35] (step=0022575) Train Loss: 5.4960, Train Steps/Sec: 1.18 + 96%|████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████▏ | 22599/23458 [6:33:55<12:05, 1.18it/s][2025-04-23 18:58:57] (step=0022600) Train Loss: 5.5473, Train Steps/Sec: 1.17 + 96%|████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████▍ | 22624/23458 [6:34:17<11:59, 1.16it/s][2025-04-23 18:59:18] (step=0022625) Train Loss: 5.4619, Train Steps/Sec: 1.18 + 97%|████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████▌ | 22649/23458 [6:34:38<11:29, 1.17it/s][2025-04-23 18:59:39] (step=0022650) Train Loss: 5.3881, Train Steps/Sec: 1.18 + 97%|████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████▋ | 22674/23458 [6:35:00<11:04, 1.18it/s][2025-04-23 19:00:01] (step=0022675) Train Loss: 5.5086, Train Steps/Sec: 1.13 + 97%|████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████▉ | 22699/23458 [6:35:21<10:38, 1.19it/s][2025-04-23 19:00:23] (step=0022700) Train Loss: 5.4392, Train Steps/Sec: 1.17 + 97%|█████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████ | 22724/23458 [6:35:43<10:34, 1.16it/s][2025-04-23 19:00:44] (step=0022725) Train Loss: 5.5065, Train Steps/Sec: 1.18 + 97%|█████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████▏ | 22749/23458 [6:36:04<10:03, 1.17it/s][2025-04-23 19:01:05] (step=0022750) Train Loss: 5.5123, Train Steps/Sec: 1.18 + 97%|█████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████▍ | 22774/23458 [6:36:25<09:37, 1.19it/s][2025-04-23 19:01:27] (step=0022775) Train Loss: 5.5216, Train Steps/Sec: 1.18 + 97%|█████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████▌ | 22799/23458 [6:36:47<09:14, 1.19it/s][2025-04-23 19:01:48] (step=0022800) Train Loss: 5.5073, Train Steps/Sec: 1.17 + 97%|█████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████▋ | 22824/23458 [6:37:08<09:07, 1.16it/s][2025-04-23 19:02:09] (step=0022825) Train Loss: 5.4362, Train Steps/Sec: 1.18 + 97%|█████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████▉ | 22849/23458 [6:37:29<08:37, 1.18it/s][2025-04-23 19:02:30] (step=0022850) Train Loss: 5.4648, Train Steps/Sec: 1.18 + 98%|██████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████ | 22874/23458 [6:37:51<08:12, 1.18it/s][2025-04-23 19:02:53] (step=0022875) Train Loss: 5.3853, Train Steps/Sec: 1.13 + 98%|██████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████▏ | 22899/23458 [6:38:13<07:49, 1.19it/s][2025-04-23 19:03:14] (step=0022900) Train Loss: 5.5334, Train Steps/Sec: 1.17 + 98%|██████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████▍ | 22924/23458 [6:38:34<07:43, 1.15it/s][2025-04-23 19:03:35] (step=0022925) Train Loss: 5.3682, Train Steps/Sec: 1.17 + 98%|██████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████▌ | 22949/23458 [6:38:56<08:15, 1.03it/s][2025-04-23 19:03:57] (step=0022950) Train Loss: 5.4453, Train Steps/Sec: 1.13 + 98%|██████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████▋ | 22974/23458 [6:39:19<06:55, 1.16it/s][2025-04-23 19:04:20] (step=0022975) Train Loss: 5.4295, Train Steps/Sec: 1.09 + 98%|██████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████▉ | 22999/23458 [6:39:40<06:25, 1.19it/s][2025-04-23 19:04:42] (step=0023000) Train Loss: 5.4528, Train Steps/Sec: 1.17 + 98%|███████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████ | 23024/23458 [6:40:02<06:15, 1.16it/s][2025-04-23 19:05:05] (step=0023025) Train Loss: 5.5646, Train Steps/Sec: 1.09 + 98%|███████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████▏ | 23049/23458 [6:40:25<05:49, 1.17it/s][2025-04-23 19:05:26] (step=0023050) Train Loss: 5.5188, Train Steps/Sec: 1.18 + 98%|███████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████▍ | 23074/23458 [6:40:46<05:23, 1.19it/s][2025-04-23 19:05:47] (step=0023075) Train Loss: 5.4824, Train Steps/Sec: 1.18 + 98%|███████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████▌ | 23099/23458 [6:41:08<05:04, 1.18it/s][2025-04-23 19:06:09] (step=0023100) Train Loss: 5.5094, Train Steps/Sec: 1.13 + 99%|███████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████▊ | 23124/23458 [6:41:29<04:49, 1.15it/s][2025-04-23 19:06:31] (step=0023125) Train Loss: 5.4275, Train Steps/Sec: 1.18 + 99%|███████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████▉ | 23149/23458 [6:41:50<04:23, 1.17it/s][2025-04-23 19:06:52] (step=0023150) Train Loss: 5.5436, Train Steps/Sec: 1.18 + 99%|████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████ | 23174/23458 [6:42:12<04:00, 1.18it/s][2025-04-23 19:07:13] (step=0023175) Train Loss: 5.5329, Train Steps/Sec: 1.18 + 99%|████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████▎ | 23199/23458 [6:42:33<03:37, 1.19it/s][2025-04-23 19:07:34] (step=0023200) Train Loss: 5.4307, Train Steps/Sec: 1.17 + 99%|████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████▍ | 23224/23458 [6:42:54<03:22, 1.16it/s][2025-04-23 19:07:56] (step=0023225) Train Loss: 5.5028, Train Steps/Sec: 1.18 + 99%|████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████▌ | 23249/23458 [6:43:16<02:58, 1.17it/s][2025-04-23 19:08:17] (step=0023250) Train Loss: 5.4717, Train Steps/Sec: 1.18 + 99%|████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████▊ | 23274/23458 [6:43:37<02:35, 1.18it/s][2025-04-23 19:08:38] (step=0023275) Train Loss: 5.4428, Train Steps/Sec: 1.18 + 99%|████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████▉ | 23299/23458 [6:43:58<02:13, 1.19it/s][2025-04-23 19:08:59] (step=0023300) Train Loss: 5.5185, Train Steps/Sec: 1.17 + 99%|█████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████ | 23324/23458 [6:44:20<01:56, 1.15it/s][2025-04-23 19:09:22] (step=0023325) Train Loss: 5.4731, Train Steps/Sec: 1.13 +100%|█████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████▎| 23349/23458 [6:44:42<01:32, 1.17it/s][2025-04-23 19:09:43] (step=0023350) Train Loss: 5.4346, Train Steps/Sec: 1.17 +100%|█████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████▍| 23374/23458 [6:45:03<01:10, 1.19it/s][2025-04-23 19:10:04] (step=0023375) Train Loss: 5.4568, Train Steps/Sec: 1.18 +100%|█████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████▌| 23399/23458 [6:45:24<00:49, 1.19it/s][2025-04-23 19:10:26] (step=0023400) Train Loss: 5.4664, Train Steps/Sec: 1.17 +100%|█████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████▊| 23424/23458 [6:45:45<00:29, 1.15it/s][2025-04-23 19:10:47] (step=0023425) Train Loss: 5.4016, Train Steps/Sec: 1.17 +100%|█████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████▉| 23449/23458 [6:46:07<00:07, 1.18it/s][2025-04-23 19:11:08] (step=0023450) Train Loss: 5.4485, Train Steps/Sec: 1.18 +100%|██████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████| 23458/23458 [6:46:15<00:00, 1.04s/it] +[2025-04-23 19:11:16] Beginning epoch 1... + 0%| | 16/23458 [00:16<5:40:45, 1.15it/s][2025-04-23 19:11:34] (step=0023475) Train Loss: 5.4174, Train Steps/Sec: 0.98 + 0%|▎ | 41/23458 [00:38<5:28:50, 1.19it/s][2025-04-23 19:11:55] (step=0023500) Train Loss: 5.3740, Train Steps/Sec: 1.16 + 0%|▍ | 66/23458 [00:59<5:38:49, 1.15it/s][2025-04-23 19:12:17] (step=0023525) Train Loss: 5.4172, Train Steps/Sec: 1.17 + 0%|▌ | 91/23458 [01:20<5:32:29, 1.17it/s][2025-04-23 19:12:38] (step=0023550) Train Loss: 5.3438, Train Steps/Sec: 1.18 + 0%|▋ | 116/23458 [01:42<5:30:06, 1.18it/s][2025-04-23 19:12:59] (step=0023575) Train Loss: 5.4250, Train Steps/Sec: 1.17 + 1%|▊ | 141/23458 [02:04<5:28:04, 1.18it/s][2025-04-23 19:13:22] (step=0023600) Train Loss: 5.4292, Train Steps/Sec: 1.11 + 1%|█ | 166/23458 [02:28<7:01:57, 1.09s/it][2025-04-23 19:13:45] (step=0023625) Train Loss: 5.5044, Train Steps/Sec: 1.07 + 1%|█▏ | 191/23458 [02:49<5:31:17, 1.17it/s][2025-04-23 19:14:06] (step=0023650) Train Loss: 5.3244, Train Steps/Sec: 1.17 + 1%|█▎ | 216/23458 [03:11<5:28:13, 1.18it/s][2025-04-23 19:14:29] (step=0023675) Train Loss: 5.4025, Train Steps/Sec: 1.12 + 1%|█▍ | 241/23458 [03:35<5:27:08, 1.18it/s][2025-04-23 19:14:52] (step=0023700) Train Loss: 5.3761, Train Steps/Sec: 1.06 + 1%|█▌ | 266/23458 [03:56<5:35:52, 1.15it/s][2025-04-23 19:15:14] (step=0023725) Train Loss: 5.4133, Train Steps/Sec: 1.17 + 1%|█▊ | 291/23458 [04:19<7:27:15, 1.16s/it][2025-04-23 19:15:36] (step=0023750) Train Loss: 5.3435, Train Steps/Sec: 1.12 + 1%|█▉ | 316/23458 [04:40<5:27:09, 1.18it/s][2025-04-23 19:15:57] (step=0023775) Train Loss: 5.4328, Train Steps/Sec: 1.17 + 1%|██ | 341/23458 [05:01<5:23:37, 1.19it/s][2025-04-23 19:16:19] (step=0023800) Train Loss: 5.4057, Train Steps/Sec: 1.17 + 2%|██▏ | 366/23458 [05:23<5:34:10, 1.15it/s][2025-04-23 19:16:40] (step=0023825) Train Loss: 5.4560, Train Steps/Sec: 1.17 + 2%|██▍ | 391/23458 [05:44<5:28:30, 1.17it/s][2025-04-23 19:17:01] (step=0023850) Train Loss: 5.3452, Train Steps/Sec: 1.18 + 2%|██▌ | 416/23458 [06:05<5:25:00, 1.18it/s][2025-04-23 19:17:23] (step=0023875) Train Loss: 5.3795, Train Steps/Sec: 1.17 + 2%|██▋ | 441/23458 [06:27<5:24:51, 1.18it/s][2025-04-23 19:17:44] (step=0023900) Train Loss: 5.3339, Train Steps/Sec: 1.16 + 2%|██▊ | 466/23458 [06:48<5:33:51, 1.15it/s][2025-04-23 19:18:05] (step=0023925) Train Loss: 5.4558, Train Steps/Sec: 1.17 + 2%|██▉ | 491/23458 [07:09<5:27:19, 1.17it/s][2025-04-23 19:18:27] (step=0023950) Train Loss: 5.3740, Train Steps/Sec: 1.17 + 2%|███▏ | 516/23458 [07:32<6:03:19, 1.05it/s][2025-04-23 19:18:49] (step=0023975) Train Loss: 5.3980, Train Steps/Sec: 1.12 + 2%|███▎ | 541/23458 [07:53<5:23:19, 1.18it/s][2025-04-23 19:19:10] (step=0024000) Train Loss: 5.4179, Train Steps/Sec: 1.17 +You are using a model of type blip-2 to instantiate a model of type instructblip. This is not supported for all configurations of models and can yield errors. +[2025-04-23 19:19:10] text_config is None. Initializing the text config with default values (`OPTConfig`). +model_name_or_path:/tmp/haozhezhao/model/blip2-flan-t5-xl +image_place_holder: +tokenizer length after expend 32101 +tokenizer length before expend 32100 +100%|███████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████| 4/4 [03:59<00:00, 59.95s/it] +[2025-04-23 19:24:18] Finish Eval in 24000 steps...████████████████████████████████████████████████████████████████████████████████████████████████| 4/4 [03:59<00:00, 59.50s/it] +processing left-padding... +processing left-padding... +processing left-padding... +processing left-padding... +[2025-04-23 19:24:37] Saved checkpoint to checkpoint/Afteri2i_100kckpt_LLAVAT5_empty_Fixed_just_segment_pretrain_stage3_dreambench_recap_400k_t2i_400k_flux_200k_midjourney_150k_recovery_150k_3e4_epoch2_half_rotated/005-GPT-XL/checkpoints/0024000.pt +[2025-04-23 19:24:39] Removed old checkpoint: checkpoint/Afteri2i_100kckpt_LLAVAT5_empty_Fixed_just_segment_pretrain_stage3_dreambench_recap_400k_t2i_400k_flux_200k_midjourney_150k_recovery_150k_3e4_epoch2_half_rotated/005-GPT-XL/checkpoints/0022000.pt + 2%|███▍ | 566/23458 [13:43<5:38:16, 1.13it/s][2025-04-23 19:25:00] (step=0024025) Train Loss: 5.3983, Train Steps/Sec: 0.07 + 3%|███▌ | 591/23458 [14:04<5:25:41, 1.17it/s][2025-04-23 19:25:21] (step=0024050) Train Loss: 5.3482, Train Steps/Sec: 1.18 + 3%|███▊ | 616/23458 [14:25<5:21:19, 1.18it/s][2025-04-23 19:25:43] (step=0024075) Train Loss: 5.3873, Train Steps/Sec: 1.18 + 3%|███▉ | 641/23458 [14:47<5:21:14, 1.18it/s][2025-04-23 19:26:04] (step=0024100) Train Loss: 5.3387, Train Steps/Sec: 1.16 + 3%|████ | 666/23458 [15:08<5:31:11, 1.15it/s][2025-04-23 19:26:25] (step=0024125) Train Loss: 5.3996, Train Steps/Sec: 1.18 + 3%|████▏ | 691/23458 [15:29<5:24:29, 1.17it/s][2025-04-23 19:26:47] (step=0024150) Train Loss: 5.3409, Train Steps/Sec: 1.17 + 3%|████▎ | 716/23458 [15:51<5:21:02, 1.18it/s][2025-04-23 19:27:08] (step=0024175) Train Loss: 5.4733, Train Steps/Sec: 1.17 + 3%|████▌ | 741/23458 [16:12<5:19:02, 1.19it/s][2025-04-23 19:27:29] (step=0024200) Train Loss: 5.3975, Train Steps/Sec: 1.17 + 3%|████▋ | 766/23458 [16:33<5:27:28, 1.15it/s][2025-04-23 19:27:51] (step=0024225) Train Loss: 5.4183, Train Steps/Sec: 1.17 + 3%|████▊ | 791/23458 [16:55<5:22:34, 1.17it/s][2025-04-23 19:28:12] (step=0024250) Train Loss: 5.3978, Train Steps/Sec: 1.17 + 3%|████▉ | 816/23458 [17:17<5:19:55, 1.18it/s][2025-04-23 19:28:34] (step=0024275) Train Loss: 5.3387, Train Steps/Sec: 1.13 + 4%|█████▏ | 841/23458 [17:39<5:16:53, 1.19it/s][2025-04-23 19:28:56] (step=0024300) Train Loss: 5.4247, Train Steps/Sec: 1.12 + 4%|█████▎ | 866/23458 [18:03<5:55:41, 1.06it/s][2025-04-23 19:29:20] (step=0024325) Train Loss: 5.4288, Train Steps/Sec: 1.06 + 4%|█████▍ | 891/23458 [18:25<5:43:35, 1.09it/s][2025-04-23 19:29:42] (step=0024350) Train Loss: 5.4009, Train Steps/Sec: 1.13 + 4%|█████▌ | 916/23458 [18:46<5:17:57, 1.18it/s][2025-04-23 19:30:03] (step=0024375) Train Loss: 5.3887, Train Steps/Sec: 1.17 + 4%|█████▋ | 941/23458 [19:07<5:16:34, 1.19it/s][2025-04-23 19:30:25] (step=0024400) Train Loss: 5.3427, Train Steps/Sec: 1.17 + 4%|█████▉ | 966/23458 [19:31<5:25:27, 1.15it/s][2025-04-23 19:30:48] (step=0024425) Train Loss: 5.3827, Train Steps/Sec: 1.08 + 4%|██████ | 991/23458 [19:52<5:20:47, 1.17it/s][2025-04-23 19:31:09] (step=0024450) Train Loss: 5.4196, Train Steps/Sec: 1.17 + 4%|██████▏ | 1016/23458 [20:13<5:18:06, 1.18it/s][2025-04-23 19:31:31] (step=0024475) Train Loss: 5.4046, Train Steps/Sec: 1.17 + 4%|██████▎ | 1041/23458 [20:34<5:15:01, 1.19it/s][2025-04-23 19:31:52] (step=0024500) Train Loss: 5.3989, Train Steps/Sec: 1.17 + 5%|██████▍ | 1066/23458 [20:56<5:25:44, 1.15it/s][2025-04-23 19:32:13] (step=0024525) Train Loss: 5.4122, Train Steps/Sec: 1.17 + 5%|██████▌ | 1091/23458 [21:17<5:17:53, 1.17it/s][2025-04-23 19:32:35] (step=0024550) Train Loss: 5.4750, Train Steps/Sec: 1.17 + 5%|██████▊ | 1116/23458 [21:39<5:15:01, 1.18it/s][2025-04-23 19:32:56] (step=0024575) Train Loss: 5.3793, Train Steps/Sec: 1.17 + 5%|██████▉ | 1141/23458 [22:00<5:13:13, 1.19it/s][2025-04-23 19:33:17] (step=0024600) Train Loss: 5.3428, Train Steps/Sec: 1.17 + 5%|███████ | 1166/23458 [22:21<5:21:13, 1.16it/s][2025-04-23 19:33:38] (step=0024625) Train Loss: 5.3406, Train Steps/Sec: 1.18 + 5%|███████▏ | 1191/23458 [22:43<6:24:18, 1.04s/it][2025-04-23 19:34:01] (step=0024650) Train Loss: 5.3333, Train Steps/Sec: 1.13 + 5%|███████▎ | 1216/23458 [23:05<5:12:53, 1.18it/s][2025-04-23 19:34:22] (step=0024675) Train Loss: 5.3736, Train Steps/Sec: 1.18 + 5%|███████▌ | 1241/23458 [23:26<5:13:09, 1.18it/s][2025-04-23 19:34:43] (step=0024700) Train Loss: 5.3979, Train Steps/Sec: 1.17 + 5%|███████▋ | 1266/23458 [23:47<5:19:32, 1.16it/s][2025-04-23 19:35:05] (step=0024725) Train Loss: 5.3841, Train Steps/Sec: 1.17 + 6%|███████▊ | 1291/23458 [24:09<5:19:06, 1.16it/s][2025-04-23 19:35:26] (step=0024750) Train Loss: 5.4464, Train Steps/Sec: 1.17 + 6%|███████▉ | 1316/23458 [24:30<5:13:07, 1.18it/s][2025-04-23 19:35:47] (step=0024775) Train Loss: 5.3683, Train Steps/Sec: 1.17 + 6%|████████ | 1341/23458 [24:51<5:09:57, 1.19it/s][2025-04-23 19:36:09] (step=0024800) Train Loss: 5.4800, Train Steps/Sec: 1.17 + 6%|████████▎ | 1366/23458 [25:13<5:19:54, 1.15it/s][2025-04-23 19:36:30] (step=0024825) Train Loss: 5.4058, Train Steps/Sec: 1.17 + 6%|████████▍ | 1391/23458 [25:34<5:14:41, 1.17it/s][2025-04-23 19:36:51] (step=0024850) Train Loss: 5.3878, Train Steps/Sec: 1.17 + 6%|████████▌ | 1416/23458 [25:55<5:11:12, 1.18it/s][2025-04-23 19:37:13] (step=0024875) Train Loss: 5.4238, Train Steps/Sec: 1.17 + 6%|████████▋ | 1441/23458 [26:17<5:09:25, 1.19it/s][2025-04-23 19:37:34] (step=0024900) Train Loss: 5.3983, Train Steps/Sec: 1.17 + 6%|████████▊ | 1466/23458 [26:39<5:25:47, 1.13it/s][2025-04-23 19:37:56] (step=0024925) Train Loss: 5.4086, Train Steps/Sec: 1.13 + 6%|█████████ | 1491/23458 [27:01<5:13:58, 1.17it/s][2025-04-23 19:38:18] (step=0024950) Train Loss: 5.4784, Train Steps/Sec: 1.13 + 6%|█████████▏ | 1516/23458 [27:22<5:09:22, 1.18it/s][2025-04-23 19:38:40] (step=0024975) Train Loss: 5.3757, Train Steps/Sec: 1.17 + 7%|█████████▎ | 1541/23458 [27:44<5:09:01, 1.18it/s][2025-04-23 19:39:02] (step=0025000) Train Loss: 5.4066, Train Steps/Sec: 1.13 + 7%|█████████▍ | 1566/23458 [28:07<5:16:25, 1.15it/s][2025-04-23 19:39:24] (step=0025025) Train Loss: 5.4579, Train Steps/Sec: 1.13 + 7%|█████████▋ | 1591/23458 [28:29<5:09:38, 1.18it/s][2025-04-23 19:39:46] (step=0025050) Train Loss: 5.3656, Train Steps/Sec: 1.12 + 7%|█████████▊ | 1616/23458 [28:51<5:13:23, 1.16it/s][2025-04-23 19:40:08] (step=0025075) Train Loss: 5.4259, Train Steps/Sec: 1.13 + 7%|█████████▉ | 1641/23458 [29:12<5:05:59, 1.19it/s][2025-04-23 19:40:30] (step=0025100) Train Loss: 5.3540, Train Steps/Sec: 1.17 + 7%|██████████ | 1666/23458 [29:34<5:13:58, 1.16it/s][2025-04-23 19:40:51] (step=0025125) Train Loss: 5.3896, Train Steps/Sec: 1.17 + 7%|██████████▏ | 1691/23458 [29:56<5:10:03, 1.17it/s][2025-04-23 19:41:13] (step=0025150) Train Loss: 5.4091, Train Steps/Sec: 1.12 + 7%|██████████▍ | 1716/23458 [30:17<5:05:55, 1.18it/s][2025-04-23 19:41:35] (step=0025175) Train Loss: 5.3545, Train Steps/Sec: 1.17 + 7%|██████████▌ | 1741/23458 [30:39<5:04:42, 1.19it/s][2025-04-23 19:41:56] (step=0025200) Train Loss: 5.4128, Train Steps/Sec: 1.17 + 8%|██████████▋ | 1766/23458 [31:00<5:14:08, 1.15it/s][2025-04-23 19:42:17] (step=0025225) Train Loss: 5.3742, Train Steps/Sec: 1.17 + 8%|██████████▊ | 1791/23458 [31:21<5:07:37, 1.17it/s][2025-04-23 19:42:39] (step=0025250) Train Loss: 5.3809, Train Steps/Sec: 1.18 + 8%|██████████▉ | 1816/23458 [31:43<5:04:45, 1.18it/s][2025-04-23 19:43:00] (step=0025275) Train Loss: 5.3863, Train Steps/Sec: 1.17 + 8%|███████████▏ | 1841/23458 [32:04<5:04:39, 1.18it/s][2025-04-23 19:43:21] (step=0025300) Train Loss: 5.4147, Train Steps/Sec: 1.17 + 8%|███████████▎ | 1866/23458 [32:26<5:11:35, 1.15it/s][2025-04-23 19:43:43] (step=0025325) Train Loss: 5.3152, Train Steps/Sec: 1.13 + 8%|███████████▍ | 1891/23458 [32:47<5:07:13, 1.17it/s][2025-04-23 19:44:05] (step=0025350) Train Loss: 5.4485, Train Steps/Sec: 1.17 + 8%|███████████▌ | 1916/23458 [33:09<5:04:26, 1.18it/s][2025-04-23 19:44:26] (step=0025375) Train Loss: 5.3609, Train Steps/Sec: 1.17 + 8%|███████████▋ | 1941/23458 [33:30<5:04:18, 1.18it/s][2025-04-23 19:44:47] (step=0025400) Train Loss: 5.4033, Train Steps/Sec: 1.17 + 8%|███████████▉ | 1966/23458 [33:52<5:10:47, 1.15it/s][2025-04-23 19:45:09] (step=0025425) Train Loss: 5.4181, Train Steps/Sec: 1.17 + 8%|████████████ | 1991/23458 [34:13<5:04:39, 1.17it/s][2025-04-23 19:45:30] (step=0025450) Train Loss: 5.4180, Train Steps/Sec: 1.17 + 9%|████████████▏ | 2016/23458 [34:34<5:03:00, 1.18it/s][2025-04-23 19:45:51] (step=0025475) Train Loss: 5.3186, Train Steps/Sec: 1.17 + 9%|████████████▎ | 2041/23458 [34:55<5:00:18, 1.19it/s][2025-04-23 19:46:13] (step=0025500) Train Loss: 5.4063, Train Steps/Sec: 1.17 + 9%|████████████▌ | 2066/23458 [35:17<5:10:30, 1.15it/s][2025-04-23 19:46:34] (step=0025525) Train Loss: 5.4142, Train Steps/Sec: 1.17 + 9%|████████████▋ | 2091/23458 [35:38<5:05:09, 1.17it/s][2025-04-23 19:46:55] (step=0025550) Train Loss: 5.3491, Train Steps/Sec: 1.17 + 9%|████████████▊ | 2116/23458 [36:00<5:59:00, 1.01s/it][2025-04-23 19:47:18] (step=0025575) Train Loss: 5.4041, Train Steps/Sec: 1.13 + 9%|████████████▉ | 2141/23458 [36:22<6:25:37, 1.09s/it][2025-04-23 19:47:40] (step=0025600) Train Loss: 5.3443, Train Steps/Sec: 1.13 + 9%|█████████████ | 2166/23458 [36:44<5:07:28, 1.15it/s][2025-04-23 19:48:01] (step=0025625) Train Loss: 5.4332, Train Steps/Sec: 1.18 + 9%|█████████████▎ | 2191/23458 [37:06<5:02:10, 1.17it/s][2025-04-23 19:48:23] (step=0025650) Train Loss: 5.4041, Train Steps/Sec: 1.13 + 9%|█████████████▍ | 2216/23458 [37:28<5:01:21, 1.17it/s][2025-04-23 19:48:45] (step=0025675) Train Loss: 5.3836, Train Steps/Sec: 1.13 + 10%|█████████████▌ | 2241/23458 [37:49<4:59:46, 1.18it/s][2025-04-23 19:49:07] (step=0025700) Train Loss: 5.4514, Train Steps/Sec: 1.17 + 10%|█████████████▋ | 2266/23458 [38:11<5:07:25, 1.15it/s][2025-04-23 19:49:28] (step=0025725) Train Loss: 5.3894, Train Steps/Sec: 1.17 + 10%|█████████████▊ | 2291/23458 [38:33<5:00:53, 1.17it/s][2025-04-23 19:49:50] (step=0025750) Train Loss: 5.3401, Train Steps/Sec: 1.13 + 10%|██████████████ | 2316/23458 [38:55<5:00:26, 1.17it/s][2025-04-23 19:50:12] (step=0025775) Train Loss: 5.4372, Train Steps/Sec: 1.12 + 10%|██████████████▏ | 2341/23458 [39:16<4:55:32, 1.19it/s][2025-04-23 19:50:34] (step=0025800) Train Loss: 5.3973, Train Steps/Sec: 1.17 + 10%|██████████████▎ | 2366/23458 [39:38<5:05:16, 1.15it/s][2025-04-23 19:50:55] (step=0025825) Train Loss: 5.3967, Train Steps/Sec: 1.17 + 10%|██████████████▍ | 2391/23458 [39:59<4:59:38, 1.17it/s][2025-04-23 19:51:16] (step=0025850) Train Loss: 5.3730, Train Steps/Sec: 1.17 + 10%|██████████████▌ | 2416/23458 [40:21<4:56:26, 1.18it/s][2025-04-23 19:51:39] (step=0025875) Train Loss: 5.4737, Train Steps/Sec: 1.12 + 10%|██████████████▊ | 2441/23458 [40:43<4:55:54, 1.18it/s][2025-04-23 19:52:00] (step=0025900) Train Loss: 5.4434, Train Steps/Sec: 1.17 + 11%|██████████████▉ | 2466/23458 [41:04<5:03:38, 1.15it/s][2025-04-23 19:52:21] (step=0025925) Train Loss: 5.3944, Train Steps/Sec: 1.17 + 11%|███████████████ | 2491/23458 [41:25<4:59:22, 1.17it/s][2025-04-23 19:52:43] (step=0025950) Train Loss: 5.3865, Train Steps/Sec: 1.17 + 11%|███████████████▏ | 2516/23458 [41:47<5:05:37, 1.14it/s][2025-04-23 19:53:05] (step=0025975) Train Loss: 5.3950, Train Steps/Sec: 1.13 + 11%|███████████████▍ | 2541/23458 [42:09<4:53:07, 1.19it/s][2025-04-23 19:53:26] (step=0026000) Train Loss: 5.3654, Train Steps/Sec: 1.17 +You are using a model of type blip-2 to instantiate a model of type instructblip. This is not supported for all configurations of models and can yield errors. +[2025-04-23 19:53:26] text_config is None. Initializing the text config with default values (`OPTConfig`). +model_name_or_path:/tmp/haozhezhao/model/blip2-flan-t5-xl +image_place_holder: +tokenizer length after expend 32101 +tokenizer length before expend 32100 +100%|███████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████| 4/4 [03:57<00:00, 59.47s/it] +[2025-04-23 19:58:33] Finish Eval in 26000 steps...████████████████████████████████████████████████████████████████████████████████████████████████| 4/4 [03:57<00:00, 59.16s/it] +processing left-padding... +processing left-padding... +processing left-padding... +processing left-padding... +[2025-04-23 19:58:52] Saved checkpoint to checkpoint/Afteri2i_100kckpt_LLAVAT5_empty_Fixed_just_segment_pretrain_stage3_dreambench_recap_400k_t2i_400k_flux_200k_midjourney_150k_recovery_150k_3e4_epoch2_half_rotated/005-GPT-XL/checkpoints/0026000.pt +[2025-04-23 19:58:54] Removed old checkpoint: checkpoint/Afteri2i_100kckpt_LLAVAT5_empty_Fixed_just_segment_pretrain_stage3_dreambench_recap_400k_t2i_400k_flux_200k_midjourney_150k_recovery_150k_3e4_epoch2_half_rotated/005-GPT-XL/checkpoints/0024000.pt + 11%|███████████████▌ | 2566/23458 [47:58<5:07:50, 1.13it/s][2025-04-23 19:59:16] (step=0026025) Train Loss: 5.4119, Train Steps/Sec: 0.07 + 11%|███████████████▋ | 2591/23458 [48:20<4:55:57, 1.18it/s][2025-04-23 19:59:37] (step=0026050) Train Loss: 5.4172, Train Steps/Sec: 1.18 + 11%|███████████████▊ | 2616/23458 [48:41<4:52:41, 1.19it/s][2025-04-23 19:59:58] (step=0026075) Train Loss: 5.4157, Train Steps/Sec: 1.18 + 11%|███████████████▉ | 2641/23458 [49:02<4:52:15, 1.19it/s][2025-04-23 20:00:20] (step=0026100) Train Loss: 5.3657, Train Steps/Sec: 1.17 + 11%|████████████████▏ | 2666/23458 [49:24<5:01:45, 1.15it/s][2025-04-23 20:00:41] (step=0026125) Train Loss: 5.3820, Train Steps/Sec: 1.17 + 11%|████████████████▎ | 2691/23458 [49:45<4:54:55, 1.17it/s][2025-04-23 20:01:02] (step=0026150) Train Loss: 5.5298, Train Steps/Sec: 1.18 + 12%|████████████████▍ | 2716/23458 [50:06<4:54:14, 1.17it/s][2025-04-23 20:01:24] (step=0026175) Train Loss: 5.3919, Train Steps/Sec: 1.17 + 12%|████████████████▌ | 2741/23458 [50:27<4:51:17, 1.19it/s][2025-04-23 20:01:45] (step=0026200) Train Loss: 5.3736, Train Steps/Sec: 1.17 + 12%|████████████████▋ | 2766/23458 [50:49<4:58:39, 1.15it/s][2025-04-23 20:02:06] (step=0026225) Train Loss: 5.4061, Train Steps/Sec: 1.17 + 12%|████████████████▉ | 2791/23458 [51:11<4:53:36, 1.17it/s][2025-04-23 20:02:28] (step=0026250) Train Loss: 5.4054, Train Steps/Sec: 1.13 + 12%|█████████████████ | 2816/23458 [51:33<4:50:49, 1.18it/s][2025-04-23 20:02:50] (step=0026275) Train Loss: 5.4132, Train Steps/Sec: 1.13 + 12%|█████████████████▏ | 2841/23458 [51:55<4:51:43, 1.18it/s][2025-04-23 20:03:13] (step=0026300) Train Loss: 5.3719, Train Steps/Sec: 1.13 + 12%|█████████████████▎ | 2866/23458 [52:17<5:10:34, 1.11it/s][2025-04-23 20:03:35] (step=0026325) Train Loss: 5.4100, Train Steps/Sec: 1.13 + 12%|█████████████████▌ | 2891/23458 [52:39<4:52:03, 1.17it/s][2025-04-23 20:03:56] (step=0026350) Train Loss: 5.4325, Train Steps/Sec: 1.18 + 12%|█████████████████▋ | 2916/23458 [53:00<4:50:07, 1.18it/s][2025-04-23 20:04:17] (step=0026375) Train Loss: 5.3857, Train Steps/Sec: 1.17 + 13%|█████████████████▊ | 2941/23458 [53:22<4:50:20, 1.18it/s][2025-04-23 20:04:40] (step=0026400) Train Loss: 5.4480, Train Steps/Sec: 1.12 + 13%|█████████████████▉ | 2966/23458 [53:44<4:55:27, 1.16it/s][2025-04-23 20:05:01] (step=0026425) Train Loss: 5.4550, Train Steps/Sec: 1.18 + 13%|██████████████████ | 2991/23458 [54:05<4:50:41, 1.17it/s][2025-04-23 20:05:22] (step=0026450) Train Loss: 5.3832, Train Steps/Sec: 1.17 + 13%|██████████████████▎ | 3016/23458 [54:26<4:47:33, 1.18it/s][2025-04-23 20:05:44] (step=0026475) Train Loss: 5.3467, Train Steps/Sec: 1.17 + 13%|██████████████████▍ | 3041/23458 [54:49<4:47:33, 1.18it/s][2025-04-23 20:06:06] (step=0026500) Train Loss: 5.4369, Train Steps/Sec: 1.12 + 13%|██████████████████▌ | 3066/23458 [55:10<4:55:38, 1.15it/s][2025-04-23 20:06:27] (step=0026525) Train Loss: 5.4723, Train Steps/Sec: 1.17 + 13%|██████████████████▋ | 3091/23458 [55:31<4:49:07, 1.17it/s][2025-04-23 20:06:49] (step=0026550) Train Loss: 5.4152, Train Steps/Sec: 1.18 + 13%|██████████████████▊ | 3116/23458 [55:53<4:47:50, 1.18it/s][2025-04-23 20:07:10] (step=0026575) Train Loss: 5.3301, Train Steps/Sec: 1.17 + 13%|███████████████████ | 3141/23458 [56:15<4:44:23, 1.19it/s][2025-04-23 20:07:32] (step=0026600) Train Loss: 5.5070, Train Steps/Sec: 1.12 + 13%|███████████████████▏ | 3166/23458 [56:36<4:52:11, 1.16it/s][2025-04-23 20:07:54] (step=0026625) Train Loss: 5.4112, Train Steps/Sec: 1.18 + 14%|███████████████████▎ | 3191/23458 [56:57<4:47:00, 1.18it/s][2025-04-23 20:08:15] (step=0026650) Train Loss: 5.4180, Train Steps/Sec: 1.18 + 14%|███████████████████▍ | 3216/23458 [57:20<4:48:27, 1.17it/s][2025-04-23 20:08:37] (step=0026675) Train Loss: 5.4051, Train Steps/Sec: 1.13 + 14%|███████████████████▌ | 3241/23458 [57:41<4:44:12, 1.19it/s][2025-04-23 20:08:58] (step=0026700) Train Loss: 5.4687, Train Steps/Sec: 1.17 + 14%|███████████████████▊ | 3266/23458 [58:02<4:52:02, 1.15it/s][2025-04-23 20:09:20] (step=0026725) Train Loss: 5.3545, Train Steps/Sec: 1.17 + 14%|███████████████████▉ | 3291/23458 [58:24<4:47:18, 1.17it/s][2025-04-23 20:09:41] (step=0026750) Train Loss: 5.3963, Train Steps/Sec: 1.17 + 14%|████████████████████ | 3316/23458 [58:45<4:44:49, 1.18it/s][2025-04-23 20:10:02] (step=0026775) Train Loss: 5.4072, Train Steps/Sec: 1.17 + 14%|████████████████████▏ | 3341/23458 [59:06<4:41:16, 1.19it/s][2025-04-23 20:10:24] (step=0026800) Train Loss: 5.3293, Train Steps/Sec: 1.17 + 14%|████████████████████▍ | 3366/23458 [59:28<4:50:36, 1.15it/s][2025-04-23 20:10:45] (step=0026825) Train Loss: 5.4767, Train Steps/Sec: 1.18 + 14%|████████████████████▌ | 3391/23458 [59:49<4:45:06, 1.17it/s][2025-04-23 20:11:06] (step=0026850) Train Loss: 5.3216, Train Steps/Sec: 1.17 + 15%|████████████████████▍ | 3416/23458 [1:00:10<4:43:32, 1.18it/s][2025-04-23 20:11:28] (step=0026875) Train Loss: 5.3796, Train Steps/Sec: 1.17 + 15%|████████████████████▌ | 3441/23458 [1:00:32<4:45:59, 1.17it/s][2025-04-23 20:11:50] (step=0026900) Train Loss: 5.3617, Train Steps/Sec: 1.12 + 15%|████████████████████▋ | 3466/23458 [1:00:55<5:10:26, 1.07it/s][2025-04-23 20:12:12] (step=0026925) Train Loss: 5.4032, Train Steps/Sec: 1.13 + 15%|████████████████████▊ | 3491/23458 [1:01:17<5:13:10, 1.06it/s][2025-04-23 20:12:34] (step=0026950) Train Loss: 5.3905, Train Steps/Sec: 1.13 + 15%|████████████████████▉ | 3516/23458 [1:01:38<4:41:12, 1.18it/s][2025-04-23 20:12:56] (step=0026975) Train Loss: 5.4509, Train Steps/Sec: 1.17 + 15%|█████████████████████▏ | 3541/23458 [1:02:00<4:40:13, 1.18it/s][2025-04-23 20:13:18] (step=0027000) Train Loss: 5.3270, Train Steps/Sec: 1.13 + 15%|█████████████████████▎ | 3566/23458 [1:02:22<4:46:12, 1.16it/s][2025-04-23 20:13:39] (step=0027025) Train Loss: 5.4122, Train Steps/Sec: 1.18 + 15%|█████████████████████▍ | 3591/23458 [1:02:44<5:05:00, 1.09it/s][2025-04-23 20:14:01] (step=0027050) Train Loss: 5.3150, Train Steps/Sec: 1.13 + 15%|█████████████████████▌ | 3616/23458 [1:03:05<4:39:24, 1.18it/s][2025-04-23 20:14:22] (step=0027075) Train Loss: 5.3616, Train Steps/Sec: 1.17 + 16%|█████████████████████▋ | 3641/23458 [1:03:26<4:37:29, 1.19it/s][2025-04-23 20:14:44] (step=0027100) Train Loss: 5.4413, Train Steps/Sec: 1.17 + 16%|█████████████████████▉ | 3666/23458 [1:03:48<4:45:49, 1.15it/s][2025-04-23 20:15:05] (step=0027125) Train Loss: 5.3499, Train Steps/Sec: 1.17 + 16%|██████████████████████ | 3691/23458 [1:04:09<4:40:49, 1.17it/s][2025-04-23 20:15:26] (step=0027150) Train Loss: 5.5160, Train Steps/Sec: 1.17 + 16%|██████████████████████▏ | 3716/23458 [1:04:30<4:38:58, 1.18it/s][2025-04-23 20:15:48] (step=0027175) Train Loss: 5.4341, Train Steps/Sec: 1.17 + 16%|██████████████████████▎ | 3741/23458 [1:04:53<5:44:08, 1.05s/it][2025-04-23 20:16:10] (step=0027200) Train Loss: 5.3179, Train Steps/Sec: 1.12 + 16%|██████████████████████▍ | 3766/23458 [1:05:14<4:44:10, 1.15it/s][2025-04-23 20:16:31] (step=0027225) Train Loss: 5.4310, Train Steps/Sec: 1.17 + 16%|██████████████████████▋ | 3791/23458 [1:05:35<4:40:54, 1.17it/s][2025-04-23 20:16:53] (step=0027250) Train Loss: 5.3930, Train Steps/Sec: 1.17 + 16%|██████████████████████▊ | 3816/23458 [1:05:57<4:36:25, 1.18it/s][2025-04-23 20:17:14] (step=0027275) Train Loss: 5.3980, Train Steps/Sec: 1.17 + 16%|██████████████████████▉ | 3841/23458 [1:06:18<4:35:17, 1.19it/s][2025-04-23 20:17:35] (step=0027300) Train Loss: 5.4707, Train Steps/Sec: 1.17 + 16%|███████████████████████ | 3866/23458 [1:06:41<4:43:55, 1.15it/s][2025-04-23 20:17:58] (step=0027325) Train Loss: 5.4527, Train Steps/Sec: 1.08 + 17%|███████████████████████▏ | 3891/23458 [1:07:02<4:38:36, 1.17it/s][2025-04-23 20:18:20] (step=0027350) Train Loss: 5.5001, Train Steps/Sec: 1.17 + 17%|███████████████████████▎ | 3916/23458 [1:07:24<4:36:24, 1.18it/s][2025-04-23 20:18:41] (step=0027375) Train Loss: 5.3263, Train Steps/Sec: 1.17 + 17%|███████████████████████▌ | 3941/23458 [1:07:45<4:35:31, 1.18it/s][2025-04-23 20:19:02] (step=0027400) Train Loss: 5.4937, Train Steps/Sec: 1.17 + 17%|███████████████████████▋ | 3966/23458 [1:08:06<4:40:39, 1.16it/s][2025-04-23 20:19:24] (step=0027425) Train Loss: 5.3632, Train Steps/Sec: 1.18 + 17%|███████████████████████▊ | 3991/23458 [1:08:28<4:37:55, 1.17it/s][2025-04-23 20:19:45] (step=0027450) Train Loss: 5.3613, Train Steps/Sec: 1.17 + 17%|███████████████████████▉ | 4016/23458 [1:08:49<4:33:56, 1.18it/s][2025-04-23 20:20:06] (step=0027475) Train Loss: 5.3969, Train Steps/Sec: 1.17 + 17%|████████████████████████ | 4041/23458 [1:09:10<4:32:03, 1.19it/s][2025-04-23 20:20:28] (step=0027500) Train Loss: 5.3874, Train Steps/Sec: 1.17 + 17%|████████████████████████▎ | 4066/23458 [1:09:32<4:39:56, 1.15it/s][2025-04-23 20:20:49] (step=0027525) Train Loss: 5.3163, Train Steps/Sec: 1.17 + 17%|████████████████████████▍ | 4091/23458 [1:09:53<4:36:44, 1.17it/s][2025-04-23 20:21:10] (step=0027550) Train Loss: 5.3904, Train Steps/Sec: 1.17 + 18%|████████████████████████▌ | 4116/23458 [1:10:15<4:32:49, 1.18it/s][2025-04-23 20:21:32] (step=0027575) Train Loss: 5.3699, Train Steps/Sec: 1.13 + 18%|████████████████████████▋ | 4141/23458 [1:10:37<4:31:10, 1.19it/s][2025-04-23 20:21:55] (step=0027600) Train Loss: 5.4471, Train Steps/Sec: 1.13 + 18%|████████████████████████▊ | 4166/23458 [1:10:59<4:38:30, 1.15it/s][2025-04-23 20:22:17] (step=0027625) Train Loss: 5.3816, Train Steps/Sec: 1.13 + 18%|█████████████████████████ | 4191/23458 [1:11:21<4:35:00, 1.17it/s][2025-04-23 20:22:39] (step=0027650) Train Loss: 5.4404, Train Steps/Sec: 1.13 + 18%|█████████████████████████▏ | 4216/23458 [1:11:43<4:32:39, 1.18it/s][2025-04-23 20:23:00] (step=0027675) Train Loss: 5.4443, Train Steps/Sec: 1.17 + 18%|█████████████████████████▎ | 4241/23458 [1:12:04<4:29:10, 1.19it/s][2025-04-23 20:23:21] (step=0027700) Train Loss: 5.3018, Train Steps/Sec: 1.17 + 18%|█████████████████████████▍ | 4266/23458 [1:12:26<4:37:56, 1.15it/s][2025-04-23 20:23:44] (step=0027725) Train Loss: 5.3741, Train Steps/Sec: 1.13 + 18%|█████████████████████████▌ | 4291/23458 [1:12:48<4:33:01, 1.17it/s][2025-04-23 20:24:05] (step=0027750) Train Loss: 5.4134, Train Steps/Sec: 1.17 + 18%|█████████████████████████▊ | 4316/23458 [1:13:09<4:30:51, 1.18it/s][2025-04-23 20:24:26] (step=0027775) Train Loss: 5.4315, Train Steps/Sec: 1.17 + 19%|█████████████████████████▉ | 4341/23458 [1:13:30<4:28:17, 1.19it/s][2025-04-23 20:24:48] (step=0027800) Train Loss: 5.3562, Train Steps/Sec: 1.17 + 19%|██████████████████████████ | 4366/23458 [1:13:52<4:35:27, 1.16it/s][2025-04-23 20:25:09] (step=0027825) Train Loss: 5.3671, Train Steps/Sec: 1.18 + 19%|██████████████████████████▏ | 4391/23458 [1:14:13<4:31:02, 1.17it/s][2025-04-23 20:25:30] (step=0027850) Train Loss: 5.4406, Train Steps/Sec: 1.18 + 19%|██████████████████████████▎ | 4416/23458 [1:14:34<4:29:52, 1.18it/s][2025-04-23 20:25:51] (step=0027875) Train Loss: 5.4056, Train Steps/Sec: 1.17 + 19%|██████████████████████████▌ | 4441/23458 [1:14:55<4:26:19, 1.19it/s][2025-04-23 20:26:13] (step=0027900) Train Loss: 5.4272, Train Steps/Sec: 1.17 + 19%|██████████████████████████▋ | 4466/23458 [1:15:18<5:06:46, 1.03it/s][2025-04-23 20:26:35] (step=0027925) Train Loss: 5.4302, Train Steps/Sec: 1.12 + 19%|██████████████████████████▊ | 4491/23458 [1:15:39<4:29:06, 1.17it/s][2025-04-23 20:26:56] (step=0027950) Train Loss: 5.3244, Train Steps/Sec: 1.18 + 19%|██████████████████████████▉ | 4516/23458 [1:16:01<4:37:07, 1.14it/s][2025-04-23 20:27:19] (step=0027975) Train Loss: 5.4145, Train Steps/Sec: 1.13 + 19%|███████████████████████████ | 4541/23458 [1:16:23<4:25:06, 1.19it/s][2025-04-23 20:27:40] (step=0028000) Train Loss: 5.3865, Train Steps/Sec: 1.17 +You are using a model of type blip-2 to instantiate a model of type instructblip. This is not supported for all configurations of models and can yield errors. +[2025-04-23 20:27:40] text_config is None. Initializing the text config with default values (`OPTConfig`). +model_name_or_path:/tmp/haozhezhao/model/blip2-flan-t5-xl +image_place_holder: +tokenizer length after expend 32101 +tokenizer length before expend 32100 +100%|███████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████| 4/4 [03:58<00:00, 59.63s/it] +[2025-04-23 20:32:47] Finish Eval in 28000 steps...████████████████████████████████████████████████████████████████████████████████████████████████| 4/4 [03:58<00:00, 59.32s/it] +processing left-padding... +processing left-padding... +processing left-padding... +processing left-padding... +[2025-04-23 20:33:07] Saved checkpoint to checkpoint/Afteri2i_100kckpt_LLAVAT5_empty_Fixed_just_segment_pretrain_stage3_dreambench_recap_400k_t2i_400k_flux_200k_midjourney_150k_recovery_150k_3e4_epoch2_half_rotated/005-GPT-XL/checkpoints/0028000.pt +[2025-04-23 20:33:09] Removed old checkpoint: checkpoint/Afteri2i_100kckpt_LLAVAT5_empty_Fixed_just_segment_pretrain_stage3_dreambench_recap_400k_t2i_400k_flux_200k_midjourney_150k_recovery_150k_3e4_epoch2_half_rotated/005-GPT-XL/checkpoints/0026000.pt + 19%|███████████████████████████▎ | 4566/23458 [1:22:13<4:38:01, 1.13it/s][2025-04-23 20:33:30] (step=0028025) Train Loss: 5.4250, Train Steps/Sec: 0.07 + 20%|███████████████████████████▍ | 4591/23458 [1:22:35<4:35:09, 1.14it/s][2025-04-23 20:33:52] (step=0028050) Train Loss: 5.3656, Train Steps/Sec: 1.13 + 20%|███████████████████████████▌ | 4616/23458 [1:22:56<4:25:32, 1.18it/s][2025-04-23 20:34:13] (step=0028075) Train Loss: 5.3309, Train Steps/Sec: 1.18 + 20%|███████████████████████████▋ | 4641/23458 [1:23:17<4:24:32, 1.19it/s][2025-04-23 20:34:35] (step=0028100) Train Loss: 5.4065, Train Steps/Sec: 1.17 + 20%|███████████████████████████▊ | 4666/23458 [1:23:39<4:32:41, 1.15it/s][2025-04-23 20:34:56] (step=0028125) Train Loss: 5.4231, Train Steps/Sec: 1.18 + 20%|███████████████████████████▉ | 4691/23458 [1:24:00<4:26:53, 1.17it/s][2025-04-23 20:35:17] (step=0028150) Train Loss: 5.4082, Train Steps/Sec: 1.18 + 20%|████████████████████████████▏ | 4716/23458 [1:24:21<4:23:45, 1.18it/s][2025-04-23 20:35:39] (step=0028175) Train Loss: 5.4198, Train Steps/Sec: 1.18 + 20%|████████████████████████████▎ | 4741/23458 [1:24:43<4:22:48, 1.19it/s][2025-04-23 20:36:00] (step=0028200) Train Loss: 5.3020, Train Steps/Sec: 1.17 + 20%|████████████████████████████▍ | 4766/23458 [1:25:05<4:29:03, 1.16it/s][2025-04-23 20:36:22] (step=0028225) Train Loss: 5.3681, Train Steps/Sec: 1.13 + 20%|████████████████████████████▌ | 4791/23458 [1:25:27<4:28:35, 1.16it/s][2025-04-23 20:36:44] (step=0028250) Train Loss: 5.4337, Train Steps/Sec: 1.13 + 21%|████████████████████████████▋ | 4816/23458 [1:25:49<4:24:26, 1.17it/s][2025-04-23 20:37:06] (step=0028275) Train Loss: 5.4037, Train Steps/Sec: 1.13 + 21%|████████████████████████████▉ | 4841/23458 [1:26:11<4:29:13, 1.15it/s][2025-04-23 20:37:28] (step=0028300) Train Loss: 5.3903, Train Steps/Sec: 1.13 + 21%|█████████████████████████████ | 4866/23458 [1:26:32<4:28:31, 1.15it/s][2025-04-23 20:37:50] (step=0028325) Train Loss: 5.4367, Train Steps/Sec: 1.17 + 21%|█████████████████████████████▏ | 4891/23458 [1:26:54<4:23:21, 1.18it/s][2025-04-23 20:38:11] (step=0028350) Train Loss: 5.4005, Train Steps/Sec: 1.18 + 21%|█████████████████████████████▎ | 4916/23458 [1:27:16<4:23:01, 1.17it/s][2025-04-23 20:38:33] (step=0028375) Train Loss: 5.3581, Train Steps/Sec: 1.13 + 21%|█████████████████████████████▍ | 4941/23458 [1:27:37<4:21:13, 1.18it/s][2025-04-23 20:38:55] (step=0028400) Train Loss: 5.3840, Train Steps/Sec: 1.17 + 21%|█████████████████████████████▋ | 4966/23458 [1:27:59<4:27:42, 1.15it/s][2025-04-23 20:39:16] (step=0028425) Train Loss: 5.4066, Train Steps/Sec: 1.17 + 21%|█████████████████████████████▊ | 4991/23458 [1:28:20<4:22:06, 1.17it/s][2025-04-23 20:39:37] (step=0028450) Train Loss: 5.3970, Train Steps/Sec: 1.18 + 21%|█████████████████████████████▉ | 5016/23458 [1:28:41<4:20:47, 1.18it/s][2025-04-23 20:39:58] (step=0028475) Train Loss: 5.4128, Train Steps/Sec: 1.17 + 21%|██████████████████████████████ | 5041/23458 [1:29:02<4:20:31, 1.18it/s][2025-04-23 20:40:20] (step=0028500) Train Loss: 5.4726, Train Steps/Sec: 1.17 + 22%|██████████████████████████████▏ | 5066/23458 [1:29:24<4:26:37, 1.15it/s][2025-04-23 20:40:41] (step=0028525) Train Loss: 5.4221, Train Steps/Sec: 1.17 + 22%|██████████████████████████████▍ | 5091/23458 [1:29:45<4:20:30, 1.18it/s][2025-04-23 20:41:02] (step=0028550) Train Loss: 5.3688, Train Steps/Sec: 1.17 + 22%|██████████████████████████████▌ | 5116/23458 [1:30:06<4:18:30, 1.18it/s][2025-04-23 20:41:24] (step=0028575) Train Loss: 5.4715, Train Steps/Sec: 1.17 + 22%|██████████████████████████████▋ | 5141/23458 [1:30:28<4:29:08, 1.13it/s][2025-04-23 20:41:45] (step=0028600) Train Loss: 5.4031, Train Steps/Sec: 1.16 + 22%|██████████████████████████████▊ | 5166/23458 [1:30:49<4:24:39, 1.15it/s][2025-04-23 20:42:07] (step=0028625) Train Loss: 5.4668, Train Steps/Sec: 1.17 + 22%|██████████████████████████████▉ | 5191/23458 [1:31:12<4:34:16, 1.11it/s][2025-04-23 20:42:29] (step=0028650) Train Loss: 5.3720, Train Steps/Sec: 1.12 + 22%|███████████████████████████████▏ | 5216/23458 [1:31:34<4:17:04, 1.18it/s][2025-04-23 20:42:51] (step=0028675) Train Loss: 5.4440, Train Steps/Sec: 1.13 + 22%|███████████████████████████████▎ | 5241/23458 [1:31:55<4:16:01, 1.19it/s][2025-04-23 20:43:13] (step=0028700) Train Loss: 5.3702, Train Steps/Sec: 1.17 + 22%|███████████████████████████████▍ | 5266/23458 [1:32:16<4:23:13, 1.15it/s][2025-04-23 20:43:34] (step=0028725) Train Loss: 5.3752, Train Steps/Sec: 1.18 + 23%|███████████████████████████████▌ | 5291/23458 [1:32:38<4:18:19, 1.17it/s][2025-04-23 20:43:55] (step=0028750) Train Loss: 5.4572, Train Steps/Sec: 1.17 + 23%|███████████████████████████████▋ | 5316/23458 [1:33:00<4:39:24, 1.08it/s][2025-04-23 20:44:17] (step=0028775) Train Loss: 5.3379, Train Steps/Sec: 1.12 + 23%|███████████████████████████████▉ | 5341/23458 [1:33:21<4:16:15, 1.18it/s][2025-04-23 20:44:39] (step=0028800) Train Loss: 5.2850, Train Steps/Sec: 1.17 + 23%|████████████████████████████████ | 5366/23458 [1:33:43<4:21:35, 1.15it/s][2025-04-23 20:45:00] (step=0028825) Train Loss: 5.3998, Train Steps/Sec: 1.18 + 23%|████████████████████████████████▏ | 5391/23458 [1:34:04<4:16:31, 1.17it/s][2025-04-23 20:45:21] (step=0028850) Train Loss: 5.3774, Train Steps/Sec: 1.17 + 23%|████████████████████████████████▎ | 5416/23458 [1:34:26<4:16:46, 1.17it/s][2025-04-23 20:45:44] (step=0028875) Train Loss: 5.3579, Train Steps/Sec: 1.13 + 23%|████████████████████████████████▍ | 5441/23458 [1:34:48<4:51:42, 1.03it/s][2025-04-23 20:46:06] (step=0028900) Train Loss: 5.3886, Train Steps/Sec: 1.12 + 23%|████████████████████████████████▌ | 5466/23458 [1:35:11<5:30:59, 1.10s/it][2025-04-23 20:46:28] (step=0028925) Train Loss: 5.4147, Train Steps/Sec: 1.13 + 23%|████████████████████████████████▊ | 5491/23458 [1:35:32<4:16:03, 1.17it/s][2025-04-23 20:46:49] (step=0028950) Train Loss: 5.4051, Train Steps/Sec: 1.17 + 24%|████████████████████████████████▉ | 5516/23458 [1:35:54<4:13:34, 1.18it/s][2025-04-23 20:47:11] (step=0028975) Train Loss: 5.3700, Train Steps/Sec: 1.13 + 24%|█████████████████████████████████ | 5541/23458 [1:36:15<4:12:17, 1.18it/s][2025-04-23 20:47:33] (step=0029000) Train Loss: 5.3681, Train Steps/Sec: 1.17 + 24%|█████████████████████████████████▏ | 5566/23458 [1:36:38<4:27:43, 1.11it/s][2025-04-23 20:47:55] (step=0029025) Train Loss: 5.3941, Train Steps/Sec: 1.13 + 24%|█████████████████████████████████▎ | 5591/23458 [1:36:59<4:13:11, 1.18it/s][2025-04-23 20:48:16] (step=0029050) Train Loss: 5.3807, Train Steps/Sec: 1.18 + 24%|█████████████████████████████████▌ | 5616/23458 [1:37:20<4:12:34, 1.18it/s][2025-04-23 20:48:38] (step=0029075) Train Loss: 5.3420, Train Steps/Sec: 1.17 + 24%|█████████████████████████████████▋ | 5641/23458 [1:37:41<4:10:06, 1.19it/s][2025-04-23 20:48:59] (step=0029100) Train Loss: 5.3802, Train Steps/Sec: 1.17 + 24%|█████████████████████████████████▊ | 5666/23458 [1:38:03<4:17:00, 1.15it/s][2025-04-23 20:49:20] (step=0029125) Train Loss: 5.3791, Train Steps/Sec: 1.18 + 24%|█████████████████████████████████▉ | 5691/23458 [1:38:24<4:11:53, 1.18it/s][2025-04-23 20:49:41] (step=0029150) Train Loss: 5.4458, Train Steps/Sec: 1.17 + 24%|██████████████████████████████████ | 5716/23458 [1:38:45<4:10:41, 1.18it/s][2025-04-23 20:50:03] (step=0029175) Train Loss: 5.4045, Train Steps/Sec: 1.17 + 24%|██████████████████████████████████▎ | 5741/23458 [1:39:07<4:08:27, 1.19it/s][2025-04-23 20:50:24] (step=0029200) Train Loss: 5.3776, Train Steps/Sec: 1.17 + 25%|██████████████████████████████████▍ | 5766/23458 [1:39:28<4:14:57, 1.16it/s][2025-04-23 20:50:45] (step=0029225) Train Loss: 5.4324, Train Steps/Sec: 1.18 + 25%|██████████████████████████████████▌ | 5791/23458 [1:39:49<4:10:45, 1.17it/s][2025-04-23 20:51:07] (step=0029250) Train Loss: 5.4198, Train Steps/Sec: 1.18 + 25%|██████████████████████████████████▋ | 5816/23458 [1:40:11<4:08:48, 1.18it/s][2025-04-23 20:51:28] (step=0029275) Train Loss: 5.4694, Train Steps/Sec: 1.17 + 25%|██████████████████████████████████▊ | 5841/23458 [1:40:32<4:06:31, 1.19it/s][2025-04-23 20:51:49] (step=0029300) Train Loss: 5.4150, Train Steps/Sec: 1.17 + 25%|███████████████████████████████████ | 5866/23458 [1:40:54<4:15:09, 1.15it/s][2025-04-23 20:52:11] (step=0029325) Train Loss: 5.4157, Train Steps/Sec: 1.13 + 25%|███████████████████████████████████▏ | 5891/23458 [1:41:15<4:10:42, 1.17it/s][2025-04-23 20:52:33] (step=0029350) Train Loss: 5.4133, Train Steps/Sec: 1.17 + 25%|███████████████████████████████████▎ | 5916/23458 [1:41:38<4:17:08, 1.14it/s][2025-04-23 20:52:55] (step=0029375) Train Loss: 5.4303, Train Steps/Sec: 1.13 + 25%|███████████████████████████████████▍ | 5941/23458 [1:41:59<4:06:41, 1.18it/s][2025-04-23 20:53:16] (step=0029400) Train Loss: 5.4748, Train Steps/Sec: 1.17 + 25%|███████████████████████████████████▌ | 5966/23458 [1:42:20<4:13:15, 1.15it/s][2025-04-23 20:53:38] (step=0029425) Train Loss: 5.4604, Train Steps/Sec: 1.17 + 26%|███████████████████████████████████▊ | 5991/23458 [1:42:42<4:08:49, 1.17it/s][2025-04-23 20:53:59] (step=0029450) Train Loss: 5.3906, Train Steps/Sec: 1.17 + 26%|███████████████████████████████████▉ | 6016/23458 [1:43:03<4:05:43, 1.18it/s][2025-04-23 20:54:20] (step=0029475) Train Loss: 5.3669, Train Steps/Sec: 1.17 + 26%|████████████████████████████████████ | 6041/23458 [1:43:25<4:47:33, 1.01it/s][2025-04-23 20:54:43] (step=0029500) Train Loss: 5.3674, Train Steps/Sec: 1.12 + 26%|████████████████████████████████████▏ | 6066/23458 [1:43:47<5:18:01, 1.10s/it][2025-04-23 20:55:05] (step=0029525) Train Loss: 5.3818, Train Steps/Sec: 1.13 + 26%|████████████████████████████████████▎ | 6091/23458 [1:44:09<4:06:44, 1.17it/s][2025-04-23 20:55:26] (step=0029550) Train Loss: 5.3160, Train Steps/Sec: 1.17 + 26%|████████████████████████████████████▌ | 6116/23458 [1:44:31<4:04:12, 1.18it/s][2025-04-23 20:55:48] (step=0029575) Train Loss: 5.3156, Train Steps/Sec: 1.13 + 26%|████████████████████████████████████▋ | 6141/23458 [1:44:53<4:03:37, 1.18it/s][2025-04-23 20:56:10] (step=0029600) Train Loss: 5.3053, Train Steps/Sec: 1.13 + 26%|████████████████████████████████████▊ | 6166/23458 [1:45:15<4:12:06, 1.14it/s][2025-04-23 20:56:32] (step=0029625) Train Loss: 5.3710, Train Steps/Sec: 1.13 + 26%|████████████████████████████████████▉ | 6191/23458 [1:45:36<4:07:05, 1.16it/s][2025-04-23 20:56:54] (step=0029650) Train Loss: 5.4710, Train Steps/Sec: 1.17 + 26%|█████████████████████████████████████ | 6216/23458 [1:45:58<4:04:22, 1.18it/s][2025-04-23 20:57:15] (step=0029675) Train Loss: 5.4087, Train Steps/Sec: 1.17 + 27%|█████████████████████████████████████▏ | 6241/23458 [1:46:20<4:02:37, 1.18it/s][2025-04-23 20:57:37] (step=0029700) Train Loss: 5.3816, Train Steps/Sec: 1.13 + 27%|█████████████████████████████████████▍ | 6266/23458 [1:46:41<4:08:38, 1.15it/s][2025-04-23 20:57:59] (step=0029725) Train Loss: 5.3148, Train Steps/Sec: 1.17 + 27%|█████████████████████████████████████▌ | 6291/23458 [1:47:03<4:03:48, 1.17it/s][2025-04-23 20:58:20] (step=0029750) Train Loss: 5.4222, Train Steps/Sec: 1.18 + 27%|█████████████████████████████████████▋ | 6316/23458 [1:47:24<4:00:53, 1.19it/s][2025-04-23 20:58:41] (step=0029775) Train Loss: 5.4219, Train Steps/Sec: 1.18 + 27%|█████████████████████████████████████▊ | 6341/23458 [1:47:45<4:01:20, 1.18it/s][2025-04-23 20:59:03] (step=0029800) Train Loss: 5.3816, Train Steps/Sec: 1.17 + 27%|█████████████████████████████████████▉ | 6366/23458 [1:48:06<4:06:20, 1.16it/s][2025-04-23 20:59:24] (step=0029825) Train Loss: 5.3279, Train Steps/Sec: 1.18 + 27%|██████████████████████████████████████▏ | 6391/23458 [1:48:28<4:04:20, 1.16it/s][2025-04-23 20:59:45] (step=0029850) Train Loss: 5.3966, Train Steps/Sec: 1.17 + 27%|██████████████████████████████████████▎ | 6416/23458 [1:48:49<4:02:12, 1.17it/s][2025-04-23 21:00:06] (step=0029875) Train Loss: 5.3477, Train Steps/Sec: 1.17 + 27%|██████████████████████████████████████▍ | 6441/23458 [1:49:10<3:58:30, 1.19it/s][2025-04-23 21:00:28] (step=0029900) Train Loss: 5.3581, Train Steps/Sec: 1.17 + 28%|██████████████████████████████████████▌ | 6466/23458 [1:49:32<4:06:40, 1.15it/s][2025-04-23 21:00:49] (step=0029925) Train Loss: 5.3700, Train Steps/Sec: 1.17 + 28%|██████████████████████████████████████▋ | 6491/23458 [1:49:53<4:02:52, 1.16it/s][2025-04-23 21:01:11] (step=0029950) Train Loss: 5.4150, Train Steps/Sec: 1.17 + 28%|██████████████████████████████████████▉ | 6516/23458 [1:50:15<5:11:12, 1.10s/it][2025-04-23 21:01:33] (step=0029975) Train Loss: 5.3065, Train Steps/Sec: 1.13 + 28%|███████████████████████████████████████ | 6541/23458 [1:50:37<3:56:49, 1.19it/s][2025-04-23 21:01:54] (step=0030000) Train Loss: 5.4136, Train Steps/Sec: 1.17 +You are using a model of type blip-2 to instantiate a model of type instructblip. This is not supported for all configurations of models and can yield errors. +[2025-04-23 21:01:54] text_config is None. Initializing the text config with default values (`OPTConfig`). +model_name_or_path:/tmp/haozhezhao/model/blip2-flan-t5-xl +image_place_holder: +tokenizer length after expend 32101 +tokenizer length before expend 32100 +100%|███████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████| 4/4 [03:56<00:00, 59.21s/it] +[2025-04-23 21:07:00] Finish Eval in 30000 steps...████████████████████████████████████████████████████████████████████████████████████████████████| 4/4 [03:56<00:00, 58.93s/it] +processing left-padding... +processing left-padding... +processing left-padding... +processing left-padding... +[2025-04-23 21:07:18] Saved checkpoint to checkpoint/Afteri2i_100kckpt_LLAVAT5_empty_Fixed_just_segment_pretrain_stage3_dreambench_recap_400k_t2i_400k_flux_200k_midjourney_150k_recovery_150k_3e4_epoch2_half_rotated/005-GPT-XL/checkpoints/0030000.pt +[2025-04-23 21:07:20] Removed old checkpoint: checkpoint/Afteri2i_100kckpt_LLAVAT5_empty_Fixed_just_segment_pretrain_stage3_dreambench_recap_400k_t2i_400k_flux_200k_midjourney_150k_recovery_150k_3e4_epoch2_half_rotated/005-GPT-XL/checkpoints/0028000.pt + 28%|███████████████████████████████████████▏ | 6566/23458 [1:56:24<4:07:43, 1.14it/s][2025-04-23 21:07:41] (step=0030025) Train Loss: 5.4096, Train Steps/Sec: 0.07 + 28%|███████████████████████████████████████▎ | 6591/23458 [1:56:45<3:58:56, 1.18it/s][2025-04-23 21:08:03] (step=0030050) Train Loss: 5.4076, Train Steps/Sec: 1.18 + 28%|███████████████████████████████████████▍ | 6616/23458 [1:57:07<3:59:27, 1.17it/s][2025-04-23 21:08:24] (step=0030075) Train Loss: 5.3457, Train Steps/Sec: 1.17 + 28%|███████████████████████████████████████▋ | 6641/23458 [1:57:29<4:25:31, 1.06it/s][2025-04-23 21:08:46] (step=0030100) Train Loss: 5.3870, Train Steps/Sec: 1.12 + 28%|███████████████████████████████████████▊ | 6666/23458 [1:57:50<4:02:38, 1.15it/s][2025-04-23 21:09:08] (step=0030125) Train Loss: 5.4202, Train Steps/Sec: 1.17 + 29%|███████████████████████████████████████▉ | 6691/23458 [1:58:12<3:58:04, 1.17it/s][2025-04-23 21:09:29] (step=0030150) Train Loss: 5.3886, Train Steps/Sec: 1.18 + 29%|████████████████████████████████████████ | 6716/23458 [1:58:33<3:56:49, 1.18it/s][2025-04-23 21:09:50] (step=0030175) Train Loss: 5.3820, Train Steps/Sec: 1.17 + 29%|████████████████████████████████████████▏ | 6741/23458 [1:58:55<3:55:01, 1.19it/s][2025-04-23 21:10:13] (step=0030200) Train Loss: 5.4192, Train Steps/Sec: 1.13 + 29%|████████████████████████████████████████▍ | 6766/23458 [1:59:18<5:16:35, 1.14s/it][2025-04-23 21:10:36] (step=0030225) Train Loss: 5.4410, Train Steps/Sec: 1.08 + 29%|████████████████████████████████████████▌ | 6791/23458 [1:59:40<4:14:34, 1.09it/s][2025-04-23 21:10:58] (step=0030250) Train Loss: 5.3164, Train Steps/Sec: 1.13 + 29%|████████████████████████████████████████▋ | 6816/23458 [2:00:03<5:03:27, 1.09s/it][2025-04-23 21:11:20] (step=0030275) Train Loss: 5.3762, Train Steps/Sec: 1.13 + 29%|████████████████████████████████████████▊ | 6841/23458 [2:00:24<3:52:22, 1.19it/s][2025-04-23 21:11:41] (step=0030300) Train Loss: 5.4445, Train Steps/Sec: 1.17 + 29%|████████████████████████████████████████▉ | 6866/23458 [2:00:45<4:00:40, 1.15it/s][2025-04-23 21:12:03] (step=0030325) Train Loss: 5.3777, Train Steps/Sec: 1.17 + 29%|█████████████████████████████████████████▏ | 6891/23458 [2:01:07<3:58:33, 1.16it/s][2025-04-23 21:12:25] (step=0030350) Train Loss: 5.3615, Train Steps/Sec: 1.13 + 29%|█████████████████████████████████████████▎ | 6916/23458 [2:01:29<3:53:20, 1.18it/s][2025-04-23 21:12:46] (step=0030375) Train Loss: 5.3787, Train Steps/Sec: 1.17 + 30%|█████████████████████████████████████████▍ | 6941/23458 [2:01:50<3:52:12, 1.19it/s][2025-04-23 21:13:07] (step=0030400) Train Loss: 5.4333, Train Steps/Sec: 1.17 + 30%|█████████████████████████████████████████▌ | 6966/23458 [2:02:11<3:58:35, 1.15it/s][2025-04-23 21:13:29] (step=0030425) Train Loss: 5.3668, Train Steps/Sec: 1.17 + 30%|█████████████████████████████████████████▋ | 6991/23458 [2:02:33<3:53:55, 1.17it/s][2025-04-23 21:13:50] (step=0030450) Train Loss: 5.4656, Train Steps/Sec: 1.18 + 30%|█████████████████████████████████████████▊ | 7016/23458 [2:02:54<3:52:26, 1.18it/s][2025-04-23 21:14:11] (step=0030475) Train Loss: 5.4235, Train Steps/Sec: 1.17 + 30%|██████████████████████████████████████████ | 7041/23458 [2:03:15<3:51:03, 1.18it/s][2025-04-23 21:14:33] (step=0030500) Train Loss: 5.4513, Train Steps/Sec: 1.17 + 30%|██████████████████████████████████████████▏ | 7066/23458 [2:03:37<4:02:49, 1.13it/s][2025-04-23 21:14:54] (step=0030525) Train Loss: 5.4046, Train Steps/Sec: 1.17 + 30%|██████████████████████████████████████████▎ | 7091/23458 [2:03:58<3:52:23, 1.17it/s][2025-04-23 21:15:16] (step=0030550) Train Loss: 5.3750, Train Steps/Sec: 1.17 + 30%|██████████████████████████████████████████▍ | 7116/23458 [2:04:19<3:50:50, 1.18it/s][2025-04-23 21:15:37] (step=0030575) Train Loss: 5.3534, Train Steps/Sec: 1.17 + 30%|██████████████████████████████████████████▌ | 7141/23458 [2:04:41<3:48:55, 1.19it/s][2025-04-23 21:15:58] (step=0030600) Train Loss: 5.4231, Train Steps/Sec: 1.17 + 31%|██████████████████████████████████████████▊ | 7166/23458 [2:05:02<3:54:31, 1.16it/s][2025-04-23 21:16:19] (step=0030625) Train Loss: 5.3057, Train Steps/Sec: 1.17 + 31%|██████████████████████████████████████████▉ | 7191/23458 [2:05:23<3:50:30, 1.18it/s][2025-04-23 21:16:41] (step=0030650) Train Loss: 5.3776, Train Steps/Sec: 1.17 + 31%|███████████████████████████████████████████ | 7216/23458 [2:05:46<3:49:53, 1.18it/s][2025-04-23 21:17:03] (step=0030675) Train Loss: 5.3863, Train Steps/Sec: 1.13 + 31%|███████████████████████████████████████████▏ | 7241/23458 [2:06:07<3:47:29, 1.19it/s][2025-04-23 21:17:24] (step=0030700) Train Loss: 5.4075, Train Steps/Sec: 1.17 + 31%|███████████████████████████████████████████▎ | 7266/23458 [2:06:28<3:55:26, 1.15it/s][2025-04-23 21:17:46] (step=0030725) Train Loss: 5.3710, Train Steps/Sec: 1.17 + 31%|███████████████████████████████████████████▌ | 7291/23458 [2:06:50<3:51:22, 1.16it/s][2025-04-23 21:18:07] (step=0030750) Train Loss: 5.3375, Train Steps/Sec: 1.17 + 31%|███████████████████████████████████████████▋ | 7316/23458 [2:07:11<3:47:53, 1.18it/s][2025-04-23 21:18:28] (step=0030775) Train Loss: 5.3561, Train Steps/Sec: 1.18 + 31%|███████████████████████████████████████████▊ | 7341/23458 [2:07:32<3:46:46, 1.18it/s][2025-04-23 21:18:50] (step=0030800) Train Loss: 5.3653, Train Steps/Sec: 1.17 + 31%|███████████████████████████████████████████▉ | 7366/23458 [2:07:55<4:01:52, 1.11it/s][2025-04-23 21:19:12] (step=0030825) Train Loss: 5.4238, Train Steps/Sec: 1.12 + 32%|████████████████████████████████████████████ | 7391/23458 [2:08:17<3:53:15, 1.15it/s][2025-04-23 21:19:34] (step=0030850) Train Loss: 5.3402, Train Steps/Sec: 1.13 + 32%|████████████████████████████████████████████▎ | 7416/23458 [2:08:39<4:09:39, 1.07it/s][2025-04-23 21:19:56] (step=0030875) Train Loss: 5.4087, Train Steps/Sec: 1.13 + 32%|████████████████████████████████████████████▍ | 7441/23458 [2:09:00<3:44:17, 1.19it/s][2025-04-23 21:20:17] (step=0030900) Train Loss: 5.3780, Train Steps/Sec: 1.17 + 32%|████████████████████████████████████████████▌ | 7466/23458 [2:09:22<3:51:34, 1.15it/s][2025-04-23 21:20:40] (step=0030925) Train Loss: 5.3767, Train Steps/Sec: 1.13 + 32%|████████████████████████████████████████████▋ | 7491/23458 [2:09:44<3:48:21, 1.17it/s][2025-04-23 21:21:02] (step=0030950) Train Loss: 5.4041, Train Steps/Sec: 1.13 + 32%|████████████████████████████████████████████▊ | 7516/23458 [2:10:07<3:44:39, 1.18it/s][2025-04-23 21:21:24] (step=0030975) Train Loss: 5.4028, Train Steps/Sec: 1.12 + 32%|█████████████████████████████████████████████ | 7541/23458 [2:10:28<3:43:44, 1.19it/s][2025-04-23 21:21:45] (step=0031000) Train Loss: 5.3647, Train Steps/Sec: 1.17 + 32%|█████████████████████████████████████████████▏ | 7566/23458 [2:10:50<3:49:19, 1.15it/s][2025-04-23 21:22:07] (step=0031025) Train Loss: 5.3483, Train Steps/Sec: 1.13 + 32%|█████████████████████████████████████████████▎ | 7591/23458 [2:11:12<3:45:58, 1.17it/s][2025-04-23 21:22:29] (step=0031050) Train Loss: 5.3735, Train Steps/Sec: 1.17 + 32%|█████████████████████████████████████████████▍ | 7616/23458 [2:11:33<3:44:18, 1.18it/s][2025-04-23 21:22:50] (step=0031075) Train Loss: 5.3554, Train Steps/Sec: 1.17 + 33%|█████████████████████████████████████████████▌ | 7641/23458 [2:11:54<3:41:53, 1.19it/s][2025-04-23 21:23:12] (step=0031100) Train Loss: 5.4916, Train Steps/Sec: 1.17 + 33%|█████████████████████████████████████████████▊ | 7666/23458 [2:12:16<3:47:42, 1.16it/s][2025-04-23 21:23:33] (step=0031125) Train Loss: 5.4115, Train Steps/Sec: 1.17 + 33%|█████████████████████████████████████████████▉ | 7691/23458 [2:12:37<3:44:45, 1.17it/s][2025-04-23 21:23:54] (step=0031150) Train Loss: 5.3993, Train Steps/Sec: 1.17 + 33%|██████████████████████████████████████████████ | 7716/23458 [2:12:58<3:41:55, 1.18it/s][2025-04-23 21:24:15] (step=0031175) Train Loss: 5.4073, Train Steps/Sec: 1.17 + 33%|██████████████████████████████████████████████▏ | 7741/23458 [2:13:20<3:44:04, 1.17it/s][2025-04-23 21:24:37] (step=0031200) Train Loss: 5.4690, Train Steps/Sec: 1.16 + 33%|██████████████████████████████████████████████▎ | 7766/23458 [2:13:41<3:47:14, 1.15it/s][2025-04-23 21:24:58] (step=0031225) Train Loss: 5.3896, Train Steps/Sec: 1.17 + 33%|██████████████████████████████████████████████▍ | 7791/23458 [2:14:02<3:43:15, 1.17it/s][2025-04-23 21:25:20] (step=0031250) Train Loss: 5.4227, Train Steps/Sec: 1.18 + 33%|██████████████████████████████████████████████▋ | 7816/23458 [2:14:23<3:41:19, 1.18it/s][2025-04-23 21:25:41] (step=0031275) Train Loss: 5.3561, Train Steps/Sec: 1.17 + 33%|██████████████████████████████████████████████▊ | 7841/23458 [2:14:45<3:39:33, 1.19it/s][2025-04-23 21:26:02] (step=0031300) Train Loss: 5.4647, Train Steps/Sec: 1.17 + 34%|██████████████████████████████████████████████▉ | 7866/23458 [2:15:07<3:45:57, 1.15it/s][2025-04-23 21:26:24] (step=0031325) Train Loss: 5.4018, Train Steps/Sec: 1.13 + 34%|███████████████████████████████████████████████ | 7891/23458 [2:15:28<3:41:43, 1.17it/s][2025-04-23 21:26:46] (step=0031350) Train Loss: 5.3700, Train Steps/Sec: 1.18 + 34%|███████████████████████████████████████████████▏ | 7916/23458 [2:15:50<3:39:11, 1.18it/s][2025-04-23 21:27:07] (step=0031375) Train Loss: 5.4462, Train Steps/Sec: 1.17 + 34%|███████████████████████████████████████████████▍ | 7941/23458 [2:16:11<3:38:13, 1.19it/s][2025-04-23 21:27:28] (step=0031400) Train Loss: 5.3412, Train Steps/Sec: 1.17 + 34%|███████████████████████████████████████████████▌ | 7966/23458 [2:16:32<3:43:36, 1.15it/s][2025-04-23 21:27:50] (step=0031425) Train Loss: 5.3855, Train Steps/Sec: 1.17 + 34%|███████████████████████████████████████████████▋ | 7991/23458 [2:16:54<3:39:54, 1.17it/s][2025-04-23 21:28:11] (step=0031450) Train Loss: 5.4602, Train Steps/Sec: 1.17 + 34%|███████████████████████████████████████████████▊ | 8016/23458 [2:17:15<3:39:16, 1.17it/s][2025-04-23 21:28:32] (step=0031475) Train Loss: 5.3626, Train Steps/Sec: 1.17 + 34%|███████████████████████████████████████████████▉ | 8041/23458 [2:17:37<4:35:28, 1.07s/it][2025-04-23 21:28:54] (step=0031500) Train Loss: 5.2661, Train Steps/Sec: 1.13 + 34%|████████████████████████████████████████████████▏ | 8066/23458 [2:17:58<3:43:33, 1.15it/s][2025-04-23 21:29:16] (step=0031525) Train Loss: 5.4389, Train Steps/Sec: 1.17 + 34%|████████████████████████████████████████████████▎ | 8091/23458 [2:18:22<3:43:18, 1.15it/s][2025-04-23 21:29:39] (step=0031550) Train Loss: 5.3640, Train Steps/Sec: 1.08 + 35%|████████████████████████████████████████████████▍ | 8116/23458 [2:18:44<3:37:12, 1.18it/s][2025-04-23 21:30:01] (step=0031575) Train Loss: 5.3886, Train Steps/Sec: 1.13 + 35%|████████████████████████████████████████████████▌ | 8141/23458 [2:19:06<3:38:37, 1.17it/s][2025-04-23 21:30:23] (step=0031600) Train Loss: 5.4189, Train Steps/Sec: 1.13 + 35%|████████████████████████████████████████████████▋ | 8166/23458 [2:19:27<3:40:35, 1.16it/s][2025-04-23 21:30:44] (step=0031625) Train Loss: 5.3754, Train Steps/Sec: 1.18 + 35%|████████████████████████████████████████████████▉ | 8191/23458 [2:19:48<3:38:16, 1.17it/s][2025-04-23 21:31:06] (step=0031650) Train Loss: 5.3548, Train Steps/Sec: 1.17 + 35%|█████████████████████████████████████████████████ | 8216/23458 [2:20:11<3:36:24, 1.17it/s][2025-04-23 21:31:28] (step=0031675) Train Loss: 5.3486, Train Steps/Sec: 1.13 + 35%|█████████████████████████████████████████████████▏ | 8241/23458 [2:20:33<3:32:45, 1.19it/s][2025-04-23 21:31:50] (step=0031700) Train Loss: 5.4211, Train Steps/Sec: 1.12 + 35%|█████████████████████████████████████████████████▎ | 8266/23458 [2:20:54<3:39:45, 1.15it/s][2025-04-23 21:32:12] (step=0031725) Train Loss: 5.3881, Train Steps/Sec: 1.17 + 35%|█████████████████████████████████████████████████▍ | 8291/23458 [2:21:16<3:35:53, 1.17it/s][2025-04-23 21:32:33] (step=0031750) Train Loss: 5.3662, Train Steps/Sec: 1.18 + 35%|█████████████████████████████████████████████████▋ | 8316/23458 [2:21:37<3:33:36, 1.18it/s][2025-04-23 21:32:54] (step=0031775) Train Loss: 5.4181, Train Steps/Sec: 1.17 + 36%|█████████████████████████████████████████████████▊ | 8341/23458 [2:21:58<3:32:40, 1.18it/s][2025-04-23 21:33:16] (step=0031800) Train Loss: 5.4754, Train Steps/Sec: 1.17 + 36%|█████████████████████████████████████████████████▉ | 8366/23458 [2:22:20<3:38:11, 1.15it/s][2025-04-23 21:33:37] (step=0031825) Train Loss: 5.4031, Train Steps/Sec: 1.18 + 36%|██████████████████████████████████████████████████ | 8391/23458 [2:22:41<3:33:53, 1.17it/s][2025-04-23 21:33:58] (step=0031850) Train Loss: 5.4572, Train Steps/Sec: 1.17 + 36%|██████████████████████████████████████████████████▏ | 8416/23458 [2:23:02<3:32:06, 1.18it/s][2025-04-23 21:34:20] (step=0031875) Train Loss: 5.4129, Train Steps/Sec: 1.17 + 36%|██████████████████████████████████████████████████▍ | 8441/23458 [2:23:24<3:31:00, 1.19it/s][2025-04-23 21:34:41] (step=0031900) Train Loss: 5.3975, Train Steps/Sec: 1.17 + 36%|██████████████████████████████████████████████████▌ | 8466/23458 [2:23:45<3:36:05, 1.16it/s][2025-04-23 21:35:02] (step=0031925) Train Loss: 5.3735, Train Steps/Sec: 1.17 + 36%|██████████████████████████████████████████████████▋ | 8491/23458 [2:24:06<3:32:31, 1.17it/s][2025-04-23 21:35:24] (step=0031950) Train Loss: 5.3686, Train Steps/Sec: 1.17 + 36%|██████████████████████████████████████████████████▊ | 8516/23458 [2:24:28<4:02:27, 1.03it/s][2025-04-23 21:35:46] (step=0031975) Train Loss: 5.4216, Train Steps/Sec: 1.13 + 36%|██████████████████████████████████████████████████▉ | 8541/23458 [2:24:50<3:29:44, 1.19it/s][2025-04-23 21:36:07] (step=0032000) Train Loss: 5.3514, Train Steps/Sec: 1.17 +You are using a model of type blip-2 to instantiate a model of type instructblip. This is not supported for all configurations of models and can yield errors. +[2025-04-23 21:36:07] text_config is None. Initializing the text config with default values (`OPTConfig`). +model_name_or_path:/tmp/haozhezhao/model/blip2-flan-t5-xl +image_place_holder: +tokenizer length after expend 32101 +tokenizer length before expend 32100 +100%|███████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████| 4/4 [03:58<00:00, 59.50s/it] +[2025-04-23 21:41:15] Finish Eval in 32000 steps...████████████████████████████████████████████████████████████████████████████████████████████████| 4/4 [03:57<00:00, 59.08s/it] +processing left-padding... +processing left-padding... +processing left-padding... +processing left-padding... +[2025-04-23 21:41:33] Saved checkpoint to checkpoint/Afteri2i_100kckpt_LLAVAT5_empty_Fixed_just_segment_pretrain_stage3_dreambench_recap_400k_t2i_400k_flux_200k_midjourney_150k_recovery_150k_3e4_epoch2_half_rotated/005-GPT-XL/checkpoints/0032000.pt +[2025-04-23 21:41:34] Removed old checkpoint: checkpoint/Afteri2i_100kckpt_LLAVAT5_empty_Fixed_just_segment_pretrain_stage3_dreambench_recap_400k_t2i_400k_flux_200k_midjourney_150k_recovery_150k_3e4_epoch2_half_rotated/005-GPT-XL/checkpoints/0030000.pt + 37%|███████████████████████████████████████████████████ | 8566/23458 [2:30:39<3:39:29, 1.13it/s][2025-04-23 21:41:56] (step=0032025) Train Loss: 5.4417, Train Steps/Sec: 0.07 + 37%|███████████████████████████████████████████████████▎ | 8591/23458 [2:31:00<3:32:13, 1.17it/s][2025-04-23 21:42:17] (step=0032050) Train Loss: 5.3834, Train Steps/Sec: 1.18 + 37%|███████████████████████████████████████████████████▍ | 8616/23458 [2:31:21<3:30:18, 1.18it/s][2025-04-23 21:42:38] (step=0032075) Train Loss: 5.3745, Train Steps/Sec: 1.17 + 37%|███████████████████████████████████████████████████▌ | 8641/23458 [2:31:42<3:26:59, 1.19it/s][2025-04-23 21:43:00] (step=0032100) Train Loss: 5.3968, Train Steps/Sec: 1.17 + 37%|███████████████████████████████████████████████████▋ | 8666/23458 [2:32:04<3:34:05, 1.15it/s][2025-04-23 21:43:21] (step=0032125) Train Loss: 5.3523, Train Steps/Sec: 1.17 + 37%|███████████████████████████████████████████████████▊ | 8691/23458 [2:32:25<3:29:50, 1.17it/s][2025-04-23 21:43:42] (step=0032150) Train Loss: 5.4187, Train Steps/Sec: 1.18 + 37%|████████████████████████████████████████████████████ | 8716/23458 [2:32:47<3:28:22, 1.18it/s][2025-04-23 21:44:04] (step=0032175) Train Loss: 5.3914, Train Steps/Sec: 1.13 + 37%|████████████████████████████████████████████████████▏ | 8741/23458 [2:33:09<3:31:09, 1.16it/s][2025-04-23 21:44:27] (step=0032200) Train Loss: 5.4057, Train Steps/Sec: 1.12 + 37%|████████████████████████████████████████████████████▎ | 8766/23458 [2:33:31<3:42:08, 1.10it/s][2025-04-23 21:44:49] (step=0032225) Train Loss: 5.4031, Train Steps/Sec: 1.13 + 37%|████████████████████████████████████████████████████▍ | 8791/23458 [2:33:53<3:29:04, 1.17it/s][2025-04-23 21:45:10] (step=0032250) Train Loss: 5.3868, Train Steps/Sec: 1.18 + 38%|████████████████████████████████████████████████████▌ | 8816/23458 [2:34:16<3:30:36, 1.16it/s][2025-04-23 21:45:33] (step=0032275) Train Loss: 5.3993, Train Steps/Sec: 1.08 + 38%|████████████████████████████████████████████████████▊ | 8841/23458 [2:34:37<3:24:51, 1.19it/s][2025-04-23 21:45:55] (step=0032300) Train Loss: 5.4455, Train Steps/Sec: 1.17 + 38%|████████████████████████████████████████████████████▉ | 8866/23458 [2:34:59<4:33:29, 1.12s/it][2025-04-23 21:46:17] (step=0032325) Train Loss: 5.3910, Train Steps/Sec: 1.13 + 38%|█████████████████████████████████████████████████████ | 8891/23458 [2:35:21<3:27:45, 1.17it/s][2025-04-23 21:46:38] (step=0032350) Train Loss: 5.4280, Train Steps/Sec: 1.17 + 38%|█████████████████████████████████████████████████████▏ | 8916/23458 [2:35:42<3:25:08, 1.18it/s][2025-04-23 21:46:59] (step=0032375) Train Loss: 5.3548, Train Steps/Sec: 1.18 + 38%|█████████████████████████████████████████████████████▎ | 8941/23458 [2:36:03<3:23:11, 1.19it/s][2025-04-23 21:47:21] (step=0032400) Train Loss: 5.4170, Train Steps/Sec: 1.17 + 38%|█████████████████████████████████████████████████████▌ | 8966/23458 [2:36:26<3:30:13, 1.15it/s][2025-04-23 21:47:43] (step=0032425) Train Loss: 5.3917, Train Steps/Sec: 1.12 + 38%|█████████████████████████████████████████████████████▋ | 8991/23458 [2:36:47<3:30:52, 1.14it/s][2025-04-23 21:48:04] (step=0032450) Train Loss: 5.4098, Train Steps/Sec: 1.16 + 38%|█████████████████████████████████████████████████████▊ | 9016/23458 [2:37:08<3:24:13, 1.18it/s][2025-04-23 21:48:26] (step=0032475) Train Loss: 5.4718, Train Steps/Sec: 1.17 + 39%|█████████████████████████████████████████████████████▉ | 9041/23458 [2:37:30<3:22:20, 1.19it/s][2025-04-23 21:48:47] (step=0032500) Train Loss: 5.4001, Train Steps/Sec: 1.17 + 39%|██████████████████████████████████████████████████████ | 9066/23458 [2:37:51<3:27:50, 1.15it/s][2025-04-23 21:49:08] (step=0032525) Train Loss: 5.4274, Train Steps/Sec: 1.18 + 39%|██████████████████████████████████████████████████████▎ | 9091/23458 [2:38:12<3:24:36, 1.17it/s][2025-04-23 21:49:30] (step=0032550) Train Loss: 5.3383, Train Steps/Sec: 1.17 + 39%|██████████████████████████████████████████████████████▍ | 9116/23458 [2:38:34<3:22:09, 1.18it/s][2025-04-23 21:49:51] (step=0032575) Train Loss: 5.3653, Train Steps/Sec: 1.18 + 39%|██████████████████████████████████████████████████████▌ | 9141/23458 [2:38:55<3:20:46, 1.19it/s][2025-04-23 21:50:12] (step=0032600) Train Loss: 5.3794, Train Steps/Sec: 1.17 + 39%|██████████████████████████████████████████████████████▋ | 9166/23458 [2:39:16<3:27:02, 1.15it/s][2025-04-23 21:50:34] (step=0032625) Train Loss: 5.3595, Train Steps/Sec: 1.18 + 39%|██████████████████████████████████████████████████████▊ | 9191/23458 [2:39:38<4:06:29, 1.04s/it][2025-04-23 21:50:56] (step=0032650) Train Loss: 5.2826, Train Steps/Sec: 1.13 + 39%|███████████████████████████████████████████████████████ | 9216/23458 [2:40:00<3:21:14, 1.18it/s][2025-04-23 21:51:17] (step=0032675) Train Loss: 5.3793, Train Steps/Sec: 1.17 + 39%|███████████████████████████████████████████████████████▏ | 9241/23458 [2:40:21<3:20:04, 1.18it/s][2025-04-23 21:51:38] (step=0032700) Train Loss: 5.3620, Train Steps/Sec: 1.17 + 40%|███████████████████████████████████████████████████████▎ | 9266/23458 [2:40:42<3:25:06, 1.15it/s][2025-04-23 21:52:00] (step=0032725) Train Loss: 5.3940, Train Steps/Sec: 1.18 + 40%|███████████████████████████████████████████████████████▍ | 9291/23458 [2:41:04<3:21:20, 1.17it/s][2025-04-23 21:52:21] (step=0032750) Train Loss: 5.3954, Train Steps/Sec: 1.17 + 40%|███████████████████████████████████████████████████████▌ | 9316/23458 [2:41:25<3:20:15, 1.18it/s][2025-04-23 21:52:42] (step=0032775) Train Loss: 5.4081, Train Steps/Sec: 1.17 + 40%|███████████████████████████████████████████████████████▋ | 9341/23458 [2:41:46<3:19:22, 1.18it/s][2025-04-23 21:53:04] (step=0032800) Train Loss: 5.4412, Train Steps/Sec: 1.17 + 40%|███████████████████████████████████████████████████████▉ | 9366/23458 [2:42:09<3:28:18, 1.13it/s][2025-04-23 21:53:26] (step=0032825) Train Loss: 5.3997, Train Steps/Sec: 1.13 + 40%|████████████████████████████████████████████████████████ | 9391/23458 [2:42:30<3:20:25, 1.17it/s][2025-04-23 21:53:47] (step=0032850) Train Loss: 5.4044, Train Steps/Sec: 1.17 + 40%|████████████████████████████████████████████████████████▏ | 9416/23458 [2:42:52<3:18:00, 1.18it/s][2025-04-23 21:54:09] (step=0032875) Train Loss: 5.3911, Train Steps/Sec: 1.13 + 40%|████████████████████████████████████████████████████████▎ | 9441/23458 [2:43:14<3:16:28, 1.19it/s][2025-04-23 21:54:32] (step=0032900) Train Loss: 5.3904, Train Steps/Sec: 1.13 + 40%|████████████████████████████████████████████████████████▍ | 9466/23458 [2:43:36<3:23:12, 1.15it/s][2025-04-23 21:54:54] (step=0032925) Train Loss: 5.3529, Train Steps/Sec: 1.13 + 40%|████████████████████████████████████████████████████████▋ | 9491/23458 [2:43:58<3:18:33, 1.17it/s][2025-04-23 21:55:15] (step=0032950) Train Loss: 5.3633, Train Steps/Sec: 1.17 + 41%|████████████████████████████████████████████████████████▊ | 9516/23458 [2:44:19<3:18:01, 1.17it/s][2025-04-23 21:55:36] (step=0032975) Train Loss: 5.4431, Train Steps/Sec: 1.17 + 41%|████████████████████████████████████████████████████████▉ | 9541/23458 [2:44:41<3:15:22, 1.19it/s][2025-04-23 21:55:59] (step=0033000) Train Loss: 5.3853, Train Steps/Sec: 1.12 + 41%|█████████████████████████████████████████████████████████ | 9566/23458 [2:45:03<3:20:51, 1.15it/s][2025-04-23 21:56:20] (step=0033025) Train Loss: 5.4077, Train Steps/Sec: 1.17 + 41%|█████████████████████████████████████████████████████████▏ | 9591/23458 [2:45:24<3:18:04, 1.17it/s][2025-04-23 21:56:41] (step=0033050) Train Loss: 5.4354, Train Steps/Sec: 1.17 + 41%|█████████████████████████████████████████████████████████▍ | 9616/23458 [2:45:45<3:14:13, 1.19it/s][2025-04-23 21:57:03] (step=0033075) Train Loss: 5.3923, Train Steps/Sec: 1.17 + 41%|█████████████████████████████████████████████████████████▌ | 9641/23458 [2:46:07<3:14:03, 1.19it/s][2025-04-23 21:57:24] (step=0033100) Train Loss: 5.4223, Train Steps/Sec: 1.17 + 41%|█████████████████████████████████████████████████████████▋ | 9666/23458 [2:46:28<3:19:21, 1.15it/s][2025-04-23 21:57:45] (step=0033125) Train Loss: 5.3510, Train Steps/Sec: 1.18 + 41%|█████████████████████████████████████████████████████████▊ | 9691/23458 [2:46:50<3:16:01, 1.17it/s][2025-04-23 21:58:08] (step=0033150) Train Loss: 5.3875, Train Steps/Sec: 1.12 + 41%|█████████████████████████████████████████████████████████▉ | 9716/23458 [2:47:11<3:13:59, 1.18it/s][2025-04-23 21:58:29] (step=0033175) Train Loss: 5.4351, Train Steps/Sec: 1.18 + 42%|██████████████████████████████████████████████████████████▏ | 9741/23458 [2:47:33<3:13:19, 1.18it/s][2025-04-23 21:58:50] (step=0033200) Train Loss: 5.4083, Train Steps/Sec: 1.17 + 42%|██████████████████████████████████████████████████████████▎ | 9766/23458 [2:47:54<3:17:58, 1.15it/s][2025-04-23 21:59:11] (step=0033225) Train Loss: 5.3731, Train Steps/Sec: 1.17 + 42%|██████████████████████████████████████████████████████████▍ | 9791/23458 [2:48:15<3:15:01, 1.17it/s][2025-04-23 21:59:33] (step=0033250) Train Loss: 5.3368, Train Steps/Sec: 1.17 + 42%|██████████████████████████████████████████████████████████▌ | 9816/23458 [2:48:37<3:12:52, 1.18it/s][2025-04-23 21:59:54] (step=0033275) Train Loss: 5.3518, Train Steps/Sec: 1.17 + 42%|██████████████████████████████████████████████████████████▋ | 9841/23458 [2:48:58<3:10:29, 1.19it/s][2025-04-23 22:00:15] (step=0033300) Train Loss: 5.3395, Train Steps/Sec: 1.17 + 42%|██████████████████████████████████████████████████████████▉ | 9866/23458 [2:49:20<3:16:27, 1.15it/s][2025-04-23 22:00:38] (step=0033325) Train Loss: 5.4656, Train Steps/Sec: 1.13 + 42%|███████████████████████████████████████████████████████████ | 9891/23458 [2:49:42<3:12:34, 1.17it/s][2025-04-23 22:00:59] (step=0033350) Train Loss: 5.3995, Train Steps/Sec: 1.17 + 42%|███████████████████████████████████████████████████████████▏ | 9916/23458 [2:50:03<3:10:52, 1.18it/s][2025-04-23 22:01:20] (step=0033375) Train Loss: 5.2446, Train Steps/Sec: 1.17 + 42%|███████████████████████████████████████████████████████████▎ | 9941/23458 [2:50:24<3:09:59, 1.19it/s][2025-04-23 22:01:42] (step=0033400) Train Loss: 5.4042, Train Steps/Sec: 1.17 + 42%|███████████████████████████████████████████████████████████▍ | 9966/23458 [2:50:45<3:14:40, 1.16it/s][2025-04-23 22:02:03] (step=0033425) Train Loss: 5.4499, Train Steps/Sec: 1.18 + 43%|███████████████████████████████████████████████████████████▋ | 9991/23458 [2:51:07<3:11:43, 1.17it/s][2025-04-23 22:02:24] (step=0033450) Train Loss: 5.3568, Train Steps/Sec: 1.17 + 43%|███████████████████████████████████████████████████████████▎ | 10016/23458 [2:51:29<3:46:00, 1.01s/it][2025-04-23 22:02:46] (step=0033475) Train Loss: 5.4555, Train Steps/Sec: 1.14 + 43%|███████████████████████████████████████████████████████████▍ | 10041/23458 [2:51:50<3:08:14, 1.19it/s][2025-04-23 22:03:08] (step=0033500) Train Loss: 5.4428, Train Steps/Sec: 1.17 + 43%|███████████████████████████████████████████████████████████▋ | 10066/23458 [2:52:12<3:14:55, 1.15it/s][2025-04-23 22:03:30] (step=0033525) Train Loss: 5.3390, Train Steps/Sec: 1.13 + 43%|███████████████████████████████████████████████████████████▊ | 10091/23458 [2:52:35<3:15:46, 1.14it/s][2025-04-23 22:03:52] (step=0033550) Train Loss: 5.4137, Train Steps/Sec: 1.13 + 43%|███████████████████████████████████████████████████████████▉ | 10116/23458 [2:52:57<3:35:02, 1.03it/s][2025-04-23 22:04:14] (step=0033575) Train Loss: 5.4014, Train Steps/Sec: 1.13 + 43%|████████████████████████████████████████████████████████████ | 10141/23458 [2:53:18<3:09:03, 1.17it/s][2025-04-23 22:04:35] (step=0033600) Train Loss: 5.3448, Train Steps/Sec: 1.17 + 43%|████████████████████████████████████████████████████████████▏ | 10166/23458 [2:53:39<3:11:58, 1.15it/s][2025-04-23 22:04:57] (step=0033625) Train Loss: 5.3621, Train Steps/Sec: 1.18 + 43%|████████████████████████████████████████████████████████████▍ | 10191/23458 [2:54:02<3:19:21, 1.11it/s][2025-04-23 22:05:19] (step=0033650) Train Loss: 5.4091, Train Steps/Sec: 1.13 + 44%|████████████████████████████████████████████████████████████▌ | 10216/23458 [2:54:23<3:07:07, 1.18it/s][2025-04-23 22:05:40] (step=0033675) Train Loss: 5.4435, Train Steps/Sec: 1.17 + 44%|████████████████████████████████████████████████████████████▋ | 10241/23458 [2:54:44<3:05:13, 1.19it/s][2025-04-23 22:06:01] (step=0033700) Train Loss: 5.3973, Train Steps/Sec: 1.17 + 44%|████████████████████████████████████████████████████████████▊ | 10266/23458 [2:55:06<3:11:34, 1.15it/s][2025-04-23 22:06:24] (step=0033725) Train Loss: 5.3883, Train Steps/Sec: 1.12 + 44%|████████████████████████████████████████████████████████████▉ | 10291/23458 [2:55:28<3:07:07, 1.17it/s][2025-04-23 22:06:45] (step=0033750) Train Loss: 5.3613, Train Steps/Sec: 1.17 + 44%|█████████████████████████████████████████████████████████████▏ | 10316/23458 [2:55:49<3:05:20, 1.18it/s][2025-04-23 22:07:06] (step=0033775) Train Loss: 5.4016, Train Steps/Sec: 1.17 + 44%|█████████████████████████████████████████████████████████████▎ | 10341/23458 [2:56:10<3:04:27, 1.19it/s][2025-04-23 22:07:28] (step=0033800) Train Loss: 5.4425, Train Steps/Sec: 1.17 + 44%|█████████████████████████████████████████████████████████████▍ | 10366/23458 [2:56:32<3:08:50, 1.16it/s][2025-04-23 22:07:49] (step=0033825) Train Loss: 5.3679, Train Steps/Sec: 1.17 + 44%|█████████████████████████████████████████████████████████████▌ | 10391/23458 [2:56:53<3:06:07, 1.17it/s][2025-04-23 22:08:10] (step=0033850) Train Loss: 5.3571, Train Steps/Sec: 1.17 + 44%|█████████████████████████████████████████████████████████████▋ | 10416/23458 [2:57:15<3:05:37, 1.17it/s][2025-04-23 22:08:33] (step=0033875) Train Loss: 5.4002, Train Steps/Sec: 1.12 + 45%|█████████████████████████████████████████████████████████████▊ | 10441/23458 [2:57:37<3:01:21, 1.20it/s][2025-04-23 22:08:54] (step=0033900) Train Loss: 5.3632, Train Steps/Sec: 1.17 + 45%|██████████████████████████████████████████████████████████████ | 10466/23458 [2:57:58<3:07:36, 1.15it/s][2025-04-23 22:09:15] (step=0033925) Train Loss: 5.4580, Train Steps/Sec: 1.17 + 45%|██████████████████████████████████████████████████████████████▏ | 10491/23458 [2:58:19<3:04:56, 1.17it/s][2025-04-23 22:09:37] (step=0033950) Train Loss: 5.3472, Train Steps/Sec: 1.17 + 45%|██████████████████████████████████████████████████████████████▎ | 10516/23458 [2:58:41<3:03:46, 1.17it/s][2025-04-23 22:09:59] (step=0033975) Train Loss: 5.4271, Train Steps/Sec: 1.13 + 45%|██████████████████████████████████████████████████████████████▍ | 10541/23458 [2:59:03<3:01:38, 1.19it/s][2025-04-23 22:10:20] (step=0034000) Train Loss: 5.3878, Train Steps/Sec: 1.17 +You are using a model of type blip-2 to instantiate a model of type instructblip. This is not supported for all configurations of models and can yield errors. +[2025-04-23 22:10:20] text_config is None. Initializing the text config with default values (`OPTConfig`). +model_name_or_path:/tmp/haozhezhao/model/blip2-flan-t5-xl +image_place_holder: +tokenizer length after expend 32101 +tokenizer length before expend 32100 +100%|███████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████| 4/4 [03:56<00:00, 59.12s/it] +[2025-04-23 22:15:24] Finish Eval in 34000 steps...████████████████████████████████████████████████████████████████████████████████████████████████| 4/4 [03:56<00:00, 58.85s/it] +processing left-padding... +processing left-padding... +processing left-padding... +processing left-padding... +[2025-04-23 22:15:42] Saved checkpoint to checkpoint/Afteri2i_100kckpt_LLAVAT5_empty_Fixed_just_segment_pretrain_stage3_dreambench_recap_400k_t2i_400k_flux_200k_midjourney_150k_recovery_150k_3e4_epoch2_half_rotated/005-GPT-XL/checkpoints/0034000.pt +[2025-04-23 22:15:44] Removed old checkpoint: checkpoint/Afteri2i_100kckpt_LLAVAT5_empty_Fixed_just_segment_pretrain_stage3_dreambench_recap_400k_t2i_400k_flux_200k_midjourney_150k_recovery_150k_3e4_epoch2_half_rotated/005-GPT-XL/checkpoints/0032000.pt + 45%|██████████████████████████████████████████████████████████████▌ | 10566/23458 [3:04:48<3:09:47, 1.13it/s][2025-04-23 22:16:05] (step=0034025) Train Loss: 5.4371, Train Steps/Sec: 0.07 + 45%|██████████████████████████████████████████████████████████████▊ | 10591/23458 [3:05:09<3:02:50, 1.17it/s][2025-04-23 22:16:27] (step=0034050) Train Loss: 5.2774, Train Steps/Sec: 1.18 + 45%|██████████████████████████████████████████████████████████████▉ | 10616/23458 [3:05:30<3:00:45, 1.18it/s][2025-04-23 22:16:48] (step=0034075) Train Loss: 5.4196, Train Steps/Sec: 1.17 + 45%|███████████████████████████████████████████████████████████████ | 10641/23458 [3:05:52<2:59:20, 1.19it/s][2025-04-23 22:17:09] (step=0034100) Train Loss: 5.3792, Train Steps/Sec: 1.17 + 45%|███████████████████████████████████████████████████████████████▏ | 10666/23458 [3:06:13<3:04:37, 1.15it/s][2025-04-23 22:17:30] (step=0034125) Train Loss: 5.3469, Train Steps/Sec: 1.17 + 46%|███████████████████████████████████████████████████████████████▎ | 10691/23458 [3:06:35<3:02:17, 1.17it/s][2025-04-23 22:17:53] (step=0034150) Train Loss: 5.3619, Train Steps/Sec: 1.13 + 46%|███████████████████████████████████████████████████████████████▍ | 10716/23458 [3:06:57<3:25:35, 1.03it/s][2025-04-23 22:18:15] (step=0034175) Train Loss: 5.4123, Train Steps/Sec: 1.13 + 46%|███████████████████████████████████████████████████████████████▋ | 10741/23458 [3:07:19<2:58:26, 1.19it/s][2025-04-23 22:18:36] (step=0034200) Train Loss: 5.3822, Train Steps/Sec: 1.17 + 46%|███████████████████████████████████████████████████████████████▊ | 10766/23458 [3:07:41<3:03:27, 1.15it/s][2025-04-23 22:18:58] (step=0034225) Train Loss: 5.3243, Train Steps/Sec: 1.13 + 46%|███████████████████████████████████████████████████████████████▉ | 10791/23458 [3:08:03<3:00:01, 1.17it/s][2025-04-23 22:19:20] (step=0034250) Train Loss: 5.3464, Train Steps/Sec: 1.13 + 46%|████████████████████████████████████████████████████████████████ | 10816/23458 [3:08:24<2:58:22, 1.18it/s][2025-04-23 22:19:42] (step=0034275) Train Loss: 5.4133, Train Steps/Sec: 1.17 + 46%|████████████████████████████████████████████████████████████████▏ | 10841/23458 [3:08:46<2:57:08, 1.19it/s][2025-04-23 22:20:04] (step=0034300) Train Loss: 5.4281, Train Steps/Sec: 1.12 + 46%|████████████████████████████████████████████████████████████████▍ | 10866/23458 [3:09:08<3:01:55, 1.15it/s][2025-04-23 22:20:25] (step=0034325) Train Loss: 5.4190, Train Steps/Sec: 1.17 + 46%|████████████████████████████████████████████████████████████████▌ | 10891/23458 [3:09:29<2:59:20, 1.17it/s][2025-04-23 22:20:47] (step=0034350) Train Loss: 5.4853, Train Steps/Sec: 1.17 + 47%|████████████████████████████████████████████████████████████████▋ | 10916/23458 [3:09:51<3:01:24, 1.15it/s][2025-04-23 22:21:08] (step=0034375) Train Loss: 5.4432, Train Steps/Sec: 1.17 + 47%|████████████████████████████████████████████████████████████████▊ | 10941/23458 [3:10:12<2:55:40, 1.19it/s][2025-04-23 22:21:29] (step=0034400) Train Loss: 5.2998, Train Steps/Sec: 1.17 + 47%|████████████████████████████████████████████████████████████████▉ | 10966/23458 [3:10:33<3:00:45, 1.15it/s][2025-04-23 22:21:51] (step=0034425) Train Loss: 5.3369, Train Steps/Sec: 1.17 + 47%|█████████████████████████████████████████████████████████████████▏ | 10991/23458 [3:10:56<2:57:15, 1.17it/s][2025-04-23 22:22:13] (step=0034450) Train Loss: 5.3543, Train Steps/Sec: 1.12 + 47%|█████████████████████████████████████████████████████████████████▎ | 11016/23458 [3:11:17<2:55:57, 1.18it/s][2025-04-23 22:22:34] (step=0034475) Train Loss: 5.4444, Train Steps/Sec: 1.17 + 47%|█████████████████████████████████████████████████████████████████▍ | 11041/23458 [3:11:38<2:55:03, 1.18it/s][2025-04-23 22:22:56] (step=0034500) Train Loss: 5.3747, Train Steps/Sec: 1.17 + 47%|█████████████████████████████████████████████████████████████████▌ | 11066/23458 [3:12:00<2:59:00, 1.15it/s][2025-04-23 22:23:17] (step=0034525) Train Loss: 5.3406, Train Steps/Sec: 1.18 + 47%|█████████████████████████████████████████████████████████████████▋ | 11091/23458 [3:12:21<2:56:11, 1.17it/s][2025-04-23 22:23:38] (step=0034550) Train Loss: 5.3214, Train Steps/Sec: 1.17 + 47%|█████████████████████████████████████████████████████████████████▊ | 11116/23458 [3:12:42<2:54:50, 1.18it/s][2025-04-23 22:24:00] (step=0034575) Train Loss: 5.3612, Train Steps/Sec: 1.18 + 47%|██████████████████████████████████████████████████████████████████ | 11141/23458 [3:13:05<2:53:24, 1.18it/s][2025-04-23 22:24:22] (step=0034600) Train Loss: 5.4126, Train Steps/Sec: 1.12 + 48%|██████████████████████████████████████████████████████████████████▏ | 11166/23458 [3:13:26<2:57:33, 1.15it/s][2025-04-23 22:24:43] (step=0034625) Train Loss: 5.4218, Train Steps/Sec: 1.18 + 48%|██████████████████████████████████████████████████████████████████▎ | 11191/23458 [3:13:48<3:31:24, 1.03s/it][2025-04-23 22:25:05] (step=0034650) Train Loss: 5.4286, Train Steps/Sec: 1.13 + 48%|██████████████████████████████████████████████████████████████████▍ | 11216/23458 [3:14:09<2:52:54, 1.18it/s][2025-04-23 22:25:27] (step=0034675) Train Loss: 5.3075, Train Steps/Sec: 1.17 + 48%|██████████████████████████████████████████████████████████████████▌ | 11241/23458 [3:14:31<2:51:47, 1.19it/s][2025-04-23 22:25:48] (step=0034700) Train Loss: 5.3645, Train Steps/Sec: 1.17 + 48%|██████████████████████████████████████████████████████████████████▊ | 11266/23458 [3:14:52<2:55:48, 1.16it/s][2025-04-23 22:26:09] (step=0034725) Train Loss: 5.4036, Train Steps/Sec: 1.17 + 48%|██████████████████████████████████████████████████████████████████▉ | 11291/23458 [3:15:13<2:52:44, 1.17it/s][2025-04-23 22:26:31] (step=0034750) Train Loss: 5.3741, Train Steps/Sec: 1.17 + 48%|███████████████████████████████████████████████████████████████████ | 11316/23458 [3:15:35<2:50:56, 1.18it/s][2025-04-23 22:26:52] (step=0034775) Train Loss: 5.4892, Train Steps/Sec: 1.17 + 48%|███████████████████████████████████████████████████████████████████▏ | 11341/23458 [3:15:57<2:50:37, 1.18it/s][2025-04-23 22:27:14] (step=0034800) Train Loss: 5.3743, Train Steps/Sec: 1.13 + 48%|███████████████████████████████████████████████████████████████████▎ | 11366/23458 [3:16:18<2:54:56, 1.15it/s][2025-04-23 22:27:36] (step=0034825) Train Loss: 5.3553, Train Steps/Sec: 1.17 + 49%|███████████████████████████████████████████████████████████████████▍ | 11391/23458 [3:16:40<2:51:44, 1.17it/s][2025-04-23 22:27:58] (step=0034850) Train Loss: 5.4150, Train Steps/Sec: 1.13 + 49%|███████████████████████████████████████████████████████████████████▋ | 11416/23458 [3:17:02<2:50:06, 1.18it/s][2025-04-23 22:28:20] (step=0034875) Train Loss: 5.3872, Train Steps/Sec: 1.13 + 49%|███████████████████████████████████████████████████████████████████▊ | 11441/23458 [3:17:25<2:54:33, 1.15it/s][2025-04-23 22:28:42] (step=0034900) Train Loss: 5.4290, Train Steps/Sec: 1.13 + 49%|███████████████████████████████████████████████████████████████████▉ | 11466/23458 [3:17:46<2:52:57, 1.16it/s][2025-04-23 22:29:03] (step=0034925) Train Loss: 5.2310, Train Steps/Sec: 1.17 + 49%|████████████████████████████████████████████████████████████████████ | 11491/23458 [3:18:07<2:50:07, 1.17it/s][2025-04-23 22:29:25] (step=0034950) Train Loss: 5.3820, Train Steps/Sec: 1.17 + 49%|████████████████████████████████████████████████████████████████████▏ | 11516/23458 [3:18:29<2:48:34, 1.18it/s][2025-04-23 22:29:47] (step=0034975) Train Loss: 5.3105, Train Steps/Sec: 1.13 + 49%|████████████████████████████████████████████████████████████████████▍ | 11541/23458 [3:18:51<2:47:00, 1.19it/s][2025-04-23 22:30:08] (step=0035000) Train Loss: 5.4366, Train Steps/Sec: 1.17 + 49%|████████████████████████████████████████████████████████████████████▌ | 11566/23458 [3:19:12<2:51:59, 1.15it/s][2025-04-23 22:30:29] (step=0035025) Train Loss: 5.3499, Train Steps/Sec: 1.17 + 49%|████████████████████████████████████████████████████████████████████▋ | 11591/23458 [3:19:33<2:48:42, 1.17it/s][2025-04-23 22:30:51] (step=0035050) Train Loss: 5.4256, Train Steps/Sec: 1.17 + 50%|████████████████████████████████████████████████████████████████████▊ | 11616/23458 [3:19:55<2:46:57, 1.18it/s][2025-04-23 22:31:12] (step=0035075) Train Loss: 5.4234, Train Steps/Sec: 1.18 + 50%|████████████████████████████████████████████████████████████████████▉ | 11641/23458 [3:20:16<2:45:44, 1.19it/s][2025-04-23 22:31:33] (step=0035100) Train Loss: 5.3765, Train Steps/Sec: 1.17 + 50%|█████████████████████████████████████████████████████████████████████▏ | 11666/23458 [3:20:37<2:51:18, 1.15it/s][2025-04-23 22:31:55] (step=0035125) Train Loss: 5.4680, Train Steps/Sec: 1.17 + 50%|█████████████████████████████████████████████████████████████████████▎ | 11691/23458 [3:20:59<2:47:22, 1.17it/s][2025-04-23 22:32:16] (step=0035150) Train Loss: 5.4583, Train Steps/Sec: 1.17 + 50%|█████████████████████████████████████████████████████████████████████▍ | 11716/23458 [3:21:21<2:45:35, 1.18it/s][2025-04-23 22:32:38] (step=0035175) Train Loss: 5.3711, Train Steps/Sec: 1.13 + 50%|█████████████████████████████████████████████████████████████████████▌ | 11741/23458 [3:21:42<2:45:10, 1.18it/s][2025-04-23 22:33:00] (step=0035200) Train Loss: 5.4463, Train Steps/Sec: 1.17 + 50%|█████████████████████████████████████████████████████████████████████▋ | 11766/23458 [3:22:04<2:48:46, 1.15it/s][2025-04-23 22:33:21] (step=0035225) Train Loss: 5.3599, Train Steps/Sec: 1.17 + 50%|█████████████████████████████████████████████████████████████████████▊ | 11791/23458 [3:22:25<2:45:55, 1.17it/s][2025-04-23 22:33:42] (step=0035250) Train Loss: 5.3913, Train Steps/Sec: 1.17 + 50%|██████████████████████████████████████████████████████████████████████ | 11816/23458 [3:22:46<2:44:08, 1.18it/s][2025-04-23 22:34:04] (step=0035275) Train Loss: 5.4934, Train Steps/Sec: 1.18 + 50%|██████████████████████████████████████████████████████████████████████▏ | 11841/23458 [3:23:08<2:42:42, 1.19it/s][2025-04-23 22:34:25] (step=0035300) Train Loss: 5.4533, Train Steps/Sec: 1.17 + 51%|██████████████████████████████████████████████████████████████████████▎ | 11866/23458 [3:23:31<2:47:24, 1.15it/s][2025-04-23 22:34:48] (step=0035325) Train Loss: 5.3313, Train Steps/Sec: 1.08 + 51%|██████████████████████████████████████████████████████████████████████▍ | 11891/23458 [3:23:52<2:44:59, 1.17it/s][2025-04-23 22:35:09] (step=0035350) Train Loss: 5.4213, Train Steps/Sec: 1.17 + 51%|██████████████████████████████████████████████████████████████████████▌ | 11916/23458 [3:24:13<2:42:53, 1.18it/s][2025-04-23 22:35:31] (step=0035375) Train Loss: 5.3647, Train Steps/Sec: 1.18 + 51%|██████████████████████████████████████████████████████████████████████▊ | 11941/23458 [3:24:35<2:42:00, 1.18it/s][2025-04-23 22:35:52] (step=0035400) Train Loss: 5.4382, Train Steps/Sec: 1.17 + 51%|██████████████████████████████████████████████████████████████████████▉ | 11966/23458 [3:24:56<2:46:09, 1.15it/s][2025-04-23 22:36:13] (step=0035425) Train Loss: 5.3685, Train Steps/Sec: 1.18 + 51%|███████████████████████████████████████████████████████████████████████ | 11991/23458 [3:25:18<2:48:00, 1.14it/s][2025-04-23 22:36:35] (step=0035450) Train Loss: 5.3200, Train Steps/Sec: 1.13 + 51%|███████████████████████████████████████████████████████████████████████▏ | 12016/23458 [3:25:39<2:40:45, 1.19it/s][2025-04-23 22:36:57] (step=0035475) Train Loss: 5.3561, Train Steps/Sec: 1.18 + 51%|███████████████████████████████████████████████████████████████████████▎ | 12041/23458 [3:26:01<2:40:56, 1.18it/s][2025-04-23 22:37:19] (step=0035500) Train Loss: 5.3949, Train Steps/Sec: 1.13 + 51%|███████████████████████████████████████████████████████████████████████▍ | 12066/23458 [3:26:23<3:18:08, 1.04s/it][2025-04-23 22:37:41] (step=0035525) Train Loss: 5.4336, Train Steps/Sec: 1.13 + 52%|███████████████████████████████████████████████████████████████████████▋ | 12091/23458 [3:26:46<3:28:57, 1.10s/it][2025-04-23 22:38:03] (step=0035550) Train Loss: 5.3783, Train Steps/Sec: 1.13 + 52%|███████████████████████████████████████████████████████████████████████▊ | 12116/23458 [3:27:07<2:39:44, 1.18it/s][2025-04-23 22:38:24] (step=0035575) Train Loss: 5.3440, Train Steps/Sec: 1.17 + 52%|███████████████████████████████████████████████████████████████████████▉ | 12141/23458 [3:27:28<2:38:41, 1.19it/s][2025-04-23 22:38:46] (step=0035600) Train Loss: 5.3149, Train Steps/Sec: 1.17 + 52%|████████████████████████████████████████████████████████████████████████ | 12166/23458 [3:27:50<2:44:22, 1.14it/s][2025-04-23 22:39:08] (step=0035625) Train Loss: 5.4315, Train Steps/Sec: 1.13 + 52%|████████████████████████████████████████████████████████████████████████▏ | 12191/23458 [3:28:12<2:39:26, 1.18it/s][2025-04-23 22:39:29] (step=0035650) Train Loss: 5.4243, Train Steps/Sec: 1.18 + 52%|████████████████████████████████████████████████████████████████████████▍ | 12216/23458 [3:28:33<2:38:28, 1.18it/s][2025-04-23 22:39:50] (step=0035675) Train Loss: 5.3682, Train Steps/Sec: 1.17 + 52%|████████████████████████████████████████████████████████████████████████▌ | 12241/23458 [3:28:54<2:38:10, 1.18it/s][2025-04-23 22:40:12] (step=0035700) Train Loss: 5.3575, Train Steps/Sec: 1.17 + 52%|████████████████████████████████████████████████████████████████████████▋ | 12266/23458 [3:29:16<2:41:17, 1.16it/s][2025-04-23 22:40:33] (step=0035725) Train Loss: 5.3390, Train Steps/Sec: 1.17 + 52%|████████████████████████████████████████████████████████████████████████▊ | 12291/23458 [3:29:37<2:38:27, 1.17it/s][2025-04-23 22:40:54] (step=0035750) Train Loss: 5.4031, Train Steps/Sec: 1.18 + 53%|████████████████████████████████████████████████████████████████████████▉ | 12316/23458 [3:29:58<2:37:15, 1.18it/s][2025-04-23 22:41:16] (step=0035775) Train Loss: 5.3380, Train Steps/Sec: 1.18 + 53%|█████████████████████████████████████████████████████████████████████████▏ | 12341/23458 [3:30:20<2:35:32, 1.19it/s][2025-04-23 22:41:37] (step=0035800) Train Loss: 5.3929, Train Steps/Sec: 1.17 + 53%|█████████████████████████████████████████████████████████████████████████▎ | 12366/23458 [3:30:41<2:40:34, 1.15it/s][2025-04-23 22:41:58] (step=0035825) Train Loss: 5.3560, Train Steps/Sec: 1.17 + 53%|█████████████████████████████████████████████████████████████████████████▍ | 12391/23458 [3:31:02<2:38:30, 1.16it/s][2025-04-23 22:42:20] (step=0035850) Train Loss: 5.3544, Train Steps/Sec: 1.17 + 53%|█████████████████████████████████████████████████████████████████████████▌ | 12416/23458 [3:31:24<2:35:32, 1.18it/s][2025-04-23 22:42:41] (step=0035875) Train Loss: 5.3694, Train Steps/Sec: 1.18 + 53%|█████████████████████████████████████████████████████████████████████████▋ | 12441/23458 [3:31:46<2:34:40, 1.19it/s][2025-04-23 22:43:03] (step=0035900) Train Loss: 5.3476, Train Steps/Sec: 1.12 + 53%|█████████████████████████████████████████████████████████████████████████▊ | 12466/23458 [3:32:07<2:38:44, 1.15it/s][2025-04-23 22:43:25] (step=0035925) Train Loss: 5.4198, Train Steps/Sec: 1.17 + 53%|██████████████████████████████████████████████████████████████████████████ | 12491/23458 [3:32:29<2:35:56, 1.17it/s][2025-04-23 22:43:46] (step=0035950) Train Loss: 5.3488, Train Steps/Sec: 1.18 + 53%|██████████████████████████████████████████████████████████████████████████▏ | 12516/23458 [3:32:51<2:39:05, 1.15it/s][2025-04-23 22:44:08] (step=0035975) Train Loss: 5.2390, Train Steps/Sec: 1.13 + 53%|██████████████████████████████████████████████████████████████████████████▎ | 12541/23458 [3:33:12<2:32:39, 1.19it/s][2025-04-23 22:44:29] (step=0036000) Train Loss: 5.3828, Train Steps/Sec: 1.17 +You are using a model of type blip-2 to instantiate a model of type instructblip. This is not supported for all configurations of models and can yield errors. +[2025-04-23 22:44:29] text_config is None. Initializing the text config with default values (`OPTConfig`). +model_name_or_path:/tmp/haozhezhao/model/blip2-flan-t5-xl +image_place_holder: +tokenizer length after expend 32101 +tokenizer length before expend 32100 +100%|███████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████| 4/4 [03:57<00:00, 59.42s/it] +[2025-04-23 22:49:36] Finish Eval in 36000 steps...████████████████████████████████████████████████████████████████████████████████████████████████| 4/4 [03:57<00:00, 59.09s/it] +processing left-padding... +processing left-padding... +processing left-padding... +processing left-padding... +[2025-04-23 22:49:54] Saved checkpoint to checkpoint/Afteri2i_100kckpt_LLAVAT5_empty_Fixed_just_segment_pretrain_stage3_dreambench_recap_400k_t2i_400k_flux_200k_midjourney_150k_recovery_150k_3e4_epoch2_half_rotated/005-GPT-XL/checkpoints/0036000.pt +[2025-04-23 22:49:56] Removed old checkpoint: checkpoint/Afteri2i_100kckpt_LLAVAT5_empty_Fixed_just_segment_pretrain_stage3_dreambench_recap_400k_t2i_400k_flux_200k_midjourney_150k_recovery_150k_3e4_epoch2_half_rotated/005-GPT-XL/checkpoints/0034000.pt + 54%|██████████████████████████████████████████████████████████████████████████▍ | 12566/23458 [3:39:00<2:40:23, 1.13it/s][2025-04-23 22:50:17] (step=0036025) Train Loss: 5.3790, Train Steps/Sec: 0.07 + 54%|██████████████████████████████████████████████████████████████████████████▌ | 12591/23458 [3:39:22<2:34:25, 1.17it/s][2025-04-23 22:50:39] (step=0036050) Train Loss: 5.3539, Train Steps/Sec: 1.13 + 54%|██████████████████████████████████████████████████████████████████████████▊ | 12616/23458 [3:39:43<2:32:11, 1.19it/s][2025-04-23 22:51:01] (step=0036075) Train Loss: 5.3567, Train Steps/Sec: 1.18 + 54%|██████████████████████████████████████████████████████████████████████████▉ | 12641/23458 [3:40:04<2:31:17, 1.19it/s][2025-04-23 22:51:22] (step=0036100) Train Loss: 5.3533, Train Steps/Sec: 1.17 + 54%|███████████████████████████████████████████████████████████████████████████ | 12666/23458 [3:40:27<2:35:39, 1.16it/s][2025-04-23 22:51:44] (step=0036125) Train Loss: 5.4074, Train Steps/Sec: 1.13 + 54%|███████████████████████████████████████████████████████████████████████████▏ | 12691/23458 [3:40:48<2:32:57, 1.17it/s][2025-04-23 22:52:06] (step=0036150) Train Loss: 5.3441, Train Steps/Sec: 1.13 + 54%|███████████████████████████████████████████████████████████████████████████▎ | 12716/23458 [3:41:10<2:31:12, 1.18it/s][2025-04-23 22:52:27] (step=0036175) Train Loss: 5.3494, Train Steps/Sec: 1.18 + 54%|███████████████████████████████████████████████████████████████████████████▍ | 12741/23458 [3:41:32<2:29:56, 1.19it/s][2025-04-23 22:52:50] (step=0036200) Train Loss: 5.4470, Train Steps/Sec: 1.12 + 54%|███████████████████████████████████████████████████████████████████████████▋ | 12766/23458 [3:41:54<2:34:28, 1.15it/s][2025-04-23 22:53:12] (step=0036225) Train Loss: 5.3582, Train Steps/Sec: 1.13 + 55%|███████████████████████████████████████████████████████████████████████████▊ | 12791/23458 [3:42:16<2:31:13, 1.18it/s][2025-04-23 22:53:33] (step=0036250) Train Loss: 5.3350, Train Steps/Sec: 1.18 + 55%|███████████████████████████████████████████████████████████████████████████▉ | 12816/23458 [3:42:38<2:45:39, 1.07it/s][2025-04-23 22:53:55] (step=0036275) Train Loss: 5.4208, Train Steps/Sec: 1.13 + 55%|████████████████████████████████████████████████████████████████████████████ | 12841/23458 [3:42:59<2:35:05, 1.14it/s][2025-04-23 22:54:17] (step=0036300) Train Loss: 5.3479, Train Steps/Sec: 1.16 + 55%|████████████████████████████████████████████████████████████████████████████▏ | 12866/23458 [3:43:21<2:32:20, 1.16it/s][2025-04-23 22:54:38] (step=0036325) Train Loss: 5.3859, Train Steps/Sec: 1.18 + 55%|████████████████████████████████████████████████████████████████████████████▍ | 12891/23458 [3:43:42<2:29:37, 1.18it/s][2025-04-23 22:54:59] (step=0036350) Train Loss: 5.4346, Train Steps/Sec: 1.18 + 55%|████████████████████████████████████████████████████████████████████████████▌ | 12916/23458 [3:44:03<2:28:56, 1.18it/s][2025-04-23 22:55:20] (step=0036375) Train Loss: 5.3912, Train Steps/Sec: 1.18 + 55%|████████████████████████████████████████████████████████████████████████████▋ | 12941/23458 [3:44:24<2:27:30, 1.19it/s][2025-04-23 22:55:42] (step=0036400) Train Loss: 5.4131, Train Steps/Sec: 1.17 + 55%|████████████████████████████████████████████████████████████████████████████▊ | 12966/23458 [3:44:46<2:31:50, 1.15it/s][2025-04-23 22:56:03] (step=0036425) Train Loss: 5.4164, Train Steps/Sec: 1.17 + 55%|████████████████████████████████████████████████████████████████████████████▉ | 12991/23458 [3:45:07<2:28:48, 1.17it/s][2025-04-23 22:56:24] (step=0036450) Train Loss: 5.3990, Train Steps/Sec: 1.17 + 55%|█████████████████████████████████████████████████████████████████████████████▏ | 13016/23458 [3:45:28<2:27:18, 1.18it/s][2025-04-23 22:56:46] (step=0036475) Train Loss: 5.3417, Train Steps/Sec: 1.17 + 56%|█████████████████████████████████████████████████████████████████████████████▎ | 13041/23458 [3:45:50<2:26:08, 1.19it/s][2025-04-23 22:57:07] (step=0036500) Train Loss: 5.3738, Train Steps/Sec: 1.17 + 56%|█████████████████████████████████████████████████████████████████████████████▍ | 13066/23458 [3:46:11<2:29:50, 1.16it/s][2025-04-23 22:57:28] (step=0036525) Train Loss: 5.2784, Train Steps/Sec: 1.17 + 56%|█████████████████████████████████████████████████████████████████████████████▌ | 13091/23458 [3:46:32<2:27:36, 1.17it/s][2025-04-23 22:57:50] (step=0036550) Train Loss: 5.3725, Train Steps/Sec: 1.18 + 56%|█████████████████████████████████████████████████████████████████████████████▋ | 13116/23458 [3:46:54<2:26:55, 1.17it/s][2025-04-23 22:58:11] (step=0036575) Train Loss: 5.3762, Train Steps/Sec: 1.17 + 56%|█████████████████████████████████████████████████████████████████████████████▊ | 13141/23458 [3:47:15<2:25:10, 1.18it/s][2025-04-23 22:58:32] (step=0036600) Train Loss: 5.4691, Train Steps/Sec: 1.17 + 56%|██████████████████████████████████████████████████████████████████████████████ | 13166/23458 [3:47:37<2:29:39, 1.15it/s][2025-04-23 22:58:55] (step=0036625) Train Loss: 5.3568, Train Steps/Sec: 1.12 + 56%|██████████████████████████████████████████████████████████████████████████████▏ | 13191/23458 [3:48:00<2:46:33, 1.03it/s][2025-04-23 22:59:17] (step=0036650) Train Loss: 5.3300, Train Steps/Sec: 1.13 + 56%|██████████████████████████████████████████████████████████████████████████████▎ | 13216/23458 [3:48:21<2:24:45, 1.18it/s][2025-04-23 22:59:38] (step=0036675) Train Loss: 5.3973, Train Steps/Sec: 1.17 + 56%|██████████████████████████████████████████████████████████████████████████████▍ | 13241/23458 [3:48:42<2:24:20, 1.18it/s][2025-04-23 23:00:00] (step=0036700) Train Loss: 5.4537, Train Steps/Sec: 1.17 + 57%|██████████████████████████████████████████████████████████████████████████████▌ | 13266/23458 [3:49:04<2:27:28, 1.15it/s][2025-04-23 23:00:21] (step=0036725) Train Loss: 5.3412, Train Steps/Sec: 1.17 + 57%|██████████████████████████████████████████████████████████████████████████████▊ | 13291/23458 [3:49:25<2:25:16, 1.17it/s][2025-04-23 23:00:42] (step=0036750) Train Loss: 5.4091, Train Steps/Sec: 1.17 + 57%|██████████████████████████████████████████████████████████████████████████████▉ | 13316/23458 [3:49:48<2:24:14, 1.17it/s][2025-04-23 23:01:05] (step=0036775) Train Loss: 5.3964, Train Steps/Sec: 1.08 + 57%|███████████████████████████████████████████████████████████████████████████████ | 13341/23458 [3:50:09<2:22:19, 1.18it/s][2025-04-23 23:01:27] (step=0036800) Train Loss: 5.4315, Train Steps/Sec: 1.16 + 57%|███████████████████████████████████████████████████████████████████████████████▏ | 13366/23458 [3:50:32<2:26:35, 1.15it/s][2025-04-23 23:01:49] (step=0036825) Train Loss: 5.3615, Train Steps/Sec: 1.13 + 57%|███████████████████████████████████████████████████████████████████████████████▎ | 13391/23458 [3:50:54<2:24:25, 1.16it/s][2025-04-23 23:02:11] (step=0036850) Train Loss: 5.3504, Train Steps/Sec: 1.13 + 57%|███████████████████████████████████████████████████████████████████████████████▍ | 13416/23458 [3:51:16<2:32:36, 1.10it/s][2025-04-23 23:02:33] (step=0036875) Train Loss: 5.3631, Train Steps/Sec: 1.13 + 57%|███████████████████████████████████████████████████████████████████████████████▋ | 13441/23458 [3:51:37<2:20:43, 1.19it/s][2025-04-23 23:02:55] (step=0036900) Train Loss: 5.4203, Train Steps/Sec: 1.17 + 57%|███████████████████████████████████████████████████████████████████████████████▊ | 13466/23458 [3:51:58<2:24:15, 1.15it/s][2025-04-23 23:03:16] (step=0036925) Train Loss: 5.3840, Train Steps/Sec: 1.18 + 58%|███████████████████████████████████████████████████████████████████████████████▉ | 13491/23458 [3:52:21<2:21:49, 1.17it/s][2025-04-23 23:03:38] (step=0036950) Train Loss: 5.3981, Train Steps/Sec: 1.13 + 58%|████████████████████████████████████████████████████████████████████████████████ | 13516/23458 [3:52:42<2:20:04, 1.18it/s][2025-04-23 23:03:59] (step=0036975) Train Loss: 5.3261, Train Steps/Sec: 1.17 + 58%|████████████████████████████████████████████████████████████████████████████████▏ | 13541/23458 [3:53:03<2:19:33, 1.18it/s][2025-04-23 23:04:21] (step=0037000) Train Loss: 5.4062, Train Steps/Sec: 1.17 + 58%|████████████████████████████████████████████████████████████████████████████████▍ | 13566/23458 [3:53:25<2:23:21, 1.15it/s][2025-04-23 23:04:42] (step=0037025) Train Loss: 5.3930, Train Steps/Sec: 1.17 + 58%|████████████████████████████████████████████████████████████████████████████████▌ | 13591/23458 [3:53:46<2:20:34, 1.17it/s][2025-04-23 23:05:03] (step=0037050) Train Loss: 5.4281, Train Steps/Sec: 1.18 + 58%|████████████████████████████████████████████████████████████████████████████████▋ | 13616/23458 [3:54:07<2:19:38, 1.17it/s][2025-04-23 23:05:25] (step=0037075) Train Loss: 5.3868, Train Steps/Sec: 1.17 + 58%|████████████████████████████████████████████████████████████████████████████████▊ | 13641/23458 [3:54:29<2:17:52, 1.19it/s][2025-04-23 23:05:46] (step=0037100) Train Loss: 5.4008, Train Steps/Sec: 1.17 + 58%|████████████████████████████████████████████████████████████████████████████████▉ | 13666/23458 [3:54:50<2:21:37, 1.15it/s][2025-04-23 23:06:07] (step=0037125) Train Loss: 5.3622, Train Steps/Sec: 1.17 + 58%|█████████████████████████████████████████████████████████████████████████████████▏ | 13691/23458 [3:55:11<2:19:02, 1.17it/s][2025-04-23 23:06:29] (step=0037150) Train Loss: 5.3278, Train Steps/Sec: 1.18 + 58%|█████████████████████████████████████████████████████████████████████████████████▎ | 13716/23458 [3:55:33<2:17:32, 1.18it/s][2025-04-23 23:06:50] (step=0037175) Train Loss: 5.4297, Train Steps/Sec: 1.17 + 59%|█████████████████████████████████████████████████████████████████████████████████▍ | 13741/23458 [3:55:54<2:16:29, 1.19it/s][2025-04-23 23:07:11] (step=0037200) Train Loss: 5.4077, Train Steps/Sec: 1.17 + 59%|█████████████████████████████████████████████████████████████████████████████████▌ | 13766/23458 [3:56:15<2:20:48, 1.15it/s][2025-04-23 23:07:33] (step=0037225) Train Loss: 5.4055, Train Steps/Sec: 1.17 + 59%|█████████████████████████████████████████████████████████████████████████████████▋ | 13791/23458 [3:56:36<2:17:33, 1.17it/s][2025-04-23 23:07:54] (step=0037250) Train Loss: 5.3539, Train Steps/Sec: 1.17 + 59%|█████████████████████████████████████████████████████████████████████████████████▊ | 13816/23458 [3:56:58<2:15:45, 1.18it/s][2025-04-23 23:08:15] (step=0037275) Train Loss: 5.3229, Train Steps/Sec: 1.17 + 59%|██████████████████████████████████████████████████████████████████████████████████ | 13841/23458 [3:57:19<2:15:10, 1.19it/s][2025-04-23 23:08:37] (step=0037300) Train Loss: 5.3898, Train Steps/Sec: 1.17 + 59%|██████████████████████████████████████████████████████████████████████████████████▏ | 13866/23458 [3:57:41<2:18:36, 1.15it/s][2025-04-23 23:08:59] (step=0037325) Train Loss: 5.4445, Train Steps/Sec: 1.13 + 59%|██████████████████████████████████████████████████████████████████████████████████▎ | 13891/23458 [3:58:04<2:16:23, 1.17it/s][2025-04-23 23:09:21] (step=0037350) Train Loss: 5.3467, Train Steps/Sec: 1.12 + 59%|██████████████████████████████████████████████████████████████████████████████████▍ | 13916/23458 [3:58:25<2:14:33, 1.18it/s][2025-04-23 23:09:42] (step=0037375) Train Loss: 5.3916, Train Steps/Sec: 1.17 + 59%|██████████████████████████████████████████████████████████████████████████████████▌ | 13941/23458 [3:58:46<2:13:19, 1.19it/s][2025-04-23 23:10:04] (step=0037400) Train Loss: 5.4618, Train Steps/Sec: 1.17 + 60%|██████████████████████████████████████████████████████████████████████████████████▊ | 13966/23458 [3:59:08<2:19:56, 1.13it/s][2025-04-23 23:10:26] (step=0037425) Train Loss: 5.4282, Train Steps/Sec: 1.13 + 60%|██████████████████████████████████████████████████████████████████████████████████▉ | 13991/23458 [3:59:30<2:14:25, 1.17it/s][2025-04-23 23:10:47] (step=0037450) Train Loss: 5.4454, Train Steps/Sec: 1.17 + 60%|███████████████████████████████████████████████████████████████████████████████████ | 14016/23458 [3:59:52<2:13:59, 1.17it/s][2025-04-23 23:11:09] (step=0037475) Train Loss: 5.3074, Train Steps/Sec: 1.13 + 60%|███████████████████████████████████████████████████████████████████████████████████▏ | 14041/23458 [4:00:14<2:13:42, 1.17it/s][2025-04-23 23:11:32] (step=0037500) Train Loss: 5.3737, Train Steps/Sec: 1.08 + 60%|███████████████████████████████████████████████████████████████████████████████████▎ | 14066/23458 [4:00:36<2:15:28, 1.16it/s][2025-04-23 23:11:54] (step=0037525) Train Loss: 5.4218, Train Steps/Sec: 1.18 + 60%|███████████████████████████████████████████████████████████████████████████████████▍ | 14091/23458 [4:00:58<2:13:25, 1.17it/s][2025-04-23 23:12:16] (step=0037550) Train Loss: 5.4343, Train Steps/Sec: 1.13 + 60%|███████████████████████████████████████████████████████████████████████████████████▋ | 14116/23458 [4:01:20<2:11:52, 1.18it/s][2025-04-23 23:12:37] (step=0037575) Train Loss: 5.3233, Train Steps/Sec: 1.18 + 60%|███████████████████████████████████████████████████████████████████████████████████▊ | 14141/23458 [4:01:42<2:12:34, 1.17it/s][2025-04-23 23:12:59] (step=0037600) Train Loss: 5.3864, Train Steps/Sec: 1.13 + 60%|███████████████████████████████████████████████████████████████████████████████████▉ | 14166/23458 [4:02:03<2:14:17, 1.15it/s][2025-04-23 23:13:20] (step=0037625) Train Loss: 5.4162, Train Steps/Sec: 1.17 + 60%|████████████████████████████████████████████████████████████████████████████████████ | 14191/23458 [4:02:24<2:11:41, 1.17it/s][2025-04-23 23:13:42] (step=0037650) Train Loss: 5.3804, Train Steps/Sec: 1.18 + 61%|████████████████████████████████████████████████████████████████████████████████████▏ | 14216/23458 [4:02:46<2:10:11, 1.18it/s][2025-04-23 23:14:03] (step=0037675) Train Loss: 5.3594, Train Steps/Sec: 1.17 + 61%|████████████████████████████████████████████████████████████████████████████████████▍ | 14241/23458 [4:03:07<2:09:34, 1.19it/s][2025-04-23 23:14:24] (step=0037700) Train Loss: 5.4147, Train Steps/Sec: 1.17 + 61%|████████████████████████████████████████████████████████████████████████████████████▌ | 14266/23458 [4:03:28<2:13:52, 1.14it/s][2025-04-23 23:14:46] (step=0037725) Train Loss: 5.4446, Train Steps/Sec: 1.17 + 61%|████████████████████████████████████████████████████████████████████████████████████▋ | 14291/23458 [4:03:50<2:10:10, 1.17it/s][2025-04-23 23:15:07] (step=0037750) Train Loss: 5.4378, Train Steps/Sec: 1.17 + 61%|████████████████████████████████████████████████████████████████████████████████████▊ | 14316/23458 [4:04:11<2:08:53, 1.18it/s][2025-04-23 23:15:28] (step=0037775) Train Loss: 5.4126, Train Steps/Sec: 1.17 + 61%|████████████████████████████████████████████████████████████████████████████████████▉ | 14341/23458 [4:04:32<2:08:10, 1.19it/s][2025-04-23 23:15:50] (step=0037800) Train Loss: 5.3200, Train Steps/Sec: 1.17 + 61%|█████████████████████████████████████████████████████████████████████████████████████▏ | 14366/23458 [4:04:54<2:10:58, 1.16it/s][2025-04-23 23:16:11] (step=0037825) Train Loss: 5.3891, Train Steps/Sec: 1.17 + 61%|█████████████████████████████████████████████████████████████████████████████████████▎ | 14391/23458 [4:05:15<2:09:04, 1.17it/s][2025-04-23 23:16:32] (step=0037850) Train Loss: 5.3157, Train Steps/Sec: 1.17 + 61%|█████████████████████████████████████████████████████████████████████████████████████▍ | 14416/23458 [4:05:36<2:07:48, 1.18it/s][2025-04-23 23:16:54] (step=0037875) Train Loss: 5.3974, Train Steps/Sec: 1.17 + 62%|█████████████████████████████████████████████████████████████████████████████████████▌ | 14441/23458 [4:05:58<2:06:11, 1.19it/s][2025-04-23 23:17:15] (step=0037900) Train Loss: 5.3568, Train Steps/Sec: 1.17 + 62%|█████████████████████████████████████████████████████████████████████████████████████▋ | 14466/23458 [4:06:19<2:09:50, 1.15it/s][2025-04-23 23:17:36] (step=0037925) Train Loss: 5.4022, Train Steps/Sec: 1.17 + 62%|█████████████████████████████████████████████████████████████████████████████████████▊ | 14491/23458 [4:06:40<2:08:06, 1.17it/s][2025-04-23 23:17:58] (step=0037950) Train Loss: 5.3916, Train Steps/Sec: 1.17 + 62%|██████████████████████████████████████████████████████████████████████████████████████ | 14516/23458 [4:07:02<2:09:15, 1.15it/s][2025-04-23 23:18:20] (step=0037975) Train Loss: 5.3339, Train Steps/Sec: 1.13 + 62%|██████████████████████████████████████████████████████████████████████████████████████▏ | 14541/23458 [4:07:24<2:05:45, 1.18it/s][2025-04-23 23:18:41] (step=0038000) Train Loss: 5.3766, Train Steps/Sec: 1.17 +You are using a model of type blip-2 to instantiate a model of type instructblip. This is not supported for all configurations of models and can yield errors. +[2025-04-23 23:18:41] text_config is None. Initializing the text config with default values (`OPTConfig`). +model_name_or_path:/tmp/haozhezhao/model/blip2-flan-t5-xl +image_place_holder: +tokenizer length after expend 32101 +tokenizer length before expend 32100 +100%|███████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████| 4/4 [03:56<00:00, 59.23s/it] +[2025-04-23 23:23:47] Finish Eval in 38000 steps...████████████████████████████████████████████████████████████████████████████████████████████████| 4/4 [03:56<00:00, 58.93s/it] +processing left-padding... +processing left-padding... +processing left-padding... +processing left-padding... +[2025-04-23 23:24:07] Saved checkpoint to checkpoint/Afteri2i_100kckpt_LLAVAT5_empty_Fixed_just_segment_pretrain_stage3_dreambench_recap_400k_t2i_400k_flux_200k_midjourney_150k_recovery_150k_3e4_epoch2_half_rotated/005-GPT-XL/checkpoints/0038000.pt +[2025-04-23 23:24:09] Removed old checkpoint: checkpoint/Afteri2i_100kckpt_LLAVAT5_empty_Fixed_just_segment_pretrain_stage3_dreambench_recap_400k_t2i_400k_flux_200k_midjourney_150k_recovery_150k_3e4_epoch2_half_rotated/005-GPT-XL/checkpoints/0036000.pt + 62%|██████████████████████████████████████████████████████████████████████████████████████▎ | 14566/23458 [4:13:13<2:11:03, 1.13it/s][2025-04-23 23:24:30] (step=0038025) Train Loss: 5.3590, Train Steps/Sec: 0.07 + 62%|██████████████████████████████████████████████████████████████████████████████████████▍ | 14591/23458 [4:13:34<2:05:52, 1.17it/s][2025-04-23 23:24:51] (step=0038050) Train Loss: 5.3531, Train Steps/Sec: 1.17 + 62%|██████████████████████████████████████████████████████████████████████████████████████▌ | 14616/23458 [4:13:57<2:30:34, 1.02s/it][2025-04-23 23:25:15] (step=0038075) Train Loss: 5.3644, Train Steps/Sec: 1.08 + 62%|██████████████████████████████████████████████████████████████████████████████████████▊ | 14641/23458 [4:14:19<2:03:50, 1.19it/s][2025-04-23 23:25:36] (step=0038100) Train Loss: 5.3416, Train Steps/Sec: 1.17 + 63%|██████████████████████████████████████████████████████████████████████████████████████▉ | 14666/23458 [4:14:41<2:44:58, 1.13s/it][2025-04-23 23:25:58] (step=0038125) Train Loss: 5.3045, Train Steps/Sec: 1.13 + 63%|███████████████████████████████████████████████████████████████████████████████████████ | 14691/23458 [4:15:02<2:04:41, 1.17it/s][2025-04-23 23:26:19] (step=0038150) Train Loss: 5.4094, Train Steps/Sec: 1.18 + 63%|███████████████████████████████████████████████████████████████████████████████████████▏ | 14716/23458 [4:15:24<2:03:58, 1.18it/s][2025-04-23 23:26:41] (step=0038175) Train Loss: 5.3995, Train Steps/Sec: 1.13 + 63%|███████████████████████████████████████████████████████████████████████████████████████▎ | 14741/23458 [4:15:46<2:02:44, 1.18it/s][2025-04-23 23:27:04] (step=0038200) Train Loss: 5.3725, Train Steps/Sec: 1.13 + 63%|███████████████████████████████████████████████████████████████████████████████████████▍ | 14766/23458 [4:16:09<2:11:18, 1.10it/s][2025-04-23 23:27:26] (step=0038225) Train Loss: 5.3797, Train Steps/Sec: 1.11 + 63%|███████████████████████████████████████████████████████████████████████████████████████▋ | 14791/23458 [4:16:30<2:03:12, 1.17it/s][2025-04-23 23:27:47] (step=0038250) Train Loss: 5.3976, Train Steps/Sec: 1.18 + 63%|███████████████████████████████████████████████████████████████████████████████████████▊ | 14816/23458 [4:16:52<2:02:02, 1.18it/s][2025-04-23 23:28:09] (step=0038275) Train Loss: 5.4130, Train Steps/Sec: 1.13 + 63%|███████████████████████████████████████████████████████████████████████████████████████▉ | 14841/23458 [4:17:13<2:01:14, 1.18it/s][2025-04-23 23:28:31] (step=0038300) Train Loss: 5.3126, Train Steps/Sec: 1.17 + 63%|████████████████████████████████████████████████████████████████████████████████████████ | 14866/23458 [4:17:35<2:05:27, 1.14it/s][2025-04-23 23:28:52] (step=0038325) Train Loss: 5.4095, Train Steps/Sec: 1.17 + 63%|████████████████████████████████████████████████████████████████████████████████████████▏ | 14891/23458 [4:17:56<2:01:53, 1.17it/s][2025-04-23 23:29:13] (step=0038350) Train Loss: 5.3758, Train Steps/Sec: 1.18 + 64%|████████████████████████████████████████████████████████████████████████████████████████▍ | 14916/23458 [4:18:17<2:00:31, 1.18it/s][2025-04-23 23:29:35] (step=0038375) Train Loss: 5.4124, Train Steps/Sec: 1.17 + 64%|████████████████████████████████████████████████████████████████████████████████████████▌ | 14941/23458 [4:18:39<1:59:47, 1.18it/s][2025-04-23 23:29:56] (step=0038400) Train Loss: 5.4171, Train Steps/Sec: 1.17 + 64%|████████████████████████████████████████████████████████████████████████████████████████▋ | 14966/23458 [4:19:00<2:03:16, 1.15it/s][2025-04-23 23:30:17] (step=0038425) Train Loss: 5.4059, Train Steps/Sec: 1.17 + 64%|████████████████████████████████████████████████████████████████████████████████████████▊ | 14991/23458 [4:19:21<2:00:34, 1.17it/s][2025-04-23 23:30:39] (step=0038450) Train Loss: 5.3667, Train Steps/Sec: 1.17 + 64%|████████████████████████████████████████████████████████████████████████████████████████▉ | 15016/23458 [4:19:43<1:59:42, 1.18it/s][2025-04-23 23:31:00] (step=0038475) Train Loss: 5.3690, Train Steps/Sec: 1.17 + 64%|█████████████████████████████████████████████████████████████████████████████████████████▏ | 15041/23458 [4:20:04<1:57:45, 1.19it/s][2025-04-23 23:31:21] (step=0038500) Train Loss: 5.3078, Train Steps/Sec: 1.17 + 64%|█████████████████████████████████████████████████████████████████████████████████████████▎ | 15066/23458 [4:20:25<2:00:51, 1.16it/s][2025-04-23 23:31:43] (step=0038525) Train Loss: 5.3146, Train Steps/Sec: 1.18 + 64%|█████████████████████████████████████████████████████████████████████████████████████████▍ | 15091/23458 [4:20:47<1:58:53, 1.17it/s][2025-04-23 23:32:04] (step=0038550) Train Loss: 5.3358, Train Steps/Sec: 1.17 + 64%|█████████████████████████████████████████████████████████████████████████████████████████▌ | 15116/23458 [4:21:08<1:58:02, 1.18it/s][2025-04-23 23:32:25] (step=0038575) Train Loss: 5.3623, Train Steps/Sec: 1.17 + 65%|█████████████████████████████████████████████████████████████████████████████████████████▋ | 15141/23458 [4:21:29<1:56:43, 1.19it/s][2025-04-23 23:32:47] (step=0038600) Train Loss: 5.3100, Train Steps/Sec: 1.17 + 65%|█████████████████████████████████████████████████████████████████████████████████████████▊ | 15166/23458 [4:21:51<1:59:43, 1.15it/s][2025-04-23 23:33:08] (step=0038625) Train Loss: 5.3963, Train Steps/Sec: 1.18 + 65%|██████████████████████████████████████████████████████████████████████████████████████████ | 15191/23458 [4:22:13<2:25:13, 1.05s/it][2025-04-23 23:33:30] (step=0038650) Train Loss: 5.3278, Train Steps/Sec: 1.12 + 65%|██████████████████████████████████████████████████████████████████████████████████████████▏ | 15216/23458 [4:22:34<1:55:47, 1.19it/s][2025-04-23 23:33:52] (step=0038675) Train Loss: 5.3523, Train Steps/Sec: 1.18 + 65%|██████████████████████████████████████████████████████████████████████████████████████████▎ | 15241/23458 [4:22:55<1:55:11, 1.19it/s][2025-04-23 23:34:13] (step=0038700) Train Loss: 5.3518, Train Steps/Sec: 1.17 + 65%|██████████████████████████████████████████████████████████████████████████████████████████▍ | 15266/23458 [4:23:17<1:58:15, 1.15it/s][2025-04-23 23:34:34] (step=0038725) Train Loss: 5.3352, Train Steps/Sec: 1.18 + 65%|██████████████████████████████████████████████████████████████████████████████████████████▌ | 15291/23458 [4:23:39<1:56:21, 1.17it/s][2025-04-23 23:34:56] (step=0038750) Train Loss: 5.2914, Train Steps/Sec: 1.13 + 65%|██████████████████████████████████████████████████████████████████████████████████████████▊ | 15316/23458 [4:24:00<1:55:13, 1.18it/s][2025-04-23 23:35:19] (step=0038775) Train Loss: 5.3661, Train Steps/Sec: 1.12 + 65%|██████████████████████████████████████████████████████████████████████████████████████████▉ | 15341/23458 [4:24:23<1:54:05, 1.19it/s][2025-04-23 23:35:41] (step=0038800) Train Loss: 5.2478, Train Steps/Sec: 1.12 + 66%|███████████████████████████████████████████████████████████████████████████████████████████ | 15366/23458 [4:24:46<2:09:18, 1.04it/s][2025-04-23 23:36:03] (step=0038825) Train Loss: 5.3765, Train Steps/Sec: 1.13 + 66%|███████████████████████████████████████████████████████████████████████████████████████████▏ | 15391/23458 [4:25:08<2:03:11, 1.09it/s][2025-04-23 23:36:25] (step=0038850) Train Loss: 5.3632, Train Steps/Sec: 1.13 + 66%|███████████████████████████████████████████████████████████████████████████████████████████▎ | 15416/23458 [4:25:29<1:54:09, 1.17it/s][2025-04-23 23:36:47] (step=0038875) Train Loss: 5.4172, Train Steps/Sec: 1.17 + 66%|███████████████████████████████████████████████████████████████████████████████████████████▍ | 15441/23458 [4:25:51<1:52:31, 1.19it/s][2025-04-23 23:37:08] (step=0038900) Train Loss: 5.4116, Train Steps/Sec: 1.17 + 66%|███████████████████████████████████████████████████████████████████████████████████████████▋ | 15466/23458 [4:26:13<1:55:40, 1.15it/s][2025-04-23 23:37:30] (step=0038925) Train Loss: 5.3879, Train Steps/Sec: 1.13 + 66%|███████████████████████████████████████████████████████████████████████████████████████████▊ | 15491/23458 [4:26:35<1:55:53, 1.15it/s][2025-04-23 23:37:52] (step=0038950) Train Loss: 5.3630, Train Steps/Sec: 1.13 + 66%|███████████████████████████████████████████████████████████████████████████████████████████▉ | 15516/23458 [4:26:56<1:51:54, 1.18it/s][2025-04-23 23:38:14] (step=0038975) Train Loss: 5.4462, Train Steps/Sec: 1.18 + 66%|████████████████████████████████████████████████████████████████████████████████████████████ | 15541/23458 [4:27:18<1:51:29, 1.18it/s][2025-04-23 23:38:35] (step=0039000) Train Loss: 5.5286, Train Steps/Sec: 1.17 + 66%|████████████████████████████████████████████████████████████████████████████████████████████▏ | 15566/23458 [4:27:39<1:53:53, 1.15it/s][2025-04-23 23:38:56] (step=0039025) Train Loss: 5.4227, Train Steps/Sec: 1.18 + 66%|████████████████████████████████████████████████████████████████████████████████████████████▍ | 15591/23458 [4:28:00<1:51:55, 1.17it/s][2025-04-23 23:39:18] (step=0039050) Train Loss: 5.3918, Train Steps/Sec: 1.17 + 67%|████████████████████████████████████████████████████████████████████████████████████████████▌ | 15616/23458 [4:28:21<1:50:35, 1.18it/s][2025-04-23 23:39:39] (step=0039075) Train Loss: 5.3715, Train Steps/Sec: 1.17 + 67%|████████████████████████████████████████████████████████████████████████████████████████████▋ | 15641/23458 [4:28:43<1:49:59, 1.18it/s][2025-04-23 23:40:00] (step=0039100) Train Loss: 5.3515, Train Steps/Sec: 1.17 + 67%|████████████████████████████████████████████████████████████████████████████████████████████▊ | 15666/23458 [4:29:04<1:52:16, 1.16it/s][2025-04-23 23:40:21] (step=0039125) Train Loss: 5.3846, Train Steps/Sec: 1.18 + 67%|████████████████████████████████████████████████████████████████████████████████████████████▉ | 15691/23458 [4:29:25<1:50:22, 1.17it/s][2025-04-23 23:40:43] (step=0039150) Train Loss: 5.3918, Train Steps/Sec: 1.17 + 67%|█████████████████████████████████████████████████████████████████████████████████████████████ | 15716/23458 [4:29:47<1:49:35, 1.18it/s][2025-04-23 23:41:04] (step=0039175) Train Loss: 5.3484, Train Steps/Sec: 1.17 + 67%|█████████████████████████████████████████████████████████████████████████████████████████████▎ | 15741/23458 [4:30:08<1:48:45, 1.18it/s][2025-04-23 23:41:26] (step=0039200) Train Loss: 5.3339, Train Steps/Sec: 1.17 + 67%|█████████████████████████████████████████████████████████████████████████████████████████████▍ | 15766/23458 [4:30:29<1:51:27, 1.15it/s][2025-04-23 23:41:47] (step=0039225) Train Loss: 5.3835, Train Steps/Sec: 1.17 + 67%|█████████████████████████████████████████████████████████████████████████████████████████████▌ | 15791/23458 [4:30:51<1:48:48, 1.17it/s][2025-04-23 23:42:08] (step=0039250) Train Loss: 5.4423, Train Steps/Sec: 1.17 + 67%|█████████████████████████████████████████████████████████████████████████████████████████████▋ | 15816/23458 [4:31:12<1:47:38, 1.18it/s][2025-04-23 23:42:29] (step=0039275) Train Loss: 5.4178, Train Steps/Sec: 1.17 + 68%|█████████████████████████████████████████████████████████████████████████████████████████████▊ | 15841/23458 [4:31:33<1:47:06, 1.19it/s][2025-04-23 23:42:51] (step=0039300) Train Loss: 5.3216, Train Steps/Sec: 1.17 + 68%|██████████████████████████████████████████████████████████████████████████████████████████████ | 15866/23458 [4:31:56<1:49:48, 1.15it/s][2025-04-23 23:43:13] (step=0039325) Train Loss: 5.3191, Train Steps/Sec: 1.13 + 68%|██████████████████████████████████████████████████████████████████████████████████████████████▏ | 15891/23458 [4:32:17<1:47:31, 1.17it/s][2025-04-23 23:43:34] (step=0039350) Train Loss: 5.3856, Train Steps/Sec: 1.17 + 68%|██████████████████████████████████████████████████████████████████████████████████████████████▎ | 15916/23458 [4:32:38<1:46:25, 1.18it/s][2025-04-23 23:43:56] (step=0039375) Train Loss: 5.3545, Train Steps/Sec: 1.17 + 68%|██████████████████████████████████████████████████████████████████████████████████████████████▍ | 15941/23458 [4:33:00<1:45:24, 1.19it/s][2025-04-23 23:44:18] (step=0039400) Train Loss: 5.4015, Train Steps/Sec: 1.13 + 68%|██████████████████████████████████████████████████████████████████████████████████████████████▌ | 15966/23458 [4:33:22<1:48:13, 1.15it/s][2025-04-23 23:44:39] (step=0039425) Train Loss: 5.3459, Train Steps/Sec: 1.18 + 68%|██████████████████████████████████████████████████████████████████████████████████████████████▊ | 15991/23458 [4:33:44<1:47:58, 1.15it/s][2025-04-23 23:45:01] (step=0039450) Train Loss: 5.2856, Train Steps/Sec: 1.13 + 68%|██████████████████████████████████████████████████████████████████████████████████████████████▉ | 16016/23458 [4:34:05<1:44:29, 1.19it/s][2025-04-23 23:45:23] (step=0039475) Train Loss: 5.3826, Train Steps/Sec: 1.18 + 68%|███████████████████████████████████████████████████████████████████████████████████████████████ | 16041/23458 [4:34:28<2:10:46, 1.06s/it][2025-04-23 23:45:46] (step=0039500) Train Loss: 5.3143, Train Steps/Sec: 1.08 + 68%|███████████████████████████████████████████████████████████████████████████████████████████████▏ | 16066/23458 [4:34:51<1:47:26, 1.15it/s][2025-04-23 23:46:08] (step=0039525) Train Loss: 5.3453, Train Steps/Sec: 1.13 + 69%|███████████████████████████████████████████████████████████████████████████████████████████████▎ | 16091/23458 [4:35:12<1:45:36, 1.16it/s][2025-04-23 23:46:29] (step=0039550) Train Loss: 5.3632, Train Steps/Sec: 1.17 + 69%|███████████████████████████████████████████████████████████████████████████████████████████████▍ | 16116/23458 [4:35:34<2:06:15, 1.03s/it][2025-04-23 23:46:51] (step=0039575) Train Loss: 5.3582, Train Steps/Sec: 1.12 + 69%|███████████████████████████████████████████████████████████████████████████████████████████████▋ | 16141/23458 [4:35:55<1:42:30, 1.19it/s][2025-04-23 23:47:13] (step=0039600) Train Loss: 5.4030, Train Steps/Sec: 1.17 + 69%|███████████████████████████████████████████████████████████████████████████████████████████████▊ | 16166/23458 [4:36:17<1:45:34, 1.15it/s][2025-04-23 23:47:34] (step=0039625) Train Loss: 5.3576, Train Steps/Sec: 1.17 + 69%|███████████████████████████████████████████████████████████████████████████████████████████████▉ | 16191/23458 [4:36:38<1:43:16, 1.17it/s][2025-04-23 23:47:55] (step=0039650) Train Loss: 5.4064, Train Steps/Sec: 1.17 + 69%|████████████████████████████████████████████████████████████████████████████████████████████████ | 16216/23458 [4:37:00<1:47:56, 1.12it/s][2025-04-23 23:48:18] (step=0039675) Train Loss: 5.3791, Train Steps/Sec: 1.12 + 69%|████████████████████████████████████████████████████████████████████████████████████████████████▏ | 16241/23458 [4:37:22<1:41:24, 1.19it/s][2025-04-23 23:48:39] (step=0039700) Train Loss: 5.3443, Train Steps/Sec: 1.17 + 69%|████████████████████████████████████████████████████████████████████████████████████████████████▍ | 16266/23458 [4:37:43<1:43:56, 1.15it/s][2025-04-23 23:49:00] (step=0039725) Train Loss: 5.4084, Train Steps/Sec: 1.17 + 69%|████████████████████████████████████████████████████████████████████████████████████████████████▌ | 16291/23458 [4:38:04<1:42:07, 1.17it/s][2025-04-23 23:49:22] (step=0039750) Train Loss: 5.2893, Train Steps/Sec: 1.17 + 70%|████████████████████████████████████████████████████████████████████████████████████████████████▋ | 16316/23458 [4:38:26<1:41:09, 1.18it/s][2025-04-23 23:49:43] (step=0039775) Train Loss: 5.3140, Train Steps/Sec: 1.17 + 70%|████████████████████████████████████████████████████████████████████████████████████████████████▊ | 16341/23458 [4:38:47<1:40:06, 1.18it/s][2025-04-23 23:50:05] (step=0039800) Train Loss: 5.3849, Train Steps/Sec: 1.17 + 70%|████████████████████████████████████████████████████████████████████████████████████████████████▉ | 16366/23458 [4:39:09<1:42:40, 1.15it/s][2025-04-23 23:50:26] (step=0039825) Train Loss: 5.3833, Train Steps/Sec: 1.17 + 70%|█████████████████████████████████████████████████████████████████████████████████████████████████ | 16391/23458 [4:39:30<1:40:51, 1.17it/s][2025-04-23 23:50:47] (step=0039850) Train Loss: 5.3417, Train Steps/Sec: 1.18 + 70%|█████████████████████████████████████████████████████████████████████████████████████████████████▎ | 16416/23458 [4:39:51<1:38:58, 1.19it/s][2025-04-23 23:51:08] (step=0039875) Train Loss: 5.3438, Train Steps/Sec: 1.18 + 70%|█████████████████████████████████████████████████████████████████████████████████████████████████▍ | 16441/23458 [4:40:12<1:38:36, 1.19it/s][2025-04-23 23:51:30] (step=0039900) Train Loss: 5.3542, Train Steps/Sec: 1.17 + 70%|█████████████████████████████████████████████████████████████████████████████████████████████████▌ | 16466/23458 [4:40:34<1:41:12, 1.15it/s][2025-04-23 23:51:51] (step=0039925) Train Loss: 5.3098, Train Steps/Sec: 1.17 + 70%|█████████████████████████████████████████████████████████████████████████████████████████████████▋ | 16491/23458 [4:40:55<1:39:24, 1.17it/s][2025-04-23 23:52:12] (step=0039950) Train Loss: 5.3280, Train Steps/Sec: 1.17 + 70%|█████████████████████████████████████████████████████████████████████████████████████████████████▊ | 16516/23458 [4:41:17<1:41:46, 1.14it/s][2025-04-23 23:52:35] (step=0039975) Train Loss: 5.4016, Train Steps/Sec: 1.13 + 71%|██████████████████████████████████████████████████████████████████████████████████████████████████ | 16541/23458 [4:41:39<1:36:59, 1.19it/s][2025-04-23 23:52:56] (step=0040000) Train Loss: 5.3193, Train Steps/Sec: 1.17 +You are using a model of type blip-2 to instantiate a model of type instructblip. This is not supported for all configurations of models and can yield errors. +[2025-04-23 23:52:56] text_config is None. Initializing the text config with default values (`OPTConfig`). +model_name_or_path:/tmp/haozhezhao/model/blip2-flan-t5-xl +image_place_holder: +tokenizer length after expend 32101 +tokenizer length before expend 32100 +100%|███████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████| 4/4 [03:57<00:00, 59.34s/it] +[2025-04-23 23:58:01] Finish Eval in 40000 steps...████████████████████████████████████████████████████████████████████████████████████████████████| 4/4 [03:56<00:00, 59.07s/it] +processing left-padding... +processing left-padding... +processing left-padding... +processing left-padding... +[2025-04-23 23:58:19] Saved checkpoint to checkpoint/Afteri2i_100kckpt_LLAVAT5_empty_Fixed_just_segment_pretrain_stage3_dreambench_recap_400k_t2i_400k_flux_200k_midjourney_150k_recovery_150k_3e4_epoch2_half_rotated/005-GPT-XL/checkpoints/0040000.pt +[2025-04-23 23:58:21] Removed old checkpoint: checkpoint/Afteri2i_100kckpt_LLAVAT5_empty_Fixed_just_segment_pretrain_stage3_dreambench_recap_400k_t2i_400k_flux_200k_midjourney_150k_recovery_150k_3e4_epoch2_half_rotated/005-GPT-XL/checkpoints/0038000.pt + 71%|██████████████████████████████████████████████████████████████████████████████████████████████████▏ | 16566/23458 [4:47:25<1:41:27, 1.13it/s][2025-04-23 23:58:42] (step=0040025) Train Loss: 5.4095, Train Steps/Sec: 0.07 + 71%|██████████████████████████████████████████████████████████████████████████████████████████████████▎ | 16591/23458 [4:47:47<1:41:10, 1.13it/s][2025-04-23 23:59:04] (step=0040050) Train Loss: 5.2760, Train Steps/Sec: 1.13 + 71%|██████████████████████████████████████████████████████████████████████████████████████████████████▍ | 16616/23458 [4:48:08<1:36:28, 1.18it/s][2025-04-23 23:59:25] (step=0040075) Train Loss: 5.4388, Train Steps/Sec: 1.18 + 71%|██████████████████████████████████████████████████████████████████████████████████████████████████▌ | 16641/23458 [4:48:29<1:36:22, 1.18it/s][2025-04-23 23:59:47] (step=0040100) Train Loss: 5.3207, Train Steps/Sec: 1.17 + 71%|██████████████████████████████████████████████████████████████████████████████████████████████████▊ | 16666/23458 [4:48:52<1:38:04, 1.15it/s][2025-04-24 00:00:09] (step=0040125) Train Loss: 5.3305, Train Steps/Sec: 1.13 + 71%|██████████████████████████████████████████████████████████████████████████████████████████████████▉ | 16691/23458 [4:49:14<1:40:28, 1.12it/s][2025-04-24 00:00:31] (step=0040150) Train Loss: 5.3170, Train Steps/Sec: 1.12 + 71%|███████████████████████████████████████████████████████████████████████████████████████████████████ | 16716/23458 [4:49:36<1:35:36, 1.18it/s][2025-04-24 00:00:53] (step=0040175) Train Loss: 5.3081, Train Steps/Sec: 1.13 + 71%|███████████████████████████████████████████████████████████████████████████████████████████████████▏ | 16741/23458 [4:49:57<1:34:12, 1.19it/s][2025-04-24 00:01:15] (step=0040200) Train Loss: 5.4608, Train Steps/Sec: 1.17 + 71%|███████████████████████████████████████████████████████████████████████████████████████████████████▎ | 16766/23458 [4:50:20<1:47:57, 1.03it/s][2025-04-24 00:01:37] (step=0040225) Train Loss: 5.2915, Train Steps/Sec: 1.12 + 72%|███████████████████████████████████████████████████████████████████████████████████████████████████▍ | 16791/23458 [4:50:42<1:35:08, 1.17it/s][2025-04-24 00:01:59] (step=0040250) Train Loss: 5.3805, Train Steps/Sec: 1.13 + 72%|███████████████████████████████████████████████████████████████████████████████████████████████████▋ | 16816/23458 [4:51:03<1:34:05, 1.18it/s][2025-04-24 00:02:20] (step=0040275) Train Loss: 5.3991, Train Steps/Sec: 1.17 + 72%|███████████████████████████████████████████████████████████████████████████████████████████████████▊ | 16841/23458 [4:51:24<1:32:54, 1.19it/s][2025-04-24 00:02:42] (step=0040300) Train Loss: 5.3909, Train Steps/Sec: 1.17 + 72%|███████████████████████████████████████████████████████████████████████████████████████████████████▉ | 16866/23458 [4:51:46<1:35:12, 1.15it/s][2025-04-24 00:03:03] (step=0040325) Train Loss: 5.3365, Train Steps/Sec: 1.17 + 72%|████████████████████████████████████████████████████████████████████████████████████████████████████ | 16891/23458 [4:52:07<1:33:28, 1.17it/s][2025-04-24 00:03:24] (step=0040350) Train Loss: 5.3318, Train Steps/Sec: 1.18 + 72%|████████████████████████████████████████████████████████████████████████████████████████████████████▏ | 16916/23458 [4:52:28<1:32:16, 1.18it/s][2025-04-24 00:03:46] (step=0040375) Train Loss: 5.4299, Train Steps/Sec: 1.17 + 72%|████████████████████████████████████████████████████████████████████████████████████████████████████▍ | 16941/23458 [4:52:51<1:39:12, 1.09it/s][2025-04-24 00:04:08] (step=0040400) Train Loss: 5.3854, Train Steps/Sec: 1.12 + 72%|████████████████████████████████████████████████████████████████████████████████████████████████████▌ | 16966/23458 [4:53:12<1:33:38, 1.16it/s][2025-04-24 00:04:29] (step=0040425) Train Loss: 5.3985, Train Steps/Sec: 1.18 + 72%|████████████████████████████████████████████████████████████████████████████████████████████████████▋ | 16991/23458 [4:53:33<1:32:09, 1.17it/s][2025-04-24 00:04:51] (step=0040450) Train Loss: 5.3305, Train Steps/Sec: 1.18 + 73%|████████████████████████████████████████████████████████████████████████████████████████████████████▊ | 17016/23458 [4:53:54<1:30:37, 1.18it/s][2025-04-24 00:05:12] (step=0040475) Train Loss: 5.4187, Train Steps/Sec: 1.18 + 73%|████████████████████████████████████████████████████████████████████████████████████████████████████▉ | 17041/23458 [4:54:16<1:30:00, 1.19it/s][2025-04-24 00:05:33] (step=0040500) Train Loss: 5.3614, Train Steps/Sec: 1.17 + 73%|█████████████████████████████████████████████████████████████████████████████████████████████████████ | 17066/23458 [4:54:37<1:32:13, 1.16it/s][2025-04-24 00:05:54] (step=0040525) Train Loss: 5.3796, Train Steps/Sec: 1.18 + 73%|█████████████████████████████████████████████████████████████████████████████████████████████████████▎ | 17091/23458 [4:54:58<1:30:38, 1.17it/s][2025-04-24 00:06:16] (step=0040550) Train Loss: 5.3545, Train Steps/Sec: 1.17 + 73%|█████████████████████████████████████████████████████████████████████████████████████████████████████▍ | 17116/23458 [4:55:20<1:29:42, 1.18it/s][2025-04-24 00:06:37] (step=0040575) Train Loss: 5.4509, Train Steps/Sec: 1.17 + 73%|█████████████████████████████████████████████████████████████████████████████████████████████████████▌ | 17141/23458 [4:55:41<1:28:47, 1.19it/s][2025-04-24 00:06:58] (step=0040600) Train Loss: 5.4385, Train Steps/Sec: 1.17 + 73%|█████████████████████████████████████████████████████████████████████████████████████████████████████▋ | 17166/23458 [4:56:02<1:30:40, 1.16it/s][2025-04-24 00:07:20] (step=0040625) Train Loss: 5.3605, Train Steps/Sec: 1.17 + 73%|█████████████████████████████████████████████████████████████████████████████████████████████████████▊ | 17191/23458 [4:56:25<1:48:01, 1.03s/it][2025-04-24 00:07:42] (step=0040650) Train Loss: 5.3840, Train Steps/Sec: 1.13 + 73%|██████████████████████████████████████████████████████████████████████████████████████████████████████ | 17216/23458 [4:56:46<1:27:52, 1.18it/s][2025-04-24 00:08:03] (step=0040675) Train Loss: 5.4537, Train Steps/Sec: 1.17 + 73%|██████████████████████████████████████████████████████████████████████████████████████████████████████▏ | 17241/23458 [4:57:07<1:27:25, 1.19it/s][2025-04-24 00:08:25] (step=0040700) Train Loss: 5.3196, Train Steps/Sec: 1.13 + 74%|██████████████████████████████████████████████████████████████████████████████████████████████████████▎ | 17266/23458 [4:57:29<1:29:32, 1.15it/s][2025-04-24 00:08:47] (step=0040725) Train Loss: 5.3378, Train Steps/Sec: 1.17 + 74%|██████████████████████████████████████████████████████████████████████████████████████████████████████▍ | 17291/23458 [4:57:51<1:27:40, 1.17it/s][2025-04-24 00:09:08] (step=0040750) Train Loss: 5.4329, Train Steps/Sec: 1.17 + 74%|██████████████████████████████████████████████████████████████████████████████████████████████████████▌ | 17316/23458 [4:58:13<1:26:55, 1.18it/s][2025-04-24 00:09:30] (step=0040775) Train Loss: 5.3876, Train Steps/Sec: 1.13 + 74%|██████████████████████████████████████████████████████████████████████████████████████████████████████▊ | 17341/23458 [4:58:34<1:25:29, 1.19it/s][2025-04-24 00:09:52] (step=0040800) Train Loss: 5.3579, Train Steps/Sec: 1.17 + 74%|██████████████████████████████████████████████████████████████████████████████████████████████████████▉ | 17366/23458 [4:58:57<1:31:04, 1.11it/s][2025-04-24 00:10:14] (step=0040825) Train Loss: 5.3172, Train Steps/Sec: 1.09 + 74%|███████████████████████████████████████████████████████████████████████████████████████████████████████ | 17391/23458 [4:59:18<1:26:11, 1.17it/s][2025-04-24 00:10:36] (step=0040850) Train Loss: 5.3524, Train Steps/Sec: 1.17 + 74%|███████████████████████████████████████████████████████████████████████████████████████████████████████▏ | 17416/23458 [4:59:40<1:25:10, 1.18it/s][2025-04-24 00:10:57] (step=0040875) Train Loss: 5.3476, Train Steps/Sec: 1.17 + 74%|███████████████████████████████████████████████████████████████████████████████████████████████████████▎ | 17441/23458 [5:00:02<1:25:49, 1.17it/s][2025-04-24 00:11:19] (step=0040900) Train Loss: 5.3729, Train Steps/Sec: 1.12 + 74%|███████████████████████████████████████████████████████████████████████████████████████████████████████▍ | 17466/23458 [5:00:23<1:26:33, 1.15it/s][2025-04-24 00:11:41] (step=0040925) Train Loss: 5.4439, Train Steps/Sec: 1.17 + 75%|███████████████████████████████████████████████████████████████████████████████████████████████████████▋ | 17491/23458 [5:00:46<1:29:21, 1.11it/s][2025-04-24 00:12:03] (step=0040950) Train Loss: 5.2763, Train Steps/Sec: 1.13 + 75%|███████████████████████████████████████████████████████████████████████████████████████████████████████▊ | 17516/23458 [5:01:07<1:23:57, 1.18it/s][2025-04-24 00:12:24] (step=0040975) Train Loss: 5.4001, Train Steps/Sec: 1.17 + 75%|███████████████████████████████████████████████████████████████████████████████████████████████████████▉ | 17541/23458 [5:01:28<1:23:04, 1.19it/s][2025-04-24 00:12:46] (step=0041000) Train Loss: 5.3769, Train Steps/Sec: 1.17 + 75%|████████████████████████████████████████████████████████████████████████████████████████████████████████ | 17566/23458 [5:01:50<1:25:08, 1.15it/s][2025-04-24 00:13:07] (step=0041025) Train Loss: 5.4464, Train Steps/Sec: 1.18 + 75%|████████████████████████████████████████████████████████████████████████████████████████████████████████▏ | 17591/23458 [5:02:11<1:23:26, 1.17it/s][2025-04-24 00:13:28] (step=0041050) Train Loss: 5.3393, Train Steps/Sec: 1.17 + 75%|████████████████████████████████████████████████████████████████████████████████████████████████████████▍ | 17616/23458 [5:02:32<1:22:20, 1.18it/s][2025-04-24 00:13:49] (step=0041075) Train Loss: 5.3334, Train Steps/Sec: 1.18 + 75%|████████████████████████████████████████████████████████████████████████████████████████████████████████▌ | 17641/23458 [5:02:53<1:21:30, 1.19it/s][2025-04-24 00:14:11] (step=0041100) Train Loss: 5.3468, Train Steps/Sec: 1.17 + 75%|████████████████████████████████████████████████████████████████████████████████████████████████████████▋ | 17666/23458 [5:03:16<1:26:44, 1.11it/s][2025-04-24 00:14:33] (step=0041125) Train Loss: 5.3493, Train Steps/Sec: 1.13 + 75%|████████████████████████████████████████████████████████████████████████████████████████████████████████▊ | 17691/23458 [5:03:37<1:21:49, 1.17it/s][2025-04-24 00:14:54] (step=0041150) Train Loss: 5.3088, Train Steps/Sec: 1.17 + 76%|████████████████████████████████████████████████████████████████████████████████████████████████████████▉ | 17716/23458 [5:03:58<1:20:55, 1.18it/s][2025-04-24 00:15:16] (step=0041175) Train Loss: 5.3304, Train Steps/Sec: 1.17 + 76%|█████████████████████████████████████████████████████████████████████████████████████████████████████████ | 17741/23458 [5:04:20<1:20:20, 1.19it/s][2025-04-24 00:15:37] (step=0041200) Train Loss: 5.3949, Train Steps/Sec: 1.17 + 76%|█████████████████████████████████████████████████████████████████████████████████████████████████████████▎ | 17766/23458 [5:04:41<1:22:02, 1.16it/s][2025-04-24 00:15:58] (step=0041225) Train Loss: 5.3861, Train Steps/Sec: 1.17 + 76%|█████████████████████████████████████████████████████████████████████████████████████████████████████████▍ | 17791/23458 [5:05:02<1:20:27, 1.17it/s][2025-04-24 00:16:20] (step=0041250) Train Loss: 5.4214, Train Steps/Sec: 1.18 + 76%|█████████████████████████████████████████████████████████████████████████████████████████████████████████▌ | 17816/23458 [5:05:23<1:19:25, 1.18it/s][2025-04-24 00:16:41] (step=0041275) Train Loss: 5.3366, Train Steps/Sec: 1.17 + 76%|█████████████████████████████████████████████████████████████████████████████████████████████████████████▋ | 17841/23458 [5:05:45<1:18:54, 1.19it/s][2025-04-24 00:17:02] (step=0041300) Train Loss: 5.3624, Train Steps/Sec: 1.17 + 76%|█████████████████████████████████████████████████████████████████████████████████████████████████████████▊ | 17866/23458 [5:06:07<1:20:57, 1.15it/s][2025-04-24 00:17:24] (step=0041325) Train Loss: 5.4542, Train Steps/Sec: 1.13 + 76%|██████████████████████████████████████████████████████████████████████████████████████████████████████████ | 17891/23458 [5:06:28<1:19:41, 1.16it/s][2025-04-24 00:17:46] (step=0041350) Train Loss: 5.4167, Train Steps/Sec: 1.17 + 76%|██████████████████████████████████████████████████████████████████████████████████████████████████████████▏ | 17916/23458 [5:06:50<1:18:16, 1.18it/s][2025-04-24 00:18:08] (step=0041375) Train Loss: 5.3460, Train Steps/Sec: 1.13 + 76%|██████████████████████████████████████████████████████████████████████████████████████████████████████████▎ | 17941/23458 [5:07:12<1:17:14, 1.19it/s][2025-04-24 00:18:29] (step=0041400) Train Loss: 5.3523, Train Steps/Sec: 1.17 + 77%|██████████████████████████████████████████████████████████████████████████████████████████████████████████▍ | 17966/23458 [5:07:34<1:27:42, 1.04it/s][2025-04-24 00:18:51] (step=0041425) Train Loss: 5.4027, Train Steps/Sec: 1.13 + 77%|██████████████████████████████████████████████████████████████████████████████████████████████████████████▌ | 17991/23458 [5:07:55<1:17:28, 1.18it/s][2025-04-24 00:19:13] (step=0041450) Train Loss: 5.2846, Train Steps/Sec: 1.18 + 77%|██████████████████████████████████████████████████████████████████████████████████████████████████████████▊ | 18016/23458 [5:08:17<1:16:47, 1.18it/s][2025-04-24 00:19:35] (step=0041475) Train Loss: 5.3474, Train Steps/Sec: 1.13 + 77%|██████████████████████████████████████████████████████████████████████████████████████████████████████████▉ | 18041/23458 [5:08:39<1:16:00, 1.19it/s][2025-04-24 00:19:57] (step=0041500) Train Loss: 5.3155, Train Steps/Sec: 1.13 + 77%|███████████████████████████████████████████████████████████████████████████████████████████████████████████ | 18066/23458 [5:09:01<1:17:45, 1.16it/s][2025-04-24 00:20:18] (step=0041525) Train Loss: 5.3846, Train Steps/Sec: 1.18 + 77%|███████████████████████████████████████████████████████████████████████████████████████████████████████████▏ | 18091/23458 [5:09:23<1:27:54, 1.02it/s][2025-04-24 00:20:40] (step=0041550) Train Loss: 5.3794, Train Steps/Sec: 1.13 + 77%|███████████████████████████████████████████████████████████████████████████████████████████████████████████▎ | 18116/23458 [5:09:44<1:15:42, 1.18it/s][2025-04-24 00:21:02] (step=0041575) Train Loss: 5.3146, Train Steps/Sec: 1.17 + 77%|███████████████████████████████████████████████████████████████████████████████████████████████████████████▍ | 18141/23458 [5:10:06<1:15:00, 1.18it/s][2025-04-24 00:21:23] (step=0041600) Train Loss: 5.4006, Train Steps/Sec: 1.17 + 77%|███████████████████████████████████████████████████████████████████████████████████████████████████████████▋ | 18166/23458 [5:10:27<1:16:36, 1.15it/s][2025-04-24 00:21:44] (step=0041625) Train Loss: 5.3050, Train Steps/Sec: 1.18 + 78%|███████████████████████████████████████████████████████████████████████████████████████████████████████████▊ | 18191/23458 [5:10:48<1:15:02, 1.17it/s][2025-04-24 00:22:06] (step=0041650) Train Loss: 5.3408, Train Steps/Sec: 1.17 + 78%|███████████████████████████████████████████████████████████████████████████████████████████████████████████▉ | 18216/23458 [5:11:10<1:15:50, 1.15it/s][2025-04-24 00:22:28] (step=0041675) Train Loss: 5.3566, Train Steps/Sec: 1.13 + 78%|████████████████████████████████████████████████████████████████████████████████████████████████████████████ | 18241/23458 [5:11:32<1:13:19, 1.19it/s][2025-04-24 00:22:49] (step=0041700) Train Loss: 5.4591, Train Steps/Sec: 1.17 + 78%|████████████████████████████████████████████████████████████████████████████████████████████████████████████▏ | 18266/23458 [5:11:53<1:14:53, 1.16it/s][2025-04-24 00:23:10] (step=0041725) Train Loss: 5.3439, Train Steps/Sec: 1.18 + 78%|████████████████████████████████████████████████████████████████████████████████████████████████████████████▍ | 18291/23458 [5:12:14<1:13:29, 1.17it/s][2025-04-24 00:23:32] (step=0041750) Train Loss: 5.3379, Train Steps/Sec: 1.18 + 78%|████████████████████████████████████████████████████████████████████████████████████████████████████████████▌ | 18316/23458 [5:12:36<1:12:20, 1.18it/s][2025-04-24 00:23:53] (step=0041775) Train Loss: 5.3794, Train Steps/Sec: 1.17 + 78%|████████████████████████████████████████████████████████████████████████████████████████████████████████████▋ | 18341/23458 [5:12:57<1:11:52, 1.19it/s][2025-04-24 00:24:14] (step=0041800) Train Loss: 5.3415, Train Steps/Sec: 1.17 + 78%|████████████████████████████████████████████████████████████████████████████████████████████████████████████▊ | 18366/23458 [5:13:18<1:13:44, 1.15it/s][2025-04-24 00:24:36] (step=0041825) Train Loss: 5.3305, Train Steps/Sec: 1.17 + 78%|████████████████████████████████████████████████████████████████████████████████████████████████████████████▉ | 18391/23458 [5:13:40<1:11:43, 1.18it/s][2025-04-24 00:24:58] (step=0041850) Train Loss: 5.3871, Train Steps/Sec: 1.13 + 79%|█████████████████████████████████████████████████████████████████████████████████████████████████████████████ | 18416/23458 [5:14:02<1:11:01, 1.18it/s][2025-04-24 00:25:19] (step=0041875) Train Loss: 5.3249, Train Steps/Sec: 1.18 + 79%|█████████████████████████████████████████████████████████████████████████████████████████████████████████████▎ | 18441/23458 [5:14:23<1:10:27, 1.19it/s][2025-04-24 00:25:40] (step=0041900) Train Loss: 5.3297, Train Steps/Sec: 1.17 + 79%|█████████████████████████████████████████████████████████████████████████████████████████████████████████████▍ | 18466/23458 [5:14:44<1:12:12, 1.15it/s][2025-04-24 00:26:02] (step=0041925) Train Loss: 5.3853, Train Steps/Sec: 1.17 + 79%|█████████████████████████████████████████████████████████████████████████████████████████████████████████████▌ | 18491/23458 [5:15:06<1:10:22, 1.18it/s][2025-04-24 00:26:23] (step=0041950) Train Loss: 5.4272, Train Steps/Sec: 1.17 + 79%|█████████████████████████████████████████████████████████████████████████████████████████████████████████████▋ | 18516/23458 [5:15:28<1:12:12, 1.14it/s][2025-04-24 00:26:45] (step=0041975) Train Loss: 5.3421, Train Steps/Sec: 1.13 + 79%|█████████████████████████████████████████████████████████████████████████████████████████████████████████████▊ | 18541/23458 [5:15:49<1:09:17, 1.18it/s][2025-04-24 00:27:07] (step=0042000) Train Loss: 5.3171, Train Steps/Sec: 1.17 +You are using a model of type blip-2 to instantiate a model of type instructblip. This is not supported for all configurations of models and can yield errors. +[2025-04-24 00:27:07] text_config is None. Initializing the text config with default values (`OPTConfig`). +model_name_or_path:/tmp/haozhezhao/model/blip2-flan-t5-xl +image_place_holder: +tokenizer length after expend 32101 +tokenizer length before expend 32100 +100%|███████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████| 4/4 [03:58<00:00, 59.68s/it] +[2025-04-24 00:32:14] Finish Eval in 42000 steps...████████████████████████████████████████████████████████████████████████████████████████████████| 4/4 [03:58<00:00, 59.21s/it] +processing left-padding... +processing left-padding... +processing left-padding... +processing left-padding... +[2025-04-24 00:32:32] Saved checkpoint to checkpoint/Afteri2i_100kckpt_LLAVAT5_empty_Fixed_just_segment_pretrain_stage3_dreambench_recap_400k_t2i_400k_flux_200k_midjourney_150k_recovery_150k_3e4_epoch2_half_rotated/005-GPT-XL/checkpoints/0042000.pt +[2025-04-24 00:32:34] Removed old checkpoint: checkpoint/Afteri2i_100kckpt_LLAVAT5_empty_Fixed_just_segment_pretrain_stage3_dreambench_recap_400k_t2i_400k_flux_200k_midjourney_150k_recovery_150k_3e4_epoch2_half_rotated/005-GPT-XL/checkpoints/0040000.pt + 79%|██████████████████████████████████████████████████████████████████████████████████████████████████████████████ | 18566/23458 [5:21:39<1:13:37, 1.11it/s][2025-04-24 00:32:56] (step=0042025) Train Loss: 5.3470, Train Steps/Sec: 0.07 + 79%|██████████████████████████████████████████████████████████████████████████████████████████████████████████████▏ | 18591/23458 [5:22:00<1:09:00, 1.18it/s][2025-04-24 00:33:17] (step=0042050) Train Loss: 5.4467, Train Steps/Sec: 1.18 + 79%|██████████████████████████████████████████████████████████████████████████████████████████████████████████████▎ | 18616/23458 [5:22:21<1:10:08, 1.15it/s][2025-04-24 00:33:39] (step=0042075) Train Loss: 5.3431, Train Steps/Sec: 1.17 + 79%|██████████████████████████████████████████████████████████████████████████████████████████████████████████████▍ | 18641/23458 [5:22:44<1:07:34, 1.19it/s][2025-04-24 00:34:01] (step=0042100) Train Loss: 5.3825, Train Steps/Sec: 1.13 + 80%|██████████████████████████████████████████████████████████████████████████████████████████████████████████████▌ | 18666/23458 [5:23:06<1:11:32, 1.12it/s][2025-04-24 00:34:23] (step=0042125) Train Loss: 5.3305, Train Steps/Sec: 1.13 + 80%|██████████████████████████████████████████████████████████████████████████████████████████████████████████████▊ | 18691/23458 [5:23:28<1:08:15, 1.16it/s][2025-04-24 00:34:45] (step=0042150) Train Loss: 5.3939, Train Steps/Sec: 1.13 + 80%|██████████████████████████████████████████████████████████████████████████████████████████████████████████████▉ | 18716/23458 [5:23:49<1:07:11, 1.18it/s][2025-04-24 00:35:06] (step=0042175) Train Loss: 5.4662, Train Steps/Sec: 1.17 + 80%|███████████████████████████████████████████████████████████████████████████████████████████████████████████████ | 18741/23458 [5:24:10<1:06:03, 1.19it/s][2025-04-24 00:35:28] (step=0042200) Train Loss: 5.3089, Train Steps/Sec: 1.17 + 80%|███████████████████████████████████████████████████████████████████████████████████████████████████████████████▏ | 18766/23458 [5:24:33<1:07:45, 1.15it/s][2025-04-24 00:35:50] (step=0042225) Train Loss: 5.3543, Train Steps/Sec: 1.13 + 80%|███████████████████████████████████████████████████████████████████████████████████████████████████████████████▎ | 18791/23458 [5:24:54<1:06:09, 1.18it/s][2025-04-24 00:36:11] (step=0042250) Train Loss: 5.4207, Train Steps/Sec: 1.17 + 80%|███████████████████████████████████████████████████████████████████████████████████████████████████████████████▍ | 18816/23458 [5:25:15<1:05:24, 1.18it/s][2025-04-24 00:36:33] (step=0042275) Train Loss: 5.4137, Train Steps/Sec: 1.17 + 80%|███████████████████████████████████████████████████████████████████████████████████████████████████████████████▋ | 18841/23458 [5:25:37<1:05:13, 1.18it/s][2025-04-24 00:36:54] (step=0042300) Train Loss: 5.4344, Train Steps/Sec: 1.16 + 80%|███████████████████████████████████████████████████████████████████████████████████████████████████████████████▊ | 18866/23458 [5:25:58<1:06:11, 1.16it/s][2025-04-24 00:37:15] (step=0042325) Train Loss: 5.2703, Train Steps/Sec: 1.18 + 81%|███████████████████████████████████████████████████████████████████████████████████████████████████████████████▉ | 18891/23458 [5:26:19<1:04:55, 1.17it/s][2025-04-24 00:37:37] (step=0042350) Train Loss: 5.3861, Train Steps/Sec: 1.17 + 81%|████████████████████████████████████████████████████████████████████████████████████████████████████████████████ | 18916/23458 [5:26:41<1:04:02, 1.18it/s][2025-04-24 00:37:58] (step=0042375) Train Loss: 5.4347, Train Steps/Sec: 1.18 + 81%|████████████████████████████████████████████████████████████████████████████████████████████████████████████████▏ | 18941/23458 [5:27:03<1:04:39, 1.16it/s][2025-04-24 00:38:20] (step=0042400) Train Loss: 5.3871, Train Steps/Sec: 1.12 + 81%|████████████████████████████████████████████████████████████████████████████████████████████████████████████████▍ | 18966/23458 [5:27:24<1:04:47, 1.16it/s][2025-04-24 00:38:42] (step=0042425) Train Loss: 5.4073, Train Steps/Sec: 1.17 + 81%|████████████████████████████████████████████████████████████████████████████████████████████████████████████████▌ | 18991/23458 [5:27:46<1:03:20, 1.18it/s][2025-04-24 00:39:03] (step=0042450) Train Loss: 5.3576, Train Steps/Sec: 1.17 + 81%|████████████████████████████████████████████████████████████████████████████████████████████████████████████████▋ | 19016/23458 [5:28:07<1:02:41, 1.18it/s][2025-04-24 00:39:24] (step=0042475) Train Loss: 5.3949, Train Steps/Sec: 1.17 + 81%|████████████████████████████████████████████████████████████████████████████████████████████████████████████████▊ | 19041/23458 [5:28:28<1:02:26, 1.18it/s][2025-04-24 00:39:46] (step=0042500) Train Loss: 5.3970, Train Steps/Sec: 1.17 + 81%|████████████████████████████████████████████████████████████████████████████████████████████████████████████████▉ | 19066/23458 [5:28:50<1:03:22, 1.16it/s][2025-04-24 00:40:07] (step=0042525) Train Loss: 5.3570, Train Steps/Sec: 1.18 + 81%|█████████████████████████████████████████████████████████████████████████████████████████████████████████████████ | 19091/23458 [5:29:11<1:02:08, 1.17it/s][2025-04-24 00:40:28] (step=0042550) Train Loss: 5.3524, Train Steps/Sec: 1.17 + 81%|█████████████████████████████████████████████████████████████████████████████████████████████████████████████████▎ | 19116/23458 [5:29:33<1:15:10, 1.04s/it][2025-04-24 00:40:50] (step=0042575) Train Loss: 5.3619, Train Steps/Sec: 1.13 + 82%|█████████████████████████████████████████████████████████████████████████████████████████████████████████████████▍ | 19141/23458 [5:29:54<1:00:28, 1.19it/s][2025-04-24 00:41:12] (step=0042600) Train Loss: 5.3608, Train Steps/Sec: 1.17 + 82%|█████████████████████████████████████████████████████████████████████████████████████████████████████████████████▌ | 19166/23458 [5:30:16<1:02:02, 1.15it/s][2025-04-24 00:41:33] (step=0042625) Train Loss: 5.4109, Train Steps/Sec: 1.17 + 82%|█████████████████████████████████████████████████████████████████████████████████████████████████████████████████▋ | 19191/23458 [5:30:38<1:13:14, 1.03s/it][2025-04-24 00:41:55] (step=0042650) Train Loss: 5.3627, Train Steps/Sec: 1.13 + 82%|███████████████████████████████████████████████████████████████████████████████████████████████████████████████████▌ | 19216/23458 [5:30:59<59:48, 1.18it/s][2025-04-24 00:42:16] (step=0042675) Train Loss: 5.3263, Train Steps/Sec: 1.18 + 82%|███████████████████████████████████████████████████████████████████████████████████████████████████████████████████▋ | 19241/23458 [5:31:21<59:09, 1.19it/s][2025-04-24 00:42:39] (step=0042700) Train Loss: 5.4153, Train Steps/Sec: 1.12 + 82%|██████████████████████████████████████████████████████████████████████████████████████████████████████████████████▏ | 19266/23458 [5:31:43<1:01:00, 1.15it/s][2025-04-24 00:43:00] (step=0042725) Train Loss: 5.3874, Train Steps/Sec: 1.17 + 82%|███████████████████████████████████████████████████████████████████████████████████████████████████████████████████▉ | 19291/23458 [5:32:05<59:58, 1.16it/s][2025-04-24 00:43:22] (step=0042750) Train Loss: 5.3222, Train Steps/Sec: 1.12 + 82%|████████████████████████████████████████████████████████████████████████████████████████████████████████████████████ | 19316/23458 [5:32:26<58:12, 1.19it/s][2025-04-24 00:43:44] (step=0042775) Train Loss: 5.3801, Train Steps/Sec: 1.18 + 82%|██████████████████████████████████████████████████████████████████████████████████████████████████████████████████▌ | 19341/23458 [5:32:49<1:06:09, 1.04it/s][2025-04-24 00:44:07] (step=0042800) Train Loss: 5.4633, Train Steps/Sec: 1.09 + 83%|████████████████████████████████████████████████████████████████████████████████████████████████████████████████████▍ | 19366/23458 [5:33:11<59:19, 1.15it/s][2025-04-24 00:44:28] (step=0042825) Train Loss: 5.3519, Train Steps/Sec: 1.17 + 83%|████████████████████████████████████████████████████████████████████████████████████████████████████████████████████▌ | 19391/23458 [5:33:32<57:41, 1.17it/s][2025-04-24 00:44:49] (step=0042850) Train Loss: 5.4045, Train Steps/Sec: 1.18 + 83%|████████████████████████████████████████████████████████████████████████████████████████████████████████████████████▋ | 19416/23458 [5:33:54<58:10, 1.16it/s][2025-04-24 00:45:11] (step=0042875) Train Loss: 5.4304, Train Steps/Sec: 1.13 + 83%|████████████████████████████████████████████████████████████████████████████████████████████████████████████████████▊ | 19441/23458 [5:34:15<56:25, 1.19it/s][2025-04-24 00:45:33] (step=0042900) Train Loss: 5.3971, Train Steps/Sec: 1.17 + 83%|█████████████████████████████████████████████████████████████████████████████████████████████████████████████████████ | 19466/23458 [5:34:37<57:51, 1.15it/s][2025-04-24 00:45:54] (step=0042925) Train Loss: 5.3408, Train Steps/Sec: 1.17 + 83%|█████████████████████████████████████████████████████████████████████████████████████████████████████████████████████▏ | 19491/23458 [5:34:58<56:29, 1.17it/s][2025-04-24 00:46:15] (step=0042950) Train Loss: 5.3681, Train Steps/Sec: 1.17 + 83%|█████████████████████████████████████████████████████████████████████████████████████████████████████████████████████▎ | 19516/23458 [5:35:19<55:47, 1.18it/s][2025-04-24 00:46:37] (step=0042975) Train Loss: 5.2992, Train Steps/Sec: 1.17 + 83%|█████████████████████████████████████████████████████████████████████████████████████████████████████████████████████▍ | 19541/23458 [5:35:40<54:49, 1.19it/s][2025-04-24 00:46:58] (step=0043000) Train Loss: 5.3539, Train Steps/Sec: 1.17 + 83%|█████████████████████████████████████████████████████████████████████████████████████████████████████████████████████▌ | 19566/23458 [5:36:02<56:27, 1.15it/s][2025-04-24 00:47:19] (step=0043025) Train Loss: 5.4103, Train Steps/Sec: 1.17 + 84%|█████████████████████████████████████████████████████████████████████████████████████████████████████████████████████▊ | 19591/23458 [5:36:23<55:20, 1.16it/s][2025-04-24 00:47:41] (step=0043050) Train Loss: 5.3787, Train Steps/Sec: 1.17 + 84%|█████████████████████████████████████████████████████████████████████████████████████████████████████████████████████▉ | 19616/23458 [5:36:45<54:10, 1.18it/s][2025-04-24 00:48:02] (step=0043075) Train Loss: 5.3696, Train Steps/Sec: 1.17 + 84%|██████████████████████████████████████████████████████████████████████████████████████████████████████████████████████ | 19641/23458 [5:37:06<53:37, 1.19it/s][2025-04-24 00:48:23] (step=0043100) Train Loss: 5.4051, Train Steps/Sec: 1.17 + 84%|██████████████████████████████████████████████████████████████████████████████████████████████████████████████████████▏ | 19666/23458 [5:37:28<55:17, 1.14it/s][2025-04-24 00:48:46] (step=0043125) Train Loss: 5.4003, Train Steps/Sec: 1.12 + 84%|██████████████████████████████████████████████████████████████████████████████████████████████████████████████████████▎ | 19691/23458 [5:37:49<53:41, 1.17it/s][2025-04-24 00:49:07] (step=0043150) Train Loss: 5.4427, Train Steps/Sec: 1.18 + 84%|██████████████████████████████████████████████████████████████████████████████████████████████████████████████████████▌ | 19716/23458 [5:38:11<52:49, 1.18it/s][2025-04-24 00:49:28] (step=0043175) Train Loss: 5.3010, Train Steps/Sec: 1.18 + 84%|██████████████████████████████████████████████████████████████████████████████████████████████████████████████████████▋ | 19741/23458 [5:38:32<52:13, 1.19it/s][2025-04-24 00:49:49] (step=0043200) Train Loss: 5.3214, Train Steps/Sec: 1.17 + 84%|██████████████████████████████████████████████████████████████████████████████████████████████████████████████████████▊ | 19766/23458 [5:38:53<53:19, 1.15it/s][2025-04-24 00:50:11] (step=0043225) Train Loss: 5.3099, Train Steps/Sec: 1.18 + 84%|██████████████████████████████████████████████████████████████████████████████████████████████████████████████████████▉ | 19791/23458 [5:39:15<52:06, 1.17it/s][2025-04-24 00:50:32] (step=0043250) Train Loss: 5.3984, Train Steps/Sec: 1.17 + 84%|███████████████████████████████████████████████████████████████████████████████████████████████████████████████████████ | 19816/23458 [5:39:36<51:38, 1.18it/s][2025-04-24 00:50:53] (step=0043275) Train Loss: 5.3031, Train Steps/Sec: 1.17 + 85%|███████████████████████████████████████████████████████████████████████████████████████████████████████████████████████▎ | 19841/23458 [5:39:57<50:49, 1.19it/s][2025-04-24 00:51:16] (step=0043300) Train Loss: 5.3809, Train Steps/Sec: 1.17 + 85%|███████████████████████████████████████████████████████████████████████████████████████████████████████████████████████▍ | 19866/23458 [5:40:21<51:57, 1.15it/s][2025-04-24 00:51:38] (step=0043325) Train Loss: 5.3812, Train Steps/Sec: 1.12 + 85%|███████████████████████████████████████████████████████████████████████████████████████████████████████████████████████▌ | 19891/23458 [5:40:43<50:40, 1.17it/s][2025-04-24 00:52:00] (step=0043350) Train Loss: 5.4348, Train Steps/Sec: 1.13 + 85%|███████████████████████████████████████████████████████████████████████████████████████████████████████████████████████▋ | 19916/23458 [5:41:04<50:01, 1.18it/s][2025-04-24 00:52:21] (step=0043375) Train Loss: 5.2752, Train Steps/Sec: 1.17 + 85%|███████████████████████████████████████████████████████████████████████████████████████████████████████████████████████▊ | 19941/23458 [5:41:26<50:36, 1.16it/s][2025-04-24 00:52:44] (step=0043400) Train Loss: 5.3548, Train Steps/Sec: 1.13 + 85%|████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████ | 19966/23458 [5:41:48<50:19, 1.16it/s][2025-04-24 00:53:05] (step=0043425) Train Loss: 5.2957, Train Steps/Sec: 1.17 + 85%|████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████▏ | 19991/23458 [5:42:10<49:53, 1.16it/s][2025-04-24 00:53:27] (step=0043450) Train Loss: 5.4098, Train Steps/Sec: 1.13 + 85%|████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████▎ | 20016/23458 [5:42:32<48:32, 1.18it/s][2025-04-24 00:53:49] (step=0043475) Train Loss: 5.4316, Train Steps/Sec: 1.13 + 85%|████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████▍ | 20041/23458 [5:42:53<47:56, 1.19it/s][2025-04-24 00:54:11] (step=0043500) Train Loss: 5.3742, Train Steps/Sec: 1.17 + 86%|████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████▌ | 20066/23458 [5:43:15<48:57, 1.15it/s][2025-04-24 00:54:32] (step=0043525) Train Loss: 5.3088, Train Steps/Sec: 1.18 + 86%|████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████▊ | 20091/23458 [5:43:37<48:04, 1.17it/s][2025-04-24 00:54:54] (step=0043550) Train Loss: 5.3163, Train Steps/Sec: 1.13 + 86%|████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████▉ | 20116/23458 [5:43:58<47:12, 1.18it/s][2025-04-24 00:55:15] (step=0043575) Train Loss: 5.3271, Train Steps/Sec: 1.17 + 86%|█████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████ | 20141/23458 [5:44:19<46:45, 1.18it/s][2025-04-24 00:55:37] (step=0043600) Train Loss: 5.3670, Train Steps/Sec: 1.16 + 86%|█████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████▏ | 20166/23458 [5:44:41<47:42, 1.15it/s][2025-04-24 00:55:58] (step=0043625) Train Loss: 5.3834, Train Steps/Sec: 1.17 + 86%|█████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████▎ | 20191/23458 [5:45:02<46:28, 1.17it/s][2025-04-24 00:56:19] (step=0043650) Train Loss: 5.3511, Train Steps/Sec: 1.17 + 86%|█████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████▌ | 20216/23458 [5:45:23<45:36, 1.18it/s][2025-04-24 00:56:41] (step=0043675) Train Loss: 5.3287, Train Steps/Sec: 1.17 + 86%|█████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████▋ | 20241/23458 [5:45:45<45:13, 1.19it/s][2025-04-24 00:57:02] (step=0043700) Train Loss: 5.3573, Train Steps/Sec: 1.17 + 86%|█████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████▊ | 20266/23458 [5:46:06<46:05, 1.15it/s][2025-04-24 00:57:23] (step=0043725) Train Loss: 5.2892, Train Steps/Sec: 1.18 + 86%|█████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████▉ | 20291/23458 [5:46:27<45:17, 1.17it/s][2025-04-24 00:57:45] (step=0043750) Train Loss: 5.4193, Train Steps/Sec: 1.17 + 87%|██████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████ | 20316/23458 [5:46:49<44:30, 1.18it/s][2025-04-24 00:58:06] (step=0043775) Train Loss: 5.3373, Train Steps/Sec: 1.17 + 87%|██████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████▎ | 20341/23458 [5:47:10<43:43, 1.19it/s][2025-04-24 00:58:27] (step=0043800) Train Loss: 5.4285, Train Steps/Sec: 1.17 + 87%|██████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████▍ | 20366/23458 [5:47:31<44:39, 1.15it/s][2025-04-24 00:58:49] (step=0043825) Train Loss: 5.3405, Train Steps/Sec: 1.17 + 87%|██████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████▌ | 20391/23458 [5:47:54<43:47, 1.17it/s][2025-04-24 00:59:11] (step=0043850) Train Loss: 5.3053, Train Steps/Sec: 1.12 + 87%|██████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████▋ | 20416/23458 [5:48:15<42:51, 1.18it/s][2025-04-24 00:59:32] (step=0043875) Train Loss: 5.3937, Train Steps/Sec: 1.18 + 87%|██████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████▊ | 20441/23458 [5:48:36<42:16, 1.19it/s][2025-04-24 00:59:54] (step=0043900) Train Loss: 5.3497, Train Steps/Sec: 1.17 + 87%|███████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████ | 20466/23458 [5:48:58<43:17, 1.15it/s][2025-04-24 01:00:15] (step=0043925) Train Loss: 5.3421, Train Steps/Sec: 1.18 + 87%|███████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████▏ | 20491/23458 [5:49:19<42:08, 1.17it/s][2025-04-24 01:00:36] (step=0043950) Train Loss: 5.3362, Train Steps/Sec: 1.18 + 87%|███████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████▎ | 20516/23458 [5:49:41<43:03, 1.14it/s][2025-04-24 01:00:58] (step=0043975) Train Loss: 5.4042, Train Steps/Sec: 1.13 + 88%|███████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████▍ | 20541/23458 [5:50:03<41:23, 1.17it/s][2025-04-24 01:01:20] (step=0044000) Train Loss: 5.3956, Train Steps/Sec: 1.13 +You are using a model of type blip-2 to instantiate a model of type instructblip. This is not supported for all configurations of models and can yield errors. +[2025-04-24 01:01:20] text_config is None. Initializing the text config with default values (`OPTConfig`). +model_name_or_path:/tmp/haozhezhao/model/blip2-flan-t5-xl +image_place_holder: +tokenizer length after expend 32101 +tokenizer length before expend 32100 +100%|███████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████| 4/4 [03:57<00:00, 59.29s/it] +[2025-04-24 01:06:25] Finish Eval in 44000 steps...████████████████████████████████████████████████████████████████████████████████████████████████| 4/4 [03:56<00:00, 59.02s/it] +processing left-padding... +processing left-padding... +processing left-padding... +processing left-padding... +[2025-04-24 01:06:43] Saved checkpoint to checkpoint/Afteri2i_100kckpt_LLAVAT5_empty_Fixed_just_segment_pretrain_stage3_dreambench_recap_400k_t2i_400k_flux_200k_midjourney_150k_recovery_150k_3e4_epoch2_half_rotated/005-GPT-XL/checkpoints/0044000.pt +[2025-04-24 01:06:45] Removed old checkpoint: checkpoint/Afteri2i_100kckpt_LLAVAT5_empty_Fixed_just_segment_pretrain_stage3_dreambench_recap_400k_t2i_400k_flux_200k_midjourney_150k_recovery_150k_3e4_epoch2_half_rotated/005-GPT-XL/checkpoints/0042000.pt + 88%|███████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████▌ | 20566/23458 [5:55:49<42:31, 1.13it/s][2025-04-24 01:07:07] (step=0044025) Train Loss: 5.2977, Train Steps/Sec: 0.07 + 88%|███████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████▊ | 20591/23458 [5:56:11<40:46, 1.17it/s][2025-04-24 01:07:29] (step=0044050) Train Loss: 5.3354, Train Steps/Sec: 1.12 + 88%|███████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████▉ | 20616/23458 [5:56:34<40:01, 1.18it/s][2025-04-24 01:07:51] (step=0044075) Train Loss: 5.3169, Train Steps/Sec: 1.13 + 88%|████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████ | 20641/23458 [5:56:56<43:31, 1.08it/s][2025-04-24 01:08:13] (step=0044100) Train Loss: 5.4186, Train Steps/Sec: 1.13 + 88%|████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████▏ | 20666/23458 [5:57:18<40:42, 1.14it/s][2025-04-24 01:08:35] (step=0044125) Train Loss: 5.3317, Train Steps/Sec: 1.13 + 88%|████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████▎ | 20691/23458 [5:57:39<39:17, 1.17it/s][2025-04-24 01:08:57] (step=0044150) Train Loss: 5.4193, Train Steps/Sec: 1.18 + 88%|████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████▌ | 20716/23458 [5:58:01<38:44, 1.18it/s][2025-04-24 01:09:18] (step=0044175) Train Loss: 5.4134, Train Steps/Sec: 1.17 + 88%|████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████▋ | 20741/23458 [5:58:23<38:24, 1.18it/s][2025-04-24 01:09:40] (step=0044200) Train Loss: 5.3286, Train Steps/Sec: 1.12 + 89%|████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████▊ | 20766/23458 [5:58:44<38:51, 1.15it/s][2025-04-24 01:10:01] (step=0044225) Train Loss: 5.4532, Train Steps/Sec: 1.18 + 89%|████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████▉ | 20791/23458 [5:59:05<38:03, 1.17it/s][2025-04-24 01:10:23] (step=0044250) Train Loss: 5.2563, Train Steps/Sec: 1.17 + 89%|█████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████ | 20816/23458 [5:59:27<37:09, 1.18it/s][2025-04-24 01:10:44] (step=0044275) Train Loss: 5.4491, Train Steps/Sec: 1.17 + 89%|█████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████▎ | 20841/23458 [5:59:48<36:44, 1.19it/s][2025-04-24 01:11:05] (step=0044300) Train Loss: 5.3308, Train Steps/Sec: 1.17 + 89%|█████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████▍ | 20866/23458 [6:00:09<37:28, 1.15it/s][2025-04-24 01:11:27] (step=0044325) Train Loss: 5.3398, Train Steps/Sec: 1.17 + 89%|█████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████▌ | 20891/23458 [6:00:31<36:20, 1.18it/s][2025-04-24 01:11:48] (step=0044350) Train Loss: 5.3435, Train Steps/Sec: 1.18 + 89%|█████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████▋ | 20916/23458 [6:00:52<35:57, 1.18it/s][2025-04-24 01:12:09] (step=0044375) Train Loss: 5.3156, Train Steps/Sec: 1.17 + 89%|█████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████▊ | 20941/23458 [6:01:13<35:15, 1.19it/s][2025-04-24 01:12:31] (step=0044400) Train Loss: 5.3139, Train Steps/Sec: 1.17 + 89%|██████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████ | 20966/23458 [6:01:35<35:55, 1.16it/s][2025-04-24 01:12:52] (step=0044425) Train Loss: 5.3983, Train Steps/Sec: 1.18 + 89%|██████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████▏ | 20991/23458 [6:01:56<35:10, 1.17it/s][2025-04-24 01:13:13] (step=0044450) Train Loss: 5.4016, Train Steps/Sec: 1.17 + 90%|██████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████▎ | 21016/23458 [6:02:17<34:34, 1.18it/s][2025-04-24 01:13:35] (step=0044475) Train Loss: 5.2697, Train Steps/Sec: 1.17 + 90%|██████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████▍ | 21041/23458 [6:02:39<33:49, 1.19it/s][2025-04-24 01:13:56] (step=0044500) Train Loss: 5.3671, Train Steps/Sec: 1.17 + 90%|██████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████▌ | 21066/23458 [6:03:00<34:36, 1.15it/s][2025-04-24 01:14:17] (step=0044525) Train Loss: 5.3122, Train Steps/Sec: 1.18 + 90%|██████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████▊ | 21091/23458 [6:03:21<33:43, 1.17it/s][2025-04-24 01:14:39] (step=0044550) Train Loss: 5.2865, Train Steps/Sec: 1.17 + 90%|██████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████▉ | 21116/23458 [6:03:44<33:10, 1.18it/s][2025-04-24 01:15:01] (step=0044575) Train Loss: 5.3465, Train Steps/Sec: 1.12 + 90%|███████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████ | 21141/23458 [6:04:05<32:26, 1.19it/s][2025-04-24 01:15:22] (step=0044600) Train Loss: 5.4583, Train Steps/Sec: 1.17 + 90%|███████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████▏ | 21166/23458 [6:04:26<33:09, 1.15it/s][2025-04-24 01:15:44] (step=0044625) Train Loss: 5.3009, Train Steps/Sec: 1.18 + 90%|███████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████▎ | 21191/23458 [6:04:48<36:59, 1.02it/s][2025-04-24 01:16:06] (step=0044650) Train Loss: 5.2811, Train Steps/Sec: 1.09 + 90%|███████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████▌ | 21216/23458 [6:05:10<31:44, 1.18it/s][2025-04-24 01:16:28] (step=0044675) Train Loss: 5.3601, Train Steps/Sec: 1.17 + 91%|███████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████▋ | 21241/23458 [6:05:32<31:03, 1.19it/s][2025-04-24 01:16:49] (step=0044700) Train Loss: 5.2750, Train Steps/Sec: 1.17 + 91%|███████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████▊ | 21266/23458 [6:05:54<31:38, 1.15it/s][2025-04-24 01:17:11] (step=0044725) Train Loss: 5.3450, Train Steps/Sec: 1.13 + 91%|███████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████▉ | 21291/23458 [6:06:15<30:50, 1.17it/s][2025-04-24 01:17:34] (step=0044750) Train Loss: 5.4142, Train Steps/Sec: 1.13 + 91%|████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████ | 21316/23458 [6:06:38<30:17, 1.18it/s][2025-04-24 01:17:57] (step=0044775) Train Loss: 5.4143, Train Steps/Sec: 1.09 + 91%|████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████▎ | 21341/23458 [6:07:01<29:43, 1.19it/s][2025-04-24 01:18:18] (step=0044800) Train Loss: 5.3673, Train Steps/Sec: 1.17 + 91%|████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████▍ | 21366/23458 [6:07:22<30:12, 1.15it/s][2025-04-24 01:18:39] (step=0044825) Train Loss: 5.4454, Train Steps/Sec: 1.18 + 91%|████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████▌ | 21391/23458 [6:07:44<31:28, 1.09it/s][2025-04-24 01:19:01] (step=0044850) Train Loss: 5.3339, Train Steps/Sec: 1.13 + 91%|████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████▋ | 21416/23458 [6:08:05<28:44, 1.18it/s][2025-04-24 01:19:23] (step=0044875) Train Loss: 5.3611, Train Steps/Sec: 1.18 + 91%|████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████▉ | 21441/23458 [6:08:27<28:13, 1.19it/s][2025-04-24 01:19:44] (step=0044900) Train Loss: 5.4296, Train Steps/Sec: 1.17 + 92%|█████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████ | 21466/23458 [6:08:48<28:39, 1.16it/s][2025-04-24 01:20:05] (step=0044925) Train Loss: 5.3284, Train Steps/Sec: 1.18 + 92%|█████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████▏ | 21491/23458 [6:09:09<28:08, 1.17it/s][2025-04-24 01:20:27] (step=0044950) Train Loss: 5.3542, Train Steps/Sec: 1.17 + 92%|█████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████▎ | 21516/23458 [6:09:31<27:19, 1.18it/s][2025-04-24 01:20:48] (step=0044975) Train Loss: 5.4280, Train Steps/Sec: 1.17 + 92%|█████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████▍ | 21541/23458 [6:09:52<26:56, 1.19it/s][2025-04-24 01:21:09] (step=0045000) Train Loss: 5.3534, Train Steps/Sec: 1.17 + 92%|█████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████▋ | 21566/23458 [6:10:13<27:28, 1.15it/s][2025-04-24 01:21:31] (step=0045025) Train Loss: 5.3394, Train Steps/Sec: 1.17 + 92%|█████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████▊ | 21591/23458 [6:10:35<26:31, 1.17it/s][2025-04-24 01:21:52] (step=0045050) Train Loss: 5.3765, Train Steps/Sec: 1.17 + 92%|█████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████▉ | 21616/23458 [6:10:56<26:00, 1.18it/s][2025-04-24 01:22:13] (step=0045075) Train Loss: 5.2853, Train Steps/Sec: 1.17 + 92%|██████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████ | 21641/23458 [6:11:17<25:35, 1.18it/s][2025-04-24 01:22:35] (step=0045100) Train Loss: 5.3460, Train Steps/Sec: 1.17 + 92%|██████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████▏ | 21666/23458 [6:11:39<25:52, 1.15it/s][2025-04-24 01:22:56] (step=0045125) Train Loss: 5.2925, Train Steps/Sec: 1.18 + 92%|██████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████▍ | 21691/23458 [6:12:00<25:08, 1.17it/s][2025-04-24 01:23:17] (step=0045150) Train Loss: 5.4047, Train Steps/Sec: 1.17 + 93%|██████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████▌ | 21716/23458 [6:12:21<24:42, 1.18it/s][2025-04-24 01:23:39] (step=0045175) Train Loss: 5.3833, Train Steps/Sec: 1.17 + 93%|██████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████▋ | 21741/23458 [6:12:43<24:11, 1.18it/s][2025-04-24 01:24:00] (step=0045200) Train Loss: 5.3247, Train Steps/Sec: 1.17 + 93%|██████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████▊ | 21766/23458 [6:13:04<24:28, 1.15it/s][2025-04-24 01:24:21] (step=0045225) Train Loss: 5.3694, Train Steps/Sec: 1.17 + 93%|██████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████▉ | 21791/23458 [6:13:25<23:38, 1.18it/s][2025-04-24 01:24:43] (step=0045250) Train Loss: 5.4376, Train Steps/Sec: 1.18 + 93%|███████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████▏ | 21816/23458 [6:13:47<23:03, 1.19it/s][2025-04-24 01:25:04] (step=0045275) Train Loss: 5.2446, Train Steps/Sec: 1.18 + 93%|███████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████▎ | 21841/23458 [6:14:09<22:45, 1.18it/s][2025-04-24 01:25:26] (step=0045300) Train Loss: 5.3384, Train Steps/Sec: 1.11 + 93%|███████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████▍ | 21866/23458 [6:14:32<22:59, 1.15it/s][2025-04-24 01:25:49] (step=0045325) Train Loss: 5.3731, Train Steps/Sec: 1.09 + 93%|███████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████▌ | 21891/23458 [6:14:53<22:21, 1.17it/s][2025-04-24 01:26:11] (step=0045350) Train Loss: 5.4790, Train Steps/Sec: 1.17 + 93%|███████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████▋ | 21916/23458 [6:15:15<22:59, 1.12it/s][2025-04-24 01:26:33] (step=0045375) Train Loss: 5.3663, Train Steps/Sec: 1.13 + 94%|███████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████▉ | 21941/23458 [6:15:37<21:19, 1.19it/s][2025-04-24 01:26:54] (step=0045400) Train Loss: 5.2886, Train Steps/Sec: 1.17 + 94%|████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████ | 21966/23458 [6:15:59<21:44, 1.14it/s][2025-04-24 01:27:16] (step=0045425) Train Loss: 5.3127, Train Steps/Sec: 1.13 + 94%|████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████▏ | 21991/23458 [6:16:21<20:52, 1.17it/s][2025-04-24 01:27:38] (step=0045450) Train Loss: 5.3928, Train Steps/Sec: 1.13 + 94%|████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████▎ | 22016/23458 [6:16:42<20:22, 1.18it/s][2025-04-24 01:28:00] (step=0045475) Train Loss: 5.4182, Train Steps/Sec: 1.17 + 94%|████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████▍ | 22041/23458 [6:17:05<19:54, 1.19it/s][2025-04-24 01:28:22] (step=0045500) Train Loss: 5.4539, Train Steps/Sec: 1.12 + 94%|████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████▋ | 22066/23458 [6:17:27<20:06, 1.15it/s][2025-04-24 01:28:44] (step=0045525) Train Loss: 5.3889, Train Steps/Sec: 1.13 + 94%|████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████▊ | 22091/23458 [6:17:48<19:26, 1.17it/s][2025-04-24 01:29:06] (step=0045550) Train Loss: 5.3678, Train Steps/Sec: 1.17 + 94%|████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████▉ | 22116/23458 [6:18:10<18:55, 1.18it/s][2025-04-24 01:29:27] (step=0045575) Train Loss: 5.3170, Train Steps/Sec: 1.17 + 94%|█████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████ | 22141/23458 [6:18:31<18:26, 1.19it/s][2025-04-24 01:29:48] (step=0045600) Train Loss: 5.3305, Train Steps/Sec: 1.17 + 94%|█████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████▏ | 22166/23458 [6:18:52<18:37, 1.16it/s][2025-04-24 01:30:10] (step=0045625) Train Loss: 5.3841, Train Steps/Sec: 1.17 + 95%|█████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████▍ | 22191/23458 [6:19:13<18:00, 1.17it/s][2025-04-24 01:30:31] (step=0045650) Train Loss: 5.3210, Train Steps/Sec: 1.17 + 95%|█████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████▌ | 22216/23458 [6:19:35<17:30, 1.18it/s][2025-04-24 01:30:52] (step=0045675) Train Loss: 5.4832, Train Steps/Sec: 1.17 + 95%|█████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████▋ | 22241/23458 [6:19:56<17:03, 1.19it/s][2025-04-24 01:31:14] (step=0045700) Train Loss: 5.4124, Train Steps/Sec: 1.17 + 95%|█████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████▊ | 22266/23458 [6:20:17<17:15, 1.15it/s][2025-04-24 01:31:35] (step=0045725) Train Loss: 5.3231, Train Steps/Sec: 1.18 + 95%|█████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████▉ | 22291/23458 [6:20:39<16:36, 1.17it/s][2025-04-24 01:31:56] (step=0045750) Train Loss: 5.3632, Train Steps/Sec: 1.17 + 95%|██████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████▏ | 22316/23458 [6:21:00<16:05, 1.18it/s][2025-04-24 01:32:17] (step=0045775) Train Loss: 5.3009, Train Steps/Sec: 1.17 + 95%|██████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████▎ | 22341/23458 [6:21:21<15:39, 1.19it/s][2025-04-24 01:32:39] (step=0045800) Train Loss: 5.3580, Train Steps/Sec: 1.17 + 95%|██████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████▍ | 22366/23458 [6:21:43<15:47, 1.15it/s][2025-04-24 01:33:00] (step=0045825) Train Loss: 5.4282, Train Steps/Sec: 1.17 + 95%|██████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████▌ | 22391/23458 [6:22:04<15:11, 1.17it/s][2025-04-24 01:33:21] (step=0045850) Train Loss: 5.3535, Train Steps/Sec: 1.18 + 96%|██████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████▋ | 22416/23458 [6:22:25<14:43, 1.18it/s][2025-04-24 01:33:43] (step=0045875) Train Loss: 5.3838, Train Steps/Sec: 1.18 + 96%|██████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████▉ | 22441/23458 [6:22:47<14:15, 1.19it/s][2025-04-24 01:34:04] (step=0045900) Train Loss: 5.3273, Train Steps/Sec: 1.17 + 96%|███████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████ | 22466/23458 [6:23:08<14:21, 1.15it/s][2025-04-24 01:34:25] (step=0045925) Train Loss: 5.3174, Train Steps/Sec: 1.18 + 96%|███████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████▏ | 22491/23458 [6:23:29<13:43, 1.17it/s][2025-04-24 01:34:47] (step=0045950) Train Loss: 5.4085, Train Steps/Sec: 1.17 + 96%|███████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████▎ | 22516/23458 [6:23:52<15:14, 1.03it/s][2025-04-24 01:35:10] (step=0045975) Train Loss: 5.3665, Train Steps/Sec: 1.09 + 96%|███████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████▍ | 22541/23458 [6:24:14<12:53, 1.19it/s][2025-04-24 01:35:31] (step=0046000) Train Loss: 5.3453, Train Steps/Sec: 1.17 +You are using a model of type blip-2 to instantiate a model of type instructblip. This is not supported for all configurations of models and can yield errors. +[2025-04-24 01:35:31] text_config is None. Initializing the text config with default values (`OPTConfig`). +model_name_or_path:/tmp/haozhezhao/model/blip2-flan-t5-xl +image_place_holder: +tokenizer length after expend 32101 +tokenizer length before expend 32100 +100%|███████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████| 4/4 [03:55<00:00, 58.84s/it] +[2025-04-24 01:40:35] Finish Eval in 46000 steps...████████████████████████████████████████████████████████████████████████████████████████████████| 4/4 [03:54<00:00, 58.53s/it] +processing left-padding... +processing left-padding... +processing left-padding... +processing left-padding... +[2025-04-24 01:40:54] Saved checkpoint to checkpoint/Afteri2i_100kckpt_LLAVAT5_empty_Fixed_just_segment_pretrain_stage3_dreambench_recap_400k_t2i_400k_flux_200k_midjourney_150k_recovery_150k_3e4_epoch2_half_rotated/005-GPT-XL/checkpoints/0046000.pt +[2025-04-24 01:40:56] Removed old checkpoint: checkpoint/Afteri2i_100kckpt_LLAVAT5_empty_Fixed_just_segment_pretrain_stage3_dreambench_recap_400k_t2i_400k_flux_200k_midjourney_150k_recovery_150k_3e4_epoch2_half_rotated/005-GPT-XL/checkpoints/0044000.pt + 96%|███████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████▋ | 22566/23458 [6:30:01<13:12, 1.13it/s][2025-04-24 01:41:18] (step=0046025) Train Loss: 5.2180, Train Steps/Sec: 0.07 + 96%|███████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████▊ | 22591/23458 [6:30:23<12:17, 1.18it/s][2025-04-24 01:41:40] (step=0046050) Train Loss: 5.3345, Train Steps/Sec: 1.13 + 96%|███████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████▉ | 22616/23458 [6:30:45<15:25, 1.10s/it][2025-04-24 01:42:02] (step=0046075) Train Loss: 5.2980, Train Steps/Sec: 1.13 + 97%|████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████ | 22641/23458 [6:31:07<12:15, 1.11it/s][2025-04-24 01:42:25] (step=0046100) Train Loss: 5.3652, Train Steps/Sec: 1.13 + 97%|████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████▏ | 22666/23458 [6:31:28<11:24, 1.16it/s][2025-04-24 01:42:46] (step=0046125) Train Loss: 5.3861, Train Steps/Sec: 1.18 + 97%|████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████▍ | 22691/23458 [6:31:50<10:56, 1.17it/s][2025-04-24 01:43:07] (step=0046150) Train Loss: 5.2832, Train Steps/Sec: 1.18 + 97%|████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████▌ | 22716/23458 [6:32:12<10:38, 1.16it/s][2025-04-24 01:43:29] (step=0046175) Train Loss: 5.3837, Train Steps/Sec: 1.13 + 97%|████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████▋ | 22741/23458 [6:32:33<10:00, 1.19it/s][2025-04-24 01:43:51] (step=0046200) Train Loss: 5.3489, Train Steps/Sec: 1.17 + 97%|████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████▊ | 22766/23458 [6:32:56<10:05, 1.14it/s][2025-04-24 01:44:13] (step=0046225) Train Loss: 5.3975, Train Steps/Sec: 1.12 + 97%|████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████▉ | 22791/23458 [6:33:17<09:27, 1.17it/s][2025-04-24 01:44:34] (step=0046250) Train Loss: 5.2774, Train Steps/Sec: 1.17 + 97%|█████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████▏ | 22816/23458 [6:33:38<09:03, 1.18it/s][2025-04-24 01:44:55] (step=0046275) Train Loss: 5.3690, Train Steps/Sec: 1.18 + 97%|█████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████▎ | 22841/23458 [6:33:59<08:39, 1.19it/s][2025-04-24 01:45:17] (step=0046300) Train Loss: 5.2927, Train Steps/Sec: 1.17 + 97%|█████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████▍ | 22866/23458 [6:34:21<08:30, 1.16it/s][2025-04-24 01:45:38] (step=0046325) Train Loss: 5.3940, Train Steps/Sec: 1.18 + 98%|█████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████▌ | 22891/23458 [6:34:42<08:03, 1.17it/s][2025-04-24 01:45:59] (step=0046350) Train Loss: 5.4671, Train Steps/Sec: 1.17 + 98%|█████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████▋ | 22916/23458 [6:35:03<07:42, 1.17it/s][2025-04-24 01:46:21] (step=0046375) Train Loss: 5.3494, Train Steps/Sec: 1.17 + 98%|█████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████▉ | 22941/23458 [6:35:25<07:15, 1.19it/s][2025-04-24 01:46:42] (step=0046400) Train Loss: 5.3517, Train Steps/Sec: 1.17 + 98%|██████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████ | 22966/23458 [6:35:46<07:05, 1.16it/s][2025-04-24 01:47:03] (step=0046425) Train Loss: 5.4245, Train Steps/Sec: 1.17 + 98%|██████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████▏ | 22991/23458 [6:36:07<06:39, 1.17it/s][2025-04-24 01:47:25] (step=0046450) Train Loss: 5.3157, Train Steps/Sec: 1.17 + 98%|██████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████▎ | 23016/23458 [6:36:29<06:15, 1.18it/s][2025-04-24 01:47:46] (step=0046475) Train Loss: 5.3498, Train Steps/Sec: 1.17 + 98%|██████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████▍ | 23041/23458 [6:36:50<05:52, 1.18it/s][2025-04-24 01:48:07] (step=0046500) Train Loss: 5.2964, Train Steps/Sec: 1.17 + 98%|██████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████▋ | 23066/23458 [6:37:11<05:40, 1.15it/s][2025-04-24 01:48:29] (step=0046525) Train Loss: 5.3717, Train Steps/Sec: 1.17 + 98%|██████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████▊ | 23091/23458 [6:37:33<05:12, 1.18it/s][2025-04-24 01:48:50] (step=0046550) Train Loss: 5.3683, Train Steps/Sec: 1.18 + 99%|██████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████▉ | 23116/23458 [6:37:54<04:48, 1.18it/s][2025-04-24 01:49:11] (step=0046575) Train Loss: 5.2902, Train Steps/Sec: 1.18 + 99%|███████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████ | 23141/23458 [6:38:15<04:27, 1.18it/s][2025-04-24 01:49:33] (step=0046600) Train Loss: 5.3673, Train Steps/Sec: 1.16 + 99%|███████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████▏ | 23166/23458 [6:38:38<05:25, 1.11s/it][2025-04-24 01:49:55] (step=0046625) Train Loss: 5.3834, Train Steps/Sec: 1.13 + 99%|███████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████▍ | 23191/23458 [6:39:00<04:38, 1.04s/it][2025-04-24 01:50:17] (step=0046650) Train Loss: 5.4046, Train Steps/Sec: 1.13 + 99%|███████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████▌ | 23216/23458 [6:39:21<03:25, 1.18it/s][2025-04-24 01:50:38] (step=0046675) Train Loss: 5.3893, Train Steps/Sec: 1.18 + 99%|███████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████▋ | 23241/23458 [6:39:43<03:03, 1.18it/s][2025-04-24 01:51:01] (step=0046700) Train Loss: 5.3200, Train Steps/Sec: 1.12 + 99%|███████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████▊ | 23266/23458 [6:40:05<02:46, 1.15it/s][2025-04-24 01:51:22] (step=0046725) Train Loss: 5.2936, Train Steps/Sec: 1.18 + 99%|███████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████▉ | 23291/23458 [6:40:28<02:22, 1.17it/s][2025-04-24 01:51:45] (step=0046750) Train Loss: 5.3798, Train Steps/Sec: 1.08 + 99%|████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████▏| 23316/23458 [6:40:50<02:00, 1.18it/s][2025-04-24 01:52:07] (step=0046775) Train Loss: 5.4106, Train Steps/Sec: 1.13 +100%|████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████▎| 23341/23458 [6:41:11<01:38, 1.19it/s][2025-04-24 01:52:28] (step=0046800) Train Loss: 5.2462, Train Steps/Sec: 1.17 +100%|████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████▍| 23366/23458 [6:41:33<01:37, 1.05s/it][2025-04-24 01:52:51] (step=0046825) Train Loss: 5.3853, Train Steps/Sec: 1.13 +100%|████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████▌| 23391/23458 [6:41:55<00:57, 1.17it/s][2025-04-24 01:53:12] (step=0046850) Train Loss: 5.3750, Train Steps/Sec: 1.17 +100%|████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████▋| 23416/23458 [6:42:16<00:35, 1.18it/s][2025-04-24 01:53:33] (step=0046875) Train Loss: 5.3776, Train Steps/Sec: 1.17 +100%|████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████▉| 23441/23458 [6:42:37<00:14, 1.19it/s][2025-04-24 01:53:55] (step=0046900) Train Loss: 5.4468, Train Steps/Sec: 1.17 +100%|█████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████| 23458/23458 [6:42:53<00:00, 1.03s/it] +[2025-04-24 01:54:09] Done! diff --git a/CKPTS/Afteri2i_100kckpt_LLAVAT5_empty_Fixed_just_segment_pretrain_stage3_dreambench_recap_400k_t2i_400k_flux_200k_midjourney_150k_recovery_150k_3e4_epoch2_half_rotated/wandb/run-20250423_122459-k5belznu/files/requirements.txt b/CKPTS/Afteri2i_100kckpt_LLAVAT5_empty_Fixed_just_segment_pretrain_stage3_dreambench_recap_400k_t2i_400k_flux_200k_midjourney_150k_recovery_150k_3e4_epoch2_half_rotated/wandb/run-20250423_122459-k5belznu/files/requirements.txt new file mode 100644 index 0000000000000000000000000000000000000000..06dc78369ffff807b210006a0e79d705ffe2a7d7 --- /dev/null +++ b/CKPTS/Afteri2i_100kckpt_LLAVAT5_empty_Fixed_just_segment_pretrain_stage3_dreambench_recap_400k_t2i_400k_flux_200k_midjourney_150k_recovery_150k_3e4_epoch2_half_rotated/wandb/run-20250423_122459-k5belznu/files/requirements.txt @@ -0,0 +1,131 @@ +typing_extensions==4.12.2 +pyzmq==26.3.0 +nvidia-cufft-cu12==11.0.2.54 +triton==3.1.0 +nvidia-cublas-cu12==12.1.3.1 +psutil==7.0.0 +nvidia-cuda-cupti-cu12==12.1.105 +smmap==5.0.2 +nvidia-cuda-runtime-cu12==12.1.105 +aiohappyeyeballs==2.6.1 +asttokens==3.0.0 +huggingface-hub==0.29.3 +pyarrow==19.0.1 +fonttools==4.56.0 +python-dateutil==2.9.0.post0 +GitPython==3.1.44 +aiohttp==3.11.14 +wandb==0.19.8 +setproctitle==1.3.5 +PyYAML==6.0.2 +pydantic_core==2.27.2 +safetensors==0.5.3 +nvidia-nvjitlink-cu12==12.1.105 +aiosignal==1.3.2 +dill==0.3.8 +nvidia-cuda-nvrtc-cu12==12.1.105 +multiprocess==0.70.16 +pure_eval==0.2.3 +stack_data==0.6.3 +pydantic==2.10.6 +MarkupSafe==2.1.5 +tornado==6.4.2 +executing==2.1.0 +executing==2.2.0 +opencv-python==4.11.0.86 +nvitop==1.4.2 +multidict==6.2.0 +Jinja2==3.1.4 +torch==2.5.1+cu121 +nvidia-curand-cu12==10.3.2.106 +platformdirs==4.3.6 +six==1.17.0 +mpmath==1.3.0 +zipp==3.21.0 +packaging==24.2 +requests==2.32.3 +certifi==2025.1.31 +docker-pycreds==0.4.0 +torchvision==0.20.1+cu121 +pandas==2.2.3 +networkx==3.3 +exceptiongroup==1.2.2 +pickleshare==0.7.5 +tokenizers==0.21.1 +charset-normalizer==3.4.1 +jupyter_core==5.7.2 +wcwidth==0.2.13 +nvidia-nvtx-cu12==12.1.105 +prompt_toolkit==3.0.50 +fsspec==2024.12.0 +pillow==11.1.0 +propcache==0.3.0 +regex==2024.11.6 +ptyprocess==0.7.0 +contourpy==1.3.1 +importlib_metadata==8.6.1 +idna==3.10 +comm==0.2.2 +protobuf==5.29.3 +yarl==1.18.3 +ipython_pygments_lexers==1.1.1 +pip==25.0 +parso==0.8.4 +joblib==1.4.2 +nvidia-nccl-cu12==2.21.5 +hf_transfer==0.1.9 +Pygments==2.19.1 +decorator==5.2.1 +filelock==3.18.0 +nvidia-cusparse-cu12==12.1.0.106 +debugpy==1.8.13 +urllib3==2.3.0 +traitlets==5.14.3 +tzdata==2025.1 +matplotlib-inline==0.1.7 +matplotlib==3.10.1 +kiwisolver==1.4.8 +nest_asyncio==1.6.0 +frozenlist==1.5.0 +nvidia-ml-py==12.570.86 +transformers==4.49.0 +nltk==3.9.1 +ipykernel==6.29.5 +click==8.1.8 +gitdb==4.0.12 +pyparsing==3.2.1 +attrs==25.3.0 +jedi==0.19.2 +ipython==9.0.2 +nvidia-cudnn-cu12==9.1.0.70 +pexpect==4.9.0 +nvidia-cusolver-cu12==11.4.5.107 +numpy==2.2.4 +tqdm==4.67.1 +pytz==2025.1 +wheel==0.45.1 +sentry-sdk==2.23.1 +torchaudio==2.5.1+cu121 +jupyter_client==8.6.3 +cycler==0.12.1 +annotated-types==0.7.0 +sympy==1.13.1 +xxhash==3.5.0 +datasets==3.4.1 +setuptools==75.8.0 +typing_extensions==4.12.2 +wheel==0.43.0 +importlib_metadata==8.0.0 +backports.tarfile==1.2.0 +autocommand==2.2.2 +packaging==24.2 +tomli==2.0.1 +typeguard==4.3.0 +zipp==3.19.2 +jaraco.context==5.3.0 +jaraco.functools==4.0.1 +more-itertools==10.3.0 +platformdirs==4.2.2 +jaraco.text==3.12.1 +jaraco.collections==5.1.0 +inflect==7.3.1 diff --git a/CKPTS/Afteri2i_100kckpt_LLAVAT5_empty_Fixed_just_segment_pretrain_stage3_dreambench_recap_400k_t2i_400k_flux_200k_midjourney_150k_recovery_150k_3e4_epoch2_half_rotated/wandb/run-20250423_122459-k5belznu/files/wandb-metadata.json b/CKPTS/Afteri2i_100kckpt_LLAVAT5_empty_Fixed_just_segment_pretrain_stage3_dreambench_recap_400k_t2i_400k_flux_200k_midjourney_150k_recovery_150k_3e4_epoch2_half_rotated/wandb/run-20250423_122459-k5belznu/files/wandb-metadata.json new file mode 100644 index 0000000000000000000000000000000000000000..85a9d1cf919f15de82ff925eb3371c6ccee21fe9 --- /dev/null +++ b/CKPTS/Afteri2i_100kckpt_LLAVAT5_empty_Fixed_just_segment_pretrain_stage3_dreambench_recap_400k_t2i_400k_flux_200k_midjourney_150k_recovery_150k_3e4_epoch2_half_rotated/wandb/run-20250423_122459-k5belznu/files/wandb-metadata.json @@ -0,0 +1,149 @@ +{ + "os": "Linux-5.15.0-1064-azure-x86_64-with-glibc2.31", + "python": "CPython 3.11.11", + "startedAt": "2025-04-23T12:24:59.614777Z", + "args": [ + "--vq-ckpt", + "/tmp/haozhezhao/model/llamagen_t2i/vq_ds16_t2i.pt", + "--data-path", + "/tmp/haozhezhao/MLLMG/jsonl_data/merged_train_set_set_subject_400k_recap_t2i_400k_flux_200k_midjourney_150k_recovery_150k_grounding_100fluxseg_50samseg.jsonl", + "--dataset", + "ti2i", + "--image-size", + "512", + "--results-dir", + "checkpoint/Afteri2i_100kckpt_LLAVAT5_empty_Fixed_just_segment_pretrain_stage3_dreambench_recap_400k_t2i_400k_flux_200k_midjourney_150k_recovery_150k_3e4_epoch2_half_rotated", + "--cloud-save-path", + "/tmp/haozhezhao/checkpoint", + "--lr", + "3e-4", + "--val_data_path", + "/tmp/haozhezhao/MLLMG/jsonl_data/dreambench_plus_valid.jsonl", + "--use_vision_tower", + "--model_name_or_path", + "/tmp/haozhezhao/model/blip2-flan-t5-xl", + "--image_place_holder", + "", + "--do_eval", + "--eval_steps", + "2000", + "--max_eval_samples", + "250", + "--cfg-scale", + "7.5", + "--top-k", + "16384", + "--load_from_checkpoint", + "/tmp/haozhezhao/MLLMG/checkpoint/FIXed_3M_1epoch_step32000.pt", + "--global-batch-size", + "56", + "--num-workers", + "8", + "--warmup", + "0.05", + "--gradient-accumulation-steps", + "4", + "--train_text_encoder", + "--ckpt-every", + "2000", + "--epochs", + "2", + "--subject_driven", + "--reference_data_path", + "/tmp/haozhezhao/MLLMG/cc12m_reference_tunnel.jsonl", + "--multimodal_encoder", + "llava", + "--do_recovery", + "--find_unused_parameters", + "--cls-token-num", + "512", + "--dreambench_eval", + "--save_total_limit", + "1", + "--load_language_projection", + "/tmp/haozhezhao/MLLMG/llava-v1.5-flant5_fixed-pretrain/mm_projector.bin", + "--gpt-ckpt", + "/tmp/haozhezhao/MLLMG/MLLMG_ckpts/checkpoints/AfterI2I_llavat5_subject_train_all_T5_subject_t2i_ti2i_120_w_flux_segment_noimageonly_empty_fix_train_all/0100000.pt", + "--mm_vision_tower", + "openai/clip-vit-large-patch14", + "--train_all", + "--load_fixed_llamagen", + "--fix", + "gpt-empty-fix" + ], + "program": "/tmp/haozhezhao/MLLMG/autoregressive/train/train_t2i.py", + "codePath": "autoregressive/train/train_t2i.py", + "email": "mimazhe55360@gmail.com", + "root": "checkpoint/Afteri2i_100kckpt_LLAVAT5_empty_Fixed_just_segment_pretrain_stage3_dreambench_recap_400k_t2i_400k_flux_200k_midjourney_150k_recovery_150k_3e4_epoch2_half_rotated", + "host": "447cc403a8794092814259713c51c1df00001X", + "executable": "/tmp/haozhezhao/anaconda3/envs/nlp/bin/python", + "codePathLocal": "autoregressive/train/train_t2i.py", + "cpu_count": 96, + "cpu_count_logical": 96, + "gpu": "NVIDIA A100-SXM4-80GB", + "gpu_count": 8, + "disk": { + "/": { + "total": "133003395072", + "used": "65555976192" + } + }, + "memory": { + "total": "1902387884032" + }, + "cpu": { + "count": 96, + "countLogical": 96 + }, + "gpu_nvidia": [ + { + "name": "NVIDIA A100-SXM4-80GB", + "memoryTotal": "85899345920", + "cudaCores": 6912, + "architecture": "Ampere" + }, + { + "name": "NVIDIA A100-SXM4-80GB", + "memoryTotal": "85899345920", + "cudaCores": 6912, + "architecture": "Ampere" + }, + { + "name": "NVIDIA A100-SXM4-80GB", + "memoryTotal": "85899345920", + "cudaCores": 6912, + "architecture": "Ampere" + }, + { + "name": "NVIDIA A100-SXM4-80GB", + "memoryTotal": "85899345920", + "cudaCores": 6912, + "architecture": "Ampere" + }, + { + "name": "NVIDIA A100-SXM4-80GB", + "memoryTotal": "85899345920", + "cudaCores": 6912, + "architecture": "Ampere" + }, + { + "name": "NVIDIA A100-SXM4-80GB", + "memoryTotal": "85899345920", + "cudaCores": 6912, + "architecture": "Ampere" + }, + { + "name": "NVIDIA A100-SXM4-80GB", + "memoryTotal": "85899345920", + "cudaCores": 6912, + "architecture": "Ampere" + }, + { + "name": "NVIDIA A100-SXM4-80GB", + "memoryTotal": "85899345920", + "cudaCores": 6912, + "architecture": "Ampere" + } + ], + "cudaVersion": "12.2" +} \ No newline at end of file diff --git a/CKPTS/Afteri2i_100kckpt_LLAVAT5_empty_Fixed_just_segment_pretrain_stage3_dreambench_recap_400k_t2i_400k_flux_200k_midjourney_150k_recovery_150k_3e4_epoch2_half_rotated/wandb/run-20250423_122459-k5belznu/files/wandb-summary.json b/CKPTS/Afteri2i_100kckpt_LLAVAT5_empty_Fixed_just_segment_pretrain_stage3_dreambench_recap_400k_t2i_400k_flux_200k_midjourney_150k_recovery_150k_3e4_epoch2_half_rotated/wandb/run-20250423_122459-k5belznu/files/wandb-summary.json new file mode 100644 index 0000000000000000000000000000000000000000..9252742582675515970ae721fc0cf6977762d46b --- /dev/null +++ b/CKPTS/Afteri2i_100kckpt_LLAVAT5_empty_Fixed_just_segment_pretrain_stage3_dreambench_recap_400k_t2i_400k_flux_200k_midjourney_150k_recovery_150k_3e4_epoch2_half_rotated/wandb/run-20250423_122459-k5belznu/files/wandb-summary.json @@ -0,0 +1 @@ +{"train lr":0.000274238544355085,"train loss":5.446779251098633,"Train Steps/Sec":1.1705568105399053,"_timestamp":1.745459635127043e+09,"_wandb":{"runtime":48553},"_runtime":48553.172666543,"_step":46900} \ No newline at end of file diff --git a/CKPTS/Afteri2i_100kckpt_LLAVAT5_empty_Fixed_just_segment_pretrain_stage3_dreambench_recap_400k_t2i_400k_flux_200k_midjourney_150k_recovery_150k_3e4_epoch2_half_rotated/wandb/run-20250423_122459-k5belznu/logs/debug-core.log b/CKPTS/Afteri2i_100kckpt_LLAVAT5_empty_Fixed_just_segment_pretrain_stage3_dreambench_recap_400k_t2i_400k_flux_200k_midjourney_150k_recovery_150k_3e4_epoch2_half_rotated/wandb/run-20250423_122459-k5belznu/logs/debug-core.log new file mode 100644 index 0000000000000000000000000000000000000000..3ee2573ebfce47631415a9a22e995d16293c5bda --- /dev/null +++ b/CKPTS/Afteri2i_100kckpt_LLAVAT5_empty_Fixed_just_segment_pretrain_stage3_dreambench_recap_400k_t2i_400k_flux_200k_midjourney_150k_recovery_150k_3e4_epoch2_half_rotated/wandb/run-20250423_122459-k5belznu/logs/debug-core.log @@ -0,0 +1,13 @@ +{"time":"2025-04-23T12:24:59.0944758Z","level":"INFO","msg":"main: starting server","port-filename":"/tmp/tmpa5spey7p/port-3176497.txt","pid":3176497,"log-level":0,"disable-analytics":false,"shutdown-on-parent-exit":false} +{"time":"2025-04-23T12:24:59.096262008Z","level":"INFO","msg":"Will exit if parent process dies.","ppid":3176497} +{"time":"2025-04-23T12:24:59.096224673Z","level":"INFO","msg":"server is running","addr":{"IP":"127.0.0.1","Port":43615,"Zone":""}} +{"time":"2025-04-23T12:24:59.281989589Z","level":"INFO","msg":"connection: ManageConnectionData: new connection created","id":"127.0.0.1:36744"} +{"time":"2025-04-23T12:24:59.620297512Z","level":"INFO","msg":"handleInformInit: received","streamId":"k5belznu","id":"127.0.0.1:36744"} +{"time":"2025-04-23T12:24:59.946834458Z","level":"INFO","msg":"handleInformInit: stream started","streamId":"k5belznu","id":"127.0.0.1:36744"} +{"time":"2025-04-24T01:54:12.787271802Z","level":"INFO","msg":"handleInformTeardown: server teardown initiated","id":"127.0.0.1:36744"} +{"time":"2025-04-24T01:54:12.787427855Z","level":"INFO","msg":"server is shutting down"} +{"time":"2025-04-24T01:54:12.787429037Z","level":"INFO","msg":"connection: closing","id":"127.0.0.1:36744"} +{"time":"2025-04-24T01:54:12.787631566Z","level":"INFO","msg":"connection: closed successfully","id":"127.0.0.1:36744"} +{"time":"2025-04-24T01:54:14.221024582Z","level":"INFO","msg":"handleInformTeardown: server shutdown complete","id":"127.0.0.1:36744"} +{"time":"2025-04-24T01:54:14.221051051Z","level":"INFO","msg":"connection: ManageConnectionData: connection closed","id":"127.0.0.1:36744"} +{"time":"2025-04-24T01:54:14.22106643Z","level":"INFO","msg":"server is closed"} diff --git a/CKPTS/Afteri2i_100kckpt_LLAVAT5_empty_Fixed_just_segment_pretrain_stage3_dreambench_recap_400k_t2i_400k_flux_200k_midjourney_150k_recovery_150k_3e4_epoch2_half_rotated/wandb/run-20250423_122459-k5belznu/logs/debug-internal.log b/CKPTS/Afteri2i_100kckpt_LLAVAT5_empty_Fixed_just_segment_pretrain_stage3_dreambench_recap_400k_t2i_400k_flux_200k_midjourney_150k_recovery_150k_3e4_epoch2_half_rotated/wandb/run-20250423_122459-k5belznu/logs/debug-internal.log new file mode 100644 index 0000000000000000000000000000000000000000..a30a26e7a381db2249e8240109339da415939267 --- /dev/null +++ b/CKPTS/Afteri2i_100kckpt_LLAVAT5_empty_Fixed_just_segment_pretrain_stage3_dreambench_recap_400k_t2i_400k_flux_200k_midjourney_150k_recovery_150k_3e4_epoch2_half_rotated/wandb/run-20250423_122459-k5belznu/logs/debug-internal.log @@ -0,0 +1,18 @@ +{"time":"2025-04-23T12:24:59.620484542Z","level":"INFO","msg":"stream: starting","core version":"0.19.8","symlink path":"checkpoint/Afteri2i_100kckpt_LLAVAT5_empty_Fixed_just_segment_pretrain_stage3_dreambench_recap_400k_t2i_400k_flux_200k_midjourney_150k_recovery_150k_3e4_epoch2_half_rotated/wandb/run-20250423_122459-k5belznu/logs/debug-core.log"} +{"time":"2025-04-23T12:24:59.946787591Z","level":"INFO","msg":"created new stream","id":"k5belznu"} +{"time":"2025-04-23T12:24:59.946828497Z","level":"INFO","msg":"stream: started","id":"k5belznu"} +{"time":"2025-04-23T12:24:59.946878972Z","level":"INFO","msg":"writer: Do: started","stream_id":"k5belznu"} +{"time":"2025-04-23T12:24:59.947187961Z","level":"INFO","msg":"handler: started","stream_id":"k5belznu"} +{"time":"2025-04-23T12:24:59.947210323Z","level":"INFO","msg":"sender: started","stream_id":"k5belznu"} +{"time":"2025-04-23T12:25:00.224703755Z","level":"INFO","msg":"Starting system monitor"} +{"time":"2025-04-23T12:38:21.68929416Z","level":"INFO","msg":"api: retrying HTTP error","status":500,"url":"https://api.wandb.ai/files/haozhezhao/llamagen_ti2i/k5belznu/file_stream","body":"{\"error\":\"context deadline exceeded\"}"} +{"time":"2025-04-23T19:50:15.838080604Z","level":"INFO","msg":"api: retrying HTTP error","status":502,"url":"https://api.wandb.ai/files/haozhezhao/llamagen_ti2i/k5belznu/file_stream","body":"\n\n\n502 Server Error\n\n\n

Error: Server Error

\n

The server encountered a temporary error and could not complete your request.

Please try again in 30 seconds.

\n

\n\n"} +{"time":"2025-04-23T21:12:19.07764583Z","level":"INFO","msg":"api: retrying HTTP error","status":502,"url":"https://api.wandb.ai/files/haozhezhao/llamagen_ti2i/k5belznu/file_stream","body":"\n\n\n502 Server Error\n\n\n

Error: Server Error

\n

The server encountered a temporary error and could not complete your request.

Please try again in 30 seconds.

\n

\n\n"} +{"time":"2025-04-24T01:54:12.787428225Z","level":"INFO","msg":"stream: closing","id":"k5belznu"} +{"time":"2025-04-24T01:54:12.787458982Z","level":"INFO","msg":"Stopping system monitor"} +{"time":"2025-04-24T01:54:12.788155851Z","level":"INFO","msg":"Stopped system monitor"} +{"time":"2025-04-24T01:54:13.990702152Z","level":"INFO","msg":"fileTransfer: Close: file transfer manager closed"} +{"time":"2025-04-24T01:54:14.220871084Z","level":"INFO","msg":"handler: closed","stream_id":"k5belznu"} +{"time":"2025-04-24T01:54:14.22088996Z","level":"INFO","msg":"writer: Close: closed","stream_id":"k5belznu"} +{"time":"2025-04-24T01:54:14.220940735Z","level":"INFO","msg":"sender: closed","stream_id":"k5belznu"} +{"time":"2025-04-24T01:54:14.220951605Z","level":"INFO","msg":"stream: closed","id":"k5belznu"} diff --git a/CKPTS/Afteri2i_100kckpt_LLAVAT5_empty_Fixed_just_segment_pretrain_stage3_dreambench_recap_400k_t2i_400k_flux_200k_midjourney_150k_recovery_150k_3e4_epoch2_half_rotated/wandb/run-20250423_122459-k5belznu/logs/debug.log b/CKPTS/Afteri2i_100kckpt_LLAVAT5_empty_Fixed_just_segment_pretrain_stage3_dreambench_recap_400k_t2i_400k_flux_200k_midjourney_150k_recovery_150k_3e4_epoch2_half_rotated/wandb/run-20250423_122459-k5belznu/logs/debug.log new file mode 100644 index 0000000000000000000000000000000000000000..9b182da5f456197a5acd00209f6c64b4b6ad21ad --- /dev/null +++ b/CKPTS/Afteri2i_100kckpt_LLAVAT5_empty_Fixed_just_segment_pretrain_stage3_dreambench_recap_400k_t2i_400k_flux_200k_midjourney_150k_recovery_150k_3e4_epoch2_half_rotated/wandb/run-20250423_122459-k5belznu/logs/debug.log @@ -0,0 +1,23 @@ +2025-04-23 12:24:59,608 INFO MainThread:3176497 [wandb_setup.py:_flush():67] Current SDK version is 0.19.8 +2025-04-23 12:24:59,608 INFO MainThread:3176497 [wandb_setup.py:_flush():67] Configure stats pid to 3176497 +2025-04-23 12:24:59,608 INFO MainThread:3176497 [wandb_setup.py:_flush():67] Loading settings from /tmp/haozhezhao/.config/wandb/settings +2025-04-23 12:24:59,608 INFO MainThread:3176497 [wandb_setup.py:_flush():67] Loading settings from /tmp/haozhezhao/MLLMG/wandb/settings +2025-04-23 12:24:59,608 INFO MainThread:3176497 [wandb_setup.py:_flush():67] Loading settings from environment variables +2025-04-23 12:24:59,608 INFO MainThread:3176497 [wandb_init.py:setup_run_log_directory():647] Logging user logs to checkpoint/Afteri2i_100kckpt_LLAVAT5_empty_Fixed_just_segment_pretrain_stage3_dreambench_recap_400k_t2i_400k_flux_200k_midjourney_150k_recovery_150k_3e4_epoch2_half_rotated/wandb/run-20250423_122459-k5belznu/logs/debug.log +2025-04-23 12:24:59,608 INFO MainThread:3176497 [wandb_init.py:setup_run_log_directory():648] Logging internal logs to checkpoint/Afteri2i_100kckpt_LLAVAT5_empty_Fixed_just_segment_pretrain_stage3_dreambench_recap_400k_t2i_400k_flux_200k_midjourney_150k_recovery_150k_3e4_epoch2_half_rotated/wandb/run-20250423_122459-k5belznu/logs/debug-internal.log +2025-04-23 12:24:59,608 INFO MainThread:3176497 [wandb_init.py:init():761] calling init triggers +2025-04-23 12:24:59,609 INFO MainThread:3176497 [wandb_init.py:init():766] wandb.init called with sweep_config: {} +config: {'data_path': '/tmp/haozhezhao/MLLMG/jsonl_data/merged_train_set_set_subject_400k_recap_t2i_400k_flux_200k_midjourney_150k_recovery_150k_grounding_100fluxseg_50samseg.jsonl', 'cloud_save_path': '/tmp/haozhezhao/checkpoint', 'no_local_save': False, 'vq_model': 'VQ-16', 'vq_ckpt': '/tmp/haozhezhao/model/llamagen_t2i/vq_ds16_t2i.pt', 'codebook_size': 16384, 'codebook_embed_dim': 8, 'gpt_model': 'GPT-XL', 'gpt_ckpt': '/tmp/haozhezhao/MLLMG/MLLMG_ckpts/checkpoints/AfterI2I_llavat5_subject_train_all_T5_subject_t2i_ti2i_120_w_flux_segment_noimageonly_empty_fix_train_all/0100000.pt', 'gpt_type': 't2i', 'vocab_size': 16384, 'cls_token_num': 512, 'dropout_p': 0.1, 'token_dropout_p': 0.1, 'drop_path': 0.0, 'no_compile': False, 'results_dir': 'checkpoint/Afteri2i_100kckpt_LLAVAT5_empty_Fixed_just_segment_pretrain_stage3_dreambench_recap_400k_t2i_400k_flux_200k_midjourney_150k_recovery_150k_3e4_epoch2_half_rotated', 'dataset': 'ti2i', 'image_size': 512, 'downsample_size': 16, 'num_classes': 1000, 'epochs': 2, 'lr': 0.0003, 'weight_decay': 0.05, 'beta1': 0.9, 'beta2': 0.95, 'max_grad_norm': 1.0, 'global_batch_size': 56, 'global_seed': 0, 'num_workers': 8, 'log_every': 25, 'ckpt_every': 2000, 'gradient_accumulation_steps': 4, 'mixed_precision': 'bf16', 'val_data_path': '/tmp/haozhezhao/MLLMG/jsonl_data/dreambench_plus_valid.jsonl', 'use_vision_tower': True, 'model_name_or_path': '/tmp/haozhezhao/model/blip2-flan-t5-xl', 'image_place_holder': '', 'processor_path': None, 'do_eval': True, 'max_eval_samples': 250, 'train_text_encoder': True, 'no_left_padding': False, 'cfg_scale': 7.5, 'top_k': 16384, 'temperature': 0.9, 'top_p': 1.0, 'eval_steps': 2000, 'project_name': 'llamagen_ti2i', 'load_from_checkpoint': '/tmp/haozhezhao/MLLMG/checkpoint/FIXed_3M_1epoch_step32000.pt', 'warmup': 0.05, 'lr_decay_style': 'cosine', 'lr_decay_ratio': 0.1, 'train_iters': 500000, 'class_dropout_prob': 0.1, 'with_image_only': False, 'image_only_rate': 0.1, 'stage2': False, 'subject_driven': True, 'load_subject_embedding': None, 'reference_data_path': '/tmp/haozhezhao/MLLMG/cc12m_reference_tunnel.jsonl', 'multimodal_encoder': 'llava', 'do_recovery': True, 'no_replace': False, 'resume': False, 'dreambench_eval': True, 'find_unused_parameters': True, 'load_visual_encoder': False, 'continue_stage1': False, 'replace_subject': False, 'train_all': True, 'save_total_limit': 1, 'load_language_projection': '/tmp/haozhezhao/MLLMG/llava-v1.5-flant5_fixed-pretrain/mm_projector.bin', 'mm_vision_tower': 'openai/clip-vit-large-patch14', 'load_fixed_llamagen': True, 'unfreeze_output': False, 'fix': 'gpt-empty-fix', 'rank': 0, 'world_size': 8, 'gpu': 0, 'dist_url': 'env://', 'distributed': True, 'dist_backend': 'nccl', '_wandb': {}} +2025-04-23 12:24:59,609 INFO MainThread:3176497 [wandb_init.py:init():784] starting backend +2025-04-23 12:24:59,609 INFO MainThread:3176497 [wandb_init.py:init():788] sending inform_init request +2025-04-23 12:24:59,614 INFO MainThread:3176497 [backend.py:_multiprocessing_setup():101] multiprocessing start_methods=fork,spawn,forkserver, using: spawn +2025-04-23 12:24:59,614 INFO MainThread:3176497 [wandb_init.py:init():798] backend started and connected +2025-04-23 12:24:59,616 INFO MainThread:3176497 [wandb_init.py:init():891] updated telemetry +2025-04-23 12:24:59,616 INFO MainThread:3176497 [wandb_init.py:init():915] communicating run to backend with 90.0 second timeout +2025-04-23 12:25:00,221 INFO MainThread:3176497 [wandb_init.py:init():990] starting run threads in backend +2025-04-23 12:25:00,347 INFO MainThread:3176497 [wandb_run.py:_console_start():2375] atexit reg +2025-04-23 12:25:00,347 INFO MainThread:3176497 [wandb_run.py:_redirect():2227] redirect: wrap_raw +2025-04-23 12:25:00,348 INFO MainThread:3176497 [wandb_run.py:_redirect():2292] Wrapping output streams. +2025-04-23 12:25:00,348 INFO MainThread:3176497 [wandb_run.py:_redirect():2315] Redirects installed. +2025-04-23 12:25:00,349 INFO MainThread:3176497 [wandb_init.py:init():1032] run started, returning control to user process +2025-04-24 01:54:12,786 INFO MsgRouterThr:3176497 [mailbox.py:close():129] Closing mailbox, abandoning 1 handles. diff --git a/CKPTS/Afteri2i_100kckpt_LLAVAT5_empty_Fixed_just_segment_pretrain_stage3_dreambench_recap_400k_t2i_400k_flux_200k_midjourney_150k_recovery_150k_3e4_epoch2_half_rotated/wandb/run-20250423_122459-k5belznu/run-k5belznu.wandb b/CKPTS/Afteri2i_100kckpt_LLAVAT5_empty_Fixed_just_segment_pretrain_stage3_dreambench_recap_400k_t2i_400k_flux_200k_midjourney_150k_recovery_150k_3e4_epoch2_half_rotated/wandb/run-20250423_122459-k5belznu/run-k5belznu.wandb new file mode 100644 index 0000000000000000000000000000000000000000..dc29afb8be2b7177f53f93e2274ff07bfde904d0 --- /dev/null +++ b/CKPTS/Afteri2i_100kckpt_LLAVAT5_empty_Fixed_just_segment_pretrain_stage3_dreambench_recap_400k_t2i_400k_flux_200k_midjourney_150k_recovery_150k_3e4_epoch2_half_rotated/wandb/run-20250423_122459-k5belznu/run-k5belznu.wandb @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:f33cd0caa6b82e02ed9c6dbf11003dec4e7f5bc0170935bcd4b3674a935d378e +size 31662811 diff --git a/CKPTS/Afteri2i_100kckpt_LLAVAT5_empty_Fixed_just_segment_pretrain_stage3_dreambench_recap_400k_t2i_400k_flux_200k_midjourney_150k_recovery_150k_3e4_epoch2_no_rotated/000-GPT-XL/checkpoints/0046000.pt b/CKPTS/Afteri2i_100kckpt_LLAVAT5_empty_Fixed_just_segment_pretrain_stage3_dreambench_recap_400k_t2i_400k_flux_200k_midjourney_150k_recovery_150k_3e4_epoch2_no_rotated/000-GPT-XL/checkpoints/0046000.pt new file mode 100644 index 0000000000000000000000000000000000000000..ed1c13b1314b969851c3ebd3163ff4506aa4b098 --- /dev/null +++ b/CKPTS/Afteri2i_100kckpt_LLAVAT5_empty_Fixed_just_segment_pretrain_stage3_dreambench_recap_400k_t2i_400k_flux_200k_midjourney_150k_recovery_150k_3e4_epoch2_no_rotated/000-GPT-XL/checkpoints/0046000.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:a1b0a1e3b5089e7ca097ea75c56aac2c078c60fe4ba49173e22bc111b0a8b4d9 +size 17322479662 diff --git a/CKPTS/Afteri2i_100kckpt_LLAVAT5_empty_Fixed_just_segment_pretrain_stage3_dreambench_recap_400k_t2i_400k_flux_200k_midjourney_150k_recovery_150k_3e4_epoch2_no_rotated/000-GPT-XL/checkpoints/eval_step_10000/batch_0_cfg_7.5_topk_16384.jpg b/CKPTS/Afteri2i_100kckpt_LLAVAT5_empty_Fixed_just_segment_pretrain_stage3_dreambench_recap_400k_t2i_400k_flux_200k_midjourney_150k_recovery_150k_3e4_epoch2_no_rotated/000-GPT-XL/checkpoints/eval_step_10000/batch_0_cfg_7.5_topk_16384.jpg new file mode 100644 index 0000000000000000000000000000000000000000..3dc83e60d6b41b617f8fbd3534554b008af9be8b --- /dev/null +++ b/CKPTS/Afteri2i_100kckpt_LLAVAT5_empty_Fixed_just_segment_pretrain_stage3_dreambench_recap_400k_t2i_400k_flux_200k_midjourney_150k_recovery_150k_3e4_epoch2_no_rotated/000-GPT-XL/checkpoints/eval_step_10000/batch_0_cfg_7.5_topk_16384.jpg @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:5db389592ac13a96926201f743e0ddd801f055fe3a877e6e1d00903522bb2f75 +size 718366 diff --git a/CKPTS/Afteri2i_100kckpt_LLAVAT5_empty_Fixed_just_segment_pretrain_stage3_dreambench_recap_400k_t2i_400k_flux_200k_midjourney_150k_recovery_150k_3e4_epoch2_no_rotated/000-GPT-XL/checkpoints/eval_step_10000/batch_1_cfg_7.5_topk_16384.jpg b/CKPTS/Afteri2i_100kckpt_LLAVAT5_empty_Fixed_just_segment_pretrain_stage3_dreambench_recap_400k_t2i_400k_flux_200k_midjourney_150k_recovery_150k_3e4_epoch2_no_rotated/000-GPT-XL/checkpoints/eval_step_10000/batch_1_cfg_7.5_topk_16384.jpg new file mode 100644 index 0000000000000000000000000000000000000000..c27a2c2898445777c77819697149cd00ccd7e67c --- /dev/null +++ b/CKPTS/Afteri2i_100kckpt_LLAVAT5_empty_Fixed_just_segment_pretrain_stage3_dreambench_recap_400k_t2i_400k_flux_200k_midjourney_150k_recovery_150k_3e4_epoch2_no_rotated/000-GPT-XL/checkpoints/eval_step_10000/batch_1_cfg_7.5_topk_16384.jpg @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:1c62030fc8c77b07809a15715c599bd722421824e2c8cd6185d28bd314d18539 +size 702287 diff --git a/CKPTS/Afteri2i_100kckpt_LLAVAT5_empty_Fixed_just_segment_pretrain_stage3_dreambench_recap_400k_t2i_400k_flux_200k_midjourney_150k_recovery_150k_3e4_epoch2_no_rotated/000-GPT-XL/checkpoints/eval_step_10000/batch_2_cfg_7.5_topk_16384.jpg b/CKPTS/Afteri2i_100kckpt_LLAVAT5_empty_Fixed_just_segment_pretrain_stage3_dreambench_recap_400k_t2i_400k_flux_200k_midjourney_150k_recovery_150k_3e4_epoch2_no_rotated/000-GPT-XL/checkpoints/eval_step_10000/batch_2_cfg_7.5_topk_16384.jpg new file mode 100644 index 0000000000000000000000000000000000000000..f4e861d08571bf4f75639e33298d4ee8d03f5132 --- /dev/null +++ b/CKPTS/Afteri2i_100kckpt_LLAVAT5_empty_Fixed_just_segment_pretrain_stage3_dreambench_recap_400k_t2i_400k_flux_200k_midjourney_150k_recovery_150k_3e4_epoch2_no_rotated/000-GPT-XL/checkpoints/eval_step_10000/batch_2_cfg_7.5_topk_16384.jpg @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:37517c0188f5842e55049c471dbde8d8e53fb8002091a5b532e13ebc056842c6 +size 708316 diff --git a/CKPTS/Afteri2i_100kckpt_LLAVAT5_empty_Fixed_just_segment_pretrain_stage3_dreambench_recap_400k_t2i_400k_flux_200k_midjourney_150k_recovery_150k_3e4_epoch2_no_rotated/000-GPT-XL/checkpoints/eval_step_10000/batch_3_cfg_7.5_topk_16384.jpg b/CKPTS/Afteri2i_100kckpt_LLAVAT5_empty_Fixed_just_segment_pretrain_stage3_dreambench_recap_400k_t2i_400k_flux_200k_midjourney_150k_recovery_150k_3e4_epoch2_no_rotated/000-GPT-XL/checkpoints/eval_step_10000/batch_3_cfg_7.5_topk_16384.jpg new file mode 100644 index 0000000000000000000000000000000000000000..d83ae7f0cca69a44e6befbbb9c1935416d45cb15 --- /dev/null +++ b/CKPTS/Afteri2i_100kckpt_LLAVAT5_empty_Fixed_just_segment_pretrain_stage3_dreambench_recap_400k_t2i_400k_flux_200k_midjourney_150k_recovery_150k_3e4_epoch2_no_rotated/000-GPT-XL/checkpoints/eval_step_10000/batch_3_cfg_7.5_topk_16384.jpg @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:a6ac8ece7c76353c5269ccfa8fbec3d5095473f6003a3dd3aa5387c291558d45 +size 839214 diff --git a/CKPTS/Afteri2i_100kckpt_LLAVAT5_empty_Fixed_just_segment_pretrain_stage3_dreambench_recap_400k_t2i_400k_flux_200k_midjourney_150k_recovery_150k_3e4_epoch2_no_rotated/000-GPT-XL/checkpoints/eval_step_12000/batch_0_cfg_7.5_topk_16384.jpg b/CKPTS/Afteri2i_100kckpt_LLAVAT5_empty_Fixed_just_segment_pretrain_stage3_dreambench_recap_400k_t2i_400k_flux_200k_midjourney_150k_recovery_150k_3e4_epoch2_no_rotated/000-GPT-XL/checkpoints/eval_step_12000/batch_0_cfg_7.5_topk_16384.jpg new file mode 100644 index 0000000000000000000000000000000000000000..e07977dedbdb8955ee861246986fb668395d1f49 --- /dev/null +++ b/CKPTS/Afteri2i_100kckpt_LLAVAT5_empty_Fixed_just_segment_pretrain_stage3_dreambench_recap_400k_t2i_400k_flux_200k_midjourney_150k_recovery_150k_3e4_epoch2_no_rotated/000-GPT-XL/checkpoints/eval_step_12000/batch_0_cfg_7.5_topk_16384.jpg @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:153754feddb54a620350dcbe447698381bcfafa2d61808d9461f064fe93fa09f +size 755500 diff --git a/CKPTS/Afteri2i_100kckpt_LLAVAT5_empty_Fixed_just_segment_pretrain_stage3_dreambench_recap_400k_t2i_400k_flux_200k_midjourney_150k_recovery_150k_3e4_epoch2_no_rotated/000-GPT-XL/checkpoints/eval_step_12000/batch_1_cfg_7.5_topk_16384.jpg b/CKPTS/Afteri2i_100kckpt_LLAVAT5_empty_Fixed_just_segment_pretrain_stage3_dreambench_recap_400k_t2i_400k_flux_200k_midjourney_150k_recovery_150k_3e4_epoch2_no_rotated/000-GPT-XL/checkpoints/eval_step_12000/batch_1_cfg_7.5_topk_16384.jpg new file mode 100644 index 0000000000000000000000000000000000000000..d0762a6fd8b3d3d4943ec893ed813c5baba1114f --- /dev/null +++ b/CKPTS/Afteri2i_100kckpt_LLAVAT5_empty_Fixed_just_segment_pretrain_stage3_dreambench_recap_400k_t2i_400k_flux_200k_midjourney_150k_recovery_150k_3e4_epoch2_no_rotated/000-GPT-XL/checkpoints/eval_step_12000/batch_1_cfg_7.5_topk_16384.jpg @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:a3d59f13d29d9b13b2e3288d718c6429b61cd67cdbcf05fd37a25eb0437fd6a7 +size 710290 diff --git a/CKPTS/Afteri2i_100kckpt_LLAVAT5_empty_Fixed_just_segment_pretrain_stage3_dreambench_recap_400k_t2i_400k_flux_200k_midjourney_150k_recovery_150k_3e4_epoch2_no_rotated/000-GPT-XL/checkpoints/eval_step_12000/batch_2_cfg_7.5_topk_16384.jpg b/CKPTS/Afteri2i_100kckpt_LLAVAT5_empty_Fixed_just_segment_pretrain_stage3_dreambench_recap_400k_t2i_400k_flux_200k_midjourney_150k_recovery_150k_3e4_epoch2_no_rotated/000-GPT-XL/checkpoints/eval_step_12000/batch_2_cfg_7.5_topk_16384.jpg new file mode 100644 index 0000000000000000000000000000000000000000..44edf0acfe1f974d733acb0f6fe3f9515f343750 --- /dev/null +++ b/CKPTS/Afteri2i_100kckpt_LLAVAT5_empty_Fixed_just_segment_pretrain_stage3_dreambench_recap_400k_t2i_400k_flux_200k_midjourney_150k_recovery_150k_3e4_epoch2_no_rotated/000-GPT-XL/checkpoints/eval_step_12000/batch_2_cfg_7.5_topk_16384.jpg @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:3dd23a73c999c510f9ae1929f03e7cad32677c122b689a939b620c4f3124c5fc +size 752254 diff --git a/CKPTS/Afteri2i_100kckpt_LLAVAT5_empty_Fixed_just_segment_pretrain_stage3_dreambench_recap_400k_t2i_400k_flux_200k_midjourney_150k_recovery_150k_3e4_epoch2_no_rotated/000-GPT-XL/checkpoints/eval_step_12000/batch_3_cfg_7.5_topk_16384.jpg b/CKPTS/Afteri2i_100kckpt_LLAVAT5_empty_Fixed_just_segment_pretrain_stage3_dreambench_recap_400k_t2i_400k_flux_200k_midjourney_150k_recovery_150k_3e4_epoch2_no_rotated/000-GPT-XL/checkpoints/eval_step_12000/batch_3_cfg_7.5_topk_16384.jpg new file mode 100644 index 0000000000000000000000000000000000000000..a56f53decb7a22b6448f8c8a9ac6d44bd669aee4 --- /dev/null +++ b/CKPTS/Afteri2i_100kckpt_LLAVAT5_empty_Fixed_just_segment_pretrain_stage3_dreambench_recap_400k_t2i_400k_flux_200k_midjourney_150k_recovery_150k_3e4_epoch2_no_rotated/000-GPT-XL/checkpoints/eval_step_12000/batch_3_cfg_7.5_topk_16384.jpg @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:f50f0100921439ffa6e42ce7672aace43b2fd6138ffb4a23f3e417b2012cdfbe +size 842360 diff --git a/CKPTS/Afteri2i_100kckpt_LLAVAT5_empty_Fixed_just_segment_pretrain_stage3_dreambench_recap_400k_t2i_400k_flux_200k_midjourney_150k_recovery_150k_3e4_epoch2_no_rotated/000-GPT-XL/checkpoints/eval_step_14000/batch_0_cfg_7.5_topk_16384.jpg b/CKPTS/Afteri2i_100kckpt_LLAVAT5_empty_Fixed_just_segment_pretrain_stage3_dreambench_recap_400k_t2i_400k_flux_200k_midjourney_150k_recovery_150k_3e4_epoch2_no_rotated/000-GPT-XL/checkpoints/eval_step_14000/batch_0_cfg_7.5_topk_16384.jpg new file mode 100644 index 0000000000000000000000000000000000000000..86450e7b6063ba6573c1e351d2d1ed1a5c0661e8 --- /dev/null +++ b/CKPTS/Afteri2i_100kckpt_LLAVAT5_empty_Fixed_just_segment_pretrain_stage3_dreambench_recap_400k_t2i_400k_flux_200k_midjourney_150k_recovery_150k_3e4_epoch2_no_rotated/000-GPT-XL/checkpoints/eval_step_14000/batch_0_cfg_7.5_topk_16384.jpg @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:5e18e61adcc6d2fa8350624acaba5c4ec6fa74f672c98b98ae851df4f3cac767 +size 744327 diff --git a/CKPTS/Afteri2i_100kckpt_LLAVAT5_empty_Fixed_just_segment_pretrain_stage3_dreambench_recap_400k_t2i_400k_flux_200k_midjourney_150k_recovery_150k_3e4_epoch2_no_rotated/000-GPT-XL/checkpoints/eval_step_14000/batch_1_cfg_7.5_topk_16384.jpg b/CKPTS/Afteri2i_100kckpt_LLAVAT5_empty_Fixed_just_segment_pretrain_stage3_dreambench_recap_400k_t2i_400k_flux_200k_midjourney_150k_recovery_150k_3e4_epoch2_no_rotated/000-GPT-XL/checkpoints/eval_step_14000/batch_1_cfg_7.5_topk_16384.jpg new file mode 100644 index 0000000000000000000000000000000000000000..6dfa8f697eb67169708b70fb8bbf8d1d5cae51cd --- /dev/null +++ b/CKPTS/Afteri2i_100kckpt_LLAVAT5_empty_Fixed_just_segment_pretrain_stage3_dreambench_recap_400k_t2i_400k_flux_200k_midjourney_150k_recovery_150k_3e4_epoch2_no_rotated/000-GPT-XL/checkpoints/eval_step_14000/batch_1_cfg_7.5_topk_16384.jpg @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:0a98c0c2efe045c2e220bb581c61474b4914aea356f373bedf9af8f87bd79046 +size 716429 diff --git a/CKPTS/Afteri2i_100kckpt_LLAVAT5_empty_Fixed_just_segment_pretrain_stage3_dreambench_recap_400k_t2i_400k_flux_200k_midjourney_150k_recovery_150k_3e4_epoch2_no_rotated/000-GPT-XL/checkpoints/eval_step_14000/batch_2_cfg_7.5_topk_16384.jpg b/CKPTS/Afteri2i_100kckpt_LLAVAT5_empty_Fixed_just_segment_pretrain_stage3_dreambench_recap_400k_t2i_400k_flux_200k_midjourney_150k_recovery_150k_3e4_epoch2_no_rotated/000-GPT-XL/checkpoints/eval_step_14000/batch_2_cfg_7.5_topk_16384.jpg new file mode 100644 index 0000000000000000000000000000000000000000..98440a50dcaaf9039b69981d1cd11d6dea090cfe --- /dev/null +++ b/CKPTS/Afteri2i_100kckpt_LLAVAT5_empty_Fixed_just_segment_pretrain_stage3_dreambench_recap_400k_t2i_400k_flux_200k_midjourney_150k_recovery_150k_3e4_epoch2_no_rotated/000-GPT-XL/checkpoints/eval_step_14000/batch_2_cfg_7.5_topk_16384.jpg @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:609da1bd17f6e432a61c32dd69d4824c9c4d7a61091e3da295b04a197906acec +size 715386 diff --git a/CKPTS/Afteri2i_100kckpt_LLAVAT5_empty_Fixed_just_segment_pretrain_stage3_dreambench_recap_400k_t2i_400k_flux_200k_midjourney_150k_recovery_150k_3e4_epoch2_no_rotated/000-GPT-XL/checkpoints/eval_step_14000/batch_3_cfg_7.5_topk_16384.jpg b/CKPTS/Afteri2i_100kckpt_LLAVAT5_empty_Fixed_just_segment_pretrain_stage3_dreambench_recap_400k_t2i_400k_flux_200k_midjourney_150k_recovery_150k_3e4_epoch2_no_rotated/000-GPT-XL/checkpoints/eval_step_14000/batch_3_cfg_7.5_topk_16384.jpg new file mode 100644 index 0000000000000000000000000000000000000000..fed2fd477ed3c4091fa3c0d3afaba51075101025 --- /dev/null +++ b/CKPTS/Afteri2i_100kckpt_LLAVAT5_empty_Fixed_just_segment_pretrain_stage3_dreambench_recap_400k_t2i_400k_flux_200k_midjourney_150k_recovery_150k_3e4_epoch2_no_rotated/000-GPT-XL/checkpoints/eval_step_14000/batch_3_cfg_7.5_topk_16384.jpg @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:7ad85d1a78c9b40c1d4ff959ecd202328809b6a72b48d9cf797d0b33792a6c23 +size 840398 diff --git a/CKPTS/Afteri2i_100kckpt_LLAVAT5_empty_Fixed_just_segment_pretrain_stage3_dreambench_recap_400k_t2i_400k_flux_200k_midjourney_150k_recovery_150k_3e4_epoch2_no_rotated/000-GPT-XL/checkpoints/eval_step_16000/batch_0_cfg_7.5_topk_16384.jpg b/CKPTS/Afteri2i_100kckpt_LLAVAT5_empty_Fixed_just_segment_pretrain_stage3_dreambench_recap_400k_t2i_400k_flux_200k_midjourney_150k_recovery_150k_3e4_epoch2_no_rotated/000-GPT-XL/checkpoints/eval_step_16000/batch_0_cfg_7.5_topk_16384.jpg new file mode 100644 index 0000000000000000000000000000000000000000..e6954c49f68eb1f147b6e23f20f8a23d98f339d5 --- /dev/null +++ b/CKPTS/Afteri2i_100kckpt_LLAVAT5_empty_Fixed_just_segment_pretrain_stage3_dreambench_recap_400k_t2i_400k_flux_200k_midjourney_150k_recovery_150k_3e4_epoch2_no_rotated/000-GPT-XL/checkpoints/eval_step_16000/batch_0_cfg_7.5_topk_16384.jpg @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:c1cee4f6d633692b7fa40a5b1cc812986521a81122a540a3c68c6c0ff26ed73f +size 737026 diff --git a/CKPTS/Afteri2i_100kckpt_LLAVAT5_empty_Fixed_just_segment_pretrain_stage3_dreambench_recap_400k_t2i_400k_flux_200k_midjourney_150k_recovery_150k_3e4_epoch2_no_rotated/000-GPT-XL/checkpoints/eval_step_16000/batch_1_cfg_7.5_topk_16384.jpg b/CKPTS/Afteri2i_100kckpt_LLAVAT5_empty_Fixed_just_segment_pretrain_stage3_dreambench_recap_400k_t2i_400k_flux_200k_midjourney_150k_recovery_150k_3e4_epoch2_no_rotated/000-GPT-XL/checkpoints/eval_step_16000/batch_1_cfg_7.5_topk_16384.jpg new file mode 100644 index 0000000000000000000000000000000000000000..b2c759f8d575dc76d53cfcd98e6e1c092b166f60 --- /dev/null +++ b/CKPTS/Afteri2i_100kckpt_LLAVAT5_empty_Fixed_just_segment_pretrain_stage3_dreambench_recap_400k_t2i_400k_flux_200k_midjourney_150k_recovery_150k_3e4_epoch2_no_rotated/000-GPT-XL/checkpoints/eval_step_16000/batch_1_cfg_7.5_topk_16384.jpg @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:000cdd692f3b9d24f3e176ca409ce04823fae5fb3286a0e047014b0b3884ffdb +size 703541 diff --git a/CKPTS/Afteri2i_100kckpt_LLAVAT5_empty_Fixed_just_segment_pretrain_stage3_dreambench_recap_400k_t2i_400k_flux_200k_midjourney_150k_recovery_150k_3e4_epoch2_no_rotated/000-GPT-XL/checkpoints/eval_step_16000/batch_2_cfg_7.5_topk_16384.jpg b/CKPTS/Afteri2i_100kckpt_LLAVAT5_empty_Fixed_just_segment_pretrain_stage3_dreambench_recap_400k_t2i_400k_flux_200k_midjourney_150k_recovery_150k_3e4_epoch2_no_rotated/000-GPT-XL/checkpoints/eval_step_16000/batch_2_cfg_7.5_topk_16384.jpg new file mode 100644 index 0000000000000000000000000000000000000000..f65039359882b57c3a38361ef74cef11f5180c89 --- /dev/null +++ b/CKPTS/Afteri2i_100kckpt_LLAVAT5_empty_Fixed_just_segment_pretrain_stage3_dreambench_recap_400k_t2i_400k_flux_200k_midjourney_150k_recovery_150k_3e4_epoch2_no_rotated/000-GPT-XL/checkpoints/eval_step_16000/batch_2_cfg_7.5_topk_16384.jpg @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:93535f0b2e767a093ad4e5dd7c30f49f3cd977b61ef05161db046a9eed856db7 +size 707261 diff --git a/CKPTS/Afteri2i_100kckpt_LLAVAT5_empty_Fixed_just_segment_pretrain_stage3_dreambench_recap_400k_t2i_400k_flux_200k_midjourney_150k_recovery_150k_3e4_epoch2_no_rotated/000-GPT-XL/checkpoints/eval_step_16000/batch_3_cfg_7.5_topk_16384.jpg b/CKPTS/Afteri2i_100kckpt_LLAVAT5_empty_Fixed_just_segment_pretrain_stage3_dreambench_recap_400k_t2i_400k_flux_200k_midjourney_150k_recovery_150k_3e4_epoch2_no_rotated/000-GPT-XL/checkpoints/eval_step_16000/batch_3_cfg_7.5_topk_16384.jpg new file mode 100644 index 0000000000000000000000000000000000000000..62e7446f19cfa791d7a2b548e6c1eaa50dc2d414 --- /dev/null +++ b/CKPTS/Afteri2i_100kckpt_LLAVAT5_empty_Fixed_just_segment_pretrain_stage3_dreambench_recap_400k_t2i_400k_flux_200k_midjourney_150k_recovery_150k_3e4_epoch2_no_rotated/000-GPT-XL/checkpoints/eval_step_16000/batch_3_cfg_7.5_topk_16384.jpg @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:e456077c15cd7134968824a003d6861b8c721886552e2d6390508cf8ce7dc48b +size 868497 diff --git a/CKPTS/Afteri2i_100kckpt_LLAVAT5_empty_Fixed_just_segment_pretrain_stage3_dreambench_recap_400k_t2i_400k_flux_200k_midjourney_150k_recovery_150k_3e4_epoch2_no_rotated/000-GPT-XL/checkpoints/eval_step_18000/batch_0_cfg_7.5_topk_16384.jpg b/CKPTS/Afteri2i_100kckpt_LLAVAT5_empty_Fixed_just_segment_pretrain_stage3_dreambench_recap_400k_t2i_400k_flux_200k_midjourney_150k_recovery_150k_3e4_epoch2_no_rotated/000-GPT-XL/checkpoints/eval_step_18000/batch_0_cfg_7.5_topk_16384.jpg new file mode 100644 index 0000000000000000000000000000000000000000..1297d018ec203c1a6a8d8d0867be5d9180ee3c4c --- /dev/null +++ b/CKPTS/Afteri2i_100kckpt_LLAVAT5_empty_Fixed_just_segment_pretrain_stage3_dreambench_recap_400k_t2i_400k_flux_200k_midjourney_150k_recovery_150k_3e4_epoch2_no_rotated/000-GPT-XL/checkpoints/eval_step_18000/batch_0_cfg_7.5_topk_16384.jpg @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:b1a587e6ac7748187e92f2e5a7331b7793bc300cc46ecf7b5b54ce6bede828f4 +size 770469 diff --git a/CKPTS/Afteri2i_100kckpt_LLAVAT5_empty_Fixed_just_segment_pretrain_stage3_dreambench_recap_400k_t2i_400k_flux_200k_midjourney_150k_recovery_150k_3e4_epoch2_no_rotated/000-GPT-XL/checkpoints/eval_step_18000/batch_1_cfg_7.5_topk_16384.jpg b/CKPTS/Afteri2i_100kckpt_LLAVAT5_empty_Fixed_just_segment_pretrain_stage3_dreambench_recap_400k_t2i_400k_flux_200k_midjourney_150k_recovery_150k_3e4_epoch2_no_rotated/000-GPT-XL/checkpoints/eval_step_18000/batch_1_cfg_7.5_topk_16384.jpg new file mode 100644 index 0000000000000000000000000000000000000000..cae896e873bfeba1366947cbee379aa102b077f3 --- /dev/null +++ b/CKPTS/Afteri2i_100kckpt_LLAVAT5_empty_Fixed_just_segment_pretrain_stage3_dreambench_recap_400k_t2i_400k_flux_200k_midjourney_150k_recovery_150k_3e4_epoch2_no_rotated/000-GPT-XL/checkpoints/eval_step_18000/batch_1_cfg_7.5_topk_16384.jpg @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:fbf13726f333e301cda8fbbac8e41054ca0e8e7478db44f4bda3aa59e8e2422f +size 705801 diff --git a/CKPTS/Afteri2i_100kckpt_LLAVAT5_empty_Fixed_just_segment_pretrain_stage3_dreambench_recap_400k_t2i_400k_flux_200k_midjourney_150k_recovery_150k_3e4_epoch2_no_rotated/000-GPT-XL/checkpoints/eval_step_18000/batch_2_cfg_7.5_topk_16384.jpg b/CKPTS/Afteri2i_100kckpt_LLAVAT5_empty_Fixed_just_segment_pretrain_stage3_dreambench_recap_400k_t2i_400k_flux_200k_midjourney_150k_recovery_150k_3e4_epoch2_no_rotated/000-GPT-XL/checkpoints/eval_step_18000/batch_2_cfg_7.5_topk_16384.jpg new file mode 100644 index 0000000000000000000000000000000000000000..cbc7585c9dc47079838f3b692feb8fa7da5757d7 --- /dev/null +++ b/CKPTS/Afteri2i_100kckpt_LLAVAT5_empty_Fixed_just_segment_pretrain_stage3_dreambench_recap_400k_t2i_400k_flux_200k_midjourney_150k_recovery_150k_3e4_epoch2_no_rotated/000-GPT-XL/checkpoints/eval_step_18000/batch_2_cfg_7.5_topk_16384.jpg @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:5f0e5dfd43e49c7f0635299078b1469231f8f0b285cc245011dd5d1890c3433e +size 773014 diff --git a/CKPTS/Afteri2i_100kckpt_LLAVAT5_empty_Fixed_just_segment_pretrain_stage3_dreambench_recap_400k_t2i_400k_flux_200k_midjourney_150k_recovery_150k_3e4_epoch2_no_rotated/000-GPT-XL/checkpoints/eval_step_18000/batch_3_cfg_7.5_topk_16384.jpg b/CKPTS/Afteri2i_100kckpt_LLAVAT5_empty_Fixed_just_segment_pretrain_stage3_dreambench_recap_400k_t2i_400k_flux_200k_midjourney_150k_recovery_150k_3e4_epoch2_no_rotated/000-GPT-XL/checkpoints/eval_step_18000/batch_3_cfg_7.5_topk_16384.jpg new file mode 100644 index 0000000000000000000000000000000000000000..6e3fbef5956f9f8e12b9cc4ecc8c795aff896507 --- /dev/null +++ b/CKPTS/Afteri2i_100kckpt_LLAVAT5_empty_Fixed_just_segment_pretrain_stage3_dreambench_recap_400k_t2i_400k_flux_200k_midjourney_150k_recovery_150k_3e4_epoch2_no_rotated/000-GPT-XL/checkpoints/eval_step_18000/batch_3_cfg_7.5_topk_16384.jpg @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:a0c800363e273ca84fb9e4d68d0d5f46eb1159888d78308693fbde43c5f17940 +size 831308 diff --git a/CKPTS/Afteri2i_100kckpt_LLAVAT5_empty_Fixed_just_segment_pretrain_stage3_dreambench_recap_400k_t2i_400k_flux_200k_midjourney_150k_recovery_150k_3e4_epoch2_no_rotated/000-GPT-XL/checkpoints/eval_step_2000/batch_0_cfg_7.5_topk_16384.jpg b/CKPTS/Afteri2i_100kckpt_LLAVAT5_empty_Fixed_just_segment_pretrain_stage3_dreambench_recap_400k_t2i_400k_flux_200k_midjourney_150k_recovery_150k_3e4_epoch2_no_rotated/000-GPT-XL/checkpoints/eval_step_2000/batch_0_cfg_7.5_topk_16384.jpg new file mode 100644 index 0000000000000000000000000000000000000000..45db57741cf70caaa8dfcdad52f54eda4d3c0633 --- /dev/null +++ b/CKPTS/Afteri2i_100kckpt_LLAVAT5_empty_Fixed_just_segment_pretrain_stage3_dreambench_recap_400k_t2i_400k_flux_200k_midjourney_150k_recovery_150k_3e4_epoch2_no_rotated/000-GPT-XL/checkpoints/eval_step_2000/batch_0_cfg_7.5_topk_16384.jpg @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:51a4e97bad24c254894c4111e12353fd7c2cfbdd90bc2621e2cbc4dceca2d21b +size 753759 diff --git a/CKPTS/Afteri2i_100kckpt_LLAVAT5_empty_Fixed_just_segment_pretrain_stage3_dreambench_recap_400k_t2i_400k_flux_200k_midjourney_150k_recovery_150k_3e4_epoch2_no_rotated/000-GPT-XL/checkpoints/eval_step_2000/batch_1_cfg_7.5_topk_16384.jpg b/CKPTS/Afteri2i_100kckpt_LLAVAT5_empty_Fixed_just_segment_pretrain_stage3_dreambench_recap_400k_t2i_400k_flux_200k_midjourney_150k_recovery_150k_3e4_epoch2_no_rotated/000-GPT-XL/checkpoints/eval_step_2000/batch_1_cfg_7.5_topk_16384.jpg new file mode 100644 index 0000000000000000000000000000000000000000..dfc6a01d744a758de1f659f3a712f5050f155ab5 --- /dev/null +++ b/CKPTS/Afteri2i_100kckpt_LLAVAT5_empty_Fixed_just_segment_pretrain_stage3_dreambench_recap_400k_t2i_400k_flux_200k_midjourney_150k_recovery_150k_3e4_epoch2_no_rotated/000-GPT-XL/checkpoints/eval_step_2000/batch_1_cfg_7.5_topk_16384.jpg @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:6643a80f97c36e0ed6ac2b4435452dcfb89a2e212fb6135efb43a2f1005f9aff +size 702996 diff --git a/CKPTS/Afteri2i_100kckpt_LLAVAT5_empty_Fixed_just_segment_pretrain_stage3_dreambench_recap_400k_t2i_400k_flux_200k_midjourney_150k_recovery_150k_3e4_epoch2_no_rotated/000-GPT-XL/checkpoints/eval_step_2000/batch_2_cfg_7.5_topk_16384.jpg b/CKPTS/Afteri2i_100kckpt_LLAVAT5_empty_Fixed_just_segment_pretrain_stage3_dreambench_recap_400k_t2i_400k_flux_200k_midjourney_150k_recovery_150k_3e4_epoch2_no_rotated/000-GPT-XL/checkpoints/eval_step_2000/batch_2_cfg_7.5_topk_16384.jpg new file mode 100644 index 0000000000000000000000000000000000000000..8c76dc68c4e70a48390037f0e124fca557afd4d9 --- /dev/null +++ b/CKPTS/Afteri2i_100kckpt_LLAVAT5_empty_Fixed_just_segment_pretrain_stage3_dreambench_recap_400k_t2i_400k_flux_200k_midjourney_150k_recovery_150k_3e4_epoch2_no_rotated/000-GPT-XL/checkpoints/eval_step_2000/batch_2_cfg_7.5_topk_16384.jpg @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:85c57c892abf46248b26f2413eedf41bc04297e76f64bce5ff943a7fb576a63c +size 645957 diff --git a/CKPTS/Afteri2i_100kckpt_LLAVAT5_empty_Fixed_just_segment_pretrain_stage3_dreambench_recap_400k_t2i_400k_flux_200k_midjourney_150k_recovery_150k_3e4_epoch2_no_rotated/000-GPT-XL/checkpoints/eval_step_2000/batch_3_cfg_7.5_topk_16384.jpg b/CKPTS/Afteri2i_100kckpt_LLAVAT5_empty_Fixed_just_segment_pretrain_stage3_dreambench_recap_400k_t2i_400k_flux_200k_midjourney_150k_recovery_150k_3e4_epoch2_no_rotated/000-GPT-XL/checkpoints/eval_step_2000/batch_3_cfg_7.5_topk_16384.jpg new file mode 100644 index 0000000000000000000000000000000000000000..4794cd56273c6af849a1f210e1d0f81d29de92a1 --- /dev/null +++ b/CKPTS/Afteri2i_100kckpt_LLAVAT5_empty_Fixed_just_segment_pretrain_stage3_dreambench_recap_400k_t2i_400k_flux_200k_midjourney_150k_recovery_150k_3e4_epoch2_no_rotated/000-GPT-XL/checkpoints/eval_step_2000/batch_3_cfg_7.5_topk_16384.jpg @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:64d772226340f32b5d3403edd4a2b792d3e22364173ba3304625cab35a70a1ff +size 769054 diff --git a/CKPTS/Afteri2i_100kckpt_LLAVAT5_empty_Fixed_just_segment_pretrain_stage3_dreambench_recap_400k_t2i_400k_flux_200k_midjourney_150k_recovery_150k_3e4_epoch2_no_rotated/000-GPT-XL/checkpoints/eval_step_20000/batch_0_cfg_7.5_topk_16384.jpg b/CKPTS/Afteri2i_100kckpt_LLAVAT5_empty_Fixed_just_segment_pretrain_stage3_dreambench_recap_400k_t2i_400k_flux_200k_midjourney_150k_recovery_150k_3e4_epoch2_no_rotated/000-GPT-XL/checkpoints/eval_step_20000/batch_0_cfg_7.5_topk_16384.jpg new file mode 100644 index 0000000000000000000000000000000000000000..bf95faf06c608edd889fa5291db927abe2cc99b8 --- /dev/null +++ b/CKPTS/Afteri2i_100kckpt_LLAVAT5_empty_Fixed_just_segment_pretrain_stage3_dreambench_recap_400k_t2i_400k_flux_200k_midjourney_150k_recovery_150k_3e4_epoch2_no_rotated/000-GPT-XL/checkpoints/eval_step_20000/batch_0_cfg_7.5_topk_16384.jpg @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:f2aa33491641f2ff7a7ce0fc25448364875f0855ace8bb76864950cc94f71327 +size 729560 diff --git a/CKPTS/Afteri2i_100kckpt_LLAVAT5_empty_Fixed_just_segment_pretrain_stage3_dreambench_recap_400k_t2i_400k_flux_200k_midjourney_150k_recovery_150k_3e4_epoch2_no_rotated/000-GPT-XL/checkpoints/eval_step_20000/batch_1_cfg_7.5_topk_16384.jpg b/CKPTS/Afteri2i_100kckpt_LLAVAT5_empty_Fixed_just_segment_pretrain_stage3_dreambench_recap_400k_t2i_400k_flux_200k_midjourney_150k_recovery_150k_3e4_epoch2_no_rotated/000-GPT-XL/checkpoints/eval_step_20000/batch_1_cfg_7.5_topk_16384.jpg new file mode 100644 index 0000000000000000000000000000000000000000..6cb21875f5fc380da7a51a20b13af17d0c9a3b1f --- /dev/null +++ b/CKPTS/Afteri2i_100kckpt_LLAVAT5_empty_Fixed_just_segment_pretrain_stage3_dreambench_recap_400k_t2i_400k_flux_200k_midjourney_150k_recovery_150k_3e4_epoch2_no_rotated/000-GPT-XL/checkpoints/eval_step_20000/batch_1_cfg_7.5_topk_16384.jpg @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:f33e3e6d36bea0f928775b98c8e7e721af1c45a4e7b289584ab1a441c25ee8d2 +size 719506 diff --git a/CKPTS/Afteri2i_100kckpt_LLAVAT5_empty_Fixed_just_segment_pretrain_stage3_dreambench_recap_400k_t2i_400k_flux_200k_midjourney_150k_recovery_150k_3e4_epoch2_no_rotated/000-GPT-XL/checkpoints/eval_step_20000/batch_2_cfg_7.5_topk_16384.jpg b/CKPTS/Afteri2i_100kckpt_LLAVAT5_empty_Fixed_just_segment_pretrain_stage3_dreambench_recap_400k_t2i_400k_flux_200k_midjourney_150k_recovery_150k_3e4_epoch2_no_rotated/000-GPT-XL/checkpoints/eval_step_20000/batch_2_cfg_7.5_topk_16384.jpg new file mode 100644 index 0000000000000000000000000000000000000000..83842174f171465c6ac522b6e3820e9a4f82c22d --- /dev/null +++ b/CKPTS/Afteri2i_100kckpt_LLAVAT5_empty_Fixed_just_segment_pretrain_stage3_dreambench_recap_400k_t2i_400k_flux_200k_midjourney_150k_recovery_150k_3e4_epoch2_no_rotated/000-GPT-XL/checkpoints/eval_step_20000/batch_2_cfg_7.5_topk_16384.jpg @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:753709b6a763ce79bf36f7aecad0c09effbebad628d24d5705303ae43688d81a +size 733263 diff --git a/CKPTS/Afteri2i_100kckpt_LLAVAT5_empty_Fixed_just_segment_pretrain_stage3_dreambench_recap_400k_t2i_400k_flux_200k_midjourney_150k_recovery_150k_3e4_epoch2_no_rotated/000-GPT-XL/checkpoints/eval_step_20000/batch_3_cfg_7.5_topk_16384.jpg b/CKPTS/Afteri2i_100kckpt_LLAVAT5_empty_Fixed_just_segment_pretrain_stage3_dreambench_recap_400k_t2i_400k_flux_200k_midjourney_150k_recovery_150k_3e4_epoch2_no_rotated/000-GPT-XL/checkpoints/eval_step_20000/batch_3_cfg_7.5_topk_16384.jpg new file mode 100644 index 0000000000000000000000000000000000000000..a522f30e4f858b775e861b7c189f893155e99676 --- /dev/null +++ b/CKPTS/Afteri2i_100kckpt_LLAVAT5_empty_Fixed_just_segment_pretrain_stage3_dreambench_recap_400k_t2i_400k_flux_200k_midjourney_150k_recovery_150k_3e4_epoch2_no_rotated/000-GPT-XL/checkpoints/eval_step_20000/batch_3_cfg_7.5_topk_16384.jpg @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:7ecc3e3cfc86aa74a060519549efeb406cc5b1c994af36097b411e2bbb2ab27f +size 905568 diff --git a/CKPTS/Afteri2i_100kckpt_LLAVAT5_empty_Fixed_just_segment_pretrain_stage3_dreambench_recap_400k_t2i_400k_flux_200k_midjourney_150k_recovery_150k_3e4_epoch2_no_rotated/000-GPT-XL/checkpoints/eval_step_22000/batch_0_cfg_7.5_topk_16384.jpg b/CKPTS/Afteri2i_100kckpt_LLAVAT5_empty_Fixed_just_segment_pretrain_stage3_dreambench_recap_400k_t2i_400k_flux_200k_midjourney_150k_recovery_150k_3e4_epoch2_no_rotated/000-GPT-XL/checkpoints/eval_step_22000/batch_0_cfg_7.5_topk_16384.jpg new file mode 100644 index 0000000000000000000000000000000000000000..d12a1407179cae4abc09bfb1f9bea666bda0e9f1 --- /dev/null +++ b/CKPTS/Afteri2i_100kckpt_LLAVAT5_empty_Fixed_just_segment_pretrain_stage3_dreambench_recap_400k_t2i_400k_flux_200k_midjourney_150k_recovery_150k_3e4_epoch2_no_rotated/000-GPT-XL/checkpoints/eval_step_22000/batch_0_cfg_7.5_topk_16384.jpg @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:f858aab576fbd7ac909ad122d8100c6318691b012cd391eb7e98a2fdda9cc2be +size 715258 diff --git a/CKPTS/Afteri2i_100kckpt_LLAVAT5_empty_Fixed_just_segment_pretrain_stage3_dreambench_recap_400k_t2i_400k_flux_200k_midjourney_150k_recovery_150k_3e4_epoch2_no_rotated/000-GPT-XL/checkpoints/eval_step_22000/batch_1_cfg_7.5_topk_16384.jpg b/CKPTS/Afteri2i_100kckpt_LLAVAT5_empty_Fixed_just_segment_pretrain_stage3_dreambench_recap_400k_t2i_400k_flux_200k_midjourney_150k_recovery_150k_3e4_epoch2_no_rotated/000-GPT-XL/checkpoints/eval_step_22000/batch_1_cfg_7.5_topk_16384.jpg new file mode 100644 index 0000000000000000000000000000000000000000..598e0183afa32d0119e34c2516ac52a85c73209c --- /dev/null +++ b/CKPTS/Afteri2i_100kckpt_LLAVAT5_empty_Fixed_just_segment_pretrain_stage3_dreambench_recap_400k_t2i_400k_flux_200k_midjourney_150k_recovery_150k_3e4_epoch2_no_rotated/000-GPT-XL/checkpoints/eval_step_22000/batch_1_cfg_7.5_topk_16384.jpg @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:aedd920894a9e658f325dee354f56205a38127735c2c70b1737f2903addd8d15 +size 703085 diff --git a/CKPTS/Afteri2i_100kckpt_LLAVAT5_empty_Fixed_just_segment_pretrain_stage3_dreambench_recap_400k_t2i_400k_flux_200k_midjourney_150k_recovery_150k_3e4_epoch2_no_rotated/000-GPT-XL/checkpoints/eval_step_22000/batch_2_cfg_7.5_topk_16384.jpg b/CKPTS/Afteri2i_100kckpt_LLAVAT5_empty_Fixed_just_segment_pretrain_stage3_dreambench_recap_400k_t2i_400k_flux_200k_midjourney_150k_recovery_150k_3e4_epoch2_no_rotated/000-GPT-XL/checkpoints/eval_step_22000/batch_2_cfg_7.5_topk_16384.jpg new file mode 100644 index 0000000000000000000000000000000000000000..396bfa19378b3dae9379d17b3827b0bbcd3e2152 --- /dev/null +++ b/CKPTS/Afteri2i_100kckpt_LLAVAT5_empty_Fixed_just_segment_pretrain_stage3_dreambench_recap_400k_t2i_400k_flux_200k_midjourney_150k_recovery_150k_3e4_epoch2_no_rotated/000-GPT-XL/checkpoints/eval_step_22000/batch_2_cfg_7.5_topk_16384.jpg @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:f96f9f3d60b3f07cacedaf8ef50bb86b527e122eedf3fc2a074efebd3a6741e3 +size 723555 diff --git a/CKPTS/Afteri2i_100kckpt_LLAVAT5_empty_Fixed_just_segment_pretrain_stage3_dreambench_recap_400k_t2i_400k_flux_200k_midjourney_150k_recovery_150k_3e4_epoch2_no_rotated/000-GPT-XL/checkpoints/eval_step_22000/batch_3_cfg_7.5_topk_16384.jpg b/CKPTS/Afteri2i_100kckpt_LLAVAT5_empty_Fixed_just_segment_pretrain_stage3_dreambench_recap_400k_t2i_400k_flux_200k_midjourney_150k_recovery_150k_3e4_epoch2_no_rotated/000-GPT-XL/checkpoints/eval_step_22000/batch_3_cfg_7.5_topk_16384.jpg new file mode 100644 index 0000000000000000000000000000000000000000..6d3fff9d4029932806f8327bd5aa3193c54930f6 --- /dev/null +++ b/CKPTS/Afteri2i_100kckpt_LLAVAT5_empty_Fixed_just_segment_pretrain_stage3_dreambench_recap_400k_t2i_400k_flux_200k_midjourney_150k_recovery_150k_3e4_epoch2_no_rotated/000-GPT-XL/checkpoints/eval_step_22000/batch_3_cfg_7.5_topk_16384.jpg @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:7b210c74075d0a203957f192a56346c762d7f1b56ae0c9737541e9ccd8f36bc8 +size 843246 diff --git a/CKPTS/Afteri2i_100kckpt_LLAVAT5_empty_Fixed_just_segment_pretrain_stage3_dreambench_recap_400k_t2i_400k_flux_200k_midjourney_150k_recovery_150k_3e4_epoch2_no_rotated/000-GPT-XL/checkpoints/eval_step_24000/batch_0_cfg_7.5_topk_16384.jpg b/CKPTS/Afteri2i_100kckpt_LLAVAT5_empty_Fixed_just_segment_pretrain_stage3_dreambench_recap_400k_t2i_400k_flux_200k_midjourney_150k_recovery_150k_3e4_epoch2_no_rotated/000-GPT-XL/checkpoints/eval_step_24000/batch_0_cfg_7.5_topk_16384.jpg new file mode 100644 index 0000000000000000000000000000000000000000..4cd066475704d0e8404f9c3a3113662571b89117 --- /dev/null +++ b/CKPTS/Afteri2i_100kckpt_LLAVAT5_empty_Fixed_just_segment_pretrain_stage3_dreambench_recap_400k_t2i_400k_flux_200k_midjourney_150k_recovery_150k_3e4_epoch2_no_rotated/000-GPT-XL/checkpoints/eval_step_24000/batch_0_cfg_7.5_topk_16384.jpg @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:c4b57eac7ff0214ed14e775ad1a618f6261a610f112997f9cf3dd999c5b6f380 +size 740186 diff --git a/CKPTS/Afteri2i_100kckpt_LLAVAT5_empty_Fixed_just_segment_pretrain_stage3_dreambench_recap_400k_t2i_400k_flux_200k_midjourney_150k_recovery_150k_3e4_epoch2_no_rotated/000-GPT-XL/checkpoints/eval_step_24000/batch_1_cfg_7.5_topk_16384.jpg b/CKPTS/Afteri2i_100kckpt_LLAVAT5_empty_Fixed_just_segment_pretrain_stage3_dreambench_recap_400k_t2i_400k_flux_200k_midjourney_150k_recovery_150k_3e4_epoch2_no_rotated/000-GPT-XL/checkpoints/eval_step_24000/batch_1_cfg_7.5_topk_16384.jpg new file mode 100644 index 0000000000000000000000000000000000000000..bafa7ed7fa085933a26239f98ede657e0a06ed80 --- /dev/null +++ b/CKPTS/Afteri2i_100kckpt_LLAVAT5_empty_Fixed_just_segment_pretrain_stage3_dreambench_recap_400k_t2i_400k_flux_200k_midjourney_150k_recovery_150k_3e4_epoch2_no_rotated/000-GPT-XL/checkpoints/eval_step_24000/batch_1_cfg_7.5_topk_16384.jpg @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:6018eda1d888a9425998d1bddb6ff5da54dd3be4a89f6e2f2a41433e0370e496 +size 716434 diff --git a/CKPTS/Afteri2i_100kckpt_LLAVAT5_empty_Fixed_just_segment_pretrain_stage3_dreambench_recap_400k_t2i_400k_flux_200k_midjourney_150k_recovery_150k_3e4_epoch2_no_rotated/000-GPT-XL/checkpoints/eval_step_24000/batch_2_cfg_7.5_topk_16384.jpg b/CKPTS/Afteri2i_100kckpt_LLAVAT5_empty_Fixed_just_segment_pretrain_stage3_dreambench_recap_400k_t2i_400k_flux_200k_midjourney_150k_recovery_150k_3e4_epoch2_no_rotated/000-GPT-XL/checkpoints/eval_step_24000/batch_2_cfg_7.5_topk_16384.jpg new file mode 100644 index 0000000000000000000000000000000000000000..272dbc85ab8b72521ed53964d981ee1d1b2a0f65 --- /dev/null +++ b/CKPTS/Afteri2i_100kckpt_LLAVAT5_empty_Fixed_just_segment_pretrain_stage3_dreambench_recap_400k_t2i_400k_flux_200k_midjourney_150k_recovery_150k_3e4_epoch2_no_rotated/000-GPT-XL/checkpoints/eval_step_24000/batch_2_cfg_7.5_topk_16384.jpg @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:ec9e7de7bf2fb3205c1e8f8582391470257dc8ec0d05441e7ece9774898e8b4f +size 737332 diff --git a/CKPTS/Afteri2i_100kckpt_LLAVAT5_empty_Fixed_just_segment_pretrain_stage3_dreambench_recap_400k_t2i_400k_flux_200k_midjourney_150k_recovery_150k_3e4_epoch2_no_rotated/000-GPT-XL/checkpoints/eval_step_24000/batch_3_cfg_7.5_topk_16384.jpg b/CKPTS/Afteri2i_100kckpt_LLAVAT5_empty_Fixed_just_segment_pretrain_stage3_dreambench_recap_400k_t2i_400k_flux_200k_midjourney_150k_recovery_150k_3e4_epoch2_no_rotated/000-GPT-XL/checkpoints/eval_step_24000/batch_3_cfg_7.5_topk_16384.jpg new file mode 100644 index 0000000000000000000000000000000000000000..4db710323d6c6b2d3fae4f6348f49cf5ce241085 --- /dev/null +++ b/CKPTS/Afteri2i_100kckpt_LLAVAT5_empty_Fixed_just_segment_pretrain_stage3_dreambench_recap_400k_t2i_400k_flux_200k_midjourney_150k_recovery_150k_3e4_epoch2_no_rotated/000-GPT-XL/checkpoints/eval_step_24000/batch_3_cfg_7.5_topk_16384.jpg @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:3702bb0f65c747480ea32c4a691b26be140cbf50f7c805e288427e5f93ba23b5 +size 880000 diff --git a/CKPTS/Afteri2i_100kckpt_LLAVAT5_empty_Fixed_just_segment_pretrain_stage3_dreambench_recap_400k_t2i_400k_flux_200k_midjourney_150k_recovery_150k_3e4_epoch2_no_rotated/000-GPT-XL/checkpoints/eval_step_26000/batch_0_cfg_7.5_topk_16384.jpg b/CKPTS/Afteri2i_100kckpt_LLAVAT5_empty_Fixed_just_segment_pretrain_stage3_dreambench_recap_400k_t2i_400k_flux_200k_midjourney_150k_recovery_150k_3e4_epoch2_no_rotated/000-GPT-XL/checkpoints/eval_step_26000/batch_0_cfg_7.5_topk_16384.jpg new file mode 100644 index 0000000000000000000000000000000000000000..d5a808ac2a9c9c8e3fe007b6e83eaf2ff86ad153 --- /dev/null +++ b/CKPTS/Afteri2i_100kckpt_LLAVAT5_empty_Fixed_just_segment_pretrain_stage3_dreambench_recap_400k_t2i_400k_flux_200k_midjourney_150k_recovery_150k_3e4_epoch2_no_rotated/000-GPT-XL/checkpoints/eval_step_26000/batch_0_cfg_7.5_topk_16384.jpg @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:fa15919e5fcec2b26c25905d2fe0d069a2839610361809621211ac6bfa99592b +size 756475 diff --git a/CKPTS/Afteri2i_100kckpt_LLAVAT5_empty_Fixed_just_segment_pretrain_stage3_dreambench_recap_400k_t2i_400k_flux_200k_midjourney_150k_recovery_150k_3e4_epoch2_no_rotated/000-GPT-XL/checkpoints/eval_step_26000/batch_1_cfg_7.5_topk_16384.jpg b/CKPTS/Afteri2i_100kckpt_LLAVAT5_empty_Fixed_just_segment_pretrain_stage3_dreambench_recap_400k_t2i_400k_flux_200k_midjourney_150k_recovery_150k_3e4_epoch2_no_rotated/000-GPT-XL/checkpoints/eval_step_26000/batch_1_cfg_7.5_topk_16384.jpg new file mode 100644 index 0000000000000000000000000000000000000000..dd2533d7fcac63c20729c67a23e14859296d7c36 --- /dev/null +++ b/CKPTS/Afteri2i_100kckpt_LLAVAT5_empty_Fixed_just_segment_pretrain_stage3_dreambench_recap_400k_t2i_400k_flux_200k_midjourney_150k_recovery_150k_3e4_epoch2_no_rotated/000-GPT-XL/checkpoints/eval_step_26000/batch_1_cfg_7.5_topk_16384.jpg @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:e03e9016a1934b7d9057b2c237bb659f1fcbc0040869e3ce3725719bc83f36b4 +size 693240 diff --git a/CKPTS/Afteri2i_100kckpt_LLAVAT5_empty_Fixed_just_segment_pretrain_stage3_dreambench_recap_400k_t2i_400k_flux_200k_midjourney_150k_recovery_150k_3e4_epoch2_no_rotated/000-GPT-XL/checkpoints/eval_step_26000/batch_2_cfg_7.5_topk_16384.jpg b/CKPTS/Afteri2i_100kckpt_LLAVAT5_empty_Fixed_just_segment_pretrain_stage3_dreambench_recap_400k_t2i_400k_flux_200k_midjourney_150k_recovery_150k_3e4_epoch2_no_rotated/000-GPT-XL/checkpoints/eval_step_26000/batch_2_cfg_7.5_topk_16384.jpg new file mode 100644 index 0000000000000000000000000000000000000000..9a81782c6649dc89c50d75d2dd0f094717e7e438 --- /dev/null +++ b/CKPTS/Afteri2i_100kckpt_LLAVAT5_empty_Fixed_just_segment_pretrain_stage3_dreambench_recap_400k_t2i_400k_flux_200k_midjourney_150k_recovery_150k_3e4_epoch2_no_rotated/000-GPT-XL/checkpoints/eval_step_26000/batch_2_cfg_7.5_topk_16384.jpg @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:61c1b37751ccfaae5c6efbc2cf5eec4fd95fb1b5d6de73d46e0098a59a64d687 +size 704914 diff --git a/CKPTS/Afteri2i_100kckpt_LLAVAT5_empty_Fixed_just_segment_pretrain_stage3_dreambench_recap_400k_t2i_400k_flux_200k_midjourney_150k_recovery_150k_3e4_epoch2_no_rotated/000-GPT-XL/checkpoints/eval_step_26000/batch_3_cfg_7.5_topk_16384.jpg b/CKPTS/Afteri2i_100kckpt_LLAVAT5_empty_Fixed_just_segment_pretrain_stage3_dreambench_recap_400k_t2i_400k_flux_200k_midjourney_150k_recovery_150k_3e4_epoch2_no_rotated/000-GPT-XL/checkpoints/eval_step_26000/batch_3_cfg_7.5_topk_16384.jpg new file mode 100644 index 0000000000000000000000000000000000000000..a9e283a19a1d617854c56d63f6dff187bfb1886d --- /dev/null +++ b/CKPTS/Afteri2i_100kckpt_LLAVAT5_empty_Fixed_just_segment_pretrain_stage3_dreambench_recap_400k_t2i_400k_flux_200k_midjourney_150k_recovery_150k_3e4_epoch2_no_rotated/000-GPT-XL/checkpoints/eval_step_26000/batch_3_cfg_7.5_topk_16384.jpg @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:712e6edd5830fd1268c79deff02612fbc3dde86ad390353aaf8c4082df2fb8f7 +size 854480 diff --git a/CKPTS/Afteri2i_100kckpt_LLAVAT5_empty_Fixed_just_segment_pretrain_stage3_dreambench_recap_400k_t2i_400k_flux_200k_midjourney_150k_recovery_150k_3e4_epoch2_no_rotated/000-GPT-XL/checkpoints/eval_step_28000/batch_0_cfg_7.5_topk_16384.jpg b/CKPTS/Afteri2i_100kckpt_LLAVAT5_empty_Fixed_just_segment_pretrain_stage3_dreambench_recap_400k_t2i_400k_flux_200k_midjourney_150k_recovery_150k_3e4_epoch2_no_rotated/000-GPT-XL/checkpoints/eval_step_28000/batch_0_cfg_7.5_topk_16384.jpg new file mode 100644 index 0000000000000000000000000000000000000000..a1ffe17271304f2d3366bf19144710e633210914 --- /dev/null +++ b/CKPTS/Afteri2i_100kckpt_LLAVAT5_empty_Fixed_just_segment_pretrain_stage3_dreambench_recap_400k_t2i_400k_flux_200k_midjourney_150k_recovery_150k_3e4_epoch2_no_rotated/000-GPT-XL/checkpoints/eval_step_28000/batch_0_cfg_7.5_topk_16384.jpg @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:b2303e0b2942caeb3b3abe5671d8bbd4071fe7f94afd7497e3ebe565aec07965 +size 728437 diff --git a/CKPTS/Afteri2i_100kckpt_LLAVAT5_empty_Fixed_just_segment_pretrain_stage3_dreambench_recap_400k_t2i_400k_flux_200k_midjourney_150k_recovery_150k_3e4_epoch2_no_rotated/000-GPT-XL/checkpoints/eval_step_28000/batch_1_cfg_7.5_topk_16384.jpg b/CKPTS/Afteri2i_100kckpt_LLAVAT5_empty_Fixed_just_segment_pretrain_stage3_dreambench_recap_400k_t2i_400k_flux_200k_midjourney_150k_recovery_150k_3e4_epoch2_no_rotated/000-GPT-XL/checkpoints/eval_step_28000/batch_1_cfg_7.5_topk_16384.jpg new file mode 100644 index 0000000000000000000000000000000000000000..3e9913336f0c00872f4d486af2aa9a5fe6e59d6c --- /dev/null +++ b/CKPTS/Afteri2i_100kckpt_LLAVAT5_empty_Fixed_just_segment_pretrain_stage3_dreambench_recap_400k_t2i_400k_flux_200k_midjourney_150k_recovery_150k_3e4_epoch2_no_rotated/000-GPT-XL/checkpoints/eval_step_28000/batch_1_cfg_7.5_topk_16384.jpg @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:7f67d0683814900e69ea7b169ec7e436bd2ea589f1e40292def3398677a77b3d +size 691827 diff --git a/CKPTS/Afteri2i_100kckpt_LLAVAT5_empty_Fixed_just_segment_pretrain_stage3_dreambench_recap_400k_t2i_400k_flux_200k_midjourney_150k_recovery_150k_3e4_epoch2_no_rotated/000-GPT-XL/checkpoints/eval_step_28000/batch_2_cfg_7.5_topk_16384.jpg b/CKPTS/Afteri2i_100kckpt_LLAVAT5_empty_Fixed_just_segment_pretrain_stage3_dreambench_recap_400k_t2i_400k_flux_200k_midjourney_150k_recovery_150k_3e4_epoch2_no_rotated/000-GPT-XL/checkpoints/eval_step_28000/batch_2_cfg_7.5_topk_16384.jpg new file mode 100644 index 0000000000000000000000000000000000000000..9a2e6f7cd5f8342b46080bdbee3e062c097c24df --- /dev/null +++ b/CKPTS/Afteri2i_100kckpt_LLAVAT5_empty_Fixed_just_segment_pretrain_stage3_dreambench_recap_400k_t2i_400k_flux_200k_midjourney_150k_recovery_150k_3e4_epoch2_no_rotated/000-GPT-XL/checkpoints/eval_step_28000/batch_2_cfg_7.5_topk_16384.jpg @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:1a8509bb159fd186cc5f87d37d5c64385ba12bdf0e196d40bdb8dade54ac6268 +size 732974 diff --git a/CKPTS/Afteri2i_100kckpt_LLAVAT5_empty_Fixed_just_segment_pretrain_stage3_dreambench_recap_400k_t2i_400k_flux_200k_midjourney_150k_recovery_150k_3e4_epoch2_no_rotated/000-GPT-XL/checkpoints/eval_step_28000/batch_3_cfg_7.5_topk_16384.jpg b/CKPTS/Afteri2i_100kckpt_LLAVAT5_empty_Fixed_just_segment_pretrain_stage3_dreambench_recap_400k_t2i_400k_flux_200k_midjourney_150k_recovery_150k_3e4_epoch2_no_rotated/000-GPT-XL/checkpoints/eval_step_28000/batch_3_cfg_7.5_topk_16384.jpg new file mode 100644 index 0000000000000000000000000000000000000000..a66c0f62f6fdd79174430ca03e0a16ab0ed02141 --- /dev/null +++ b/CKPTS/Afteri2i_100kckpt_LLAVAT5_empty_Fixed_just_segment_pretrain_stage3_dreambench_recap_400k_t2i_400k_flux_200k_midjourney_150k_recovery_150k_3e4_epoch2_no_rotated/000-GPT-XL/checkpoints/eval_step_28000/batch_3_cfg_7.5_topk_16384.jpg @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:aa114786f5cc95094f76f30588e5fa0b34e38c2487591384081f28c6c55009d5 +size 869117 diff --git a/CKPTS/Afteri2i_100kckpt_LLAVAT5_empty_Fixed_just_segment_pretrain_stage3_dreambench_recap_400k_t2i_400k_flux_200k_midjourney_150k_recovery_150k_3e4_epoch2_no_rotated/000-GPT-XL/checkpoints/eval_step_30000/batch_0_cfg_7.5_topk_16384.jpg b/CKPTS/Afteri2i_100kckpt_LLAVAT5_empty_Fixed_just_segment_pretrain_stage3_dreambench_recap_400k_t2i_400k_flux_200k_midjourney_150k_recovery_150k_3e4_epoch2_no_rotated/000-GPT-XL/checkpoints/eval_step_30000/batch_0_cfg_7.5_topk_16384.jpg new file mode 100644 index 0000000000000000000000000000000000000000..b4f0e24a7004a36599b3996df6384b7fea4e7617 --- /dev/null +++ b/CKPTS/Afteri2i_100kckpt_LLAVAT5_empty_Fixed_just_segment_pretrain_stage3_dreambench_recap_400k_t2i_400k_flux_200k_midjourney_150k_recovery_150k_3e4_epoch2_no_rotated/000-GPT-XL/checkpoints/eval_step_30000/batch_0_cfg_7.5_topk_16384.jpg @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:c855c0310ea5acb664fe539b7822d279c175305298096c39e7450a8e3fccab41 +size 748083 diff --git a/CKPTS/Afteri2i_100kckpt_LLAVAT5_empty_Fixed_just_segment_pretrain_stage3_dreambench_recap_400k_t2i_400k_flux_200k_midjourney_150k_recovery_150k_3e4_epoch2_no_rotated/000-GPT-XL/checkpoints/eval_step_30000/batch_1_cfg_7.5_topk_16384.jpg b/CKPTS/Afteri2i_100kckpt_LLAVAT5_empty_Fixed_just_segment_pretrain_stage3_dreambench_recap_400k_t2i_400k_flux_200k_midjourney_150k_recovery_150k_3e4_epoch2_no_rotated/000-GPT-XL/checkpoints/eval_step_30000/batch_1_cfg_7.5_topk_16384.jpg new file mode 100644 index 0000000000000000000000000000000000000000..dc1324e995cc449623a0549183ba651c0630b160 --- /dev/null +++ b/CKPTS/Afteri2i_100kckpt_LLAVAT5_empty_Fixed_just_segment_pretrain_stage3_dreambench_recap_400k_t2i_400k_flux_200k_midjourney_150k_recovery_150k_3e4_epoch2_no_rotated/000-GPT-XL/checkpoints/eval_step_30000/batch_1_cfg_7.5_topk_16384.jpg @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:1b72ed1d480ff22e6dd53a318a1825f17f0892890f8a1c4829c5f5eeffb573c6 +size 707152 diff --git a/CKPTS/Afteri2i_100kckpt_LLAVAT5_empty_Fixed_just_segment_pretrain_stage3_dreambench_recap_400k_t2i_400k_flux_200k_midjourney_150k_recovery_150k_3e4_epoch2_no_rotated/000-GPT-XL/checkpoints/eval_step_30000/batch_2_cfg_7.5_topk_16384.jpg b/CKPTS/Afteri2i_100kckpt_LLAVAT5_empty_Fixed_just_segment_pretrain_stage3_dreambench_recap_400k_t2i_400k_flux_200k_midjourney_150k_recovery_150k_3e4_epoch2_no_rotated/000-GPT-XL/checkpoints/eval_step_30000/batch_2_cfg_7.5_topk_16384.jpg new file mode 100644 index 0000000000000000000000000000000000000000..614155ccc30df3bbff58c91907a55865f30f096a --- /dev/null +++ b/CKPTS/Afteri2i_100kckpt_LLAVAT5_empty_Fixed_just_segment_pretrain_stage3_dreambench_recap_400k_t2i_400k_flux_200k_midjourney_150k_recovery_150k_3e4_epoch2_no_rotated/000-GPT-XL/checkpoints/eval_step_30000/batch_2_cfg_7.5_topk_16384.jpg @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:390b8eaef6e0f6ae36873a0762bdfd15d80e246dd079532a623890a8b00803ef +size 710380 diff --git a/CKPTS/Afteri2i_100kckpt_LLAVAT5_empty_Fixed_just_segment_pretrain_stage3_dreambench_recap_400k_t2i_400k_flux_200k_midjourney_150k_recovery_150k_3e4_epoch2_no_rotated/000-GPT-XL/checkpoints/eval_step_30000/batch_3_cfg_7.5_topk_16384.jpg b/CKPTS/Afteri2i_100kckpt_LLAVAT5_empty_Fixed_just_segment_pretrain_stage3_dreambench_recap_400k_t2i_400k_flux_200k_midjourney_150k_recovery_150k_3e4_epoch2_no_rotated/000-GPT-XL/checkpoints/eval_step_30000/batch_3_cfg_7.5_topk_16384.jpg new file mode 100644 index 0000000000000000000000000000000000000000..a11f727a2d74ef6ba34333b0f14dee53c0b64526 --- /dev/null +++ b/CKPTS/Afteri2i_100kckpt_LLAVAT5_empty_Fixed_just_segment_pretrain_stage3_dreambench_recap_400k_t2i_400k_flux_200k_midjourney_150k_recovery_150k_3e4_epoch2_no_rotated/000-GPT-XL/checkpoints/eval_step_30000/batch_3_cfg_7.5_topk_16384.jpg @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:c9138717ee66394fecf616852873378d1412c580a5332c6c503c49687b12fb4d +size 859173 diff --git a/CKPTS/Afteri2i_100kckpt_LLAVAT5_empty_Fixed_just_segment_pretrain_stage3_dreambench_recap_400k_t2i_400k_flux_200k_midjourney_150k_recovery_150k_3e4_epoch2_no_rotated/000-GPT-XL/checkpoints/eval_step_32000/batch_0_cfg_7.5_topk_16384.jpg b/CKPTS/Afteri2i_100kckpt_LLAVAT5_empty_Fixed_just_segment_pretrain_stage3_dreambench_recap_400k_t2i_400k_flux_200k_midjourney_150k_recovery_150k_3e4_epoch2_no_rotated/000-GPT-XL/checkpoints/eval_step_32000/batch_0_cfg_7.5_topk_16384.jpg new file mode 100644 index 0000000000000000000000000000000000000000..127614b9ae78780f5f24d1173e632888b66bbc1d --- /dev/null +++ b/CKPTS/Afteri2i_100kckpt_LLAVAT5_empty_Fixed_just_segment_pretrain_stage3_dreambench_recap_400k_t2i_400k_flux_200k_midjourney_150k_recovery_150k_3e4_epoch2_no_rotated/000-GPT-XL/checkpoints/eval_step_32000/batch_0_cfg_7.5_topk_16384.jpg @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:fd1dcc696a51aa4016aa10a1d7852fb845370e5d9750e4fdcc990af6eceb1a6b +size 708935 diff --git a/CKPTS/Afteri2i_100kckpt_LLAVAT5_empty_Fixed_just_segment_pretrain_stage3_dreambench_recap_400k_t2i_400k_flux_200k_midjourney_150k_recovery_150k_3e4_epoch2_no_rotated/000-GPT-XL/checkpoints/eval_step_32000/batch_1_cfg_7.5_topk_16384.jpg b/CKPTS/Afteri2i_100kckpt_LLAVAT5_empty_Fixed_just_segment_pretrain_stage3_dreambench_recap_400k_t2i_400k_flux_200k_midjourney_150k_recovery_150k_3e4_epoch2_no_rotated/000-GPT-XL/checkpoints/eval_step_32000/batch_1_cfg_7.5_topk_16384.jpg new file mode 100644 index 0000000000000000000000000000000000000000..a52c09b51ca04ef0d871ce30ab4044d44e641522 --- /dev/null +++ b/CKPTS/Afteri2i_100kckpt_LLAVAT5_empty_Fixed_just_segment_pretrain_stage3_dreambench_recap_400k_t2i_400k_flux_200k_midjourney_150k_recovery_150k_3e4_epoch2_no_rotated/000-GPT-XL/checkpoints/eval_step_32000/batch_1_cfg_7.5_topk_16384.jpg @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:de3989b3c4ef15155155331f4a3e4cff490ee49f6407c3af6b89898f23559b1a +size 705367 diff --git a/CKPTS/Afteri2i_100kckpt_LLAVAT5_empty_Fixed_just_segment_pretrain_stage3_dreambench_recap_400k_t2i_400k_flux_200k_midjourney_150k_recovery_150k_3e4_epoch2_no_rotated/000-GPT-XL/checkpoints/eval_step_32000/batch_2_cfg_7.5_topk_16384.jpg b/CKPTS/Afteri2i_100kckpt_LLAVAT5_empty_Fixed_just_segment_pretrain_stage3_dreambench_recap_400k_t2i_400k_flux_200k_midjourney_150k_recovery_150k_3e4_epoch2_no_rotated/000-GPT-XL/checkpoints/eval_step_32000/batch_2_cfg_7.5_topk_16384.jpg new file mode 100644 index 0000000000000000000000000000000000000000..3983930550df436da17a0e77c46f212dc21a6cd9 --- /dev/null +++ b/CKPTS/Afteri2i_100kckpt_LLAVAT5_empty_Fixed_just_segment_pretrain_stage3_dreambench_recap_400k_t2i_400k_flux_200k_midjourney_150k_recovery_150k_3e4_epoch2_no_rotated/000-GPT-XL/checkpoints/eval_step_32000/batch_2_cfg_7.5_topk_16384.jpg @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:19b8d70bae59c982b460e7506c55edb89725150edf14311832fedfee063f2ff7 +size 716449 diff --git a/CKPTS/Afteri2i_100kckpt_LLAVAT5_empty_Fixed_just_segment_pretrain_stage3_dreambench_recap_400k_t2i_400k_flux_200k_midjourney_150k_recovery_150k_3e4_epoch2_no_rotated/000-GPT-XL/checkpoints/eval_step_32000/batch_3_cfg_7.5_topk_16384.jpg b/CKPTS/Afteri2i_100kckpt_LLAVAT5_empty_Fixed_just_segment_pretrain_stage3_dreambench_recap_400k_t2i_400k_flux_200k_midjourney_150k_recovery_150k_3e4_epoch2_no_rotated/000-GPT-XL/checkpoints/eval_step_32000/batch_3_cfg_7.5_topk_16384.jpg new file mode 100644 index 0000000000000000000000000000000000000000..0f4703b24db787cf671cd6e9901da79ca6d2f5f3 --- /dev/null +++ b/CKPTS/Afteri2i_100kckpt_LLAVAT5_empty_Fixed_just_segment_pretrain_stage3_dreambench_recap_400k_t2i_400k_flux_200k_midjourney_150k_recovery_150k_3e4_epoch2_no_rotated/000-GPT-XL/checkpoints/eval_step_32000/batch_3_cfg_7.5_topk_16384.jpg @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:62f7f30c5335ed602157d4443bc1debdb326f2ba32f75f4427e43b641d074bf1 +size 857634 diff --git a/CKPTS/Afteri2i_100kckpt_LLAVAT5_empty_Fixed_just_segment_pretrain_stage3_dreambench_recap_400k_t2i_400k_flux_200k_midjourney_150k_recovery_150k_3e4_epoch2_no_rotated/000-GPT-XL/checkpoints/eval_step_34000/batch_0_cfg_7.5_topk_16384.jpg b/CKPTS/Afteri2i_100kckpt_LLAVAT5_empty_Fixed_just_segment_pretrain_stage3_dreambench_recap_400k_t2i_400k_flux_200k_midjourney_150k_recovery_150k_3e4_epoch2_no_rotated/000-GPT-XL/checkpoints/eval_step_34000/batch_0_cfg_7.5_topk_16384.jpg new file mode 100644 index 0000000000000000000000000000000000000000..61b59b3f21bc10c32544820b6e91fc06329780e2 --- /dev/null +++ b/CKPTS/Afteri2i_100kckpt_LLAVAT5_empty_Fixed_just_segment_pretrain_stage3_dreambench_recap_400k_t2i_400k_flux_200k_midjourney_150k_recovery_150k_3e4_epoch2_no_rotated/000-GPT-XL/checkpoints/eval_step_34000/batch_0_cfg_7.5_topk_16384.jpg @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:7bd31da4cbec4868cee7f9ce8292a1cf0b4890eecfbc074461dcc16946344f9b +size 735646 diff --git a/CKPTS/Afteri2i_100kckpt_LLAVAT5_empty_Fixed_just_segment_pretrain_stage3_dreambench_recap_400k_t2i_400k_flux_200k_midjourney_150k_recovery_150k_3e4_epoch2_no_rotated/000-GPT-XL/checkpoints/eval_step_34000/batch_1_cfg_7.5_topk_16384.jpg b/CKPTS/Afteri2i_100kckpt_LLAVAT5_empty_Fixed_just_segment_pretrain_stage3_dreambench_recap_400k_t2i_400k_flux_200k_midjourney_150k_recovery_150k_3e4_epoch2_no_rotated/000-GPT-XL/checkpoints/eval_step_34000/batch_1_cfg_7.5_topk_16384.jpg new file mode 100644 index 0000000000000000000000000000000000000000..8efa1028cdeeb100ba8f2cd59808cbf714c25510 --- /dev/null +++ b/CKPTS/Afteri2i_100kckpt_LLAVAT5_empty_Fixed_just_segment_pretrain_stage3_dreambench_recap_400k_t2i_400k_flux_200k_midjourney_150k_recovery_150k_3e4_epoch2_no_rotated/000-GPT-XL/checkpoints/eval_step_34000/batch_1_cfg_7.5_topk_16384.jpg @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:a1a0e224c4610cd93925f268c98408398e830d7166727dd7c7b1917b0f01904a +size 725664 diff --git a/CKPTS/Afteri2i_100kckpt_LLAVAT5_empty_Fixed_just_segment_pretrain_stage3_dreambench_recap_400k_t2i_400k_flux_200k_midjourney_150k_recovery_150k_3e4_epoch2_no_rotated/000-GPT-XL/checkpoints/eval_step_34000/batch_2_cfg_7.5_topk_16384.jpg b/CKPTS/Afteri2i_100kckpt_LLAVAT5_empty_Fixed_just_segment_pretrain_stage3_dreambench_recap_400k_t2i_400k_flux_200k_midjourney_150k_recovery_150k_3e4_epoch2_no_rotated/000-GPT-XL/checkpoints/eval_step_34000/batch_2_cfg_7.5_topk_16384.jpg new file mode 100644 index 0000000000000000000000000000000000000000..dc0e8da4815ab87c7e87b4d87f9f017bddf9f71a --- /dev/null +++ b/CKPTS/Afteri2i_100kckpt_LLAVAT5_empty_Fixed_just_segment_pretrain_stage3_dreambench_recap_400k_t2i_400k_flux_200k_midjourney_150k_recovery_150k_3e4_epoch2_no_rotated/000-GPT-XL/checkpoints/eval_step_34000/batch_2_cfg_7.5_topk_16384.jpg @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:319b30d9440fa0e301298ba389e0b120edd14805678e3445563ed49c019357dc +size 698141 diff --git a/CKPTS/Afteri2i_100kckpt_LLAVAT5_empty_Fixed_just_segment_pretrain_stage3_dreambench_recap_400k_t2i_400k_flux_200k_midjourney_150k_recovery_150k_3e4_epoch2_no_rotated/000-GPT-XL/checkpoints/eval_step_34000/batch_3_cfg_7.5_topk_16384.jpg b/CKPTS/Afteri2i_100kckpt_LLAVAT5_empty_Fixed_just_segment_pretrain_stage3_dreambench_recap_400k_t2i_400k_flux_200k_midjourney_150k_recovery_150k_3e4_epoch2_no_rotated/000-GPT-XL/checkpoints/eval_step_34000/batch_3_cfg_7.5_topk_16384.jpg new file mode 100644 index 0000000000000000000000000000000000000000..ad4e3e14219c8756a068737c0025cd09eeba0c94 --- /dev/null +++ b/CKPTS/Afteri2i_100kckpt_LLAVAT5_empty_Fixed_just_segment_pretrain_stage3_dreambench_recap_400k_t2i_400k_flux_200k_midjourney_150k_recovery_150k_3e4_epoch2_no_rotated/000-GPT-XL/checkpoints/eval_step_34000/batch_3_cfg_7.5_topk_16384.jpg @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:656e5bfdeb1f3d4ac20e30de29801665188a31bb78110e217c0f9ed9a44f135f +size 851569 diff --git a/CKPTS/Afteri2i_100kckpt_LLAVAT5_empty_Fixed_just_segment_pretrain_stage3_dreambench_recap_400k_t2i_400k_flux_200k_midjourney_150k_recovery_150k_3e4_epoch2_no_rotated/000-GPT-XL/checkpoints/eval_step_36000/batch_0_cfg_7.5_topk_16384.jpg b/CKPTS/Afteri2i_100kckpt_LLAVAT5_empty_Fixed_just_segment_pretrain_stage3_dreambench_recap_400k_t2i_400k_flux_200k_midjourney_150k_recovery_150k_3e4_epoch2_no_rotated/000-GPT-XL/checkpoints/eval_step_36000/batch_0_cfg_7.5_topk_16384.jpg new file mode 100644 index 0000000000000000000000000000000000000000..b4f0e947ac3095dd27c7cb99eda4fec439b3ba70 --- /dev/null +++ b/CKPTS/Afteri2i_100kckpt_LLAVAT5_empty_Fixed_just_segment_pretrain_stage3_dreambench_recap_400k_t2i_400k_flux_200k_midjourney_150k_recovery_150k_3e4_epoch2_no_rotated/000-GPT-XL/checkpoints/eval_step_36000/batch_0_cfg_7.5_topk_16384.jpg @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:78a9b6890fc230bcfde3e3cf7ba16ef7f05deff84b14f3d8c3b7f4bf60bbe8c9 +size 716678 diff --git a/CKPTS/Afteri2i_100kckpt_LLAVAT5_empty_Fixed_just_segment_pretrain_stage3_dreambench_recap_400k_t2i_400k_flux_200k_midjourney_150k_recovery_150k_3e4_epoch2_no_rotated/000-GPT-XL/checkpoints/eval_step_36000/batch_1_cfg_7.5_topk_16384.jpg b/CKPTS/Afteri2i_100kckpt_LLAVAT5_empty_Fixed_just_segment_pretrain_stage3_dreambench_recap_400k_t2i_400k_flux_200k_midjourney_150k_recovery_150k_3e4_epoch2_no_rotated/000-GPT-XL/checkpoints/eval_step_36000/batch_1_cfg_7.5_topk_16384.jpg new file mode 100644 index 0000000000000000000000000000000000000000..3b8259a1e3a10c82e11e24d818bd9f2a08adce0d --- /dev/null +++ b/CKPTS/Afteri2i_100kckpt_LLAVAT5_empty_Fixed_just_segment_pretrain_stage3_dreambench_recap_400k_t2i_400k_flux_200k_midjourney_150k_recovery_150k_3e4_epoch2_no_rotated/000-GPT-XL/checkpoints/eval_step_36000/batch_1_cfg_7.5_topk_16384.jpg @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:0c9ed26abbea92bfa37d97cf91087b10d4b160e7be922ec1e2440c848b14723c +size 704926 diff --git a/CKPTS/Afteri2i_100kckpt_LLAVAT5_empty_Fixed_just_segment_pretrain_stage3_dreambench_recap_400k_t2i_400k_flux_200k_midjourney_150k_recovery_150k_3e4_epoch2_no_rotated/000-GPT-XL/checkpoints/eval_step_36000/batch_2_cfg_7.5_topk_16384.jpg b/CKPTS/Afteri2i_100kckpt_LLAVAT5_empty_Fixed_just_segment_pretrain_stage3_dreambench_recap_400k_t2i_400k_flux_200k_midjourney_150k_recovery_150k_3e4_epoch2_no_rotated/000-GPT-XL/checkpoints/eval_step_36000/batch_2_cfg_7.5_topk_16384.jpg new file mode 100644 index 0000000000000000000000000000000000000000..3aa0fef04011deb56d69faf43d688e66245737f7 --- /dev/null +++ b/CKPTS/Afteri2i_100kckpt_LLAVAT5_empty_Fixed_just_segment_pretrain_stage3_dreambench_recap_400k_t2i_400k_flux_200k_midjourney_150k_recovery_150k_3e4_epoch2_no_rotated/000-GPT-XL/checkpoints/eval_step_36000/batch_2_cfg_7.5_topk_16384.jpg @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:5ae52478a47e82fa50aac8a4ec77335e2313a777a984c8864af8490c1373c9f9 +size 725739 diff --git a/CKPTS/Afteri2i_100kckpt_LLAVAT5_empty_Fixed_just_segment_pretrain_stage3_dreambench_recap_400k_t2i_400k_flux_200k_midjourney_150k_recovery_150k_3e4_epoch2_no_rotated/000-GPT-XL/checkpoints/eval_step_36000/batch_3_cfg_7.5_topk_16384.jpg b/CKPTS/Afteri2i_100kckpt_LLAVAT5_empty_Fixed_just_segment_pretrain_stage3_dreambench_recap_400k_t2i_400k_flux_200k_midjourney_150k_recovery_150k_3e4_epoch2_no_rotated/000-GPT-XL/checkpoints/eval_step_36000/batch_3_cfg_7.5_topk_16384.jpg new file mode 100644 index 0000000000000000000000000000000000000000..a1ef7bc64e4e006480594a8448cecc3c4e672d09 --- /dev/null +++ b/CKPTS/Afteri2i_100kckpt_LLAVAT5_empty_Fixed_just_segment_pretrain_stage3_dreambench_recap_400k_t2i_400k_flux_200k_midjourney_150k_recovery_150k_3e4_epoch2_no_rotated/000-GPT-XL/checkpoints/eval_step_36000/batch_3_cfg_7.5_topk_16384.jpg @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:5d443b9d068d115d926d5149a09e963dac4d344dc7462b686d39c7cb2449e0f0 +size 866692 diff --git a/CKPTS/Afteri2i_100kckpt_LLAVAT5_empty_Fixed_just_segment_pretrain_stage3_dreambench_recap_400k_t2i_400k_flux_200k_midjourney_150k_recovery_150k_3e4_epoch2_no_rotated/000-GPT-XL/checkpoints/eval_step_38000/batch_0_cfg_7.5_topk_16384.jpg b/CKPTS/Afteri2i_100kckpt_LLAVAT5_empty_Fixed_just_segment_pretrain_stage3_dreambench_recap_400k_t2i_400k_flux_200k_midjourney_150k_recovery_150k_3e4_epoch2_no_rotated/000-GPT-XL/checkpoints/eval_step_38000/batch_0_cfg_7.5_topk_16384.jpg new file mode 100644 index 0000000000000000000000000000000000000000..71c480306d0f0453c0425855181c2f224dcea40a --- /dev/null +++ b/CKPTS/Afteri2i_100kckpt_LLAVAT5_empty_Fixed_just_segment_pretrain_stage3_dreambench_recap_400k_t2i_400k_flux_200k_midjourney_150k_recovery_150k_3e4_epoch2_no_rotated/000-GPT-XL/checkpoints/eval_step_38000/batch_0_cfg_7.5_topk_16384.jpg @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:3f0d968994ec584a501977ab290112171896a0434063fbfaef4f29b035b9a964 +size 718443 diff --git a/CKPTS/Afteri2i_100kckpt_LLAVAT5_empty_Fixed_just_segment_pretrain_stage3_dreambench_recap_400k_t2i_400k_flux_200k_midjourney_150k_recovery_150k_3e4_epoch2_no_rotated/000-GPT-XL/checkpoints/eval_step_38000/batch_1_cfg_7.5_topk_16384.jpg b/CKPTS/Afteri2i_100kckpt_LLAVAT5_empty_Fixed_just_segment_pretrain_stage3_dreambench_recap_400k_t2i_400k_flux_200k_midjourney_150k_recovery_150k_3e4_epoch2_no_rotated/000-GPT-XL/checkpoints/eval_step_38000/batch_1_cfg_7.5_topk_16384.jpg new file mode 100644 index 0000000000000000000000000000000000000000..e205c1406e10e86dc8d10d81816ad3df6794427f --- /dev/null +++ b/CKPTS/Afteri2i_100kckpt_LLAVAT5_empty_Fixed_just_segment_pretrain_stage3_dreambench_recap_400k_t2i_400k_flux_200k_midjourney_150k_recovery_150k_3e4_epoch2_no_rotated/000-GPT-XL/checkpoints/eval_step_38000/batch_1_cfg_7.5_topk_16384.jpg @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:6b0f4a03e82e34434584218e58ba3803b773424fca0fdf9de85d4b57c2df30bd +size 719673 diff --git a/CKPTS/Afteri2i_100kckpt_LLAVAT5_empty_Fixed_just_segment_pretrain_stage3_dreambench_recap_400k_t2i_400k_flux_200k_midjourney_150k_recovery_150k_3e4_epoch2_no_rotated/000-GPT-XL/checkpoints/eval_step_38000/batch_2_cfg_7.5_topk_16384.jpg b/CKPTS/Afteri2i_100kckpt_LLAVAT5_empty_Fixed_just_segment_pretrain_stage3_dreambench_recap_400k_t2i_400k_flux_200k_midjourney_150k_recovery_150k_3e4_epoch2_no_rotated/000-GPT-XL/checkpoints/eval_step_38000/batch_2_cfg_7.5_topk_16384.jpg new file mode 100644 index 0000000000000000000000000000000000000000..604d3d6c197005d0325b71ef878f43180c54a72e --- /dev/null +++ b/CKPTS/Afteri2i_100kckpt_LLAVAT5_empty_Fixed_just_segment_pretrain_stage3_dreambench_recap_400k_t2i_400k_flux_200k_midjourney_150k_recovery_150k_3e4_epoch2_no_rotated/000-GPT-XL/checkpoints/eval_step_38000/batch_2_cfg_7.5_topk_16384.jpg @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:7347be2ca4eac87d0580e49a1dae8c864193a8e827379ea1d94246c2702a5a26 +size 723455 diff --git a/CKPTS/Afteri2i_100kckpt_LLAVAT5_empty_Fixed_just_segment_pretrain_stage3_dreambench_recap_400k_t2i_400k_flux_200k_midjourney_150k_recovery_150k_3e4_epoch2_no_rotated/000-GPT-XL/checkpoints/eval_step_38000/batch_3_cfg_7.5_topk_16384.jpg b/CKPTS/Afteri2i_100kckpt_LLAVAT5_empty_Fixed_just_segment_pretrain_stage3_dreambench_recap_400k_t2i_400k_flux_200k_midjourney_150k_recovery_150k_3e4_epoch2_no_rotated/000-GPT-XL/checkpoints/eval_step_38000/batch_3_cfg_7.5_topk_16384.jpg new file mode 100644 index 0000000000000000000000000000000000000000..9f41a47b22ebda6a0d6b8b677b1b349cca07660e --- /dev/null +++ b/CKPTS/Afteri2i_100kckpt_LLAVAT5_empty_Fixed_just_segment_pretrain_stage3_dreambench_recap_400k_t2i_400k_flux_200k_midjourney_150k_recovery_150k_3e4_epoch2_no_rotated/000-GPT-XL/checkpoints/eval_step_38000/batch_3_cfg_7.5_topk_16384.jpg @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:23bf55874cdaa1359782cce406efd0a44b946b0377227335a4120c1796b0dfe6 +size 863613 diff --git a/CKPTS/Afteri2i_100kckpt_LLAVAT5_empty_Fixed_just_segment_pretrain_stage3_dreambench_recap_400k_t2i_400k_flux_200k_midjourney_150k_recovery_150k_3e4_epoch2_no_rotated/000-GPT-XL/checkpoints/eval_step_4000/batch_0_cfg_7.5_topk_16384.jpg b/CKPTS/Afteri2i_100kckpt_LLAVAT5_empty_Fixed_just_segment_pretrain_stage3_dreambench_recap_400k_t2i_400k_flux_200k_midjourney_150k_recovery_150k_3e4_epoch2_no_rotated/000-GPT-XL/checkpoints/eval_step_4000/batch_0_cfg_7.5_topk_16384.jpg new file mode 100644 index 0000000000000000000000000000000000000000..8a226e78b1a2e972e61573130d4dec3f293df784 --- /dev/null +++ b/CKPTS/Afteri2i_100kckpt_LLAVAT5_empty_Fixed_just_segment_pretrain_stage3_dreambench_recap_400k_t2i_400k_flux_200k_midjourney_150k_recovery_150k_3e4_epoch2_no_rotated/000-GPT-XL/checkpoints/eval_step_4000/batch_0_cfg_7.5_topk_16384.jpg @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:18865169c58b9ab478839722a4a82b842f9496f9e0428aecd61cc87b67939e75 +size 729452 diff --git a/CKPTS/Afteri2i_100kckpt_LLAVAT5_empty_Fixed_just_segment_pretrain_stage3_dreambench_recap_400k_t2i_400k_flux_200k_midjourney_150k_recovery_150k_3e4_epoch2_no_rotated/000-GPT-XL/checkpoints/eval_step_4000/batch_1_cfg_7.5_topk_16384.jpg b/CKPTS/Afteri2i_100kckpt_LLAVAT5_empty_Fixed_just_segment_pretrain_stage3_dreambench_recap_400k_t2i_400k_flux_200k_midjourney_150k_recovery_150k_3e4_epoch2_no_rotated/000-GPT-XL/checkpoints/eval_step_4000/batch_1_cfg_7.5_topk_16384.jpg new file mode 100644 index 0000000000000000000000000000000000000000..d33cd2a93aee310463a18f83ae3ac689381dd139 --- /dev/null +++ b/CKPTS/Afteri2i_100kckpt_LLAVAT5_empty_Fixed_just_segment_pretrain_stage3_dreambench_recap_400k_t2i_400k_flux_200k_midjourney_150k_recovery_150k_3e4_epoch2_no_rotated/000-GPT-XL/checkpoints/eval_step_4000/batch_1_cfg_7.5_topk_16384.jpg @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:779fa6647cf7b2bc2f24942d143314f8ae3e34dcfd51912a20becf4949e3b183 +size 684071 diff --git a/CKPTS/Afteri2i_100kckpt_LLAVAT5_empty_Fixed_just_segment_pretrain_stage3_dreambench_recap_400k_t2i_400k_flux_200k_midjourney_150k_recovery_150k_3e4_epoch2_no_rotated/000-GPT-XL/checkpoints/eval_step_4000/batch_2_cfg_7.5_topk_16384.jpg b/CKPTS/Afteri2i_100kckpt_LLAVAT5_empty_Fixed_just_segment_pretrain_stage3_dreambench_recap_400k_t2i_400k_flux_200k_midjourney_150k_recovery_150k_3e4_epoch2_no_rotated/000-GPT-XL/checkpoints/eval_step_4000/batch_2_cfg_7.5_topk_16384.jpg new file mode 100644 index 0000000000000000000000000000000000000000..927d0eb5fa9f01da61a4edcde495f6534c7f7d56 --- /dev/null +++ b/CKPTS/Afteri2i_100kckpt_LLAVAT5_empty_Fixed_just_segment_pretrain_stage3_dreambench_recap_400k_t2i_400k_flux_200k_midjourney_150k_recovery_150k_3e4_epoch2_no_rotated/000-GPT-XL/checkpoints/eval_step_4000/batch_2_cfg_7.5_topk_16384.jpg @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:e03e324b873fad5441876b415cfde09f23112a067bb4d62bd17a44a6bf560bfc +size 675488 diff --git a/CKPTS/Afteri2i_100kckpt_LLAVAT5_empty_Fixed_just_segment_pretrain_stage3_dreambench_recap_400k_t2i_400k_flux_200k_midjourney_150k_recovery_150k_3e4_epoch2_no_rotated/000-GPT-XL/checkpoints/eval_step_4000/batch_3_cfg_7.5_topk_16384.jpg b/CKPTS/Afteri2i_100kckpt_LLAVAT5_empty_Fixed_just_segment_pretrain_stage3_dreambench_recap_400k_t2i_400k_flux_200k_midjourney_150k_recovery_150k_3e4_epoch2_no_rotated/000-GPT-XL/checkpoints/eval_step_4000/batch_3_cfg_7.5_topk_16384.jpg new file mode 100644 index 0000000000000000000000000000000000000000..ea85410ac2854c4c0e120829b0f034b060bde745 --- /dev/null +++ b/CKPTS/Afteri2i_100kckpt_LLAVAT5_empty_Fixed_just_segment_pretrain_stage3_dreambench_recap_400k_t2i_400k_flux_200k_midjourney_150k_recovery_150k_3e4_epoch2_no_rotated/000-GPT-XL/checkpoints/eval_step_4000/batch_3_cfg_7.5_topk_16384.jpg @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:20fea677116974641df713b3fe233d510cb11fb31c6b73f0f281317a6115f2e5 +size 812744 diff --git a/CKPTS/Afteri2i_100kckpt_LLAVAT5_empty_Fixed_just_segment_pretrain_stage3_dreambench_recap_400k_t2i_400k_flux_200k_midjourney_150k_recovery_150k_3e4_epoch2_no_rotated/000-GPT-XL/checkpoints/eval_step_40000/batch_0_cfg_7.5_topk_16384.jpg b/CKPTS/Afteri2i_100kckpt_LLAVAT5_empty_Fixed_just_segment_pretrain_stage3_dreambench_recap_400k_t2i_400k_flux_200k_midjourney_150k_recovery_150k_3e4_epoch2_no_rotated/000-GPT-XL/checkpoints/eval_step_40000/batch_0_cfg_7.5_topk_16384.jpg new file mode 100644 index 0000000000000000000000000000000000000000..3f31b9336dce59a9425594d535eb7e97eca14bef --- /dev/null +++ b/CKPTS/Afteri2i_100kckpt_LLAVAT5_empty_Fixed_just_segment_pretrain_stage3_dreambench_recap_400k_t2i_400k_flux_200k_midjourney_150k_recovery_150k_3e4_epoch2_no_rotated/000-GPT-XL/checkpoints/eval_step_40000/batch_0_cfg_7.5_topk_16384.jpg @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:c111f1342ab1d647062e009f7d11b42e134d562d47af2130374d1639c35fa151 +size 750707 diff --git a/CKPTS/Afteri2i_100kckpt_LLAVAT5_empty_Fixed_just_segment_pretrain_stage3_dreambench_recap_400k_t2i_400k_flux_200k_midjourney_150k_recovery_150k_3e4_epoch2_no_rotated/000-GPT-XL/checkpoints/eval_step_40000/batch_1_cfg_7.5_topk_16384.jpg b/CKPTS/Afteri2i_100kckpt_LLAVAT5_empty_Fixed_just_segment_pretrain_stage3_dreambench_recap_400k_t2i_400k_flux_200k_midjourney_150k_recovery_150k_3e4_epoch2_no_rotated/000-GPT-XL/checkpoints/eval_step_40000/batch_1_cfg_7.5_topk_16384.jpg new file mode 100644 index 0000000000000000000000000000000000000000..6184aff87a8c4f13952d084117e714e2b28140c5 --- /dev/null +++ b/CKPTS/Afteri2i_100kckpt_LLAVAT5_empty_Fixed_just_segment_pretrain_stage3_dreambench_recap_400k_t2i_400k_flux_200k_midjourney_150k_recovery_150k_3e4_epoch2_no_rotated/000-GPT-XL/checkpoints/eval_step_40000/batch_1_cfg_7.5_topk_16384.jpg @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:1d481c4c75f311391a2b925a8e3969e4503712908b8ba49ed7426f80888635e9 +size 704917 diff --git a/CKPTS/Afteri2i_100kckpt_LLAVAT5_empty_Fixed_just_segment_pretrain_stage3_dreambench_recap_400k_t2i_400k_flux_200k_midjourney_150k_recovery_150k_3e4_epoch2_no_rotated/000-GPT-XL/checkpoints/eval_step_40000/batch_2_cfg_7.5_topk_16384.jpg b/CKPTS/Afteri2i_100kckpt_LLAVAT5_empty_Fixed_just_segment_pretrain_stage3_dreambench_recap_400k_t2i_400k_flux_200k_midjourney_150k_recovery_150k_3e4_epoch2_no_rotated/000-GPT-XL/checkpoints/eval_step_40000/batch_2_cfg_7.5_topk_16384.jpg new file mode 100644 index 0000000000000000000000000000000000000000..9de41fac096ac88e25f79a8afe6ddf686ef90f13 --- /dev/null +++ b/CKPTS/Afteri2i_100kckpt_LLAVAT5_empty_Fixed_just_segment_pretrain_stage3_dreambench_recap_400k_t2i_400k_flux_200k_midjourney_150k_recovery_150k_3e4_epoch2_no_rotated/000-GPT-XL/checkpoints/eval_step_40000/batch_2_cfg_7.5_topk_16384.jpg @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:9cdb25babb2a8f2e353bd5f2fa8ef3661a545b8a556055d8923e559a4320a95b +size 735456 diff --git a/CKPTS/Afteri2i_100kckpt_LLAVAT5_empty_Fixed_just_segment_pretrain_stage3_dreambench_recap_400k_t2i_400k_flux_200k_midjourney_150k_recovery_150k_3e4_epoch2_no_rotated/000-GPT-XL/checkpoints/eval_step_40000/batch_3_cfg_7.5_topk_16384.jpg b/CKPTS/Afteri2i_100kckpt_LLAVAT5_empty_Fixed_just_segment_pretrain_stage3_dreambench_recap_400k_t2i_400k_flux_200k_midjourney_150k_recovery_150k_3e4_epoch2_no_rotated/000-GPT-XL/checkpoints/eval_step_40000/batch_3_cfg_7.5_topk_16384.jpg new file mode 100644 index 0000000000000000000000000000000000000000..a3bc103bc8cb21e81f4263a2fdc186b8c0c673bb --- /dev/null +++ b/CKPTS/Afteri2i_100kckpt_LLAVAT5_empty_Fixed_just_segment_pretrain_stage3_dreambench_recap_400k_t2i_400k_flux_200k_midjourney_150k_recovery_150k_3e4_epoch2_no_rotated/000-GPT-XL/checkpoints/eval_step_40000/batch_3_cfg_7.5_topk_16384.jpg @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:f19271be026074c13eb725467ad4482274708b393d6d6c72317fdb535ef805a8 +size 844586 diff --git a/CKPTS/Afteri2i_100kckpt_LLAVAT5_empty_Fixed_just_segment_pretrain_stage3_dreambench_recap_400k_t2i_400k_flux_200k_midjourney_150k_recovery_150k_3e4_epoch2_no_rotated/000-GPT-XL/checkpoints/eval_step_42000/batch_0_cfg_7.5_topk_16384.jpg b/CKPTS/Afteri2i_100kckpt_LLAVAT5_empty_Fixed_just_segment_pretrain_stage3_dreambench_recap_400k_t2i_400k_flux_200k_midjourney_150k_recovery_150k_3e4_epoch2_no_rotated/000-GPT-XL/checkpoints/eval_step_42000/batch_0_cfg_7.5_topk_16384.jpg new file mode 100644 index 0000000000000000000000000000000000000000..bac81dde2d23332d9a2d512ca24bbf2cfc71efec --- /dev/null +++ b/CKPTS/Afteri2i_100kckpt_LLAVAT5_empty_Fixed_just_segment_pretrain_stage3_dreambench_recap_400k_t2i_400k_flux_200k_midjourney_150k_recovery_150k_3e4_epoch2_no_rotated/000-GPT-XL/checkpoints/eval_step_42000/batch_0_cfg_7.5_topk_16384.jpg @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:7b44eb59b7f6c948503d32d0824cc5e8fa17899b3861577dee57267817f61e4c +size 749311 diff --git a/CKPTS/Afteri2i_100kckpt_LLAVAT5_empty_Fixed_just_segment_pretrain_stage3_dreambench_recap_400k_t2i_400k_flux_200k_midjourney_150k_recovery_150k_3e4_epoch2_no_rotated/000-GPT-XL/checkpoints/eval_step_42000/batch_1_cfg_7.5_topk_16384.jpg b/CKPTS/Afteri2i_100kckpt_LLAVAT5_empty_Fixed_just_segment_pretrain_stage3_dreambench_recap_400k_t2i_400k_flux_200k_midjourney_150k_recovery_150k_3e4_epoch2_no_rotated/000-GPT-XL/checkpoints/eval_step_42000/batch_1_cfg_7.5_topk_16384.jpg new file mode 100644 index 0000000000000000000000000000000000000000..5c6690220443d4bf846426828418ebec607dcfa5 --- /dev/null +++ b/CKPTS/Afteri2i_100kckpt_LLAVAT5_empty_Fixed_just_segment_pretrain_stage3_dreambench_recap_400k_t2i_400k_flux_200k_midjourney_150k_recovery_150k_3e4_epoch2_no_rotated/000-GPT-XL/checkpoints/eval_step_42000/batch_1_cfg_7.5_topk_16384.jpg @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:dc9dbd78f33486b3a0ac11c4d4200e90b51a9476061c5af1a3bf4f5a3322ec52 +size 712038 diff --git a/CKPTS/Afteri2i_100kckpt_LLAVAT5_empty_Fixed_just_segment_pretrain_stage3_dreambench_recap_400k_t2i_400k_flux_200k_midjourney_150k_recovery_150k_3e4_epoch2_no_rotated/000-GPT-XL/checkpoints/eval_step_42000/batch_2_cfg_7.5_topk_16384.jpg b/CKPTS/Afteri2i_100kckpt_LLAVAT5_empty_Fixed_just_segment_pretrain_stage3_dreambench_recap_400k_t2i_400k_flux_200k_midjourney_150k_recovery_150k_3e4_epoch2_no_rotated/000-GPT-XL/checkpoints/eval_step_42000/batch_2_cfg_7.5_topk_16384.jpg new file mode 100644 index 0000000000000000000000000000000000000000..6ab2999c44014e5432430f6f40a1e3d272bb1bf1 --- /dev/null +++ b/CKPTS/Afteri2i_100kckpt_LLAVAT5_empty_Fixed_just_segment_pretrain_stage3_dreambench_recap_400k_t2i_400k_flux_200k_midjourney_150k_recovery_150k_3e4_epoch2_no_rotated/000-GPT-XL/checkpoints/eval_step_42000/batch_2_cfg_7.5_topk_16384.jpg @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:7c2cdaf0a5975867f0d26010f9fe0fc0b7203ec0805e5a0b2cae401b364e6d1a +size 717094 diff --git a/CKPTS/Afteri2i_100kckpt_LLAVAT5_empty_Fixed_just_segment_pretrain_stage3_dreambench_recap_400k_t2i_400k_flux_200k_midjourney_150k_recovery_150k_3e4_epoch2_no_rotated/000-GPT-XL/checkpoints/eval_step_42000/batch_3_cfg_7.5_topk_16384.jpg b/CKPTS/Afteri2i_100kckpt_LLAVAT5_empty_Fixed_just_segment_pretrain_stage3_dreambench_recap_400k_t2i_400k_flux_200k_midjourney_150k_recovery_150k_3e4_epoch2_no_rotated/000-GPT-XL/checkpoints/eval_step_42000/batch_3_cfg_7.5_topk_16384.jpg new file mode 100644 index 0000000000000000000000000000000000000000..8a78631d135c03d0aa459fb59b1dc7ded5f5f2a4 --- /dev/null +++ b/CKPTS/Afteri2i_100kckpt_LLAVAT5_empty_Fixed_just_segment_pretrain_stage3_dreambench_recap_400k_t2i_400k_flux_200k_midjourney_150k_recovery_150k_3e4_epoch2_no_rotated/000-GPT-XL/checkpoints/eval_step_42000/batch_3_cfg_7.5_topk_16384.jpg @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:4a586aa6b83ea5600ab0de626b495fbc08ab98fbcc37526eb3002f3139e8bd2d +size 861640 diff --git a/CKPTS/Afteri2i_100kckpt_LLAVAT5_empty_Fixed_just_segment_pretrain_stage3_dreambench_recap_400k_t2i_400k_flux_200k_midjourney_150k_recovery_150k_3e4_epoch2_no_rotated/000-GPT-XL/checkpoints/eval_step_44000/batch_0_cfg_7.5_topk_16384.jpg b/CKPTS/Afteri2i_100kckpt_LLAVAT5_empty_Fixed_just_segment_pretrain_stage3_dreambench_recap_400k_t2i_400k_flux_200k_midjourney_150k_recovery_150k_3e4_epoch2_no_rotated/000-GPT-XL/checkpoints/eval_step_44000/batch_0_cfg_7.5_topk_16384.jpg new file mode 100644 index 0000000000000000000000000000000000000000..e51f3f7392bb7745c44de2f0c0d8414742f2d297 --- /dev/null +++ b/CKPTS/Afteri2i_100kckpt_LLAVAT5_empty_Fixed_just_segment_pretrain_stage3_dreambench_recap_400k_t2i_400k_flux_200k_midjourney_150k_recovery_150k_3e4_epoch2_no_rotated/000-GPT-XL/checkpoints/eval_step_44000/batch_0_cfg_7.5_topk_16384.jpg @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:735b156311289d1c908c7a9354d2531f7c2e278bcf1b4c698d7d419db9683dd7 +size 738505 diff --git a/CKPTS/Afteri2i_100kckpt_LLAVAT5_empty_Fixed_just_segment_pretrain_stage3_dreambench_recap_400k_t2i_400k_flux_200k_midjourney_150k_recovery_150k_3e4_epoch2_no_rotated/000-GPT-XL/checkpoints/eval_step_44000/batch_1_cfg_7.5_topk_16384.jpg b/CKPTS/Afteri2i_100kckpt_LLAVAT5_empty_Fixed_just_segment_pretrain_stage3_dreambench_recap_400k_t2i_400k_flux_200k_midjourney_150k_recovery_150k_3e4_epoch2_no_rotated/000-GPT-XL/checkpoints/eval_step_44000/batch_1_cfg_7.5_topk_16384.jpg new file mode 100644 index 0000000000000000000000000000000000000000..d3fa06f5a9360a6f6d6e07739113401cd1dc57e8 --- /dev/null +++ b/CKPTS/Afteri2i_100kckpt_LLAVAT5_empty_Fixed_just_segment_pretrain_stage3_dreambench_recap_400k_t2i_400k_flux_200k_midjourney_150k_recovery_150k_3e4_epoch2_no_rotated/000-GPT-XL/checkpoints/eval_step_44000/batch_1_cfg_7.5_topk_16384.jpg @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:210aa83033195d6f0c385da881dabd79fb5d4d8494255e335231c954c068aa82 +size 717917 diff --git a/CKPTS/Afteri2i_100kckpt_LLAVAT5_empty_Fixed_just_segment_pretrain_stage3_dreambench_recap_400k_t2i_400k_flux_200k_midjourney_150k_recovery_150k_3e4_epoch2_no_rotated/000-GPT-XL/checkpoints/eval_step_44000/batch_2_cfg_7.5_topk_16384.jpg b/CKPTS/Afteri2i_100kckpt_LLAVAT5_empty_Fixed_just_segment_pretrain_stage3_dreambench_recap_400k_t2i_400k_flux_200k_midjourney_150k_recovery_150k_3e4_epoch2_no_rotated/000-GPT-XL/checkpoints/eval_step_44000/batch_2_cfg_7.5_topk_16384.jpg new file mode 100644 index 0000000000000000000000000000000000000000..09899e03396322576359d7691880179a63720d35 --- /dev/null +++ b/CKPTS/Afteri2i_100kckpt_LLAVAT5_empty_Fixed_just_segment_pretrain_stage3_dreambench_recap_400k_t2i_400k_flux_200k_midjourney_150k_recovery_150k_3e4_epoch2_no_rotated/000-GPT-XL/checkpoints/eval_step_44000/batch_2_cfg_7.5_topk_16384.jpg @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:56e0dc4c3a0c3f8b0498751783fb760e858f3177c24c9de822681ed54ed97b96 +size 708695 diff --git a/CKPTS/Afteri2i_100kckpt_LLAVAT5_empty_Fixed_just_segment_pretrain_stage3_dreambench_recap_400k_t2i_400k_flux_200k_midjourney_150k_recovery_150k_3e4_epoch2_no_rotated/000-GPT-XL/checkpoints/eval_step_44000/batch_3_cfg_7.5_topk_16384.jpg b/CKPTS/Afteri2i_100kckpt_LLAVAT5_empty_Fixed_just_segment_pretrain_stage3_dreambench_recap_400k_t2i_400k_flux_200k_midjourney_150k_recovery_150k_3e4_epoch2_no_rotated/000-GPT-XL/checkpoints/eval_step_44000/batch_3_cfg_7.5_topk_16384.jpg new file mode 100644 index 0000000000000000000000000000000000000000..54abd4f1e15b3960ed2dd82011fe7784d1d32b93 --- /dev/null +++ b/CKPTS/Afteri2i_100kckpt_LLAVAT5_empty_Fixed_just_segment_pretrain_stage3_dreambench_recap_400k_t2i_400k_flux_200k_midjourney_150k_recovery_150k_3e4_epoch2_no_rotated/000-GPT-XL/checkpoints/eval_step_44000/batch_3_cfg_7.5_topk_16384.jpg @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:3603ca59b76eeadad220464ab649a5384b9b43f193d44b52a62b31c6832ee7c0 +size 872842 diff --git a/CKPTS/Afteri2i_100kckpt_LLAVAT5_empty_Fixed_just_segment_pretrain_stage3_dreambench_recap_400k_t2i_400k_flux_200k_midjourney_150k_recovery_150k_3e4_epoch2_no_rotated/000-GPT-XL/checkpoints/eval_step_46000/batch_0_cfg_7.5_topk_16384.jpg b/CKPTS/Afteri2i_100kckpt_LLAVAT5_empty_Fixed_just_segment_pretrain_stage3_dreambench_recap_400k_t2i_400k_flux_200k_midjourney_150k_recovery_150k_3e4_epoch2_no_rotated/000-GPT-XL/checkpoints/eval_step_46000/batch_0_cfg_7.5_topk_16384.jpg new file mode 100644 index 0000000000000000000000000000000000000000..f3550792e24d0b4d24c5643ddee20885bcc8e142 --- /dev/null +++ b/CKPTS/Afteri2i_100kckpt_LLAVAT5_empty_Fixed_just_segment_pretrain_stage3_dreambench_recap_400k_t2i_400k_flux_200k_midjourney_150k_recovery_150k_3e4_epoch2_no_rotated/000-GPT-XL/checkpoints/eval_step_46000/batch_0_cfg_7.5_topk_16384.jpg @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:49d8ab38e940bf31034218869f4481972150fe1bcfc660ecc6860fa26b868eef +size 758954 diff --git a/CKPTS/Afteri2i_100kckpt_LLAVAT5_empty_Fixed_just_segment_pretrain_stage3_dreambench_recap_400k_t2i_400k_flux_200k_midjourney_150k_recovery_150k_3e4_epoch2_no_rotated/000-GPT-XL/checkpoints/eval_step_46000/batch_1_cfg_7.5_topk_16384.jpg b/CKPTS/Afteri2i_100kckpt_LLAVAT5_empty_Fixed_just_segment_pretrain_stage3_dreambench_recap_400k_t2i_400k_flux_200k_midjourney_150k_recovery_150k_3e4_epoch2_no_rotated/000-GPT-XL/checkpoints/eval_step_46000/batch_1_cfg_7.5_topk_16384.jpg new file mode 100644 index 0000000000000000000000000000000000000000..709a334a46a9934a2e882ea76f2c47bb7bd93e73 --- /dev/null +++ b/CKPTS/Afteri2i_100kckpt_LLAVAT5_empty_Fixed_just_segment_pretrain_stage3_dreambench_recap_400k_t2i_400k_flux_200k_midjourney_150k_recovery_150k_3e4_epoch2_no_rotated/000-GPT-XL/checkpoints/eval_step_46000/batch_1_cfg_7.5_topk_16384.jpg @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:b08be6086320a1c6849852b68577c3246fe46a05b6722d250bf674a3dbf33ce4 +size 715274 diff --git a/CKPTS/Afteri2i_100kckpt_LLAVAT5_empty_Fixed_just_segment_pretrain_stage3_dreambench_recap_400k_t2i_400k_flux_200k_midjourney_150k_recovery_150k_3e4_epoch2_no_rotated/000-GPT-XL/checkpoints/eval_step_46000/batch_2_cfg_7.5_topk_16384.jpg b/CKPTS/Afteri2i_100kckpt_LLAVAT5_empty_Fixed_just_segment_pretrain_stage3_dreambench_recap_400k_t2i_400k_flux_200k_midjourney_150k_recovery_150k_3e4_epoch2_no_rotated/000-GPT-XL/checkpoints/eval_step_46000/batch_2_cfg_7.5_topk_16384.jpg new file mode 100644 index 0000000000000000000000000000000000000000..904b4e1e2fba87f09d03092e89072a10f99681b5 --- /dev/null +++ b/CKPTS/Afteri2i_100kckpt_LLAVAT5_empty_Fixed_just_segment_pretrain_stage3_dreambench_recap_400k_t2i_400k_flux_200k_midjourney_150k_recovery_150k_3e4_epoch2_no_rotated/000-GPT-XL/checkpoints/eval_step_46000/batch_2_cfg_7.5_topk_16384.jpg @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:d197e01c3c06270f49701cd7aa2dc050c109d2099707b119aece856d41d77249 +size 721300 diff --git a/CKPTS/Afteri2i_100kckpt_LLAVAT5_empty_Fixed_just_segment_pretrain_stage3_dreambench_recap_400k_t2i_400k_flux_200k_midjourney_150k_recovery_150k_3e4_epoch2_no_rotated/000-GPT-XL/checkpoints/eval_step_46000/batch_3_cfg_7.5_topk_16384.jpg b/CKPTS/Afteri2i_100kckpt_LLAVAT5_empty_Fixed_just_segment_pretrain_stage3_dreambench_recap_400k_t2i_400k_flux_200k_midjourney_150k_recovery_150k_3e4_epoch2_no_rotated/000-GPT-XL/checkpoints/eval_step_46000/batch_3_cfg_7.5_topk_16384.jpg new file mode 100644 index 0000000000000000000000000000000000000000..b22839d717ca86c038ba84a406059af3cde093fb --- /dev/null +++ b/CKPTS/Afteri2i_100kckpt_LLAVAT5_empty_Fixed_just_segment_pretrain_stage3_dreambench_recap_400k_t2i_400k_flux_200k_midjourney_150k_recovery_150k_3e4_epoch2_no_rotated/000-GPT-XL/checkpoints/eval_step_46000/batch_3_cfg_7.5_topk_16384.jpg @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:7bd591cfab7efd026b2c54a87f9ecf7c5271fc40c1719c64d4be86256093ca03 +size 871492 diff --git a/CKPTS/Afteri2i_100kckpt_LLAVAT5_empty_Fixed_just_segment_pretrain_stage3_dreambench_recap_400k_t2i_400k_flux_200k_midjourney_150k_recovery_150k_3e4_epoch2_no_rotated/000-GPT-XL/checkpoints/eval_step_6000/batch_0_cfg_7.5_topk_16384.jpg b/CKPTS/Afteri2i_100kckpt_LLAVAT5_empty_Fixed_just_segment_pretrain_stage3_dreambench_recap_400k_t2i_400k_flux_200k_midjourney_150k_recovery_150k_3e4_epoch2_no_rotated/000-GPT-XL/checkpoints/eval_step_6000/batch_0_cfg_7.5_topk_16384.jpg new file mode 100644 index 0000000000000000000000000000000000000000..9f4f9f649dae7c43d95b9f095a1b6eed175a8c5e --- /dev/null +++ b/CKPTS/Afteri2i_100kckpt_LLAVAT5_empty_Fixed_just_segment_pretrain_stage3_dreambench_recap_400k_t2i_400k_flux_200k_midjourney_150k_recovery_150k_3e4_epoch2_no_rotated/000-GPT-XL/checkpoints/eval_step_6000/batch_0_cfg_7.5_topk_16384.jpg @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:b78150b2a5c33edbfddccd6538c9189373454a1f6e8b48e9ceebae9e922b9bc2 +size 760274 diff --git a/CKPTS/Afteri2i_100kckpt_LLAVAT5_empty_Fixed_just_segment_pretrain_stage3_dreambench_recap_400k_t2i_400k_flux_200k_midjourney_150k_recovery_150k_3e4_epoch2_no_rotated/000-GPT-XL/checkpoints/eval_step_6000/batch_1_cfg_7.5_topk_16384.jpg b/CKPTS/Afteri2i_100kckpt_LLAVAT5_empty_Fixed_just_segment_pretrain_stage3_dreambench_recap_400k_t2i_400k_flux_200k_midjourney_150k_recovery_150k_3e4_epoch2_no_rotated/000-GPT-XL/checkpoints/eval_step_6000/batch_1_cfg_7.5_topk_16384.jpg new file mode 100644 index 0000000000000000000000000000000000000000..2cf9b6b087e8eac9be59832971921588d2a08386 --- /dev/null +++ b/CKPTS/Afteri2i_100kckpt_LLAVAT5_empty_Fixed_just_segment_pretrain_stage3_dreambench_recap_400k_t2i_400k_flux_200k_midjourney_150k_recovery_150k_3e4_epoch2_no_rotated/000-GPT-XL/checkpoints/eval_step_6000/batch_1_cfg_7.5_topk_16384.jpg @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:933005016c82da3b3be3bded5dfac60dccbd1cff950fab16f9d7536892f80b3a +size 716755 diff --git a/CKPTS/Afteri2i_100kckpt_LLAVAT5_empty_Fixed_just_segment_pretrain_stage3_dreambench_recap_400k_t2i_400k_flux_200k_midjourney_150k_recovery_150k_3e4_epoch2_no_rotated/000-GPT-XL/checkpoints/eval_step_6000/batch_2_cfg_7.5_topk_16384.jpg b/CKPTS/Afteri2i_100kckpt_LLAVAT5_empty_Fixed_just_segment_pretrain_stage3_dreambench_recap_400k_t2i_400k_flux_200k_midjourney_150k_recovery_150k_3e4_epoch2_no_rotated/000-GPT-XL/checkpoints/eval_step_6000/batch_2_cfg_7.5_topk_16384.jpg new file mode 100644 index 0000000000000000000000000000000000000000..997ad4e65d5c3d51db2e47caebf5cd062c3238a8 --- /dev/null +++ b/CKPTS/Afteri2i_100kckpt_LLAVAT5_empty_Fixed_just_segment_pretrain_stage3_dreambench_recap_400k_t2i_400k_flux_200k_midjourney_150k_recovery_150k_3e4_epoch2_no_rotated/000-GPT-XL/checkpoints/eval_step_6000/batch_2_cfg_7.5_topk_16384.jpg @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:8f436d0100b8c1602f1a2b85e5ce43d50050cd25ab7c1e2bc5fd2ac60877f783 +size 715148 diff --git a/CKPTS/Afteri2i_100kckpt_LLAVAT5_empty_Fixed_just_segment_pretrain_stage3_dreambench_recap_400k_t2i_400k_flux_200k_midjourney_150k_recovery_150k_3e4_epoch2_no_rotated/000-GPT-XL/checkpoints/eval_step_6000/batch_3_cfg_7.5_topk_16384.jpg b/CKPTS/Afteri2i_100kckpt_LLAVAT5_empty_Fixed_just_segment_pretrain_stage3_dreambench_recap_400k_t2i_400k_flux_200k_midjourney_150k_recovery_150k_3e4_epoch2_no_rotated/000-GPT-XL/checkpoints/eval_step_6000/batch_3_cfg_7.5_topk_16384.jpg new file mode 100644 index 0000000000000000000000000000000000000000..62f4a35c3e6ee392883f6ae897726191a7ba32a9 --- /dev/null +++ b/CKPTS/Afteri2i_100kckpt_LLAVAT5_empty_Fixed_just_segment_pretrain_stage3_dreambench_recap_400k_t2i_400k_flux_200k_midjourney_150k_recovery_150k_3e4_epoch2_no_rotated/000-GPT-XL/checkpoints/eval_step_6000/batch_3_cfg_7.5_topk_16384.jpg @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:43363937e39980a76c09a817c21e7234bb90d4714ec24dae786a9797913ef8e6 +size 863755 diff --git a/CKPTS/Afteri2i_100kckpt_LLAVAT5_empty_Fixed_just_segment_pretrain_stage3_dreambench_recap_400k_t2i_400k_flux_200k_midjourney_150k_recovery_150k_3e4_epoch2_no_rotated/000-GPT-XL/checkpoints/eval_step_8000/batch_0_cfg_7.5_topk_16384.jpg b/CKPTS/Afteri2i_100kckpt_LLAVAT5_empty_Fixed_just_segment_pretrain_stage3_dreambench_recap_400k_t2i_400k_flux_200k_midjourney_150k_recovery_150k_3e4_epoch2_no_rotated/000-GPT-XL/checkpoints/eval_step_8000/batch_0_cfg_7.5_topk_16384.jpg new file mode 100644 index 0000000000000000000000000000000000000000..6ed9113ef4b7893c8b7837593be325a777e25395 --- /dev/null +++ b/CKPTS/Afteri2i_100kckpt_LLAVAT5_empty_Fixed_just_segment_pretrain_stage3_dreambench_recap_400k_t2i_400k_flux_200k_midjourney_150k_recovery_150k_3e4_epoch2_no_rotated/000-GPT-XL/checkpoints/eval_step_8000/batch_0_cfg_7.5_topk_16384.jpg @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:54857a247dbaaf86493d08ae5ca44680865e0c72aec9ae8acc55f585705b4e8c +size 753222 diff --git a/CKPTS/Afteri2i_100kckpt_LLAVAT5_empty_Fixed_just_segment_pretrain_stage3_dreambench_recap_400k_t2i_400k_flux_200k_midjourney_150k_recovery_150k_3e4_epoch2_no_rotated/000-GPT-XL/checkpoints/eval_step_8000/batch_1_cfg_7.5_topk_16384.jpg b/CKPTS/Afteri2i_100kckpt_LLAVAT5_empty_Fixed_just_segment_pretrain_stage3_dreambench_recap_400k_t2i_400k_flux_200k_midjourney_150k_recovery_150k_3e4_epoch2_no_rotated/000-GPT-XL/checkpoints/eval_step_8000/batch_1_cfg_7.5_topk_16384.jpg new file mode 100644 index 0000000000000000000000000000000000000000..eebce21c471289b5e0365cdf4eba18c570726d12 --- /dev/null +++ b/CKPTS/Afteri2i_100kckpt_LLAVAT5_empty_Fixed_just_segment_pretrain_stage3_dreambench_recap_400k_t2i_400k_flux_200k_midjourney_150k_recovery_150k_3e4_epoch2_no_rotated/000-GPT-XL/checkpoints/eval_step_8000/batch_1_cfg_7.5_topk_16384.jpg @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:fd2142a1160587dd5810b8793e8916975c0b654c284d11932b13441697dcda7f +size 707307 diff --git a/CKPTS/Afteri2i_100kckpt_LLAVAT5_empty_Fixed_just_segment_pretrain_stage3_dreambench_recap_400k_t2i_400k_flux_200k_midjourney_150k_recovery_150k_3e4_epoch2_no_rotated/000-GPT-XL/checkpoints/eval_step_8000/batch_2_cfg_7.5_topk_16384.jpg b/CKPTS/Afteri2i_100kckpt_LLAVAT5_empty_Fixed_just_segment_pretrain_stage3_dreambench_recap_400k_t2i_400k_flux_200k_midjourney_150k_recovery_150k_3e4_epoch2_no_rotated/000-GPT-XL/checkpoints/eval_step_8000/batch_2_cfg_7.5_topk_16384.jpg new file mode 100644 index 0000000000000000000000000000000000000000..39b16bf65c7aca806e9ec7d88fe7e66ceb64471d --- /dev/null +++ b/CKPTS/Afteri2i_100kckpt_LLAVAT5_empty_Fixed_just_segment_pretrain_stage3_dreambench_recap_400k_t2i_400k_flux_200k_midjourney_150k_recovery_150k_3e4_epoch2_no_rotated/000-GPT-XL/checkpoints/eval_step_8000/batch_2_cfg_7.5_topk_16384.jpg @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:f6ffd97c23b296c0f942cb2cf4df7389883bbabedaa7bbf718ab418b597333f2 +size 747164 diff --git a/CKPTS/Afteri2i_100kckpt_LLAVAT5_empty_Fixed_just_segment_pretrain_stage3_dreambench_recap_400k_t2i_400k_flux_200k_midjourney_150k_recovery_150k_3e4_epoch2_no_rotated/000-GPT-XL/checkpoints/eval_step_8000/batch_3_cfg_7.5_topk_16384.jpg b/CKPTS/Afteri2i_100kckpt_LLAVAT5_empty_Fixed_just_segment_pretrain_stage3_dreambench_recap_400k_t2i_400k_flux_200k_midjourney_150k_recovery_150k_3e4_epoch2_no_rotated/000-GPT-XL/checkpoints/eval_step_8000/batch_3_cfg_7.5_topk_16384.jpg new file mode 100644 index 0000000000000000000000000000000000000000..16ffeec763581929c45fb09e24e647fca996891a --- /dev/null +++ b/CKPTS/Afteri2i_100kckpt_LLAVAT5_empty_Fixed_just_segment_pretrain_stage3_dreambench_recap_400k_t2i_400k_flux_200k_midjourney_150k_recovery_150k_3e4_epoch2_no_rotated/000-GPT-XL/checkpoints/eval_step_8000/batch_3_cfg_7.5_topk_16384.jpg @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:f3187d709a06eee7f7bc96f95c2ac3229c621e54317c5e4880bf1a44a1a177ce +size 845300 diff --git a/CKPTS/Afteri2i_100kckpt_LLAVAT5_empty_Fixed_just_segment_pretrain_stage3_dreambench_recap_400k_t2i_400k_flux_200k_midjourney_150k_recovery_150k_3e4_epoch2_no_rotated/000-GPT-XL/log.txt b/CKPTS/Afteri2i_100kckpt_LLAVAT5_empty_Fixed_just_segment_pretrain_stage3_dreambench_recap_400k_t2i_400k_flux_200k_midjourney_150k_recovery_150k_3e4_epoch2_no_rotated/000-GPT-XL/log.txt new file mode 100644 index 0000000000000000000000000000000000000000..08ee78f55044027ada28fd8d723a1ffa1b89a006 --- /dev/null +++ b/CKPTS/Afteri2i_100kckpt_LLAVAT5_empty_Fixed_just_segment_pretrain_stage3_dreambench_recap_400k_t2i_400k_flux_200k_midjourney_150k_recovery_150k_3e4_epoch2_no_rotated/000-GPT-XL/log.txt @@ -0,0 +1,1987 @@ +[2025-04-24 02:08:56] Experiment directory created at checkpoint/Afteri2i_100kckpt_LLAVAT5_empty_Fixed_just_segment_pretrain_stage3_dreambench_recap_400k_t2i_400k_flux_200k_midjourney_150k_recovery_150k_3e4_epoch2_no_rotated/000-GPT-XL +[2025-04-24 02:08:56] Namespace(data_path='/tmp/haozhezhao/MLLMG/jsonl_data/merged_train_set_set_subject_400k_recap_t2i_400k_flux_200k_midjourney_150k_recovery_150k_grounding_100fluxseg_50samseg.jsonl', cloud_save_path='/tmp/haozhezhao/checkpoint', no_local_save=False, vq_model='VQ-16', vq_ckpt='/tmp/haozhezhao/model/llamagen_t2i/vq_ds16_t2i.pt', codebook_size=16384, codebook_embed_dim=8, gpt_model='GPT-XL', gpt_ckpt='/tmp/haozhezhao/MLLMG/MLLMG_ckpts/checkpoints/AfterI2I_llavat5_subject_train_all_T5_subject_t2i_ti2i_120_w_flux_segment_noimageonly_empty_fix_train_all/0100000.pt', gpt_type='t2i', vocab_size=16384, cls_token_num=512, dropout_p=0.1, token_dropout_p=0.1, drop_path=0.0, no_compile=False, results_dir='checkpoint/Afteri2i_100kckpt_LLAVAT5_empty_Fixed_just_segment_pretrain_stage3_dreambench_recap_400k_t2i_400k_flux_200k_midjourney_150k_recovery_150k_3e4_epoch2_no_rotated', dataset='ti2i', image_size=512, downsample_size=16, num_classes=1000, epochs=2, lr=0.0003, weight_decay=0.05, beta1=0.9, beta2=0.95, max_grad_norm=1.0, global_batch_size=56, global_seed=0, num_workers=8, log_every=25, ckpt_every=2000, gradient_accumulation_steps=4, mixed_precision='bf16', val_data_path='/tmp/haozhezhao/MLLMG/jsonl_data/dreambench_plus_valid.jsonl', use_vision_tower=True, model_name_or_path='/tmp/haozhezhao/model/blip2-flan-t5-xl', image_place_holder='', processor_path=None, do_eval=True, max_eval_samples=250, train_text_encoder=True, no_left_padding=False, cfg_scale=7.5, top_k=16384, temperature=0.9, top_p=1.0, eval_steps=2000, project_name='llamagen_ti2i', load_from_checkpoint='/tmp/haozhezhao/MLLMG/checkpoint/FIXed_3M_1epoch_step32000.pt', warmup=0.05, lr_decay_style='cosine', lr_decay_ratio=0.1, train_iters=500000, class_dropout_prob=0.1, with_image_only=False, image_only_rate=0.1, stage2=False, subject_driven=True, load_subject_embedding=None, reference_data_path='/tmp/haozhezhao/MLLMG/cc12m_reference_tunnel.jsonl', multimodal_encoder='llava', do_recovery=True, no_replace=False, resume=False, dreambench_eval=True, find_unused_parameters=True, load_visual_encoder=False, continue_stage1=False, replace_subject=False, train_all=True, save_total_limit=1, load_language_projection='/tmp/haozhezhao/MLLMG/llava-v1.5-flant5_fixed-pretrain/mm_projector.bin', mm_vision_tower='openai/clip-vit-large-patch14', load_fixed_llamagen=True, unfreeze_output=False, fix='gpt-empty-fix', rank=0, world_size=8, gpu=0, dist_url='env://', distributed=True, dist_backend='nccl') +[2025-04-24 02:08:56] Starting rank=0, seed=0, world_size=8. +[2025-04-24 02:08:56] text_config is None. Initializing the text config with default values (`OPTConfig`). +[2025-04-24 02:10:01] GPT Parameters: 2,310,680,832 +[2025-04-24 02:10:01] num decayed parameter tensors: 356, with 2,007,303,168 parameters +[2025-04-24 02:10:01] num non-decayed parameter tensors: 124, with 197,888 parameters +[2025-04-24 02:10:01] using fused AdamW: True +[2025-04-24 02:10:10] Dataset contains 1,313,682 images +[2025-04-24 02:10:10] Train iters 46916 , warmup 2345.8, len of loader 23458 +[2025-04-24 02:10:25] ### LOAD pretraining weights from checkpoint: /tmp/haozhezhao/MLLMG/MLLMG_ckpts/checkpoints/AfterI2I_llavat5_subject_train_all_T5_subject_t2i_ti2i_120_w_flux_segment_noimageonly_empty_fix_train_all/0100000.pt +[2025-04-24 02:10:25] Initial state: steps=0, epochs=0 +[2025-04-24 02:10:25] compiling the model... (may take several minutes) +[2025-04-24 02:10:26] freeze the vit +[2025-04-24 02:10:26] ***** total param is 2310680832 ***** +[2025-04-24 02:10:26] ***** total trained param is 2007501056 ***** +[2025-04-24 02:10:35] Training for 2 epochs... +[2025-04-24 02:10:35] Beginning epoch 0... +[2025-04-24 02:15:03] (step=0000025) Train Loss: 6.5957, Train Steps/Sec: 0.09 +[2025-04-24 02:15:52] (step=0000050) Train Loss: 6.6553, Train Steps/Sec: 0.52 +[2025-04-24 02:17:21] (step=0000075) Train Loss: 6.4592, Train Steps/Sec: 0.28 +[2025-04-24 02:18:15] (step=0000100) Train Loss: 6.3507, Train Steps/Sec: 0.46 +[2025-04-24 02:18:36] (step=0000125) Train Loss: 6.2555, Train Steps/Sec: 1.17 +[2025-04-24 02:19:25] (step=0000150) Train Loss: 6.1031, Train Steps/Sec: 0.51 +[2025-04-24 02:20:13] (step=0000175) Train Loss: 6.1222, Train Steps/Sec: 0.52 +[2025-04-24 02:20:34] (step=0000200) Train Loss: 6.0427, Train Steps/Sec: 1.16 +[2025-04-24 02:20:56] (step=0000225) Train Loss: 5.9289, Train Steps/Sec: 1.17 +[2025-04-24 02:21:42] (step=0000250) Train Loss: 5.8817, Train Steps/Sec: 0.54 +[2025-04-24 02:22:55] (step=0000275) Train Loss: 5.9753, Train Steps/Sec: 0.34 +[2025-04-24 02:23:17] (step=0000300) Train Loss: 5.8086, Train Steps/Sec: 1.16 +[2025-04-24 02:23:38] (step=0000325) Train Loss: 5.9533, Train Steps/Sec: 1.17 +[2025-04-24 02:24:00] (step=0000350) Train Loss: 5.8435, Train Steps/Sec: 1.17 +[2025-04-24 02:24:21] (step=0000375) Train Loss: 5.8652, Train Steps/Sec: 1.17 +[2025-04-24 02:25:10] (step=0000400) Train Loss: 5.7729, Train Steps/Sec: 0.51 +[2025-04-24 02:25:35] (step=0000425) Train Loss: 5.8456, Train Steps/Sec: 0.99 +[2025-04-24 02:25:56] (step=0000450) Train Loss: 5.8262, Train Steps/Sec: 1.17 +[2025-04-24 02:26:18] (step=0000475) Train Loss: 5.8046, Train Steps/Sec: 1.17 +[2025-04-24 02:26:39] (step=0000500) Train Loss: 5.8188, Train Steps/Sec: 1.17 +[2025-04-24 02:27:01] (step=0000525) Train Loss: 5.8230, Train Steps/Sec: 1.17 +[2025-04-24 02:27:22] (step=0000550) Train Loss: 5.8371, Train Steps/Sec: 1.17 +[2025-04-24 02:27:43] (step=0000575) Train Loss: 5.8416, Train Steps/Sec: 1.17 +[2025-04-24 02:28:09] (step=0000600) Train Loss: 5.7896, Train Steps/Sec: 0.99 +[2025-04-24 02:28:30] (step=0000625) Train Loss: 5.8656, Train Steps/Sec: 1.16 +[2025-04-24 02:28:52] (step=0000650) Train Loss: 5.7768, Train Steps/Sec: 1.17 +[2025-04-24 02:29:13] (step=0000675) Train Loss: 5.7927, Train Steps/Sec: 1.17 +[2025-04-24 02:29:35] (step=0000700) Train Loss: 5.9221, Train Steps/Sec: 1.12 +[2025-04-24 02:29:57] (step=0000725) Train Loss: 5.7608, Train Steps/Sec: 1.17 +[2025-04-24 02:30:18] (step=0000750) Train Loss: 5.8462, Train Steps/Sec: 1.17 +[2025-04-24 02:30:40] (step=0000775) Train Loss: 5.8319, Train Steps/Sec: 1.17 +[2025-04-24 02:31:01] (step=0000800) Train Loss: 5.7974, Train Steps/Sec: 1.16 +[2025-04-24 02:31:23] (step=0000825) Train Loss: 5.7650, Train Steps/Sec: 1.17 +[2025-04-24 02:31:44] (step=0000850) Train Loss: 5.7241, Train Steps/Sec: 1.17 +[2025-04-24 02:32:05] (step=0000875) Train Loss: 5.7821, Train Steps/Sec: 1.17 +[2025-04-24 02:32:27] (step=0000900) Train Loss: 5.7709, Train Steps/Sec: 1.16 +[2025-04-24 02:32:48] (step=0000925) Train Loss: 5.8272, Train Steps/Sec: 1.17 +[2025-04-24 02:33:10] (step=0000950) Train Loss: 5.8730, Train Steps/Sec: 1.17 +[2025-04-24 02:33:31] (step=0000975) Train Loss: 5.7750, Train Steps/Sec: 1.17 +[2025-04-24 02:33:53] (step=0001000) Train Loss: 5.7671, Train Steps/Sec: 1.16 +[2025-04-24 02:34:14] (step=0001025) Train Loss: 5.7232, Train Steps/Sec: 1.17 +[2025-04-24 02:34:35] (step=0001050) Train Loss: 5.7602, Train Steps/Sec: 1.17 +[2025-04-24 02:34:57] (step=0001075) Train Loss: 5.7207, Train Steps/Sec: 1.17 +[2025-04-24 02:35:18] (step=0001100) Train Loss: 5.7756, Train Steps/Sec: 1.16 +[2025-04-24 02:35:40] (step=0001125) Train Loss: 5.7312, Train Steps/Sec: 1.17 +[2025-04-24 02:36:02] (step=0001150) Train Loss: 5.7570, Train Steps/Sec: 1.11 +[2025-04-24 02:36:25] (step=0001175) Train Loss: 5.7889, Train Steps/Sec: 1.11 +[2025-04-24 02:36:46] (step=0001200) Train Loss: 5.7174, Train Steps/Sec: 1.16 +[2025-04-24 02:37:08] (step=0001225) Train Loss: 5.7553, Train Steps/Sec: 1.16 +[2025-04-24 02:37:31] (step=0001250) Train Loss: 5.6890, Train Steps/Sec: 1.06 +[2025-04-24 02:37:55] (step=0001275) Train Loss: 5.7127, Train Steps/Sec: 1.04 +[2025-04-24 02:38:17] (step=0001300) Train Loss: 5.7368, Train Steps/Sec: 1.16 +[2025-04-24 02:38:38] (step=0001325) Train Loss: 5.8127, Train Steps/Sec: 1.17 +[2025-04-24 02:38:59] (step=0001350) Train Loss: 5.7408, Train Steps/Sec: 1.17 +[2025-04-24 02:39:22] (step=0001375) Train Loss: 5.7906, Train Steps/Sec: 1.09 +[2025-04-24 02:39:44] (step=0001400) Train Loss: 5.6655, Train Steps/Sec: 1.16 +[2025-04-24 02:40:05] (step=0001425) Train Loss: 5.7180, Train Steps/Sec: 1.17 +[2025-04-24 02:40:27] (step=0001450) Train Loss: 5.8401, Train Steps/Sec: 1.17 +[2025-04-24 02:40:48] (step=0001475) Train Loss: 5.7429, Train Steps/Sec: 1.17 +[2025-04-24 02:41:10] (step=0001500) Train Loss: 5.6887, Train Steps/Sec: 1.16 +[2025-04-24 02:41:31] (step=0001525) Train Loss: 5.7400, Train Steps/Sec: 1.17 +[2025-04-24 02:41:53] (step=0001550) Train Loss: 5.7293, Train Steps/Sec: 1.17 +[2025-04-24 02:42:14] (step=0001575) Train Loss: 5.7392, Train Steps/Sec: 1.17 +[2025-04-24 02:42:35] (step=0001600) Train Loss: 5.7208, Train Steps/Sec: 1.16 +[2025-04-24 02:42:57] (step=0001625) Train Loss: 5.7277, Train Steps/Sec: 1.17 +[2025-04-24 02:43:18] (step=0001650) Train Loss: 5.7647, Train Steps/Sec: 1.17 +[2025-04-24 02:43:40] (step=0001675) Train Loss: 5.7128, Train Steps/Sec: 1.17 +[2025-04-24 02:44:01] (step=0001700) Train Loss: 5.8438, Train Steps/Sec: 1.16 +[2025-04-24 02:44:22] (step=0001725) Train Loss: 5.7536, Train Steps/Sec: 1.17 +[2025-04-24 02:44:44] (step=0001750) Train Loss: 5.6801, Train Steps/Sec: 1.17 +[2025-04-24 02:45:05] (step=0001775) Train Loss: 5.7108, Train Steps/Sec: 1.17 +[2025-04-24 02:45:27] (step=0001800) Train Loss: 5.7746, Train Steps/Sec: 1.17 +[2025-04-24 02:45:48] (step=0001825) Train Loss: 5.7193, Train Steps/Sec: 1.17 +[2025-04-24 02:46:09] (step=0001850) Train Loss: 5.7679, Train Steps/Sec: 1.17 +[2025-04-24 02:46:32] (step=0001875) Train Loss: 5.7137, Train Steps/Sec: 1.11 +[2025-04-24 02:46:54] (step=0001900) Train Loss: 5.6731, Train Steps/Sec: 1.11 +[2025-04-24 02:47:19] (step=0001925) Train Loss: 5.8252, Train Steps/Sec: 1.00 +[2025-04-24 02:47:41] (step=0001950) Train Loss: 5.7324, Train Steps/Sec: 1.17 +[2025-04-24 02:48:02] (step=0001975) Train Loss: 5.6646, Train Steps/Sec: 1.17 +[2025-04-24 02:48:24] (step=0002000) Train Loss: 5.6796, Train Steps/Sec: 1.17 +[2025-04-24 02:48:24] text_config is None. Initializing the text config with default values (`OPTConfig`). +[2025-04-24 02:53:44] Finish Eval in 2000 steps... +[2025-04-24 02:54:48] Saved checkpoint to checkpoint/Afteri2i_100kckpt_LLAVAT5_empty_Fixed_just_segment_pretrain_stage3_dreambench_recap_400k_t2i_400k_flux_200k_midjourney_150k_recovery_150k_3e4_epoch2_no_rotated/000-GPT-XL/checkpoints/0002000.pt +[2025-04-24 02:55:10] (step=0002025) Train Loss: 5.7594, Train Steps/Sec: 0.06 +[2025-04-24 02:55:32] (step=0002050) Train Loss: 5.6316, Train Steps/Sec: 1.17 +[2025-04-24 02:55:53] (step=0002075) Train Loss: 5.6661, Train Steps/Sec: 1.16 +[2025-04-24 02:56:15] (step=0002100) Train Loss: 5.6931, Train Steps/Sec: 1.16 +[2025-04-24 02:56:36] (step=0002125) Train Loss: 5.6037, Train Steps/Sec: 1.17 +[2025-04-24 02:56:57] (step=0002150) Train Loss: 5.7843, Train Steps/Sec: 1.17 +[2025-04-24 02:57:19] (step=0002175) Train Loss: 5.5708, Train Steps/Sec: 1.17 +[2025-04-24 02:57:40] (step=0002200) Train Loss: 5.7404, Train Steps/Sec: 1.16 +[2025-04-24 02:58:02] (step=0002225) Train Loss: 5.6501, Train Steps/Sec: 1.16 +[2025-04-24 02:58:23] (step=0002250) Train Loss: 5.7567, Train Steps/Sec: 1.16 +[2025-04-24 02:58:45] (step=0002275) Train Loss: 5.7399, Train Steps/Sec: 1.17 +[2025-04-24 02:59:06] (step=0002300) Train Loss: 5.6678, Train Steps/Sec: 1.16 +[2025-04-24 02:59:28] (step=0002325) Train Loss: 5.6738, Train Steps/Sec: 1.17 +[2025-04-24 02:59:49] (step=0002350) Train Loss: 5.7381, Train Steps/Sec: 1.17 +[2025-04-24 03:00:11] (step=0002375) Train Loss: 5.6478, Train Steps/Sec: 1.16 +[2025-04-24 03:00:32] (step=0002400) Train Loss: 5.6091, Train Steps/Sec: 1.16 +[2025-04-24 03:00:54] (step=0002425) Train Loss: 5.7378, Train Steps/Sec: 1.17 +[2025-04-24 03:01:15] (step=0002450) Train Loss: 5.7220, Train Steps/Sec: 1.17 +[2025-04-24 03:01:36] (step=0002475) Train Loss: 5.7484, Train Steps/Sec: 1.17 +[2025-04-24 03:01:58] (step=0002500) Train Loss: 5.7335, Train Steps/Sec: 1.17 +[2025-04-24 03:02:19] (step=0002525) Train Loss: 5.6860, Train Steps/Sec: 1.15 +[2025-04-24 03:02:41] (step=0002550) Train Loss: 5.7739, Train Steps/Sec: 1.17 +[2025-04-24 03:03:06] (step=0002575) Train Loss: 5.6798, Train Steps/Sec: 0.98 +[2025-04-24 03:03:29] (step=0002600) Train Loss: 5.6500, Train Steps/Sec: 1.11 +[2025-04-24 03:03:51] (step=0002625) Train Loss: 5.6982, Train Steps/Sec: 1.11 +[2025-04-24 03:04:14] (step=0002650) Train Loss: 5.5997, Train Steps/Sec: 1.12 +[2025-04-24 03:04:36] (step=0002675) Train Loss: 5.6861, Train Steps/Sec: 1.11 +[2025-04-24 03:04:58] (step=0002700) Train Loss: 5.6614, Train Steps/Sec: 1.16 +[2025-04-24 03:05:19] (step=0002725) Train Loss: 5.6786, Train Steps/Sec: 1.17 +[2025-04-24 03:05:40] (step=0002750) Train Loss: 5.7428, Train Steps/Sec: 1.17 +[2025-04-24 03:06:02] (step=0002775) Train Loss: 5.6406, Train Steps/Sec: 1.17 +[2025-04-24 03:06:23] (step=0002800) Train Loss: 5.6809, Train Steps/Sec: 1.16 +[2025-04-24 03:06:44] (step=0002825) Train Loss: 5.7662, Train Steps/Sec: 1.17 +[2025-04-24 03:07:06] (step=0002850) Train Loss: 5.6854, Train Steps/Sec: 1.17 +[2025-04-24 03:07:27] (step=0002875) Train Loss: 5.7506, Train Steps/Sec: 1.17 +[2025-04-24 03:07:49] (step=0002900) Train Loss: 5.6676, Train Steps/Sec: 1.17 +[2025-04-24 03:08:10] (step=0002925) Train Loss: 5.6462, Train Steps/Sec: 1.17 +[2025-04-24 03:08:31] (step=0002950) Train Loss: 5.5837, Train Steps/Sec: 1.17 +[2025-04-24 03:08:53] (step=0002975) Train Loss: 5.6462, Train Steps/Sec: 1.17 +[2025-04-24 03:09:14] (step=0003000) Train Loss: 5.6839, Train Steps/Sec: 1.17 +[2025-04-24 03:09:36] (step=0003025) Train Loss: 5.6783, Train Steps/Sec: 1.17 +[2025-04-24 03:09:57] (step=0003050) Train Loss: 5.7179, Train Steps/Sec: 1.17 +[2025-04-24 03:10:18] (step=0003075) Train Loss: 5.6046, Train Steps/Sec: 1.17 +[2025-04-24 03:10:40] (step=0003100) Train Loss: 5.8085, Train Steps/Sec: 1.16 +[2025-04-24 03:11:01] (step=0003125) Train Loss: 5.7583, Train Steps/Sec: 1.17 +[2025-04-24 03:11:22] (step=0003150) Train Loss: 5.7029, Train Steps/Sec: 1.17 +[2025-04-24 03:11:44] (step=0003175) Train Loss: 5.6699, Train Steps/Sec: 1.17 +[2025-04-24 03:12:05] (step=0003200) Train Loss: 5.6563, Train Steps/Sec: 1.16 +[2025-04-24 03:12:28] (step=0003225) Train Loss: 5.7033, Train Steps/Sec: 1.13 +[2025-04-24 03:12:52] (step=0003250) Train Loss: 5.6183, Train Steps/Sec: 1.04 +[2025-04-24 03:13:13] (step=0003275) Train Loss: 5.6353, Train Steps/Sec: 1.17 +[2025-04-24 03:13:34] (step=0003300) Train Loss: 5.5692, Train Steps/Sec: 1.16 +[2025-04-24 03:13:58] (step=0003325) Train Loss: 5.5634, Train Steps/Sec: 1.07 +[2025-04-24 03:14:20] (step=0003350) Train Loss: 5.7090, Train Steps/Sec: 1.11 +[2025-04-24 03:14:42] (step=0003375) Train Loss: 5.6541, Train Steps/Sec: 1.17 +[2025-04-24 03:15:03] (step=0003400) Train Loss: 5.6853, Train Steps/Sec: 1.16 +[2025-04-24 03:15:25] (step=0003425) Train Loss: 5.6970, Train Steps/Sec: 1.17 +[2025-04-24 03:15:46] (step=0003450) Train Loss: 5.7134, Train Steps/Sec: 1.17 +[2025-04-24 03:16:07] (step=0003475) Train Loss: 5.6570, Train Steps/Sec: 1.17 +[2025-04-24 03:16:29] (step=0003500) Train Loss: 5.6432, Train Steps/Sec: 1.16 +[2025-04-24 03:16:50] (step=0003525) Train Loss: 5.6803, Train Steps/Sec: 1.17 +[2025-04-24 03:17:12] (step=0003550) Train Loss: 5.7212, Train Steps/Sec: 1.17 +[2025-04-24 03:17:33] (step=0003575) Train Loss: 5.6999, Train Steps/Sec: 1.17 +[2025-04-24 03:17:54] (step=0003600) Train Loss: 5.6854, Train Steps/Sec: 1.16 +[2025-04-24 03:18:16] (step=0003625) Train Loss: 5.6661, Train Steps/Sec: 1.17 +[2025-04-24 03:18:37] (step=0003650) Train Loss: 5.5655, Train Steps/Sec: 1.17 +[2025-04-24 03:18:59] (step=0003675) Train Loss: 5.6526, Train Steps/Sec: 1.17 +[2025-04-24 03:19:20] (step=0003700) Train Loss: 5.7180, Train Steps/Sec: 1.16 +[2025-04-24 03:19:42] (step=0003725) Train Loss: 5.6502, Train Steps/Sec: 1.16 +[2025-04-24 03:20:03] (step=0003750) Train Loss: 5.6715, Train Steps/Sec: 1.17 +[2025-04-24 03:20:25] (step=0003775) Train Loss: 5.6307, Train Steps/Sec: 1.16 +[2025-04-24 03:20:46] (step=0003800) Train Loss: 5.6172, Train Steps/Sec: 1.16 +[2025-04-24 03:21:08] (step=0003825) Train Loss: 5.7500, Train Steps/Sec: 1.16 +[2025-04-24 03:21:29] (step=0003850) Train Loss: 5.5991, Train Steps/Sec: 1.17 +[2025-04-24 03:21:51] (step=0003875) Train Loss: 5.6052, Train Steps/Sec: 1.16 +[2025-04-24 03:22:16] (step=0003900) Train Loss: 5.6585, Train Steps/Sec: 1.00 +[2025-04-24 03:22:37] (step=0003925) Train Loss: 5.6553, Train Steps/Sec: 1.17 +[2025-04-24 03:22:59] (step=0003950) Train Loss: 5.7087, Train Steps/Sec: 1.17 +[2025-04-24 03:23:21] (step=0003975) Train Loss: 5.6191, Train Steps/Sec: 1.12 +[2025-04-24 03:23:43] (step=0004000) Train Loss: 5.6471, Train Steps/Sec: 1.12 +[2025-04-24 03:23:43] text_config is None. Initializing the text config with default values (`OPTConfig`). +[2025-04-24 03:28:57] Finish Eval in 4000 steps... +[2025-04-24 03:30:02] Saved checkpoint to checkpoint/Afteri2i_100kckpt_LLAVAT5_empty_Fixed_just_segment_pretrain_stage3_dreambench_recap_400k_t2i_400k_flux_200k_midjourney_150k_recovery_150k_3e4_epoch2_no_rotated/000-GPT-XL/checkpoints/0004000.pt +[2025-04-24 03:30:04] Removed old checkpoint: checkpoint/Afteri2i_100kckpt_LLAVAT5_empty_Fixed_just_segment_pretrain_stage3_dreambench_recap_400k_t2i_400k_flux_200k_midjourney_150k_recovery_150k_3e4_epoch2_no_rotated/000-GPT-XL/checkpoints/0002000.pt +[2025-04-24 03:30:25] (step=0004025) Train Loss: 5.6916, Train Steps/Sec: 0.06 +[2025-04-24 03:30:48] (step=0004050) Train Loss: 5.6571, Train Steps/Sec: 1.12 +[2025-04-24 03:31:10] (step=0004075) Train Loss: 5.6856, Train Steps/Sec: 1.12 +[2025-04-24 03:31:32] (step=0004100) Train Loss: 5.5924, Train Steps/Sec: 1.17 +[2025-04-24 03:31:53] (step=0004125) Train Loss: 5.5697, Train Steps/Sec: 1.17 +[2025-04-24 03:32:14] (step=0004150) Train Loss: 5.6732, Train Steps/Sec: 1.17 +[2025-04-24 03:32:36] (step=0004175) Train Loss: 5.6684, Train Steps/Sec: 1.17 +[2025-04-24 03:32:57] (step=0004200) Train Loss: 5.6094, Train Steps/Sec: 1.15 +[2025-04-24 03:33:19] (step=0004225) Train Loss: 5.6585, Train Steps/Sec: 1.15 +[2025-04-24 03:33:40] (step=0004250) Train Loss: 5.5982, Train Steps/Sec: 1.17 +[2025-04-24 03:34:02] (step=0004275) Train Loss: 5.6848, Train Steps/Sec: 1.17 +[2025-04-24 03:34:23] (step=0004300) Train Loss: 5.6966, Train Steps/Sec: 1.16 +[2025-04-24 03:34:45] (step=0004325) Train Loss: 5.6266, Train Steps/Sec: 1.15 +[2025-04-24 03:35:07] (step=0004350) Train Loss: 5.6196, Train Steps/Sec: 1.15 +[2025-04-24 03:35:28] (step=0004375) Train Loss: 5.6525, Train Steps/Sec: 1.17 +[2025-04-24 03:35:50] (step=0004400) Train Loss: 5.5551, Train Steps/Sec: 1.16 +[2025-04-24 03:36:11] (step=0004425) Train Loss: 5.6523, Train Steps/Sec: 1.16 +[2025-04-24 03:36:33] (step=0004450) Train Loss: 5.6658, Train Steps/Sec: 1.15 +[2025-04-24 03:36:54] (step=0004475) Train Loss: 5.6252, Train Steps/Sec: 1.17 +[2025-04-24 03:37:16] (step=0004500) Train Loss: 5.5928, Train Steps/Sec: 1.16 +[2025-04-24 03:37:37] (step=0004525) Train Loss: 5.7613, Train Steps/Sec: 1.17 +[2025-04-24 03:38:01] (step=0004550) Train Loss: 5.6549, Train Steps/Sec: 1.02 +[2025-04-24 03:38:24] (step=0004575) Train Loss: 5.5792, Train Steps/Sec: 1.11 +[2025-04-24 03:38:46] (step=0004600) Train Loss: 5.6540, Train Steps/Sec: 1.15 +[2025-04-24 03:39:07] (step=0004625) Train Loss: 5.6687, Train Steps/Sec: 1.17 +[2025-04-24 03:39:28] (step=0004650) Train Loss: 5.6963, Train Steps/Sec: 1.16 +[2025-04-24 03:39:52] (step=0004675) Train Loss: 5.6864, Train Steps/Sec: 1.08 +[2025-04-24 03:40:13] (step=0004700) Train Loss: 5.5643, Train Steps/Sec: 1.17 +[2025-04-24 03:40:34] (step=0004725) Train Loss: 5.6133, Train Steps/Sec: 1.17 +[2025-04-24 03:40:56] (step=0004750) Train Loss: 5.6152, Train Steps/Sec: 1.17 +[2025-04-24 03:41:18] (step=0004775) Train Loss: 5.6003, Train Steps/Sec: 1.12 +[2025-04-24 03:41:41] (step=0004800) Train Loss: 5.5961, Train Steps/Sec: 1.11 +[2025-04-24 03:42:02] (step=0004825) Train Loss: 5.5705, Train Steps/Sec: 1.17 +[2025-04-24 03:42:23] (step=0004850) Train Loss: 5.5740, Train Steps/Sec: 1.17 +[2025-04-24 03:42:45] (step=0004875) Train Loss: 5.6113, Train Steps/Sec: 1.17 +[2025-04-24 03:43:06] (step=0004900) Train Loss: 5.5909, Train Steps/Sec: 1.16 +[2025-04-24 03:43:28] (step=0004925) Train Loss: 5.6852, Train Steps/Sec: 1.17 +[2025-04-24 03:43:49] (step=0004950) Train Loss: 5.6314, Train Steps/Sec: 1.17 +[2025-04-24 03:44:10] (step=0004975) Train Loss: 5.6388, Train Steps/Sec: 1.17 +[2025-04-24 03:44:32] (step=0005000) Train Loss: 5.7043, Train Steps/Sec: 1.16 +[2025-04-24 03:44:53] (step=0005025) Train Loss: 5.5631, Train Steps/Sec: 1.17 +[2025-04-24 03:45:15] (step=0005050) Train Loss: 5.5957, Train Steps/Sec: 1.17 +[2025-04-24 03:45:36] (step=0005075) Train Loss: 5.6212, Train Steps/Sec: 1.17 +[2025-04-24 03:45:57] (step=0005100) Train Loss: 5.6674, Train Steps/Sec: 1.17 +[2025-04-24 03:46:19] (step=0005125) Train Loss: 5.6051, Train Steps/Sec: 1.17 +[2025-04-24 03:46:40] (step=0005150) Train Loss: 5.5861, Train Steps/Sec: 1.17 +[2025-04-24 03:47:02] (step=0005175) Train Loss: 5.6565, Train Steps/Sec: 1.17 +[2025-04-24 03:47:25] (step=0005200) Train Loss: 5.6731, Train Steps/Sec: 1.05 +[2025-04-24 03:47:49] (step=0005225) Train Loss: 5.5777, Train Steps/Sec: 1.07 +[2025-04-24 03:48:10] (step=0005250) Train Loss: 5.6420, Train Steps/Sec: 1.17 +[2025-04-24 03:48:32] (step=0005275) Train Loss: 5.6104, Train Steps/Sec: 1.17 +[2025-04-24 03:48:53] (step=0005300) Train Loss: 5.5681, Train Steps/Sec: 1.16 +[2025-04-24 03:49:16] (step=0005325) Train Loss: 5.6294, Train Steps/Sec: 1.09 +[2025-04-24 03:49:37] (step=0005350) Train Loss: 5.6259, Train Steps/Sec: 1.17 +[2025-04-24 03:49:59] (step=0005375) Train Loss: 5.6297, Train Steps/Sec: 1.16 +[2025-04-24 03:50:21] (step=0005400) Train Loss: 5.6250, Train Steps/Sec: 1.16 +[2025-04-24 03:50:42] (step=0005425) Train Loss: 5.6357, Train Steps/Sec: 1.17 +[2025-04-24 03:51:03] (step=0005450) Train Loss: 5.7066, Train Steps/Sec: 1.17 +[2025-04-24 03:51:25] (step=0005475) Train Loss: 5.5916, Train Steps/Sec: 1.17 +[2025-04-24 03:51:47] (step=0005500) Train Loss: 5.6334, Train Steps/Sec: 1.11 +[2025-04-24 03:52:10] (step=0005525) Train Loss: 5.5726, Train Steps/Sec: 1.11 +[2025-04-24 03:52:31] (step=0005550) Train Loss: 5.6380, Train Steps/Sec: 1.17 +[2025-04-24 03:52:52] (step=0005575) Train Loss: 5.5884, Train Steps/Sec: 1.17 +[2025-04-24 03:53:14] (step=0005600) Train Loss: 5.6941, Train Steps/Sec: 1.16 +[2025-04-24 03:53:35] (step=0005625) Train Loss: 5.5545, Train Steps/Sec: 1.17 +[2025-04-24 03:53:57] (step=0005650) Train Loss: 5.6760, Train Steps/Sec: 1.17 +[2025-04-24 03:54:18] (step=0005675) Train Loss: 5.6761, Train Steps/Sec: 1.17 +[2025-04-24 03:54:39] (step=0005700) Train Loss: 5.6432, Train Steps/Sec: 1.17 +[2025-04-24 03:55:01] (step=0005725) Train Loss: 5.6273, Train Steps/Sec: 1.17 +[2025-04-24 03:55:22] (step=0005750) Train Loss: 5.5605, Train Steps/Sec: 1.17 +[2025-04-24 03:55:44] (step=0005775) Train Loss: 5.6771, Train Steps/Sec: 1.17 +[2025-04-24 03:56:05] (step=0005800) Train Loss: 5.5572, Train Steps/Sec: 1.17 +[2025-04-24 03:56:26] (step=0005825) Train Loss: 5.6315, Train Steps/Sec: 1.17 +[2025-04-24 03:56:48] (step=0005850) Train Loss: 5.6225, Train Steps/Sec: 1.17 +[2025-04-24 03:57:12] (step=0005875) Train Loss: 5.5693, Train Steps/Sec: 1.03 +[2025-04-24 03:57:34] (step=0005900) Train Loss: 5.5906, Train Steps/Sec: 1.17 +[2025-04-24 03:57:55] (step=0005925) Train Loss: 5.5422, Train Steps/Sec: 1.17 +[2025-04-24 03:58:16] (step=0005950) Train Loss: 5.6422, Train Steps/Sec: 1.17 +[2025-04-24 03:58:39] (step=0005975) Train Loss: 5.6053, Train Steps/Sec: 1.08 +[2025-04-24 03:59:01] (step=0006000) Train Loss: 5.6260, Train Steps/Sec: 1.17 +[2025-04-24 03:59:01] text_config is None. Initializing the text config with default values (`OPTConfig`). +[2025-04-24 04:04:07] Finish Eval in 6000 steps... +[2025-04-24 04:04:27] Saved checkpoint to checkpoint/Afteri2i_100kckpt_LLAVAT5_empty_Fixed_just_segment_pretrain_stage3_dreambench_recap_400k_t2i_400k_flux_200k_midjourney_150k_recovery_150k_3e4_epoch2_no_rotated/000-GPT-XL/checkpoints/0006000.pt +[2025-04-24 04:04:29] Removed old checkpoint: checkpoint/Afteri2i_100kckpt_LLAVAT5_empty_Fixed_just_segment_pretrain_stage3_dreambench_recap_400k_t2i_400k_flux_200k_midjourney_150k_recovery_150k_3e4_epoch2_no_rotated/000-GPT-XL/checkpoints/0004000.pt +[2025-04-24 04:04:51] (step=0006025) Train Loss: 5.6376, Train Steps/Sec: 0.07 +[2025-04-24 04:05:12] (step=0006050) Train Loss: 5.6994, Train Steps/Sec: 1.17 +[2025-04-24 04:05:33] (step=0006075) Train Loss: 5.5185, Train Steps/Sec: 1.17 +[2025-04-24 04:05:55] (step=0006100) Train Loss: 5.6487, Train Steps/Sec: 1.16 +[2025-04-24 04:06:16] (step=0006125) Train Loss: 5.5975, Train Steps/Sec: 1.17 +[2025-04-24 04:06:38] (step=0006150) Train Loss: 5.6118, Train Steps/Sec: 1.17 +[2025-04-24 04:06:59] (step=0006175) Train Loss: 5.6022, Train Steps/Sec: 1.17 +[2025-04-24 04:07:20] (step=0006200) Train Loss: 5.6536, Train Steps/Sec: 1.17 +[2025-04-24 04:07:43] (step=0006225) Train Loss: 5.6016, Train Steps/Sec: 1.11 +[2025-04-24 04:08:05] (step=0006250) Train Loss: 5.6505, Train Steps/Sec: 1.12 +[2025-04-24 04:08:27] (step=0006275) Train Loss: 5.5216, Train Steps/Sec: 1.16 +[2025-04-24 04:08:48] (step=0006300) Train Loss: 5.5498, Train Steps/Sec: 1.16 +[2025-04-24 04:09:10] (step=0006325) Train Loss: 5.6056, Train Steps/Sec: 1.17 +[2025-04-24 04:09:31] (step=0006350) Train Loss: 5.5886, Train Steps/Sec: 1.17 +[2025-04-24 04:09:53] (step=0006375) Train Loss: 5.6752, Train Steps/Sec: 1.15 +[2025-04-24 04:10:14] (step=0006400) Train Loss: 5.5540, Train Steps/Sec: 1.16 +[2025-04-24 04:10:36] (step=0006425) Train Loss: 5.6114, Train Steps/Sec: 1.17 +[2025-04-24 04:10:57] (step=0006450) Train Loss: 5.5641, Train Steps/Sec: 1.17 +[2025-04-24 04:11:18] (step=0006475) Train Loss: 5.5893, Train Steps/Sec: 1.17 +[2025-04-24 04:11:40] (step=0006500) Train Loss: 5.5561, Train Steps/Sec: 1.17 +[2025-04-24 04:12:03] (step=0006525) Train Loss: 5.6092, Train Steps/Sec: 1.07 +[2025-04-24 04:12:26] (step=0006550) Train Loss: 5.6367, Train Steps/Sec: 1.07 +[2025-04-24 04:12:48] (step=0006575) Train Loss: 5.6732, Train Steps/Sec: 1.16 +[2025-04-24 04:13:09] (step=0006600) Train Loss: 5.5629, Train Steps/Sec: 1.16 +[2025-04-24 04:13:31] (step=0006625) Train Loss: 5.5156, Train Steps/Sec: 1.16 +[2025-04-24 04:13:54] (step=0006650) Train Loss: 5.5275, Train Steps/Sec: 1.09 +[2025-04-24 04:14:15] (step=0006675) Train Loss: 5.6160, Train Steps/Sec: 1.17 +[2025-04-24 04:14:37] (step=0006700) Train Loss: 5.5878, Train Steps/Sec: 1.16 +[2025-04-24 04:14:58] (step=0006725) Train Loss: 5.5768, Train Steps/Sec: 1.17 +[2025-04-24 04:15:20] (step=0006750) Train Loss: 5.5450, Train Steps/Sec: 1.17 +[2025-04-24 04:15:41] (step=0006775) Train Loss: 5.6187, Train Steps/Sec: 1.17 +[2025-04-24 04:16:02] (step=0006800) Train Loss: 5.6300, Train Steps/Sec: 1.17 +[2025-04-24 04:16:24] (step=0006825) Train Loss: 5.6439, Train Steps/Sec: 1.17 +[2025-04-24 04:16:45] (step=0006850) Train Loss: 5.6374, Train Steps/Sec: 1.17 +[2025-04-24 04:17:07] (step=0006875) Train Loss: 5.5296, Train Steps/Sec: 1.17 +[2025-04-24 04:17:28] (step=0006900) Train Loss: 5.6635, Train Steps/Sec: 1.17 +[2025-04-24 04:17:49] (step=0006925) Train Loss: 5.5907, Train Steps/Sec: 1.17 +[2025-04-24 04:18:12] (step=0006950) Train Loss: 5.5792, Train Steps/Sec: 1.12 +[2025-04-24 04:18:34] (step=0006975) Train Loss: 5.5694, Train Steps/Sec: 1.11 +[2025-04-24 04:18:56] (step=0007000) Train Loss: 5.6567, Train Steps/Sec: 1.16 +[2025-04-24 04:19:17] (step=0007025) Train Loss: 5.5618, Train Steps/Sec: 1.17 +[2025-04-24 04:19:39] (step=0007050) Train Loss: 5.5351, Train Steps/Sec: 1.16 +[2025-04-24 04:20:00] (step=0007075) Train Loss: 5.4690, Train Steps/Sec: 1.17 +[2025-04-24 04:20:22] (step=0007100) Train Loss: 5.6560, Train Steps/Sec: 1.16 +[2025-04-24 04:20:43] (step=0007125) Train Loss: 5.6232, Train Steps/Sec: 1.17 +[2025-04-24 04:21:04] (step=0007150) Train Loss: 5.6039, Train Steps/Sec: 1.17 +[2025-04-24 04:21:27] (step=0007175) Train Loss: 5.5603, Train Steps/Sec: 1.12 +[2025-04-24 04:21:51] (step=0007200) Train Loss: 5.5331, Train Steps/Sec: 1.02 +[2025-04-24 04:22:13] (step=0007225) Train Loss: 5.5614, Train Steps/Sec: 1.17 +[2025-04-24 04:22:34] (step=0007250) Train Loss: 5.6110, Train Steps/Sec: 1.17 +[2025-04-24 04:22:55] (step=0007275) Train Loss: 5.5729, Train Steps/Sec: 1.17 +[2025-04-24 04:23:18] (step=0007300) Train Loss: 5.6187, Train Steps/Sec: 1.12 +[2025-04-24 04:23:40] (step=0007325) Train Loss: 5.6047, Train Steps/Sec: 1.13 +[2025-04-24 04:24:01] (step=0007350) Train Loss: 5.5474, Train Steps/Sec: 1.17 +[2025-04-24 04:24:23] (step=0007375) Train Loss: 5.6287, Train Steps/Sec: 1.17 +[2025-04-24 04:24:44] (step=0007400) Train Loss: 5.5149, Train Steps/Sec: 1.16 +[2025-04-24 04:25:06] (step=0007425) Train Loss: 5.6450, Train Steps/Sec: 1.17 +[2025-04-24 04:25:27] (step=0007450) Train Loss: 5.5700, Train Steps/Sec: 1.17 +[2025-04-24 04:25:48] (step=0007475) Train Loss: 5.6698, Train Steps/Sec: 1.17 +[2025-04-24 04:26:10] (step=0007500) Train Loss: 5.6018, Train Steps/Sec: 1.16 +[2025-04-24 04:26:31] (step=0007525) Train Loss: 5.5281, Train Steps/Sec: 1.17 +[2025-04-24 04:26:53] (step=0007550) Train Loss: 5.5381, Train Steps/Sec: 1.17 +[2025-04-24 04:27:14] (step=0007575) Train Loss: 5.6411, Train Steps/Sec: 1.17 +[2025-04-24 04:27:36] (step=0007600) Train Loss: 5.5474, Train Steps/Sec: 1.17 +[2025-04-24 04:27:57] (step=0007625) Train Loss: 5.6712, Train Steps/Sec: 1.17 +[2025-04-24 04:28:18] (step=0007650) Train Loss: 5.5934, Train Steps/Sec: 1.17 +[2025-04-24 04:28:41] (step=0007675) Train Loss: 5.5648, Train Steps/Sec: 1.12 +[2025-04-24 04:29:03] (step=0007700) Train Loss: 5.4692, Train Steps/Sec: 1.11 +[2025-04-24 04:29:25] (step=0007725) Train Loss: 5.6143, Train Steps/Sec: 1.17 +[2025-04-24 04:29:46] (step=0007750) Train Loss: 5.5223, Train Steps/Sec: 1.17 +[2025-04-24 04:30:08] (step=0007775) Train Loss: 5.5483, Train Steps/Sec: 1.17 +[2025-04-24 04:30:29] (step=0007800) Train Loss: 5.6109, Train Steps/Sec: 1.16 +[2025-04-24 04:30:50] (step=0007825) Train Loss: 5.5703, Train Steps/Sec: 1.17 +[2025-04-24 04:31:14] (step=0007850) Train Loss: 5.6036, Train Steps/Sec: 1.04 +[2025-04-24 04:31:37] (step=0007875) Train Loss: 5.5698, Train Steps/Sec: 1.11 +[2025-04-24 04:31:59] (step=0007900) Train Loss: 5.6066, Train Steps/Sec: 1.16 +[2025-04-24 04:32:20] (step=0007925) Train Loss: 5.5746, Train Steps/Sec: 1.16 +[2025-04-24 04:32:42] (step=0007950) Train Loss: 5.6176, Train Steps/Sec: 1.16 +[2025-04-24 04:33:05] (step=0007975) Train Loss: 5.5800, Train Steps/Sec: 1.08 +[2025-04-24 04:33:26] (step=0008000) Train Loss: 5.6072, Train Steps/Sec: 1.16 +[2025-04-24 04:33:26] text_config is None. Initializing the text config with default values (`OPTConfig`). +[2025-04-24 04:38:34] Finish Eval in 8000 steps... +[2025-04-24 04:38:53] Saved checkpoint to checkpoint/Afteri2i_100kckpt_LLAVAT5_empty_Fixed_just_segment_pretrain_stage3_dreambench_recap_400k_t2i_400k_flux_200k_midjourney_150k_recovery_150k_3e4_epoch2_no_rotated/000-GPT-XL/checkpoints/0008000.pt +[2025-04-24 04:38:55] Removed old checkpoint: checkpoint/Afteri2i_100kckpt_LLAVAT5_empty_Fixed_just_segment_pretrain_stage3_dreambench_recap_400k_t2i_400k_flux_200k_midjourney_150k_recovery_150k_3e4_epoch2_no_rotated/000-GPT-XL/checkpoints/0006000.pt +[2025-04-24 04:39:16] (step=0008025) Train Loss: 5.5394, Train Steps/Sec: 0.07 +[2025-04-24 04:39:38] (step=0008050) Train Loss: 5.7179, Train Steps/Sec: 1.17 +[2025-04-24 04:39:59] (step=0008075) Train Loss: 5.6368, Train Steps/Sec: 1.17 +[2025-04-24 04:40:21] (step=0008100) Train Loss: 5.7061, Train Steps/Sec: 1.16 +[2025-04-24 04:40:42] (step=0008125) Train Loss: 5.5919, Train Steps/Sec: 1.17 +[2025-04-24 04:41:03] (step=0008150) Train Loss: 5.5466, Train Steps/Sec: 1.17 +[2025-04-24 04:41:25] (step=0008175) Train Loss: 5.5152, Train Steps/Sec: 1.17 +[2025-04-24 04:41:46] (step=0008200) Train Loss: 5.5640, Train Steps/Sec: 1.17 +[2025-04-24 04:42:08] (step=0008225) Train Loss: 5.5893, Train Steps/Sec: 1.17 +[2025-04-24 04:42:29] (step=0008250) Train Loss: 5.6468, Train Steps/Sec: 1.17 +[2025-04-24 04:42:50] (step=0008275) Train Loss: 5.5346, Train Steps/Sec: 1.17 +[2025-04-24 04:43:12] (step=0008300) Train Loss: 5.6475, Train Steps/Sec: 1.14 +[2025-04-24 04:43:33] (step=0008325) Train Loss: 5.6025, Train Steps/Sec: 1.17 +[2025-04-24 04:43:55] (step=0008350) Train Loss: 5.6005, Train Steps/Sec: 1.17 +[2025-04-24 04:44:17] (step=0008375) Train Loss: 5.5665, Train Steps/Sec: 1.11 +[2025-04-24 04:44:39] (step=0008400) Train Loss: 5.5267, Train Steps/Sec: 1.16 +[2025-04-24 04:45:01] (step=0008425) Train Loss: 5.5358, Train Steps/Sec: 1.11 +[2025-04-24 04:45:23] (step=0008450) Train Loss: 5.6232, Train Steps/Sec: 1.17 +[2025-04-24 04:45:44] (step=0008475) Train Loss: 5.6399, Train Steps/Sec: 1.17 +[2025-04-24 04:46:07] (step=0008500) Train Loss: 5.5162, Train Steps/Sec: 1.07 +[2025-04-24 04:46:30] (step=0008525) Train Loss: 5.6092, Train Steps/Sec: 1.08 +[2025-04-24 04:46:52] (step=0008550) Train Loss: 5.6503, Train Steps/Sec: 1.17 +[2025-04-24 04:47:13] (step=0008575) Train Loss: 5.5699, Train Steps/Sec: 1.17 +[2025-04-24 04:47:35] (step=0008600) Train Loss: 5.5621, Train Steps/Sec: 1.17 +[2025-04-24 04:47:57] (step=0008625) Train Loss: 5.5764, Train Steps/Sec: 1.12 +[2025-04-24 04:48:19] (step=0008650) Train Loss: 5.5777, Train Steps/Sec: 1.13 +[2025-04-24 04:48:40] (step=0008675) Train Loss: 5.5463, Train Steps/Sec: 1.17 +[2025-04-24 04:49:02] (step=0008700) Train Loss: 5.5819, Train Steps/Sec: 1.16 +[2025-04-24 04:49:23] (step=0008725) Train Loss: 5.5966, Train Steps/Sec: 1.17 +[2025-04-24 04:49:45] (step=0008750) Train Loss: 5.6097, Train Steps/Sec: 1.17 +[2025-04-24 04:50:06] (step=0008775) Train Loss: 5.6725, Train Steps/Sec: 1.17 +[2025-04-24 04:50:28] (step=0008800) Train Loss: 5.5910, Train Steps/Sec: 1.16 +[2025-04-24 04:50:49] (step=0008825) Train Loss: 5.5610, Train Steps/Sec: 1.17 +[2025-04-24 04:51:10] (step=0008850) Train Loss: 5.6187, Train Steps/Sec: 1.17 +[2025-04-24 04:51:32] (step=0008875) Train Loss: 5.5719, Train Steps/Sec: 1.17 +[2025-04-24 04:51:53] (step=0008900) Train Loss: 5.6100, Train Steps/Sec: 1.17 +[2025-04-24 04:52:15] (step=0008925) Train Loss: 5.7115, Train Steps/Sec: 1.17 +[2025-04-24 04:52:36] (step=0008950) Train Loss: 5.4613, Train Steps/Sec: 1.17 +[2025-04-24 04:52:57] (step=0008975) Train Loss: 5.5932, Train Steps/Sec: 1.17 +[2025-04-24 04:53:19] (step=0009000) Train Loss: 5.5512, Train Steps/Sec: 1.16 +[2025-04-24 04:53:40] (step=0009025) Train Loss: 5.5649, Train Steps/Sec: 1.17 +[2025-04-24 04:54:02] (step=0009050) Train Loss: 5.4963, Train Steps/Sec: 1.17 +[2025-04-24 04:54:23] (step=0009075) Train Loss: 5.6044, Train Steps/Sec: 1.17 +[2025-04-24 04:54:45] (step=0009100) Train Loss: 5.5861, Train Steps/Sec: 1.11 +[2025-04-24 04:55:07] (step=0009125) Train Loss: 5.5719, Train Steps/Sec: 1.17 +[2025-04-24 04:55:30] (step=0009150) Train Loss: 5.6192, Train Steps/Sec: 1.07 +[2025-04-24 04:55:54] (step=0009175) Train Loss: 5.5947, Train Steps/Sec: 1.05 +[2025-04-24 04:56:15] (step=0009200) Train Loss: 5.6048, Train Steps/Sec: 1.17 +[2025-04-24 04:56:37] (step=0009225) Train Loss: 5.6334, Train Steps/Sec: 1.17 +[2025-04-24 04:56:58] (step=0009250) Train Loss: 5.4839, Train Steps/Sec: 1.17 +[2025-04-24 04:57:21] (step=0009275) Train Loss: 5.6271, Train Steps/Sec: 1.13 +[2025-04-24 04:57:42] (step=0009300) Train Loss: 5.6049, Train Steps/Sec: 1.17 +[2025-04-24 04:58:04] (step=0009325) Train Loss: 5.5851, Train Steps/Sec: 1.13 +[2025-04-24 04:58:26] (step=0009350) Train Loss: 5.5594, Train Steps/Sec: 1.17 +[2025-04-24 04:58:47] (step=0009375) Train Loss: 5.5903, Train Steps/Sec: 1.17 +[2025-04-24 04:59:08] (step=0009400) Train Loss: 5.5579, Train Steps/Sec: 1.17 +[2025-04-24 04:59:30] (step=0009425) Train Loss: 5.5823, Train Steps/Sec: 1.17 +[2025-04-24 04:59:51] (step=0009450) Train Loss: 5.5779, Train Steps/Sec: 1.17 +[2025-04-24 05:00:13] (step=0009475) Train Loss: 5.5650, Train Steps/Sec: 1.17 +[2025-04-24 05:00:34] (step=0009500) Train Loss: 5.6458, Train Steps/Sec: 1.16 +[2025-04-24 05:00:55] (step=0009525) Train Loss: 5.5350, Train Steps/Sec: 1.17 +[2025-04-24 05:01:17] (step=0009550) Train Loss: 5.5697, Train Steps/Sec: 1.17 +[2025-04-24 05:01:38] (step=0009575) Train Loss: 5.5861, Train Steps/Sec: 1.17 +[2025-04-24 05:02:00] (step=0009600) Train Loss: 5.5548, Train Steps/Sec: 1.16 +[2025-04-24 05:02:21] (step=0009625) Train Loss: 5.6191, Train Steps/Sec: 1.17 +[2025-04-24 05:02:42] (step=0009650) Train Loss: 5.5496, Train Steps/Sec: 1.17 +[2025-04-24 05:03:04] (step=0009675) Train Loss: 5.5891, Train Steps/Sec: 1.17 +[2025-04-24 05:03:25] (step=0009700) Train Loss: 5.5376, Train Steps/Sec: 1.16 +[2025-04-24 05:03:46] (step=0009725) Train Loss: 5.5166, Train Steps/Sec: 1.17 +[2025-04-24 05:04:08] (step=0009750) Train Loss: 5.5685, Train Steps/Sec: 1.17 +[2025-04-24 05:04:29] (step=0009775) Train Loss: 5.5803, Train Steps/Sec: 1.17 +[2025-04-24 05:04:52] (step=0009800) Train Loss: 5.5779, Train Steps/Sec: 1.17 +[2025-04-24 05:05:16] (step=0009825) Train Loss: 5.5227, Train Steps/Sec: 1.04 +[2025-04-24 05:05:38] (step=0009850) Train Loss: 5.5784, Train Steps/Sec: 1.12 +[2025-04-24 05:06:01] (step=0009875) Train Loss: 5.5709, Train Steps/Sec: 1.11 +[2025-04-24 05:06:22] (step=0009900) Train Loss: 5.5935, Train Steps/Sec: 1.17 +[2025-04-24 05:06:43] (step=0009925) Train Loss: 5.5775, Train Steps/Sec: 1.17 +[2025-04-24 05:07:06] (step=0009950) Train Loss: 5.5773, Train Steps/Sec: 1.12 +[2025-04-24 05:07:28] (step=0009975) Train Loss: 5.5334, Train Steps/Sec: 1.13 +[2025-04-24 05:07:49] (step=0010000) Train Loss: 5.5647, Train Steps/Sec: 1.17 +[2025-04-24 05:07:49] text_config is None. Initializing the text config with default values (`OPTConfig`). +[2025-04-24 05:12:57] Finish Eval in 10000 steps... +[2025-04-24 05:13:17] Saved checkpoint to checkpoint/Afteri2i_100kckpt_LLAVAT5_empty_Fixed_just_segment_pretrain_stage3_dreambench_recap_400k_t2i_400k_flux_200k_midjourney_150k_recovery_150k_3e4_epoch2_no_rotated/000-GPT-XL/checkpoints/0010000.pt +[2025-04-24 05:13:20] Removed old checkpoint: checkpoint/Afteri2i_100kckpt_LLAVAT5_empty_Fixed_just_segment_pretrain_stage3_dreambench_recap_400k_t2i_400k_flux_200k_midjourney_150k_recovery_150k_3e4_epoch2_no_rotated/000-GPT-XL/checkpoints/0008000.pt +[2025-04-24 05:13:41] (step=0010025) Train Loss: 5.6128, Train Steps/Sec: 0.07 +[2025-04-24 05:14:03] (step=0010050) Train Loss: 5.5361, Train Steps/Sec: 1.16 +[2025-04-24 05:14:24] (step=0010075) Train Loss: 5.5859, Train Steps/Sec: 1.16 +[2025-04-24 05:14:46] (step=0010100) Train Loss: 5.4551, Train Steps/Sec: 1.16 +[2025-04-24 05:15:07] (step=0010125) Train Loss: 5.6075, Train Steps/Sec: 1.17 +[2025-04-24 05:15:29] (step=0010150) Train Loss: 5.4969, Train Steps/Sec: 1.17 +[2025-04-24 05:15:50] (step=0010175) Train Loss: 5.5945, Train Steps/Sec: 1.17 +[2025-04-24 05:16:12] (step=0010200) Train Loss: 5.5507, Train Steps/Sec: 1.16 +[2025-04-24 05:16:34] (step=0010225) Train Loss: 5.6699, Train Steps/Sec: 1.14 +[2025-04-24 05:16:55] (step=0010250) Train Loss: 5.5454, Train Steps/Sec: 1.17 +[2025-04-24 05:17:16] (step=0010275) Train Loss: 5.5839, Train Steps/Sec: 1.17 +[2025-04-24 05:17:38] (step=0010300) Train Loss: 5.5582, Train Steps/Sec: 1.16 +[2025-04-24 05:17:59] (step=0010325) Train Loss: 5.5427, Train Steps/Sec: 1.16 +[2025-04-24 05:18:21] (step=0010350) Train Loss: 5.4767, Train Steps/Sec: 1.17 +[2025-04-24 05:18:42] (step=0010375) Train Loss: 5.5055, Train Steps/Sec: 1.17 +[2025-04-24 05:19:04] (step=0010400) Train Loss: 5.5088, Train Steps/Sec: 1.16 +[2025-04-24 05:19:25] (step=0010425) Train Loss: 5.5144, Train Steps/Sec: 1.17 +[2025-04-24 05:19:46] (step=0010450) Train Loss: 5.5395, Train Steps/Sec: 1.17 +[2025-04-24 05:20:10] (step=0010475) Train Loss: 5.5179, Train Steps/Sec: 1.07 +[2025-04-24 05:20:33] (step=0010500) Train Loss: 5.5821, Train Steps/Sec: 1.07 +[2025-04-24 05:20:54] (step=0010525) Train Loss: 5.5002, Train Steps/Sec: 1.17 +[2025-04-24 05:21:17] (step=0010550) Train Loss: 5.5111, Train Steps/Sec: 1.12 +[2025-04-24 05:21:38] (step=0010575) Train Loss: 5.5364, Train Steps/Sec: 1.17 +[2025-04-24 05:22:02] (step=0010600) Train Loss: 5.5712, Train Steps/Sec: 1.06 +[2025-04-24 05:22:23] (step=0010625) Train Loss: 5.5174, Train Steps/Sec: 1.17 +[2025-04-24 05:22:46] (step=0010650) Train Loss: 5.5729, Train Steps/Sec: 1.12 +[2025-04-24 05:23:07] (step=0010675) Train Loss: 5.4921, Train Steps/Sec: 1.17 +[2025-04-24 05:23:28] (step=0010700) Train Loss: 5.5949, Train Steps/Sec: 1.16 +[2025-04-24 05:23:50] (step=0010725) Train Loss: 5.4964, Train Steps/Sec: 1.17 +[2025-04-24 05:24:11] (step=0010750) Train Loss: 5.6323, Train Steps/Sec: 1.17 +[2025-04-24 05:24:33] (step=0010775) Train Loss: 5.5652, Train Steps/Sec: 1.17 +[2025-04-24 05:24:54] (step=0010800) Train Loss: 5.4863, Train Steps/Sec: 1.16 +[2025-04-24 05:25:15] (step=0010825) Train Loss: 5.4864, Train Steps/Sec: 1.17 +[2025-04-24 05:25:37] (step=0010850) Train Loss: 5.6307, Train Steps/Sec: 1.17 +[2025-04-24 05:25:58] (step=0010875) Train Loss: 5.6250, Train Steps/Sec: 1.17 +[2025-04-24 05:26:20] (step=0010900) Train Loss: 5.6807, Train Steps/Sec: 1.16 +[2025-04-24 05:26:41] (step=0010925) Train Loss: 5.5064, Train Steps/Sec: 1.17 +[2025-04-24 05:27:02] (step=0010950) Train Loss: 5.4973, Train Steps/Sec: 1.17 +[2025-04-24 05:27:24] (step=0010975) Train Loss: 5.6616, Train Steps/Sec: 1.17 +[2025-04-24 05:27:45] (step=0011000) Train Loss: 5.5108, Train Steps/Sec: 1.16 +[2025-04-24 05:28:07] (step=0011025) Train Loss: 5.5245, Train Steps/Sec: 1.17 +[2025-04-24 05:28:28] (step=0011050) Train Loss: 5.5323, Train Steps/Sec: 1.17 +[2025-04-24 05:28:50] (step=0011075) Train Loss: 5.5379, Train Steps/Sec: 1.17 +[2025-04-24 05:29:11] (step=0011100) Train Loss: 5.5367, Train Steps/Sec: 1.16 +[2025-04-24 05:29:33] (step=0011125) Train Loss: 5.5582, Train Steps/Sec: 1.12 +[2025-04-24 05:29:56] (step=0011150) Train Loss: 5.5823, Train Steps/Sec: 1.08 +[2025-04-24 05:30:19] (step=0011175) Train Loss: 5.5513, Train Steps/Sec: 1.12 +[2025-04-24 05:30:40] (step=0011200) Train Loss: 5.4976, Train Steps/Sec: 1.17 +[2025-04-24 05:31:02] (step=0011225) Train Loss: 5.5454, Train Steps/Sec: 1.17 +[2025-04-24 05:31:24] (step=0011250) Train Loss: 5.5721, Train Steps/Sec: 1.12 +[2025-04-24 05:31:46] (step=0011275) Train Loss: 5.5922, Train Steps/Sec: 1.11 +[2025-04-24 05:32:08] (step=0011300) Train Loss: 5.6184, Train Steps/Sec: 1.17 +[2025-04-24 05:32:32] (step=0011325) Train Loss: 5.5439, Train Steps/Sec: 1.05 +[2025-04-24 05:32:53] (step=0011350) Train Loss: 5.4864, Train Steps/Sec: 1.17 +[2025-04-24 05:33:14] (step=0011375) Train Loss: 5.5239, Train Steps/Sec: 1.17 +[2025-04-24 05:33:36] (step=0011400) Train Loss: 5.5655, Train Steps/Sec: 1.16 +[2025-04-24 05:33:57] (step=0011425) Train Loss: 5.5337, Train Steps/Sec: 1.17 +[2025-04-24 05:34:19] (step=0011450) Train Loss: 5.4405, Train Steps/Sec: 1.17 +[2025-04-24 05:34:40] (step=0011475) Train Loss: 5.5468, Train Steps/Sec: 1.17 +[2025-04-24 05:35:01] (step=0011500) Train Loss: 5.5810, Train Steps/Sec: 1.16 +[2025-04-24 05:35:23] (step=0011525) Train Loss: 5.4924, Train Steps/Sec: 1.17 +[2025-04-24 05:35:44] (step=0011550) Train Loss: 5.5264, Train Steps/Sec: 1.17 +[2025-04-24 05:36:06] (step=0011575) Train Loss: 5.5708, Train Steps/Sec: 1.17 +[2025-04-24 05:36:27] (step=0011600) Train Loss: 5.5370, Train Steps/Sec: 1.17 +[2025-04-24 05:36:48] (step=0011625) Train Loss: 5.5699, Train Steps/Sec: 1.17 +[2025-04-24 05:37:10] (step=0011650) Train Loss: 5.5066, Train Steps/Sec: 1.17 +[2025-04-24 05:37:31] (step=0011675) Train Loss: 5.5464, Train Steps/Sec: 1.17 +[2025-04-24 05:37:53] (step=0011700) Train Loss: 5.4979, Train Steps/Sec: 1.16 +[2025-04-24 05:38:14] (step=0011725) Train Loss: 5.5132, Train Steps/Sec: 1.17 +[2025-04-24 05:38:35] (step=0011750) Train Loss: 5.5854, Train Steps/Sec: 1.17 +[2025-04-24 05:38:58] (step=0011775) Train Loss: 5.6056, Train Steps/Sec: 1.12 +[2025-04-24 05:39:20] (step=0011800) Train Loss: 5.5701, Train Steps/Sec: 1.12 +[2025-04-24 05:39:43] (step=0011825) Train Loss: 5.5820, Train Steps/Sec: 1.08 +[2025-04-24 05:40:04] (step=0011850) Train Loss: 5.5347, Train Steps/Sec: 1.17 +[2025-04-24 05:40:26] (step=0011875) Train Loss: 5.5597, Train Steps/Sec: 1.17 +[2025-04-24 05:40:47] (step=0011900) Train Loss: 5.6397, Train Steps/Sec: 1.17 +[2025-04-24 05:41:09] (step=0011925) Train Loss: 5.4462, Train Steps/Sec: 1.12 +[2025-04-24 05:41:31] (step=0011950) Train Loss: 5.5621, Train Steps/Sec: 1.17 +[2025-04-24 05:41:53] (step=0011975) Train Loss: 5.5928, Train Steps/Sec: 1.12 +[2025-04-24 05:42:16] (step=0012000) Train Loss: 5.5831, Train Steps/Sec: 1.11 +[2025-04-24 05:42:16] text_config is None. Initializing the text config with default values (`OPTConfig`). +[2025-04-24 05:47:29] Finish Eval in 12000 steps... +[2025-04-24 05:47:48] Saved checkpoint to checkpoint/Afteri2i_100kckpt_LLAVAT5_empty_Fixed_just_segment_pretrain_stage3_dreambench_recap_400k_t2i_400k_flux_200k_midjourney_150k_recovery_150k_3e4_epoch2_no_rotated/000-GPT-XL/checkpoints/0012000.pt +[2025-04-24 05:47:50] Removed old checkpoint: checkpoint/Afteri2i_100kckpt_LLAVAT5_empty_Fixed_just_segment_pretrain_stage3_dreambench_recap_400k_t2i_400k_flux_200k_midjourney_150k_recovery_150k_3e4_epoch2_no_rotated/000-GPT-XL/checkpoints/0010000.pt +[2025-04-24 05:48:11] (step=0012025) Train Loss: 5.5987, Train Steps/Sec: 0.07 +[2025-04-24 05:48:34] (step=0012050) Train Loss: 5.5446, Train Steps/Sec: 1.11 +[2025-04-24 05:48:55] (step=0012075) Train Loss: 5.5930, Train Steps/Sec: 1.17 +[2025-04-24 05:49:16] (step=0012100) Train Loss: 5.5341, Train Steps/Sec: 1.16 +[2025-04-24 05:49:38] (step=0012125) Train Loss: 5.5410, Train Steps/Sec: 1.17 +[2025-04-24 05:49:59] (step=0012150) Train Loss: 5.5226, Train Steps/Sec: 1.16 +[2025-04-24 05:50:21] (step=0012175) Train Loss: 5.5763, Train Steps/Sec: 1.17 +[2025-04-24 05:50:42] (step=0012200) Train Loss: 5.5687, Train Steps/Sec: 1.16 +[2025-04-24 05:51:04] (step=0012225) Train Loss: 5.6387, Train Steps/Sec: 1.17 +[2025-04-24 05:51:25] (step=0012250) Train Loss: 5.5751, Train Steps/Sec: 1.17 +[2025-04-24 05:51:46] (step=0012275) Train Loss: 5.4511, Train Steps/Sec: 1.17 +[2025-04-24 05:52:08] (step=0012300) Train Loss: 5.4989, Train Steps/Sec: 1.17 +[2025-04-24 05:52:29] (step=0012325) Train Loss: 5.5317, Train Steps/Sec: 1.17 +[2025-04-24 05:52:50] (step=0012350) Train Loss: 5.5611, Train Steps/Sec: 1.17 +[2025-04-24 05:53:12] (step=0012375) Train Loss: 5.6331, Train Steps/Sec: 1.17 +[2025-04-24 05:53:33] (step=0012400) Train Loss: 5.5454, Train Steps/Sec: 1.17 +[2025-04-24 05:53:54] (step=0012425) Train Loss: 5.5489, Train Steps/Sec: 1.17 +[2025-04-24 05:54:17] (step=0012450) Train Loss: 5.5369, Train Steps/Sec: 1.12 +[2025-04-24 05:54:41] (step=0012475) Train Loss: 5.5585, Train Steps/Sec: 1.04 +[2025-04-24 05:55:02] (step=0012500) Train Loss: 5.5377, Train Steps/Sec: 1.16 +[2025-04-24 05:55:24] (step=0012525) Train Loss: 5.5340, Train Steps/Sec: 1.17 +[2025-04-24 05:55:45] (step=0012550) Train Loss: 5.4382, Train Steps/Sec: 1.17 +[2025-04-24 05:56:07] (step=0012575) Train Loss: 5.5709, Train Steps/Sec: 1.13 +[2025-04-24 05:56:28] (step=0012600) Train Loss: 5.5039, Train Steps/Sec: 1.17 +[2025-04-24 05:56:50] (step=0012625) Train Loss: 5.5009, Train Steps/Sec: 1.17 +[2025-04-24 05:57:11] (step=0012650) Train Loss: 5.5406, Train Steps/Sec: 1.17 +[2025-04-24 05:57:34] (step=0012675) Train Loss: 5.4848, Train Steps/Sec: 1.12 +[2025-04-24 05:57:55] (step=0012700) Train Loss: 5.5173, Train Steps/Sec: 1.17 +[2025-04-24 05:58:17] (step=0012725) Train Loss: 5.4820, Train Steps/Sec: 1.12 +[2025-04-24 05:58:39] (step=0012750) Train Loss: 5.5819, Train Steps/Sec: 1.17 +[2025-04-24 05:59:01] (step=0012775) Train Loss: 5.5643, Train Steps/Sec: 1.11 +[2025-04-24 05:59:23] (step=0012800) Train Loss: 5.5847, Train Steps/Sec: 1.16 +[2025-04-24 05:59:44] (step=0012825) Train Loss: 5.5839, Train Steps/Sec: 1.17 +[2025-04-24 06:00:05] (step=0012850) Train Loss: 5.5342, Train Steps/Sec: 1.17 +[2025-04-24 06:00:27] (step=0012875) Train Loss: 5.5779, Train Steps/Sec: 1.17 +[2025-04-24 06:00:48] (step=0012900) Train Loss: 5.5836, Train Steps/Sec: 1.16 +[2025-04-24 06:01:10] (step=0012925) Train Loss: 5.6191, Train Steps/Sec: 1.17 +[2025-04-24 06:01:31] (step=0012950) Train Loss: 5.4604, Train Steps/Sec: 1.17 +[2025-04-24 06:01:52] (step=0012975) Train Loss: 5.4950, Train Steps/Sec: 1.17 +[2025-04-24 06:02:14] (step=0013000) Train Loss: 5.4169, Train Steps/Sec: 1.17 +[2025-04-24 06:02:35] (step=0013025) Train Loss: 5.5978, Train Steps/Sec: 1.17 +[2025-04-24 06:02:56] (step=0013050) Train Loss: 5.4675, Train Steps/Sec: 1.17 +[2025-04-24 06:03:18] (step=0013075) Train Loss: 5.5210, Train Steps/Sec: 1.17 +[2025-04-24 06:03:40] (step=0013100) Train Loss: 5.5366, Train Steps/Sec: 1.12 +[2025-04-24 06:04:03] (step=0013125) Train Loss: 5.4783, Train Steps/Sec: 1.09 +[2025-04-24 06:04:25] (step=0013150) Train Loss: 5.4682, Train Steps/Sec: 1.12 +[2025-04-24 06:04:47] (step=0013175) Train Loss: 5.5113, Train Steps/Sec: 1.17 +[2025-04-24 06:05:08] (step=0013200) Train Loss: 5.5449, Train Steps/Sec: 1.17 +[2025-04-24 06:05:30] (step=0013225) Train Loss: 5.5020, Train Steps/Sec: 1.13 +[2025-04-24 06:05:52] (step=0013250) Train Loss: 5.5087, Train Steps/Sec: 1.17 +[2025-04-24 06:06:13] (step=0013275) Train Loss: 5.5594, Train Steps/Sec: 1.17 +[2025-04-24 06:06:34] (step=0013300) Train Loss: 5.5388, Train Steps/Sec: 1.17 +[2025-04-24 06:06:57] (step=0013325) Train Loss: 5.6025, Train Steps/Sec: 1.12 +[2025-04-24 06:07:18] (step=0013350) Train Loss: 5.5242, Train Steps/Sec: 1.17 +[2025-04-24 06:07:40] (step=0013375) Train Loss: 5.6302, Train Steps/Sec: 1.17 +[2025-04-24 06:08:01] (step=0013400) Train Loss: 5.5842, Train Steps/Sec: 1.17 +[2025-04-24 06:08:22] (step=0013425) Train Loss: 5.5440, Train Steps/Sec: 1.17 +[2025-04-24 06:08:45] (step=0013450) Train Loss: 5.4954, Train Steps/Sec: 1.12 +[2025-04-24 06:09:06] (step=0013475) Train Loss: 5.4986, Train Steps/Sec: 1.17 +[2025-04-24 06:09:27] (step=0013500) Train Loss: 5.5067, Train Steps/Sec: 1.17 +[2025-04-24 06:09:50] (step=0013525) Train Loss: 5.4794, Train Steps/Sec: 1.12 +[2025-04-24 06:10:11] (step=0013550) Train Loss: 5.5198, Train Steps/Sec: 1.18 +[2025-04-24 06:10:32] (step=0013575) Train Loss: 5.4643, Train Steps/Sec: 1.17 +[2025-04-24 06:10:54] (step=0013600) Train Loss: 5.5655, Train Steps/Sec: 1.17 +[2025-04-24 06:11:15] (step=0013625) Train Loss: 5.5171, Train Steps/Sec: 1.17 +[2025-04-24 06:11:37] (step=0013650) Train Loss: 5.5217, Train Steps/Sec: 1.17 +[2025-04-24 06:11:58] (step=0013675) Train Loss: 5.4415, Train Steps/Sec: 1.17 +[2025-04-24 06:12:19] (step=0013700) Train Loss: 5.5033, Train Steps/Sec: 1.17 +[2025-04-24 06:12:41] (step=0013725) Train Loss: 5.5418, Train Steps/Sec: 1.17 +[2025-04-24 06:13:03] (step=0013750) Train Loss: 5.4834, Train Steps/Sec: 1.12 +[2025-04-24 06:13:25] (step=0013775) Train Loss: 5.5646, Train Steps/Sec: 1.13 +[2025-04-24 06:13:48] (step=0013800) Train Loss: 5.6057, Train Steps/Sec: 1.08 +[2025-04-24 06:14:10] (step=0013825) Train Loss: 5.4401, Train Steps/Sec: 1.17 +[2025-04-24 06:14:31] (step=0013850) Train Loss: 5.5813, Train Steps/Sec: 1.17 +[2025-04-24 06:14:52] (step=0013875) Train Loss: 5.5548, Train Steps/Sec: 1.17 +[2025-04-24 06:15:15] (step=0013900) Train Loss: 5.5498, Train Steps/Sec: 1.12 +[2025-04-24 06:15:36] (step=0013925) Train Loss: 5.4976, Train Steps/Sec: 1.17 +[2025-04-24 06:15:57] (step=0013950) Train Loss: 5.5556, Train Steps/Sec: 1.17 +[2025-04-24 06:16:20] (step=0013975) Train Loss: 5.5339, Train Steps/Sec: 1.12 +[2025-04-24 06:16:41] (step=0014000) Train Loss: 5.4978, Train Steps/Sec: 1.16 +[2025-04-24 06:16:41] text_config is None. Initializing the text config with default values (`OPTConfig`). +[2025-04-24 06:21:53] Finish Eval in 14000 steps... +[2025-04-24 06:22:13] Saved checkpoint to checkpoint/Afteri2i_100kckpt_LLAVAT5_empty_Fixed_just_segment_pretrain_stage3_dreambench_recap_400k_t2i_400k_flux_200k_midjourney_150k_recovery_150k_3e4_epoch2_no_rotated/000-GPT-XL/checkpoints/0014000.pt +[2025-04-24 06:22:15] Removed old checkpoint: checkpoint/Afteri2i_100kckpt_LLAVAT5_empty_Fixed_just_segment_pretrain_stage3_dreambench_recap_400k_t2i_400k_flux_200k_midjourney_150k_recovery_150k_3e4_epoch2_no_rotated/000-GPT-XL/checkpoints/0012000.pt +[2025-04-24 06:22:36] (step=0014025) Train Loss: 5.4919, Train Steps/Sec: 0.07 +[2025-04-24 06:22:57] (step=0014050) Train Loss: 5.5194, Train Steps/Sec: 1.17 +[2025-04-24 06:23:19] (step=0014075) Train Loss: 5.5446, Train Steps/Sec: 1.16 +[2025-04-24 06:23:41] (step=0014100) Train Loss: 5.5655, Train Steps/Sec: 1.17 +[2025-04-24 06:24:02] (step=0014125) Train Loss: 5.5254, Train Steps/Sec: 1.17 +[2025-04-24 06:24:23] (step=0014150) Train Loss: 5.5130, Train Steps/Sec: 1.17 +[2025-04-24 06:24:46] (step=0014175) Train Loss: 5.5385, Train Steps/Sec: 1.12 +[2025-04-24 06:25:07] (step=0014200) Train Loss: 5.4828, Train Steps/Sec: 1.17 +[2025-04-24 06:25:28] (step=0014225) Train Loss: 5.5338, Train Steps/Sec: 1.17 +[2025-04-24 06:25:51] (step=0014250) Train Loss: 5.5779, Train Steps/Sec: 1.11 +[2025-04-24 06:26:12] (step=0014275) Train Loss: 5.4490, Train Steps/Sec: 1.17 +[2025-04-24 06:26:34] (step=0014300) Train Loss: 5.4498, Train Steps/Sec: 1.17 +[2025-04-24 06:26:55] (step=0014325) Train Loss: 5.4488, Train Steps/Sec: 1.17 +[2025-04-24 06:27:16] (step=0014350) Train Loss: 5.3821, Train Steps/Sec: 1.17 +[2025-04-24 06:27:38] (step=0014375) Train Loss: 5.4413, Train Steps/Sec: 1.17 +[2025-04-24 06:27:59] (step=0014400) Train Loss: 5.5049, Train Steps/Sec: 1.17 +[2025-04-24 06:28:21] (step=0014425) Train Loss: 5.5118, Train Steps/Sec: 1.12 +[2025-04-24 06:28:44] (step=0014450) Train Loss: 5.4822, Train Steps/Sec: 1.08 +[2025-04-24 06:29:07] (step=0014475) Train Loss: 5.5181, Train Steps/Sec: 1.12 +[2025-04-24 06:29:28] (step=0014500) Train Loss: 5.5869, Train Steps/Sec: 1.17 +[2025-04-24 06:29:49] (step=0014525) Train Loss: 5.5846, Train Steps/Sec: 1.17 +[2025-04-24 06:30:12] (step=0014550) Train Loss: 5.4971, Train Steps/Sec: 1.13 +[2025-04-24 06:30:33] (step=0014575) Train Loss: 5.4679, Train Steps/Sec: 1.17 +[2025-04-24 06:30:55] (step=0014600) Train Loss: 5.4909, Train Steps/Sec: 1.17 +[2025-04-24 06:31:16] (step=0014625) Train Loss: 5.4876, Train Steps/Sec: 1.17 +[2025-04-24 06:31:38] (step=0014650) Train Loss: 5.4965, Train Steps/Sec: 1.12 +[2025-04-24 06:32:00] (step=0014675) Train Loss: 5.5523, Train Steps/Sec: 1.17 +[2025-04-24 06:32:21] (step=0014700) Train Loss: 5.4943, Train Steps/Sec: 1.17 +[2025-04-24 06:32:42] (step=0014725) Train Loss: 5.5455, Train Steps/Sec: 1.17 +[2025-04-24 06:33:04] (step=0014750) Train Loss: 5.5248, Train Steps/Sec: 1.17 +[2025-04-24 06:33:25] (step=0014775) Train Loss: 5.4871, Train Steps/Sec: 1.17 +[2025-04-24 06:33:46] (step=0014800) Train Loss: 5.5036, Train Steps/Sec: 1.17 +[2025-04-24 06:34:08] (step=0014825) Train Loss: 5.5412, Train Steps/Sec: 1.17 +[2025-04-24 06:34:29] (step=0014850) Train Loss: 5.5355, Train Steps/Sec: 1.17 +[2025-04-24 06:34:50] (step=0014875) Train Loss: 5.5076, Train Steps/Sec: 1.17 +[2025-04-24 06:35:13] (step=0014900) Train Loss: 5.5077, Train Steps/Sec: 1.11 +[2025-04-24 06:35:34] (step=0014925) Train Loss: 5.5678, Train Steps/Sec: 1.17 +[2025-04-24 06:35:56] (step=0014950) Train Loss: 5.5746, Train Steps/Sec: 1.17 +[2025-04-24 06:36:18] (step=0014975) Train Loss: 5.5748, Train Steps/Sec: 1.11 +[2025-04-24 06:36:40] (step=0015000) Train Loss: 5.4713, Train Steps/Sec: 1.17 +[2025-04-24 06:37:01] (step=0015025) Train Loss: 5.5223, Train Steps/Sec: 1.17 +[2025-04-24 06:37:22] (step=0015050) Train Loss: 5.6078, Train Steps/Sec: 1.17 +[2025-04-24 06:37:45] (step=0015075) Train Loss: 5.5112, Train Steps/Sec: 1.13 +[2025-04-24 06:38:07] (step=0015100) Train Loss: 5.4279, Train Steps/Sec: 1.13 +[2025-04-24 06:38:30] (step=0015125) Train Loss: 5.4736, Train Steps/Sec: 1.08 +[2025-04-24 06:38:51] (step=0015150) Train Loss: 5.5534, Train Steps/Sec: 1.17 +[2025-04-24 06:39:13] (step=0015175) Train Loss: 5.5289, Train Steps/Sec: 1.17 +[2025-04-24 06:39:35] (step=0015200) Train Loss: 5.5192, Train Steps/Sec: 1.12 +[2025-04-24 06:39:56] (step=0015225) Train Loss: 5.5116, Train Steps/Sec: 1.17 +[2025-04-24 06:40:18] (step=0015250) Train Loss: 5.4879, Train Steps/Sec: 1.17 +[2025-04-24 06:40:39] (step=0015275) Train Loss: 5.6239, Train Steps/Sec: 1.17 +[2025-04-24 06:41:00] (step=0015300) Train Loss: 5.4383, Train Steps/Sec: 1.16 +[2025-04-24 06:41:23] (step=0015325) Train Loss: 5.4595, Train Steps/Sec: 1.12 +[2025-04-24 06:41:44] (step=0015350) Train Loss: 5.5223, Train Steps/Sec: 1.17 +[2025-04-24 06:42:05] (step=0015375) Train Loss: 5.5820, Train Steps/Sec: 1.17 +[2025-04-24 06:42:27] (step=0015400) Train Loss: 5.5653, Train Steps/Sec: 1.17 +[2025-04-24 06:42:48] (step=0015425) Train Loss: 5.4641, Train Steps/Sec: 1.17 +[2025-04-24 06:43:10] (step=0015450) Train Loss: 5.5484, Train Steps/Sec: 1.17 +[2025-04-24 06:43:31] (step=0015475) Train Loss: 5.5013, Train Steps/Sec: 1.17 +[2025-04-24 06:43:52] (step=0015500) Train Loss: 5.5357, Train Steps/Sec: 1.17 +[2025-04-24 06:44:14] (step=0015525) Train Loss: 5.6173, Train Steps/Sec: 1.17 +[2025-04-24 06:44:35] (step=0015550) Train Loss: 5.5293, Train Steps/Sec: 1.17 +[2025-04-24 06:44:57] (step=0015575) Train Loss: 5.5117, Train Steps/Sec: 1.17 +[2025-04-24 06:45:19] (step=0015600) Train Loss: 5.4896, Train Steps/Sec: 1.11 +[2025-04-24 06:45:40] (step=0015625) Train Loss: 5.5461, Train Steps/Sec: 1.17 +[2025-04-24 06:46:02] (step=0015650) Train Loss: 5.5639, Train Steps/Sec: 1.17 +[2025-04-24 06:46:23] (step=0015675) Train Loss: 5.4943, Train Steps/Sec: 1.17 +[2025-04-24 06:46:46] (step=0015700) Train Loss: 5.5440, Train Steps/Sec: 1.11 +[2025-04-24 06:47:08] (step=0015725) Train Loss: 5.4317, Train Steps/Sec: 1.12 +[2025-04-24 06:47:30] (step=0015750) Train Loss: 5.4574, Train Steps/Sec: 1.13 +[2025-04-24 06:47:53] (step=0015775) Train Loss: 5.5298, Train Steps/Sec: 1.08 +[2025-04-24 06:48:15] (step=0015800) Train Loss: 5.5907, Train Steps/Sec: 1.16 +[2025-04-24 06:48:36] (step=0015825) Train Loss: 5.5807, Train Steps/Sec: 1.17 +[2025-04-24 06:48:58] (step=0015850) Train Loss: 5.4671, Train Steps/Sec: 1.17 +[2025-04-24 06:49:20] (step=0015875) Train Loss: 5.5392, Train Steps/Sec: 1.13 +[2025-04-24 06:49:41] (step=0015900) Train Loss: 5.4741, Train Steps/Sec: 1.16 +[2025-04-24 06:50:03] (step=0015925) Train Loss: 5.5490, Train Steps/Sec: 1.17 +[2025-04-24 06:50:24] (step=0015950) Train Loss: 5.5225, Train Steps/Sec: 1.17 +[2025-04-24 06:50:46] (step=0015975) Train Loss: 5.5246, Train Steps/Sec: 1.12 +[2025-04-24 06:51:08] (step=0016000) Train Loss: 5.5019, Train Steps/Sec: 1.17 +[2025-04-24 06:51:08] text_config is None. Initializing the text config with default values (`OPTConfig`). +[2025-04-24 06:56:27] Finish Eval in 16000 steps... +[2025-04-24 06:56:47] Saved checkpoint to checkpoint/Afteri2i_100kckpt_LLAVAT5_empty_Fixed_just_segment_pretrain_stage3_dreambench_recap_400k_t2i_400k_flux_200k_midjourney_150k_recovery_150k_3e4_epoch2_no_rotated/000-GPT-XL/checkpoints/0016000.pt +[2025-04-24 06:56:48] Removed old checkpoint: checkpoint/Afteri2i_100kckpt_LLAVAT5_empty_Fixed_just_segment_pretrain_stage3_dreambench_recap_400k_t2i_400k_flux_200k_midjourney_150k_recovery_150k_3e4_epoch2_no_rotated/000-GPT-XL/checkpoints/0014000.pt +[2025-04-24 06:57:10] (step=0016025) Train Loss: 5.5051, Train Steps/Sec: 0.07 +[2025-04-24 06:57:31] (step=0016050) Train Loss: 5.4464, Train Steps/Sec: 1.17 +[2025-04-24 06:57:53] (step=0016075) Train Loss: 5.4626, Train Steps/Sec: 1.17 +[2025-04-24 06:58:14] (step=0016100) Train Loss: 5.5685, Train Steps/Sec: 1.16 +[2025-04-24 06:58:35] (step=0016125) Train Loss: 5.5126, Train Steps/Sec: 1.17 +[2025-04-24 06:58:57] (step=0016150) Train Loss: 5.4935, Train Steps/Sec: 1.17 +[2025-04-24 06:59:18] (step=0016175) Train Loss: 5.3838, Train Steps/Sec: 1.17 +[2025-04-24 06:59:39] (step=0016200) Train Loss: 5.5826, Train Steps/Sec: 1.17 +[2025-04-24 07:00:01] (step=0016225) Train Loss: 5.4875, Train Steps/Sec: 1.17 +[2025-04-24 07:00:22] (step=0016250) Train Loss: 5.4966, Train Steps/Sec: 1.17 +[2025-04-24 07:00:44] (step=0016275) Train Loss: 5.4893, Train Steps/Sec: 1.17 +[2025-04-24 07:01:05] (step=0016300) Train Loss: 5.4548, Train Steps/Sec: 1.17 +[2025-04-24 07:01:27] (step=0016325) Train Loss: 5.4922, Train Steps/Sec: 1.12 +[2025-04-24 07:01:49] (step=0016350) Train Loss: 5.4465, Train Steps/Sec: 1.17 +[2025-04-24 07:02:10] (step=0016375) Train Loss: 5.5063, Train Steps/Sec: 1.17 +[2025-04-24 07:02:32] (step=0016400) Train Loss: 5.5979, Train Steps/Sec: 1.12 +[2025-04-24 07:02:56] (step=0016425) Train Loss: 5.4638, Train Steps/Sec: 1.04 +[2025-04-24 07:03:19] (step=0016450) Train Loss: 5.5447, Train Steps/Sec: 1.12 +[2025-04-24 07:03:40] (step=0016475) Train Loss: 5.4832, Train Steps/Sec: 1.17 +[2025-04-24 07:04:01] (step=0016500) Train Loss: 5.5053, Train Steps/Sec: 1.17 +[2025-04-24 07:04:24] (step=0016525) Train Loss: 5.4677, Train Steps/Sec: 1.13 +[2025-04-24 07:04:45] (step=0016550) Train Loss: 5.5246, Train Steps/Sec: 1.17 +[2025-04-24 07:05:06] (step=0016575) Train Loss: 5.4827, Train Steps/Sec: 1.17 +[2025-04-24 07:05:28] (step=0016600) Train Loss: 5.5285, Train Steps/Sec: 1.17 +[2025-04-24 07:05:49] (step=0016625) Train Loss: 5.4629, Train Steps/Sec: 1.17 +[2025-04-24 07:06:10] (step=0016650) Train Loss: 5.5421, Train Steps/Sec: 1.17 +[2025-04-24 07:06:33] (step=0016675) Train Loss: 5.4885, Train Steps/Sec: 1.12 +[2025-04-24 07:06:54] (step=0016700) Train Loss: 5.5197, Train Steps/Sec: 1.16 +[2025-04-24 07:07:15] (step=0016725) Train Loss: 5.4876, Train Steps/Sec: 1.17 +[2025-04-24 07:07:37] (step=0016750) Train Loss: 5.4569, Train Steps/Sec: 1.17 +[2025-04-24 07:07:58] (step=0016775) Train Loss: 5.4885, Train Steps/Sec: 1.17 +[2025-04-24 07:08:19] (step=0016800) Train Loss: 5.5632, Train Steps/Sec: 1.17 +[2025-04-24 07:08:41] (step=0016825) Train Loss: 5.5166, Train Steps/Sec: 1.17 +[2025-04-24 07:09:02] (step=0016850) Train Loss: 5.4503, Train Steps/Sec: 1.17 +[2025-04-24 07:09:24] (step=0016875) Train Loss: 5.4826, Train Steps/Sec: 1.17 +[2025-04-24 07:09:45] (step=0016900) Train Loss: 5.6190, Train Steps/Sec: 1.16 +[2025-04-24 07:10:06] (step=0016925) Train Loss: 5.5137, Train Steps/Sec: 1.17 +[2025-04-24 07:10:28] (step=0016950) Train Loss: 5.4950, Train Steps/Sec: 1.17 +[2025-04-24 07:10:49] (step=0016975) Train Loss: 5.5063, Train Steps/Sec: 1.17 +[2025-04-24 07:11:11] (step=0017000) Train Loss: 5.5108, Train Steps/Sec: 1.16 +[2025-04-24 07:11:32] (step=0017025) Train Loss: 5.4433, Train Steps/Sec: 1.17 +[2025-04-24 07:11:55] (step=0017050) Train Loss: 5.5164, Train Steps/Sec: 1.07 +[2025-04-24 07:12:17] (step=0017075) Train Loss: 5.5241, Train Steps/Sec: 1.13 +[2025-04-24 07:12:41] (step=0017100) Train Loss: 5.5162, Train Steps/Sec: 1.08 +[2025-04-24 07:13:02] (step=0017125) Train Loss: 5.4866, Train Steps/Sec: 1.17 +[2025-04-24 07:13:24] (step=0017150) Train Loss: 5.6039, Train Steps/Sec: 1.11 +[2025-04-24 07:13:46] (step=0017175) Train Loss: 5.4777, Train Steps/Sec: 1.17 +[2025-04-24 07:14:08] (step=0017200) Train Loss: 5.4156, Train Steps/Sec: 1.12 +[2025-04-24 07:14:29] (step=0017225) Train Loss: 5.5043, Train Steps/Sec: 1.17 +[2025-04-24 07:14:51] (step=0017250) Train Loss: 5.5061, Train Steps/Sec: 1.17 +[2025-04-24 07:15:12] (step=0017275) Train Loss: 5.5259, Train Steps/Sec: 1.17 +[2025-04-24 07:15:34] (step=0017300) Train Loss: 5.5143, Train Steps/Sec: 1.17 +[2025-04-24 07:15:56] (step=0017325) Train Loss: 5.5173, Train Steps/Sec: 1.12 +[2025-04-24 07:16:17] (step=0017350) Train Loss: 5.4510, Train Steps/Sec: 1.17 +[2025-04-24 07:16:39] (step=0017375) Train Loss: 5.5847, Train Steps/Sec: 1.17 +[2025-04-24 07:17:00] (step=0017400) Train Loss: 5.4573, Train Steps/Sec: 1.17 +[2025-04-24 07:17:21] (step=0017425) Train Loss: 5.4729, Train Steps/Sec: 1.17 +[2025-04-24 07:17:43] (step=0017450) Train Loss: 5.5323, Train Steps/Sec: 1.17 +[2025-04-24 07:18:04] (step=0017475) Train Loss: 5.5635, Train Steps/Sec: 1.17 +[2025-04-24 07:18:26] (step=0017500) Train Loss: 5.5245, Train Steps/Sec: 1.17 +[2025-04-24 07:18:47] (step=0017525) Train Loss: 5.5077, Train Steps/Sec: 1.17 +[2025-04-24 07:19:08] (step=0017550) Train Loss: 5.5774, Train Steps/Sec: 1.17 +[2025-04-24 07:19:30] (step=0017575) Train Loss: 5.4494, Train Steps/Sec: 1.17 +[2025-04-24 07:19:51] (step=0017600) Train Loss: 5.5178, Train Steps/Sec: 1.16 +[2025-04-24 07:20:12] (step=0017625) Train Loss: 5.4888, Train Steps/Sec: 1.17 +[2025-04-24 07:20:34] (step=0017650) Train Loss: 5.5101, Train Steps/Sec: 1.17 +[2025-04-24 07:20:55] (step=0017675) Train Loss: 5.4688, Train Steps/Sec: 1.17 +[2025-04-24 07:21:17] (step=0017700) Train Loss: 5.4802, Train Steps/Sec: 1.12 +[2025-04-24 07:21:40] (step=0017725) Train Loss: 5.5327, Train Steps/Sec: 1.13 +[2025-04-24 07:22:03] (step=0017750) Train Loss: 5.4576, Train Steps/Sec: 1.08 +[2025-04-24 07:22:25] (step=0017775) Train Loss: 5.4912, Train Steps/Sec: 1.12 +[2025-04-24 07:22:46] (step=0017800) Train Loss: 5.5289, Train Steps/Sec: 1.17 +[2025-04-24 07:23:08] (step=0017825) Train Loss: 5.5371, Train Steps/Sec: 1.17 +[2025-04-24 07:23:30] (step=0017850) Train Loss: 5.4750, Train Steps/Sec: 1.13 +[2025-04-24 07:23:52] (step=0017875) Train Loss: 5.4674, Train Steps/Sec: 1.11 +[2025-04-24 07:24:14] (step=0017900) Train Loss: 5.4952, Train Steps/Sec: 1.17 +[2025-04-24 07:24:35] (step=0017925) Train Loss: 5.5112, Train Steps/Sec: 1.17 +[2025-04-24 07:24:57] (step=0017950) Train Loss: 5.4589, Train Steps/Sec: 1.17 +[2025-04-24 07:25:18] (step=0017975) Train Loss: 5.4212, Train Steps/Sec: 1.17 +[2025-04-24 07:25:40] (step=0018000) Train Loss: 5.4668, Train Steps/Sec: 1.12 +[2025-04-24 07:25:40] text_config is None. Initializing the text config with default values (`OPTConfig`). +[2025-04-24 07:30:52] Finish Eval in 18000 steps... +[2025-04-24 07:31:11] Saved checkpoint to checkpoint/Afteri2i_100kckpt_LLAVAT5_empty_Fixed_just_segment_pretrain_stage3_dreambench_recap_400k_t2i_400k_flux_200k_midjourney_150k_recovery_150k_3e4_epoch2_no_rotated/000-GPT-XL/checkpoints/0018000.pt +[2025-04-24 07:31:13] Removed old checkpoint: checkpoint/Afteri2i_100kckpt_LLAVAT5_empty_Fixed_just_segment_pretrain_stage3_dreambench_recap_400k_t2i_400k_flux_200k_midjourney_150k_recovery_150k_3e4_epoch2_no_rotated/000-GPT-XL/checkpoints/0016000.pt +[2025-04-24 07:31:34] (step=0018025) Train Loss: 5.4358, Train Steps/Sec: 0.07 +[2025-04-24 07:31:56] (step=0018050) Train Loss: 5.4906, Train Steps/Sec: 1.17 +[2025-04-24 07:32:17] (step=0018075) Train Loss: 5.4534, Train Steps/Sec: 1.17 +[2025-04-24 07:32:38] (step=0018100) Train Loss: 5.5204, Train Steps/Sec: 1.17 +[2025-04-24 07:33:00] (step=0018125) Train Loss: 5.4805, Train Steps/Sec: 1.17 +[2025-04-24 07:33:21] (step=0018150) Train Loss: 5.5645, Train Steps/Sec: 1.16 +[2025-04-24 07:33:43] (step=0018175) Train Loss: 5.3972, Train Steps/Sec: 1.17 +[2025-04-24 07:34:04] (step=0018200) Train Loss: 5.5897, Train Steps/Sec: 1.17 +[2025-04-24 07:34:25] (step=0018225) Train Loss: 5.5371, Train Steps/Sec: 1.17 +[2025-04-24 07:34:47] (step=0018250) Train Loss: 5.5952, Train Steps/Sec: 1.17 +[2025-04-24 07:35:08] (step=0018275) Train Loss: 5.3934, Train Steps/Sec: 1.17 +[2025-04-24 07:35:30] (step=0018300) Train Loss: 5.5189, Train Steps/Sec: 1.17 +[2025-04-24 07:35:51] (step=0018325) Train Loss: 5.4553, Train Steps/Sec: 1.17 +[2025-04-24 07:36:13] (step=0018350) Train Loss: 5.4789, Train Steps/Sec: 1.12 +[2025-04-24 07:36:35] (step=0018375) Train Loss: 5.4963, Train Steps/Sec: 1.17 +[2025-04-24 07:36:58] (step=0018400) Train Loss: 5.5375, Train Steps/Sec: 1.08 +[2025-04-24 07:37:20] (step=0018425) Train Loss: 5.4807, Train Steps/Sec: 1.13 +[2025-04-24 07:37:41] (step=0018450) Train Loss: 5.3620, Train Steps/Sec: 1.17 +[2025-04-24 07:38:03] (step=0018475) Train Loss: 5.4450, Train Steps/Sec: 1.16 +[2025-04-24 07:38:26] (step=0018500) Train Loss: 5.4931, Train Steps/Sec: 1.07 +[2025-04-24 07:38:47] (step=0018525) Train Loss: 5.4474, Train Steps/Sec: 1.17 +[2025-04-24 07:39:09] (step=0018550) Train Loss: 5.4844, Train Steps/Sec: 1.17 +[2025-04-24 07:39:30] (step=0018575) Train Loss: 5.4188, Train Steps/Sec: 1.16 +[2025-04-24 07:39:53] (step=0018600) Train Loss: 5.5376, Train Steps/Sec: 1.11 +[2025-04-24 07:40:14] (step=0018625) Train Loss: 5.5167, Train Steps/Sec: 1.17 +[2025-04-24 07:40:37] (step=0018650) Train Loss: 5.4393, Train Steps/Sec: 1.12 +[2025-04-24 07:40:58] (step=0018675) Train Loss: 5.3458, Train Steps/Sec: 1.17 +[2025-04-24 07:41:19] (step=0018700) Train Loss: 5.4778, Train Steps/Sec: 1.17 +[2025-04-24 07:41:41] (step=0018725) Train Loss: 5.4822, Train Steps/Sec: 1.17 +[2025-04-24 07:42:02] (step=0018750) Train Loss: 5.5625, Train Steps/Sec: 1.17 +[2025-04-24 07:42:23] (step=0018775) Train Loss: 5.5072, Train Steps/Sec: 1.17 +[2025-04-24 07:42:45] (step=0018800) Train Loss: 5.5198, Train Steps/Sec: 1.17 +[2025-04-24 07:43:06] (step=0018825) Train Loss: 5.5383, Train Steps/Sec: 1.17 +[2025-04-24 07:43:28] (step=0018850) Train Loss: 5.4402, Train Steps/Sec: 1.17 +[2025-04-24 07:43:49] (step=0018875) Train Loss: 5.4847, Train Steps/Sec: 1.17 +[2025-04-24 07:44:10] (step=0018900) Train Loss: 5.4426, Train Steps/Sec: 1.16 +[2025-04-24 07:44:32] (step=0018925) Train Loss: 5.4358, Train Steps/Sec: 1.17 +[2025-04-24 07:44:53] (step=0018950) Train Loss: 5.4731, Train Steps/Sec: 1.17 +[2025-04-24 07:45:14] (step=0018975) Train Loss: 5.5241, Train Steps/Sec: 1.17 +[2025-04-24 07:45:36] (step=0019000) Train Loss: 5.4190, Train Steps/Sec: 1.17 +[2025-04-24 07:45:58] (step=0019025) Train Loss: 5.5091, Train Steps/Sec: 1.13 +[2025-04-24 07:46:20] (step=0019050) Train Loss: 5.5391, Train Steps/Sec: 1.13 +[2025-04-24 07:46:43] (step=0019075) Train Loss: 5.5753, Train Steps/Sec: 1.08 +[2025-04-24 07:47:05] (step=0019100) Train Loss: 5.5134, Train Steps/Sec: 1.16 +[2025-04-24 07:47:26] (step=0019125) Train Loss: 5.4791, Train Steps/Sec: 1.17 +[2025-04-24 07:47:47] (step=0019150) Train Loss: 5.4918, Train Steps/Sec: 1.17 +[2025-04-24 07:48:09] (step=0019175) Train Loss: 5.5401, Train Steps/Sec: 1.13 +[2025-04-24 07:48:31] (step=0019200) Train Loss: 5.4794, Train Steps/Sec: 1.17 +[2025-04-24 07:48:53] (step=0019225) Train Loss: 5.4962, Train Steps/Sec: 1.12 +[2025-04-24 07:49:15] (step=0019250) Train Loss: 5.4318, Train Steps/Sec: 1.17 +[2025-04-24 07:49:36] (step=0019275) Train Loss: 5.4728, Train Steps/Sec: 1.17 +[2025-04-24 07:49:57] (step=0019300) Train Loss: 5.4955, Train Steps/Sec: 1.17 +[2025-04-24 07:50:21] (step=0019325) Train Loss: 5.4629, Train Steps/Sec: 1.07 +[2025-04-24 07:50:42] (step=0019350) Train Loss: 5.4892, Train Steps/Sec: 1.17 +[2025-04-24 07:51:04] (step=0019375) Train Loss: 5.4656, Train Steps/Sec: 1.17 +[2025-04-24 07:51:25] (step=0019400) Train Loss: 5.4915, Train Steps/Sec: 1.16 +[2025-04-24 07:51:46] (step=0019425) Train Loss: 5.5224, Train Steps/Sec: 1.17 +[2025-04-24 07:52:08] (step=0019450) Train Loss: 5.5405, Train Steps/Sec: 1.17 +[2025-04-24 07:52:29] (step=0019475) Train Loss: 5.4621, Train Steps/Sec: 1.17 +[2025-04-24 07:52:51] (step=0019500) Train Loss: 5.4519, Train Steps/Sec: 1.17 +[2025-04-24 07:53:12] (step=0019525) Train Loss: 5.5039, Train Steps/Sec: 1.17 +[2025-04-24 07:53:33] (step=0019550) Train Loss: 5.4655, Train Steps/Sec: 1.17 +[2025-04-24 07:53:55] (step=0019575) Train Loss: 5.5005, Train Steps/Sec: 1.17 +[2025-04-24 07:54:16] (step=0019600) Train Loss: 5.4640, Train Steps/Sec: 1.17 +[2025-04-24 07:54:38] (step=0019625) Train Loss: 5.5170, Train Steps/Sec: 1.17 +[2025-04-24 07:54:59] (step=0019650) Train Loss: 5.5773, Train Steps/Sec: 1.17 +[2025-04-24 07:55:21] (step=0019675) Train Loss: 5.4250, Train Steps/Sec: 1.12 +[2025-04-24 07:55:43] (step=0019700) Train Loss: 5.5086, Train Steps/Sec: 1.12 +[2025-04-24 07:56:06] (step=0019725) Train Loss: 5.4626, Train Steps/Sec: 1.13 +[2025-04-24 07:56:28] (step=0019750) Train Loss: 5.4114, Train Steps/Sec: 1.13 +[2025-04-24 07:56:49] (step=0019775) Train Loss: 5.5689, Train Steps/Sec: 1.17 +[2025-04-24 07:57:11] (step=0019800) Train Loss: 5.4083, Train Steps/Sec: 1.17 +[2025-04-24 07:57:33] (step=0019825) Train Loss: 5.5481, Train Steps/Sec: 1.12 +[2025-04-24 07:57:54] (step=0019850) Train Loss: 5.5007, Train Steps/Sec: 1.17 +[2025-04-24 07:58:16] (step=0019875) Train Loss: 5.4171, Train Steps/Sec: 1.17 +[2025-04-24 07:58:37] (step=0019900) Train Loss: 5.4718, Train Steps/Sec: 1.17 +[2025-04-24 07:58:58] (step=0019925) Train Loss: 5.5123, Train Steps/Sec: 1.17 +[2025-04-24 07:59:21] (step=0019950) Train Loss: 5.5131, Train Steps/Sec: 1.12 +[2025-04-24 07:59:43] (step=0019975) Train Loss: 5.5085, Train Steps/Sec: 1.12 +[2025-04-24 08:00:04] (step=0020000) Train Loss: 5.3823, Train Steps/Sec: 1.17 +[2025-04-24 08:00:05] text_config is None. Initializing the text config with default values (`OPTConfig`). +[2025-04-24 08:05:17] Finish Eval in 20000 steps... +[2025-04-24 08:05:35] Saved checkpoint to checkpoint/Afteri2i_100kckpt_LLAVAT5_empty_Fixed_just_segment_pretrain_stage3_dreambench_recap_400k_t2i_400k_flux_200k_midjourney_150k_recovery_150k_3e4_epoch2_no_rotated/000-GPT-XL/checkpoints/0020000.pt +[2025-04-24 08:05:37] Removed old checkpoint: checkpoint/Afteri2i_100kckpt_LLAVAT5_empty_Fixed_just_segment_pretrain_stage3_dreambench_recap_400k_t2i_400k_flux_200k_midjourney_150k_recovery_150k_3e4_epoch2_no_rotated/000-GPT-XL/checkpoints/0018000.pt +[2025-04-24 08:05:59] (step=0020025) Train Loss: 5.5133, Train Steps/Sec: 0.07 +[2025-04-24 08:06:21] (step=0020050) Train Loss: 5.5377, Train Steps/Sec: 1.12 +[2025-04-24 08:06:42] (step=0020075) Train Loss: 5.4973, Train Steps/Sec: 1.17 +[2025-04-24 08:07:04] (step=0020100) Train Loss: 5.4359, Train Steps/Sec: 1.17 +[2025-04-24 08:07:25] (step=0020125) Train Loss: 5.4953, Train Steps/Sec: 1.17 +[2025-04-24 08:07:46] (step=0020150) Train Loss: 5.3692, Train Steps/Sec: 1.17 +[2025-04-24 08:08:08] (step=0020175) Train Loss: 5.4092, Train Steps/Sec: 1.17 +[2025-04-24 08:08:29] (step=0020200) Train Loss: 5.5509, Train Steps/Sec: 1.16 +[2025-04-24 08:08:51] (step=0020225) Train Loss: 5.4333, Train Steps/Sec: 1.17 +[2025-04-24 08:09:12] (step=0020250) Train Loss: 5.3948, Train Steps/Sec: 1.17 +[2025-04-24 08:09:33] (step=0020275) Train Loss: 5.5149, Train Steps/Sec: 1.17 +[2025-04-24 08:09:55] (step=0020300) Train Loss: 5.4302, Train Steps/Sec: 1.16 +[2025-04-24 08:10:17] (step=0020325) Train Loss: 5.4272, Train Steps/Sec: 1.12 +[2025-04-24 08:10:38] (step=0020350) Train Loss: 5.5617, Train Steps/Sec: 1.17 +[2025-04-24 08:11:01] (step=0020375) Train Loss: 5.4860, Train Steps/Sec: 1.13 +[2025-04-24 08:11:24] (step=0020400) Train Loss: 5.4642, Train Steps/Sec: 1.08 +[2025-04-24 08:11:45] (step=0020425) Train Loss: 5.5198, Train Steps/Sec: 1.17 +[2025-04-24 08:12:06] (step=0020450) Train Loss: 5.5216, Train Steps/Sec: 1.17 +[2025-04-24 08:12:28] (step=0020475) Train Loss: 5.5134, Train Steps/Sec: 1.17 +[2025-04-24 08:12:50] (step=0020500) Train Loss: 5.4728, Train Steps/Sec: 1.11 +[2025-04-24 08:13:12] (step=0020525) Train Loss: 5.4495, Train Steps/Sec: 1.17 +[2025-04-24 08:13:33] (step=0020550) Train Loss: 5.5141, Train Steps/Sec: 1.16 +[2025-04-24 08:13:55] (step=0020575) Train Loss: 5.5725, Train Steps/Sec: 1.17 +[2025-04-24 08:14:16] (step=0020600) Train Loss: 5.4600, Train Steps/Sec: 1.17 +[2025-04-24 08:14:38] (step=0020625) Train Loss: 5.3859, Train Steps/Sec: 1.17 +[2025-04-24 08:14:59] (step=0020650) Train Loss: 5.4379, Train Steps/Sec: 1.17 +[2025-04-24 08:15:22] (step=0020675) Train Loss: 5.4990, Train Steps/Sec: 1.07 +[2025-04-24 08:15:44] (step=0020700) Train Loss: 5.4891, Train Steps/Sec: 1.17 +[2025-04-24 08:16:05] (step=0020725) Train Loss: 5.4595, Train Steps/Sec: 1.17 +[2025-04-24 08:16:26] (step=0020750) Train Loss: 5.4715, Train Steps/Sec: 1.17 +[2025-04-24 08:16:49] (step=0020775) Train Loss: 5.5092, Train Steps/Sec: 1.12 +[2025-04-24 08:17:10] (step=0020800) Train Loss: 5.5078, Train Steps/Sec: 1.17 +[2025-04-24 08:17:31] (step=0020825) Train Loss: 5.5520, Train Steps/Sec: 1.17 +[2025-04-24 08:17:53] (step=0020850) Train Loss: 5.4424, Train Steps/Sec: 1.17 +[2025-04-24 08:18:14] (step=0020875) Train Loss: 5.5171, Train Steps/Sec: 1.17 +[2025-04-24 08:18:36] (step=0020900) Train Loss: 5.5085, Train Steps/Sec: 1.16 +[2025-04-24 08:18:57] (step=0020925) Train Loss: 5.4829, Train Steps/Sec: 1.17 +[2025-04-24 08:19:18] (step=0020950) Train Loss: 5.5743, Train Steps/Sec: 1.17 +[2025-04-24 08:19:40] (step=0020975) Train Loss: 5.4730, Train Steps/Sec: 1.17 +[2025-04-24 08:20:02] (step=0021000) Train Loss: 5.4709, Train Steps/Sec: 1.12 +[2025-04-24 08:20:24] (step=0021025) Train Loss: 5.4145, Train Steps/Sec: 1.13 +[2025-04-24 08:20:48] (step=0021050) Train Loss: 5.5274, Train Steps/Sec: 1.08 +[2025-04-24 08:21:09] (step=0021075) Train Loss: 5.4070, Train Steps/Sec: 1.17 +[2025-04-24 08:21:30] (step=0021100) Train Loss: 5.4601, Train Steps/Sec: 1.17 +[2025-04-24 08:21:52] (step=0021125) Train Loss: 5.4321, Train Steps/Sec: 1.17 +[2025-04-24 08:22:14] (step=0021150) Train Loss: 5.4045, Train Steps/Sec: 1.13 +[2025-04-24 08:22:35] (step=0021175) Train Loss: 5.4734, Train Steps/Sec: 1.17 +[2025-04-24 08:22:57] (step=0021200) Train Loss: 5.5089, Train Steps/Sec: 1.17 +[2025-04-24 08:23:18] (step=0021225) Train Loss: 5.5798, Train Steps/Sec: 1.17 +[2025-04-24 08:23:39] (step=0021250) Train Loss: 5.4149, Train Steps/Sec: 1.17 +[2025-04-24 08:24:01] (step=0021275) Train Loss: 5.4229, Train Steps/Sec: 1.17 +[2025-04-24 08:24:22] (step=0021300) Train Loss: 5.4381, Train Steps/Sec: 1.17 +[2025-04-24 08:24:44] (step=0021325) Train Loss: 5.5042, Train Steps/Sec: 1.12 +[2025-04-24 08:25:06] (step=0021350) Train Loss: 5.5044, Train Steps/Sec: 1.17 +[2025-04-24 08:25:27] (step=0021375) Train Loss: 5.4743, Train Steps/Sec: 1.17 +[2025-04-24 08:25:50] (step=0021400) Train Loss: 5.4124, Train Steps/Sec: 1.11 +[2025-04-24 08:26:11] (step=0021425) Train Loss: 5.4300, Train Steps/Sec: 1.17 +[2025-04-24 08:26:32] (step=0021450) Train Loss: 5.5121, Train Steps/Sec: 1.17 +[2025-04-24 08:26:54] (step=0021475) Train Loss: 5.4615, Train Steps/Sec: 1.17 +[2025-04-24 08:27:16] (step=0021500) Train Loss: 5.4793, Train Steps/Sec: 1.11 +[2025-04-24 08:27:38] (step=0021525) Train Loss: 5.4409, Train Steps/Sec: 1.17 +[2025-04-24 08:27:59] (step=0021550) Train Loss: 5.5200, Train Steps/Sec: 1.17 +[2025-04-24 08:28:20] (step=0021575) Train Loss: 5.4614, Train Steps/Sec: 1.17 +[2025-04-24 08:28:42] (step=0021600) Train Loss: 5.5253, Train Steps/Sec: 1.17 +[2025-04-24 08:29:03] (step=0021625) Train Loss: 5.4831, Train Steps/Sec: 1.17 +[2025-04-24 08:29:26] (step=0021650) Train Loss: 5.4975, Train Steps/Sec: 1.12 +[2025-04-24 08:29:47] (step=0021675) Train Loss: 5.4511, Train Steps/Sec: 1.16 +[2025-04-24 08:30:11] (step=0021700) Train Loss: 5.5029, Train Steps/Sec: 1.04 +[2025-04-24 08:30:33] (step=0021725) Train Loss: 5.4794, Train Steps/Sec: 1.16 +[2025-04-24 08:30:54] (step=0021750) Train Loss: 5.4671, Train Steps/Sec: 1.17 +[2025-04-24 08:31:15] (step=0021775) Train Loss: 5.5973, Train Steps/Sec: 1.17 +[2025-04-24 08:31:38] (step=0021800) Train Loss: 5.4924, Train Steps/Sec: 1.12 +[2025-04-24 08:31:59] (step=0021825) Train Loss: 5.4545, Train Steps/Sec: 1.16 +[2025-04-24 08:32:21] (step=0021850) Train Loss: 5.4740, Train Steps/Sec: 1.17 +[2025-04-24 08:32:42] (step=0021875) Train Loss: 5.4343, Train Steps/Sec: 1.17 +[2025-04-24 08:33:03] (step=0021900) Train Loss: 5.4504, Train Steps/Sec: 1.16 +[2025-04-24 08:33:25] (step=0021925) Train Loss: 5.5281, Train Steps/Sec: 1.17 +[2025-04-24 08:33:46] (step=0021950) Train Loss: 5.4968, Train Steps/Sec: 1.17 +[2025-04-24 08:34:09] (step=0021975) Train Loss: 5.4799, Train Steps/Sec: 1.12 +[2025-04-24 08:34:30] (step=0022000) Train Loss: 5.4684, Train Steps/Sec: 1.17 +[2025-04-24 08:34:30] text_config is None. Initializing the text config with default values (`OPTConfig`). +[2025-04-24 08:39:45] Finish Eval in 22000 steps... +[2025-04-24 08:40:04] Saved checkpoint to checkpoint/Afteri2i_100kckpt_LLAVAT5_empty_Fixed_just_segment_pretrain_stage3_dreambench_recap_400k_t2i_400k_flux_200k_midjourney_150k_recovery_150k_3e4_epoch2_no_rotated/000-GPT-XL/checkpoints/0022000.pt +[2025-04-24 08:40:06] Removed old checkpoint: checkpoint/Afteri2i_100kckpt_LLAVAT5_empty_Fixed_just_segment_pretrain_stage3_dreambench_recap_400k_t2i_400k_flux_200k_midjourney_150k_recovery_150k_3e4_epoch2_no_rotated/000-GPT-XL/checkpoints/0020000.pt +[2025-04-24 08:40:27] (step=0022025) Train Loss: 5.4818, Train Steps/Sec: 0.07 +[2025-04-24 08:40:48] (step=0022050) Train Loss: 5.4911, Train Steps/Sec: 1.17 +[2025-04-24 08:41:10] (step=0022075) Train Loss: 5.4607, Train Steps/Sec: 1.17 +[2025-04-24 08:41:31] (step=0022100) Train Loss: 5.5138, Train Steps/Sec: 1.17 +[2025-04-24 08:41:54] (step=0022125) Train Loss: 5.5113, Train Steps/Sec: 1.12 +[2025-04-24 08:42:15] (step=0022150) Train Loss: 5.4715, Train Steps/Sec: 1.17 +[2025-04-24 08:42:36] (step=0022175) Train Loss: 5.5074, Train Steps/Sec: 1.17 +[2025-04-24 08:42:58] (step=0022200) Train Loss: 5.5234, Train Steps/Sec: 1.16 +[2025-04-24 08:43:20] (step=0022225) Train Loss: 5.5074, Train Steps/Sec: 1.11 +[2025-04-24 08:43:42] (step=0022250) Train Loss: 5.4257, Train Steps/Sec: 1.17 +[2025-04-24 08:44:03] (step=0022275) Train Loss: 5.4651, Train Steps/Sec: 1.17 +[2025-04-24 08:44:25] (step=0022300) Train Loss: 5.5406, Train Steps/Sec: 1.12 +[2025-04-24 08:44:47] (step=0022325) Train Loss: 5.5015, Train Steps/Sec: 1.17 +[2025-04-24 08:45:09] (step=0022350) Train Loss: 5.4660, Train Steps/Sec: 1.13 +[2025-04-24 08:45:32] (step=0022375) Train Loss: 5.5126, Train Steps/Sec: 1.08 +[2025-04-24 08:45:54] (step=0022400) Train Loss: 5.5021, Train Steps/Sec: 1.16 +[2025-04-24 08:46:15] (step=0022425) Train Loss: 5.4861, Train Steps/Sec: 1.15 +[2025-04-24 08:46:37] (step=0022450) Train Loss: 5.5172, Train Steps/Sec: 1.17 +[2025-04-24 08:46:59] (step=0022475) Train Loss: 5.4981, Train Steps/Sec: 1.12 +[2025-04-24 08:47:20] (step=0022500) Train Loss: 5.5080, Train Steps/Sec: 1.17 +[2025-04-24 08:47:42] (step=0022525) Train Loss: 5.4191, Train Steps/Sec: 1.17 +[2025-04-24 08:48:03] (step=0022550) Train Loss: 5.4878, Train Steps/Sec: 1.17 +[2025-04-24 08:48:24] (step=0022575) Train Loss: 5.4937, Train Steps/Sec: 1.17 +[2025-04-24 08:48:46] (step=0022600) Train Loss: 5.5459, Train Steps/Sec: 1.17 +[2025-04-24 08:49:07] (step=0022625) Train Loss: 5.4589, Train Steps/Sec: 1.17 +[2025-04-24 08:49:29] (step=0022650) Train Loss: 5.3859, Train Steps/Sec: 1.17 +[2025-04-24 08:49:51] (step=0022675) Train Loss: 5.5071, Train Steps/Sec: 1.12 +[2025-04-24 08:50:13] (step=0022700) Train Loss: 5.4370, Train Steps/Sec: 1.16 +[2025-04-24 08:50:34] (step=0022725) Train Loss: 5.5046, Train Steps/Sec: 1.17 +[2025-04-24 08:50:55] (step=0022750) Train Loss: 5.5108, Train Steps/Sec: 1.17 +[2025-04-24 08:51:17] (step=0022775) Train Loss: 5.5214, Train Steps/Sec: 1.17 +[2025-04-24 08:51:38] (step=0022800) Train Loss: 5.5054, Train Steps/Sec: 1.16 +[2025-04-24 08:52:00] (step=0022825) Train Loss: 5.4326, Train Steps/Sec: 1.12 +[2025-04-24 08:52:22] (step=0022850) Train Loss: 5.4649, Train Steps/Sec: 1.17 +[2025-04-24 08:52:43] (step=0022875) Train Loss: 5.3830, Train Steps/Sec: 1.17 +[2025-04-24 08:53:05] (step=0022900) Train Loss: 5.5336, Train Steps/Sec: 1.17 +[2025-04-24 08:53:26] (step=0022925) Train Loss: 5.3681, Train Steps/Sec: 1.17 +[2025-04-24 08:53:49] (step=0022950) Train Loss: 5.4452, Train Steps/Sec: 1.11 +[2025-04-24 08:54:10] (step=0022975) Train Loss: 5.4281, Train Steps/Sec: 1.17 +[2025-04-24 08:54:32] (step=0023000) Train Loss: 5.4518, Train Steps/Sec: 1.12 +[2025-04-24 08:54:55] (step=0023025) Train Loss: 5.5616, Train Steps/Sec: 1.09 +[2025-04-24 08:55:17] (step=0023050) Train Loss: 5.5169, Train Steps/Sec: 1.17 +[2025-04-24 08:55:38] (step=0023075) Train Loss: 5.4815, Train Steps/Sec: 1.17 +[2025-04-24 08:55:59] (step=0023100) Train Loss: 5.5091, Train Steps/Sec: 1.17 +[2025-04-24 08:56:22] (step=0023125) Train Loss: 5.4254, Train Steps/Sec: 1.13 +[2025-04-24 08:56:43] (step=0023150) Train Loss: 5.5435, Train Steps/Sec: 1.17 +[2025-04-24 08:57:04] (step=0023175) Train Loss: 5.5297, Train Steps/Sec: 1.17 +[2025-04-24 08:57:26] (step=0023200) Train Loss: 5.4291, Train Steps/Sec: 1.16 +[2025-04-24 08:57:47] (step=0023225) Train Loss: 5.5005, Train Steps/Sec: 1.17 +[2025-04-24 08:58:09] (step=0023250) Train Loss: 5.4685, Train Steps/Sec: 1.17 +[2025-04-24 08:58:30] (step=0023275) Train Loss: 5.4398, Train Steps/Sec: 1.17 +[2025-04-24 08:58:51] (step=0023300) Train Loss: 5.5183, Train Steps/Sec: 1.16 +[2025-04-24 08:59:14] (step=0023325) Train Loss: 5.4721, Train Steps/Sec: 1.11 +[2025-04-24 08:59:35] (step=0023350) Train Loss: 5.4324, Train Steps/Sec: 1.17 +[2025-04-24 08:59:57] (step=0023375) Train Loss: 5.4557, Train Steps/Sec: 1.17 +[2025-04-24 09:00:18] (step=0023400) Train Loss: 5.4655, Train Steps/Sec: 1.16 +[2025-04-24 09:00:39] (step=0023425) Train Loss: 5.4009, Train Steps/Sec: 1.17 +[2025-04-24 09:01:01] (step=0023450) Train Loss: 5.4469, Train Steps/Sec: 1.17 +[2025-04-24 09:01:09] Beginning epoch 1... +[2025-04-24 09:01:26] (step=0023475) Train Loss: 5.4170, Train Steps/Sec: 0.97 +[2025-04-24 09:01:48] (step=0023500) Train Loss: 5.3715, Train Steps/Sec: 1.15 +[2025-04-24 09:02:10] (step=0023525) Train Loss: 5.4157, Train Steps/Sec: 1.16 +[2025-04-24 09:02:33] (step=0023550) Train Loss: 5.3424, Train Steps/Sec: 1.09 +[2025-04-24 09:02:54] (step=0023575) Train Loss: 5.4245, Train Steps/Sec: 1.16 +[2025-04-24 09:03:16] (step=0023600) Train Loss: 5.4285, Train Steps/Sec: 1.16 +[2025-04-24 09:03:38] (step=0023625) Train Loss: 5.5040, Train Steps/Sec: 1.11 +[2025-04-24 09:04:00] (step=0023650) Train Loss: 5.3243, Train Steps/Sec: 1.17 +[2025-04-24 09:04:25] (step=0023675) Train Loss: 5.3999, Train Steps/Sec: 1.00 +[2025-04-24 09:04:47] (step=0023700) Train Loss: 5.3756, Train Steps/Sec: 1.11 +[2025-04-24 09:05:09] (step=0023725) Train Loss: 5.4116, Train Steps/Sec: 1.16 +[2025-04-24 09:05:30] (step=0023750) Train Loss: 5.3427, Train Steps/Sec: 1.16 +[2025-04-24 09:05:53] (step=0023775) Train Loss: 5.4304, Train Steps/Sec: 1.11 +[2025-04-24 09:06:14] (step=0023800) Train Loss: 5.4047, Train Steps/Sec: 1.17 +[2025-04-24 09:06:36] (step=0023825) Train Loss: 5.4554, Train Steps/Sec: 1.17 +[2025-04-24 09:06:57] (step=0023850) Train Loss: 5.3423, Train Steps/Sec: 1.17 +[2025-04-24 09:07:18] (step=0023875) Train Loss: 5.3771, Train Steps/Sec: 1.17 +[2025-04-24 09:07:40] (step=0023900) Train Loss: 5.3327, Train Steps/Sec: 1.17 +[2025-04-24 09:08:01] (step=0023925) Train Loss: 5.4558, Train Steps/Sec: 1.17 +[2025-04-24 09:08:23] (step=0023950) Train Loss: 5.3731, Train Steps/Sec: 1.17 +[2025-04-24 09:08:45] (step=0023975) Train Loss: 5.3961, Train Steps/Sec: 1.11 +[2025-04-24 09:09:07] (step=0024000) Train Loss: 5.4169, Train Steps/Sec: 1.16 +[2025-04-24 09:09:07] text_config is None. Initializing the text config with default values (`OPTConfig`). +[2025-04-24 09:14:21] Finish Eval in 24000 steps... +[2025-04-24 09:14:40] Saved checkpoint to checkpoint/Afteri2i_100kckpt_LLAVAT5_empty_Fixed_just_segment_pretrain_stage3_dreambench_recap_400k_t2i_400k_flux_200k_midjourney_150k_recovery_150k_3e4_epoch2_no_rotated/000-GPT-XL/checkpoints/0024000.pt +[2025-04-24 09:14:42] Removed old checkpoint: checkpoint/Afteri2i_100kckpt_LLAVAT5_empty_Fixed_just_segment_pretrain_stage3_dreambench_recap_400k_t2i_400k_flux_200k_midjourney_150k_recovery_150k_3e4_epoch2_no_rotated/000-GPT-XL/checkpoints/0022000.pt +[2025-04-24 09:15:03] (step=0024025) Train Loss: 5.3980, Train Steps/Sec: 0.07 +[2025-04-24 09:15:24] (step=0024050) Train Loss: 5.3459, Train Steps/Sec: 1.17 +[2025-04-24 09:15:46] (step=0024075) Train Loss: 5.3863, Train Steps/Sec: 1.17 +[2025-04-24 09:16:07] (step=0024100) Train Loss: 5.3357, Train Steps/Sec: 1.17 +[2025-04-24 09:16:28] (step=0024125) Train Loss: 5.3984, Train Steps/Sec: 1.17 +[2025-04-24 09:16:50] (step=0024150) Train Loss: 5.3400, Train Steps/Sec: 1.17 +[2025-04-24 09:17:11] (step=0024175) Train Loss: 5.4724, Train Steps/Sec: 1.16 +[2025-04-24 09:17:33] (step=0024200) Train Loss: 5.3977, Train Steps/Sec: 1.15 +[2025-04-24 09:17:55] (step=0024225) Train Loss: 5.4184, Train Steps/Sec: 1.17 +[2025-04-24 09:18:16] (step=0024250) Train Loss: 5.3963, Train Steps/Sec: 1.17 +[2025-04-24 09:18:39] (step=0024275) Train Loss: 5.3373, Train Steps/Sec: 1.07 +[2025-04-24 09:19:01] (step=0024300) Train Loss: 5.4231, Train Steps/Sec: 1.17 +[2025-04-24 09:19:23] (step=0024325) Train Loss: 5.4282, Train Steps/Sec: 1.13 +[2025-04-24 09:19:46] (step=0024350) Train Loss: 5.3997, Train Steps/Sec: 1.07 +[2025-04-24 09:20:08] (step=0024375) Train Loss: 5.3876, Train Steps/Sec: 1.17 +[2025-04-24 09:20:30] (step=0024400) Train Loss: 5.3416, Train Steps/Sec: 1.14 +[2025-04-24 09:20:52] (step=0024425) Train Loss: 5.3808, Train Steps/Sec: 1.10 +[2025-04-24 09:21:15] (step=0024450) Train Loss: 5.4180, Train Steps/Sec: 1.12 +[2025-04-24 09:21:36] (step=0024475) Train Loss: 5.4024, Train Steps/Sec: 1.17 +[2025-04-24 09:21:57] (step=0024500) Train Loss: 5.3979, Train Steps/Sec: 1.16 +[2025-04-24 09:22:19] (step=0024525) Train Loss: 5.4117, Train Steps/Sec: 1.17 +[2025-04-24 09:22:40] (step=0024550) Train Loss: 5.4733, Train Steps/Sec: 1.17 +[2025-04-24 09:23:01] (step=0024575) Train Loss: 5.3784, Train Steps/Sec: 1.17 +[2025-04-24 09:23:23] (step=0024600) Train Loss: 5.3431, Train Steps/Sec: 1.17 +[2025-04-24 09:23:44] (step=0024625) Train Loss: 5.3395, Train Steps/Sec: 1.17 +[2025-04-24 09:24:06] (step=0024650) Train Loss: 5.3325, Train Steps/Sec: 1.17 +[2025-04-24 09:24:28] (step=0024675) Train Loss: 5.3717, Train Steps/Sec: 1.11 +[2025-04-24 09:24:50] (step=0024700) Train Loss: 5.3978, Train Steps/Sec: 1.17 +[2025-04-24 09:25:11] (step=0024725) Train Loss: 5.3836, Train Steps/Sec: 1.17 +[2025-04-24 09:25:32] (step=0024750) Train Loss: 5.4446, Train Steps/Sec: 1.17 +[2025-04-24 09:25:54] (step=0024775) Train Loss: 5.3656, Train Steps/Sec: 1.17 +[2025-04-24 09:26:15] (step=0024800) Train Loss: 5.4777, Train Steps/Sec: 1.17 +[2025-04-24 09:26:37] (step=0024825) Train Loss: 5.4036, Train Steps/Sec: 1.17 +[2025-04-24 09:26:58] (step=0024850) Train Loss: 5.3854, Train Steps/Sec: 1.17 +[2025-04-24 09:27:19] (step=0024875) Train Loss: 5.4243, Train Steps/Sec: 1.17 +[2025-04-24 09:27:41] (step=0024900) Train Loss: 5.3952, Train Steps/Sec: 1.16 +[2025-04-24 09:28:02] (step=0024925) Train Loss: 5.4075, Train Steps/Sec: 1.17 +[2025-04-24 09:28:24] (step=0024950) Train Loss: 5.4776, Train Steps/Sec: 1.12 +[2025-04-24 09:28:47] (step=0024975) Train Loss: 5.3747, Train Steps/Sec: 1.13 +[2025-04-24 09:29:10] (step=0025000) Train Loss: 5.4054, Train Steps/Sec: 1.07 +[2025-04-24 09:29:32] (step=0025025) Train Loss: 5.4568, Train Steps/Sec: 1.13 +[2025-04-24 09:29:54] (step=0025050) Train Loss: 5.3641, Train Steps/Sec: 1.17 +[2025-04-24 09:30:15] (step=0025075) Train Loss: 5.4242, Train Steps/Sec: 1.17 +[2025-04-24 09:30:37] (step=0025100) Train Loss: 5.3522, Train Steps/Sec: 1.12 +[2025-04-24 09:30:59] (step=0025125) Train Loss: 5.3878, Train Steps/Sec: 1.17 +[2025-04-24 09:31:21] (step=0025150) Train Loss: 5.4051, Train Steps/Sec: 1.12 +[2025-04-24 09:31:42] (step=0025175) Train Loss: 5.3538, Train Steps/Sec: 1.17 +[2025-04-24 09:32:04] (step=0025200) Train Loss: 5.4114, Train Steps/Sec: 1.17 +[2025-04-24 09:32:25] (step=0025225) Train Loss: 5.3730, Train Steps/Sec: 1.17 +[2025-04-24 09:32:46] (step=0025250) Train Loss: 5.3789, Train Steps/Sec: 1.17 +[2025-04-24 09:33:08] (step=0025275) Train Loss: 5.3845, Train Steps/Sec: 1.17 +[2025-04-24 09:33:29] (step=0025300) Train Loss: 5.4134, Train Steps/Sec: 1.16 +[2025-04-24 09:33:52] (step=0025325) Train Loss: 5.3136, Train Steps/Sec: 1.11 +[2025-04-24 09:34:13] (step=0025350) Train Loss: 5.4474, Train Steps/Sec: 1.17 +[2025-04-24 09:34:34] (step=0025375) Train Loss: 5.3589, Train Steps/Sec: 1.17 +[2025-04-24 09:34:56] (step=0025400) Train Loss: 5.4045, Train Steps/Sec: 1.16 +[2025-04-24 09:35:17] (step=0025425) Train Loss: 5.4170, Train Steps/Sec: 1.17 +[2025-04-24 09:35:39] (step=0025450) Train Loss: 5.4153, Train Steps/Sec: 1.17 +[2025-04-24 09:36:00] (step=0025475) Train Loss: 5.3156, Train Steps/Sec: 1.17 +[2025-04-24 09:36:21] (step=0025500) Train Loss: 5.4037, Train Steps/Sec: 1.17 +[2025-04-24 09:36:43] (step=0025525) Train Loss: 5.4132, Train Steps/Sec: 1.17 +[2025-04-24 09:37:04] (step=0025550) Train Loss: 5.3476, Train Steps/Sec: 1.17 +[2025-04-24 09:37:26] (step=0025575) Train Loss: 5.4033, Train Steps/Sec: 1.17 +[2025-04-24 09:37:48] (step=0025600) Train Loss: 5.3420, Train Steps/Sec: 1.12 +[2025-04-24 09:38:09] (step=0025625) Train Loss: 5.4317, Train Steps/Sec: 1.17 +[2025-04-24 09:38:32] (step=0025650) Train Loss: 5.4007, Train Steps/Sec: 1.08 +[2025-04-24 09:38:55] (step=0025675) Train Loss: 5.3824, Train Steps/Sec: 1.13 +[2025-04-24 09:39:16] (step=0025700) Train Loss: 5.4513, Train Steps/Sec: 1.17 +[2025-04-24 09:39:38] (step=0025725) Train Loss: 5.3883, Train Steps/Sec: 1.12 +[2025-04-24 09:40:00] (step=0025750) Train Loss: 5.3391, Train Steps/Sec: 1.17 +[2025-04-24 09:40:22] (step=0025775) Train Loss: 5.4353, Train Steps/Sec: 1.12 +[2025-04-24 09:40:44] (step=0025800) Train Loss: 5.3932, Train Steps/Sec: 1.17 +[2025-04-24 09:41:05] (step=0025825) Train Loss: 5.3949, Train Steps/Sec: 1.17 +[2025-04-24 09:41:26] (step=0025850) Train Loss: 5.3722, Train Steps/Sec: 1.17 +[2025-04-24 09:41:49] (step=0025875) Train Loss: 5.4734, Train Steps/Sec: 1.12 +[2025-04-24 09:42:10] (step=0025900) Train Loss: 5.4412, Train Steps/Sec: 1.17 +[2025-04-24 09:42:31] (step=0025925) Train Loss: 5.3930, Train Steps/Sec: 1.17 +[2025-04-24 09:42:53] (step=0025950) Train Loss: 5.3832, Train Steps/Sec: 1.17 +[2025-04-24 09:43:15] (step=0025975) Train Loss: 5.3942, Train Steps/Sec: 1.12 +[2025-04-24 09:43:36] (step=0026000) Train Loss: 5.3592, Train Steps/Sec: 1.16 +[2025-04-24 09:43:36] text_config is None. Initializing the text config with default values (`OPTConfig`). +[2025-04-24 09:48:45] Finish Eval in 26000 steps... +[2025-04-24 09:49:04] Saved checkpoint to checkpoint/Afteri2i_100kckpt_LLAVAT5_empty_Fixed_just_segment_pretrain_stage3_dreambench_recap_400k_t2i_400k_flux_200k_midjourney_150k_recovery_150k_3e4_epoch2_no_rotated/000-GPT-XL/checkpoints/0026000.pt +[2025-04-24 09:49:06] Removed old checkpoint: checkpoint/Afteri2i_100kckpt_LLAVAT5_empty_Fixed_just_segment_pretrain_stage3_dreambench_recap_400k_t2i_400k_flux_200k_midjourney_150k_recovery_150k_3e4_epoch2_no_rotated/000-GPT-XL/checkpoints/0024000.pt +[2025-04-24 09:49:28] (step=0026025) Train Loss: 5.4082, Train Steps/Sec: 0.07 +[2025-04-24 09:49:49] (step=0026050) Train Loss: 5.4153, Train Steps/Sec: 1.17 +[2025-04-24 09:50:10] (step=0026075) Train Loss: 5.4127, Train Steps/Sec: 1.17 +[2025-04-24 09:50:32] (step=0026100) Train Loss: 5.3668, Train Steps/Sec: 1.17 +[2025-04-24 09:50:53] (step=0026125) Train Loss: 5.3799, Train Steps/Sec: 1.17 +[2025-04-24 09:51:14] (step=0026150) Train Loss: 5.5288, Train Steps/Sec: 1.17 +[2025-04-24 09:51:36] (step=0026175) Train Loss: 5.3914, Train Steps/Sec: 1.17 +[2025-04-24 09:51:57] (step=0026200) Train Loss: 5.3718, Train Steps/Sec: 1.17 +[2025-04-24 09:52:18] (step=0026225) Train Loss: 5.4055, Train Steps/Sec: 1.17 +[2025-04-24 09:52:41] (step=0026250) Train Loss: 5.4038, Train Steps/Sec: 1.12 +[2025-04-24 09:53:02] (step=0026275) Train Loss: 5.4092, Train Steps/Sec: 1.17 +[2025-04-24 09:53:24] (step=0026300) Train Loss: 5.3703, Train Steps/Sec: 1.12 +[2025-04-24 09:53:48] (step=0026325) Train Loss: 5.4083, Train Steps/Sec: 1.08 +[2025-04-24 09:54:09] (step=0026350) Train Loss: 5.4313, Train Steps/Sec: 1.17 +[2025-04-24 09:54:31] (step=0026375) Train Loss: 5.3828, Train Steps/Sec: 1.17 +[2025-04-24 09:54:52] (step=0026400) Train Loss: 5.4450, Train Steps/Sec: 1.16 +[2025-04-24 09:55:14] (step=0026425) Train Loss: 5.4536, Train Steps/Sec: 1.12 +[2025-04-24 09:55:37] (step=0026450) Train Loss: 5.3820, Train Steps/Sec: 1.12 +[2025-04-24 09:55:58] (step=0026475) Train Loss: 5.3449, Train Steps/Sec: 1.17 +[2025-04-24 09:56:20] (step=0026500) Train Loss: 5.4351, Train Steps/Sec: 1.17 +[2025-04-24 09:56:41] (step=0026525) Train Loss: 5.4697, Train Steps/Sec: 1.17 +[2025-04-24 09:57:02] (step=0026550) Train Loss: 5.4127, Train Steps/Sec: 1.17 +[2025-04-24 09:57:24] (step=0026575) Train Loss: 5.3295, Train Steps/Sec: 1.17 +[2025-04-24 09:57:46] (step=0026600) Train Loss: 5.5068, Train Steps/Sec: 1.11 +[2025-04-24 09:58:08] (step=0026625) Train Loss: 5.4076, Train Steps/Sec: 1.17 +[2025-04-24 09:58:29] (step=0026650) Train Loss: 5.4158, Train Steps/Sec: 1.17 +[2025-04-24 09:58:51] (step=0026675) Train Loss: 5.4059, Train Steps/Sec: 1.12 +[2025-04-24 09:59:13] (step=0026700) Train Loss: 5.4661, Train Steps/Sec: 1.17 +[2025-04-24 09:59:34] (step=0026725) Train Loss: 5.3516, Train Steps/Sec: 1.17 +[2025-04-24 09:59:56] (step=0026750) Train Loss: 5.3949, Train Steps/Sec: 1.17 +[2025-04-24 10:00:17] (step=0026775) Train Loss: 5.4029, Train Steps/Sec: 1.17 +[2025-04-24 10:00:38] (step=0026800) Train Loss: 5.3295, Train Steps/Sec: 1.17 +[2025-04-24 10:01:00] (step=0026825) Train Loss: 5.4752, Train Steps/Sec: 1.17 +[2025-04-24 10:01:21] (step=0026850) Train Loss: 5.3200, Train Steps/Sec: 1.17 +[2025-04-24 10:01:43] (step=0026875) Train Loss: 5.3777, Train Steps/Sec: 1.17 +[2025-04-24 10:02:05] (step=0026900) Train Loss: 5.3604, Train Steps/Sec: 1.11 +[2025-04-24 10:02:26] (step=0026925) Train Loss: 5.4015, Train Steps/Sec: 1.17 +[2025-04-24 10:02:48] (step=0026950) Train Loss: 5.3886, Train Steps/Sec: 1.13 +[2025-04-24 10:03:11] (step=0026975) Train Loss: 5.4493, Train Steps/Sec: 1.12 +[2025-04-24 10:03:33] (step=0027000) Train Loss: 5.3256, Train Steps/Sec: 1.12 +[2025-04-24 10:03:54] (step=0027025) Train Loss: 5.4102, Train Steps/Sec: 1.17 +[2025-04-24 10:04:16] (step=0027050) Train Loss: 5.3140, Train Steps/Sec: 1.17 +[2025-04-24 10:04:38] (step=0027075) Train Loss: 5.3612, Train Steps/Sec: 1.13 +[2025-04-24 10:04:59] (step=0027100) Train Loss: 5.4387, Train Steps/Sec: 1.17 +[2025-04-24 10:05:21] (step=0027125) Train Loss: 5.3492, Train Steps/Sec: 1.17 +[2025-04-24 10:05:42] (step=0027150) Train Loss: 5.5132, Train Steps/Sec: 1.17 +[2025-04-24 10:06:04] (step=0027175) Train Loss: 5.4334, Train Steps/Sec: 1.12 +[2025-04-24 10:06:26] (step=0027200) Train Loss: 5.3170, Train Steps/Sec: 1.17 +[2025-04-24 10:06:47] (step=0027225) Train Loss: 5.4304, Train Steps/Sec: 1.17 +[2025-04-24 10:07:09] (step=0027250) Train Loss: 5.3896, Train Steps/Sec: 1.17 +[2025-04-24 10:07:30] (step=0027275) Train Loss: 5.3996, Train Steps/Sec: 1.17 +[2025-04-24 10:07:51] (step=0027300) Train Loss: 5.4702, Train Steps/Sec: 1.17 +[2025-04-24 10:08:15] (step=0027325) Train Loss: 5.4505, Train Steps/Sec: 1.07 +[2025-04-24 10:08:36] (step=0027350) Train Loss: 5.4990, Train Steps/Sec: 1.17 +[2025-04-24 10:08:57] (step=0027375) Train Loss: 5.3243, Train Steps/Sec: 1.17 +[2025-04-24 10:09:19] (step=0027400) Train Loss: 5.4923, Train Steps/Sec: 1.17 +[2025-04-24 10:09:40] (step=0027425) Train Loss: 5.3611, Train Steps/Sec: 1.17 +[2025-04-24 10:10:02] (step=0027450) Train Loss: 5.3598, Train Steps/Sec: 1.17 +[2025-04-24 10:10:23] (step=0027475) Train Loss: 5.3965, Train Steps/Sec: 1.17 +[2025-04-24 10:10:44] (step=0027500) Train Loss: 5.3855, Train Steps/Sec: 1.17 +[2025-04-24 10:11:06] (step=0027525) Train Loss: 5.3154, Train Steps/Sec: 1.17 +[2025-04-24 10:11:27] (step=0027550) Train Loss: 5.3897, Train Steps/Sec: 1.17 +[2025-04-24 10:11:49] (step=0027575) Train Loss: 5.3680, Train Steps/Sec: 1.12 +[2025-04-24 10:12:11] (step=0027600) Train Loss: 5.4462, Train Steps/Sec: 1.17 +[2025-04-24 10:12:33] (step=0027625) Train Loss: 5.3809, Train Steps/Sec: 1.13 +[2025-04-24 10:12:56] (step=0027650) Train Loss: 5.4394, Train Steps/Sec: 1.08 +[2025-04-24 10:13:17] (step=0027675) Train Loss: 5.4391, Train Steps/Sec: 1.17 +[2025-04-24 10:13:39] (step=0027700) Train Loss: 5.3009, Train Steps/Sec: 1.17 +[2025-04-24 10:14:00] (step=0027725) Train Loss: 5.3732, Train Steps/Sec: 1.17 +[2025-04-24 10:14:23] (step=0027750) Train Loss: 5.4107, Train Steps/Sec: 1.12 +[2025-04-24 10:14:44] (step=0027775) Train Loss: 5.4288, Train Steps/Sec: 1.17 +[2025-04-24 10:15:05] (step=0027800) Train Loss: 5.3547, Train Steps/Sec: 1.17 +[2025-04-24 10:15:27] (step=0027825) Train Loss: 5.3669, Train Steps/Sec: 1.17 +[2025-04-24 10:15:48] (step=0027850) Train Loss: 5.4399, Train Steps/Sec: 1.17 +[2025-04-24 10:16:09] (step=0027875) Train Loss: 5.4048, Train Steps/Sec: 1.17 +[2025-04-24 10:16:32] (step=0027900) Train Loss: 5.4257, Train Steps/Sec: 1.11 +[2025-04-24 10:16:53] (step=0027925) Train Loss: 5.4280, Train Steps/Sec: 1.17 +[2025-04-24 10:17:14] (step=0027950) Train Loss: 5.3229, Train Steps/Sec: 1.17 +[2025-04-24 10:17:37] (step=0027975) Train Loss: 5.4126, Train Steps/Sec: 1.12 +[2025-04-24 10:17:58] (step=0028000) Train Loss: 5.3863, Train Steps/Sec: 1.16 +[2025-04-24 10:17:58] text_config is None. Initializing the text config with default values (`OPTConfig`). +[2025-04-24 10:23:09] Finish Eval in 28000 steps... +[2025-04-24 10:23:28] Saved checkpoint to checkpoint/Afteri2i_100kckpt_LLAVAT5_empty_Fixed_just_segment_pretrain_stage3_dreambench_recap_400k_t2i_400k_flux_200k_midjourney_150k_recovery_150k_3e4_epoch2_no_rotated/000-GPT-XL/checkpoints/0028000.pt +[2025-04-24 10:23:29] Removed old checkpoint: checkpoint/Afteri2i_100kckpt_LLAVAT5_empty_Fixed_just_segment_pretrain_stage3_dreambench_recap_400k_t2i_400k_flux_200k_midjourney_150k_recovery_150k_3e4_epoch2_no_rotated/000-GPT-XL/checkpoints/0026000.pt +[2025-04-24 10:23:51] (step=0028025) Train Loss: 5.4236, Train Steps/Sec: 0.07 +[2025-04-24 10:24:13] (step=0028050) Train Loss: 5.3656, Train Steps/Sec: 1.12 +[2025-04-24 10:24:35] (step=0028075) Train Loss: 5.3292, Train Steps/Sec: 1.17 +[2025-04-24 10:24:56] (step=0028100) Train Loss: 5.4061, Train Steps/Sec: 1.17 +[2025-04-24 10:25:17] (step=0028125) Train Loss: 5.4235, Train Steps/Sec: 1.17 +[2025-04-24 10:25:39] (step=0028150) Train Loss: 5.4079, Train Steps/Sec: 1.17 +[2025-04-24 10:26:00] (step=0028175) Train Loss: 5.4202, Train Steps/Sec: 1.17 +[2025-04-24 10:26:22] (step=0028200) Train Loss: 5.3009, Train Steps/Sec: 1.17 +[2025-04-24 10:26:44] (step=0028225) Train Loss: 5.3656, Train Steps/Sec: 1.12 +[2025-04-24 10:27:06] (step=0028250) Train Loss: 5.4310, Train Steps/Sec: 1.17 +[2025-04-24 10:27:28] (step=0028275) Train Loss: 5.4025, Train Steps/Sec: 1.13 +[2025-04-24 10:27:51] (step=0028300) Train Loss: 5.3900, Train Steps/Sec: 1.08 +[2025-04-24 10:28:12] (step=0028325) Train Loss: 5.4338, Train Steps/Sec: 1.17 +[2025-04-24 10:28:34] (step=0028350) Train Loss: 5.4017, Train Steps/Sec: 1.17 +[2025-04-24 10:28:55] (step=0028375) Train Loss: 5.3586, Train Steps/Sec: 1.17 +[2025-04-24 10:29:17] (step=0028400) Train Loss: 5.3816, Train Steps/Sec: 1.12 +[2025-04-24 10:29:38] (step=0028425) Train Loss: 5.4064, Train Steps/Sec: 1.17 +[2025-04-24 10:30:00] (step=0028450) Train Loss: 5.3963, Train Steps/Sec: 1.17 +[2025-04-24 10:30:21] (step=0028475) Train Loss: 5.4108, Train Steps/Sec: 1.17 +[2025-04-24 10:30:43] (step=0028500) Train Loss: 5.4721, Train Steps/Sec: 1.16 +[2025-04-24 10:31:04] (step=0028525) Train Loss: 5.4207, Train Steps/Sec: 1.17 +[2025-04-24 10:31:25] (step=0028550) Train Loss: 5.3678, Train Steps/Sec: 1.17 +[2025-04-24 10:31:47] (step=0028575) Train Loss: 5.4702, Train Steps/Sec: 1.17 +[2025-04-24 10:32:08] (step=0028600) Train Loss: 5.4019, Train Steps/Sec: 1.16 +[2025-04-24 10:32:31] (step=0028625) Train Loss: 5.4637, Train Steps/Sec: 1.12 +[2025-04-24 10:32:53] (step=0028650) Train Loss: 5.3704, Train Steps/Sec: 1.13 +[2025-04-24 10:33:14] (step=0028675) Train Loss: 5.4430, Train Steps/Sec: 1.17 +[2025-04-24 10:33:36] (step=0028700) Train Loss: 5.3690, Train Steps/Sec: 1.16 +[2025-04-24 10:33:57] (step=0028725) Train Loss: 5.3758, Train Steps/Sec: 1.17 +[2025-04-24 10:34:18] (step=0028750) Train Loss: 5.4547, Train Steps/Sec: 1.17 +[2025-04-24 10:34:41] (step=0028775) Train Loss: 5.3379, Train Steps/Sec: 1.12 +[2025-04-24 10:35:02] (step=0028800) Train Loss: 5.2841, Train Steps/Sec: 1.16 +[2025-04-24 10:35:23] (step=0028825) Train Loss: 5.3991, Train Steps/Sec: 1.17 +[2025-04-24 10:35:45] (step=0028850) Train Loss: 5.3742, Train Steps/Sec: 1.17 +[2025-04-24 10:36:07] (step=0028875) Train Loss: 5.3572, Train Steps/Sec: 1.12 +[2025-04-24 10:36:29] (step=0028900) Train Loss: 5.3873, Train Steps/Sec: 1.16 +[2025-04-24 10:36:50] (step=0028925) Train Loss: 5.4119, Train Steps/Sec: 1.17 +[2025-04-24 10:37:14] (step=0028950) Train Loss: 5.4029, Train Steps/Sec: 1.04 +[2025-04-24 10:37:35] (step=0028975) Train Loss: 5.3675, Train Steps/Sec: 1.17 +[2025-04-24 10:37:57] (step=0029000) Train Loss: 5.3676, Train Steps/Sec: 1.17 +[2025-04-24 10:38:18] (step=0029025) Train Loss: 5.3921, Train Steps/Sec: 1.17 +[2025-04-24 10:38:40] (step=0029050) Train Loss: 5.3798, Train Steps/Sec: 1.13 +[2025-04-24 10:39:02] (step=0029075) Train Loss: 5.3402, Train Steps/Sec: 1.17 +[2025-04-24 10:39:23] (step=0029100) Train Loss: 5.3791, Train Steps/Sec: 1.16 +[2025-04-24 10:39:44] (step=0029125) Train Loss: 5.3776, Train Steps/Sec: 1.17 +[2025-04-24 10:40:06] (step=0029150) Train Loss: 5.4449, Train Steps/Sec: 1.17 +[2025-04-24 10:40:27] (step=0029175) Train Loss: 5.4016, Train Steps/Sec: 1.17 +[2025-04-24 10:40:49] (step=0029200) Train Loss: 5.3775, Train Steps/Sec: 1.17 +[2025-04-24 10:41:10] (step=0029225) Train Loss: 5.4309, Train Steps/Sec: 1.17 +[2025-04-24 10:41:31] (step=0029250) Train Loss: 5.4203, Train Steps/Sec: 1.17 +[2025-04-24 10:41:53] (step=0029275) Train Loss: 5.4673, Train Steps/Sec: 1.17 +[2025-04-24 10:42:14] (step=0029300) Train Loss: 5.4146, Train Steps/Sec: 1.17 +[2025-04-24 10:42:37] (step=0029325) Train Loss: 5.4129, Train Steps/Sec: 1.12 +[2025-04-24 10:42:59] (step=0029350) Train Loss: 5.4137, Train Steps/Sec: 1.11 +[2025-04-24 10:43:20] (step=0029375) Train Loss: 5.4292, Train Steps/Sec: 1.17 +[2025-04-24 10:43:42] (step=0029400) Train Loss: 5.4733, Train Steps/Sec: 1.16 +[2025-04-24 10:44:03] (step=0029425) Train Loss: 5.4613, Train Steps/Sec: 1.17 +[2025-04-24 10:44:25] (step=0029450) Train Loss: 5.3897, Train Steps/Sec: 1.17 +[2025-04-24 10:44:46] (step=0029475) Train Loss: 5.3642, Train Steps/Sec: 1.17 +[2025-04-24 10:45:08] (step=0029500) Train Loss: 5.3665, Train Steps/Sec: 1.11 +[2025-04-24 10:45:30] (step=0029525) Train Loss: 5.3826, Train Steps/Sec: 1.17 +[2025-04-24 10:45:52] (step=0029550) Train Loss: 5.3154, Train Steps/Sec: 1.12 +[2025-04-24 10:46:14] (step=0029575) Train Loss: 5.3159, Train Steps/Sec: 1.17 +[2025-04-24 10:46:36] (step=0029600) Train Loss: 5.3028, Train Steps/Sec: 1.12 +[2025-04-24 10:46:59] (step=0029625) Train Loss: 5.3691, Train Steps/Sec: 1.08 +[2025-04-24 10:47:20] (step=0029650) Train Loss: 5.4709, Train Steps/Sec: 1.17 +[2025-04-24 10:47:42] (step=0029675) Train Loss: 5.4077, Train Steps/Sec: 1.17 +[2025-04-24 10:48:03] (step=0029700) Train Loss: 5.3807, Train Steps/Sec: 1.16 +[2025-04-24 10:48:25] (step=0029725) Train Loss: 5.3125, Train Steps/Sec: 1.13 +[2025-04-24 10:48:47] (step=0029750) Train Loss: 5.4205, Train Steps/Sec: 1.17 +[2025-04-24 10:49:08] (step=0029775) Train Loss: 5.4188, Train Steps/Sec: 1.17 +[2025-04-24 10:49:30] (step=0029800) Train Loss: 5.3808, Train Steps/Sec: 1.16 +[2025-04-24 10:49:51] (step=0029825) Train Loss: 5.3266, Train Steps/Sec: 1.17 +[2025-04-24 10:50:13] (step=0029850) Train Loss: 5.3969, Train Steps/Sec: 1.15 +[2025-04-24 10:50:34] (step=0029875) Train Loss: 5.3466, Train Steps/Sec: 1.17 +[2025-04-24 10:50:56] (step=0029900) Train Loss: 5.3587, Train Steps/Sec: 1.16 +[2025-04-24 10:51:17] (step=0029925) Train Loss: 5.3669, Train Steps/Sec: 1.17 +[2025-04-24 10:51:38] (step=0029950) Train Loss: 5.4132, Train Steps/Sec: 1.17 +[2025-04-24 10:52:00] (step=0029975) Train Loss: 5.3061, Train Steps/Sec: 1.13 +[2025-04-24 10:52:22] (step=0030000) Train Loss: 5.4135, Train Steps/Sec: 1.17 +[2025-04-24 10:52:22] text_config is None. Initializing the text config with default values (`OPTConfig`). +[2025-04-24 10:57:30] Finish Eval in 30000 steps... +[2025-04-24 10:57:49] Saved checkpoint to checkpoint/Afteri2i_100kckpt_LLAVAT5_empty_Fixed_just_segment_pretrain_stage3_dreambench_recap_400k_t2i_400k_flux_200k_midjourney_150k_recovery_150k_3e4_epoch2_no_rotated/000-GPT-XL/checkpoints/0030000.pt +[2025-04-24 10:57:51] Removed old checkpoint: checkpoint/Afteri2i_100kckpt_LLAVAT5_empty_Fixed_just_segment_pretrain_stage3_dreambench_recap_400k_t2i_400k_flux_200k_midjourney_150k_recovery_150k_3e4_epoch2_no_rotated/000-GPT-XL/checkpoints/0028000.pt +[2025-04-24 10:58:12] (step=0030025) Train Loss: 5.4087, Train Steps/Sec: 0.07 +[2025-04-24 10:58:34] (step=0030050) Train Loss: 5.4061, Train Steps/Sec: 1.17 +[2025-04-24 10:58:56] (step=0030075) Train Loss: 5.3454, Train Steps/Sec: 1.12 +[2025-04-24 10:59:17] (step=0030100) Train Loss: 5.3862, Train Steps/Sec: 1.16 +[2025-04-24 10:59:39] (step=0030125) Train Loss: 5.4183, Train Steps/Sec: 1.17 +[2025-04-24 11:00:00] (step=0030150) Train Loss: 5.3874, Train Steps/Sec: 1.15 +[2025-04-24 11:00:22] (step=0030175) Train Loss: 5.3817, Train Steps/Sec: 1.17 +[2025-04-24 11:00:44] (step=0030200) Train Loss: 5.4180, Train Steps/Sec: 1.12 +[2025-04-24 11:01:06] (step=0030225) Train Loss: 5.4399, Train Steps/Sec: 1.12 +[2025-04-24 11:01:29] (step=0030250) Train Loss: 5.3166, Train Steps/Sec: 1.12 +[2025-04-24 11:01:51] (step=0030275) Train Loss: 5.3753, Train Steps/Sec: 1.12 +[2025-04-24 11:02:12] (step=0030300) Train Loss: 5.4426, Train Steps/Sec: 1.17 +[2025-04-24 11:02:34] (step=0030325) Train Loss: 5.3768, Train Steps/Sec: 1.17 +[2025-04-24 11:02:55] (step=0030350) Train Loss: 5.3610, Train Steps/Sec: 1.17 +[2025-04-24 11:03:17] (step=0030375) Train Loss: 5.3753, Train Steps/Sec: 1.13 +[2025-04-24 11:03:39] (step=0030400) Train Loss: 5.4298, Train Steps/Sec: 1.16 +[2025-04-24 11:04:00] (step=0030425) Train Loss: 5.3649, Train Steps/Sec: 1.17 +[2025-04-24 11:04:22] (step=0030450) Train Loss: 5.4641, Train Steps/Sec: 1.17 +[2025-04-24 11:04:43] (step=0030475) Train Loss: 5.4222, Train Steps/Sec: 1.17 +[2025-04-24 11:05:04] (step=0030500) Train Loss: 5.4516, Train Steps/Sec: 1.16 +[2025-04-24 11:05:26] (step=0030525) Train Loss: 5.4039, Train Steps/Sec: 1.17 +[2025-04-24 11:05:47] (step=0030550) Train Loss: 5.3727, Train Steps/Sec: 1.17 +[2025-04-24 11:06:09] (step=0030575) Train Loss: 5.3517, Train Steps/Sec: 1.17 +[2025-04-24 11:06:30] (step=0030600) Train Loss: 5.4230, Train Steps/Sec: 1.17 +[2025-04-24 11:06:51] (step=0030625) Train Loss: 5.3040, Train Steps/Sec: 1.17 +[2025-04-24 11:07:13] (step=0030650) Train Loss: 5.3773, Train Steps/Sec: 1.17 +[2025-04-24 11:07:35] (step=0030675) Train Loss: 5.3824, Train Steps/Sec: 1.13 +[2025-04-24 11:07:56] (step=0030700) Train Loss: 5.4066, Train Steps/Sec: 1.17 +[2025-04-24 11:08:18] (step=0030725) Train Loss: 5.3707, Train Steps/Sec: 1.17 +[2025-04-24 11:08:39] (step=0030750) Train Loss: 5.3374, Train Steps/Sec: 1.17 +[2025-04-24 11:09:00] (step=0030775) Train Loss: 5.3552, Train Steps/Sec: 1.17 +[2025-04-24 11:09:23] (step=0030800) Train Loss: 5.3624, Train Steps/Sec: 1.11 +[2025-04-24 11:09:44] (step=0030825) Train Loss: 5.4207, Train Steps/Sec: 1.17 +[2025-04-24 11:10:07] (step=0030850) Train Loss: 5.3388, Train Steps/Sec: 1.12 +[2025-04-24 11:10:28] (step=0030875) Train Loss: 5.4066, Train Steps/Sec: 1.17 +[2025-04-24 11:10:49] (step=0030900) Train Loss: 5.3789, Train Steps/Sec: 1.16 +[2025-04-24 11:11:12] (step=0030925) Train Loss: 5.3748, Train Steps/Sec: 1.09 +[2025-04-24 11:11:36] (step=0030950) Train Loss: 5.4025, Train Steps/Sec: 1.07 +[2025-04-24 11:11:57] (step=0030975) Train Loss: 5.4025, Train Steps/Sec: 1.17 +[2025-04-24 11:12:18] (step=0031000) Train Loss: 5.3641, Train Steps/Sec: 1.17 +[2025-04-24 11:12:41] (step=0031025) Train Loss: 5.3488, Train Steps/Sec: 1.13 +[2025-04-24 11:13:02] (step=0031050) Train Loss: 5.3697, Train Steps/Sec: 1.17 +[2025-04-24 11:13:23] (step=0031075) Train Loss: 5.3547, Train Steps/Sec: 1.17 +[2025-04-24 11:13:45] (step=0031100) Train Loss: 5.4885, Train Steps/Sec: 1.16 +[2025-04-24 11:14:06] (step=0031125) Train Loss: 5.4072, Train Steps/Sec: 1.17 +[2025-04-24 11:14:28] (step=0031150) Train Loss: 5.3982, Train Steps/Sec: 1.17 +[2025-04-24 11:14:49] (step=0031175) Train Loss: 5.4047, Train Steps/Sec: 1.17 +[2025-04-24 11:15:10] (step=0031200) Train Loss: 5.4679, Train Steps/Sec: 1.17 +[2025-04-24 11:15:32] (step=0031225) Train Loss: 5.3877, Train Steps/Sec: 1.17 +[2025-04-24 11:15:53] (step=0031250) Train Loss: 5.4221, Train Steps/Sec: 1.17 +[2025-04-24 11:16:14] (step=0031275) Train Loss: 5.3549, Train Steps/Sec: 1.17 +[2025-04-24 11:16:36] (step=0031300) Train Loss: 5.4628, Train Steps/Sec: 1.17 +[2025-04-24 11:16:58] (step=0031325) Train Loss: 5.3999, Train Steps/Sec: 1.12 +[2025-04-24 11:17:19] (step=0031350) Train Loss: 5.3702, Train Steps/Sec: 1.17 +[2025-04-24 11:17:41] (step=0031375) Train Loss: 5.4457, Train Steps/Sec: 1.17 +[2025-04-24 11:18:02] (step=0031400) Train Loss: 5.3383, Train Steps/Sec: 1.17 +[2025-04-24 11:18:24] (step=0031425) Train Loss: 5.3851, Train Steps/Sec: 1.17 +[2025-04-24 11:18:45] (step=0031450) Train Loss: 5.4595, Train Steps/Sec: 1.17 +[2025-04-24 11:19:06] (step=0031475) Train Loss: 5.3589, Train Steps/Sec: 1.17 +[2025-04-24 11:19:29] (step=0031500) Train Loss: 5.2657, Train Steps/Sec: 1.11 +[2025-04-24 11:19:51] (step=0031525) Train Loss: 5.4379, Train Steps/Sec: 1.12 +[2025-04-24 11:20:13] (step=0031550) Train Loss: 5.3626, Train Steps/Sec: 1.17 +[2025-04-24 11:20:35] (step=0031575) Train Loss: 5.3889, Train Steps/Sec: 1.13 +[2025-04-24 11:20:58] (step=0031600) Train Loss: 5.4181, Train Steps/Sec: 1.08 +[2025-04-24 11:21:19] (step=0031625) Train Loss: 5.3756, Train Steps/Sec: 1.17 +[2025-04-24 11:21:41] (step=0031650) Train Loss: 5.3532, Train Steps/Sec: 1.17 +[2025-04-24 11:22:03] (step=0031675) Train Loss: 5.3470, Train Steps/Sec: 1.12 +[2025-04-24 11:22:25] (step=0031700) Train Loss: 5.4187, Train Steps/Sec: 1.12 +[2025-04-24 11:22:47] (step=0031725) Train Loss: 5.3863, Train Steps/Sec: 1.17 +[2025-04-24 11:23:08] (step=0031750) Train Loss: 5.3666, Train Steps/Sec: 1.17 +[2025-04-24 11:23:29] (step=0031775) Train Loss: 5.4164, Train Steps/Sec: 1.17 +[2025-04-24 11:23:51] (step=0031800) Train Loss: 5.4742, Train Steps/Sec: 1.17 +[2025-04-24 11:24:12] (step=0031825) Train Loss: 5.4010, Train Steps/Sec: 1.18 +[2025-04-24 11:24:33] (step=0031850) Train Loss: 5.4572, Train Steps/Sec: 1.17 +[2025-04-24 11:24:55] (step=0031875) Train Loss: 5.4126, Train Steps/Sec: 1.17 +[2025-04-24 11:25:16] (step=0031900) Train Loss: 5.3960, Train Steps/Sec: 1.16 +[2025-04-24 11:25:38] (step=0031925) Train Loss: 5.3739, Train Steps/Sec: 1.17 +[2025-04-24 11:25:59] (step=0031950) Train Loss: 5.3678, Train Steps/Sec: 1.17 +[2025-04-24 11:26:20] (step=0031975) Train Loss: 5.4218, Train Steps/Sec: 1.17 +[2025-04-24 11:26:43] (step=0032000) Train Loss: 5.3500, Train Steps/Sec: 1.12 +[2025-04-24 11:26:43] text_config is None. Initializing the text config with default values (`OPTConfig`). +[2025-04-24 11:31:50] Finish Eval in 32000 steps... +[2025-04-24 11:32:11] Saved checkpoint to checkpoint/Afteri2i_100kckpt_LLAVAT5_empty_Fixed_just_segment_pretrain_stage3_dreambench_recap_400k_t2i_400k_flux_200k_midjourney_150k_recovery_150k_3e4_epoch2_no_rotated/000-GPT-XL/checkpoints/0032000.pt +[2025-04-24 11:32:13] Removed old checkpoint: checkpoint/Afteri2i_100kckpt_LLAVAT5_empty_Fixed_just_segment_pretrain_stage3_dreambench_recap_400k_t2i_400k_flux_200k_midjourney_150k_recovery_150k_3e4_epoch2_no_rotated/000-GPT-XL/checkpoints/0030000.pt +[2025-04-24 11:32:34] (step=0032025) Train Loss: 5.4388, Train Steps/Sec: 0.07 +[2025-04-24 11:32:55] (step=0032050) Train Loss: 5.3813, Train Steps/Sec: 1.17 +[2025-04-24 11:33:17] (step=0032075) Train Loss: 5.3727, Train Steps/Sec: 1.15 +[2025-04-24 11:33:39] (step=0032100) Train Loss: 5.3919, Train Steps/Sec: 1.17 +[2025-04-24 11:34:00] (step=0032125) Train Loss: 5.3524, Train Steps/Sec: 1.17 +[2025-04-24 11:34:21] (step=0032150) Train Loss: 5.4180, Train Steps/Sec: 1.17 +[2025-04-24 11:34:44] (step=0032175) Train Loss: 5.3888, Train Steps/Sec: 1.12 +[2025-04-24 11:35:05] (step=0032200) Train Loss: 5.4044, Train Steps/Sec: 1.17 +[2025-04-24 11:35:28] (step=0032225) Train Loss: 5.4023, Train Steps/Sec: 1.07 +[2025-04-24 11:35:52] (step=0032250) Train Loss: 5.3870, Train Steps/Sec: 1.08 +[2025-04-24 11:36:13] (step=0032275) Train Loss: 5.3983, Train Steps/Sec: 1.17 +[2025-04-24 11:36:34] (step=0032300) Train Loss: 5.4428, Train Steps/Sec: 1.17 +[2025-04-24 11:36:56] (step=0032325) Train Loss: 5.3910, Train Steps/Sec: 1.17 +[2025-04-24 11:37:18] (step=0032350) Train Loss: 5.4299, Train Steps/Sec: 1.12 +[2025-04-24 11:37:39] (step=0032375) Train Loss: 5.3534, Train Steps/Sec: 1.17 +[2025-04-24 11:38:02] (step=0032400) Train Loss: 5.4174, Train Steps/Sec: 1.11 +[2025-04-24 11:38:23] (step=0032425) Train Loss: 5.3892, Train Steps/Sec: 1.17 +[2025-04-24 11:38:45] (step=0032450) Train Loss: 5.4092, Train Steps/Sec: 1.17 +[2025-04-24 11:39:06] (step=0032475) Train Loss: 5.4718, Train Steps/Sec: 1.17 +[2025-04-24 11:39:27] (step=0032500) Train Loss: 5.3994, Train Steps/Sec: 1.17 +[2025-04-24 11:39:49] (step=0032525) Train Loss: 5.4240, Train Steps/Sec: 1.17 +[2025-04-24 11:40:10] (step=0032550) Train Loss: 5.3366, Train Steps/Sec: 1.17 +[2025-04-24 11:40:31] (step=0032575) Train Loss: 5.3629, Train Steps/Sec: 1.17 +[2025-04-24 11:40:53] (step=0032600) Train Loss: 5.3764, Train Steps/Sec: 1.17 +[2025-04-24 11:41:14] (step=0032625) Train Loss: 5.3574, Train Steps/Sec: 1.17 +[2025-04-24 11:41:37] (step=0032650) Train Loss: 5.2824, Train Steps/Sec: 1.12 +[2025-04-24 11:41:58] (step=0032675) Train Loss: 5.3770, Train Steps/Sec: 1.17 +[2025-04-24 11:42:19] (step=0032700) Train Loss: 5.3612, Train Steps/Sec: 1.17 +[2025-04-24 11:42:41] (step=0032725) Train Loss: 5.3932, Train Steps/Sec: 1.17 +[2025-04-24 11:43:02] (step=0032750) Train Loss: 5.3931, Train Steps/Sec: 1.16 +[2025-04-24 11:43:24] (step=0032775) Train Loss: 5.4068, Train Steps/Sec: 1.17 +[2025-04-24 11:43:45] (step=0032800) Train Loss: 5.4416, Train Steps/Sec: 1.17 +[2025-04-24 11:44:07] (step=0032825) Train Loss: 5.3992, Train Steps/Sec: 1.12 +[2025-04-24 11:44:29] (step=0032850) Train Loss: 5.4042, Train Steps/Sec: 1.17 +[2025-04-24 11:44:50] (step=0032875) Train Loss: 5.3902, Train Steps/Sec: 1.17 +[2025-04-24 11:45:14] (step=0032900) Train Loss: 5.3895, Train Steps/Sec: 1.04 +[2025-04-24 11:45:35] (step=0032925) Train Loss: 5.3514, Train Steps/Sec: 1.17 +[2025-04-24 11:45:58] (step=0032950) Train Loss: 5.3612, Train Steps/Sec: 1.12 +[2025-04-24 11:46:19] (step=0032975) Train Loss: 5.4407, Train Steps/Sec: 1.17 +[2025-04-24 11:46:41] (step=0033000) Train Loss: 5.3833, Train Steps/Sec: 1.12 +[2025-04-24 11:47:03] (step=0033025) Train Loss: 5.4045, Train Steps/Sec: 1.17 +[2025-04-24 11:47:24] (step=0033050) Train Loss: 5.4354, Train Steps/Sec: 1.17 +[2025-04-24 11:47:45] (step=0033075) Train Loss: 5.3907, Train Steps/Sec: 1.17 +[2025-04-24 11:48:07] (step=0033100) Train Loss: 5.4221, Train Steps/Sec: 1.17 +[2025-04-24 11:48:29] (step=0033125) Train Loss: 5.3490, Train Steps/Sec: 1.12 +[2025-04-24 11:48:51] (step=0033150) Train Loss: 5.3860, Train Steps/Sec: 1.17 +[2025-04-24 11:49:12] (step=0033175) Train Loss: 5.4320, Train Steps/Sec: 1.17 +[2025-04-24 11:49:33] (step=0033200) Train Loss: 5.4085, Train Steps/Sec: 1.17 +[2025-04-24 11:49:55] (step=0033225) Train Loss: 5.3728, Train Steps/Sec: 1.17 +[2025-04-24 11:50:16] (step=0033250) Train Loss: 5.3358, Train Steps/Sec: 1.17 +[2025-04-24 11:50:37] (step=0033275) Train Loss: 5.3519, Train Steps/Sec: 1.17 +[2025-04-24 11:50:59] (step=0033300) Train Loss: 5.3360, Train Steps/Sec: 1.17 +[2025-04-24 11:51:21] (step=0033325) Train Loss: 5.4604, Train Steps/Sec: 1.12 +[2025-04-24 11:51:42] (step=0033350) Train Loss: 5.3979, Train Steps/Sec: 1.17 +[2025-04-24 11:52:04] (step=0033375) Train Loss: 5.2429, Train Steps/Sec: 1.17 +[2025-04-24 11:52:25] (step=0033400) Train Loss: 5.4023, Train Steps/Sec: 1.16 +[2025-04-24 11:52:47] (step=0033425) Train Loss: 5.4474, Train Steps/Sec: 1.17 +[2025-04-24 11:53:08] (step=0033450) Train Loss: 5.3556, Train Steps/Sec: 1.17 +[2025-04-24 11:53:30] (step=0033475) Train Loss: 5.4534, Train Steps/Sec: 1.12 +[2025-04-24 11:53:52] (step=0033500) Train Loss: 5.4402, Train Steps/Sec: 1.17 +[2025-04-24 11:54:13] (step=0033525) Train Loss: 5.3372, Train Steps/Sec: 1.17 +[2025-04-24 11:54:35] (step=0033550) Train Loss: 5.4119, Train Steps/Sec: 1.13 +[2025-04-24 11:54:58] (step=0033575) Train Loss: 5.4003, Train Steps/Sec: 1.08 +[2025-04-24 11:55:20] (step=0033600) Train Loss: 5.3447, Train Steps/Sec: 1.17 +[2025-04-24 11:55:41] (step=0033625) Train Loss: 5.3598, Train Steps/Sec: 1.17 +[2025-04-24 11:56:03] (step=0033650) Train Loss: 5.4084, Train Steps/Sec: 1.17 +[2025-04-24 11:56:26] (step=0033675) Train Loss: 5.4417, Train Steps/Sec: 1.07 +[2025-04-24 11:56:47] (step=0033700) Train Loss: 5.3953, Train Steps/Sec: 1.17 +[2025-04-24 11:57:09] (step=0033725) Train Loss: 5.3879, Train Steps/Sec: 1.17 +[2025-04-24 11:57:30] (step=0033750) Train Loss: 5.3586, Train Steps/Sec: 1.17 +[2025-04-24 11:57:51] (step=0033775) Train Loss: 5.4001, Train Steps/Sec: 1.17 +[2025-04-24 11:58:13] (step=0033800) Train Loss: 5.4391, Train Steps/Sec: 1.17 +[2025-04-24 11:58:34] (step=0033825) Train Loss: 5.3674, Train Steps/Sec: 1.17 +[2025-04-24 11:58:56] (step=0033850) Train Loss: 5.3565, Train Steps/Sec: 1.12 +[2025-04-24 11:59:18] (step=0033875) Train Loss: 5.3990, Train Steps/Sec: 1.17 +[2025-04-24 11:59:39] (step=0033900) Train Loss: 5.3598, Train Steps/Sec: 1.17 +[2025-04-24 12:00:01] (step=0033925) Train Loss: 5.4551, Train Steps/Sec: 1.17 +[2025-04-24 12:00:22] (step=0033950) Train Loss: 5.3448, Train Steps/Sec: 1.17 +[2025-04-24 12:00:44] (step=0033975) Train Loss: 5.4253, Train Steps/Sec: 1.13 +[2025-04-24 12:01:05] (step=0034000) Train Loss: 5.3884, Train Steps/Sec: 1.17 +[2025-04-24 12:01:05] text_config is None. Initializing the text config with default values (`OPTConfig`). +[2025-04-24 12:06:13] Finish Eval in 34000 steps... +[2025-04-24 12:06:33] Saved checkpoint to checkpoint/Afteri2i_100kckpt_LLAVAT5_empty_Fixed_just_segment_pretrain_stage3_dreambench_recap_400k_t2i_400k_flux_200k_midjourney_150k_recovery_150k_3e4_epoch2_no_rotated/000-GPT-XL/checkpoints/0034000.pt +[2025-04-24 12:06:35] Removed old checkpoint: checkpoint/Afteri2i_100kckpt_LLAVAT5_empty_Fixed_just_segment_pretrain_stage3_dreambench_recap_400k_t2i_400k_flux_200k_midjourney_150k_recovery_150k_3e4_epoch2_no_rotated/000-GPT-XL/checkpoints/0032000.pt +[2025-04-24 12:06:57] (step=0034025) Train Loss: 5.4368, Train Steps/Sec: 0.07 +[2025-04-24 12:07:18] (step=0034050) Train Loss: 5.2764, Train Steps/Sec: 1.17 +[2025-04-24 12:07:39] (step=0034075) Train Loss: 5.4176, Train Steps/Sec: 1.17 +[2025-04-24 12:08:01] (step=0034100) Train Loss: 5.3790, Train Steps/Sec: 1.17 +[2025-04-24 12:08:22] (step=0034125) Train Loss: 5.3458, Train Steps/Sec: 1.17 +[2025-04-24 12:08:44] (step=0034150) Train Loss: 5.3604, Train Steps/Sec: 1.12 +[2025-04-24 12:09:06] (step=0034175) Train Loss: 5.4141, Train Steps/Sec: 1.17 +[2025-04-24 12:09:28] (step=0034200) Train Loss: 5.3816, Train Steps/Sec: 1.12 +[2025-04-24 12:09:51] (step=0034225) Train Loss: 5.3233, Train Steps/Sec: 1.09 +[2025-04-24 12:10:12] (step=0034250) Train Loss: 5.3481, Train Steps/Sec: 1.17 +[2025-04-24 12:10:34] (step=0034275) Train Loss: 5.4116, Train Steps/Sec: 1.17 +[2025-04-24 12:10:55] (step=0034300) Train Loss: 5.4265, Train Steps/Sec: 1.17 +[2025-04-24 12:11:17] (step=0034325) Train Loss: 5.4159, Train Steps/Sec: 1.13 +[2025-04-24 12:11:39] (step=0034350) Train Loss: 5.4816, Train Steps/Sec: 1.17 +[2025-04-24 12:12:00] (step=0034375) Train Loss: 5.4426, Train Steps/Sec: 1.17 +[2025-04-24 12:12:23] (step=0034400) Train Loss: 5.3004, Train Steps/Sec: 1.11 +[2025-04-24 12:12:44] (step=0034425) Train Loss: 5.3355, Train Steps/Sec: 1.17 +[2025-04-24 12:13:05] (step=0034450) Train Loss: 5.3511, Train Steps/Sec: 1.17 +[2025-04-24 12:13:27] (step=0034475) Train Loss: 5.4426, Train Steps/Sec: 1.17 +[2025-04-24 12:13:48] (step=0034500) Train Loss: 5.3739, Train Steps/Sec: 1.16 +[2025-04-24 12:14:09] (step=0034525) Train Loss: 5.3407, Train Steps/Sec: 1.17 +[2025-04-24 12:14:31] (step=0034550) Train Loss: 5.3222, Train Steps/Sec: 1.17 +[2025-04-24 12:14:53] (step=0034575) Train Loss: 5.3590, Train Steps/Sec: 1.12 +[2025-04-24 12:15:15] (step=0034600) Train Loss: 5.4122, Train Steps/Sec: 1.16 +[2025-04-24 12:15:36] (step=0034625) Train Loss: 5.4179, Train Steps/Sec: 1.17 +[2025-04-24 12:15:57] (step=0034650) Train Loss: 5.4255, Train Steps/Sec: 1.17 +[2025-04-24 12:16:20] (step=0034675) Train Loss: 5.3072, Train Steps/Sec: 1.12 +[2025-04-24 12:16:41] (step=0034700) Train Loss: 5.3647, Train Steps/Sec: 1.16 +[2025-04-24 12:17:03] (step=0034725) Train Loss: 5.4027, Train Steps/Sec: 1.17 +[2025-04-24 12:17:24] (step=0034750) Train Loss: 5.3723, Train Steps/Sec: 1.17 +[2025-04-24 12:17:45] (step=0034775) Train Loss: 5.4882, Train Steps/Sec: 1.17 +[2025-04-24 12:18:07] (step=0034800) Train Loss: 5.3720, Train Steps/Sec: 1.12 +[2025-04-24 12:18:29] (step=0034825) Train Loss: 5.3544, Train Steps/Sec: 1.17 +[2025-04-24 12:18:50] (step=0034850) Train Loss: 5.4132, Train Steps/Sec: 1.17 +[2025-04-24 12:19:13] (step=0034875) Train Loss: 5.3859, Train Steps/Sec: 1.08 +[2025-04-24 12:19:36] (step=0034900) Train Loss: 5.4274, Train Steps/Sec: 1.12 +[2025-04-24 12:19:57] (step=0034925) Train Loss: 5.2312, Train Steps/Sec: 1.17 +[2025-04-24 12:20:18] (step=0034950) Train Loss: 5.3792, Train Steps/Sec: 1.17 +[2025-04-24 12:20:40] (step=0034975) Train Loss: 5.3106, Train Steps/Sec: 1.12 +[2025-04-24 12:21:02] (step=0035000) Train Loss: 5.4351, Train Steps/Sec: 1.16 +[2025-04-24 12:21:23] (step=0035025) Train Loss: 5.3471, Train Steps/Sec: 1.17 +[2025-04-24 12:21:45] (step=0035050) Train Loss: 5.4253, Train Steps/Sec: 1.17 +[2025-04-24 12:22:06] (step=0035075) Train Loss: 5.4213, Train Steps/Sec: 1.17 +[2025-04-24 12:22:28] (step=0035100) Train Loss: 5.3740, Train Steps/Sec: 1.17 +[2025-04-24 12:22:50] (step=0035125) Train Loss: 5.4672, Train Steps/Sec: 1.12 +[2025-04-24 12:23:11] (step=0035150) Train Loss: 5.4553, Train Steps/Sec: 1.17 +[2025-04-24 12:23:33] (step=0035175) Train Loss: 5.3695, Train Steps/Sec: 1.17 +[2025-04-24 12:23:54] (step=0035200) Train Loss: 5.4445, Train Steps/Sec: 1.16 +[2025-04-24 12:24:16] (step=0035225) Train Loss: 5.3601, Train Steps/Sec: 1.17 +[2025-04-24 12:24:37] (step=0035250) Train Loss: 5.3907, Train Steps/Sec: 1.17 +[2025-04-24 12:24:58] (step=0035275) Train Loss: 5.4913, Train Steps/Sec: 1.17 +[2025-04-24 12:25:21] (step=0035300) Train Loss: 5.4522, Train Steps/Sec: 1.11 +[2025-04-24 12:25:43] (step=0035325) Train Loss: 5.3313, Train Steps/Sec: 1.12 +[2025-04-24 12:26:04] (step=0035350) Train Loss: 5.4184, Train Steps/Sec: 1.17 +[2025-04-24 12:26:26] (step=0035375) Train Loss: 5.3622, Train Steps/Sec: 1.17 +[2025-04-24 12:26:47] (step=0035400) Train Loss: 5.4354, Train Steps/Sec: 1.16 +[2025-04-24 12:27:09] (step=0035425) Train Loss: 5.3684, Train Steps/Sec: 1.17 +[2025-04-24 12:27:31] (step=0035450) Train Loss: 5.3190, Train Steps/Sec: 1.12 +[2025-04-24 12:27:52] (step=0035475) Train Loss: 5.3557, Train Steps/Sec: 1.17 +[2025-04-24 12:28:14] (step=0035500) Train Loss: 5.3941, Train Steps/Sec: 1.17 +[2025-04-24 12:28:36] (step=0035525) Train Loss: 5.4323, Train Steps/Sec: 1.13 +[2025-04-24 12:28:59] (step=0035550) Train Loss: 5.3760, Train Steps/Sec: 1.08 +[2025-04-24 12:29:20] (step=0035575) Train Loss: 5.3458, Train Steps/Sec: 1.17 +[2025-04-24 12:29:42] (step=0035600) Train Loss: 5.3111, Train Steps/Sec: 1.17 +[2025-04-24 12:30:03] (step=0035625) Train Loss: 5.4294, Train Steps/Sec: 1.17 +[2025-04-24 12:30:25] (step=0035650) Train Loss: 5.4234, Train Steps/Sec: 1.13 +[2025-04-24 12:30:47] (step=0035675) Train Loss: 5.3680, Train Steps/Sec: 1.17 +[2025-04-24 12:31:08] (step=0035700) Train Loss: 5.3564, Train Steps/Sec: 1.17 +[2025-04-24 12:31:29] (step=0035725) Train Loss: 5.3365, Train Steps/Sec: 1.17 +[2025-04-24 12:31:51] (step=0035750) Train Loss: 5.4007, Train Steps/Sec: 1.17 +[2025-04-24 12:32:12] (step=0035775) Train Loss: 5.3369, Train Steps/Sec: 1.17 +[2025-04-24 12:32:33] (step=0035800) Train Loss: 5.3929, Train Steps/Sec: 1.17 +[2025-04-24 12:32:55] (step=0035825) Train Loss: 5.3558, Train Steps/Sec: 1.17 +[2025-04-24 12:33:17] (step=0035850) Train Loss: 5.3544, Train Steps/Sec: 1.12 +[2025-04-24 12:33:39] (step=0035875) Train Loss: 5.3696, Train Steps/Sec: 1.17 +[2025-04-24 12:34:00] (step=0035900) Train Loss: 5.3462, Train Steps/Sec: 1.17 +[2025-04-24 12:34:21] (step=0035925) Train Loss: 5.4178, Train Steps/Sec: 1.17 +[2025-04-24 12:34:43] (step=0035950) Train Loss: 5.3463, Train Steps/Sec: 1.17 +[2025-04-24 12:35:05] (step=0035975) Train Loss: 5.2341, Train Steps/Sec: 1.13 +[2025-04-24 12:35:26] (step=0036000) Train Loss: 5.3826, Train Steps/Sec: 1.17 +[2025-04-24 12:35:26] text_config is None. Initializing the text config with default values (`OPTConfig`). +[2025-04-24 12:40:35] Finish Eval in 36000 steps... +[2025-04-24 12:40:54] Saved checkpoint to checkpoint/Afteri2i_100kckpt_LLAVAT5_empty_Fixed_just_segment_pretrain_stage3_dreambench_recap_400k_t2i_400k_flux_200k_midjourney_150k_recovery_150k_3e4_epoch2_no_rotated/000-GPT-XL/checkpoints/0036000.pt +[2025-04-24 12:40:56] Removed old checkpoint: checkpoint/Afteri2i_100kckpt_LLAVAT5_empty_Fixed_just_segment_pretrain_stage3_dreambench_recap_400k_t2i_400k_flux_200k_midjourney_150k_recovery_150k_3e4_epoch2_no_rotated/000-GPT-XL/checkpoints/0034000.pt +[2025-04-24 12:41:18] (step=0036025) Train Loss: 5.3762, Train Steps/Sec: 0.07 +[2025-04-24 12:41:40] (step=0036050) Train Loss: 5.3532, Train Steps/Sec: 1.17 +[2025-04-24 12:42:01] (step=0036075) Train Loss: 5.3544, Train Steps/Sec: 1.17 +[2025-04-24 12:42:23] (step=0036100) Train Loss: 5.3530, Train Steps/Sec: 1.17 +[2025-04-24 12:42:45] (step=0036125) Train Loss: 5.4050, Train Steps/Sec: 1.12 +[2025-04-24 12:43:06] (step=0036150) Train Loss: 5.3430, Train Steps/Sec: 1.17 +[2025-04-24 12:43:27] (step=0036175) Train Loss: 5.3489, Train Steps/Sec: 1.17 +[2025-04-24 12:43:51] (step=0036200) Train Loss: 5.4462, Train Steps/Sec: 1.04 +[2025-04-24 12:44:13] (step=0036225) Train Loss: 5.3558, Train Steps/Sec: 1.17 +[2025-04-24 12:44:34] (step=0036250) Train Loss: 5.3332, Train Steps/Sec: 1.17 +[2025-04-24 12:44:56] (step=0036275) Train Loss: 5.4207, Train Steps/Sec: 1.17 +[2025-04-24 12:45:18] (step=0036300) Train Loss: 5.3440, Train Steps/Sec: 1.12 +[2025-04-24 12:45:39] (step=0036325) Train Loss: 5.3846, Train Steps/Sec: 1.17 +[2025-04-24 12:46:01] (step=0036350) Train Loss: 5.4336, Train Steps/Sec: 1.17 +[2025-04-24 12:46:22] (step=0036375) Train Loss: 5.3890, Train Steps/Sec: 1.17 +[2025-04-24 12:46:43] (step=0036400) Train Loss: 5.4087, Train Steps/Sec: 1.17 +[2025-04-24 12:47:05] (step=0036425) Train Loss: 5.4143, Train Steps/Sec: 1.17 +[2025-04-24 12:47:26] (step=0036450) Train Loss: 5.3947, Train Steps/Sec: 1.17 +[2025-04-24 12:47:47] (step=0036475) Train Loss: 5.3427, Train Steps/Sec: 1.17 +[2025-04-24 12:48:09] (step=0036500) Train Loss: 5.3745, Train Steps/Sec: 1.17 +[2025-04-24 12:48:30] (step=0036525) Train Loss: 5.2770, Train Steps/Sec: 1.17 +[2025-04-24 12:48:52] (step=0036550) Train Loss: 5.3725, Train Steps/Sec: 1.17 +[2025-04-24 12:49:14] (step=0036575) Train Loss: 5.3753, Train Steps/Sec: 1.12 +[2025-04-24 12:49:36] (step=0036600) Train Loss: 5.4663, Train Steps/Sec: 1.15 +[2025-04-24 12:49:57] (step=0036625) Train Loss: 5.3558, Train Steps/Sec: 1.17 +[2025-04-24 12:50:19] (step=0036650) Train Loss: 5.3299, Train Steps/Sec: 1.17 +[2025-04-24 12:50:41] (step=0036675) Train Loss: 5.3964, Train Steps/Sec: 1.12 +[2025-04-24 12:51:02] (step=0036700) Train Loss: 5.4503, Train Steps/Sec: 1.17 +[2025-04-24 12:51:24] (step=0036725) Train Loss: 5.3401, Train Steps/Sec: 1.17 +[2025-04-24 12:51:46] (step=0036750) Train Loss: 5.4083, Train Steps/Sec: 1.12 +[2025-04-24 12:52:08] (step=0036775) Train Loss: 5.3950, Train Steps/Sec: 1.12 +[2025-04-24 12:52:30] (step=0036800) Train Loss: 5.4297, Train Steps/Sec: 1.17 +[2025-04-24 12:52:51] (step=0036825) Train Loss: 5.3609, Train Steps/Sec: 1.17 +[2025-04-24 12:53:14] (step=0036850) Train Loss: 5.3463, Train Steps/Sec: 1.08 +[2025-04-24 12:53:36] (step=0036875) Train Loss: 5.3616, Train Steps/Sec: 1.13 +[2025-04-24 12:53:58] (step=0036900) Train Loss: 5.4154, Train Steps/Sec: 1.17 +[2025-04-24 12:54:19] (step=0036925) Train Loss: 5.3826, Train Steps/Sec: 1.17 +[2025-04-24 12:54:41] (step=0036950) Train Loss: 5.3976, Train Steps/Sec: 1.13 +[2025-04-24 12:55:03] (step=0036975) Train Loss: 5.3232, Train Steps/Sec: 1.17 +[2025-04-24 12:55:24] (step=0037000) Train Loss: 5.4052, Train Steps/Sec: 1.17 +[2025-04-24 12:55:45] (step=0037025) Train Loss: 5.3924, Train Steps/Sec: 1.17 +[2025-04-24 12:56:07] (step=0037050) Train Loss: 5.4273, Train Steps/Sec: 1.17 +[2025-04-24 12:56:28] (step=0037075) Train Loss: 5.3861, Train Steps/Sec: 1.17 +[2025-04-24 12:56:50] (step=0037100) Train Loss: 5.4010, Train Steps/Sec: 1.16 +[2025-04-24 12:57:11] (step=0037125) Train Loss: 5.3616, Train Steps/Sec: 1.17 +[2025-04-24 12:57:32] (step=0037150) Train Loss: 5.3275, Train Steps/Sec: 1.17 +[2025-04-24 12:57:54] (step=0037175) Train Loss: 5.4265, Train Steps/Sec: 1.17 +[2025-04-24 12:58:15] (step=0037200) Train Loss: 5.4046, Train Steps/Sec: 1.17 +[2025-04-24 12:58:36] (step=0037225) Train Loss: 5.4050, Train Steps/Sec: 1.17 +[2025-04-24 12:58:58] (step=0037250) Train Loss: 5.3538, Train Steps/Sec: 1.17 +[2025-04-24 12:59:19] (step=0037275) Train Loss: 5.3219, Train Steps/Sec: 1.17 +[2025-04-24 12:59:42] (step=0037300) Train Loss: 5.3880, Train Steps/Sec: 1.11 +[2025-04-24 13:00:04] (step=0037325) Train Loss: 5.4442, Train Steps/Sec: 1.13 +[2025-04-24 13:00:25] (step=0037350) Train Loss: 5.3454, Train Steps/Sec: 1.17 +[2025-04-24 13:00:46] (step=0037375) Train Loss: 5.3918, Train Steps/Sec: 1.17 +[2025-04-24 13:01:08] (step=0037400) Train Loss: 5.4611, Train Steps/Sec: 1.17 +[2025-04-24 13:01:30] (step=0037425) Train Loss: 5.4276, Train Steps/Sec: 1.12 +[2025-04-24 13:01:51] (step=0037450) Train Loss: 5.4460, Train Steps/Sec: 1.17 +[2025-04-24 13:02:14] (step=0037475) Train Loss: 5.3067, Train Steps/Sec: 1.12 +[2025-04-24 13:02:36] (step=0037500) Train Loss: 5.3716, Train Steps/Sec: 1.13 +[2025-04-24 13:02:59] (step=0037525) Train Loss: 5.4218, Train Steps/Sec: 1.08 +[2025-04-24 13:03:20] (step=0037550) Train Loss: 5.4343, Train Steps/Sec: 1.17 +[2025-04-24 13:03:42] (step=0037575) Train Loss: 5.3231, Train Steps/Sec: 1.17 +[2025-04-24 13:04:03] (step=0037600) Train Loss: 5.3819, Train Steps/Sec: 1.17 +[2025-04-24 13:04:25] (step=0037625) Train Loss: 5.4150, Train Steps/Sec: 1.13 +[2025-04-24 13:04:47] (step=0037650) Train Loss: 5.3797, Train Steps/Sec: 1.17 +[2025-04-24 13:05:08] (step=0037675) Train Loss: 5.3587, Train Steps/Sec: 1.17 +[2025-04-24 13:05:29] (step=0037700) Train Loss: 5.4128, Train Steps/Sec: 1.17 +[2025-04-24 13:05:51] (step=0037725) Train Loss: 5.4440, Train Steps/Sec: 1.17 +[2025-04-24 13:06:12] (step=0037750) Train Loss: 5.4347, Train Steps/Sec: 1.17 +[2025-04-24 13:06:34] (step=0037775) Train Loss: 5.4135, Train Steps/Sec: 1.17 +[2025-04-24 13:06:55] (step=0037800) Train Loss: 5.3153, Train Steps/Sec: 1.16 +[2025-04-24 13:07:16] (step=0037825) Train Loss: 5.3887, Train Steps/Sec: 1.17 +[2025-04-24 13:07:38] (step=0037850) Train Loss: 5.3146, Train Steps/Sec: 1.17 +[2025-04-24 13:07:59] (step=0037875) Train Loss: 5.3970, Train Steps/Sec: 1.17 +[2025-04-24 13:08:21] (step=0037900) Train Loss: 5.3551, Train Steps/Sec: 1.17 +[2025-04-24 13:08:42] (step=0037925) Train Loss: 5.3976, Train Steps/Sec: 1.17 +[2025-04-24 13:09:03] (step=0037950) Train Loss: 5.3898, Train Steps/Sec: 1.17 +[2025-04-24 13:09:25] (step=0037975) Train Loss: 5.3322, Train Steps/Sec: 1.13 +[2025-04-24 13:09:47] (step=0038000) Train Loss: 5.3768, Train Steps/Sec: 1.17 +[2025-04-24 13:09:47] text_config is None. Initializing the text config with default values (`OPTConfig`). +[2025-04-24 13:14:58] Finish Eval in 38000 steps... +[2025-04-24 13:15:18] Saved checkpoint to checkpoint/Afteri2i_100kckpt_LLAVAT5_empty_Fixed_just_segment_pretrain_stage3_dreambench_recap_400k_t2i_400k_flux_200k_midjourney_150k_recovery_150k_3e4_epoch2_no_rotated/000-GPT-XL/checkpoints/0038000.pt +[2025-04-24 13:15:20] Removed old checkpoint: checkpoint/Afteri2i_100kckpt_LLAVAT5_empty_Fixed_just_segment_pretrain_stage3_dreambench_recap_400k_t2i_400k_flux_200k_midjourney_150k_recovery_150k_3e4_epoch2_no_rotated/000-GPT-XL/checkpoints/0036000.pt +[2025-04-24 13:15:43] (step=0038025) Train Loss: 5.3576, Train Steps/Sec: 0.07 +[2025-04-24 13:16:04] (step=0038050) Train Loss: 5.3539, Train Steps/Sec: 1.17 +[2025-04-24 13:16:26] (step=0038075) Train Loss: 5.3652, Train Steps/Sec: 1.17 +[2025-04-24 13:16:48] (step=0038100) Train Loss: 5.3404, Train Steps/Sec: 1.11 +[2025-04-24 13:17:10] (step=0038125) Train Loss: 5.3054, Train Steps/Sec: 1.17 +[2025-04-24 13:17:31] (step=0038150) Train Loss: 5.4078, Train Steps/Sec: 1.17 +[2025-04-24 13:17:55] (step=0038175) Train Loss: 5.3976, Train Steps/Sec: 1.05 +[2025-04-24 13:18:17] (step=0038200) Train Loss: 5.3705, Train Steps/Sec: 1.11 +[2025-04-24 13:18:39] (step=0038225) Train Loss: 5.3780, Train Steps/Sec: 1.17 +[2025-04-24 13:19:00] (step=0038250) Train Loss: 5.3937, Train Steps/Sec: 1.17 +[2025-04-24 13:19:22] (step=0038275) Train Loss: 5.4124, Train Steps/Sec: 1.13 +[2025-04-24 13:19:44] (step=0038300) Train Loss: 5.3106, Train Steps/Sec: 1.16 +[2025-04-24 13:20:05] (step=0038325) Train Loss: 5.4085, Train Steps/Sec: 1.17 +[2025-04-24 13:20:27] (step=0038350) Train Loss: 5.3736, Train Steps/Sec: 1.17 +[2025-04-24 13:20:48] (step=0038375) Train Loss: 5.4102, Train Steps/Sec: 1.17 +[2025-04-24 13:21:09] (step=0038400) Train Loss: 5.4151, Train Steps/Sec: 1.17 +[2025-04-24 13:21:31] (step=0038425) Train Loss: 5.4022, Train Steps/Sec: 1.17 +[2025-04-24 13:21:52] (step=0038450) Train Loss: 5.3627, Train Steps/Sec: 1.17 +[2025-04-24 13:22:13] (step=0038475) Train Loss: 5.3657, Train Steps/Sec: 1.17 +[2025-04-24 13:22:35] (step=0038500) Train Loss: 5.3080, Train Steps/Sec: 1.16 +[2025-04-24 13:22:57] (step=0038525) Train Loss: 5.3129, Train Steps/Sec: 1.15 +[2025-04-24 13:23:18] (step=0038550) Train Loss: 5.3336, Train Steps/Sec: 1.17 +[2025-04-24 13:23:39] (step=0038575) Train Loss: 5.3620, Train Steps/Sec: 1.17 +[2025-04-24 13:24:01] (step=0038600) Train Loss: 5.3070, Train Steps/Sec: 1.16 +[2025-04-24 13:24:22] (step=0038625) Train Loss: 5.3940, Train Steps/Sec: 1.17 +[2025-04-24 13:24:44] (step=0038650) Train Loss: 5.3262, Train Steps/Sec: 1.12 +[2025-04-24 13:25:06] (step=0038675) Train Loss: 5.3525, Train Steps/Sec: 1.17 +[2025-04-24 13:25:27] (step=0038700) Train Loss: 5.3479, Train Steps/Sec: 1.17 +[2025-04-24 13:25:50] (step=0038725) Train Loss: 5.3327, Train Steps/Sec: 1.12 +[2025-04-24 13:26:12] (step=0038750) Train Loss: 5.2901, Train Steps/Sec: 1.12 +[2025-04-24 13:26:33] (step=0038775) Train Loss: 5.3641, Train Steps/Sec: 1.17 +[2025-04-24 13:26:55] (step=0038800) Train Loss: 5.2454, Train Steps/Sec: 1.16 +[2025-04-24 13:27:17] (step=0038825) Train Loss: 5.3751, Train Steps/Sec: 1.13 +[2025-04-24 13:27:40] (step=0038850) Train Loss: 5.3626, Train Steps/Sec: 1.08 +[2025-04-24 13:28:01] (step=0038875) Train Loss: 5.4141, Train Steps/Sec: 1.17 +[2025-04-24 13:28:23] (step=0038900) Train Loss: 5.4106, Train Steps/Sec: 1.17 +[2025-04-24 13:28:46] (step=0038925) Train Loss: 5.3852, Train Steps/Sec: 1.07 +[2025-04-24 13:29:08] (step=0038950) Train Loss: 5.3623, Train Steps/Sec: 1.17 +[2025-04-24 13:29:29] (step=0038975) Train Loss: 5.4443, Train Steps/Sec: 1.17 +[2025-04-24 13:29:50] (step=0039000) Train Loss: 5.5243, Train Steps/Sec: 1.17 +[2025-04-24 13:30:12] (step=0039025) Train Loss: 5.4217, Train Steps/Sec: 1.17 +[2025-04-24 13:30:33] (step=0039050) Train Loss: 5.3892, Train Steps/Sec: 1.17 +[2025-04-24 13:30:54] (step=0039075) Train Loss: 5.3695, Train Steps/Sec: 1.17 +[2025-04-24 13:31:16] (step=0039100) Train Loss: 5.3494, Train Steps/Sec: 1.17 +[2025-04-24 13:31:37] (step=0039125) Train Loss: 5.3832, Train Steps/Sec: 1.17 +[2025-04-24 13:31:59] (step=0039150) Train Loss: 5.3891, Train Steps/Sec: 1.17 +[2025-04-24 13:32:20] (step=0039175) Train Loss: 5.3458, Train Steps/Sec: 1.17 +[2025-04-24 13:32:41] (step=0039200) Train Loss: 5.3330, Train Steps/Sec: 1.16 +[2025-04-24 13:33:03] (step=0039225) Train Loss: 5.3804, Train Steps/Sec: 1.17 +[2025-04-24 13:33:24] (step=0039250) Train Loss: 5.4404, Train Steps/Sec: 1.17 +[2025-04-24 13:33:46] (step=0039275) Train Loss: 5.4176, Train Steps/Sec: 1.17 +[2025-04-24 13:34:07] (step=0039300) Train Loss: 5.3215, Train Steps/Sec: 1.17 +[2025-04-24 13:34:29] (step=0039325) Train Loss: 5.3189, Train Steps/Sec: 1.13 +[2025-04-24 13:34:50] (step=0039350) Train Loss: 5.3835, Train Steps/Sec: 1.17 +[2025-04-24 13:35:12] (step=0039375) Train Loss: 5.3502, Train Steps/Sec: 1.17 +[2025-04-24 13:35:34] (step=0039400) Train Loss: 5.3997, Train Steps/Sec: 1.12 +[2025-04-24 13:35:56] (step=0039425) Train Loss: 5.3441, Train Steps/Sec: 1.17 +[2025-04-24 13:36:18] (step=0039450) Train Loss: 5.2829, Train Steps/Sec: 1.11 +[2025-04-24 13:36:39] (step=0039475) Train Loss: 5.3788, Train Steps/Sec: 1.17 +[2025-04-24 13:37:03] (step=0039500) Train Loss: 5.3119, Train Steps/Sec: 1.04 +[2025-04-24 13:37:25] (step=0039525) Train Loss: 5.3431, Train Steps/Sec: 1.17 +[2025-04-24 13:37:46] (step=0039550) Train Loss: 5.3628, Train Steps/Sec: 1.17 +[2025-04-24 13:38:07] (step=0039575) Train Loss: 5.3576, Train Steps/Sec: 1.17 +[2025-04-24 13:38:30] (step=0039600) Train Loss: 5.4015, Train Steps/Sec: 1.12 +[2025-04-24 13:38:51] (step=0039625) Train Loss: 5.3550, Train Steps/Sec: 1.17 +[2025-04-24 13:39:13] (step=0039650) Train Loss: 5.4042, Train Steps/Sec: 1.12 +[2025-04-24 13:39:35] (step=0039675) Train Loss: 5.3776, Train Steps/Sec: 1.17 +[2025-04-24 13:39:56] (step=0039700) Train Loss: 5.3404, Train Steps/Sec: 1.16 +[2025-04-24 13:40:18] (step=0039725) Train Loss: 5.4076, Train Steps/Sec: 1.17 +[2025-04-24 13:40:39] (step=0039750) Train Loss: 5.2880, Train Steps/Sec: 1.17 +[2025-04-24 13:41:00] (step=0039775) Train Loss: 5.3139, Train Steps/Sec: 1.17 +[2025-04-24 13:41:22] (step=0039800) Train Loss: 5.3846, Train Steps/Sec: 1.17 +[2025-04-24 13:41:43] (step=0039825) Train Loss: 5.3812, Train Steps/Sec: 1.17 +[2025-04-24 13:42:04] (step=0039850) Train Loss: 5.3420, Train Steps/Sec: 1.17 +[2025-04-24 13:42:26] (step=0039875) Train Loss: 5.3414, Train Steps/Sec: 1.17 +[2025-04-24 13:42:47] (step=0039900) Train Loss: 5.3531, Train Steps/Sec: 1.17 +[2025-04-24 13:43:09] (step=0039925) Train Loss: 5.3093, Train Steps/Sec: 1.17 +[2025-04-24 13:43:30] (step=0039950) Train Loss: 5.3284, Train Steps/Sec: 1.16 +[2025-04-24 13:43:52] (step=0039975) Train Loss: 5.4014, Train Steps/Sec: 1.12 +[2025-04-24 13:44:14] (step=0040000) Train Loss: 5.3164, Train Steps/Sec: 1.16 +[2025-04-24 13:44:14] text_config is None. Initializing the text config with default values (`OPTConfig`). +[2025-04-24 13:49:21] Finish Eval in 40000 steps... +[2025-04-24 13:49:39] Saved checkpoint to checkpoint/Afteri2i_100kckpt_LLAVAT5_empty_Fixed_just_segment_pretrain_stage3_dreambench_recap_400k_t2i_400k_flux_200k_midjourney_150k_recovery_150k_3e4_epoch2_no_rotated/000-GPT-XL/checkpoints/0040000.pt +[2025-04-24 13:49:42] Removed old checkpoint: checkpoint/Afteri2i_100kckpt_LLAVAT5_empty_Fixed_just_segment_pretrain_stage3_dreambench_recap_400k_t2i_400k_flux_200k_midjourney_150k_recovery_150k_3e4_epoch2_no_rotated/000-GPT-XL/checkpoints/0038000.pt +[2025-04-24 13:50:03] (step=0040025) Train Loss: 5.4095, Train Steps/Sec: 0.07 +[2025-04-24 13:50:25] (step=0040050) Train Loss: 5.2762, Train Steps/Sec: 1.17 +[2025-04-24 13:50:47] (step=0040075) Train Loss: 5.4382, Train Steps/Sec: 1.12 +[2025-04-24 13:51:08] (step=0040100) Train Loss: 5.3194, Train Steps/Sec: 1.17 +[2025-04-24 13:51:30] (step=0040125) Train Loss: 5.3296, Train Steps/Sec: 1.18 +[2025-04-24 13:51:52] (step=0040150) Train Loss: 5.3140, Train Steps/Sec: 1.13 +[2025-04-24 13:52:16] (step=0040175) Train Loss: 5.3069, Train Steps/Sec: 1.04 +[2025-04-24 13:52:37] (step=0040200) Train Loss: 5.4563, Train Steps/Sec: 1.17 +[2025-04-24 13:52:59] (step=0040225) Train Loss: 5.2905, Train Steps/Sec: 1.17 +[2025-04-24 13:53:21] (step=0040250) Train Loss: 5.3793, Train Steps/Sec: 1.13 +[2025-04-24 13:53:42] (step=0040275) Train Loss: 5.3974, Train Steps/Sec: 1.17 +[2025-04-24 13:54:03] (step=0040300) Train Loss: 5.3865, Train Steps/Sec: 1.17 +[2025-04-24 13:54:25] (step=0040325) Train Loss: 5.3317, Train Steps/Sec: 1.17 +[2025-04-24 13:54:46] (step=0040350) Train Loss: 5.3318, Train Steps/Sec: 1.17 +[2025-04-24 13:55:08] (step=0040375) Train Loss: 5.4306, Train Steps/Sec: 1.12 +[2025-04-24 13:55:30] (step=0040400) Train Loss: 5.3807, Train Steps/Sec: 1.17 +[2025-04-24 13:55:51] (step=0040425) Train Loss: 5.3971, Train Steps/Sec: 1.17 +[2025-04-24 13:56:13] (step=0040450) Train Loss: 5.3281, Train Steps/Sec: 1.15 +[2025-04-24 13:56:34] (step=0040475) Train Loss: 5.4163, Train Steps/Sec: 1.17 +[2025-04-24 13:56:56] (step=0040500) Train Loss: 5.3598, Train Steps/Sec: 1.17 +[2025-04-24 13:57:17] (step=0040525) Train Loss: 5.3764, Train Steps/Sec: 1.17 +[2025-04-24 13:57:39] (step=0040550) Train Loss: 5.3526, Train Steps/Sec: 1.17 +[2025-04-24 13:58:00] (step=0040575) Train Loss: 5.4485, Train Steps/Sec: 1.17 +[2025-04-24 13:58:21] (step=0040600) Train Loss: 5.4342, Train Steps/Sec: 1.17 +[2025-04-24 13:58:43] (step=0040625) Train Loss: 5.3581, Train Steps/Sec: 1.17 +[2025-04-24 13:59:05] (step=0040650) Train Loss: 5.3813, Train Steps/Sec: 1.12 +[2025-04-24 13:59:26] (step=0040675) Train Loss: 5.4520, Train Steps/Sec: 1.17 +[2025-04-24 13:59:48] (step=0040700) Train Loss: 5.3197, Train Steps/Sec: 1.17 +[2025-04-24 14:00:10] (step=0040725) Train Loss: 5.3347, Train Steps/Sec: 1.12 +[2025-04-24 14:00:31] (step=0040750) Train Loss: 5.4309, Train Steps/Sec: 1.17 +[2025-04-24 14:00:53] (step=0040775) Train Loss: 5.3861, Train Steps/Sec: 1.17 +[2025-04-24 14:01:15] (step=0040800) Train Loss: 5.3549, Train Steps/Sec: 1.12 +[2025-04-24 14:01:38] (step=0040825) Train Loss: 5.3173, Train Steps/Sec: 1.08 +[2025-04-24 14:02:00] (step=0040850) Train Loss: 5.3498, Train Steps/Sec: 1.17 +[2025-04-24 14:02:21] (step=0040875) Train Loss: 5.3461, Train Steps/Sec: 1.17 +[2025-04-24 14:02:44] (step=0040900) Train Loss: 5.3719, Train Steps/Sec: 1.12 +[2025-04-24 14:03:06] (step=0040925) Train Loss: 5.4406, Train Steps/Sec: 1.17 +[2025-04-24 14:03:27] (step=0040950) Train Loss: 5.2755, Train Steps/Sec: 1.17 +[2025-04-24 14:03:48] (step=0040975) Train Loss: 5.3991, Train Steps/Sec: 1.17 +[2025-04-24 14:04:10] (step=0041000) Train Loss: 5.3725, Train Steps/Sec: 1.17 +[2025-04-24 14:04:31] (step=0041025) Train Loss: 5.4448, Train Steps/Sec: 1.17 +[2025-04-24 14:04:52] (step=0041050) Train Loss: 5.3389, Train Steps/Sec: 1.17 +[2025-04-24 14:05:14] (step=0041075) Train Loss: 5.3352, Train Steps/Sec: 1.17 +[2025-04-24 14:05:35] (step=0041100) Train Loss: 5.3442, Train Steps/Sec: 1.16 +[2025-04-24 14:05:58] (step=0041125) Train Loss: 5.3483, Train Steps/Sec: 1.12 +[2025-04-24 14:06:19] (step=0041150) Train Loss: 5.3065, Train Steps/Sec: 1.17 +[2025-04-24 14:06:40] (step=0041175) Train Loss: 5.3290, Train Steps/Sec: 1.17 +[2025-04-24 14:07:02] (step=0041200) Train Loss: 5.3915, Train Steps/Sec: 1.17 +[2025-04-24 14:07:23] (step=0041225) Train Loss: 5.3850, Train Steps/Sec: 1.17 +[2025-04-24 14:07:44] (step=0041250) Train Loss: 5.4192, Train Steps/Sec: 1.17 +[2025-04-24 14:08:06] (step=0041275) Train Loss: 5.3372, Train Steps/Sec: 1.17 +[2025-04-24 14:08:27] (step=0041300) Train Loss: 5.3605, Train Steps/Sec: 1.17 +[2025-04-24 14:08:49] (step=0041325) Train Loss: 5.4512, Train Steps/Sec: 1.13 +[2025-04-24 14:09:11] (step=0041350) Train Loss: 5.4149, Train Steps/Sec: 1.17 +[2025-04-24 14:09:33] (step=0041375) Train Loss: 5.3431, Train Steps/Sec: 1.12 +[2025-04-24 14:09:54] (step=0041400) Train Loss: 5.3505, Train Steps/Sec: 1.17 +[2025-04-24 14:10:16] (step=0041425) Train Loss: 5.3994, Train Steps/Sec: 1.17 +[2025-04-24 14:10:38] (step=0041450) Train Loss: 5.2830, Train Steps/Sec: 1.13 +[2025-04-24 14:11:01] (step=0041475) Train Loss: 5.3473, Train Steps/Sec: 1.08 +[2025-04-24 14:11:22] (step=0041500) Train Loss: 5.3119, Train Steps/Sec: 1.17 +[2025-04-24 14:11:44] (step=0041525) Train Loss: 5.3814, Train Steps/Sec: 1.17 +[2025-04-24 14:12:05] (step=0041550) Train Loss: 5.3775, Train Steps/Sec: 1.17 +[2025-04-24 14:12:27] (step=0041575) Train Loss: 5.3127, Train Steps/Sec: 1.12 +[2025-04-24 14:12:49] (step=0041600) Train Loss: 5.3991, Train Steps/Sec: 1.16 +[2025-04-24 14:13:11] (step=0041625) Train Loss: 5.3020, Train Steps/Sec: 1.12 +[2025-04-24 14:13:32] (step=0041650) Train Loss: 5.3394, Train Steps/Sec: 1.17 +[2025-04-24 14:13:54] (step=0041675) Train Loss: 5.3548, Train Steps/Sec: 1.17 +[2025-04-24 14:14:15] (step=0041700) Train Loss: 5.4572, Train Steps/Sec: 1.17 +[2025-04-24 14:14:37] (step=0041725) Train Loss: 5.3402, Train Steps/Sec: 1.17 +[2025-04-24 14:14:58] (step=0041750) Train Loss: 5.3364, Train Steps/Sec: 1.17 +[2025-04-24 14:15:19] (step=0041775) Train Loss: 5.3755, Train Steps/Sec: 1.17 +[2025-04-24 14:15:41] (step=0041800) Train Loss: 5.3409, Train Steps/Sec: 1.17 +[2025-04-24 14:16:02] (step=0041825) Train Loss: 5.3260, Train Steps/Sec: 1.17 +[2025-04-24 14:16:24] (step=0041850) Train Loss: 5.3839, Train Steps/Sec: 1.12 +[2025-04-24 14:16:46] (step=0041875) Train Loss: 5.3205, Train Steps/Sec: 1.17 +[2025-04-24 14:17:07] (step=0041900) Train Loss: 5.3277, Train Steps/Sec: 1.17 +[2025-04-24 14:17:29] (step=0041925) Train Loss: 5.3813, Train Steps/Sec: 1.17 +[2025-04-24 14:17:50] (step=0041950) Train Loss: 5.4256, Train Steps/Sec: 1.17 +[2025-04-24 14:18:12] (step=0041975) Train Loss: 5.3405, Train Steps/Sec: 1.13 +[2025-04-24 14:18:34] (step=0042000) Train Loss: 5.3171, Train Steps/Sec: 1.17 +[2025-04-24 14:18:34] text_config is None. Initializing the text config with default values (`OPTConfig`). +[2025-04-24 14:23:40] Finish Eval in 42000 steps... +[2025-04-24 14:23:59] Saved checkpoint to checkpoint/Afteri2i_100kckpt_LLAVAT5_empty_Fixed_just_segment_pretrain_stage3_dreambench_recap_400k_t2i_400k_flux_200k_midjourney_150k_recovery_150k_3e4_epoch2_no_rotated/000-GPT-XL/checkpoints/0042000.pt +[2025-04-24 14:24:01] Removed old checkpoint: checkpoint/Afteri2i_100kckpt_LLAVAT5_empty_Fixed_just_segment_pretrain_stage3_dreambench_recap_400k_t2i_400k_flux_200k_midjourney_150k_recovery_150k_3e4_epoch2_no_rotated/000-GPT-XL/checkpoints/0040000.pt +[2025-04-24 14:24:23] (step=0042025) Train Loss: 5.3458, Train Steps/Sec: 0.07 +[2025-04-24 14:24:44] (step=0042050) Train Loss: 5.4435, Train Steps/Sec: 1.17 +[2025-04-24 14:25:06] (step=0042075) Train Loss: 5.3411, Train Steps/Sec: 1.17 +[2025-04-24 14:25:27] (step=0042100) Train Loss: 5.3802, Train Steps/Sec: 1.17 +[2025-04-24 14:25:50] (step=0042125) Train Loss: 5.3292, Train Steps/Sec: 1.09 +[2025-04-24 14:26:12] (step=0042150) Train Loss: 5.3917, Train Steps/Sec: 1.13 +[2025-04-24 14:26:34] (step=0042175) Train Loss: 5.4638, Train Steps/Sec: 1.17 +[2025-04-24 14:26:55] (step=0042200) Train Loss: 5.3049, Train Steps/Sec: 1.16 +[2025-04-24 14:27:17] (step=0042225) Train Loss: 5.3507, Train Steps/Sec: 1.12 +[2025-04-24 14:27:39] (step=0042250) Train Loss: 5.4194, Train Steps/Sec: 1.17 +[2025-04-24 14:28:00] (step=0042275) Train Loss: 5.4105, Train Steps/Sec: 1.17 +[2025-04-24 14:28:22] (step=0042300) Train Loss: 5.4330, Train Steps/Sec: 1.17 +[2025-04-24 14:28:43] (step=0042325) Train Loss: 5.2673, Train Steps/Sec: 1.17 +[2025-04-24 14:29:05] (step=0042350) Train Loss: 5.3846, Train Steps/Sec: 1.12 +[2025-04-24 14:29:27] (step=0042375) Train Loss: 5.4311, Train Steps/Sec: 1.15 +[2025-04-24 14:29:48] (step=0042400) Train Loss: 5.3851, Train Steps/Sec: 1.17 +[2025-04-24 14:30:10] (step=0042425) Train Loss: 5.4065, Train Steps/Sec: 1.17 +[2025-04-24 14:30:31] (step=0042450) Train Loss: 5.3578, Train Steps/Sec: 1.17 +[2025-04-24 14:30:52] (step=0042475) Train Loss: 5.3888, Train Steps/Sec: 1.17 +[2025-04-24 14:31:14] (step=0042500) Train Loss: 5.3962, Train Steps/Sec: 1.16 +[2025-04-24 14:31:35] (step=0042525) Train Loss: 5.3548, Train Steps/Sec: 1.17 +[2025-04-24 14:31:57] (step=0042550) Train Loss: 5.3534, Train Steps/Sec: 1.17 +[2025-04-24 14:32:19] (step=0042575) Train Loss: 5.3578, Train Steps/Sec: 1.12 +[2025-04-24 14:32:40] (step=0042600) Train Loss: 5.3585, Train Steps/Sec: 1.17 +[2025-04-24 14:33:02] (step=0042625) Train Loss: 5.4102, Train Steps/Sec: 1.17 +[2025-04-24 14:33:24] (step=0042650) Train Loss: 5.3603, Train Steps/Sec: 1.13 +[2025-04-24 14:33:45] (step=0042675) Train Loss: 5.3248, Train Steps/Sec: 1.17 +[2025-04-24 14:34:08] (step=0042700) Train Loss: 5.4126, Train Steps/Sec: 1.12 +[2025-04-24 14:34:29] (step=0042725) Train Loss: 5.3865, Train Steps/Sec: 1.17 +[2025-04-24 14:34:50] (step=0042750) Train Loss: 5.3205, Train Steps/Sec: 1.17 +[2025-04-24 14:35:13] (step=0042775) Train Loss: 5.3785, Train Steps/Sec: 1.13 +[2025-04-24 14:35:36] (step=0042800) Train Loss: 5.4610, Train Steps/Sec: 1.08 +[2025-04-24 14:35:57] (step=0042825) Train Loss: 5.3499, Train Steps/Sec: 1.17 +[2025-04-24 14:36:18] (step=0042850) Train Loss: 5.4028, Train Steps/Sec: 1.17 +[2025-04-24 14:36:40] (step=0042875) Train Loss: 5.4273, Train Steps/Sec: 1.17 +[2025-04-24 14:37:02] (step=0042900) Train Loss: 5.3952, Train Steps/Sec: 1.12 +[2025-04-24 14:37:23] (step=0042925) Train Loss: 5.3373, Train Steps/Sec: 1.17 +[2025-04-24 14:37:45] (step=0042950) Train Loss: 5.3690, Train Steps/Sec: 1.17 +[2025-04-24 14:38:06] (step=0042975) Train Loss: 5.2976, Train Steps/Sec: 1.17 +[2025-04-24 14:38:27] (step=0043000) Train Loss: 5.3527, Train Steps/Sec: 1.17 +[2025-04-24 14:38:49] (step=0043025) Train Loss: 5.4105, Train Steps/Sec: 1.17 +[2025-04-24 14:39:10] (step=0043050) Train Loss: 5.3778, Train Steps/Sec: 1.17 +[2025-04-24 14:39:32] (step=0043075) Train Loss: 5.3676, Train Steps/Sec: 1.12 +[2025-04-24 14:39:54] (step=0043100) Train Loss: 5.4025, Train Steps/Sec: 1.17 +[2025-04-24 14:40:15] (step=0043125) Train Loss: 5.3961, Train Steps/Sec: 1.17 +[2025-04-24 14:40:37] (step=0043150) Train Loss: 5.4422, Train Steps/Sec: 1.17 +[2025-04-24 14:40:58] (step=0043175) Train Loss: 5.3020, Train Steps/Sec: 1.17 +[2025-04-24 14:41:19] (step=0043200) Train Loss: 5.3199, Train Steps/Sec: 1.17 +[2025-04-24 14:41:41] (step=0043225) Train Loss: 5.3092, Train Steps/Sec: 1.17 +[2025-04-24 14:42:02] (step=0043250) Train Loss: 5.3972, Train Steps/Sec: 1.17 +[2025-04-24 14:42:23] (step=0043275) Train Loss: 5.3018, Train Steps/Sec: 1.17 +[2025-04-24 14:42:46] (step=0043300) Train Loss: 5.3800, Train Steps/Sec: 1.11 +[2025-04-24 14:43:08] (step=0043325) Train Loss: 5.3792, Train Steps/Sec: 1.13 +[2025-04-24 14:43:30] (step=0043350) Train Loss: 5.4345, Train Steps/Sec: 1.12 +[2025-04-24 14:43:52] (step=0043375) Train Loss: 5.2724, Train Steps/Sec: 1.17 +[2025-04-24 14:44:13] (step=0043400) Train Loss: 5.3531, Train Steps/Sec: 1.16 +[2025-04-24 14:44:34] (step=0043425) Train Loss: 5.2922, Train Steps/Sec: 1.17 +[2025-04-24 14:44:58] (step=0043450) Train Loss: 5.4083, Train Steps/Sec: 1.08 +[2025-04-24 14:45:20] (step=0043475) Train Loss: 5.4297, Train Steps/Sec: 1.12 +[2025-04-24 14:45:41] (step=0043500) Train Loss: 5.3746, Train Steps/Sec: 1.17 +[2025-04-24 14:46:03] (step=0043525) Train Loss: 5.3094, Train Steps/Sec: 1.17 +[2025-04-24 14:46:25] (step=0043550) Train Loss: 5.3138, Train Steps/Sec: 1.13 +[2025-04-24 14:46:46] (step=0043575) Train Loss: 5.3246, Train Steps/Sec: 1.17 +[2025-04-24 14:47:08] (step=0043600) Train Loss: 5.3643, Train Steps/Sec: 1.17 +[2025-04-24 14:47:29] (step=0043625) Train Loss: 5.3798, Train Steps/Sec: 1.17 +[2025-04-24 14:47:50] (step=0043650) Train Loss: 5.3471, Train Steps/Sec: 1.17 +[2025-04-24 14:48:12] (step=0043675) Train Loss: 5.3267, Train Steps/Sec: 1.17 +[2025-04-24 14:48:33] (step=0043700) Train Loss: 5.3575, Train Steps/Sec: 1.16 +[2025-04-24 14:48:54] (step=0043725) Train Loss: 5.2876, Train Steps/Sec: 1.17 +[2025-04-24 14:49:16] (step=0043750) Train Loss: 5.4190, Train Steps/Sec: 1.17 +[2025-04-24 14:49:37] (step=0043775) Train Loss: 5.3367, Train Steps/Sec: 1.17 +[2025-04-24 14:50:00] (step=0043800) Train Loss: 5.4266, Train Steps/Sec: 1.11 +[2025-04-24 14:50:21] (step=0043825) Train Loss: 5.3375, Train Steps/Sec: 1.17 +[2025-04-24 14:50:42] (step=0043850) Train Loss: 5.3039, Train Steps/Sec: 1.17 +[2025-04-24 14:51:04] (step=0043875) Train Loss: 5.3923, Train Steps/Sec: 1.17 +[2025-04-24 14:51:25] (step=0043900) Train Loss: 5.3499, Train Steps/Sec: 1.17 +[2025-04-24 14:51:47] (step=0043925) Train Loss: 5.3395, Train Steps/Sec: 1.17 +[2025-04-24 14:52:08] (step=0043950) Train Loss: 5.3378, Train Steps/Sec: 1.17 +[2025-04-24 14:52:30] (step=0043975) Train Loss: 5.4040, Train Steps/Sec: 1.13 +[2025-04-24 14:52:53] (step=0044000) Train Loss: 5.3958, Train Steps/Sec: 1.12 +[2025-04-24 14:52:53] text_config is None. Initializing the text config with default values (`OPTConfig`). +[2025-04-24 14:58:00] Finish Eval in 44000 steps... +[2025-04-24 14:58:19] Saved checkpoint to checkpoint/Afteri2i_100kckpt_LLAVAT5_empty_Fixed_just_segment_pretrain_stage3_dreambench_recap_400k_t2i_400k_flux_200k_midjourney_150k_recovery_150k_3e4_epoch2_no_rotated/000-GPT-XL/checkpoints/0044000.pt +[2025-04-24 14:58:21] Removed old checkpoint: checkpoint/Afteri2i_100kckpt_LLAVAT5_empty_Fixed_just_segment_pretrain_stage3_dreambench_recap_400k_t2i_400k_flux_200k_midjourney_150k_recovery_150k_3e4_epoch2_no_rotated/000-GPT-XL/checkpoints/0042000.pt +[2025-04-24 14:58:43] (step=0044025) Train Loss: 5.2961, Train Steps/Sec: 0.07 +[2025-04-24 14:59:05] (step=0044050) Train Loss: 5.3310, Train Steps/Sec: 1.17 +[2025-04-24 14:59:26] (step=0044075) Train Loss: 5.3132, Train Steps/Sec: 1.18 +[2025-04-24 14:59:49] (step=0044100) Train Loss: 5.4157, Train Steps/Sec: 1.08 +[2025-04-24 15:00:11] (step=0044125) Train Loss: 5.3271, Train Steps/Sec: 1.13 +[2025-04-24 15:00:33] (step=0044150) Train Loss: 5.4176, Train Steps/Sec: 1.17 +[2025-04-24 15:00:54] (step=0044175) Train Loss: 5.4108, Train Steps/Sec: 1.17 +[2025-04-24 15:01:16] (step=0044200) Train Loss: 5.3260, Train Steps/Sec: 1.12 +[2025-04-24 15:01:38] (step=0044225) Train Loss: 5.4521, Train Steps/Sec: 1.17 +[2025-04-24 15:01:59] (step=0044250) Train Loss: 5.2540, Train Steps/Sec: 1.17 +[2025-04-24 15:02:21] (step=0044275) Train Loss: 5.4447, Train Steps/Sec: 1.17 +[2025-04-24 15:02:42] (step=0044300) Train Loss: 5.3279, Train Steps/Sec: 1.15 +[2025-04-24 15:03:04] (step=0044325) Train Loss: 5.3360, Train Steps/Sec: 1.17 +[2025-04-24 15:03:25] (step=0044350) Train Loss: 5.3412, Train Steps/Sec: 1.17 +[2025-04-24 15:03:46] (step=0044375) Train Loss: 5.3127, Train Steps/Sec: 1.17 +[2025-04-24 15:04:08] (step=0044400) Train Loss: 5.3130, Train Steps/Sec: 1.17 +[2025-04-24 15:04:29] (step=0044425) Train Loss: 5.3956, Train Steps/Sec: 1.17 +[2025-04-24 15:04:50] (step=0044450) Train Loss: 5.4003, Train Steps/Sec: 1.17 +[2025-04-24 15:05:12] (step=0044475) Train Loss: 5.2661, Train Steps/Sec: 1.17 +[2025-04-24 15:05:33] (step=0044500) Train Loss: 5.3642, Train Steps/Sec: 1.16 +[2025-04-24 15:05:56] (step=0044525) Train Loss: 5.3086, Train Steps/Sec: 1.11 +[2025-04-24 15:06:17] (step=0044550) Train Loss: 5.2828, Train Steps/Sec: 1.17 +[2025-04-24 15:06:38] (step=0044575) Train Loss: 5.3477, Train Steps/Sec: 1.17 +[2025-04-24 15:07:00] (step=0044600) Train Loss: 5.4552, Train Steps/Sec: 1.17 +[2025-04-24 15:07:21] (step=0044625) Train Loss: 5.2984, Train Steps/Sec: 1.17 +[2025-04-24 15:07:44] (step=0044650) Train Loss: 5.2785, Train Steps/Sec: 1.11 +[2025-04-24 15:08:06] (step=0044675) Train Loss: 5.3587, Train Steps/Sec: 1.12 +[2025-04-24 15:08:27] (step=0044700) Train Loss: 5.2732, Train Steps/Sec: 1.16 +[2025-04-24 15:08:49] (step=0044725) Train Loss: 5.3421, Train Steps/Sec: 1.17 +[2025-04-24 15:09:12] (step=0044750) Train Loss: 5.4088, Train Steps/Sec: 1.08 +[2025-04-24 15:09:34] (step=0044775) Train Loss: 5.4134, Train Steps/Sec: 1.12 +[2025-04-24 15:09:57] (step=0044800) Train Loss: 5.3649, Train Steps/Sec: 1.12 +[2025-04-24 15:10:18] (step=0044825) Train Loss: 5.4438, Train Steps/Sec: 1.17 +[2025-04-24 15:10:39] (step=0044850) Train Loss: 5.3291, Train Steps/Sec: 1.17 +[2025-04-24 15:11:01] (step=0044875) Train Loss: 5.3579, Train Steps/Sec: 1.12 +[2025-04-24 15:11:23] (step=0044900) Train Loss: 5.4260, Train Steps/Sec: 1.17 +[2025-04-24 15:11:44] (step=0044925) Train Loss: 5.3263, Train Steps/Sec: 1.17 +[2025-04-24 15:12:06] (step=0044950) Train Loss: 5.3517, Train Steps/Sec: 1.17 +[2025-04-24 15:12:27] (step=0044975) Train Loss: 5.4278, Train Steps/Sec: 1.17 +[2025-04-24 15:12:48] (step=0045000) Train Loss: 5.3542, Train Steps/Sec: 1.17 +[2025-04-24 15:13:10] (step=0045025) Train Loss: 5.3356, Train Steps/Sec: 1.17 +[2025-04-24 15:13:31] (step=0045050) Train Loss: 5.3747, Train Steps/Sec: 1.17 +[2025-04-24 15:13:52] (step=0045075) Train Loss: 5.2835, Train Steps/Sec: 1.17 +[2025-04-24 15:14:14] (step=0045100) Train Loss: 5.3420, Train Steps/Sec: 1.17 +[2025-04-24 15:14:35] (step=0045125) Train Loss: 5.2902, Train Steps/Sec: 1.17 +[2025-04-24 15:14:56] (step=0045150) Train Loss: 5.4017, Train Steps/Sec: 1.17 +[2025-04-24 15:15:18] (step=0045175) Train Loss: 5.3824, Train Steps/Sec: 1.17 +[2025-04-24 15:15:39] (step=0045200) Train Loss: 5.3239, Train Steps/Sec: 1.17 +[2025-04-24 15:16:01] (step=0045225) Train Loss: 5.3674, Train Steps/Sec: 1.17 +[2025-04-24 15:16:23] (step=0045250) Train Loss: 5.4348, Train Steps/Sec: 1.11 +[2025-04-24 15:16:44] (step=0045275) Train Loss: 5.2441, Train Steps/Sec: 1.17 +[2025-04-24 15:17:06] (step=0045300) Train Loss: 5.3340, Train Steps/Sec: 1.17 +[2025-04-24 15:17:29] (step=0045325) Train Loss: 5.3702, Train Steps/Sec: 1.08 +[2025-04-24 15:17:50] (step=0045350) Train Loss: 5.4778, Train Steps/Sec: 1.17 +[2025-04-24 15:18:12] (step=0045375) Train Loss: 5.3637, Train Steps/Sec: 1.17 +[2025-04-24 15:18:33] (step=0045400) Train Loss: 5.2872, Train Steps/Sec: 1.16 +[2025-04-24 15:18:56] (step=0045425) Train Loss: 5.3114, Train Steps/Sec: 1.09 +[2025-04-24 15:19:18] (step=0045450) Train Loss: 5.3929, Train Steps/Sec: 1.13 +[2025-04-24 15:19:41] (step=0045475) Train Loss: 5.4169, Train Steps/Sec: 1.12 +[2025-04-24 15:20:02] (step=0045500) Train Loss: 5.4487, Train Steps/Sec: 1.17 +[2025-04-24 15:20:24] (step=0045525) Train Loss: 5.3873, Train Steps/Sec: 1.13 +[2025-04-24 15:20:46] (step=0045550) Train Loss: 5.3651, Train Steps/Sec: 1.17 +[2025-04-24 15:21:07] (step=0045575) Train Loss: 5.3151, Train Steps/Sec: 1.17 +[2025-04-24 15:21:28] (step=0045600) Train Loss: 5.3295, Train Steps/Sec: 1.17 +[2025-04-24 15:21:50] (step=0045625) Train Loss: 5.3815, Train Steps/Sec: 1.17 +[2025-04-24 15:22:11] (step=0045650) Train Loss: 5.3210, Train Steps/Sec: 1.17 +[2025-04-24 15:22:32] (step=0045675) Train Loss: 5.4804, Train Steps/Sec: 1.17 +[2025-04-24 15:22:54] (step=0045700) Train Loss: 5.4092, Train Steps/Sec: 1.16 +[2025-04-24 15:23:15] (step=0045725) Train Loss: 5.3221, Train Steps/Sec: 1.17 +[2025-04-24 15:23:37] (step=0045750) Train Loss: 5.3596, Train Steps/Sec: 1.17 +[2025-04-24 15:23:58] (step=0045775) Train Loss: 5.2999, Train Steps/Sec: 1.17 +[2025-04-24 15:24:19] (step=0045800) Train Loss: 5.3564, Train Steps/Sec: 1.17 +[2025-04-24 15:24:41] (step=0045825) Train Loss: 5.4261, Train Steps/Sec: 1.17 +[2025-04-24 15:25:02] (step=0045850) Train Loss: 5.3527, Train Steps/Sec: 1.17 +[2025-04-24 15:25:23] (step=0045875) Train Loss: 5.3850, Train Steps/Sec: 1.17 +[2025-04-24 15:25:45] (step=0045900) Train Loss: 5.3263, Train Steps/Sec: 1.17 +[2025-04-24 15:26:06] (step=0045925) Train Loss: 5.3156, Train Steps/Sec: 1.17 +[2025-04-24 15:26:27] (step=0045950) Train Loss: 5.4064, Train Steps/Sec: 1.17 +[2025-04-24 15:26:51] (step=0045975) Train Loss: 5.3635, Train Steps/Sec: 1.07 +[2025-04-24 15:27:13] (step=0046000) Train Loss: 5.3436, Train Steps/Sec: 1.12 +[2025-04-24 15:27:13] text_config is None. Initializing the text config with default values (`OPTConfig`). +[2025-04-24 15:32:19] Finish Eval in 46000 steps... +[2025-04-24 15:32:38] Saved checkpoint to checkpoint/Afteri2i_100kckpt_LLAVAT5_empty_Fixed_just_segment_pretrain_stage3_dreambench_recap_400k_t2i_400k_flux_200k_midjourney_150k_recovery_150k_3e4_epoch2_no_rotated/000-GPT-XL/checkpoints/0046000.pt +[2025-04-24 15:32:40] Removed old checkpoint: checkpoint/Afteri2i_100kckpt_LLAVAT5_empty_Fixed_just_segment_pretrain_stage3_dreambench_recap_400k_t2i_400k_flux_200k_midjourney_150k_recovery_150k_3e4_epoch2_no_rotated/000-GPT-XL/checkpoints/0044000.pt +[2025-04-24 15:33:02] (step=0046025) Train Loss: 5.2158, Train Steps/Sec: 0.07 +[2025-04-24 15:33:23] (step=0046050) Train Loss: 5.3332, Train Steps/Sec: 1.17 +[2025-04-24 15:33:46] (step=0046075) Train Loss: 5.2963, Train Steps/Sec: 1.08 +[2025-04-24 15:34:08] (step=0046100) Train Loss: 5.3621, Train Steps/Sec: 1.12 +[2025-04-24 15:34:30] (step=0046125) Train Loss: 5.3827, Train Steps/Sec: 1.17 +[2025-04-24 15:34:51] (step=0046150) Train Loss: 5.2810, Train Steps/Sec: 1.17 +[2025-04-24 15:35:12] (step=0046175) Train Loss: 5.3794, Train Steps/Sec: 1.17 +[2025-04-24 15:35:36] (step=0046200) Train Loss: 5.3471, Train Steps/Sec: 1.07 +[2025-04-24 15:35:57] (step=0046225) Train Loss: 5.3950, Train Steps/Sec: 1.15 +[2025-04-24 15:36:19] (step=0046250) Train Loss: 5.2749, Train Steps/Sec: 1.17 +[2025-04-24 15:36:40] (step=0046275) Train Loss: 5.3665, Train Steps/Sec: 1.17 +[2025-04-24 15:37:01] (step=0046300) Train Loss: 5.2892, Train Steps/Sec: 1.17 +[2025-04-24 15:37:23] (step=0046325) Train Loss: 5.3947, Train Steps/Sec: 1.17 +[2025-04-24 15:37:44] (step=0046350) Train Loss: 5.4648, Train Steps/Sec: 1.17 +[2025-04-24 15:38:05] (step=0046375) Train Loss: 5.3487, Train Steps/Sec: 1.17 +[2025-04-24 15:38:27] (step=0046400) Train Loss: 5.3504, Train Steps/Sec: 1.16 +[2025-04-24 15:38:48] (step=0046425) Train Loss: 5.4223, Train Steps/Sec: 1.17 +[2025-04-24 15:39:10] (step=0046450) Train Loss: 5.3115, Train Steps/Sec: 1.17 +[2025-04-24 15:39:31] (step=0046475) Train Loss: 5.3462, Train Steps/Sec: 1.17 +[2025-04-24 15:39:52] (step=0046500) Train Loss: 5.2936, Train Steps/Sec: 1.17 +[2025-04-24 15:40:14] (step=0046525) Train Loss: 5.3681, Train Steps/Sec: 1.17 +[2025-04-24 15:40:35] (step=0046550) Train Loss: 5.3644, Train Steps/Sec: 1.17 +[2025-04-24 15:40:56] (step=0046575) Train Loss: 5.2854, Train Steps/Sec: 1.17 +[2025-04-24 15:41:18] (step=0046600) Train Loss: 5.3650, Train Steps/Sec: 1.17 +[2025-04-24 15:41:39] (step=0046625) Train Loss: 5.3799, Train Steps/Sec: 1.17 +[2025-04-24 15:42:02] (step=0046650) Train Loss: 5.4018, Train Steps/Sec: 1.08 +[2025-04-24 15:42:24] (step=0046675) Train Loss: 5.3859, Train Steps/Sec: 1.17 +[2025-04-24 15:42:46] (step=0046700) Train Loss: 5.3203, Train Steps/Sec: 1.11 +[2025-04-24 15:43:08] (step=0046725) Train Loss: 5.2928, Train Steps/Sec: 1.13 +[2025-04-24 15:43:30] (step=0046750) Train Loss: 5.3758, Train Steps/Sec: 1.12 +[2025-04-24 15:43:53] (step=0046775) Train Loss: 5.4073, Train Steps/Sec: 1.12 +[2025-04-24 15:44:14] (step=0046800) Train Loss: 5.2407, Train Steps/Sec: 1.16 +[2025-04-24 15:44:36] (step=0046825) Train Loss: 5.3809, Train Steps/Sec: 1.17 +[2025-04-24 15:44:58] (step=0046850) Train Loss: 5.3736, Train Steps/Sec: 1.12 +[2025-04-24 15:45:19] (step=0046875) Train Loss: 5.3742, Train Steps/Sec: 1.17 +[2025-04-24 15:45:41] (step=0046900) Train Loss: 5.4445, Train Steps/Sec: 1.17 +[2025-04-24 15:45:55] Done! diff --git a/CKPTS/Afteri2i_100kckpt_LLAVAT5_empty_Fixed_just_segment_pretrain_stage3_dreambench_recap_400k_t2i_400k_flux_200k_midjourney_150k_recovery_150k_3e4_epoch2_no_rotated/002-GPT-XL/checkpoints/eval_step_48000/batch_0_cfg_7.5_topk_16384.jpg b/CKPTS/Afteri2i_100kckpt_LLAVAT5_empty_Fixed_just_segment_pretrain_stage3_dreambench_recap_400k_t2i_400k_flux_200k_midjourney_150k_recovery_150k_3e4_epoch2_no_rotated/002-GPT-XL/checkpoints/eval_step_48000/batch_0_cfg_7.5_topk_16384.jpg new file mode 100644 index 0000000000000000000000000000000000000000..0b6a4cc19cc1acb62445f691720eecd60c910566 --- /dev/null +++ b/CKPTS/Afteri2i_100kckpt_LLAVAT5_empty_Fixed_just_segment_pretrain_stage3_dreambench_recap_400k_t2i_400k_flux_200k_midjourney_150k_recovery_150k_3e4_epoch2_no_rotated/002-GPT-XL/checkpoints/eval_step_48000/batch_0_cfg_7.5_topk_16384.jpg @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:b0db6d83680bc519fd6c7aec5500e1a5161026db41ae0024099aebb4d829e66b +size 720414 diff --git a/CKPTS/Afteri2i_100kckpt_LLAVAT5_empty_Fixed_just_segment_pretrain_stage3_dreambench_recap_400k_t2i_400k_flux_200k_midjourney_150k_recovery_150k_3e4_epoch2_no_rotated/002-GPT-XL/checkpoints/eval_step_48000/batch_1_cfg_7.5_topk_16384.jpg b/CKPTS/Afteri2i_100kckpt_LLAVAT5_empty_Fixed_just_segment_pretrain_stage3_dreambench_recap_400k_t2i_400k_flux_200k_midjourney_150k_recovery_150k_3e4_epoch2_no_rotated/002-GPT-XL/checkpoints/eval_step_48000/batch_1_cfg_7.5_topk_16384.jpg new file mode 100644 index 0000000000000000000000000000000000000000..2abb7e6345d606df58c2e944e75e83bd12411ecf --- /dev/null +++ b/CKPTS/Afteri2i_100kckpt_LLAVAT5_empty_Fixed_just_segment_pretrain_stage3_dreambench_recap_400k_t2i_400k_flux_200k_midjourney_150k_recovery_150k_3e4_epoch2_no_rotated/002-GPT-XL/checkpoints/eval_step_48000/batch_1_cfg_7.5_topk_16384.jpg @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:1bcf5fc0784645975bfe0dc1ed9da992f1e092d426f26b6c77fdf20ab25ca3e7 +size 710307 diff --git a/CKPTS/Afteri2i_100kckpt_LLAVAT5_empty_Fixed_just_segment_pretrain_stage3_dreambench_recap_400k_t2i_400k_flux_200k_midjourney_150k_recovery_150k_3e4_epoch2_no_rotated/002-GPT-XL/checkpoints/eval_step_48000/batch_2_cfg_7.5_topk_16384.jpg b/CKPTS/Afteri2i_100kckpt_LLAVAT5_empty_Fixed_just_segment_pretrain_stage3_dreambench_recap_400k_t2i_400k_flux_200k_midjourney_150k_recovery_150k_3e4_epoch2_no_rotated/002-GPT-XL/checkpoints/eval_step_48000/batch_2_cfg_7.5_topk_16384.jpg new file mode 100644 index 0000000000000000000000000000000000000000..db1b814cf867fa7aec5a1175ddd78a17b19c7606 --- /dev/null +++ b/CKPTS/Afteri2i_100kckpt_LLAVAT5_empty_Fixed_just_segment_pretrain_stage3_dreambench_recap_400k_t2i_400k_flux_200k_midjourney_150k_recovery_150k_3e4_epoch2_no_rotated/002-GPT-XL/checkpoints/eval_step_48000/batch_2_cfg_7.5_topk_16384.jpg @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:9e83e3e81d65ffd643cde1eb9ad95841b2c1bae0490225289756a8b8e6dd5e9e +size 723102 diff --git a/CKPTS/Afteri2i_100kckpt_LLAVAT5_empty_Fixed_just_segment_pretrain_stage3_dreambench_recap_400k_t2i_400k_flux_200k_midjourney_150k_recovery_150k_3e4_epoch2_no_rotated/002-GPT-XL/checkpoints/eval_step_48000/batch_3_cfg_7.5_topk_16384.jpg b/CKPTS/Afteri2i_100kckpt_LLAVAT5_empty_Fixed_just_segment_pretrain_stage3_dreambench_recap_400k_t2i_400k_flux_200k_midjourney_150k_recovery_150k_3e4_epoch2_no_rotated/002-GPT-XL/checkpoints/eval_step_48000/batch_3_cfg_7.5_topk_16384.jpg new file mode 100644 index 0000000000000000000000000000000000000000..0562fb3e56075c603f7e375a7ee475107664861c --- /dev/null +++ b/CKPTS/Afteri2i_100kckpt_LLAVAT5_empty_Fixed_just_segment_pretrain_stage3_dreambench_recap_400k_t2i_400k_flux_200k_midjourney_150k_recovery_150k_3e4_epoch2_no_rotated/002-GPT-XL/checkpoints/eval_step_48000/batch_3_cfg_7.5_topk_16384.jpg @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:6c3379a2ebd221dfd03318198385c075b62249ed0fce748b75ae2ce76cf25d49 +size 846919 diff --git a/CKPTS/Afteri2i_100kckpt_LLAVAT5_empty_Fixed_just_segment_pretrain_stage3_dreambench_recap_400k_t2i_400k_flux_200k_midjourney_150k_recovery_150k_3e4_epoch2_no_rotated/002-GPT-XL/checkpoints/eval_step_50000/batch_0_cfg_7.5_topk_16384.jpg b/CKPTS/Afteri2i_100kckpt_LLAVAT5_empty_Fixed_just_segment_pretrain_stage3_dreambench_recap_400k_t2i_400k_flux_200k_midjourney_150k_recovery_150k_3e4_epoch2_no_rotated/002-GPT-XL/checkpoints/eval_step_50000/batch_0_cfg_7.5_topk_16384.jpg new file mode 100644 index 0000000000000000000000000000000000000000..a7c4547eba7e684d45919111f35edfca02645fcc --- /dev/null +++ b/CKPTS/Afteri2i_100kckpt_LLAVAT5_empty_Fixed_just_segment_pretrain_stage3_dreambench_recap_400k_t2i_400k_flux_200k_midjourney_150k_recovery_150k_3e4_epoch2_no_rotated/002-GPT-XL/checkpoints/eval_step_50000/batch_0_cfg_7.5_topk_16384.jpg @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:e7d7e33bbbaf390f9902b3492c884f1763e10d20978637890dd8be8afb21cdd9 +size 737178 diff --git a/CKPTS/Afteri2i_100kckpt_LLAVAT5_empty_Fixed_just_segment_pretrain_stage3_dreambench_recap_400k_t2i_400k_flux_200k_midjourney_150k_recovery_150k_3e4_epoch2_no_rotated/002-GPT-XL/checkpoints/eval_step_50000/batch_1_cfg_7.5_topk_16384.jpg b/CKPTS/Afteri2i_100kckpt_LLAVAT5_empty_Fixed_just_segment_pretrain_stage3_dreambench_recap_400k_t2i_400k_flux_200k_midjourney_150k_recovery_150k_3e4_epoch2_no_rotated/002-GPT-XL/checkpoints/eval_step_50000/batch_1_cfg_7.5_topk_16384.jpg new file mode 100644 index 0000000000000000000000000000000000000000..19c85a18b00d3390f373c90c9255cf557dd4c123 --- /dev/null +++ b/CKPTS/Afteri2i_100kckpt_LLAVAT5_empty_Fixed_just_segment_pretrain_stage3_dreambench_recap_400k_t2i_400k_flux_200k_midjourney_150k_recovery_150k_3e4_epoch2_no_rotated/002-GPT-XL/checkpoints/eval_step_50000/batch_1_cfg_7.5_topk_16384.jpg @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:e89dcb9511bf0a300919b3bcff682c05765f2c3b9f0c5e4e43a8cecf93eda75a +size 709173 diff --git a/CKPTS/Afteri2i_100kckpt_LLAVAT5_empty_Fixed_just_segment_pretrain_stage3_dreambench_recap_400k_t2i_400k_flux_200k_midjourney_150k_recovery_150k_3e4_epoch2_no_rotated/002-GPT-XL/checkpoints/eval_step_50000/batch_2_cfg_7.5_topk_16384.jpg b/CKPTS/Afteri2i_100kckpt_LLAVAT5_empty_Fixed_just_segment_pretrain_stage3_dreambench_recap_400k_t2i_400k_flux_200k_midjourney_150k_recovery_150k_3e4_epoch2_no_rotated/002-GPT-XL/checkpoints/eval_step_50000/batch_2_cfg_7.5_topk_16384.jpg new file mode 100644 index 0000000000000000000000000000000000000000..4e7ac686c5d3b98cdbda952b57944c5a4366523e --- /dev/null +++ b/CKPTS/Afteri2i_100kckpt_LLAVAT5_empty_Fixed_just_segment_pretrain_stage3_dreambench_recap_400k_t2i_400k_flux_200k_midjourney_150k_recovery_150k_3e4_epoch2_no_rotated/002-GPT-XL/checkpoints/eval_step_50000/batch_2_cfg_7.5_topk_16384.jpg @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:cdf8d562b5244f13729b80ed2d3dae5aeb0b8020ca61f32f9784898a197c161f +size 692489 diff --git a/CKPTS/Afteri2i_100kckpt_LLAVAT5_empty_Fixed_just_segment_pretrain_stage3_dreambench_recap_400k_t2i_400k_flux_200k_midjourney_150k_recovery_150k_3e4_epoch2_no_rotated/002-GPT-XL/checkpoints/eval_step_50000/batch_3_cfg_7.5_topk_16384.jpg b/CKPTS/Afteri2i_100kckpt_LLAVAT5_empty_Fixed_just_segment_pretrain_stage3_dreambench_recap_400k_t2i_400k_flux_200k_midjourney_150k_recovery_150k_3e4_epoch2_no_rotated/002-GPT-XL/checkpoints/eval_step_50000/batch_3_cfg_7.5_topk_16384.jpg new file mode 100644 index 0000000000000000000000000000000000000000..54de5fcef14531d81ff9dc9fe31e20dbe8988e5c --- /dev/null +++ b/CKPTS/Afteri2i_100kckpt_LLAVAT5_empty_Fixed_just_segment_pretrain_stage3_dreambench_recap_400k_t2i_400k_flux_200k_midjourney_150k_recovery_150k_3e4_epoch2_no_rotated/002-GPT-XL/checkpoints/eval_step_50000/batch_3_cfg_7.5_topk_16384.jpg @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:75c26caa8ed28ff99a2dbf8f5945b23511ce6fc8a95ad5c576bc40c6de8ac101 +size 841423 diff --git a/CKPTS/Afteri2i_100kckpt_LLAVAT5_empty_Fixed_just_segment_pretrain_stage3_dreambench_recap_400k_t2i_400k_flux_200k_midjourney_150k_recovery_150k_3e4_epoch2_no_rotated/002-GPT-XL/checkpoints/eval_step_52000/batch_0_cfg_7.5_topk_16384.jpg b/CKPTS/Afteri2i_100kckpt_LLAVAT5_empty_Fixed_just_segment_pretrain_stage3_dreambench_recap_400k_t2i_400k_flux_200k_midjourney_150k_recovery_150k_3e4_epoch2_no_rotated/002-GPT-XL/checkpoints/eval_step_52000/batch_0_cfg_7.5_topk_16384.jpg new file mode 100644 index 0000000000000000000000000000000000000000..5325581004a5450d5d468d4e12a9b64d920b6923 --- /dev/null +++ b/CKPTS/Afteri2i_100kckpt_LLAVAT5_empty_Fixed_just_segment_pretrain_stage3_dreambench_recap_400k_t2i_400k_flux_200k_midjourney_150k_recovery_150k_3e4_epoch2_no_rotated/002-GPT-XL/checkpoints/eval_step_52000/batch_0_cfg_7.5_topk_16384.jpg @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:5dcf6e542a3f975d6c32ce9a40820e96ca16fda7f553d9f4dcbc93de990d41fb +size 730286 diff --git a/CKPTS/Afteri2i_100kckpt_LLAVAT5_empty_Fixed_just_segment_pretrain_stage3_dreambench_recap_400k_t2i_400k_flux_200k_midjourney_150k_recovery_150k_3e4_epoch2_no_rotated/002-GPT-XL/checkpoints/eval_step_52000/batch_1_cfg_7.5_topk_16384.jpg b/CKPTS/Afteri2i_100kckpt_LLAVAT5_empty_Fixed_just_segment_pretrain_stage3_dreambench_recap_400k_t2i_400k_flux_200k_midjourney_150k_recovery_150k_3e4_epoch2_no_rotated/002-GPT-XL/checkpoints/eval_step_52000/batch_1_cfg_7.5_topk_16384.jpg new file mode 100644 index 0000000000000000000000000000000000000000..0fcd9d2042972c08283834c4e0fe2fa7aa6b5e7d --- /dev/null +++ b/CKPTS/Afteri2i_100kckpt_LLAVAT5_empty_Fixed_just_segment_pretrain_stage3_dreambench_recap_400k_t2i_400k_flux_200k_midjourney_150k_recovery_150k_3e4_epoch2_no_rotated/002-GPT-XL/checkpoints/eval_step_52000/batch_1_cfg_7.5_topk_16384.jpg @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:2510286e6d121f1a50f962483a735bf4a00701cf3c6fb37475cc400f07cf0616 +size 722407 diff --git a/CKPTS/Afteri2i_100kckpt_LLAVAT5_empty_Fixed_just_segment_pretrain_stage3_dreambench_recap_400k_t2i_400k_flux_200k_midjourney_150k_recovery_150k_3e4_epoch2_no_rotated/002-GPT-XL/checkpoints/eval_step_52000/batch_2_cfg_7.5_topk_16384.jpg b/CKPTS/Afteri2i_100kckpt_LLAVAT5_empty_Fixed_just_segment_pretrain_stage3_dreambench_recap_400k_t2i_400k_flux_200k_midjourney_150k_recovery_150k_3e4_epoch2_no_rotated/002-GPT-XL/checkpoints/eval_step_52000/batch_2_cfg_7.5_topk_16384.jpg new file mode 100644 index 0000000000000000000000000000000000000000..8e2b5ef959ab2c5e220f0ea13070f3a55b653a6e --- /dev/null +++ b/CKPTS/Afteri2i_100kckpt_LLAVAT5_empty_Fixed_just_segment_pretrain_stage3_dreambench_recap_400k_t2i_400k_flux_200k_midjourney_150k_recovery_150k_3e4_epoch2_no_rotated/002-GPT-XL/checkpoints/eval_step_52000/batch_2_cfg_7.5_topk_16384.jpg @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:5c1a5ce1aaf370d429d624acdf169ee7f5786cbb28f787896c10b2698374da63 +size 723545 diff --git a/CKPTS/Afteri2i_100kckpt_LLAVAT5_empty_Fixed_just_segment_pretrain_stage3_dreambench_recap_400k_t2i_400k_flux_200k_midjourney_150k_recovery_150k_3e4_epoch2_no_rotated/002-GPT-XL/checkpoints/eval_step_52000/batch_3_cfg_7.5_topk_16384.jpg b/CKPTS/Afteri2i_100kckpt_LLAVAT5_empty_Fixed_just_segment_pretrain_stage3_dreambench_recap_400k_t2i_400k_flux_200k_midjourney_150k_recovery_150k_3e4_epoch2_no_rotated/002-GPT-XL/checkpoints/eval_step_52000/batch_3_cfg_7.5_topk_16384.jpg new file mode 100644 index 0000000000000000000000000000000000000000..afe7f544d366e962f72f2fa2a9e4b9e62557c9f0 --- /dev/null +++ b/CKPTS/Afteri2i_100kckpt_LLAVAT5_empty_Fixed_just_segment_pretrain_stage3_dreambench_recap_400k_t2i_400k_flux_200k_midjourney_150k_recovery_150k_3e4_epoch2_no_rotated/002-GPT-XL/checkpoints/eval_step_52000/batch_3_cfg_7.5_topk_16384.jpg @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:fc68a7d1c3e3137bdeb19e8b814a891ba2eba360d238293b79c3514634c3ca7e +size 852446 diff --git a/CKPTS/Afteri2i_100kckpt_LLAVAT5_empty_Fixed_just_segment_pretrain_stage3_dreambench_recap_400k_t2i_400k_flux_200k_midjourney_150k_recovery_150k_3e4_epoch2_no_rotated/002-GPT-XL/checkpoints/eval_step_54000/batch_0_cfg_7.5_topk_16384.jpg b/CKPTS/Afteri2i_100kckpt_LLAVAT5_empty_Fixed_just_segment_pretrain_stage3_dreambench_recap_400k_t2i_400k_flux_200k_midjourney_150k_recovery_150k_3e4_epoch2_no_rotated/002-GPT-XL/checkpoints/eval_step_54000/batch_0_cfg_7.5_topk_16384.jpg new file mode 100644 index 0000000000000000000000000000000000000000..64eaca47e92462e6a4433f610016e209486e6860 --- /dev/null +++ b/CKPTS/Afteri2i_100kckpt_LLAVAT5_empty_Fixed_just_segment_pretrain_stage3_dreambench_recap_400k_t2i_400k_flux_200k_midjourney_150k_recovery_150k_3e4_epoch2_no_rotated/002-GPT-XL/checkpoints/eval_step_54000/batch_0_cfg_7.5_topk_16384.jpg @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:07d9fab596d47697560a5012dbf68e0f5f280202a0690f12991962ae759b34d2 +size 728719 diff --git a/CKPTS/Afteri2i_100kckpt_LLAVAT5_empty_Fixed_just_segment_pretrain_stage3_dreambench_recap_400k_t2i_400k_flux_200k_midjourney_150k_recovery_150k_3e4_epoch2_no_rotated/002-GPT-XL/checkpoints/eval_step_54000/batch_1_cfg_7.5_topk_16384.jpg b/CKPTS/Afteri2i_100kckpt_LLAVAT5_empty_Fixed_just_segment_pretrain_stage3_dreambench_recap_400k_t2i_400k_flux_200k_midjourney_150k_recovery_150k_3e4_epoch2_no_rotated/002-GPT-XL/checkpoints/eval_step_54000/batch_1_cfg_7.5_topk_16384.jpg new file mode 100644 index 0000000000000000000000000000000000000000..d8a7b0a609eea079b2deb7fc841adb945be28b29 --- /dev/null +++ b/CKPTS/Afteri2i_100kckpt_LLAVAT5_empty_Fixed_just_segment_pretrain_stage3_dreambench_recap_400k_t2i_400k_flux_200k_midjourney_150k_recovery_150k_3e4_epoch2_no_rotated/002-GPT-XL/checkpoints/eval_step_54000/batch_1_cfg_7.5_topk_16384.jpg @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:459f722e2807684176cc6b35d087b59b08a19358e1af6218fa7163954444537e +size 710581 diff --git a/CKPTS/Afteri2i_100kckpt_LLAVAT5_empty_Fixed_just_segment_pretrain_stage3_dreambench_recap_400k_t2i_400k_flux_200k_midjourney_150k_recovery_150k_3e4_epoch2_no_rotated/002-GPT-XL/checkpoints/eval_step_54000/batch_2_cfg_7.5_topk_16384.jpg b/CKPTS/Afteri2i_100kckpt_LLAVAT5_empty_Fixed_just_segment_pretrain_stage3_dreambench_recap_400k_t2i_400k_flux_200k_midjourney_150k_recovery_150k_3e4_epoch2_no_rotated/002-GPT-XL/checkpoints/eval_step_54000/batch_2_cfg_7.5_topk_16384.jpg new file mode 100644 index 0000000000000000000000000000000000000000..1a51794682b412be0f32b6804d3cedb2c848f57c --- /dev/null +++ b/CKPTS/Afteri2i_100kckpt_LLAVAT5_empty_Fixed_just_segment_pretrain_stage3_dreambench_recap_400k_t2i_400k_flux_200k_midjourney_150k_recovery_150k_3e4_epoch2_no_rotated/002-GPT-XL/checkpoints/eval_step_54000/batch_2_cfg_7.5_topk_16384.jpg @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:9836dd2d7e794097f58910aca350bf74871d50c0bd74aa675def81fc477b970f +size 716740 diff --git a/CKPTS/Afteri2i_100kckpt_LLAVAT5_empty_Fixed_just_segment_pretrain_stage3_dreambench_recap_400k_t2i_400k_flux_200k_midjourney_150k_recovery_150k_3e4_epoch2_no_rotated/002-GPT-XL/checkpoints/eval_step_54000/batch_3_cfg_7.5_topk_16384.jpg b/CKPTS/Afteri2i_100kckpt_LLAVAT5_empty_Fixed_just_segment_pretrain_stage3_dreambench_recap_400k_t2i_400k_flux_200k_midjourney_150k_recovery_150k_3e4_epoch2_no_rotated/002-GPT-XL/checkpoints/eval_step_54000/batch_3_cfg_7.5_topk_16384.jpg new file mode 100644 index 0000000000000000000000000000000000000000..38b2a1c6fb054729a8f56e20ed44c519a88f1e09 --- /dev/null +++ b/CKPTS/Afteri2i_100kckpt_LLAVAT5_empty_Fixed_just_segment_pretrain_stage3_dreambench_recap_400k_t2i_400k_flux_200k_midjourney_150k_recovery_150k_3e4_epoch2_no_rotated/002-GPT-XL/checkpoints/eval_step_54000/batch_3_cfg_7.5_topk_16384.jpg @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:1526676a4ea20c203890a12b1b3d6d58dd86f55a76b4bdba6763f8d68d356aef +size 878559 diff --git a/CKPTS/Afteri2i_100kckpt_LLAVAT5_empty_Fixed_just_segment_pretrain_stage3_dreambench_recap_400k_t2i_400k_flux_200k_midjourney_150k_recovery_150k_3e4_epoch2_no_rotated/002-GPT-XL/checkpoints/eval_step_56000/batch_0_cfg_7.5_topk_16384.jpg b/CKPTS/Afteri2i_100kckpt_LLAVAT5_empty_Fixed_just_segment_pretrain_stage3_dreambench_recap_400k_t2i_400k_flux_200k_midjourney_150k_recovery_150k_3e4_epoch2_no_rotated/002-GPT-XL/checkpoints/eval_step_56000/batch_0_cfg_7.5_topk_16384.jpg new file mode 100644 index 0000000000000000000000000000000000000000..3ff7828e785c364fd3e719fa1517ae500638e607 --- /dev/null +++ b/CKPTS/Afteri2i_100kckpt_LLAVAT5_empty_Fixed_just_segment_pretrain_stage3_dreambench_recap_400k_t2i_400k_flux_200k_midjourney_150k_recovery_150k_3e4_epoch2_no_rotated/002-GPT-XL/checkpoints/eval_step_56000/batch_0_cfg_7.5_topk_16384.jpg @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:2ef1c46a03224e433a6640a7cea09a57b426c920838070aaf3e3348b5b0dea8c +size 742976 diff --git a/CKPTS/Afteri2i_100kckpt_LLAVAT5_empty_Fixed_just_segment_pretrain_stage3_dreambench_recap_400k_t2i_400k_flux_200k_midjourney_150k_recovery_150k_3e4_epoch2_no_rotated/002-GPT-XL/checkpoints/eval_step_56000/batch_1_cfg_7.5_topk_16384.jpg b/CKPTS/Afteri2i_100kckpt_LLAVAT5_empty_Fixed_just_segment_pretrain_stage3_dreambench_recap_400k_t2i_400k_flux_200k_midjourney_150k_recovery_150k_3e4_epoch2_no_rotated/002-GPT-XL/checkpoints/eval_step_56000/batch_1_cfg_7.5_topk_16384.jpg new file mode 100644 index 0000000000000000000000000000000000000000..5a46182a51c40dda6acb0409d1bf276f71f53baa --- /dev/null +++ b/CKPTS/Afteri2i_100kckpt_LLAVAT5_empty_Fixed_just_segment_pretrain_stage3_dreambench_recap_400k_t2i_400k_flux_200k_midjourney_150k_recovery_150k_3e4_epoch2_no_rotated/002-GPT-XL/checkpoints/eval_step_56000/batch_1_cfg_7.5_topk_16384.jpg @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:2232a21fb29e8b8f46dbfd22d89a3caa76cae2c2137cd039f447d1165df7798a +size 712987 diff --git a/CKPTS/Afteri2i_100kckpt_LLAVAT5_empty_Fixed_just_segment_pretrain_stage3_dreambench_recap_400k_t2i_400k_flux_200k_midjourney_150k_recovery_150k_3e4_epoch2_no_rotated/002-GPT-XL/checkpoints/eval_step_56000/batch_2_cfg_7.5_topk_16384.jpg b/CKPTS/Afteri2i_100kckpt_LLAVAT5_empty_Fixed_just_segment_pretrain_stage3_dreambench_recap_400k_t2i_400k_flux_200k_midjourney_150k_recovery_150k_3e4_epoch2_no_rotated/002-GPT-XL/checkpoints/eval_step_56000/batch_2_cfg_7.5_topk_16384.jpg new file mode 100644 index 0000000000000000000000000000000000000000..f3b3d2b13261bd844b37bce3c4175c10ff45bf65 --- /dev/null +++ b/CKPTS/Afteri2i_100kckpt_LLAVAT5_empty_Fixed_just_segment_pretrain_stage3_dreambench_recap_400k_t2i_400k_flux_200k_midjourney_150k_recovery_150k_3e4_epoch2_no_rotated/002-GPT-XL/checkpoints/eval_step_56000/batch_2_cfg_7.5_topk_16384.jpg @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:d596608bad5a139ce93b5efcc19f0f8a6b7ce1105cc17bcfbd6ac40238e1e56a +size 737250 diff --git a/CKPTS/Afteri2i_100kckpt_LLAVAT5_empty_Fixed_just_segment_pretrain_stage3_dreambench_recap_400k_t2i_400k_flux_200k_midjourney_150k_recovery_150k_3e4_epoch2_no_rotated/002-GPT-XL/checkpoints/eval_step_56000/batch_3_cfg_7.5_topk_16384.jpg b/CKPTS/Afteri2i_100kckpt_LLAVAT5_empty_Fixed_just_segment_pretrain_stage3_dreambench_recap_400k_t2i_400k_flux_200k_midjourney_150k_recovery_150k_3e4_epoch2_no_rotated/002-GPT-XL/checkpoints/eval_step_56000/batch_3_cfg_7.5_topk_16384.jpg new file mode 100644 index 0000000000000000000000000000000000000000..3b252b7380f6d8731707312c258b47f687cc8b62 --- /dev/null +++ b/CKPTS/Afteri2i_100kckpt_LLAVAT5_empty_Fixed_just_segment_pretrain_stage3_dreambench_recap_400k_t2i_400k_flux_200k_midjourney_150k_recovery_150k_3e4_epoch2_no_rotated/002-GPT-XL/checkpoints/eval_step_56000/batch_3_cfg_7.5_topk_16384.jpg @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:1f7d576b3e03fc6c8e807cf3c1c7f66da3d83a1db49b88a19a05d011db31fc8d +size 840551 diff --git a/CKPTS/Afteri2i_100kckpt_LLAVAT5_empty_Fixed_just_segment_pretrain_stage3_dreambench_recap_400k_t2i_400k_flux_200k_midjourney_150k_recovery_150k_3e4_epoch2_no_rotated/002-GPT-XL/checkpoints/eval_step_58000/batch_0_cfg_7.5_topk_16384.jpg b/CKPTS/Afteri2i_100kckpt_LLAVAT5_empty_Fixed_just_segment_pretrain_stage3_dreambench_recap_400k_t2i_400k_flux_200k_midjourney_150k_recovery_150k_3e4_epoch2_no_rotated/002-GPT-XL/checkpoints/eval_step_58000/batch_0_cfg_7.5_topk_16384.jpg new file mode 100644 index 0000000000000000000000000000000000000000..8287b6774d806e462b9128f1835bb242c6eedb81 --- /dev/null +++ b/CKPTS/Afteri2i_100kckpt_LLAVAT5_empty_Fixed_just_segment_pretrain_stage3_dreambench_recap_400k_t2i_400k_flux_200k_midjourney_150k_recovery_150k_3e4_epoch2_no_rotated/002-GPT-XL/checkpoints/eval_step_58000/batch_0_cfg_7.5_topk_16384.jpg @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:399dea3025387995293d87b143dc637644c22f31ae40564ae0690bed1e3281a8 +size 749841 diff --git a/CKPTS/Afteri2i_100kckpt_LLAVAT5_empty_Fixed_just_segment_pretrain_stage3_dreambench_recap_400k_t2i_400k_flux_200k_midjourney_150k_recovery_150k_3e4_epoch2_no_rotated/002-GPT-XL/checkpoints/eval_step_58000/batch_1_cfg_7.5_topk_16384.jpg b/CKPTS/Afteri2i_100kckpt_LLAVAT5_empty_Fixed_just_segment_pretrain_stage3_dreambench_recap_400k_t2i_400k_flux_200k_midjourney_150k_recovery_150k_3e4_epoch2_no_rotated/002-GPT-XL/checkpoints/eval_step_58000/batch_1_cfg_7.5_topk_16384.jpg new file mode 100644 index 0000000000000000000000000000000000000000..739c43045667ac0d0ea5638fc526165076a3cfc4 --- /dev/null +++ b/CKPTS/Afteri2i_100kckpt_LLAVAT5_empty_Fixed_just_segment_pretrain_stage3_dreambench_recap_400k_t2i_400k_flux_200k_midjourney_150k_recovery_150k_3e4_epoch2_no_rotated/002-GPT-XL/checkpoints/eval_step_58000/batch_1_cfg_7.5_topk_16384.jpg @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:1f66cb5a88ebe059b62f27fba483b24b843b5739d3617c0be70383544b686d72 +size 710879 diff --git a/CKPTS/Afteri2i_100kckpt_LLAVAT5_empty_Fixed_just_segment_pretrain_stage3_dreambench_recap_400k_t2i_400k_flux_200k_midjourney_150k_recovery_150k_3e4_epoch2_no_rotated/002-GPT-XL/checkpoints/eval_step_58000/batch_2_cfg_7.5_topk_16384.jpg b/CKPTS/Afteri2i_100kckpt_LLAVAT5_empty_Fixed_just_segment_pretrain_stage3_dreambench_recap_400k_t2i_400k_flux_200k_midjourney_150k_recovery_150k_3e4_epoch2_no_rotated/002-GPT-XL/checkpoints/eval_step_58000/batch_2_cfg_7.5_topk_16384.jpg new file mode 100644 index 0000000000000000000000000000000000000000..43929616ac207aa13533d72bcf1d87e4ab2e3ca6 --- /dev/null +++ b/CKPTS/Afteri2i_100kckpt_LLAVAT5_empty_Fixed_just_segment_pretrain_stage3_dreambench_recap_400k_t2i_400k_flux_200k_midjourney_150k_recovery_150k_3e4_epoch2_no_rotated/002-GPT-XL/checkpoints/eval_step_58000/batch_2_cfg_7.5_topk_16384.jpg @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:3416059f6cf42b0d261a134d7a059620af38281693b3c18d3840b0e00168920c +size 733891 diff --git a/CKPTS/Afteri2i_100kckpt_LLAVAT5_empty_Fixed_just_segment_pretrain_stage3_dreambench_recap_400k_t2i_400k_flux_200k_midjourney_150k_recovery_150k_3e4_epoch2_no_rotated/002-GPT-XL/checkpoints/eval_step_58000/batch_3_cfg_7.5_topk_16384.jpg b/CKPTS/Afteri2i_100kckpt_LLAVAT5_empty_Fixed_just_segment_pretrain_stage3_dreambench_recap_400k_t2i_400k_flux_200k_midjourney_150k_recovery_150k_3e4_epoch2_no_rotated/002-GPT-XL/checkpoints/eval_step_58000/batch_3_cfg_7.5_topk_16384.jpg new file mode 100644 index 0000000000000000000000000000000000000000..0c16ef0887f31a2924acf2d63eb53a9e7042a21d --- /dev/null +++ b/CKPTS/Afteri2i_100kckpt_LLAVAT5_empty_Fixed_just_segment_pretrain_stage3_dreambench_recap_400k_t2i_400k_flux_200k_midjourney_150k_recovery_150k_3e4_epoch2_no_rotated/002-GPT-XL/checkpoints/eval_step_58000/batch_3_cfg_7.5_topk_16384.jpg @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:c0f66ac28ab7642b6470f1a7f4d4b70332c650b298c74f2bb3ad3f4b9dbd3769 +size 853456 diff --git a/CKPTS/Afteri2i_100kckpt_LLAVAT5_empty_Fixed_just_segment_pretrain_stage3_dreambench_recap_400k_t2i_400k_flux_200k_midjourney_150k_recovery_150k_3e4_epoch2_no_rotated/002-GPT-XL/checkpoints/eval_step_60000/batch_0_cfg_7.5_topk_16384.jpg b/CKPTS/Afteri2i_100kckpt_LLAVAT5_empty_Fixed_just_segment_pretrain_stage3_dreambench_recap_400k_t2i_400k_flux_200k_midjourney_150k_recovery_150k_3e4_epoch2_no_rotated/002-GPT-XL/checkpoints/eval_step_60000/batch_0_cfg_7.5_topk_16384.jpg new file mode 100644 index 0000000000000000000000000000000000000000..686061f22e9027c9bfa9c08a244f1263d2cbf12a --- /dev/null +++ b/CKPTS/Afteri2i_100kckpt_LLAVAT5_empty_Fixed_just_segment_pretrain_stage3_dreambench_recap_400k_t2i_400k_flux_200k_midjourney_150k_recovery_150k_3e4_epoch2_no_rotated/002-GPT-XL/checkpoints/eval_step_60000/batch_0_cfg_7.5_topk_16384.jpg @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:530e33714c62819f9a2560d9c6983f281f6ae8ce0da9462ff6598ed1be293eba +size 734258 diff --git a/CKPTS/Afteri2i_100kckpt_LLAVAT5_empty_Fixed_just_segment_pretrain_stage3_dreambench_recap_400k_t2i_400k_flux_200k_midjourney_150k_recovery_150k_3e4_epoch2_no_rotated/002-GPT-XL/checkpoints/eval_step_60000/batch_1_cfg_7.5_topk_16384.jpg b/CKPTS/Afteri2i_100kckpt_LLAVAT5_empty_Fixed_just_segment_pretrain_stage3_dreambench_recap_400k_t2i_400k_flux_200k_midjourney_150k_recovery_150k_3e4_epoch2_no_rotated/002-GPT-XL/checkpoints/eval_step_60000/batch_1_cfg_7.5_topk_16384.jpg new file mode 100644 index 0000000000000000000000000000000000000000..8c9c3d8f3d9ae349ae962257f2f0167be0a8e49c --- /dev/null +++ b/CKPTS/Afteri2i_100kckpt_LLAVAT5_empty_Fixed_just_segment_pretrain_stage3_dreambench_recap_400k_t2i_400k_flux_200k_midjourney_150k_recovery_150k_3e4_epoch2_no_rotated/002-GPT-XL/checkpoints/eval_step_60000/batch_1_cfg_7.5_topk_16384.jpg @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:2303a3c84b798d4ccffebcc90320107823077cd2a7c17dc80a0553ff094f25e6 +size 696938 diff --git a/CKPTS/Afteri2i_100kckpt_LLAVAT5_empty_Fixed_just_segment_pretrain_stage3_dreambench_recap_400k_t2i_400k_flux_200k_midjourney_150k_recovery_150k_3e4_epoch2_no_rotated/002-GPT-XL/checkpoints/eval_step_60000/batch_2_cfg_7.5_topk_16384.jpg b/CKPTS/Afteri2i_100kckpt_LLAVAT5_empty_Fixed_just_segment_pretrain_stage3_dreambench_recap_400k_t2i_400k_flux_200k_midjourney_150k_recovery_150k_3e4_epoch2_no_rotated/002-GPT-XL/checkpoints/eval_step_60000/batch_2_cfg_7.5_topk_16384.jpg new file mode 100644 index 0000000000000000000000000000000000000000..2cbd52b463deb0b9ff5709738f328449920aa31d --- /dev/null +++ b/CKPTS/Afteri2i_100kckpt_LLAVAT5_empty_Fixed_just_segment_pretrain_stage3_dreambench_recap_400k_t2i_400k_flux_200k_midjourney_150k_recovery_150k_3e4_epoch2_no_rotated/002-GPT-XL/checkpoints/eval_step_60000/batch_2_cfg_7.5_topk_16384.jpg @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:f1e51e2c64ec10e2d622e11959b7c63aa7e9b8f0a51960c88f1f408a0412faf2 +size 722256 diff --git a/CKPTS/Afteri2i_100kckpt_LLAVAT5_empty_Fixed_just_segment_pretrain_stage3_dreambench_recap_400k_t2i_400k_flux_200k_midjourney_150k_recovery_150k_3e4_epoch2_no_rotated/002-GPT-XL/checkpoints/eval_step_60000/batch_3_cfg_7.5_topk_16384.jpg b/CKPTS/Afteri2i_100kckpt_LLAVAT5_empty_Fixed_just_segment_pretrain_stage3_dreambench_recap_400k_t2i_400k_flux_200k_midjourney_150k_recovery_150k_3e4_epoch2_no_rotated/002-GPT-XL/checkpoints/eval_step_60000/batch_3_cfg_7.5_topk_16384.jpg new file mode 100644 index 0000000000000000000000000000000000000000..4ebafc594beb44498f69a40d93f0e0ec17c7f008 --- /dev/null +++ b/CKPTS/Afteri2i_100kckpt_LLAVAT5_empty_Fixed_just_segment_pretrain_stage3_dreambench_recap_400k_t2i_400k_flux_200k_midjourney_150k_recovery_150k_3e4_epoch2_no_rotated/002-GPT-XL/checkpoints/eval_step_60000/batch_3_cfg_7.5_topk_16384.jpg @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:83c16685dd7b2c5c86f6f1c516e1c2f1ea5ac8aa8f05a4bf61833af46d0c38a6 +size 867947 diff --git a/CKPTS/Afteri2i_100kckpt_LLAVAT5_empty_Fixed_just_segment_pretrain_stage3_dreambench_recap_400k_t2i_400k_flux_200k_midjourney_150k_recovery_150k_3e4_epoch2_no_rotated/002-GPT-XL/checkpoints/eval_step_62000/batch_0_cfg_7.5_topk_16384.jpg b/CKPTS/Afteri2i_100kckpt_LLAVAT5_empty_Fixed_just_segment_pretrain_stage3_dreambench_recap_400k_t2i_400k_flux_200k_midjourney_150k_recovery_150k_3e4_epoch2_no_rotated/002-GPT-XL/checkpoints/eval_step_62000/batch_0_cfg_7.5_topk_16384.jpg new file mode 100644 index 0000000000000000000000000000000000000000..3be715671ed92932b97ee60d2e5ed917c09792a2 --- /dev/null +++ b/CKPTS/Afteri2i_100kckpt_LLAVAT5_empty_Fixed_just_segment_pretrain_stage3_dreambench_recap_400k_t2i_400k_flux_200k_midjourney_150k_recovery_150k_3e4_epoch2_no_rotated/002-GPT-XL/checkpoints/eval_step_62000/batch_0_cfg_7.5_topk_16384.jpg @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:52e3789ed7f74336b751504d3bb7682ee86e8fc1b0d00a689fdb899c958b1484 +size 739005 diff --git a/CKPTS/Afteri2i_100kckpt_LLAVAT5_empty_Fixed_just_segment_pretrain_stage3_dreambench_recap_400k_t2i_400k_flux_200k_midjourney_150k_recovery_150k_3e4_epoch2_no_rotated/002-GPT-XL/checkpoints/eval_step_62000/batch_1_cfg_7.5_topk_16384.jpg b/CKPTS/Afteri2i_100kckpt_LLAVAT5_empty_Fixed_just_segment_pretrain_stage3_dreambench_recap_400k_t2i_400k_flux_200k_midjourney_150k_recovery_150k_3e4_epoch2_no_rotated/002-GPT-XL/checkpoints/eval_step_62000/batch_1_cfg_7.5_topk_16384.jpg new file mode 100644 index 0000000000000000000000000000000000000000..fe1cbdb4b4b711044a075240a06ec3147e07e2be --- /dev/null +++ b/CKPTS/Afteri2i_100kckpt_LLAVAT5_empty_Fixed_just_segment_pretrain_stage3_dreambench_recap_400k_t2i_400k_flux_200k_midjourney_150k_recovery_150k_3e4_epoch2_no_rotated/002-GPT-XL/checkpoints/eval_step_62000/batch_1_cfg_7.5_topk_16384.jpg @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:cf81b5c50efe0c50f2de2eca1e27234b1bdb1092545bbde924b98fd392032c86 +size 716826 diff --git a/CKPTS/Afteri2i_100kckpt_LLAVAT5_empty_Fixed_just_segment_pretrain_stage3_dreambench_recap_400k_t2i_400k_flux_200k_midjourney_150k_recovery_150k_3e4_epoch2_no_rotated/002-GPT-XL/checkpoints/eval_step_62000/batch_2_cfg_7.5_topk_16384.jpg b/CKPTS/Afteri2i_100kckpt_LLAVAT5_empty_Fixed_just_segment_pretrain_stage3_dreambench_recap_400k_t2i_400k_flux_200k_midjourney_150k_recovery_150k_3e4_epoch2_no_rotated/002-GPT-XL/checkpoints/eval_step_62000/batch_2_cfg_7.5_topk_16384.jpg new file mode 100644 index 0000000000000000000000000000000000000000..0ff9cfcb025e01ca39e82c8265187930ccc4e76a --- /dev/null +++ b/CKPTS/Afteri2i_100kckpt_LLAVAT5_empty_Fixed_just_segment_pretrain_stage3_dreambench_recap_400k_t2i_400k_flux_200k_midjourney_150k_recovery_150k_3e4_epoch2_no_rotated/002-GPT-XL/checkpoints/eval_step_62000/batch_2_cfg_7.5_topk_16384.jpg @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:4508f2a417bd6abaceb7bbb8fe970a301713340876b88b929ad982d55a0ac705 +size 736308 diff --git a/CKPTS/Afteri2i_100kckpt_LLAVAT5_empty_Fixed_just_segment_pretrain_stage3_dreambench_recap_400k_t2i_400k_flux_200k_midjourney_150k_recovery_150k_3e4_epoch2_no_rotated/002-GPT-XL/checkpoints/eval_step_62000/batch_3_cfg_7.5_topk_16384.jpg b/CKPTS/Afteri2i_100kckpt_LLAVAT5_empty_Fixed_just_segment_pretrain_stage3_dreambench_recap_400k_t2i_400k_flux_200k_midjourney_150k_recovery_150k_3e4_epoch2_no_rotated/002-GPT-XL/checkpoints/eval_step_62000/batch_3_cfg_7.5_topk_16384.jpg new file mode 100644 index 0000000000000000000000000000000000000000..2ef2e4e0b7e111480ebe119880700ea69604b707 --- /dev/null +++ b/CKPTS/Afteri2i_100kckpt_LLAVAT5_empty_Fixed_just_segment_pretrain_stage3_dreambench_recap_400k_t2i_400k_flux_200k_midjourney_150k_recovery_150k_3e4_epoch2_no_rotated/002-GPT-XL/checkpoints/eval_step_62000/batch_3_cfg_7.5_topk_16384.jpg @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:2674b8931d2386865037e8d0e2ce9d5cefed649ee4495f7925d997c6fdedbac3 +size 880788 diff --git a/CKPTS/Afteri2i_100kckpt_LLAVAT5_empty_Fixed_just_segment_pretrain_stage3_dreambench_recap_400k_t2i_400k_flux_200k_midjourney_150k_recovery_150k_3e4_epoch2_no_rotated/002-GPT-XL/checkpoints/eval_step_64000/batch_0_cfg_7.5_topk_16384.jpg b/CKPTS/Afteri2i_100kckpt_LLAVAT5_empty_Fixed_just_segment_pretrain_stage3_dreambench_recap_400k_t2i_400k_flux_200k_midjourney_150k_recovery_150k_3e4_epoch2_no_rotated/002-GPT-XL/checkpoints/eval_step_64000/batch_0_cfg_7.5_topk_16384.jpg new file mode 100644 index 0000000000000000000000000000000000000000..1a19ee8b3cb1820aba94bfdc103f13519ce91617 --- /dev/null +++ b/CKPTS/Afteri2i_100kckpt_LLAVAT5_empty_Fixed_just_segment_pretrain_stage3_dreambench_recap_400k_t2i_400k_flux_200k_midjourney_150k_recovery_150k_3e4_epoch2_no_rotated/002-GPT-XL/checkpoints/eval_step_64000/batch_0_cfg_7.5_topk_16384.jpg @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:b545e66c7499ff6bdc0d79047ec61f94f5209a67ea98faafac8305ebbc9eef31 +size 734087 diff --git a/CKPTS/Afteri2i_100kckpt_LLAVAT5_empty_Fixed_just_segment_pretrain_stage3_dreambench_recap_400k_t2i_400k_flux_200k_midjourney_150k_recovery_150k_3e4_epoch2_no_rotated/002-GPT-XL/checkpoints/eval_step_64000/batch_1_cfg_7.5_topk_16384.jpg b/CKPTS/Afteri2i_100kckpt_LLAVAT5_empty_Fixed_just_segment_pretrain_stage3_dreambench_recap_400k_t2i_400k_flux_200k_midjourney_150k_recovery_150k_3e4_epoch2_no_rotated/002-GPT-XL/checkpoints/eval_step_64000/batch_1_cfg_7.5_topk_16384.jpg new file mode 100644 index 0000000000000000000000000000000000000000..3f7db0648e4425fe6af86749d1c9aa23b65e8215 --- /dev/null +++ b/CKPTS/Afteri2i_100kckpt_LLAVAT5_empty_Fixed_just_segment_pretrain_stage3_dreambench_recap_400k_t2i_400k_flux_200k_midjourney_150k_recovery_150k_3e4_epoch2_no_rotated/002-GPT-XL/checkpoints/eval_step_64000/batch_1_cfg_7.5_topk_16384.jpg @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:07a09cf419d595f358f61508ce87f3bd9f298f9e87d65a2c7f47b7c52b8207ad +size 699056 diff --git a/CKPTS/Afteri2i_100kckpt_LLAVAT5_empty_Fixed_just_segment_pretrain_stage3_dreambench_recap_400k_t2i_400k_flux_200k_midjourney_150k_recovery_150k_3e4_epoch2_no_rotated/002-GPT-XL/checkpoints/eval_step_64000/batch_2_cfg_7.5_topk_16384.jpg b/CKPTS/Afteri2i_100kckpt_LLAVAT5_empty_Fixed_just_segment_pretrain_stage3_dreambench_recap_400k_t2i_400k_flux_200k_midjourney_150k_recovery_150k_3e4_epoch2_no_rotated/002-GPT-XL/checkpoints/eval_step_64000/batch_2_cfg_7.5_topk_16384.jpg new file mode 100644 index 0000000000000000000000000000000000000000..c126169f11d3fb109c5ea0a6125ab08dcc828076 --- /dev/null +++ b/CKPTS/Afteri2i_100kckpt_LLAVAT5_empty_Fixed_just_segment_pretrain_stage3_dreambench_recap_400k_t2i_400k_flux_200k_midjourney_150k_recovery_150k_3e4_epoch2_no_rotated/002-GPT-XL/checkpoints/eval_step_64000/batch_2_cfg_7.5_topk_16384.jpg @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:93aca35f90be0a06ad07dbc55f7fe512f991c6d886e6fa6c1a66bfcb2ea41e1b +size 742580 diff --git a/CKPTS/Afteri2i_100kckpt_LLAVAT5_empty_Fixed_just_segment_pretrain_stage3_dreambench_recap_400k_t2i_400k_flux_200k_midjourney_150k_recovery_150k_3e4_epoch2_no_rotated/002-GPT-XL/checkpoints/eval_step_64000/batch_3_cfg_7.5_topk_16384.jpg b/CKPTS/Afteri2i_100kckpt_LLAVAT5_empty_Fixed_just_segment_pretrain_stage3_dreambench_recap_400k_t2i_400k_flux_200k_midjourney_150k_recovery_150k_3e4_epoch2_no_rotated/002-GPT-XL/checkpoints/eval_step_64000/batch_3_cfg_7.5_topk_16384.jpg new file mode 100644 index 0000000000000000000000000000000000000000..0bd84a587f8af36b2763d7ee5b794a270f4001a9 --- /dev/null +++ b/CKPTS/Afteri2i_100kckpt_LLAVAT5_empty_Fixed_just_segment_pretrain_stage3_dreambench_recap_400k_t2i_400k_flux_200k_midjourney_150k_recovery_150k_3e4_epoch2_no_rotated/002-GPT-XL/checkpoints/eval_step_64000/batch_3_cfg_7.5_topk_16384.jpg @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:ddfda1433bece95ede16d5456ed2215ec2fa423d1dcda3d6fb1420d944c85a29 +size 861241 diff --git a/CKPTS/Afteri2i_100kckpt_LLAVAT5_empty_Fixed_just_segment_pretrain_stage3_dreambench_recap_400k_t2i_400k_flux_200k_midjourney_150k_recovery_150k_3e4_epoch2_no_rotated/002-GPT-XL/checkpoints/eval_step_66000/batch_0_cfg_7.5_topk_16384.jpg b/CKPTS/Afteri2i_100kckpt_LLAVAT5_empty_Fixed_just_segment_pretrain_stage3_dreambench_recap_400k_t2i_400k_flux_200k_midjourney_150k_recovery_150k_3e4_epoch2_no_rotated/002-GPT-XL/checkpoints/eval_step_66000/batch_0_cfg_7.5_topk_16384.jpg new file mode 100644 index 0000000000000000000000000000000000000000..8e74c176c25c17b7b8286906a4045222672f4243 --- /dev/null +++ b/CKPTS/Afteri2i_100kckpt_LLAVAT5_empty_Fixed_just_segment_pretrain_stage3_dreambench_recap_400k_t2i_400k_flux_200k_midjourney_150k_recovery_150k_3e4_epoch2_no_rotated/002-GPT-XL/checkpoints/eval_step_66000/batch_0_cfg_7.5_topk_16384.jpg @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:b734d7dc7e48ff49bf00c81237da083712515f282450360bcdbfbfd64925e454 +size 764452 diff --git a/CKPTS/Afteri2i_100kckpt_LLAVAT5_empty_Fixed_just_segment_pretrain_stage3_dreambench_recap_400k_t2i_400k_flux_200k_midjourney_150k_recovery_150k_3e4_epoch2_no_rotated/002-GPT-XL/checkpoints/eval_step_66000/batch_1_cfg_7.5_topk_16384.jpg b/CKPTS/Afteri2i_100kckpt_LLAVAT5_empty_Fixed_just_segment_pretrain_stage3_dreambench_recap_400k_t2i_400k_flux_200k_midjourney_150k_recovery_150k_3e4_epoch2_no_rotated/002-GPT-XL/checkpoints/eval_step_66000/batch_1_cfg_7.5_topk_16384.jpg new file mode 100644 index 0000000000000000000000000000000000000000..7c93947a7b52dc878d4dcb8183451592a4f5643c --- /dev/null +++ b/CKPTS/Afteri2i_100kckpt_LLAVAT5_empty_Fixed_just_segment_pretrain_stage3_dreambench_recap_400k_t2i_400k_flux_200k_midjourney_150k_recovery_150k_3e4_epoch2_no_rotated/002-GPT-XL/checkpoints/eval_step_66000/batch_1_cfg_7.5_topk_16384.jpg @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:f54d975dd811e41e8fc22c57434e6f38bc126e60c661ee40db86d5c97766b1e2 +size 720284 diff --git a/CKPTS/Afteri2i_100kckpt_LLAVAT5_empty_Fixed_just_segment_pretrain_stage3_dreambench_recap_400k_t2i_400k_flux_200k_midjourney_150k_recovery_150k_3e4_epoch2_no_rotated/002-GPT-XL/checkpoints/eval_step_66000/batch_2_cfg_7.5_topk_16384.jpg b/CKPTS/Afteri2i_100kckpt_LLAVAT5_empty_Fixed_just_segment_pretrain_stage3_dreambench_recap_400k_t2i_400k_flux_200k_midjourney_150k_recovery_150k_3e4_epoch2_no_rotated/002-GPT-XL/checkpoints/eval_step_66000/batch_2_cfg_7.5_topk_16384.jpg new file mode 100644 index 0000000000000000000000000000000000000000..3099d9b467db695293b024e4ea88236f713e341d --- /dev/null +++ b/CKPTS/Afteri2i_100kckpt_LLAVAT5_empty_Fixed_just_segment_pretrain_stage3_dreambench_recap_400k_t2i_400k_flux_200k_midjourney_150k_recovery_150k_3e4_epoch2_no_rotated/002-GPT-XL/checkpoints/eval_step_66000/batch_2_cfg_7.5_topk_16384.jpg @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:493d85bb55e25ffb1efbf0dd66b0b934752bf2959853c90d31873a8f43a2b6c0 +size 766768 diff --git a/CKPTS/Afteri2i_100kckpt_LLAVAT5_empty_Fixed_just_segment_pretrain_stage3_dreambench_recap_400k_t2i_400k_flux_200k_midjourney_150k_recovery_150k_3e4_epoch2_no_rotated/002-GPT-XL/checkpoints/eval_step_66000/batch_3_cfg_7.5_topk_16384.jpg b/CKPTS/Afteri2i_100kckpt_LLAVAT5_empty_Fixed_just_segment_pretrain_stage3_dreambench_recap_400k_t2i_400k_flux_200k_midjourney_150k_recovery_150k_3e4_epoch2_no_rotated/002-GPT-XL/checkpoints/eval_step_66000/batch_3_cfg_7.5_topk_16384.jpg new file mode 100644 index 0000000000000000000000000000000000000000..1e3750ff8e5b4dbedec41a368ee8f57c5066ea11 --- /dev/null +++ b/CKPTS/Afteri2i_100kckpt_LLAVAT5_empty_Fixed_just_segment_pretrain_stage3_dreambench_recap_400k_t2i_400k_flux_200k_midjourney_150k_recovery_150k_3e4_epoch2_no_rotated/002-GPT-XL/checkpoints/eval_step_66000/batch_3_cfg_7.5_topk_16384.jpg @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:b7efe736e59d04de8bfe3db7c8882cc996b1c6bad53e1be8b3fc0dc554125376 +size 874429 diff --git a/CKPTS/Afteri2i_100kckpt_LLAVAT5_empty_Fixed_just_segment_pretrain_stage3_dreambench_recap_400k_t2i_400k_flux_200k_midjourney_150k_recovery_150k_3e4_epoch2_no_rotated/002-GPT-XL/checkpoints/eval_step_68000/batch_0_cfg_7.5_topk_16384.jpg b/CKPTS/Afteri2i_100kckpt_LLAVAT5_empty_Fixed_just_segment_pretrain_stage3_dreambench_recap_400k_t2i_400k_flux_200k_midjourney_150k_recovery_150k_3e4_epoch2_no_rotated/002-GPT-XL/checkpoints/eval_step_68000/batch_0_cfg_7.5_topk_16384.jpg new file mode 100644 index 0000000000000000000000000000000000000000..7c2c6d7e2c12ed0256a492ea90fba7ef447436ed --- /dev/null +++ b/CKPTS/Afteri2i_100kckpt_LLAVAT5_empty_Fixed_just_segment_pretrain_stage3_dreambench_recap_400k_t2i_400k_flux_200k_midjourney_150k_recovery_150k_3e4_epoch2_no_rotated/002-GPT-XL/checkpoints/eval_step_68000/batch_0_cfg_7.5_topk_16384.jpg @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:7ee1e0711fa30f4a2b0d42d834aa9f4d3a6af020c639f0793a18ea0b8802c05b +size 736914 diff --git a/CKPTS/Afteri2i_100kckpt_LLAVAT5_empty_Fixed_just_segment_pretrain_stage3_dreambench_recap_400k_t2i_400k_flux_200k_midjourney_150k_recovery_150k_3e4_epoch2_no_rotated/002-GPT-XL/checkpoints/eval_step_68000/batch_1_cfg_7.5_topk_16384.jpg b/CKPTS/Afteri2i_100kckpt_LLAVAT5_empty_Fixed_just_segment_pretrain_stage3_dreambench_recap_400k_t2i_400k_flux_200k_midjourney_150k_recovery_150k_3e4_epoch2_no_rotated/002-GPT-XL/checkpoints/eval_step_68000/batch_1_cfg_7.5_topk_16384.jpg new file mode 100644 index 0000000000000000000000000000000000000000..87b2b02ab448cff73d77b78998e8504399c0c70e --- /dev/null +++ b/CKPTS/Afteri2i_100kckpt_LLAVAT5_empty_Fixed_just_segment_pretrain_stage3_dreambench_recap_400k_t2i_400k_flux_200k_midjourney_150k_recovery_150k_3e4_epoch2_no_rotated/002-GPT-XL/checkpoints/eval_step_68000/batch_1_cfg_7.5_topk_16384.jpg @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:1c9a58f3c3f5adfce443796eba4b442718e428713c9248ed33b1af8d20ec8f32 +size 721999 diff --git a/CKPTS/Afteri2i_100kckpt_LLAVAT5_empty_Fixed_just_segment_pretrain_stage3_dreambench_recap_400k_t2i_400k_flux_200k_midjourney_150k_recovery_150k_3e4_epoch2_no_rotated/002-GPT-XL/checkpoints/eval_step_68000/batch_2_cfg_7.5_topk_16384.jpg b/CKPTS/Afteri2i_100kckpt_LLAVAT5_empty_Fixed_just_segment_pretrain_stage3_dreambench_recap_400k_t2i_400k_flux_200k_midjourney_150k_recovery_150k_3e4_epoch2_no_rotated/002-GPT-XL/checkpoints/eval_step_68000/batch_2_cfg_7.5_topk_16384.jpg new file mode 100644 index 0000000000000000000000000000000000000000..6753761a769e08f677d47199908824a07d003d51 --- /dev/null +++ b/CKPTS/Afteri2i_100kckpt_LLAVAT5_empty_Fixed_just_segment_pretrain_stage3_dreambench_recap_400k_t2i_400k_flux_200k_midjourney_150k_recovery_150k_3e4_epoch2_no_rotated/002-GPT-XL/checkpoints/eval_step_68000/batch_2_cfg_7.5_topk_16384.jpg @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:9411b00958bb25a63c1d25f8a0fea21c30a7cba35433876d65ed25981955cfd4 +size 740089 diff --git a/CKPTS/Afteri2i_100kckpt_LLAVAT5_empty_Fixed_just_segment_pretrain_stage3_dreambench_recap_400k_t2i_400k_flux_200k_midjourney_150k_recovery_150k_3e4_epoch2_no_rotated/002-GPT-XL/checkpoints/eval_step_68000/batch_3_cfg_7.5_topk_16384.jpg b/CKPTS/Afteri2i_100kckpt_LLAVAT5_empty_Fixed_just_segment_pretrain_stage3_dreambench_recap_400k_t2i_400k_flux_200k_midjourney_150k_recovery_150k_3e4_epoch2_no_rotated/002-GPT-XL/checkpoints/eval_step_68000/batch_3_cfg_7.5_topk_16384.jpg new file mode 100644 index 0000000000000000000000000000000000000000..34c48bface5b34b858bd3e746193ae8b5f333f83 --- /dev/null +++ b/CKPTS/Afteri2i_100kckpt_LLAVAT5_empty_Fixed_just_segment_pretrain_stage3_dreambench_recap_400k_t2i_400k_flux_200k_midjourney_150k_recovery_150k_3e4_epoch2_no_rotated/002-GPT-XL/checkpoints/eval_step_68000/batch_3_cfg_7.5_topk_16384.jpg @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:190a95eed683cd3b4ad7aa317a1da55896ca09ba89ea12f51eed335393e897b1 +size 871382 diff --git a/CKPTS/Afteri2i_100kckpt_LLAVAT5_empty_Fixed_just_segment_pretrain_stage3_dreambench_recap_400k_t2i_400k_flux_200k_midjourney_150k_recovery_150k_3e4_epoch2_no_rotated/002-GPT-XL/checkpoints/eval_step_70000/batch_0_cfg_7.5_topk_16384.jpg b/CKPTS/Afteri2i_100kckpt_LLAVAT5_empty_Fixed_just_segment_pretrain_stage3_dreambench_recap_400k_t2i_400k_flux_200k_midjourney_150k_recovery_150k_3e4_epoch2_no_rotated/002-GPT-XL/checkpoints/eval_step_70000/batch_0_cfg_7.5_topk_16384.jpg new file mode 100644 index 0000000000000000000000000000000000000000..07293f9885891c78985cbf8602cca66b7f1452c6 --- /dev/null +++ b/CKPTS/Afteri2i_100kckpt_LLAVAT5_empty_Fixed_just_segment_pretrain_stage3_dreambench_recap_400k_t2i_400k_flux_200k_midjourney_150k_recovery_150k_3e4_epoch2_no_rotated/002-GPT-XL/checkpoints/eval_step_70000/batch_0_cfg_7.5_topk_16384.jpg @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:3c0fb47b23a193d5275c4bdfb18f407c5afa1a5d05a00d173eaba483d750fabb +size 728788 diff --git a/CKPTS/Afteri2i_100kckpt_LLAVAT5_empty_Fixed_just_segment_pretrain_stage3_dreambench_recap_400k_t2i_400k_flux_200k_midjourney_150k_recovery_150k_3e4_epoch2_no_rotated/002-GPT-XL/checkpoints/eval_step_70000/batch_1_cfg_7.5_topk_16384.jpg b/CKPTS/Afteri2i_100kckpt_LLAVAT5_empty_Fixed_just_segment_pretrain_stage3_dreambench_recap_400k_t2i_400k_flux_200k_midjourney_150k_recovery_150k_3e4_epoch2_no_rotated/002-GPT-XL/checkpoints/eval_step_70000/batch_1_cfg_7.5_topk_16384.jpg new file mode 100644 index 0000000000000000000000000000000000000000..6522419059c8d65b85c382f518ea23f78dc37c3b --- /dev/null +++ b/CKPTS/Afteri2i_100kckpt_LLAVAT5_empty_Fixed_just_segment_pretrain_stage3_dreambench_recap_400k_t2i_400k_flux_200k_midjourney_150k_recovery_150k_3e4_epoch2_no_rotated/002-GPT-XL/checkpoints/eval_step_70000/batch_1_cfg_7.5_topk_16384.jpg @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:d70c110d201ee4759cb65b1d525a7dcbdae7e75a86764b2e968e8085d40c046b +size 701036 diff --git a/CKPTS/Afteri2i_100kckpt_LLAVAT5_empty_Fixed_just_segment_pretrain_stage3_dreambench_recap_400k_t2i_400k_flux_200k_midjourney_150k_recovery_150k_3e4_epoch2_no_rotated/002-GPT-XL/checkpoints/eval_step_70000/batch_2_cfg_7.5_topk_16384.jpg b/CKPTS/Afteri2i_100kckpt_LLAVAT5_empty_Fixed_just_segment_pretrain_stage3_dreambench_recap_400k_t2i_400k_flux_200k_midjourney_150k_recovery_150k_3e4_epoch2_no_rotated/002-GPT-XL/checkpoints/eval_step_70000/batch_2_cfg_7.5_topk_16384.jpg new file mode 100644 index 0000000000000000000000000000000000000000..a7d717295ed0a79fdad6b2fc564e61bf67ff1ce6 --- /dev/null +++ b/CKPTS/Afteri2i_100kckpt_LLAVAT5_empty_Fixed_just_segment_pretrain_stage3_dreambench_recap_400k_t2i_400k_flux_200k_midjourney_150k_recovery_150k_3e4_epoch2_no_rotated/002-GPT-XL/checkpoints/eval_step_70000/batch_2_cfg_7.5_topk_16384.jpg @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:efbde8d3b1c7ae6af7f1b5c4f65d864265c2b713cabfdc5044190b87e245a941 +size 741074 diff --git a/CKPTS/Afteri2i_100kckpt_LLAVAT5_empty_Fixed_just_segment_pretrain_stage3_dreambench_recap_400k_t2i_400k_flux_200k_midjourney_150k_recovery_150k_3e4_epoch2_no_rotated/002-GPT-XL/checkpoints/eval_step_70000/batch_3_cfg_7.5_topk_16384.jpg b/CKPTS/Afteri2i_100kckpt_LLAVAT5_empty_Fixed_just_segment_pretrain_stage3_dreambench_recap_400k_t2i_400k_flux_200k_midjourney_150k_recovery_150k_3e4_epoch2_no_rotated/002-GPT-XL/checkpoints/eval_step_70000/batch_3_cfg_7.5_topk_16384.jpg new file mode 100644 index 0000000000000000000000000000000000000000..d8fe63a0ba97c62e5a854e366bf2b41c3bf75b63 --- /dev/null +++ b/CKPTS/Afteri2i_100kckpt_LLAVAT5_empty_Fixed_just_segment_pretrain_stage3_dreambench_recap_400k_t2i_400k_flux_200k_midjourney_150k_recovery_150k_3e4_epoch2_no_rotated/002-GPT-XL/checkpoints/eval_step_70000/batch_3_cfg_7.5_topk_16384.jpg @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:79ab6957e8cb918c150ef583b13d4c27425aa8947bfee680b0403a182307f2a7 +size 873408 diff --git a/CKPTS/Afteri2i_100kckpt_LLAVAT5_empty_Fixed_just_segment_pretrain_stage3_dreambench_recap_400k_t2i_400k_flux_200k_midjourney_150k_recovery_150k_3e4_epoch2_no_rotated/002-GPT-XL/log.txt b/CKPTS/Afteri2i_100kckpt_LLAVAT5_empty_Fixed_just_segment_pretrain_stage3_dreambench_recap_400k_t2i_400k_flux_200k_midjourney_150k_recovery_150k_3e4_epoch2_no_rotated/002-GPT-XL/log.txt new file mode 100644 index 0000000000000000000000000000000000000000..6b62bbdd9e29cd2c029b39fe707d0a971685ec06 --- /dev/null +++ b/CKPTS/Afteri2i_100kckpt_LLAVAT5_empty_Fixed_just_segment_pretrain_stage3_dreambench_recap_400k_t2i_400k_flux_200k_midjourney_150k_recovery_150k_3e4_epoch2_no_rotated/002-GPT-XL/log.txt @@ -0,0 +1,1041 @@ +[2025-04-25 06:05:59] Experiment directory created at checkpoint/Afteri2i_100kckpt_LLAVAT5_empty_Fixed_just_segment_pretrain_stage3_dreambench_recap_400k_t2i_400k_flux_200k_midjourney_150k_recovery_150k_3e4_epoch2_no_rotated/002-GPT-XL +[2025-04-25 06:05:59] Namespace(data_path='/tmp/haozhezhao/MLLMG/jsonl_data/merged_train_set_set_subject_400k_recap_t2i_400k_flux_200k_midjourney_150k_recovery_150k_grounding_100fluxseg_50samseg.jsonl', cloud_save_path='/tmp/haozhezhao/checkpoint', no_local_save=False, vq_model='VQ-16', vq_ckpt='/tmp/haozhezhao/model/llamagen_t2i/vq_ds16_t2i.pt', codebook_size=16384, codebook_embed_dim=8, gpt_model='GPT-XL', gpt_ckpt='/tmp/haozhezhao/MLLMG/checkpoint/Afteri2i_100kckpt_LLAVAT5_empty_Fixed_just_segment_pretrain_stage3_dreambench_recap_400k_t2i_400k_flux_200k_midjourney_150k_recovery_150k_3e4_epoch2_no_rotated/000-GPT-XL/checkpoints/0046000.pt', gpt_type='t2i', vocab_size=16384, cls_token_num=512, dropout_p=0.1, token_dropout_p=0.1, drop_path=0.0, no_compile=False, results_dir='checkpoint/Afteri2i_100kckpt_LLAVAT5_empty_Fixed_just_segment_pretrain_stage3_dreambench_recap_400k_t2i_400k_flux_200k_midjourney_150k_recovery_150k_3e4_epoch2_no_rotated', dataset='ti2i', image_size=512, downsample_size=16, num_classes=1000, epochs=3, lr=0.0003, weight_decay=0.05, beta1=0.9, beta2=0.95, max_grad_norm=1.0, global_batch_size=56, global_seed=0, num_workers=8, log_every=25, ckpt_every=2000, gradient_accumulation_steps=4, mixed_precision='bf16', val_data_path='/tmp/haozhezhao/MLLMG/jsonl_data/dreambench_plus_valid.jsonl', use_vision_tower=True, model_name_or_path='/tmp/haozhezhao/model/blip2-flan-t5-xl', image_place_holder='', processor_path=None, do_eval=True, max_eval_samples=250, train_text_encoder=True, no_left_padding=False, cfg_scale=7.5, top_k=16384, temperature=0.9, top_p=1.0, eval_steps=2000, project_name='llamagen_ti2i', load_from_checkpoint='/tmp/haozhezhao/MLLMG/checkpoint/FIXed_3M_1epoch_step32000.pt', warmup=0.05, lr_decay_style='cosine', lr_decay_ratio=0.1, train_iters=500000, class_dropout_prob=0.1, with_image_only=False, image_only_rate=0.1, stage2=False, subject_driven=True, load_subject_embedding=None, reference_data_path='/tmp/haozhezhao/MLLMG/cc12m_reference_tunnel.jsonl', multimodal_encoder='llava', do_recovery=True, no_replace=False, resume=True, dreambench_eval=True, find_unused_parameters=True, load_visual_encoder=False, continue_stage1=False, replace_subject=False, train_all=True, save_total_limit=1, load_language_projection='/tmp/haozhezhao/MLLMG/llava-v1.5-flant5_fixed-pretrain/mm_projector.bin', mm_vision_tower='openai/clip-vit-large-patch14', load_fixed_llamagen=True, unfreeze_output=False, fix='gpt-empty-fix', rank=0, world_size=8, gpu=0, dist_url='env://', distributed=True, dist_backend='nccl') +[2025-04-25 06:05:59] Starting rank=0, seed=0, world_size=8. +[2025-04-25 06:05:59] text_config is None. Initializing the text config with default values (`OPTConfig`). +[2025-04-25 06:07:05] GPT Parameters: 2,310,680,832 +[2025-04-25 06:07:05] num decayed parameter tensors: 356, with 2,007,303,168 parameters +[2025-04-25 06:07:05] num non-decayed parameter tensors: 124, with 197,888 parameters +[2025-04-25 06:07:05] using fused AdamW: True +[2025-04-25 06:07:13] Dataset contains 1,313,682 images +[2025-04-25 06:07:13] Train iters 70374 , warmup 3518.7000000000003, len of loader 23458 +[2025-04-25 06:07:29] Resume training from checkpoint: /tmp/haozhezhao/MLLMG/checkpoint/Afteri2i_100kckpt_LLAVAT5_empty_Fixed_just_segment_pretrain_stage3_dreambench_recap_400k_t2i_400k_flux_200k_midjourney_150k_recovery_150k_3e4_epoch2_no_rotated/000-GPT-XL/checkpoints/0046000.pt +[2025-04-25 06:07:29] Initial state: steps=46000, epochs=1 +[2025-04-25 06:07:29] compiling the model... (may take several minutes) +[2025-04-25 06:07:30] freeze the vit +[2025-04-25 06:07:30] ***** total param is 2310680832 ***** +[2025-04-25 06:07:30] ***** total trained param is 2007501056 ***** +[2025-04-25 06:07:35] Training for 3 epochs... +[2025-04-25 06:07:35] Beginning epoch 1... +[2025-04-25 06:36:07] (step=0046025) Train Loss: 5.2182, Train Steps/Sec: 0.01 +[2025-04-25 06:37:01] (step=0046050) Train Loss: 5.3363, Train Steps/Sec: 0.47 +[2025-04-25 06:38:20] (step=0046075) Train Loss: 5.2915, Train Steps/Sec: 0.31 +[2025-04-25 06:38:42] (step=0046100) Train Loss: 5.3664, Train Steps/Sec: 1.16 +[2025-04-25 06:39:03] (step=0046125) Train Loss: 5.3841, Train Steps/Sec: 1.17 +[2025-04-25 06:40:48] (step=0046150) Train Loss: 5.2847, Train Steps/Sec: 0.24 +[2025-04-25 06:41:09] (step=0046175) Train Loss: 5.3772, Train Steps/Sec: 1.17 +[2025-04-25 06:41:30] (step=0046200) Train Loss: 5.3538, Train Steps/Sec: 1.17 +[2025-04-25 06:42:17] (step=0046225) Train Loss: 5.3947, Train Steps/Sec: 0.54 +[2025-04-25 06:42:39] (step=0046250) Train Loss: 5.2745, Train Steps/Sec: 1.12 +[2025-04-25 06:43:02] (step=0046275) Train Loss: 5.3726, Train Steps/Sec: 1.11 +[2025-04-25 06:43:48] (step=0046300) Train Loss: 5.2970, Train Steps/Sec: 0.54 +[2025-04-25 06:44:34] (step=0046325) Train Loss: 5.3957, Train Steps/Sec: 0.54 +[2025-04-25 06:44:56] (step=0046350) Train Loss: 5.4616, Train Steps/Sec: 1.18 +[2025-04-25 06:45:17] (step=0046375) Train Loss: 5.3499, Train Steps/Sec: 1.17 +[2025-04-25 06:46:04] (step=0046400) Train Loss: 5.3552, Train Steps/Sec: 0.53 +[2025-04-25 06:46:26] (step=0046425) Train Loss: 5.4262, Train Steps/Sec: 1.18 +[2025-04-25 06:46:47] (step=0046450) Train Loss: 5.3122, Train Steps/Sec: 1.17 +[2025-04-25 06:47:08] (step=0046475) Train Loss: 5.3518, Train Steps/Sec: 1.17 +[2025-04-25 06:47:56] (step=0046500) Train Loss: 5.2905, Train Steps/Sec: 0.53 +[2025-04-25 06:48:17] (step=0046525) Train Loss: 5.3737, Train Steps/Sec: 1.17 +[2025-04-25 06:48:39] (step=0046550) Train Loss: 5.3690, Train Steps/Sec: 1.13 +[2025-04-25 06:49:02] (step=0046575) Train Loss: 5.2869, Train Steps/Sec: 1.11 +[2025-04-25 06:49:24] (step=0046600) Train Loss: 5.3717, Train Steps/Sec: 1.11 +[2025-04-25 06:49:48] (step=0046625) Train Loss: 5.3783, Train Steps/Sec: 1.03 +[2025-04-25 06:50:10] (step=0046650) Train Loss: 5.4032, Train Steps/Sec: 1.17 +[2025-04-25 06:50:31] (step=0046675) Train Loss: 5.3840, Train Steps/Sec: 1.17 +[2025-04-25 06:50:52] (step=0046700) Train Loss: 5.3200, Train Steps/Sec: 1.17 +[2025-04-25 06:51:14] (step=0046725) Train Loss: 5.2928, Train Steps/Sec: 1.17 +[2025-04-25 06:51:35] (step=0046750) Train Loss: 5.3795, Train Steps/Sec: 1.17 +[2025-04-25 06:51:56] (step=0046775) Train Loss: 5.4092, Train Steps/Sec: 1.17 +[2025-04-25 06:52:18] (step=0046800) Train Loss: 5.2414, Train Steps/Sec: 1.17 +[2025-04-25 06:52:39] (step=0046825) Train Loss: 5.3846, Train Steps/Sec: 1.17 +[2025-04-25 06:53:01] (step=0046850) Train Loss: 5.3781, Train Steps/Sec: 1.17 +[2025-04-25 06:53:22] (step=0046875) Train Loss: 5.3731, Train Steps/Sec: 1.17 +[2025-04-25 06:53:43] (step=0046900) Train Loss: 5.4406, Train Steps/Sec: 1.17 +[2025-04-25 06:53:58] Beginning epoch 2... +[2025-04-25 06:54:09] (step=0046925) Train Loss: 5.3348, Train Steps/Sec: 0.98 +[2025-04-25 06:54:30] (step=0046950) Train Loss: 5.3262, Train Steps/Sec: 1.17 +[2025-04-25 06:54:52] (step=0046975) Train Loss: 5.3311, Train Steps/Sec: 1.17 +[2025-04-25 06:55:15] (step=0047000) Train Loss: 5.2322, Train Steps/Sec: 1.05 +[2025-04-25 06:55:37] (step=0047025) Train Loss: 5.2292, Train Steps/Sec: 1.17 +[2025-04-25 06:55:58] (step=0047050) Train Loss: 5.2233, Train Steps/Sec: 1.17 +[2025-04-25 06:56:20] (step=0047075) Train Loss: 5.2715, Train Steps/Sec: 1.17 +[2025-04-25 06:56:41] (step=0047100) Train Loss: 5.2030, Train Steps/Sec: 1.16 +[2025-04-25 06:57:02] (step=0047125) Train Loss: 5.1688, Train Steps/Sec: 1.17 +[2025-04-25 06:57:24] (step=0047150) Train Loss: 5.2820, Train Steps/Sec: 1.17 +[2025-04-25 06:57:45] (step=0047175) Train Loss: 5.2931, Train Steps/Sec: 1.17 +[2025-04-25 06:58:08] (step=0047200) Train Loss: 5.2749, Train Steps/Sec: 1.11 +[2025-04-25 06:58:29] (step=0047225) Train Loss: 5.2923, Train Steps/Sec: 1.17 +[2025-04-25 06:58:51] (step=0047250) Train Loss: 5.2284, Train Steps/Sec: 1.12 +[2025-04-25 06:59:15] (step=0047275) Train Loss: 5.2331, Train Steps/Sec: 1.06 +[2025-04-25 06:59:38] (step=0047300) Train Loss: 5.2246, Train Steps/Sec: 1.10 +[2025-04-25 07:00:00] (step=0047325) Train Loss: 5.3062, Train Steps/Sec: 1.11 +[2025-04-25 07:00:22] (step=0047350) Train Loss: 5.2587, Train Steps/Sec: 1.17 +[2025-04-25 07:00:43] (step=0047375) Train Loss: 5.2664, Train Steps/Sec: 1.17 +[2025-04-25 07:01:04] (step=0047400) Train Loss: 5.2718, Train Steps/Sec: 1.16 +[2025-04-25 07:01:26] (step=0047425) Train Loss: 5.2578, Train Steps/Sec: 1.17 +[2025-04-25 07:01:47] (step=0047450) Train Loss: 5.2491, Train Steps/Sec: 1.17 +[2025-04-25 07:02:09] (step=0047475) Train Loss: 5.2689, Train Steps/Sec: 1.17 +[2025-04-25 07:02:30] (step=0047500) Train Loss: 5.2657, Train Steps/Sec: 1.17 +[2025-04-25 07:02:51] (step=0047525) Train Loss: 5.3215, Train Steps/Sec: 1.17 +[2025-04-25 07:03:13] (step=0047550) Train Loss: 5.2861, Train Steps/Sec: 1.17 +[2025-04-25 07:03:34] (step=0047575) Train Loss: 5.3298, Train Steps/Sec: 1.17 +[2025-04-25 07:03:56] (step=0047600) Train Loss: 5.3455, Train Steps/Sec: 1.16 +[2025-04-25 07:04:17] (step=0047625) Train Loss: 5.2444, Train Steps/Sec: 1.17 +[2025-04-25 07:04:39] (step=0047650) Train Loss: 5.2882, Train Steps/Sec: 1.16 +[2025-04-25 07:05:00] (step=0047675) Train Loss: 5.3281, Train Steps/Sec: 1.17 +[2025-04-25 07:05:22] (step=0047700) Train Loss: 5.2791, Train Steps/Sec: 1.16 +[2025-04-25 07:05:45] (step=0047725) Train Loss: 5.2672, Train Steps/Sec: 1.08 +[2025-04-25 07:06:06] (step=0047750) Train Loss: 5.2602, Train Steps/Sec: 1.17 +[2025-04-25 07:06:27] (step=0047775) Train Loss: 5.2706, Train Steps/Sec: 1.17 +[2025-04-25 07:06:49] (step=0047800) Train Loss: 5.2113, Train Steps/Sec: 1.16 +[2025-04-25 07:07:10] (step=0047825) Train Loss: 5.2626, Train Steps/Sec: 1.17 +[2025-04-25 07:07:33] (step=0047850) Train Loss: 5.2194, Train Steps/Sec: 1.11 +[2025-04-25 07:07:54] (step=0047875) Train Loss: 5.3253, Train Steps/Sec: 1.17 +[2025-04-25 07:08:17] (step=0047900) Train Loss: 5.3148, Train Steps/Sec: 1.12 +[2025-04-25 07:08:40] (step=0047925) Train Loss: 5.3373, Train Steps/Sec: 1.08 +[2025-04-25 07:09:02] (step=0047950) Train Loss: 5.2988, Train Steps/Sec: 1.12 +[2025-04-25 07:09:23] (step=0047975) Train Loss: 5.2819, Train Steps/Sec: 1.17 +[2025-04-25 07:09:45] (step=0048000) Train Loss: 5.2750, Train Steps/Sec: 1.16 +[2025-04-25 07:09:45] text_config is None. Initializing the text config with default values (`OPTConfig`). +[2025-04-25 07:14:50] Finish Eval in 48000 steps... +[2025-04-25 07:15:08] Saved checkpoint to checkpoint/Afteri2i_100kckpt_LLAVAT5_empty_Fixed_just_segment_pretrain_stage3_dreambench_recap_400k_t2i_400k_flux_200k_midjourney_150k_recovery_150k_3e4_epoch2_no_rotated/002-GPT-XL/checkpoints/0048000.pt +[2025-04-25 07:15:29] (step=0048025) Train Loss: 5.2351, Train Steps/Sec: 0.07 +[2025-04-25 07:15:51] (step=0048050) Train Loss: 5.3013, Train Steps/Sec: 1.17 +[2025-04-25 07:16:12] (step=0048075) Train Loss: 5.3307, Train Steps/Sec: 1.17 +[2025-04-25 07:16:34] (step=0048100) Train Loss: 5.2136, Train Steps/Sec: 1.16 +[2025-04-25 07:16:55] (step=0048125) Train Loss: 5.2373, Train Steps/Sec: 1.17 +[2025-04-25 07:17:17] (step=0048150) Train Loss: 5.3068, Train Steps/Sec: 1.17 +[2025-04-25 07:17:38] (step=0048175) Train Loss: 5.2439, Train Steps/Sec: 1.17 +[2025-04-25 07:17:59] (step=0048200) Train Loss: 5.3323, Train Steps/Sec: 1.16 +[2025-04-25 07:18:21] (step=0048225) Train Loss: 5.1257, Train Steps/Sec: 1.17 +[2025-04-25 07:18:42] (step=0048250) Train Loss: 5.2314, Train Steps/Sec: 1.17 +[2025-04-25 07:19:04] (step=0048275) Train Loss: 5.2166, Train Steps/Sec: 1.17 +[2025-04-25 07:19:25] (step=0048300) Train Loss: 5.2908, Train Steps/Sec: 1.16 +[2025-04-25 07:19:46] (step=0048325) Train Loss: 5.2416, Train Steps/Sec: 1.17 +[2025-04-25 07:20:08] (step=0048350) Train Loss: 5.3068, Train Steps/Sec: 1.17 +[2025-04-25 07:20:29] (step=0048375) Train Loss: 5.2801, Train Steps/Sec: 1.17 +[2025-04-25 07:20:51] (step=0048400) Train Loss: 5.2122, Train Steps/Sec: 1.16 +[2025-04-25 07:21:12] (step=0048425) Train Loss: 5.2322, Train Steps/Sec: 1.17 +[2025-04-25 07:21:35] (step=0048450) Train Loss: 5.2675, Train Steps/Sec: 1.12 +[2025-04-25 07:21:57] (step=0048475) Train Loss: 5.2939, Train Steps/Sec: 1.10 +[2025-04-25 07:22:20] (step=0048500) Train Loss: 5.3127, Train Steps/Sec: 1.11 +[2025-04-25 07:22:41] (step=0048525) Train Loss: 5.2923, Train Steps/Sec: 1.17 +[2025-04-25 07:23:03] (step=0048550) Train Loss: 5.2369, Train Steps/Sec: 1.12 +[2025-04-25 07:23:25] (step=0048575) Train Loss: 5.2860, Train Steps/Sec: 1.17 +[2025-04-25 07:23:49] (step=0048600) Train Loss: 5.2642, Train Steps/Sec: 1.04 +[2025-04-25 07:24:10] (step=0048625) Train Loss: 5.3143, Train Steps/Sec: 1.17 +[2025-04-25 07:24:32] (step=0048650) Train Loss: 5.2668, Train Steps/Sec: 1.17 +[2025-04-25 07:24:53] (step=0048675) Train Loss: 5.2718, Train Steps/Sec: 1.17 +[2025-04-25 07:25:14] (step=0048700) Train Loss: 5.3269, Train Steps/Sec: 1.16 +[2025-04-25 07:25:36] (step=0048725) Train Loss: 5.2737, Train Steps/Sec: 1.17 +[2025-04-25 07:25:58] (step=0048750) Train Loss: 5.2746, Train Steps/Sec: 1.12 +[2025-04-25 07:26:19] (step=0048775) Train Loss: 5.2917, Train Steps/Sec: 1.17 +[2025-04-25 07:26:41] (step=0048800) Train Loss: 5.2774, Train Steps/Sec: 1.16 +[2025-04-25 07:27:02] (step=0048825) Train Loss: 5.2921, Train Steps/Sec: 1.17 +[2025-04-25 07:27:24] (step=0048850) Train Loss: 5.2409, Train Steps/Sec: 1.17 +[2025-04-25 07:27:45] (step=0048875) Train Loss: 5.3059, Train Steps/Sec: 1.17 +[2025-04-25 07:28:07] (step=0048900) Train Loss: 5.3326, Train Steps/Sec: 1.16 +[2025-04-25 07:28:28] (step=0048925) Train Loss: 5.3097, Train Steps/Sec: 1.17 +[2025-04-25 07:28:49] (step=0048950) Train Loss: 5.2551, Train Steps/Sec: 1.17 +[2025-04-25 07:29:11] (step=0048975) Train Loss: 5.2611, Train Steps/Sec: 1.17 +[2025-04-25 07:29:32] (step=0049000) Train Loss: 5.2770, Train Steps/Sec: 1.16 +[2025-04-25 07:29:54] (step=0049025) Train Loss: 5.2191, Train Steps/Sec: 1.17 +[2025-04-25 07:30:15] (step=0049050) Train Loss: 5.3676, Train Steps/Sec: 1.17 +[2025-04-25 07:30:36] (step=0049075) Train Loss: 5.2996, Train Steps/Sec: 1.17 +[2025-04-25 07:30:58] (step=0049100) Train Loss: 5.2345, Train Steps/Sec: 1.16 +[2025-04-25 07:31:19] (step=0049125) Train Loss: 5.2624, Train Steps/Sec: 1.17 +[2025-04-25 07:31:42] (step=0049150) Train Loss: 5.2233, Train Steps/Sec: 1.12 +[2025-04-25 07:32:04] (step=0049175) Train Loss: 5.3195, Train Steps/Sec: 1.12 +[2025-04-25 07:32:26] (step=0049200) Train Loss: 5.1764, Train Steps/Sec: 1.11 +[2025-04-25 07:32:49] (step=0049225) Train Loss: 5.2885, Train Steps/Sec: 1.13 +[2025-04-25 07:33:12] (step=0049250) Train Loss: 5.1969, Train Steps/Sec: 1.05 +[2025-04-25 07:33:34] (step=0049275) Train Loss: 5.2862, Train Steps/Sec: 1.17 +[2025-04-25 07:33:55] (step=0049300) Train Loss: 5.3223, Train Steps/Sec: 1.16 +[2025-04-25 07:34:17] (step=0049325) Train Loss: 5.3354, Train Steps/Sec: 1.17 +[2025-04-25 07:34:38] (step=0049350) Train Loss: 5.2902, Train Steps/Sec: 1.17 +[2025-04-25 07:34:59] (step=0049375) Train Loss: 5.3162, Train Steps/Sec: 1.17 +[2025-04-25 07:35:21] (step=0049400) Train Loss: 5.2568, Train Steps/Sec: 1.16 +[2025-04-25 07:35:42] (step=0049425) Train Loss: 5.2498, Train Steps/Sec: 1.17 +[2025-04-25 07:36:04] (step=0049450) Train Loss: 5.2931, Train Steps/Sec: 1.17 +[2025-04-25 07:36:25] (step=0049475) Train Loss: 5.3109, Train Steps/Sec: 1.17 +[2025-04-25 07:36:48] (step=0049500) Train Loss: 5.2926, Train Steps/Sec: 1.11 +[2025-04-25 07:37:09] (step=0049525) Train Loss: 5.3444, Train Steps/Sec: 1.17 +[2025-04-25 07:37:30] (step=0049550) Train Loss: 5.2933, Train Steps/Sec: 1.17 +[2025-04-25 07:37:52] (step=0049575) Train Loss: 5.3218, Train Steps/Sec: 1.17 +[2025-04-25 07:38:13] (step=0049600) Train Loss: 5.2536, Train Steps/Sec: 1.17 +[2025-04-25 07:38:35] (step=0049625) Train Loss: 5.2446, Train Steps/Sec: 1.17 +[2025-04-25 07:38:56] (step=0049650) Train Loss: 5.2745, Train Steps/Sec: 1.16 +[2025-04-25 07:39:18] (step=0049675) Train Loss: 5.2865, Train Steps/Sec: 1.16 +[2025-04-25 07:39:39] (step=0049700) Train Loss: 5.2231, Train Steps/Sec: 1.16 +[2025-04-25 07:40:01] (step=0049725) Train Loss: 5.2320, Train Steps/Sec: 1.17 +[2025-04-25 07:40:22] (step=0049750) Train Loss: 5.4063, Train Steps/Sec: 1.17 +[2025-04-25 07:40:44] (step=0049775) Train Loss: 5.3695, Train Steps/Sec: 1.17 +[2025-04-25 07:41:06] (step=0049800) Train Loss: 5.2475, Train Steps/Sec: 1.12 +[2025-04-25 07:41:27] (step=0049825) Train Loss: 5.2169, Train Steps/Sec: 1.17 +[2025-04-25 07:41:49] (step=0049850) Train Loss: 5.3037, Train Steps/Sec: 1.17 +[2025-04-25 07:42:11] (step=0049875) Train Loss: 5.3558, Train Steps/Sec: 1.13 +[2025-04-25 07:42:35] (step=0049900) Train Loss: 5.2146, Train Steps/Sec: 1.04 +[2025-04-25 07:42:58] (step=0049925) Train Loss: 5.3232, Train Steps/Sec: 1.08 +[2025-04-25 07:43:19] (step=0049950) Train Loss: 5.3100, Train Steps/Sec: 1.17 +[2025-04-25 07:43:41] (step=0049975) Train Loss: 5.2127, Train Steps/Sec: 1.17 +[2025-04-25 07:44:02] (step=0050000) Train Loss: 5.2904, Train Steps/Sec: 1.16 +[2025-04-25 07:44:02] text_config is None. Initializing the text config with default values (`OPTConfig`). +[2025-04-25 07:49:08] Finish Eval in 50000 steps... +[2025-04-25 07:49:27] Saved checkpoint to checkpoint/Afteri2i_100kckpt_LLAVAT5_empty_Fixed_just_segment_pretrain_stage3_dreambench_recap_400k_t2i_400k_flux_200k_midjourney_150k_recovery_150k_3e4_epoch2_no_rotated/002-GPT-XL/checkpoints/0050000.pt +[2025-04-25 07:49:30] Removed old checkpoint: checkpoint/Afteri2i_100kckpt_LLAVAT5_empty_Fixed_just_segment_pretrain_stage3_dreambench_recap_400k_t2i_400k_flux_200k_midjourney_150k_recovery_150k_3e4_epoch2_no_rotated/002-GPT-XL/checkpoints/0048000.pt +[2025-04-25 07:49:51] (step=0050025) Train Loss: 5.3138, Train Steps/Sec: 0.07 +[2025-04-25 07:50:13] (step=0050050) Train Loss: 5.2595, Train Steps/Sec: 1.16 +[2025-04-25 07:50:34] (step=0050075) Train Loss: 5.2120, Train Steps/Sec: 1.17 +[2025-04-25 07:50:56] (step=0050100) Train Loss: 5.2992, Train Steps/Sec: 1.16 +[2025-04-25 07:51:17] (step=0050125) Train Loss: 5.3257, Train Steps/Sec: 1.17 +[2025-04-25 07:51:39] (step=0050150) Train Loss: 5.3392, Train Steps/Sec: 1.16 +[2025-04-25 07:52:00] (step=0050175) Train Loss: 5.3054, Train Steps/Sec: 1.17 +[2025-04-25 07:52:22] (step=0050200) Train Loss: 5.2217, Train Steps/Sec: 1.16 +[2025-04-25 07:52:43] (step=0050225) Train Loss: 5.3270, Train Steps/Sec: 1.16 +[2025-04-25 07:53:05] (step=0050250) Train Loss: 5.2269, Train Steps/Sec: 1.17 +[2025-04-25 07:53:26] (step=0050275) Train Loss: 5.2466, Train Steps/Sec: 1.17 +[2025-04-25 07:53:47] (step=0050300) Train Loss: 5.3594, Train Steps/Sec: 1.16 +[2025-04-25 07:54:09] (step=0050325) Train Loss: 5.3077, Train Steps/Sec: 1.17 +[2025-04-25 07:54:30] (step=0050350) Train Loss: 5.2708, Train Steps/Sec: 1.17 +[2025-04-25 07:54:52] (step=0050375) Train Loss: 5.3305, Train Steps/Sec: 1.17 +[2025-04-25 07:55:13] (step=0050400) Train Loss: 5.3107, Train Steps/Sec: 1.15 +[2025-04-25 07:55:35] (step=0050425) Train Loss: 5.2263, Train Steps/Sec: 1.17 +[2025-04-25 07:55:57] (step=0050450) Train Loss: 5.2646, Train Steps/Sec: 1.12 +[2025-04-25 07:56:18] (step=0050475) Train Loss: 5.2875, Train Steps/Sec: 1.17 +[2025-04-25 07:56:40] (step=0050500) Train Loss: 5.3474, Train Steps/Sec: 1.17 +[2025-04-25 07:57:02] (step=0050525) Train Loss: 5.1713, Train Steps/Sec: 1.12 +[2025-04-25 07:57:24] (step=0050550) Train Loss: 5.3388, Train Steps/Sec: 1.12 +[2025-04-25 07:57:47] (step=0050575) Train Loss: 5.2772, Train Steps/Sec: 1.09 +[2025-04-25 07:58:09] (step=0050600) Train Loss: 5.2134, Train Steps/Sec: 1.16 +[2025-04-25 07:58:31] (step=0050625) Train Loss: 5.3416, Train Steps/Sec: 1.12 +[2025-04-25 07:58:54] (step=0050650) Train Loss: 5.2410, Train Steps/Sec: 1.12 +[2025-04-25 07:59:15] (step=0050675) Train Loss: 5.2899, Train Steps/Sec: 1.17 +[2025-04-25 07:59:36] (step=0050700) Train Loss: 5.3115, Train Steps/Sec: 1.17 +[2025-04-25 07:59:58] (step=0050725) Train Loss: 5.2461, Train Steps/Sec: 1.17 +[2025-04-25 08:00:20] (step=0050750) Train Loss: 5.2945, Train Steps/Sec: 1.11 +[2025-04-25 08:00:41] (step=0050775) Train Loss: 5.3122, Train Steps/Sec: 1.17 +[2025-04-25 08:01:03] (step=0050800) Train Loss: 5.2003, Train Steps/Sec: 1.17 +[2025-04-25 08:01:24] (step=0050825) Train Loss: 5.3067, Train Steps/Sec: 1.17 +[2025-04-25 08:01:46] (step=0050850) Train Loss: 5.2811, Train Steps/Sec: 1.17 +[2025-04-25 08:02:07] (step=0050875) Train Loss: 5.2294, Train Steps/Sec: 1.17 +[2025-04-25 08:02:28] (step=0050900) Train Loss: 5.3407, Train Steps/Sec: 1.17 +[2025-04-25 08:02:50] (step=0050925) Train Loss: 5.3036, Train Steps/Sec: 1.17 +[2025-04-25 08:03:11] (step=0050950) Train Loss: 5.3031, Train Steps/Sec: 1.17 +[2025-04-25 08:03:33] (step=0050975) Train Loss: 5.2697, Train Steps/Sec: 1.17 +[2025-04-25 08:03:54] (step=0051000) Train Loss: 5.3382, Train Steps/Sec: 1.16 +[2025-04-25 08:04:15] (step=0051025) Train Loss: 5.3313, Train Steps/Sec: 1.17 +[2025-04-25 08:04:37] (step=0051050) Train Loss: 5.2845, Train Steps/Sec: 1.17 +[2025-04-25 08:04:58] (step=0051075) Train Loss: 5.2751, Train Steps/Sec: 1.17 +[2025-04-25 08:05:20] (step=0051100) Train Loss: 5.3010, Train Steps/Sec: 1.16 +[2025-04-25 08:05:42] (step=0051125) Train Loss: 5.2881, Train Steps/Sec: 1.12 +[2025-04-25 08:06:03] (step=0051150) Train Loss: 5.2731, Train Steps/Sec: 1.17 +[2025-04-25 08:06:25] (step=0051175) Train Loss: 5.2481, Train Steps/Sec: 1.13 +[2025-04-25 08:06:48] (step=0051200) Train Loss: 5.2926, Train Steps/Sec: 1.12 +[2025-04-25 08:07:11] (step=0051225) Train Loss: 5.3127, Train Steps/Sec: 1.09 +[2025-04-25 08:07:32] (step=0051250) Train Loss: 5.2915, Train Steps/Sec: 1.17 +[2025-04-25 08:07:54] (step=0051275) Train Loss: 5.2624, Train Steps/Sec: 1.16 +[2025-04-25 08:08:15] (step=0051300) Train Loss: 5.2621, Train Steps/Sec: 1.17 +[2025-04-25 08:08:36] (step=0051325) Train Loss: 5.2343, Train Steps/Sec: 1.17 +[2025-04-25 08:08:59] (step=0051350) Train Loss: 5.2943, Train Steps/Sec: 1.13 +[2025-04-25 08:09:20] (step=0051375) Train Loss: 5.3008, Train Steps/Sec: 1.17 +[2025-04-25 08:09:43] (step=0051400) Train Loss: 5.2584, Train Steps/Sec: 1.10 +[2025-04-25 08:10:04] (step=0051425) Train Loss: 5.2868, Train Steps/Sec: 1.17 +[2025-04-25 08:10:25] (step=0051450) Train Loss: 5.2585, Train Steps/Sec: 1.17 +[2025-04-25 08:10:47] (step=0051475) Train Loss: 5.3129, Train Steps/Sec: 1.17 +[2025-04-25 08:11:09] (step=0051500) Train Loss: 5.2160, Train Steps/Sec: 1.10 +[2025-04-25 08:11:31] (step=0051525) Train Loss: 5.2878, Train Steps/Sec: 1.17 +[2025-04-25 08:11:52] (step=0051550) Train Loss: 5.3765, Train Steps/Sec: 1.17 +[2025-04-25 08:12:13] (step=0051575) Train Loss: 5.2994, Train Steps/Sec: 1.17 +[2025-04-25 08:12:35] (step=0051600) Train Loss: 5.2954, Train Steps/Sec: 1.16 +[2025-04-25 08:12:57] (step=0051625) Train Loss: 5.2695, Train Steps/Sec: 1.16 +[2025-04-25 08:13:18] (step=0051650) Train Loss: 5.3088, Train Steps/Sec: 1.17 +[2025-04-25 08:13:39] (step=0051675) Train Loss: 5.2754, Train Steps/Sec: 1.17 +[2025-04-25 08:14:01] (step=0051700) Train Loss: 5.2860, Train Steps/Sec: 1.17 +[2025-04-25 08:14:22] (step=0051725) Train Loss: 5.2600, Train Steps/Sec: 1.17 +[2025-04-25 08:14:43] (step=0051750) Train Loss: 5.2866, Train Steps/Sec: 1.17 +[2025-04-25 08:15:06] (step=0051775) Train Loss: 5.3317, Train Steps/Sec: 1.12 +[2025-04-25 08:15:27] (step=0051800) Train Loss: 5.3018, Train Steps/Sec: 1.16 +[2025-04-25 08:15:49] (step=0051825) Train Loss: 5.3064, Train Steps/Sec: 1.13 +[2025-04-25 08:16:12] (step=0051850) Train Loss: 5.2582, Train Steps/Sec: 1.12 +[2025-04-25 08:16:33] (step=0051875) Train Loss: 5.2837, Train Steps/Sec: 1.17 +[2025-04-25 08:16:56] (step=0051900) Train Loss: 5.3158, Train Steps/Sec: 1.08 +[2025-04-25 08:17:18] (step=0051925) Train Loss: 5.3236, Train Steps/Sec: 1.17 +[2025-04-25 08:17:39] (step=0051950) Train Loss: 5.2775, Train Steps/Sec: 1.17 +[2025-04-25 08:18:00] (step=0051975) Train Loss: 5.2215, Train Steps/Sec: 1.17 +[2025-04-25 08:18:22] (step=0052000) Train Loss: 5.3349, Train Steps/Sec: 1.16 +[2025-04-25 08:18:22] text_config is None. Initializing the text config with default values (`OPTConfig`). +[2025-04-25 08:23:28] Finish Eval in 52000 steps... +[2025-04-25 08:23:46] Saved checkpoint to checkpoint/Afteri2i_100kckpt_LLAVAT5_empty_Fixed_just_segment_pretrain_stage3_dreambench_recap_400k_t2i_400k_flux_200k_midjourney_150k_recovery_150k_3e4_epoch2_no_rotated/002-GPT-XL/checkpoints/0052000.pt +[2025-04-25 08:23:48] Removed old checkpoint: checkpoint/Afteri2i_100kckpt_LLAVAT5_empty_Fixed_just_segment_pretrain_stage3_dreambench_recap_400k_t2i_400k_flux_200k_midjourney_150k_recovery_150k_3e4_epoch2_no_rotated/002-GPT-XL/checkpoints/0050000.pt +[2025-04-25 08:24:10] (step=0052025) Train Loss: 5.3486, Train Steps/Sec: 0.07 +[2025-04-25 08:24:31] (step=0052050) Train Loss: 5.2924, Train Steps/Sec: 1.17 +[2025-04-25 08:24:53] (step=0052075) Train Loss: 5.2657, Train Steps/Sec: 1.13 +[2025-04-25 08:25:15] (step=0052100) Train Loss: 5.3027, Train Steps/Sec: 1.17 +[2025-04-25 08:25:37] (step=0052125) Train Loss: 5.2392, Train Steps/Sec: 1.12 +[2025-04-25 08:25:58] (step=0052150) Train Loss: 5.2757, Train Steps/Sec: 1.17 +[2025-04-25 08:26:20] (step=0052175) Train Loss: 5.2209, Train Steps/Sec: 1.17 +[2025-04-25 08:26:41] (step=0052200) Train Loss: 5.3072, Train Steps/Sec: 1.16 +[2025-04-25 08:27:03] (step=0052225) Train Loss: 5.3042, Train Steps/Sec: 1.17 +[2025-04-25 08:27:24] (step=0052250) Train Loss: 5.2794, Train Steps/Sec: 1.17 +[2025-04-25 08:27:45] (step=0052275) Train Loss: 5.2672, Train Steps/Sec: 1.17 +[2025-04-25 08:28:07] (step=0052300) Train Loss: 5.2680, Train Steps/Sec: 1.16 +[2025-04-25 08:28:29] (step=0052325) Train Loss: 5.3534, Train Steps/Sec: 1.16 +[2025-04-25 08:28:50] (step=0052350) Train Loss: 5.2623, Train Steps/Sec: 1.17 +[2025-04-25 08:29:11] (step=0052375) Train Loss: 5.2816, Train Steps/Sec: 1.17 +[2025-04-25 08:29:33] (step=0052400) Train Loss: 5.2983, Train Steps/Sec: 1.17 +[2025-04-25 08:29:55] (step=0052425) Train Loss: 5.3378, Train Steps/Sec: 1.13 +[2025-04-25 08:30:16] (step=0052450) Train Loss: 5.3607, Train Steps/Sec: 1.17 +[2025-04-25 08:30:38] (step=0052475) Train Loss: 5.3965, Train Steps/Sec: 1.13 +[2025-04-25 08:31:01] (step=0052500) Train Loss: 5.2747, Train Steps/Sec: 1.12 +[2025-04-25 08:31:22] (step=0052525) Train Loss: 5.3493, Train Steps/Sec: 1.17 +[2025-04-25 08:31:45] (step=0052550) Train Loss: 5.2575, Train Steps/Sec: 1.08 +[2025-04-25 08:32:06] (step=0052575) Train Loss: 5.2266, Train Steps/Sec: 1.17 +[2025-04-25 08:32:28] (step=0052600) Train Loss: 5.2579, Train Steps/Sec: 1.17 +[2025-04-25 08:32:49] (step=0052625) Train Loss: 5.2887, Train Steps/Sec: 1.17 +[2025-04-25 08:33:11] (step=0052650) Train Loss: 5.2585, Train Steps/Sec: 1.17 +[2025-04-25 08:33:32] (step=0052675) Train Loss: 5.2892, Train Steps/Sec: 1.17 +[2025-04-25 08:33:53] (step=0052700) Train Loss: 5.2784, Train Steps/Sec: 1.17 +[2025-04-25 08:34:15] (step=0052725) Train Loss: 5.3006, Train Steps/Sec: 1.17 +[2025-04-25 08:34:37] (step=0052750) Train Loss: 5.2882, Train Steps/Sec: 1.12 +[2025-04-25 08:34:59] (step=0052775) Train Loss: 5.2165, Train Steps/Sec: 1.17 +[2025-04-25 08:35:21] (step=0052800) Train Loss: 5.2068, Train Steps/Sec: 1.12 +[2025-04-25 08:35:42] (step=0052825) Train Loss: 5.2873, Train Steps/Sec: 1.17 +[2025-04-25 08:36:05] (step=0052850) Train Loss: 5.3201, Train Steps/Sec: 1.12 +[2025-04-25 08:36:26] (step=0052875) Train Loss: 5.2943, Train Steps/Sec: 1.17 +[2025-04-25 08:36:47] (step=0052900) Train Loss: 5.2710, Train Steps/Sec: 1.17 +[2025-04-25 08:37:09] (step=0052925) Train Loss: 5.3268, Train Steps/Sec: 1.17 +[2025-04-25 08:37:30] (step=0052950) Train Loss: 5.3488, Train Steps/Sec: 1.17 +[2025-04-25 08:37:51] (step=0052975) Train Loss: 5.2678, Train Steps/Sec: 1.17 +[2025-04-25 08:38:13] (step=0053000) Train Loss: 5.2899, Train Steps/Sec: 1.16 +[2025-04-25 08:38:34] (step=0053025) Train Loss: 5.2765, Train Steps/Sec: 1.17 +[2025-04-25 08:38:56] (step=0053050) Train Loss: 5.2875, Train Steps/Sec: 1.17 +[2025-04-25 08:39:18] (step=0053075) Train Loss: 5.2570, Train Steps/Sec: 1.12 +[2025-04-25 08:39:39] (step=0053100) Train Loss: 5.2685, Train Steps/Sec: 1.16 +[2025-04-25 08:40:02] (step=0053125) Train Loss: 5.2305, Train Steps/Sec: 1.13 +[2025-04-25 08:40:24] (step=0053150) Train Loss: 5.2854, Train Steps/Sec: 1.13 +[2025-04-25 08:40:45] (step=0053175) Train Loss: 5.2939, Train Steps/Sec: 1.17 +[2025-04-25 08:41:07] (step=0053200) Train Loss: 5.2526, Train Steps/Sec: 1.12 +[2025-04-25 08:41:30] (step=0053225) Train Loss: 5.3854, Train Steps/Sec: 1.12 +[2025-04-25 08:41:51] (step=0053250) Train Loss: 5.3054, Train Steps/Sec: 1.17 +[2025-04-25 08:42:12] (step=0053275) Train Loss: 5.3671, Train Steps/Sec: 1.17 +[2025-04-25 08:42:34] (step=0053300) Train Loss: 5.3695, Train Steps/Sec: 1.17 +[2025-04-25 08:42:55] (step=0053325) Train Loss: 5.2711, Train Steps/Sec: 1.17 +[2025-04-25 08:43:17] (step=0053350) Train Loss: 5.2769, Train Steps/Sec: 1.17 +[2025-04-25 08:43:38] (step=0053375) Train Loss: 5.2386, Train Steps/Sec: 1.17 +[2025-04-25 08:43:59] (step=0053400) Train Loss: 5.2945, Train Steps/Sec: 1.17 +[2025-04-25 08:44:21] (step=0053425) Train Loss: 5.2858, Train Steps/Sec: 1.17 +[2025-04-25 08:44:42] (step=0053450) Train Loss: 5.2631, Train Steps/Sec: 1.17 +[2025-04-25 08:45:04] (step=0053475) Train Loss: 5.3600, Train Steps/Sec: 1.17 +[2025-04-25 08:45:26] (step=0053500) Train Loss: 5.2505, Train Steps/Sec: 1.11 +[2025-04-25 08:45:49] (step=0053525) Train Loss: 5.2966, Train Steps/Sec: 1.12 +[2025-04-25 08:46:10] (step=0053550) Train Loss: 5.2848, Train Steps/Sec: 1.17 +[2025-04-25 08:46:32] (step=0053575) Train Loss: 5.2486, Train Steps/Sec: 1.12 +[2025-04-25 08:46:54] (step=0053600) Train Loss: 5.2235, Train Steps/Sec: 1.16 +[2025-04-25 08:47:15] (step=0053625) Train Loss: 5.2916, Train Steps/Sec: 1.17 +[2025-04-25 08:47:36] (step=0053650) Train Loss: 5.3147, Train Steps/Sec: 1.17 +[2025-04-25 08:47:58] (step=0053675) Train Loss: 5.2024, Train Steps/Sec: 1.17 +[2025-04-25 08:48:19] (step=0053700) Train Loss: 5.3005, Train Steps/Sec: 1.17 +[2025-04-25 08:48:42] (step=0053725) Train Loss: 5.3213, Train Steps/Sec: 1.12 +[2025-04-25 08:49:03] (step=0053750) Train Loss: 5.2576, Train Steps/Sec: 1.17 +[2025-04-25 08:49:25] (step=0053775) Train Loss: 5.3459, Train Steps/Sec: 1.13 +[2025-04-25 08:49:47] (step=0053800) Train Loss: 5.2487, Train Steps/Sec: 1.12 +[2025-04-25 08:50:09] (step=0053825) Train Loss: 5.2200, Train Steps/Sec: 1.17 +[2025-04-25 08:50:30] (step=0053850) Train Loss: 5.3440, Train Steps/Sec: 1.17 +[2025-04-25 08:50:53] (step=0053875) Train Loss: 5.3593, Train Steps/Sec: 1.09 +[2025-04-25 08:51:15] (step=0053900) Train Loss: 5.2815, Train Steps/Sec: 1.17 +[2025-04-25 08:51:36] (step=0053925) Train Loss: 5.3861, Train Steps/Sec: 1.17 +[2025-04-25 08:51:57] (step=0053950) Train Loss: 5.2165, Train Steps/Sec: 1.17 +[2025-04-25 08:52:19] (step=0053975) Train Loss: 5.2800, Train Steps/Sec: 1.16 +[2025-04-25 08:52:41] (step=0054000) Train Loss: 5.2128, Train Steps/Sec: 1.16 +[2025-04-25 08:52:41] text_config is None. Initializing the text config with default values (`OPTConfig`). +[2025-04-25 08:57:46] Finish Eval in 54000 steps... +[2025-04-25 08:58:05] Saved checkpoint to checkpoint/Afteri2i_100kckpt_LLAVAT5_empty_Fixed_just_segment_pretrain_stage3_dreambench_recap_400k_t2i_400k_flux_200k_midjourney_150k_recovery_150k_3e4_epoch2_no_rotated/002-GPT-XL/checkpoints/0054000.pt +[2025-04-25 08:58:08] Removed old checkpoint: checkpoint/Afteri2i_100kckpt_LLAVAT5_empty_Fixed_just_segment_pretrain_stage3_dreambench_recap_400k_t2i_400k_flux_200k_midjourney_150k_recovery_150k_3e4_epoch2_no_rotated/002-GPT-XL/checkpoints/0052000.pt +[2025-04-25 08:58:29] (step=0054025) Train Loss: 5.3777, Train Steps/Sec: 0.07 +[2025-04-25 08:58:51] (step=0054050) Train Loss: 5.4221, Train Steps/Sec: 1.16 +[2025-04-25 08:59:12] (step=0054075) Train Loss: 5.2302, Train Steps/Sec: 1.16 +[2025-04-25 08:59:34] (step=0054100) Train Loss: 5.2255, Train Steps/Sec: 1.16 +[2025-04-25 08:59:55] (step=0054125) Train Loss: 5.2972, Train Steps/Sec: 1.17 +[2025-04-25 09:00:17] (step=0054150) Train Loss: 5.2248, Train Steps/Sec: 1.17 +[2025-04-25 09:00:38] (step=0054175) Train Loss: 5.3726, Train Steps/Sec: 1.17 +[2025-04-25 09:00:59] (step=0054200) Train Loss: 5.2850, Train Steps/Sec: 1.16 +[2025-04-25 09:01:21] (step=0054225) Train Loss: 5.2925, Train Steps/Sec: 1.17 +[2025-04-25 09:01:43] (step=0054250) Train Loss: 5.3419, Train Steps/Sec: 1.10 +[2025-04-25 09:02:05] (step=0054275) Train Loss: 5.2194, Train Steps/Sec: 1.15 +[2025-04-25 09:02:27] (step=0054300) Train Loss: 5.3211, Train Steps/Sec: 1.15 +[2025-04-25 09:02:50] (step=0054325) Train Loss: 5.3794, Train Steps/Sec: 1.08 +[2025-04-25 09:03:12] (step=0054350) Train Loss: 5.2115, Train Steps/Sec: 1.16 +[2025-04-25 09:03:34] (step=0054375) Train Loss: 5.1621, Train Steps/Sec: 1.11 +[2025-04-25 09:03:56] (step=0054400) Train Loss: 5.3115, Train Steps/Sec: 1.16 +[2025-04-25 09:04:17] (step=0054425) Train Loss: 5.2375, Train Steps/Sec: 1.16 +[2025-04-25 09:04:39] (step=0054450) Train Loss: 5.2847, Train Steps/Sec: 1.12 +[2025-04-25 09:05:02] (step=0054475) Train Loss: 5.2918, Train Steps/Sec: 1.12 +[2025-04-25 09:05:23] (step=0054500) Train Loss: 5.3568, Train Steps/Sec: 1.16 +[2025-04-25 09:05:46] (step=0054525) Train Loss: 5.3004, Train Steps/Sec: 1.09 +[2025-04-25 09:06:08] (step=0054550) Train Loss: 5.2616, Train Steps/Sec: 1.17 +[2025-04-25 09:06:29] (step=0054575) Train Loss: 5.3386, Train Steps/Sec: 1.17 +[2025-04-25 09:06:51] (step=0054600) Train Loss: 5.2800, Train Steps/Sec: 1.17 +[2025-04-25 09:07:12] (step=0054625) Train Loss: 5.3447, Train Steps/Sec: 1.17 +[2025-04-25 09:07:33] (step=0054650) Train Loss: 5.2766, Train Steps/Sec: 1.17 +[2025-04-25 09:07:55] (step=0054675) Train Loss: 5.2347, Train Steps/Sec: 1.17 +[2025-04-25 09:08:16] (step=0054700) Train Loss: 5.2645, Train Steps/Sec: 1.16 +[2025-04-25 09:08:37] (step=0054725) Train Loss: 5.3187, Train Steps/Sec: 1.17 +[2025-04-25 09:09:00] (step=0054750) Train Loss: 5.3027, Train Steps/Sec: 1.12 +[2025-04-25 09:09:21] (step=0054775) Train Loss: 5.2667, Train Steps/Sec: 1.17 +[2025-04-25 09:09:43] (step=0054800) Train Loss: 5.2595, Train Steps/Sec: 1.17 +[2025-04-25 09:10:04] (step=0054825) Train Loss: 5.3338, Train Steps/Sec: 1.17 +[2025-04-25 09:10:25] (step=0054850) Train Loss: 5.2845, Train Steps/Sec: 1.16 +[2025-04-25 09:10:47] (step=0054875) Train Loss: 5.2682, Train Steps/Sec: 1.16 +[2025-04-25 09:11:08] (step=0054900) Train Loss: 5.2875, Train Steps/Sec: 1.17 +[2025-04-25 09:11:30] (step=0054925) Train Loss: 5.3312, Train Steps/Sec: 1.17 +[2025-04-25 09:11:51] (step=0054950) Train Loss: 5.2811, Train Steps/Sec: 1.17 +[2025-04-25 09:12:13] (step=0054975) Train Loss: 5.3055, Train Steps/Sec: 1.12 +[2025-04-25 09:12:35] (step=0055000) Train Loss: 5.2660, Train Steps/Sec: 1.17 +[2025-04-25 09:12:57] (step=0055025) Train Loss: 5.2717, Train Steps/Sec: 1.13 +[2025-04-25 09:13:19] (step=0055050) Train Loss: 5.2567, Train Steps/Sec: 1.13 +[2025-04-25 09:13:41] (step=0055075) Train Loss: 5.2771, Train Steps/Sec: 1.17 +[2025-04-25 09:14:03] (step=0055100) Train Loss: 5.3122, Train Steps/Sec: 1.12 +[2025-04-25 09:14:25] (step=0055125) Train Loss: 5.2791, Train Steps/Sec: 1.13 +[2025-04-25 09:14:47] (step=0055150) Train Loss: 5.3104, Train Steps/Sec: 1.17 +[2025-04-25 09:15:09] (step=0055175) Train Loss: 5.2952, Train Steps/Sec: 1.13 +[2025-04-25 09:15:31] (step=0055200) Train Loss: 5.2603, Train Steps/Sec: 1.12 +[2025-04-25 09:15:52] (step=0055225) Train Loss: 5.3251, Train Steps/Sec: 1.17 +[2025-04-25 09:16:14] (step=0055250) Train Loss: 5.2390, Train Steps/Sec: 1.17 +[2025-04-25 09:16:35] (step=0055275) Train Loss: 5.2610, Train Steps/Sec: 1.17 +[2025-04-25 09:16:57] (step=0055300) Train Loss: 5.3269, Train Steps/Sec: 1.16 +[2025-04-25 09:17:18] (step=0055325) Train Loss: 5.2442, Train Steps/Sec: 1.17 +[2025-04-25 09:17:39] (step=0055350) Train Loss: 5.2754, Train Steps/Sec: 1.17 +[2025-04-25 09:18:01] (step=0055375) Train Loss: 5.2155, Train Steps/Sec: 1.17 +[2025-04-25 09:18:22] (step=0055400) Train Loss: 5.2695, Train Steps/Sec: 1.17 +[2025-04-25 09:18:43] (step=0055425) Train Loss: 5.3190, Train Steps/Sec: 1.17 +[2025-04-25 09:19:05] (step=0055450) Train Loss: 5.2577, Train Steps/Sec: 1.17 +[2025-04-25 09:19:26] (step=0055475) Train Loss: 5.3184, Train Steps/Sec: 1.17 +[2025-04-25 09:19:49] (step=0055500) Train Loss: 5.3463, Train Steps/Sec: 1.12 +[2025-04-25 09:20:10] (step=0055525) Train Loss: 5.2282, Train Steps/Sec: 1.17 +[2025-04-25 09:20:31] (step=0055550) Train Loss: 5.2771, Train Steps/Sec: 1.17 +[2025-04-25 09:20:53] (step=0055575) Train Loss: 5.3466, Train Steps/Sec: 1.17 +[2025-04-25 09:21:14] (step=0055600) Train Loss: 5.3142, Train Steps/Sec: 1.16 +[2025-04-25 09:21:36] (step=0055625) Train Loss: 5.3022, Train Steps/Sec: 1.17 +[2025-04-25 09:21:57] (step=0055650) Train Loss: 5.2214, Train Steps/Sec: 1.17 +[2025-04-25 09:22:19] (step=0055675) Train Loss: 5.2672, Train Steps/Sec: 1.12 +[2025-04-25 09:22:42] (step=0055700) Train Loss: 5.3390, Train Steps/Sec: 1.12 +[2025-04-25 09:23:03] (step=0055725) Train Loss: 5.1906, Train Steps/Sec: 1.17 +[2025-04-25 09:23:25] (step=0055750) Train Loss: 5.2746, Train Steps/Sec: 1.12 +[2025-04-25 09:23:48] (step=0055775) Train Loss: 5.3198, Train Steps/Sec: 1.08 +[2025-04-25 09:24:10] (step=0055800) Train Loss: 5.1801, Train Steps/Sec: 1.17 +[2025-04-25 09:24:32] (step=0055825) Train Loss: 5.1296, Train Steps/Sec: 1.13 +[2025-04-25 09:24:54] (step=0055850) Train Loss: 5.2243, Train Steps/Sec: 1.12 +[2025-04-25 09:25:16] (step=0055875) Train Loss: 5.3029, Train Steps/Sec: 1.17 +[2025-04-25 09:25:37] (step=0055900) Train Loss: 5.2990, Train Steps/Sec: 1.16 +[2025-04-25 09:25:58] (step=0055925) Train Loss: 5.1814, Train Steps/Sec: 1.17 +[2025-04-25 09:26:20] (step=0055950) Train Loss: 5.3687, Train Steps/Sec: 1.17 +[2025-04-25 09:26:41] (step=0055975) Train Loss: 5.3579, Train Steps/Sec: 1.17 +[2025-04-25 09:27:03] (step=0056000) Train Loss: 5.3560, Train Steps/Sec: 1.16 +[2025-04-25 09:27:03] text_config is None. Initializing the text config with default values (`OPTConfig`). +[2025-04-25 09:32:09] Finish Eval in 56000 steps... +[2025-04-25 09:32:29] Saved checkpoint to checkpoint/Afteri2i_100kckpt_LLAVAT5_empty_Fixed_just_segment_pretrain_stage3_dreambench_recap_400k_t2i_400k_flux_200k_midjourney_150k_recovery_150k_3e4_epoch2_no_rotated/002-GPT-XL/checkpoints/0056000.pt +[2025-04-25 09:32:31] Removed old checkpoint: checkpoint/Afteri2i_100kckpt_LLAVAT5_empty_Fixed_just_segment_pretrain_stage3_dreambench_recap_400k_t2i_400k_flux_200k_midjourney_150k_recovery_150k_3e4_epoch2_no_rotated/002-GPT-XL/checkpoints/0054000.pt +[2025-04-25 09:32:53] (step=0056025) Train Loss: 5.3169, Train Steps/Sec: 0.07 +[2025-04-25 09:33:14] (step=0056050) Train Loss: 5.2756, Train Steps/Sec: 1.17 +[2025-04-25 09:33:35] (step=0056075) Train Loss: 5.3036, Train Steps/Sec: 1.17 +[2025-04-25 09:33:57] (step=0056100) Train Loss: 5.3553, Train Steps/Sec: 1.16 +[2025-04-25 09:34:18] (step=0056125) Train Loss: 5.3022, Train Steps/Sec: 1.17 +[2025-04-25 09:34:39] (step=0056150) Train Loss: 5.3276, Train Steps/Sec: 1.17 +[2025-04-25 09:35:01] (step=0056175) Train Loss: 5.2486, Train Steps/Sec: 1.15 +[2025-04-25 09:35:23] (step=0056200) Train Loss: 5.3283, Train Steps/Sec: 1.17 +[2025-04-25 09:35:44] (step=0056225) Train Loss: 5.2839, Train Steps/Sec: 1.17 +[2025-04-25 09:36:05] (step=0056250) Train Loss: 5.2823, Train Steps/Sec: 1.17 +[2025-04-25 09:36:27] (step=0056275) Train Loss: 5.2822, Train Steps/Sec: 1.17 +[2025-04-25 09:36:48] (step=0056300) Train Loss: 5.2985, Train Steps/Sec: 1.16 +[2025-04-25 09:37:10] (step=0056325) Train Loss: 5.3006, Train Steps/Sec: 1.13 +[2025-04-25 09:37:32] (step=0056350) Train Loss: 5.3926, Train Steps/Sec: 1.17 +[2025-04-25 09:37:53] (step=0056375) Train Loss: 5.3270, Train Steps/Sec: 1.17 +[2025-04-25 09:38:15] (step=0056400) Train Loss: 5.2389, Train Steps/Sec: 1.12 +[2025-04-25 09:38:38] (step=0056425) Train Loss: 5.2750, Train Steps/Sec: 1.12 +[2025-04-25 09:39:00] (step=0056450) Train Loss: 5.3009, Train Steps/Sec: 1.12 +[2025-04-25 09:39:22] (step=0056475) Train Loss: 5.1975, Train Steps/Sec: 1.13 +[2025-04-25 09:39:44] (step=0056500) Train Loss: 5.2619, Train Steps/Sec: 1.16 +[2025-04-25 09:40:07] (step=0056525) Train Loss: 5.2976, Train Steps/Sec: 1.12 +[2025-04-25 09:40:28] (step=0056550) Train Loss: 5.2536, Train Steps/Sec: 1.17 +[2025-04-25 09:40:50] (step=0056575) Train Loss: 5.2491, Train Steps/Sec: 1.17 +[2025-04-25 09:41:11] (step=0056600) Train Loss: 5.3557, Train Steps/Sec: 1.16 +[2025-04-25 09:41:32] (step=0056625) Train Loss: 5.3269, Train Steps/Sec: 1.17 +[2025-04-25 09:41:54] (step=0056650) Train Loss: 5.3061, Train Steps/Sec: 1.17 +[2025-04-25 09:42:15] (step=0056675) Train Loss: 5.2498, Train Steps/Sec: 1.17 +[2025-04-25 09:42:37] (step=0056700) Train Loss: 5.2482, Train Steps/Sec: 1.17 +[2025-04-25 09:42:58] (step=0056725) Train Loss: 5.2492, Train Steps/Sec: 1.17 +[2025-04-25 09:43:20] (step=0056750) Train Loss: 5.1916, Train Steps/Sec: 1.12 +[2025-04-25 09:43:42] (step=0056775) Train Loss: 5.3029, Train Steps/Sec: 1.17 +[2025-04-25 09:44:03] (step=0056800) Train Loss: 5.3020, Train Steps/Sec: 1.17 +[2025-04-25 09:44:25] (step=0056825) Train Loss: 5.1958, Train Steps/Sec: 1.17 +[2025-04-25 09:44:46] (step=0056850) Train Loss: 5.2183, Train Steps/Sec: 1.17 +[2025-04-25 09:45:07] (step=0056875) Train Loss: 5.3026, Train Steps/Sec: 1.17 +[2025-04-25 09:45:29] (step=0056900) Train Loss: 5.3129, Train Steps/Sec: 1.16 +[2025-04-25 09:45:50] (step=0056925) Train Loss: 5.3192, Train Steps/Sec: 1.17 +[2025-04-25 09:46:12] (step=0056950) Train Loss: 5.2815, Train Steps/Sec: 1.17 +[2025-04-25 09:46:34] (step=0056975) Train Loss: 5.3201, Train Steps/Sec: 1.12 +[2025-04-25 09:46:55] (step=0057000) Train Loss: 5.2743, Train Steps/Sec: 1.17 +[2025-04-25 09:47:17] (step=0057025) Train Loss: 5.2622, Train Steps/Sec: 1.17 +[2025-04-25 09:47:39] (step=0057050) Train Loss: 5.2619, Train Steps/Sec: 1.13 +[2025-04-25 09:48:01] (step=0057075) Train Loss: 5.2092, Train Steps/Sec: 1.12 +[2025-04-25 09:48:23] (step=0057100) Train Loss: 5.3599, Train Steps/Sec: 1.17 +[2025-04-25 09:48:44] (step=0057125) Train Loss: 5.2205, Train Steps/Sec: 1.17 +[2025-04-25 09:49:06] (step=0057150) Train Loss: 5.2800, Train Steps/Sec: 1.13 +[2025-04-25 09:49:29] (step=0057175) Train Loss: 5.2760, Train Steps/Sec: 1.08 +[2025-04-25 09:49:51] (step=0057200) Train Loss: 5.3249, Train Steps/Sec: 1.17 +[2025-04-25 09:50:12] (step=0057225) Train Loss: 5.3148, Train Steps/Sec: 1.17 +[2025-04-25 09:50:34] (step=0057250) Train Loss: 5.2735, Train Steps/Sec: 1.12 +[2025-04-25 09:50:56] (step=0057275) Train Loss: 5.3209, Train Steps/Sec: 1.17 +[2025-04-25 09:51:17] (step=0057300) Train Loss: 5.2391, Train Steps/Sec: 1.16 +[2025-04-25 09:51:38] (step=0057325) Train Loss: 5.3539, Train Steps/Sec: 1.17 +[2025-04-25 09:52:00] (step=0057350) Train Loss: 5.3386, Train Steps/Sec: 1.17 +[2025-04-25 09:52:21] (step=0057375) Train Loss: 5.3473, Train Steps/Sec: 1.17 +[2025-04-25 09:52:43] (step=0057400) Train Loss: 5.2929, Train Steps/Sec: 1.16 +[2025-04-25 09:53:04] (step=0057425) Train Loss: 5.3097, Train Steps/Sec: 1.17 +[2025-04-25 09:53:25] (step=0057450) Train Loss: 5.3929, Train Steps/Sec: 1.17 +[2025-04-25 09:53:47] (step=0057475) Train Loss: 5.2993, Train Steps/Sec: 1.17 +[2025-04-25 09:54:09] (step=0057500) Train Loss: 5.3005, Train Steps/Sec: 1.11 +[2025-04-25 09:54:31] (step=0057525) Train Loss: 5.3090, Train Steps/Sec: 1.17 +[2025-04-25 09:54:52] (step=0057550) Train Loss: 5.3536, Train Steps/Sec: 1.17 +[2025-04-25 09:55:13] (step=0057575) Train Loss: 5.2989, Train Steps/Sec: 1.17 +[2025-04-25 09:55:35] (step=0057600) Train Loss: 5.2439, Train Steps/Sec: 1.17 +[2025-04-25 09:55:57] (step=0057625) Train Loss: 5.2547, Train Steps/Sec: 1.13 +[2025-04-25 09:56:18] (step=0057650) Train Loss: 5.2678, Train Steps/Sec: 1.17 +[2025-04-25 09:56:40] (step=0057675) Train Loss: 5.3069, Train Steps/Sec: 1.16 +[2025-04-25 09:57:02] (step=0057700) Train Loss: 5.2753, Train Steps/Sec: 1.12 +[2025-04-25 09:57:24] (step=0057725) Train Loss: 5.2937, Train Steps/Sec: 1.12 +[2025-04-25 09:57:46] (step=0057750) Train Loss: 5.3537, Train Steps/Sec: 1.17 +[2025-04-25 09:58:07] (step=0057775) Train Loss: 5.2501, Train Steps/Sec: 1.17 +[2025-04-25 09:58:29] (step=0057800) Train Loss: 5.3242, Train Steps/Sec: 1.13 +[2025-04-25 09:58:52] (step=0057825) Train Loss: 5.2653, Train Steps/Sec: 1.12 +[2025-04-25 09:59:13] (step=0057850) Train Loss: 5.2615, Train Steps/Sec: 1.17 +[2025-04-25 09:59:34] (step=0057875) Train Loss: 5.3389, Train Steps/Sec: 1.17 +[2025-04-25 09:59:57] (step=0057900) Train Loss: 5.2761, Train Steps/Sec: 1.12 +[2025-04-25 10:00:18] (step=0057925) Train Loss: 5.2504, Train Steps/Sec: 1.17 +[2025-04-25 10:00:40] (step=0057950) Train Loss: 5.3483, Train Steps/Sec: 1.17 +[2025-04-25 10:01:02] (step=0057975) Train Loss: 5.2758, Train Steps/Sec: 1.12 +[2025-04-25 10:01:23] (step=0058000) Train Loss: 5.3085, Train Steps/Sec: 1.17 +[2025-04-25 10:01:23] text_config is None. Initializing the text config with default values (`OPTConfig`). +[2025-04-25 10:06:29] Finish Eval in 58000 steps... +[2025-04-25 10:06:49] Saved checkpoint to checkpoint/Afteri2i_100kckpt_LLAVAT5_empty_Fixed_just_segment_pretrain_stage3_dreambench_recap_400k_t2i_400k_flux_200k_midjourney_150k_recovery_150k_3e4_epoch2_no_rotated/002-GPT-XL/checkpoints/0058000.pt +[2025-04-25 10:06:52] Removed old checkpoint: checkpoint/Afteri2i_100kckpt_LLAVAT5_empty_Fixed_just_segment_pretrain_stage3_dreambench_recap_400k_t2i_400k_flux_200k_midjourney_150k_recovery_150k_3e4_epoch2_no_rotated/002-GPT-XL/checkpoints/0056000.pt +[2025-04-25 10:07:13] (step=0058025) Train Loss: 5.2773, Train Steps/Sec: 0.07 +[2025-04-25 10:07:35] (step=0058050) Train Loss: 5.3203, Train Steps/Sec: 1.17 +[2025-04-25 10:07:56] (step=0058075) Train Loss: 5.2922, Train Steps/Sec: 1.17 +[2025-04-25 10:08:18] (step=0058100) Train Loss: 5.2621, Train Steps/Sec: 1.15 +[2025-04-25 10:08:39] (step=0058125) Train Loss: 5.2763, Train Steps/Sec: 1.17 +[2025-04-25 10:09:00] (step=0058150) Train Loss: 5.3120, Train Steps/Sec: 1.17 +[2025-04-25 10:09:22] (step=0058175) Train Loss: 5.2789, Train Steps/Sec: 1.17 +[2025-04-25 10:09:43] (step=0058200) Train Loss: 5.2849, Train Steps/Sec: 1.16 +[2025-04-25 10:10:05] (step=0058225) Train Loss: 5.2859, Train Steps/Sec: 1.17 +[2025-04-25 10:10:26] (step=0058250) Train Loss: 5.2359, Train Steps/Sec: 1.17 +[2025-04-25 10:10:48] (step=0058275) Train Loss: 5.2938, Train Steps/Sec: 1.12 +[2025-04-25 10:11:10] (step=0058300) Train Loss: 5.3189, Train Steps/Sec: 1.17 +[2025-04-25 10:11:31] (step=0058325) Train Loss: 5.3662, Train Steps/Sec: 1.17 +[2025-04-25 10:11:53] (step=0058350) Train Loss: 5.3598, Train Steps/Sec: 1.12 +[2025-04-25 10:12:15] (step=0058375) Train Loss: 5.2802, Train Steps/Sec: 1.13 +[2025-04-25 10:12:37] (step=0058400) Train Loss: 5.1913, Train Steps/Sec: 1.17 +[2025-04-25 10:12:58] (step=0058425) Train Loss: 5.3371, Train Steps/Sec: 1.17 +[2025-04-25 10:13:20] (step=0058450) Train Loss: 5.2303, Train Steps/Sec: 1.13 +[2025-04-25 10:13:42] (step=0058475) Train Loss: 5.2475, Train Steps/Sec: 1.17 +[2025-04-25 10:14:04] (step=0058500) Train Loss: 5.3057, Train Steps/Sec: 1.12 +[2025-04-25 10:14:26] (step=0058525) Train Loss: 5.3227, Train Steps/Sec: 1.17 +[2025-04-25 10:14:47] (step=0058550) Train Loss: 5.3476, Train Steps/Sec: 1.17 +[2025-04-25 10:15:08] (step=0058575) Train Loss: 5.3026, Train Steps/Sec: 1.17 +[2025-04-25 10:15:30] (step=0058600) Train Loss: 5.2403, Train Steps/Sec: 1.16 +[2025-04-25 10:15:52] (step=0058625) Train Loss: 5.2837, Train Steps/Sec: 1.12 +[2025-04-25 10:16:14] (step=0058650) Train Loss: 5.3110, Train Steps/Sec: 1.17 +[2025-04-25 10:16:35] (step=0058675) Train Loss: 5.2776, Train Steps/Sec: 1.17 +[2025-04-25 10:16:57] (step=0058700) Train Loss: 5.2267, Train Steps/Sec: 1.12 +[2025-04-25 10:17:19] (step=0058725) Train Loss: 5.3002, Train Steps/Sec: 1.17 +[2025-04-25 10:17:41] (step=0058750) Train Loss: 5.3081, Train Steps/Sec: 1.12 +[2025-04-25 10:18:03] (step=0058775) Train Loss: 5.2659, Train Steps/Sec: 1.17 +[2025-04-25 10:18:24] (step=0058800) Train Loss: 5.2432, Train Steps/Sec: 1.17 +[2025-04-25 10:18:45] (step=0058825) Train Loss: 5.2899, Train Steps/Sec: 1.17 +[2025-04-25 10:19:07] (step=0058850) Train Loss: 5.2921, Train Steps/Sec: 1.17 +[2025-04-25 10:19:28] (step=0058875) Train Loss: 5.3497, Train Steps/Sec: 1.17 +[2025-04-25 10:19:49] (step=0058900) Train Loss: 5.2507, Train Steps/Sec: 1.17 +[2025-04-25 10:20:12] (step=0058925) Train Loss: 5.3443, Train Steps/Sec: 1.13 +[2025-04-25 10:20:33] (step=0058950) Train Loss: 5.2995, Train Steps/Sec: 1.17 +[2025-04-25 10:20:54] (step=0058975) Train Loss: 5.2991, Train Steps/Sec: 1.17 +[2025-04-25 10:21:17] (step=0059000) Train Loss: 5.3176, Train Steps/Sec: 1.12 +[2025-04-25 10:21:38] (step=0059025) Train Loss: 5.1812, Train Steps/Sec: 1.17 +[2025-04-25 10:22:00] (step=0059050) Train Loss: 5.2424, Train Steps/Sec: 1.12 +[2025-04-25 10:22:22] (step=0059075) Train Loss: 5.3235, Train Steps/Sec: 1.17 +[2025-04-25 10:22:43] (step=0059100) Train Loss: 5.2927, Train Steps/Sec: 1.16 +[2025-04-25 10:23:05] (step=0059125) Train Loss: 5.1694, Train Steps/Sec: 1.13 +[2025-04-25 10:23:27] (step=0059150) Train Loss: 5.2597, Train Steps/Sec: 1.13 +[2025-04-25 10:23:49] (step=0059175) Train Loss: 5.3108, Train Steps/Sec: 1.17 +[2025-04-25 10:24:10] (step=0059200) Train Loss: 5.3129, Train Steps/Sec: 1.17 +[2025-04-25 10:24:32] (step=0059225) Train Loss: 5.3318, Train Steps/Sec: 1.17 +[2025-04-25 10:24:53] (step=0059250) Train Loss: 5.2440, Train Steps/Sec: 1.17 +[2025-04-25 10:25:14] (step=0059275) Train Loss: 5.3401, Train Steps/Sec: 1.17 +[2025-04-25 10:25:36] (step=0059300) Train Loss: 5.2764, Train Steps/Sec: 1.17 +[2025-04-25 10:25:57] (step=0059325) Train Loss: 5.3568, Train Steps/Sec: 1.17 +[2025-04-25 10:26:19] (step=0059350) Train Loss: 5.3294, Train Steps/Sec: 1.13 +[2025-04-25 10:26:41] (step=0059375) Train Loss: 5.3026, Train Steps/Sec: 1.17 +[2025-04-25 10:27:02] (step=0059400) Train Loss: 5.3288, Train Steps/Sec: 1.17 +[2025-04-25 10:27:24] (step=0059425) Train Loss: 5.2847, Train Steps/Sec: 1.17 +[2025-04-25 10:27:46] (step=0059450) Train Loss: 5.3240, Train Steps/Sec: 1.12 +[2025-04-25 10:28:07] (step=0059475) Train Loss: 5.3223, Train Steps/Sec: 1.17 +[2025-04-25 10:28:30] (step=0059500) Train Loss: 5.2546, Train Steps/Sec: 1.11 +[2025-04-25 10:28:51] (step=0059525) Train Loss: 5.2779, Train Steps/Sec: 1.17 +[2025-04-25 10:29:12] (step=0059550) Train Loss: 5.2911, Train Steps/Sec: 1.17 +[2025-04-25 10:29:35] (step=0059575) Train Loss: 5.2197, Train Steps/Sec: 1.12 +[2025-04-25 10:29:56] (step=0059600) Train Loss: 5.3015, Train Steps/Sec: 1.17 +[2025-04-25 10:30:17] (step=0059625) Train Loss: 5.3149, Train Steps/Sec: 1.17 +[2025-04-25 10:30:39] (step=0059650) Train Loss: 5.2136, Train Steps/Sec: 1.17 +[2025-04-25 10:31:01] (step=0059675) Train Loss: 5.2819, Train Steps/Sec: 1.13 +[2025-04-25 10:31:23] (step=0059700) Train Loss: 5.3448, Train Steps/Sec: 1.12 +[2025-04-25 10:31:45] (step=0059725) Train Loss: 5.3353, Train Steps/Sec: 1.17 +[2025-04-25 10:32:06] (step=0059750) Train Loss: 5.2608, Train Steps/Sec: 1.17 +[2025-04-25 10:32:28] (step=0059775) Train Loss: 5.2741, Train Steps/Sec: 1.12 +[2025-04-25 10:32:51] (step=0059800) Train Loss: 5.3171, Train Steps/Sec: 1.12 +[2025-04-25 10:33:12] (step=0059825) Train Loss: 5.2556, Train Steps/Sec: 1.17 +[2025-04-25 10:33:33] (step=0059850) Train Loss: 5.2499, Train Steps/Sec: 1.17 +[2025-04-25 10:33:55] (step=0059875) Train Loss: 5.2841, Train Steps/Sec: 1.17 +[2025-04-25 10:34:16] (step=0059900) Train Loss: 5.1980, Train Steps/Sec: 1.17 +[2025-04-25 10:34:37] (step=0059925) Train Loss: 5.2898, Train Steps/Sec: 1.17 +[2025-04-25 10:34:59] (step=0059950) Train Loss: 5.2988, Train Steps/Sec: 1.17 +[2025-04-25 10:35:20] (step=0059975) Train Loss: 5.3223, Train Steps/Sec: 1.17 +[2025-04-25 10:35:42] (step=0060000) Train Loss: 5.3202, Train Steps/Sec: 1.16 +[2025-04-25 10:35:42] text_config is None. Initializing the text config with default values (`OPTConfig`). +[2025-04-25 10:40:47] Finish Eval in 60000 steps... +[2025-04-25 10:41:06] Saved checkpoint to checkpoint/Afteri2i_100kckpt_LLAVAT5_empty_Fixed_just_segment_pretrain_stage3_dreambench_recap_400k_t2i_400k_flux_200k_midjourney_150k_recovery_150k_3e4_epoch2_no_rotated/002-GPT-XL/checkpoints/0060000.pt +[2025-04-25 10:41:09] Removed old checkpoint: checkpoint/Afteri2i_100kckpt_LLAVAT5_empty_Fixed_just_segment_pretrain_stage3_dreambench_recap_400k_t2i_400k_flux_200k_midjourney_150k_recovery_150k_3e4_epoch2_no_rotated/002-GPT-XL/checkpoints/0058000.pt +[2025-04-25 10:41:30] (step=0060025) Train Loss: 5.2903, Train Steps/Sec: 0.07 +[2025-04-25 10:41:52] (step=0060050) Train Loss: 5.2546, Train Steps/Sec: 1.17 +[2025-04-25 10:42:14] (step=0060075) Train Loss: 5.3219, Train Steps/Sec: 1.12 +[2025-04-25 10:42:35] (step=0060100) Train Loss: 5.2247, Train Steps/Sec: 1.17 +[2025-04-25 10:42:57] (step=0060125) Train Loss: 5.3042, Train Steps/Sec: 1.17 +[2025-04-25 10:43:18] (step=0060150) Train Loss: 5.2465, Train Steps/Sec: 1.17 +[2025-04-25 10:43:40] (step=0060175) Train Loss: 5.2824, Train Steps/Sec: 1.12 +[2025-04-25 10:44:02] (step=0060200) Train Loss: 5.3293, Train Steps/Sec: 1.17 +[2025-04-25 10:44:24] (step=0060225) Train Loss: 5.2412, Train Steps/Sec: 1.12 +[2025-04-25 10:44:45] (step=0060250) Train Loss: 5.2938, Train Steps/Sec: 1.17 +[2025-04-25 10:45:07] (step=0060275) Train Loss: 5.2849, Train Steps/Sec: 1.17 +[2025-04-25 10:45:28] (step=0060300) Train Loss: 5.2285, Train Steps/Sec: 1.17 +[2025-04-25 10:45:50] (step=0060325) Train Loss: 5.2706, Train Steps/Sec: 1.12 +[2025-04-25 10:46:13] (step=0060350) Train Loss: 5.2950, Train Steps/Sec: 1.13 +[2025-04-25 10:46:34] (step=0060375) Train Loss: 5.1872, Train Steps/Sec: 1.17 +[2025-04-25 10:46:55] (step=0060400) Train Loss: 5.3249, Train Steps/Sec: 1.16 +[2025-04-25 10:47:18] (step=0060425) Train Loss: 5.2726, Train Steps/Sec: 1.13 +[2025-04-25 10:47:39] (step=0060450) Train Loss: 5.2659, Train Steps/Sec: 1.16 +[2025-04-25 10:48:02] (step=0060475) Train Loss: 5.2618, Train Steps/Sec: 1.12 +[2025-04-25 10:48:23] (step=0060500) Train Loss: 5.2968, Train Steps/Sec: 1.16 +[2025-04-25 10:48:44] (step=0060525) Train Loss: 5.2961, Train Steps/Sec: 1.17 +[2025-04-25 10:49:06] (step=0060550) Train Loss: 5.2770, Train Steps/Sec: 1.17 +[2025-04-25 10:49:27] (step=0060575) Train Loss: 5.3047, Train Steps/Sec: 1.17 +[2025-04-25 10:49:49] (step=0060600) Train Loss: 5.3593, Train Steps/Sec: 1.16 +[2025-04-25 10:50:10] (step=0060625) Train Loss: 5.2503, Train Steps/Sec: 1.17 +[2025-04-25 10:50:31] (step=0060650) Train Loss: 5.2576, Train Steps/Sec: 1.17 +[2025-04-25 10:50:53] (step=0060675) Train Loss: 5.2836, Train Steps/Sec: 1.17 +[2025-04-25 10:51:14] (step=0060700) Train Loss: 5.2749, Train Steps/Sec: 1.16 +[2025-04-25 10:51:36] (step=0060725) Train Loss: 5.3321, Train Steps/Sec: 1.17 +[2025-04-25 10:51:58] (step=0060750) Train Loss: 5.2364, Train Steps/Sec: 1.11 +[2025-04-25 10:52:20] (step=0060775) Train Loss: 5.2285, Train Steps/Sec: 1.17 +[2025-04-25 10:52:42] (step=0060800) Train Loss: 5.2454, Train Steps/Sec: 1.12 +[2025-04-25 10:53:03] (step=0060825) Train Loss: 5.3321, Train Steps/Sec: 1.17 +[2025-04-25 10:53:25] (step=0060850) Train Loss: 5.2449, Train Steps/Sec: 1.17 +[2025-04-25 10:53:47] (step=0060875) Train Loss: 5.2102, Train Steps/Sec: 1.12 +[2025-04-25 10:54:09] (step=0060900) Train Loss: 5.2369, Train Steps/Sec: 1.12 +[2025-04-25 10:54:31] (step=0060925) Train Loss: 5.3053, Train Steps/Sec: 1.17 +[2025-04-25 10:54:52] (step=0060950) Train Loss: 5.2585, Train Steps/Sec: 1.17 +[2025-04-25 10:55:14] (step=0060975) Train Loss: 5.3317, Train Steps/Sec: 1.13 +[2025-04-25 10:55:37] (step=0061000) Train Loss: 5.2482, Train Steps/Sec: 1.12 +[2025-04-25 10:55:58] (step=0061025) Train Loss: 5.2760, Train Steps/Sec: 1.17 +[2025-04-25 10:56:19] (step=0061050) Train Loss: 5.2656, Train Steps/Sec: 1.17 +[2025-04-25 10:56:41] (step=0061075) Train Loss: 5.2647, Train Steps/Sec: 1.17 +[2025-04-25 10:57:03] (step=0061100) Train Loss: 5.2231, Train Steps/Sec: 1.12 +[2025-04-25 10:57:25] (step=0061125) Train Loss: 5.2489, Train Steps/Sec: 1.13 +[2025-04-25 10:57:47] (step=0061150) Train Loss: 5.3300, Train Steps/Sec: 1.17 +[2025-04-25 10:58:08] (step=0061175) Train Loss: 5.1952, Train Steps/Sec: 1.17 +[2025-04-25 10:58:29] (step=0061200) Train Loss: 5.2213, Train Steps/Sec: 1.17 +[2025-04-25 10:58:51] (step=0061225) Train Loss: 5.2123, Train Steps/Sec: 1.17 +[2025-04-25 10:59:12] (step=0061250) Train Loss: 5.3096, Train Steps/Sec: 1.17 +[2025-04-25 10:59:34] (step=0061275) Train Loss: 5.2840, Train Steps/Sec: 1.17 +[2025-04-25 10:59:55] (step=0061300) Train Loss: 5.3036, Train Steps/Sec: 1.16 +[2025-04-25 11:00:16] (step=0061325) Train Loss: 5.2937, Train Steps/Sec: 1.17 +[2025-04-25 11:00:38] (step=0061350) Train Loss: 5.2981, Train Steps/Sec: 1.17 +[2025-04-25 11:00:59] (step=0061375) Train Loss: 5.2620, Train Steps/Sec: 1.17 +[2025-04-25 11:01:21] (step=0061400) Train Loss: 5.2764, Train Steps/Sec: 1.16 +[2025-04-25 11:01:42] (step=0061425) Train Loss: 5.3370, Train Steps/Sec: 1.17 +[2025-04-25 11:02:03] (step=0061450) Train Loss: 5.2107, Train Steps/Sec: 1.17 +[2025-04-25 11:02:25] (step=0061475) Train Loss: 5.2517, Train Steps/Sec: 1.17 +[2025-04-25 11:02:47] (step=0061500) Train Loss: 5.2253, Train Steps/Sec: 1.11 +[2025-04-25 11:03:10] (step=0061525) Train Loss: 5.3081, Train Steps/Sec: 1.08 +[2025-04-25 11:03:32] (step=0061550) Train Loss: 5.3916, Train Steps/Sec: 1.17 +[2025-04-25 11:03:53] (step=0061575) Train Loss: 5.2753, Train Steps/Sec: 1.17 +[2025-04-25 11:04:15] (step=0061600) Train Loss: 5.2418, Train Steps/Sec: 1.17 +[2025-04-25 11:04:38] (step=0061625) Train Loss: 5.3001, Train Steps/Sec: 1.08 +[2025-04-25 11:05:00] (step=0061650) Train Loss: 5.2804, Train Steps/Sec: 1.13 +[2025-04-25 11:05:21] (step=0061675) Train Loss: 5.2134, Train Steps/Sec: 1.17 +[2025-04-25 11:05:43] (step=0061700) Train Loss: 5.2887, Train Steps/Sec: 1.16 +[2025-04-25 11:06:04] (step=0061725) Train Loss: 5.3041, Train Steps/Sec: 1.16 +[2025-04-25 11:06:27] (step=0061750) Train Loss: 5.2179, Train Steps/Sec: 1.13 +[2025-04-25 11:06:48] (step=0061775) Train Loss: 5.2847, Train Steps/Sec: 1.17 +[2025-04-25 11:07:10] (step=0061800) Train Loss: 5.2911, Train Steps/Sec: 1.12 +[2025-04-25 11:07:32] (step=0061825) Train Loss: 5.3185, Train Steps/Sec: 1.17 +[2025-04-25 11:07:53] (step=0061850) Train Loss: 5.2480, Train Steps/Sec: 1.17 +[2025-04-25 11:08:14] (step=0061875) Train Loss: 5.2934, Train Steps/Sec: 1.17 +[2025-04-25 11:08:36] (step=0061900) Train Loss: 5.3014, Train Steps/Sec: 1.17 +[2025-04-25 11:08:57] (step=0061925) Train Loss: 5.3941, Train Steps/Sec: 1.17 +[2025-04-25 11:09:19] (step=0061950) Train Loss: 5.2803, Train Steps/Sec: 1.17 +[2025-04-25 11:09:40] (step=0061975) Train Loss: 5.3309, Train Steps/Sec: 1.17 +[2025-04-25 11:10:01] (step=0062000) Train Loss: 5.2699, Train Steps/Sec: 1.17 +[2025-04-25 11:10:01] text_config is None. Initializing the text config with default values (`OPTConfig`). +[2025-04-25 11:15:09] Finish Eval in 62000 steps... +[2025-04-25 11:15:28] Saved checkpoint to checkpoint/Afteri2i_100kckpt_LLAVAT5_empty_Fixed_just_segment_pretrain_stage3_dreambench_recap_400k_t2i_400k_flux_200k_midjourney_150k_recovery_150k_3e4_epoch2_no_rotated/002-GPT-XL/checkpoints/0062000.pt +[2025-04-25 11:15:31] Removed old checkpoint: checkpoint/Afteri2i_100kckpt_LLAVAT5_empty_Fixed_just_segment_pretrain_stage3_dreambench_recap_400k_t2i_400k_flux_200k_midjourney_150k_recovery_150k_3e4_epoch2_no_rotated/002-GPT-XL/checkpoints/0060000.pt +[2025-04-25 11:15:52] (step=0062025) Train Loss: 5.2381, Train Steps/Sec: 0.07 +[2025-04-25 11:16:13] (step=0062050) Train Loss: 5.2925, Train Steps/Sec: 1.17 +[2025-04-25 11:16:35] (step=0062075) Train Loss: 5.3144, Train Steps/Sec: 1.17 +[2025-04-25 11:16:56] (step=0062100) Train Loss: 5.3473, Train Steps/Sec: 1.16 +[2025-04-25 11:17:18] (step=0062125) Train Loss: 5.2766, Train Steps/Sec: 1.17 +[2025-04-25 11:17:39] (step=0062150) Train Loss: 5.2953, Train Steps/Sec: 1.17 +[2025-04-25 11:18:01] (step=0062175) Train Loss: 5.2777, Train Steps/Sec: 1.12 +[2025-04-25 11:18:23] (step=0062200) Train Loss: 5.2398, Train Steps/Sec: 1.16 +[2025-04-25 11:18:44] (step=0062225) Train Loss: 5.3130, Train Steps/Sec: 1.17 +[2025-04-25 11:19:06] (step=0062250) Train Loss: 5.2198, Train Steps/Sec: 1.12 +[2025-04-25 11:19:28] (step=0062275) Train Loss: 5.3072, Train Steps/Sec: 1.13 +[2025-04-25 11:19:51] (step=0062300) Train Loss: 5.2469, Train Steps/Sec: 1.12 +[2025-04-25 11:20:12] (step=0062325) Train Loss: 5.3269, Train Steps/Sec: 1.17 +[2025-04-25 11:20:34] (step=0062350) Train Loss: 5.2984, Train Steps/Sec: 1.17 +[2025-04-25 11:20:56] (step=0062375) Train Loss: 5.2452, Train Steps/Sec: 1.12 +[2025-04-25 11:21:18] (step=0062400) Train Loss: 5.2130, Train Steps/Sec: 1.12 +[2025-04-25 11:21:39] (step=0062425) Train Loss: 5.3062, Train Steps/Sec: 1.17 +[2025-04-25 11:22:02] (step=0062450) Train Loss: 5.2167, Train Steps/Sec: 1.12 +[2025-04-25 11:22:23] (step=0062475) Train Loss: 5.2977, Train Steps/Sec: 1.17 +[2025-04-25 11:22:45] (step=0062500) Train Loss: 5.2355, Train Steps/Sec: 1.17 +[2025-04-25 11:23:06] (step=0062525) Train Loss: 5.3225, Train Steps/Sec: 1.17 +[2025-04-25 11:23:27] (step=0062550) Train Loss: 5.2079, Train Steps/Sec: 1.17 +[2025-04-25 11:23:49] (step=0062575) Train Loss: 5.2371, Train Steps/Sec: 1.17 +[2025-04-25 11:24:10] (step=0062600) Train Loss: 5.2944, Train Steps/Sec: 1.16 +[2025-04-25 11:24:32] (step=0062625) Train Loss: 5.3301, Train Steps/Sec: 1.15 +[2025-04-25 11:24:53] (step=0062650) Train Loss: 5.2319, Train Steps/Sec: 1.17 +[2025-04-25 11:25:15] (step=0062675) Train Loss: 5.3203, Train Steps/Sec: 1.17 +[2025-04-25 11:25:36] (step=0062700) Train Loss: 5.2948, Train Steps/Sec: 1.16 +[2025-04-25 11:25:57] (step=0062725) Train Loss: 5.2873, Train Steps/Sec: 1.17 +[2025-04-25 11:26:20] (step=0062750) Train Loss: 5.3012, Train Steps/Sec: 1.12 +[2025-04-25 11:26:41] (step=0062775) Train Loss: 5.2741, Train Steps/Sec: 1.17 +[2025-04-25 11:27:03] (step=0062800) Train Loss: 5.2888, Train Steps/Sec: 1.17 +[2025-04-25 11:27:25] (step=0062825) Train Loss: 5.2637, Train Steps/Sec: 1.12 +[2025-04-25 11:27:46] (step=0062850) Train Loss: 5.2891, Train Steps/Sec: 1.17 +[2025-04-25 11:28:08] (step=0062875) Train Loss: 5.3436, Train Steps/Sec: 1.17 +[2025-04-25 11:28:29] (step=0062900) Train Loss: 5.2466, Train Steps/Sec: 1.16 +[2025-04-25 11:28:51] (step=0062925) Train Loss: 5.3076, Train Steps/Sec: 1.13 +[2025-04-25 11:29:14] (step=0062950) Train Loss: 5.2473, Train Steps/Sec: 1.12 +[2025-04-25 11:29:35] (step=0062975) Train Loss: 5.2367, Train Steps/Sec: 1.17 +[2025-04-25 11:29:57] (step=0063000) Train Loss: 5.2279, Train Steps/Sec: 1.12 +[2025-04-25 11:30:19] (step=0063025) Train Loss: 5.3075, Train Steps/Sec: 1.17 +[2025-04-25 11:30:40] (step=0063050) Train Loss: 5.3049, Train Steps/Sec: 1.17 +[2025-04-25 11:31:02] (step=0063075) Train Loss: 5.2513, Train Steps/Sec: 1.13 +[2025-04-25 11:31:25] (step=0063100) Train Loss: 5.3263, Train Steps/Sec: 1.11 +[2025-04-25 11:31:47] (step=0063125) Train Loss: 5.3226, Train Steps/Sec: 1.13 +[2025-04-25 11:32:08] (step=0063150) Train Loss: 5.3336, Train Steps/Sec: 1.17 +[2025-04-25 11:32:30] (step=0063175) Train Loss: 5.3286, Train Steps/Sec: 1.17 +[2025-04-25 11:32:51] (step=0063200) Train Loss: 5.2025, Train Steps/Sec: 1.16 +[2025-04-25 11:33:12] (step=0063225) Train Loss: 5.2721, Train Steps/Sec: 1.17 +[2025-04-25 11:33:34] (step=0063250) Train Loss: 5.2499, Train Steps/Sec: 1.17 +[2025-04-25 11:33:55] (step=0063275) Train Loss: 5.2251, Train Steps/Sec: 1.16 +[2025-04-25 11:34:17] (step=0063300) Train Loss: 5.3077, Train Steps/Sec: 1.16 +[2025-04-25 11:34:38] (step=0063325) Train Loss: 5.3372, Train Steps/Sec: 1.17 +[2025-04-25 11:34:59] (step=0063350) Train Loss: 5.2926, Train Steps/Sec: 1.17 +[2025-04-25 11:35:21] (step=0063375) Train Loss: 5.2895, Train Steps/Sec: 1.17 +[2025-04-25 11:35:42] (step=0063400) Train Loss: 5.2981, Train Steps/Sec: 1.16 +[2025-04-25 11:36:04] (step=0063425) Train Loss: 5.2249, Train Steps/Sec: 1.17 +[2025-04-25 11:36:25] (step=0063450) Train Loss: 5.3208, Train Steps/Sec: 1.17 +[2025-04-25 11:36:47] (step=0063475) Train Loss: 5.2535, Train Steps/Sec: 1.12 +[2025-04-25 11:37:10] (step=0063500) Train Loss: 5.1907, Train Steps/Sec: 1.11 +[2025-04-25 11:37:31] (step=0063525) Train Loss: 5.3159, Train Steps/Sec: 1.17 +[2025-04-25 11:37:53] (step=0063550) Train Loss: 5.2436, Train Steps/Sec: 1.17 +[2025-04-25 11:38:15] (step=0063575) Train Loss: 5.2907, Train Steps/Sec: 1.12 +[2025-04-25 11:38:37] (step=0063600) Train Loss: 5.3135, Train Steps/Sec: 1.16 +[2025-04-25 11:38:59] (step=0063625) Train Loss: 5.2554, Train Steps/Sec: 1.17 +[2025-04-25 11:39:20] (step=0063650) Train Loss: 5.3573, Train Steps/Sec: 1.17 +[2025-04-25 11:39:41] (step=0063675) Train Loss: 5.2699, Train Steps/Sec: 1.17 +[2025-04-25 11:40:03] (step=0063700) Train Loss: 5.2673, Train Steps/Sec: 1.16 +[2025-04-25 11:40:26] (step=0063725) Train Loss: 5.2347, Train Steps/Sec: 1.08 +[2025-04-25 11:40:47] (step=0063750) Train Loss: 5.3367, Train Steps/Sec: 1.17 +[2025-04-25 11:41:09] (step=0063775) Train Loss: 5.2790, Train Steps/Sec: 1.12 +[2025-04-25 11:41:31] (step=0063800) Train Loss: 5.2655, Train Steps/Sec: 1.16 +[2025-04-25 11:41:53] (step=0063825) Train Loss: 5.2647, Train Steps/Sec: 1.12 +[2025-04-25 11:42:15] (step=0063850) Train Loss: 5.2725, Train Steps/Sec: 1.17 +[2025-04-25 11:42:36] (step=0063875) Train Loss: 5.3431, Train Steps/Sec: 1.17 +[2025-04-25 11:42:57] (step=0063900) Train Loss: 5.2987, Train Steps/Sec: 1.16 +[2025-04-25 11:43:19] (step=0063925) Train Loss: 5.3018, Train Steps/Sec: 1.17 +[2025-04-25 11:43:40] (step=0063950) Train Loss: 5.2648, Train Steps/Sec: 1.16 +[2025-04-25 11:44:02] (step=0063975) Train Loss: 5.3315, Train Steps/Sec: 1.17 +[2025-04-25 11:44:23] (step=0064000) Train Loss: 5.2440, Train Steps/Sec: 1.16 +[2025-04-25 11:44:23] text_config is None. Initializing the text config with default values (`OPTConfig`). +[2025-04-25 11:49:33] Finish Eval in 64000 steps... +[2025-04-25 11:49:54] Saved checkpoint to checkpoint/Afteri2i_100kckpt_LLAVAT5_empty_Fixed_just_segment_pretrain_stage3_dreambench_recap_400k_t2i_400k_flux_200k_midjourney_150k_recovery_150k_3e4_epoch2_no_rotated/002-GPT-XL/checkpoints/0064000.pt +[2025-04-25 11:49:56] Removed old checkpoint: checkpoint/Afteri2i_100kckpt_LLAVAT5_empty_Fixed_just_segment_pretrain_stage3_dreambench_recap_400k_t2i_400k_flux_200k_midjourney_150k_recovery_150k_3e4_epoch2_no_rotated/002-GPT-XL/checkpoints/0062000.pt +[2025-04-25 11:50:18] (step=0064025) Train Loss: 5.2728, Train Steps/Sec: 0.07 +[2025-04-25 11:50:39] (step=0064050) Train Loss: 5.2697, Train Steps/Sec: 1.17 +[2025-04-25 11:51:00] (step=0064075) Train Loss: 5.3011, Train Steps/Sec: 1.17 +[2025-04-25 11:51:22] (step=0064100) Train Loss: 5.2911, Train Steps/Sec: 1.16 +[2025-04-25 11:51:44] (step=0064125) Train Loss: 5.2715, Train Steps/Sec: 1.12 +[2025-04-25 11:52:05] (step=0064150) Train Loss: 5.2665, Train Steps/Sec: 1.17 +[2025-04-25 11:52:27] (step=0064175) Train Loss: 5.3542, Train Steps/Sec: 1.17 +[2025-04-25 11:52:48] (step=0064200) Train Loss: 5.2871, Train Steps/Sec: 1.16 +[2025-04-25 11:53:10] (step=0064225) Train Loss: 5.2985, Train Steps/Sec: 1.17 +[2025-04-25 11:53:32] (step=0064250) Train Loss: 5.2705, Train Steps/Sec: 1.13 +[2025-04-25 11:53:54] (step=0064275) Train Loss: 5.2875, Train Steps/Sec: 1.12 +[2025-04-25 11:54:16] (step=0064300) Train Loss: 5.2449, Train Steps/Sec: 1.16 +[2025-04-25 11:54:37] (step=0064325) Train Loss: 5.3039, Train Steps/Sec: 1.16 +[2025-04-25 11:54:59] (step=0064350) Train Loss: 5.2686, Train Steps/Sec: 1.16 +[2025-04-25 11:55:21] (step=0064375) Train Loss: 5.2847, Train Steps/Sec: 1.13 +[2025-04-25 11:55:43] (step=0064400) Train Loss: 5.3239, Train Steps/Sec: 1.16 +[2025-04-25 11:56:05] (step=0064425) Train Loss: 5.2520, Train Steps/Sec: 1.13 +[2025-04-25 11:56:27] (step=0064450) Train Loss: 5.3115, Train Steps/Sec: 1.12 +[2025-04-25 11:56:48] (step=0064475) Train Loss: 5.3204, Train Steps/Sec: 1.17 +[2025-04-25 11:57:10] (step=0064500) Train Loss: 5.2275, Train Steps/Sec: 1.16 +[2025-04-25 11:57:31] (step=0064525) Train Loss: 5.3257, Train Steps/Sec: 1.17 +[2025-04-25 11:57:54] (step=0064550) Train Loss: 5.2728, Train Steps/Sec: 1.09 +[2025-04-25 11:58:16] (step=0064575) Train Loss: 5.3542, Train Steps/Sec: 1.17 +[2025-04-25 11:58:37] (step=0064600) Train Loss: 5.3135, Train Steps/Sec: 1.16 +[2025-04-25 11:58:58] (step=0064625) Train Loss: 5.3048, Train Steps/Sec: 1.17 +[2025-04-25 11:59:20] (step=0064650) Train Loss: 5.2762, Train Steps/Sec: 1.17 +[2025-04-25 11:59:41] (step=0064675) Train Loss: 5.2831, Train Steps/Sec: 1.17 +[2025-04-25 12:00:03] (step=0064700) Train Loss: 5.3384, Train Steps/Sec: 1.16 +[2025-04-25 12:00:24] (step=0064725) Train Loss: 5.3184, Train Steps/Sec: 1.17 +[2025-04-25 12:00:47] (step=0064750) Train Loss: 5.3196, Train Steps/Sec: 1.11 +[2025-04-25 12:01:08] (step=0064775) Train Loss: 5.2784, Train Steps/Sec: 1.17 +[2025-04-25 12:01:30] (step=0064800) Train Loss: 5.2653, Train Steps/Sec: 1.12 +[2025-04-25 12:01:52] (step=0064825) Train Loss: 5.3298, Train Steps/Sec: 1.17 +[2025-04-25 12:02:13] (step=0064850) Train Loss: 5.3237, Train Steps/Sec: 1.17 +[2025-04-25 12:02:35] (step=0064875) Train Loss: 5.2528, Train Steps/Sec: 1.16 +[2025-04-25 12:02:57] (step=0064900) Train Loss: 5.3451, Train Steps/Sec: 1.11 +[2025-04-25 12:03:20] (step=0064925) Train Loss: 5.2912, Train Steps/Sec: 1.11 +[2025-04-25 12:03:41] (step=0064950) Train Loss: 5.3011, Train Steps/Sec: 1.16 +[2025-04-25 12:04:03] (step=0064975) Train Loss: 5.3494, Train Steps/Sec: 1.16 +[2025-04-25 12:04:24] (step=0065000) Train Loss: 5.3323, Train Steps/Sec: 1.16 +[2025-04-25 12:04:46] (step=0065025) Train Loss: 5.2564, Train Steps/Sec: 1.17 +[2025-04-25 12:05:08] (step=0065050) Train Loss: 5.2641, Train Steps/Sec: 1.12 +[2025-04-25 12:05:30] (step=0065075) Train Loss: 5.2982, Train Steps/Sec: 1.17 +[2025-04-25 12:05:52] (step=0065100) Train Loss: 5.2524, Train Steps/Sec: 1.12 +[2025-04-25 12:06:14] (step=0065125) Train Loss: 5.2975, Train Steps/Sec: 1.16 +[2025-04-25 12:06:35] (step=0065150) Train Loss: 5.3399, Train Steps/Sec: 1.16 +[2025-04-25 12:06:57] (step=0065175) Train Loss: 5.3151, Train Steps/Sec: 1.12 +[2025-04-25 12:07:19] (step=0065200) Train Loss: 5.3030, Train Steps/Sec: 1.16 +[2025-04-25 12:07:40] (step=0065225) Train Loss: 5.3290, Train Steps/Sec: 1.17 +[2025-04-25 12:08:02] (step=0065250) Train Loss: 5.1906, Train Steps/Sec: 1.17 +[2025-04-25 12:08:23] (step=0065275) Train Loss: 5.2619, Train Steps/Sec: 1.17 +[2025-04-25 12:08:46] (step=0065300) Train Loss: 5.2434, Train Steps/Sec: 1.12 +[2025-04-25 12:09:07] (step=0065325) Train Loss: 5.2541, Train Steps/Sec: 1.17 +[2025-04-25 12:09:28] (step=0065350) Train Loss: 5.2584, Train Steps/Sec: 1.17 +[2025-04-25 12:09:50] (step=0065375) Train Loss: 5.2180, Train Steps/Sec: 1.17 +[2025-04-25 12:10:11] (step=0065400) Train Loss: 5.3258, Train Steps/Sec: 1.16 +[2025-04-25 12:10:33] (step=0065425) Train Loss: 5.3915, Train Steps/Sec: 1.17 +[2025-04-25 12:10:55] (step=0065450) Train Loss: 5.2688, Train Steps/Sec: 1.13 +[2025-04-25 12:11:16] (step=0065475) Train Loss: 5.2849, Train Steps/Sec: 1.17 +[2025-04-25 12:11:39] (step=0065500) Train Loss: 5.2339, Train Steps/Sec: 1.10 +[2025-04-25 12:12:00] (step=0065525) Train Loss: 5.2441, Train Steps/Sec: 1.17 +[2025-04-25 12:12:22] (step=0065550) Train Loss: 5.2555, Train Steps/Sec: 1.13 +[2025-04-25 12:12:44] (step=0065575) Train Loss: 5.2488, Train Steps/Sec: 1.13 +[2025-04-25 12:13:06] (step=0065600) Train Loss: 5.2734, Train Steps/Sec: 1.16 +[2025-04-25 12:13:27] (step=0065625) Train Loss: 5.3257, Train Steps/Sec: 1.17 +[2025-04-25 12:13:49] (step=0065650) Train Loss: 5.2347, Train Steps/Sec: 1.17 +[2025-04-25 12:14:10] (step=0065675) Train Loss: 5.2852, Train Steps/Sec: 1.17 +[2025-04-25 12:14:32] (step=0065700) Train Loss: 5.2503, Train Steps/Sec: 1.12 +[2025-04-25 12:14:54] (step=0065725) Train Loss: 5.2519, Train Steps/Sec: 1.17 +[2025-04-25 12:15:16] (step=0065750) Train Loss: 5.2354, Train Steps/Sec: 1.13 +[2025-04-25 12:15:37] (step=0065775) Train Loss: 5.2931, Train Steps/Sec: 1.17 +[2025-04-25 12:15:59] (step=0065800) Train Loss: 5.2365, Train Steps/Sec: 1.16 +[2025-04-25 12:16:20] (step=0065825) Train Loss: 5.2977, Train Steps/Sec: 1.16 +[2025-04-25 12:16:42] (step=0065850) Train Loss: 5.2538, Train Steps/Sec: 1.17 +[2025-04-25 12:17:03] (step=0065875) Train Loss: 5.3097, Train Steps/Sec: 1.17 +[2025-04-25 12:17:25] (step=0065900) Train Loss: 5.2988, Train Steps/Sec: 1.12 +[2025-04-25 12:17:47] (step=0065925) Train Loss: 5.2806, Train Steps/Sec: 1.17 +[2025-04-25 12:18:08] (step=0065950) Train Loss: 5.2496, Train Steps/Sec: 1.17 +[2025-04-25 12:18:29] (step=0065975) Train Loss: 5.2462, Train Steps/Sec: 1.17 +[2025-04-25 12:18:51] (step=0066000) Train Loss: 5.3236, Train Steps/Sec: 1.16 +[2025-04-25 12:18:51] text_config is None. Initializing the text config with default values (`OPTConfig`). +[2025-04-25 12:23:58] Finish Eval in 66000 steps... +[2025-04-25 12:24:17] Saved checkpoint to checkpoint/Afteri2i_100kckpt_LLAVAT5_empty_Fixed_just_segment_pretrain_stage3_dreambench_recap_400k_t2i_400k_flux_200k_midjourney_150k_recovery_150k_3e4_epoch2_no_rotated/002-GPT-XL/checkpoints/0066000.pt +[2025-04-25 12:24:20] Removed old checkpoint: checkpoint/Afteri2i_100kckpt_LLAVAT5_empty_Fixed_just_segment_pretrain_stage3_dreambench_recap_400k_t2i_400k_flux_200k_midjourney_150k_recovery_150k_3e4_epoch2_no_rotated/002-GPT-XL/checkpoints/0064000.pt +[2025-04-25 12:24:42] (step=0066025) Train Loss: 5.3122, Train Steps/Sec: 0.07 +[2025-04-25 12:25:03] (step=0066050) Train Loss: 5.2722, Train Steps/Sec: 1.17 +[2025-04-25 12:25:25] (step=0066075) Train Loss: 5.3333, Train Steps/Sec: 1.17 +[2025-04-25 12:25:47] (step=0066100) Train Loss: 5.2389, Train Steps/Sec: 1.12 +[2025-04-25 12:26:09] (step=0066125) Train Loss: 5.2791, Train Steps/Sec: 1.17 +[2025-04-25 12:26:30] (step=0066150) Train Loss: 5.2844, Train Steps/Sec: 1.17 +[2025-04-25 12:26:51] (step=0066175) Train Loss: 5.2506, Train Steps/Sec: 1.17 +[2025-04-25 12:27:13] (step=0066200) Train Loss: 5.3416, Train Steps/Sec: 1.12 +[2025-04-25 12:27:36] (step=0066225) Train Loss: 5.2626, Train Steps/Sec: 1.13 +[2025-04-25 12:27:57] (step=0066250) Train Loss: 5.2669, Train Steps/Sec: 1.17 +[2025-04-25 12:28:18] (step=0066275) Train Loss: 5.2152, Train Steps/Sec: 1.17 +[2025-04-25 12:28:40] (step=0066300) Train Loss: 5.3041, Train Steps/Sec: 1.17 +[2025-04-25 12:29:01] (step=0066325) Train Loss: 5.2513, Train Steps/Sec: 1.17 +[2025-04-25 12:29:23] (step=0066350) Train Loss: 5.3461, Train Steps/Sec: 1.13 +[2025-04-25 12:29:45] (step=0066375) Train Loss: 5.2607, Train Steps/Sec: 1.17 +[2025-04-25 12:30:06] (step=0066400) Train Loss: 5.2777, Train Steps/Sec: 1.17 +[2025-04-25 12:30:28] (step=0066425) Train Loss: 5.2717, Train Steps/Sec: 1.13 +[2025-04-25 12:30:50] (step=0066450) Train Loss: 5.3400, Train Steps/Sec: 1.17 +[2025-04-25 12:31:12] (step=0066475) Train Loss: 5.2986, Train Steps/Sec: 1.15 +[2025-04-25 12:31:33] (step=0066500) Train Loss: 5.2846, Train Steps/Sec: 1.17 +[2025-04-25 12:31:54] (step=0066525) Train Loss: 5.3451, Train Steps/Sec: 1.17 +[2025-04-25 12:32:16] (step=0066550) Train Loss: 5.1930, Train Steps/Sec: 1.17 +[2025-04-25 12:32:37] (step=0066575) Train Loss: 5.2591, Train Steps/Sec: 1.17 +[2025-04-25 12:32:59] (step=0066600) Train Loss: 5.2801, Train Steps/Sec: 1.16 +[2025-04-25 12:33:21] (step=0066625) Train Loss: 5.2892, Train Steps/Sec: 1.12 +[2025-04-25 12:33:42] (step=0066650) Train Loss: 5.2893, Train Steps/Sec: 1.17 +[2025-04-25 12:34:04] (step=0066675) Train Loss: 5.2175, Train Steps/Sec: 1.17 +[2025-04-25 12:34:25] (step=0066700) Train Loss: 5.2385, Train Steps/Sec: 1.16 +[2025-04-25 12:34:47] (step=0066725) Train Loss: 5.2278, Train Steps/Sec: 1.17 +[2025-04-25 12:35:10] (step=0066750) Train Loss: 5.2007, Train Steps/Sec: 1.07 +[2025-04-25 12:35:31] (step=0066775) Train Loss: 5.2182, Train Steps/Sec: 1.17 +[2025-04-25 12:35:53] (step=0066800) Train Loss: 5.2558, Train Steps/Sec: 1.17 +[2025-04-25 12:36:14] (step=0066825) Train Loss: 5.2631, Train Steps/Sec: 1.17 +[2025-04-25 12:36:36] (step=0066850) Train Loss: 5.3439, Train Steps/Sec: 1.13 +[2025-04-25 12:36:59] (step=0066875) Train Loss: 5.3521, Train Steps/Sec: 1.13 +[2025-04-25 12:37:20] (step=0066900) Train Loss: 5.2771, Train Steps/Sec: 1.16 +[2025-04-25 12:37:41] (step=0066925) Train Loss: 5.1950, Train Steps/Sec: 1.17 +[2025-04-25 12:38:03] (step=0066950) Train Loss: 5.2399, Train Steps/Sec: 1.17 +[2025-04-25 12:38:24] (step=0066975) Train Loss: 5.2870, Train Steps/Sec: 1.17 +[2025-04-25 12:38:46] (step=0067000) Train Loss: 5.3785, Train Steps/Sec: 1.17 +[2025-04-25 12:39:08] (step=0067025) Train Loss: 5.3322, Train Steps/Sec: 1.13 +[2025-04-25 12:39:29] (step=0067050) Train Loss: 5.2929, Train Steps/Sec: 1.17 +[2025-04-25 12:39:51] (step=0067075) Train Loss: 5.2622, Train Steps/Sec: 1.13 +[2025-04-25 12:40:13] (step=0067100) Train Loss: 5.2364, Train Steps/Sec: 1.17 +[2025-04-25 12:40:34] (step=0067125) Train Loss: 5.3073, Train Steps/Sec: 1.17 +[2025-04-25 12:40:56] (step=0067150) Train Loss: 5.2526, Train Steps/Sec: 1.17 +[2025-04-25 12:41:17] (step=0067175) Train Loss: 5.2587, Train Steps/Sec: 1.17 +[2025-04-25 12:41:38] (step=0067200) Train Loss: 5.2721, Train Steps/Sec: 1.17 +[2025-04-25 12:42:00] (step=0067225) Train Loss: 5.2212, Train Steps/Sec: 1.17 +[2025-04-25 12:42:21] (step=0067250) Train Loss: 5.3060, Train Steps/Sec: 1.17 +[2025-04-25 12:42:43] (step=0067275) Train Loss: 5.3428, Train Steps/Sec: 1.17 +[2025-04-25 12:43:04] (step=0067300) Train Loss: 5.2951, Train Steps/Sec: 1.16 +[2025-04-25 12:43:26] (step=0067325) Train Loss: 5.1986, Train Steps/Sec: 1.16 +[2025-04-25 12:43:48] (step=0067350) Train Loss: 5.3209, Train Steps/Sec: 1.11 +[2025-04-25 12:44:10] (step=0067375) Train Loss: 5.2948, Train Steps/Sec: 1.16 +[2025-04-25 12:44:32] (step=0067400) Train Loss: 5.2959, Train Steps/Sec: 1.12 +[2025-04-25 12:44:53] (step=0067425) Train Loss: 5.2064, Train Steps/Sec: 1.17 +[2025-04-25 12:45:15] (step=0067450) Train Loss: 5.3147, Train Steps/Sec: 1.17 +[2025-04-25 12:45:37] (step=0067475) Train Loss: 5.2386, Train Steps/Sec: 1.12 +[2025-04-25 12:46:00] (step=0067500) Train Loss: 5.2690, Train Steps/Sec: 1.07 +[2025-04-25 12:46:23] (step=0067525) Train Loss: 5.2822, Train Steps/Sec: 1.13 +[2025-04-25 12:46:44] (step=0067550) Train Loss: 5.2633, Train Steps/Sec: 1.17 +[2025-04-25 12:47:05] (step=0067575) Train Loss: 5.3224, Train Steps/Sec: 1.17 +[2025-04-25 12:47:27] (step=0067600) Train Loss: 5.2718, Train Steps/Sec: 1.17 +[2025-04-25 12:47:48] (step=0067625) Train Loss: 5.3447, Train Steps/Sec: 1.17 +[2025-04-25 12:48:10] (step=0067650) Train Loss: 5.2827, Train Steps/Sec: 1.17 +[2025-04-25 12:48:32] (step=0067675) Train Loss: 5.2485, Train Steps/Sec: 1.13 +[2025-04-25 12:48:53] (step=0067700) Train Loss: 5.3151, Train Steps/Sec: 1.17 +[2025-04-25 12:49:15] (step=0067725) Train Loss: 5.2613, Train Steps/Sec: 1.13 +[2025-04-25 12:49:37] (step=0067750) Train Loss: 5.2742, Train Steps/Sec: 1.17 +[2025-04-25 12:49:58] (step=0067775) Train Loss: 5.3248, Train Steps/Sec: 1.17 +[2025-04-25 12:50:20] (step=0067800) Train Loss: 5.2314, Train Steps/Sec: 1.16 +[2025-04-25 12:50:41] (step=0067825) Train Loss: 5.2049, Train Steps/Sec: 1.17 +[2025-04-25 12:51:02] (step=0067850) Train Loss: 5.3367, Train Steps/Sec: 1.17 +[2025-04-25 12:51:24] (step=0067875) Train Loss: 5.2525, Train Steps/Sec: 1.17 +[2025-04-25 12:51:45] (step=0067900) Train Loss: 5.2579, Train Steps/Sec: 1.17 +[2025-04-25 12:52:07] (step=0067925) Train Loss: 5.2871, Train Steps/Sec: 1.17 +[2025-04-25 12:52:28] (step=0067950) Train Loss: 5.2530, Train Steps/Sec: 1.17 +[2025-04-25 12:52:49] (step=0067975) Train Loss: 5.2696, Train Steps/Sec: 1.17 +[2025-04-25 12:53:11] (step=0068000) Train Loss: 5.3478, Train Steps/Sec: 1.16 +[2025-04-25 12:53:11] text_config is None. Initializing the text config with default values (`OPTConfig`). +[2025-04-25 12:58:15] Finish Eval in 68000 steps... +[2025-04-25 12:58:35] Saved checkpoint to checkpoint/Afteri2i_100kckpt_LLAVAT5_empty_Fixed_just_segment_pretrain_stage3_dreambench_recap_400k_t2i_400k_flux_200k_midjourney_150k_recovery_150k_3e4_epoch2_no_rotated/002-GPT-XL/checkpoints/0068000.pt +[2025-04-25 12:58:37] Removed old checkpoint: checkpoint/Afteri2i_100kckpt_LLAVAT5_empty_Fixed_just_segment_pretrain_stage3_dreambench_recap_400k_t2i_400k_flux_200k_midjourney_150k_recovery_150k_3e4_epoch2_no_rotated/002-GPT-XL/checkpoints/0066000.pt +[2025-04-25 12:58:59] (step=0068025) Train Loss: 5.3284, Train Steps/Sec: 0.07 +[2025-04-25 12:59:21] (step=0068050) Train Loss: 5.2549, Train Steps/Sec: 1.13 +[2025-04-25 12:59:43] (step=0068075) Train Loss: 5.3152, Train Steps/Sec: 1.13 +[2025-04-25 13:00:04] (step=0068100) Train Loss: 5.2639, Train Steps/Sec: 1.17 +[2025-04-25 13:00:26] (step=0068125) Train Loss: 5.3326, Train Steps/Sec: 1.17 +[2025-04-25 13:00:48] (step=0068150) Train Loss: 5.2317, Train Steps/Sec: 1.12 +[2025-04-25 13:01:09] (step=0068175) Train Loss: 5.2620, Train Steps/Sec: 1.17 +[2025-04-25 13:01:32] (step=0068200) Train Loss: 5.2920, Train Steps/Sec: 1.12 +[2025-04-25 13:01:54] (step=0068225) Train Loss: 5.2847, Train Steps/Sec: 1.11 +[2025-04-25 13:02:15] (step=0068250) Train Loss: 5.2895, Train Steps/Sec: 1.17 +[2025-04-25 13:02:37] (step=0068275) Train Loss: 5.2453, Train Steps/Sec: 1.17 +[2025-04-25 13:02:58] (step=0068300) Train Loss: 5.2884, Train Steps/Sec: 1.17 +[2025-04-25 13:03:20] (step=0068325) Train Loss: 5.2455, Train Steps/Sec: 1.17 +[2025-04-25 13:03:42] (step=0068350) Train Loss: 5.2737, Train Steps/Sec: 1.13 +[2025-04-25 13:04:03] (step=0068375) Train Loss: 5.2648, Train Steps/Sec: 1.17 +[2025-04-25 13:04:26] (step=0068400) Train Loss: 5.2624, Train Steps/Sec: 1.10 +[2025-04-25 13:04:47] (step=0068425) Train Loss: 5.2105, Train Steps/Sec: 1.17 +[2025-04-25 13:05:09] (step=0068450) Train Loss: 5.2519, Train Steps/Sec: 1.17 +[2025-04-25 13:05:30] (step=0068475) Train Loss: 5.2855, Train Steps/Sec: 1.17 +[2025-04-25 13:05:51] (step=0068500) Train Loss: 5.2132, Train Steps/Sec: 1.16 +[2025-04-25 13:06:13] (step=0068525) Train Loss: 5.3203, Train Steps/Sec: 1.17 +[2025-04-25 13:06:34] (step=0068550) Train Loss: 5.2444, Train Steps/Sec: 1.17 +[2025-04-25 13:06:56] (step=0068575) Train Loss: 5.2654, Train Steps/Sec: 1.17 +[2025-04-25 13:07:17] (step=0068600) Train Loss: 5.2516, Train Steps/Sec: 1.16 +[2025-04-25 13:07:38] (step=0068625) Train Loss: 5.2787, Train Steps/Sec: 1.17 +[2025-04-25 13:08:00] (step=0068650) Train Loss: 5.2895, Train Steps/Sec: 1.17 +[2025-04-25 13:08:21] (step=0068675) Train Loss: 5.2549, Train Steps/Sec: 1.17 +[2025-04-25 13:08:44] (step=0068700) Train Loss: 5.1919, Train Steps/Sec: 1.11 +[2025-04-25 13:09:05] (step=0068725) Train Loss: 5.2938, Train Steps/Sec: 1.17 +[2025-04-25 13:09:27] (step=0068750) Train Loss: 5.2770, Train Steps/Sec: 1.12 +[2025-04-25 13:09:49] (step=0068775) Train Loss: 5.1915, Train Steps/Sec: 1.17 +[2025-04-25 13:10:11] (step=0068800) Train Loss: 5.2342, Train Steps/Sec: 1.12 +[2025-04-25 13:10:33] (step=0068825) Train Loss: 5.2811, Train Steps/Sec: 1.13 +[2025-04-25 13:10:55] (step=0068850) Train Loss: 5.2294, Train Steps/Sec: 1.13 +[2025-04-25 13:11:17] (step=0068875) Train Loss: 5.3303, Train Steps/Sec: 1.17 +[2025-04-25 13:11:38] (step=0068900) Train Loss: 5.2596, Train Steps/Sec: 1.17 +[2025-04-25 13:12:00] (step=0068925) Train Loss: 5.2830, Train Steps/Sec: 1.17 +[2025-04-25 13:12:22] (step=0068950) Train Loss: 5.2907, Train Steps/Sec: 1.12 +[2025-04-25 13:12:43] (step=0068975) Train Loss: 5.3233, Train Steps/Sec: 1.17 +[2025-04-25 13:13:06] (step=0069000) Train Loss: 5.2842, Train Steps/Sec: 1.12 +[2025-04-25 13:13:27] (step=0069025) Train Loss: 5.2869, Train Steps/Sec: 1.17 +[2025-04-25 13:13:49] (step=0069050) Train Loss: 5.2529, Train Steps/Sec: 1.12 +[2025-04-25 13:14:11] (step=0069075) Train Loss: 5.3417, Train Steps/Sec: 1.17 +[2025-04-25 13:14:32] (step=0069100) Train Loss: 5.2065, Train Steps/Sec: 1.15 +[2025-04-25 13:14:54] (step=0069125) Train Loss: 5.3106, Train Steps/Sec: 1.17 +[2025-04-25 13:15:15] (step=0069150) Train Loss: 5.2046, Train Steps/Sec: 1.17 +[2025-04-25 13:15:37] (step=0069175) Train Loss: 5.2311, Train Steps/Sec: 1.17 +[2025-04-25 13:15:58] (step=0069200) Train Loss: 5.2488, Train Steps/Sec: 1.15 +[2025-04-25 13:16:20] (step=0069225) Train Loss: 5.2498, Train Steps/Sec: 1.17 +[2025-04-25 13:16:41] (step=0069250) Train Loss: 5.3456, Train Steps/Sec: 1.17 +[2025-04-25 13:17:02] (step=0069275) Train Loss: 5.3414, Train Steps/Sec: 1.17 +[2025-04-25 13:17:24] (step=0069300) Train Loss: 5.2328, Train Steps/Sec: 1.17 +[2025-04-25 13:17:45] (step=0069325) Train Loss: 5.2631, Train Steps/Sec: 1.17 +[2025-04-25 13:18:08] (step=0069350) Train Loss: 5.2852, Train Steps/Sec: 1.11 +[2025-04-25 13:18:29] (step=0069375) Train Loss: 5.2896, Train Steps/Sec: 1.16 +[2025-04-25 13:18:51] (step=0069400) Train Loss: 5.2807, Train Steps/Sec: 1.16 +[2025-04-25 13:19:12] (step=0069425) Train Loss: 5.2748, Train Steps/Sec: 1.17 +[2025-04-25 13:19:34] (step=0069450) Train Loss: 5.2175, Train Steps/Sec: 1.16 +[2025-04-25 13:19:56] (step=0069475) Train Loss: 5.2765, Train Steps/Sec: 1.12 +[2025-04-25 13:20:19] (step=0069500) Train Loss: 5.3443, Train Steps/Sec: 1.07 +[2025-04-25 13:20:41] (step=0069525) Train Loss: 5.3521, Train Steps/Sec: 1.17 +[2025-04-25 13:21:03] (step=0069550) Train Loss: 5.2703, Train Steps/Sec: 1.12 +[2025-04-25 13:21:24] (step=0069575) Train Loss: 5.3039, Train Steps/Sec: 1.17 +[2025-04-25 13:21:46] (step=0069600) Train Loss: 5.2823, Train Steps/Sec: 1.17 +[2025-04-25 13:22:07] (step=0069625) Train Loss: 5.2650, Train Steps/Sec: 1.17 +[2025-04-25 13:22:29] (step=0069650) Train Loss: 5.3182, Train Steps/Sec: 1.13 +[2025-04-25 13:22:52] (step=0069675) Train Loss: 5.1984, Train Steps/Sec: 1.12 +[2025-04-25 13:23:13] (step=0069700) Train Loss: 5.2053, Train Steps/Sec: 1.16 +[2025-04-25 13:23:35] (step=0069725) Train Loss: 5.2719, Train Steps/Sec: 1.13 +[2025-04-25 13:23:57] (step=0069750) Train Loss: 5.2552, Train Steps/Sec: 1.17 +[2025-04-25 13:24:18] (step=0069775) Train Loss: 5.3546, Train Steps/Sec: 1.17 +[2025-04-25 13:24:40] (step=0069800) Train Loss: 5.2663, Train Steps/Sec: 1.17 +[2025-04-25 13:25:01] (step=0069825) Train Loss: 5.3270, Train Steps/Sec: 1.16 +[2025-04-25 13:25:22] (step=0069850) Train Loss: 5.3055, Train Steps/Sec: 1.17 +[2025-04-25 13:25:44] (step=0069875) Train Loss: 5.2550, Train Steps/Sec: 1.17 +[2025-04-25 13:26:05] (step=0069900) Train Loss: 5.2711, Train Steps/Sec: 1.16 +[2025-04-25 13:26:27] (step=0069925) Train Loss: 5.2078, Train Steps/Sec: 1.17 +[2025-04-25 13:26:48] (step=0069950) Train Loss: 5.2604, Train Steps/Sec: 1.17 +[2025-04-25 13:27:09] (step=0069975) Train Loss: 5.1566, Train Steps/Sec: 1.17 +[2025-04-25 13:27:32] (step=0070000) Train Loss: 5.2783, Train Steps/Sec: 1.12 +[2025-04-25 13:27:32] text_config is None. Initializing the text config with default values (`OPTConfig`). +[2025-04-25 13:32:38] Finish Eval in 70000 steps... +[2025-04-25 13:32:58] Saved checkpoint to checkpoint/Afteri2i_100kckpt_LLAVAT5_empty_Fixed_just_segment_pretrain_stage3_dreambench_recap_400k_t2i_400k_flux_200k_midjourney_150k_recovery_150k_3e4_epoch2_no_rotated/002-GPT-XL/checkpoints/0070000.pt +[2025-04-25 13:33:00] Removed old checkpoint: checkpoint/Afteri2i_100kckpt_LLAVAT5_empty_Fixed_just_segment_pretrain_stage3_dreambench_recap_400k_t2i_400k_flux_200k_midjourney_150k_recovery_150k_3e4_epoch2_no_rotated/002-GPT-XL/checkpoints/0068000.pt +[2025-04-25 13:33:21] (step=0070025) Train Loss: 5.3229, Train Steps/Sec: 0.07 +[2025-04-25 13:33:42] (step=0070050) Train Loss: 5.2708, Train Steps/Sec: 1.17 +[2025-04-25 13:34:04] (step=0070075) Train Loss: 5.3289, Train Steps/Sec: 1.17 +[2025-04-25 13:34:25] (step=0070100) Train Loss: 5.3080, Train Steps/Sec: 1.16 +[2025-04-25 13:34:47] (step=0070125) Train Loss: 5.1834, Train Steps/Sec: 1.12 +[2025-04-25 13:35:10] (step=0070150) Train Loss: 5.2757, Train Steps/Sec: 1.13 +[2025-04-25 13:35:31] (step=0070175) Train Loss: 5.2015, Train Steps/Sec: 1.17 +[2025-04-25 13:35:52] (step=0070200) Train Loss: 5.2483, Train Steps/Sec: 1.17 +[2025-04-25 13:36:14] (step=0070225) Train Loss: 5.1853, Train Steps/Sec: 1.17 +[2025-04-25 13:36:35] (step=0070250) Train Loss: 5.2452, Train Steps/Sec: 1.17 +[2025-04-25 13:36:57] (step=0070275) Train Loss: 5.2581, Train Steps/Sec: 1.13 +[2025-04-25 13:37:19] (step=0070300) Train Loss: 5.2202, Train Steps/Sec: 1.16 +[2025-04-25 13:37:42] (step=0070325) Train Loss: 5.2565, Train Steps/Sec: 1.10 +[2025-04-25 13:38:03] (step=0070350) Train Loss: 5.3007, Train Steps/Sec: 1.16 +[2025-04-25 13:38:25] Done! diff --git a/CKPTS/Afteri2i_100kckpt_LLAVAT5_empty_Fixed_just_segment_pretrain_stage3_dreambench_recap_400k_t2i_400k_flux_200k_midjourney_150k_recovery_150k_3e4_epoch2_no_rotated/003-GPT-XL/checkpoints/0092000.pt b/CKPTS/Afteri2i_100kckpt_LLAVAT5_empty_Fixed_just_segment_pretrain_stage3_dreambench_recap_400k_t2i_400k_flux_200k_midjourney_150k_recovery_150k_3e4_epoch2_no_rotated/003-GPT-XL/checkpoints/0092000.pt new file mode 100644 index 0000000000000000000000000000000000000000..d20562ef54a981944c20d4f8cc1cadd0e10cf32a --- /dev/null +++ b/CKPTS/Afteri2i_100kckpt_LLAVAT5_empty_Fixed_just_segment_pretrain_stage3_dreambench_recap_400k_t2i_400k_flux_200k_midjourney_150k_recovery_150k_3e4_epoch2_no_rotated/003-GPT-XL/checkpoints/0092000.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:d05732192794cddbc46ecc8de6149b4f25b98b64db548f713c94ddb680308ac9 +size 17322479726 diff --git a/CKPTS/Afteri2i_100kckpt_LLAVAT5_empty_Fixed_just_segment_pretrain_stage3_dreambench_recap_400k_t2i_400k_flux_200k_midjourney_150k_recovery_150k_3e4_epoch2_no_rotated/003-GPT-XL/checkpoints/eval_step_72000/batch_0_cfg_7.5_topk_16384.jpg b/CKPTS/Afteri2i_100kckpt_LLAVAT5_empty_Fixed_just_segment_pretrain_stage3_dreambench_recap_400k_t2i_400k_flux_200k_midjourney_150k_recovery_150k_3e4_epoch2_no_rotated/003-GPT-XL/checkpoints/eval_step_72000/batch_0_cfg_7.5_topk_16384.jpg new file mode 100644 index 0000000000000000000000000000000000000000..e20426542d8124f28b90a440a1eada01b318ef65 --- /dev/null +++ b/CKPTS/Afteri2i_100kckpt_LLAVAT5_empty_Fixed_just_segment_pretrain_stage3_dreambench_recap_400k_t2i_400k_flux_200k_midjourney_150k_recovery_150k_3e4_epoch2_no_rotated/003-GPT-XL/checkpoints/eval_step_72000/batch_0_cfg_7.5_topk_16384.jpg @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:091bfdff5dc10e58affe93ddd4c6d84596a91cd2ebfc1b242f6a8013bf8c2615 +size 740380 diff --git a/CKPTS/Afteri2i_100kckpt_LLAVAT5_empty_Fixed_just_segment_pretrain_stage3_dreambench_recap_400k_t2i_400k_flux_200k_midjourney_150k_recovery_150k_3e4_epoch2_no_rotated/003-GPT-XL/checkpoints/eval_step_72000/batch_1_cfg_7.5_topk_16384.jpg b/CKPTS/Afteri2i_100kckpt_LLAVAT5_empty_Fixed_just_segment_pretrain_stage3_dreambench_recap_400k_t2i_400k_flux_200k_midjourney_150k_recovery_150k_3e4_epoch2_no_rotated/003-GPT-XL/checkpoints/eval_step_72000/batch_1_cfg_7.5_topk_16384.jpg new file mode 100644 index 0000000000000000000000000000000000000000..c5eb4ebb1e7725bc25bbbbe8a81c184414d49209 --- /dev/null +++ b/CKPTS/Afteri2i_100kckpt_LLAVAT5_empty_Fixed_just_segment_pretrain_stage3_dreambench_recap_400k_t2i_400k_flux_200k_midjourney_150k_recovery_150k_3e4_epoch2_no_rotated/003-GPT-XL/checkpoints/eval_step_72000/batch_1_cfg_7.5_topk_16384.jpg @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:b1f5c610919d205d375aedddc77103ea87b1bfceb858af1d927a7838be262c36 +size 715945 diff --git a/CKPTS/Afteri2i_100kckpt_LLAVAT5_empty_Fixed_just_segment_pretrain_stage3_dreambench_recap_400k_t2i_400k_flux_200k_midjourney_150k_recovery_150k_3e4_epoch2_no_rotated/003-GPT-XL/checkpoints/eval_step_72000/batch_2_cfg_7.5_topk_16384.jpg b/CKPTS/Afteri2i_100kckpt_LLAVAT5_empty_Fixed_just_segment_pretrain_stage3_dreambench_recap_400k_t2i_400k_flux_200k_midjourney_150k_recovery_150k_3e4_epoch2_no_rotated/003-GPT-XL/checkpoints/eval_step_72000/batch_2_cfg_7.5_topk_16384.jpg new file mode 100644 index 0000000000000000000000000000000000000000..81f80d252c508cc56717cd128e7370987a5d3685 --- /dev/null +++ b/CKPTS/Afteri2i_100kckpt_LLAVAT5_empty_Fixed_just_segment_pretrain_stage3_dreambench_recap_400k_t2i_400k_flux_200k_midjourney_150k_recovery_150k_3e4_epoch2_no_rotated/003-GPT-XL/checkpoints/eval_step_72000/batch_2_cfg_7.5_topk_16384.jpg @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:784a36a884ff32dda3fe9d4a893eb33cb3ca220e8a9e8932afa3acba233cd8c2 +size 718984 diff --git a/CKPTS/Afteri2i_100kckpt_LLAVAT5_empty_Fixed_just_segment_pretrain_stage3_dreambench_recap_400k_t2i_400k_flux_200k_midjourney_150k_recovery_150k_3e4_epoch2_no_rotated/003-GPT-XL/checkpoints/eval_step_72000/batch_3_cfg_7.5_topk_16384.jpg b/CKPTS/Afteri2i_100kckpt_LLAVAT5_empty_Fixed_just_segment_pretrain_stage3_dreambench_recap_400k_t2i_400k_flux_200k_midjourney_150k_recovery_150k_3e4_epoch2_no_rotated/003-GPT-XL/checkpoints/eval_step_72000/batch_3_cfg_7.5_topk_16384.jpg new file mode 100644 index 0000000000000000000000000000000000000000..6c4afa463d011d2ffde0cf4ed15ad6f375c129f4 --- /dev/null +++ b/CKPTS/Afteri2i_100kckpt_LLAVAT5_empty_Fixed_just_segment_pretrain_stage3_dreambench_recap_400k_t2i_400k_flux_200k_midjourney_150k_recovery_150k_3e4_epoch2_no_rotated/003-GPT-XL/checkpoints/eval_step_72000/batch_3_cfg_7.5_topk_16384.jpg @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:063d93e8102807ec9624f313def1cfca9b0774f2b76bccf26beee56e31eeadd8 +size 889037 diff --git a/CKPTS/Afteri2i_100kckpt_LLAVAT5_empty_Fixed_just_segment_pretrain_stage3_dreambench_recap_400k_t2i_400k_flux_200k_midjourney_150k_recovery_150k_3e4_epoch2_no_rotated/003-GPT-XL/checkpoints/eval_step_74000/batch_0_cfg_7.5_topk_16384.jpg b/CKPTS/Afteri2i_100kckpt_LLAVAT5_empty_Fixed_just_segment_pretrain_stage3_dreambench_recap_400k_t2i_400k_flux_200k_midjourney_150k_recovery_150k_3e4_epoch2_no_rotated/003-GPT-XL/checkpoints/eval_step_74000/batch_0_cfg_7.5_topk_16384.jpg new file mode 100644 index 0000000000000000000000000000000000000000..e03bcb8d337b4131e9e54c03ed56e1da80baa519 --- /dev/null +++ b/CKPTS/Afteri2i_100kckpt_LLAVAT5_empty_Fixed_just_segment_pretrain_stage3_dreambench_recap_400k_t2i_400k_flux_200k_midjourney_150k_recovery_150k_3e4_epoch2_no_rotated/003-GPT-XL/checkpoints/eval_step_74000/batch_0_cfg_7.5_topk_16384.jpg @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:8e4997ae58a138d9e5ae53a8bf16696031fa09979a4271485c9d1c7e1a27062f +size 735303 diff --git a/CKPTS/Afteri2i_100kckpt_LLAVAT5_empty_Fixed_just_segment_pretrain_stage3_dreambench_recap_400k_t2i_400k_flux_200k_midjourney_150k_recovery_150k_3e4_epoch2_no_rotated/003-GPT-XL/checkpoints/eval_step_74000/batch_1_cfg_7.5_topk_16384.jpg b/CKPTS/Afteri2i_100kckpt_LLAVAT5_empty_Fixed_just_segment_pretrain_stage3_dreambench_recap_400k_t2i_400k_flux_200k_midjourney_150k_recovery_150k_3e4_epoch2_no_rotated/003-GPT-XL/checkpoints/eval_step_74000/batch_1_cfg_7.5_topk_16384.jpg new file mode 100644 index 0000000000000000000000000000000000000000..f6930d31745236a43013ed5f7bbd7461950534f7 --- /dev/null +++ b/CKPTS/Afteri2i_100kckpt_LLAVAT5_empty_Fixed_just_segment_pretrain_stage3_dreambench_recap_400k_t2i_400k_flux_200k_midjourney_150k_recovery_150k_3e4_epoch2_no_rotated/003-GPT-XL/checkpoints/eval_step_74000/batch_1_cfg_7.5_topk_16384.jpg @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:4f4abebb9e1f159e3cd9df8ca97c5f3bff8898f5625dd9e69706f245581513b6 +size 704466 diff --git a/CKPTS/Afteri2i_100kckpt_LLAVAT5_empty_Fixed_just_segment_pretrain_stage3_dreambench_recap_400k_t2i_400k_flux_200k_midjourney_150k_recovery_150k_3e4_epoch2_no_rotated/003-GPT-XL/checkpoints/eval_step_74000/batch_2_cfg_7.5_topk_16384.jpg b/CKPTS/Afteri2i_100kckpt_LLAVAT5_empty_Fixed_just_segment_pretrain_stage3_dreambench_recap_400k_t2i_400k_flux_200k_midjourney_150k_recovery_150k_3e4_epoch2_no_rotated/003-GPT-XL/checkpoints/eval_step_74000/batch_2_cfg_7.5_topk_16384.jpg new file mode 100644 index 0000000000000000000000000000000000000000..5c32c8a10bd0356f0182d3b43bbc4d00ff0388f1 --- /dev/null +++ b/CKPTS/Afteri2i_100kckpt_LLAVAT5_empty_Fixed_just_segment_pretrain_stage3_dreambench_recap_400k_t2i_400k_flux_200k_midjourney_150k_recovery_150k_3e4_epoch2_no_rotated/003-GPT-XL/checkpoints/eval_step_74000/batch_2_cfg_7.5_topk_16384.jpg @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:66e07fa9b12cce592426ee20ed161757c8b2675ce35ad0d6213befe0393f41b5 +size 723967 diff --git a/CKPTS/Afteri2i_100kckpt_LLAVAT5_empty_Fixed_just_segment_pretrain_stage3_dreambench_recap_400k_t2i_400k_flux_200k_midjourney_150k_recovery_150k_3e4_epoch2_no_rotated/003-GPT-XL/checkpoints/eval_step_74000/batch_3_cfg_7.5_topk_16384.jpg b/CKPTS/Afteri2i_100kckpt_LLAVAT5_empty_Fixed_just_segment_pretrain_stage3_dreambench_recap_400k_t2i_400k_flux_200k_midjourney_150k_recovery_150k_3e4_epoch2_no_rotated/003-GPT-XL/checkpoints/eval_step_74000/batch_3_cfg_7.5_topk_16384.jpg new file mode 100644 index 0000000000000000000000000000000000000000..cb32c587cc7b4c183baf9e416f83c9c9c827a844 --- /dev/null +++ b/CKPTS/Afteri2i_100kckpt_LLAVAT5_empty_Fixed_just_segment_pretrain_stage3_dreambench_recap_400k_t2i_400k_flux_200k_midjourney_150k_recovery_150k_3e4_epoch2_no_rotated/003-GPT-XL/checkpoints/eval_step_74000/batch_3_cfg_7.5_topk_16384.jpg @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:6a5678873af7100ef7604988dc256427a21a652587d36f51ada54a4eaa43272a +size 876762 diff --git a/CKPTS/Afteri2i_100kckpt_LLAVAT5_empty_Fixed_just_segment_pretrain_stage3_dreambench_recap_400k_t2i_400k_flux_200k_midjourney_150k_recovery_150k_3e4_epoch2_no_rotated/003-GPT-XL/checkpoints/eval_step_76000/batch_0_cfg_7.5_topk_16384.jpg b/CKPTS/Afteri2i_100kckpt_LLAVAT5_empty_Fixed_just_segment_pretrain_stage3_dreambench_recap_400k_t2i_400k_flux_200k_midjourney_150k_recovery_150k_3e4_epoch2_no_rotated/003-GPT-XL/checkpoints/eval_step_76000/batch_0_cfg_7.5_topk_16384.jpg new file mode 100644 index 0000000000000000000000000000000000000000..5e59d020b93d26c972e73f6744ec337244aca12f --- /dev/null +++ b/CKPTS/Afteri2i_100kckpt_LLAVAT5_empty_Fixed_just_segment_pretrain_stage3_dreambench_recap_400k_t2i_400k_flux_200k_midjourney_150k_recovery_150k_3e4_epoch2_no_rotated/003-GPT-XL/checkpoints/eval_step_76000/batch_0_cfg_7.5_topk_16384.jpg @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:ca31cc3ce3a34d0b01780722ca7b5231cb248af96e38de86e6cd3bb516df2393 +size 736794 diff --git a/CKPTS/Afteri2i_100kckpt_LLAVAT5_empty_Fixed_just_segment_pretrain_stage3_dreambench_recap_400k_t2i_400k_flux_200k_midjourney_150k_recovery_150k_3e4_epoch2_no_rotated/003-GPT-XL/checkpoints/eval_step_76000/batch_1_cfg_7.5_topk_16384.jpg b/CKPTS/Afteri2i_100kckpt_LLAVAT5_empty_Fixed_just_segment_pretrain_stage3_dreambench_recap_400k_t2i_400k_flux_200k_midjourney_150k_recovery_150k_3e4_epoch2_no_rotated/003-GPT-XL/checkpoints/eval_step_76000/batch_1_cfg_7.5_topk_16384.jpg new file mode 100644 index 0000000000000000000000000000000000000000..b785c0ebf65274f6c8b619257ea77feb47db01aa --- /dev/null +++ b/CKPTS/Afteri2i_100kckpt_LLAVAT5_empty_Fixed_just_segment_pretrain_stage3_dreambench_recap_400k_t2i_400k_flux_200k_midjourney_150k_recovery_150k_3e4_epoch2_no_rotated/003-GPT-XL/checkpoints/eval_step_76000/batch_1_cfg_7.5_topk_16384.jpg @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:e2e99ceba823354087ef5cc21ee8c04b6d418fa5606eaf38170be28cdc4b0ab5 +size 721733 diff --git a/CKPTS/Afteri2i_100kckpt_LLAVAT5_empty_Fixed_just_segment_pretrain_stage3_dreambench_recap_400k_t2i_400k_flux_200k_midjourney_150k_recovery_150k_3e4_epoch2_no_rotated/003-GPT-XL/checkpoints/eval_step_76000/batch_2_cfg_7.5_topk_16384.jpg b/CKPTS/Afteri2i_100kckpt_LLAVAT5_empty_Fixed_just_segment_pretrain_stage3_dreambench_recap_400k_t2i_400k_flux_200k_midjourney_150k_recovery_150k_3e4_epoch2_no_rotated/003-GPT-XL/checkpoints/eval_step_76000/batch_2_cfg_7.5_topk_16384.jpg new file mode 100644 index 0000000000000000000000000000000000000000..f9924efd8511310746ba0f4bbc20b581bb3b006f --- /dev/null +++ b/CKPTS/Afteri2i_100kckpt_LLAVAT5_empty_Fixed_just_segment_pretrain_stage3_dreambench_recap_400k_t2i_400k_flux_200k_midjourney_150k_recovery_150k_3e4_epoch2_no_rotated/003-GPT-XL/checkpoints/eval_step_76000/batch_2_cfg_7.5_topk_16384.jpg @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:05f75371eed973a7de862c258ae78fbc1b5b0fb29ffe77e643506e3462cfb39b +size 733754 diff --git a/CKPTS/Afteri2i_100kckpt_LLAVAT5_empty_Fixed_just_segment_pretrain_stage3_dreambench_recap_400k_t2i_400k_flux_200k_midjourney_150k_recovery_150k_3e4_epoch2_no_rotated/003-GPT-XL/checkpoints/eval_step_76000/batch_3_cfg_7.5_topk_16384.jpg b/CKPTS/Afteri2i_100kckpt_LLAVAT5_empty_Fixed_just_segment_pretrain_stage3_dreambench_recap_400k_t2i_400k_flux_200k_midjourney_150k_recovery_150k_3e4_epoch2_no_rotated/003-GPT-XL/checkpoints/eval_step_76000/batch_3_cfg_7.5_topk_16384.jpg new file mode 100644 index 0000000000000000000000000000000000000000..b6cc5cc18c7e9c5ebfa34e8efdf817a510030373 --- /dev/null +++ b/CKPTS/Afteri2i_100kckpt_LLAVAT5_empty_Fixed_just_segment_pretrain_stage3_dreambench_recap_400k_t2i_400k_flux_200k_midjourney_150k_recovery_150k_3e4_epoch2_no_rotated/003-GPT-XL/checkpoints/eval_step_76000/batch_3_cfg_7.5_topk_16384.jpg @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:f85c638e6be5ffd5d4967dca8ab182873905f3785b503e6ff5b497f74a782226 +size 869267 diff --git a/CKPTS/Afteri2i_100kckpt_LLAVAT5_empty_Fixed_just_segment_pretrain_stage3_dreambench_recap_400k_t2i_400k_flux_200k_midjourney_150k_recovery_150k_3e4_epoch2_no_rotated/003-GPT-XL/checkpoints/eval_step_78000/batch_0_cfg_7.5_topk_16384.jpg b/CKPTS/Afteri2i_100kckpt_LLAVAT5_empty_Fixed_just_segment_pretrain_stage3_dreambench_recap_400k_t2i_400k_flux_200k_midjourney_150k_recovery_150k_3e4_epoch2_no_rotated/003-GPT-XL/checkpoints/eval_step_78000/batch_0_cfg_7.5_topk_16384.jpg new file mode 100644 index 0000000000000000000000000000000000000000..7a0dd24a45f078579ab6a63de4a38d849b3a7bc0 --- /dev/null +++ b/CKPTS/Afteri2i_100kckpt_LLAVAT5_empty_Fixed_just_segment_pretrain_stage3_dreambench_recap_400k_t2i_400k_flux_200k_midjourney_150k_recovery_150k_3e4_epoch2_no_rotated/003-GPT-XL/checkpoints/eval_step_78000/batch_0_cfg_7.5_topk_16384.jpg @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:bf9cee493adf851d9d365875148d16193ceb67af0eb6654f7296841d68a84115 +size 729050 diff --git a/CKPTS/Afteri2i_100kckpt_LLAVAT5_empty_Fixed_just_segment_pretrain_stage3_dreambench_recap_400k_t2i_400k_flux_200k_midjourney_150k_recovery_150k_3e4_epoch2_no_rotated/003-GPT-XL/checkpoints/eval_step_78000/batch_1_cfg_7.5_topk_16384.jpg b/CKPTS/Afteri2i_100kckpt_LLAVAT5_empty_Fixed_just_segment_pretrain_stage3_dreambench_recap_400k_t2i_400k_flux_200k_midjourney_150k_recovery_150k_3e4_epoch2_no_rotated/003-GPT-XL/checkpoints/eval_step_78000/batch_1_cfg_7.5_topk_16384.jpg new file mode 100644 index 0000000000000000000000000000000000000000..1a1adc17e18c2b6eb834b3180c2d9b5ae47b7f9a --- /dev/null +++ b/CKPTS/Afteri2i_100kckpt_LLAVAT5_empty_Fixed_just_segment_pretrain_stage3_dreambench_recap_400k_t2i_400k_flux_200k_midjourney_150k_recovery_150k_3e4_epoch2_no_rotated/003-GPT-XL/checkpoints/eval_step_78000/batch_1_cfg_7.5_topk_16384.jpg @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:f8d485db8ccd2f3f9b040047188f8339b83fe17add1e857d02f98d97e6169ef3 +size 709459 diff --git a/CKPTS/Afteri2i_100kckpt_LLAVAT5_empty_Fixed_just_segment_pretrain_stage3_dreambench_recap_400k_t2i_400k_flux_200k_midjourney_150k_recovery_150k_3e4_epoch2_no_rotated/003-GPT-XL/checkpoints/eval_step_78000/batch_2_cfg_7.5_topk_16384.jpg b/CKPTS/Afteri2i_100kckpt_LLAVAT5_empty_Fixed_just_segment_pretrain_stage3_dreambench_recap_400k_t2i_400k_flux_200k_midjourney_150k_recovery_150k_3e4_epoch2_no_rotated/003-GPT-XL/checkpoints/eval_step_78000/batch_2_cfg_7.5_topk_16384.jpg new file mode 100644 index 0000000000000000000000000000000000000000..9389cb1db604e7876f6d7c4c470210fd5d2b65a2 --- /dev/null +++ b/CKPTS/Afteri2i_100kckpt_LLAVAT5_empty_Fixed_just_segment_pretrain_stage3_dreambench_recap_400k_t2i_400k_flux_200k_midjourney_150k_recovery_150k_3e4_epoch2_no_rotated/003-GPT-XL/checkpoints/eval_step_78000/batch_2_cfg_7.5_topk_16384.jpg @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:1f2666948460c59ea5939612a98a7ded2e2bbb5e6946bf36ff6221215d9cb83b +size 718235 diff --git a/CKPTS/Afteri2i_100kckpt_LLAVAT5_empty_Fixed_just_segment_pretrain_stage3_dreambench_recap_400k_t2i_400k_flux_200k_midjourney_150k_recovery_150k_3e4_epoch2_no_rotated/003-GPT-XL/checkpoints/eval_step_78000/batch_3_cfg_7.5_topk_16384.jpg b/CKPTS/Afteri2i_100kckpt_LLAVAT5_empty_Fixed_just_segment_pretrain_stage3_dreambench_recap_400k_t2i_400k_flux_200k_midjourney_150k_recovery_150k_3e4_epoch2_no_rotated/003-GPT-XL/checkpoints/eval_step_78000/batch_3_cfg_7.5_topk_16384.jpg new file mode 100644 index 0000000000000000000000000000000000000000..29834d99ffa00f897af6b9e3106120c49075440b --- /dev/null +++ b/CKPTS/Afteri2i_100kckpt_LLAVAT5_empty_Fixed_just_segment_pretrain_stage3_dreambench_recap_400k_t2i_400k_flux_200k_midjourney_150k_recovery_150k_3e4_epoch2_no_rotated/003-GPT-XL/checkpoints/eval_step_78000/batch_3_cfg_7.5_topk_16384.jpg @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:d67cf4d2d2015faca92ebb4a4a48679a00233d03fe52b4ee5e7b884fb1889c0c +size 877637 diff --git a/CKPTS/Afteri2i_100kckpt_LLAVAT5_empty_Fixed_just_segment_pretrain_stage3_dreambench_recap_400k_t2i_400k_flux_200k_midjourney_150k_recovery_150k_3e4_epoch2_no_rotated/003-GPT-XL/checkpoints/eval_step_80000/batch_0_cfg_7.5_topk_16384.jpg b/CKPTS/Afteri2i_100kckpt_LLAVAT5_empty_Fixed_just_segment_pretrain_stage3_dreambench_recap_400k_t2i_400k_flux_200k_midjourney_150k_recovery_150k_3e4_epoch2_no_rotated/003-GPT-XL/checkpoints/eval_step_80000/batch_0_cfg_7.5_topk_16384.jpg new file mode 100644 index 0000000000000000000000000000000000000000..4339274b0f4e47fe497b9d8e249218ff61044a65 --- /dev/null +++ b/CKPTS/Afteri2i_100kckpt_LLAVAT5_empty_Fixed_just_segment_pretrain_stage3_dreambench_recap_400k_t2i_400k_flux_200k_midjourney_150k_recovery_150k_3e4_epoch2_no_rotated/003-GPT-XL/checkpoints/eval_step_80000/batch_0_cfg_7.5_topk_16384.jpg @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:ea48c55236e79c30c59841ddf950fbb9b85ce085f9ddba17fe486ab195ba113c +size 714379 diff --git a/CKPTS/Afteri2i_100kckpt_LLAVAT5_empty_Fixed_just_segment_pretrain_stage3_dreambench_recap_400k_t2i_400k_flux_200k_midjourney_150k_recovery_150k_3e4_epoch2_no_rotated/003-GPT-XL/checkpoints/eval_step_80000/batch_1_cfg_7.5_topk_16384.jpg b/CKPTS/Afteri2i_100kckpt_LLAVAT5_empty_Fixed_just_segment_pretrain_stage3_dreambench_recap_400k_t2i_400k_flux_200k_midjourney_150k_recovery_150k_3e4_epoch2_no_rotated/003-GPT-XL/checkpoints/eval_step_80000/batch_1_cfg_7.5_topk_16384.jpg new file mode 100644 index 0000000000000000000000000000000000000000..b4445c3f62cb3ad13a81ad7e5ea279667a413c04 --- /dev/null +++ b/CKPTS/Afteri2i_100kckpt_LLAVAT5_empty_Fixed_just_segment_pretrain_stage3_dreambench_recap_400k_t2i_400k_flux_200k_midjourney_150k_recovery_150k_3e4_epoch2_no_rotated/003-GPT-XL/checkpoints/eval_step_80000/batch_1_cfg_7.5_topk_16384.jpg @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:402f53f5dfc430cb1770c02715a3bb7b17c352f8bf265675c050fd2c8b5f1e6a +size 707218 diff --git a/CKPTS/Afteri2i_100kckpt_LLAVAT5_empty_Fixed_just_segment_pretrain_stage3_dreambench_recap_400k_t2i_400k_flux_200k_midjourney_150k_recovery_150k_3e4_epoch2_no_rotated/003-GPT-XL/checkpoints/eval_step_80000/batch_2_cfg_7.5_topk_16384.jpg b/CKPTS/Afteri2i_100kckpt_LLAVAT5_empty_Fixed_just_segment_pretrain_stage3_dreambench_recap_400k_t2i_400k_flux_200k_midjourney_150k_recovery_150k_3e4_epoch2_no_rotated/003-GPT-XL/checkpoints/eval_step_80000/batch_2_cfg_7.5_topk_16384.jpg new file mode 100644 index 0000000000000000000000000000000000000000..61c4ff349d58a010c725693746ddee7401a586af --- /dev/null +++ b/CKPTS/Afteri2i_100kckpt_LLAVAT5_empty_Fixed_just_segment_pretrain_stage3_dreambench_recap_400k_t2i_400k_flux_200k_midjourney_150k_recovery_150k_3e4_epoch2_no_rotated/003-GPT-XL/checkpoints/eval_step_80000/batch_2_cfg_7.5_topk_16384.jpg @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:6630b5b717a9384315805e15f6fef08571936b2a88021e0a909aa4ac982a777e +size 715476 diff --git a/CKPTS/Afteri2i_100kckpt_LLAVAT5_empty_Fixed_just_segment_pretrain_stage3_dreambench_recap_400k_t2i_400k_flux_200k_midjourney_150k_recovery_150k_3e4_epoch2_no_rotated/003-GPT-XL/checkpoints/eval_step_80000/batch_3_cfg_7.5_topk_16384.jpg b/CKPTS/Afteri2i_100kckpt_LLAVAT5_empty_Fixed_just_segment_pretrain_stage3_dreambench_recap_400k_t2i_400k_flux_200k_midjourney_150k_recovery_150k_3e4_epoch2_no_rotated/003-GPT-XL/checkpoints/eval_step_80000/batch_3_cfg_7.5_topk_16384.jpg new file mode 100644 index 0000000000000000000000000000000000000000..d0afd11aca8d0baeb06640763d02e24c6c67618b --- /dev/null +++ b/CKPTS/Afteri2i_100kckpt_LLAVAT5_empty_Fixed_just_segment_pretrain_stage3_dreambench_recap_400k_t2i_400k_flux_200k_midjourney_150k_recovery_150k_3e4_epoch2_no_rotated/003-GPT-XL/checkpoints/eval_step_80000/batch_3_cfg_7.5_topk_16384.jpg @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:ea38e8c006c6da47ffd9fc648e0e0d7c9f6fd468892f4459c18db59f4dc441bf +size 826772 diff --git a/CKPTS/Afteri2i_100kckpt_LLAVAT5_empty_Fixed_just_segment_pretrain_stage3_dreambench_recap_400k_t2i_400k_flux_200k_midjourney_150k_recovery_150k_3e4_epoch2_no_rotated/003-GPT-XL/checkpoints/eval_step_82000/batch_0_cfg_7.5_topk_16384.jpg b/CKPTS/Afteri2i_100kckpt_LLAVAT5_empty_Fixed_just_segment_pretrain_stage3_dreambench_recap_400k_t2i_400k_flux_200k_midjourney_150k_recovery_150k_3e4_epoch2_no_rotated/003-GPT-XL/checkpoints/eval_step_82000/batch_0_cfg_7.5_topk_16384.jpg new file mode 100644 index 0000000000000000000000000000000000000000..9b44173c21c2bf5a26544fb21108f10e96dce657 --- /dev/null +++ b/CKPTS/Afteri2i_100kckpt_LLAVAT5_empty_Fixed_just_segment_pretrain_stage3_dreambench_recap_400k_t2i_400k_flux_200k_midjourney_150k_recovery_150k_3e4_epoch2_no_rotated/003-GPT-XL/checkpoints/eval_step_82000/batch_0_cfg_7.5_topk_16384.jpg @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:ddef871cf1df6a2917f4577a6b10a94c4b6b6cc657797299182f1598267649d0 +size 737177 diff --git a/CKPTS/Afteri2i_100kckpt_LLAVAT5_empty_Fixed_just_segment_pretrain_stage3_dreambench_recap_400k_t2i_400k_flux_200k_midjourney_150k_recovery_150k_3e4_epoch2_no_rotated/003-GPT-XL/checkpoints/eval_step_82000/batch_1_cfg_7.5_topk_16384.jpg b/CKPTS/Afteri2i_100kckpt_LLAVAT5_empty_Fixed_just_segment_pretrain_stage3_dreambench_recap_400k_t2i_400k_flux_200k_midjourney_150k_recovery_150k_3e4_epoch2_no_rotated/003-GPT-XL/checkpoints/eval_step_82000/batch_1_cfg_7.5_topk_16384.jpg new file mode 100644 index 0000000000000000000000000000000000000000..5821622d51dee6c4649f0dd0e8d4c6b063cd7fe3 --- /dev/null +++ b/CKPTS/Afteri2i_100kckpt_LLAVAT5_empty_Fixed_just_segment_pretrain_stage3_dreambench_recap_400k_t2i_400k_flux_200k_midjourney_150k_recovery_150k_3e4_epoch2_no_rotated/003-GPT-XL/checkpoints/eval_step_82000/batch_1_cfg_7.5_topk_16384.jpg @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:7f0f3ffb6ac0da967a0da3b7129d2a4992fe416d5811af00522ec9193980a2d3 +size 695841 diff --git a/CKPTS/Afteri2i_100kckpt_LLAVAT5_empty_Fixed_just_segment_pretrain_stage3_dreambench_recap_400k_t2i_400k_flux_200k_midjourney_150k_recovery_150k_3e4_epoch2_no_rotated/003-GPT-XL/checkpoints/eval_step_82000/batch_2_cfg_7.5_topk_16384.jpg b/CKPTS/Afteri2i_100kckpt_LLAVAT5_empty_Fixed_just_segment_pretrain_stage3_dreambench_recap_400k_t2i_400k_flux_200k_midjourney_150k_recovery_150k_3e4_epoch2_no_rotated/003-GPT-XL/checkpoints/eval_step_82000/batch_2_cfg_7.5_topk_16384.jpg new file mode 100644 index 0000000000000000000000000000000000000000..2f772229e77356ab2d4f1ec9357bc8d23a0546fb --- /dev/null +++ b/CKPTS/Afteri2i_100kckpt_LLAVAT5_empty_Fixed_just_segment_pretrain_stage3_dreambench_recap_400k_t2i_400k_flux_200k_midjourney_150k_recovery_150k_3e4_epoch2_no_rotated/003-GPT-XL/checkpoints/eval_step_82000/batch_2_cfg_7.5_topk_16384.jpg @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:d488cc77a9b6ee61cba4ae3f8be8646765793a3fcab3481faa044d9be5ee3d9b +size 731919 diff --git a/CKPTS/Afteri2i_100kckpt_LLAVAT5_empty_Fixed_just_segment_pretrain_stage3_dreambench_recap_400k_t2i_400k_flux_200k_midjourney_150k_recovery_150k_3e4_epoch2_no_rotated/003-GPT-XL/checkpoints/eval_step_82000/batch_3_cfg_7.5_topk_16384.jpg b/CKPTS/Afteri2i_100kckpt_LLAVAT5_empty_Fixed_just_segment_pretrain_stage3_dreambench_recap_400k_t2i_400k_flux_200k_midjourney_150k_recovery_150k_3e4_epoch2_no_rotated/003-GPT-XL/checkpoints/eval_step_82000/batch_3_cfg_7.5_topk_16384.jpg new file mode 100644 index 0000000000000000000000000000000000000000..f904c176c85861304e0df56455f7daf5ef93cbce --- /dev/null +++ b/CKPTS/Afteri2i_100kckpt_LLAVAT5_empty_Fixed_just_segment_pretrain_stage3_dreambench_recap_400k_t2i_400k_flux_200k_midjourney_150k_recovery_150k_3e4_epoch2_no_rotated/003-GPT-XL/checkpoints/eval_step_82000/batch_3_cfg_7.5_topk_16384.jpg @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:b9e5190876109dff0a454e33a89cfd2a55393ecdcc54e3bdea81776322bf9b66 +size 857573 diff --git a/CKPTS/Afteri2i_100kckpt_LLAVAT5_empty_Fixed_just_segment_pretrain_stage3_dreambench_recap_400k_t2i_400k_flux_200k_midjourney_150k_recovery_150k_3e4_epoch2_no_rotated/003-GPT-XL/checkpoints/eval_step_84000/batch_0_cfg_7.5_topk_16384.jpg b/CKPTS/Afteri2i_100kckpt_LLAVAT5_empty_Fixed_just_segment_pretrain_stage3_dreambench_recap_400k_t2i_400k_flux_200k_midjourney_150k_recovery_150k_3e4_epoch2_no_rotated/003-GPT-XL/checkpoints/eval_step_84000/batch_0_cfg_7.5_topk_16384.jpg new file mode 100644 index 0000000000000000000000000000000000000000..c9ff2d9303906378bc395d6c09cb23eaf504bf9f --- /dev/null +++ b/CKPTS/Afteri2i_100kckpt_LLAVAT5_empty_Fixed_just_segment_pretrain_stage3_dreambench_recap_400k_t2i_400k_flux_200k_midjourney_150k_recovery_150k_3e4_epoch2_no_rotated/003-GPT-XL/checkpoints/eval_step_84000/batch_0_cfg_7.5_topk_16384.jpg @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:e43c179593e2394023f2d1cf8aeae5b0888e90d3bc58db9ceb46d54e7db55f3d +size 740784 diff --git a/CKPTS/Afteri2i_100kckpt_LLAVAT5_empty_Fixed_just_segment_pretrain_stage3_dreambench_recap_400k_t2i_400k_flux_200k_midjourney_150k_recovery_150k_3e4_epoch2_no_rotated/003-GPT-XL/checkpoints/eval_step_84000/batch_1_cfg_7.5_topk_16384.jpg b/CKPTS/Afteri2i_100kckpt_LLAVAT5_empty_Fixed_just_segment_pretrain_stage3_dreambench_recap_400k_t2i_400k_flux_200k_midjourney_150k_recovery_150k_3e4_epoch2_no_rotated/003-GPT-XL/checkpoints/eval_step_84000/batch_1_cfg_7.5_topk_16384.jpg new file mode 100644 index 0000000000000000000000000000000000000000..c1f83575772d2563023dfe738c841343183791d1 --- /dev/null +++ b/CKPTS/Afteri2i_100kckpt_LLAVAT5_empty_Fixed_just_segment_pretrain_stage3_dreambench_recap_400k_t2i_400k_flux_200k_midjourney_150k_recovery_150k_3e4_epoch2_no_rotated/003-GPT-XL/checkpoints/eval_step_84000/batch_1_cfg_7.5_topk_16384.jpg @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:24c37a1a3ca661375c12636c389b7c319de83e91a8095002adf92f385f057b40 +size 697796 diff --git a/CKPTS/Afteri2i_100kckpt_LLAVAT5_empty_Fixed_just_segment_pretrain_stage3_dreambench_recap_400k_t2i_400k_flux_200k_midjourney_150k_recovery_150k_3e4_epoch2_no_rotated/003-GPT-XL/checkpoints/eval_step_84000/batch_2_cfg_7.5_topk_16384.jpg b/CKPTS/Afteri2i_100kckpt_LLAVAT5_empty_Fixed_just_segment_pretrain_stage3_dreambench_recap_400k_t2i_400k_flux_200k_midjourney_150k_recovery_150k_3e4_epoch2_no_rotated/003-GPT-XL/checkpoints/eval_step_84000/batch_2_cfg_7.5_topk_16384.jpg new file mode 100644 index 0000000000000000000000000000000000000000..a9a69683a8cbdab6c78fc502ef08217cc55a3ae6 --- /dev/null +++ b/CKPTS/Afteri2i_100kckpt_LLAVAT5_empty_Fixed_just_segment_pretrain_stage3_dreambench_recap_400k_t2i_400k_flux_200k_midjourney_150k_recovery_150k_3e4_epoch2_no_rotated/003-GPT-XL/checkpoints/eval_step_84000/batch_2_cfg_7.5_topk_16384.jpg @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:ed679e760fb3116ed7e01ad83a3ea9608fc9563274c99d11ddcdb050b47b914a +size 733732 diff --git a/CKPTS/Afteri2i_100kckpt_LLAVAT5_empty_Fixed_just_segment_pretrain_stage3_dreambench_recap_400k_t2i_400k_flux_200k_midjourney_150k_recovery_150k_3e4_epoch2_no_rotated/003-GPT-XL/checkpoints/eval_step_84000/batch_3_cfg_7.5_topk_16384.jpg b/CKPTS/Afteri2i_100kckpt_LLAVAT5_empty_Fixed_just_segment_pretrain_stage3_dreambench_recap_400k_t2i_400k_flux_200k_midjourney_150k_recovery_150k_3e4_epoch2_no_rotated/003-GPT-XL/checkpoints/eval_step_84000/batch_3_cfg_7.5_topk_16384.jpg new file mode 100644 index 0000000000000000000000000000000000000000..d6afd870b33697d265ec23eeb3c0e69266dab35d --- /dev/null +++ b/CKPTS/Afteri2i_100kckpt_LLAVAT5_empty_Fixed_just_segment_pretrain_stage3_dreambench_recap_400k_t2i_400k_flux_200k_midjourney_150k_recovery_150k_3e4_epoch2_no_rotated/003-GPT-XL/checkpoints/eval_step_84000/batch_3_cfg_7.5_topk_16384.jpg @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:3205b1659ba056b65d36d08ff42880d500d0b474e37453378dcceda547b526c3 +size 826052 diff --git a/CKPTS/Afteri2i_100kckpt_LLAVAT5_empty_Fixed_just_segment_pretrain_stage3_dreambench_recap_400k_t2i_400k_flux_200k_midjourney_150k_recovery_150k_3e4_epoch2_no_rotated/003-GPT-XL/checkpoints/eval_step_86000/batch_0_cfg_7.5_topk_16384.jpg b/CKPTS/Afteri2i_100kckpt_LLAVAT5_empty_Fixed_just_segment_pretrain_stage3_dreambench_recap_400k_t2i_400k_flux_200k_midjourney_150k_recovery_150k_3e4_epoch2_no_rotated/003-GPT-XL/checkpoints/eval_step_86000/batch_0_cfg_7.5_topk_16384.jpg new file mode 100644 index 0000000000000000000000000000000000000000..208b52e7af71f73ab3ca8ed7b0804ed8875d5c84 --- /dev/null +++ b/CKPTS/Afteri2i_100kckpt_LLAVAT5_empty_Fixed_just_segment_pretrain_stage3_dreambench_recap_400k_t2i_400k_flux_200k_midjourney_150k_recovery_150k_3e4_epoch2_no_rotated/003-GPT-XL/checkpoints/eval_step_86000/batch_0_cfg_7.5_topk_16384.jpg @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:6ea64d7541ed895dff8049897e6bc0cd0cfb01347a90e98628ebf05aca0ad1bd +size 722997 diff --git a/CKPTS/Afteri2i_100kckpt_LLAVAT5_empty_Fixed_just_segment_pretrain_stage3_dreambench_recap_400k_t2i_400k_flux_200k_midjourney_150k_recovery_150k_3e4_epoch2_no_rotated/003-GPT-XL/checkpoints/eval_step_86000/batch_1_cfg_7.5_topk_16384.jpg b/CKPTS/Afteri2i_100kckpt_LLAVAT5_empty_Fixed_just_segment_pretrain_stage3_dreambench_recap_400k_t2i_400k_flux_200k_midjourney_150k_recovery_150k_3e4_epoch2_no_rotated/003-GPT-XL/checkpoints/eval_step_86000/batch_1_cfg_7.5_topk_16384.jpg new file mode 100644 index 0000000000000000000000000000000000000000..a9e43611415f515dba083d4e64b6e0ac75d8edc2 --- /dev/null +++ b/CKPTS/Afteri2i_100kckpt_LLAVAT5_empty_Fixed_just_segment_pretrain_stage3_dreambench_recap_400k_t2i_400k_flux_200k_midjourney_150k_recovery_150k_3e4_epoch2_no_rotated/003-GPT-XL/checkpoints/eval_step_86000/batch_1_cfg_7.5_topk_16384.jpg @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:8d65a8af67cf113e4e8688c310ae34d460859356f6a39473d1d5dbd2ff058301 +size 700581 diff --git a/CKPTS/Afteri2i_100kckpt_LLAVAT5_empty_Fixed_just_segment_pretrain_stage3_dreambench_recap_400k_t2i_400k_flux_200k_midjourney_150k_recovery_150k_3e4_epoch2_no_rotated/003-GPT-XL/checkpoints/eval_step_86000/batch_2_cfg_7.5_topk_16384.jpg b/CKPTS/Afteri2i_100kckpt_LLAVAT5_empty_Fixed_just_segment_pretrain_stage3_dreambench_recap_400k_t2i_400k_flux_200k_midjourney_150k_recovery_150k_3e4_epoch2_no_rotated/003-GPT-XL/checkpoints/eval_step_86000/batch_2_cfg_7.5_topk_16384.jpg new file mode 100644 index 0000000000000000000000000000000000000000..02704c28d4c65326135a5fc4fbe57481a264ae15 --- /dev/null +++ b/CKPTS/Afteri2i_100kckpt_LLAVAT5_empty_Fixed_just_segment_pretrain_stage3_dreambench_recap_400k_t2i_400k_flux_200k_midjourney_150k_recovery_150k_3e4_epoch2_no_rotated/003-GPT-XL/checkpoints/eval_step_86000/batch_2_cfg_7.5_topk_16384.jpg @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:a7e8d3b923dcdca6ddfc2c17b60f575dbb2fdede98eade496855d1d261f458da +size 739571 diff --git a/CKPTS/Afteri2i_100kckpt_LLAVAT5_empty_Fixed_just_segment_pretrain_stage3_dreambench_recap_400k_t2i_400k_flux_200k_midjourney_150k_recovery_150k_3e4_epoch2_no_rotated/003-GPT-XL/checkpoints/eval_step_86000/batch_3_cfg_7.5_topk_16384.jpg b/CKPTS/Afteri2i_100kckpt_LLAVAT5_empty_Fixed_just_segment_pretrain_stage3_dreambench_recap_400k_t2i_400k_flux_200k_midjourney_150k_recovery_150k_3e4_epoch2_no_rotated/003-GPT-XL/checkpoints/eval_step_86000/batch_3_cfg_7.5_topk_16384.jpg new file mode 100644 index 0000000000000000000000000000000000000000..59b087f37a66a8be9b7786860476038d1277987e --- /dev/null +++ b/CKPTS/Afteri2i_100kckpt_LLAVAT5_empty_Fixed_just_segment_pretrain_stage3_dreambench_recap_400k_t2i_400k_flux_200k_midjourney_150k_recovery_150k_3e4_epoch2_no_rotated/003-GPT-XL/checkpoints/eval_step_86000/batch_3_cfg_7.5_topk_16384.jpg @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:9ac91f1bd2754bd1357500565bb6b2f3c2fac7b923c50ec25d98f120ba07f6ab +size 845745 diff --git a/CKPTS/Afteri2i_100kckpt_LLAVAT5_empty_Fixed_just_segment_pretrain_stage3_dreambench_recap_400k_t2i_400k_flux_200k_midjourney_150k_recovery_150k_3e4_epoch2_no_rotated/003-GPT-XL/checkpoints/eval_step_88000/batch_0_cfg_7.5_topk_16384.jpg b/CKPTS/Afteri2i_100kckpt_LLAVAT5_empty_Fixed_just_segment_pretrain_stage3_dreambench_recap_400k_t2i_400k_flux_200k_midjourney_150k_recovery_150k_3e4_epoch2_no_rotated/003-GPT-XL/checkpoints/eval_step_88000/batch_0_cfg_7.5_topk_16384.jpg new file mode 100644 index 0000000000000000000000000000000000000000..8ab140db551711bc1053a5c295ec89bdc6c1dd6c --- /dev/null +++ b/CKPTS/Afteri2i_100kckpt_LLAVAT5_empty_Fixed_just_segment_pretrain_stage3_dreambench_recap_400k_t2i_400k_flux_200k_midjourney_150k_recovery_150k_3e4_epoch2_no_rotated/003-GPT-XL/checkpoints/eval_step_88000/batch_0_cfg_7.5_topk_16384.jpg @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:8e235388662b4a046cf5dd1f90d9e5350b24c5b39f16b3b9fcc2864161b97ce5 +size 755610 diff --git a/CKPTS/Afteri2i_100kckpt_LLAVAT5_empty_Fixed_just_segment_pretrain_stage3_dreambench_recap_400k_t2i_400k_flux_200k_midjourney_150k_recovery_150k_3e4_epoch2_no_rotated/003-GPT-XL/checkpoints/eval_step_88000/batch_1_cfg_7.5_topk_16384.jpg b/CKPTS/Afteri2i_100kckpt_LLAVAT5_empty_Fixed_just_segment_pretrain_stage3_dreambench_recap_400k_t2i_400k_flux_200k_midjourney_150k_recovery_150k_3e4_epoch2_no_rotated/003-GPT-XL/checkpoints/eval_step_88000/batch_1_cfg_7.5_topk_16384.jpg new file mode 100644 index 0000000000000000000000000000000000000000..a7fba7971546c3b7bff29c44ca99409a1f7d058d --- /dev/null +++ b/CKPTS/Afteri2i_100kckpt_LLAVAT5_empty_Fixed_just_segment_pretrain_stage3_dreambench_recap_400k_t2i_400k_flux_200k_midjourney_150k_recovery_150k_3e4_epoch2_no_rotated/003-GPT-XL/checkpoints/eval_step_88000/batch_1_cfg_7.5_topk_16384.jpg @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:f7e75cf2df676444da6f564f2c1f0485c393ceec5f70c2d0e6989bcd7c5f58b9 +size 726248 diff --git a/CKPTS/Afteri2i_100kckpt_LLAVAT5_empty_Fixed_just_segment_pretrain_stage3_dreambench_recap_400k_t2i_400k_flux_200k_midjourney_150k_recovery_150k_3e4_epoch2_no_rotated/003-GPT-XL/checkpoints/eval_step_88000/batch_2_cfg_7.5_topk_16384.jpg b/CKPTS/Afteri2i_100kckpt_LLAVAT5_empty_Fixed_just_segment_pretrain_stage3_dreambench_recap_400k_t2i_400k_flux_200k_midjourney_150k_recovery_150k_3e4_epoch2_no_rotated/003-GPT-XL/checkpoints/eval_step_88000/batch_2_cfg_7.5_topk_16384.jpg new file mode 100644 index 0000000000000000000000000000000000000000..18e95a85f6cde6ee42e970f42f7b84fc2ebd2943 --- /dev/null +++ b/CKPTS/Afteri2i_100kckpt_LLAVAT5_empty_Fixed_just_segment_pretrain_stage3_dreambench_recap_400k_t2i_400k_flux_200k_midjourney_150k_recovery_150k_3e4_epoch2_no_rotated/003-GPT-XL/checkpoints/eval_step_88000/batch_2_cfg_7.5_topk_16384.jpg @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:391d620795894ca2b7a1424934e15beb309da8573f6c92645a3d4a71010a8f89 +size 720462 diff --git a/CKPTS/Afteri2i_100kckpt_LLAVAT5_empty_Fixed_just_segment_pretrain_stage3_dreambench_recap_400k_t2i_400k_flux_200k_midjourney_150k_recovery_150k_3e4_epoch2_no_rotated/003-GPT-XL/checkpoints/eval_step_88000/batch_3_cfg_7.5_topk_16384.jpg b/CKPTS/Afteri2i_100kckpt_LLAVAT5_empty_Fixed_just_segment_pretrain_stage3_dreambench_recap_400k_t2i_400k_flux_200k_midjourney_150k_recovery_150k_3e4_epoch2_no_rotated/003-GPT-XL/checkpoints/eval_step_88000/batch_3_cfg_7.5_topk_16384.jpg new file mode 100644 index 0000000000000000000000000000000000000000..5cbc1f7cb9575cdf0ac5a090273af63c7f63d1a6 --- /dev/null +++ b/CKPTS/Afteri2i_100kckpt_LLAVAT5_empty_Fixed_just_segment_pretrain_stage3_dreambench_recap_400k_t2i_400k_flux_200k_midjourney_150k_recovery_150k_3e4_epoch2_no_rotated/003-GPT-XL/checkpoints/eval_step_88000/batch_3_cfg_7.5_topk_16384.jpg @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:34973133d33a1938f5c70e12290afe99c666688c868868b08bc2edabec18c966 +size 869980 diff --git a/CKPTS/Afteri2i_100kckpt_LLAVAT5_empty_Fixed_just_segment_pretrain_stage3_dreambench_recap_400k_t2i_400k_flux_200k_midjourney_150k_recovery_150k_3e4_epoch2_no_rotated/003-GPT-XL/checkpoints/eval_step_90000/batch_0_cfg_7.5_topk_16384.jpg b/CKPTS/Afteri2i_100kckpt_LLAVAT5_empty_Fixed_just_segment_pretrain_stage3_dreambench_recap_400k_t2i_400k_flux_200k_midjourney_150k_recovery_150k_3e4_epoch2_no_rotated/003-GPT-XL/checkpoints/eval_step_90000/batch_0_cfg_7.5_topk_16384.jpg new file mode 100644 index 0000000000000000000000000000000000000000..eb278f5fac2ac666dd60b2300b437a2f218c23d4 --- /dev/null +++ b/CKPTS/Afteri2i_100kckpt_LLAVAT5_empty_Fixed_just_segment_pretrain_stage3_dreambench_recap_400k_t2i_400k_flux_200k_midjourney_150k_recovery_150k_3e4_epoch2_no_rotated/003-GPT-XL/checkpoints/eval_step_90000/batch_0_cfg_7.5_topk_16384.jpg @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:45ede40ac73cb2db303afea2325ebd783f0445044f4bec93e5ac204bd42ebe1e +size 716520 diff --git a/CKPTS/Afteri2i_100kckpt_LLAVAT5_empty_Fixed_just_segment_pretrain_stage3_dreambench_recap_400k_t2i_400k_flux_200k_midjourney_150k_recovery_150k_3e4_epoch2_no_rotated/003-GPT-XL/checkpoints/eval_step_90000/batch_1_cfg_7.5_topk_16384.jpg b/CKPTS/Afteri2i_100kckpt_LLAVAT5_empty_Fixed_just_segment_pretrain_stage3_dreambench_recap_400k_t2i_400k_flux_200k_midjourney_150k_recovery_150k_3e4_epoch2_no_rotated/003-GPT-XL/checkpoints/eval_step_90000/batch_1_cfg_7.5_topk_16384.jpg new file mode 100644 index 0000000000000000000000000000000000000000..df1be7ba9fd90a66ebb9d7e9413216f8bca7835a --- /dev/null +++ b/CKPTS/Afteri2i_100kckpt_LLAVAT5_empty_Fixed_just_segment_pretrain_stage3_dreambench_recap_400k_t2i_400k_flux_200k_midjourney_150k_recovery_150k_3e4_epoch2_no_rotated/003-GPT-XL/checkpoints/eval_step_90000/batch_1_cfg_7.5_topk_16384.jpg @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:8b0d03f5adc5e59eee24e0081173a8cc335e8e2c58c3a1c2a2ca478815b4fa1a +size 696183 diff --git a/CKPTS/Afteri2i_100kckpt_LLAVAT5_empty_Fixed_just_segment_pretrain_stage3_dreambench_recap_400k_t2i_400k_flux_200k_midjourney_150k_recovery_150k_3e4_epoch2_no_rotated/003-GPT-XL/checkpoints/eval_step_90000/batch_2_cfg_7.5_topk_16384.jpg b/CKPTS/Afteri2i_100kckpt_LLAVAT5_empty_Fixed_just_segment_pretrain_stage3_dreambench_recap_400k_t2i_400k_flux_200k_midjourney_150k_recovery_150k_3e4_epoch2_no_rotated/003-GPT-XL/checkpoints/eval_step_90000/batch_2_cfg_7.5_topk_16384.jpg new file mode 100644 index 0000000000000000000000000000000000000000..413a936dc095657cef21500f37db2505a9e79fdd --- /dev/null +++ b/CKPTS/Afteri2i_100kckpt_LLAVAT5_empty_Fixed_just_segment_pretrain_stage3_dreambench_recap_400k_t2i_400k_flux_200k_midjourney_150k_recovery_150k_3e4_epoch2_no_rotated/003-GPT-XL/checkpoints/eval_step_90000/batch_2_cfg_7.5_topk_16384.jpg @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:0fa51e5cef4f6ad8babebe7c904e1b59078e42e130d73d10262ce611c0df94a2 +size 730166 diff --git a/CKPTS/Afteri2i_100kckpt_LLAVAT5_empty_Fixed_just_segment_pretrain_stage3_dreambench_recap_400k_t2i_400k_flux_200k_midjourney_150k_recovery_150k_3e4_epoch2_no_rotated/003-GPT-XL/checkpoints/eval_step_90000/batch_3_cfg_7.5_topk_16384.jpg b/CKPTS/Afteri2i_100kckpt_LLAVAT5_empty_Fixed_just_segment_pretrain_stage3_dreambench_recap_400k_t2i_400k_flux_200k_midjourney_150k_recovery_150k_3e4_epoch2_no_rotated/003-GPT-XL/checkpoints/eval_step_90000/batch_3_cfg_7.5_topk_16384.jpg new file mode 100644 index 0000000000000000000000000000000000000000..f0516cd229aa048572c4239245421960cd6dfecc --- /dev/null +++ b/CKPTS/Afteri2i_100kckpt_LLAVAT5_empty_Fixed_just_segment_pretrain_stage3_dreambench_recap_400k_t2i_400k_flux_200k_midjourney_150k_recovery_150k_3e4_epoch2_no_rotated/003-GPT-XL/checkpoints/eval_step_90000/batch_3_cfg_7.5_topk_16384.jpg @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:395ba0bde49c08654fd31f8ba710bfb933ade4f8e60d3b244975f4788931128c +size 869847 diff --git a/CKPTS/Afteri2i_100kckpt_LLAVAT5_empty_Fixed_just_segment_pretrain_stage3_dreambench_recap_400k_t2i_400k_flux_200k_midjourney_150k_recovery_150k_3e4_epoch2_no_rotated/003-GPT-XL/checkpoints/eval_step_92000/batch_0_cfg_7.5_topk_16384.jpg b/CKPTS/Afteri2i_100kckpt_LLAVAT5_empty_Fixed_just_segment_pretrain_stage3_dreambench_recap_400k_t2i_400k_flux_200k_midjourney_150k_recovery_150k_3e4_epoch2_no_rotated/003-GPT-XL/checkpoints/eval_step_92000/batch_0_cfg_7.5_topk_16384.jpg new file mode 100644 index 0000000000000000000000000000000000000000..701b78c23a659c394167474778131759caa38235 --- /dev/null +++ b/CKPTS/Afteri2i_100kckpt_LLAVAT5_empty_Fixed_just_segment_pretrain_stage3_dreambench_recap_400k_t2i_400k_flux_200k_midjourney_150k_recovery_150k_3e4_epoch2_no_rotated/003-GPT-XL/checkpoints/eval_step_92000/batch_0_cfg_7.5_topk_16384.jpg @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:0a908ff58c41ec29ccc10df71f467bd8bb457a4f2dded0c3512b3eddaf3dda7c +size 744978 diff --git a/CKPTS/Afteri2i_100kckpt_LLAVAT5_empty_Fixed_just_segment_pretrain_stage3_dreambench_recap_400k_t2i_400k_flux_200k_midjourney_150k_recovery_150k_3e4_epoch2_no_rotated/003-GPT-XL/checkpoints/eval_step_92000/batch_1_cfg_7.5_topk_16384.jpg b/CKPTS/Afteri2i_100kckpt_LLAVAT5_empty_Fixed_just_segment_pretrain_stage3_dreambench_recap_400k_t2i_400k_flux_200k_midjourney_150k_recovery_150k_3e4_epoch2_no_rotated/003-GPT-XL/checkpoints/eval_step_92000/batch_1_cfg_7.5_topk_16384.jpg new file mode 100644 index 0000000000000000000000000000000000000000..b08ac6443a54e3196214927430c1443e200e1430 --- /dev/null +++ b/CKPTS/Afteri2i_100kckpt_LLAVAT5_empty_Fixed_just_segment_pretrain_stage3_dreambench_recap_400k_t2i_400k_flux_200k_midjourney_150k_recovery_150k_3e4_epoch2_no_rotated/003-GPT-XL/checkpoints/eval_step_92000/batch_1_cfg_7.5_topk_16384.jpg @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:e2a91dc26dc76e322c9deedf9b8c50d7f136a37105b3a0000845265c110c5745 +size 726104 diff --git a/CKPTS/Afteri2i_100kckpt_LLAVAT5_empty_Fixed_just_segment_pretrain_stage3_dreambench_recap_400k_t2i_400k_flux_200k_midjourney_150k_recovery_150k_3e4_epoch2_no_rotated/003-GPT-XL/checkpoints/eval_step_92000/batch_2_cfg_7.5_topk_16384.jpg b/CKPTS/Afteri2i_100kckpt_LLAVAT5_empty_Fixed_just_segment_pretrain_stage3_dreambench_recap_400k_t2i_400k_flux_200k_midjourney_150k_recovery_150k_3e4_epoch2_no_rotated/003-GPT-XL/checkpoints/eval_step_92000/batch_2_cfg_7.5_topk_16384.jpg new file mode 100644 index 0000000000000000000000000000000000000000..107e36c12c3fc20e950c4ec0b295eac1bcfc16c4 --- /dev/null +++ b/CKPTS/Afteri2i_100kckpt_LLAVAT5_empty_Fixed_just_segment_pretrain_stage3_dreambench_recap_400k_t2i_400k_flux_200k_midjourney_150k_recovery_150k_3e4_epoch2_no_rotated/003-GPT-XL/checkpoints/eval_step_92000/batch_2_cfg_7.5_topk_16384.jpg @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:aba9767d32ae7d0355817e1f13da5be5985938183123f97e22e390d57459cd71 +size 737406 diff --git a/CKPTS/Afteri2i_100kckpt_LLAVAT5_empty_Fixed_just_segment_pretrain_stage3_dreambench_recap_400k_t2i_400k_flux_200k_midjourney_150k_recovery_150k_3e4_epoch2_no_rotated/003-GPT-XL/checkpoints/eval_step_92000/batch_3_cfg_7.5_topk_16384.jpg b/CKPTS/Afteri2i_100kckpt_LLAVAT5_empty_Fixed_just_segment_pretrain_stage3_dreambench_recap_400k_t2i_400k_flux_200k_midjourney_150k_recovery_150k_3e4_epoch2_no_rotated/003-GPT-XL/checkpoints/eval_step_92000/batch_3_cfg_7.5_topk_16384.jpg new file mode 100644 index 0000000000000000000000000000000000000000..0ce3aefb14b33dfed7b400586dec7dc292c7fa72 --- /dev/null +++ b/CKPTS/Afteri2i_100kckpt_LLAVAT5_empty_Fixed_just_segment_pretrain_stage3_dreambench_recap_400k_t2i_400k_flux_200k_midjourney_150k_recovery_150k_3e4_epoch2_no_rotated/003-GPT-XL/checkpoints/eval_step_92000/batch_3_cfg_7.5_topk_16384.jpg @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:6986060be2cd498d9290663a89b5ff67e5b921f55458b7cdd73a89896a19f2f5 +size 867991 diff --git a/CKPTS/Afteri2i_100kckpt_LLAVAT5_empty_Fixed_just_segment_pretrain_stage3_dreambench_recap_400k_t2i_400k_flux_200k_midjourney_150k_recovery_150k_3e4_epoch2_no_rotated/003-GPT-XL/log.txt b/CKPTS/Afteri2i_100kckpt_LLAVAT5_empty_Fixed_just_segment_pretrain_stage3_dreambench_recap_400k_t2i_400k_flux_200k_midjourney_150k_recovery_150k_3e4_epoch2_no_rotated/003-GPT-XL/log.txt new file mode 100644 index 0000000000000000000000000000000000000000..d537b43061f6538a10c0ce73ad281a17da5708d8 --- /dev/null +++ b/CKPTS/Afteri2i_100kckpt_LLAVAT5_empty_Fixed_just_segment_pretrain_stage3_dreambench_recap_400k_t2i_400k_flux_200k_midjourney_150k_recovery_150k_3e4_epoch2_no_rotated/003-GPT-XL/log.txt @@ -0,0 +1,1016 @@ +[2025-04-26 08:02:27] Experiment directory created at checkpoint/Afteri2i_100kckpt_LLAVAT5_empty_Fixed_just_segment_pretrain_stage3_dreambench_recap_400k_t2i_400k_flux_200k_midjourney_150k_recovery_150k_3e4_epoch2_no_rotated/003-GPT-XL +[2025-04-26 08:02:27] Namespace(data_path='/tmp/haozhezhao/MLLMG/jsonl_data/merged_train_set_set_subject_400k_recap_t2i_400k_flux_200k_midjourney_150k_recovery_150k_grounding_100fluxseg_50samseg.jsonl', cloud_save_path='/tmp/haozhezhao/checkpoint', no_local_save=False, vq_model='VQ-16', vq_ckpt='/tmp/haozhezhao/model/llamagen_t2i/vq_ds16_t2i.pt', codebook_size=16384, codebook_embed_dim=8, gpt_model='GPT-XL', gpt_ckpt='/tmp/haozhezhao/MLLMG/checkpoint/Afteri2i_100kckpt_LLAVAT5_empty_Fixed_just_segment_pretrain_stage3_dreambench_recap_400k_t2i_400k_flux_200k_midjourney_150k_recovery_150k_3e4_epoch2_no_rotated/002-GPT-XL/checkpoints/0070000.pt', gpt_type='t2i', vocab_size=16384, cls_token_num=512, dropout_p=0.1, token_dropout_p=0.1, drop_path=0.0, no_compile=False, results_dir='checkpoint/Afteri2i_100kckpt_LLAVAT5_empty_Fixed_just_segment_pretrain_stage3_dreambench_recap_400k_t2i_400k_flux_200k_midjourney_150k_recovery_150k_3e4_epoch2_no_rotated', dataset='ti2i', image_size=512, downsample_size=16, num_classes=1000, epochs=4, lr=0.0003, weight_decay=0.05, beta1=0.9, beta2=0.95, max_grad_norm=1.0, global_batch_size=56, global_seed=0, num_workers=8, log_every=25, ckpt_every=2000, gradient_accumulation_steps=4, mixed_precision='bf16', val_data_path='/tmp/haozhezhao/MLLMG/jsonl_data/dreambench_plus_valid.jsonl', use_vision_tower=True, model_name_or_path='/tmp/haozhezhao/model/blip2-flan-t5-xl', image_place_holder='', processor_path=None, do_eval=True, max_eval_samples=250, train_text_encoder=True, no_left_padding=False, cfg_scale=7.5, top_k=16384, temperature=0.9, top_p=1.0, eval_steps=2000, project_name='llamagen_ti2i', load_from_checkpoint='/tmp/haozhezhao/MLLMG/checkpoint/FIXed_3M_1epoch_step32000.pt', warmup=0.05, lr_decay_style='cosine', lr_decay_ratio=0.1, train_iters=500000, class_dropout_prob=0.1, with_image_only=False, image_only_rate=0.1, stage2=False, subject_driven=True, load_subject_embedding=None, reference_data_path='/tmp/haozhezhao/MLLMG/cc12m_reference_tunnel.jsonl', multimodal_encoder='llava', do_recovery=True, no_replace=False, resume=True, dreambench_eval=True, find_unused_parameters=True, load_visual_encoder=False, continue_stage1=False, replace_subject=False, train_all=True, save_total_limit=1, load_language_projection='/tmp/haozhezhao/MLLMG/llava-v1.5-flant5_fixed-pretrain/mm_projector.bin', mm_vision_tower='openai/clip-vit-large-patch14', load_fixed_llamagen=True, unfreeze_output=False, fix='gpt-empty-fix', rank=0, world_size=8, gpu=0, dist_url='env://', distributed=True, dist_backend='nccl') +[2025-04-26 08:02:27] Starting rank=0, seed=0, world_size=8. +[2025-04-26 08:02:27] text_config is None. Initializing the text config with default values (`OPTConfig`). +[2025-04-26 08:03:34] GPT Parameters: 2,310,680,832 +[2025-04-26 08:03:34] num decayed parameter tensors: 356, with 2,007,303,168 parameters +[2025-04-26 08:03:34] num non-decayed parameter tensors: 124, with 197,888 parameters +[2025-04-26 08:03:34] using fused AdamW: True +[2025-04-26 08:03:43] Dataset contains 1,313,682 images +[2025-04-26 08:03:43] Train iters 93832 , warmup 4691.6, len of loader 23458 +[2025-04-26 08:03:59] Resume training from checkpoint: /tmp/haozhezhao/MLLMG/checkpoint/Afteri2i_100kckpt_LLAVAT5_empty_Fixed_just_segment_pretrain_stage3_dreambench_recap_400k_t2i_400k_flux_200k_midjourney_150k_recovery_150k_3e4_epoch2_no_rotated/002-GPT-XL/checkpoints/0070000.pt +[2025-04-26 08:03:59] Initial state: steps=70000, epochs=2 +[2025-04-26 08:03:59] compiling the model... (may take several minutes) +[2025-04-26 08:04:00] freeze the vit +[2025-04-26 08:04:00] ***** total param is 2310680832 ***** +[2025-04-26 08:04:00] ***** total trained param is 2007501056 ***** +[2025-04-26 08:04:04] Training for 4 epochs... +[2025-04-26 08:04:04] Beginning epoch 2... +[2025-04-26 08:31:04] (step=0070025) Train Loss: 5.3177, Train Steps/Sec: 0.02 +[2025-04-26 08:34:21] (step=0070050) Train Loss: 5.2728, Train Steps/Sec: 0.13 +[2025-04-26 08:36:17] (step=0070075) Train Loss: 5.3276, Train Steps/Sec: 0.22 +[2025-04-26 08:37:13] (step=0070100) Train Loss: 5.3134, Train Steps/Sec: 0.44 +[2025-04-26 08:37:35] (step=0070125) Train Loss: 5.1849, Train Steps/Sec: 1.18 +[2025-04-26 08:37:56] (step=0070150) Train Loss: 5.2783, Train Steps/Sec: 1.17 +[2025-04-26 08:38:17] (step=0070175) Train Loss: 5.1995, Train Steps/Sec: 1.18 +[2025-04-26 08:38:39] (step=0070200) Train Loss: 5.2465, Train Steps/Sec: 1.17 +[2025-04-26 08:39:00] (step=0070225) Train Loss: 5.1871, Train Steps/Sec: 1.18 +[2025-04-26 08:39:21] (step=0070250) Train Loss: 5.2435, Train Steps/Sec: 1.17 +[2025-04-26 08:39:42] (step=0070275) Train Loss: 5.2690, Train Steps/Sec: 1.17 +[2025-04-26 08:40:04] (step=0070300) Train Loss: 5.2306, Train Steps/Sec: 1.17 +[2025-04-26 08:40:25] (step=0070325) Train Loss: 5.2591, Train Steps/Sec: 1.17 +[2025-04-26 08:40:46] (step=0070350) Train Loss: 5.3037, Train Steps/Sec: 1.17 +[2025-04-26 08:41:34] Beginning epoch 3... +[2025-04-26 08:41:38] (step=0070375) Train Loss: 5.2023, Train Steps/Sec: 0.49 +[2025-04-26 08:42:00] (step=0070400) Train Loss: 5.2237, Train Steps/Sec: 1.15 +[2025-04-26 08:42:48] (step=0070425) Train Loss: 5.1489, Train Steps/Sec: 0.51 +[2025-04-26 08:43:10] (step=0070450) Train Loss: 5.2047, Train Steps/Sec: 1.17 +[2025-04-26 08:43:58] (step=0070475) Train Loss: 5.1460, Train Steps/Sec: 0.52 +[2025-04-26 08:44:46] (step=0070500) Train Loss: 5.2218, Train Steps/Sec: 0.52 +[2025-04-26 08:45:08] (step=0070525) Train Loss: 5.1163, Train Steps/Sec: 1.17 +[2025-04-26 08:45:29] (step=0070550) Train Loss: 5.2263, Train Steps/Sec: 1.17 +[2025-04-26 08:45:50] (step=0070575) Train Loss: 5.2134, Train Steps/Sec: 1.17 +[2025-04-26 08:46:13] (step=0070600) Train Loss: 5.1674, Train Steps/Sec: 1.11 +[2025-04-26 08:46:34] (step=0070625) Train Loss: 5.1579, Train Steps/Sec: 1.17 +[2025-04-26 08:46:57] (step=0070650) Train Loss: 5.2015, Train Steps/Sec: 1.11 +[2025-04-26 08:47:20] (step=0070675) Train Loss: 5.1822, Train Steps/Sec: 1.06 +[2025-04-26 08:47:42] (step=0070700) Train Loss: 5.1702, Train Steps/Sec: 1.17 +[2025-04-26 08:48:04] (step=0070725) Train Loss: 5.2097, Train Steps/Sec: 1.12 +[2025-04-26 08:48:25] (step=0070750) Train Loss: 5.1938, Train Steps/Sec: 1.17 +[2025-04-26 08:48:47] (step=0070775) Train Loss: 5.1679, Train Steps/Sec: 1.17 +[2025-04-26 08:49:08] (step=0070800) Train Loss: 5.1441, Train Steps/Sec: 1.16 +[2025-04-26 08:49:30] (step=0070825) Train Loss: 5.1766, Train Steps/Sec: 1.17 +[2025-04-26 08:49:51] (step=0070850) Train Loss: 5.0985, Train Steps/Sec: 1.17 +[2025-04-26 08:50:14] (step=0070875) Train Loss: 5.2293, Train Steps/Sec: 1.11 +[2025-04-26 08:50:35] (step=0070900) Train Loss: 5.1960, Train Steps/Sec: 1.17 +[2025-04-26 08:50:57] (step=0070925) Train Loss: 5.2425, Train Steps/Sec: 1.17 +[2025-04-26 08:51:18] (step=0070950) Train Loss: 5.0804, Train Steps/Sec: 1.17 +[2025-04-26 08:51:39] (step=0070975) Train Loss: 5.1564, Train Steps/Sec: 1.17 +[2025-04-26 08:52:02] (step=0071000) Train Loss: 5.1597, Train Steps/Sec: 1.11 +[2025-04-26 08:52:23] (step=0071025) Train Loss: 5.1195, Train Steps/Sec: 1.17 +[2025-04-26 08:52:45] (step=0071050) Train Loss: 5.2757, Train Steps/Sec: 1.12 +[2025-04-26 08:53:07] (step=0071075) Train Loss: 5.1781, Train Steps/Sec: 1.17 +[2025-04-26 08:53:28] (step=0071100) Train Loss: 5.1939, Train Steps/Sec: 1.16 +[2025-04-26 08:53:50] (step=0071125) Train Loss: 5.1881, Train Steps/Sec: 1.17 +[2025-04-26 08:54:11] (step=0071150) Train Loss: 5.1737, Train Steps/Sec: 1.17 +[2025-04-26 08:54:32] (step=0071175) Train Loss: 5.2379, Train Steps/Sec: 1.17 +[2025-04-26 08:54:54] (step=0071200) Train Loss: 5.1470, Train Steps/Sec: 1.17 +[2025-04-26 08:55:15] (step=0071225) Train Loss: 5.2287, Train Steps/Sec: 1.17 +[2025-04-26 08:55:37] (step=0071250) Train Loss: 5.2393, Train Steps/Sec: 1.13 +[2025-04-26 08:55:59] (step=0071275) Train Loss: 5.1175, Train Steps/Sec: 1.17 +[2025-04-26 08:56:20] (step=0071300) Train Loss: 5.1864, Train Steps/Sec: 1.16 +[2025-04-26 08:56:44] (step=0071325) Train Loss: 5.2623, Train Steps/Sec: 1.07 +[2025-04-26 08:57:06] (step=0071350) Train Loss: 5.1197, Train Steps/Sec: 1.11 +[2025-04-26 08:57:27] (step=0071375) Train Loss: 5.2252, Train Steps/Sec: 1.17 +[2025-04-26 08:57:50] (step=0071400) Train Loss: 5.2624, Train Steps/Sec: 1.12 +[2025-04-26 08:58:11] (step=0071425) Train Loss: 5.2681, Train Steps/Sec: 1.17 +[2025-04-26 08:58:32] (step=0071450) Train Loss: 5.1750, Train Steps/Sec: 1.17 +[2025-04-26 08:58:54] (step=0071475) Train Loss: 5.2488, Train Steps/Sec: 1.17 +[2025-04-26 08:59:15] (step=0071500) Train Loss: 5.1950, Train Steps/Sec: 1.17 +[2025-04-26 08:59:36] (step=0071525) Train Loss: 5.2644, Train Steps/Sec: 1.17 +[2025-04-26 08:59:58] (step=0071550) Train Loss: 5.2776, Train Steps/Sec: 1.17 +[2025-04-26 09:00:19] (step=0071575) Train Loss: 5.2145, Train Steps/Sec: 1.17 +[2025-04-26 09:00:41] (step=0071600) Train Loss: 5.2214, Train Steps/Sec: 1.17 +[2025-04-26 09:01:02] (step=0071625) Train Loss: 5.1590, Train Steps/Sec: 1.17 +[2025-04-26 09:01:23] (step=0071650) Train Loss: 5.2093, Train Steps/Sec: 1.17 +[2025-04-26 09:01:46] (step=0071675) Train Loss: 5.2189, Train Steps/Sec: 1.10 +[2025-04-26 09:02:07] (step=0071700) Train Loss: 5.1737, Train Steps/Sec: 1.17 +[2025-04-26 09:02:30] (step=0071725) Train Loss: 5.2125, Train Steps/Sec: 1.12 +[2025-04-26 09:02:51] (step=0071750) Train Loss: 5.2269, Train Steps/Sec: 1.17 +[2025-04-26 09:03:13] (step=0071775) Train Loss: 5.1398, Train Steps/Sec: 1.12 +[2025-04-26 09:03:35] (step=0071800) Train Loss: 5.1689, Train Steps/Sec: 1.17 +[2025-04-26 09:03:56] (step=0071825) Train Loss: 5.2288, Train Steps/Sec: 1.17 +[2025-04-26 09:04:17] (step=0071850) Train Loss: 5.1150, Train Steps/Sec: 1.17 +[2025-04-26 09:04:39] (step=0071875) Train Loss: 5.2020, Train Steps/Sec: 1.17 +[2025-04-26 09:05:01] (step=0071900) Train Loss: 5.2502, Train Steps/Sec: 1.12 +[2025-04-26 09:05:22] (step=0071925) Train Loss: 5.1836, Train Steps/Sec: 1.18 +[2025-04-26 09:05:44] (step=0071950) Train Loss: 5.3045, Train Steps/Sec: 1.17 +[2025-04-26 09:06:05] (step=0071975) Train Loss: 5.2267, Train Steps/Sec: 1.17 +[2025-04-26 09:06:29] (step=0072000) Train Loss: 5.2362, Train Steps/Sec: 1.04 +[2025-04-26 09:06:29] text_config is None. Initializing the text config with default values (`OPTConfig`). +[2025-04-26 09:11:33] Finish Eval in 72000 steps... +[2025-04-26 09:11:52] Saved checkpoint to checkpoint/Afteri2i_100kckpt_LLAVAT5_empty_Fixed_just_segment_pretrain_stage3_dreambench_recap_400k_t2i_400k_flux_200k_midjourney_150k_recovery_150k_3e4_epoch2_no_rotated/003-GPT-XL/checkpoints/0072000.pt +[2025-04-26 09:12:13] (step=0072025) Train Loss: 5.2627, Train Steps/Sec: 0.07 +[2025-04-26 09:12:34] (step=0072050) Train Loss: 5.1603, Train Steps/Sec: 1.18 +[2025-04-26 09:12:56] (step=0072075) Train Loss: 5.1936, Train Steps/Sec: 1.13 +[2025-04-26 09:13:18] (step=0072100) Train Loss: 5.1464, Train Steps/Sec: 1.17 +[2025-04-26 09:13:39] (step=0072125) Train Loss: 5.1983, Train Steps/Sec: 1.17 +[2025-04-26 09:14:01] (step=0072150) Train Loss: 5.1974, Train Steps/Sec: 1.17 +[2025-04-26 09:14:22] (step=0072175) Train Loss: 5.2031, Train Steps/Sec: 1.16 +[2025-04-26 09:14:43] (step=0072200) Train Loss: 5.1981, Train Steps/Sec: 1.16 +[2025-04-26 09:15:05] (step=0072225) Train Loss: 5.2199, Train Steps/Sec: 1.17 +[2025-04-26 09:15:26] (step=0072250) Train Loss: 5.2361, Train Steps/Sec: 1.17 +[2025-04-26 09:15:48] (step=0072275) Train Loss: 5.1935, Train Steps/Sec: 1.17 +[2025-04-26 09:16:09] (step=0072300) Train Loss: 5.1987, Train Steps/Sec: 1.17 +[2025-04-26 09:16:30] (step=0072325) Train Loss: 5.2612, Train Steps/Sec: 1.17 +[2025-04-26 09:16:52] (step=0072350) Train Loss: 5.1871, Train Steps/Sec: 1.17 +[2025-04-26 09:17:13] (step=0072375) Train Loss: 5.2231, Train Steps/Sec: 1.17 +[2025-04-26 09:17:34] (step=0072400) Train Loss: 5.2451, Train Steps/Sec: 1.17 +[2025-04-26 09:17:56] (step=0072425) Train Loss: 5.2731, Train Steps/Sec: 1.17 +[2025-04-26 09:18:17] (step=0072450) Train Loss: 5.1899, Train Steps/Sec: 1.17 +[2025-04-26 09:18:40] (step=0072475) Train Loss: 5.2209, Train Steps/Sec: 1.11 +[2025-04-26 09:19:02] (step=0072500) Train Loss: 5.2227, Train Steps/Sec: 1.12 +[2025-04-26 09:19:23] (step=0072525) Train Loss: 5.2354, Train Steps/Sec: 1.17 +[2025-04-26 09:19:45] (step=0072550) Train Loss: 5.2398, Train Steps/Sec: 1.13 +[2025-04-26 09:20:07] (step=0072575) Train Loss: 5.1794, Train Steps/Sec: 1.17 +[2025-04-26 09:20:28] (step=0072600) Train Loss: 5.2150, Train Steps/Sec: 1.17 +[2025-04-26 09:20:50] (step=0072625) Train Loss: 5.2829, Train Steps/Sec: 1.17 +[2025-04-26 09:21:13] (step=0072650) Train Loss: 5.1619, Train Steps/Sec: 1.09 +[2025-04-26 09:21:35] (step=0072675) Train Loss: 5.2027, Train Steps/Sec: 1.13 +[2025-04-26 09:21:56] (step=0072700) Train Loss: 5.2222, Train Steps/Sec: 1.17 +[2025-04-26 09:22:19] (step=0072725) Train Loss: 5.2303, Train Steps/Sec: 1.12 +[2025-04-26 09:22:41] (step=0072750) Train Loss: 5.2464, Train Steps/Sec: 1.13 +[2025-04-26 09:23:02] (step=0072775) Train Loss: 5.2538, Train Steps/Sec: 1.17 +[2025-04-26 09:23:23] (step=0072800) Train Loss: 5.2122, Train Steps/Sec: 1.16 +[2025-04-26 09:23:45] (step=0072825) Train Loss: 5.2032, Train Steps/Sec: 1.17 +[2025-04-26 09:24:06] (step=0072850) Train Loss: 5.2186, Train Steps/Sec: 1.17 +[2025-04-26 09:24:27] (step=0072875) Train Loss: 5.1958, Train Steps/Sec: 1.17 +[2025-04-26 09:24:49] (step=0072900) Train Loss: 5.1992, Train Steps/Sec: 1.17 +[2025-04-26 09:25:10] (step=0072925) Train Loss: 5.1741, Train Steps/Sec: 1.17 +[2025-04-26 09:25:32] (step=0072950) Train Loss: 5.2045, Train Steps/Sec: 1.17 +[2025-04-26 09:25:53] (step=0072975) Train Loss: 5.1716, Train Steps/Sec: 1.17 +[2025-04-26 09:26:14] (step=0073000) Train Loss: 5.2231, Train Steps/Sec: 1.17 +[2025-04-26 09:26:36] (step=0073025) Train Loss: 5.2236, Train Steps/Sec: 1.18 +[2025-04-26 09:26:57] (step=0073050) Train Loss: 5.2391, Train Steps/Sec: 1.17 +[2025-04-26 09:27:18] (step=0073075) Train Loss: 5.1965, Train Steps/Sec: 1.17 +[2025-04-26 09:27:40] (step=0073100) Train Loss: 5.2307, Train Steps/Sec: 1.16 +[2025-04-26 09:28:01] (step=0073125) Train Loss: 5.2199, Train Steps/Sec: 1.17 +[2025-04-26 09:28:22] (step=0073150) Train Loss: 5.2918, Train Steps/Sec: 1.17 +[2025-04-26 09:28:44] (step=0073175) Train Loss: 5.2242, Train Steps/Sec: 1.17 +[2025-04-26 09:29:06] (step=0073200) Train Loss: 5.2876, Train Steps/Sec: 1.12 +[2025-04-26 09:29:28] (step=0073225) Train Loss: 5.1798, Train Steps/Sec: 1.12 +[2025-04-26 09:29:50] (step=0073250) Train Loss: 5.2056, Train Steps/Sec: 1.17 +[2025-04-26 09:30:12] (step=0073275) Train Loss: 5.1746, Train Steps/Sec: 1.11 +[2025-04-26 09:30:35] (step=0073300) Train Loss: 5.2564, Train Steps/Sec: 1.08 +[2025-04-26 09:30:57] (step=0073325) Train Loss: 5.2722, Train Steps/Sec: 1.17 +[2025-04-26 09:31:19] (step=0073350) Train Loss: 5.1969, Train Steps/Sec: 1.13 +[2025-04-26 09:31:40] (step=0073375) Train Loss: 5.1830, Train Steps/Sec: 1.17 +[2025-04-26 09:32:01] (step=0073400) Train Loss: 5.2953, Train Steps/Sec: 1.17 +[2025-04-26 09:32:24] (step=0073425) Train Loss: 5.2161, Train Steps/Sec: 1.12 +[2025-04-26 09:32:46] (step=0073450) Train Loss: 5.1851, Train Steps/Sec: 1.12 +[2025-04-26 09:33:07] (step=0073475) Train Loss: 5.1782, Train Steps/Sec: 1.17 +[2025-04-26 09:33:29] (step=0073500) Train Loss: 5.2191, Train Steps/Sec: 1.17 +[2025-04-26 09:33:50] (step=0073525) Train Loss: 5.2895, Train Steps/Sec: 1.17 +[2025-04-26 09:34:12] (step=0073550) Train Loss: 5.1918, Train Steps/Sec: 1.17 +[2025-04-26 09:34:33] (step=0073575) Train Loss: 5.1989, Train Steps/Sec: 1.17 +[2025-04-26 09:34:54] (step=0073600) Train Loss: 5.2376, Train Steps/Sec: 1.16 +[2025-04-26 09:35:16] (step=0073625) Train Loss: 5.0749, Train Steps/Sec: 1.17 +[2025-04-26 09:35:37] (step=0073650) Train Loss: 5.2055, Train Steps/Sec: 1.17 +[2025-04-26 09:35:58] (step=0073675) Train Loss: 5.1818, Train Steps/Sec: 1.17 +[2025-04-26 09:36:20] (step=0073700) Train Loss: 5.2265, Train Steps/Sec: 1.17 +[2025-04-26 09:36:41] (step=0073725) Train Loss: 5.1732, Train Steps/Sec: 1.17 +[2025-04-26 09:37:03] (step=0073750) Train Loss: 5.1412, Train Steps/Sec: 1.17 +[2025-04-26 09:37:24] (step=0073775) Train Loss: 5.2333, Train Steps/Sec: 1.17 +[2025-04-26 09:37:45] (step=0073800) Train Loss: 5.2395, Train Steps/Sec: 1.16 +[2025-04-26 09:38:07] (step=0073825) Train Loss: 5.2150, Train Steps/Sec: 1.17 +[2025-04-26 09:38:28] (step=0073850) Train Loss: 5.2702, Train Steps/Sec: 1.17 +[2025-04-26 09:38:50] (step=0073875) Train Loss: 5.1005, Train Steps/Sec: 1.13 +[2025-04-26 09:39:12] (step=0073900) Train Loss: 5.2028, Train Steps/Sec: 1.17 +[2025-04-26 09:39:33] (step=0073925) Train Loss: 5.2127, Train Steps/Sec: 1.17 +[2025-04-26 09:39:56] (step=0073950) Train Loss: 5.2715, Train Steps/Sec: 1.09 +[2025-04-26 09:40:18] (step=0073975) Train Loss: 5.1776, Train Steps/Sec: 1.13 +[2025-04-26 09:40:40] (step=0074000) Train Loss: 5.1680, Train Steps/Sec: 1.12 +[2025-04-26 09:40:41] text_config is None. Initializing the text config with default values (`OPTConfig`). +[2025-04-26 09:45:38] Finish Eval in 74000 steps... +[2025-04-26 09:45:57] Saved checkpoint to checkpoint/Afteri2i_100kckpt_LLAVAT5_empty_Fixed_just_segment_pretrain_stage3_dreambench_recap_400k_t2i_400k_flux_200k_midjourney_150k_recovery_150k_3e4_epoch2_no_rotated/003-GPT-XL/checkpoints/0074000.pt +[2025-04-26 09:45:59] Removed old checkpoint: checkpoint/Afteri2i_100kckpt_LLAVAT5_empty_Fixed_just_segment_pretrain_stage3_dreambench_recap_400k_t2i_400k_flux_200k_midjourney_150k_recovery_150k_3e4_epoch2_no_rotated/003-GPT-XL/checkpoints/0072000.pt +[2025-04-26 09:46:21] (step=0074025) Train Loss: 5.1184, Train Steps/Sec: 0.07 +[2025-04-26 09:46:42] (step=0074050) Train Loss: 5.3005, Train Steps/Sec: 1.17 +[2025-04-26 09:47:04] (step=0074075) Train Loss: 5.1684, Train Steps/Sec: 1.12 +[2025-04-26 09:47:27] (step=0074100) Train Loss: 5.2449, Train Steps/Sec: 1.11 +[2025-04-26 09:47:48] (step=0074125) Train Loss: 5.1284, Train Steps/Sec: 1.17 +[2025-04-26 09:48:09] (step=0074150) Train Loss: 5.2036, Train Steps/Sec: 1.17 +[2025-04-26 09:48:31] (step=0074175) Train Loss: 5.2195, Train Steps/Sec: 1.17 +[2025-04-26 09:48:52] (step=0074200) Train Loss: 5.2426, Train Steps/Sec: 1.17 +[2025-04-26 09:49:13] (step=0074225) Train Loss: 5.2129, Train Steps/Sec: 1.17 +[2025-04-26 09:49:35] (step=0074250) Train Loss: 5.1796, Train Steps/Sec: 1.17 +[2025-04-26 09:49:56] (step=0074275) Train Loss: 5.2302, Train Steps/Sec: 1.17 +[2025-04-26 09:50:18] (step=0074300) Train Loss: 5.2911, Train Steps/Sec: 1.16 +[2025-04-26 09:50:39] (step=0074325) Train Loss: 5.2462, Train Steps/Sec: 1.17 +[2025-04-26 09:51:00] (step=0074350) Train Loss: 5.2051, Train Steps/Sec: 1.17 +[2025-04-26 09:51:22] (step=0074375) Train Loss: 5.2195, Train Steps/Sec: 1.17 +[2025-04-26 09:51:43] (step=0074400) Train Loss: 5.2866, Train Steps/Sec: 1.16 +[2025-04-26 09:52:04] (step=0074425) Train Loss: 5.2964, Train Steps/Sec: 1.17 +[2025-04-26 09:52:26] (step=0074450) Train Loss: 5.2190, Train Steps/Sec: 1.17 +[2025-04-26 09:52:47] (step=0074475) Train Loss: 5.1818, Train Steps/Sec: 1.17 +[2025-04-26 09:53:09] (step=0074500) Train Loss: 5.2220, Train Steps/Sec: 1.17 +[2025-04-26 09:53:31] (step=0074525) Train Loss: 5.1763, Train Steps/Sec: 1.13 +[2025-04-26 09:53:52] (step=0074550) Train Loss: 5.2442, Train Steps/Sec: 1.17 +[2025-04-26 09:54:13] (step=0074575) Train Loss: 5.2165, Train Steps/Sec: 1.17 +[2025-04-26 09:54:35] (step=0074600) Train Loss: 5.1595, Train Steps/Sec: 1.17 +[2025-04-26 09:54:58] (step=0074625) Train Loss: 5.2413, Train Steps/Sec: 1.09 +[2025-04-26 09:55:19] (step=0074650) Train Loss: 5.1682, Train Steps/Sec: 1.17 +[2025-04-26 09:55:42] (step=0074675) Train Loss: 5.2214, Train Steps/Sec: 1.08 +[2025-04-26 09:56:04] (step=0074700) Train Loss: 5.2202, Train Steps/Sec: 1.17 +[2025-04-26 09:56:26] (step=0074725) Train Loss: 5.3591, Train Steps/Sec: 1.12 +[2025-04-26 09:56:47] (step=0074750) Train Loss: 5.1127, Train Steps/Sec: 1.17 +[2025-04-26 09:57:10] (step=0074775) Train Loss: 5.1991, Train Steps/Sec: 1.12 +[2025-04-26 09:57:31] (step=0074800) Train Loss: 5.2166, Train Steps/Sec: 1.16 +[2025-04-26 09:57:52] (step=0074825) Train Loss: 5.1958, Train Steps/Sec: 1.17 +[2025-04-26 09:58:14] (step=0074850) Train Loss: 5.1683, Train Steps/Sec: 1.17 +[2025-04-26 09:58:36] (step=0074875) Train Loss: 5.1631, Train Steps/Sec: 1.11 +[2025-04-26 09:58:58] (step=0074900) Train Loss: 5.2026, Train Steps/Sec: 1.17 +[2025-04-26 09:59:19] (step=0074925) Train Loss: 5.1976, Train Steps/Sec: 1.17 +[2025-04-26 09:59:40] (step=0074950) Train Loss: 5.1268, Train Steps/Sec: 1.17 +[2025-04-26 10:00:02] (step=0074975) Train Loss: 5.2329, Train Steps/Sec: 1.17 +[2025-04-26 10:00:23] (step=0075000) Train Loss: 5.2005, Train Steps/Sec: 1.17 +[2025-04-26 10:00:44] (step=0075025) Train Loss: 5.2735, Train Steps/Sec: 1.17 +[2025-04-26 10:01:06] (step=0075050) Train Loss: 5.1927, Train Steps/Sec: 1.17 +[2025-04-26 10:01:27] (step=0075075) Train Loss: 5.2109, Train Steps/Sec: 1.17 +[2025-04-26 10:01:49] (step=0075100) Train Loss: 5.1940, Train Steps/Sec: 1.17 +[2025-04-26 10:02:10] (step=0075125) Train Loss: 5.2243, Train Steps/Sec: 1.17 +[2025-04-26 10:02:31] (step=0075150) Train Loss: 5.1904, Train Steps/Sec: 1.17 +[2025-04-26 10:02:53] (step=0075175) Train Loss: 5.2919, Train Steps/Sec: 1.13 +[2025-04-26 10:03:15] (step=0075200) Train Loss: 5.2684, Train Steps/Sec: 1.17 +[2025-04-26 10:03:36] (step=0075225) Train Loss: 5.2402, Train Steps/Sec: 1.17 +[2025-04-26 10:03:57] (step=0075250) Train Loss: 5.2662, Train Steps/Sec: 1.17 +[2025-04-26 10:04:21] (step=0075275) Train Loss: 5.2182, Train Steps/Sec: 1.08 +[2025-04-26 10:04:42] (step=0075300) Train Loss: 5.2925, Train Steps/Sec: 1.17 +[2025-04-26 10:05:03] (step=0075325) Train Loss: 5.3134, Train Steps/Sec: 1.17 +[2025-04-26 10:05:26] (step=0075350) Train Loss: 5.2629, Train Steps/Sec: 1.13 +[2025-04-26 10:05:47] (step=0075375) Train Loss: 5.2108, Train Steps/Sec: 1.17 +[2025-04-26 10:06:09] (step=0075400) Train Loss: 5.2688, Train Steps/Sec: 1.12 +[2025-04-26 10:06:31] (step=0075425) Train Loss: 5.1907, Train Steps/Sec: 1.17 +[2025-04-26 10:06:54] (step=0075450) Train Loss: 5.1866, Train Steps/Sec: 1.08 +[2025-04-26 10:07:15] (step=0075475) Train Loss: 5.2531, Train Steps/Sec: 1.17 +[2025-04-26 10:07:36] (step=0075500) Train Loss: 5.2518, Train Steps/Sec: 1.17 +[2025-04-26 10:07:58] (step=0075525) Train Loss: 5.2192, Train Steps/Sec: 1.17 +[2025-04-26 10:08:19] (step=0075550) Train Loss: 5.2572, Train Steps/Sec: 1.17 +[2025-04-26 10:08:41] (step=0075575) Train Loss: 5.1905, Train Steps/Sec: 1.17 +[2025-04-26 10:09:02] (step=0075600) Train Loss: 5.1872, Train Steps/Sec: 1.16 +[2025-04-26 10:09:23] (step=0075625) Train Loss: 5.2429, Train Steps/Sec: 1.17 +[2025-04-26 10:09:45] (step=0075650) Train Loss: 5.2182, Train Steps/Sec: 1.17 +[2025-04-26 10:10:07] (step=0075675) Train Loss: 5.2292, Train Steps/Sec: 1.12 +[2025-04-26 10:10:29] (step=0075700) Train Loss: 5.1356, Train Steps/Sec: 1.17 +[2025-04-26 10:10:50] (step=0075725) Train Loss: 5.2368, Train Steps/Sec: 1.17 +[2025-04-26 10:11:11] (step=0075750) Train Loss: 5.1997, Train Steps/Sec: 1.17 +[2025-04-26 10:11:32] (step=0075775) Train Loss: 5.2296, Train Steps/Sec: 1.17 +[2025-04-26 10:11:54] (step=0075800) Train Loss: 5.2874, Train Steps/Sec: 1.17 +[2025-04-26 10:12:16] (step=0075825) Train Loss: 5.2431, Train Steps/Sec: 1.13 +[2025-04-26 10:12:37] (step=0075850) Train Loss: 5.3591, Train Steps/Sec: 1.17 +[2025-04-26 10:12:59] (step=0075875) Train Loss: 5.3043, Train Steps/Sec: 1.17 +[2025-04-26 10:13:20] (step=0075900) Train Loss: 5.1970, Train Steps/Sec: 1.17 +[2025-04-26 10:13:42] (step=0075925) Train Loss: 5.2334, Train Steps/Sec: 1.13 +[2025-04-26 10:14:05] (step=0075950) Train Loss: 5.1530, Train Steps/Sec: 1.13 +[2025-04-26 10:14:26] (step=0075975) Train Loss: 5.2604, Train Steps/Sec: 1.17 +[2025-04-26 10:14:47] (step=0076000) Train Loss: 5.1788, Train Steps/Sec: 1.17 +[2025-04-26 10:14:47] text_config is None. Initializing the text config with default values (`OPTConfig`). +[2025-04-26 10:19:51] Finish Eval in 76000 steps... +[2025-04-26 10:20:10] Saved checkpoint to checkpoint/Afteri2i_100kckpt_LLAVAT5_empty_Fixed_just_segment_pretrain_stage3_dreambench_recap_400k_t2i_400k_flux_200k_midjourney_150k_recovery_150k_3e4_epoch2_no_rotated/003-GPT-XL/checkpoints/0076000.pt +[2025-04-26 10:20:12] Removed old checkpoint: checkpoint/Afteri2i_100kckpt_LLAVAT5_empty_Fixed_just_segment_pretrain_stage3_dreambench_recap_400k_t2i_400k_flux_200k_midjourney_150k_recovery_150k_3e4_epoch2_no_rotated/003-GPT-XL/checkpoints/0074000.pt +[2025-04-26 10:20:35] (step=0076025) Train Loss: 5.2282, Train Steps/Sec: 0.07 +[2025-04-26 10:20:56] (step=0076050) Train Loss: 5.1730, Train Steps/Sec: 1.17 +[2025-04-26 10:21:17] (step=0076075) Train Loss: 5.2761, Train Steps/Sec: 1.17 +[2025-04-26 10:21:39] (step=0076100) Train Loss: 5.2569, Train Steps/Sec: 1.17 +[2025-04-26 10:22:02] (step=0076125) Train Loss: 5.3060, Train Steps/Sec: 1.08 +[2025-04-26 10:22:23] (step=0076150) Train Loss: 5.2034, Train Steps/Sec: 1.17 +[2025-04-26 10:22:44] (step=0076175) Train Loss: 5.2180, Train Steps/Sec: 1.17 +[2025-04-26 10:23:06] (step=0076200) Train Loss: 5.2756, Train Steps/Sec: 1.17 +[2025-04-26 10:23:27] (step=0076225) Train Loss: 5.2242, Train Steps/Sec: 1.17 +[2025-04-26 10:23:48] (step=0076250) Train Loss: 5.2432, Train Steps/Sec: 1.17 +[2025-04-26 10:24:10] (step=0076275) Train Loss: 5.2239, Train Steps/Sec: 1.17 +[2025-04-26 10:24:31] (step=0076300) Train Loss: 5.2556, Train Steps/Sec: 1.17 +[2025-04-26 10:24:53] (step=0076325) Train Loss: 5.2450, Train Steps/Sec: 1.17 +[2025-04-26 10:25:14] (step=0076350) Train Loss: 5.2379, Train Steps/Sec: 1.17 +[2025-04-26 10:25:35] (step=0076375) Train Loss: 5.2819, Train Steps/Sec: 1.17 +[2025-04-26 10:25:57] (step=0076400) Train Loss: 5.1849, Train Steps/Sec: 1.17 +[2025-04-26 10:26:18] (step=0076425) Train Loss: 5.1960, Train Steps/Sec: 1.17 +[2025-04-26 10:26:39] (step=0076450) Train Loss: 5.1921, Train Steps/Sec: 1.17 +[2025-04-26 10:27:02] (step=0076475) Train Loss: 5.1592, Train Steps/Sec: 1.08 +[2025-04-26 10:27:24] (step=0076500) Train Loss: 5.1632, Train Steps/Sec: 1.17 +[2025-04-26 10:27:45] (step=0076525) Train Loss: 5.3221, Train Steps/Sec: 1.17 +[2025-04-26 10:28:06] (step=0076550) Train Loss: 5.2393, Train Steps/Sec: 1.18 +[2025-04-26 10:28:29] (step=0076575) Train Loss: 5.1996, Train Steps/Sec: 1.13 +[2025-04-26 10:28:51] (step=0076600) Train Loss: 5.2227, Train Steps/Sec: 1.12 +[2025-04-26 10:29:12] (step=0076625) Train Loss: 5.2115, Train Steps/Sec: 1.17 +[2025-04-26 10:29:34] (step=0076650) Train Loss: 5.1501, Train Steps/Sec: 1.17 +[2025-04-26 10:29:55] (step=0076675) Train Loss: 5.1772, Train Steps/Sec: 1.17 +[2025-04-26 10:30:17] (step=0076700) Train Loss: 5.1620, Train Steps/Sec: 1.12 +[2025-04-26 10:30:39] (step=0076725) Train Loss: 5.1974, Train Steps/Sec: 1.12 +[2025-04-26 10:31:01] (step=0076750) Train Loss: 5.2154, Train Steps/Sec: 1.17 +[2025-04-26 10:31:22] (step=0076775) Train Loss: 5.1880, Train Steps/Sec: 1.17 +[2025-04-26 10:31:44] (step=0076800) Train Loss: 5.2167, Train Steps/Sec: 1.12 +[2025-04-26 10:32:06] (step=0076825) Train Loss: 5.2630, Train Steps/Sec: 1.17 +[2025-04-26 10:32:28] (step=0076850) Train Loss: 5.2477, Train Steps/Sec: 1.13 +[2025-04-26 10:32:49] (step=0076875) Train Loss: 5.3000, Train Steps/Sec: 1.17 +[2025-04-26 10:33:11] (step=0076900) Train Loss: 5.2153, Train Steps/Sec: 1.17 +[2025-04-26 10:33:32] (step=0076925) Train Loss: 5.2121, Train Steps/Sec: 1.17 +[2025-04-26 10:33:53] (step=0076950) Train Loss: 5.1852, Train Steps/Sec: 1.17 +[2025-04-26 10:34:15] (step=0076975) Train Loss: 5.2703, Train Steps/Sec: 1.17 +[2025-04-26 10:34:36] (step=0077000) Train Loss: 5.1002, Train Steps/Sec: 1.17 +[2025-04-26 10:34:57] (step=0077025) Train Loss: 5.2017, Train Steps/Sec: 1.18 +[2025-04-26 10:35:19] (step=0077050) Train Loss: 5.2576, Train Steps/Sec: 1.17 +[2025-04-26 10:35:40] (step=0077075) Train Loss: 5.2865, Train Steps/Sec: 1.17 +[2025-04-26 10:36:01] (step=0077100) Train Loss: 5.2455, Train Steps/Sec: 1.16 +[2025-04-26 10:36:23] (step=0077125) Train Loss: 5.2395, Train Steps/Sec: 1.17 +[2025-04-26 10:36:45] (step=0077150) Train Loss: 5.2484, Train Steps/Sec: 1.13 +[2025-04-26 10:37:06] (step=0077175) Train Loss: 5.2104, Train Steps/Sec: 1.17 +[2025-04-26 10:37:28] (step=0077200) Train Loss: 5.2271, Train Steps/Sec: 1.17 +[2025-04-26 10:37:49] (step=0077225) Train Loss: 5.1997, Train Steps/Sec: 1.18 +[2025-04-26 10:38:12] (step=0077250) Train Loss: 5.2665, Train Steps/Sec: 1.09 +[2025-04-26 10:38:34] (step=0077275) Train Loss: 5.2167, Train Steps/Sec: 1.12 +[2025-04-26 10:38:56] (step=0077300) Train Loss: 5.2560, Train Steps/Sec: 1.17 +[2025-04-26 10:39:17] (step=0077325) Train Loss: 5.2550, Train Steps/Sec: 1.18 +[2025-04-26 10:39:38] (step=0077350) Train Loss: 5.2165, Train Steps/Sec: 1.17 +[2025-04-26 10:40:00] (step=0077375) Train Loss: 5.2506, Train Steps/Sec: 1.13 +[2025-04-26 10:40:22] (step=0077400) Train Loss: 5.2076, Train Steps/Sec: 1.17 +[2025-04-26 10:40:43] (step=0077425) Train Loss: 5.2469, Train Steps/Sec: 1.17 +[2025-04-26 10:41:05] (step=0077450) Train Loss: 5.1338, Train Steps/Sec: 1.13 +[2025-04-26 10:41:27] (step=0077475) Train Loss: 5.1505, Train Steps/Sec: 1.17 +[2025-04-26 10:41:49] (step=0077500) Train Loss: 5.1970, Train Steps/Sec: 1.12 +[2025-04-26 10:42:10] (step=0077525) Train Loss: 5.1828, Train Steps/Sec: 1.17 +[2025-04-26 10:42:32] (step=0077550) Train Loss: 5.3007, Train Steps/Sec: 1.17 +[2025-04-26 10:42:54] (step=0077575) Train Loss: 5.3253, Train Steps/Sec: 1.13 +[2025-04-26 10:43:15] (step=0077600) Train Loss: 5.1145, Train Steps/Sec: 1.17 +[2025-04-26 10:43:36] (step=0077625) Train Loss: 5.1850, Train Steps/Sec: 1.18 +[2025-04-26 10:43:58] (step=0077650) Train Loss: 5.2001, Train Steps/Sec: 1.17 +[2025-04-26 10:44:19] (step=0077675) Train Loss: 5.1618, Train Steps/Sec: 1.17 +[2025-04-26 10:44:40] (step=0077700) Train Loss: 5.2704, Train Steps/Sec: 1.17 +[2025-04-26 10:45:02] (step=0077725) Train Loss: 5.2656, Train Steps/Sec: 1.17 +[2025-04-26 10:45:23] (step=0077750) Train Loss: 5.2015, Train Steps/Sec: 1.17 +[2025-04-26 10:45:44] (step=0077775) Train Loss: 5.2372, Train Steps/Sec: 1.17 +[2025-04-26 10:46:07] (step=0077800) Train Loss: 5.1600, Train Steps/Sec: 1.12 +[2025-04-26 10:46:28] (step=0077825) Train Loss: 5.2210, Train Steps/Sec: 1.17 +[2025-04-26 10:46:49] (step=0077850) Train Loss: 5.2890, Train Steps/Sec: 1.18 +[2025-04-26 10:47:11] (step=0077875) Train Loss: 5.1744, Train Steps/Sec: 1.17 +[2025-04-26 10:47:34] (step=0077900) Train Loss: 5.1005, Train Steps/Sec: 1.08 +[2025-04-26 10:47:55] (step=0077925) Train Loss: 5.2191, Train Steps/Sec: 1.17 +[2025-04-26 10:48:16] (step=0077950) Train Loss: 5.2554, Train Steps/Sec: 1.17 +[2025-04-26 10:48:38] (step=0077975) Train Loss: 5.2580, Train Steps/Sec: 1.17 +[2025-04-26 10:48:59] (step=0078000) Train Loss: 5.2949, Train Steps/Sec: 1.17 +[2025-04-26 10:48:59] text_config is None. Initializing the text config with default values (`OPTConfig`). +[2025-04-26 10:54:03] Finish Eval in 78000 steps... +[2025-04-26 10:54:21] Saved checkpoint to checkpoint/Afteri2i_100kckpt_LLAVAT5_empty_Fixed_just_segment_pretrain_stage3_dreambench_recap_400k_t2i_400k_flux_200k_midjourney_150k_recovery_150k_3e4_epoch2_no_rotated/003-GPT-XL/checkpoints/0078000.pt +[2025-04-26 10:54:23] Removed old checkpoint: checkpoint/Afteri2i_100kckpt_LLAVAT5_empty_Fixed_just_segment_pretrain_stage3_dreambench_recap_400k_t2i_400k_flux_200k_midjourney_150k_recovery_150k_3e4_epoch2_no_rotated/003-GPT-XL/checkpoints/0076000.pt +[2025-04-26 10:54:45] (step=0078025) Train Loss: 5.0942, Train Steps/Sec: 0.07 +[2025-04-26 10:55:06] (step=0078050) Train Loss: 5.2269, Train Steps/Sec: 1.17 +[2025-04-26 10:55:29] (step=0078075) Train Loss: 5.2480, Train Steps/Sec: 1.12 +[2025-04-26 10:55:50] (step=0078100) Train Loss: 5.1732, Train Steps/Sec: 1.17 +[2025-04-26 10:56:11] (step=0078125) Train Loss: 5.2350, Train Steps/Sec: 1.17 +[2025-04-26 10:56:33] (step=0078150) Train Loss: 5.2776, Train Steps/Sec: 1.17 +[2025-04-26 10:56:55] (step=0078175) Train Loss: 5.2423, Train Steps/Sec: 1.13 +[2025-04-26 10:57:16] (step=0078200) Train Loss: 5.2205, Train Steps/Sec: 1.17 +[2025-04-26 10:57:38] (step=0078225) Train Loss: 5.2980, Train Steps/Sec: 1.17 +[2025-04-26 10:57:59] (step=0078250) Train Loss: 5.2624, Train Steps/Sec: 1.17 +[2025-04-26 10:58:20] (step=0078275) Train Loss: 5.2611, Train Steps/Sec: 1.17 +[2025-04-26 10:58:43] (step=0078300) Train Loss: 5.2976, Train Steps/Sec: 1.12 +[2025-04-26 10:59:04] (step=0078325) Train Loss: 5.2013, Train Steps/Sec: 1.17 +[2025-04-26 10:59:25] (step=0078350) Train Loss: 5.1889, Train Steps/Sec: 1.17 +[2025-04-26 10:59:46] (step=0078375) Train Loss: 5.2154, Train Steps/Sec: 1.17 +[2025-04-26 11:00:08] (step=0078400) Train Loss: 5.2249, Train Steps/Sec: 1.17 +[2025-04-26 11:00:29] (step=0078425) Train Loss: 5.1799, Train Steps/Sec: 1.17 +[2025-04-26 11:00:51] (step=0078450) Train Loss: 5.2537, Train Steps/Sec: 1.13 +[2025-04-26 11:01:13] (step=0078475) Train Loss: 5.1908, Train Steps/Sec: 1.17 +[2025-04-26 11:01:34] (step=0078500) Train Loss: 5.2150, Train Steps/Sec: 1.17 +[2025-04-26 11:01:55] (step=0078525) Train Loss: 5.2139, Train Steps/Sec: 1.17 +[2025-04-26 11:02:19] (step=0078550) Train Loss: 5.2097, Train Steps/Sec: 1.08 +[2025-04-26 11:02:40] (step=0078575) Train Loss: 5.1725, Train Steps/Sec: 1.17 +[2025-04-26 11:03:01] (step=0078600) Train Loss: 5.1654, Train Steps/Sec: 1.17 +[2025-04-26 11:03:23] (step=0078625) Train Loss: 5.2390, Train Steps/Sec: 1.17 +[2025-04-26 11:03:44] (step=0078650) Train Loss: 5.2043, Train Steps/Sec: 1.18 +[2025-04-26 11:04:05] (step=0078675) Train Loss: 5.2323, Train Steps/Sec: 1.17 +[2025-04-26 11:04:28] (step=0078700) Train Loss: 5.1712, Train Steps/Sec: 1.12 +[2025-04-26 11:04:50] (step=0078725) Train Loss: 5.3020, Train Steps/Sec: 1.12 +[2025-04-26 11:05:11] (step=0078750) Train Loss: 5.2100, Train Steps/Sec: 1.17 +[2025-04-26 11:05:32] (step=0078775) Train Loss: 5.1714, Train Steps/Sec: 1.17 +[2025-04-26 11:05:54] (step=0078800) Train Loss: 5.2345, Train Steps/Sec: 1.17 +[2025-04-26 11:06:15] (step=0078825) Train Loss: 5.1673, Train Steps/Sec: 1.17 +[2025-04-26 11:06:38] (step=0078850) Train Loss: 5.2388, Train Steps/Sec: 1.12 +[2025-04-26 11:07:00] (step=0078875) Train Loss: 5.1118, Train Steps/Sec: 1.11 +[2025-04-26 11:07:21] (step=0078900) Train Loss: 5.1860, Train Steps/Sec: 1.17 +[2025-04-26 11:07:43] (step=0078925) Train Loss: 5.3031, Train Steps/Sec: 1.17 +[2025-04-26 11:08:04] (step=0078950) Train Loss: 5.2052, Train Steps/Sec: 1.17 +[2025-04-26 11:08:25] (step=0078975) Train Loss: 5.3103, Train Steps/Sec: 1.17 +[2025-04-26 11:08:47] (step=0079000) Train Loss: 5.2245, Train Steps/Sec: 1.17 +[2025-04-26 11:09:09] (step=0079025) Train Loss: 5.2455, Train Steps/Sec: 1.13 +[2025-04-26 11:09:30] (step=0079050) Train Loss: 5.1736, Train Steps/Sec: 1.17 +[2025-04-26 11:09:52] (step=0079075) Train Loss: 5.2344, Train Steps/Sec: 1.17 +[2025-04-26 11:10:14] (step=0079100) Train Loss: 5.1818, Train Steps/Sec: 1.12 +[2025-04-26 11:10:35] (step=0079125) Train Loss: 5.1617, Train Steps/Sec: 1.17 +[2025-04-26 11:10:56] (step=0079150) Train Loss: 5.2646, Train Steps/Sec: 1.17 +[2025-04-26 11:11:18] (step=0079175) Train Loss: 5.2288, Train Steps/Sec: 1.18 +[2025-04-26 11:11:39] (step=0079200) Train Loss: 5.1792, Train Steps/Sec: 1.17 +[2025-04-26 11:12:02] (step=0079225) Train Loss: 5.1763, Train Steps/Sec: 1.09 +[2025-04-26 11:12:23] (step=0079250) Train Loss: 5.1914, Train Steps/Sec: 1.17 +[2025-04-26 11:12:45] (step=0079275) Train Loss: 5.2450, Train Steps/Sec: 1.17 +[2025-04-26 11:13:06] (step=0079300) Train Loss: 5.2746, Train Steps/Sec: 1.17 +[2025-04-26 11:13:27] (step=0079325) Train Loss: 5.1077, Train Steps/Sec: 1.17 +[2025-04-26 11:13:49] (step=0079350) Train Loss: 5.1906, Train Steps/Sec: 1.17 +[2025-04-26 11:14:11] (step=0079375) Train Loss: 5.2699, Train Steps/Sec: 1.13 +[2025-04-26 11:14:32] (step=0079400) Train Loss: 5.2240, Train Steps/Sec: 1.17 +[2025-04-26 11:14:54] (step=0079425) Train Loss: 5.2064, Train Steps/Sec: 1.18 +[2025-04-26 11:15:16] (step=0079450) Train Loss: 5.2317, Train Steps/Sec: 1.12 +[2025-04-26 11:15:37] (step=0079475) Train Loss: 5.2154, Train Steps/Sec: 1.17 +[2025-04-26 11:15:59] (step=0079500) Train Loss: 5.2910, Train Steps/Sec: 1.17 +[2025-04-26 11:16:21] (step=0079525) Train Loss: 5.1139, Train Steps/Sec: 1.13 +[2025-04-26 11:16:42] (step=0079550) Train Loss: 5.2217, Train Steps/Sec: 1.17 +[2025-04-26 11:17:03] (step=0079575) Train Loss: 5.1969, Train Steps/Sec: 1.18 +[2025-04-26 11:17:25] (step=0079600) Train Loss: 5.1557, Train Steps/Sec: 1.17 +[2025-04-26 11:17:46] (step=0079625) Train Loss: 5.1815, Train Steps/Sec: 1.17 +[2025-04-26 11:18:07] (step=0079650) Train Loss: 5.2411, Train Steps/Sec: 1.17 +[2025-04-26 11:18:30] (step=0079675) Train Loss: 5.1448, Train Steps/Sec: 1.12 +[2025-04-26 11:18:51] (step=0079700) Train Loss: 5.2429, Train Steps/Sec: 1.17 +[2025-04-26 11:19:13] (step=0079725) Train Loss: 5.1949, Train Steps/Sec: 1.17 +[2025-04-26 11:19:35] (step=0079750) Train Loss: 5.1465, Train Steps/Sec: 1.12 +[2025-04-26 11:19:57] (step=0079775) Train Loss: 5.2275, Train Steps/Sec: 1.13 +[2025-04-26 11:20:18] (step=0079800) Train Loss: 5.2449, Train Steps/Sec: 1.17 +[2025-04-26 11:20:40] (step=0079825) Train Loss: 5.1761, Train Steps/Sec: 1.17 +[2025-04-26 11:21:01] (step=0079850) Train Loss: 5.2577, Train Steps/Sec: 1.17 +[2025-04-26 11:21:24] (step=0079875) Train Loss: 5.1251, Train Steps/Sec: 1.08 +[2025-04-26 11:21:46] (step=0079900) Train Loss: 5.1968, Train Steps/Sec: 1.17 +[2025-04-26 11:22:07] (step=0079925) Train Loss: 5.2369, Train Steps/Sec: 1.17 +[2025-04-26 11:22:28] (step=0079950) Train Loss: 5.2255, Train Steps/Sec: 1.17 +[2025-04-26 11:22:49] (step=0079975) Train Loss: 5.2699, Train Steps/Sec: 1.17 +[2025-04-26 11:23:11] (step=0080000) Train Loss: 5.1260, Train Steps/Sec: 1.17 +[2025-04-26 11:23:11] text_config is None. Initializing the text config with default values (`OPTConfig`). +[2025-04-26 11:28:14] Finish Eval in 80000 steps... +[2025-04-26 11:28:33] Saved checkpoint to checkpoint/Afteri2i_100kckpt_LLAVAT5_empty_Fixed_just_segment_pretrain_stage3_dreambench_recap_400k_t2i_400k_flux_200k_midjourney_150k_recovery_150k_3e4_epoch2_no_rotated/003-GPT-XL/checkpoints/0080000.pt +[2025-04-26 11:28:35] Removed old checkpoint: checkpoint/Afteri2i_100kckpt_LLAVAT5_empty_Fixed_just_segment_pretrain_stage3_dreambench_recap_400k_t2i_400k_flux_200k_midjourney_150k_recovery_150k_3e4_epoch2_no_rotated/003-GPT-XL/checkpoints/0078000.pt +[2025-04-26 11:28:57] (step=0080025) Train Loss: 5.2299, Train Steps/Sec: 0.07 +[2025-04-26 11:29:19] (step=0080050) Train Loss: 5.2434, Train Steps/Sec: 1.13 +[2025-04-26 11:29:40] (step=0080075) Train Loss: 5.2021, Train Steps/Sec: 1.18 +[2025-04-26 11:30:02] (step=0080100) Train Loss: 5.2045, Train Steps/Sec: 1.17 +[2025-04-26 11:30:23] (step=0080125) Train Loss: 5.1867, Train Steps/Sec: 1.18 +[2025-04-26 11:30:44] (step=0080150) Train Loss: 5.2372, Train Steps/Sec: 1.17 +[2025-04-26 11:31:05] (step=0080175) Train Loss: 5.1763, Train Steps/Sec: 1.17 +[2025-04-26 11:31:28] (step=0080200) Train Loss: 5.2027, Train Steps/Sec: 1.12 +[2025-04-26 11:31:49] (step=0080225) Train Loss: 5.2090, Train Steps/Sec: 1.17 +[2025-04-26 11:32:10] (step=0080250) Train Loss: 5.1535, Train Steps/Sec: 1.17 +[2025-04-26 11:32:32] (step=0080275) Train Loss: 5.2156, Train Steps/Sec: 1.17 +[2025-04-26 11:32:53] (step=0080300) Train Loss: 5.2084, Train Steps/Sec: 1.17 +[2025-04-26 11:33:14] (step=0080325) Train Loss: 5.1890, Train Steps/Sec: 1.17 +[2025-04-26 11:33:36] (step=0080350) Train Loss: 5.2679, Train Steps/Sec: 1.17 +[2025-04-26 11:33:57] (step=0080375) Train Loss: 5.1822, Train Steps/Sec: 1.17 +[2025-04-26 11:34:18] (step=0080400) Train Loss: 5.2923, Train Steps/Sec: 1.17 +[2025-04-26 11:34:41] (step=0080425) Train Loss: 5.2111, Train Steps/Sec: 1.12 +[2025-04-26 11:35:02] (step=0080450) Train Loss: 5.2618, Train Steps/Sec: 1.17 +[2025-04-26 11:35:25] (step=0080475) Train Loss: 5.2126, Train Steps/Sec: 1.07 +[2025-04-26 11:35:47] (step=0080500) Train Loss: 5.2458, Train Steps/Sec: 1.17 +[2025-04-26 11:36:10] (step=0080525) Train Loss: 5.2094, Train Steps/Sec: 1.08 +[2025-04-26 11:36:31] (step=0080550) Train Loss: 5.1727, Train Steps/Sec: 1.16 +[2025-04-26 11:36:53] (step=0080575) Train Loss: 5.2490, Train Steps/Sec: 1.18 +[2025-04-26 11:37:14] (step=0080600) Train Loss: 5.1792, Train Steps/Sec: 1.17 +[2025-04-26 11:37:35] (step=0080625) Train Loss: 5.2775, Train Steps/Sec: 1.18 +[2025-04-26 11:37:57] (step=0080650) Train Loss: 5.2371, Train Steps/Sec: 1.17 +[2025-04-26 11:38:18] (step=0080675) Train Loss: 5.2461, Train Steps/Sec: 1.17 +[2025-04-26 11:38:39] (step=0080700) Train Loss: 5.2001, Train Steps/Sec: 1.17 +[2025-04-26 11:39:02] (step=0080725) Train Loss: 5.2294, Train Steps/Sec: 1.08 +[2025-04-26 11:39:24] (step=0080750) Train Loss: 5.1969, Train Steps/Sec: 1.17 +[2025-04-26 11:39:45] (step=0080775) Train Loss: 5.2025, Train Steps/Sec: 1.17 +[2025-04-26 11:40:06] (step=0080800) Train Loss: 5.2036, Train Steps/Sec: 1.17 +[2025-04-26 11:40:28] (step=0080825) Train Loss: 5.2235, Train Steps/Sec: 1.18 +[2025-04-26 11:40:49] (step=0080850) Train Loss: 5.2662, Train Steps/Sec: 1.17 +[2025-04-26 11:41:11] (step=0080875) Train Loss: 5.2085, Train Steps/Sec: 1.13 +[2025-04-26 11:41:33] (step=0080900) Train Loss: 5.2199, Train Steps/Sec: 1.17 +[2025-04-26 11:41:54] (step=0080925) Train Loss: 5.2501, Train Steps/Sec: 1.17 +[2025-04-26 11:42:15] (step=0080950) Train Loss: 5.2358, Train Steps/Sec: 1.17 +[2025-04-26 11:42:37] (step=0080975) Train Loss: 5.1623, Train Steps/Sec: 1.17 +[2025-04-26 11:42:58] (step=0081000) Train Loss: 5.1754, Train Steps/Sec: 1.17 +[2025-04-26 11:43:19] (step=0081025) Train Loss: 5.2735, Train Steps/Sec: 1.17 +[2025-04-26 11:43:41] (step=0081050) Train Loss: 5.1815, Train Steps/Sec: 1.17 +[2025-04-26 11:44:03] (step=0081075) Train Loss: 5.2468, Train Steps/Sec: 1.13 +[2025-04-26 11:44:24] (step=0081100) Train Loss: 5.2042, Train Steps/Sec: 1.17 +[2025-04-26 11:44:46] (step=0081125) Train Loss: 5.2870, Train Steps/Sec: 1.17 +[2025-04-26 11:45:07] (step=0081150) Train Loss: 5.2561, Train Steps/Sec: 1.17 +[2025-04-26 11:45:28] (step=0081175) Train Loss: 5.2443, Train Steps/Sec: 1.17 +[2025-04-26 11:45:52] (step=0081200) Train Loss: 5.2681, Train Steps/Sec: 1.04 +[2025-04-26 11:46:13] (step=0081225) Train Loss: 5.2193, Train Steps/Sec: 1.17 +[2025-04-26 11:46:35] (step=0081250) Train Loss: 5.2152, Train Steps/Sec: 1.17 +[2025-04-26 11:46:57] (step=0081275) Train Loss: 5.2812, Train Steps/Sec: 1.12 +[2025-04-26 11:47:18] (step=0081300) Train Loss: 5.2006, Train Steps/Sec: 1.17 +[2025-04-26 11:47:40] (step=0081325) Train Loss: 5.2496, Train Steps/Sec: 1.17 +[2025-04-26 11:48:01] (step=0081350) Train Loss: 5.1429, Train Steps/Sec: 1.17 +[2025-04-26 11:48:22] (step=0081375) Train Loss: 5.2397, Train Steps/Sec: 1.17 +[2025-04-26 11:48:45] (step=0081400) Train Loss: 5.2700, Train Steps/Sec: 1.12 +[2025-04-26 11:49:06] (step=0081425) Train Loss: 5.2294, Train Steps/Sec: 1.17 +[2025-04-26 11:49:28] (step=0081450) Train Loss: 5.1557, Train Steps/Sec: 1.12 +[2025-04-26 11:49:50] (step=0081475) Train Loss: 5.2303, Train Steps/Sec: 1.17 +[2025-04-26 11:50:11] (step=0081500) Train Loss: 5.2469, Train Steps/Sec: 1.17 +[2025-04-26 11:50:32] (step=0081525) Train Loss: 5.2575, Train Steps/Sec: 1.17 +[2025-04-26 11:50:55] (step=0081550) Train Loss: 5.1735, Train Steps/Sec: 1.12 +[2025-04-26 11:51:16] (step=0081575) Train Loss: 5.1104, Train Steps/Sec: 1.17 +[2025-04-26 11:51:37] (step=0081600) Train Loss: 5.2384, Train Steps/Sec: 1.17 +[2025-04-26 11:51:59] (step=0081625) Train Loss: 5.1480, Train Steps/Sec: 1.18 +[2025-04-26 11:52:20] (step=0081650) Train Loss: 5.2557, Train Steps/Sec: 1.18 +[2025-04-26 11:52:41] (step=0081675) Train Loss: 5.1780, Train Steps/Sec: 1.17 +[2025-04-26 11:53:03] (step=0081700) Train Loss: 5.2164, Train Steps/Sec: 1.17 +[2025-04-26 11:53:25] (step=0081725) Train Loss: 5.1688, Train Steps/Sec: 1.13 +[2025-04-26 11:53:46] (step=0081750) Train Loss: 5.2312, Train Steps/Sec: 1.18 +[2025-04-26 11:54:07] (step=0081775) Train Loss: 5.1833, Train Steps/Sec: 1.17 +[2025-04-26 11:54:29] (step=0081800) Train Loss: 5.2069, Train Steps/Sec: 1.17 +[2025-04-26 11:54:50] (step=0081825) Train Loss: 5.1078, Train Steps/Sec: 1.17 +[2025-04-26 11:55:13] (step=0081850) Train Loss: 5.2393, Train Steps/Sec: 1.09 +[2025-04-26 11:55:34] (step=0081875) Train Loss: 5.2020, Train Steps/Sec: 1.17 +[2025-04-26 11:55:56] (step=0081900) Train Loss: 5.2522, Train Steps/Sec: 1.17 +[2025-04-26 11:56:18] (step=0081925) Train Loss: 5.1425, Train Steps/Sec: 1.13 +[2025-04-26 11:56:39] (step=0081950) Train Loss: 5.1746, Train Steps/Sec: 1.17 +[2025-04-26 11:57:00] (step=0081975) Train Loss: 5.2081, Train Steps/Sec: 1.17 +[2025-04-26 11:57:22] (step=0082000) Train Loss: 5.1475, Train Steps/Sec: 1.17 +[2025-04-26 11:57:22] text_config is None. Initializing the text config with default values (`OPTConfig`). +[2025-04-26 12:02:25] Finish Eval in 82000 steps... +[2025-04-26 12:02:44] Saved checkpoint to checkpoint/Afteri2i_100kckpt_LLAVAT5_empty_Fixed_just_segment_pretrain_stage3_dreambench_recap_400k_t2i_400k_flux_200k_midjourney_150k_recovery_150k_3e4_epoch2_no_rotated/003-GPT-XL/checkpoints/0082000.pt +[2025-04-26 12:02:46] Removed old checkpoint: checkpoint/Afteri2i_100kckpt_LLAVAT5_empty_Fixed_just_segment_pretrain_stage3_dreambench_recap_400k_t2i_400k_flux_200k_midjourney_150k_recovery_150k_3e4_epoch2_no_rotated/003-GPT-XL/checkpoints/0080000.pt +[2025-04-26 12:03:08] (step=0082025) Train Loss: 5.2146, Train Steps/Sec: 0.07 +[2025-04-26 12:03:29] (step=0082050) Train Loss: 5.2572, Train Steps/Sec: 1.18 +[2025-04-26 12:03:52] (step=0082075) Train Loss: 5.2018, Train Steps/Sec: 1.08 +[2025-04-26 12:04:14] (step=0082100) Train Loss: 5.1821, Train Steps/Sec: 1.17 +[2025-04-26 12:04:35] (step=0082125) Train Loss: 5.2398, Train Steps/Sec: 1.17 +[2025-04-26 12:04:56] (step=0082150) Train Loss: 5.1624, Train Steps/Sec: 1.17 +[2025-04-26 12:05:18] (step=0082175) Train Loss: 5.1693, Train Steps/Sec: 1.17 +[2025-04-26 12:05:39] (step=0082200) Train Loss: 5.1855, Train Steps/Sec: 1.17 +[2025-04-26 12:06:01] (step=0082225) Train Loss: 5.2456, Train Steps/Sec: 1.13 +[2025-04-26 12:06:23] (step=0082250) Train Loss: 5.3109, Train Steps/Sec: 1.17 +[2025-04-26 12:06:44] (step=0082275) Train Loss: 5.2153, Train Steps/Sec: 1.17 +[2025-04-26 12:07:05] (step=0082300) Train Loss: 5.1786, Train Steps/Sec: 1.17 +[2025-04-26 12:07:27] (step=0082325) Train Loss: 5.2174, Train Steps/Sec: 1.17 +[2025-04-26 12:07:48] (step=0082350) Train Loss: 5.1947, Train Steps/Sec: 1.17 +[2025-04-26 12:08:10] (step=0082375) Train Loss: 5.2321, Train Steps/Sec: 1.13 +[2025-04-26 12:08:31] (step=0082400) Train Loss: 5.2587, Train Steps/Sec: 1.17 +[2025-04-26 12:08:53] (step=0082425) Train Loss: 5.3106, Train Steps/Sec: 1.17 +[2025-04-26 12:09:14] (step=0082450) Train Loss: 5.2623, Train Steps/Sec: 1.17 +[2025-04-26 12:09:35] (step=0082475) Train Loss: 5.2453, Train Steps/Sec: 1.17 +[2025-04-26 12:09:58] (step=0082500) Train Loss: 5.2369, Train Steps/Sec: 1.12 +[2025-04-26 12:10:20] (step=0082525) Train Loss: 5.1854, Train Steps/Sec: 1.13 +[2025-04-26 12:10:41] (step=0082550) Train Loss: 5.2246, Train Steps/Sec: 1.17 +[2025-04-26 12:11:03] (step=0082575) Train Loss: 5.2645, Train Steps/Sec: 1.17 +[2025-04-26 12:11:24] (step=0082600) Train Loss: 5.2601, Train Steps/Sec: 1.17 +[2025-04-26 12:11:45] (step=0082625) Train Loss: 5.2164, Train Steps/Sec: 1.17 +[2025-04-26 12:12:07] (step=0082650) Train Loss: 5.2003, Train Steps/Sec: 1.13 +[2025-04-26 12:12:29] (step=0082675) Train Loss: 5.2575, Train Steps/Sec: 1.17 +[2025-04-26 12:12:50] (step=0082700) Train Loss: 5.1848, Train Steps/Sec: 1.17 +[2025-04-26 12:13:13] (step=0082725) Train Loss: 5.2034, Train Steps/Sec: 1.08 +[2025-04-26 12:13:34] (step=0082750) Train Loss: 5.1924, Train Steps/Sec: 1.17 +[2025-04-26 12:13:56] (step=0082775) Train Loss: 5.1899, Train Steps/Sec: 1.17 +[2025-04-26 12:14:17] (step=0082800) Train Loss: 5.2436, Train Steps/Sec: 1.17 +[2025-04-26 12:14:38] (step=0082825) Train Loss: 5.2469, Train Steps/Sec: 1.17 +[2025-04-26 12:15:00] (step=0082850) Train Loss: 5.1976, Train Steps/Sec: 1.17 +[2025-04-26 12:15:22] (step=0082875) Train Loss: 5.2191, Train Steps/Sec: 1.12 +[2025-04-26 12:15:44] (step=0082900) Train Loss: 5.1782, Train Steps/Sec: 1.12 +[2025-04-26 12:16:06] (step=0082925) Train Loss: 5.2027, Train Steps/Sec: 1.17 +[2025-04-26 12:16:27] (step=0082950) Train Loss: 5.2438, Train Steps/Sec: 1.18 +[2025-04-26 12:16:48] (step=0082975) Train Loss: 5.2097, Train Steps/Sec: 1.17 +[2025-04-26 12:17:10] (step=0083000) Train Loss: 5.2168, Train Steps/Sec: 1.17 +[2025-04-26 12:17:32] (step=0083025) Train Loss: 5.1443, Train Steps/Sec: 1.13 +[2025-04-26 12:17:53] (step=0083050) Train Loss: 5.1646, Train Steps/Sec: 1.17 +[2025-04-26 12:18:14] (step=0083075) Train Loss: 5.1859, Train Steps/Sec: 1.17 +[2025-04-26 12:18:36] (step=0083100) Train Loss: 5.2762, Train Steps/Sec: 1.17 +[2025-04-26 12:18:57] (step=0083125) Train Loss: 5.2262, Train Steps/Sec: 1.17 +[2025-04-26 12:19:19] (step=0083150) Train Loss: 5.2372, Train Steps/Sec: 1.12 +[2025-04-26 12:19:41] (step=0083175) Train Loss: 5.2105, Train Steps/Sec: 1.13 +[2025-04-26 12:20:03] (step=0083200) Train Loss: 5.2879, Train Steps/Sec: 1.17 +[2025-04-26 12:20:24] (step=0083225) Train Loss: 5.2919, Train Steps/Sec: 1.18 +[2025-04-26 12:20:45] (step=0083250) Train Loss: 5.2661, Train Steps/Sec: 1.18 +[2025-04-26 12:21:07] (step=0083275) Train Loss: 5.2156, Train Steps/Sec: 1.18 +[2025-04-26 12:21:28] (step=0083300) Train Loss: 5.1600, Train Steps/Sec: 1.17 +[2025-04-26 12:21:49] (step=0083325) Train Loss: 5.2731, Train Steps/Sec: 1.18 +[2025-04-26 12:22:11] (step=0083350) Train Loss: 5.1684, Train Steps/Sec: 1.17 +[2025-04-26 12:22:33] (step=0083375) Train Loss: 5.2394, Train Steps/Sec: 1.13 +[2025-04-26 12:22:55] (step=0083400) Train Loss: 5.3234, Train Steps/Sec: 1.12 +[2025-04-26 12:23:16] (step=0083425) Train Loss: 5.1660, Train Steps/Sec: 1.17 +[2025-04-26 12:23:39] (step=0083450) Train Loss: 5.2341, Train Steps/Sec: 1.12 +[2025-04-26 12:24:00] (step=0083475) Train Loss: 5.3115, Train Steps/Sec: 1.18 +[2025-04-26 12:24:21] (step=0083500) Train Loss: 5.1411, Train Steps/Sec: 1.17 +[2025-04-26 12:24:43] (step=0083525) Train Loss: 5.2177, Train Steps/Sec: 1.17 +[2025-04-26 12:25:04] (step=0083550) Train Loss: 5.2463, Train Steps/Sec: 1.17 +[2025-04-26 12:25:26] (step=0083575) Train Loss: 5.2573, Train Steps/Sec: 1.13 +[2025-04-26 12:25:47] (step=0083600) Train Loss: 5.1722, Train Steps/Sec: 1.17 +[2025-04-26 12:26:09] (step=0083625) Train Loss: 5.2237, Train Steps/Sec: 1.17 +[2025-04-26 12:26:30] (step=0083650) Train Loss: 5.2694, Train Steps/Sec: 1.17 +[2025-04-26 12:26:52] (step=0083675) Train Loss: 5.2693, Train Steps/Sec: 1.13 +[2025-04-26 12:27:15] (step=0083700) Train Loss: 5.2825, Train Steps/Sec: 1.12 +[2025-04-26 12:27:36] (step=0083725) Train Loss: 5.2050, Train Steps/Sec: 1.17 +[2025-04-26 12:27:57] (step=0083750) Train Loss: 5.1875, Train Steps/Sec: 1.17 +[2025-04-26 12:28:19] (step=0083775) Train Loss: 5.2583, Train Steps/Sec: 1.17 +[2025-04-26 12:28:40] (step=0083800) Train Loss: 5.2006, Train Steps/Sec: 1.16 +[2025-04-26 12:29:02] (step=0083825) Train Loss: 5.2362, Train Steps/Sec: 1.13 +[2025-04-26 12:29:24] (step=0083850) Train Loss: 5.2831, Train Steps/Sec: 1.13 +[2025-04-26 12:29:46] (step=0083875) Train Loss: 5.2392, Train Steps/Sec: 1.17 +[2025-04-26 12:30:07] (step=0083900) Train Loss: 5.1922, Train Steps/Sec: 1.17 +[2025-04-26 12:30:28] (step=0083925) Train Loss: 5.1389, Train Steps/Sec: 1.17 +[2025-04-26 12:30:50] (step=0083950) Train Loss: 5.2510, Train Steps/Sec: 1.17 +[2025-04-26 12:31:11] (step=0083975) Train Loss: 5.2281, Train Steps/Sec: 1.17 +[2025-04-26 12:31:32] (step=0084000) Train Loss: 5.1968, Train Steps/Sec: 1.17 +[2025-04-26 12:31:32] text_config is None. Initializing the text config with default values (`OPTConfig`). +[2025-04-26 12:36:36] Finish Eval in 84000 steps... +[2025-04-26 12:36:57] Saved checkpoint to checkpoint/Afteri2i_100kckpt_LLAVAT5_empty_Fixed_just_segment_pretrain_stage3_dreambench_recap_400k_t2i_400k_flux_200k_midjourney_150k_recovery_150k_3e4_epoch2_no_rotated/003-GPT-XL/checkpoints/0084000.pt +[2025-04-26 12:36:59] Removed old checkpoint: checkpoint/Afteri2i_100kckpt_LLAVAT5_empty_Fixed_just_segment_pretrain_stage3_dreambench_recap_400k_t2i_400k_flux_200k_midjourney_150k_recovery_150k_3e4_epoch2_no_rotated/003-GPT-XL/checkpoints/0082000.pt +[2025-04-26 12:37:20] (step=0084025) Train Loss: 5.2378, Train Steps/Sec: 0.07 +[2025-04-26 12:37:42] (step=0084050) Train Loss: 5.1547, Train Steps/Sec: 1.17 +[2025-04-26 12:38:03] (step=0084075) Train Loss: 5.2813, Train Steps/Sec: 1.17 +[2025-04-26 12:38:25] (step=0084100) Train Loss: 5.1336, Train Steps/Sec: 1.12 +[2025-04-26 12:38:47] (step=0084125) Train Loss: 5.3587, Train Steps/Sec: 1.13 +[2025-04-26 12:39:09] (step=0084150) Train Loss: 5.1854, Train Steps/Sec: 1.17 +[2025-04-26 12:39:30] (step=0084175) Train Loss: 5.2343, Train Steps/Sec: 1.17 +[2025-04-26 12:39:51] (step=0084200) Train Loss: 5.2323, Train Steps/Sec: 1.17 +[2025-04-26 12:40:12] (step=0084225) Train Loss: 5.2228, Train Steps/Sec: 1.17 +[2025-04-26 12:40:35] (step=0084250) Train Loss: 5.1654, Train Steps/Sec: 1.13 +[2025-04-26 12:40:56] (step=0084275) Train Loss: 5.3421, Train Steps/Sec: 1.17 +[2025-04-26 12:41:17] (step=0084300) Train Loss: 5.2484, Train Steps/Sec: 1.17 +[2025-04-26 12:41:39] (step=0084325) Train Loss: 5.1976, Train Steps/Sec: 1.13 +[2025-04-26 12:42:01] (step=0084350) Train Loss: 5.2352, Train Steps/Sec: 1.17 +[2025-04-26 12:42:22] (step=0084375) Train Loss: 5.2759, Train Steps/Sec: 1.17 +[2025-04-26 12:42:44] (step=0084400) Train Loss: 5.2078, Train Steps/Sec: 1.16 +[2025-04-26 12:43:05] (step=0084425) Train Loss: 5.2184, Train Steps/Sec: 1.17 +[2025-04-26 12:43:26] (step=0084450) Train Loss: 5.2634, Train Steps/Sec: 1.17 +[2025-04-26 12:43:49] (step=0084475) Train Loss: 5.1496, Train Steps/Sec: 1.12 +[2025-04-26 12:44:12] (step=0084500) Train Loss: 5.2304, Train Steps/Sec: 1.08 +[2025-04-26 12:44:33] (step=0084525) Train Loss: 5.2540, Train Steps/Sec: 1.17 +[2025-04-26 12:44:54] (step=0084550) Train Loss: 5.2330, Train Steps/Sec: 1.18 +[2025-04-26 12:45:16] (step=0084575) Train Loss: 5.1535, Train Steps/Sec: 1.18 +[2025-04-26 12:45:37] (step=0084600) Train Loss: 5.1684, Train Steps/Sec: 1.17 +[2025-04-26 12:45:58] (step=0084625) Train Loss: 5.1829, Train Steps/Sec: 1.17 +[2025-04-26 12:46:20] (step=0084650) Train Loss: 5.2168, Train Steps/Sec: 1.17 +[2025-04-26 12:46:41] (step=0084675) Train Loss: 5.2803, Train Steps/Sec: 1.17 +[2025-04-26 12:47:02] (step=0084700) Train Loss: 5.2972, Train Steps/Sec: 1.17 +[2025-04-26 12:47:25] (step=0084725) Train Loss: 5.1646, Train Steps/Sec: 1.11 +[2025-04-26 12:47:47] (step=0084750) Train Loss: 5.2072, Train Steps/Sec: 1.12 +[2025-04-26 12:48:08] (step=0084775) Train Loss: 5.1674, Train Steps/Sec: 1.17 +[2025-04-26 12:48:30] (step=0084800) Train Loss: 5.3115, Train Steps/Sec: 1.17 +[2025-04-26 12:48:51] (step=0084825) Train Loss: 5.2484, Train Steps/Sec: 1.17 +[2025-04-26 12:49:13] (step=0084850) Train Loss: 5.2287, Train Steps/Sec: 1.13 +[2025-04-26 12:49:35] (step=0084875) Train Loss: 5.1467, Train Steps/Sec: 1.18 +[2025-04-26 12:49:56] (step=0084900) Train Loss: 5.2955, Train Steps/Sec: 1.16 +[2025-04-26 12:50:18] (step=0084925) Train Loss: 5.2105, Train Steps/Sec: 1.13 +[2025-04-26 12:50:39] (step=0084950) Train Loss: 5.1113, Train Steps/Sec: 1.17 +[2025-04-26 12:51:01] (step=0084975) Train Loss: 5.1397, Train Steps/Sec: 1.17 +[2025-04-26 12:51:23] (step=0085000) Train Loss: 5.1740, Train Steps/Sec: 1.12 +[2025-04-26 12:51:44] (step=0085025) Train Loss: 5.2634, Train Steps/Sec: 1.17 +[2025-04-26 12:52:06] (step=0085050) Train Loss: 5.2193, Train Steps/Sec: 1.17 +[2025-04-26 12:52:27] (step=0085075) Train Loss: 5.2725, Train Steps/Sec: 1.17 +[2025-04-26 12:52:48] (step=0085100) Train Loss: 5.2572, Train Steps/Sec: 1.17 +[2025-04-26 12:53:10] (step=0085125) Train Loss: 5.1184, Train Steps/Sec: 1.17 +[2025-04-26 12:53:33] (step=0085150) Train Loss: 5.2053, Train Steps/Sec: 1.09 +[2025-04-26 12:53:54] (step=0085175) Train Loss: 5.2049, Train Steps/Sec: 1.18 +[2025-04-26 12:54:15] (step=0085200) Train Loss: 5.2862, Train Steps/Sec: 1.17 +[2025-04-26 12:54:37] (step=0085225) Train Loss: 5.2122, Train Steps/Sec: 1.18 +[2025-04-26 12:54:58] (step=0085250) Train Loss: 5.2593, Train Steps/Sec: 1.17 +[2025-04-26 12:55:19] (step=0085275) Train Loss: 5.2075, Train Steps/Sec: 1.17 +[2025-04-26 12:55:42] (step=0085300) Train Loss: 5.1088, Train Steps/Sec: 1.11 +[2025-04-26 12:56:03] (step=0085325) Train Loss: 5.2390, Train Steps/Sec: 1.17 +[2025-04-26 12:56:24] (step=0085350) Train Loss: 5.2014, Train Steps/Sec: 1.17 +[2025-04-26 12:56:46] (step=0085375) Train Loss: 5.2358, Train Steps/Sec: 1.17 +[2025-04-26 12:57:07] (step=0085400) Train Loss: 5.2248, Train Steps/Sec: 1.17 +[2025-04-26 12:57:29] (step=0085425) Train Loss: 5.2480, Train Steps/Sec: 1.13 +[2025-04-26 12:57:52] (step=0085450) Train Loss: 5.2202, Train Steps/Sec: 1.11 +[2025-04-26 12:58:13] (step=0085475) Train Loss: 5.2568, Train Steps/Sec: 1.18 +[2025-04-26 12:58:34] (step=0085500) Train Loss: 5.1838, Train Steps/Sec: 1.17 +[2025-04-26 12:58:56] (step=0085525) Train Loss: 5.2175, Train Steps/Sec: 1.17 +[2025-04-26 12:59:17] (step=0085550) Train Loss: 5.2222, Train Steps/Sec: 1.17 +[2025-04-26 12:59:39] (step=0085575) Train Loss: 5.2632, Train Steps/Sec: 1.13 +[2025-04-26 13:00:01] (step=0085600) Train Loss: 5.2269, Train Steps/Sec: 1.12 +[2025-04-26 13:00:23] (step=0085625) Train Loss: 5.2224, Train Steps/Sec: 1.17 +[2025-04-26 13:00:45] (step=0085650) Train Loss: 5.1772, Train Steps/Sec: 1.13 +[2025-04-26 13:01:06] (step=0085675) Train Loss: 5.1836, Train Steps/Sec: 1.18 +[2025-04-26 13:01:27] (step=0085700) Train Loss: 5.3223, Train Steps/Sec: 1.17 +[2025-04-26 13:01:49] (step=0085725) Train Loss: 5.2029, Train Steps/Sec: 1.17 +[2025-04-26 13:02:10] (step=0085750) Train Loss: 5.1804, Train Steps/Sec: 1.17 +[2025-04-26 13:02:31] (step=0085775) Train Loss: 5.2954, Train Steps/Sec: 1.17 +[2025-04-26 13:02:54] (step=0085800) Train Loss: 5.2202, Train Steps/Sec: 1.12 +[2025-04-26 13:03:16] (step=0085825) Train Loss: 5.2971, Train Steps/Sec: 1.13 +[2025-04-26 13:03:37] (step=0085850) Train Loss: 5.2220, Train Steps/Sec: 1.17 +[2025-04-26 13:03:59] (step=0085875) Train Loss: 5.2304, Train Steps/Sec: 1.17 +[2025-04-26 13:04:20] (step=0085900) Train Loss: 5.2367, Train Steps/Sec: 1.17 +[2025-04-26 13:04:41] (step=0085925) Train Loss: 5.2397, Train Steps/Sec: 1.17 +[2025-04-26 13:05:03] (step=0085950) Train Loss: 5.2447, Train Steps/Sec: 1.17 +[2025-04-26 13:05:24] (step=0085975) Train Loss: 5.2599, Train Steps/Sec: 1.17 +[2025-04-26 13:05:45] (step=0086000) Train Loss: 5.2693, Train Steps/Sec: 1.17 +[2025-04-26 13:05:45] text_config is None. Initializing the text config with default values (`OPTConfig`). +[2025-04-26 13:10:49] Finish Eval in 86000 steps... +[2025-04-26 13:11:08] Saved checkpoint to checkpoint/Afteri2i_100kckpt_LLAVAT5_empty_Fixed_just_segment_pretrain_stage3_dreambench_recap_400k_t2i_400k_flux_200k_midjourney_150k_recovery_150k_3e4_epoch2_no_rotated/003-GPT-XL/checkpoints/0086000.pt +[2025-04-26 13:11:10] Removed old checkpoint: checkpoint/Afteri2i_100kckpt_LLAVAT5_empty_Fixed_just_segment_pretrain_stage3_dreambench_recap_400k_t2i_400k_flux_200k_midjourney_150k_recovery_150k_3e4_epoch2_no_rotated/003-GPT-XL/checkpoints/0084000.pt +[2025-04-26 13:11:32] (step=0086025) Train Loss: 5.2534, Train Steps/Sec: 0.07 +[2025-04-26 13:11:53] (step=0086050) Train Loss: 5.1996, Train Steps/Sec: 1.17 +[2025-04-26 13:12:14] (step=0086075) Train Loss: 5.2011, Train Steps/Sec: 1.17 +[2025-04-26 13:12:38] (step=0086100) Train Loss: 5.2901, Train Steps/Sec: 1.07 +[2025-04-26 13:12:59] (step=0086125) Train Loss: 5.1875, Train Steps/Sec: 1.17 +[2025-04-26 13:13:20] (step=0086150) Train Loss: 5.1956, Train Steps/Sec: 1.17 +[2025-04-26 13:13:42] (step=0086175) Train Loss: 5.2781, Train Steps/Sec: 1.17 +[2025-04-26 13:14:03] (step=0086200) Train Loss: 5.2062, Train Steps/Sec: 1.17 +[2025-04-26 13:14:25] (step=0086225) Train Loss: 5.2333, Train Steps/Sec: 1.17 +[2025-04-26 13:14:46] (step=0086250) Train Loss: 5.2621, Train Steps/Sec: 1.17 +[2025-04-26 13:15:09] (step=0086275) Train Loss: 5.2492, Train Steps/Sec: 1.11 +[2025-04-26 13:15:32] (step=0086300) Train Loss: 5.1617, Train Steps/Sec: 1.08 +[2025-04-26 13:15:53] (step=0086325) Train Loss: 5.2847, Train Steps/Sec: 1.16 +[2025-04-26 13:16:15] (step=0086350) Train Loss: 5.2623, Train Steps/Sec: 1.16 +[2025-04-26 13:16:36] (step=0086375) Train Loss: 5.1861, Train Steps/Sec: 1.17 +[2025-04-26 13:16:58] (step=0086400) Train Loss: 5.2379, Train Steps/Sec: 1.17 +[2025-04-26 13:17:19] (step=0086425) Train Loss: 5.1802, Train Steps/Sec: 1.18 +[2025-04-26 13:17:41] (step=0086450) Train Loss: 5.2266, Train Steps/Sec: 1.13 +[2025-04-26 13:18:03] (step=0086475) Train Loss: 5.2178, Train Steps/Sec: 1.13 +[2025-04-26 13:18:25] (step=0086500) Train Loss: 5.2630, Train Steps/Sec: 1.17 +[2025-04-26 13:18:46] (step=0086525) Train Loss: 5.1679, Train Steps/Sec: 1.17 +[2025-04-26 13:19:07] (step=0086550) Train Loss: 5.2826, Train Steps/Sec: 1.17 +[2025-04-26 13:19:29] (step=0086575) Train Loss: 5.2457, Train Steps/Sec: 1.17 +[2025-04-26 13:19:50] (step=0086600) Train Loss: 5.1706, Train Steps/Sec: 1.17 +[2025-04-26 13:20:11] (step=0086625) Train Loss: 5.2522, Train Steps/Sec: 1.17 +[2025-04-26 13:20:33] (step=0086650) Train Loss: 5.2711, Train Steps/Sec: 1.17 +[2025-04-26 13:20:54] (step=0086675) Train Loss: 5.2326, Train Steps/Sec: 1.17 +[2025-04-26 13:21:16] (step=0086700) Train Loss: 5.2066, Train Steps/Sec: 1.17 +[2025-04-26 13:21:38] (step=0086725) Train Loss: 5.2541, Train Steps/Sec: 1.12 +[2025-04-26 13:21:59] (step=0086750) Train Loss: 5.2955, Train Steps/Sec: 1.17 +[2025-04-26 13:22:21] (step=0086775) Train Loss: 5.2549, Train Steps/Sec: 1.12 +[2025-04-26 13:22:43] (step=0086800) Train Loss: 5.1885, Train Steps/Sec: 1.17 +[2025-04-26 13:23:04] (step=0086825) Train Loss: 5.1905, Train Steps/Sec: 1.17 +[2025-04-26 13:23:25] (step=0086850) Train Loss: 5.2981, Train Steps/Sec: 1.18 +[2025-04-26 13:23:47] (step=0086875) Train Loss: 5.2581, Train Steps/Sec: 1.17 +[2025-04-26 13:24:09] (step=0086900) Train Loss: 5.2495, Train Steps/Sec: 1.12 +[2025-04-26 13:24:30] (step=0086925) Train Loss: 5.2624, Train Steps/Sec: 1.17 +[2025-04-26 13:24:53] (step=0086950) Train Loss: 5.2537, Train Steps/Sec: 1.08 +[2025-04-26 13:25:15] (step=0086975) Train Loss: 5.1669, Train Steps/Sec: 1.18 +[2025-04-26 13:25:36] (step=0087000) Train Loss: 5.2642, Train Steps/Sec: 1.17 +[2025-04-26 13:25:58] (step=0087025) Train Loss: 5.2317, Train Steps/Sec: 1.13 +[2025-04-26 13:26:19] (step=0087050) Train Loss: 5.2547, Train Steps/Sec: 1.17 +[2025-04-26 13:26:41] (step=0087075) Train Loss: 5.2482, Train Steps/Sec: 1.18 +[2025-04-26 13:27:03] (step=0087100) Train Loss: 5.2513, Train Steps/Sec: 1.12 +[2025-04-26 13:27:25] (step=0087125) Train Loss: 5.2012, Train Steps/Sec: 1.13 +[2025-04-26 13:27:47] (step=0087150) Train Loss: 5.3794, Train Steps/Sec: 1.17 +[2025-04-26 13:28:08] (step=0087175) Train Loss: 5.2513, Train Steps/Sec: 1.18 +[2025-04-26 13:28:29] (step=0087200) Train Loss: 5.2792, Train Steps/Sec: 1.17 +[2025-04-26 13:28:50] (step=0087225) Train Loss: 5.2440, Train Steps/Sec: 1.17 +[2025-04-26 13:29:12] (step=0087250) Train Loss: 5.2771, Train Steps/Sec: 1.17 +[2025-04-26 13:29:33] (step=0087275) Train Loss: 5.2096, Train Steps/Sec: 1.17 +[2025-04-26 13:29:58] (step=0087300) Train Loss: 5.1818, Train Steps/Sec: 1.01 +[2025-04-26 13:30:19] (step=0087325) Train Loss: 5.2443, Train Steps/Sec: 1.17 +[2025-04-26 13:30:41] (step=0087350) Train Loss: 5.2058, Train Steps/Sec: 1.17 +[2025-04-26 13:31:02] (step=0087375) Train Loss: 5.2147, Train Steps/Sec: 1.17 +[2025-04-26 13:31:23] (step=0087400) Train Loss: 5.2235, Train Steps/Sec: 1.17 +[2025-04-26 13:31:45] (step=0087425) Train Loss: 5.2480, Train Steps/Sec: 1.18 +[2025-04-26 13:32:08] (step=0087450) Train Loss: 5.2324, Train Steps/Sec: 1.07 +[2025-04-26 13:32:29] (step=0087475) Train Loss: 5.2938, Train Steps/Sec: 1.18 +[2025-04-26 13:32:50] (step=0087500) Train Loss: 5.3411, Train Steps/Sec: 1.17 +[2025-04-26 13:33:12] (step=0087525) Train Loss: 5.2652, Train Steps/Sec: 1.17 +[2025-04-26 13:33:33] (step=0087550) Train Loss: 5.2339, Train Steps/Sec: 1.18 +[2025-04-26 13:33:54] (step=0087575) Train Loss: 5.1793, Train Steps/Sec: 1.17 +[2025-04-26 13:34:17] (step=0087600) Train Loss: 5.2982, Train Steps/Sec: 1.12 +[2025-04-26 13:34:39] (step=0087625) Train Loss: 5.1681, Train Steps/Sec: 1.13 +[2025-04-26 13:35:00] (step=0087650) Train Loss: 5.1411, Train Steps/Sec: 1.17 +[2025-04-26 13:35:21] (step=0087675) Train Loss: 5.1968, Train Steps/Sec: 1.17 +[2025-04-26 13:35:44] (step=0087700) Train Loss: 5.2241, Train Steps/Sec: 1.11 +[2025-04-26 13:36:05] (step=0087725) Train Loss: 5.1682, Train Steps/Sec: 1.17 +[2025-04-26 13:36:27] (step=0087750) Train Loss: 5.2304, Train Steps/Sec: 1.13 +[2025-04-26 13:36:50] (step=0087775) Train Loss: 5.2376, Train Steps/Sec: 1.12 +[2025-04-26 13:37:12] (step=0087800) Train Loss: 5.3087, Train Steps/Sec: 1.12 +[2025-04-26 13:37:33] (step=0087825) Train Loss: 5.2120, Train Steps/Sec: 1.17 +[2025-04-26 13:37:55] (step=0087850) Train Loss: 5.2760, Train Steps/Sec: 1.17 +[2025-04-26 13:38:16] (step=0087875) Train Loss: 5.2433, Train Steps/Sec: 1.17 +[2025-04-26 13:38:38] (step=0087900) Train Loss: 5.1919, Train Steps/Sec: 1.17 +[2025-04-26 13:38:59] (step=0087925) Train Loss: 5.3212, Train Steps/Sec: 1.17 +[2025-04-26 13:39:20] (step=0087950) Train Loss: 5.2428, Train Steps/Sec: 1.17 +[2025-04-26 13:39:41] (step=0087975) Train Loss: 5.1635, Train Steps/Sec: 1.17 +[2025-04-26 13:40:03] (step=0088000) Train Loss: 5.1698, Train Steps/Sec: 1.17 +[2025-04-26 13:40:03] text_config is None. Initializing the text config with default values (`OPTConfig`). +[2025-04-26 13:45:09] Finish Eval in 88000 steps... +[2025-04-26 13:45:28] Saved checkpoint to checkpoint/Afteri2i_100kckpt_LLAVAT5_empty_Fixed_just_segment_pretrain_stage3_dreambench_recap_400k_t2i_400k_flux_200k_midjourney_150k_recovery_150k_3e4_epoch2_no_rotated/003-GPT-XL/checkpoints/0088000.pt +[2025-04-26 13:45:30] Removed old checkpoint: checkpoint/Afteri2i_100kckpt_LLAVAT5_empty_Fixed_just_segment_pretrain_stage3_dreambench_recap_400k_t2i_400k_flux_200k_midjourney_150k_recovery_150k_3e4_epoch2_no_rotated/003-GPT-XL/checkpoints/0086000.pt +[2025-04-26 13:45:52] (step=0088025) Train Loss: 5.1892, Train Steps/Sec: 0.07 +[2025-04-26 13:46:13] (step=0088050) Train Loss: 5.2951, Train Steps/Sec: 1.18 +[2025-04-26 13:46:34] (step=0088075) Train Loss: 5.2612, Train Steps/Sec: 1.17 +[2025-04-26 13:46:57] (step=0088100) Train Loss: 5.2667, Train Steps/Sec: 1.12 +[2025-04-26 13:47:18] (step=0088125) Train Loss: 5.2525, Train Steps/Sec: 1.17 +[2025-04-26 13:47:39] (step=0088150) Train Loss: 5.2326, Train Steps/Sec: 1.17 +[2025-04-26 13:48:00] (step=0088175) Train Loss: 5.2339, Train Steps/Sec: 1.17 +[2025-04-26 13:48:22] (step=0088200) Train Loss: 5.2267, Train Steps/Sec: 1.17 +[2025-04-26 13:48:43] (step=0088225) Train Loss: 5.3199, Train Steps/Sec: 1.17 +[2025-04-26 13:49:05] (step=0088250) Train Loss: 5.2083, Train Steps/Sec: 1.16 +[2025-04-26 13:49:27] (step=0088275) Train Loss: 5.2533, Train Steps/Sec: 1.13 +[2025-04-26 13:49:48] (step=0088300) Train Loss: 5.1575, Train Steps/Sec: 1.17 +[2025-04-26 13:50:10] (step=0088325) Train Loss: 5.3107, Train Steps/Sec: 1.13 +[2025-04-26 13:50:32] (step=0088350) Train Loss: 5.2492, Train Steps/Sec: 1.17 +[2025-04-26 13:50:53] (step=0088375) Train Loss: 5.2424, Train Steps/Sec: 1.17 +[2025-04-26 13:51:15] (step=0088400) Train Loss: 5.2286, Train Steps/Sec: 1.17 +[2025-04-26 13:51:37] (step=0088425) Train Loss: 5.1782, Train Steps/Sec: 1.13 +[2025-04-26 13:51:59] (step=0088450) Train Loss: 5.1716, Train Steps/Sec: 1.13 +[2025-04-26 13:52:21] (step=0088475) Train Loss: 5.1746, Train Steps/Sec: 1.13 +[2025-04-26 13:52:43] (step=0088500) Train Loss: 5.1612, Train Steps/Sec: 1.12 +[2025-04-26 13:53:05] (step=0088525) Train Loss: 5.1721, Train Steps/Sec: 1.17 +[2025-04-26 13:53:26] (step=0088550) Train Loss: 5.2572, Train Steps/Sec: 1.17 +[2025-04-26 13:53:47] (step=0088575) Train Loss: 5.2578, Train Steps/Sec: 1.17 +[2025-04-26 13:54:09] (step=0088600) Train Loss: 5.2059, Train Steps/Sec: 1.17 +[2025-04-26 13:54:30] (step=0088625) Train Loss: 5.1815, Train Steps/Sec: 1.17 +[2025-04-26 13:54:51] (step=0088650) Train Loss: 5.1503, Train Steps/Sec: 1.17 +[2025-04-26 13:55:13] (step=0088675) Train Loss: 5.2386, Train Steps/Sec: 1.17 +[2025-04-26 13:55:34] (step=0088700) Train Loss: 5.1779, Train Steps/Sec: 1.17 +[2025-04-26 13:55:56] (step=0088725) Train Loss: 5.2298, Train Steps/Sec: 1.12 +[2025-04-26 13:56:18] (step=0088750) Train Loss: 5.1594, Train Steps/Sec: 1.17 +[2025-04-26 13:56:40] (step=0088775) Train Loss: 5.2273, Train Steps/Sec: 1.13 +[2025-04-26 13:57:01] (step=0088800) Train Loss: 5.1788, Train Steps/Sec: 1.16 +[2025-04-26 13:57:23] (step=0088825) Train Loss: 5.2487, Train Steps/Sec: 1.17 +[2025-04-26 13:57:44] (step=0088850) Train Loss: 5.2479, Train Steps/Sec: 1.17 +[2025-04-26 13:58:06] (step=0088875) Train Loss: 5.2078, Train Steps/Sec: 1.16 +[2025-04-26 13:58:27] (step=0088900) Train Loss: 5.2527, Train Steps/Sec: 1.15 +[2025-04-26 13:58:50] (step=0088925) Train Loss: 5.2223, Train Steps/Sec: 1.11 +[2025-04-26 13:59:11] (step=0088950) Train Loss: 5.2307, Train Steps/Sec: 1.16 +[2025-04-26 13:59:33] (step=0088975) Train Loss: 5.2801, Train Steps/Sec: 1.17 +[2025-04-26 13:59:55] (step=0089000) Train Loss: 5.2540, Train Steps/Sec: 1.12 +[2025-04-26 14:00:17] (step=0089025) Train Loss: 5.1873, Train Steps/Sec: 1.16 +[2025-04-26 14:00:38] (step=0089050) Train Loss: 5.2626, Train Steps/Sec: 1.17 +[2025-04-26 14:01:00] (step=0089075) Train Loss: 5.2666, Train Steps/Sec: 1.12 +[2025-04-26 14:01:23] (step=0089100) Train Loss: 5.1995, Train Steps/Sec: 1.12 +[2025-04-26 14:01:44] (step=0089125) Train Loss: 5.2048, Train Steps/Sec: 1.17 +[2025-04-26 14:02:05] (step=0089150) Train Loss: 5.2233, Train Steps/Sec: 1.17 +[2025-04-26 14:02:27] (step=0089175) Train Loss: 5.2017, Train Steps/Sec: 1.16 +[2025-04-26 14:02:49] (step=0089200) Train Loss: 5.2886, Train Steps/Sec: 1.12 +[2025-04-26 14:03:10] (step=0089225) Train Loss: 5.3054, Train Steps/Sec: 1.17 +[2025-04-26 14:03:32] (step=0089250) Train Loss: 5.2428, Train Steps/Sec: 1.17 +[2025-04-26 14:03:53] (step=0089275) Train Loss: 5.1640, Train Steps/Sec: 1.16 +[2025-04-26 14:04:16] (step=0089300) Train Loss: 5.1989, Train Steps/Sec: 1.10 +[2025-04-26 14:04:38] (step=0089325) Train Loss: 5.1712, Train Steps/Sec: 1.17 +[2025-04-26 14:04:59] (step=0089350) Train Loss: 5.1662, Train Steps/Sec: 1.17 +[2025-04-26 14:05:20] (step=0089375) Train Loss: 5.2355, Train Steps/Sec: 1.17 +[2025-04-26 14:05:42] (step=0089400) Train Loss: 5.3000, Train Steps/Sec: 1.17 +[2025-04-26 14:06:03] (step=0089425) Train Loss: 5.2197, Train Steps/Sec: 1.16 +[2025-04-26 14:06:27] (step=0089450) Train Loss: 5.2155, Train Steps/Sec: 1.06 +[2025-04-26 14:06:48] (step=0089475) Train Loss: 5.2430, Train Steps/Sec: 1.17 +[2025-04-26 14:07:10] (step=0089500) Train Loss: 5.3112, Train Steps/Sec: 1.15 +[2025-04-26 14:07:31] (step=0089525) Train Loss: 5.2604, Train Steps/Sec: 1.16 +[2025-04-26 14:07:53] (step=0089550) Train Loss: 5.2270, Train Steps/Sec: 1.17 +[2025-04-26 14:08:15] (step=0089575) Train Loss: 5.2362, Train Steps/Sec: 1.13 +[2025-04-26 14:08:37] (step=0089600) Train Loss: 5.2615, Train Steps/Sec: 1.15 +[2025-04-26 14:08:58] (step=0089625) Train Loss: 5.1774, Train Steps/Sec: 1.16 +[2025-04-26 14:09:20] (step=0089650) Train Loss: 5.2355, Train Steps/Sec: 1.16 +[2025-04-26 14:09:42] (step=0089675) Train Loss: 5.2369, Train Steps/Sec: 1.12 +[2025-04-26 14:10:03] (step=0089700) Train Loss: 5.2447, Train Steps/Sec: 1.17 +[2025-04-26 14:10:25] (step=0089725) Train Loss: 5.1787, Train Steps/Sec: 1.16 +[2025-04-26 14:10:48] (step=0089750) Train Loss: 5.2261, Train Steps/Sec: 1.07 +[2025-04-26 14:11:10] (step=0089775) Train Loss: 5.2511, Train Steps/Sec: 1.17 +[2025-04-26 14:11:31] (step=0089800) Train Loss: 5.2347, Train Steps/Sec: 1.16 +[2025-04-26 14:11:53] (step=0089825) Train Loss: 5.1934, Train Steps/Sec: 1.17 +[2025-04-26 14:12:14] (step=0089850) Train Loss: 5.2450, Train Steps/Sec: 1.17 +[2025-04-26 14:12:35] (step=0089875) Train Loss: 5.1600, Train Steps/Sec: 1.17 +[2025-04-26 14:12:57] (step=0089900) Train Loss: 5.2461, Train Steps/Sec: 1.17 +[2025-04-26 14:13:19] (step=0089925) Train Loss: 5.2053, Train Steps/Sec: 1.12 +[2025-04-26 14:13:41] (step=0089950) Train Loss: 5.2325, Train Steps/Sec: 1.17 +[2025-04-26 14:14:02] (step=0089975) Train Loss: 5.1925, Train Steps/Sec: 1.17 +[2025-04-26 14:14:23] (step=0090000) Train Loss: 5.1632, Train Steps/Sec: 1.16 +[2025-04-26 14:14:23] text_config is None. Initializing the text config with default values (`OPTConfig`). +[2025-04-26 14:19:25] Finish Eval in 90000 steps... +[2025-04-26 14:19:53] Saved checkpoint to checkpoint/Afteri2i_100kckpt_LLAVAT5_empty_Fixed_just_segment_pretrain_stage3_dreambench_recap_400k_t2i_400k_flux_200k_midjourney_150k_recovery_150k_3e4_epoch2_no_rotated/003-GPT-XL/checkpoints/0090000.pt +[2025-04-26 14:19:56] Removed old checkpoint: checkpoint/Afteri2i_100kckpt_LLAVAT5_empty_Fixed_just_segment_pretrain_stage3_dreambench_recap_400k_t2i_400k_flux_200k_midjourney_150k_recovery_150k_3e4_epoch2_no_rotated/003-GPT-XL/checkpoints/0088000.pt +[2025-04-26 14:20:17] (step=0090025) Train Loss: 5.2262, Train Steps/Sec: 0.07 +[2025-04-26 14:20:39] (step=0090050) Train Loss: 5.2542, Train Steps/Sec: 1.15 +[2025-04-26 14:21:00] (step=0090075) Train Loss: 5.1689, Train Steps/Sec: 1.17 +[2025-04-26 14:21:23] (step=0090100) Train Loss: 5.2298, Train Steps/Sec: 1.12 +[2025-04-26 14:21:45] (step=0090125) Train Loss: 5.2348, Train Steps/Sec: 1.11 +[2025-04-26 14:22:06] (step=0090150) Train Loss: 5.3167, Train Steps/Sec: 1.16 +[2025-04-26 14:22:28] (step=0090175) Train Loss: 5.2560, Train Steps/Sec: 1.16 +[2025-04-26 14:22:49] (step=0090200) Train Loss: 5.2767, Train Steps/Sec: 1.17 +[2025-04-26 14:23:12] (step=0090225) Train Loss: 5.2540, Train Steps/Sec: 1.12 +[2025-04-26 14:23:33] (step=0090250) Train Loss: 5.2135, Train Steps/Sec: 1.17 +[2025-04-26 14:23:55] (step=0090275) Train Loss: 5.1393, Train Steps/Sec: 1.16 +[2025-04-26 14:24:16] (step=0090300) Train Loss: 5.2205, Train Steps/Sec: 1.16 +[2025-04-26 14:24:38] (step=0090325) Train Loss: 5.2361, Train Steps/Sec: 1.17 +[2025-04-26 14:25:00] (step=0090350) Train Loss: 5.2362, Train Steps/Sec: 1.12 +[2025-04-26 14:25:21] (step=0090375) Train Loss: 5.2021, Train Steps/Sec: 1.16 +[2025-04-26 14:25:44] (step=0090400) Train Loss: 5.2192, Train Steps/Sec: 1.11 +[2025-04-26 14:26:06] (step=0090425) Train Loss: 5.2019, Train Steps/Sec: 1.12 +[2025-04-26 14:26:27] (step=0090450) Train Loss: 5.2561, Train Steps/Sec: 1.17 +[2025-04-26 14:26:49] (step=0090475) Train Loss: 5.2621, Train Steps/Sec: 1.16 +[2025-04-26 14:27:11] (step=0090500) Train Loss: 5.1460, Train Steps/Sec: 1.16 +[2025-04-26 14:27:32] (step=0090525) Train Loss: 5.1856, Train Steps/Sec: 1.17 +[2025-04-26 14:27:53] (step=0090550) Train Loss: 5.2238, Train Steps/Sec: 1.17 +[2025-04-26 14:28:15] (step=0090575) Train Loss: 5.2297, Train Steps/Sec: 1.17 +[2025-04-26 14:28:36] (step=0090600) Train Loss: 5.2220, Train Steps/Sec: 1.17 +[2025-04-26 14:28:57] (step=0090625) Train Loss: 5.2372, Train Steps/Sec: 1.18 +[2025-04-26 14:29:19] (step=0090650) Train Loss: 5.2688, Train Steps/Sec: 1.12 +[2025-04-26 14:29:41] (step=0090675) Train Loss: 5.1891, Train Steps/Sec: 1.18 +[2025-04-26 14:30:02] (step=0090700) Train Loss: 5.3241, Train Steps/Sec: 1.17 +[2025-04-26 14:30:24] (step=0090725) Train Loss: 5.2136, Train Steps/Sec: 1.12 +[2025-04-26 14:30:46] (step=0090750) Train Loss: 5.2470, Train Steps/Sec: 1.18 +[2025-04-26 14:31:07] (step=0090775) Train Loss: 5.1749, Train Steps/Sec: 1.17 +[2025-04-26 14:31:29] (step=0090800) Train Loss: 5.2014, Train Steps/Sec: 1.12 +[2025-04-26 14:31:51] (step=0090825) Train Loss: 5.2019, Train Steps/Sec: 1.17 +[2025-04-26 14:32:12] (step=0090850) Train Loss: 5.3517, Train Steps/Sec: 1.17 +[2025-04-26 14:32:34] (step=0090875) Train Loss: 5.1937, Train Steps/Sec: 1.13 +[2025-04-26 14:32:57] (step=0090900) Train Loss: 5.1731, Train Steps/Sec: 1.11 +[2025-04-26 14:33:18] (step=0090925) Train Loss: 5.2446, Train Steps/Sec: 1.17 +[2025-04-26 14:33:39] (step=0090950) Train Loss: 5.1775, Train Steps/Sec: 1.17 +[2025-04-26 14:34:00] (step=0090975) Train Loss: 5.2638, Train Steps/Sec: 1.18 +[2025-04-26 14:34:22] (step=0091000) Train Loss: 5.3206, Train Steps/Sec: 1.17 +[2025-04-26 14:34:44] (step=0091025) Train Loss: 5.2054, Train Steps/Sec: 1.13 +[2025-04-26 14:35:06] (step=0091050) Train Loss: 5.2451, Train Steps/Sec: 1.13 +[2025-04-26 14:35:28] (step=0091075) Train Loss: 5.1870, Train Steps/Sec: 1.13 +[2025-04-26 14:35:50] (step=0091100) Train Loss: 5.2540, Train Steps/Sec: 1.17 +[2025-04-26 14:36:11] (step=0091125) Train Loss: 5.2366, Train Steps/Sec: 1.17 +[2025-04-26 14:36:32] (step=0091150) Train Loss: 5.1328, Train Steps/Sec: 1.17 +[2025-04-26 14:36:54] (step=0091175) Train Loss: 5.1533, Train Steps/Sec: 1.17 +[2025-04-26 14:37:15] (step=0091200) Train Loss: 5.2468, Train Steps/Sec: 1.17 +[2025-04-26 14:37:36] (step=0091225) Train Loss: 5.1544, Train Steps/Sec: 1.17 +[2025-04-26 14:37:58] (step=0091250) Train Loss: 5.2958, Train Steps/Sec: 1.17 +[2025-04-26 14:38:19] (step=0091275) Train Loss: 5.2441, Train Steps/Sec: 1.17 +[2025-04-26 14:38:40] (step=0091300) Train Loss: 5.2316, Train Steps/Sec: 1.17 +[2025-04-26 14:39:02] (step=0091325) Train Loss: 5.2198, Train Steps/Sec: 1.17 +[2025-04-26 14:39:23] (step=0091350) Train Loss: 5.2055, Train Steps/Sec: 1.18 +[2025-04-26 14:39:45] (step=0091375) Train Loss: 5.2436, Train Steps/Sec: 1.13 +[2025-04-26 14:40:07] (step=0091400) Train Loss: 5.1907, Train Steps/Sec: 1.17 +[2025-04-26 14:40:28] (step=0091425) Train Loss: 5.1758, Train Steps/Sec: 1.18 +[2025-04-26 14:40:50] (step=0091450) Train Loss: 5.2669, Train Steps/Sec: 1.12 +[2025-04-26 14:41:12] (step=0091475) Train Loss: 5.1017, Train Steps/Sec: 1.13 +[2025-04-26 14:41:34] (step=0091500) Train Loss: 5.1643, Train Steps/Sec: 1.17 +[2025-04-26 14:41:56] (step=0091525) Train Loss: 5.2491, Train Steps/Sec: 1.13 +[2025-04-26 14:42:17] (step=0091550) Train Loss: 5.2693, Train Steps/Sec: 1.18 +[2025-04-26 14:42:38] (step=0091575) Train Loss: 5.2570, Train Steps/Sec: 1.17 +[2025-04-26 14:43:00] (step=0091600) Train Loss: 5.2349, Train Steps/Sec: 1.17 +[2025-04-26 14:43:21] (step=0091625) Train Loss: 5.2476, Train Steps/Sec: 1.17 +[2025-04-26 14:43:42] (step=0091650) Train Loss: 5.2776, Train Steps/Sec: 1.17 +[2025-04-26 14:44:04] (step=0091675) Train Loss: 5.1855, Train Steps/Sec: 1.17 +[2025-04-26 14:44:26] (step=0091700) Train Loss: 5.1685, Train Steps/Sec: 1.12 +[2025-04-26 14:44:49] (step=0091725) Train Loss: 5.2990, Train Steps/Sec: 1.08 +[2025-04-26 14:45:10] (step=0091750) Train Loss: 5.3128, Train Steps/Sec: 1.18 +[2025-04-26 14:45:32] (step=0091775) Train Loss: 5.1800, Train Steps/Sec: 1.17 +[2025-04-26 14:45:53] (step=0091800) Train Loss: 5.1306, Train Steps/Sec: 1.17 +[2025-04-26 14:46:14] (step=0091825) Train Loss: 5.3233, Train Steps/Sec: 1.17 +[2025-04-26 14:46:36] (step=0091850) Train Loss: 5.2241, Train Steps/Sec: 1.17 +[2025-04-26 14:46:57] (step=0091875) Train Loss: 5.2547, Train Steps/Sec: 1.17 +[2025-04-26 14:47:19] (step=0091900) Train Loss: 5.2830, Train Steps/Sec: 1.17 +[2025-04-26 14:47:40] (step=0091925) Train Loss: 5.1818, Train Steps/Sec: 1.17 +[2025-04-26 14:48:01] (step=0091950) Train Loss: 5.2696, Train Steps/Sec: 1.17 +[2025-04-26 14:48:22] (step=0091975) Train Loss: 5.1632, Train Steps/Sec: 1.18 +[2025-04-26 14:48:44] (step=0092000) Train Loss: 5.2357, Train Steps/Sec: 1.17 +[2025-04-26 14:48:44] text_config is None. Initializing the text config with default values (`OPTConfig`). +[2025-04-26 14:53:47] Finish Eval in 92000 steps... +[2025-04-26 14:54:06] Saved checkpoint to checkpoint/Afteri2i_100kckpt_LLAVAT5_empty_Fixed_just_segment_pretrain_stage3_dreambench_recap_400k_t2i_400k_flux_200k_midjourney_150k_recovery_150k_3e4_epoch2_no_rotated/003-GPT-XL/checkpoints/0092000.pt +[2025-04-26 14:54:08] Removed old checkpoint: checkpoint/Afteri2i_100kckpt_LLAVAT5_empty_Fixed_just_segment_pretrain_stage3_dreambench_recap_400k_t2i_400k_flux_200k_midjourney_150k_recovery_150k_3e4_epoch2_no_rotated/003-GPT-XL/checkpoints/0090000.pt +[2025-04-26 14:54:30] (step=0092025) Train Loss: 5.2550, Train Steps/Sec: 0.07 +[2025-04-26 14:54:51] (step=0092050) Train Loss: 5.2718, Train Steps/Sec: 1.18 +[2025-04-26 14:55:12] (step=0092075) Train Loss: 5.1775, Train Steps/Sec: 1.17 +[2025-04-26 14:55:34] (step=0092100) Train Loss: 5.2944, Train Steps/Sec: 1.11 +[2025-04-26 14:55:56] (step=0092125) Train Loss: 5.2479, Train Steps/Sec: 1.18 +[2025-04-26 14:56:18] (step=0092150) Train Loss: 5.2769, Train Steps/Sec: 1.13 +[2025-04-26 14:56:39] (step=0092175) Train Loss: 5.2369, Train Steps/Sec: 1.18 +[2025-04-26 14:57:01] (step=0092200) Train Loss: 5.2095, Train Steps/Sec: 1.13 +[2025-04-26 14:57:23] (step=0092225) Train Loss: 5.2055, Train Steps/Sec: 1.17 +[2025-04-26 14:57:44] (step=0092250) Train Loss: 5.2420, Train Steps/Sec: 1.17 +[2025-04-26 14:58:05] (step=0092275) Train Loss: 5.2455, Train Steps/Sec: 1.17 +[2025-04-26 14:58:27] (step=0092300) Train Loss: 5.2560, Train Steps/Sec: 1.17 +[2025-04-26 14:58:48] (step=0092325) Train Loss: 5.2372, Train Steps/Sec: 1.18 +[2025-04-26 14:59:09] (step=0092350) Train Loss: 5.2496, Train Steps/Sec: 1.17 +[2025-04-26 14:59:32] (step=0092375) Train Loss: 5.1715, Train Steps/Sec: 1.08 +[2025-04-26 14:59:55] (step=0092400) Train Loss: 5.2935, Train Steps/Sec: 1.12 +[2025-04-26 15:00:16] (step=0092425) Train Loss: 5.2222, Train Steps/Sec: 1.17 +[2025-04-26 15:00:37] (step=0092450) Train Loss: 5.1902, Train Steps/Sec: 1.17 +[2025-04-26 15:00:58] (step=0092475) Train Loss: 5.1754, Train Steps/Sec: 1.17 +[2025-04-26 15:01:21] (step=0092500) Train Loss: 5.2052, Train Steps/Sec: 1.12 +[2025-04-26 15:01:42] (step=0092525) Train Loss: 5.1529, Train Steps/Sec: 1.17 +[2025-04-26 15:02:03] (step=0092550) Train Loss: 5.3493, Train Steps/Sec: 1.17 +[2025-04-26 15:02:25] (step=0092575) Train Loss: 5.1634, Train Steps/Sec: 1.17 +[2025-04-26 15:02:46] (step=0092600) Train Loss: 5.2780, Train Steps/Sec: 1.17 +[2025-04-26 15:03:07] (step=0092625) Train Loss: 5.1629, Train Steps/Sec: 1.17 +[2025-04-26 15:03:29] (step=0092650) Train Loss: 5.1931, Train Steps/Sec: 1.17 +[2025-04-26 15:03:50] (step=0092675) Train Loss: 5.1880, Train Steps/Sec: 1.17 +[2025-04-26 15:04:11] (step=0092700) Train Loss: 5.2338, Train Steps/Sec: 1.17 +[2025-04-26 15:04:34] (step=0092725) Train Loss: 5.2412, Train Steps/Sec: 1.13 +[2025-04-26 15:04:55] (step=0092750) Train Loss: 5.2120, Train Steps/Sec: 1.17 +[2025-04-26 15:05:16] (step=0092775) Train Loss: 5.3538, Train Steps/Sec: 1.17 +[2025-04-26 15:05:38] (step=0092800) Train Loss: 5.2353, Train Steps/Sec: 1.17 +[2025-04-26 15:06:01] (step=0092825) Train Loss: 5.1691, Train Steps/Sec: 1.08 +[2025-04-26 15:06:23] (step=0092850) Train Loss: 5.2859, Train Steps/Sec: 1.13 +[2025-04-26 15:06:44] (step=0092875) Train Loss: 5.2087, Train Steps/Sec: 1.17 +[2025-04-26 15:07:06] (step=0092900) Train Loss: 5.1838, Train Steps/Sec: 1.17 +[2025-04-26 15:07:27] (step=0092925) Train Loss: 5.2958, Train Steps/Sec: 1.17 +[2025-04-26 15:07:48] (step=0092950) Train Loss: 5.2527, Train Steps/Sec: 1.17 +[2025-04-26 15:08:09] (step=0092975) Train Loss: 5.2661, Train Steps/Sec: 1.17 +[2025-04-26 15:08:31] (step=0093000) Train Loss: 5.1949, Train Steps/Sec: 1.17 +[2025-04-26 15:08:52] (step=0093025) Train Loss: 5.2165, Train Steps/Sec: 1.17 +[2025-04-26 15:09:16] (step=0093050) Train Loss: 5.1622, Train Steps/Sec: 1.05 +[2025-04-26 15:09:37] (step=0093075) Train Loss: 5.2077, Train Steps/Sec: 1.17 +[2025-04-26 15:09:59] (step=0093100) Train Loss: 5.2386, Train Steps/Sec: 1.17 +[2025-04-26 15:10:20] (step=0093125) Train Loss: 5.2036, Train Steps/Sec: 1.17 +[2025-04-26 15:10:41] (step=0093150) Train Loss: 5.2691, Train Steps/Sec: 1.17 +[2025-04-26 15:11:03] (step=0093175) Train Loss: 5.1493, Train Steps/Sec: 1.17 +[2025-04-26 15:11:24] (step=0093200) Train Loss: 5.2619, Train Steps/Sec: 1.17 +[2025-04-26 15:11:45] (step=0093225) Train Loss: 5.2152, Train Steps/Sec: 1.17 +[2025-04-26 15:12:07] (step=0093250) Train Loss: 5.2636, Train Steps/Sec: 1.17 +[2025-04-26 15:12:28] (step=0093275) Train Loss: 5.2517, Train Steps/Sec: 1.17 +[2025-04-26 15:12:50] (step=0093300) Train Loss: 5.2671, Train Steps/Sec: 1.12 +[2025-04-26 15:13:12] (step=0093325) Train Loss: 5.2703, Train Steps/Sec: 1.17 +[2025-04-26 15:13:33] (step=0093350) Train Loss: 5.1847, Train Steps/Sec: 1.17 +[2025-04-26 15:13:54] (step=0093375) Train Loss: 5.2008, Train Steps/Sec: 1.17 +[2025-04-26 15:14:16] (step=0093400) Train Loss: 5.2751, Train Steps/Sec: 1.17 +[2025-04-26 15:14:37] (step=0093425) Train Loss: 5.2426, Train Steps/Sec: 1.17 +[2025-04-26 15:14:59] (step=0093450) Train Loss: 5.2634, Train Steps/Sec: 1.13 +[2025-04-26 15:15:21] (step=0093475) Train Loss: 5.2814, Train Steps/Sec: 1.13 +[2025-04-26 15:15:43] (step=0093500) Train Loss: 5.2686, Train Steps/Sec: 1.13 +[2025-04-26 15:16:05] (step=0093525) Train Loss: 5.2574, Train Steps/Sec: 1.18 +[2025-04-26 15:16:27] (step=0093550) Train Loss: 5.1546, Train Steps/Sec: 1.13 +[2025-04-26 15:16:48] (step=0093575) Train Loss: 5.2106, Train Steps/Sec: 1.17 +[2025-04-26 15:17:10] (step=0093600) Train Loss: 5.2060, Train Steps/Sec: 1.17 +[2025-04-26 15:17:31] (step=0093625) Train Loss: 5.2009, Train Steps/Sec: 1.18 +[2025-04-26 15:17:52] (step=0093650) Train Loss: 5.2195, Train Steps/Sec: 1.17 +[2025-04-26 15:18:14] (step=0093675) Train Loss: 5.2121, Train Steps/Sec: 1.17 +[2025-04-26 15:18:36] (step=0093700) Train Loss: 5.1729, Train Steps/Sec: 1.12 +[2025-04-26 15:18:58] (step=0093725) Train Loss: 5.2559, Train Steps/Sec: 1.13 +[2025-04-26 15:19:19] (step=0093750) Train Loss: 5.2123, Train Steps/Sec: 1.18 +[2025-04-26 15:19:41] (step=0093775) Train Loss: 5.1502, Train Steps/Sec: 1.18 +[2025-04-26 15:20:02] (step=0093800) Train Loss: 5.1662, Train Steps/Sec: 1.17 +[2025-04-26 15:20:23] (step=0093825) Train Loss: 5.1947, Train Steps/Sec: 1.18 +[2025-04-26 15:20:30] Done! diff --git a/CKPTS/Afteri2i_100kckpt_LLAVAT5_empty_Fixed_just_segment_pretrain_stage3_dreambench_recap_400k_t2i_400k_flux_200k_midjourney_150k_recovery_150k_3e4_epoch2_no_rotated/wandb/debug-internal.log b/CKPTS/Afteri2i_100kckpt_LLAVAT5_empty_Fixed_just_segment_pretrain_stage3_dreambench_recap_400k_t2i_400k_flux_200k_midjourney_150k_recovery_150k_3e4_epoch2_no_rotated/wandb/debug-internal.log new file mode 100644 index 0000000000000000000000000000000000000000..512261d9109fab3b8772044697823bc9f6cbcaf9 --- /dev/null +++ b/CKPTS/Afteri2i_100kckpt_LLAVAT5_empty_Fixed_just_segment_pretrain_stage3_dreambench_recap_400k_t2i_400k_flux_200k_midjourney_150k_recovery_150k_3e4_epoch2_no_rotated/wandb/debug-internal.log @@ -0,0 +1,15 @@ +{"time":"2025-04-26T08:04:03.466771029Z","level":"INFO","msg":"stream: starting","core version":"0.19.8","symlink path":"checkpoint/Afteri2i_100kckpt_LLAVAT5_empty_Fixed_just_segment_pretrain_stage3_dreambench_recap_400k_t2i_400k_flux_200k_midjourney_150k_recovery_150k_3e4_epoch2_no_rotated/wandb/run-20250426_080403-2j21z9h0/logs/debug-core.log"} +{"time":"2025-04-26T08:04:03.67302256Z","level":"INFO","msg":"created new stream","id":"2j21z9h0"} +{"time":"2025-04-26T08:04:03.673052226Z","level":"INFO","msg":"stream: started","id":"2j21z9h0"} +{"time":"2025-04-26T08:04:03.673070455Z","level":"INFO","msg":"writer: Do: started","stream_id":"2j21z9h0"} +{"time":"2025-04-26T08:04:03.673092306Z","level":"INFO","msg":"sender: started","stream_id":"2j21z9h0"} +{"time":"2025-04-26T08:04:03.673315729Z","level":"INFO","msg":"handler: started","stream_id":"2j21z9h0"} +{"time":"2025-04-26T08:04:04.102489046Z","level":"INFO","msg":"Starting system monitor"} +{"time":"2025-04-26T15:20:33.187645817Z","level":"INFO","msg":"stream: closing","id":"2j21z9h0"} +{"time":"2025-04-26T15:20:33.187688377Z","level":"INFO","msg":"Stopping system monitor"} +{"time":"2025-04-26T15:20:33.188384875Z","level":"INFO","msg":"Stopped system monitor"} +{"time":"2025-04-26T15:20:34.293026283Z","level":"INFO","msg":"fileTransfer: Close: file transfer manager closed"} +{"time":"2025-04-26T15:20:34.493877028Z","level":"INFO","msg":"handler: closed","stream_id":"2j21z9h0"} +{"time":"2025-04-26T15:20:34.49391269Z","level":"INFO","msg":"sender: closed","stream_id":"2j21z9h0"} +{"time":"2025-04-26T15:20:34.493906784Z","level":"INFO","msg":"writer: Close: closed","stream_id":"2j21z9h0"} +{"time":"2025-04-26T15:20:34.494012131Z","level":"INFO","msg":"stream: closed","id":"2j21z9h0"} diff --git a/CKPTS/Afteri2i_100kckpt_LLAVAT5_empty_Fixed_just_segment_pretrain_stage3_dreambench_recap_400k_t2i_400k_flux_200k_midjourney_150k_recovery_150k_3e4_epoch2_no_rotated/wandb/debug.log b/CKPTS/Afteri2i_100kckpt_LLAVAT5_empty_Fixed_just_segment_pretrain_stage3_dreambench_recap_400k_t2i_400k_flux_200k_midjourney_150k_recovery_150k_3e4_epoch2_no_rotated/wandb/debug.log new file mode 100644 index 0000000000000000000000000000000000000000..67daf6350cdc194f1aba2211885c4876c254f0e6 --- /dev/null +++ b/CKPTS/Afteri2i_100kckpt_LLAVAT5_empty_Fixed_just_segment_pretrain_stage3_dreambench_recap_400k_t2i_400k_flux_200k_midjourney_150k_recovery_150k_3e4_epoch2_no_rotated/wandb/debug.log @@ -0,0 +1,23 @@ +2025-04-26 08:04:03,459 INFO MainThread:3408653 [wandb_setup.py:_flush():67] Current SDK version is 0.19.8 +2025-04-26 08:04:03,460 INFO MainThread:3408653 [wandb_setup.py:_flush():67] Configure stats pid to 3408653 +2025-04-26 08:04:03,460 INFO MainThread:3408653 [wandb_setup.py:_flush():67] Loading settings from /tmp/haozhezhao/.config/wandb/settings +2025-04-26 08:04:03,460 INFO MainThread:3408653 [wandb_setup.py:_flush():67] Loading settings from /tmp/haozhezhao/MLLMG/wandb/settings +2025-04-26 08:04:03,460 INFO MainThread:3408653 [wandb_setup.py:_flush():67] Loading settings from environment variables +2025-04-26 08:04:03,460 INFO MainThread:3408653 [wandb_init.py:setup_run_log_directory():647] Logging user logs to checkpoint/Afteri2i_100kckpt_LLAVAT5_empty_Fixed_just_segment_pretrain_stage3_dreambench_recap_400k_t2i_400k_flux_200k_midjourney_150k_recovery_150k_3e4_epoch2_no_rotated/wandb/run-20250426_080403-2j21z9h0/logs/debug.log +2025-04-26 08:04:03,460 INFO MainThread:3408653 [wandb_init.py:setup_run_log_directory():648] Logging internal logs to checkpoint/Afteri2i_100kckpt_LLAVAT5_empty_Fixed_just_segment_pretrain_stage3_dreambench_recap_400k_t2i_400k_flux_200k_midjourney_150k_recovery_150k_3e4_epoch2_no_rotated/wandb/run-20250426_080403-2j21z9h0/logs/debug-internal.log +2025-04-26 08:04:03,460 INFO MainThread:3408653 [wandb_init.py:init():761] calling init triggers +2025-04-26 08:04:03,460 INFO MainThread:3408653 [wandb_init.py:init():766] wandb.init called with sweep_config: {} +config: {'data_path': '/tmp/haozhezhao/MLLMG/jsonl_data/merged_train_set_set_subject_400k_recap_t2i_400k_flux_200k_midjourney_150k_recovery_150k_grounding_100fluxseg_50samseg.jsonl', 'cloud_save_path': '/tmp/haozhezhao/checkpoint', 'no_local_save': False, 'vq_model': 'VQ-16', 'vq_ckpt': '/tmp/haozhezhao/model/llamagen_t2i/vq_ds16_t2i.pt', 'codebook_size': 16384, 'codebook_embed_dim': 8, 'gpt_model': 'GPT-XL', 'gpt_ckpt': '/tmp/haozhezhao/MLLMG/checkpoint/Afteri2i_100kckpt_LLAVAT5_empty_Fixed_just_segment_pretrain_stage3_dreambench_recap_400k_t2i_400k_flux_200k_midjourney_150k_recovery_150k_3e4_epoch2_no_rotated/002-GPT-XL/checkpoints/0070000.pt', 'gpt_type': 't2i', 'vocab_size': 16384, 'cls_token_num': 512, 'dropout_p': 0.1, 'token_dropout_p': 0.1, 'drop_path': 0.0, 'no_compile': False, 'results_dir': 'checkpoint/Afteri2i_100kckpt_LLAVAT5_empty_Fixed_just_segment_pretrain_stage3_dreambench_recap_400k_t2i_400k_flux_200k_midjourney_150k_recovery_150k_3e4_epoch2_no_rotated', 'dataset': 'ti2i', 'image_size': 512, 'downsample_size': 16, 'num_classes': 1000, 'epochs': 4, 'lr': 0.0003, 'weight_decay': 0.05, 'beta1': 0.9, 'beta2': 0.95, 'max_grad_norm': 1.0, 'global_batch_size': 56, 'global_seed': 0, 'num_workers': 8, 'log_every': 25, 'ckpt_every': 2000, 'gradient_accumulation_steps': 4, 'mixed_precision': 'bf16', 'val_data_path': '/tmp/haozhezhao/MLLMG/jsonl_data/dreambench_plus_valid.jsonl', 'use_vision_tower': True, 'model_name_or_path': '/tmp/haozhezhao/model/blip2-flan-t5-xl', 'image_place_holder': '', 'processor_path': None, 'do_eval': True, 'max_eval_samples': 250, 'train_text_encoder': True, 'no_left_padding': False, 'cfg_scale': 7.5, 'top_k': 16384, 'temperature': 0.9, 'top_p': 1.0, 'eval_steps': 2000, 'project_name': 'llamagen_ti2i', 'load_from_checkpoint': '/tmp/haozhezhao/MLLMG/checkpoint/FIXed_3M_1epoch_step32000.pt', 'warmup': 0.05, 'lr_decay_style': 'cosine', 'lr_decay_ratio': 0.1, 'train_iters': 500000, 'class_dropout_prob': 0.1, 'with_image_only': False, 'image_only_rate': 0.1, 'stage2': False, 'subject_driven': True, 'load_subject_embedding': None, 'reference_data_path': '/tmp/haozhezhao/MLLMG/cc12m_reference_tunnel.jsonl', 'multimodal_encoder': 'llava', 'do_recovery': True, 'no_replace': False, 'resume': True, 'dreambench_eval': True, 'find_unused_parameters': True, 'load_visual_encoder': False, 'continue_stage1': False, 'replace_subject': False, 'train_all': True, 'save_total_limit': 1, 'load_language_projection': '/tmp/haozhezhao/MLLMG/llava-v1.5-flant5_fixed-pretrain/mm_projector.bin', 'mm_vision_tower': 'openai/clip-vit-large-patch14', 'load_fixed_llamagen': True, 'unfreeze_output': False, 'fix': 'gpt-empty-fix', 'rank': 0, 'world_size': 8, 'gpu': 0, 'dist_url': 'env://', 'distributed': True, 'dist_backend': 'nccl', '_wandb': {}} +2025-04-26 08:04:03,460 INFO MainThread:3408653 [wandb_init.py:init():784] starting backend +2025-04-26 08:04:03,460 INFO MainThread:3408653 [wandb_init.py:init():788] sending inform_init request +2025-04-26 08:04:03,465 INFO MainThread:3408653 [backend.py:_multiprocessing_setup():101] multiprocessing start_methods=fork,spawn,forkserver, using: spawn +2025-04-26 08:04:03,465 INFO MainThread:3408653 [wandb_init.py:init():798] backend started and connected +2025-04-26 08:04:03,467 INFO MainThread:3408653 [wandb_init.py:init():891] updated telemetry +2025-04-26 08:04:03,467 INFO MainThread:3408653 [wandb_init.py:init():915] communicating run to backend with 90.0 second timeout +2025-04-26 08:04:04,100 INFO MainThread:3408653 [wandb_init.py:init():990] starting run threads in backend +2025-04-26 08:04:04,235 INFO MainThread:3408653 [wandb_run.py:_console_start():2375] atexit reg +2025-04-26 08:04:04,235 INFO MainThread:3408653 [wandb_run.py:_redirect():2227] redirect: wrap_raw +2025-04-26 08:04:04,236 INFO MainThread:3408653 [wandb_run.py:_redirect():2292] Wrapping output streams. +2025-04-26 08:04:04,236 INFO MainThread:3408653 [wandb_run.py:_redirect():2315] Redirects installed. +2025-04-26 08:04:04,237 INFO MainThread:3408653 [wandb_init.py:init():1032] run started, returning control to user process +2025-04-26 15:20:33,186 INFO MsgRouterThr:3408653 [mailbox.py:close():129] Closing mailbox, abandoning 1 handles. diff --git a/CKPTS/Afteri2i_100kckpt_LLAVAT5_empty_Fixed_just_segment_pretrain_stage3_dreambench_recap_400k_t2i_400k_flux_200k_midjourney_150k_recovery_150k_3e4_epoch2_no_rotated/wandb/run-20250424_021033-arc66v3w/files/config.yaml b/CKPTS/Afteri2i_100kckpt_LLAVAT5_empty_Fixed_just_segment_pretrain_stage3_dreambench_recap_400k_t2i_400k_flux_200k_midjourney_150k_recovery_150k_3e4_epoch2_no_rotated/wandb/run-20250424_021033-arc66v3w/files/config.yaml new file mode 100644 index 0000000000000000000000000000000000000000..2892bc141a891ca86305f2ddddb977b2a813ce27 --- /dev/null +++ b/CKPTS/Afteri2i_100kckpt_LLAVAT5_empty_Fixed_just_segment_pretrain_stage3_dreambench_recap_400k_t2i_400k_flux_200k_midjourney_150k_recovery_150k_3e4_epoch2_no_rotated/wandb/run-20250424_021033-arc66v3w/files/config.yaml @@ -0,0 +1,199 @@ +_wandb: + value: + cli_version: 0.19.8 + m: [] + python_version: 3.11.11 + t: + "1": + - 1 + - 11 + - 41 + - 49 + - 51 + - 55 + "2": + - 1 + - 11 + - 41 + - 49 + - 51 + - 55 + "3": + - 13 + - 16 + - 23 + - 55 + - 61 + "4": 3.11.11 + "5": 0.19.8 + "6": 4.49.0 + "8": + - 5 + "12": 0.19.8 + "13": linux-x86_64 +beta1: + value: 0.9 +beta2: + value: 0.95 +cfg_scale: + value: 7.5 +ckpt_every: + value: 2000 +class_dropout_prob: + value: 0.1 +cloud_save_path: + value: /tmp/haozhezhao/checkpoint +cls_token_num: + value: 512 +codebook_embed_dim: + value: 8 +codebook_size: + value: 16384 +continue_stage1: + value: false +data_path: + value: /tmp/haozhezhao/MLLMG/jsonl_data/merged_train_set_set_subject_400k_recap_t2i_400k_flux_200k_midjourney_150k_recovery_150k_grounding_100fluxseg_50samseg.jsonl +dataset: + value: ti2i +dist_backend: + value: nccl +dist_url: + value: env:// +distributed: + value: true +do_eval: + value: true +do_recovery: + value: true +downsample_size: + value: 16 +dreambench_eval: + value: true +drop_path: + value: 0 +dropout_p: + value: 0.1 +epochs: + value: 2 +eval_steps: + value: 2000 +find_unused_parameters: + value: true +fix: + value: gpt-empty-fix +global_batch_size: + value: 56 +global_seed: + value: 0 +gpt_ckpt: + value: /tmp/haozhezhao/MLLMG/MLLMG_ckpts/checkpoints/AfterI2I_llavat5_subject_train_all_T5_subject_t2i_ti2i_120_w_flux_segment_noimageonly_empty_fix_train_all/0100000.pt +gpt_model: + value: GPT-XL +gpt_type: + value: t2i +gpu: + value: 0 +gradient_accumulation_steps: + value: 4 +image_only_rate: + value: 0.1 +image_place_holder: + value: +image_size: + value: 512 +load_fixed_llamagen: + value: true +load_from_checkpoint: + value: /tmp/haozhezhao/MLLMG/checkpoint/FIXed_3M_1epoch_step32000.pt +load_language_projection: + value: /tmp/haozhezhao/MLLMG/llava-v1.5-flant5_fixed-pretrain/mm_projector.bin +load_subject_embedding: + value: null +load_visual_encoder: + value: false +log_every: + value: 25 +lr: + value: 0.0003 +lr_decay_ratio: + value: 0.1 +lr_decay_style: + value: cosine +max_eval_samples: + value: 250 +max_grad_norm: + value: 1 +mixed_precision: + value: bf16 +mm_vision_tower: + value: openai/clip-vit-large-patch14 +model_name_or_path: + value: /tmp/haozhezhao/model/blip2-flan-t5-xl +multimodal_encoder: + value: llava +no_compile: + value: false +no_left_padding: + value: false +no_local_save: + value: false +no_replace: + value: false +num_classes: + value: 1000 +num_workers: + value: 8 +processor_path: + value: null +project_name: + value: llamagen_ti2i +rank: + value: 0 +reference_data_path: + value: /tmp/haozhezhao/MLLMG/cc12m_reference_tunnel.jsonl +replace_subject: + value: false +results_dir: + value: checkpoint/Afteri2i_100kckpt_LLAVAT5_empty_Fixed_just_segment_pretrain_stage3_dreambench_recap_400k_t2i_400k_flux_200k_midjourney_150k_recovery_150k_3e4_epoch2_no_rotated +resume: + value: false +save_total_limit: + value: 1 +stage2: + value: false +subject_driven: + value: true +temperature: + value: 0.9 +token_dropout_p: + value: 0.1 +top_k: + value: 16384 +top_p: + value: 1 +train_all: + value: true +train_iters: + value: 500000 +train_text_encoder: + value: true +unfreeze_output: + value: false +use_vision_tower: + value: true +val_data_path: + value: /tmp/haozhezhao/MLLMG/jsonl_data/dreambench_plus_valid.jsonl +vocab_size: + value: 16384 +vq_ckpt: + value: /tmp/haozhezhao/model/llamagen_t2i/vq_ds16_t2i.pt +vq_model: + value: VQ-16 +warmup: + value: 0.05 +weight_decay: + value: 0.05 +with_image_only: + value: false +world_size: + value: 8 diff --git a/CKPTS/Afteri2i_100kckpt_LLAVAT5_empty_Fixed_just_segment_pretrain_stage3_dreambench_recap_400k_t2i_400k_flux_200k_midjourney_150k_recovery_150k_3e4_epoch2_no_rotated/wandb/run-20250424_021033-arc66v3w/files/output.log b/CKPTS/Afteri2i_100kckpt_LLAVAT5_empty_Fixed_just_segment_pretrain_stage3_dreambench_recap_400k_t2i_400k_flux_200k_midjourney_150k_recovery_150k_3e4_epoch2_no_rotated/wandb/run-20250424_021033-arc66v3w/files/output.log new file mode 100644 index 0000000000000000000000000000000000000000..1ddb4e78eeffaef6d73ab27552e7b97fde184049 --- /dev/null +++ b/CKPTS/Afteri2i_100kckpt_LLAVAT5_empty_Fixed_just_segment_pretrain_stage3_dreambench_recap_400k_t2i_400k_flux_200k_midjourney_150k_recovery_150k_3e4_epoch2_no_rotated/wandb/run-20250424_021033-arc66v3w/files/output.log @@ -0,0 +1,2226 @@ +[2025-04-24 02:10:35] Training for 2 epochs... +[2025-04-24 02:10:35] Beginning epoch 0... + 0%| | 0/23458 [00:00 +tokenizer length after expend 32101 +tokenizer length before expend 32100 + /tmp/haozhezhao/anaconda3/envs/nlp/lib/python3.11/contextlib.py:105: FutureWarning: `torch.backends.cuda.sdp_kernel()` is deprecated. In the future, this context manager will be removed. Please see `torch.nn.attention.sdpa_kernel()` for the new context manager, with updated signature. + self.gen = func(*args, **kwds) | 0/4 [00:00 +tokenizer length after expend 32101 +tokenizer length before expend 32100 +100%|████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████| 4/4 [03:58<00:00, 59.54s/it] +[2025-04-24 03:28:57] Finish Eval in 4000 steps...██████████████████████████████████████████████████████████████████████████████████████| 4/4 [03:57<00:00, 58.92s/it] +processing left-padding... +processing left-padding... +processing left-padding... +processing left-padding... +[2025-04-24 03:30:02] Saved checkpoint to checkpoint/Afteri2i_100kckpt_LLAVAT5_empty_Fixed_just_segment_pretrain_stage3_dreambench_recap_400k_t2i_400k_flux_200k_midjourney_150k_recovery_150k_3e4_epoch2_no_rotated/000-GPT-XL/checkpoints/0004000.pt +[2025-04-24 03:30:04] Removed old checkpoint: checkpoint/Afteri2i_100kckpt_LLAVAT5_empty_Fixed_just_segment_pretrain_stage3_dreambench_recap_400k_t2i_400k_flux_200k_midjourney_150k_recovery_150k_3e4_epoch2_no_rotated/000-GPT-XL/checkpoints/0002000.pt + 17%|██████████████████████▏ | 4024/23458 [1:19:49<4:47:33, 1.13it/s][2025-04-24 03:30:25] (step=0004025) Train Loss: 5.6916, Train Steps/Sec: 0.06 + 17%|██████████████████████▎ | 4049/23458 [1:20:12<4:37:27, 1.17it/s][2025-04-24 03:30:48] (step=0004050) Train Loss: 5.6571, Train Steps/Sec: 1.12 + 17%|██████████████████████▍ | 4074/23458 [1:20:34<4:35:55, 1.17it/s][2025-04-24 03:31:10] (step=0004075) Train Loss: 5.6856, Train Steps/Sec: 1.12 + 17%|██████████████████████▌ | 4099/23458 [1:20:55<4:32:26, 1.18it/s][2025-04-24 03:31:32] (step=0004100) Train Loss: 5.5924, Train Steps/Sec: 1.17 + 18%|██████████████████████▋ | 4124/23458 [1:21:17<4:40:24, 1.15it/s][2025-04-24 03:31:53] (step=0004125) Train Loss: 5.5697, Train Steps/Sec: 1.17 + 18%|██████████████████████▊ | 4149/23458 [1:21:38<4:36:34, 1.16it/s][2025-04-24 03:32:14] (step=0004150) Train Loss: 5.6732, Train Steps/Sec: 1.17 + 18%|██████████████████████▉ | 4174/23458 [1:21:59<4:32:50, 1.18it/s][2025-04-24 03:32:36] (step=0004175) Train Loss: 5.6684, Train Steps/Sec: 1.17 + 18%|███████████████████████ | 4199/23458 [1:22:21<4:37:11, 1.16it/s][2025-04-24 03:32:57] (step=0004200) Train Loss: 5.6094, Train Steps/Sec: 1.15 + 18%|███████████████████████▏ | 4224/23458 [1:22:43<4:42:04, 1.14it/s][2025-04-24 03:33:19] (step=0004225) Train Loss: 5.6585, Train Steps/Sec: 1.15 + 18%|███████████████████████▎ | 4249/23458 [1:23:04<4:34:06, 1.17it/s][2025-04-24 03:33:40] (step=0004250) Train Loss: 5.5982, Train Steps/Sec: 1.17 + 18%|███████████████████████▌ | 4274/23458 [1:23:26<4:32:53, 1.17it/s][2025-04-24 03:34:02] (step=0004275) Train Loss: 5.6848, Train Steps/Sec: 1.17 + 18%|███████████████████████▋ | 4299/23458 [1:23:47<4:31:30, 1.18it/s][2025-04-24 03:34:23] (step=0004300) Train Loss: 5.6966, Train Steps/Sec: 1.16 + 18%|███████████████████████▊ | 4324/23458 [1:24:09<4:43:39, 1.12it/s][2025-04-24 03:34:45] (step=0004325) Train Loss: 5.6266, Train Steps/Sec: 1.15 + 19%|███████████████████████▉ | 4349/23458 [1:24:30<4:34:33, 1.16it/s][2025-04-24 03:35:07] (step=0004350) Train Loss: 5.6196, Train Steps/Sec: 1.15 + 19%|████████████████████████ | 4374/23458 [1:24:52<4:33:10, 1.16it/s][2025-04-24 03:35:28] (step=0004375) Train Loss: 5.6525, Train Steps/Sec: 1.17 + 19%|████████████████████████▏ | 4399/23458 [1:25:13<4:30:08, 1.18it/s][2025-04-24 03:35:50] (step=0004400) Train Loss: 5.5551, Train Steps/Sec: 1.16 + 19%|████████████████████████▎ | 4424/23458 [1:25:35<4:35:40, 1.15it/s][2025-04-24 03:36:11] (step=0004425) Train Loss: 5.6523, Train Steps/Sec: 1.16 + 19%|████████████████████████▍ | 4449/23458 [1:25:57<4:31:32, 1.17it/s][2025-04-24 03:36:33] (step=0004450) Train Loss: 5.6658, Train Steps/Sec: 1.15 + 19%|████████████████████████▌ | 4474/23458 [1:26:18<4:28:51, 1.18it/s][2025-04-24 03:36:54] (step=0004475) Train Loss: 5.6252, Train Steps/Sec: 1.17 + 19%|████████████████████████▋ | 4499/23458 [1:26:39<4:28:19, 1.18it/s][2025-04-24 03:37:16] (step=0004500) Train Loss: 5.5928, Train Steps/Sec: 1.16 + 19%|████████████████████████▉ | 4524/23458 [1:27:01<4:35:29, 1.15it/s][2025-04-24 03:37:37] (step=0004525) Train Loss: 5.7613, Train Steps/Sec: 1.17 + 19%|█████████████████████████ | 4549/23458 [1:27:24<4:37:07, 1.14it/s][2025-04-24 03:38:01] (step=0004550) Train Loss: 5.6549, Train Steps/Sec: 1.02 + 19%|█████████████████████████▏ | 4574/23458 [1:27:48<4:34:24, 1.15it/s][2025-04-24 03:38:24] (step=0004575) Train Loss: 5.5792, Train Steps/Sec: 1.11 + 20%|█████████████████████████▎ | 4599/23458 [1:28:09<4:27:11, 1.18it/s][2025-04-24 03:38:46] (step=0004600) Train Loss: 5.6540, Train Steps/Sec: 1.15 + 20%|█████████████████████████▍ | 4624/23458 [1:28:31<4:33:15, 1.15it/s][2025-04-24 03:39:07] (step=0004625) Train Loss: 5.6687, Train Steps/Sec: 1.17 + 20%|█████████████████████████▌ | 4649/23458 [1:28:52<4:30:28, 1.16it/s][2025-04-24 03:39:28] (step=0004650) Train Loss: 5.6963, Train Steps/Sec: 1.16 + 20%|█████████████████████████▋ | 4674/23458 [1:29:15<4:27:40, 1.17it/s][2025-04-24 03:39:52] (step=0004675) Train Loss: 5.6864, Train Steps/Sec: 1.08 + 20%|█████████████████████████▊ | 4699/23458 [1:29:37<4:24:03, 1.18it/s][2025-04-24 03:40:13] (step=0004700) Train Loss: 5.5643, Train Steps/Sec: 1.17 + 20%|█████████████████████████▉ | 4724/23458 [1:29:58<4:32:32, 1.15it/s][2025-04-24 03:40:34] (step=0004725) Train Loss: 5.6133, Train Steps/Sec: 1.17 + 20%|██████████████████████████ | 4749/23458 [1:30:19<4:26:51, 1.17it/s][2025-04-24 03:40:56] (step=0004750) Train Loss: 5.6152, Train Steps/Sec: 1.17 + 20%|██████████████████████████▎ | 4774/23458 [1:30:42<4:25:52, 1.17it/s][2025-04-24 03:41:18] (step=0004775) Train Loss: 5.6003, Train Steps/Sec: 1.12 + 20%|██████████████████████████▍ | 4799/23458 [1:31:04<4:24:57, 1.17it/s][2025-04-24 03:41:41] (step=0004800) Train Loss: 5.5961, Train Steps/Sec: 1.11 + 21%|██████████████████████████▌ | 4824/23458 [1:31:26<4:30:42, 1.15it/s][2025-04-24 03:42:02] (step=0004825) Train Loss: 5.5705, Train Steps/Sec: 1.17 + 21%|██████████████████████████▋ | 4849/23458 [1:31:47<4:25:35, 1.17it/s][2025-04-24 03:42:23] (step=0004850) Train Loss: 5.5740, Train Steps/Sec: 1.17 + 21%|██████████████████████████▊ | 4874/23458 [1:32:08<4:22:25, 1.18it/s][2025-04-24 03:42:45] (step=0004875) Train Loss: 5.6113, Train Steps/Sec: 1.17 + 21%|██████████████████████████▉ | 4899/23458 [1:32:30<4:22:56, 1.18it/s][2025-04-24 03:43:06] (step=0004900) Train Loss: 5.5909, Train Steps/Sec: 1.16 + 21%|███████████████████████████ | 4924/23458 [1:32:51<4:28:36, 1.15it/s][2025-04-24 03:43:28] (step=0004925) Train Loss: 5.6852, Train Steps/Sec: 1.17 + 21%|███████████████████████████▏ | 4949/23458 [1:33:13<4:24:24, 1.17it/s][2025-04-24 03:43:49] (step=0004950) Train Loss: 5.6314, Train Steps/Sec: 1.17 + 21%|███████████████████████████▎ | 4974/23458 [1:33:34<4:21:55, 1.18it/s][2025-04-24 03:44:10] (step=0004975) Train Loss: 5.6388, Train Steps/Sec: 1.17 + 21%|███████████████████████████▍ | 4999/23458 [1:33:56<4:20:50, 1.18it/s][2025-04-24 03:44:32] (step=0005000) Train Loss: 5.7043, Train Steps/Sec: 1.16 + 21%|███████████████████████████▋ | 5024/23458 [1:34:17<4:27:01, 1.15it/s][2025-04-24 03:44:53] (step=0005025) Train Loss: 5.5631, Train Steps/Sec: 1.17 + 22%|███████████████████████████▊ | 5049/23458 [1:34:38<4:23:13, 1.17it/s][2025-04-24 03:45:15] (step=0005050) Train Loss: 5.5957, Train Steps/Sec: 1.17 + 22%|███████████████████████████▉ | 5074/23458 [1:35:00<4:20:14, 1.18it/s][2025-04-24 03:45:36] (step=0005075) Train Loss: 5.6212, Train Steps/Sec: 1.17 + 22%|████████████████████████████ | 5099/23458 [1:35:21<4:17:40, 1.19it/s][2025-04-24 03:45:57] (step=0005100) Train Loss: 5.6674, Train Steps/Sec: 1.17 + 22%|████████████████████████████▏ | 5124/23458 [1:35:43<4:27:11, 1.14it/s][2025-04-24 03:46:19] (step=0005125) Train Loss: 5.6051, Train Steps/Sec: 1.17 + 22%|████████████████████████████▎ | 5149/23458 [1:36:04<4:20:50, 1.17it/s][2025-04-24 03:46:40] (step=0005150) Train Loss: 5.5861, Train Steps/Sec: 1.17 + 22%|████████████████████████████▍ | 5174/23458 [1:36:25<4:18:11, 1.18it/s][2025-04-24 03:47:02] (step=0005175) Train Loss: 5.6565, Train Steps/Sec: 1.17 + 22%|████████████████████████████▌ | 5199/23458 [1:36:48<5:02:38, 1.01it/s][2025-04-24 03:47:25] (step=0005200) Train Loss: 5.6731, Train Steps/Sec: 1.05 + 22%|████████████████████████████▋ | 5224/23458 [1:37:12<4:27:37, 1.14it/s][2025-04-24 03:47:49] (step=0005225) Train Loss: 5.5777, Train Steps/Sec: 1.07 + 22%|████████████████████████████▊ | 5249/23458 [1:37:34<4:20:11, 1.17it/s][2025-04-24 03:48:10] (step=0005250) Train Loss: 5.6420, Train Steps/Sec: 1.17 + 22%|█████████████████████████████ | 5274/23458 [1:37:55<4:18:49, 1.17it/s][2025-04-24 03:48:32] (step=0005275) Train Loss: 5.6104, Train Steps/Sec: 1.17 + 23%|█████████████████████████████▏ | 5299/23458 [1:38:17<4:18:03, 1.17it/s][2025-04-24 03:48:53] (step=0005300) Train Loss: 5.5681, Train Steps/Sec: 1.16 + 23%|█████████████████████████████▎ | 5324/23458 [1:38:40<4:23:51, 1.15it/s][2025-04-24 03:49:16] (step=0005325) Train Loss: 5.6294, Train Steps/Sec: 1.09 + 23%|█████████████████████████████▍ | 5349/23458 [1:39:01<4:19:15, 1.16it/s][2025-04-24 03:49:37] (step=0005350) Train Loss: 5.6259, Train Steps/Sec: 1.17 + 23%|█████████████████████████████▌ | 5374/23458 [1:39:23<4:21:39, 1.15it/s][2025-04-24 03:49:59] (step=0005375) Train Loss: 5.6297, Train Steps/Sec: 1.16 + 23%|█████████████████████████████▋ | 5399/23458 [1:39:44<4:16:22, 1.17it/s][2025-04-24 03:50:21] (step=0005400) Train Loss: 5.6250, Train Steps/Sec: 1.16 + 23%|█████████████████████████████▊ | 5424/23458 [1:40:06<4:22:11, 1.15it/s][2025-04-24 03:50:42] (step=0005425) Train Loss: 5.6357, Train Steps/Sec: 1.17 + 23%|█████████████████████████████▉ | 5449/23458 [1:40:27<4:18:16, 1.16it/s][2025-04-24 03:51:03] (step=0005450) Train Loss: 5.7066, Train Steps/Sec: 1.17 + 23%|██████████████████████████████ | 5474/23458 [1:40:49<4:14:30, 1.18it/s][2025-04-24 03:51:25] (step=0005475) Train Loss: 5.5916, Train Steps/Sec: 1.17 + 23%|██████████████████████████████▏ | 5499/23458 [1:41:11<4:12:59, 1.18it/s][2025-04-24 03:51:47] (step=0005500) Train Loss: 5.6334, Train Steps/Sec: 1.11 + 24%|██████████████████████████████▍ | 5524/23458 [1:41:33<4:24:14, 1.13it/s][2025-04-24 03:52:10] (step=0005525) Train Loss: 5.5726, Train Steps/Sec: 1.11 + 24%|██████████████████████████████▌ | 5549/23458 [1:41:55<4:15:43, 1.17it/s][2025-04-24 03:52:31] (step=0005550) Train Loss: 5.6380, Train Steps/Sec: 1.17 + 24%|██████████████████████████████▋ | 5574/23458 [1:42:16<4:14:36, 1.17it/s][2025-04-24 03:52:52] (step=0005575) Train Loss: 5.5884, Train Steps/Sec: 1.17 + 24%|██████████████████████████████▊ | 5599/23458 [1:42:38<4:11:31, 1.18it/s][2025-04-24 03:53:14] (step=0005600) Train Loss: 5.6941, Train Steps/Sec: 1.16 + 24%|██████████████████████████████▉ | 5624/23458 [1:42:59<4:18:38, 1.15it/s][2025-04-24 03:53:35] (step=0005625) Train Loss: 5.5545, Train Steps/Sec: 1.17 + 24%|███████████████████████████████ | 5649/23458 [1:43:20<4:14:27, 1.17it/s][2025-04-24 03:53:57] (step=0005650) Train Loss: 5.6760, Train Steps/Sec: 1.17 + 24%|███████████████████████████████▏ | 5674/23458 [1:43:42<4:11:44, 1.18it/s][2025-04-24 03:54:18] (step=0005675) Train Loss: 5.6761, Train Steps/Sec: 1.17 + 24%|███████████████████████████████▎ | 5699/23458 [1:44:03<4:10:18, 1.18it/s][2025-04-24 03:54:39] (step=0005700) Train Loss: 5.6432, Train Steps/Sec: 1.17 + 24%|███████████████████████████████▍ | 5724/23458 [1:44:25<4:17:37, 1.15it/s][2025-04-24 03:55:01] (step=0005725) Train Loss: 5.6273, Train Steps/Sec: 1.17 + 25%|███████████████████████████████▌ | 5749/23458 [1:44:46<4:14:29, 1.16it/s][2025-04-24 03:55:22] (step=0005750) Train Loss: 5.5605, Train Steps/Sec: 1.17 + 25%|███████████████████████████████▊ | 5774/23458 [1:45:07<4:11:47, 1.17it/s][2025-04-24 03:55:44] (step=0005775) Train Loss: 5.6771, Train Steps/Sec: 1.17 + 25%|███████████████████████████████▉ | 5799/23458 [1:45:29<4:07:38, 1.19it/s][2025-04-24 03:56:05] (step=0005800) Train Loss: 5.5572, Train Steps/Sec: 1.17 + 25%|████████████████████████████████ | 5824/23458 [1:45:50<4:16:48, 1.14it/s][2025-04-24 03:56:26] (step=0005825) Train Loss: 5.6315, Train Steps/Sec: 1.17 + 25%|████████████████████████████████▏ | 5849/23458 [1:46:12<4:12:10, 1.16it/s][2025-04-24 03:56:48] (step=0005850) Train Loss: 5.6225, Train Steps/Sec: 1.17 + 25%|████████████████████████████████▎ | 5874/23458 [1:46:36<4:31:47, 1.08it/s][2025-04-24 03:57:12] (step=0005875) Train Loss: 5.5693, Train Steps/Sec: 1.03 + 25%|████████████████████████████████▍ | 5899/23458 [1:46:57<4:07:24, 1.18it/s][2025-04-24 03:57:34] (step=0005900) Train Loss: 5.5906, Train Steps/Sec: 1.17 + 25%|████████████████████████████████▌ | 5924/23458 [1:47:19<4:14:12, 1.15it/s][2025-04-24 03:57:55] (step=0005925) Train Loss: 5.5422, Train Steps/Sec: 1.17 + 25%|████████████████████████████████▋ | 5949/23458 [1:47:40<4:10:22, 1.17it/s][2025-04-24 03:58:16] (step=0005950) Train Loss: 5.6422, Train Steps/Sec: 1.17 + 25%|████████████████████████████████▊ | 5974/23458 [1:48:03<5:08:41, 1.06s/it][2025-04-24 03:58:39] (step=0005975) Train Loss: 5.6053, Train Steps/Sec: 1.08 + 26%|████████████████████████████████▉ | 5999/23458 [1:48:25<4:04:57, 1.19it/s][2025-04-24 03:59:01] (step=0006000) Train Loss: 5.6260, Train Steps/Sec: 1.17 +You are using a model of type blip-2 to instantiate a model of type instructblip. This is not supported for all configurations of models and can yield errors. +[2025-04-24 03:59:01] text_config is None. Initializing the text config with default values (`OPTConfig`). +model_name_or_path:/tmp/haozhezhao/model/blip2-flan-t5-xl +image_place_holder: +tokenizer length after expend 32101 +tokenizer length before expend 32100 +100%|████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████| 4/4 [03:58<00:00, 59.60s/it] +[2025-04-24 04:04:07] Finish Eval in 6000 steps...██████████████████████████████████████████████████████████████████████████████████████| 4/4 [03:57<00:00, 59.13s/it] +processing left-padding... +processing left-padding... +processing left-padding... +processing left-padding... +[2025-04-24 04:04:27] Saved checkpoint to checkpoint/Afteri2i_100kckpt_LLAVAT5_empty_Fixed_just_segment_pretrain_stage3_dreambench_recap_400k_t2i_400k_flux_200k_midjourney_150k_recovery_150k_3e4_epoch2_no_rotated/000-GPT-XL/checkpoints/0006000.pt +[2025-04-24 04:04:29] Removed old checkpoint: checkpoint/Afteri2i_100kckpt_LLAVAT5_empty_Fixed_just_segment_pretrain_stage3_dreambench_recap_400k_t2i_400k_flux_200k_midjourney_150k_recovery_150k_3e4_epoch2_no_rotated/000-GPT-XL/checkpoints/0004000.pt + 26%|█████████████████████████████████▏ | 6024/23458 [1:54:14<4:16:42, 1.13it/s][2025-04-24 04:04:51] (step=0006025) Train Loss: 5.6376, Train Steps/Sec: 0.07 + 26%|█████████████████████████████████▎ | 6049/23458 [1:54:36<4:10:02, 1.16it/s][2025-04-24 04:05:12] (step=0006050) Train Loss: 5.6994, Train Steps/Sec: 1.17 + 26%|█████████████████████████████████▍ | 6074/23458 [1:54:57<4:06:29, 1.18it/s][2025-04-24 04:05:33] (step=0006075) Train Loss: 5.5185, Train Steps/Sec: 1.17 + 26%|█████████████████████████████████▌ | 6099/23458 [1:55:19<4:04:29, 1.18it/s][2025-04-24 04:05:55] (step=0006100) Train Loss: 5.6487, Train Steps/Sec: 1.16 + 26%|█████████████████████████████████▋ | 6124/23458 [1:55:40<4:11:32, 1.15it/s][2025-04-24 04:06:16] (step=0006125) Train Loss: 5.5975, Train Steps/Sec: 1.17 + 26%|█████████████████████████████████▊ | 6149/23458 [1:56:01<4:06:26, 1.17it/s][2025-04-24 04:06:38] (step=0006150) Train Loss: 5.6118, Train Steps/Sec: 1.17 + 26%|█████████████████████████████████▉ | 6174/23458 [1:56:23<4:05:00, 1.18it/s][2025-04-24 04:06:59] (step=0006175) Train Loss: 5.6022, Train Steps/Sec: 1.17 + 26%|██████████████████████████████████ | 6199/23458 [1:56:44<4:03:02, 1.18it/s][2025-04-24 04:07:20] (step=0006200) Train Loss: 5.6536, Train Steps/Sec: 1.17 + 27%|██████████████████████████████████▏ | 6224/23458 [1:57:07<4:10:30, 1.15it/s][2025-04-24 04:07:43] (step=0006225) Train Loss: 5.6016, Train Steps/Sec: 1.11 + 27%|██████████████████████████████████▎ | 6249/23458 [1:57:29<4:12:42, 1.13it/s][2025-04-24 04:08:05] (step=0006250) Train Loss: 5.6505, Train Steps/Sec: 1.12 + 27%|██████████████████████████████████▌ | 6274/23458 [1:57:51<4:05:47, 1.17it/s][2025-04-24 04:08:27] (step=0006275) Train Loss: 5.5216, Train Steps/Sec: 1.16 + 27%|██████████████████████████████████▋ | 6299/23458 [1:58:12<4:02:37, 1.18it/s][2025-04-24 04:08:48] (step=0006300) Train Loss: 5.5498, Train Steps/Sec: 1.16 + 27%|██████████████████████████████████▊ | 6324/23458 [1:58:33<4:07:18, 1.15it/s][2025-04-24 04:09:10] (step=0006325) Train Loss: 5.6056, Train Steps/Sec: 1.17 + 27%|██████████████████████████████████▉ | 6349/23458 [1:58:55<4:05:34, 1.16it/s][2025-04-24 04:09:31] (step=0006350) Train Loss: 5.5886, Train Steps/Sec: 1.17 + 27%|███████████████████████████████████ | 6374/23458 [1:59:17<4:02:41, 1.17it/s][2025-04-24 04:09:53] (step=0006375) Train Loss: 5.6752, Train Steps/Sec: 1.15 + 27%|███████████████████████████████████▏ | 6399/23458 [1:59:38<4:01:03, 1.18it/s][2025-04-24 04:10:14] (step=0006400) Train Loss: 5.5540, Train Steps/Sec: 1.16 + 27%|███████████████████████████████████▎ | 6424/23458 [1:59:59<4:06:49, 1.15it/s][2025-04-24 04:10:36] (step=0006425) Train Loss: 5.6114, Train Steps/Sec: 1.17 + 27%|███████████████████████████████████▍ | 6449/23458 [2:00:21<4:02:47, 1.17it/s][2025-04-24 04:10:57] (step=0006450) Train Loss: 5.5641, Train Steps/Sec: 1.17 + 28%|███████████████████████████████████▌ | 6474/23458 [2:00:42<4:00:41, 1.18it/s][2025-04-24 04:11:18] (step=0006475) Train Loss: 5.5893, Train Steps/Sec: 1.17 + 28%|███████████████████████████████████▋ | 6499/23458 [2:01:03<3:58:45, 1.18it/s][2025-04-24 04:11:40] (step=0006500) Train Loss: 5.5561, Train Steps/Sec: 1.17 + 28%|███████████████████████████████████▉ | 6524/23458 [2:01:27<4:13:25, 1.11it/s][2025-04-24 04:12:03] (step=0006525) Train Loss: 5.6092, Train Steps/Sec: 1.07 + 28%|████████████████████████████████████ | 6549/23458 [2:01:50<4:01:55, 1.16it/s][2025-04-24 04:12:26] (step=0006550) Train Loss: 5.6367, Train Steps/Sec: 1.07 + 28%|████████████████████████████████████▏ | 6574/23458 [2:02:12<3:59:40, 1.17it/s][2025-04-24 04:12:48] (step=0006575) Train Loss: 5.6732, Train Steps/Sec: 1.16 + 28%|████████████████████████████████████▎ | 6599/23458 [2:02:33<3:58:19, 1.18it/s][2025-04-24 04:13:09] (step=0006600) Train Loss: 5.5629, Train Steps/Sec: 1.16 + 28%|████████████████████████████████████▍ | 6624/23458 [2:02:55<4:04:14, 1.15it/s][2025-04-24 04:13:31] (step=0006625) Train Loss: 5.5156, Train Steps/Sec: 1.16 + 28%|████████████████████████████████████▌ | 6649/23458 [2:03:18<4:33:35, 1.02it/s][2025-04-24 04:13:54] (step=0006650) Train Loss: 5.5275, Train Steps/Sec: 1.09 + 28%|████████████████████████████████████▋ | 6674/23458 [2:03:39<3:57:54, 1.18it/s][2025-04-24 04:14:15] (step=0006675) Train Loss: 5.6160, Train Steps/Sec: 1.17 + 29%|████████████████████████████████████▊ | 6699/23458 [2:04:00<3:56:27, 1.18it/s][2025-04-24 04:14:37] (step=0006700) Train Loss: 5.5878, Train Steps/Sec: 1.16 + 29%|████████████████████████████████████▉ | 6724/23458 [2:04:22<4:03:31, 1.15it/s][2025-04-24 04:14:58] (step=0006725) Train Loss: 5.5768, Train Steps/Sec: 1.17 + 29%|█████████████████████████████████████ | 6749/23458 [2:04:43<3:59:11, 1.16it/s][2025-04-24 04:15:20] (step=0006750) Train Loss: 5.5450, Train Steps/Sec: 1.17 + 29%|█████████████████████████████████████▎ | 6774/23458 [2:05:05<3:58:41, 1.16it/s][2025-04-24 04:15:41] (step=0006775) Train Loss: 5.6187, Train Steps/Sec: 1.17 + 29%|█████████████████████████████████████▍ | 6799/23458 [2:05:26<3:53:48, 1.19it/s][2025-04-24 04:16:02] (step=0006800) Train Loss: 5.6300, Train Steps/Sec: 1.17 + 29%|█████████████████████████████████████▌ | 6824/23458 [2:05:48<4:00:19, 1.15it/s][2025-04-24 04:16:24] (step=0006825) Train Loss: 5.6439, Train Steps/Sec: 1.17 + 29%|█████████████████████████████████████▋ | 6849/23458 [2:06:09<3:56:27, 1.17it/s][2025-04-24 04:16:45] (step=0006850) Train Loss: 5.6374, Train Steps/Sec: 1.17 + 29%|█████████████████████████████████████▊ | 6874/23458 [2:06:30<3:54:35, 1.18it/s][2025-04-24 04:17:07] (step=0006875) Train Loss: 5.5296, Train Steps/Sec: 1.17 + 29%|█████████████████████████████████████▉ | 6899/23458 [2:06:52<3:53:16, 1.18it/s][2025-04-24 04:17:28] (step=0006900) Train Loss: 5.6635, Train Steps/Sec: 1.17 + 30%|██████████████████████████████████████ | 6924/23458 [2:07:13<4:00:23, 1.15it/s][2025-04-24 04:17:49] (step=0006925) Train Loss: 5.5907, Train Steps/Sec: 1.17 + 30%|██████████████████████████████████████▏ | 6949/23458 [2:07:36<3:56:48, 1.16it/s][2025-04-24 04:18:12] (step=0006950) Train Loss: 5.5792, Train Steps/Sec: 1.12 + 30%|██████████████████████████████████████▎ | 6974/23458 [2:07:58<3:56:24, 1.16it/s][2025-04-24 04:18:34] (step=0006975) Train Loss: 5.5694, Train Steps/Sec: 1.11 + 30%|██████████████████████████████████████▍ | 6999/23458 [2:08:19<3:51:58, 1.18it/s][2025-04-24 04:18:56] (step=0007000) Train Loss: 5.6567, Train Steps/Sec: 1.16 + 30%|██████████████████████████████████████▋ | 7024/23458 [2:08:41<3:58:46, 1.15it/s][2025-04-24 04:19:17] (step=0007025) Train Loss: 5.5618, Train Steps/Sec: 1.17 + 30%|██████████████████████████████████████▊ | 7049/23458 [2:09:02<3:54:45, 1.16it/s][2025-04-24 04:19:39] (step=0007050) Train Loss: 5.5351, Train Steps/Sec: 1.16 + 30%|██████████████████████████████████████▉ | 7074/23458 [2:09:24<3:52:42, 1.17it/s][2025-04-24 04:20:00] (step=0007075) Train Loss: 5.4690, Train Steps/Sec: 1.17 + 30%|███████████████████████████████████████ | 7099/23458 [2:09:45<3:50:56, 1.18it/s][2025-04-24 04:20:22] (step=0007100) Train Loss: 5.6560, Train Steps/Sec: 1.16 + 30%|███████████████████████████████████████▏ | 7124/23458 [2:10:07<3:56:41, 1.15it/s][2025-04-24 04:20:43] (step=0007125) Train Loss: 5.6232, Train Steps/Sec: 1.17 + 30%|███████████████████████████████████████▎ | 7149/23458 [2:10:28<3:53:14, 1.17it/s][2025-04-24 04:21:04] (step=0007150) Train Loss: 5.6039, Train Steps/Sec: 1.17 + 31%|███████████████████████████████████████▍ | 7174/23458 [2:10:51<4:09:31, 1.09it/s][2025-04-24 04:21:27] (step=0007175) Train Loss: 5.5603, Train Steps/Sec: 1.12 + 31%|███████████████████████████████████████▌ | 7199/23458 [2:11:15<3:54:17, 1.16it/s][2025-04-24 04:21:51] (step=0007200) Train Loss: 5.5331, Train Steps/Sec: 1.02 + 31%|███████████████████████████████████████▋ | 7224/23458 [2:11:36<3:55:35, 1.15it/s][2025-04-24 04:22:13] (step=0007225) Train Loss: 5.5614, Train Steps/Sec: 1.17 + 31%|███████████████████████████████████████▊ | 7249/23458 [2:11:58<3:52:28, 1.16it/s][2025-04-24 04:22:34] (step=0007250) Train Loss: 5.6110, Train Steps/Sec: 1.17 + 31%|████████████████████████████████████████ | 7274/23458 [2:12:19<3:49:15, 1.18it/s][2025-04-24 04:22:55] (step=0007275) Train Loss: 5.5729, Train Steps/Sec: 1.17 + 31%|████████████████████████████████████████▏ | 7299/23458 [2:12:41<3:50:00, 1.17it/s][2025-04-24 04:23:18] (step=0007300) Train Loss: 5.6187, Train Steps/Sec: 1.12 + 31%|████████████████████████████████████████▎ | 7324/23458 [2:13:04<3:53:23, 1.15it/s][2025-04-24 04:23:40] (step=0007325) Train Loss: 5.6047, Train Steps/Sec: 1.13 + 31%|████████████████████████████████████████▍ | 7349/23458 [2:13:25<3:50:34, 1.16it/s][2025-04-24 04:24:01] (step=0007350) Train Loss: 5.5474, Train Steps/Sec: 1.17 + 31%|████████████████████████████████████████▌ | 7374/23458 [2:13:47<3:48:35, 1.17it/s][2025-04-24 04:24:23] (step=0007375) Train Loss: 5.6287, Train Steps/Sec: 1.17 + 32%|████████████████████████████████████████▋ | 7399/23458 [2:14:08<3:46:20, 1.18it/s][2025-04-24 04:24:44] (step=0007400) Train Loss: 5.5149, Train Steps/Sec: 1.16 + 32%|████████████████████████████████████████▊ | 7424/23458 [2:14:29<3:54:08, 1.14it/s][2025-04-24 04:25:06] (step=0007425) Train Loss: 5.6450, Train Steps/Sec: 1.17 + 32%|████████████████████████████████████████▉ | 7449/23458 [2:14:51<3:50:00, 1.16it/s][2025-04-24 04:25:27] (step=0007450) Train Loss: 5.5700, Train Steps/Sec: 1.17 + 32%|█████████████████████████████████████████ | 7474/23458 [2:15:12<3:46:49, 1.17it/s][2025-04-24 04:25:48] (step=0007475) Train Loss: 5.6698, Train Steps/Sec: 1.17 + 32%|█████████████████████████████████████████▏ | 7499/23458 [2:15:34<3:44:57, 1.18it/s][2025-04-24 04:26:10] (step=0007500) Train Loss: 5.6018, Train Steps/Sec: 1.16 + 32%|█████████████████████████████████████████▍ | 7524/23458 [2:15:55<3:51:00, 1.15it/s][2025-04-24 04:26:31] (step=0007525) Train Loss: 5.5281, Train Steps/Sec: 1.17 + 32%|█████████████████████████████████████████▌ | 7549/23458 [2:16:17<3:47:43, 1.16it/s][2025-04-24 04:26:53] (step=0007550) Train Loss: 5.5381, Train Steps/Sec: 1.17 + 32%|█████████████████████████████████████████▋ | 7574/23458 [2:16:38<3:45:08, 1.18it/s][2025-04-24 04:27:14] (step=0007575) Train Loss: 5.6411, Train Steps/Sec: 1.17 + 32%|█████████████████████████████████████████▊ | 7599/23458 [2:16:59<3:43:42, 1.18it/s][2025-04-24 04:27:36] (step=0007600) Train Loss: 5.5474, Train Steps/Sec: 1.17 + 33%|█████████████████████████████████████████▉ | 7624/23458 [2:17:21<3:48:47, 1.15it/s][2025-04-24 04:27:57] (step=0007625) Train Loss: 5.6712, Train Steps/Sec: 1.17 + 33%|██████████████████████████████████████████ | 7649/23458 [2:17:42<3:45:14, 1.17it/s][2025-04-24 04:28:18] (step=0007650) Train Loss: 5.5934, Train Steps/Sec: 1.17 + 33%|██████████████████████████████████████████▏ | 7674/23458 [2:18:05<3:43:13, 1.18it/s][2025-04-24 04:28:41] (step=0007675) Train Loss: 5.5648, Train Steps/Sec: 1.12 + 33%|██████████████████████████████████████████▎ | 7699/23458 [2:18:27<3:44:14, 1.17it/s][2025-04-24 04:29:03] (step=0007700) Train Loss: 5.4692, Train Steps/Sec: 1.11 + 33%|██████████████████████████████████████████▍ | 7724/23458 [2:18:49<3:47:47, 1.15it/s][2025-04-24 04:29:25] (step=0007725) Train Loss: 5.6143, Train Steps/Sec: 1.17 + 33%|██████████████████████████████████████████▌ | 7749/23458 [2:19:10<3:44:09, 1.17it/s][2025-04-24 04:29:46] (step=0007750) Train Loss: 5.5223, Train Steps/Sec: 1.17 + 33%|██████████████████████████████████████████▊ | 7774/23458 [2:19:31<3:42:13, 1.18it/s][2025-04-24 04:30:08] (step=0007775) Train Loss: 5.5483, Train Steps/Sec: 1.17 + 33%|██████████████████████████████████████████▉ | 7799/23458 [2:19:53<3:42:03, 1.18it/s][2025-04-24 04:30:29] (step=0007800) Train Loss: 5.6109, Train Steps/Sec: 1.16 + 33%|███████████████████████████████████████████ | 7824/23458 [2:20:14<3:46:18, 1.15it/s][2025-04-24 04:30:50] (step=0007825) Train Loss: 5.5703, Train Steps/Sec: 1.17 + 33%|███████████████████████████████████████████▏ | 7849/23458 [2:20:38<4:02:28, 1.07it/s][2025-04-24 04:31:14] (step=0007850) Train Loss: 5.6036, Train Steps/Sec: 1.04 + 34%|███████████████████████████████████████████▎ | 7874/23458 [2:21:01<3:42:35, 1.17it/s][2025-04-24 04:31:37] (step=0007875) Train Loss: 5.5698, Train Steps/Sec: 1.11 + 34%|███████████████████████████████████████████▍ | 7899/23458 [2:21:22<3:39:08, 1.18it/s][2025-04-24 04:31:59] (step=0007900) Train Loss: 5.6066, Train Steps/Sec: 1.16 + 34%|███████████████████████████████████████████▌ | 7924/23458 [2:21:44<3:46:33, 1.14it/s][2025-04-24 04:32:20] (step=0007925) Train Loss: 5.5746, Train Steps/Sec: 1.16 + 34%|███████████████████████████████████████████▋ | 7949/23458 [2:22:05<3:42:56, 1.16it/s][2025-04-24 04:32:42] (step=0007950) Train Loss: 5.6176, Train Steps/Sec: 1.16 + 34%|███████████████████████████████████████████▊ | 7974/23458 [2:22:28<3:40:56, 1.17it/s][2025-04-24 04:33:05] (step=0007975) Train Loss: 5.5800, Train Steps/Sec: 1.08 + 34%|███████████████████████████████████████████▉ | 7999/23458 [2:22:50<3:37:50, 1.18it/s][2025-04-24 04:33:26] (step=0008000) Train Loss: 5.6072, Train Steps/Sec: 1.16 +You are using a model of type blip-2 to instantiate a model of type instructblip. This is not supported for all configurations of models and can yield errors. +[2025-04-24 04:33:26] text_config is None. Initializing the text config with default values (`OPTConfig`). +model_name_or_path:/tmp/haozhezhao/model/blip2-flan-t5-xl +image_place_holder: +tokenizer length after expend 32101 +tokenizer length before expend 32100 +100%|████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████| 4/4 [03:59<00:00, 59.85s/it] +[2025-04-24 04:38:34] Finish Eval in 8000 steps...██████████████████████████████████████████████████████████████████████████████████████| 4/4 [03:58<00:00, 59.47s/it] +processing left-padding... +processing left-padding... +processing left-padding... +processing left-padding... +[2025-04-24 04:38:53] Saved checkpoint to checkpoint/Afteri2i_100kckpt_LLAVAT5_empty_Fixed_just_segment_pretrain_stage3_dreambench_recap_400k_t2i_400k_flux_200k_midjourney_150k_recovery_150k_3e4_epoch2_no_rotated/000-GPT-XL/checkpoints/0008000.pt +[2025-04-24 04:38:55] Removed old checkpoint: checkpoint/Afteri2i_100kckpt_LLAVAT5_empty_Fixed_just_segment_pretrain_stage3_dreambench_recap_400k_t2i_400k_flux_200k_midjourney_150k_recovery_150k_3e4_epoch2_no_rotated/000-GPT-XL/checkpoints/0006000.pt + 34%|████████████████████████████████████████████▏ | 8024/23458 [2:28:40<3:48:50, 1.12it/s][2025-04-24 04:39:16] (step=0008025) Train Loss: 5.5394, Train Steps/Sec: 0.07 + 34%|████████████████████████████████████████████▎ | 8049/23458 [2:29:02<3:39:51, 1.17it/s][2025-04-24 04:39:38] (step=0008050) Train Loss: 5.7179, Train Steps/Sec: 1.17 + 34%|████████████████████████████████████████████▍ | 8074/23458 [2:29:23<3:38:29, 1.17it/s][2025-04-24 04:39:59] (step=0008075) Train Loss: 5.6368, Train Steps/Sec: 1.17 + 35%|████████████████████████████████████████████▌ | 8099/23458 [2:29:44<3:37:07, 1.18it/s][2025-04-24 04:40:21] (step=0008100) Train Loss: 5.7061, Train Steps/Sec: 1.16 + 35%|████████████████████████████████████████████▋ | 8124/23458 [2:30:06<3:42:37, 1.15it/s][2025-04-24 04:40:42] (step=0008125) Train Loss: 5.5919, Train Steps/Sec: 1.17 + 35%|████████████████████████████████████████████▊ | 8149/23458 [2:30:27<3:39:30, 1.16it/s][2025-04-24 04:41:03] (step=0008150) Train Loss: 5.5466, Train Steps/Sec: 1.17 + 35%|████████████████████████████████████████████▉ | 8174/23458 [2:30:49<3:36:42, 1.18it/s][2025-04-24 04:41:25] (step=0008175) Train Loss: 5.5152, Train Steps/Sec: 1.17 + 35%|█████████████████████████████████████████████ | 8199/23458 [2:31:10<3:34:58, 1.18it/s][2025-04-24 04:41:46] (step=0008200) Train Loss: 5.5640, Train Steps/Sec: 1.17 + 35%|█████████████████████████████████████████████▏ | 8224/23458 [2:31:31<3:40:14, 1.15it/s][2025-04-24 04:42:08] (step=0008225) Train Loss: 5.5893, Train Steps/Sec: 1.17 + 35%|█████████████████████████████████████████████▎ | 8249/23458 [2:31:53<3:36:53, 1.17it/s][2025-04-24 04:42:29] (step=0008250) Train Loss: 5.6468, Train Steps/Sec: 1.17 + 35%|█████████████████████████████████████████████▌ | 8274/23458 [2:32:14<3:35:14, 1.18it/s][2025-04-24 04:42:50] (step=0008275) Train Loss: 5.5346, Train Steps/Sec: 1.17 + 35%|█████████████████████████████████████████████▋ | 8299/23458 [2:32:36<3:34:25, 1.18it/s][2025-04-24 04:43:12] (step=0008300) Train Loss: 5.6475, Train Steps/Sec: 1.14 + 35%|█████████████████████████████████████████████▊ | 8324/23458 [2:32:57<3:39:47, 1.15it/s][2025-04-24 04:43:33] (step=0008325) Train Loss: 5.6025, Train Steps/Sec: 1.17 + 36%|█████████████████████████████████████████████▉ | 8349/23458 [2:33:19<3:35:46, 1.17it/s][2025-04-24 04:43:55] (step=0008350) Train Loss: 5.6005, Train Steps/Sec: 1.17 + 36%|██████████████████████████████████████████████ | 8374/23458 [2:33:40<3:34:19, 1.17it/s][2025-04-24 04:44:17] (step=0008375) Train Loss: 5.5665, Train Steps/Sec: 1.11 + 36%|██████████████████████████████████████████████▏ | 8399/23458 [2:34:02<3:32:40, 1.18it/s][2025-04-24 04:44:39] (step=0008400) Train Loss: 5.5267, Train Steps/Sec: 1.16 + 36%|██████████████████████████████████████████████▎ | 8424/23458 [2:34:25<3:42:28, 1.13it/s][2025-04-24 04:45:01] (step=0008425) Train Loss: 5.5358, Train Steps/Sec: 1.11 + 36%|██████████████████████████████████████████████▍ | 8449/23458 [2:34:46<3:33:53, 1.17it/s][2025-04-24 04:45:23] (step=0008450) Train Loss: 5.6232, Train Steps/Sec: 1.17 + 36%|██████████████████████████████████████████████▌ | 8474/23458 [2:35:08<3:32:00, 1.18it/s][2025-04-24 04:45:44] (step=0008475) Train Loss: 5.6399, Train Steps/Sec: 1.17 + 36%|██████████████████████████████████████████████▋ | 8499/23458 [2:35:31<4:15:10, 1.02s/it][2025-04-24 04:46:07] (step=0008500) Train Loss: 5.5162, Train Steps/Sec: 1.07 + 36%|██████████████████████████████████████████████▉ | 8524/23458 [2:35:54<3:41:16, 1.12it/s][2025-04-24 04:46:30] (step=0008525) Train Loss: 5.6092, Train Steps/Sec: 1.08 + 36%|███████████████████████████████████████████████ | 8549/23458 [2:36:16<3:32:56, 1.17it/s][2025-04-24 04:46:52] (step=0008550) Train Loss: 5.6503, Train Steps/Sec: 1.17 + 37%|███████████████████████████████████████████████▏ | 8574/23458 [2:36:37<3:30:54, 1.18it/s][2025-04-24 04:47:13] (step=0008575) Train Loss: 5.5699, Train Steps/Sec: 1.17 + 37%|███████████████████████████████████████████████▎ | 8599/23458 [2:36:58<3:29:22, 1.18it/s][2025-04-24 04:47:35] (step=0008600) Train Loss: 5.5621, Train Steps/Sec: 1.17 + 37%|███████████████████████████████████████████████▍ | 8624/23458 [2:37:21<3:35:50, 1.15it/s][2025-04-24 04:47:57] (step=0008625) Train Loss: 5.5764, Train Steps/Sec: 1.12 + 37%|███████████████████████████████████████████████▌ | 8649/23458 [2:37:43<4:00:38, 1.03it/s][2025-04-24 04:48:19] (step=0008650) Train Loss: 5.5777, Train Steps/Sec: 1.13 + 37%|███████████████████████████████████████████████▋ | 8674/23458 [2:38:04<3:29:39, 1.18it/s][2025-04-24 04:48:40] (step=0008675) Train Loss: 5.5463, Train Steps/Sec: 1.17 + 37%|███████████████████████████████████████████████▊ | 8699/23458 [2:38:26<3:28:34, 1.18it/s][2025-04-24 04:49:02] (step=0008700) Train Loss: 5.5819, Train Steps/Sec: 1.16 + 37%|███████████████████████████████████████████████▉ | 8724/23458 [2:38:47<3:35:48, 1.14it/s][2025-04-24 04:49:23] (step=0008725) Train Loss: 5.5966, Train Steps/Sec: 1.17 + 37%|████████████████████████████████████████████████ | 8749/23458 [2:39:09<3:29:50, 1.17it/s][2025-04-24 04:49:45] (step=0008750) Train Loss: 5.6097, Train Steps/Sec: 1.17 + 37%|████████████████████████████████████████████████▏ | 8774/23458 [2:39:30<3:28:25, 1.17it/s][2025-04-24 04:50:06] (step=0008775) Train Loss: 5.6725, Train Steps/Sec: 1.17 + 38%|████████████████████████████████████████████████▍ | 8799/23458 [2:39:51<3:26:46, 1.18it/s][2025-04-24 04:50:28] (step=0008800) Train Loss: 5.5910, Train Steps/Sec: 1.16 + 38%|████████████████████████████████████████████████▌ | 8824/23458 [2:40:13<3:33:38, 1.14it/s][2025-04-24 04:50:49] (step=0008825) Train Loss: 5.5610, Train Steps/Sec: 1.17 + 38%|████████████████████████████████████████████████▋ | 8849/23458 [2:40:34<3:28:40, 1.17it/s][2025-04-24 04:51:10] (step=0008850) Train Loss: 5.6187, Train Steps/Sec: 1.17 + 38%|████████████████████████████████████████████████▊ | 8874/23458 [2:40:56<3:26:58, 1.17it/s][2025-04-24 04:51:32] (step=0008875) Train Loss: 5.5719, Train Steps/Sec: 1.17 + 38%|████████████████████████████████████████████████▉ | 8899/23458 [2:41:17<3:24:57, 1.18it/s][2025-04-24 04:51:53] (step=0008900) Train Loss: 5.6100, Train Steps/Sec: 1.17 + 38%|█████████████████████████████████████████████████ | 8924/23458 [2:41:38<3:31:30, 1.15it/s][2025-04-24 04:52:15] (step=0008925) Train Loss: 5.7115, Train Steps/Sec: 1.17 + 38%|█████████████████████████████████████████████████▏ | 8949/23458 [2:42:00<3:26:56, 1.17it/s][2025-04-24 04:52:36] (step=0008950) Train Loss: 5.4613, Train Steps/Sec: 1.17 + 38%|█████████████████████████████████████████████████▎ | 8974/23458 [2:42:21<3:25:22, 1.18it/s][2025-04-24 04:52:57] (step=0008975) Train Loss: 5.5932, Train Steps/Sec: 1.17 + 38%|█████████████████████████████████████████████████▍ | 8999/23458 [2:42:43<3:24:10, 1.18it/s][2025-04-24 04:53:19] (step=0009000) Train Loss: 5.5512, Train Steps/Sec: 1.16 + 38%|█████████████████████████████████████████████████▌ | 9024/23458 [2:43:04<3:29:37, 1.15it/s][2025-04-24 04:53:40] (step=0009025) Train Loss: 5.5649, Train Steps/Sec: 1.17 + 39%|█████████████████████████████████████████████████▊ | 9049/23458 [2:43:25<3:25:59, 1.17it/s][2025-04-24 04:54:02] (step=0009050) Train Loss: 5.4963, Train Steps/Sec: 1.17 + 39%|█████████████████████████████████████████████████▉ | 9074/23458 [2:43:47<3:23:22, 1.18it/s][2025-04-24 04:54:23] (step=0009075) Train Loss: 5.6044, Train Steps/Sec: 1.17 + 39%|██████████████████████████████████████████████████ | 9099/23458 [2:44:09<4:14:08, 1.06s/it][2025-04-24 04:54:45] (step=0009100) Train Loss: 5.5861, Train Steps/Sec: 1.11 + 39%|██████████████████████████████████████████████████▏ | 9124/23458 [2:44:31<3:27:37, 1.15it/s][2025-04-24 04:55:07] (step=0009125) Train Loss: 5.5719, Train Steps/Sec: 1.17 + 39%|██████████████████████████████████████████████████▎ | 9149/23458 [2:44:54<3:28:45, 1.14it/s][2025-04-24 04:55:30] (step=0009150) Train Loss: 5.6192, Train Steps/Sec: 1.07 + 39%|██████████████████████████████████████████████████▍ | 9174/23458 [2:45:18<3:56:37, 1.01it/s][2025-04-24 04:55:54] (step=0009175) Train Loss: 5.5947, Train Steps/Sec: 1.05 + 39%|██████████████████████████████████████████████████▌ | 9199/23458 [2:45:39<3:20:48, 1.18it/s][2025-04-24 04:56:15] (step=0009200) Train Loss: 5.6048, Train Steps/Sec: 1.17 + 39%|██████████████████████████████████████████████████▋ | 9224/23458 [2:46:01<3:26:15, 1.15it/s][2025-04-24 04:56:37] (step=0009225) Train Loss: 5.6334, Train Steps/Sec: 1.17 + 39%|██████████████████████████████████████████████████▊ | 9249/23458 [2:46:22<3:22:32, 1.17it/s][2025-04-24 04:56:58] (step=0009250) Train Loss: 5.4839, Train Steps/Sec: 1.17 + 40%|██████████████████████████████████████████████████▉ | 9274/23458 [2:46:44<3:25:56, 1.15it/s][2025-04-24 04:57:21] (step=0009275) Train Loss: 5.6271, Train Steps/Sec: 1.13 + 40%|███████████████████████████████████████████████████▏ | 9299/23458 [2:47:06<3:20:42, 1.18it/s][2025-04-24 04:57:42] (step=0009300) Train Loss: 5.6049, Train Steps/Sec: 1.17 + 40%|███████████████████████████████████████████████████▎ | 9324/23458 [2:47:28<3:25:52, 1.14it/s][2025-04-24 04:58:04] (step=0009325) Train Loss: 5.5851, Train Steps/Sec: 1.13 + 40%|███████████████████████████████████████████████████▍ | 9349/23458 [2:47:49<3:21:36, 1.17it/s][2025-04-24 04:58:26] (step=0009350) Train Loss: 5.5594, Train Steps/Sec: 1.17 + 40%|███████████████████████████████████████████████████▌ | 9374/23458 [2:48:11<3:19:16, 1.18it/s][2025-04-24 04:58:47] (step=0009375) Train Loss: 5.5903, Train Steps/Sec: 1.17 + 40%|███████████████████████████████████████████████████▋ | 9399/23458 [2:48:32<3:18:25, 1.18it/s][2025-04-24 04:59:08] (step=0009400) Train Loss: 5.5579, Train Steps/Sec: 1.17 + 40%|███████████████████████████████████████████████████▊ | 9424/23458 [2:48:54<3:23:28, 1.15it/s][2025-04-24 04:59:30] (step=0009425) Train Loss: 5.5823, Train Steps/Sec: 1.17 + 40%|███████████████████████████████████████████████████▉ | 9449/23458 [2:49:15<3:20:03, 1.17it/s][2025-04-24 04:59:51] (step=0009450) Train Loss: 5.5779, Train Steps/Sec: 1.17 + 40%|████████████████████████████████████████████████████ | 9474/23458 [2:49:36<3:18:57, 1.17it/s][2025-04-24 05:00:13] (step=0009475) Train Loss: 5.5650, Train Steps/Sec: 1.17 + 40%|████████████████████████████████████████████████████▏ | 9499/23458 [2:49:58<3:16:55, 1.18it/s][2025-04-24 05:00:34] (step=0009500) Train Loss: 5.6458, Train Steps/Sec: 1.16 + 41%|████████████████████████████████████████████████████▎ | 9524/23458 [2:50:19<3:20:59, 1.16it/s][2025-04-24 05:00:55] (step=0009525) Train Loss: 5.5350, Train Steps/Sec: 1.17 + 41%|████████████████████████████████████████████████████▌ | 9549/23458 [2:50:40<3:18:36, 1.17it/s][2025-04-24 05:01:17] (step=0009550) Train Loss: 5.5697, Train Steps/Sec: 1.17 + 41%|████████████████████████████████████████████████████▋ | 9574/23458 [2:51:02<3:16:41, 1.18it/s][2025-04-24 05:01:38] (step=0009575) Train Loss: 5.5861, Train Steps/Sec: 1.17 + 41%|████████████████████████████████████████████████████▊ | 9599/23458 [2:51:23<3:14:47, 1.19it/s][2025-04-24 05:02:00] (step=0009600) Train Loss: 5.5548, Train Steps/Sec: 1.16 + 41%|████████████████████████████████████████████████████▉ | 9624/23458 [2:51:45<3:20:44, 1.15it/s][2025-04-24 05:02:21] (step=0009625) Train Loss: 5.6191, Train Steps/Sec: 1.17 + 41%|█████████████████████████████████████████████████████ | 9649/23458 [2:52:06<3:16:38, 1.17it/s][2025-04-24 05:02:42] (step=0009650) Train Loss: 5.5496, Train Steps/Sec: 1.17 + 41%|█████████████████████████████████████████████████████▏ | 9674/23458 [2:52:27<3:15:52, 1.17it/s][2025-04-24 05:03:04] (step=0009675) Train Loss: 5.5891, Train Steps/Sec: 1.17 + 41%|█████████████████████████████████████████████████████▎ | 9699/23458 [2:52:49<3:14:16, 1.18it/s][2025-04-24 05:03:25] (step=0009700) Train Loss: 5.5376, Train Steps/Sec: 1.16 + 41%|█████████████████████████████████████████████████████▍ | 9724/23458 [2:53:10<3:18:58, 1.15it/s][2025-04-24 05:03:46] (step=0009725) Train Loss: 5.5166, Train Steps/Sec: 1.17 + 42%|█████████████████████████████████████████████████████▌ | 9749/23458 [2:53:32<3:16:37, 1.16it/s][2025-04-24 05:04:08] (step=0009750) Train Loss: 5.5685, Train Steps/Sec: 1.17 + 42%|█████████████████████████████████████████████████████▋ | 9774/23458 [2:53:53<3:14:10, 1.17it/s][2025-04-24 05:04:29] (step=0009775) Train Loss: 5.5803, Train Steps/Sec: 1.17 + 42%|█████████████████████████████████████████████████████▉ | 9799/23458 [2:54:14<3:11:41, 1.19it/s][2025-04-24 05:04:52] (step=0009800) Train Loss: 5.5779, Train Steps/Sec: 1.17 + 42%|██████████████████████████████████████████████████████ | 9824/23458 [2:54:39<3:56:53, 1.04s/it][2025-04-24 05:05:16] (step=0009825) Train Loss: 5.5227, Train Steps/Sec: 1.04 + 42%|██████████████████████████████████████████████████████▏ | 9849/23458 [2:55:02<3:14:17, 1.17it/s][2025-04-24 05:05:38] (step=0009850) Train Loss: 5.5784, Train Steps/Sec: 1.12 + 42%|██████████████████████████████████████████████████████▎ | 9874/23458 [2:55:24<3:13:51, 1.17it/s][2025-04-24 05:06:01] (step=0009875) Train Loss: 5.5709, Train Steps/Sec: 1.11 + 42%|██████████████████████████████████████████████████████▍ | 9899/23458 [2:55:46<3:10:24, 1.19it/s][2025-04-24 05:06:22] (step=0009900) Train Loss: 5.5935, Train Steps/Sec: 1.17 + 42%|██████████████████████████████████████████████████████▌ | 9924/23458 [2:56:07<3:16:09, 1.15it/s][2025-04-24 05:06:43] (step=0009925) Train Loss: 5.5775, Train Steps/Sec: 1.17 + 42%|██████████████████████████████████████████████████████▋ | 9949/23458 [2:56:30<3:12:57, 1.17it/s][2025-04-24 05:07:06] (step=0009950) Train Loss: 5.5773, Train Steps/Sec: 1.12 + 43%|██████████████████████████████████████████████████████▊ | 9974/23458 [2:56:52<3:17:17, 1.14it/s][2025-04-24 05:07:28] (step=0009975) Train Loss: 5.5334, Train Steps/Sec: 1.13 + 43%|██████████████████████████████████████████████████████▉ | 9999/23458 [2:57:13<3:09:27, 1.18it/s][2025-04-24 05:07:49] (step=0010000) Train Loss: 5.5647, Train Steps/Sec: 1.17 +You are using a model of type blip-2 to instantiate a model of type instructblip. This is not supported for all configurations of models and can yield errors. +[2025-04-24 05:07:49] text_config is None. Initializing the text config with default values (`OPTConfig`). +model_name_or_path:/tmp/haozhezhao/model/blip2-flan-t5-xl +image_place_holder: +tokenizer length after expend 32101 +tokenizer length before expend 32100 +100%|████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████| 4/4 [03:57<00:00, 59.49s/it] +[2025-04-24 05:12:57] Finish Eval in 10000 steps...█████████████████████████████████████████████████████████████████████████████████████| 4/4 [03:57<00:00, 59.18s/it] +processing left-padding... +processing left-padding... +processing left-padding... +processing left-padding... +[2025-04-24 05:13:17] Saved checkpoint to checkpoint/Afteri2i_100kckpt_LLAVAT5_empty_Fixed_just_segment_pretrain_stage3_dreambench_recap_400k_t2i_400k_flux_200k_midjourney_150k_recovery_150k_3e4_epoch2_no_rotated/000-GPT-XL/checkpoints/0010000.pt +[2025-04-24 05:13:20] Removed old checkpoint: checkpoint/Afteri2i_100kckpt_LLAVAT5_empty_Fixed_just_segment_pretrain_stage3_dreambench_recap_400k_t2i_400k_flux_200k_midjourney_150k_recovery_150k_3e4_epoch2_no_rotated/000-GPT-XL/checkpoints/0008000.pt + 43%|██████████████████████████████████████████████████████▋ | 10024/23458 [3:03:05<3:20:24, 1.12it/s][2025-04-24 05:13:41] (step=0010025) Train Loss: 5.6128, Train Steps/Sec: 0.07 + 43%|██████████████████████████████████████████████████████▊ | 10049/23458 [3:03:27<3:12:22, 1.16it/s][2025-04-24 05:14:03] (step=0010050) Train Loss: 5.5361, Train Steps/Sec: 1.16 + 43%|██████████████████████████████████████████████████████▉ | 10074/23458 [3:03:48<3:11:25, 1.17it/s][2025-04-24 05:14:24] (step=0010075) Train Loss: 5.5859, Train Steps/Sec: 1.16 + 43%|███████████████████████████████████████████████████████ | 10099/23458 [3:04:10<3:07:48, 1.19it/s][2025-04-24 05:14:46] (step=0010100) Train Loss: 5.4551, Train Steps/Sec: 1.16 + 43%|███████████████████████████████████████████████████████▏ | 10124/23458 [3:04:31<3:14:08, 1.14it/s][2025-04-24 05:15:07] (step=0010125) Train Loss: 5.6075, Train Steps/Sec: 1.17 + 43%|███████████████████████████████████████████████████████▍ | 10149/23458 [3:04:52<3:10:42, 1.16it/s][2025-04-24 05:15:29] (step=0010150) Train Loss: 5.4969, Train Steps/Sec: 1.17 + 43%|███████████████████████████████████████████████████████▌ | 10174/23458 [3:05:14<3:08:10, 1.18it/s][2025-04-24 05:15:50] (step=0010175) Train Loss: 5.5945, Train Steps/Sec: 1.17 + 43%|███████████████████████████████████████████████████████▋ | 10199/23458 [3:05:35<3:07:50, 1.18it/s][2025-04-24 05:16:12] (step=0010200) Train Loss: 5.5507, Train Steps/Sec: 1.16 + 44%|███████████████████████████████████████████████████████▊ | 10224/23458 [3:05:57<3:12:12, 1.15it/s][2025-04-24 05:16:34] (step=0010225) Train Loss: 5.6699, Train Steps/Sec: 1.14 + 44%|███████████████████████████████████████████████████████▉ | 10249/23458 [3:06:19<3:08:15, 1.17it/s][2025-04-24 05:16:55] (step=0010250) Train Loss: 5.5454, Train Steps/Sec: 1.17 + 44%|████████████████████████████████████████████████████████ | 10274/23458 [3:06:40<3:06:30, 1.18it/s][2025-04-24 05:17:16] (step=0010275) Train Loss: 5.5839, Train Steps/Sec: 1.17 + 44%|████████████████████████████████████████████████████████▏ | 10299/23458 [3:07:01<3:05:01, 1.19it/s][2025-04-24 05:17:38] (step=0010300) Train Loss: 5.5582, Train Steps/Sec: 1.16 + 44%|████████████████████████████████████████████████████████▎ | 10324/23458 [3:07:23<3:12:10, 1.14it/s][2025-04-24 05:17:59] (step=0010325) Train Loss: 5.5427, Train Steps/Sec: 1.16 + 44%|████████████████████████████████████████████████████████▍ | 10349/23458 [3:07:44<3:07:52, 1.16it/s][2025-04-24 05:18:21] (step=0010350) Train Loss: 5.4767, Train Steps/Sec: 1.17 + 44%|████████████████████████████████████████████████████████▌ | 10374/23458 [3:08:06<3:06:51, 1.17it/s][2025-04-24 05:18:42] (step=0010375) Train Loss: 5.5055, Train Steps/Sec: 1.17 + 44%|████████████████████████████████████████████████████████▋ | 10399/23458 [3:08:27<3:04:23, 1.18it/s][2025-04-24 05:19:04] (step=0010400) Train Loss: 5.5088, Train Steps/Sec: 1.16 + 44%|████████████████████████████████████████████████████████▉ | 10424/23458 [3:08:49<3:08:40, 1.15it/s][2025-04-24 05:19:25] (step=0010425) Train Loss: 5.5144, Train Steps/Sec: 1.17 + 45%|█████████████████████████████████████████████████████████ | 10449/23458 [3:09:10<3:05:18, 1.17it/s][2025-04-24 05:19:46] (step=0010450) Train Loss: 5.5395, Train Steps/Sec: 1.17 + 45%|█████████████████████████████████████████████████████████▏ | 10474/23458 [3:09:33<3:44:23, 1.04s/it][2025-04-24 05:20:10] (step=0010475) Train Loss: 5.5179, Train Steps/Sec: 1.07 + 45%|█████████████████████████████████████████████████████████▎ | 10499/23458 [3:09:57<3:10:52, 1.13it/s][2025-04-24 05:20:33] (step=0010500) Train Loss: 5.5821, Train Steps/Sec: 1.07 + 45%|█████████████████████████████████████████████████████████▍ | 10524/23458 [3:10:18<3:08:18, 1.14it/s][2025-04-24 05:20:54] (step=0010525) Train Loss: 5.5002, Train Steps/Sec: 1.17 + 45%|█████████████████████████████████████████████████████████▌ | 10549/23458 [3:10:41<3:08:30, 1.14it/s][2025-04-24 05:21:17] (step=0010550) Train Loss: 5.5111, Train Steps/Sec: 1.12 + 45%|█████████████████████████████████████████████████████████▋ | 10574/23458 [3:11:02<3:02:17, 1.18it/s][2025-04-24 05:21:38] (step=0010575) Train Loss: 5.5364, Train Steps/Sec: 1.17 + 45%|█████████████████████████████████████████████████████████▊ | 10599/23458 [3:11:25<3:14:15, 1.10it/s][2025-04-24 05:22:02] (step=0010600) Train Loss: 5.5712, Train Steps/Sec: 1.06 + 45%|█████████████████████████████████████████████████████████▉ | 10624/23458 [3:11:47<3:07:43, 1.14it/s][2025-04-24 05:22:23] (step=0010625) Train Loss: 5.5174, Train Steps/Sec: 1.17 + 45%|██████████████████████████████████████████████████████████ | 10649/23458 [3:12:09<3:49:44, 1.08s/it][2025-04-24 05:22:46] (step=0010650) Train Loss: 5.5729, Train Steps/Sec: 1.12 + 46%|██████████████████████████████████████████████████████████▏ | 10674/23458 [3:12:31<3:01:04, 1.18it/s][2025-04-24 05:23:07] (step=0010675) Train Loss: 5.4921, Train Steps/Sec: 1.17 + 46%|██████████████████████████████████████████████████████████▍ | 10699/23458 [3:12:52<3:00:22, 1.18it/s][2025-04-24 05:23:28] (step=0010700) Train Loss: 5.5949, Train Steps/Sec: 1.16 + 46%|██████████████████████████████████████████████████████████▌ | 10724/23458 [3:13:14<3:05:48, 1.14it/s][2025-04-24 05:23:50] (step=0010725) Train Loss: 5.4964, Train Steps/Sec: 1.17 + 46%|██████████████████████████████████████████████████████████▋ | 10749/23458 [3:13:35<3:01:44, 1.17it/s][2025-04-24 05:24:11] (step=0010750) Train Loss: 5.6323, Train Steps/Sec: 1.17 + 46%|██████████████████████████████████████████████████████████▊ | 10774/23458 [3:13:56<2:59:43, 1.18it/s][2025-04-24 05:24:33] (step=0010775) Train Loss: 5.5652, Train Steps/Sec: 1.17 + 46%|██████████████████████████████████████████████████████████▉ | 10799/23458 [3:14:18<2:58:09, 1.18it/s][2025-04-24 05:24:54] (step=0010800) Train Loss: 5.4863, Train Steps/Sec: 1.16 + 46%|███████████████████████████████████████████████████████████ | 10824/23458 [3:14:39<3:03:59, 1.14it/s][2025-04-24 05:25:15] (step=0010825) Train Loss: 5.4864, Train Steps/Sec: 1.17 + 46%|███████████████████████████████████████████████████████████▏ | 10849/23458 [3:15:01<3:00:30, 1.16it/s][2025-04-24 05:25:37] (step=0010850) Train Loss: 5.6307, Train Steps/Sec: 1.17 + 46%|███████████████████████████████████████████████████████████▎ | 10874/23458 [3:15:22<2:59:42, 1.17it/s][2025-04-24 05:25:58] (step=0010875) Train Loss: 5.6250, Train Steps/Sec: 1.17 + 46%|███████████████████████████████████████████████████████████▍ | 10899/23458 [3:15:43<2:58:02, 1.18it/s][2025-04-24 05:26:20] (step=0010900) Train Loss: 5.6807, Train Steps/Sec: 1.16 + 47%|███████████████████████████████████████████████████████████▌ | 10924/23458 [3:16:05<3:01:43, 1.15it/s][2025-04-24 05:26:41] (step=0010925) Train Loss: 5.5064, Train Steps/Sec: 1.17 + 47%|███████████████████████████████████████████████████████████▋ | 10949/23458 [3:16:26<2:57:53, 1.17it/s][2025-04-24 05:27:02] (step=0010950) Train Loss: 5.4973, Train Steps/Sec: 1.17 + 47%|███████████████████████████████████████████████████████████▉ | 10974/23458 [3:16:48<2:57:08, 1.17it/s][2025-04-24 05:27:24] (step=0010975) Train Loss: 5.6616, Train Steps/Sec: 1.17 + 47%|████████████████████████████████████████████████████████████ | 10999/23458 [3:17:09<2:56:24, 1.18it/s][2025-04-24 05:27:45] (step=0011000) Train Loss: 5.5108, Train Steps/Sec: 1.16 + 47%|████████████████████████████████████████████████████████████▏ | 11024/23458 [3:17:31<2:59:53, 1.15it/s][2025-04-24 05:28:07] (step=0011025) Train Loss: 5.5245, Train Steps/Sec: 1.17 + 47%|████████████████████████████████████████████████████████████▎ | 11049/23458 [3:17:52<2:56:50, 1.17it/s][2025-04-24 05:28:28] (step=0011050) Train Loss: 5.5323, Train Steps/Sec: 1.17 + 47%|████████████████████████████████████████████████████████████▍ | 11074/23458 [3:18:13<2:56:11, 1.17it/s][2025-04-24 05:28:50] (step=0011075) Train Loss: 5.5379, Train Steps/Sec: 1.17 + 47%|████████████████████████████████████████████████████████████▌ | 11099/23458 [3:18:35<2:53:37, 1.19it/s][2025-04-24 05:29:11] (step=0011100) Train Loss: 5.5367, Train Steps/Sec: 1.16 + 47%|████████████████████████████████████████████████████████████▋ | 11124/23458 [3:18:57<3:00:33, 1.14it/s][2025-04-24 05:29:33] (step=0011125) Train Loss: 5.5582, Train Steps/Sec: 1.12 + 48%|████████████████████████████████████████████████████████████▊ | 11149/23458 [3:19:20<2:59:58, 1.14it/s][2025-04-24 05:29:56] (step=0011150) Train Loss: 5.5823, Train Steps/Sec: 1.08 + 48%|████████████████████████████████████████████████████████████▉ | 11174/23458 [3:19:42<2:54:45, 1.17it/s][2025-04-24 05:30:19] (step=0011175) Train Loss: 5.5513, Train Steps/Sec: 1.12 + 48%|█████████████████████████████████████████████████████████████ | 11199/23458 [3:20:04<2:52:33, 1.18it/s][2025-04-24 05:30:40] (step=0011200) Train Loss: 5.4976, Train Steps/Sec: 1.17 + 48%|█████████████████████████████████████████████████████████████▏ | 11224/23458 [3:20:25<2:58:11, 1.14it/s][2025-04-24 05:31:02] (step=0011225) Train Loss: 5.5454, Train Steps/Sec: 1.17 + 48%|█████████████████████████████████████████████████████████████▍ | 11249/23458 [3:20:48<3:48:23, 1.12s/it][2025-04-24 05:31:24] (step=0011250) Train Loss: 5.5721, Train Steps/Sec: 1.12 + 48%|█████████████████████████████████████████████████████████████▌ | 11274/23458 [3:21:10<2:54:57, 1.16it/s][2025-04-24 05:31:46] (step=0011275) Train Loss: 5.5922, Train Steps/Sec: 1.11 + 48%|█████████████████████████████████████████████████████████████▋ | 11299/23458 [3:21:31<2:51:06, 1.18it/s][2025-04-24 05:32:08] (step=0011300) Train Loss: 5.6184, Train Steps/Sec: 1.17 + 48%|█████████████████████████████████████████████████████████████▊ | 11324/23458 [3:21:55<3:19:00, 1.02it/s][2025-04-24 05:32:32] (step=0011325) Train Loss: 5.5439, Train Steps/Sec: 1.05 + 48%|█████████████████████████████████████████████████████████████▉ | 11349/23458 [3:22:17<2:52:47, 1.17it/s][2025-04-24 05:32:53] (step=0011350) Train Loss: 5.4864, Train Steps/Sec: 1.17 + 48%|██████████████████████████████████████████████████████████████ | 11374/23458 [3:22:38<2:50:49, 1.18it/s][2025-04-24 05:33:14] (step=0011375) Train Loss: 5.5239, Train Steps/Sec: 1.17 + 49%|██████████████████████████████████████████████████████████████▏ | 11399/23458 [3:22:59<2:50:38, 1.18it/s][2025-04-24 05:33:36] (step=0011400) Train Loss: 5.5655, Train Steps/Sec: 1.16 + 49%|██████████████████████████████████████████████████████████████▎ | 11424/23458 [3:23:21<2:54:49, 1.15it/s][2025-04-24 05:33:57] (step=0011425) Train Loss: 5.5337, Train Steps/Sec: 1.17 + 49%|██████████████████████████████████████████████████████████████▍ | 11449/23458 [3:23:42<2:51:35, 1.17it/s][2025-04-24 05:34:19] (step=0011450) Train Loss: 5.4405, Train Steps/Sec: 1.17 + 49%|██████████████████████████████████████████████████████████████▌ | 11474/23458 [3:24:04<2:51:07, 1.17it/s][2025-04-24 05:34:40] (step=0011475) Train Loss: 5.5468, Train Steps/Sec: 1.17 + 49%|██████████████████████████████████████████████████████████████▋ | 11499/23458 [3:24:25<2:49:27, 1.18it/s][2025-04-24 05:35:01] (step=0011500) Train Loss: 5.5810, Train Steps/Sec: 1.16 + 49%|██████████████████████████████████████████████████████████████▉ | 11524/23458 [3:24:47<2:53:02, 1.15it/s][2025-04-24 05:35:23] (step=0011525) Train Loss: 5.4924, Train Steps/Sec: 1.17 + 49%|███████████████████████████████████████████████████████████████ | 11549/23458 [3:25:08<2:49:52, 1.17it/s][2025-04-24 05:35:44] (step=0011550) Train Loss: 5.5264, Train Steps/Sec: 1.17 + 49%|███████████████████████████████████████████████████████████████▏ | 11574/23458 [3:25:29<2:48:52, 1.17it/s][2025-04-24 05:36:06] (step=0011575) Train Loss: 5.5708, Train Steps/Sec: 1.17 + 49%|███████████████████████████████████████████████████████████████▎ | 11599/23458 [3:25:51<2:46:41, 1.19it/s][2025-04-24 05:36:27] (step=0011600) Train Loss: 5.5370, Train Steps/Sec: 1.17 + 50%|███████████████████████████████████████████████████████████████▍ | 11624/23458 [3:26:12<2:51:48, 1.15it/s][2025-04-24 05:36:48] (step=0011625) Train Loss: 5.5699, Train Steps/Sec: 1.17 + 50%|███████████████████████████████████████████████████████████████▌ | 11649/23458 [3:26:34<2:48:16, 1.17it/s][2025-04-24 05:37:10] (step=0011650) Train Loss: 5.5066, Train Steps/Sec: 1.17 + 50%|███████████████████████████████████████████████████████████████▋ | 11674/23458 [3:26:55<2:47:09, 1.17it/s][2025-04-24 05:37:31] (step=0011675) Train Loss: 5.5464, Train Steps/Sec: 1.17 + 50%|███████████████████████████████████████████████████████████████▊ | 11699/23458 [3:27:16<2:46:02, 1.18it/s][2025-04-24 05:37:53] (step=0011700) Train Loss: 5.4979, Train Steps/Sec: 1.16 + 50%|███████████████████████████████████████████████████████████████▉ | 11724/23458 [3:27:38<2:50:10, 1.15it/s][2025-04-24 05:38:14] (step=0011725) Train Loss: 5.5132, Train Steps/Sec: 1.17 + 50%|████████████████████████████████████████████████████████████████ | 11749/23458 [3:27:59<2:47:46, 1.16it/s][2025-04-24 05:38:35] (step=0011750) Train Loss: 5.5854, Train Steps/Sec: 1.17 + 50%|████████████████████████████████████████████████████████████████▏ | 11774/23458 [3:28:20<2:45:30, 1.18it/s][2025-04-24 05:38:58] (step=0011775) Train Loss: 5.6056, Train Steps/Sec: 1.12 + 50%|████████████████████████████████████████████████████████████████▍ | 11799/23458 [3:28:44<2:48:57, 1.15it/s][2025-04-24 05:39:20] (step=0011800) Train Loss: 5.5701, Train Steps/Sec: 1.12 + 50%|████████████████████████████████████████████████████████████████▌ | 11824/23458 [3:29:07<2:50:09, 1.14it/s][2025-04-24 05:39:43] (step=0011825) Train Loss: 5.5820, Train Steps/Sec: 1.08 + 51%|████████████████████████████████████████████████████████████████▋ | 11849/23458 [3:29:28<2:45:41, 1.17it/s][2025-04-24 05:40:04] (step=0011850) Train Loss: 5.5347, Train Steps/Sec: 1.17 + 51%|████████████████████████████████████████████████████████████████▊ | 11874/23458 [3:29:50<2:44:15, 1.18it/s][2025-04-24 05:40:26] (step=0011875) Train Loss: 5.5597, Train Steps/Sec: 1.17 + 51%|████████████████████████████████████████████████████████████████▉ | 11899/23458 [3:30:11<2:42:16, 1.19it/s][2025-04-24 05:40:47] (step=0011900) Train Loss: 5.6397, Train Steps/Sec: 1.17 + 51%|█████████████████████████████████████████████████████████████████ | 11924/23458 [3:30:33<2:47:31, 1.15it/s][2025-04-24 05:41:09] (step=0011925) Train Loss: 5.4462, Train Steps/Sec: 1.12 + 51%|█████████████████████████████████████████████████████████████████▏ | 11949/23458 [3:30:55<2:44:06, 1.17it/s][2025-04-24 05:41:31] (step=0011950) Train Loss: 5.5621, Train Steps/Sec: 1.17 + 51%|█████████████████████████████████████████████████████████████████▎ | 11974/23458 [3:31:17<2:45:13, 1.16it/s][2025-04-24 05:41:53] (step=0011975) Train Loss: 5.5928, Train Steps/Sec: 1.12 + 51%|█████████████████████████████████████████████████████████████████▍ | 11999/23458 [3:31:39<2:42:29, 1.18it/s][2025-04-24 05:42:16] (step=0012000) Train Loss: 5.5831, Train Steps/Sec: 1.11 +You are using a model of type blip-2 to instantiate a model of type instructblip. This is not supported for all configurations of models and can yield errors. +[2025-04-24 05:42:16] text_config is None. Initializing the text config with default values (`OPTConfig`). +model_name_or_path:/tmp/haozhezhao/model/blip2-flan-t5-xl +image_place_holder: +tokenizer length after expend 32101 +tokenizer length before expend 32100 +100%|████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████| 4/4 [04:05<00:00, 61.42s/it] +[2025-04-24 05:47:29] Finish Eval in 12000 steps...█████████████████████████████████████████████████████████████████████████████████████| 4/4 [04:05<00:00, 61.08s/it] +processing left-padding... +processing left-padding... +processing left-padding... +processing left-padding... +[2025-04-24 05:47:48] Saved checkpoint to checkpoint/Afteri2i_100kckpt_LLAVAT5_empty_Fixed_just_segment_pretrain_stage3_dreambench_recap_400k_t2i_400k_flux_200k_midjourney_150k_recovery_150k_3e4_epoch2_no_rotated/000-GPT-XL/checkpoints/0012000.pt +[2025-04-24 05:47:50] Removed old checkpoint: checkpoint/Afteri2i_100kckpt_LLAVAT5_empty_Fixed_just_segment_pretrain_stage3_dreambench_recap_400k_t2i_400k_flux_200k_midjourney_150k_recovery_150k_3e4_epoch2_no_rotated/000-GPT-XL/checkpoints/0010000.pt + 51%|█████████████████████████████████████████████████████████████████▌ | 12024/23458 [3:37:35<2:48:56, 1.13it/s][2025-04-24 05:48:11] (step=0012025) Train Loss: 5.5987, Train Steps/Sec: 0.07 + 51%|█████████████████████████████████████████████████████████████████▋ | 12049/23458 [3:37:57<3:25:33, 1.08s/it][2025-04-24 05:48:34] (step=0012050) Train Loss: 5.5446, Train Steps/Sec: 1.11 + 51%|█████████████████████████████████████████████████████████████████▉ | 12074/23458 [3:38:19<2:41:33, 1.17it/s][2025-04-24 05:48:55] (step=0012075) Train Loss: 5.5930, Train Steps/Sec: 1.17 + 52%|██████████████████████████████████████████████████████████████████ | 12099/23458 [3:38:40<2:40:09, 1.18it/s][2025-04-24 05:49:16] (step=0012100) Train Loss: 5.5341, Train Steps/Sec: 1.16 + 52%|██████████████████████████████████████████████████████████████████▏ | 12124/23458 [3:39:02<2:44:36, 1.15it/s][2025-04-24 05:49:38] (step=0012125) Train Loss: 5.5410, Train Steps/Sec: 1.17 + 52%|██████████████████████████████████████████████████████████████████▎ | 12149/23458 [3:39:23<2:40:51, 1.17it/s][2025-04-24 05:49:59] (step=0012150) Train Loss: 5.5226, Train Steps/Sec: 1.16 + 52%|██████████████████████████████████████████████████████████████████▍ | 12174/23458 [3:39:44<2:39:59, 1.18it/s][2025-04-24 05:50:21] (step=0012175) Train Loss: 5.5763, Train Steps/Sec: 1.17 + 52%|██████████████████████████████████████████████████████████████████▌ | 12199/23458 [3:40:06<2:38:52, 1.18it/s][2025-04-24 05:50:42] (step=0012200) Train Loss: 5.5687, Train Steps/Sec: 1.16 + 52%|██████████████████████████████████████████████████████████████████▋ | 12224/23458 [3:40:27<2:42:15, 1.15it/s][2025-04-24 05:51:04] (step=0012225) Train Loss: 5.6387, Train Steps/Sec: 1.17 + 52%|██████████████████████████████████████████████████████████████████▊ | 12249/23458 [3:40:49<2:39:25, 1.17it/s][2025-04-24 05:51:25] (step=0012250) Train Loss: 5.5751, Train Steps/Sec: 1.17 + 52%|██████████████████████████████████████████████████████████████████▉ | 12274/23458 [3:41:10<2:37:59, 1.18it/s][2025-04-24 05:51:46] (step=0012275) Train Loss: 5.4511, Train Steps/Sec: 1.17 + 52%|███████████████████████████████████████████████████████████████████ | 12299/23458 [3:41:31<2:36:59, 1.18it/s][2025-04-24 05:52:08] (step=0012300) Train Loss: 5.4989, Train Steps/Sec: 1.17 + 53%|███████████████████████████████████████████████████████████████████▏ | 12324/23458 [3:41:53<2:40:49, 1.15it/s][2025-04-24 05:52:29] (step=0012325) Train Loss: 5.5317, Train Steps/Sec: 1.17 + 53%|███████████████████████████████████████████████████████████████████▍ | 12349/23458 [3:42:14<2:38:07, 1.17it/s][2025-04-24 05:52:50] (step=0012350) Train Loss: 5.5611, Train Steps/Sec: 1.17 + 53%|███████████████████████████████████████████████████████████████████▌ | 12374/23458 [3:42:35<2:37:26, 1.17it/s][2025-04-24 05:53:12] (step=0012375) Train Loss: 5.6331, Train Steps/Sec: 1.17 + 53%|███████████████████████████████████████████████████████████████████▋ | 12399/23458 [3:42:57<2:35:34, 1.18it/s][2025-04-24 05:53:33] (step=0012400) Train Loss: 5.5454, Train Steps/Sec: 1.17 + 53%|███████████████████████████████████████████████████████████████████▊ | 12424/23458 [3:43:18<2:39:55, 1.15it/s][2025-04-24 05:53:54] (step=0012425) Train Loss: 5.5489, Train Steps/Sec: 1.17 + 53%|███████████████████████████████████████████████████████████████████▉ | 12449/23458 [3:43:40<2:37:03, 1.17it/s][2025-04-24 05:54:17] (step=0012450) Train Loss: 5.5369, Train Steps/Sec: 1.12 + 53%|████████████████████████████████████████████████████████████████████ | 12474/23458 [3:44:04<3:31:25, 1.15s/it][2025-04-24 05:54:41] (step=0012475) Train Loss: 5.5585, Train Steps/Sec: 1.04 + 53%|████████████████████████████████████████████████████████████████████▏ | 12499/23458 [3:44:26<2:34:15, 1.18it/s][2025-04-24 05:55:02] (step=0012500) Train Loss: 5.5377, Train Steps/Sec: 1.16 + 53%|████████████████████████████████████████████████████████████████████▎ | 12524/23458 [3:44:47<2:39:23, 1.14it/s][2025-04-24 05:55:24] (step=0012525) Train Loss: 5.5340, Train Steps/Sec: 1.17 + 53%|████████████████████████████████████████████████████████████████████▍ | 12549/23458 [3:45:09<2:35:51, 1.17it/s][2025-04-24 05:55:45] (step=0012550) Train Loss: 5.4382, Train Steps/Sec: 1.17 + 54%|████████████████████████████████████████████████████████████████████▌ | 12574/23458 [3:45:31<2:33:37, 1.18it/s][2025-04-24 05:56:07] (step=0012575) Train Loss: 5.5709, Train Steps/Sec: 1.13 + 54%|████████████████████████████████████████████████████████████████████▋ | 12599/23458 [3:45:52<2:32:28, 1.19it/s][2025-04-24 05:56:28] (step=0012600) Train Loss: 5.5039, Train Steps/Sec: 1.17 + 54%|████████████████████████████████████████████████████████████████████▉ | 12624/23458 [3:46:14<2:36:55, 1.15it/s][2025-04-24 05:56:50] (step=0012625) Train Loss: 5.5009, Train Steps/Sec: 1.17 + 54%|█████████████████████████████████████████████████████████████████████ | 12649/23458 [3:46:35<2:34:14, 1.17it/s][2025-04-24 05:57:11] (step=0012650) Train Loss: 5.5406, Train Steps/Sec: 1.17 + 54%|█████████████████████████████████████████████████████████████████████▏ | 12674/23458 [3:46:57<2:32:27, 1.18it/s][2025-04-24 05:57:34] (step=0012675) Train Loss: 5.4848, Train Steps/Sec: 1.12 + 54%|█████████████████████████████████████████████████████████████████████▎ | 12699/23458 [3:47:19<2:31:21, 1.18it/s][2025-04-24 05:57:55] (step=0012700) Train Loss: 5.5173, Train Steps/Sec: 1.17 + 54%|█████████████████████████████████████████████████████████████████████▍ | 12724/23458 [3:47:41<2:36:00, 1.15it/s][2025-04-24 05:58:17] (step=0012725) Train Loss: 5.4820, Train Steps/Sec: 1.12 + 54%|█████████████████████████████████████████████████████████████████████▌ | 12749/23458 [3:48:02<2:33:27, 1.16it/s][2025-04-24 05:58:39] (step=0012750) Train Loss: 5.5819, Train Steps/Sec: 1.17 + 54%|█████████████████████████████████████████████████████████████████████▋ | 12774/23458 [3:48:24<2:31:03, 1.18it/s][2025-04-24 05:59:01] (step=0012775) Train Loss: 5.5643, Train Steps/Sec: 1.11 + 55%|█████████████████████████████████████████████████████████████████████▊ | 12799/23458 [3:48:46<2:31:42, 1.17it/s][2025-04-24 05:59:23] (step=0012800) Train Loss: 5.5847, Train Steps/Sec: 1.16 + 55%|█████████████████████████████████████████████████████████████████████▉ | 12824/23458 [3:49:08<2:33:35, 1.15it/s][2025-04-24 05:59:44] (step=0012825) Train Loss: 5.5839, Train Steps/Sec: 1.17 + 55%|██████████████████████████████████████████████████████████████████████ | 12849/23458 [3:49:29<2:31:43, 1.17it/s][2025-04-24 06:00:05] (step=0012850) Train Loss: 5.5342, Train Steps/Sec: 1.17 + 55%|██████████████████████████████████████████████████████████████████████▏ | 12874/23458 [3:49:50<2:29:25, 1.18it/s][2025-04-24 06:00:27] (step=0012875) Train Loss: 5.5779, Train Steps/Sec: 1.17 + 55%|██████████████████████████████████████████████████████████████████████▍ | 12899/23458 [3:50:12<2:29:01, 1.18it/s][2025-04-24 06:00:48] (step=0012900) Train Loss: 5.5836, Train Steps/Sec: 1.16 + 55%|██████████████████████████████████████████████████████████████████████▌ | 12924/23458 [3:50:33<2:33:07, 1.15it/s][2025-04-24 06:01:10] (step=0012925) Train Loss: 5.6191, Train Steps/Sec: 1.17 + 55%|██████████████████████████████████████████████████████████████████████▋ | 12949/23458 [3:50:55<2:29:28, 1.17it/s][2025-04-24 06:01:31] (step=0012950) Train Loss: 5.4604, Train Steps/Sec: 1.17 + 55%|██████████████████████████████████████████████████████████████████████▊ | 12974/23458 [3:51:16<2:28:15, 1.18it/s][2025-04-24 06:01:52] (step=0012975) Train Loss: 5.4950, Train Steps/Sec: 1.17 + 55%|██████████████████████████████████████████████████████████████████████▉ | 12999/23458 [3:51:37<2:27:28, 1.18it/s][2025-04-24 06:02:14] (step=0013000) Train Loss: 5.4169, Train Steps/Sec: 1.17 + 56%|███████████████████████████████████████████████████████████████████████ | 13024/23458 [3:51:59<2:30:48, 1.15it/s][2025-04-24 06:02:35] (step=0013025) Train Loss: 5.5978, Train Steps/Sec: 1.17 + 56%|███████████████████████████████████████████████████████████████████████▏ | 13049/23458 [3:52:20<2:29:18, 1.16it/s][2025-04-24 06:02:56] (step=0013050) Train Loss: 5.4675, Train Steps/Sec: 1.17 + 56%|███████████████████████████████████████████████████████████████████████▎ | 13074/23458 [3:52:42<2:26:55, 1.18it/s][2025-04-24 06:03:18] (step=0013075) Train Loss: 5.5210, Train Steps/Sec: 1.17 + 56%|███████████████████████████████████████████████████████████████████████▍ | 13099/23458 [3:53:04<2:29:26, 1.16it/s][2025-04-24 06:03:40] (step=0013100) Train Loss: 5.5366, Train Steps/Sec: 1.12 + 56%|███████████████████████████████████████████████████████████████████████▌ | 13124/23458 [3:53:27<3:14:04, 1.13s/it][2025-04-24 06:04:03] (step=0013125) Train Loss: 5.4783, Train Steps/Sec: 1.09 + 56%|███████████████████████████████████████████████████████████████████████▋ | 13149/23458 [3:53:49<2:26:55, 1.17it/s][2025-04-24 06:04:25] (step=0013150) Train Loss: 5.4682, Train Steps/Sec: 1.12 + 56%|███████████████████████████████████████████████████████████████████████▉ | 13174/23458 [3:54:11<2:25:43, 1.18it/s][2025-04-24 06:04:47] (step=0013175) Train Loss: 5.5113, Train Steps/Sec: 1.17 + 56%|████████████████████████████████████████████████████████████████████████ | 13199/23458 [3:54:32<2:24:07, 1.19it/s][2025-04-24 06:05:08] (step=0013200) Train Loss: 5.5449, Train Steps/Sec: 1.17 + 56%|████████████████████████████████████████████████████████████████████████▏ | 13224/23458 [3:54:54<2:33:11, 1.11it/s][2025-04-24 06:05:30] (step=0013225) Train Loss: 5.5020, Train Steps/Sec: 1.13 + 56%|████████████████████████████████████████████████████████████████████████▎ | 13249/23458 [3:55:16<2:25:05, 1.17it/s][2025-04-24 06:05:52] (step=0013250) Train Loss: 5.5087, Train Steps/Sec: 1.17 + 57%|████████████████████████████████████████████████████████████████████████▍ | 13274/23458 [3:55:37<2:23:31, 1.18it/s][2025-04-24 06:06:13] (step=0013275) Train Loss: 5.5594, Train Steps/Sec: 1.17 + 57%|████████████████████████████████████████████████████████████████████████▌ | 13299/23458 [3:55:58<2:22:32, 1.19it/s][2025-04-24 06:06:34] (step=0013300) Train Loss: 5.5388, Train Steps/Sec: 1.17 + 57%|████████████████████████████████████████████████████████████████████████▋ | 13324/23458 [3:56:21<2:26:33, 1.15it/s][2025-04-24 06:06:57] (step=0013325) Train Loss: 5.6025, Train Steps/Sec: 1.12 + 57%|████████████████████████████████████████████████████████████████████████▊ | 13349/23458 [3:56:42<2:24:17, 1.17it/s][2025-04-24 06:07:18] (step=0013350) Train Loss: 5.5242, Train Steps/Sec: 1.17 + 57%|████████████████████████████████████████████████████████████████████████▉ | 13374/23458 [3:57:03<2:22:35, 1.18it/s][2025-04-24 06:07:40] (step=0013375) Train Loss: 5.6302, Train Steps/Sec: 1.17 + 57%|█████████████████████████████████████████████████████████████████████████ | 13399/23458 [3:57:25<2:22:17, 1.18it/s][2025-04-24 06:08:01] (step=0013400) Train Loss: 5.5842, Train Steps/Sec: 1.17 + 57%|█████████████████████████████████████████████████████████████████████████▏ | 13424/23458 [3:57:46<2:25:29, 1.15it/s][2025-04-24 06:08:22] (step=0013425) Train Loss: 5.5440, Train Steps/Sec: 1.17 + 57%|█████████████████████████████████████████████████████████████████████████▍ | 13449/23458 [3:58:08<2:23:02, 1.17it/s][2025-04-24 06:08:45] (step=0013450) Train Loss: 5.4954, Train Steps/Sec: 1.12 + 57%|█████████████████████████████████████████████████████████████████████████▌ | 13474/23458 [3:58:30<2:20:44, 1.18it/s][2025-04-24 06:09:06] (step=0013475) Train Loss: 5.4986, Train Steps/Sec: 1.17 + 58%|█████████████████████████████████████████████████████████████████████████▋ | 13499/23458 [3:58:51<2:19:56, 1.19it/s][2025-04-24 06:09:27] (step=0013500) Train Loss: 5.5067, Train Steps/Sec: 1.17 + 58%|█████████████████████████████████████████████████████████████████████████▊ | 13524/23458 [3:59:14<2:23:47, 1.15it/s][2025-04-24 06:09:50] (step=0013525) Train Loss: 5.4794, Train Steps/Sec: 1.12 + 58%|█████████████████████████████████████████████████████████████████████████▉ | 13549/23458 [3:59:35<2:20:34, 1.17it/s][2025-04-24 06:10:11] (step=0013550) Train Loss: 5.5198, Train Steps/Sec: 1.18 + 58%|██████████████████████████████████████████████████████████████████████████ | 13574/23458 [3:59:56<2:19:56, 1.18it/s][2025-04-24 06:10:32] (step=0013575) Train Loss: 5.4643, Train Steps/Sec: 1.17 + 58%|██████████████████████████████████████████████████████████████████████████▏ | 13599/23458 [4:00:17<2:18:17, 1.19it/s][2025-04-24 06:10:54] (step=0013600) Train Loss: 5.5655, Train Steps/Sec: 1.17 + 58%|██████████████████████████████████████████████████████████████████████████▎ | 13624/23458 [4:00:39<2:23:02, 1.15it/s][2025-04-24 06:11:15] (step=0013625) Train Loss: 5.5171, Train Steps/Sec: 1.17 + 58%|██████████████████████████████████████████████████████████████████████████▍ | 13649/23458 [4:01:00<2:19:37, 1.17it/s][2025-04-24 06:11:37] (step=0013650) Train Loss: 5.5217, Train Steps/Sec: 1.17 + 58%|██████████████████████████████████████████████████████████████████████████▌ | 13674/23458 [4:01:22<2:18:35, 1.18it/s][2025-04-24 06:11:58] (step=0013675) Train Loss: 5.4415, Train Steps/Sec: 1.17 + 58%|██████████████████████████████████████████████████████████████████████████▋ | 13699/23458 [4:01:43<2:17:05, 1.19it/s][2025-04-24 06:12:19] (step=0013700) Train Loss: 5.5033, Train Steps/Sec: 1.17 + 59%|██████████████████████████████████████████████████████████████████████████▉ | 13724/23458 [4:02:04<2:20:51, 1.15it/s][2025-04-24 06:12:41] (step=0013725) Train Loss: 5.5418, Train Steps/Sec: 1.17 + 59%|███████████████████████████████████████████████████████████████████████████ | 13749/23458 [4:02:27<2:51:38, 1.06s/it][2025-04-24 06:13:03] (step=0013750) Train Loss: 5.4834, Train Steps/Sec: 1.12 + 59%|███████████████████████████████████████████████████████████████████████████▏ | 13774/23458 [4:02:49<2:24:31, 1.12it/s][2025-04-24 06:13:25] (step=0013775) Train Loss: 5.5646, Train Steps/Sec: 1.13 + 59%|███████████████████████████████████████████████████████████████████████████▎ | 13799/23458 [4:03:12<2:26:35, 1.10it/s][2025-04-24 06:13:48] (step=0013800) Train Loss: 5.6057, Train Steps/Sec: 1.08 + 59%|███████████████████████████████████████████████████████████████████████████▍ | 13824/23458 [4:03:33<2:19:26, 1.15it/s][2025-04-24 06:14:10] (step=0013825) Train Loss: 5.4401, Train Steps/Sec: 1.17 + 59%|███████████████████████████████████████████████████████████████████████████▌ | 13849/23458 [4:03:55<2:16:52, 1.17it/s][2025-04-24 06:14:31] (step=0013850) Train Loss: 5.5813, Train Steps/Sec: 1.17 + 59%|███████████████████████████████████████████████████████████████████████████▋ | 13874/23458 [4:04:16<2:15:25, 1.18it/s][2025-04-24 06:14:52] (step=0013875) Train Loss: 5.5548, Train Steps/Sec: 1.17 + 59%|███████████████████████████████████████████████████████████████████████████▊ | 13899/23458 [4:04:38<2:14:26, 1.19it/s][2025-04-24 06:15:15] (step=0013900) Train Loss: 5.5498, Train Steps/Sec: 1.12 + 59%|███████████████████████████████████████████████████████████████████████████▉ | 13924/23458 [4:05:00<2:17:37, 1.15it/s][2025-04-24 06:15:36] (step=0013925) Train Loss: 5.4976, Train Steps/Sec: 1.17 + 59%|████████████████████████████████████████████████████████████████████████████ | 13949/23458 [4:05:21<2:15:32, 1.17it/s][2025-04-24 06:15:57] (step=0013950) Train Loss: 5.5556, Train Steps/Sec: 1.17 + 60%|████████████████████████████████████████████████████████████████████████████▏ | 13974/23458 [4:05:43<2:15:44, 1.16it/s][2025-04-24 06:16:20] (step=0013975) Train Loss: 5.5339, Train Steps/Sec: 1.12 + 60%|████████████████████████████████████████████████████████████████████████████▍ | 13999/23458 [4:06:05<2:13:07, 1.18it/s][2025-04-24 06:16:41] (step=0014000) Train Loss: 5.4978, Train Steps/Sec: 1.16 +You are using a model of type blip-2 to instantiate a model of type instructblip. This is not supported for all configurations of models and can yield errors. +[2025-04-24 06:16:41] text_config is None. Initializing the text config with default values (`OPTConfig`). +model_name_or_path:/tmp/haozhezhao/model/blip2-flan-t5-xl +image_place_holder: +tokenizer length after expend 32101 +tokenizer length before expend 32100 +100%|████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████| 4/4 [04:04<00:00, 61.12s/it] +[2025-04-24 06:21:53] Finish Eval in 14000 steps...█████████████████████████████████████████████████████████████████████████████████████| 4/4 [04:04<00:00, 60.76s/it] +processing left-padding... +processing left-padding... +processing left-padding... +processing left-padding... +[2025-04-24 06:22:13] Saved checkpoint to checkpoint/Afteri2i_100kckpt_LLAVAT5_empty_Fixed_just_segment_pretrain_stage3_dreambench_recap_400k_t2i_400k_flux_200k_midjourney_150k_recovery_150k_3e4_epoch2_no_rotated/000-GPT-XL/checkpoints/0014000.pt +[2025-04-24 06:22:15] Removed old checkpoint: checkpoint/Afteri2i_100kckpt_LLAVAT5_empty_Fixed_just_segment_pretrain_stage3_dreambench_recap_400k_t2i_400k_flux_200k_midjourney_150k_recovery_150k_3e4_epoch2_no_rotated/000-GPT-XL/checkpoints/0012000.pt + 60%|████████████████████████████████████████████████████████████████████████████▌ | 14024/23458 [4:12:00<2:19:29, 1.13it/s][2025-04-24 06:22:36] (step=0014025) Train Loss: 5.4919, Train Steps/Sec: 0.07 + 60%|████████████████████████████████████████████████████████████████████████████▋ | 14049/23458 [4:12:21<2:14:10, 1.17it/s][2025-04-24 06:22:57] (step=0014050) Train Loss: 5.5194, Train Steps/Sec: 1.17 + 60%|████████████████████████████████████████████████████████████████████████████▊ | 14074/23458 [4:12:43<2:12:36, 1.18it/s][2025-04-24 06:23:19] (step=0014075) Train Loss: 5.5446, Train Steps/Sec: 1.16 + 60%|████████████████████████████████████████████████████████████████████████████▉ | 14099/23458 [4:13:04<2:11:33, 1.19it/s][2025-04-24 06:23:41] (step=0014100) Train Loss: 5.5655, Train Steps/Sec: 1.17 + 60%|█████████████████████████████████████████████████████████████████████████████ | 14124/23458 [4:13:26<2:15:04, 1.15it/s][2025-04-24 06:24:02] (step=0014125) Train Loss: 5.5254, Train Steps/Sec: 1.17 + 60%|█████████████████████████████████████████████████████████████████████████████▏ | 14149/23458 [4:13:47<2:12:32, 1.17it/s][2025-04-24 06:24:23] (step=0014150) Train Loss: 5.5130, Train Steps/Sec: 1.17 + 60%|█████████████████████████████████████████████████████████████████████████████▎ | 14174/23458 [4:14:09<2:10:50, 1.18it/s][2025-04-24 06:24:46] (step=0014175) Train Loss: 5.5385, Train Steps/Sec: 1.12 + 61%|█████████████████████████████████████████████████████████████████████████████▍ | 14199/23458 [4:14:31<2:10:21, 1.18it/s][2025-04-24 06:25:07] (step=0014200) Train Loss: 5.4828, Train Steps/Sec: 1.17 + 61%|█████████████████████████████████████████████████████████████████████████████▌ | 14224/23458 [4:14:52<2:14:08, 1.15it/s][2025-04-24 06:25:28] (step=0014225) Train Loss: 5.5338, Train Steps/Sec: 1.17 + 61%|█████████████████████████████████████████████████████████████████████████████▊ | 14249/23458 [4:15:15<2:11:45, 1.16it/s][2025-04-24 06:25:51] (step=0014250) Train Loss: 5.5779, Train Steps/Sec: 1.11 + 61%|█████████████████████████████████████████████████████████████████████████████▉ | 14274/23458 [4:15:36<2:09:52, 1.18it/s][2025-04-24 06:26:12] (step=0014275) Train Loss: 5.4490, Train Steps/Sec: 1.17 + 61%|██████████████████████████████████████████████████████████████████████████████ | 14299/23458 [4:15:57<2:08:43, 1.19it/s][2025-04-24 06:26:34] (step=0014300) Train Loss: 5.4498, Train Steps/Sec: 1.17 + 61%|██████████████████████████████████████████████████████████████████████████████▏ | 14324/23458 [4:16:19<2:12:41, 1.15it/s][2025-04-24 06:26:55] (step=0014325) Train Loss: 5.4488, Train Steps/Sec: 1.17 + 61%|██████████████████████████████████████████████████████████████████████████████▎ | 14349/23458 [4:16:40<2:10:21, 1.16it/s][2025-04-24 06:27:16] (step=0014350) Train Loss: 5.3821, Train Steps/Sec: 1.17 + 61%|██████████████████████████████████████████████████████████████████████████████▍ | 14374/23458 [4:17:01<2:08:16, 1.18it/s][2025-04-24 06:27:38] (step=0014375) Train Loss: 5.4413, Train Steps/Sec: 1.17 + 61%|██████████████████████████████████████████████████████████████████████████████▌ | 14399/23458 [4:17:23<2:07:25, 1.18it/s][2025-04-24 06:27:59] (step=0014400) Train Loss: 5.5049, Train Steps/Sec: 1.17 + 61%|██████████████████████████████████████████████████████████████████████████████▋ | 14424/23458 [4:17:45<2:11:03, 1.15it/s][2025-04-24 06:28:21] (step=0014425) Train Loss: 5.5118, Train Steps/Sec: 1.12 + 62%|██████████████████████████████████████████████████████████████████████████████▊ | 14449/23458 [4:18:08<2:14:58, 1.11it/s][2025-04-24 06:28:44] (step=0014450) Train Loss: 5.4822, Train Steps/Sec: 1.08 + 62%|██████████████████████████████████████████████████████████████████████████████▉ | 14474/23458 [4:18:31<2:06:54, 1.18it/s][2025-04-24 06:29:07] (step=0014475) Train Loss: 5.5181, Train Steps/Sec: 1.12 + 62%|███████████████████████████████████████████████████████████████████████████████ | 14499/23458 [4:18:52<2:05:57, 1.19it/s][2025-04-24 06:29:28] (step=0014500) Train Loss: 5.5869, Train Steps/Sec: 1.17 + 62%|███████████████████████████████████████████████████████████████████████████████▎ | 14524/23458 [4:19:13<2:09:22, 1.15it/s][2025-04-24 06:29:49] (step=0014525) Train Loss: 5.5846, Train Steps/Sec: 1.17 + 62%|███████████████████████████████████████████████████████████████████████████████▍ | 14549/23458 [4:19:35<2:08:08, 1.16it/s][2025-04-24 06:30:12] (step=0014550) Train Loss: 5.4971, Train Steps/Sec: 1.13 + 62%|███████████████████████████████████████████████████████████████████████████████▌ | 14574/23458 [4:19:57<2:05:52, 1.18it/s][2025-04-24 06:30:33] (step=0014575) Train Loss: 5.4679, Train Steps/Sec: 1.17 + 62%|███████████████████████████████████████████████████████████████████████████████▋ | 14599/23458 [4:20:18<2:04:28, 1.19it/s][2025-04-24 06:30:55] (step=0014600) Train Loss: 5.4909, Train Steps/Sec: 1.17 + 62%|███████████████████████████████████████████████████████████████████████████████▊ | 14624/23458 [4:20:40<2:07:52, 1.15it/s][2025-04-24 06:31:16] (step=0014625) Train Loss: 5.4876, Train Steps/Sec: 1.17 + 62%|███████████████████████████████████████████████████████████████████████████████▉ | 14649/23458 [4:21:02<2:35:59, 1.06s/it][2025-04-24 06:31:38] (step=0014650) Train Loss: 5.4965, Train Steps/Sec: 1.12 + 63%|████████████████████████████████████████████████████████████████████████████████ | 14674/23458 [4:21:23<2:04:17, 1.18it/s][2025-04-24 06:32:00] (step=0014675) Train Loss: 5.5523, Train Steps/Sec: 1.17 + 63%|████████████████████████████████████████████████████████████████████████████████▏ | 14699/23458 [4:21:45<2:03:17, 1.18it/s][2025-04-24 06:32:21] (step=0014700) Train Loss: 5.4943, Train Steps/Sec: 1.17 + 63%|████████████████████████████████████████████████████████████████████████████████▎ | 14724/23458 [4:22:06<2:06:56, 1.15it/s][2025-04-24 06:32:42] (step=0014725) Train Loss: 5.5455, Train Steps/Sec: 1.17 + 63%|████████████████████████████████████████████████████████████████████████████████▍ | 14749/23458 [4:22:27<2:04:36, 1.16it/s][2025-04-24 06:33:04] (step=0014750) Train Loss: 5.5248, Train Steps/Sec: 1.17 + 63%|████████████████████████████████████████████████████████████████████████████████▌ | 14774/23458 [4:22:49<2:02:28, 1.18it/s][2025-04-24 06:33:25] (step=0014775) Train Loss: 5.4871, Train Steps/Sec: 1.17 + 63%|████████████████████████████████████████████████████████████████████████████████▊ | 14799/23458 [4:23:10<2:01:49, 1.18it/s][2025-04-24 06:33:46] (step=0014800) Train Loss: 5.5036, Train Steps/Sec: 1.17 + 63%|████████████████████████████████████████████████████████████████████████████████▉ | 14824/23458 [4:23:32<2:04:51, 1.15it/s][2025-04-24 06:34:08] (step=0014825) Train Loss: 5.5412, Train Steps/Sec: 1.17 + 63%|█████████████████████████████████████████████████████████████████████████████████ | 14849/23458 [4:23:53<2:02:50, 1.17it/s][2025-04-24 06:34:29] (step=0014850) Train Loss: 5.5355, Train Steps/Sec: 1.17 + 63%|█████████████████████████████████████████████████████████████████████████████████▏ | 14874/23458 [4:24:14<2:01:12, 1.18it/s][2025-04-24 06:34:50] (step=0014875) Train Loss: 5.5076, Train Steps/Sec: 1.17 + 64%|█████████████████████████████████████████████████████████████████████████████████▎ | 14899/23458 [4:24:37<1:59:58, 1.19it/s][2025-04-24 06:35:13] (step=0014900) Train Loss: 5.5077, Train Steps/Sec: 1.11 + 64%|█████████████████████████████████████████████████████████████████████████████████▍ | 14924/23458 [4:24:58<2:03:26, 1.15it/s][2025-04-24 06:35:34] (step=0014925) Train Loss: 5.5678, Train Steps/Sec: 1.17 + 64%|█████████████████████████████████████████████████████████████████████████████████▌ | 14949/23458 [4:25:19<2:01:37, 1.17it/s][2025-04-24 06:35:56] (step=0014950) Train Loss: 5.5746, Train Steps/Sec: 1.17 + 64%|█████████████████████████████████████████████████████████████████████████████████▋ | 14974/23458 [4:25:42<2:00:14, 1.18it/s][2025-04-24 06:36:18] (step=0014975) Train Loss: 5.5748, Train Steps/Sec: 1.11 + 64%|█████████████████████████████████████████████████████████████████████████████████▊ | 14999/23458 [4:26:03<1:58:53, 1.19it/s][2025-04-24 06:36:40] (step=0015000) Train Loss: 5.4713, Train Steps/Sec: 1.17 + 64%|█████████████████████████████████████████████████████████████████████████████████▉ | 15024/23458 [4:26:25<2:02:16, 1.15it/s][2025-04-24 06:37:01] (step=0015025) Train Loss: 5.5223, Train Steps/Sec: 1.17 + 64%|██████████████████████████████████████████████████████████████████████████████████ | 15049/23458 [4:26:46<2:00:07, 1.17it/s][2025-04-24 06:37:22] (step=0015050) Train Loss: 5.6078, Train Steps/Sec: 1.17 + 64%|██████████████████████████████████████████████████████████████████████████████████▎ | 15074/23458 [4:27:08<2:00:24, 1.16it/s][2025-04-24 06:37:45] (step=0015075) Train Loss: 5.5112, Train Steps/Sec: 1.13 + 64%|██████████████████████████████████████████████████████████████████████████████████▍ | 15099/23458 [4:27:30<1:58:15, 1.18it/s][2025-04-24 06:38:07] (step=0015100) Train Loss: 5.4279, Train Steps/Sec: 1.13 + 64%|██████████████████████████████████████████████████████████████████████████████████▌ | 15124/23458 [4:27:54<2:01:17, 1.15it/s][2025-04-24 06:38:30] (step=0015125) Train Loss: 5.4736, Train Steps/Sec: 1.08 + 65%|██████████████████████████████████████████████████████████████████████████████████▋ | 15149/23458 [4:28:15<1:58:21, 1.17it/s][2025-04-24 06:38:51] (step=0015150) Train Loss: 5.5534, Train Steps/Sec: 1.17 + 65%|██████████████████████████████████████████████████████████████████████████████████▊ | 15174/23458 [4:28:36<1:57:55, 1.17it/s][2025-04-24 06:39:13] (step=0015175) Train Loss: 5.5289, Train Steps/Sec: 1.17 + 65%|██████████████████████████████████████████████████████████████████████████████████▉ | 15199/23458 [4:28:59<2:31:45, 1.10s/it][2025-04-24 06:39:35] (step=0015200) Train Loss: 5.5192, Train Steps/Sec: 1.12 + 65%|███████████████████████████████████████████████████████████████████████████████████ | 15224/23458 [4:29:20<2:00:01, 1.14it/s][2025-04-24 06:39:56] (step=0015225) Train Loss: 5.5116, Train Steps/Sec: 1.17 + 65%|███████████████████████████████████████████████████████████████████████████████████▏ | 15249/23458 [4:29:41<1:57:18, 1.17it/s][2025-04-24 06:40:18] (step=0015250) Train Loss: 5.4879, Train Steps/Sec: 1.17 + 65%|███████████████████████████████████████████████████████████████████████████████████▎ | 15274/23458 [4:30:03<1:55:35, 1.18it/s][2025-04-24 06:40:39] (step=0015275) Train Loss: 5.6239, Train Steps/Sec: 1.17 + 65%|███████████████████████████████████████████████████████████████████████████████████▍ | 15299/23458 [4:30:24<1:54:44, 1.19it/s][2025-04-24 06:41:00] (step=0015300) Train Loss: 5.4383, Train Steps/Sec: 1.16 + 65%|███████████████████████████████████████████████████████████████████████████████████▌ | 15324/23458 [4:30:47<1:57:50, 1.15it/s][2025-04-24 06:41:23] (step=0015325) Train Loss: 5.4595, Train Steps/Sec: 1.12 + 65%|███████████████████████████████████████████████████████████████████████████████████▊ | 15349/23458 [4:31:08<1:55:41, 1.17it/s][2025-04-24 06:41:44] (step=0015350) Train Loss: 5.5223, Train Steps/Sec: 1.17 + 66%|███████████████████████████████████████████████████████████████████████████████████▉ | 15374/23458 [4:31:29<1:54:19, 1.18it/s][2025-04-24 06:42:05] (step=0015375) Train Loss: 5.5820, Train Steps/Sec: 1.17 + 66%|████████████████████████████████████████████████████████████████████████████████████ | 15399/23458 [4:31:51<1:53:22, 1.18it/s][2025-04-24 06:42:27] (step=0015400) Train Loss: 5.5653, Train Steps/Sec: 1.17 + 66%|████████████████████████████████████████████████████████████████████████████████████▏ | 15424/23458 [4:32:12<1:56:33, 1.15it/s][2025-04-24 06:42:48] (step=0015425) Train Loss: 5.4641, Train Steps/Sec: 1.17 + 66%|████████████████████████████████████████████████████████████████████████████████████▎ | 15449/23458 [4:32:33<1:54:11, 1.17it/s][2025-04-24 06:43:10] (step=0015450) Train Loss: 5.5484, Train Steps/Sec: 1.17 + 66%|████████████████████████████████████████████████████████████████████████████████████▍ | 15474/23458 [4:32:55<1:53:30, 1.17it/s][2025-04-24 06:43:31] (step=0015475) Train Loss: 5.5013, Train Steps/Sec: 1.17 + 66%|████████████████████████████████████████████████████████████████████████████████████▌ | 15499/23458 [4:33:16<1:52:08, 1.18it/s][2025-04-24 06:43:52] (step=0015500) Train Loss: 5.5357, Train Steps/Sec: 1.17 + 66%|████████████████████████████████████████████████████████████████████████████████████▋ | 15524/23458 [4:33:38<1:54:42, 1.15it/s][2025-04-24 06:44:14] (step=0015525) Train Loss: 5.6173, Train Steps/Sec: 1.17 + 66%|████████████████████████████████████████████████████████████████████████████████████▊ | 15549/23458 [4:33:59<1:52:22, 1.17it/s][2025-04-24 06:44:35] (step=0015550) Train Loss: 5.5293, Train Steps/Sec: 1.17 + 66%|████████████████████████████████████████████████████████████████████████████████████▉ | 15574/23458 [4:34:20<1:51:39, 1.18it/s][2025-04-24 06:44:57] (step=0015575) Train Loss: 5.5117, Train Steps/Sec: 1.17 + 66%|█████████████████████████████████████████████████████████████████████████████████████ | 15599/23458 [4:34:42<1:50:58, 1.18it/s][2025-04-24 06:45:19] (step=0015600) Train Loss: 5.4896, Train Steps/Sec: 1.11 + 67%|█████████████████████████████████████████████████████████████████████████████████████▎ | 15624/23458 [4:35:04<1:53:37, 1.15it/s][2025-04-24 06:45:40] (step=0015625) Train Loss: 5.5461, Train Steps/Sec: 1.17 + 67%|█████████████████████████████████████████████████████████████████████████████████████▍ | 15649/23458 [4:35:26<1:52:01, 1.16it/s][2025-04-24 06:46:02] (step=0015650) Train Loss: 5.5639, Train Steps/Sec: 1.17 + 67%|█████████████████████████████████████████████████████████████████████████████████████▌ | 15674/23458 [4:35:47<1:50:04, 1.18it/s][2025-04-24 06:46:23] (step=0015675) Train Loss: 5.4943, Train Steps/Sec: 1.17 + 67%|█████████████████████████████████████████████████████████████████████████████████████▋ | 15699/23458 [4:36:09<1:49:40, 1.18it/s][2025-04-24 06:46:46] (step=0015700) Train Loss: 5.5440, Train Steps/Sec: 1.11 + 67%|█████████████████████████████████████████████████████████████████████████████████████▊ | 15724/23458 [4:36:32<2:16:53, 1.06s/it][2025-04-24 06:47:08] (step=0015725) Train Loss: 5.4317, Train Steps/Sec: 1.12 + 67%|█████████████████████████████████████████████████████████████████████████████████████▉ | 15749/23458 [4:36:54<1:53:37, 1.13it/s][2025-04-24 06:47:30] (step=0015750) Train Loss: 5.4574, Train Steps/Sec: 1.13 + 67%|██████████████████████████████████████████████████████████████████████████████████████ | 15774/23458 [4:37:17<2:14:45, 1.05s/it][2025-04-24 06:47:53] (step=0015775) Train Loss: 5.5298, Train Steps/Sec: 1.08 + 67%|██████████████████████████████████████████████████████████████████████████████████████▏ | 15799/23458 [4:37:39<1:47:42, 1.19it/s][2025-04-24 06:48:15] (step=0015800) Train Loss: 5.5907, Train Steps/Sec: 1.16 + 67%|██████████████████████████████████████████████████████████████████████████████████████▎ | 15824/23458 [4:38:00<1:51:18, 1.14it/s][2025-04-24 06:48:36] (step=0015825) Train Loss: 5.5807, Train Steps/Sec: 1.17 + 68%|██████████████████████████████████████████████████████████████████████████████████████▍ | 15849/23458 [4:38:21<1:48:18, 1.17it/s][2025-04-24 06:48:58] (step=0015850) Train Loss: 5.4671, Train Steps/Sec: 1.17 + 68%|██████████████████████████████████████████████████████████████████████████████████████▌ | 15874/23458 [4:38:44<1:47:27, 1.18it/s][2025-04-24 06:49:20] (step=0015875) Train Loss: 5.5392, Train Steps/Sec: 1.13 + 68%|██████████████████████████████████████████████████████████████████████████████████████▊ | 15899/23458 [4:39:05<1:46:23, 1.18it/s][2025-04-24 06:49:41] (step=0015900) Train Loss: 5.4741, Train Steps/Sec: 1.16 + 68%|██████████████████████████████████████████████████████████████████████████████████████▉ | 15924/23458 [4:39:26<1:49:22, 1.15it/s][2025-04-24 06:50:03] (step=0015925) Train Loss: 5.5490, Train Steps/Sec: 1.17 + 68%|███████████████████████████████████████████████████████████████████████████████████████ | 15949/23458 [4:39:48<1:47:09, 1.17it/s][2025-04-24 06:50:24] (step=0015950) Train Loss: 5.5225, Train Steps/Sec: 1.17 + 68%|███████████████████████████████████████████████████████████████████████████████████████▏ | 15974/23458 [4:40:10<1:46:47, 1.17it/s][2025-04-24 06:50:46] (step=0015975) Train Loss: 5.5246, Train Steps/Sec: 1.12 + 68%|███████████████████████████████████████████████████████████████████████████████████████▎ | 15999/23458 [4:40:31<1:45:05, 1.18it/s][2025-04-24 06:51:08] (step=0016000) Train Loss: 5.5019, Train Steps/Sec: 1.17 +You are using a model of type blip-2 to instantiate a model of type instructblip. This is not supported for all configurations of models and can yield errors. +[2025-04-24 06:51:08] text_config is None. Initializing the text config with default values (`OPTConfig`). +model_name_or_path:/tmp/haozhezhao/model/blip2-flan-t5-xl +image_place_holder: +tokenizer length after expend 32101 +tokenizer length before expend 32100 +100%|████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████| 4/4 [04:07<00:00, 61.77s/it] +[2025-04-24 06:56:27] Finish Eval in 16000 steps...█████████████████████████████████████████████████████████████████████████████████████| 4/4 [04:06<00:00, 61.24s/it] +processing left-padding... +processing left-padding... +processing left-padding... +processing left-padding... +[2025-04-24 06:56:47] Saved checkpoint to checkpoint/Afteri2i_100kckpt_LLAVAT5_empty_Fixed_just_segment_pretrain_stage3_dreambench_recap_400k_t2i_400k_flux_200k_midjourney_150k_recovery_150k_3e4_epoch2_no_rotated/000-GPT-XL/checkpoints/0016000.pt +[2025-04-24 06:56:48] Removed old checkpoint: checkpoint/Afteri2i_100kckpt_LLAVAT5_empty_Fixed_just_segment_pretrain_stage3_dreambench_recap_400k_t2i_400k_flux_200k_midjourney_150k_recovery_150k_3e4_epoch2_no_rotated/000-GPT-XL/checkpoints/0014000.pt + 68%|███████████████████████████████████████████████████████████████████████████████████████▍ | 16024/23458 [4:46:34<1:49:31, 1.13it/s][2025-04-24 06:57:10] (step=0016025) Train Loss: 5.5051, Train Steps/Sec: 0.07 + 68%|███████████████████████████████████████████████████████████████████████████████████████▌ | 16049/23458 [4:46:55<1:45:29, 1.17it/s][2025-04-24 06:57:31] (step=0016050) Train Loss: 5.4464, Train Steps/Sec: 1.17 + 69%|███████████████████████████████████████████████████████████████████████████████████████▋ | 16074/23458 [4:47:16<1:44:50, 1.17it/s][2025-04-24 06:57:53] (step=0016075) Train Loss: 5.4626, Train Steps/Sec: 1.17 + 69%|███████████████████████████████████████████████████████████████████████████████████████▊ | 16099/23458 [4:47:38<1:43:43, 1.18it/s][2025-04-24 06:58:14] (step=0016100) Train Loss: 5.5685, Train Steps/Sec: 1.16 + 69%|███████████████████████████████████████████████████████████████████████████████████████▉ | 16124/23458 [4:47:59<1:45:52, 1.15it/s][2025-04-24 06:58:35] (step=0016125) Train Loss: 5.5126, Train Steps/Sec: 1.17 + 69%|████████████████████████████████████████████████████████████████████████████████████████ | 16149/23458 [4:48:20<1:44:06, 1.17it/s][2025-04-24 06:58:57] (step=0016150) Train Loss: 5.4935, Train Steps/Sec: 1.17 + 69%|████████████████████████████████████████████████████████████████████████████████████████▎ | 16174/23458 [4:48:42<1:42:54, 1.18it/s][2025-04-24 06:59:18] (step=0016175) Train Loss: 5.3838, Train Steps/Sec: 1.17 + 69%|████████████████████████████████████████████████████████████████████████████████████████▍ | 16199/23458 [4:49:03<1:41:53, 1.19it/s][2025-04-24 06:59:39] (step=0016200) Train Loss: 5.5826, Train Steps/Sec: 1.17 + 69%|████████████████████████████████████████████████████████████████████████████████████████▌ | 16224/23458 [4:49:25<1:44:59, 1.15it/s][2025-04-24 07:00:01] (step=0016225) Train Loss: 5.4875, Train Steps/Sec: 1.17 + 69%|████████████████████████████████████████████████████████████████████████████████████████▋ | 16249/23458 [4:49:46<1:42:31, 1.17it/s][2025-04-24 07:00:22] (step=0016250) Train Loss: 5.4966, Train Steps/Sec: 1.17 + 69%|████████████████████████████████████████████████████████████████████████████████████████▊ | 16274/23458 [4:50:07<1:41:26, 1.18it/s][2025-04-24 07:00:44] (step=0016275) Train Loss: 5.4893, Train Steps/Sec: 1.17 + 69%|████████████████████████████████████████████████████████████████████████████████████████▉ | 16299/23458 [4:50:29<1:40:29, 1.19it/s][2025-04-24 07:01:05] (step=0016300) Train Loss: 5.4548, Train Steps/Sec: 1.17 + 70%|█████████████████████████████████████████████████████████████████████████████████████████ | 16324/23458 [4:50:51<2:08:30, 1.08s/it][2025-04-24 07:01:27] (step=0016325) Train Loss: 5.4922, Train Steps/Sec: 1.12 + 70%|█████████████████████████████████████████████████████████████████████████████████████████▏ | 16349/23458 [4:51:12<1:41:20, 1.17it/s][2025-04-24 07:01:49] (step=0016350) Train Loss: 5.4465, Train Steps/Sec: 1.17 + 70%|█████████████████████████████████████████████████████████████████████████████████████████▎ | 16374/23458 [4:51:34<1:39:46, 1.18it/s][2025-04-24 07:02:10] (step=0016375) Train Loss: 5.5063, Train Steps/Sec: 1.17 + 70%|█████████████████████████████████████████████████████████████████████████████████████████▍ | 16399/23458 [4:51:56<1:39:27, 1.18it/s][2025-04-24 07:02:32] (step=0016400) Train Loss: 5.5979, Train Steps/Sec: 1.12 + 70%|█████████████████████████████████████████████████████████████████████████████████████████▌ | 16424/23458 [4:52:20<1:52:31, 1.04it/s][2025-04-24 07:02:56] (step=0016425) Train Loss: 5.4638, Train Steps/Sec: 1.04 + 70%|█████████████████████████████████████████████████████████████████████████████████████████▊ | 16449/23458 [4:52:42<1:40:31, 1.16it/s][2025-04-24 07:03:19] (step=0016450) Train Loss: 5.5447, Train Steps/Sec: 1.12 + 70%|█████████████████████████████████████████████████████████████████████████████████████████▉ | 16474/23458 [4:53:04<1:38:32, 1.18it/s][2025-04-24 07:03:40] (step=0016475) Train Loss: 5.4832, Train Steps/Sec: 1.17 + 70%|██████████████████████████████████████████████████████████████████████████████████████████ | 16499/23458 [4:53:25<1:37:25, 1.19it/s][2025-04-24 07:04:01] (step=0016500) Train Loss: 5.5053, Train Steps/Sec: 1.17 + 70%|██████████████████████████████████████████████████████████████████████████████████████████▏ | 16524/23458 [4:53:47<1:50:51, 1.04it/s][2025-04-24 07:04:24] (step=0016525) Train Loss: 5.4677, Train Steps/Sec: 1.13 + 71%|██████████████████████████████████████████████████████████████████████████████████████████▎ | 16549/23458 [4:54:09<1:38:29, 1.17it/s][2025-04-24 07:04:45] (step=0016550) Train Loss: 5.5246, Train Steps/Sec: 1.17 + 71%|██████████████████████████████████████████████████████████████████████████████████████████▍ | 16574/23458 [4:54:30<1:37:20, 1.18it/s][2025-04-24 07:05:06] (step=0016575) Train Loss: 5.4827, Train Steps/Sec: 1.17 + 71%|██████████████████████████████████████████████████████████████████████████████████████████▌ | 16599/23458 [4:54:51<1:36:41, 1.18it/s][2025-04-24 07:05:28] (step=0016600) Train Loss: 5.5285, Train Steps/Sec: 1.17 + 71%|██████████████████████████████████████████████████████████████████████████████████████████▋ | 16624/23458 [4:55:13<1:39:01, 1.15it/s][2025-04-24 07:05:49] (step=0016625) Train Loss: 5.4629, Train Steps/Sec: 1.17 + 71%|██████████████████████████████████████████████████████████████████████████████████████████▊ | 16649/23458 [4:55:34<1:37:12, 1.17it/s][2025-04-24 07:06:10] (step=0016650) Train Loss: 5.5421, Train Steps/Sec: 1.17 + 71%|██████████████████████████████████████████████████████████████████████████████████████████▉ | 16674/23458 [4:55:56<1:36:12, 1.18it/s][2025-04-24 07:06:33] (step=0016675) Train Loss: 5.4885, Train Steps/Sec: 1.12 + 71%|███████████████████████████████████████████████████████████████████████████████████████████ | 16699/23458 [4:56:18<1:35:13, 1.18it/s][2025-04-24 07:06:54] (step=0016700) Train Loss: 5.5197, Train Steps/Sec: 1.16 + 71%|███████████████████████████████████████████████████████████████████████████████████████████▎ | 16724/23458 [4:56:39<1:37:18, 1.15it/s][2025-04-24 07:07:15] (step=0016725) Train Loss: 5.4876, Train Steps/Sec: 1.17 + 71%|███████████████████████████████████████████████████████████████████████████████████████████▍ | 16749/23458 [4:57:01<1:35:56, 1.17it/s][2025-04-24 07:07:37] (step=0016750) Train Loss: 5.4569, Train Steps/Sec: 1.17 + 72%|███████████████████████████████████████████████████████████████████████████████████████████▌ | 16774/23458 [4:57:22<1:34:31, 1.18it/s][2025-04-24 07:07:58] (step=0016775) Train Loss: 5.4885, Train Steps/Sec: 1.17 + 72%|███████████████████████████████████████████████████████████████████████████████████████████▋ | 16799/23458 [4:57:43<1:33:47, 1.18it/s][2025-04-24 07:08:19] (step=0016800) Train Loss: 5.5632, Train Steps/Sec: 1.17 + 72%|███████████████████████████████████████████████████████████████████████████████████████████▊ | 16824/23458 [4:58:05<1:35:59, 1.15it/s][2025-04-24 07:08:41] (step=0016825) Train Loss: 5.5166, Train Steps/Sec: 1.17 + 72%|███████████████████████████████████████████████████████████████████████████████████████████▉ | 16849/23458 [4:58:26<1:34:14, 1.17it/s][2025-04-24 07:09:02] (step=0016850) Train Loss: 5.4503, Train Steps/Sec: 1.17 + 72%|████████████████████████████████████████████████████████████████████████████████████████████ | 16874/23458 [4:58:47<1:33:02, 1.18it/s][2025-04-24 07:09:24] (step=0016875) Train Loss: 5.4826, Train Steps/Sec: 1.17 + 72%|████████████████████████████████████████████████████████████████████████████████████████████▏ | 16899/23458 [4:59:09<1:32:05, 1.19it/s][2025-04-24 07:09:45] (step=0016900) Train Loss: 5.6190, Train Steps/Sec: 1.16 + 72%|████████████████████████████████████████████████████████████████████████████████████████████▎ | 16924/23458 [4:59:30<1:34:30, 1.15it/s][2025-04-24 07:10:06] (step=0016925) Train Loss: 5.5137, Train Steps/Sec: 1.17 + 72%|████████████████████████████████████████████████████████████████████████████████████████████▍ | 16949/23458 [4:59:51<1:32:55, 1.17it/s][2025-04-24 07:10:28] (step=0016950) Train Loss: 5.4950, Train Steps/Sec: 1.17 + 72%|████████████████████████████████████████████████████████████████████████████████████████████▌ | 16974/23458 [5:00:13<1:31:40, 1.18it/s][2025-04-24 07:10:49] (step=0016975) Train Loss: 5.5063, Train Steps/Sec: 1.17 + 72%|████████████████████████████████████████████████████████████████████████████████████████████▊ | 16999/23458 [5:00:34<1:30:56, 1.18it/s][2025-04-24 07:11:11] (step=0017000) Train Loss: 5.5108, Train Steps/Sec: 1.16 + 73%|████████████████████████████████████████████████████████████████████████████████████████████▉ | 17024/23458 [5:00:56<1:33:19, 1.15it/s][2025-04-24 07:11:32] (step=0017025) Train Loss: 5.4433, Train Steps/Sec: 1.17 + 73%|█████████████████████████████████████████████████████████████████████████████████████████████ | 17049/23458 [5:01:19<1:36:06, 1.11it/s][2025-04-24 07:11:55] (step=0017050) Train Loss: 5.5164, Train Steps/Sec: 1.07 + 73%|█████████████████████████████████████████████████████████████████████████████████████████████▏ | 17074/23458 [5:01:41<1:30:45, 1.17it/s][2025-04-24 07:12:17] (step=0017075) Train Loss: 5.5241, Train Steps/Sec: 1.13 + 73%|█████████████████████████████████████████████████████████████████████████████████████████████▎ | 17099/23458 [5:02:04<1:30:18, 1.17it/s][2025-04-24 07:12:41] (step=0017100) Train Loss: 5.5162, Train Steps/Sec: 1.08 + 73%|█████████████████████████████████████████████████████████████████████████████████████████████▍ | 17124/23458 [5:02:26<1:32:10, 1.15it/s][2025-04-24 07:13:02] (step=0017125) Train Loss: 5.4866, Train Steps/Sec: 1.17 + 73%|█████████████████████████████████████████████████████████████████████████████████████████████▌ | 17149/23458 [5:02:48<1:30:07, 1.17it/s][2025-04-24 07:13:24] (step=0017150) Train Loss: 5.6039, Train Steps/Sec: 1.11 + 73%|█████████████████████████████████████████████████████████████████████████████████████████████▋ | 17174/23458 [5:03:10<1:29:14, 1.17it/s][2025-04-24 07:13:46] (step=0017175) Train Loss: 5.4777, Train Steps/Sec: 1.17 + 73%|█████████████████████████████████████████████████████████████████████████████████████████████▊ | 17199/23458 [5:03:32<1:27:55, 1.19it/s][2025-04-24 07:14:08] (step=0017200) Train Loss: 5.4156, Train Steps/Sec: 1.12 + 73%|█████████████████████████████████████████████████████████████████████████████████████████████▉ | 17224/23458 [5:03:53<1:30:48, 1.14it/s][2025-04-24 07:14:29] (step=0017225) Train Loss: 5.5043, Train Steps/Sec: 1.17 + 74%|██████████████████████████████████████████████████████████████████████████████████████████████ | 17249/23458 [5:04:15<1:28:41, 1.17it/s][2025-04-24 07:14:51] (step=0017250) Train Loss: 5.5061, Train Steps/Sec: 1.17 + 74%|██████████████████████████████████████████████████████████████████████████████████████████████▎ | 17274/23458 [5:04:36<1:27:21, 1.18it/s][2025-04-24 07:15:12] (step=0017275) Train Loss: 5.5259, Train Steps/Sec: 1.17 + 74%|██████████████████████████████████████████████████████████████████████████████████████████████▍ | 17299/23458 [5:04:57<1:26:37, 1.18it/s][2025-04-24 07:15:34] (step=0017300) Train Loss: 5.5143, Train Steps/Sec: 1.17 + 74%|██████████████████████████████████████████████████████████████████████████████████████████████▌ | 17324/23458 [5:05:20<1:29:09, 1.15it/s][2025-04-24 07:15:56] (step=0017325) Train Loss: 5.5173, Train Steps/Sec: 1.12 + 74%|██████████████████████████████████████████████████████████████████████████████████████████████▋ | 17349/23458 [5:05:41<1:27:14, 1.17it/s][2025-04-24 07:16:17] (step=0017350) Train Loss: 5.4510, Train Steps/Sec: 1.17 + 74%|██████████████████████████████████████████████████████████████████████████████████████████████▊ | 17374/23458 [5:06:02<1:25:48, 1.18it/s][2025-04-24 07:16:39] (step=0017375) Train Loss: 5.5847, Train Steps/Sec: 1.17 + 74%|██████████████████████████████████████████████████████████████████████████████████████████████▉ | 17399/23458 [5:06:24<1:25:14, 1.18it/s][2025-04-24 07:17:00] (step=0017400) Train Loss: 5.4573, Train Steps/Sec: 1.17 + 74%|███████████████████████████████████████████████████████████████████████████████████████████████ | 17424/23458 [5:06:45<1:27:42, 1.15it/s][2025-04-24 07:17:21] (step=0017425) Train Loss: 5.4729, Train Steps/Sec: 1.17 + 74%|███████████████████████████████████████████████████████████████████████████████████████████████▏ | 17449/23458 [5:07:07<1:25:51, 1.17it/s][2025-04-24 07:17:43] (step=0017450) Train Loss: 5.5323, Train Steps/Sec: 1.17 + 74%|███████████████████████████████████████████████████████████████████████████████████████████████▎ | 17474/23458 [5:07:28<1:24:42, 1.18it/s][2025-04-24 07:18:04] (step=0017475) Train Loss: 5.5635, Train Steps/Sec: 1.17 + 75%|███████████████████████████████████████████████████████████████████████████████████████████████▍ | 17499/23458 [5:07:49<1:23:51, 1.18it/s][2025-04-24 07:18:26] (step=0017500) Train Loss: 5.5245, Train Steps/Sec: 1.17 + 75%|███████████████████████████████████████████████████████████████████████████████████████████████▌ | 17524/23458 [5:08:11<1:26:12, 1.15it/s][2025-04-24 07:18:47] (step=0017525) Train Loss: 5.5077, Train Steps/Sec: 1.17 + 75%|███████████████████████████████████████████████████████████████████████████████████████████████▊ | 17549/23458 [5:08:32<1:24:45, 1.16it/s][2025-04-24 07:19:08] (step=0017550) Train Loss: 5.5774, Train Steps/Sec: 1.17 + 75%|███████████████████████████████████████████████████████████████████████████████████████████████▉ | 17574/23458 [5:08:53<1:23:25, 1.18it/s][2025-04-24 07:19:30] (step=0017575) Train Loss: 5.4494, Train Steps/Sec: 1.17 + 75%|████████████████████████████████████████████████████████████████████████████████████████████████ | 17599/23458 [5:09:15<1:22:31, 1.18it/s][2025-04-24 07:19:51] (step=0017600) Train Loss: 5.5178, Train Steps/Sec: 1.16 + 75%|████████████████████████████████████████████████████████████████████████████████████████████████▏ | 17624/23458 [5:09:36<1:24:18, 1.15it/s][2025-04-24 07:20:12] (step=0017625) Train Loss: 5.4888, Train Steps/Sec: 1.17 + 75%|████████████████████████████████████████████████████████████████████████████████████████████████▎ | 17649/23458 [5:09:57<1:22:58, 1.17it/s][2025-04-24 07:20:34] (step=0017650) Train Loss: 5.5101, Train Steps/Sec: 1.17 + 75%|████████████████████████████████████████████████████████████████████████████████████████████████▍ | 17674/23458 [5:10:19<1:21:35, 1.18it/s][2025-04-24 07:20:55] (step=0017675) Train Loss: 5.4688, Train Steps/Sec: 1.17 + 75%|████████████████████████████████████████████████████████████████████████████████████████████████▌ | 17699/23458 [5:10:41<1:24:11, 1.14it/s][2025-04-24 07:21:17] (step=0017700) Train Loss: 5.4802, Train Steps/Sec: 1.12 + 76%|████████████████████████████████████████████████████████████████████████████████████████████████▋ | 17724/23458 [5:11:03<1:31:04, 1.05it/s][2025-04-24 07:21:40] (step=0017725) Train Loss: 5.5327, Train Steps/Sec: 1.13 + 76%|████████████████████████████████████████████████████████████████████████████████████████████████▊ | 17749/23458 [5:11:25<1:22:01, 1.16it/s][2025-04-24 07:22:03] (step=0017750) Train Loss: 5.4576, Train Steps/Sec: 1.08 + 76%|████████████████████████████████████████████████████████████████████████████████████████████████▉ | 17774/23458 [5:11:49<1:21:31, 1.16it/s][2025-04-24 07:22:25] (step=0017775) Train Loss: 5.4912, Train Steps/Sec: 1.12 + 76%|█████████████████████████████████████████████████████████████████████████████████████████████████ | 17799/23458 [5:12:10<1:19:41, 1.18it/s][2025-04-24 07:22:46] (step=0017800) Train Loss: 5.5289, Train Steps/Sec: 1.17 + 76%|█████████████████████████████████████████████████████████████████████████████████████████████████▎ | 17824/23458 [5:12:32<1:21:29, 1.15it/s][2025-04-24 07:23:08] (step=0017825) Train Loss: 5.5371, Train Steps/Sec: 1.17 + 76%|█████████████████████████████████████████████████████████████████████████████████████████████████▍ | 17849/23458 [5:12:54<1:21:37, 1.15it/s][2025-04-24 07:23:30] (step=0017850) Train Loss: 5.4750, Train Steps/Sec: 1.13 + 76%|█████████████████████████████████████████████████████████████████████████████████████████████████▌ | 17874/23458 [5:13:16<1:20:19, 1.16it/s][2025-04-24 07:23:52] (step=0017875) Train Loss: 5.4674, Train Steps/Sec: 1.11 + 76%|█████████████████████████████████████████████████████████████████████████████████████████████████▋ | 17899/23458 [5:13:38<1:18:05, 1.19it/s][2025-04-24 07:24:14] (step=0017900) Train Loss: 5.4952, Train Steps/Sec: 1.17 + 76%|█████████████████████████████████████████████████████████████████████████████████████████████████▊ | 17924/23458 [5:13:59<1:20:03, 1.15it/s][2025-04-24 07:24:35] (step=0017925) Train Loss: 5.5112, Train Steps/Sec: 1.17 + 77%|█████████████████████████████████████████████████████████████████████████████████████████████████▉ | 17949/23458 [5:14:20<1:18:30, 1.17it/s][2025-04-24 07:24:57] (step=0017950) Train Loss: 5.4589, Train Steps/Sec: 1.17 + 77%|██████████████████████████████████████████████████████████████████████████████████████████████████ | 17974/23458 [5:14:42<1:17:54, 1.17it/s][2025-04-24 07:25:18] (step=0017975) Train Loss: 5.4212, Train Steps/Sec: 1.17 + 77%|██████████████████████████████████████████████████████████████████████████████████████████████████▏ | 17999/23458 [5:15:04<1:16:52, 1.18it/s][2025-04-24 07:25:40] (step=0018000) Train Loss: 5.4668, Train Steps/Sec: 1.12 +You are using a model of type blip-2 to instantiate a model of type instructblip. This is not supported for all configurations of models and can yield errors. +[2025-04-24 07:25:40] text_config is None. Initializing the text config with default values (`OPTConfig`). +model_name_or_path:/tmp/haozhezhao/model/blip2-flan-t5-xl +image_place_holder: +tokenizer length after expend 32101 +tokenizer length before expend 32100 +100%|████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████| 4/4 [04:02<00:00, 60.71s/it] +[2025-04-24 07:30:52] Finish Eval in 18000 steps...█████████████████████████████████████████████████████████████████████████████████████| 4/4 [04:02<00:00, 60.34s/it] +processing left-padding... +processing left-padding... +processing left-padding... +processing left-padding... +[2025-04-24 07:31:11] Saved checkpoint to checkpoint/Afteri2i_100kckpt_LLAVAT5_empty_Fixed_just_segment_pretrain_stage3_dreambench_recap_400k_t2i_400k_flux_200k_midjourney_150k_recovery_150k_3e4_epoch2_no_rotated/000-GPT-XL/checkpoints/0018000.pt +[2025-04-24 07:31:13] Removed old checkpoint: checkpoint/Afteri2i_100kckpt_LLAVAT5_empty_Fixed_just_segment_pretrain_stage3_dreambench_recap_400k_t2i_400k_flux_200k_midjourney_150k_recovery_150k_3e4_epoch2_no_rotated/000-GPT-XL/checkpoints/0016000.pt + 77%|██████████████████████████████████████████████████████████████████████████████████████████████████▎ | 18024/23458 [5:20:58<1:20:17, 1.13it/s][2025-04-24 07:31:34] (step=0018025) Train Loss: 5.4358, Train Steps/Sec: 0.07 + 77%|██████████████████████████████████████████████████████████████████████████████████████████████████▍ | 18049/23458 [5:21:19<1:17:03, 1.17it/s][2025-04-24 07:31:56] (step=0018050) Train Loss: 5.4906, Train Steps/Sec: 1.17 + 77%|██████████████████████████████████████████████████████████████████████████████████████████████████▌ | 18074/23458 [5:21:41<1:16:39, 1.17it/s][2025-04-24 07:32:17] (step=0018075) Train Loss: 5.4534, Train Steps/Sec: 1.17 + 77%|██████████████████████████████████████████████████████████████████████████████████████████████████▊ | 18099/23458 [5:22:02<1:15:20, 1.19it/s][2025-04-24 07:32:38] (step=0018100) Train Loss: 5.5204, Train Steps/Sec: 1.17 + 77%|██████████████████████████████████████████████████████████████████████████████████████████████████▉ | 18124/23458 [5:22:24<1:17:16, 1.15it/s][2025-04-24 07:33:00] (step=0018125) Train Loss: 5.4805, Train Steps/Sec: 1.17 + 77%|███████████████████████████████████████████████████████████████████████████████████████████████████ | 18149/23458 [5:22:45<1:15:40, 1.17it/s][2025-04-24 07:33:21] (step=0018150) Train Loss: 5.5645, Train Steps/Sec: 1.16 + 77%|███████████████████████████████████████████████████████████████████████████████████████████████████▏ | 18174/23458 [5:23:06<1:14:44, 1.18it/s][2025-04-24 07:33:43] (step=0018175) Train Loss: 5.3972, Train Steps/Sec: 1.17 + 78%|███████████████████████████████████████████████████████████████████████████████████████████████████▎ | 18199/23458 [5:23:28<1:14:02, 1.18it/s][2025-04-24 07:34:04] (step=0018200) Train Loss: 5.5897, Train Steps/Sec: 1.17 + 78%|███████████████████████████████████████████████████████████████████████████████████████████████████▍ | 18224/23458 [5:23:49<1:15:49, 1.15it/s][2025-04-24 07:34:25] (step=0018225) Train Loss: 5.5371, Train Steps/Sec: 1.17 + 78%|███████████████████████████████████████████████████████████████████████████████████████████████████▌ | 18249/23458 [5:24:11<1:14:26, 1.17it/s][2025-04-24 07:34:47] (step=0018250) Train Loss: 5.5952, Train Steps/Sec: 1.17 + 78%|███████████████████████████████████████████████████████████████████████████████████████████████████▋ | 18274/23458 [5:24:32<1:13:14, 1.18it/s][2025-04-24 07:35:08] (step=0018275) Train Loss: 5.3934, Train Steps/Sec: 1.17 + 78%|███████████████████████████████████████████████████████████████████████████████████████████████████▊ | 18299/23458 [5:24:53<1:12:37, 1.18it/s][2025-04-24 07:35:30] (step=0018300) Train Loss: 5.5189, Train Steps/Sec: 1.17 + 78%|███████████████████████████████████████████████████████████████████████████████████████████████████▉ | 18324/23458 [5:25:15<1:14:12, 1.15it/s][2025-04-24 07:35:51] (step=0018325) Train Loss: 5.4553, Train Steps/Sec: 1.17 + 78%|████████████████████████████████████████████████████████████████████████████████████████████████████ | 18349/23458 [5:25:36<1:12:49, 1.17it/s][2025-04-24 07:36:13] (step=0018350) Train Loss: 5.4789, Train Steps/Sec: 1.12 + 78%|████████████████████████████████████████████████████████████████████████████████████████████████████▎ | 18374/23458 [5:25:58<1:11:44, 1.18it/s][2025-04-24 07:36:35] (step=0018375) Train Loss: 5.4963, Train Steps/Sec: 1.17 + 78%|████████████████████████████████████████████████████████████████████████████████████████████████████▍ | 18399/23458 [5:26:21<1:11:23, 1.18it/s][2025-04-24 07:36:58] (step=0018400) Train Loss: 5.5375, Train Steps/Sec: 1.08 + 79%|████████████████████████████████████████████████████████████████████████████████████████████████████▌ | 18424/23458 [5:26:44<1:13:11, 1.15it/s][2025-04-24 07:37:20] (step=0018425) Train Loss: 5.4807, Train Steps/Sec: 1.13 + 79%|████████████████████████████████████████████████████████████████████████████████████████████████████▋ | 18449/23458 [5:27:05<1:11:55, 1.16it/s][2025-04-24 07:37:41] (step=0018450) Train Loss: 5.3620, Train Steps/Sec: 1.17 + 79%|████████████████████████████████████████████████████████████████████████████████████████████████████▊ | 18474/23458 [5:27:27<1:10:34, 1.18it/s][2025-04-24 07:38:03] (step=0018475) Train Loss: 5.4450, Train Steps/Sec: 1.16 + 79%|████████████████████████████████████████████████████████████████████████████████████████████████████▉ | 18499/23458 [5:27:50<1:19:48, 1.04it/s][2025-04-24 07:38:26] (step=0018500) Train Loss: 5.4931, Train Steps/Sec: 1.07 + 79%|█████████████████████████████████████████████████████████████████████████████████████████████████████ | 18524/23458 [5:28:11<1:11:30, 1.15it/s][2025-04-24 07:38:47] (step=0018525) Train Loss: 5.4474, Train Steps/Sec: 1.17 + 79%|█████████████████████████████████████████████████████████████████████████████████████████████████████▏ | 18549/23458 [5:28:33<1:09:49, 1.17it/s][2025-04-24 07:39:09] (step=0018550) Train Loss: 5.4844, Train Steps/Sec: 1.17 + 79%|█████████████████████████████████████████████████████████████████████████████████████████████████████▎ | 18574/23458 [5:28:54<1:09:16, 1.18it/s][2025-04-24 07:39:30] (step=0018575) Train Loss: 5.4188, Train Steps/Sec: 1.16 + 79%|█████████████████████████████████████████████████████████████████████████████████████████████████████▍ | 18599/23458 [5:29:17<1:08:35, 1.18it/s][2025-04-24 07:39:53] (step=0018600) Train Loss: 5.5376, Train Steps/Sec: 1.11 + 79%|█████████████████████████████████████████████████████████████████████████████████████████████████████▌ | 18624/23458 [5:29:38<1:09:51, 1.15it/s][2025-04-24 07:40:14] (step=0018625) Train Loss: 5.5167, Train Steps/Sec: 1.17 + 79%|█████████████████████████████████████████████████████████████████████████████████████████████████████▊ | 18649/23458 [5:30:00<1:24:43, 1.06s/it][2025-04-24 07:40:37] (step=0018650) Train Loss: 5.4393, Train Steps/Sec: 1.12 + 80%|█████████████████████████████████████████████████████████████████████████████████████████████████████▉ | 18674/23458 [5:30:22<1:07:47, 1.18it/s][2025-04-24 07:40:58] (step=0018675) Train Loss: 5.3458, Train Steps/Sec: 1.17 + 80%|██████████████████████████████████████████████████████████████████████████████████████████████████████ | 18699/23458 [5:30:43<1:06:54, 1.19it/s][2025-04-24 07:41:19] (step=0018700) Train Loss: 5.4778, Train Steps/Sec: 1.17 + 80%|██████████████████████████████████████████████████████████████████████████████████████████████████████▏ | 18724/23458 [5:31:04<1:08:40, 1.15it/s][2025-04-24 07:41:41] (step=0018725) Train Loss: 5.4822, Train Steps/Sec: 1.17 + 80%|██████████████████████████████████████████████████████████████████████████████████████████████████████▎ | 18749/23458 [5:31:26<1:07:12, 1.17it/s][2025-04-24 07:42:02] (step=0018750) Train Loss: 5.5625, Train Steps/Sec: 1.17 + 80%|██████████████████████████████████████████████████████████████████████████████████████████████████████▍ | 18774/23458 [5:31:47<1:06:17, 1.18it/s][2025-04-24 07:42:23] (step=0018775) Train Loss: 5.5072, Train Steps/Sec: 1.17 + 80%|██████████████████████████████████████████████████████████████████████████████████████████████████████▌ | 18799/23458 [5:32:09<1:05:29, 1.19it/s][2025-04-24 07:42:45] (step=0018800) Train Loss: 5.5198, Train Steps/Sec: 1.17 + 80%|██████████████████████████████████████████████████████████████████████████████████████████████████████▋ | 18824/23458 [5:32:30<1:06:58, 1.15it/s][2025-04-24 07:43:06] (step=0018825) Train Loss: 5.5383, Train Steps/Sec: 1.17 + 80%|██████████████████████████████████████████████████████████████████████████████████████████████████████▊ | 18849/23458 [5:32:51<1:05:49, 1.17it/s][2025-04-24 07:43:28] (step=0018850) Train Loss: 5.4402, Train Steps/Sec: 1.17 + 80%|██████████████████████████████████████████████████████████████████████████████████████████████████████▉ | 18874/23458 [5:33:13<1:04:57, 1.18it/s][2025-04-24 07:43:49] (step=0018875) Train Loss: 5.4847, Train Steps/Sec: 1.17 + 81%|███████████████████████████████████████████████████████████████████████████████████████████████████████ | 18899/23458 [5:33:34<1:04:00, 1.19it/s][2025-04-24 07:44:10] (step=0018900) Train Loss: 5.4426, Train Steps/Sec: 1.16 + 81%|███████████████████████████████████████████████████████████████████████████████████████████████████████▎ | 18924/23458 [5:33:55<1:05:46, 1.15it/s][2025-04-24 07:44:32] (step=0018925) Train Loss: 5.4358, Train Steps/Sec: 1.17 + 81%|███████████████████████████████████████████████████████████████████████████████████████████████████████▍ | 18949/23458 [5:34:17<1:04:12, 1.17it/s][2025-04-24 07:44:53] (step=0018950) Train Loss: 5.4731, Train Steps/Sec: 1.17 + 81%|███████████████████████████████████████████████████████████████████████████████████████████████████████▌ | 18974/23458 [5:34:38<1:03:26, 1.18it/s][2025-04-24 07:45:14] (step=0018975) Train Loss: 5.5241, Train Steps/Sec: 1.17 + 81%|███████████████████████████████████████████████████████████████████████████████████████████████████████▋ | 18999/23458 [5:34:59<1:02:50, 1.18it/s][2025-04-24 07:45:36] (step=0019000) Train Loss: 5.4190, Train Steps/Sec: 1.17 + 81%|███████████████████████████████████████████████████████████████████████████████████████████████████████▊ | 19024/23458 [5:35:22<1:04:06, 1.15it/s][2025-04-24 07:45:58] (step=0019025) Train Loss: 5.5091, Train Steps/Sec: 1.13 + 81%|███████████████████████████████████████████████████████████████████████████████████████████████████████▉ | 19049/23458 [5:35:44<1:03:21, 1.16it/s][2025-04-24 07:46:20] (step=0019050) Train Loss: 5.5391, Train Steps/Sec: 1.13 + 81%|████████████████████████████████████████████████████████████████████████████████████████████████████████ | 19074/23458 [5:36:07<1:08:38, 1.06it/s][2025-04-24 07:46:43] (step=0019075) Train Loss: 5.5753, Train Steps/Sec: 1.08 + 81%|████████████████████████████████████████████████████████████████████████████████████████████████████████▏ | 19099/23458 [5:36:28<1:01:37, 1.18it/s][2025-04-24 07:47:05] (step=0019100) Train Loss: 5.5134, Train Steps/Sec: 1.16 + 82%|████████████████████████████████████████████████████████████████████████████████████████████████████████▎ | 19124/23458 [5:36:50<1:02:41, 1.15it/s][2025-04-24 07:47:26] (step=0019125) Train Loss: 5.4791, Train Steps/Sec: 1.17 + 82%|████████████████████████████████████████████████████████████████████████████████████████████████████████▍ | 19149/23458 [5:37:11<1:01:22, 1.17it/s][2025-04-24 07:47:47] (step=0019150) Train Loss: 5.4918, Train Steps/Sec: 1.17 + 82%|████████████████████████████████████████████████████████████████████████████████████████████████████████▌ | 19174/23458 [5:37:33<1:00:36, 1.18it/s][2025-04-24 07:48:09] (step=0019175) Train Loss: 5.5401, Train Steps/Sec: 1.13 + 82%|██████████████████████████████████████████████████████████████████████████████████████████████████████████▍ | 19199/23458 [5:37:55<59:48, 1.19it/s][2025-04-24 07:48:31] (step=0019200) Train Loss: 5.4794, Train Steps/Sec: 1.17 + 82%|████████████████████████████████████████████████████████████████████████████████████████████████████████▉ | 19224/23458 [5:38:17<1:01:18, 1.15it/s][2025-04-24 07:48:53] (step=0019225) Train Loss: 5.4962, Train Steps/Sec: 1.12 + 82%|█████████████████████████████████████████████████████████████████████████████████████████████████████████ | 19249/23458 [5:38:38<1:00:13, 1.16it/s][2025-04-24 07:49:15] (step=0019250) Train Loss: 5.4318, Train Steps/Sec: 1.17 + 82%|██████████████████████████████████████████████████████████████████████████████████████████████████████████▊ | 19274/23458 [5:39:00<59:06, 1.18it/s][2025-04-24 07:49:36] (step=0019275) Train Loss: 5.4728, Train Steps/Sec: 1.17 + 82%|██████████████████████████████████████████████████████████████████████████████████████████████████████████▉ | 19299/23458 [5:39:21<58:18, 1.19it/s][2025-04-24 07:49:57] (step=0019300) Train Loss: 5.4955, Train Steps/Sec: 1.17 + 82%|█████████████████████████████████████████████████████████████████████████████████████████████████████████▍ | 19324/23458 [5:39:45<1:00:08, 1.15it/s][2025-04-24 07:50:21] (step=0019325) Train Loss: 5.4629, Train Steps/Sec: 1.07 + 82%|███████████████████████████████████████████████████████████████████████████████████████████████████████████▏ | 19349/23458 [5:40:06<58:44, 1.17it/s][2025-04-24 07:50:42] (step=0019350) Train Loss: 5.4892, Train Steps/Sec: 1.17 + 83%|███████████████████████████████████████████████████████████████████████████████████████████████████████████▎ | 19374/23458 [5:40:27<57:42, 1.18it/s][2025-04-24 07:51:04] (step=0019375) Train Loss: 5.4656, Train Steps/Sec: 1.17 + 83%|███████████████████████████████████████████████████████████████████████████████████████████████████████████▌ | 19399/23458 [5:40:49<57:15, 1.18it/s][2025-04-24 07:51:25] (step=0019400) Train Loss: 5.4915, Train Steps/Sec: 1.16 + 83%|███████████████████████████████████████████████████████████████████████████████████████████████████████████▋ | 19424/23458 [5:41:10<58:29, 1.15it/s][2025-04-24 07:51:46] (step=0019425) Train Loss: 5.5224, Train Steps/Sec: 1.17 + 83%|███████████████████████████████████████████████████████████████████████████████████████████████████████████▊ | 19449/23458 [5:41:32<57:28, 1.16it/s][2025-04-24 07:52:08] (step=0019450) Train Loss: 5.5405, Train Steps/Sec: 1.17 + 83%|███████████████████████████████████████████████████████████████████████████████████████████████████████████▉ | 19474/23458 [5:41:53<56:38, 1.17it/s][2025-04-24 07:52:29] (step=0019475) Train Loss: 5.4621, Train Steps/Sec: 1.17 + 83%|████████████████████████████████████████████████████████████████████████████████████████████████████████████ | 19499/23458 [5:42:14<55:43, 1.18it/s][2025-04-24 07:52:51] (step=0019500) Train Loss: 5.4519, Train Steps/Sec: 1.17 + 83%|████████████████████████████████████████████████████████████████████████████████████████████████████████████▏ | 19524/23458 [5:42:36<57:00, 1.15it/s][2025-04-24 07:53:12] (step=0019525) Train Loss: 5.5039, Train Steps/Sec: 1.17 + 83%|████████████████████████████████████████████████████████████████████████████████████████████████████████████▎ | 19549/23458 [5:42:57<55:55, 1.17it/s][2025-04-24 07:53:33] (step=0019550) Train Loss: 5.4655, Train Steps/Sec: 1.17 + 83%|████████████████████████████████████████████████████████████████████████████████████████████████████████████▍ | 19574/23458 [5:43:19<54:50, 1.18it/s][2025-04-24 07:53:55] (step=0019575) Train Loss: 5.5005, Train Steps/Sec: 1.17 + 84%|████████████████████████████████████████████████████████████████████████████████████████████████████████████▌ | 19599/23458 [5:43:40<54:14, 1.19it/s][2025-04-24 07:54:16] (step=0019600) Train Loss: 5.4640, Train Steps/Sec: 1.17 + 84%|████████████████████████████████████████████████████████████████████████████████████████████████████████████▊ | 19624/23458 [5:44:01<55:38, 1.15it/s][2025-04-24 07:54:38] (step=0019625) Train Loss: 5.5170, Train Steps/Sec: 1.17 + 84%|████████████████████████████████████████████████████████████████████████████████████████████████████████████▉ | 19649/23458 [5:44:23<54:26, 1.17it/s][2025-04-24 07:54:59] (step=0019650) Train Loss: 5.5773, Train Steps/Sec: 1.17 + 84%|█████████████████████████████████████████████████████████████████████████████████████████████████████████████ | 19674/23458 [5:44:45<54:06, 1.17it/s][2025-04-24 07:55:21] (step=0019675) Train Loss: 5.4250, Train Steps/Sec: 1.12 + 84%|█████████████████████████████████████████████████████████████████████████████████████████████████████████████▏ | 19699/23458 [5:45:07<55:25, 1.13it/s][2025-04-24 07:55:43] (step=0019700) Train Loss: 5.5086, Train Steps/Sec: 1.12 + 84%|█████████████████████████████████████████████████████████████████████████████████████████████████████████████▎ | 19724/23458 [5:45:29<59:09, 1.05it/s][2025-04-24 07:56:06] (step=0019725) Train Loss: 5.4626, Train Steps/Sec: 1.13 + 84%|█████████████████████████████████████████████████████████████████████████████████████████████████████████████▍ | 19749/23458 [5:45:52<52:57, 1.17it/s][2025-04-24 07:56:28] (step=0019750) Train Loss: 5.4114, Train Steps/Sec: 1.13 + 84%|█████████████████████████████████████████████████████████████████████████████████████████████████████████████▌ | 19774/23458 [5:46:13<52:26, 1.17it/s][2025-04-24 07:56:49] (step=0019775) Train Loss: 5.5689, Train Steps/Sec: 1.17 + 84%|█████████████████████████████████████████████████████████████████████████████████████████████████████████████▋ | 19799/23458 [5:46:34<51:30, 1.18it/s][2025-04-24 07:57:11] (step=0019800) Train Loss: 5.4083, Train Steps/Sec: 1.17 + 85%|█████████████████████████████████████████████████████████████████████████████████████████████████████████████▊ | 19824/23458 [5:46:57<55:32, 1.09it/s][2025-04-24 07:57:33] (step=0019825) Train Loss: 5.5481, Train Steps/Sec: 1.12 + 85%|█████████████████████████████████████████████████████████████████████████████████████████████████████████████▉ | 19849/23458 [5:47:18<51:30, 1.17it/s][2025-04-24 07:57:54] (step=0019850) Train Loss: 5.5007, Train Steps/Sec: 1.17 + 85%|██████████████████████████████████████████████████████████████████████████████████████████████████████████████▏ | 19874/23458 [5:47:39<50:51, 1.17it/s][2025-04-24 07:58:16] (step=0019875) Train Loss: 5.4171, Train Steps/Sec: 1.17 + 85%|██████████████████████████████████████████████████████████████████████████████████████████████████████████████▎ | 19899/23458 [5:48:01<50:18, 1.18it/s][2025-04-24 07:58:37] (step=0019900) Train Loss: 5.4718, Train Steps/Sec: 1.17 + 85%|██████████████████████████████████████████████████████████████████████████████████████████████████████████████▍ | 19924/23458 [5:48:22<51:15, 1.15it/s][2025-04-24 07:58:58] (step=0019925) Train Loss: 5.5123, Train Steps/Sec: 1.17 + 85%|██████████████████████████████████████████████████████████████████████████████████████████████████████████████▌ | 19949/23458 [5:48:44<50:10, 1.17it/s][2025-04-24 07:59:21] (step=0019950) Train Loss: 5.5131, Train Steps/Sec: 1.12 + 85%|██████████████████████████████████████████████████████████████████████████████████████████████████████████████▋ | 19974/23458 [5:49:07<49:55, 1.16it/s][2025-04-24 07:59:43] (step=0019975) Train Loss: 5.5085, Train Steps/Sec: 1.12 + 85%|██████████████████████████████████████████████████████████████████████████████████████████████████████████████▊ | 19999/23458 [5:49:28<48:52, 1.18it/s][2025-04-24 08:00:04] (step=0020000) Train Loss: 5.3823, Train Steps/Sec: 1.17 +You are using a model of type blip-2 to instantiate a model of type instructblip. This is not supported for all configurations of models and can yield errors. +[2025-04-24 08:00:05] text_config is None. Initializing the text config with default values (`OPTConfig`). +model_name_or_path:/tmp/haozhezhao/model/blip2-flan-t5-xl +image_place_holder: +tokenizer length after expend 32101 +tokenizer length before expend 32100 +100%|████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████| 4/4 [04:04<00:00, 61.07s/it] +[2025-04-24 08:05:17] Finish Eval in 20000 steps...█████████████████████████████████████████████████████████████████████████████████████| 4/4 [04:03<00:00, 60.71s/it] +processing left-padding... +processing left-padding... +processing left-padding... +processing left-padding... +[2025-04-24 08:05:35] Saved checkpoint to checkpoint/Afteri2i_100kckpt_LLAVAT5_empty_Fixed_just_segment_pretrain_stage3_dreambench_recap_400k_t2i_400k_flux_200k_midjourney_150k_recovery_150k_3e4_epoch2_no_rotated/000-GPT-XL/checkpoints/0020000.pt +[2025-04-24 08:05:37] Removed old checkpoint: checkpoint/Afteri2i_100kckpt_LLAVAT5_empty_Fixed_just_segment_pretrain_stage3_dreambench_recap_400k_t2i_400k_flux_200k_midjourney_150k_recovery_150k_3e4_epoch2_no_rotated/000-GPT-XL/checkpoints/0018000.pt + 85%|██████████████████████████████████████████████████████████████████████████████████████████████████████████████▉ | 20024/23458 [5:55:22<50:57, 1.12it/s][2025-04-24 08:05:59] (step=0020025) Train Loss: 5.5133, Train Steps/Sec: 0.07 + 85%|███████████████████████████████████████████████████████████████████████████████████████████████████████████████ | 20049/23458 [5:55:45<49:01, 1.16it/s][2025-04-24 08:06:21] (step=0020050) Train Loss: 5.5377, Train Steps/Sec: 1.12 + 86%|███████████████████████████████████████████████████████████████████████████████████████████████████████████████▏ | 20074/23458 [5:56:06<47:45, 1.18it/s][2025-04-24 08:06:42] (step=0020075) Train Loss: 5.4973, Train Steps/Sec: 1.17 + 86%|███████████████████████████████████████████████████████████████████████████████████████████████████████████████▍ | 20099/23458 [5:56:27<47:09, 1.19it/s][2025-04-24 08:07:04] (step=0020100) Train Loss: 5.4359, Train Steps/Sec: 1.17 + 86%|███████████████████████████████████████████████████████████████████████████████████████████████████████████████▌ | 20124/23458 [5:56:49<48:22, 1.15it/s][2025-04-24 08:07:25] (step=0020125) Train Loss: 5.4953, Train Steps/Sec: 1.17 + 86%|███████████████████████████████████████████████████████████████████████████████████████████████████████████████▋ | 20149/23458 [5:57:10<47:10, 1.17it/s][2025-04-24 08:07:46] (step=0020150) Train Loss: 5.3692, Train Steps/Sec: 1.17 + 86%|███████████████████████████████████████████████████████████████████████████████████████████████████████████████▊ | 20174/23458 [5:57:31<46:26, 1.18it/s][2025-04-24 08:08:08] (step=0020175) Train Loss: 5.4092, Train Steps/Sec: 1.17 + 86%|███████████████████████████████████████████████████████████████████████████████████████████████████████████████▉ | 20199/23458 [5:57:53<46:07, 1.18it/s][2025-04-24 08:08:29] (step=0020200) Train Loss: 5.5509, Train Steps/Sec: 1.16 + 86%|████████████████████████████████████████████████████████████████████████████████████████████████████████████████ | 20224/23458 [5:58:14<46:59, 1.15it/s][2025-04-24 08:08:51] (step=0020225) Train Loss: 5.4333, Train Steps/Sec: 1.17 + 86%|████████████████████████████████████████████████████████████████████████████████████████████████████████████████▏ | 20249/23458 [5:58:36<45:46, 1.17it/s][2025-04-24 08:09:12] (step=0020250) Train Loss: 5.3948, Train Steps/Sec: 1.17 + 86%|████████████████████████████████████████████████████████████████████████████████████████████████████████████████▎ | 20274/23458 [5:58:57<45:04, 1.18it/s][2025-04-24 08:09:33] (step=0020275) Train Loss: 5.5149, Train Steps/Sec: 1.17 + 87%|████████████████████████████████████████████████████████████████████████████████████████████████████████████████▍ | 20299/23458 [5:59:18<44:33, 1.18it/s][2025-04-24 08:09:55] (step=0020300) Train Loss: 5.4302, Train Steps/Sec: 1.16 + 87%|████████████████████████████████████████████████████████████████████████████████████████████████████████████████▋ | 20324/23458 [5:59:41<48:46, 1.07it/s][2025-04-24 08:10:17] (step=0020325) Train Loss: 5.4272, Train Steps/Sec: 1.12 + 87%|████████████████████████████████████████████████████████████████████████████████████████████████████████████████▊ | 20349/23458 [6:00:02<44:21, 1.17it/s][2025-04-24 08:10:38] (step=0020350) Train Loss: 5.5617, Train Steps/Sec: 1.17 + 87%|████████████████████████████████████████████████████████████████████████████████████████████████████████████████▉ | 20374/23458 [6:00:24<43:42, 1.18it/s][2025-04-24 08:11:01] (step=0020375) Train Loss: 5.4860, Train Steps/Sec: 1.13 + 87%|█████████████████████████████████████████████████████████████████████████████████████████████████████████████████ | 20399/23458 [6:00:47<43:11, 1.18it/s][2025-04-24 08:11:24] (step=0020400) Train Loss: 5.4642, Train Steps/Sec: 1.08 + 87%|█████████████████████████████████████████████████████████████████████████████████████████████████████████████████▏ | 20424/23458 [6:01:09<43:50, 1.15it/s][2025-04-24 08:11:45] (step=0020425) Train Loss: 5.5198, Train Steps/Sec: 1.17 + 87%|█████████████████████████████████████████████████████████████████████████████████████████████████████████████████▎ | 20449/23458 [6:01:30<42:58, 1.17it/s][2025-04-24 08:12:06] (step=0020450) Train Loss: 5.5216, Train Steps/Sec: 1.17 + 87%|█████████████████████████████████████████████████████████████████████████████████████████████████████████████████▍ | 20474/23458 [6:01:52<42:16, 1.18it/s][2025-04-24 08:12:28] (step=0020475) Train Loss: 5.5134, Train Steps/Sec: 1.17 + 87%|█████████████████████████████████████████████████████████████████████████████████████████████████████████████████▌ | 20499/23458 [6:02:14<41:48, 1.18it/s][2025-04-24 08:12:50] (step=0020500) Train Loss: 5.4728, Train Steps/Sec: 1.11 + 87%|█████████████████████████████████████████████████████████████████████████████████████████████████████████████████▋ | 20524/23458 [6:02:36<42:35, 1.15it/s][2025-04-24 08:13:12] (step=0020525) Train Loss: 5.4495, Train Steps/Sec: 1.17 + 88%|█████████████████████████████████████████████████████████████████████████████████████████████████████████████████▉ | 20549/23458 [6:02:57<43:53, 1.10it/s][2025-04-24 08:13:33] (step=0020550) Train Loss: 5.5141, Train Steps/Sec: 1.16 + 88%|██████████████████████████████████████████████████████████████████████████████████████████████████████████████████ | 20574/23458 [6:03:19<40:51, 1.18it/s][2025-04-24 08:13:55] (step=0020575) Train Loss: 5.5725, Train Steps/Sec: 1.17 + 88%|██████████████████████████████████████████████████████████████████████████████████████████████████████████████████▏ | 20599/23458 [6:03:40<40:13, 1.18it/s][2025-04-24 08:14:16] (step=0020600) Train Loss: 5.4600, Train Steps/Sec: 1.17 + 88%|██████████████████████████████████████████████████████████████████████████████████████████████████████████████████▎ | 20624/23458 [6:04:01<41:08, 1.15it/s][2025-04-24 08:14:38] (step=0020625) Train Loss: 5.3859, Train Steps/Sec: 1.17 + 88%|██████████████████████████████████████████████████████████████████████████████████████████████████████████████████▍ | 20649/23458 [6:04:23<39:59, 1.17it/s][2025-04-24 08:14:59] (step=0020650) Train Loss: 5.4379, Train Steps/Sec: 1.17 + 88%|██████████████████████████████████████████████████████████████████████████████████████████████████████████████████▌ | 20674/23458 [6:04:46<39:28, 1.18it/s][2025-04-24 08:15:22] (step=0020675) Train Loss: 5.4990, Train Steps/Sec: 1.07 + 88%|██████████████████████████████████████████████████████████████████████████████████████████████████████████████████▋ | 20699/23458 [6:05:07<38:50, 1.18it/s][2025-04-24 08:15:44] (step=0020700) Train Loss: 5.4891, Train Steps/Sec: 1.17 + 88%|██████████████████████████████████████████████████████████████████████████████████████████████████████████████████▊ | 20724/23458 [6:05:29<39:35, 1.15it/s][2025-04-24 08:16:05] (step=0020725) Train Loss: 5.4595, Train Steps/Sec: 1.17 + 88%|██████████████████████████████████████████████████████████████████████████████████████████████████████████████████▉ | 20749/23458 [6:05:50<38:38, 1.17it/s][2025-04-24 08:16:26] (step=0020750) Train Loss: 5.4715, Train Steps/Sec: 1.17 + 89%|███████████████████████████████████████████████████████████████████████████████████████████████████████████████████▏ | 20774/23458 [6:06:13<38:49, 1.15it/s][2025-04-24 08:16:49] (step=0020775) Train Loss: 5.5092, Train Steps/Sec: 1.12 + 89%|███████████████████████████████████████████████████████████████████████████████████████████████████████████████████▎ | 20799/23458 [6:06:34<37:16, 1.19it/s][2025-04-24 08:17:10] (step=0020800) Train Loss: 5.5078, Train Steps/Sec: 1.17 + 89%|███████████████████████████████████████████████████████████████████████████████████████████████████████████████████▍ | 20824/23458 [6:06:55<38:05, 1.15it/s][2025-04-24 08:17:31] (step=0020825) Train Loss: 5.5520, Train Steps/Sec: 1.17 + 89%|███████████████████████████████████████████████████████████████████████████████████████████████████████████████████▌ | 20849/23458 [6:07:17<37:19, 1.17it/s][2025-04-24 08:17:53] (step=0020850) Train Loss: 5.4424, Train Steps/Sec: 1.17 + 89%|███████████████████████████████████████████████████████████████████████████████████████████████████████████████████▋ | 20874/23458 [6:07:38<36:40, 1.17it/s][2025-04-24 08:18:14] (step=0020875) Train Loss: 5.5171, Train Steps/Sec: 1.17 + 89%|███████████████████████████████████████████████████████████████████████████████████████████████████████████████████▊ | 20899/23458 [6:07:59<36:00, 1.18it/s][2025-04-24 08:18:36] (step=0020900) Train Loss: 5.5085, Train Steps/Sec: 1.16 + 89%|███████████████████████████████████████████████████████████████████████████████████████████████████████████████████▉ | 20924/23458 [6:08:21<36:48, 1.15it/s][2025-04-24 08:18:57] (step=0020925) Train Loss: 5.4829, Train Steps/Sec: 1.17 + 89%|████████████████████████████████████████████████████████████████████████████████████████████████████████████████████ | 20949/23458 [6:08:42<35:51, 1.17it/s][2025-04-24 08:19:18] (step=0020950) Train Loss: 5.5743, Train Steps/Sec: 1.17 + 89%|████████████████████████████████████████████████████████████████████████████████████████████████████████████████████▏ | 20974/23458 [6:09:04<35:11, 1.18it/s][2025-04-24 08:19:40] (step=0020975) Train Loss: 5.4730, Train Steps/Sec: 1.17 + 90%|████████████████████████████████████████████████████████████████████████████████████████████████████████████████████▎ | 20999/23458 [6:09:26<34:36, 1.18it/s][2025-04-24 08:20:02] (step=0021000) Train Loss: 5.4709, Train Steps/Sec: 1.12 + 90%|████████████████████████████████████████████████████████████████████████████████████████████████████████████████████▌ | 21024/23458 [6:09:48<35:40, 1.14it/s][2025-04-24 08:20:24] (step=0021025) Train Loss: 5.4145, Train Steps/Sec: 1.13 + 90%|████████████████████████████████████████████████████████████████████████████████████████████████████████████████████▋ | 21049/23458 [6:10:11<36:21, 1.10it/s][2025-04-24 08:20:48] (step=0021050) Train Loss: 5.5274, Train Steps/Sec: 1.08 + 90%|████████████████████████████████████████████████████████████████████████████████████████████████████████████████████▊ | 21074/23458 [6:10:33<33:44, 1.18it/s][2025-04-24 08:21:09] (step=0021075) Train Loss: 5.4070, Train Steps/Sec: 1.17 + 90%|████████████████████████████████████████████████████████████████████████████████████████████████████████████████████▉ | 21099/23458 [6:10:54<33:12, 1.18it/s][2025-04-24 08:21:30] (step=0021100) Train Loss: 5.4601, Train Steps/Sec: 1.17 + 90%|█████████████████████████████████████████████████████████████████████████████████████████████████████████████████████ | 21124/23458 [6:11:15<33:51, 1.15it/s][2025-04-24 08:21:52] (step=0021125) Train Loss: 5.4321, Train Steps/Sec: 1.17 + 90%|█████████████████████████████████████████████████████████████████████████████████████████████████████████████████████▏ | 21149/23458 [6:11:38<32:59, 1.17it/s][2025-04-24 08:22:14] (step=0021150) Train Loss: 5.4045, Train Steps/Sec: 1.13 + 90%|█████████████████████████████████████████████████████████████████████████████████████████████████████████████████████▎ | 21174/23458 [6:11:59<32:23, 1.18it/s][2025-04-24 08:22:35] (step=0021175) Train Loss: 5.4734, Train Steps/Sec: 1.17 + 90%|█████████████████████████████████████████████████████████████████████████████████████████████████████████████████████▍ | 21199/23458 [6:12:20<31:51, 1.18it/s][2025-04-24 08:22:57] (step=0021200) Train Loss: 5.5089, Train Steps/Sec: 1.17 + 90%|█████████████████████████████████████████████████████████████████████████████████████████████████████████████████████▌ | 21224/23458 [6:12:42<32:18, 1.15it/s][2025-04-24 08:23:18] (step=0021225) Train Loss: 5.5798, Train Steps/Sec: 1.17 + 91%|█████████████████████████████████████████████████████████████████████████████████████████████████████████████████████▊ | 21249/23458 [6:13:03<31:29, 1.17it/s][2025-04-24 08:23:39] (step=0021250) Train Loss: 5.4149, Train Steps/Sec: 1.17 + 91%|█████████████████████████████████████████████████████████████████████████████████████████████████████████████████████▉ | 21274/23458 [6:13:24<30:55, 1.18it/s][2025-04-24 08:24:01] (step=0021275) Train Loss: 5.4229, Train Steps/Sec: 1.17 + 91%|██████████████████████████████████████████████████████████████████████████████████████████████████████████████████████ | 21299/23458 [6:13:46<30:17, 1.19it/s][2025-04-24 08:24:22] (step=0021300) Train Loss: 5.4381, Train Steps/Sec: 1.17 + 91%|██████████████████████████████████████████████████████████████████████████████████████████████████████████████████████▏ | 21324/23458 [6:14:08<31:00, 1.15it/s][2025-04-24 08:24:44] (step=0021325) Train Loss: 5.5042, Train Steps/Sec: 1.12 + 91%|██████████████████████████████████████████████████████████████████████████████████████████████████████████████████████▎ | 21349/23458 [6:14:30<30:03, 1.17it/s][2025-04-24 08:25:06] (step=0021350) Train Loss: 5.5044, Train Steps/Sec: 1.17 + 91%|██████████████████████████████████████████████████████████████████████████████████████████████████████████████████████▍ | 21374/23458 [6:14:51<29:40, 1.17it/s][2025-04-24 08:25:27] (step=0021375) Train Loss: 5.4743, Train Steps/Sec: 1.17 + 91%|██████████████████████████████████████████████████████████████████████████████████████████████████████████████████████▌ | 21399/23458 [6:15:13<28:54, 1.19it/s][2025-04-24 08:25:50] (step=0021400) Train Loss: 5.4124, Train Steps/Sec: 1.11 + 91%|██████████████████████████████████████████████████████████████████████████████████████████████████████████████████████▋ | 21424/23458 [6:15:35<29:28, 1.15it/s][2025-04-24 08:26:11] (step=0021425) Train Loss: 5.4300, Train Steps/Sec: 1.17 + 91%|██████████████████████████████████████████████████████████████████████████████████████████████████████████████████████▊ | 21449/23458 [6:15:56<28:52, 1.16it/s][2025-04-24 08:26:32] (step=0021450) Train Loss: 5.5121, Train Steps/Sec: 1.17 + 92%|███████████████████████████████████████████████████████████████████████████████████████████████████████████████████████ | 21474/23458 [6:16:18<28:11, 1.17it/s][2025-04-24 08:26:54] (step=0021475) Train Loss: 5.4615, Train Steps/Sec: 1.17 + 92%|███████████████████████████████████████████████████████████████████████████████████████████████████████████████████████▏ | 21499/23458 [6:16:40<29:24, 1.11it/s][2025-04-24 08:27:16] (step=0021500) Train Loss: 5.4793, Train Steps/Sec: 1.11 + 92%|███████████████████████████████████████████████████████████████████████████████████████████████████████████████████████▎ | 21524/23458 [6:17:01<27:59, 1.15it/s][2025-04-24 08:27:38] (step=0021525) Train Loss: 5.4409, Train Steps/Sec: 1.17 + 92%|███████████████████████████████████████████████████████████████████████████████████████████████████████████████████████▍ | 21549/23458 [6:17:23<27:17, 1.17it/s][2025-04-24 08:27:59] (step=0021550) Train Loss: 5.5200, Train Steps/Sec: 1.17 + 92%|███████████████████████████████████████████████████████████████████████████████████████████████████████████████████████▌ | 21574/23458 [6:17:44<26:46, 1.17it/s][2025-04-24 08:28:20] (step=0021575) Train Loss: 5.4614, Train Steps/Sec: 1.17 + 92%|███████████████████████████████████████████████████████████████████████████████████████████████████████████████████████▋ | 21599/23458 [6:18:05<26:04, 1.19it/s][2025-04-24 08:28:42] (step=0021600) Train Loss: 5.5253, Train Steps/Sec: 1.17 + 92%|███████████████████████████████████████████████████████████████████████████████████████████████████████████████████████▊ | 21624/23458 [6:18:27<26:43, 1.14it/s][2025-04-24 08:29:03] (step=0021625) Train Loss: 5.4831, Train Steps/Sec: 1.17 + 92%|███████████████████████████████████████████████████████████████████████████████████████████████████████████████████████▉ | 21649/23458 [6:18:49<26:01, 1.16it/s][2025-04-24 08:29:26] (step=0021650) Train Loss: 5.4975, Train Steps/Sec: 1.12 + 92%|████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████ | 21674/23458 [6:19:11<25:27, 1.17it/s][2025-04-24 08:29:47] (step=0021675) Train Loss: 5.4511, Train Steps/Sec: 1.16 + 93%|████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████▎ | 21699/23458 [6:19:34<32:24, 1.11s/it][2025-04-24 08:30:11] (step=0021700) Train Loss: 5.5029, Train Steps/Sec: 1.04 + 93%|████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████▍ | 21724/23458 [6:19:57<25:25, 1.14it/s][2025-04-24 08:30:33] (step=0021725) Train Loss: 5.4794, Train Steps/Sec: 1.16 + 93%|████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████▌ | 21749/23458 [6:20:18<24:20, 1.17it/s][2025-04-24 08:30:54] (step=0021750) Train Loss: 5.4671, Train Steps/Sec: 1.17 + 93%|████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████▋ | 21774/23458 [6:20:39<23:50, 1.18it/s][2025-04-24 08:31:15] (step=0021775) Train Loss: 5.5973, Train Steps/Sec: 1.17 + 93%|████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████▊ | 21799/23458 [6:21:01<25:47, 1.07it/s][2025-04-24 08:31:38] (step=0021800) Train Loss: 5.4924, Train Steps/Sec: 1.12 + 93%|████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████▉ | 21824/23458 [6:21:23<23:47, 1.14it/s][2025-04-24 08:31:59] (step=0021825) Train Loss: 5.4545, Train Steps/Sec: 1.16 + 93%|█████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████ | 21849/23458 [6:21:44<22:58, 1.17it/s][2025-04-24 08:32:21] (step=0021850) Train Loss: 5.4740, Train Steps/Sec: 1.17 + 93%|█████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████▏ | 21874/23458 [6:22:06<22:23, 1.18it/s][2025-04-24 08:32:42] (step=0021875) Train Loss: 5.4343, Train Steps/Sec: 1.17 + 93%|█████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████▎ | 21899/23458 [6:22:27<22:04, 1.18it/s][2025-04-24 08:33:03] (step=0021900) Train Loss: 5.4504, Train Steps/Sec: 1.16 + 93%|█████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████▍ | 21924/23458 [6:22:49<22:14, 1.15it/s][2025-04-24 08:33:25] (step=0021925) Train Loss: 5.5281, Train Steps/Sec: 1.17 + 94%|█████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████▋ | 21949/23458 [6:23:10<21:33, 1.17it/s][2025-04-24 08:33:46] (step=0021950) Train Loss: 5.4968, Train Steps/Sec: 1.17 + 94%|█████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████▊ | 21974/23458 [6:23:32<24:36, 1.01it/s][2025-04-24 08:34:09] (step=0021975) Train Loss: 5.4799, Train Steps/Sec: 1.12 + 94%|█████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████▉ | 21999/23458 [6:23:54<20:32, 1.18it/s][2025-04-24 08:34:30] (step=0022000) Train Loss: 5.4684, Train Steps/Sec: 1.17 +You are using a model of type blip-2 to instantiate a model of type instructblip. This is not supported for all configurations of models and can yield errors. +[2025-04-24 08:34:30] text_config is None. Initializing the text config with default values (`OPTConfig`). +model_name_or_path:/tmp/haozhezhao/model/blip2-flan-t5-xl +image_place_holder: +tokenizer length after expend 32101 +tokenizer length before expend 32100 +100%|████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████| 4/4 [04:05<00:00, 61.44s/it] +[2025-04-24 08:39:45] Finish Eval in 22000 steps...█████████████████████████████████████████████████████████████████████████████████████| 4/4 [04:05<00:00, 61.08s/it] +processing left-padding... +processing left-padding... +processing left-padding... +processing left-padding... +[2025-04-24 08:40:04] Saved checkpoint to checkpoint/Afteri2i_100kckpt_LLAVAT5_empty_Fixed_just_segment_pretrain_stage3_dreambench_recap_400k_t2i_400k_flux_200k_midjourney_150k_recovery_150k_3e4_epoch2_no_rotated/000-GPT-XL/checkpoints/0022000.pt +[2025-04-24 08:40:06] Removed old checkpoint: checkpoint/Afteri2i_100kckpt_LLAVAT5_empty_Fixed_just_segment_pretrain_stage3_dreambench_recap_400k_t2i_400k_flux_200k_midjourney_150k_recovery_150k_3e4_epoch2_no_rotated/000-GPT-XL/checkpoints/0020000.pt + 94%|██████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████ | 22024/23458 [6:29:51<21:09, 1.13it/s][2025-04-24 08:40:27] (step=0022025) Train Loss: 5.4818, Train Steps/Sec: 0.07 + 94%|██████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████▏ | 22049/23458 [6:30:12<20:07, 1.17it/s][2025-04-24 08:40:48] (step=0022050) Train Loss: 5.4911, Train Steps/Sec: 1.17 + 94%|██████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████▎ | 22074/23458 [6:30:34<19:39, 1.17it/s][2025-04-24 08:41:10] (step=0022075) Train Loss: 5.4607, Train Steps/Sec: 1.17 + 94%|██████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████▍ | 22099/23458 [6:30:55<19:10, 1.18it/s][2025-04-24 08:41:31] (step=0022100) Train Loss: 5.5138, Train Steps/Sec: 1.17 + 94%|██████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████▌ | 22124/23458 [6:31:17<19:19, 1.15it/s][2025-04-24 08:41:54] (step=0022125) Train Loss: 5.5113, Train Steps/Sec: 1.12 + 94%|██████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████▋ | 22149/23458 [6:31:39<18:38, 1.17it/s][2025-04-24 08:42:15] (step=0022150) Train Loss: 5.4715, Train Steps/Sec: 1.17 + 95%|██████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████▉ | 22174/23458 [6:32:00<18:11, 1.18it/s][2025-04-24 08:42:36] (step=0022175) Train Loss: 5.5074, Train Steps/Sec: 1.17 + 95%|███████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████ | 22199/23458 [6:32:22<17:50, 1.18it/s][2025-04-24 08:42:58] (step=0022200) Train Loss: 5.5234, Train Steps/Sec: 1.16 + 95%|███████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████▏ | 22224/23458 [6:32:44<20:22, 1.01it/s][2025-04-24 08:43:20] (step=0022225) Train Loss: 5.5074, Train Steps/Sec: 1.11 + 95%|███████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████▎ | 22249/23458 [6:33:06<17:14, 1.17it/s][2025-04-24 08:43:42] (step=0022250) Train Loss: 5.4257, Train Steps/Sec: 1.17 + 95%|███████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████▍ | 22274/23458 [6:33:27<16:45, 1.18it/s][2025-04-24 08:44:03] (step=0022275) Train Loss: 5.4651, Train Steps/Sec: 1.17 + 95%|███████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████▌ | 22299/23458 [6:33:49<16:34, 1.17it/s][2025-04-24 08:44:25] (step=0022300) Train Loss: 5.5406, Train Steps/Sec: 1.12 + 95%|███████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████▋ | 22324/23458 [6:34:11<16:25, 1.15it/s][2025-04-24 08:44:47] (step=0022325) Train Loss: 5.5015, Train Steps/Sec: 1.17 + 95%|███████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████▊ | 22349/23458 [6:34:33<15:47, 1.17it/s][2025-04-24 08:45:09] (step=0022350) Train Loss: 5.4660, Train Steps/Sec: 1.13 + 95%|███████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████▉ | 22374/23458 [6:34:56<15:24, 1.17it/s][2025-04-24 08:45:32] (step=0022375) Train Loss: 5.5126, Train Steps/Sec: 1.08 + 95%|████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████▏ | 22399/23458 [6:35:17<14:56, 1.18it/s][2025-04-24 08:45:54] (step=0022400) Train Loss: 5.5021, Train Steps/Sec: 1.16 + 96%|████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████▎ | 22424/23458 [6:35:39<15:06, 1.14it/s][2025-04-24 08:46:15] (step=0022425) Train Loss: 5.4861, Train Steps/Sec: 1.15 + 96%|████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████▍ | 22449/23458 [6:36:00<14:26, 1.16it/s][2025-04-24 08:46:37] (step=0022450) Train Loss: 5.5172, Train Steps/Sec: 1.17 + 96%|████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████▌ | 22474/23458 [6:36:23<13:55, 1.18it/s][2025-04-24 08:46:59] (step=0022475) Train Loss: 5.4981, Train Steps/Sec: 1.12 + 96%|████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████▋ | 22499/23458 [6:36:44<13:31, 1.18it/s][2025-04-24 08:47:20] (step=0022500) Train Loss: 5.5080, Train Steps/Sec: 1.17 + 96%|████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████▊ | 22524/23458 [6:37:05<13:35, 1.15it/s][2025-04-24 08:47:42] (step=0022525) Train Loss: 5.4191, Train Steps/Sec: 1.17 + 96%|████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████▉ | 22549/23458 [6:37:27<12:58, 1.17it/s][2025-04-24 08:48:03] (step=0022550) Train Loss: 5.4878, Train Steps/Sec: 1.17 + 96%|█████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████ | 22574/23458 [6:37:48<12:32, 1.17it/s][2025-04-24 08:48:24] (step=0022575) Train Loss: 5.4937, Train Steps/Sec: 1.17 + 96%|█████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████▏ | 22599/23458 [6:38:10<12:03, 1.19it/s][2025-04-24 08:48:46] (step=0022600) Train Loss: 5.5459, Train Steps/Sec: 1.17 + 96%|█████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████▍ | 22624/23458 [6:38:31<12:07, 1.15it/s][2025-04-24 08:49:07] (step=0022625) Train Loss: 5.4589, Train Steps/Sec: 1.17 + 97%|█████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████▌ | 22649/23458 [6:38:52<11:33, 1.17it/s][2025-04-24 08:49:29] (step=0022650) Train Loss: 5.3859, Train Steps/Sec: 1.17 + 97%|█████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████▋ | 22674/23458 [6:39:15<11:07, 1.17it/s][2025-04-24 08:49:51] (step=0022675) Train Loss: 5.5071, Train Steps/Sec: 1.12 + 97%|█████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████▊ | 22699/23458 [6:39:36<10:39, 1.19it/s][2025-04-24 08:50:13] (step=0022700) Train Loss: 5.4370, Train Steps/Sec: 1.16 + 97%|█████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████▉ | 22724/23458 [6:39:58<10:39, 1.15it/s][2025-04-24 08:50:34] (step=0022725) Train Loss: 5.5046, Train Steps/Sec: 1.17 + 97%|██████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████ | 22749/23458 [6:40:19<10:13, 1.16it/s][2025-04-24 08:50:55] (step=0022750) Train Loss: 5.5108, Train Steps/Sec: 1.17 + 97%|██████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████▏ | 22774/23458 [6:40:40<09:39, 1.18it/s][2025-04-24 08:51:17] (step=0022775) Train Loss: 5.5214, Train Steps/Sec: 1.17 + 97%|██████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████▎ | 22799/23458 [6:41:02<09:13, 1.19it/s][2025-04-24 08:51:38] (step=0022800) Train Loss: 5.5054, Train Steps/Sec: 1.16 + 97%|██████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████▍ | 22824/23458 [6:41:24<12:23, 1.17s/it][2025-04-24 08:52:00] (step=0022825) Train Loss: 5.4326, Train Steps/Sec: 1.12 + 97%|██████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████▋ | 22849/23458 [6:41:46<08:41, 1.17it/s][2025-04-24 08:52:22] (step=0022850) Train Loss: 5.4649, Train Steps/Sec: 1.17 + 98%|██████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████▊ | 22874/23458 [6:42:07<08:15, 1.18it/s][2025-04-24 08:52:43] (step=0022875) Train Loss: 5.3830, Train Steps/Sec: 1.17 + 98%|██████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████▉ | 22899/23458 [6:42:28<07:52, 1.18it/s][2025-04-24 08:53:05] (step=0022900) Train Loss: 5.5336, Train Steps/Sec: 1.17 + 98%|███████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████ | 22924/23458 [6:42:50<07:44, 1.15it/s][2025-04-24 08:53:26] (step=0022925) Train Loss: 5.3681, Train Steps/Sec: 1.17 + 98%|███████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████▏ | 22949/23458 [6:43:12<09:13, 1.09s/it][2025-04-24 08:53:49] (step=0022950) Train Loss: 5.4452, Train Steps/Sec: 1.11 + 98%|███████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████▎ | 22974/23458 [6:43:34<06:50, 1.18it/s][2025-04-24 08:54:10] (step=0022975) Train Loss: 5.4281, Train Steps/Sec: 1.17 + 98%|███████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████▍ | 22999/23458 [6:43:56<06:41, 1.14it/s][2025-04-24 08:54:32] (step=0023000) Train Loss: 5.4518, Train Steps/Sec: 1.12 + 98%|███████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████▌ | 23024/23458 [6:44:19<06:48, 1.06it/s][2025-04-24 08:54:55] (step=0023025) Train Loss: 5.5616, Train Steps/Sec: 1.09 + 98%|███████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████▋ | 23049/23458 [6:44:40<05:49, 1.17it/s][2025-04-24 08:55:17] (step=0023050) Train Loss: 5.5169, Train Steps/Sec: 1.17 + 98%|███████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████▊ | 23074/23458 [6:45:02<05:25, 1.18it/s][2025-04-24 08:55:38] (step=0023075) Train Loss: 5.4815, Train Steps/Sec: 1.17 + 98%|████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████ | 23099/23458 [6:45:23<05:03, 1.18it/s][2025-04-24 08:55:59] (step=0023100) Train Loss: 5.5091, Train Steps/Sec: 1.17 + 99%|████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████▏ | 23124/23458 [6:45:45<04:52, 1.14it/s][2025-04-24 08:56:22] (step=0023125) Train Loss: 5.4254, Train Steps/Sec: 1.13 + 99%|████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████▎ | 23149/23458 [6:46:07<04:24, 1.17it/s][2025-04-24 08:56:43] (step=0023150) Train Loss: 5.5435, Train Steps/Sec: 1.17 + 99%|████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████▍ | 23174/23458 [6:46:28<04:00, 1.18it/s][2025-04-24 08:57:04] (step=0023175) Train Loss: 5.5297, Train Steps/Sec: 1.17 + 99%|████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████▌ | 23199/23458 [6:46:49<03:38, 1.18it/s][2025-04-24 08:57:26] (step=0023200) Train Loss: 5.4291, Train Steps/Sec: 1.16 + 99%|████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████▋ | 23224/23458 [6:47:11<03:23, 1.15it/s][2025-04-24 08:57:47] (step=0023225) Train Loss: 5.5005, Train Steps/Sec: 1.17 + 99%|████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████▊ | 23249/23458 [6:47:32<03:00, 1.16it/s][2025-04-24 08:58:09] (step=0023250) Train Loss: 5.4685, Train Steps/Sec: 1.17 + 99%|████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████▉ | 23274/23458 [6:47:54<02:36, 1.18it/s][2025-04-24 08:58:30] (step=0023275) Train Loss: 5.4398, Train Steps/Sec: 1.17 + 99%|█████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████ | 23299/23458 [6:48:15<02:15, 1.18it/s][2025-04-24 08:58:51] (step=0023300) Train Loss: 5.5183, Train Steps/Sec: 1.16 + 99%|█████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████▎| 23324/23458 [6:48:38<01:57, 1.14it/s][2025-04-24 08:59:14] (step=0023325) Train Loss: 5.4721, Train Steps/Sec: 1.11 +100%|█████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████▍| 23349/23458 [6:48:59<01:33, 1.17it/s][2025-04-24 08:59:35] (step=0023350) Train Loss: 5.4324, Train Steps/Sec: 1.17 +100%|█████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████▌| 23374/23458 [6:49:20<01:11, 1.18it/s][2025-04-24 08:59:57] (step=0023375) Train Loss: 5.4557, Train Steps/Sec: 1.17 +100%|█████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████▋| 23399/23458 [6:49:42<00:49, 1.18it/s][2025-04-24 09:00:18] (step=0023400) Train Loss: 5.4655, Train Steps/Sec: 1.16 +100%|█████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████▊| 23424/23458 [6:50:03<00:29, 1.15it/s][2025-04-24 09:00:39] (step=0023425) Train Loss: 5.4009, Train Steps/Sec: 1.17 +100%|█████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████▉| 23449/23458 [6:50:25<00:07, 1.17it/s][2025-04-24 09:01:01] (step=0023450) Train Loss: 5.4469, Train Steps/Sec: 1.17 +100%|██████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████| 23458/23458 [6:50:33<00:00, 1.05s/it] +[2025-04-24 09:01:09] Beginning epoch 1... + 0%| | 16/23458 [00:16<5:35:00, 1.17it/s][2025-04-24 09:01:26] (step=0023475) Train Loss: 5.4170, Train Steps/Sec: 0.97 + 0%|▏ | 41/23458 [00:38<5:39:09, 1.15it/s][2025-04-24 09:01:48] (step=0023500) Train Loss: 5.3715, Train Steps/Sec: 1.15 + 0%|▎ | 66/23458 [01:00<5:45:22, 1.13it/s][2025-04-24 09:02:10] (step=0023525) Train Loss: 5.4157, Train Steps/Sec: 1.16 + 0%|▌ | 91/23458 [01:23<7:23:56, 1.14s/it][2025-04-24 09:02:33] (step=0023550) Train Loss: 5.3424, Train Steps/Sec: 1.09 + 0%|▋ | 116/23458 [01:44<5:31:15, 1.17it/s][2025-04-24 09:02:54] (step=0023575) Train Loss: 5.4245, Train Steps/Sec: 1.16 + 1%|▊ | 141/23458 [02:05<5:28:41, 1.18it/s][2025-04-24 09:03:16] (step=0023600) Train Loss: 5.4285, Train Steps/Sec: 1.16 + 1%|▉ | 166/23458 [02:28<5:38:12, 1.15it/s][2025-04-24 09:03:38] (step=0023625) Train Loss: 5.5040, Train Steps/Sec: 1.11 + 1%|█ | 191/23458 [02:49<5:33:19, 1.16it/s][2025-04-24 09:04:00] (step=0023650) Train Loss: 5.3243, Train Steps/Sec: 1.17 + 1%|█▏ | 216/23458 [03:14<8:39:42, 1.34s/it][2025-04-24 09:04:25] (step=0023675) Train Loss: 5.3999, Train Steps/Sec: 1.00 + 1%|█▎ | 241/23458 [03:37<5:27:46, 1.18it/s][2025-04-24 09:04:47] (step=0023700) Train Loss: 5.3756, Train Steps/Sec: 1.11 + 1%|█▍ | 266/23458 [03:58<5:39:35, 1.14it/s][2025-04-24 09:05:09] (step=0023725) Train Loss: 5.4116, Train Steps/Sec: 1.16 + 1%|█▋ | 291/23458 [04:20<5:34:33, 1.15it/s][2025-04-24 09:05:30] (step=0023750) Train Loss: 5.3427, Train Steps/Sec: 1.16 + 1%|█▊ | 316/23458 [04:43<5:57:17, 1.08it/s][2025-04-24 09:05:53] (step=0023775) Train Loss: 5.4304, Train Steps/Sec: 1.11 + 1%|█▉ | 341/23458 [05:04<5:25:52, 1.18it/s][2025-04-24 09:06:14] (step=0023800) Train Loss: 5.4047, Train Steps/Sec: 1.17 + 2%|██ | 366/23458 [05:25<5:34:47, 1.15it/s][2025-04-24 09:06:36] (step=0023825) Train Loss: 5.4554, Train Steps/Sec: 1.17 + 2%|██▏ | 391/23458 [05:47<5:28:37, 1.17it/s][2025-04-24 09:06:57] (step=0023850) Train Loss: 5.3423, Train Steps/Sec: 1.17 + 2%|██▎ | 416/23458 [06:08<5:26:55, 1.17it/s][2025-04-24 09:07:18] (step=0023875) Train Loss: 5.3771, Train Steps/Sec: 1.17 + 2%|██▍ | 441/23458 [06:30<5:22:51, 1.19it/s][2025-04-24 09:07:40] (step=0023900) Train Loss: 5.3327, Train Steps/Sec: 1.17 + 2%|██▌ | 466/23458 [06:51<5:34:06, 1.15it/s][2025-04-24 09:08:01] (step=0023925) Train Loss: 5.4558, Train Steps/Sec: 1.17 + 2%|██▊ | 491/23458 [07:12<5:27:23, 1.17it/s][2025-04-24 09:08:23] (step=0023950) Train Loss: 5.3731, Train Steps/Sec: 1.17 + 2%|██▉ | 516/23458 [07:35<5:29:13, 1.16it/s][2025-04-24 09:08:45] (step=0023975) Train Loss: 5.3961, Train Steps/Sec: 1.11 + 2%|███ | 541/23458 [07:57<5:29:49, 1.16it/s][2025-04-24 09:09:07] (step=0024000) Train Loss: 5.4169, Train Steps/Sec: 1.16 +You are using a model of type blip-2 to instantiate a model of type instructblip. This is not supported for all configurations of models and can yield errors. +[2025-04-24 09:09:07] text_config is None. Initializing the text config with default values (`OPTConfig`). +model_name_or_path:/tmp/haozhezhao/model/blip2-flan-t5-xl +image_place_holder: +tokenizer length after expend 32101 +tokenizer length before expend 32100 +100%|████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████| 4/4 [04:05<00:00, 61.42s/it] +[2025-04-24 09:14:21] Finish Eval in 24000 steps...█████████████████████████████████████████████████████████████████████████████████████| 4/4 [04:05<00:00, 60.99s/it] +processing left-padding... +processing left-padding... +processing left-padding... +processing left-padding... +[2025-04-24 09:14:40] Saved checkpoint to checkpoint/Afteri2i_100kckpt_LLAVAT5_empty_Fixed_just_segment_pretrain_stage3_dreambench_recap_400k_t2i_400k_flux_200k_midjourney_150k_recovery_150k_3e4_epoch2_no_rotated/000-GPT-XL/checkpoints/0024000.pt +[2025-04-24 09:14:42] Removed old checkpoint: checkpoint/Afteri2i_100kckpt_LLAVAT5_empty_Fixed_just_segment_pretrain_stage3_dreambench_recap_400k_t2i_400k_flux_200k_midjourney_150k_recovery_150k_3e4_epoch2_no_rotated/000-GPT-XL/checkpoints/0022000.pt + 2%|███▏ | 566/23458 [13:53<5:38:41, 1.13it/s][2025-04-24 09:15:03] (step=0024025) Train Loss: 5.3980, Train Steps/Sec: 0.07 + 3%|███▎ | 591/23458 [14:14<5:24:55, 1.17it/s][2025-04-24 09:15:24] (step=0024050) Train Loss: 5.3459, Train Steps/Sec: 1.17 + 3%|███▍ | 616/23458 [14:36<5:24:04, 1.17it/s][2025-04-24 09:15:46] (step=0024075) Train Loss: 5.3863, Train Steps/Sec: 1.17 + 3%|███▌ | 641/23458 [14:57<5:22:41, 1.18it/s][2025-04-24 09:16:07] (step=0024100) Train Loss: 5.3357, Train Steps/Sec: 1.17 + 3%|███▋ | 666/23458 [15:18<5:30:31, 1.15it/s][2025-04-24 09:16:28] (step=0024125) Train Loss: 5.3984, Train Steps/Sec: 1.17 + 3%|███▉ | 691/23458 [15:40<5:28:50, 1.15it/s][2025-04-24 09:16:50] (step=0024150) Train Loss: 5.3400, Train Steps/Sec: 1.17 + 3%|████ | 716/23458 [16:01<5:28:09, 1.16it/s][2025-04-24 09:17:11] (step=0024175) Train Loss: 5.4724, Train Steps/Sec: 1.16 + 3%|████▏ | 741/23458 [16:23<5:24:11, 1.17it/s][2025-04-24 09:17:33] (step=0024200) Train Loss: 5.3977, Train Steps/Sec: 1.15 + 3%|████▎ | 766/23458 [16:45<5:29:54, 1.15it/s][2025-04-24 09:17:55] (step=0024225) Train Loss: 5.4184, Train Steps/Sec: 1.17 + 3%|████▍ | 791/23458 [17:06<5:23:23, 1.17it/s][2025-04-24 09:18:16] (step=0024250) Train Loss: 5.3963, Train Steps/Sec: 1.17 + 3%|████▌ | 816/23458 [17:29<6:13:29, 1.01it/s][2025-04-24 09:18:39] (step=0024275) Train Loss: 5.3373, Train Steps/Sec: 1.07 + 4%|████▋ | 841/23458 [17:51<5:19:32, 1.18it/s][2025-04-24 09:19:01] (step=0024300) Train Loss: 5.4231, Train Steps/Sec: 1.17 + 4%|████▊ | 866/23458 [18:13<5:28:47, 1.15it/s][2025-04-24 09:19:23] (step=0024325) Train Loss: 5.4282, Train Steps/Sec: 1.13 + 4%|█████ | 891/23458 [18:36<5:26:26, 1.15it/s][2025-04-24 09:19:46] (step=0024350) Train Loss: 5.3997, Train Steps/Sec: 1.07 + 4%|█████▏ | 916/23458 [18:58<5:20:20, 1.17it/s][2025-04-24 09:20:08] (step=0024375) Train Loss: 5.3876, Train Steps/Sec: 1.17 + 4%|█████▎ | 941/23458 [19:19<5:16:11, 1.19it/s][2025-04-24 09:20:30] (step=0024400) Train Loss: 5.3416, Train Steps/Sec: 1.14 + 4%|█████▍ | 966/23458 [19:42<5:25:59, 1.15it/s][2025-04-24 09:20:52] (step=0024425) Train Loss: 5.3808, Train Steps/Sec: 1.10 + 4%|█████▌ | 991/23458 [20:05<5:21:04, 1.17it/s][2025-04-24 09:21:15] (step=0024450) Train Loss: 5.4180, Train Steps/Sec: 1.12 + 4%|█████▋ | 1016/23458 [20:26<5:16:27, 1.18it/s][2025-04-24 09:21:36] (step=0024475) Train Loss: 5.4024, Train Steps/Sec: 1.17 + 4%|█████▊ | 1041/23458 [20:47<5:16:52, 1.18it/s][2025-04-24 09:21:57] (step=0024500) Train Loss: 5.3979, Train Steps/Sec: 1.16 + 5%|█████▉ | 1066/23458 [21:09<5:26:36, 1.14it/s][2025-04-24 09:22:19] (step=0024525) Train Loss: 5.4117, Train Steps/Sec: 1.17 + 5%|██████ | 1091/23458 [21:30<5:18:19, 1.17it/s][2025-04-24 09:22:40] (step=0024550) Train Loss: 5.4733, Train Steps/Sec: 1.17 + 5%|██████▏ | 1116/23458 [21:51<5:16:33, 1.18it/s][2025-04-24 09:23:01] (step=0024575) Train Loss: 5.3784, Train Steps/Sec: 1.17 + 5%|██████▎ | 1141/23458 [22:13<5:14:11, 1.18it/s][2025-04-24 09:23:23] (step=0024600) Train Loss: 5.3431, Train Steps/Sec: 1.17 + 5%|██████▌ | 1166/23458 [22:34<5:23:28, 1.15it/s][2025-04-24 09:23:44] (step=0024625) Train Loss: 5.3395, Train Steps/Sec: 1.17 + 5%|██████▋ | 1191/23458 [22:56<5:19:16, 1.16it/s][2025-04-24 09:24:06] (step=0024650) Train Loss: 5.3325, Train Steps/Sec: 1.17 + 5%|██████▊ | 1216/23458 [23:18<5:15:17, 1.18it/s][2025-04-24 09:24:28] (step=0024675) Train Loss: 5.3717, Train Steps/Sec: 1.11 + 5%|██████▉ | 1241/23458 [23:40<5:13:16, 1.18it/s][2025-04-24 09:24:50] (step=0024700) Train Loss: 5.3978, Train Steps/Sec: 1.17 + 5%|███████ | 1266/23458 [24:01<5:22:18, 1.15it/s][2025-04-24 09:25:11] (step=0024725) Train Loss: 5.3836, Train Steps/Sec: 1.17 + 6%|███████▏ | 1291/23458 [24:22<5:15:19, 1.17it/s][2025-04-24 09:25:32] (step=0024750) Train Loss: 5.4446, Train Steps/Sec: 1.17 + 6%|███████▎ | 1316/23458 [24:44<5:13:46, 1.18it/s][2025-04-24 09:25:54] (step=0024775) Train Loss: 5.3656, Train Steps/Sec: 1.17 + 6%|███████▍ | 1341/23458 [25:05<5:11:01, 1.19it/s][2025-04-24 09:26:15] (step=0024800) Train Loss: 5.4777, Train Steps/Sec: 1.17 + 6%|███████▋ | 1366/23458 [25:26<5:18:59, 1.15it/s][2025-04-24 09:26:37] (step=0024825) Train Loss: 5.4036, Train Steps/Sec: 1.17 + 6%|███████▊ | 1391/23458 [25:48<5:15:02, 1.17it/s][2025-04-24 09:26:58] (step=0024850) Train Loss: 5.3854, Train Steps/Sec: 1.17 + 6%|███████▉ | 1416/23458 [26:09<5:11:52, 1.18it/s][2025-04-24 09:27:19] (step=0024875) Train Loss: 5.4243, Train Steps/Sec: 1.17 + 6%|████████ | 1441/23458 [26:31<5:10:16, 1.18it/s][2025-04-24 09:27:41] (step=0024900) Train Loss: 5.3952, Train Steps/Sec: 1.16 + 6%|████████▏ | 1466/23458 [26:52<5:20:39, 1.14it/s][2025-04-24 09:28:02] (step=0024925) Train Loss: 5.4075, Train Steps/Sec: 1.17 + 6%|████████▎ | 1491/23458 [27:14<5:12:58, 1.17it/s][2025-04-24 09:28:24] (step=0024950) Train Loss: 5.4776, Train Steps/Sec: 1.12 + 6%|████████▍ | 1516/23458 [27:36<5:32:05, 1.10it/s][2025-04-24 09:28:47] (step=0024975) Train Loss: 5.3747, Train Steps/Sec: 1.13 + 7%|████████▌ | 1541/23458 [28:00<5:14:26, 1.16it/s][2025-04-24 09:29:10] (step=0025000) Train Loss: 5.4054, Train Steps/Sec: 1.07 + 7%|████████▋ | 1566/23458 [28:22<5:16:31, 1.15it/s][2025-04-24 09:29:32] (step=0025025) Train Loss: 5.4568, Train Steps/Sec: 1.13 + 7%|████████▉ | 1591/23458 [28:44<5:15:19, 1.16it/s][2025-04-24 09:29:54] (step=0025050) Train Loss: 5.3641, Train Steps/Sec: 1.17 + 7%|█████████ | 1616/23458 [29:05<5:08:32, 1.18it/s][2025-04-24 09:30:15] (step=0025075) Train Loss: 5.4242, Train Steps/Sec: 1.17 + 7%|█████████▏ | 1641/23458 [29:27<5:14:56, 1.15it/s][2025-04-24 09:30:37] (step=0025100) Train Loss: 5.3522, Train Steps/Sec: 1.12 + 7%|█████████▎ | 1666/23458 [29:48<5:15:31, 1.15it/s][2025-04-24 09:30:59] (step=0025125) Train Loss: 5.3878, Train Steps/Sec: 1.17 + 7%|█████████▍ | 1691/23458 [30:11<5:10:50, 1.17it/s][2025-04-24 09:31:21] (step=0025150) Train Loss: 5.4051, Train Steps/Sec: 1.12 + 7%|█████████▌ | 1716/23458 [30:32<5:07:20, 1.18it/s][2025-04-24 09:31:42] (step=0025175) Train Loss: 5.3538, Train Steps/Sec: 1.17 + 7%|█████████▋ | 1741/23458 [30:54<5:05:57, 1.18it/s][2025-04-24 09:32:04] (step=0025200) Train Loss: 5.4114, Train Steps/Sec: 1.17 + 8%|█████████▊ | 1766/23458 [31:15<5:14:27, 1.15it/s][2025-04-24 09:32:25] (step=0025225) Train Loss: 5.3730, Train Steps/Sec: 1.17 + 8%|██████████ | 1791/23458 [31:36<5:08:11, 1.17it/s][2025-04-24 09:32:46] (step=0025250) Train Loss: 5.3789, Train Steps/Sec: 1.17 + 8%|██████████▏ | 1816/23458 [31:58<5:06:04, 1.18it/s][2025-04-24 09:33:08] (step=0025275) Train Loss: 5.3845, Train Steps/Sec: 1.17 + 8%|██████████▎ | 1841/23458 [32:19<5:06:14, 1.18it/s][2025-04-24 09:33:29] (step=0025300) Train Loss: 5.4134, Train Steps/Sec: 1.16 + 8%|██████████▍ | 1866/23458 [32:42<5:13:22, 1.15it/s][2025-04-24 09:33:52] (step=0025325) Train Loss: 5.3136, Train Steps/Sec: 1.11 + 8%|██████████▌ | 1891/23458 [33:03<5:10:00, 1.16it/s][2025-04-24 09:34:13] (step=0025350) Train Loss: 5.4474, Train Steps/Sec: 1.17 + 8%|██████████▋ | 1916/23458 [33:24<5:05:30, 1.18it/s][2025-04-24 09:34:34] (step=0025375) Train Loss: 5.3589, Train Steps/Sec: 1.17 + 8%|██████████▊ | 1941/23458 [33:46<5:04:30, 1.18it/s][2025-04-24 09:34:56] (step=0025400) Train Loss: 5.4045, Train Steps/Sec: 1.16 + 8%|██████████▉ | 1966/23458 [34:07<5:11:12, 1.15it/s][2025-04-24 09:35:17] (step=0025425) Train Loss: 5.4170, Train Steps/Sec: 1.17 + 8%|███████████ | 1991/23458 [34:29<5:06:39, 1.17it/s][2025-04-24 09:35:39] (step=0025450) Train Loss: 5.4153, Train Steps/Sec: 1.17 + 9%|███████████▎ | 2016/23458 [34:50<5:02:57, 1.18it/s][2025-04-24 09:36:00] (step=0025475) Train Loss: 5.3156, Train Steps/Sec: 1.17 + 9%|███████████▍ | 2041/23458 [35:11<5:01:15, 1.18it/s][2025-04-24 09:36:21] (step=0025500) Train Loss: 5.4037, Train Steps/Sec: 1.17 + 9%|███████████▌ | 2066/23458 [35:33<5:10:33, 1.15it/s][2025-04-24 09:36:43] (step=0025525) Train Loss: 5.4132, Train Steps/Sec: 1.17 + 9%|███████████▋ | 2091/23458 [35:54<5:03:32, 1.17it/s][2025-04-24 09:37:04] (step=0025550) Train Loss: 5.3476, Train Steps/Sec: 1.17 + 9%|███████████▊ | 2116/23458 [36:15<5:02:37, 1.18it/s][2025-04-24 09:37:26] (step=0025575) Train Loss: 5.4033, Train Steps/Sec: 1.17 + 9%|███████████▉ | 2141/23458 [36:38<5:00:17, 1.18it/s][2025-04-24 09:37:48] (step=0025600) Train Loss: 5.3420, Train Steps/Sec: 1.12 + 9%|████████████ | 2166/23458 [36:59<5:09:10, 1.15it/s][2025-04-24 09:38:09] (step=0025625) Train Loss: 5.4317, Train Steps/Sec: 1.17 + 9%|████████████▏ | 2191/23458 [37:22<5:51:51, 1.01it/s][2025-04-24 09:38:32] (step=0025650) Train Loss: 5.4007, Train Steps/Sec: 1.08 + 9%|████████████▍ | 2216/23458 [37:45<5:02:16, 1.17it/s][2025-04-24 09:38:55] (step=0025675) Train Loss: 5.3824, Train Steps/Sec: 1.13 + 10%|████████████▌ | 2241/23458 [38:06<4:58:44, 1.18it/s][2025-04-24 09:39:16] (step=0025700) Train Loss: 5.4513, Train Steps/Sec: 1.17 + 10%|████████████▋ | 2266/23458 [38:28<5:10:27, 1.14it/s][2025-04-24 09:39:38] (step=0025725) Train Loss: 5.3883, Train Steps/Sec: 1.12 + 10%|████████████▊ | 2291/23458 [38:50<5:02:23, 1.17it/s][2025-04-24 09:40:00] (step=0025750) Train Loss: 5.3391, Train Steps/Sec: 1.17 + 10%|████████████▉ | 2316/23458 [39:12<4:59:23, 1.18it/s][2025-04-24 09:40:22] (step=0025775) Train Loss: 5.4353, Train Steps/Sec: 1.12 + 10%|█████████████ | 2341/23458 [39:33<4:56:51, 1.19it/s][2025-04-24 09:40:44] (step=0025800) Train Loss: 5.3932, Train Steps/Sec: 1.17 + 10%|█████████████▏ | 2366/23458 [39:55<5:04:20, 1.16it/s][2025-04-24 09:41:05] (step=0025825) Train Loss: 5.3949, Train Steps/Sec: 1.17 + 10%|█████████████▎ | 2391/23458 [40:16<5:01:59, 1.16it/s][2025-04-24 09:41:26] (step=0025850) Train Loss: 5.3722, Train Steps/Sec: 1.17 + 10%|█████████████▍ | 2416/23458 [40:38<4:57:18, 1.18it/s][2025-04-24 09:41:49] (step=0025875) Train Loss: 5.4734, Train Steps/Sec: 1.12 + 10%|█████████████▋ | 2441/23458 [41:00<4:54:44, 1.19it/s][2025-04-24 09:42:10] (step=0025900) Train Loss: 5.4412, Train Steps/Sec: 1.17 + 11%|█████████████▊ | 2466/23458 [41:21<5:03:06, 1.15it/s][2025-04-24 09:42:31] (step=0025925) Train Loss: 5.3930, Train Steps/Sec: 1.17 + 11%|█████████████▉ | 2491/23458 [41:43<4:59:21, 1.17it/s][2025-04-24 09:42:53] (step=0025950) Train Loss: 5.3832, Train Steps/Sec: 1.17 + 11%|██████████████ | 2516/23458 [42:05<5:42:34, 1.02it/s][2025-04-24 09:43:15] (step=0025975) Train Loss: 5.3942, Train Steps/Sec: 1.12 + 11%|██████████████▏ | 2541/23458 [42:26<4:54:09, 1.19it/s][2025-04-24 09:43:36] (step=0026000) Train Loss: 5.3592, Train Steps/Sec: 1.16 +You are using a model of type blip-2 to instantiate a model of type instructblip. This is not supported for all configurations of models and can yield errors. +[2025-04-24 09:43:36] text_config is None. Initializing the text config with default values (`OPTConfig`). +model_name_or_path:/tmp/haozhezhao/model/blip2-flan-t5-xl +image_place_holder: +tokenizer length after expend 32101 +tokenizer length before expend 32100 +100%|████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████| 4/4 [03:59<00:00, 59.93s/it] +[2025-04-24 09:48:45] Finish Eval in 26000 steps...█████████████████████████████████████████████████████████████████████████████████████| 4/4 [03:59<00:00, 59.56s/it] +processing left-padding... +processing left-padding... +processing left-padding... +processing left-padding... +[2025-04-24 09:49:04] Saved checkpoint to checkpoint/Afteri2i_100kckpt_LLAVAT5_empty_Fixed_just_segment_pretrain_stage3_dreambench_recap_400k_t2i_400k_flux_200k_midjourney_150k_recovery_150k_3e4_epoch2_no_rotated/000-GPT-XL/checkpoints/0026000.pt +[2025-04-24 09:49:06] Removed old checkpoint: checkpoint/Afteri2i_100kckpt_LLAVAT5_empty_Fixed_just_segment_pretrain_stage3_dreambench_recap_400k_t2i_400k_flux_200k_midjourney_150k_recovery_150k_3e4_epoch2_no_rotated/000-GPT-XL/checkpoints/0024000.pt + 11%|██████████████▎ | 2566/23458 [48:18<5:09:15, 1.13it/s][2025-04-24 09:49:28] (step=0026025) Train Loss: 5.4082, Train Steps/Sec: 0.07 + 11%|██████████████▍ | 2591/23458 [48:39<4:56:18, 1.17it/s][2025-04-24 09:49:49] (step=0026050) Train Loss: 5.4153, Train Steps/Sec: 1.17 + 11%|██████████████▌ | 2616/23458 [49:00<4:55:12, 1.18it/s][2025-04-24 09:50:10] (step=0026075) Train Loss: 5.4127, Train Steps/Sec: 1.17 + 11%|██████████████▋ | 2641/23458 [49:22<4:53:51, 1.18it/s][2025-04-24 09:50:32] (step=0026100) Train Loss: 5.3668, Train Steps/Sec: 1.17 + 11%|██████████████▉ | 2666/23458 [49:43<4:59:59, 1.16it/s][2025-04-24 09:50:53] (step=0026125) Train Loss: 5.3799, Train Steps/Sec: 1.17 + 11%|███████████████ | 2691/23458 [50:04<4:55:48, 1.17it/s][2025-04-24 09:51:14] (step=0026150) Train Loss: 5.5288, Train Steps/Sec: 1.17 + 12%|███████████████▏ | 2716/23458 [50:26<4:53:02, 1.18it/s][2025-04-24 09:51:36] (step=0026175) Train Loss: 5.3914, Train Steps/Sec: 1.17 + 12%|███████████████▎ | 2741/23458 [50:47<4:51:00, 1.19it/s][2025-04-24 09:51:57] (step=0026200) Train Loss: 5.3718, Train Steps/Sec: 1.17 + 12%|███████████████▍ | 2766/23458 [51:08<5:00:23, 1.15it/s][2025-04-24 09:52:18] (step=0026225) Train Loss: 5.4055, Train Steps/Sec: 1.17 + 12%|███████████████▌ | 2791/23458 [51:31<4:57:52, 1.16it/s][2025-04-24 09:52:41] (step=0026250) Train Loss: 5.4038, Train Steps/Sec: 1.12 + 12%|███████████████▋ | 2816/23458 [51:52<4:51:12, 1.18it/s][2025-04-24 09:53:02] (step=0026275) Train Loss: 5.4092, Train Steps/Sec: 1.17 + 12%|███████████████▊ | 2841/23458 [52:14<4:51:14, 1.18it/s][2025-04-24 09:53:24] (step=0026300) Train Loss: 5.3703, Train Steps/Sec: 1.12 + 12%|████████████████ | 2866/23458 [52:38<5:28:54, 1.04it/s][2025-04-24 09:53:48] (step=0026325) Train Loss: 5.4083, Train Steps/Sec: 1.08 + 12%|████████████████▏ | 2891/23458 [52:59<4:53:07, 1.17it/s][2025-04-24 09:54:09] (step=0026350) Train Loss: 5.4313, Train Steps/Sec: 1.17 + 12%|████████████████▎ | 2916/23458 [53:21<4:49:42, 1.18it/s][2025-04-24 09:54:31] (step=0026375) Train Loss: 5.3828, Train Steps/Sec: 1.17 + 13%|████████████████▍ | 2941/23458 [53:42<4:49:46, 1.18it/s][2025-04-24 09:54:52] (step=0026400) Train Loss: 5.4450, Train Steps/Sec: 1.16 + 13%|████████████████▌ | 2966/23458 [54:04<4:57:38, 1.15it/s][2025-04-24 09:55:14] (step=0026425) Train Loss: 5.4536, Train Steps/Sec: 1.12 + 13%|████████████████▋ | 2991/23458 [54:27<4:53:38, 1.16it/s][2025-04-24 09:55:37] (step=0026450) Train Loss: 5.3820, Train Steps/Sec: 1.12 + 13%|████████████████▊ | 3016/23458 [54:48<4:48:22, 1.18it/s][2025-04-24 09:55:58] (step=0026475) Train Loss: 5.3449, Train Steps/Sec: 1.17 + 13%|████████████████▉ | 3041/23458 [55:10<4:49:08, 1.18it/s][2025-04-24 09:56:20] (step=0026500) Train Loss: 5.4351, Train Steps/Sec: 1.17 + 13%|█████████████████ | 3066/23458 [55:31<4:55:18, 1.15it/s][2025-04-24 09:56:41] (step=0026525) Train Loss: 5.4697, Train Steps/Sec: 1.17 + 13%|█████████████████▎ | 3091/23458 [55:52<4:51:32, 1.16it/s][2025-04-24 09:57:02] (step=0026550) Train Loss: 5.4127, Train Steps/Sec: 1.17 + 13%|█████████████████▍ | 3116/23458 [56:14<4:48:47, 1.17it/s][2025-04-24 09:57:24] (step=0026575) Train Loss: 5.3295, Train Steps/Sec: 1.17 + 13%|█████████████████▌ | 3141/23458 [56:36<4:47:10, 1.18it/s][2025-04-24 09:57:46] (step=0026600) Train Loss: 5.5068, Train Steps/Sec: 1.11 + 13%|█████████████████▋ | 3166/23458 [56:58<4:55:15, 1.15it/s][2025-04-24 09:58:08] (step=0026625) Train Loss: 5.4076, Train Steps/Sec: 1.17 + 14%|█████████████████▊ | 3191/23458 [57:19<4:50:33, 1.16it/s][2025-04-24 09:58:29] (step=0026650) Train Loss: 5.4158, Train Steps/Sec: 1.17 + 14%|█████████████████▉ | 3216/23458 [57:41<4:46:33, 1.18it/s][2025-04-24 09:58:51] (step=0026675) Train Loss: 5.4059, Train Steps/Sec: 1.12 + 14%|██████████████████ | 3241/23458 [58:03<4:45:42, 1.18it/s][2025-04-24 09:59:13] (step=0026700) Train Loss: 5.4661, Train Steps/Sec: 1.17 + 14%|██████████████████▏ | 3266/23458 [58:24<4:53:30, 1.15it/s][2025-04-24 09:59:34] (step=0026725) Train Loss: 5.3516, Train Steps/Sec: 1.17 + 14%|██████████████████▍ | 3291/23458 [58:46<4:47:07, 1.17it/s][2025-04-24 09:59:56] (step=0026750) Train Loss: 5.3949, Train Steps/Sec: 1.17 + 14%|██████████████████▌ | 3316/23458 [59:07<4:44:13, 1.18it/s][2025-04-24 10:00:17] (step=0026775) Train Loss: 5.4029, Train Steps/Sec: 1.17 + 14%|██████████████████▋ | 3341/23458 [59:28<4:43:28, 1.18it/s][2025-04-24 10:00:38] (step=0026800) Train Loss: 5.3295, Train Steps/Sec: 1.17 + 14%|██████████████████▊ | 3366/23458 [59:50<4:51:07, 1.15it/s][2025-04-24 10:01:00] (step=0026825) Train Loss: 5.4752, Train Steps/Sec: 1.17 + 14%|██████████████████▋ | 3391/23458 [1:00:11<4:46:54, 1.17it/s][2025-04-24 10:01:21] (step=0026850) Train Loss: 5.3200, Train Steps/Sec: 1.17 + 15%|██████████████████▊ | 3416/23458 [1:00:32<4:43:15, 1.18it/s][2025-04-24 10:01:43] (step=0026875) Train Loss: 5.3777, Train Steps/Sec: 1.17 + 15%|██████████████████▉ | 3441/23458 [1:00:55<5:30:44, 1.01it/s][2025-04-24 10:02:05] (step=0026900) Train Loss: 5.3604, Train Steps/Sec: 1.11 + 15%|███████████████████ | 3466/23458 [1:01:16<4:48:43, 1.15it/s][2025-04-24 10:02:26] (step=0026925) Train Loss: 5.4015, Train Steps/Sec: 1.17 + 15%|███████████████████▏ | 3491/23458 [1:01:38<4:58:00, 1.12it/s][2025-04-24 10:02:48] (step=0026950) Train Loss: 5.3886, Train Steps/Sec: 1.13 + 15%|███████████████████▎ | 3516/23458 [1:02:01<4:49:49, 1.15it/s][2025-04-24 10:03:11] (step=0026975) Train Loss: 5.4493, Train Steps/Sec: 1.12 + 15%|███████████████████▍ | 3541/23458 [1:02:23<4:40:02, 1.19it/s][2025-04-24 10:03:33] (step=0027000) Train Loss: 5.3256, Train Steps/Sec: 1.12 + 15%|███████████████████▌ | 3566/23458 [1:02:44<4:48:48, 1.15it/s][2025-04-24 10:03:54] (step=0027025) Train Loss: 5.4102, Train Steps/Sec: 1.17 + 15%|███████████████████▋ | 3591/23458 [1:03:06<4:44:33, 1.16it/s][2025-04-24 10:04:16] (step=0027050) Train Loss: 5.3140, Train Steps/Sec: 1.17 + 15%|███████████████████▉ | 3616/23458 [1:03:28<5:40:06, 1.03s/it][2025-04-24 10:04:38] (step=0027075) Train Loss: 5.3612, Train Steps/Sec: 1.13 + 16%|████████████████████ | 3641/23458 [1:03:49<4:38:25, 1.19it/s][2025-04-24 10:04:59] (step=0027100) Train Loss: 5.4387, Train Steps/Sec: 1.17 + 16%|████████████████████▏ | 3666/23458 [1:04:11<4:46:55, 1.15it/s][2025-04-24 10:05:21] (step=0027125) Train Loss: 5.3492, Train Steps/Sec: 1.17 + 16%|████████████████████▎ | 3691/23458 [1:04:32<4:41:44, 1.17it/s][2025-04-24 10:05:42] (step=0027150) Train Loss: 5.5132, Train Steps/Sec: 1.17 + 16%|████████████████████▍ | 3716/23458 [1:04:54<4:39:43, 1.18it/s][2025-04-24 10:06:04] (step=0027175) Train Loss: 5.4334, Train Steps/Sec: 1.12 + 16%|████████████████████▌ | 3741/23458 [1:05:16<4:36:40, 1.19it/s][2025-04-24 10:06:26] (step=0027200) Train Loss: 5.3170, Train Steps/Sec: 1.17 + 16%|████████████████████▋ | 3766/23458 [1:05:37<4:45:41, 1.15it/s][2025-04-24 10:06:47] (step=0027225) Train Loss: 5.4304, Train Steps/Sec: 1.17 + 16%|████████████████████▊ | 3791/23458 [1:05:58<4:41:29, 1.16it/s][2025-04-24 10:07:09] (step=0027250) Train Loss: 5.3896, Train Steps/Sec: 1.17 + 16%|████████████████████▉ | 3816/23458 [1:06:20<4:39:21, 1.17it/s][2025-04-24 10:07:30] (step=0027275) Train Loss: 5.3996, Train Steps/Sec: 1.17 + 16%|█████████████████████ | 3841/23458 [1:06:41<4:35:09, 1.19it/s][2025-04-24 10:07:51] (step=0027300) Train Loss: 5.4702, Train Steps/Sec: 1.17 + 16%|█████████████████████▎ | 3866/23458 [1:07:05<4:45:09, 1.15it/s][2025-04-24 10:08:15] (step=0027325) Train Loss: 5.4505, Train Steps/Sec: 1.07 + 17%|█████████████████████▍ | 3891/23458 [1:07:26<4:38:11, 1.17it/s][2025-04-24 10:08:36] (step=0027350) Train Loss: 5.4990, Train Steps/Sec: 1.17 + 17%|█████████████████████▌ | 3916/23458 [1:07:47<4:37:16, 1.17it/s][2025-04-24 10:08:57] (step=0027375) Train Loss: 5.3243, Train Steps/Sec: 1.17 + 17%|█████████████████████▋ | 3941/23458 [1:08:09<4:34:37, 1.18it/s][2025-04-24 10:09:19] (step=0027400) Train Loss: 5.4923, Train Steps/Sec: 1.17 + 17%|█████████████████████▊ | 3966/23458 [1:08:30<4:42:02, 1.15it/s][2025-04-24 10:09:40] (step=0027425) Train Loss: 5.3611, Train Steps/Sec: 1.17 + 17%|█████████████████████▉ | 3991/23458 [1:08:52<4:37:45, 1.17it/s][2025-04-24 10:10:02] (step=0027450) Train Loss: 5.3598, Train Steps/Sec: 1.17 + 17%|██████████████████████ | 4016/23458 [1:09:13<4:33:41, 1.18it/s][2025-04-24 10:10:23] (step=0027475) Train Loss: 5.3965, Train Steps/Sec: 1.17 + 17%|██████████████████████▏ | 4041/23458 [1:09:34<4:33:00, 1.19it/s][2025-04-24 10:10:44] (step=0027500) Train Loss: 5.3855, Train Steps/Sec: 1.17 + 17%|██████████████████████▎ | 4066/23458 [1:09:56<4:41:47, 1.15it/s][2025-04-24 10:11:06] (step=0027525) Train Loss: 5.3154, Train Steps/Sec: 1.17 + 17%|██████████████████████▍ | 4091/23458 [1:10:17<4:36:33, 1.17it/s][2025-04-24 10:11:27] (step=0027550) Train Loss: 5.3897, Train Steps/Sec: 1.17 + 18%|██████████████████████▋ | 4116/23458 [1:10:39<4:34:14, 1.18it/s][2025-04-24 10:11:49] (step=0027575) Train Loss: 5.3680, Train Steps/Sec: 1.12 + 18%|██████████████████████▊ | 4141/23458 [1:11:01<4:30:29, 1.19it/s][2025-04-24 10:12:11] (step=0027600) Train Loss: 5.4462, Train Steps/Sec: 1.17 + 18%|██████████████████████▉ | 4166/23458 [1:11:23<4:39:47, 1.15it/s][2025-04-24 10:12:33] (step=0027625) Train Loss: 5.3809, Train Steps/Sec: 1.13 + 18%|███████████████████████ | 4191/23458 [1:11:46<4:34:21, 1.17it/s][2025-04-24 10:12:56] (step=0027650) Train Loss: 5.4394, Train Steps/Sec: 1.08 + 18%|███████████████████████▏ | 4216/23458 [1:12:07<4:31:26, 1.18it/s][2025-04-24 10:13:17] (step=0027675) Train Loss: 5.4391, Train Steps/Sec: 1.17 + 18%|███████████████████████▎ | 4241/23458 [1:12:29<4:31:26, 1.18it/s][2025-04-24 10:13:39] (step=0027700) Train Loss: 5.3009, Train Steps/Sec: 1.17 + 18%|███████████████████████▍ | 4266/23458 [1:12:50<4:37:43, 1.15it/s][2025-04-24 10:14:00] (step=0027725) Train Loss: 5.3732, Train Steps/Sec: 1.17 + 18%|███████████████████████▌ | 4291/23458 [1:13:12<4:33:41, 1.17it/s][2025-04-24 10:14:23] (step=0027750) Train Loss: 5.4107, Train Steps/Sec: 1.12 + 18%|███████████████████████▋ | 4316/23458 [1:13:34<4:32:22, 1.17it/s][2025-04-24 10:14:44] (step=0027775) Train Loss: 5.4288, Train Steps/Sec: 1.17 + 19%|███████████████████████▊ | 4341/23458 [1:13:55<4:27:48, 1.19it/s][2025-04-24 10:15:05] (step=0027800) Train Loss: 5.3547, Train Steps/Sec: 1.17 + 19%|████████████████████████ | 4366/23458 [1:14:17<4:36:59, 1.15it/s][2025-04-24 10:15:27] (step=0027825) Train Loss: 5.3669, Train Steps/Sec: 1.17 + 19%|████████████████████████▏ | 4391/23458 [1:14:38<4:31:55, 1.17it/s][2025-04-24 10:15:48] (step=0027850) Train Loss: 5.4399, Train Steps/Sec: 1.17 + 19%|████████████████████████▎ | 4416/23458 [1:14:59<4:29:50, 1.18it/s][2025-04-24 10:16:09] (step=0027875) Train Loss: 5.4048, Train Steps/Sec: 1.17 + 19%|████████████████████████▍ | 4441/23458 [1:15:22<4:28:58, 1.18it/s][2025-04-24 10:16:32] (step=0027900) Train Loss: 5.4257, Train Steps/Sec: 1.11 + 19%|████████████████████████▌ | 4466/23458 [1:15:43<4:36:11, 1.15it/s][2025-04-24 10:16:53] (step=0027925) Train Loss: 5.4280, Train Steps/Sec: 1.17 + 19%|████████████████████████▋ | 4491/23458 [1:16:04<4:29:51, 1.17it/s][2025-04-24 10:17:14] (step=0027950) Train Loss: 5.3229, Train Steps/Sec: 1.17 + 19%|████████████████████████▊ | 4516/23458 [1:16:27<5:10:58, 1.02it/s][2025-04-24 10:17:37] (step=0027975) Train Loss: 5.4126, Train Steps/Sec: 1.12 + 19%|████████████████████████▉ | 4541/23458 [1:16:48<4:26:22, 1.18it/s][2025-04-24 10:17:58] (step=0028000) Train Loss: 5.3863, Train Steps/Sec: 1.16 +You are using a model of type blip-2 to instantiate a model of type instructblip. This is not supported for all configurations of models and can yield errors. +[2025-04-24 10:17:58] text_config is None. Initializing the text config with default values (`OPTConfig`). +model_name_or_path:/tmp/haozhezhao/model/blip2-flan-t5-xl +image_place_holder: +tokenizer length after expend 32101 +tokenizer length before expend 32100 +100%|████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████| 4/4 [04:01<00:00, 60.28s/it] +[2025-04-24 10:23:09] Finish Eval in 28000 steps...█████████████████████████████████████████████████████████████████████████████████████| 4/4 [04:00<00:00, 59.78s/it] +processing left-padding... +processing left-padding... +processing left-padding... +processing left-padding... +[2025-04-24 10:23:28] Saved checkpoint to checkpoint/Afteri2i_100kckpt_LLAVAT5_empty_Fixed_just_segment_pretrain_stage3_dreambench_recap_400k_t2i_400k_flux_200k_midjourney_150k_recovery_150k_3e4_epoch2_no_rotated/000-GPT-XL/checkpoints/0028000.pt +[2025-04-24 10:23:29] Removed old checkpoint: checkpoint/Afteri2i_100kckpt_LLAVAT5_empty_Fixed_just_segment_pretrain_stage3_dreambench_recap_400k_t2i_400k_flux_200k_midjourney_150k_recovery_150k_3e4_epoch2_no_rotated/000-GPT-XL/checkpoints/0026000.pt + 19%|█████████████████████████ | 4566/23458 [1:22:41<4:38:58, 1.13it/s][2025-04-24 10:23:51] (step=0028025) Train Loss: 5.4236, Train Steps/Sec: 0.07 + 20%|█████████████████████████▏ | 4591/23458 [1:23:03<4:28:04, 1.17it/s][2025-04-24 10:24:13] (step=0028050) Train Loss: 5.3656, Train Steps/Sec: 1.12 + 20%|█████████████████████████▍ | 4616/23458 [1:23:25<4:26:46, 1.18it/s][2025-04-24 10:24:35] (step=0028075) Train Loss: 5.3292, Train Steps/Sec: 1.17 + 20%|█████████████████████████▌ | 4641/23458 [1:23:46<4:25:28, 1.18it/s][2025-04-24 10:24:56] (step=0028100) Train Loss: 5.4061, Train Steps/Sec: 1.17 + 20%|█████████████████████████▋ | 4666/23458 [1:24:07<4:32:19, 1.15it/s][2025-04-24 10:25:17] (step=0028125) Train Loss: 5.4235, Train Steps/Sec: 1.17 + 20%|█████████████████████████▊ | 4691/23458 [1:24:29<4:28:13, 1.17it/s][2025-04-24 10:25:39] (step=0028150) Train Loss: 5.4079, Train Steps/Sec: 1.17 + 20%|█████████████████████████▉ | 4716/23458 [1:24:50<4:27:21, 1.17it/s][2025-04-24 10:26:00] (step=0028175) Train Loss: 5.4202, Train Steps/Sec: 1.17 + 20%|██████████████████████████ | 4741/23458 [1:25:11<4:22:21, 1.19it/s][2025-04-24 10:26:22] (step=0028200) Train Loss: 5.3009, Train Steps/Sec: 1.17 + 20%|██████████████████████████▏ | 4766/23458 [1:25:34<4:31:57, 1.15it/s][2025-04-24 10:26:44] (step=0028225) Train Loss: 5.3656, Train Steps/Sec: 1.12 + 20%|██████████████████████████▎ | 4791/23458 [1:25:55<4:25:33, 1.17it/s][2025-04-24 10:27:06] (step=0028250) Train Loss: 5.4310, Train Steps/Sec: 1.17 + 21%|██████████████████████████▍ | 4816/23458 [1:26:18<4:25:50, 1.17it/s][2025-04-24 10:27:28] (step=0028275) Train Loss: 5.4025, Train Steps/Sec: 1.13 + 21%|██████████████████████████▌ | 4841/23458 [1:26:41<4:26:28, 1.16it/s][2025-04-24 10:27:51] (step=0028300) Train Loss: 5.3900, Train Steps/Sec: 1.08 + 21%|██████████████████████████▊ | 4866/23458 [1:27:02<4:29:15, 1.15it/s][2025-04-24 10:28:12] (step=0028325) Train Loss: 5.4338, Train Steps/Sec: 1.17 + 21%|██████████████████████████▉ | 4891/23458 [1:27:23<4:25:34, 1.17it/s][2025-04-24 10:28:34] (step=0028350) Train Loss: 5.4017, Train Steps/Sec: 1.17 + 21%|███████████████████████████ | 4916/23458 [1:27:45<4:21:53, 1.18it/s][2025-04-24 10:28:55] (step=0028375) Train Loss: 5.3586, Train Steps/Sec: 1.17 + 21%|███████████████████████████▏ | 4941/23458 [1:28:07<4:22:08, 1.18it/s][2025-04-24 10:29:17] (step=0028400) Train Loss: 5.3816, Train Steps/Sec: 1.12 + 21%|███████████████████████████▎ | 4966/23458 [1:28:28<4:28:06, 1.15it/s][2025-04-24 10:29:38] (step=0028425) Train Loss: 5.4064, Train Steps/Sec: 1.17 + 21%|███████████████████████████▍ | 4991/23458 [1:28:50<4:23:14, 1.17it/s][2025-04-24 10:30:00] (step=0028450) Train Loss: 5.3963, Train Steps/Sec: 1.17 + 21%|███████████████████████████▌ | 5016/23458 [1:29:11<4:22:18, 1.17it/s][2025-04-24 10:30:21] (step=0028475) Train Loss: 5.4108, Train Steps/Sec: 1.17 + 21%|███████████████████████████▋ | 5041/23458 [1:29:32<4:19:06, 1.18it/s][2025-04-24 10:30:43] (step=0028500) Train Loss: 5.4721, Train Steps/Sec: 1.16 + 22%|███████████████████████████▊ | 5066/23458 [1:29:54<4:26:00, 1.15it/s][2025-04-24 10:31:04] (step=0028525) Train Loss: 5.4207, Train Steps/Sec: 1.17 + 22%|███████████████████████████▉ | 5091/23458 [1:30:15<4:21:20, 1.17it/s][2025-04-24 10:31:25] (step=0028550) Train Loss: 5.3678, Train Steps/Sec: 1.17 + 22%|████████████████████████████▏ | 5116/23458 [1:30:37<4:20:20, 1.17it/s][2025-04-24 10:31:47] (step=0028575) Train Loss: 5.4702, Train Steps/Sec: 1.17 + 22%|████████████████████████████▎ | 5141/23458 [1:30:58<4:18:11, 1.18it/s][2025-04-24 10:32:08] (step=0028600) Train Loss: 5.4019, Train Steps/Sec: 1.16 + 22%|████████████████████████████▍ | 5166/23458 [1:31:20<4:25:35, 1.15it/s][2025-04-24 10:32:31] (step=0028625) Train Loss: 5.4637, Train Steps/Sec: 1.12 + 22%|████████████████████████████▌ | 5191/23458 [1:31:43<4:57:48, 1.02it/s][2025-04-24 10:32:53] (step=0028650) Train Loss: 5.3704, Train Steps/Sec: 1.13 + 22%|████████████████████████████▋ | 5216/23458 [1:32:04<4:17:54, 1.18it/s][2025-04-24 10:33:14] (step=0028675) Train Loss: 5.4430, Train Steps/Sec: 1.17 + 22%|████████████████████████████▊ | 5241/23458 [1:32:25<4:17:20, 1.18it/s][2025-04-24 10:33:36] (step=0028700) Train Loss: 5.3690, Train Steps/Sec: 1.16 + 22%|████████████████████████████▉ | 5266/23458 [1:32:47<4:22:15, 1.16it/s][2025-04-24 10:33:57] (step=0028725) Train Loss: 5.3758, Train Steps/Sec: 1.17 + 23%|█████████████████████████████ | 5291/23458 [1:33:08<4:19:08, 1.17it/s][2025-04-24 10:34:18] (step=0028750) Train Loss: 5.4547, Train Steps/Sec: 1.17 + 23%|█████████████████████████████▏ | 5316/23458 [1:33:30<4:16:18, 1.18it/s][2025-04-24 10:34:41] (step=0028775) Train Loss: 5.3379, Train Steps/Sec: 1.12 + 23%|█████████████████████████████▎ | 5341/23458 [1:33:52<4:14:56, 1.18it/s][2025-04-24 10:35:02] (step=0028800) Train Loss: 5.2841, Train Steps/Sec: 1.16 + 23%|█████████████████████████████▌ | 5366/23458 [1:34:13<4:22:06, 1.15it/s][2025-04-24 10:35:23] (step=0028825) Train Loss: 5.3991, Train Steps/Sec: 1.17 + 23%|█████████████████████████████▋ | 5391/23458 [1:34:35<4:17:16, 1.17it/s][2025-04-24 10:35:45] (step=0028850) Train Loss: 5.3742, Train Steps/Sec: 1.17 + 23%|█████████████████████████████▊ | 5416/23458 [1:34:57<4:30:05, 1.11it/s][2025-04-24 10:36:07] (step=0028875) Train Loss: 5.3572, Train Steps/Sec: 1.12 + 23%|█████████████████████████████▉ | 5441/23458 [1:35:18<4:13:35, 1.18it/s][2025-04-24 10:36:29] (step=0028900) Train Loss: 5.3873, Train Steps/Sec: 1.16 + 23%|██████████████████████████████ | 5466/23458 [1:35:40<4:20:07, 1.15it/s][2025-04-24 10:36:50] (step=0028925) Train Loss: 5.4119, Train Steps/Sec: 1.17 + 23%|██████████████████████████████▏ | 5491/23458 [1:36:04<5:42:13, 1.14s/it][2025-04-24 10:37:14] (step=0028950) Train Loss: 5.4029, Train Steps/Sec: 1.04 + 24%|██████████████████████████████▎ | 5516/23458 [1:36:25<4:13:31, 1.18it/s][2025-04-24 10:37:35] (step=0028975) Train Loss: 5.3675, Train Steps/Sec: 1.17 + 24%|██████████████████████████████▍ | 5541/23458 [1:36:47<4:12:28, 1.18it/s][2025-04-24 10:37:57] (step=0029000) Train Loss: 5.3676, Train Steps/Sec: 1.17 + 24%|██████████████████████████████▌ | 5566/23458 [1:37:08<4:19:27, 1.15it/s][2025-04-24 10:38:18] (step=0029025) Train Loss: 5.3921, Train Steps/Sec: 1.17 + 24%|██████████████████████████████▋ | 5591/23458 [1:37:30<4:42:30, 1.05it/s][2025-04-24 10:38:40] (step=0029050) Train Loss: 5.3798, Train Steps/Sec: 1.13 + 24%|██████████████████████████████▉ | 5616/23458 [1:37:51<4:12:51, 1.18it/s][2025-04-24 10:39:02] (step=0029075) Train Loss: 5.3402, Train Steps/Sec: 1.17 + 24%|███████████████████████████████ | 5641/23458 [1:38:13<4:13:40, 1.17it/s][2025-04-24 10:39:23] (step=0029100) Train Loss: 5.3791, Train Steps/Sec: 1.16 + 24%|███████████████████████████████▏ | 5666/23458 [1:38:34<4:17:18, 1.15it/s][2025-04-24 10:39:44] (step=0029125) Train Loss: 5.3776, Train Steps/Sec: 1.17 + 24%|███████████████████████████████▎ | 5691/23458 [1:38:56<4:15:36, 1.16it/s][2025-04-24 10:40:06] (step=0029150) Train Loss: 5.4449, Train Steps/Sec: 1.17 + 24%|███████████████████████████████▍ | 5716/23458 [1:39:17<4:11:25, 1.18it/s][2025-04-24 10:40:27] (step=0029175) Train Loss: 5.4016, Train Steps/Sec: 1.17 + 24%|███████████████████████████████▌ | 5741/23458 [1:39:39<4:09:38, 1.18it/s][2025-04-24 10:40:49] (step=0029200) Train Loss: 5.3775, Train Steps/Sec: 1.17 + 25%|███████████████████████████████▋ | 5766/23458 [1:40:00<4:16:28, 1.15it/s][2025-04-24 10:41:10] (step=0029225) Train Loss: 5.4309, Train Steps/Sec: 1.17 + 25%|███████████████████████████████▊ | 5791/23458 [1:40:21<4:11:42, 1.17it/s][2025-04-24 10:41:31] (step=0029250) Train Loss: 5.4203, Train Steps/Sec: 1.17 + 25%|███████████████████████████████▉ | 5816/23458 [1:40:43<4:09:10, 1.18it/s][2025-04-24 10:41:53] (step=0029275) Train Loss: 5.4673, Train Steps/Sec: 1.17 + 25%|████████████████████████████████ | 5841/23458 [1:41:04<4:08:18, 1.18it/s][2025-04-24 10:42:14] (step=0029300) Train Loss: 5.4146, Train Steps/Sec: 1.17 + 25%|████████████████████████████████▎ | 5866/23458 [1:41:26<4:14:37, 1.15it/s][2025-04-24 10:42:37] (step=0029325) Train Loss: 5.4129, Train Steps/Sec: 1.12 + 25%|████████████████████████████████▍ | 5891/23458 [1:41:49<4:10:21, 1.17it/s][2025-04-24 10:42:59] (step=0029350) Train Loss: 5.4137, Train Steps/Sec: 1.11 + 25%|████████████████████████████████▌ | 5916/23458 [1:42:10<4:09:36, 1.17it/s][2025-04-24 10:43:20] (step=0029375) Train Loss: 5.4292, Train Steps/Sec: 1.17 + 25%|████████████████████████████████▋ | 5941/23458 [1:42:32<4:06:59, 1.18it/s][2025-04-24 10:43:42] (step=0029400) Train Loss: 5.4733, Train Steps/Sec: 1.16 + 25%|████████████████████████████████▊ | 5966/23458 [1:42:53<4:14:27, 1.15it/s][2025-04-24 10:44:03] (step=0029425) Train Loss: 5.4613, Train Steps/Sec: 1.17 + 26%|████████████████████████████████▉ | 5991/23458 [1:43:15<4:09:30, 1.17it/s][2025-04-24 10:44:25] (step=0029450) Train Loss: 5.3897, Train Steps/Sec: 1.17 + 26%|█████████████████████████████████ | 6016/23458 [1:43:36<4:08:10, 1.17it/s][2025-04-24 10:44:46] (step=0029475) Train Loss: 5.3642, Train Steps/Sec: 1.17 + 26%|█████████████████████████████████▏ | 6041/23458 [1:43:58<4:06:33, 1.18it/s][2025-04-24 10:45:08] (step=0029500) Train Loss: 5.3665, Train Steps/Sec: 1.11 + 26%|█████████████████████████████████▎ | 6066/23458 [1:44:20<4:13:10, 1.14it/s][2025-04-24 10:45:30] (step=0029525) Train Loss: 5.3826, Train Steps/Sec: 1.17 + 26%|█████████████████████████████████▍ | 6091/23458 [1:44:42<4:08:24, 1.17it/s][2025-04-24 10:45:52] (step=0029550) Train Loss: 5.3154, Train Steps/Sec: 1.12 + 26%|█████████████████████████████████▋ | 6116/23458 [1:45:03<4:05:33, 1.18it/s][2025-04-24 10:46:14] (step=0029575) Train Loss: 5.3159, Train Steps/Sec: 1.17 + 26%|█████████████████████████████████▊ | 6141/23458 [1:45:26<4:09:49, 1.16it/s][2025-04-24 10:46:36] (step=0029600) Train Loss: 5.3028, Train Steps/Sec: 1.12 + 26%|█████████████████████████████████▉ | 6166/23458 [1:45:49<4:11:54, 1.14it/s][2025-04-24 10:46:59] (step=0029625) Train Loss: 5.3691, Train Steps/Sec: 1.08 + 26%|██████████████████████████████████ | 6191/23458 [1:46:10<4:06:15, 1.17it/s][2025-04-24 10:47:20] (step=0029650) Train Loss: 5.4709, Train Steps/Sec: 1.17 + 26%|██████████████████████████████████▏ | 6216/23458 [1:46:32<4:03:59, 1.18it/s][2025-04-24 10:47:42] (step=0029675) Train Loss: 5.4077, Train Steps/Sec: 1.17 + 27%|██████████████████████████████████▎ | 6241/23458 [1:46:53<4:02:49, 1.18it/s][2025-04-24 10:48:03] (step=0029700) Train Loss: 5.3807, Train Steps/Sec: 1.16 + 27%|██████████████████████████████████▍ | 6266/23458 [1:47:15<4:09:00, 1.15it/s][2025-04-24 10:48:25] (step=0029725) Train Loss: 5.3125, Train Steps/Sec: 1.13 + 27%|██████████████████████████████████▌ | 6291/23458 [1:47:37<4:05:00, 1.17it/s][2025-04-24 10:48:47] (step=0029750) Train Loss: 5.4205, Train Steps/Sec: 1.17 + 27%|██████████████████████████████████▋ | 6316/23458 [1:47:58<4:04:12, 1.17it/s][2025-04-24 10:49:08] (step=0029775) Train Loss: 5.4188, Train Steps/Sec: 1.17 + 27%|██████████████████████████████████▊ | 6341/23458 [1:48:19<4:01:02, 1.18it/s][2025-04-24 10:49:30] (step=0029800) Train Loss: 5.3808, Train Steps/Sec: 1.16 + 27%|███████████████████████████████████ | 6366/23458 [1:48:41<4:07:39, 1.15it/s][2025-04-24 10:49:51] (step=0029825) Train Loss: 5.3266, Train Steps/Sec: 1.17 + 27%|███████████████████████████████████▏ | 6391/23458 [1:49:03<4:03:38, 1.17it/s][2025-04-24 10:50:13] (step=0029850) Train Loss: 5.3969, Train Steps/Sec: 1.15 + 27%|███████████████████████████████████▎ | 6416/23458 [1:49:24<4:02:32, 1.17it/s][2025-04-24 10:50:34] (step=0029875) Train Loss: 5.3466, Train Steps/Sec: 1.17 + 27%|███████████████████████████████████▍ | 6441/23458 [1:49:45<4:00:14, 1.18it/s][2025-04-24 10:50:56] (step=0029900) Train Loss: 5.3587, Train Steps/Sec: 1.16 + 28%|███████████████████████████████████▌ | 6466/23458 [1:50:07<4:05:45, 1.15it/s][2025-04-24 10:51:17] (step=0029925) Train Loss: 5.3669, Train Steps/Sec: 1.17 + 28%|███████████████████████████████████▋ | 6491/23458 [1:50:28<4:01:40, 1.17it/s][2025-04-24 10:51:38] (step=0029950) Train Loss: 5.4132, Train Steps/Sec: 1.17 + 28%|███████████████████████████████████▊ | 6516/23458 [1:50:50<4:08:41, 1.14it/s][2025-04-24 10:52:00] (step=0029975) Train Loss: 5.3061, Train Steps/Sec: 1.13 + 28%|███████████████████████████████████▉ | 6541/23458 [1:51:12<3:58:34, 1.18it/s][2025-04-24 10:52:22] (step=0030000) Train Loss: 5.4135, Train Steps/Sec: 1.17 +You are using a model of type blip-2 to instantiate a model of type instructblip. This is not supported for all configurations of models and can yield errors. +[2025-04-24 10:52:22] text_config is None. Initializing the text config with default values (`OPTConfig`). +model_name_or_path:/tmp/haozhezhao/model/blip2-flan-t5-xl +image_place_holder: +tokenizer length after expend 32101 +tokenizer length before expend 32100 +100%|████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████| 4/4 [04:00<00:00, 60.01s/it] +[2025-04-24 10:57:30] Finish Eval in 30000 steps...█████████████████████████████████████████████████████████████████████████████████████| 4/4 [03:59<00:00, 59.66s/it] +processing left-padding... +processing left-padding... +processing left-padding... +processing left-padding... +[2025-04-24 10:57:49] Saved checkpoint to checkpoint/Afteri2i_100kckpt_LLAVAT5_empty_Fixed_just_segment_pretrain_stage3_dreambench_recap_400k_t2i_400k_flux_200k_midjourney_150k_recovery_150k_3e4_epoch2_no_rotated/000-GPT-XL/checkpoints/0030000.pt +[2025-04-24 10:57:51] Removed old checkpoint: checkpoint/Afteri2i_100kckpt_LLAVAT5_empty_Fixed_just_segment_pretrain_stage3_dreambench_recap_400k_t2i_400k_flux_200k_midjourney_150k_recovery_150k_3e4_epoch2_no_rotated/000-GPT-XL/checkpoints/0028000.pt + 28%|████████████████████████████████████ | 6566/23458 [1:57:02<4:09:30, 1.13it/s][2025-04-24 10:58:12] (step=0030025) Train Loss: 5.4087, Train Steps/Sec: 0.07 + 28%|████████████████████████████████████▏ | 6591/23458 [1:57:23<4:00:23, 1.17it/s][2025-04-24 10:58:34] (step=0030050) Train Loss: 5.4061, Train Steps/Sec: 1.17 + 28%|████████████████████████████████████▍ | 6616/23458 [1:57:46<3:57:16, 1.18it/s][2025-04-24 10:58:56] (step=0030075) Train Loss: 5.3454, Train Steps/Sec: 1.12 + 28%|████████████████████████████████████▌ | 6641/23458 [1:58:07<3:56:38, 1.18it/s][2025-04-24 10:59:17] (step=0030100) Train Loss: 5.3862, Train Steps/Sec: 1.16 + 28%|████████████████████████████████████▋ | 6666/23458 [1:58:29<4:02:53, 1.15it/s][2025-04-24 10:59:39] (step=0030125) Train Loss: 5.4183, Train Steps/Sec: 1.17 + 29%|████████████████████████████████████▊ | 6691/23458 [1:58:50<4:30:02, 1.03it/s][2025-04-24 11:00:00] (step=0030150) Train Loss: 5.3874, Train Steps/Sec: 1.15 + 29%|████████████████████████████████████▉ | 6716/23458 [1:59:12<3:57:05, 1.18it/s][2025-04-24 11:00:22] (step=0030175) Train Loss: 5.3817, Train Steps/Sec: 1.17 + 29%|█████████████████████████████████████ | 6741/23458 [1:59:34<3:56:15, 1.18it/s][2025-04-24 11:00:44] (step=0030200) Train Loss: 5.4180, Train Steps/Sec: 1.12 + 29%|█████████████████████████████████████▏ | 6766/23458 [1:59:56<4:01:25, 1.15it/s][2025-04-24 11:01:06] (step=0030225) Train Loss: 5.4399, Train Steps/Sec: 1.12 + 29%|█████████████████████████████████████▎ | 6791/23458 [2:00:19<4:46:01, 1.03s/it][2025-04-24 11:01:29] (step=0030250) Train Loss: 5.3166, Train Steps/Sec: 1.12 + 29%|█████████████████████████████████████▍ | 6816/23458 [2:00:41<4:00:07, 1.16it/s][2025-04-24 11:01:51] (step=0030275) Train Loss: 5.3753, Train Steps/Sec: 1.12 + 29%|█████████████████████████████████████▌ | 6841/23458 [2:01:02<3:54:38, 1.18it/s][2025-04-24 11:02:12] (step=0030300) Train Loss: 5.4426, Train Steps/Sec: 1.17 + 29%|█████████████████████████████████████▊ | 6866/23458 [2:01:24<3:59:48, 1.15it/s][2025-04-24 11:02:34] (step=0030325) Train Loss: 5.3768, Train Steps/Sec: 1.17 + 29%|█████████████████████████████████████▉ | 6891/23458 [2:01:45<3:57:16, 1.16it/s][2025-04-24 11:02:55] (step=0030350) Train Loss: 5.3610, Train Steps/Sec: 1.17 + 29%|██████████████████████████████████████ | 6916/23458 [2:02:07<3:56:13, 1.17it/s][2025-04-24 11:03:17] (step=0030375) Train Loss: 5.3753, Train Steps/Sec: 1.13 + 30%|██████████████████████████████████████▏ | 6941/23458 [2:02:29<3:54:45, 1.17it/s][2025-04-24 11:03:39] (step=0030400) Train Loss: 5.4298, Train Steps/Sec: 1.16 + 30%|██████████████████████████████████████▎ | 6966/23458 [2:02:50<3:58:26, 1.15it/s][2025-04-24 11:04:00] (step=0030425) Train Loss: 5.3649, Train Steps/Sec: 1.17 + 30%|██████████████████████████████████████▍ | 6991/23458 [2:03:11<3:54:19, 1.17it/s][2025-04-24 11:04:22] (step=0030450) Train Loss: 5.4641, Train Steps/Sec: 1.17 + 30%|██████████████████████████████████████▌ | 7016/23458 [2:03:33<3:52:49, 1.18it/s][2025-04-24 11:04:43] (step=0030475) Train Loss: 5.4222, Train Steps/Sec: 1.17 + 30%|██████████████████████████████████████▋ | 7041/23458 [2:03:54<3:50:11, 1.19it/s][2025-04-24 11:05:04] (step=0030500) Train Loss: 5.4516, Train Steps/Sec: 1.16 + 30%|██████████████████████████████████████▊ | 7066/23458 [2:04:16<3:57:48, 1.15it/s][2025-04-24 11:05:26] (step=0030525) Train Loss: 5.4039, Train Steps/Sec: 1.17 + 30%|██████████████████████████████████████▉ | 7091/23458 [2:04:37<3:54:06, 1.17it/s][2025-04-24 11:05:47] (step=0030550) Train Loss: 5.3727, Train Steps/Sec: 1.17 + 30%|███████████████████████████████████████▏ | 7116/23458 [2:04:58<3:51:43, 1.18it/s][2025-04-24 11:06:09] (step=0030575) Train Loss: 5.3517, Train Steps/Sec: 1.17 + 30%|███████████████████████████████████████▎ | 7141/23458 [2:05:20<3:49:46, 1.18it/s][2025-04-24 11:06:30] (step=0030600) Train Loss: 5.4230, Train Steps/Sec: 1.17 + 31%|███████████████████████████████████████▍ | 7166/23458 [2:05:41<3:56:36, 1.15it/s][2025-04-24 11:06:51] (step=0030625) Train Loss: 5.3040, Train Steps/Sec: 1.17 + 31%|███████████████████████████████████████▌ | 7191/23458 [2:06:03<3:51:31, 1.17it/s][2025-04-24 11:07:13] (step=0030650) Train Loss: 5.3773, Train Steps/Sec: 1.17 + 31%|███████████████████████████████████████▋ | 7216/23458 [2:06:25<3:49:55, 1.18it/s][2025-04-24 11:07:35] (step=0030675) Train Loss: 5.3824, Train Steps/Sec: 1.13 + 31%|███████████████████████████████████████▊ | 7241/23458 [2:06:46<3:47:58, 1.19it/s][2025-04-24 11:07:56] (step=0030700) Train Loss: 5.4066, Train Steps/Sec: 1.17 + 31%|███████████████████████████████████████▉ | 7266/23458 [2:07:08<3:55:24, 1.15it/s][2025-04-24 11:08:18] (step=0030725) Train Loss: 5.3707, Train Steps/Sec: 1.17 + 31%|████████████████████████████████████████ | 7291/23458 [2:07:29<3:49:45, 1.17it/s][2025-04-24 11:08:39] (step=0030750) Train Loss: 5.3374, Train Steps/Sec: 1.17 + 31%|████████████████████████████████████████▏ | 7316/23458 [2:07:50<3:49:14, 1.17it/s][2025-04-24 11:09:00] (step=0030775) Train Loss: 5.3552, Train Steps/Sec: 1.17 + 31%|████████████████████████████████████████▎ | 7341/23458 [2:08:13<3:46:19, 1.19it/s][2025-04-24 11:09:23] (step=0030800) Train Loss: 5.3624, Train Steps/Sec: 1.11 + 31%|████████████████████████████████████████▌ | 7366/23458 [2:08:34<3:53:30, 1.15it/s][2025-04-24 11:09:44] (step=0030825) Train Loss: 5.4207, Train Steps/Sec: 1.17 + 32%|████████████████████████████████████████▋ | 7391/23458 [2:08:56<4:00:46, 1.11it/s][2025-04-24 11:10:07] (step=0030850) Train Loss: 5.3388, Train Steps/Sec: 1.12 + 32%|████████████████████████████████████████▊ | 7416/23458 [2:09:18<3:46:42, 1.18it/s][2025-04-24 11:10:28] (step=0030875) Train Loss: 5.4066, Train Steps/Sec: 1.17 + 32%|████████████████████████████████████████▉ | 7441/23458 [2:09:39<3:45:32, 1.18it/s][2025-04-24 11:10:49] (step=0030900) Train Loss: 5.3789, Train Steps/Sec: 1.16 + 32%|█████████████████████████████████████████ | 7466/23458 [2:10:02<4:23:41, 1.01it/s][2025-04-24 11:11:12] (step=0030925) Train Loss: 5.3748, Train Steps/Sec: 1.09 + 32%|█████████████████████████████████████████▏ | 7491/23458 [2:10:26<3:47:46, 1.17it/s][2025-04-24 11:11:36] (step=0030950) Train Loss: 5.4025, Train Steps/Sec: 1.07 + 32%|█████████████████████████████████████████▎ | 7516/23458 [2:10:47<3:45:50, 1.18it/s][2025-04-24 11:11:57] (step=0030975) Train Loss: 5.4025, Train Steps/Sec: 1.17 + 32%|█████████████████████████████████████████▍ | 7541/23458 [2:11:08<3:43:54, 1.18it/s][2025-04-24 11:12:18] (step=0031000) Train Loss: 5.3641, Train Steps/Sec: 1.17 + 32%|█████████████████████████████████████████▌ | 7566/23458 [2:11:31<4:01:07, 1.10it/s][2025-04-24 11:12:41] (step=0031025) Train Loss: 5.3488, Train Steps/Sec: 1.13 + 32%|█████████████████████████████████████████▋ | 7591/23458 [2:11:52<3:47:01, 1.16it/s][2025-04-24 11:13:02] (step=0031050) Train Loss: 5.3697, Train Steps/Sec: 1.17 + 32%|█████████████████████████████████████████▉ | 7616/23458 [2:12:13<3:43:44, 1.18it/s][2025-04-24 11:13:23] (step=0031075) Train Loss: 5.3547, Train Steps/Sec: 1.17 + 33%|██████████████████████████████████████████ | 7641/23458 [2:12:35<3:42:23, 1.19it/s][2025-04-24 11:13:45] (step=0031100) Train Loss: 5.4885, Train Steps/Sec: 1.16 + 33%|██████████████████████████████████████████▏ | 7666/23458 [2:12:56<3:49:25, 1.15it/s][2025-04-24 11:14:06] (step=0031125) Train Loss: 5.4072, Train Steps/Sec: 1.17 + 33%|██████████████████████████████████████████▎ | 7691/23458 [2:13:17<3:47:21, 1.16it/s][2025-04-24 11:14:28] (step=0031150) Train Loss: 5.3982, Train Steps/Sec: 1.17 + 33%|██████████████████████████████████████████▍ | 7716/23458 [2:13:39<3:41:57, 1.18it/s][2025-04-24 11:14:49] (step=0031175) Train Loss: 5.4047, Train Steps/Sec: 1.17 + 33%|██████████████████████████████████████████▌ | 7741/23458 [2:14:00<3:40:52, 1.19it/s][2025-04-24 11:15:10] (step=0031200) Train Loss: 5.4679, Train Steps/Sec: 1.17 + 33%|██████████████████████████████████████████▋ | 7766/23458 [2:14:22<3:48:05, 1.15it/s][2025-04-24 11:15:32] (step=0031225) Train Loss: 5.3877, Train Steps/Sec: 1.17 + 33%|██████████████████████████████████████████▊ | 7791/23458 [2:14:43<3:43:55, 1.17it/s][2025-04-24 11:15:53] (step=0031250) Train Loss: 5.4221, Train Steps/Sec: 1.17 + 33%|██████████████████████████████████████████▉ | 7816/23458 [2:15:04<3:42:14, 1.17it/s][2025-04-24 11:16:14] (step=0031275) Train Loss: 5.3549, Train Steps/Sec: 1.17 + 33%|███████████████████████████████████████████ | 7841/23458 [2:15:26<3:39:44, 1.18it/s][2025-04-24 11:16:36] (step=0031300) Train Loss: 5.4628, Train Steps/Sec: 1.17 + 34%|███████████████████████████████████████████▎ | 7866/23458 [2:15:48<3:46:33, 1.15it/s][2025-04-24 11:16:58] (step=0031325) Train Loss: 5.3999, Train Steps/Sec: 1.12 + 34%|███████████████████████████████████████████▍ | 7891/23458 [2:16:09<3:42:01, 1.17it/s][2025-04-24 11:17:19] (step=0031350) Train Loss: 5.3702, Train Steps/Sec: 1.17 + 34%|███████████████████████████████████████████▌ | 7916/23458 [2:16:31<3:40:52, 1.17it/s][2025-04-24 11:17:41] (step=0031375) Train Loss: 5.4457, Train Steps/Sec: 1.17 + 34%|███████████████████████████████████████████▋ | 7941/23458 [2:16:52<3:38:29, 1.18it/s][2025-04-24 11:18:02] (step=0031400) Train Loss: 5.3383, Train Steps/Sec: 1.17 + 34%|███████████████████████████████████████████▊ | 7966/23458 [2:17:13<3:43:32, 1.16it/s][2025-04-24 11:18:24] (step=0031425) Train Loss: 5.3851, Train Steps/Sec: 1.17 + 34%|███████████████████████████████████████████▉ | 7991/23458 [2:17:35<3:40:27, 1.17it/s][2025-04-24 11:18:45] (step=0031450) Train Loss: 5.4595, Train Steps/Sec: 1.17 + 34%|████████████████████████████████████████████ | 8016/23458 [2:17:56<3:38:13, 1.18it/s][2025-04-24 11:19:06] (step=0031475) Train Loss: 5.3589, Train Steps/Sec: 1.17 + 34%|████████████████████████████████████████████▏ | 8041/23458 [2:18:19<4:18:19, 1.01s/it][2025-04-24 11:19:29] (step=0031500) Train Loss: 5.2657, Train Steps/Sec: 1.11 + 34%|████████████████████████████████████████████▎ | 8066/23458 [2:18:41<3:43:54, 1.15it/s][2025-04-24 11:19:51] (step=0031525) Train Loss: 5.4379, Train Steps/Sec: 1.12 + 34%|████████████████████████████████████████████▍ | 8091/23458 [2:19:02<3:38:53, 1.17it/s][2025-04-24 11:20:13] (step=0031550) Train Loss: 5.3626, Train Steps/Sec: 1.17 + 35%|████████████████████████████████████████████▋ | 8116/23458 [2:19:25<3:39:04, 1.17it/s][2025-04-24 11:20:35] (step=0031575) Train Loss: 5.3889, Train Steps/Sec: 1.13 + 35%|████████████████████████████████████████████▊ | 8141/23458 [2:19:48<3:36:11, 1.18it/s][2025-04-24 11:20:58] (step=0031600) Train Loss: 5.4181, Train Steps/Sec: 1.08 + 35%|████████████████████████████████████████████▉ | 8166/23458 [2:20:09<3:41:16, 1.15it/s][2025-04-24 11:21:19] (step=0031625) Train Loss: 5.3756, Train Steps/Sec: 1.17 + 35%|█████████████████████████████████████████████ | 8191/23458 [2:20:31<3:37:46, 1.17it/s][2025-04-24 11:21:41] (step=0031650) Train Loss: 5.3532, Train Steps/Sec: 1.17 + 35%|█████████████████████████████████████████████▏ | 8216/23458 [2:20:53<3:37:21, 1.17it/s][2025-04-24 11:22:03] (step=0031675) Train Loss: 5.3470, Train Steps/Sec: 1.12 + 35%|█████████████████████████████████████████████▎ | 8241/23458 [2:21:15<3:33:55, 1.19it/s][2025-04-24 11:22:25] (step=0031700) Train Loss: 5.4187, Train Steps/Sec: 1.12 + 35%|█████████████████████████████████████████████▍ | 8266/23458 [2:21:37<3:39:52, 1.15it/s][2025-04-24 11:22:47] (step=0031725) Train Loss: 5.3863, Train Steps/Sec: 1.17 + 35%|█████████████████████████████████████████████▌ | 8291/23458 [2:21:58<3:37:14, 1.16it/s][2025-04-24 11:23:08] (step=0031750) Train Loss: 5.3666, Train Steps/Sec: 1.17 + 35%|█████████████████████████████████████████████▋ | 8316/23458 [2:22:19<3:33:57, 1.18it/s][2025-04-24 11:23:29] (step=0031775) Train Loss: 5.4164, Train Steps/Sec: 1.17 + 36%|█████████████████████████████████████████████▊ | 8341/23458 [2:22:41<3:32:42, 1.18it/s][2025-04-24 11:23:51] (step=0031800) Train Loss: 5.4742, Train Steps/Sec: 1.17 + 36%|██████████████████████████████████████████████ | 8366/23458 [2:23:02<3:38:06, 1.15it/s][2025-04-24 11:24:12] (step=0031825) Train Loss: 5.4010, Train Steps/Sec: 1.18 + 36%|██████████████████████████████████████████████▏ | 8391/23458 [2:23:23<3:34:28, 1.17it/s][2025-04-24 11:24:33] (step=0031850) Train Loss: 5.4572, Train Steps/Sec: 1.17 + 36%|██████████████████████████████████████████████▎ | 8416/23458 [2:23:45<3:32:46, 1.18it/s][2025-04-24 11:24:55] (step=0031875) Train Loss: 5.4126, Train Steps/Sec: 1.17 + 36%|██████████████████████████████████████████████▍ | 8441/23458 [2:24:06<3:30:56, 1.19it/s][2025-04-24 11:25:16] (step=0031900) Train Loss: 5.3960, Train Steps/Sec: 1.16 + 36%|██████████████████████████████████████████████▌ | 8466/23458 [2:24:27<3:36:55, 1.15it/s][2025-04-24 11:25:38] (step=0031925) Train Loss: 5.3739, Train Steps/Sec: 1.17 + 36%|██████████████████████████████████████████████▋ | 8491/23458 [2:24:49<3:34:19, 1.16it/s][2025-04-24 11:25:59] (step=0031950) Train Loss: 5.3678, Train Steps/Sec: 1.17 + 36%|██████████████████████████████████████████████▊ | 8516/23458 [2:25:10<3:31:43, 1.18it/s][2025-04-24 11:26:20] (step=0031975) Train Loss: 5.4218, Train Steps/Sec: 1.17 + 36%|██████████████████████████████████████████████▉ | 8541/23458 [2:25:32<3:29:43, 1.19it/s][2025-04-24 11:26:43] (step=0032000) Train Loss: 5.3500, Train Steps/Sec: 1.12 +You are using a model of type blip-2 to instantiate a model of type instructblip. This is not supported for all configurations of models and can yield errors. +[2025-04-24 11:26:43] text_config is None. Initializing the text config with default values (`OPTConfig`). +model_name_or_path:/tmp/haozhezhao/model/blip2-flan-t5-xl +image_place_holder: +tokenizer length after expend 32101 +tokenizer length before expend 32100 +100%|████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████| 4/4 [03:57<00:00, 59.38s/it] +[2025-04-24 11:31:50] Finish Eval in 32000 steps...█████████████████████████████████████████████████████████████████████████████████████| 4/4 [03:57<00:00, 59.03s/it] +processing left-padding... +processing left-padding... +processing left-padding... +processing left-padding... +[2025-04-24 11:32:11] Saved checkpoint to checkpoint/Afteri2i_100kckpt_LLAVAT5_empty_Fixed_just_segment_pretrain_stage3_dreambench_recap_400k_t2i_400k_flux_200k_midjourney_150k_recovery_150k_3e4_epoch2_no_rotated/000-GPT-XL/checkpoints/0032000.pt +[2025-04-24 11:32:13] Removed old checkpoint: checkpoint/Afteri2i_100kckpt_LLAVAT5_empty_Fixed_just_segment_pretrain_stage3_dreambench_recap_400k_t2i_400k_flux_200k_midjourney_150k_recovery_150k_3e4_epoch2_no_rotated/000-GPT-XL/checkpoints/0030000.pt + 37%|███████████████████████████████████████████████ | 8566/23458 [2:31:24<3:41:07, 1.12it/s][2025-04-24 11:32:34] (step=0032025) Train Loss: 5.4388, Train Steps/Sec: 0.07 + 37%|███████████████████████████████████████████████▏ | 8591/23458 [2:31:45<3:31:26, 1.17it/s][2025-04-24 11:32:55] (step=0032050) Train Loss: 5.3813, Train Steps/Sec: 1.17 + 37%|███████████████████████████████████████████████▍ | 8616/23458 [2:32:07<3:50:41, 1.07it/s][2025-04-24 11:33:17] (step=0032075) Train Loss: 5.3727, Train Steps/Sec: 1.15 + 37%|███████████████████████████████████████████████▌ | 8641/23458 [2:32:28<3:28:55, 1.18it/s][2025-04-24 11:33:39] (step=0032100) Train Loss: 5.3919, Train Steps/Sec: 1.17 + 37%|███████████████████████████████████████████████▋ | 8666/23458 [2:32:50<3:36:24, 1.14it/s][2025-04-24 11:34:00] (step=0032125) Train Loss: 5.3524, Train Steps/Sec: 1.17 + 37%|███████████████████████████████████████████████▊ | 8691/23458 [2:33:11<3:30:26, 1.17it/s][2025-04-24 11:34:21] (step=0032150) Train Loss: 5.4180, Train Steps/Sec: 1.17 + 37%|███████████████████████████████████████████████▉ | 8716/23458 [2:33:34<3:29:51, 1.17it/s][2025-04-24 11:34:44] (step=0032175) Train Loss: 5.3888, Train Steps/Sec: 1.12 + 37%|████████████████████████████████████████████████ | 8741/23458 [2:33:55<3:26:50, 1.19it/s][2025-04-24 11:35:05] (step=0032200) Train Loss: 5.4044, Train Steps/Sec: 1.17 + 37%|████████████████████████████████████████████████▏ | 8766/23458 [2:34:18<4:34:52, 1.12s/it][2025-04-24 11:35:28] (step=0032225) Train Loss: 5.4023, Train Steps/Sec: 1.07 + 37%|████████████████████████████████████████████████▎ | 8791/23458 [2:34:41<3:47:11, 1.08it/s][2025-04-24 11:35:52] (step=0032250) Train Loss: 5.3870, Train Steps/Sec: 1.08 + 38%|████████████████████████████████████████████████▍ | 8816/23458 [2:35:03<3:26:26, 1.18it/s][2025-04-24 11:36:13] (step=0032275) Train Loss: 5.3983, Train Steps/Sec: 1.17 + 38%|████████████████████████████████████████████████▌ | 8841/23458 [2:35:24<3:25:38, 1.18it/s][2025-04-24 11:36:34] (step=0032300) Train Loss: 5.4428, Train Steps/Sec: 1.17 + 38%|████████████████████████████████████████████████▊ | 8866/23458 [2:35:46<3:31:11, 1.15it/s][2025-04-24 11:36:56] (step=0032325) Train Loss: 5.3910, Train Steps/Sec: 1.17 + 38%|████████████████████████████████████████████████▉ | 8891/23458 [2:36:08<3:28:29, 1.16it/s][2025-04-24 11:37:18] (step=0032350) Train Loss: 5.4299, Train Steps/Sec: 1.12 + 38%|█████████████████████████████████████████████████ | 8916/23458 [2:36:29<3:25:47, 1.18it/s][2025-04-24 11:37:39] (step=0032375) Train Loss: 5.3534, Train Steps/Sec: 1.17 + 38%|█████████████████████████████████████████████████▏ | 8941/23458 [2:36:52<3:26:19, 1.17it/s][2025-04-24 11:38:02] (step=0032400) Train Loss: 5.4174, Train Steps/Sec: 1.11 + 38%|█████████████████████████████████████████████████▎ | 8966/23458 [2:37:13<3:29:47, 1.15it/s][2025-04-24 11:38:23] (step=0032425) Train Loss: 5.3892, Train Steps/Sec: 1.17 + 38%|█████████████████████████████████████████████████▍ | 8991/23458 [2:37:34<3:26:21, 1.17it/s][2025-04-24 11:38:45] (step=0032450) Train Loss: 5.4092, Train Steps/Sec: 1.17 + 38%|█████████████████████████████████████████████████▌ | 9016/23458 [2:37:56<3:25:01, 1.17it/s][2025-04-24 11:39:06] (step=0032475) Train Loss: 5.4718, Train Steps/Sec: 1.17 + 39%|█████████████████████████████████████████████████▋ | 9041/23458 [2:38:17<3:23:18, 1.18it/s][2025-04-24 11:39:27] (step=0032500) Train Loss: 5.3994, Train Steps/Sec: 1.17 + 39%|█████████████████████████████████████████████████▊ | 9066/23458 [2:38:39<3:28:22, 1.15it/s][2025-04-24 11:39:49] (step=0032525) Train Loss: 5.4240, Train Steps/Sec: 1.17 + 39%|█████████████████████████████████████████████████▉ | 9091/23458 [2:39:00<3:24:27, 1.17it/s][2025-04-24 11:40:10] (step=0032550) Train Loss: 5.3366, Train Steps/Sec: 1.17 + 39%|██████████████████████████████████████████████████▏ | 9116/23458 [2:39:21<3:23:10, 1.18it/s][2025-04-24 11:40:31] (step=0032575) Train Loss: 5.3629, Train Steps/Sec: 1.17 + 39%|██████████████████████████████████████████████████▎ | 9141/23458 [2:39:43<3:21:03, 1.19it/s][2025-04-24 11:40:53] (step=0032600) Train Loss: 5.3764, Train Steps/Sec: 1.17 + 39%|██████████████████████████████████████████████████▍ | 9166/23458 [2:40:04<3:26:58, 1.15it/s][2025-04-24 11:41:14] (step=0032625) Train Loss: 5.3574, Train Steps/Sec: 1.17 + 39%|██████████████████████████████████████████████████▌ | 9191/23458 [2:40:27<4:16:20, 1.08s/it][2025-04-24 11:41:37] (step=0032650) Train Loss: 5.2824, Train Steps/Sec: 1.12 + 39%|██████████████████████████████████████████████████▋ | 9216/23458 [2:40:48<3:21:07, 1.18it/s][2025-04-24 11:41:58] (step=0032675) Train Loss: 5.3770, Train Steps/Sec: 1.17 + 39%|██████████████████████████████████████████████████▊ | 9241/23458 [2:41:09<3:20:38, 1.18it/s][2025-04-24 11:42:19] (step=0032700) Train Loss: 5.3612, Train Steps/Sec: 1.17 + 40%|██████████████████████████████████████████████████▉ | 9266/23458 [2:41:31<3:25:32, 1.15it/s][2025-04-24 11:42:41] (step=0032725) Train Loss: 5.3932, Train Steps/Sec: 1.17 + 40%|███████████████████████████████████████████████████ | 9291/23458 [2:41:52<3:22:32, 1.17it/s][2025-04-24 11:43:02] (step=0032750) Train Loss: 5.3931, Train Steps/Sec: 1.16 + 40%|███████████████████████████████████████████████████▏ | 9316/23458 [2:42:14<3:19:53, 1.18it/s][2025-04-24 11:43:24] (step=0032775) Train Loss: 5.4068, Train Steps/Sec: 1.17 + 40%|███████████████████████████████████████████████████▎ | 9341/23458 [2:42:35<3:19:45, 1.18it/s][2025-04-24 11:43:45] (step=0032800) Train Loss: 5.4416, Train Steps/Sec: 1.17 + 40%|███████████████████████████████████████████████████▌ | 9366/23458 [2:42:57<3:31:56, 1.11it/s][2025-04-24 11:44:07] (step=0032825) Train Loss: 5.3992, Train Steps/Sec: 1.12 + 40%|███████████████████████████████████████████████████▋ | 9391/23458 [2:43:19<3:20:34, 1.17it/s][2025-04-24 11:44:29] (step=0032850) Train Loss: 5.4042, Train Steps/Sec: 1.17 + 40%|███████████████████████████████████████████████████▊ | 9416/23458 [2:43:40<3:19:08, 1.18it/s][2025-04-24 11:44:50] (step=0032875) Train Loss: 5.3902, Train Steps/Sec: 1.17 + 40%|███████████████████████████████████████████████████▉ | 9441/23458 [2:44:04<4:36:13, 1.18s/it][2025-04-24 11:45:14] (step=0032900) Train Loss: 5.3895, Train Steps/Sec: 1.04 + 40%|████████████████████████████████████████████████████ | 9466/23458 [2:44:25<3:26:33, 1.13it/s][2025-04-24 11:45:35] (step=0032925) Train Loss: 5.3514, Train Steps/Sec: 1.17 + 40%|████████████████████████████████████████████████████▏ | 9491/23458 [2:44:48<3:22:26, 1.15it/s][2025-04-24 11:45:58] (step=0032950) Train Loss: 5.3612, Train Steps/Sec: 1.12 + 41%|████████████████████████████████████████████████████▎ | 9516/23458 [2:45:09<3:17:18, 1.18it/s][2025-04-24 11:46:19] (step=0032975) Train Loss: 5.4407, Train Steps/Sec: 1.17 + 41%|████████████████████████████████████████████████████▍ | 9541/23458 [2:45:31<3:22:36, 1.14it/s][2025-04-24 11:46:41] (step=0033000) Train Loss: 5.3833, Train Steps/Sec: 1.12 + 41%|████████████████████████████████████████████████████▌ | 9566/23458 [2:45:53<3:21:40, 1.15it/s][2025-04-24 11:47:03] (step=0033025) Train Loss: 5.4045, Train Steps/Sec: 1.17 + 41%|████████████████████████████████████████████████████▋ | 9591/23458 [2:46:14<3:18:04, 1.17it/s][2025-04-24 11:47:24] (step=0033050) Train Loss: 5.4354, Train Steps/Sec: 1.17 + 41%|████████████████████████████████████████████████████▉ | 9616/23458 [2:46:35<3:15:50, 1.18it/s][2025-04-24 11:47:45] (step=0033075) Train Loss: 5.3907, Train Steps/Sec: 1.17 + 41%|█████████████████████████████████████████████████████ | 9641/23458 [2:46:57<3:14:07, 1.19it/s][2025-04-24 11:48:07] (step=0033100) Train Loss: 5.4221, Train Steps/Sec: 1.17 + 41%|█████████████████████████████████████████████████████▏ | 9666/23458 [2:47:19<3:22:55, 1.13it/s][2025-04-24 11:48:29] (step=0033125) Train Loss: 5.3490, Train Steps/Sec: 1.12 + 41%|█████████████████████████████████████████████████████▎ | 9691/23458 [2:47:41<3:15:43, 1.17it/s][2025-04-24 11:48:51] (step=0033150) Train Loss: 5.3860, Train Steps/Sec: 1.17 + 41%|█████████████████████████████████████████████████████▍ | 9716/23458 [2:48:02<3:14:35, 1.18it/s][2025-04-24 11:49:12] (step=0033175) Train Loss: 5.4320, Train Steps/Sec: 1.17 + 42%|█████████████████████████████████████████████████████▌ | 9741/23458 [2:48:23<3:13:11, 1.18it/s][2025-04-24 11:49:33] (step=0033200) Train Loss: 5.4085, Train Steps/Sec: 1.17 + 42%|█████████████████████████████████████████████████████▋ | 9766/23458 [2:48:45<3:18:50, 1.15it/s][2025-04-24 11:49:55] (step=0033225) Train Loss: 5.3728, Train Steps/Sec: 1.17 + 42%|█████████████████████████████████████████████████████▊ | 9791/23458 [2:49:06<3:14:35, 1.17it/s][2025-04-24 11:50:16] (step=0033250) Train Loss: 5.3358, Train Steps/Sec: 1.17 + 42%|█████████████████████████████████████████████████████▉ | 9816/23458 [2:49:27<3:12:42, 1.18it/s][2025-04-24 11:50:37] (step=0033275) Train Loss: 5.3519, Train Steps/Sec: 1.17 + 42%|██████████████████████████████████████████████████████ | 9841/23458 [2:49:49<3:11:10, 1.19it/s][2025-04-24 11:50:59] (step=0033300) Train Loss: 5.3360, Train Steps/Sec: 1.17 + 42%|██████████████████████████████████████████████████████▎ | 9866/23458 [2:50:11<3:17:17, 1.15it/s][2025-04-24 11:51:21] (step=0033325) Train Loss: 5.4604, Train Steps/Sec: 1.12 + 42%|██████████████████████████████████████████████████████▍ | 9891/23458 [2:50:32<3:16:03, 1.15it/s][2025-04-24 11:51:42] (step=0033350) Train Loss: 5.3979, Train Steps/Sec: 1.17 + 42%|██████████████████████████████████████████████████████▌ | 9916/23458 [2:50:54<3:12:02, 1.18it/s][2025-04-24 11:52:04] (step=0033375) Train Loss: 5.2429, Train Steps/Sec: 1.17 + 42%|██████████████████████████████████████████████████████▋ | 9941/23458 [2:51:15<3:11:15, 1.18it/s][2025-04-24 11:52:25] (step=0033400) Train Loss: 5.4023, Train Steps/Sec: 1.16 + 42%|██████████████████████████████████████████████████████▊ | 9966/23458 [2:51:37<3:15:50, 1.15it/s][2025-04-24 11:52:47] (step=0033425) Train Loss: 5.4474, Train Steps/Sec: 1.17 + 43%|██████████████████████████████████████████████████████▉ | 9991/23458 [2:51:58<3:11:27, 1.17it/s][2025-04-24 11:53:08] (step=0033450) Train Loss: 5.3556, Train Steps/Sec: 1.17 + 43%|██████████████████████████████████████████████████████▋ | 10016/23458 [2:52:19<3:09:59, 1.18it/s][2025-04-24 11:53:30] (step=0033475) Train Loss: 5.4534, Train Steps/Sec: 1.12 + 43%|██████████████████████████████████████████████████████▊ | 10041/23458 [2:52:42<3:09:38, 1.18it/s][2025-04-24 11:53:52] (step=0033500) Train Loss: 5.4402, Train Steps/Sec: 1.17 + 43%|██████████████████████████████████████████████████████▉ | 10066/23458 [2:53:03<3:14:04, 1.15it/s][2025-04-24 11:54:13] (step=0033525) Train Loss: 5.3372, Train Steps/Sec: 1.17 + 43%|███████████████████████████████████████████████████████ | 10091/23458 [2:53:25<3:11:41, 1.16it/s][2025-04-24 11:54:35] (step=0033550) Train Loss: 5.4119, Train Steps/Sec: 1.13 + 43%|███████████████████████████████████████████████████████▏ | 10116/23458 [2:53:48<3:09:07, 1.18it/s][2025-04-24 11:54:58] (step=0033575) Train Loss: 5.4003, Train Steps/Sec: 1.08 + 43%|███████████████████████████████████████████████████████▎ | 10141/23458 [2:54:10<3:06:34, 1.19it/s][2025-04-24 11:55:20] (step=0033600) Train Loss: 5.3447, Train Steps/Sec: 1.17 + 43%|███████████████████████████████████████████████████████▍ | 10166/23458 [2:54:31<3:12:41, 1.15it/s][2025-04-24 11:55:41] (step=0033625) Train Loss: 5.3598, Train Steps/Sec: 1.17 + 43%|███████████████████████████████████████████████████████▌ | 10191/23458 [2:54:52<3:08:46, 1.17it/s][2025-04-24 11:56:03] (step=0033650) Train Loss: 5.4084, Train Steps/Sec: 1.17 + 44%|███████████████████████████████████████████████████████▋ | 10216/23458 [2:55:16<3:08:45, 1.17it/s][2025-04-24 11:56:26] (step=0033675) Train Loss: 5.4417, Train Steps/Sec: 1.07 + 44%|███████████████████████████████████████████████████████▉ | 10241/23458 [2:55:37<3:05:42, 1.19it/s][2025-04-24 11:56:47] (step=0033700) Train Loss: 5.3953, Train Steps/Sec: 1.17 + 44%|████████████████████████████████████████████████████████ | 10266/23458 [2:55:59<3:11:53, 1.15it/s][2025-04-24 11:57:09] (step=0033725) Train Loss: 5.3879, Train Steps/Sec: 1.17 + 44%|████████████████████████████████████████████████████████▏ | 10291/23458 [2:56:20<3:07:21, 1.17it/s][2025-04-24 11:57:30] (step=0033750) Train Loss: 5.3586, Train Steps/Sec: 1.17 + 44%|████████████████████████████████████████████████████████▎ | 10316/23458 [2:56:41<3:05:39, 1.18it/s][2025-04-24 11:57:51] (step=0033775) Train Loss: 5.4001, Train Steps/Sec: 1.17 + 44%|████████████████████████████████████████████████████████▍ | 10341/23458 [2:57:02<3:04:35, 1.18it/s][2025-04-24 11:58:13] (step=0033800) Train Loss: 5.4391, Train Steps/Sec: 1.17 + 44%|████████████████████████████████████████████████████████▌ | 10366/23458 [2:57:24<3:09:12, 1.15it/s][2025-04-24 11:58:34] (step=0033825) Train Loss: 5.3674, Train Steps/Sec: 1.17 + 44%|████████████████████████████████████████████████████████▋ | 10391/23458 [2:57:46<3:09:53, 1.15it/s][2025-04-24 11:58:56] (step=0033850) Train Loss: 5.3565, Train Steps/Sec: 1.12 + 44%|████████████████████████████████████████████████████████▊ | 10416/23458 [2:58:08<3:05:22, 1.17it/s][2025-04-24 11:59:18] (step=0033875) Train Loss: 5.3990, Train Steps/Sec: 1.17 + 45%|████████████████████████████████████████████████████████▉ | 10441/23458 [2:58:29<3:03:16, 1.18it/s][2025-04-24 11:59:39] (step=0033900) Train Loss: 5.3598, Train Steps/Sec: 1.17 + 45%|█████████████████████████████████████████████████████████ | 10466/23458 [2:58:50<3:07:21, 1.16it/s][2025-04-24 12:00:01] (step=0033925) Train Loss: 5.4551, Train Steps/Sec: 1.17 + 45%|█████████████████████████████████████████████████████████▏ | 10491/23458 [2:59:12<3:04:48, 1.17it/s][2025-04-24 12:00:22] (step=0033950) Train Loss: 5.3448, Train Steps/Sec: 1.17 + 45%|█████████████████████████████████████████████████████████▍ | 10516/23458 [2:59:34<3:09:03, 1.14it/s][2025-04-24 12:00:44] (step=0033975) Train Loss: 5.4253, Train Steps/Sec: 1.13 + 45%|█████████████████████████████████████████████████████████▌ | 10541/23458 [2:59:55<3:01:50, 1.18it/s][2025-04-24 12:01:05] (step=0034000) Train Loss: 5.3884, Train Steps/Sec: 1.17 +You are using a model of type blip-2 to instantiate a model of type instructblip. This is not supported for all configurations of models and can yield errors. +[2025-04-24 12:01:05] text_config is None. Initializing the text config with default values (`OPTConfig`). +model_name_or_path:/tmp/haozhezhao/model/blip2-flan-t5-xl +image_place_holder: +tokenizer length after expend 32101 +tokenizer length before expend 32100 +100%|████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████| 4/4 [03:59<00:00, 59.91s/it] +[2025-04-24 12:06:13] Finish Eval in 34000 steps...█████████████████████████████████████████████████████████████████████████████████████| 4/4 [03:59<00:00, 59.56s/it] +processing left-padding... +processing left-padding... +processing left-padding... +processing left-padding... +[2025-04-24 12:06:33] Saved checkpoint to checkpoint/Afteri2i_100kckpt_LLAVAT5_empty_Fixed_just_segment_pretrain_stage3_dreambench_recap_400k_t2i_400k_flux_200k_midjourney_150k_recovery_150k_3e4_epoch2_no_rotated/000-GPT-XL/checkpoints/0034000.pt +[2025-04-24 12:06:35] Removed old checkpoint: checkpoint/Afteri2i_100kckpt_LLAVAT5_empty_Fixed_just_segment_pretrain_stage3_dreambench_recap_400k_t2i_400k_flux_200k_midjourney_150k_recovery_150k_3e4_epoch2_no_rotated/000-GPT-XL/checkpoints/0032000.pt + 45%|█████████████████████████████████████████████████████████▋ | 10566/23458 [3:05:46<3:10:08, 1.13it/s][2025-04-24 12:06:57] (step=0034025) Train Loss: 5.4368, Train Steps/Sec: 0.07 + 45%|█████████████████████████████████████████████████████████▊ | 10591/23458 [3:06:08<3:04:17, 1.16it/s][2025-04-24 12:07:18] (step=0034050) Train Loss: 5.2764, Train Steps/Sec: 1.17 + 45%|█████████████████████████████████████████████████████████▉ | 10616/23458 [3:06:29<3:02:00, 1.18it/s][2025-04-24 12:07:39] (step=0034075) Train Loss: 5.4176, Train Steps/Sec: 1.17 + 45%|██████████████████████████████████████████████████████████ | 10641/23458 [3:06:50<2:59:42, 1.19it/s][2025-04-24 12:08:01] (step=0034100) Train Loss: 5.3790, Train Steps/Sec: 1.17 + 45%|██████████████████████████████████████████████████████████▏ | 10666/23458 [3:07:12<3:05:16, 1.15it/s][2025-04-24 12:08:22] (step=0034125) Train Loss: 5.3458, Train Steps/Sec: 1.17 + 46%|██████████████████████████████████████████████████████████▎ | 10691/23458 [3:07:34<3:02:03, 1.17it/s][2025-04-24 12:08:44] (step=0034150) Train Loss: 5.3604, Train Steps/Sec: 1.12 + 46%|██████████████████████████████████████████████████████████▍ | 10716/23458 [3:07:55<3:00:15, 1.18it/s][2025-04-24 12:09:06] (step=0034175) Train Loss: 5.4141, Train Steps/Sec: 1.17 + 46%|██████████████████████████████████████████████████████████▌ | 10741/23458 [3:08:18<3:07:27, 1.13it/s][2025-04-24 12:09:28] (step=0034200) Train Loss: 5.3816, Train Steps/Sec: 1.12 + 46%|██████████████████████████████████████████████████████████▋ | 10766/23458 [3:08:41<3:07:46, 1.13it/s][2025-04-24 12:09:51] (step=0034225) Train Loss: 5.3233, Train Steps/Sec: 1.09 + 46%|██████████████████████████████████████████████████████████▉ | 10791/23458 [3:09:02<3:00:58, 1.17it/s][2025-04-24 12:10:12] (step=0034250) Train Loss: 5.3481, Train Steps/Sec: 1.17 + 46%|███████████████████████████████████████████████████████████ | 10816/23458 [3:09:24<2:59:03, 1.18it/s][2025-04-24 12:10:34] (step=0034275) Train Loss: 5.4116, Train Steps/Sec: 1.17 + 46%|███████████████████████████████████████████████████████████▏ | 10841/23458 [3:09:45<2:57:11, 1.19it/s][2025-04-24 12:10:55] (step=0034300) Train Loss: 5.4265, Train Steps/Sec: 1.17 + 46%|███████████████████████████████████████████████████████████▎ | 10866/23458 [3:10:07<3:02:20, 1.15it/s][2025-04-24 12:11:17] (step=0034325) Train Loss: 5.4159, Train Steps/Sec: 1.13 + 46%|███████████████████████████████████████████████████████████▍ | 10891/23458 [3:10:29<3:00:37, 1.16it/s][2025-04-24 12:11:39] (step=0034350) Train Loss: 5.4816, Train Steps/Sec: 1.17 + 47%|███████████████████████████████████████████████████████████▌ | 10916/23458 [3:10:50<2:57:16, 1.18it/s][2025-04-24 12:12:00] (step=0034375) Train Loss: 5.4426, Train Steps/Sec: 1.17 + 47%|███████████████████████████████████████████████████████████▋ | 10941/23458 [3:11:12<2:58:30, 1.17it/s][2025-04-24 12:12:23] (step=0034400) Train Loss: 5.3004, Train Steps/Sec: 1.11 + 47%|███████████████████████████████████████████████████████████▊ | 10966/23458 [3:11:34<3:00:53, 1.15it/s][2025-04-24 12:12:44] (step=0034425) Train Loss: 5.3355, Train Steps/Sec: 1.17 + 47%|███████████████████████████████████████████████████████████▉ | 10991/23458 [3:11:55<2:57:29, 1.17it/s][2025-04-24 12:13:05] (step=0034450) Train Loss: 5.3511, Train Steps/Sec: 1.17 + 47%|████████████████████████████████████████████████████████████ | 11016/23458 [3:12:17<2:56:03, 1.18it/s][2025-04-24 12:13:27] (step=0034475) Train Loss: 5.4426, Train Steps/Sec: 1.17 + 47%|████████████████████████████████████████████████████████████▏ | 11041/23458 [3:12:38<2:54:51, 1.18it/s][2025-04-24 12:13:48] (step=0034500) Train Loss: 5.3739, Train Steps/Sec: 1.16 + 47%|████████████████████████████████████████████████████████████▍ | 11066/23458 [3:12:59<2:59:20, 1.15it/s][2025-04-24 12:14:09] (step=0034525) Train Loss: 5.3407, Train Steps/Sec: 1.17 + 47%|████████████████████████████████████████████████████████████▌ | 11091/23458 [3:13:21<2:56:20, 1.17it/s][2025-04-24 12:14:31] (step=0034550) Train Loss: 5.3222, Train Steps/Sec: 1.17 + 47%|████████████████████████████████████████████████████████████▋ | 11116/23458 [3:13:43<3:05:32, 1.11it/s][2025-04-24 12:14:53] (step=0034575) Train Loss: 5.3590, Train Steps/Sec: 1.12 + 47%|████████████████████████████████████████████████████████████▊ | 11141/23458 [3:14:05<2:53:30, 1.18it/s][2025-04-24 12:15:15] (step=0034600) Train Loss: 5.4122, Train Steps/Sec: 1.16 + 48%|████████████████████████████████████████████████████████████▉ | 11166/23458 [3:14:26<2:58:45, 1.15it/s][2025-04-24 12:15:36] (step=0034625) Train Loss: 5.4179, Train Steps/Sec: 1.17 + 48%|█████████████████████████████████████████████████████████████ | 11191/23458 [3:14:47<2:54:20, 1.17it/s][2025-04-24 12:15:57] (step=0034650) Train Loss: 5.4255, Train Steps/Sec: 1.17 + 48%|█████████████████████████████████████████████████████████████▏ | 11216/23458 [3:15:10<2:52:45, 1.18it/s][2025-04-24 12:16:20] (step=0034675) Train Loss: 5.3072, Train Steps/Sec: 1.12 + 48%|█████████████████████████████████████████████████████████████▎ | 11241/23458 [3:15:31<2:53:00, 1.18it/s][2025-04-24 12:16:41] (step=0034700) Train Loss: 5.3647, Train Steps/Sec: 1.16 + 48%|█████████████████████████████████████████████████████████████▍ | 11266/23458 [3:15:52<2:58:38, 1.14it/s][2025-04-24 12:17:03] (step=0034725) Train Loss: 5.4027, Train Steps/Sec: 1.17 + 48%|█████████████████████████████████████████████████████████████▌ | 11291/23458 [3:16:14<2:53:09, 1.17it/s][2025-04-24 12:17:24] (step=0034750) Train Loss: 5.3723, Train Steps/Sec: 1.17 + 48%|█████████████████████████████████████████████████████████████▋ | 11316/23458 [3:16:35<2:51:24, 1.18it/s][2025-04-24 12:17:45] (step=0034775) Train Loss: 5.4882, Train Steps/Sec: 1.17 + 48%|█████████████████████████████████████████████████████████████▉ | 11341/23458 [3:16:57<2:51:13, 1.18it/s][2025-04-24 12:18:07] (step=0034800) Train Loss: 5.3720, Train Steps/Sec: 1.12 + 48%|██████████████████████████████████████████████████████████████ | 11366/23458 [3:17:19<2:54:54, 1.15it/s][2025-04-24 12:18:29] (step=0034825) Train Loss: 5.3544, Train Steps/Sec: 1.17 + 49%|██████████████████████████████████████████████████████████████▏ | 11391/23458 [3:17:40<2:51:16, 1.17it/s][2025-04-24 12:18:50] (step=0034850) Train Loss: 5.4132, Train Steps/Sec: 1.17 + 49%|██████████████████████████████████████████████████████████████▎ | 11416/23458 [3:18:03<3:46:35, 1.13s/it][2025-04-24 12:19:13] (step=0034875) Train Loss: 5.3859, Train Steps/Sec: 1.08 + 49%|██████████████████████████████████████████████████████████████▍ | 11441/23458 [3:18:25<2:48:44, 1.19it/s][2025-04-24 12:19:36] (step=0034900) Train Loss: 5.4274, Train Steps/Sec: 1.12 + 49%|██████████████████████████████████████████████████████████████▌ | 11466/23458 [3:18:47<2:53:16, 1.15it/s][2025-04-24 12:19:57] (step=0034925) Train Loss: 5.2312, Train Steps/Sec: 1.17 + 49%|██████████████████████████████████████████████████████████████▋ | 11491/23458 [3:19:08<2:50:43, 1.17it/s][2025-04-24 12:20:18] (step=0034950) Train Loss: 5.3792, Train Steps/Sec: 1.17 + 49%|██████████████████████████████████████████████████████████████▊ | 11516/23458 [3:19:30<3:07:19, 1.06it/s][2025-04-24 12:20:40] (step=0034975) Train Loss: 5.3106, Train Steps/Sec: 1.12 + 49%|██████████████████████████████████████████████████████████████▉ | 11541/23458 [3:19:52<2:47:44, 1.18it/s][2025-04-24 12:21:02] (step=0035000) Train Loss: 5.4351, Train Steps/Sec: 1.16 + 49%|███████████████████████████████████████████████████████████████ | 11566/23458 [3:20:13<2:52:33, 1.15it/s][2025-04-24 12:21:23] (step=0035025) Train Loss: 5.3471, Train Steps/Sec: 1.17 + 49%|███████████████████████████████████████████████████████████████▏ | 11591/23458 [3:20:35<2:48:44, 1.17it/s][2025-04-24 12:21:45] (step=0035050) Train Loss: 5.4253, Train Steps/Sec: 1.17 + 50%|███████████████████████████████████████████████████████████████▍ | 11616/23458 [3:20:56<2:48:16, 1.17it/s][2025-04-24 12:22:06] (step=0035075) Train Loss: 5.4213, Train Steps/Sec: 1.17 + 50%|███████████████████████████████████████████████████████████████▌ | 11641/23458 [3:21:17<2:46:41, 1.18it/s][2025-04-24 12:22:28] (step=0035100) Train Loss: 5.3740, Train Steps/Sec: 1.17 + 50%|███████████████████████████████████████████████████████████████▋ | 11666/23458 [3:21:40<2:52:18, 1.14it/s][2025-04-24 12:22:50] (step=0035125) Train Loss: 5.4672, Train Steps/Sec: 1.12 + 50%|███████████████████████████████████████████████████████████████▊ | 11691/23458 [3:22:01<2:47:59, 1.17it/s][2025-04-24 12:23:11] (step=0035150) Train Loss: 5.4553, Train Steps/Sec: 1.17 + 50%|███████████████████████████████████████████████████████████████▉ | 11716/23458 [3:22:23<2:46:22, 1.18it/s][2025-04-24 12:23:33] (step=0035175) Train Loss: 5.3695, Train Steps/Sec: 1.17 + 50%|████████████████████████████████████████████████████████████████ | 11741/23458 [3:22:44<2:45:20, 1.18it/s][2025-04-24 12:23:54] (step=0035200) Train Loss: 5.4445, Train Steps/Sec: 1.16 + 50%|████████████████████████████████████████████████████████████████▏ | 11766/23458 [3:23:05<2:48:52, 1.15it/s][2025-04-24 12:24:16] (step=0035225) Train Loss: 5.3601, Train Steps/Sec: 1.17 + 50%|████████████████████████████████████████████████████████████████▎ | 11791/23458 [3:23:27<2:46:42, 1.17it/s][2025-04-24 12:24:37] (step=0035250) Train Loss: 5.3907, Train Steps/Sec: 1.17 + 50%|████████████████████████████████████████████████████████████████▍ | 11816/23458 [3:23:48<2:44:30, 1.18it/s][2025-04-24 12:24:58] (step=0035275) Train Loss: 5.4913, Train Steps/Sec: 1.17 + 50%|████████████████████████████████████████████████████████████████▌ | 11841/23458 [3:24:10<2:48:28, 1.15it/s][2025-04-24 12:25:21] (step=0035300) Train Loss: 5.4522, Train Steps/Sec: 1.11 + 51%|████████████████████████████████████████████████████████████████▋ | 11866/23458 [3:24:33<2:49:33, 1.14it/s][2025-04-24 12:25:43] (step=0035325) Train Loss: 5.3313, Train Steps/Sec: 1.12 + 51%|████████████████████████████████████████████████████████████████▉ | 11891/23458 [3:24:54<2:45:13, 1.17it/s][2025-04-24 12:26:04] (step=0035350) Train Loss: 5.4184, Train Steps/Sec: 1.17 + 51%|█████████████████████████████████████████████████████████████████ | 11916/23458 [3:25:16<2:43:52, 1.17it/s][2025-04-24 12:26:26] (step=0035375) Train Loss: 5.3622, Train Steps/Sec: 1.17 + 51%|█████████████████████████████████████████████████████████████████▏ | 11941/23458 [3:25:37<2:42:24, 1.18it/s][2025-04-24 12:26:47] (step=0035400) Train Loss: 5.4354, Train Steps/Sec: 1.16 + 51%|█████████████████████████████████████████████████████████████████▎ | 11966/23458 [3:25:58<2:46:36, 1.15it/s][2025-04-24 12:27:09] (step=0035425) Train Loss: 5.3684, Train Steps/Sec: 1.17 + 51%|█████████████████████████████████████████████████████████████████▍ | 11991/23458 [3:26:21<2:52:21, 1.11it/s][2025-04-24 12:27:31] (step=0035450) Train Loss: 5.3190, Train Steps/Sec: 1.12 + 51%|█████████████████████████████████████████████████████████████████▌ | 12016/23458 [3:26:42<2:41:42, 1.18it/s][2025-04-24 12:27:52] (step=0035475) Train Loss: 5.3557, Train Steps/Sec: 1.17 + 51%|█████████████████████████████████████████████████████████████████▋ | 12041/23458 [3:27:03<2:40:24, 1.19it/s][2025-04-24 12:28:14] (step=0035500) Train Loss: 5.3941, Train Steps/Sec: 1.17 + 51%|█████████████████████████████████████████████████████████████████▊ | 12066/23458 [3:27:26<2:46:25, 1.14it/s][2025-04-24 12:28:36] (step=0035525) Train Loss: 5.4323, Train Steps/Sec: 1.13 + 52%|█████████████████████████████████████████████████████████████████▉ | 12091/23458 [3:27:49<2:43:24, 1.16it/s][2025-04-24 12:28:59] (step=0035550) Train Loss: 5.3760, Train Steps/Sec: 1.08 + 52%|██████████████████████████████████████████████████████████████████ | 12116/23458 [3:28:10<2:41:27, 1.17it/s][2025-04-24 12:29:20] (step=0035575) Train Loss: 5.3458, Train Steps/Sec: 1.17 + 52%|██████████████████████████████████████████████████████████████████▏ | 12141/23458 [3:28:32<2:39:17, 1.18it/s][2025-04-24 12:29:42] (step=0035600) Train Loss: 5.3111, Train Steps/Sec: 1.17 + 52%|██████████████████████████████████████████████████████████████████▍ | 12166/23458 [3:28:53<2:43:49, 1.15it/s][2025-04-24 12:30:03] (step=0035625) Train Loss: 5.4294, Train Steps/Sec: 1.17 + 52%|██████████████████████████████████████████████████████████████████▌ | 12191/23458 [3:29:15<2:40:07, 1.17it/s][2025-04-24 12:30:25] (step=0035650) Train Loss: 5.4234, Train Steps/Sec: 1.13 + 52%|██████████████████████████████████████████████████████████████████▋ | 12216/23458 [3:29:37<2:38:59, 1.18it/s][2025-04-24 12:30:47] (step=0035675) Train Loss: 5.3680, Train Steps/Sec: 1.17 + 52%|██████████████████████████████████████████████████████████████████▊ | 12241/23458 [3:29:58<2:37:52, 1.18it/s][2025-04-24 12:31:08] (step=0035700) Train Loss: 5.3564, Train Steps/Sec: 1.17 + 52%|██████████████████████████████████████████████████████████████████▉ | 12266/23458 [3:30:19<2:41:44, 1.15it/s][2025-04-24 12:31:29] (step=0035725) Train Loss: 5.3365, Train Steps/Sec: 1.17 + 52%|███████████████████████████████████████████████████████████████████ | 12291/23458 [3:30:41<2:38:54, 1.17it/s][2025-04-24 12:31:51] (step=0035750) Train Loss: 5.4007, Train Steps/Sec: 1.17 + 53%|███████████████████████████████████████████████████████████████████▏ | 12316/23458 [3:31:02<2:37:50, 1.18it/s][2025-04-24 12:32:12] (step=0035775) Train Loss: 5.3369, Train Steps/Sec: 1.17 + 53%|███████████████████████████████████████████████████████████████████▎ | 12341/23458 [3:31:23<2:36:04, 1.19it/s][2025-04-24 12:32:33] (step=0035800) Train Loss: 5.3929, Train Steps/Sec: 1.17 + 53%|███████████████████████████████████████████████████████████████████▍ | 12366/23458 [3:31:45<2:41:04, 1.15it/s][2025-04-24 12:32:55] (step=0035825) Train Loss: 5.3558, Train Steps/Sec: 1.17 + 53%|███████████████████████████████████████████████████████████████████▌ | 12391/23458 [3:32:07<2:38:21, 1.16it/s][2025-04-24 12:33:17] (step=0035850) Train Loss: 5.3544, Train Steps/Sec: 1.12 + 53%|███████████████████████████████████████████████████████████████████▋ | 12416/23458 [3:32:28<2:35:54, 1.18it/s][2025-04-24 12:33:39] (step=0035875) Train Loss: 5.3696, Train Steps/Sec: 1.17 + 53%|███████████████████████████████████████████████████████████████████▉ | 12441/23458 [3:32:50<2:34:50, 1.19it/s][2025-04-24 12:34:00] (step=0035900) Train Loss: 5.3462, Train Steps/Sec: 1.17 + 53%|████████████████████████████████████████████████████████████████████ | 12466/23458 [3:33:11<2:39:00, 1.15it/s][2025-04-24 12:34:21] (step=0035925) Train Loss: 5.4178, Train Steps/Sec: 1.17 + 53%|████████████████████████████████████████████████████████████████████▏ | 12491/23458 [3:33:33<2:36:33, 1.17it/s][2025-04-24 12:34:43] (step=0035950) Train Loss: 5.3463, Train Steps/Sec: 1.17 + 53%|████████████████████████████████████████████████████████████████████▎ | 12516/23458 [3:33:54<2:34:19, 1.18it/s][2025-04-24 12:35:05] (step=0035975) Train Loss: 5.2341, Train Steps/Sec: 1.13 + 53%|████████████████████████████████████████████████████████████████████▍ | 12541/23458 [3:34:16<2:33:33, 1.18it/s][2025-04-24 12:35:26] (step=0036000) Train Loss: 5.3826, Train Steps/Sec: 1.17 +You are using a model of type blip-2 to instantiate a model of type instructblip. This is not supported for all configurations of models and can yield errors. +[2025-04-24 12:35:26] text_config is None. Initializing the text config with default values (`OPTConfig`). +model_name_or_path:/tmp/haozhezhao/model/blip2-flan-t5-xl +image_place_holder: +tokenizer length after expend 32101 +tokenizer length before expend 32100 +100%|████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████| 4/4 [03:59<00:00, 59.82s/it] +[2025-04-24 12:40:35] Finish Eval in 36000 steps...█████████████████████████████████████████████████████████████████████████████████████| 4/4 [03:58<00:00, 59.48s/it] +processing left-padding... +processing left-padding... +processing left-padding... +processing left-padding... +[2025-04-24 12:40:54] Saved checkpoint to checkpoint/Afteri2i_100kckpt_LLAVAT5_empty_Fixed_just_segment_pretrain_stage3_dreambench_recap_400k_t2i_400k_flux_200k_midjourney_150k_recovery_150k_3e4_epoch2_no_rotated/000-GPT-XL/checkpoints/0036000.pt +[2025-04-24 12:40:56] Removed old checkpoint: checkpoint/Afteri2i_100kckpt_LLAVAT5_empty_Fixed_just_segment_pretrain_stage3_dreambench_recap_400k_t2i_400k_flux_200k_midjourney_150k_recovery_150k_3e4_epoch2_no_rotated/000-GPT-XL/checkpoints/0034000.pt + 54%|████████████████████████████████████████████████████████████████████▌ | 12566/23458 [3:40:08<2:47:41, 1.08it/s][2025-04-24 12:41:18] (step=0036025) Train Loss: 5.3762, Train Steps/Sec: 0.07 + 54%|████████████████████████████████████████████████████████████████████▋ | 12591/23458 [3:40:30<2:34:16, 1.17it/s][2025-04-24 12:41:40] (step=0036050) Train Loss: 5.3532, Train Steps/Sec: 1.17 + 54%|████████████████████████████████████████████████████████████████████▊ | 12616/23458 [3:40:51<2:32:44, 1.18it/s][2025-04-24 12:42:01] (step=0036075) Train Loss: 5.3544, Train Steps/Sec: 1.17 + 54%|████████████████████████████████████████████████████████████████████▉ | 12641/23458 [3:41:12<2:31:50, 1.19it/s][2025-04-24 12:42:23] (step=0036100) Train Loss: 5.3530, Train Steps/Sec: 1.17 + 54%|█████████████████████████████████████████████████████████████████████ | 12666/23458 [3:41:35<2:36:41, 1.15it/s][2025-04-24 12:42:45] (step=0036125) Train Loss: 5.4050, Train Steps/Sec: 1.12 + 54%|█████████████████████████████████████████████████████████████████████▏ | 12691/23458 [3:41:56<2:33:20, 1.17it/s][2025-04-24 12:43:06] (step=0036150) Train Loss: 5.3430, Train Steps/Sec: 1.17 + 54%|█████████████████████████████████████████████████████████████████████▍ | 12716/23458 [3:42:17<2:31:34, 1.18it/s][2025-04-24 12:43:27] (step=0036175) Train Loss: 5.3489, Train Steps/Sec: 1.17 + 54%|█████████████████████████████████████████████████████████████████████▌ | 12741/23458 [3:42:41<2:45:46, 1.08it/s][2025-04-24 12:43:51] (step=0036200) Train Loss: 5.4462, Train Steps/Sec: 1.04 + 54%|█████████████████████████████████████████████████████████████████████▋ | 12766/23458 [3:43:03<2:34:57, 1.15it/s][2025-04-24 12:44:13] (step=0036225) Train Loss: 5.3558, Train Steps/Sec: 1.17 + 55%|█████████████████████████████████████████████████████████████████████▊ | 12791/23458 [3:43:24<2:32:01, 1.17it/s][2025-04-24 12:44:34] (step=0036250) Train Loss: 5.3332, Train Steps/Sec: 1.17 + 55%|█████████████████████████████████████████████████████████████████████▉ | 12816/23458 [3:43:45<2:32:46, 1.16it/s][2025-04-24 12:44:56] (step=0036275) Train Loss: 5.4207, Train Steps/Sec: 1.17 + 55%|██████████████████████████████████████████████████████████████████████ | 12841/23458 [3:44:08<2:30:20, 1.18it/s][2025-04-24 12:45:18] (step=0036300) Train Loss: 5.3440, Train Steps/Sec: 1.12 + 55%|██████████████████████████████████████████████████████████████████████▏ | 12866/23458 [3:44:29<2:34:44, 1.14it/s][2025-04-24 12:45:39] (step=0036325) Train Loss: 5.3846, Train Steps/Sec: 1.17 + 55%|██████████████████████████████████████████████████████████████████████▎ | 12891/23458 [3:44:51<2:31:11, 1.16it/s][2025-04-24 12:46:01] (step=0036350) Train Loss: 5.4336, Train Steps/Sec: 1.17 + 55%|██████████████████████████████████████████████████████████████████████▍ | 12916/23458 [3:45:12<2:29:29, 1.18it/s][2025-04-24 12:46:22] (step=0036375) Train Loss: 5.3890, Train Steps/Sec: 1.17 + 55%|██████████████████████████████████████████████████████████████████████▌ | 12941/23458 [3:45:33<2:27:37, 1.19it/s][2025-04-24 12:46:43] (step=0036400) Train Loss: 5.4087, Train Steps/Sec: 1.17 + 55%|██████████████████████████████████████████████████████████████████████▋ | 12966/23458 [3:45:55<2:32:02, 1.15it/s][2025-04-24 12:47:05] (step=0036425) Train Loss: 5.4143, Train Steps/Sec: 1.17 + 55%|██████████████████████████████████████████████████████████████████████▉ | 12991/23458 [3:46:16<2:29:16, 1.17it/s][2025-04-24 12:47:26] (step=0036450) Train Loss: 5.3947, Train Steps/Sec: 1.17 + 55%|███████████████████████████████████████████████████████████████████████ | 13016/23458 [3:46:37<2:27:32, 1.18it/s][2025-04-24 12:47:47] (step=0036475) Train Loss: 5.3427, Train Steps/Sec: 1.17 + 56%|███████████████████████████████████████████████████████████████████████▏ | 13041/23458 [3:46:59<2:26:30, 1.19it/s][2025-04-24 12:48:09] (step=0036500) Train Loss: 5.3745, Train Steps/Sec: 1.17 + 56%|███████████████████████████████████████████████████████████████████████▎ | 13066/23458 [3:47:20<2:31:09, 1.15it/s][2025-04-24 12:48:30] (step=0036525) Train Loss: 5.2770, Train Steps/Sec: 1.17 + 56%|███████████████████████████████████████████████████████████████████████▍ | 13091/23458 [3:47:41<2:28:00, 1.17it/s][2025-04-24 12:48:52] (step=0036550) Train Loss: 5.3725, Train Steps/Sec: 1.17 + 56%|███████████████████████████████████████████████████████████████████████▌ | 13116/23458 [3:48:04<2:26:31, 1.18it/s][2025-04-24 12:49:14] (step=0036575) Train Loss: 5.3753, Train Steps/Sec: 1.12 + 56%|███████████████████████████████████████████████████████████████████████▋ | 13141/23458 [3:48:26<2:25:23, 1.18it/s][2025-04-24 12:49:36] (step=0036600) Train Loss: 5.4663, Train Steps/Sec: 1.15 + 56%|███████████████████████████████████████████████████████████████████████▊ | 13166/23458 [3:48:47<2:28:48, 1.15it/s][2025-04-24 12:49:57] (step=0036625) Train Loss: 5.3558, Train Steps/Sec: 1.17 + 56%|███████████████████████████████████████████████████████████████████████▉ | 13191/23458 [3:49:08<2:26:44, 1.17it/s][2025-04-24 12:50:19] (step=0036650) Train Loss: 5.3299, Train Steps/Sec: 1.17 + 56%|████████████████████████████████████████████████████████████████████████ | 13216/23458 [3:49:31<2:24:48, 1.18it/s][2025-04-24 12:50:41] (step=0036675) Train Loss: 5.3964, Train Steps/Sec: 1.12 + 56%|████████████████████████████████████████████████████████████████████████▎ | 13241/23458 [3:49:52<2:23:36, 1.19it/s][2025-04-24 12:51:02] (step=0036700) Train Loss: 5.4503, Train Steps/Sec: 1.17 + 57%|████████████████████████████████████████████████████████████████████████▍ | 13266/23458 [3:50:13<2:28:20, 1.15it/s][2025-04-24 12:51:24] (step=0036725) Train Loss: 5.3401, Train Steps/Sec: 1.17 + 57%|████████████████████████████████████████████████████████████████████████▌ | 13291/23458 [3:50:36<2:37:53, 1.07it/s][2025-04-24 12:51:46] (step=0036750) Train Loss: 5.4083, Train Steps/Sec: 1.12 + 57%|████████████████████████████████████████████████████████████████████████▋ | 13316/23458 [3:50:58<2:24:44, 1.17it/s][2025-04-24 12:52:08] (step=0036775) Train Loss: 5.3950, Train Steps/Sec: 1.12 + 57%|████████████████████████████████████████████████████████████████████████▊ | 13341/23458 [3:51:19<2:22:31, 1.18it/s][2025-04-24 12:52:30] (step=0036800) Train Loss: 5.4297, Train Steps/Sec: 1.17 + 57%|████████████████████████████████████████████████████████████████████████▉ | 13366/23458 [3:51:41<2:26:28, 1.15it/s][2025-04-24 12:52:51] (step=0036825) Train Loss: 5.3609, Train Steps/Sec: 1.17 + 57%|█████████████████████████████████████████████████████████████████████████ | 13391/23458 [3:52:04<2:57:07, 1.06s/it][2025-04-24 12:53:14] (step=0036850) Train Loss: 5.3463, Train Steps/Sec: 1.08 + 57%|█████████████████████████████████████████████████████████████████████████▏ | 13416/23458 [3:52:26<2:22:29, 1.17it/s][2025-04-24 12:53:36] (step=0036875) Train Loss: 5.3616, Train Steps/Sec: 1.13 + 57%|█████████████████████████████████████████████████████████████████████████▎ | 13441/23458 [3:52:48<2:20:54, 1.18it/s][2025-04-24 12:53:58] (step=0036900) Train Loss: 5.4154, Train Steps/Sec: 1.17 + 57%|█████████████████████████████████████████████████████████████████████████▍ | 13466/23458 [3:53:09<2:25:22, 1.15it/s][2025-04-24 12:54:19] (step=0036925) Train Loss: 5.3826, Train Steps/Sec: 1.17 + 58%|█████████████████████████████████████████████████████████████████████████▌ | 13491/23458 [3:53:31<2:50:39, 1.03s/it][2025-04-24 12:54:41] (step=0036950) Train Loss: 5.3976, Train Steps/Sec: 1.13 + 58%|█████████████████████████████████████████████████████████████████████████▊ | 13516/23458 [3:53:52<2:20:18, 1.18it/s][2025-04-24 12:55:03] (step=0036975) Train Loss: 5.3232, Train Steps/Sec: 1.17 + 58%|█████████████████████████████████████████████████████████████████████████▉ | 13541/23458 [3:54:14<2:19:40, 1.18it/s][2025-04-24 12:55:24] (step=0037000) Train Loss: 5.4052, Train Steps/Sec: 1.17 + 58%|██████████████████████████████████████████████████████████████████████████ | 13566/23458 [3:54:35<2:24:16, 1.14it/s][2025-04-24 12:55:45] (step=0037025) Train Loss: 5.3924, Train Steps/Sec: 1.17 + 58%|██████████████████████████████████████████████████████████████████████████▏ | 13591/23458 [3:54:57<2:20:42, 1.17it/s][2025-04-24 12:56:07] (step=0037050) Train Loss: 5.4273, Train Steps/Sec: 1.17 + 58%|██████████████████████████████████████████████████████████████████████████▎ | 13616/23458 [3:55:18<2:19:00, 1.18it/s][2025-04-24 12:56:28] (step=0037075) Train Loss: 5.3861, Train Steps/Sec: 1.17 + 58%|██████████████████████████████████████████████████████████████████████████▍ | 13641/23458 [3:55:39<2:18:19, 1.18it/s][2025-04-24 12:56:50] (step=0037100) Train Loss: 5.4010, Train Steps/Sec: 1.16 + 58%|██████████████████████████████████████████████████████████████████████████▌ | 13666/23458 [3:56:01<2:22:28, 1.15it/s][2025-04-24 12:57:11] (step=0037125) Train Loss: 5.3616, Train Steps/Sec: 1.17 + 58%|██████████████████████████████████████████████████████████████████████████▋ | 13691/23458 [3:56:22<2:19:12, 1.17it/s][2025-04-24 12:57:32] (step=0037150) Train Loss: 5.3275, Train Steps/Sec: 1.17 + 58%|██████████████████████████████████████████████████████████████████████████▊ | 13716/23458 [3:56:43<2:17:53, 1.18it/s][2025-04-24 12:57:54] (step=0037175) Train Loss: 5.4265, Train Steps/Sec: 1.17 + 59%|██████████████████████████████████████████████████████████████████████████▉ | 13741/23458 [3:57:05<2:16:49, 1.18it/s][2025-04-24 12:58:15] (step=0037200) Train Loss: 5.4046, Train Steps/Sec: 1.17 + 59%|███████████████████████████████████████████████████████████████████████████ | 13766/23458 [3:57:26<2:20:13, 1.15it/s][2025-04-24 12:58:36] (step=0037225) Train Loss: 5.4050, Train Steps/Sec: 1.17 + 59%|███████████████████████████████████████████████████████████████████████████▎ | 13791/23458 [3:57:48<2:18:22, 1.16it/s][2025-04-24 12:58:58] (step=0037250) Train Loss: 5.3538, Train Steps/Sec: 1.17 + 59%|███████████████████████████████████████████████████████████████████████████▍ | 13816/23458 [3:58:09<2:16:25, 1.18it/s][2025-04-24 12:59:19] (step=0037275) Train Loss: 5.3219, Train Steps/Sec: 1.17 + 59%|███████████████████████████████████████████████████████████████████████████▌ | 13841/23458 [3:58:31<2:16:49, 1.17it/s][2025-04-24 12:59:42] (step=0037300) Train Loss: 5.3880, Train Steps/Sec: 1.11 + 59%|███████████████████████████████████████████████████████████████████████████▋ | 13866/23458 [3:58:54<2:19:08, 1.15it/s][2025-04-24 13:00:04] (step=0037325) Train Loss: 5.4442, Train Steps/Sec: 1.13 + 59%|███████████████████████████████████████████████████████████████████████████▊ | 13891/23458 [3:59:15<2:16:17, 1.17it/s][2025-04-24 13:00:25] (step=0037350) Train Loss: 5.3454, Train Steps/Sec: 1.17 + 59%|███████████████████████████████████████████████████████████████████████████▉ | 13916/23458 [3:59:36<2:14:16, 1.18it/s][2025-04-24 13:00:46] (step=0037375) Train Loss: 5.3918, Train Steps/Sec: 1.17 + 59%|████████████████████████████████████████████████████████████████████████████ | 13941/23458 [3:59:58<2:13:48, 1.19it/s][2025-04-24 13:01:08] (step=0037400) Train Loss: 5.4611, Train Steps/Sec: 1.17 + 60%|████████████████████████████████████████████████████████████████████████████▏ | 13966/23458 [4:00:20<2:28:52, 1.06it/s][2025-04-24 13:01:30] (step=0037425) Train Loss: 5.4276, Train Steps/Sec: 1.12 + 60%|████████████████████████████████████████████████████████████████████████████▎ | 13991/23458 [4:00:41<2:15:29, 1.16it/s][2025-04-24 13:01:51] (step=0037450) Train Loss: 5.4460, Train Steps/Sec: 1.17 + 60%|████████████████████████████████████████████████████████████████████████████▍ | 14016/23458 [4:01:04<2:29:56, 1.05it/s][2025-04-24 13:02:14] (step=0037475) Train Loss: 5.3067, Train Steps/Sec: 1.12 + 60%|████████████████████████████████████████████████████████████████████████████▌ | 14041/23458 [4:01:26<2:24:55, 1.08it/s][2025-04-24 13:02:36] (step=0037500) Train Loss: 5.3716, Train Steps/Sec: 1.13 + 60%|████████████████████████████████████████████████████████████████████████████▊ | 14066/23458 [4:01:49<2:15:59, 1.15it/s][2025-04-24 13:02:59] (step=0037525) Train Loss: 5.4218, Train Steps/Sec: 1.08 + 60%|████████████████████████████████████████████████████████████████████████████▉ | 14091/23458 [4:02:10<2:13:36, 1.17it/s][2025-04-24 13:03:20] (step=0037550) Train Loss: 5.4343, Train Steps/Sec: 1.17 + 60%|█████████████████████████████████████████████████████████████████████████████ | 14116/23458 [4:02:32<2:12:16, 1.18it/s][2025-04-24 13:03:42] (step=0037575) Train Loss: 5.3231, Train Steps/Sec: 1.17 + 60%|█████████████████████████████████████████████████████████████████████████████▏ | 14141/23458 [4:02:53<2:11:28, 1.18it/s][2025-04-24 13:04:03] (step=0037600) Train Loss: 5.3819, Train Steps/Sec: 1.17 + 60%|█████████████████████████████████████████████████████████████████████████████▎ | 14166/23458 [4:03:15<2:14:48, 1.15it/s][2025-04-24 13:04:25] (step=0037625) Train Loss: 5.4150, Train Steps/Sec: 1.13 + 60%|█████████████████████████████████████████████████████████████████████████████▍ | 14191/23458 [4:03:37<2:12:08, 1.17it/s][2025-04-24 13:04:47] (step=0037650) Train Loss: 5.3797, Train Steps/Sec: 1.17 + 61%|█████████████████████████████████████████████████████████████████████████████▌ | 14216/23458 [4:03:58<2:12:01, 1.17it/s][2025-04-24 13:05:08] (step=0037675) Train Loss: 5.3587, Train Steps/Sec: 1.17 + 61%|█████████████████████████████████████████████████████████████████████████████▋ | 14241/23458 [4:04:19<2:09:40, 1.18it/s][2025-04-24 13:05:29] (step=0037700) Train Loss: 5.4128, Train Steps/Sec: 1.17 + 61%|█████████████████████████████████████████████████████████████████████████████▊ | 14266/23458 [4:04:41<2:13:09, 1.15it/s][2025-04-24 13:05:51] (step=0037725) Train Loss: 5.4440, Train Steps/Sec: 1.17 + 61%|█████████████████████████████████████████████████████████████████████████████▉ | 14291/23458 [4:05:02<2:10:53, 1.17it/s][2025-04-24 13:06:12] (step=0037750) Train Loss: 5.4347, Train Steps/Sec: 1.17 + 61%|██████████████████████████████████████████████████████████████████████████████ | 14316/23458 [4:05:23<2:09:09, 1.18it/s][2025-04-24 13:06:34] (step=0037775) Train Loss: 5.4135, Train Steps/Sec: 1.17 + 61%|██████████████████████████████████████████████████████████████████████████████▎ | 14341/23458 [4:05:45<2:07:46, 1.19it/s][2025-04-24 13:06:55] (step=0037800) Train Loss: 5.3153, Train Steps/Sec: 1.16 + 61%|██████████████████████████████████████████████████████████████████████████████▍ | 14366/23458 [4:06:06<2:11:43, 1.15it/s][2025-04-24 13:07:16] (step=0037825) Train Loss: 5.3887, Train Steps/Sec: 1.17 + 61%|██████████████████████████████████████████████████████████████████████████████▌ | 14391/23458 [4:06:28<2:11:10, 1.15it/s][2025-04-24 13:07:38] (step=0037850) Train Loss: 5.3146, Train Steps/Sec: 1.17 + 61%|██████████████████████████████████████████████████████████████████████████████▋ | 14416/23458 [4:06:49<2:07:52, 1.18it/s][2025-04-24 13:07:59] (step=0037875) Train Loss: 5.3970, Train Steps/Sec: 1.17 + 62%|██████████████████████████████████████████████████████████████████████████████▊ | 14441/23458 [4:07:10<2:06:50, 1.18it/s][2025-04-24 13:08:21] (step=0037900) Train Loss: 5.3551, Train Steps/Sec: 1.17 + 62%|██████████████████████████████████████████████████████████████████████████████▉ | 14466/23458 [4:07:32<2:10:06, 1.15it/s][2025-04-24 13:08:42] (step=0037925) Train Loss: 5.3976, Train Steps/Sec: 1.17 + 62%|███████████████████████████████████████████████████████████████████████████████ | 14491/23458 [4:07:53<2:07:37, 1.17it/s][2025-04-24 13:09:03] (step=0037950) Train Loss: 5.3898, Train Steps/Sec: 1.17 + 62%|███████████████████████████████████████████████████████████████████████████████▏ | 14516/23458 [4:08:15<2:09:31, 1.15it/s][2025-04-24 13:09:25] (step=0037975) Train Loss: 5.3322, Train Steps/Sec: 1.13 + 62%|███████████████████████████████████████████████████████████████████████████████▎ | 14541/23458 [4:08:37<2:05:50, 1.18it/s][2025-04-24 13:09:47] (step=0038000) Train Loss: 5.3768, Train Steps/Sec: 1.17 +You are using a model of type blip-2 to instantiate a model of type instructblip. This is not supported for all configurations of models and can yield errors. +[2025-04-24 13:09:47] text_config is None. Initializing the text config with default values (`OPTConfig`). +model_name_or_path:/tmp/haozhezhao/model/blip2-flan-t5-xl +image_place_holder: +tokenizer length after expend 32101 +tokenizer length before expend 32100 +100%|████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████| 4/4 [04:00<00:00, 60.20s/it] +[2025-04-24 13:14:58] Finish Eval in 38000 steps...█████████████████████████████████████████████████████████████████████████████████████| 4/4 [04:00<00:00, 59.70s/it] +processing left-padding... +processing left-padding... +processing left-padding... +processing left-padding... +[2025-04-24 13:15:18] Saved checkpoint to checkpoint/Afteri2i_100kckpt_LLAVAT5_empty_Fixed_just_segment_pretrain_stage3_dreambench_recap_400k_t2i_400k_flux_200k_midjourney_150k_recovery_150k_3e4_epoch2_no_rotated/000-GPT-XL/checkpoints/0038000.pt +[2025-04-24 13:15:20] Removed old checkpoint: checkpoint/Afteri2i_100kckpt_LLAVAT5_empty_Fixed_just_segment_pretrain_stage3_dreambench_recap_400k_t2i_400k_flux_200k_midjourney_150k_recovery_150k_3e4_epoch2_no_rotated/000-GPT-XL/checkpoints/0036000.pt + 62%|███████████████████████████████████████████████████████████████████████████████▍ | 14566/23458 [4:14:33<2:13:16, 1.11it/s][2025-04-24 13:15:43] (step=0038025) Train Loss: 5.3576, Train Steps/Sec: 0.07 + 62%|███████████████████████████████████████████████████████████████████████████████▌ | 14591/23458 [4:14:54<2:07:09, 1.16it/s][2025-04-24 13:16:04] (step=0038050) Train Loss: 5.3539, Train Steps/Sec: 1.17 + 62%|███████████████████████████████████████████████████████████████████████████████▊ | 14616/23458 [4:15:16<2:05:44, 1.17it/s][2025-04-24 13:16:26] (step=0038075) Train Loss: 5.3652, Train Steps/Sec: 1.17 + 62%|███████████████████████████████████████████████████████████████████████████████▉ | 14641/23458 [4:15:38<2:04:02, 1.18it/s][2025-04-24 13:16:48] (step=0038100) Train Loss: 5.3404, Train Steps/Sec: 1.11 + 63%|████████████████████████████████████████████████████████████████████████████████ | 14666/23458 [4:16:00<2:07:14, 1.15it/s][2025-04-24 13:17:10] (step=0038125) Train Loss: 5.3054, Train Steps/Sec: 1.17 + 63%|████████████████████████████████████████████████████████████████████████████████▏ | 14691/23458 [4:16:21<2:04:34, 1.17it/s][2025-04-24 13:17:31] (step=0038150) Train Loss: 5.4078, Train Steps/Sec: 1.17 + 63%|████████████████████████████████████████████████████████████████████████████████▎ | 14716/23458 [4:16:45<2:22:40, 1.02it/s][2025-04-24 13:17:55] (step=0038175) Train Loss: 5.3976, Train Steps/Sec: 1.05 + 63%|████████████████████████████████████████████████████████████████████████████████▍ | 14741/23458 [4:17:07<2:48:50, 1.16s/it][2025-04-24 13:18:17] (step=0038200) Train Loss: 5.3705, Train Steps/Sec: 1.11 + 63%|████████████████████████████████████████████████████████████████████████████████▌ | 14766/23458 [4:17:29<2:06:04, 1.15it/s][2025-04-24 13:18:39] (step=0038225) Train Loss: 5.3780, Train Steps/Sec: 1.17 + 63%|████████████████████████████████████████████████████████████████████████████████▋ | 14791/23458 [4:17:50<2:03:54, 1.17it/s][2025-04-24 13:19:00] (step=0038250) Train Loss: 5.3937, Train Steps/Sec: 1.17 + 63%|████████████████████████████████████████████████████████████████████████████████▊ | 14816/23458 [4:18:12<2:03:21, 1.17it/s][2025-04-24 13:19:22] (step=0038275) Train Loss: 5.4124, Train Steps/Sec: 1.13 + 63%|████████████████████████████████████████████████████████████████████████████████▉ | 14841/23458 [4:18:34<2:01:18, 1.18it/s][2025-04-24 13:19:44] (step=0038300) Train Loss: 5.3106, Train Steps/Sec: 1.16 + 63%|█████████████████████████████████████████████████████████████████████████████████ | 14866/23458 [4:18:55<2:05:14, 1.14it/s][2025-04-24 13:20:05] (step=0038325) Train Loss: 5.4085, Train Steps/Sec: 1.17 + 63%|█████████████████████████████████████████████████████████████████████████████████▎ | 14891/23458 [4:19:16<2:01:47, 1.17it/s][2025-04-24 13:20:27] (step=0038350) Train Loss: 5.3736, Train Steps/Sec: 1.17 + 64%|█████████████████████████████████████████████████████████████████████████████████▍ | 14916/23458 [4:19:38<2:00:58, 1.18it/s][2025-04-24 13:20:48] (step=0038375) Train Loss: 5.4102, Train Steps/Sec: 1.17 + 64%|█████████████████████████████████████████████████████████████████████████████████▌ | 14941/23458 [4:19:59<1:59:38, 1.19it/s][2025-04-24 13:21:09] (step=0038400) Train Loss: 5.4151, Train Steps/Sec: 1.17 + 64%|█████████████████████████████████████████████████████████████████████████████████▋ | 14966/23458 [4:20:21<2:03:18, 1.15it/s][2025-04-24 13:21:31] (step=0038425) Train Loss: 5.4022, Train Steps/Sec: 1.17 + 64%|█████████████████████████████████████████████████████████████████████████████████▊ | 14991/23458 [4:20:42<2:00:10, 1.17it/s][2025-04-24 13:21:52] (step=0038450) Train Loss: 5.3627, Train Steps/Sec: 1.17 + 64%|█████████████████████████████████████████████████████████████████████████████████▉ | 15016/23458 [4:21:03<1:59:18, 1.18it/s][2025-04-24 13:22:13] (step=0038475) Train Loss: 5.3657, Train Steps/Sec: 1.17 + 64%|██████████████████████████████████████████████████████████████████████████████████ | 15041/23458 [4:21:25<1:58:30, 1.18it/s][2025-04-24 13:22:35] (step=0038500) Train Loss: 5.3080, Train Steps/Sec: 1.16 + 64%|██████████████████████████████████████████████████████████████████████████████████▏ | 15066/23458 [4:21:46<2:01:45, 1.15it/s][2025-04-24 13:22:57] (step=0038525) Train Loss: 5.3129, Train Steps/Sec: 1.15 + 64%|██████████████████████████████████████████████████████████████████████████████████▎ | 15091/23458 [4:22:08<2:00:06, 1.16it/s][2025-04-24 13:23:18] (step=0038550) Train Loss: 5.3336, Train Steps/Sec: 1.17 + 64%|██████████████████████████████████████████████████████████████████████████████████▍ | 15116/23458 [4:22:29<1:58:08, 1.18it/s][2025-04-24 13:23:39] (step=0038575) Train Loss: 5.3620, Train Steps/Sec: 1.17 + 65%|██████████████████████████████████████████████████████████████████████████████████▌ | 15141/23458 [4:22:51<1:56:55, 1.19it/s][2025-04-24 13:24:01] (step=0038600) Train Loss: 5.3070, Train Steps/Sec: 1.16 + 65%|██████████████████████████████████████████████████████████████████████████████████▊ | 15166/23458 [4:23:12<2:00:02, 1.15it/s][2025-04-24 13:24:22] (step=0038625) Train Loss: 5.3940, Train Steps/Sec: 1.17 + 65%|██████████████████████████████████████████████████████████████████████████████████▉ | 15191/23458 [4:23:34<2:27:52, 1.07s/it][2025-04-24 13:24:44] (step=0038650) Train Loss: 5.3262, Train Steps/Sec: 1.12 + 65%|███████████████████████████████████████████████████████████████████████████████████ | 15216/23458 [4:23:56<1:56:42, 1.18it/s][2025-04-24 13:25:06] (step=0038675) Train Loss: 5.3525, Train Steps/Sec: 1.17 + 65%|███████████████████████████████████████████████████████████████████████████████████▏ | 15241/23458 [4:24:17<1:55:15, 1.19it/s][2025-04-24 13:25:27] (step=0038700) Train Loss: 5.3479, Train Steps/Sec: 1.17 + 65%|███████████████████████████████████████████████████████████████████████████████████▎ | 15266/23458 [4:24:39<1:59:13, 1.15it/s][2025-04-24 13:25:50] (step=0038725) Train Loss: 5.3327, Train Steps/Sec: 1.12 + 65%|███████████████████████████████████████████████████████████████████████████████████▍ | 15291/23458 [4:25:02<1:57:03, 1.16it/s][2025-04-24 13:26:12] (step=0038750) Train Loss: 5.2901, Train Steps/Sec: 1.12 + 65%|███████████████████████████████████████████████████████████████████████████████████▌ | 15316/23458 [4:25:23<1:55:23, 1.18it/s][2025-04-24 13:26:33] (step=0038775) Train Loss: 5.3641, Train Steps/Sec: 1.17 + 65%|███████████████████████████████████████████████████████████████████████████████████▋ | 15341/23458 [4:25:45<1:54:43, 1.18it/s][2025-04-24 13:26:55] (step=0038800) Train Loss: 5.2454, Train Steps/Sec: 1.16 + 66%|███████████████████████████████████████████████████████████████████████████████████▊ | 15366/23458 [4:26:07<1:59:01, 1.13it/s][2025-04-24 13:27:17] (step=0038825) Train Loss: 5.3751, Train Steps/Sec: 1.13 + 66%|███████████████████████████████████████████████████████████████████████████████████▉ | 15391/23458 [4:26:30<1:55:27, 1.16it/s][2025-04-24 13:27:40] (step=0038850) Train Loss: 5.3626, Train Steps/Sec: 1.08 + 66%|████████████████████████████████████████████████████████████████████████████████████ | 15416/23458 [4:26:51<1:53:50, 1.18it/s][2025-04-24 13:28:01] (step=0038875) Train Loss: 5.4141, Train Steps/Sec: 1.17 + 66%|████████████████████████████████████████████████████████████████████████████████████▎ | 15441/23458 [4:27:13<1:52:37, 1.19it/s][2025-04-24 13:28:23] (step=0038900) Train Loss: 5.4106, Train Steps/Sec: 1.17 + 66%|████████████████████████████████████████████████████████████████████████████████████▍ | 15466/23458 [4:27:36<2:22:19, 1.07s/it][2025-04-24 13:28:46] (step=0038925) Train Loss: 5.3852, Train Steps/Sec: 1.07 + 66%|████████████████████████████████████████████████████████████████████████████████████▌ | 15491/23458 [4:27:57<1:53:45, 1.17it/s][2025-04-24 13:29:08] (step=0038950) Train Loss: 5.3623, Train Steps/Sec: 1.17 + 66%|████████████████████████████████████████████████████████████████████████████████████▋ | 15516/23458 [4:28:19<1:53:13, 1.17it/s][2025-04-24 13:29:29] (step=0038975) Train Loss: 5.4443, Train Steps/Sec: 1.17 + 66%|████████████████████████████████████████████████████████████████████████████████████▊ | 15541/23458 [4:28:40<1:51:17, 1.19it/s][2025-04-24 13:29:50] (step=0039000) Train Loss: 5.5243, Train Steps/Sec: 1.17 + 66%|████████████████████████████████████████████████████████████████████████████████████▉ | 15566/23458 [4:29:02<1:54:10, 1.15it/s][2025-04-24 13:30:12] (step=0039025) Train Loss: 5.4217, Train Steps/Sec: 1.17 + 66%|█████████████████████████████████████████████████████████████████████████████████████ | 15591/23458 [4:29:23<1:52:50, 1.16it/s][2025-04-24 13:30:33] (step=0039050) Train Loss: 5.3892, Train Steps/Sec: 1.17 + 67%|█████████████████████████████████████████████████████████████████████████████████████▏ | 15616/23458 [4:29:44<1:51:29, 1.17it/s][2025-04-24 13:30:54] (step=0039075) Train Loss: 5.3695, Train Steps/Sec: 1.17 + 67%|█████████████████████████████████████████████████████████████████████████████████████▎ | 15641/23458 [4:30:06<1:49:59, 1.18it/s][2025-04-24 13:31:16] (step=0039100) Train Loss: 5.3494, Train Steps/Sec: 1.17 + 67%|█████████████████████████████████████████████████████████████████████████████████████▍ | 15666/23458 [4:30:27<1:52:47, 1.15it/s][2025-04-24 13:31:37] (step=0039125) Train Loss: 5.3832, Train Steps/Sec: 1.17 + 67%|█████████████████████████████████████████████████████████████████████████████████████▌ | 15691/23458 [4:30:49<1:50:41, 1.17it/s][2025-04-24 13:31:59] (step=0039150) Train Loss: 5.3891, Train Steps/Sec: 1.17 + 67%|█████████████████████████████████████████████████████████████████████████████████████▊ | 15716/23458 [4:31:10<1:49:54, 1.17it/s][2025-04-24 13:32:20] (step=0039175) Train Loss: 5.3458, Train Steps/Sec: 1.17 + 67%|█████████████████████████████████████████████████████████████████████████████████████▉ | 15741/23458 [4:31:31<1:48:53, 1.18it/s][2025-04-24 13:32:41] (step=0039200) Train Loss: 5.3330, Train Steps/Sec: 1.16 + 67%|██████████████████████████████████████████████████████████████████████████████████████ | 15766/23458 [4:31:53<1:51:31, 1.15it/s][2025-04-24 13:33:03] (step=0039225) Train Loss: 5.3804, Train Steps/Sec: 1.17 + 67%|██████████████████████████████████████████████████████████████████████████████████████▏ | 15791/23458 [4:32:14<1:49:05, 1.17it/s][2025-04-24 13:33:24] (step=0039250) Train Loss: 5.4404, Train Steps/Sec: 1.17 + 67%|██████████████████████████████████████████████████████████████████████████████████████▎ | 15816/23458 [4:32:35<1:48:21, 1.18it/s][2025-04-24 13:33:46] (step=0039275) Train Loss: 5.4176, Train Steps/Sec: 1.17 + 68%|██████████████████████████████████████████████████████████████████████████████████████▍ | 15841/23458 [4:32:57<1:47:00, 1.19it/s][2025-04-24 13:34:07] (step=0039300) Train Loss: 5.3215, Train Steps/Sec: 1.17 + 68%|██████████████████████████████████████████████████████████████████████████████████████▌ | 15866/23458 [4:33:19<1:50:03, 1.15it/s][2025-04-24 13:34:29] (step=0039325) Train Loss: 5.3189, Train Steps/Sec: 1.13 + 68%|██████████████████████████████████████████████████████████████████████████████████████▋ | 15891/23458 [4:33:40<1:47:49, 1.17it/s][2025-04-24 13:34:50] (step=0039350) Train Loss: 5.3835, Train Steps/Sec: 1.17 + 68%|██████████████████████████████████████████████████████████████████████████████████████▊ | 15916/23458 [4:34:02<1:46:37, 1.18it/s][2025-04-24 13:35:12] (step=0039375) Train Loss: 5.3502, Train Steps/Sec: 1.17 + 68%|██████████████████████████████████████████████████████████████████████████████████████▉ | 15941/23458 [4:34:24<1:50:21, 1.14it/s][2025-04-24 13:35:34] (step=0039400) Train Loss: 5.3997, Train Steps/Sec: 1.12 + 68%|███████████████████████████████████████████████████████████████████████████████████████ | 15966/23458 [4:34:45<1:48:47, 1.15it/s][2025-04-24 13:35:56] (step=0039425) Train Loss: 5.3441, Train Steps/Sec: 1.17 + 68%|███████████████████████████████████████████████████████████████████████████████████████▎ | 15991/23458 [4:35:08<2:26:43, 1.18s/it][2025-04-24 13:36:18] (step=0039450) Train Loss: 5.2829, Train Steps/Sec: 1.11 + 68%|███████████████████████████████████████████████████████████████████████████████████████▍ | 16016/23458 [4:35:29<1:45:15, 1.18it/s][2025-04-24 13:36:39] (step=0039475) Train Loss: 5.3788, Train Steps/Sec: 1.17 + 68%|███████████████████████████████████████████████████████████████████████████████████████▌ | 16041/23458 [4:35:53<1:49:36, 1.13it/s][2025-04-24 13:37:03] (step=0039500) Train Loss: 5.3119, Train Steps/Sec: 1.04 + 68%|███████████████████████████████████████████████████████████████████████████████████████▋ | 16066/23458 [4:36:14<1:47:20, 1.15it/s][2025-04-24 13:37:25] (step=0039525) Train Loss: 5.3431, Train Steps/Sec: 1.17 + 69%|███████████████████████████████████████████████████████████████████████████████████████▊ | 16091/23458 [4:36:36<1:45:15, 1.17it/s][2025-04-24 13:37:46] (step=0039550) Train Loss: 5.3628, Train Steps/Sec: 1.17 + 69%|███████████████████████████████████████████████████████████████████████████████████████▉ | 16116/23458 [4:36:57<1:44:00, 1.18it/s][2025-04-24 13:38:07] (step=0039575) Train Loss: 5.3576, Train Steps/Sec: 1.17 + 69%|████████████████████████████████████████████████████████████████████████████████████████ | 16141/23458 [4:37:19<1:43:17, 1.18it/s][2025-04-24 13:38:30] (step=0039600) Train Loss: 5.4015, Train Steps/Sec: 1.12 + 69%|████████████████████████████████████████████████████████████████████████████████████████▏ | 16166/23458 [4:37:41<1:45:41, 1.15it/s][2025-04-24 13:38:51] (step=0039625) Train Loss: 5.3550, Train Steps/Sec: 1.17 + 69%|████████████████████████████████████████████████████████████████████████████████████████▎ | 16191/23458 [4:38:03<2:01:27, 1.00s/it][2025-04-24 13:39:13] (step=0039650) Train Loss: 5.4042, Train Steps/Sec: 1.12 + 69%|████████████████████████████████████████████████████████████████████████████████████████▍ | 16216/23458 [4:38:25<1:42:20, 1.18it/s][2025-04-24 13:39:35] (step=0039675) Train Loss: 5.3776, Train Steps/Sec: 1.17 + 69%|████████████████████████████████████████████████████████████████████████████████████████▌ | 16241/23458 [4:38:46<1:41:46, 1.18it/s][2025-04-24 13:39:56] (step=0039700) Train Loss: 5.3404, Train Steps/Sec: 1.16 + 69%|████████████████████████████████████████████████████████████████████████████████████████▊ | 16266/23458 [4:39:07<1:44:54, 1.14it/s][2025-04-24 13:40:18] (step=0039725) Train Loss: 5.4076, Train Steps/Sec: 1.17 + 69%|████████████████████████████████████████████████████████████████████████████████████████▉ | 16291/23458 [4:39:29<1:42:06, 1.17it/s][2025-04-24 13:40:39] (step=0039750) Train Loss: 5.2880, Train Steps/Sec: 1.17 + 70%|█████████████████████████████████████████████████████████████████████████████████████████ | 16316/23458 [4:39:50<1:41:45, 1.17it/s][2025-04-24 13:41:00] (step=0039775) Train Loss: 5.3139, Train Steps/Sec: 1.17 + 70%|█████████████████████████████████████████████████████████████████████████████████████████▏ | 16341/23458 [4:40:12<1:40:47, 1.18it/s][2025-04-24 13:41:22] (step=0039800) Train Loss: 5.3846, Train Steps/Sec: 1.17 + 70%|█████████████████████████████████████████████████████████████████████████████████████████▎ | 16366/23458 [4:40:33<1:42:54, 1.15it/s][2025-04-24 13:41:43] (step=0039825) Train Loss: 5.3812, Train Steps/Sec: 1.17 + 70%|█████████████████████████████████████████████████████████████████████████████████████████▍ | 16391/23458 [4:40:54<1:40:22, 1.17it/s][2025-04-24 13:42:04] (step=0039850) Train Loss: 5.3420, Train Steps/Sec: 1.17 + 70%|█████████████████████████████████████████████████████████████████████████████████████████▌ | 16416/23458 [4:41:16<1:39:48, 1.18it/s][2025-04-24 13:42:26] (step=0039875) Train Loss: 5.3414, Train Steps/Sec: 1.17 + 70%|█████████████████████████████████████████████████████████████████████████████████████████▋ | 16441/23458 [4:41:37<1:38:35, 1.19it/s][2025-04-24 13:42:47] (step=0039900) Train Loss: 5.3531, Train Steps/Sec: 1.17 + 70%|█████████████████████████████████████████████████████████████████████████████████████████▊ | 16466/23458 [4:41:58<1:41:15, 1.15it/s][2025-04-24 13:43:09] (step=0039925) Train Loss: 5.3093, Train Steps/Sec: 1.17 + 70%|█████████████████████████████████████████████████████████████████████████████████████████▉ | 16491/23458 [4:42:20<1:40:39, 1.15it/s][2025-04-24 13:43:30] (step=0039950) Train Loss: 5.3284, Train Steps/Sec: 1.16 + 70%|██████████████████████████████████████████████████████████████████████████████████████████ | 16516/23458 [4:42:42<1:41:57, 1.13it/s][2025-04-24 13:43:52] (step=0039975) Train Loss: 5.4014, Train Steps/Sec: 1.12 + 71%|██████████████████████████████████████████████████████████████████████████████████████████▎ | 16541/23458 [4:43:04<1:37:58, 1.18it/s][2025-04-24 13:44:14] (step=0040000) Train Loss: 5.3164, Train Steps/Sec: 1.16 +You are using a model of type blip-2 to instantiate a model of type instructblip. This is not supported for all configurations of models and can yield errors. +[2025-04-24 13:44:14] text_config is None. Initializing the text config with default values (`OPTConfig`). +model_name_or_path:/tmp/haozhezhao/model/blip2-flan-t5-xl +image_place_holder: +tokenizer length after expend 32101 +tokenizer length before expend 32100 +100%|████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████| 4/4 [03:59<00:00, 59.79s/it] +[2025-04-24 13:49:21] Finish Eval in 40000 steps...█████████████████████████████████████████████████████████████████████████████████████| 4/4 [03:58<00:00, 59.45s/it] +processing left-padding... +processing left-padding... +processing left-padding... +processing left-padding... +[2025-04-24 13:49:39] Saved checkpoint to checkpoint/Afteri2i_100kckpt_LLAVAT5_empty_Fixed_just_segment_pretrain_stage3_dreambench_recap_400k_t2i_400k_flux_200k_midjourney_150k_recovery_150k_3e4_epoch2_no_rotated/000-GPT-XL/checkpoints/0040000.pt +[2025-04-24 13:49:42] Removed old checkpoint: checkpoint/Afteri2i_100kckpt_LLAVAT5_empty_Fixed_just_segment_pretrain_stage3_dreambench_recap_400k_t2i_400k_flux_200k_midjourney_150k_recovery_150k_3e4_epoch2_no_rotated/000-GPT-XL/checkpoints/0038000.pt + 71%|██████████████████████████████████████████████████████████████████████████████████████████▍ | 16566/23458 [4:48:53<1:41:44, 1.13it/s][2025-04-24 13:50:03] (step=0040025) Train Loss: 5.4095, Train Steps/Sec: 0.07 + 71%|██████████████████████████████████████████████████████████████████████████████████████████▌ | 16591/23458 [4:49:15<1:37:53, 1.17it/s][2025-04-24 13:50:25] (step=0040050) Train Loss: 5.2762, Train Steps/Sec: 1.17 + 71%|██████████████████████████████████████████████████████████████████████████████████████████▋ | 16616/23458 [4:49:37<1:36:53, 1.18it/s][2025-04-24 13:50:47] (step=0040075) Train Loss: 5.4382, Train Steps/Sec: 1.12 + 71%|██████████████████████████████████████████████████████████████████████████████████████████▊ | 16641/23458 [4:49:58<1:35:53, 1.18it/s][2025-04-24 13:51:08] (step=0040100) Train Loss: 5.3194, Train Steps/Sec: 1.17 + 71%|██████████████████████████████████████████████████████████████████████████████████████████▉ | 16666/23458 [4:50:20<1:38:12, 1.15it/s][2025-04-24 13:51:30] (step=0040125) Train Loss: 5.3296, Train Steps/Sec: 1.18 + 71%|███████████████████████████████████████████████████████████████████████████████████████████ | 16691/23458 [4:50:42<1:37:09, 1.16it/s][2025-04-24 13:51:52] (step=0040150) Train Loss: 5.3140, Train Steps/Sec: 1.13 + 71%|███████████████████████████████████████████████████████████████████████████████████████████▏ | 16716/23458 [4:51:06<1:46:37, 1.05it/s][2025-04-24 13:52:16] (step=0040175) Train Loss: 5.3069, Train Steps/Sec: 1.04 + 71%|███████████████████████████████████████████████████████████████████████████████████████████▎ | 16741/23458 [4:51:27<1:34:06, 1.19it/s][2025-04-24 13:52:37] (step=0040200) Train Loss: 5.4563, Train Steps/Sec: 1.17 + 71%|███████████████████████████████████████████████████████████████████████████████████████████▍ | 16766/23458 [4:51:48<1:37:31, 1.14it/s][2025-04-24 13:52:59] (step=0040225) Train Loss: 5.2905, Train Steps/Sec: 1.17 + 72%|███████████████████████████████████████████████████████████████████████████████████████████▌ | 16791/23458 [4:52:11<1:36:41, 1.15it/s][2025-04-24 13:53:21] (step=0040250) Train Loss: 5.3793, Train Steps/Sec: 1.13 + 72%|███████████████████████████████████████████████████████████████████████████████████████████▊ | 16816/23458 [4:52:32<1:33:41, 1.18it/s][2025-04-24 13:53:42] (step=0040275) Train Loss: 5.3974, Train Steps/Sec: 1.17 + 72%|███████████████████████████████████████████████████████████████████████████████████████████▉ | 16841/23458 [4:52:53<1:33:09, 1.18it/s][2025-04-24 13:54:03] (step=0040300) Train Loss: 5.3865, Train Steps/Sec: 1.17 + 72%|████████████████████████████████████████████████████████████████████████████████████████████ | 16866/23458 [4:53:15<1:35:42, 1.15it/s][2025-04-24 13:54:25] (step=0040325) Train Loss: 5.3317, Train Steps/Sec: 1.17 + 72%|████████████████████████████████████████████████████████████████████████████████████████████▏ | 16891/23458 [4:53:36<1:33:42, 1.17it/s][2025-04-24 13:54:46] (step=0040350) Train Loss: 5.3318, Train Steps/Sec: 1.17 + 72%|████████████████████████████████████████████████████████████████████████████████████████████▎ | 16916/23458 [4:53:58<1:55:28, 1.06s/it][2025-04-24 13:55:08] (step=0040375) Train Loss: 5.4306, Train Steps/Sec: 1.12 + 72%|████████████████████████████████████████████████████████████████████████████████████████████▍ | 16941/23458 [4:54:20<1:31:35, 1.19it/s][2025-04-24 13:55:30] (step=0040400) Train Loss: 5.3807, Train Steps/Sec: 1.17 + 72%|████████████████████████████████████████████████████████████████████████████████████████████▌ | 16966/23458 [4:54:41<1:34:15, 1.15it/s][2025-04-24 13:55:51] (step=0040425) Train Loss: 5.3971, Train Steps/Sec: 1.17 + 72%|████████████████████████████████████████████████████████████████████████████████████████████▋ | 16991/23458 [4:55:03<1:32:04, 1.17it/s][2025-04-24 13:56:13] (step=0040450) Train Loss: 5.3281, Train Steps/Sec: 1.15 + 73%|████████████████████████████████████████████████████████████████████████████████████████████▊ | 17016/23458 [4:55:24<1:30:57, 1.18it/s][2025-04-24 13:56:34] (step=0040475) Train Loss: 5.4163, Train Steps/Sec: 1.17 + 73%|████████████████████████████████████████████████████████████████████████████████████████████▉ | 17041/23458 [4:55:46<1:30:00, 1.19it/s][2025-04-24 13:56:56] (step=0040500) Train Loss: 5.3598, Train Steps/Sec: 1.17 + 73%|█████████████████████████████████████████████████████████████████████████████████████████████ | 17066/23458 [4:56:07<1:32:38, 1.15it/s][2025-04-24 13:57:17] (step=0040525) Train Loss: 5.3764, Train Steps/Sec: 1.17 + 73%|█████████████████████████████████████████████████████████████████████████████████████████████▎ | 17091/23458 [4:56:28<1:30:52, 1.17it/s][2025-04-24 13:57:39] (step=0040550) Train Loss: 5.3526, Train Steps/Sec: 1.17 + 73%|█████████████████████████████████████████████████████████████████████████████████████████████▍ | 17116/23458 [4:56:50<1:29:55, 1.18it/s][2025-04-24 13:58:00] (step=0040575) Train Loss: 5.4485, Train Steps/Sec: 1.17 + 73%|█████████████████████████████████████████████████████████████████████████████████████████████▌ | 17141/23458 [4:57:11<1:28:47, 1.19it/s][2025-04-24 13:58:21] (step=0040600) Train Loss: 5.4342, Train Steps/Sec: 1.17 + 73%|█████████████████████████████████████████████████████████████████████████████████████████████▋ | 17166/23458 [4:57:33<1:31:00, 1.15it/s][2025-04-24 13:58:43] (step=0040625) Train Loss: 5.3581, Train Steps/Sec: 1.17 + 73%|█████████████████████████████████████████████████████████████████████████████████████████████▊ | 17191/23458 [4:57:55<1:49:06, 1.04s/it][2025-04-24 13:59:05] (step=0040650) Train Loss: 5.3813, Train Steps/Sec: 1.12 + 73%|█████████████████████████████████████████████████████████████████████████████████████████████▉ | 17216/23458 [4:58:16<1:28:24, 1.18it/s][2025-04-24 13:59:26] (step=0040675) Train Loss: 5.4520, Train Steps/Sec: 1.17 + 73%|██████████████████████████████████████████████████████████████████████████████████████████████ | 17241/23458 [4:58:37<1:27:30, 1.18it/s][2025-04-24 13:59:48] (step=0040700) Train Loss: 5.3197, Train Steps/Sec: 1.17 + 74%|██████████████████████████████████████████████████████████████████████████████████████████████▏ | 17266/23458 [4:59:00<1:29:57, 1.15it/s][2025-04-24 14:00:10] (step=0040725) Train Loss: 5.3347, Train Steps/Sec: 1.12 + 74%|██████████████████████████████████████████████████████████████████████████████████████████████▎ | 17291/23458 [4:59:21<1:27:55, 1.17it/s][2025-04-24 14:00:31] (step=0040750) Train Loss: 5.4309, Train Steps/Sec: 1.17 + 74%|██████████████████████████████████████████████████████████████████████████████████████████████▍ | 17316/23458 [4:59:43<1:26:44, 1.18it/s][2025-04-24 14:00:53] (step=0040775) Train Loss: 5.3861, Train Steps/Sec: 1.17 + 74%|██████████████████████████████████████████████████████████████████████████████████████████████▌ | 17341/23458 [5:00:05<1:28:47, 1.15it/s][2025-04-24 14:01:15] (step=0040800) Train Loss: 5.3549, Train Steps/Sec: 1.12 + 74%|██████████████████████████████████████████████████████████████████████████████████████████████▊ | 17366/23458 [5:00:28<1:29:01, 1.14it/s][2025-04-24 14:01:38] (step=0040825) Train Loss: 5.3173, Train Steps/Sec: 1.08 + 74%|██████████████████████████████████████████████████████████████████████████████████████████████▉ | 17391/23458 [5:00:49<1:26:15, 1.17it/s][2025-04-24 14:02:00] (step=0040850) Train Loss: 5.3498, Train Steps/Sec: 1.17 + 74%|███████████████████████████████████████████████████████████████████████████████████████████████ | 17416/23458 [5:01:11<1:25:27, 1.18it/s][2025-04-24 14:02:21] (step=0040875) Train Loss: 5.3461, Train Steps/Sec: 1.17 + 74%|███████████████████████████████████████████████████████████████████████████████████████████████▏ | 17441/23458 [5:01:33<1:26:17, 1.16it/s][2025-04-24 14:02:44] (step=0040900) Train Loss: 5.3719, Train Steps/Sec: 1.12 + 74%|███████████████████████████████████████████████████████████████████████████████████████████████▎ | 17466/23458 [5:01:55<1:26:50, 1.15it/s][2025-04-24 14:03:06] (step=0040925) Train Loss: 5.4406, Train Steps/Sec: 1.17 + 75%|███████████████████████████████████████████████████████████████████████████████████████████████▍ | 17491/23458 [5:02:17<1:25:09, 1.17it/s][2025-04-24 14:03:27] (step=0040950) Train Loss: 5.2755, Train Steps/Sec: 1.17 + 75%|███████████████████████████████████████████████████████████████████████████████████████████████▌ | 17516/23458 [5:02:38<1:24:16, 1.18it/s][2025-04-24 14:03:48] (step=0040975) Train Loss: 5.3991, Train Steps/Sec: 1.17 + 75%|███████████████████████████████████████████████████████████████████████████████████████████████▋ | 17541/23458 [5:03:00<1:23:18, 1.18it/s][2025-04-24 14:04:10] (step=0041000) Train Loss: 5.3725, Train Steps/Sec: 1.17 + 75%|███████████████████████████████████████████████████████████████████████████████████████████████▊ | 17566/23458 [5:03:21<1:25:15, 1.15it/s][2025-04-24 14:04:31] (step=0041025) Train Loss: 5.4448, Train Steps/Sec: 1.17 + 75%|███████████████████████████████████████████████████████████████████████████████████████████████▉ | 17591/23458 [5:03:42<1:23:38, 1.17it/s][2025-04-24 14:04:52] (step=0041050) Train Loss: 5.3389, Train Steps/Sec: 1.17 + 75%|████████████████████████████████████████████████████████████████████████████████████████████████ | 17616/23458 [5:04:04<1:22:38, 1.18it/s][2025-04-24 14:05:14] (step=0041075) Train Loss: 5.3352, Train Steps/Sec: 1.17 + 75%|████████████████████████████████████████████████████████████████████████████████████████████████▎ | 17641/23458 [5:04:25<1:22:08, 1.18it/s][2025-04-24 14:05:35] (step=0041100) Train Loss: 5.3442, Train Steps/Sec: 1.16 + 75%|████████████████████████████████████████████████████████████████████████████████████████████████▍ | 17666/23458 [5:04:48<1:24:01, 1.15it/s][2025-04-24 14:05:58] (step=0041125) Train Loss: 5.3483, Train Steps/Sec: 1.12 + 75%|████████████████████████████████████████████████████████████████████████████████████████████████▌ | 17691/23458 [5:05:09<1:22:18, 1.17it/s][2025-04-24 14:06:19] (step=0041150) Train Loss: 5.3065, Train Steps/Sec: 1.17 + 76%|████████████████████████████████████████████████████████████████████████████████████████████████▋ | 17716/23458 [5:05:30<1:21:14, 1.18it/s][2025-04-24 14:06:40] (step=0041175) Train Loss: 5.3290, Train Steps/Sec: 1.17 + 76%|████████████████████████████████████████████████████████████████████████████████████████████████▊ | 17741/23458 [5:05:52<1:20:14, 1.19it/s][2025-04-24 14:07:02] (step=0041200) Train Loss: 5.3915, Train Steps/Sec: 1.17 + 76%|████████████████████████████████████████████████████████████████████████████████████████████████▉ | 17766/23458 [5:06:13<1:22:03, 1.16it/s][2025-04-24 14:07:23] (step=0041225) Train Loss: 5.3850, Train Steps/Sec: 1.17 + 76%|█████████████████████████████████████████████████████████████████████████████████████████████████ | 17791/23458 [5:06:34<1:20:31, 1.17it/s][2025-04-24 14:07:44] (step=0041250) Train Loss: 5.4192, Train Steps/Sec: 1.17 + 76%|█████████████████████████████████████████████████████████████████████████████████████████████████▏ | 17816/23458 [5:06:56<1:19:45, 1.18it/s][2025-04-24 14:08:06] (step=0041275) Train Loss: 5.3372, Train Steps/Sec: 1.17 + 76%|█████████████████████████████████████████████████████████████████████████████████████████████████▎ | 17841/23458 [5:07:17<1:18:54, 1.19it/s][2025-04-24 14:08:27] (step=0041300) Train Loss: 5.3605, Train Steps/Sec: 1.17 + 76%|█████████████████████████████████████████████████████████████████████████████████████████████████▍ | 17866/23458 [5:07:39<1:21:07, 1.15it/s][2025-04-24 14:08:49] (step=0041325) Train Loss: 5.4512, Train Steps/Sec: 1.13 + 76%|█████████████████████████████████████████████████████████████████████████████████████████████████▌ | 17891/23458 [5:08:01<1:19:18, 1.17it/s][2025-04-24 14:09:11] (step=0041350) Train Loss: 5.4149, Train Steps/Sec: 1.17 + 76%|█████████████████████████████████████████████████████████████████████████████████████████████████▊ | 17916/23458 [5:08:23<1:20:18, 1.15it/s][2025-04-24 14:09:33] (step=0041375) Train Loss: 5.3431, Train Steps/Sec: 1.12 + 76%|█████████████████████████████████████████████████████████████████████████████████████████████████▉ | 17941/23458 [5:08:44<1:17:30, 1.19it/s][2025-04-24 14:09:54] (step=0041400) Train Loss: 5.3505, Train Steps/Sec: 1.17 + 77%|██████████████████████████████████████████████████████████████████████████████████████████████████ | 17966/23458 [5:09:06<1:19:35, 1.15it/s][2025-04-24 14:10:16] (step=0041425) Train Loss: 5.3994, Train Steps/Sec: 1.17 + 77%|██████████████████████████████████████████████████████████████████████████████████████████████████▏ | 17991/23458 [5:09:28<1:39:51, 1.10s/it][2025-04-24 14:10:38] (step=0041450) Train Loss: 5.2830, Train Steps/Sec: 1.13 + 77%|██████████████████████████████████████████████████████████████████████████████████████████████████▎ | 18016/23458 [5:09:51<1:39:45, 1.10s/it][2025-04-24 14:11:01] (step=0041475) Train Loss: 5.3473, Train Steps/Sec: 1.08 + 77%|██████████████████████████████████████████████████████████████████████████████████████████████████▍ | 18041/23458 [5:10:12<1:16:24, 1.18it/s][2025-04-24 14:11:22] (step=0041500) Train Loss: 5.3119, Train Steps/Sec: 1.17 + 77%|██████████████████████████████████████████████████████████████████████████████████████████████████▌ | 18066/23458 [5:10:34<1:17:49, 1.15it/s][2025-04-24 14:11:44] (step=0041525) Train Loss: 5.3814, Train Steps/Sec: 1.17 + 77%|██████████████████████████████████████████████████████████████████████████████████████████████████▋ | 18091/23458 [5:10:55<1:16:39, 1.17it/s][2025-04-24 14:12:05] (step=0041550) Train Loss: 5.3775, Train Steps/Sec: 1.17 + 77%|██████████████████████████████████████████████████████████████████████████████████████████████████▊ | 18116/23458 [5:11:17<1:15:52, 1.17it/s][2025-04-24 14:12:27] (step=0041575) Train Loss: 5.3127, Train Steps/Sec: 1.12 + 77%|██████████████████████████████████████████████████████████████████████████████████████████████████▉ | 18141/23458 [5:11:39<1:14:48, 1.18it/s][2025-04-24 14:12:49] (step=0041600) Train Loss: 5.3991, Train Steps/Sec: 1.16 + 77%|███████████████████████████████████████████████████████████████████████████████████████████████████ | 18166/23458 [5:12:01<1:17:37, 1.14it/s][2025-04-24 14:13:11] (step=0041625) Train Loss: 5.3020, Train Steps/Sec: 1.12 + 78%|███████████████████████████████████████████████████████████████████████████████████████████████████▎ | 18191/23458 [5:12:22<1:15:02, 1.17it/s][2025-04-24 14:13:32] (step=0041650) Train Loss: 5.3394, Train Steps/Sec: 1.17 + 78%|███████████████████████████████████████████████████████████████████████████████████████████████████▍ | 18216/23458 [5:12:44<1:14:20, 1.18it/s][2025-04-24 14:13:54] (step=0041675) Train Loss: 5.3548, Train Steps/Sec: 1.17 + 78%|███████████████████████████████████████████████████████████████████████████████████████████████████▌ | 18241/23458 [5:13:05<1:13:24, 1.18it/s][2025-04-24 14:14:15] (step=0041700) Train Loss: 5.4572, Train Steps/Sec: 1.17 + 78%|███████████████████████████████████████████████████████████████████████████████████████████████████▋ | 18266/23458 [5:13:27<1:15:08, 1.15it/s][2025-04-24 14:14:37] (step=0041725) Train Loss: 5.3402, Train Steps/Sec: 1.17 + 78%|███████████████████████████████████████████████████████████████████████████████████████████████████▊ | 18291/23458 [5:13:48<1:13:32, 1.17it/s][2025-04-24 14:14:58] (step=0041750) Train Loss: 5.3364, Train Steps/Sec: 1.17 + 78%|███████████████████████████████████████████████████████████████████████████████████████████████████▉ | 18316/23458 [5:14:09<1:12:39, 1.18it/s][2025-04-24 14:15:19] (step=0041775) Train Loss: 5.3755, Train Steps/Sec: 1.17 + 78%|████████████████████████████████████████████████████████████████████████████████████████████████████ | 18341/23458 [5:14:31<1:12:10, 1.18it/s][2025-04-24 14:15:41] (step=0041800) Train Loss: 5.3409, Train Steps/Sec: 1.17 + 78%|████████████████████████████████████████████████████████████████████████████████████████████████████▏ | 18366/23458 [5:14:52<1:14:11, 1.14it/s][2025-04-24 14:16:02] (step=0041825) Train Loss: 5.3260, Train Steps/Sec: 1.17 + 78%|████████████████████████████████████████████████████████████████████████████████████████████████████▎ | 18391/23458 [5:15:14<1:12:15, 1.17it/s][2025-04-24 14:16:24] (step=0041850) Train Loss: 5.3839, Train Steps/Sec: 1.12 + 79%|████████████████████████████████████████████████████████████████████████████████████████████████████▍ | 18416/23458 [5:15:36<1:11:18, 1.18it/s][2025-04-24 14:16:46] (step=0041875) Train Loss: 5.3205, Train Steps/Sec: 1.17 + 79%|████████████████████████████████████████████████████████████████████████████████████████████████████▌ | 18441/23458 [5:15:57<1:10:29, 1.19it/s][2025-04-24 14:17:07] (step=0041900) Train Loss: 5.3277, Train Steps/Sec: 1.17 + 79%|████████████████████████████████████████████████████████████████████████████████████████████████████▊ | 18466/23458 [5:16:18<1:12:03, 1.15it/s][2025-04-24 14:17:29] (step=0041925) Train Loss: 5.3813, Train Steps/Sec: 1.17 + 79%|████████████████████████████████████████████████████████████████████████████████████████████████████▉ | 18491/23458 [5:16:40<1:11:15, 1.16it/s][2025-04-24 14:17:50] (step=0041950) Train Loss: 5.4256, Train Steps/Sec: 1.17 + 79%|█████████████████████████████████████████████████████████████████████████████████████████████████████ | 18516/23458 [5:17:02<1:10:19, 1.17it/s][2025-04-24 14:18:12] (step=0041975) Train Loss: 5.3405, Train Steps/Sec: 1.13 + 79%|█████████████████████████████████████████████████████████████████████████████████████████████████████▏ | 18541/23458 [5:17:23<1:09:26, 1.18it/s][2025-04-24 14:18:34] (step=0042000) Train Loss: 5.3171, Train Steps/Sec: 1.17 +You are using a model of type blip-2 to instantiate a model of type instructblip. This is not supported for all configurations of models and can yield errors. +[2025-04-24 14:18:34] text_config is None. Initializing the text config with default values (`OPTConfig`). +model_name_or_path:/tmp/haozhezhao/model/blip2-flan-t5-xl +image_place_holder: +tokenizer length after expend 32101 +tokenizer length before expend 32100 +100%|██████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████| 4/4 [03:57<00:00, 59.46s/it] +[2025-04-24 14:23:40] Finish Eval in 42000 steps...███████████████████████████████████████████████████████████████████████████████████████████| 4/4 [03:57<00:00, 59.10s/it] +processing left-padding... +processing left-padding... +processing left-padding... +processing left-padding... +[2025-04-24 14:23:59] Saved checkpoint to checkpoint/Afteri2i_100kckpt_LLAVAT5_empty_Fixed_just_segment_pretrain_stage3_dreambench_recap_400k_t2i_400k_flux_200k_midjourney_150k_recovery_150k_3e4_epoch2_no_rotated/000-GPT-XL/checkpoints/0042000.pt +[2025-04-24 14:24:01] Removed old checkpoint: checkpoint/Afteri2i_100kckpt_LLAVAT5_empty_Fixed_just_segment_pretrain_stage3_dreambench_recap_400k_t2i_400k_flux_200k_midjourney_150k_recovery_150k_3e4_epoch2_no_rotated/000-GPT-XL/checkpoints/0040000.pt + 79%|█████████████████████████████████████████████████████████████████████████████████████████████████████▎ | 18566/23458 [5:23:13<1:29:05, 1.09s/it][2025-04-24 14:24:23] (step=0042025) Train Loss: 5.3458, Train Steps/Sec: 0.07 + 79%|█████████████████████████████████████████████████████████████████████████████████████████████████████▍ | 18591/23458 [5:23:34<1:09:23, 1.17it/s][2025-04-24 14:24:44] (step=0042050) Train Loss: 5.4435, Train Steps/Sec: 1.17 + 79%|█████████████████████████████████████████████████████████████████████████████████████████████████████▌ | 18616/23458 [5:23:56<1:08:21, 1.18it/s][2025-04-24 14:25:06] (step=0042075) Train Loss: 5.3411, Train Steps/Sec: 1.17 + 79%|█████████████████████████████████████████████████████████████████████████████████████████████████████▋ | 18641/23458 [5:24:17<1:07:43, 1.19it/s][2025-04-24 14:25:27] (step=0042100) Train Loss: 5.3802, Train Steps/Sec: 1.17 + 80%|█████████████████████████████████████████████████████████████████████████████████████████████████████▊ | 18666/23458 [5:24:40<1:16:43, 1.04it/s][2025-04-24 14:25:50] (step=0042125) Train Loss: 5.3292, Train Steps/Sec: 1.09 + 80%|█████████████████████████████████████████████████████████████████████████████████████████████████████▉ | 18691/23458 [5:25:02<1:08:18, 1.16it/s][2025-04-24 14:26:12] (step=0042150) Train Loss: 5.3917, Train Steps/Sec: 1.13 + 80%|██████████████████████████████████████████████████████████████████████████████████████████████████████ | 18716/23458 [5:25:24<1:07:03, 1.18it/s][2025-04-24 14:26:34] (step=0042175) Train Loss: 5.4638, Train Steps/Sec: 1.17 + 80%|██████████████████████████████████████████████████████████████████████████████████████████████████████▎ | 18741/23458 [5:25:45<1:06:23, 1.18it/s][2025-04-24 14:26:55] (step=0042200) Train Loss: 5.3049, Train Steps/Sec: 1.16 + 80%|██████████████████████████████████████████████████████████████████████████████████████████████████████▍ | 18766/23458 [5:26:07<1:10:38, 1.11it/s][2025-04-24 14:27:17] (step=0042225) Train Loss: 5.3507, Train Steps/Sec: 1.12 + 80%|██████████████████████████████████████████████████████████████████████████████████████████████████████▌ | 18791/23458 [5:26:29<1:06:18, 1.17it/s][2025-04-24 14:27:39] (step=0042250) Train Loss: 5.4194, Train Steps/Sec: 1.17 + 80%|██████████████████████████████████████████████████████████████████████████████████████████████████████▋ | 18816/23458 [5:26:50<1:05:34, 1.18it/s][2025-04-24 14:28:00] (step=0042275) Train Loss: 5.4105, Train Steps/Sec: 1.17 + 80%|██████████████████████████████████████████████████████████████████████████████████████████████████████▊ | 18841/23458 [5:27:11<1:05:02, 1.18it/s][2025-04-24 14:28:22] (step=0042300) Train Loss: 5.4330, Train Steps/Sec: 1.17 + 80%|██████████████████████████████████████████████████████████████████████████████████████████████████████▉ | 18866/23458 [5:27:33<1:06:27, 1.15it/s][2025-04-24 14:28:43] (step=0042325) Train Loss: 5.2673, Train Steps/Sec: 1.17 + 81%|███████████████████████████████████████████████████████████████████████████████████████████████████████ | 18891/23458 [5:27:55<1:05:06, 1.17it/s][2025-04-24 14:29:05] (step=0042350) Train Loss: 5.3846, Train Steps/Sec: 1.12 + 81%|███████████████████████████████████████████████████████████████████████████████████████████████████████▏ | 18916/23458 [5:28:17<1:04:32, 1.17it/s][2025-04-24 14:29:27] (step=0042375) Train Loss: 5.4311, Train Steps/Sec: 1.15 + 81%|███████████████████████████████████████████████████████████████████████████████████████████████████████▎ | 18941/23458 [5:28:38<1:03:23, 1.19it/s][2025-04-24 14:29:48] (step=0042400) Train Loss: 5.3851, Train Steps/Sec: 1.17 + 81%|███████████████████████████████████████████████████████████████████████████████████████████████████████▍ | 18966/23458 [5:29:00<1:05:01, 1.15it/s][2025-04-24 14:30:10] (step=0042425) Train Loss: 5.4065, Train Steps/Sec: 1.17 + 81%|███████████████████████████████████████████████████████████████████████████████████████████████████████▋ | 18991/23458 [5:29:21<1:03:47, 1.17it/s][2025-04-24 14:30:31] (step=0042450) Train Loss: 5.3578, Train Steps/Sec: 1.17 + 81%|███████████████████████████████████████████████████████████████████████████████████████████████████████▊ | 19016/23458 [5:29:42<1:02:40, 1.18it/s][2025-04-24 14:30:52] (step=0042475) Train Loss: 5.3888, Train Steps/Sec: 1.17 + 81%|███████████████████████████████████████████████████████████████████████████████████████████████████████▉ | 19041/23458 [5:30:04<1:02:31, 1.18it/s][2025-04-24 14:31:14] (step=0042500) Train Loss: 5.3962, Train Steps/Sec: 1.16 + 81%|████████████████████████████████████████████████████████████████████████████████████████████████████████ | 19066/23458 [5:30:25<1:03:57, 1.14it/s][2025-04-24 14:31:35] (step=0042525) Train Loss: 5.3548, Train Steps/Sec: 1.17 + 81%|████████████████████████████████████████████████████████████████████████████████████████████████████████▏ | 19091/23458 [5:30:47<1:02:06, 1.17it/s][2025-04-24 14:31:57] (step=0042550) Train Loss: 5.3534, Train Steps/Sec: 1.17 + 81%|████████████████████████████████████████████████████████████████████████████████████████████████████████▎ | 19116/23458 [5:31:09<1:01:26, 1.18it/s][2025-04-24 14:32:19] (step=0042575) Train Loss: 5.3578, Train Steps/Sec: 1.12 + 82%|████████████████████████████████████████████████████████████████████████████████████████████████████████▍ | 19141/23458 [5:31:30<1:00:50, 1.18it/s][2025-04-24 14:32:40] (step=0042600) Train Loss: 5.3585, Train Steps/Sec: 1.17 + 82%|████████████████████████████████████████████████████████████████████████████████████████████████████████▌ | 19166/23458 [5:31:52<1:02:14, 1.15it/s][2025-04-24 14:33:02] (step=0042625) Train Loss: 5.4102, Train Steps/Sec: 1.17 + 82%|████████████████████████████████████████████████████████████████████████████████████████████████████████▋ | 19191/23458 [5:32:14<1:13:51, 1.04s/it][2025-04-24 14:33:24] (step=0042650) Train Loss: 5.3603, Train Steps/Sec: 1.13 + 82%|██████████████████████████████████████████████████████████████████████████████████████████████████████████▍ | 19216/23458 [5:32:35<59:57, 1.18it/s][2025-04-24 14:33:45] (step=0042675) Train Loss: 5.3248, Train Steps/Sec: 1.17 + 82%|██████████████████████████████████████████████████████████████████████████████████████████████████████████▋ | 19241/23458 [5:32:57<59:18, 1.18it/s][2025-04-24 14:34:08] (step=0042700) Train Loss: 5.4126, Train Steps/Sec: 1.12 + 82%|█████████████████████████████████████████████████████████████████████████████████████████████████████████▏ | 19266/23458 [5:33:19<1:00:49, 1.15it/s][2025-04-24 14:34:29] (step=0042725) Train Loss: 5.3865, Train Steps/Sec: 1.17 + 82%|██████████████████████████████████████████████████████████████████████████████████████████████████████████▉ | 19291/23458 [5:33:40<59:50, 1.16it/s][2025-04-24 14:34:50] (step=0042750) Train Loss: 5.3205, Train Steps/Sec: 1.17 + 82%|███████████████████████████████████████████████████████████████████████████████████████████████████████████ | 19316/23458 [5:34:02<59:49, 1.15it/s][2025-04-24 14:35:13] (step=0042775) Train Loss: 5.3785, Train Steps/Sec: 1.13 + 82%|███████████████████████████████████████████████████████████████████████████████████████████████████████████▏ | 19341/23458 [5:34:26<59:23, 1.16it/s][2025-04-24 14:35:36] (step=0042800) Train Loss: 5.4610, Train Steps/Sec: 1.08 + 83%|███████████████████████████████████████████████████████████████████████████████████████████████████████████▎ | 19366/23458 [5:34:47<59:14, 1.15it/s][2025-04-24 14:35:57] (step=0042825) Train Loss: 5.3499, Train Steps/Sec: 1.17 + 83%|███████████████████████████████████████████████████████████████████████████████████████████████████████████▍ | 19391/23458 [5:35:08<57:54, 1.17it/s][2025-04-24 14:36:18] (step=0042850) Train Loss: 5.4028, Train Steps/Sec: 1.17 + 83%|███████████████████████████████████████████████████████████████████████████████████████████████████████████▌ | 19416/23458 [5:35:30<57:19, 1.18it/s][2025-04-24 14:36:40] (step=0042875) Train Loss: 5.4273, Train Steps/Sec: 1.17 + 83%|███████████████████████████████████████████████████████████████████████████████████████████████████████████▋ | 19441/23458 [5:35:52<56:33, 1.18it/s][2025-04-24 14:37:02] (step=0042900) Train Loss: 5.3952, Train Steps/Sec: 1.12 + 83%|███████████████████████████████████████████████████████████████████████████████████████████████████████████▉ | 19466/23458 [5:36:13<57:54, 1.15it/s][2025-04-24 14:37:23] (step=0042925) Train Loss: 5.3373, Train Steps/Sec: 1.17 + 83%|████████████████████████████████████████████████████████████████████████████████████████████████████████████ | 19491/23458 [5:36:35<56:43, 1.17it/s][2025-04-24 14:37:45] (step=0042950) Train Loss: 5.3690, Train Steps/Sec: 1.17 + 83%|████████████████████████████████████████████████████████████████████████████████████████████████████████████▏ | 19516/23458 [5:36:56<55:58, 1.17it/s][2025-04-24 14:38:06] (step=0042975) Train Loss: 5.2976, Train Steps/Sec: 1.17 + 83%|████████████████████████████████████████████████████████████████████████████████████████████████████████████▎ | 19541/23458 [5:37:17<55:02, 1.19it/s][2025-04-24 14:38:27] (step=0043000) Train Loss: 5.3527, Train Steps/Sec: 1.17 + 83%|████████████████████████████████████████████████████████████████████████████████████████████████████████████▍ | 19566/23458 [5:37:39<57:13, 1.13it/s][2025-04-24 14:38:49] (step=0043025) Train Loss: 5.4105, Train Steps/Sec: 1.17 + 84%|████████████████████████████████████████████████████████████████████████████████████████████████████████████▌ | 19591/23458 [5:38:00<55:25, 1.16it/s][2025-04-24 14:39:10] (step=0043050) Train Loss: 5.3778, Train Steps/Sec: 1.17 + 84%|████████████████████████████████████████████████████████████████████████████████████████████████████████████▋ | 19616/23458 [5:38:22<54:39, 1.17it/s][2025-04-24 14:39:32] (step=0043075) Train Loss: 5.3676, Train Steps/Sec: 1.12 + 84%|████████████████████████████████████████████████████████████████████████████████████████████████████████████▊ | 19641/23458 [5:38:44<53:47, 1.18it/s][2025-04-24 14:39:54] (step=0043100) Train Loss: 5.4025, Train Steps/Sec: 1.17 + 84%|████████████████████████████████████████████████████████████████████████████████████████████████████████████▉ | 19666/23458 [5:39:05<54:53, 1.15it/s][2025-04-24 14:40:15] (step=0043125) Train Loss: 5.3961, Train Steps/Sec: 1.17 + 84%|█████████████████████████████████████████████████████████████████████████████████████████████████████████████ | 19691/23458 [5:39:26<53:43, 1.17it/s][2025-04-24 14:40:37] (step=0043150) Train Loss: 5.4422, Train Steps/Sec: 1.17 + 84%|█████████████████████████████████████████████████████████████████████████████████████████████████████████████▎ | 19716/23458 [5:39:48<52:55, 1.18it/s][2025-04-24 14:40:58] (step=0043175) Train Loss: 5.3020, Train Steps/Sec: 1.17 + 84%|█████████████████████████████████████████████████████████████████████████████████████████████████████████████▍ | 19741/23458 [5:40:09<52:22, 1.18it/s][2025-04-24 14:41:19] (step=0043200) Train Loss: 5.3199, Train Steps/Sec: 1.17 + 84%|█████████████████████████████████████████████████████████████████████████████████████████████████████████████▌ | 19766/23458 [5:40:31<53:15, 1.16it/s][2025-04-24 14:41:41] (step=0043225) Train Loss: 5.3092, Train Steps/Sec: 1.17 + 84%|█████████████████████████████████████████████████████████████████████████████████████████████████████████████▋ | 19791/23458 [5:40:52<52:10, 1.17it/s][2025-04-24 14:42:02] (step=0043250) Train Loss: 5.3972, Train Steps/Sec: 1.17 + 84%|█████████████████████████████████████████████████████████████████████████████████████████████████████████████▊ | 19816/23458 [5:41:13<51:27, 1.18it/s][2025-04-24 14:42:23] (step=0043275) Train Loss: 5.3018, Train Steps/Sec: 1.17 + 85%|█████████████████████████████████████████████████████████████████████████████████████████████████████████████▉ | 19841/23458 [5:41:36<50:44, 1.19it/s][2025-04-24 14:42:46] (step=0043300) Train Loss: 5.3800, Train Steps/Sec: 1.11 + 85%|██████████████████████████████████████████████████████████████████████████████████████████████████████████████ | 19866/23458 [5:41:58<52:02, 1.15it/s][2025-04-24 14:43:08] (step=0043325) Train Loss: 5.3792, Train Steps/Sec: 1.13 + 85%|██████████████████████████████████████████████████████████████████████████████████████████████████████████████▏ | 19891/23458 [5:42:20<51:53, 1.15it/s][2025-04-24 14:43:30] (step=0043350) Train Loss: 5.4345, Train Steps/Sec: 1.12 + 85%|██████████████████████████████████████████████████████████████████████████████████████████████████████████████▎ | 19916/23458 [5:42:42<50:22, 1.17it/s][2025-04-24 14:43:52] (step=0043375) Train Loss: 5.2724, Train Steps/Sec: 1.17 + 85%|██████████████████████████████████████████████████████████████████████████████████████████████████████████████▌ | 19941/23458 [5:43:03<49:41, 1.18it/s][2025-04-24 14:44:13] (step=0043400) Train Loss: 5.3531, Train Steps/Sec: 1.16 + 85%|██████████████████████████████████████████████████████████████████████████████████████████████████████████████▋ | 19966/23458 [5:43:24<50:34, 1.15it/s][2025-04-24 14:44:34] (step=0043425) Train Loss: 5.2922, Train Steps/Sec: 1.17 + 85%|██████████████████████████████████████████████████████████████████████████████████████████████████████████████▊ | 19991/23458 [5:43:47<49:25, 1.17it/s][2025-04-24 14:44:58] (step=0043450) Train Loss: 5.4083, Train Steps/Sec: 1.08 + 85%|██████████████████████████████████████████████████████████████████████████████████████████████████████████████▉ | 20016/23458 [5:44:10<48:46, 1.18it/s][2025-04-24 14:45:20] (step=0043475) Train Loss: 5.4297, Train Steps/Sec: 1.12 + 85%|███████████████████████████████████████████████████████████████████████████████████████████████████████████████ | 20041/23458 [5:44:31<47:58, 1.19it/s][2025-04-24 14:45:41] (step=0043500) Train Loss: 5.3746, Train Steps/Sec: 1.17 + 86%|███████████████████████████████████████████████████████████████████████████████████████████████████████████████▏ | 20066/23458 [5:44:52<49:25, 1.14it/s][2025-04-24 14:46:03] (step=0043525) Train Loss: 5.3094, Train Steps/Sec: 1.17 + 86%|███████████████████████████████████████████████████████████████████████████████████████████████████████████████▎ | 20091/23458 [5:45:15<48:52, 1.15it/s][2025-04-24 14:46:25] (step=0043550) Train Loss: 5.3138, Train Steps/Sec: 1.13 + 86%|███████████████████████████████████████████████████████████████████████████████████████████████████████████████▍ | 20116/23458 [5:45:36<47:03, 1.18it/s][2025-04-24 14:46:46] (step=0043575) Train Loss: 5.3246, Train Steps/Sec: 1.17 + 86%|███████████████████████████████████████████████████████████████████████████████████████████████████████████████▌ | 20141/23458 [5:45:57<46:44, 1.18it/s][2025-04-24 14:47:08] (step=0043600) Train Loss: 5.3643, Train Steps/Sec: 1.17 + 86%|███████████████████████████████████████████████████████████████████████████████████████████████████████████████▊ | 20166/23458 [5:46:19<48:22, 1.13it/s][2025-04-24 14:47:29] (step=0043625) Train Loss: 5.3798, Train Steps/Sec: 1.17 + 86%|███████████████████████████████████████████████████████████████████████████████████████████████████████████████▉ | 20191/23458 [5:46:40<46:30, 1.17it/s][2025-04-24 14:47:50] (step=0043650) Train Loss: 5.3471, Train Steps/Sec: 1.17 + 86%|████████████████████████████████████████████████████████████████████████████████████████████████████████████████ | 20216/23458 [5:47:02<46:06, 1.17it/s][2025-04-24 14:48:12] (step=0043675) Train Loss: 5.3267, Train Steps/Sec: 1.17 + 86%|████████████████████████████████████████████████████████████████████████████████████████████████████████████████▏ | 20241/23458 [5:47:23<45:18, 1.18it/s][2025-04-24 14:48:33] (step=0043700) Train Loss: 5.3575, Train Steps/Sec: 1.16 + 86%|████████████████████████████████████████████████████████████████████████████████████████████████████████████████▎ | 20266/23458 [5:47:44<46:11, 1.15it/s][2025-04-24 14:48:54] (step=0043725) Train Loss: 5.2876, Train Steps/Sec: 1.17 + 86%|████████████████████████████████████████████████████████████████████████████████████████████████████████████████▍ | 20291/23458 [5:48:06<45:24, 1.16it/s][2025-04-24 14:49:16] (step=0043750) Train Loss: 5.4190, Train Steps/Sec: 1.17 + 87%|████████████████████████████████████████████████████████████████████████████████████████████████████████████████▌ | 20316/23458 [5:48:27<44:29, 1.18it/s][2025-04-24 14:49:37] (step=0043775) Train Loss: 5.3367, Train Steps/Sec: 1.17 + 87%|████████████████████████████████████████████████████████████████████████████████████████████████████████████████▋ | 20341/23458 [5:48:50<43:49, 1.19it/s][2025-04-24 14:50:00] (step=0043800) Train Loss: 5.4266, Train Steps/Sec: 1.11 + 87%|████████████████████████████████████████████████████████████████████████████████████████████████████████████████▊ | 20366/23458 [5:49:11<44:57, 1.15it/s][2025-04-24 14:50:21] (step=0043825) Train Loss: 5.3375, Train Steps/Sec: 1.17 + 87%|█████████████████████████████████████████████████████████████████████████████████████████████████████████████████ | 20391/23458 [5:49:32<43:47, 1.17it/s][2025-04-24 14:50:42] (step=0043850) Train Loss: 5.3039, Train Steps/Sec: 1.17 + 87%|█████████████████████████████████████████████████████████████████████████████████████████████████████████████████▏ | 20416/23458 [5:49:54<42:54, 1.18it/s][2025-04-24 14:51:04] (step=0043875) Train Loss: 5.3923, Train Steps/Sec: 1.17 + 87%|█████████████████████████████████████████████████████████████████████████████████████████████████████████████████▎ | 20441/23458 [5:50:15<42:40, 1.18it/s][2025-04-24 14:51:25] (step=0043900) Train Loss: 5.3499, Train Steps/Sec: 1.17 + 87%|█████████████████████████████████████████████████████████████████████████████████████████████████████████████████▍ | 20466/23458 [5:50:37<43:39, 1.14it/s][2025-04-24 14:51:47] (step=0043925) Train Loss: 5.3395, Train Steps/Sec: 1.17 + 87%|█████████████████████████████████████████████████████████████████████████████████████████████████████████████████▌ | 20491/23458 [5:50:58<42:13, 1.17it/s][2025-04-24 14:52:08] (step=0043950) Train Loss: 5.3378, Train Steps/Sec: 1.17 + 87%|█████████████████████████████████████████████████████████████████████████████████████████████████████████████████▋ | 20516/23458 [5:51:20<42:03, 1.17it/s][2025-04-24 14:52:30] (step=0043975) Train Loss: 5.4040, Train Steps/Sec: 1.13 + 88%|█████████████████████████████████████████████████████████████████████████████████████████████████████████████████▊ | 20541/23458 [5:51:42<47:58, 1.01it/s][2025-04-24 14:52:53] (step=0044000) Train Loss: 5.3958, Train Steps/Sec: 1.12 +You are using a model of type blip-2 to instantiate a model of type instructblip. This is not supported for all configurations of models and can yield errors. +[2025-04-24 14:52:53] text_config is None. Initializing the text config with default values (`OPTConfig`). +model_name_or_path:/tmp/haozhezhao/model/blip2-flan-t5-xl +image_place_holder: +tokenizer length after expend 32101 +tokenizer length before expend 32100 +100%|██████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████| 4/4 [03:58<00:00, 59.60s/it] +[2025-04-24 14:58:00] Finish Eval in 44000 steps...███████████████████████████████████████████████████████████████████████████████████████████| 4/4 [03:57<00:00, 59.24s/it] +processing left-padding... +processing left-padding... +processing left-padding... +processing left-padding... +[2025-04-24 14:58:19] Saved checkpoint to checkpoint/Afteri2i_100kckpt_LLAVAT5_empty_Fixed_just_segment_pretrain_stage3_dreambench_recap_400k_t2i_400k_flux_200k_midjourney_150k_recovery_150k_3e4_epoch2_no_rotated/000-GPT-XL/checkpoints/0044000.pt +[2025-04-24 14:58:21] Removed old checkpoint: checkpoint/Afteri2i_100kckpt_LLAVAT5_empty_Fixed_just_segment_pretrain_stage3_dreambench_recap_400k_t2i_400k_flux_200k_midjourney_150k_recovery_150k_3e4_epoch2_no_rotated/000-GPT-XL/checkpoints/0042000.pt + 88%|█████████████████████████████████████████████████████████████████████████████████████████████████████████████████▉ | 20566/23458 [5:57:33<42:41, 1.13it/s][2025-04-24 14:58:43] (step=0044025) Train Loss: 5.2961, Train Steps/Sec: 0.07 + 88%|██████████████████████████████████████████████████████████████████████████████████████████████████████████████████ | 20591/23458 [5:57:55<40:41, 1.17it/s][2025-04-24 14:59:05] (step=0044050) Train Loss: 5.3310, Train Steps/Sec: 1.17 + 88%|██████████████████████████████████████████████████████████████████████████████████████████████████████████████████▎ | 20616/23458 [5:58:16<40:04, 1.18it/s][2025-04-24 14:59:26] (step=0044075) Train Loss: 5.3132, Train Steps/Sec: 1.18 + 88%|██████████████████████████████████████████████████████████████████████████████████████████████████████████████████▍ | 20641/23458 [5:58:39<41:14, 1.14it/s][2025-04-24 14:59:49] (step=0044100) Train Loss: 5.4157, Train Steps/Sec: 1.08 + 88%|██████████████████████████████████████████████████████████████████████████████████████████████████████████████████▌ | 20666/23458 [5:59:01<41:16, 1.13it/s][2025-04-24 15:00:11] (step=0044125) Train Loss: 5.3271, Train Steps/Sec: 1.13 + 88%|██████████████████████████████████████████████████████████████████████████████████████████████████████████████████▋ | 20691/23458 [5:59:23<39:21, 1.17it/s][2025-04-24 15:00:33] (step=0044150) Train Loss: 5.4176, Train Steps/Sec: 1.17 + 88%|██████████████████████████████████████████████████████████████████████████████████████████████████████████████████▊ | 20716/23458 [5:59:44<39:06, 1.17it/s][2025-04-24 15:00:54] (step=0044175) Train Loss: 5.4108, Train Steps/Sec: 1.17 + 88%|██████████████████████████████████████████████████████████████████████████████████████████████████████████████████▉ | 20741/23458 [6:00:06<44:14, 1.02it/s][2025-04-24 15:01:16] (step=0044200) Train Loss: 5.3260, Train Steps/Sec: 1.12 + 89%|███████████████████████████████████████████████████████████████████████████████████████████████████████████████████ | 20766/23458 [6:00:28<38:58, 1.15it/s][2025-04-24 15:01:38] (step=0044225) Train Loss: 5.4521, Train Steps/Sec: 1.17 + 89%|███████████████████████████████████████████████████████████████████████████████████████████████████████████████████▏ | 20791/23458 [6:00:49<37:56, 1.17it/s][2025-04-24 15:01:59] (step=0044250) Train Loss: 5.2540, Train Steps/Sec: 1.17 + 89%|███████████████████████████████████████████████████████████████████████████████████████████████████████████████████▎ | 20816/23458 [6:01:10<37:20, 1.18it/s][2025-04-24 15:02:21] (step=0044275) Train Loss: 5.4447, Train Steps/Sec: 1.17 + 89%|███████████████████████████████████████████████████████████████████████████████████████████████████████████████████▍ | 20841/23458 [6:01:32<36:53, 1.18it/s][2025-04-24 15:02:42] (step=0044300) Train Loss: 5.3279, Train Steps/Sec: 1.15 + 89%|███████████████████████████████████████████████████████████████████████████████████████████████████████████████████▋ | 20866/23458 [6:01:54<37:28, 1.15it/s][2025-04-24 15:03:04] (step=0044325) Train Loss: 5.3360, Train Steps/Sec: 1.17 + 89%|███████████████████████████████████████████████████████████████████████████████████████████████████████████████████▊ | 20891/23458 [6:02:15<36:47, 1.16it/s][2025-04-24 15:03:25] (step=0044350) Train Loss: 5.3412, Train Steps/Sec: 1.17 + 89%|███████████████████████████████████████████████████████████████████████████████████████████████████████████████████▉ | 20916/23458 [6:02:36<35:58, 1.18it/s][2025-04-24 15:03:46] (step=0044375) Train Loss: 5.3127, Train Steps/Sec: 1.17 + 89%|████████████████████████████████████████████████████████████████████████████████████████████████████████████████████ | 20941/23458 [6:02:58<35:26, 1.18it/s][2025-04-24 15:04:08] (step=0044400) Train Loss: 5.3130, Train Steps/Sec: 1.17 + 89%|████████████████████████████████████████████████████████████████████████████████████████████████████████████████████▏ | 20966/23458 [6:03:19<36:07, 1.15it/s][2025-04-24 15:04:29] (step=0044425) Train Loss: 5.3956, Train Steps/Sec: 1.17 + 89%|████████████████████████████████████████████████████████████████████████████████████████████████████████████████████▎ | 20991/23458 [6:03:40<35:07, 1.17it/s][2025-04-24 15:04:50] (step=0044450) Train Loss: 5.4003, Train Steps/Sec: 1.17 + 90%|████████████████████████████████████████████████████████████████████████████████████████████████████████████████████▍ | 21016/23458 [6:04:02<34:38, 1.17it/s][2025-04-24 15:05:12] (step=0044475) Train Loss: 5.2661, Train Steps/Sec: 1.17 + 90%|████████████████████████████████████████████████████████████████████████████████████████████████████████████████████▌ | 21041/23458 [6:04:23<34:01, 1.18it/s][2025-04-24 15:05:33] (step=0044500) Train Loss: 5.3642, Train Steps/Sec: 1.16 + 90%|████████████████████████████████████████████████████████████████████████████████████████████████████████████████████▋ | 21066/23458 [6:04:46<34:57, 1.14it/s][2025-04-24 15:05:56] (step=0044525) Train Loss: 5.3086, Train Steps/Sec: 1.11 + 90%|████████████████████████████████████████████████████████████████████████████████████████████████████████████████████▉ | 21091/23458 [6:05:07<33:50, 1.17it/s][2025-04-24 15:06:17] (step=0044550) Train Loss: 5.2828, Train Steps/Sec: 1.17 + 90%|█████████████████████████████████████████████████████████████████████████████████████████████████████████████████████ | 21116/23458 [6:05:28<33:16, 1.17it/s][2025-04-24 15:06:38] (step=0044575) Train Loss: 5.3477, Train Steps/Sec: 1.17 + 90%|█████████████████████████████████████████████████████████████████████████████████████████████████████████████████████▏ | 21141/23458 [6:05:50<32:31, 1.19it/s][2025-04-24 15:07:00] (step=0044600) Train Loss: 5.4552, Train Steps/Sec: 1.17 + 90%|█████████████████████████████████████████████████████████████████████████████████████████████████████████████████████▎ | 21166/23458 [6:06:11<33:06, 1.15it/s][2025-04-24 15:07:21] (step=0044625) Train Loss: 5.2984, Train Steps/Sec: 1.17 + 90%|█████████████████████████████████████████████████████████████████████████████████████████████████████████████████████▍ | 21191/23458 [6:06:34<39:21, 1.04s/it][2025-04-24 15:07:44] (step=0044650) Train Loss: 5.2785, Train Steps/Sec: 1.11 + 90%|█████████████████████████████████████████████████████████████████████████████████████████████████████████████████████▌ | 21216/23458 [6:06:56<31:39, 1.18it/s][2025-04-24 15:08:06] (step=0044675) Train Loss: 5.3587, Train Steps/Sec: 1.12 + 91%|█████████████████████████████████████████████████████████████████████████████████████████████████████████████████████▋ | 21241/23458 [6:07:17<31:15, 1.18it/s][2025-04-24 15:08:27] (step=0044700) Train Loss: 5.2732, Train Steps/Sec: 1.16 + 91%|█████████████████████████████████████████████████████████████████████████████████████████████████████████████████████▊ | 21266/23458 [6:07:39<31:48, 1.15it/s][2025-04-24 15:08:49] (step=0044725) Train Loss: 5.3421, Train Steps/Sec: 1.17 + 91%|█████████████████████████████████████████████████████████████████████████████████████████████████████████████████████▉ | 21291/23458 [6:08:02<35:24, 1.02it/s][2025-04-24 15:09:12] (step=0044750) Train Loss: 5.4088, Train Steps/Sec: 1.08 + 91%|██████████████████████████████████████████████████████████████████████████████████████████████████████████████████████▏ | 21316/23458 [6:08:24<30:16, 1.18it/s][2025-04-24 15:09:34] (step=0044775) Train Loss: 5.4134, Train Steps/Sec: 1.12 + 91%|██████████████████████████████████████████████████████████████████████████████████████████████████████████████████████▎ | 21341/23458 [6:08:46<29:45, 1.19it/s][2025-04-24 15:09:57] (step=0044800) Train Loss: 5.3649, Train Steps/Sec: 1.12 + 91%|██████████████████████████████████████████████████████████████████████████████████████████████████████████████████████▍ | 21366/23458 [6:09:08<30:19, 1.15it/s][2025-04-24 15:10:18] (step=0044825) Train Loss: 5.4438, Train Steps/Sec: 1.17 + 91%|██████████████████████████████████████████████████████████████████████████████████████████████████████████████████████▌ | 21391/23458 [6:09:29<29:30, 1.17it/s][2025-04-24 15:10:39] (step=0044850) Train Loss: 5.3291, Train Steps/Sec: 1.17 + 91%|██████████████████████████████████████████████████████████████████████████████████████████████████████████████████████▋ | 21416/23458 [6:09:51<28:59, 1.17it/s][2025-04-24 15:11:01] (step=0044875) Train Loss: 5.3579, Train Steps/Sec: 1.12 + 91%|██████████████████████████████████████████████████████████████████████████████████████████████████████████████████████▊ | 21441/23458 [6:10:13<28:27, 1.18it/s][2025-04-24 15:11:23] (step=0044900) Train Loss: 5.4260, Train Steps/Sec: 1.17 + 92%|██████████████████████████████████████████████████████████████████████████████████████████████████████████████████████▉ | 21466/23458 [6:10:34<28:48, 1.15it/s][2025-04-24 15:11:44] (step=0044925) Train Loss: 5.3263, Train Steps/Sec: 1.17 + 92%|███████████████████████████████████████████████████████████████████████████████████████████████████████████████████████ | 21491/23458 [6:10:55<28:23, 1.15it/s][2025-04-24 15:12:06] (step=0044950) Train Loss: 5.3517, Train Steps/Sec: 1.17 + 92%|███████████████████████████████████████████████████████████████████████████████████████████████████████████████████████▏ | 21516/23458 [6:11:17<27:31, 1.18it/s][2025-04-24 15:12:27] (step=0044975) Train Loss: 5.4278, Train Steps/Sec: 1.17 + 92%|███████████████████████████████████████████████████████████████████████████████████████████████████████████████████████▍ | 21541/23458 [6:11:38<26:57, 1.19it/s][2025-04-24 15:12:48] (step=0045000) Train Loss: 5.3542, Train Steps/Sec: 1.17 + 92%|███████████████████████████████████████████████████████████████████████████████████████████████████████████████████████▌ | 21566/23458 [6:12:00<27:26, 1.15it/s][2025-04-24 15:13:10] (step=0045025) Train Loss: 5.3356, Train Steps/Sec: 1.17 + 92%|███████████████████████████████████████████████████████████████████████████████████████████████████████████████████████▋ | 21591/23458 [6:12:21<26:28, 1.18it/s][2025-04-24 15:13:31] (step=0045050) Train Loss: 5.3747, Train Steps/Sec: 1.17 + 92%|███████████████████████████████████████████████████████████████████████████████████████████████████████████████████████▊ | 21616/23458 [6:12:42<26:05, 1.18it/s][2025-04-24 15:13:52] (step=0045075) Train Loss: 5.2835, Train Steps/Sec: 1.17 + 92%|███████████████████████████████████████████████████████████████████████████████████████████████████████████████████████▉ | 21641/23458 [6:13:04<25:29, 1.19it/s][2025-04-24 15:14:14] (step=0045100) Train Loss: 5.3420, Train Steps/Sec: 1.17 + 92%|████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████ | 21666/23458 [6:13:25<25:57, 1.15it/s][2025-04-24 15:14:35] (step=0045125) Train Loss: 5.2902, Train Steps/Sec: 1.17 + 92%|████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████▏ | 21691/23458 [6:13:46<25:08, 1.17it/s][2025-04-24 15:14:56] (step=0045150) Train Loss: 5.4017, Train Steps/Sec: 1.17 + 93%|████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████▎ | 21716/23458 [6:14:08<24:41, 1.18it/s][2025-04-24 15:15:18] (step=0045175) Train Loss: 5.3824, Train Steps/Sec: 1.17 + 93%|████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████▍ | 21741/23458 [6:14:29<24:14, 1.18it/s][2025-04-24 15:15:39] (step=0045200) Train Loss: 5.3239, Train Steps/Sec: 1.17 + 93%|████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████▌ | 21766/23458 [6:14:50<24:32, 1.15it/s][2025-04-24 15:16:01] (step=0045225) Train Loss: 5.3674, Train Steps/Sec: 1.17 + 93%|████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████▊ | 21791/23458 [6:15:13<23:52, 1.16it/s][2025-04-24 15:16:23] (step=0045250) Train Loss: 5.4348, Train Steps/Sec: 1.11 + 93%|████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████▉ | 21816/23458 [6:15:34<23:18, 1.17it/s][2025-04-24 15:16:44] (step=0045275) Train Loss: 5.2441, Train Steps/Sec: 1.17 + 93%|█████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████ | 21841/23458 [6:15:56<22:43, 1.19it/s][2025-04-24 15:17:06] (step=0045300) Train Loss: 5.3340, Train Steps/Sec: 1.17 + 93%|█████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████▏ | 21866/23458 [6:16:19<24:00, 1.11it/s][2025-04-24 15:17:29] (step=0045325) Train Loss: 5.3702, Train Steps/Sec: 1.08 + 93%|█████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████▎ | 21891/23458 [6:16:40<22:23, 1.17it/s][2025-04-24 15:17:50] (step=0045350) Train Loss: 5.4778, Train Steps/Sec: 1.17 + 93%|█████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████▍ | 21916/23458 [6:17:02<21:46, 1.18it/s][2025-04-24 15:18:12] (step=0045375) Train Loss: 5.3637, Train Steps/Sec: 1.17 + 94%|█████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████▌ | 21941/23458 [6:17:23<21:16, 1.19it/s][2025-04-24 15:18:33] (step=0045400) Train Loss: 5.2872, Train Steps/Sec: 1.16 + 94%|█████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████▋ | 21966/23458 [6:17:46<21:36, 1.15it/s][2025-04-24 15:18:56] (step=0045425) Train Loss: 5.3114, Train Steps/Sec: 1.09 + 94%|█████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████▊ | 21991/23458 [6:18:08<20:53, 1.17it/s][2025-04-24 15:19:18] (step=0045450) Train Loss: 5.3929, Train Steps/Sec: 1.13 + 94%|██████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████ | 22016/23458 [6:18:31<20:46, 1.16it/s][2025-04-24 15:19:41] (step=0045475) Train Loss: 5.4169, Train Steps/Sec: 1.12 + 94%|██████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████▏ | 22041/23458 [6:18:52<19:53, 1.19it/s][2025-04-24 15:20:02] (step=0045500) Train Loss: 5.4487, Train Steps/Sec: 1.17 + 94%|██████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████▎ | 22066/23458 [6:19:14<20:38, 1.12it/s][2025-04-24 15:20:24] (step=0045525) Train Loss: 5.3873, Train Steps/Sec: 1.13 + 94%|██████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████▍ | 22091/23458 [6:19:36<19:29, 1.17it/s][2025-04-24 15:20:46] (step=0045550) Train Loss: 5.3651, Train Steps/Sec: 1.17 + 94%|██████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████▌ | 22116/23458 [6:19:57<19:00, 1.18it/s][2025-04-24 15:21:07] (step=0045575) Train Loss: 5.3151, Train Steps/Sec: 1.17 + 94%|██████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████▋ | 22141/23458 [6:20:18<18:30, 1.19it/s][2025-04-24 15:21:28] (step=0045600) Train Loss: 5.3295, Train Steps/Sec: 1.17 + 94%|██████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████▊ | 22166/23458 [6:20:40<18:41, 1.15it/s][2025-04-24 15:21:50] (step=0045625) Train Loss: 5.3815, Train Steps/Sec: 1.17 + 95%|██████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████▉ | 22191/23458 [6:21:01<17:58, 1.17it/s][2025-04-24 15:22:11] (step=0045650) Train Loss: 5.3210, Train Steps/Sec: 1.17 + 95%|███████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████ | 22216/23458 [6:21:22<17:31, 1.18it/s][2025-04-24 15:22:32] (step=0045675) Train Loss: 5.4804, Train Steps/Sec: 1.17 + 95%|███████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████▎ | 22241/23458 [6:21:44<17:08, 1.18it/s][2025-04-24 15:22:54] (step=0045700) Train Loss: 5.4092, Train Steps/Sec: 1.16 + 95%|███████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████▍ | 22266/23458 [6:22:05<17:16, 1.15it/s][2025-04-24 15:23:15] (step=0045725) Train Loss: 5.3221, Train Steps/Sec: 1.17 + 95%|███████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████▌ | 22291/23458 [6:22:26<16:35, 1.17it/s][2025-04-24 15:23:37] (step=0045750) Train Loss: 5.3596, Train Steps/Sec: 1.17 + 95%|███████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████▋ | 22316/23458 [6:22:48<16:07, 1.18it/s][2025-04-24 15:23:58] (step=0045775) Train Loss: 5.2999, Train Steps/Sec: 1.17 + 95%|███████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████▊ | 22341/23458 [6:23:09<15:41, 1.19it/s][2025-04-24 15:24:19] (step=0045800) Train Loss: 5.3564, Train Steps/Sec: 1.17 + 95%|███████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████▉ | 22366/23458 [6:23:31<15:52, 1.15it/s][2025-04-24 15:24:41] (step=0045825) Train Loss: 5.4261, Train Steps/Sec: 1.17 + 95%|████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████ | 22391/23458 [6:23:52<15:11, 1.17it/s][2025-04-24 15:25:02] (step=0045850) Train Loss: 5.3527, Train Steps/Sec: 1.17 + 96%|████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████▏ | 22416/23458 [6:24:13<14:42, 1.18it/s][2025-04-24 15:25:23] (step=0045875) Train Loss: 5.3850, Train Steps/Sec: 1.17 + 96%|████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████▎ | 22441/23458 [6:24:35<14:19, 1.18it/s][2025-04-24 15:25:45] (step=0045900) Train Loss: 5.3263, Train Steps/Sec: 1.17 + 96%|████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████▌ | 22466/23458 [6:24:56<14:20, 1.15it/s][2025-04-24 15:26:06] (step=0045925) Train Loss: 5.3156, Train Steps/Sec: 1.17 + 96%|████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████▋ | 22491/23458 [6:25:17<13:50, 1.16it/s][2025-04-24 15:26:27] (step=0045950) Train Loss: 5.4064, Train Steps/Sec: 1.17 + 96%|████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████▊ | 22516/23458 [6:25:41<13:39, 1.15it/s][2025-04-24 15:26:51] (step=0045975) Train Loss: 5.3635, Train Steps/Sec: 1.07 + 96%|████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████▉ | 22541/23458 [6:26:03<12:52, 1.19it/s][2025-04-24 15:27:13] (step=0046000) Train Loss: 5.3436, Train Steps/Sec: 1.12 +You are using a model of type blip-2 to instantiate a model of type instructblip. This is not supported for all configurations of models and can yield errors. +[2025-04-24 15:27:13] text_config is None. Initializing the text config with default values (`OPTConfig`). +model_name_or_path:/tmp/haozhezhao/model/blip2-flan-t5-xl +image_place_holder: +tokenizer length after expend 32101 +tokenizer length before expend 32100 +100%|██████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████| 4/4 [03:57<00:00, 59.29s/it] +[2025-04-24 15:32:19] Finish Eval in 46000 steps...███████████████████████████████████████████████████████████████████████████████████████████| 4/4 [03:56<00:00, 58.90s/it] +processing left-padding... +processing left-padding... +processing left-padding... +processing left-padding... +[2025-04-24 15:32:38] Saved checkpoint to checkpoint/Afteri2i_100kckpt_LLAVAT5_empty_Fixed_just_segment_pretrain_stage3_dreambench_recap_400k_t2i_400k_flux_200k_midjourney_150k_recovery_150k_3e4_epoch2_no_rotated/000-GPT-XL/checkpoints/0046000.pt +[2025-04-24 15:32:40] Removed old checkpoint: checkpoint/Afteri2i_100kckpt_LLAVAT5_empty_Fixed_just_segment_pretrain_stage3_dreambench_recap_400k_t2i_400k_flux_200k_midjourney_150k_recovery_150k_3e4_epoch2_no_rotated/000-GPT-XL/checkpoints/0044000.pt + 96%|█████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████ | 22566/23458 [6:31:52<13:10, 1.13it/s][2025-04-24 15:33:02] (step=0046025) Train Loss: 5.2158, Train Steps/Sec: 0.07 + 96%|█████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████▏ | 22591/23458 [6:32:13<12:23, 1.17it/s][2025-04-24 15:33:23] (step=0046050) Train Loss: 5.3332, Train Steps/Sec: 1.17 + 96%|█████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████▎ | 22616/23458 [6:32:36<12:54, 1.09it/s][2025-04-24 15:33:46] (step=0046075) Train Loss: 5.2963, Train Steps/Sec: 1.08 + 97%|█████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████▍ | 22641/23458 [6:32:58<12:18, 1.11it/s][2025-04-24 15:34:08] (step=0046100) Train Loss: 5.3621, Train Steps/Sec: 1.12 + 97%|█████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████▌ | 22666/23458 [6:33:20<11:26, 1.15it/s][2025-04-24 15:34:30] (step=0046125) Train Loss: 5.3827, Train Steps/Sec: 1.17 + 97%|█████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████▋ | 22691/23458 [6:33:41<10:56, 1.17it/s][2025-04-24 15:34:51] (step=0046150) Train Loss: 5.2810, Train Steps/Sec: 1.17 + 97%|█████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████▉ | 22716/23458 [6:34:02<10:27, 1.18it/s][2025-04-24 15:35:12] (step=0046175) Train Loss: 5.3794, Train Steps/Sec: 1.17 + 97%|██████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████ | 22741/23458 [6:34:25<12:00, 1.00s/it][2025-04-24 15:35:36] (step=0046200) Train Loss: 5.3471, Train Steps/Sec: 1.07 + 97%|██████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████▏ | 22766/23458 [6:34:47<10:00, 1.15it/s][2025-04-24 15:35:57] (step=0046225) Train Loss: 5.3950, Train Steps/Sec: 1.15 + 97%|██████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████▎ | 22791/23458 [6:35:09<09:36, 1.16it/s][2025-04-24 15:36:19] (step=0046250) Train Loss: 5.2749, Train Steps/Sec: 1.17 + 97%|██████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████▍ | 22816/23458 [6:35:30<09:03, 1.18it/s][2025-04-24 15:36:40] (step=0046275) Train Loss: 5.3665, Train Steps/Sec: 1.17 + 97%|██████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████▌ | 22841/23458 [6:35:51<08:38, 1.19it/s][2025-04-24 15:37:01] (step=0046300) Train Loss: 5.2892, Train Steps/Sec: 1.17 + 97%|██████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████▋ | 22866/23458 [6:36:13<08:34, 1.15it/s][2025-04-24 15:37:23] (step=0046325) Train Loss: 5.3947, Train Steps/Sec: 1.17 + 98%|██████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████▊ | 22891/23458 [6:36:34<08:04, 1.17it/s][2025-04-24 15:37:44] (step=0046350) Train Loss: 5.4648, Train Steps/Sec: 1.17 + 98%|██████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████▉ | 22916/23458 [6:36:55<07:39, 1.18it/s][2025-04-24 15:38:05] (step=0046375) Train Loss: 5.3487, Train Steps/Sec: 1.17 + 98%|███████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████▏ | 22941/23458 [6:37:17<07:20, 1.17it/s][2025-04-24 15:38:27] (step=0046400) Train Loss: 5.3504, Train Steps/Sec: 1.16 + 98%|███████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████▎ | 22966/23458 [6:37:38<07:10, 1.14it/s][2025-04-24 15:38:48] (step=0046425) Train Loss: 5.4223, Train Steps/Sec: 1.17 + 98%|███████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████▍ | 22991/23458 [6:37:59<06:39, 1.17it/s][2025-04-24 15:39:10] (step=0046450) Train Loss: 5.3115, Train Steps/Sec: 1.17 + 98%|███████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████▌ | 23016/23458 [6:38:21<06:14, 1.18it/s][2025-04-24 15:39:31] (step=0046475) Train Loss: 5.3462, Train Steps/Sec: 1.17 + 98%|███████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████▋ | 23041/23458 [6:38:42<05:50, 1.19it/s][2025-04-24 15:39:52] (step=0046500) Train Loss: 5.2936, Train Steps/Sec: 1.17 + 98%|███████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████▊ | 23066/23458 [6:39:04<05:41, 1.15it/s][2025-04-24 15:40:14] (step=0046525) Train Loss: 5.3681, Train Steps/Sec: 1.17 + 98%|███████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████▉ | 23091/23458 [6:39:25<05:12, 1.17it/s][2025-04-24 15:40:35] (step=0046550) Train Loss: 5.3644, Train Steps/Sec: 1.17 + 99%|████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████ | 23116/23458 [6:39:46<04:50, 1.18it/s][2025-04-24 15:40:56] (step=0046575) Train Loss: 5.2854, Train Steps/Sec: 1.17 + 99%|████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████▏ | 23141/23458 [6:40:08<04:30, 1.17it/s][2025-04-24 15:41:18] (step=0046600) Train Loss: 5.3650, Train Steps/Sec: 1.17 + 99%|████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████▍ | 23166/23458 [6:40:29<04:13, 1.15it/s][2025-04-24 15:41:39] (step=0046625) Train Loss: 5.3799, Train Steps/Sec: 1.17 + 99%|████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████▌ | 23191/23458 [6:40:52<04:38, 1.04s/it][2025-04-24 15:42:02] (step=0046650) Train Loss: 5.4018, Train Steps/Sec: 1.08 + 99%|████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████▋ | 23216/23458 [6:41:13<03:25, 1.18it/s][2025-04-24 15:42:24] (step=0046675) Train Loss: 5.3859, Train Steps/Sec: 1.17 + 99%|████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████▊ | 23241/23458 [6:41:36<03:03, 1.18it/s][2025-04-24 15:42:46] (step=0046700) Train Loss: 5.3203, Train Steps/Sec: 1.11 + 99%|████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████▉ | 23266/23458 [6:41:58<02:48, 1.14it/s][2025-04-24 15:43:08] (step=0046725) Train Loss: 5.2928, Train Steps/Sec: 1.13 + 99%|█████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████ | 23291/23458 [6:42:20<02:22, 1.17it/s][2025-04-24 15:43:30] (step=0046750) Train Loss: 5.3758, Train Steps/Sec: 1.12 + 99%|█████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████▏| 23316/23458 [6:42:43<02:01, 1.17it/s][2025-04-24 15:43:53] (step=0046775) Train Loss: 5.4073, Train Steps/Sec: 1.12 +100%|█████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████▎| 23341/23458 [6:43:04<01:38, 1.18it/s][2025-04-24 15:44:14] (step=0046800) Train Loss: 5.2407, Train Steps/Sec: 1.16 +100%|█████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████▍| 23366/23458 [6:43:25<01:19, 1.15it/s][2025-04-24 15:44:36] (step=0046825) Train Loss: 5.3809, Train Steps/Sec: 1.17 +100%|█████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████▋| 23391/23458 [6:43:48<00:57, 1.17it/s][2025-04-24 15:44:58] (step=0046850) Train Loss: 5.3736, Train Steps/Sec: 1.12 +100%|█████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████▊| 23416/23458 [6:44:09<00:36, 1.17it/s][2025-04-24 15:45:19] (step=0046875) Train Loss: 5.3742, Train Steps/Sec: 1.17 +100%|█████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████▉| 23441/23458 [6:44:30<00:14, 1.18it/s][2025-04-24 15:45:41] (step=0046900) Train Loss: 5.4445, Train Steps/Sec: 1.17 +100%|██████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████| 23458/23458 [6:44:46<00:00, 1.04s/it] +[2025-04-24 15:45:55] Done! diff --git a/CKPTS/Afteri2i_100kckpt_LLAVAT5_empty_Fixed_just_segment_pretrain_stage3_dreambench_recap_400k_t2i_400k_flux_200k_midjourney_150k_recovery_150k_3e4_epoch2_no_rotated/wandb/run-20250424_021033-arc66v3w/files/requirements.txt b/CKPTS/Afteri2i_100kckpt_LLAVAT5_empty_Fixed_just_segment_pretrain_stage3_dreambench_recap_400k_t2i_400k_flux_200k_midjourney_150k_recovery_150k_3e4_epoch2_no_rotated/wandb/run-20250424_021033-arc66v3w/files/requirements.txt new file mode 100644 index 0000000000000000000000000000000000000000..06dc78369ffff807b210006a0e79d705ffe2a7d7 --- /dev/null +++ b/CKPTS/Afteri2i_100kckpt_LLAVAT5_empty_Fixed_just_segment_pretrain_stage3_dreambench_recap_400k_t2i_400k_flux_200k_midjourney_150k_recovery_150k_3e4_epoch2_no_rotated/wandb/run-20250424_021033-arc66v3w/files/requirements.txt @@ -0,0 +1,131 @@ +typing_extensions==4.12.2 +pyzmq==26.3.0 +nvidia-cufft-cu12==11.0.2.54 +triton==3.1.0 +nvidia-cublas-cu12==12.1.3.1 +psutil==7.0.0 +nvidia-cuda-cupti-cu12==12.1.105 +smmap==5.0.2 +nvidia-cuda-runtime-cu12==12.1.105 +aiohappyeyeballs==2.6.1 +asttokens==3.0.0 +huggingface-hub==0.29.3 +pyarrow==19.0.1 +fonttools==4.56.0 +python-dateutil==2.9.0.post0 +GitPython==3.1.44 +aiohttp==3.11.14 +wandb==0.19.8 +setproctitle==1.3.5 +PyYAML==6.0.2 +pydantic_core==2.27.2 +safetensors==0.5.3 +nvidia-nvjitlink-cu12==12.1.105 +aiosignal==1.3.2 +dill==0.3.8 +nvidia-cuda-nvrtc-cu12==12.1.105 +multiprocess==0.70.16 +pure_eval==0.2.3 +stack_data==0.6.3 +pydantic==2.10.6 +MarkupSafe==2.1.5 +tornado==6.4.2 +executing==2.1.0 +executing==2.2.0 +opencv-python==4.11.0.86 +nvitop==1.4.2 +multidict==6.2.0 +Jinja2==3.1.4 +torch==2.5.1+cu121 +nvidia-curand-cu12==10.3.2.106 +platformdirs==4.3.6 +six==1.17.0 +mpmath==1.3.0 +zipp==3.21.0 +packaging==24.2 +requests==2.32.3 +certifi==2025.1.31 +docker-pycreds==0.4.0 +torchvision==0.20.1+cu121 +pandas==2.2.3 +networkx==3.3 +exceptiongroup==1.2.2 +pickleshare==0.7.5 +tokenizers==0.21.1 +charset-normalizer==3.4.1 +jupyter_core==5.7.2 +wcwidth==0.2.13 +nvidia-nvtx-cu12==12.1.105 +prompt_toolkit==3.0.50 +fsspec==2024.12.0 +pillow==11.1.0 +propcache==0.3.0 +regex==2024.11.6 +ptyprocess==0.7.0 +contourpy==1.3.1 +importlib_metadata==8.6.1 +idna==3.10 +comm==0.2.2 +protobuf==5.29.3 +yarl==1.18.3 +ipython_pygments_lexers==1.1.1 +pip==25.0 +parso==0.8.4 +joblib==1.4.2 +nvidia-nccl-cu12==2.21.5 +hf_transfer==0.1.9 +Pygments==2.19.1 +decorator==5.2.1 +filelock==3.18.0 +nvidia-cusparse-cu12==12.1.0.106 +debugpy==1.8.13 +urllib3==2.3.0 +traitlets==5.14.3 +tzdata==2025.1 +matplotlib-inline==0.1.7 +matplotlib==3.10.1 +kiwisolver==1.4.8 +nest_asyncio==1.6.0 +frozenlist==1.5.0 +nvidia-ml-py==12.570.86 +transformers==4.49.0 +nltk==3.9.1 +ipykernel==6.29.5 +click==8.1.8 +gitdb==4.0.12 +pyparsing==3.2.1 +attrs==25.3.0 +jedi==0.19.2 +ipython==9.0.2 +nvidia-cudnn-cu12==9.1.0.70 +pexpect==4.9.0 +nvidia-cusolver-cu12==11.4.5.107 +numpy==2.2.4 +tqdm==4.67.1 +pytz==2025.1 +wheel==0.45.1 +sentry-sdk==2.23.1 +torchaudio==2.5.1+cu121 +jupyter_client==8.6.3 +cycler==0.12.1 +annotated-types==0.7.0 +sympy==1.13.1 +xxhash==3.5.0 +datasets==3.4.1 +setuptools==75.8.0 +typing_extensions==4.12.2 +wheel==0.43.0 +importlib_metadata==8.0.0 +backports.tarfile==1.2.0 +autocommand==2.2.2 +packaging==24.2 +tomli==2.0.1 +typeguard==4.3.0 +zipp==3.19.2 +jaraco.context==5.3.0 +jaraco.functools==4.0.1 +more-itertools==10.3.0 +platformdirs==4.2.2 +jaraco.text==3.12.1 +jaraco.collections==5.1.0 +inflect==7.3.1 diff --git a/CKPTS/Afteri2i_100kckpt_LLAVAT5_empty_Fixed_just_segment_pretrain_stage3_dreambench_recap_400k_t2i_400k_flux_200k_midjourney_150k_recovery_150k_3e4_epoch2_no_rotated/wandb/run-20250424_021033-arc66v3w/files/wandb-metadata.json b/CKPTS/Afteri2i_100kckpt_LLAVAT5_empty_Fixed_just_segment_pretrain_stage3_dreambench_recap_400k_t2i_400k_flux_200k_midjourney_150k_recovery_150k_3e4_epoch2_no_rotated/wandb/run-20250424_021033-arc66v3w/files/wandb-metadata.json new file mode 100644 index 0000000000000000000000000000000000000000..fa20e6c7fa1a459621540169cb1a8995d99f29ad --- /dev/null +++ b/CKPTS/Afteri2i_100kckpt_LLAVAT5_empty_Fixed_just_segment_pretrain_stage3_dreambench_recap_400k_t2i_400k_flux_200k_midjourney_150k_recovery_150k_3e4_epoch2_no_rotated/wandb/run-20250424_021033-arc66v3w/files/wandb-metadata.json @@ -0,0 +1,149 @@ +{ + "os": "Linux-5.15.0-1064-azure-x86_64-with-glibc2.31", + "python": "CPython 3.11.11", + "startedAt": "2025-04-24T02:10:33.862692Z", + "args": [ + "--vq-ckpt", + "/tmp/haozhezhao/model/llamagen_t2i/vq_ds16_t2i.pt", + "--data-path", + "/tmp/haozhezhao/MLLMG/jsonl_data/merged_train_set_set_subject_400k_recap_t2i_400k_flux_200k_midjourney_150k_recovery_150k_grounding_100fluxseg_50samseg.jsonl", + "--dataset", + "ti2i", + "--image-size", + "512", + "--results-dir", + "checkpoint/Afteri2i_100kckpt_LLAVAT5_empty_Fixed_just_segment_pretrain_stage3_dreambench_recap_400k_t2i_400k_flux_200k_midjourney_150k_recovery_150k_3e4_epoch2_no_rotated", + "--cloud-save-path", + "/tmp/haozhezhao/checkpoint", + "--lr", + "3e-4", + "--val_data_path", + "/tmp/haozhezhao/MLLMG/jsonl_data/dreambench_plus_valid.jsonl", + "--use_vision_tower", + "--model_name_or_path", + "/tmp/haozhezhao/model/blip2-flan-t5-xl", + "--image_place_holder", + "", + "--do_eval", + "--eval_steps", + "2000", + "--max_eval_samples", + "250", + "--cfg-scale", + "7.5", + "--top-k", + "16384", + "--load_from_checkpoint", + "/tmp/haozhezhao/MLLMG/checkpoint/FIXed_3M_1epoch_step32000.pt", + "--global-batch-size", + "56", + "--num-workers", + "8", + "--warmup", + "0.05", + "--gradient-accumulation-steps", + "4", + "--train_text_encoder", + "--ckpt-every", + "2000", + "--epochs", + "2", + "--subject_driven", + "--reference_data_path", + "/tmp/haozhezhao/MLLMG/cc12m_reference_tunnel.jsonl", + "--multimodal_encoder", + "llava", + "--do_recovery", + "--find_unused_parameters", + "--cls-token-num", + "512", + "--dreambench_eval", + "--save_total_limit", + "1", + "--load_language_projection", + "/tmp/haozhezhao/MLLMG/llava-v1.5-flant5_fixed-pretrain/mm_projector.bin", + "--gpt-ckpt", + "/tmp/haozhezhao/MLLMG/MLLMG_ckpts/checkpoints/AfterI2I_llavat5_subject_train_all_T5_subject_t2i_ti2i_120_w_flux_segment_noimageonly_empty_fix_train_all/0100000.pt", + "--mm_vision_tower", + "openai/clip-vit-large-patch14", + "--train_all", + "--load_fixed_llamagen", + "--fix", + "gpt-empty-fix" + ], + "program": "/tmp/haozhezhao/MLLMG/autoregressive/train/train_t2i.py", + "codePath": "autoregressive/train/train_t2i.py", + "email": "mimazhe55360@gmail.com", + "root": "checkpoint/Afteri2i_100kckpt_LLAVAT5_empty_Fixed_just_segment_pretrain_stage3_dreambench_recap_400k_t2i_400k_flux_200k_midjourney_150k_recovery_150k_3e4_epoch2_no_rotated", + "host": "447cc403a8794092814259713c51c1df00001X", + "executable": "/tmp/haozhezhao/anaconda3/envs/nlp/bin/python", + "codePathLocal": "autoregressive/train/train_t2i.py", + "cpu_count": 96, + "cpu_count_logical": 96, + "gpu": "NVIDIA A100-SXM4-80GB", + "gpu_count": 8, + "disk": { + "/": { + "total": "133003395072", + "used": "65600028672" + } + }, + "memory": { + "total": "1902387884032" + }, + "cpu": { + "count": 96, + "countLogical": 96 + }, + "gpu_nvidia": [ + { + "name": "NVIDIA A100-SXM4-80GB", + "memoryTotal": "85899345920", + "cudaCores": 6912, + "architecture": "Ampere" + }, + { + "name": "NVIDIA A100-SXM4-80GB", + "memoryTotal": "85899345920", + "cudaCores": 6912, + "architecture": "Ampere" + }, + { + "name": "NVIDIA A100-SXM4-80GB", + "memoryTotal": "85899345920", + "cudaCores": 6912, + "architecture": "Ampere" + }, + { + "name": "NVIDIA A100-SXM4-80GB", + "memoryTotal": "85899345920", + "cudaCores": 6912, + "architecture": "Ampere" + }, + { + "name": "NVIDIA A100-SXM4-80GB", + "memoryTotal": "85899345920", + "cudaCores": 6912, + "architecture": "Ampere" + }, + { + "name": "NVIDIA A100-SXM4-80GB", + "memoryTotal": "85899345920", + "cudaCores": 6912, + "architecture": "Ampere" + }, + { + "name": "NVIDIA A100-SXM4-80GB", + "memoryTotal": "85899345920", + "cudaCores": 6912, + "architecture": "Ampere" + }, + { + "name": "NVIDIA A100-SXM4-80GB", + "memoryTotal": "85899345920", + "cudaCores": 6912, + "architecture": "Ampere" + } + ], + "cudaVersion": "12.2" +} \ No newline at end of file diff --git a/CKPTS/Afteri2i_100kckpt_LLAVAT5_empty_Fixed_just_segment_pretrain_stage3_dreambench_recap_400k_t2i_400k_flux_200k_midjourney_150k_recovery_150k_3e4_epoch2_no_rotated/wandb/run-20250424_021033-arc66v3w/files/wandb-summary.json b/CKPTS/Afteri2i_100kckpt_LLAVAT5_empty_Fixed_just_segment_pretrain_stage3_dreambench_recap_400k_t2i_400k_flux_200k_midjourney_150k_recovery_150k_3e4_epoch2_no_rotated/wandb/run-20250424_021033-arc66v3w/files/wandb-summary.json new file mode 100644 index 0000000000000000000000000000000000000000..ecf30161a8ccfca5f79ab9f22205a26f725b01e1 --- /dev/null +++ b/CKPTS/Afteri2i_100kckpt_LLAVAT5_empty_Fixed_just_segment_pretrain_stage3_dreambench_recap_400k_t2i_400k_flux_200k_midjourney_150k_recovery_150k_3e4_epoch2_no_rotated/wandb/run-20250424_021033-arc66v3w/files/wandb-summary.json @@ -0,0 +1 @@ +{"_runtime":48924.726811174,"_step":46900,"_wandb":{"runtime":48924},"train lr":0.000274238544355085,"train loss":5.444525241851807,"Train Steps/Sec":1.1670410665953292,"_timestamp":1.7455095411112263e+09} \ No newline at end of file diff --git a/CKPTS/Afteri2i_100kckpt_LLAVAT5_empty_Fixed_just_segment_pretrain_stage3_dreambench_recap_400k_t2i_400k_flux_200k_midjourney_150k_recovery_150k_3e4_epoch2_no_rotated/wandb/run-20250424_021033-arc66v3w/logs/debug-core.log b/CKPTS/Afteri2i_100kckpt_LLAVAT5_empty_Fixed_just_segment_pretrain_stage3_dreambench_recap_400k_t2i_400k_flux_200k_midjourney_150k_recovery_150k_3e4_epoch2_no_rotated/wandb/run-20250424_021033-arc66v3w/logs/debug-core.log new file mode 100644 index 0000000000000000000000000000000000000000..55ba2ef34fe64d827fd3648e524b5c2c56b1bc57 --- /dev/null +++ b/CKPTS/Afteri2i_100kckpt_LLAVAT5_empty_Fixed_just_segment_pretrain_stage3_dreambench_recap_400k_t2i_400k_flux_200k_midjourney_150k_recovery_150k_3e4_epoch2_no_rotated/wandb/run-20250424_021033-arc66v3w/logs/debug-core.log @@ -0,0 +1,13 @@ +{"time":"2025-04-24T02:10:33.229615861Z","level":"INFO","msg":"main: starting server","port-filename":"/tmp/tmp2lk_ymeq/port-3231966.txt","pid":3231966,"log-level":0,"disable-analytics":false,"shutdown-on-parent-exit":false} +{"time":"2025-04-24T02:10:33.230942269Z","level":"INFO","msg":"Will exit if parent process dies.","ppid":3231966} +{"time":"2025-04-24T02:10:33.23092151Z","level":"INFO","msg":"server is running","addr":{"IP":"127.0.0.1","Port":44945,"Zone":""}} +{"time":"2025-04-24T02:10:33.417358394Z","level":"INFO","msg":"connection: ManageConnectionData: new connection created","id":"127.0.0.1:35156"} +{"time":"2025-04-24T02:10:33.864578955Z","level":"INFO","msg":"handleInformInit: received","streamId":"arc66v3w","id":"127.0.0.1:35156"} +{"time":"2025-04-24T02:10:34.071410412Z","level":"INFO","msg":"handleInformInit: stream started","streamId":"arc66v3w","id":"127.0.0.1:35156"} +{"time":"2025-04-24T15:45:58.589377198Z","level":"INFO","msg":"handleInformTeardown: server teardown initiated","id":"127.0.0.1:35156"} +{"time":"2025-04-24T15:45:58.589449839Z","level":"INFO","msg":"connection: closing","id":"127.0.0.1:35156"} +{"time":"2025-04-24T15:45:58.589476574Z","level":"INFO","msg":"server is shutting down"} +{"time":"2025-04-24T15:45:58.589669439Z","level":"INFO","msg":"connection: closed successfully","id":"127.0.0.1:35156"} +{"time":"2025-04-24T15:45:59.931942666Z","level":"INFO","msg":"handleInformTeardown: server shutdown complete","id":"127.0.0.1:35156"} +{"time":"2025-04-24T15:45:59.931968524Z","level":"INFO","msg":"connection: ManageConnectionData: connection closed","id":"127.0.0.1:35156"} +{"time":"2025-04-24T15:45:59.931983001Z","level":"INFO","msg":"server is closed"} diff --git a/CKPTS/Afteri2i_100kckpt_LLAVAT5_empty_Fixed_just_segment_pretrain_stage3_dreambench_recap_400k_t2i_400k_flux_200k_midjourney_150k_recovery_150k_3e4_epoch2_no_rotated/wandb/run-20250424_021033-arc66v3w/logs/debug-internal.log b/CKPTS/Afteri2i_100kckpt_LLAVAT5_empty_Fixed_just_segment_pretrain_stage3_dreambench_recap_400k_t2i_400k_flux_200k_midjourney_150k_recovery_150k_3e4_epoch2_no_rotated/wandb/run-20250424_021033-arc66v3w/logs/debug-internal.log new file mode 100644 index 0000000000000000000000000000000000000000..ab0c2fc13629f9dabb2d6f7e7c10f734e401d0fd --- /dev/null +++ b/CKPTS/Afteri2i_100kckpt_LLAVAT5_empty_Fixed_just_segment_pretrain_stage3_dreambench_recap_400k_t2i_400k_flux_200k_midjourney_150k_recovery_150k_3e4_epoch2_no_rotated/wandb/run-20250424_021033-arc66v3w/logs/debug-internal.log @@ -0,0 +1,16 @@ +{"time":"2025-04-24T02:10:33.864856905Z","level":"INFO","msg":"stream: starting","core version":"0.19.8","symlink path":"checkpoint/Afteri2i_100kckpt_LLAVAT5_empty_Fixed_just_segment_pretrain_stage3_dreambench_recap_400k_t2i_400k_flux_200k_midjourney_150k_recovery_150k_3e4_epoch2_no_rotated/wandb/run-20250424_021033-arc66v3w/logs/debug-core.log"} +{"time":"2025-04-24T02:10:34.07137687Z","level":"INFO","msg":"created new stream","id":"arc66v3w"} +{"time":"2025-04-24T02:10:34.071404972Z","level":"INFO","msg":"stream: started","id":"arc66v3w"} +{"time":"2025-04-24T02:10:34.071450393Z","level":"INFO","msg":"writer: Do: started","stream_id":"arc66v3w"} +{"time":"2025-04-24T02:10:34.071476115Z","level":"INFO","msg":"handler: started","stream_id":"arc66v3w"} +{"time":"2025-04-24T02:10:34.071466883Z","level":"INFO","msg":"sender: started","stream_id":"arc66v3w"} +{"time":"2025-04-24T02:10:35.131795783Z","level":"INFO","msg":"Starting system monitor"} +{"time":"2025-04-24T15:13:50.841496183Z","level":"INFO","msg":"api: retrying HTTP error","status":500,"url":"https://api.wandb.ai/files/haozhezhao/llamagen_ti2i/arc66v3w/file_stream","body":"{\"error\":\"rpc error: code = Unauthenticated desc = Request had invalid euc header.\"}"} +{"time":"2025-04-24T15:45:58.589490921Z","level":"INFO","msg":"stream: closing","id":"arc66v3w"} +{"time":"2025-04-24T15:45:58.589517671Z","level":"INFO","msg":"Stopping system monitor"} +{"time":"2025-04-24T15:45:58.590418531Z","level":"INFO","msg":"Stopped system monitor"} +{"time":"2025-04-24T15:45:59.725910519Z","level":"INFO","msg":"fileTransfer: Close: file transfer manager closed"} +{"time":"2025-04-24T15:45:59.931748724Z","level":"INFO","msg":"handler: closed","stream_id":"arc66v3w"} +{"time":"2025-04-24T15:45:59.931771186Z","level":"INFO","msg":"writer: Close: closed","stream_id":"arc66v3w"} +{"time":"2025-04-24T15:45:59.931824896Z","level":"INFO","msg":"sender: closed","stream_id":"arc66v3w"} +{"time":"2025-04-24T15:45:59.931844603Z","level":"INFO","msg":"stream: closed","id":"arc66v3w"} diff --git a/CKPTS/Afteri2i_100kckpt_LLAVAT5_empty_Fixed_just_segment_pretrain_stage3_dreambench_recap_400k_t2i_400k_flux_200k_midjourney_150k_recovery_150k_3e4_epoch2_no_rotated/wandb/run-20250424_021033-arc66v3w/logs/debug.log b/CKPTS/Afteri2i_100kckpt_LLAVAT5_empty_Fixed_just_segment_pretrain_stage3_dreambench_recap_400k_t2i_400k_flux_200k_midjourney_150k_recovery_150k_3e4_epoch2_no_rotated/wandb/run-20250424_021033-arc66v3w/logs/debug.log new file mode 100644 index 0000000000000000000000000000000000000000..c719b7575f62ef416b3b7e3e619f04178274772c --- /dev/null +++ b/CKPTS/Afteri2i_100kckpt_LLAVAT5_empty_Fixed_just_segment_pretrain_stage3_dreambench_recap_400k_t2i_400k_flux_200k_midjourney_150k_recovery_150k_3e4_epoch2_no_rotated/wandb/run-20250424_021033-arc66v3w/logs/debug.log @@ -0,0 +1,23 @@ +2025-04-24 02:10:33,857 INFO MainThread:3231966 [wandb_setup.py:_flush():67] Current SDK version is 0.19.8 +2025-04-24 02:10:33,857 INFO MainThread:3231966 [wandb_setup.py:_flush():67] Configure stats pid to 3231966 +2025-04-24 02:10:33,857 INFO MainThread:3231966 [wandb_setup.py:_flush():67] Loading settings from /tmp/haozhezhao/.config/wandb/settings +2025-04-24 02:10:33,857 INFO MainThread:3231966 [wandb_setup.py:_flush():67] Loading settings from /tmp/haozhezhao/MLLMG/wandb/settings +2025-04-24 02:10:33,857 INFO MainThread:3231966 [wandb_setup.py:_flush():67] Loading settings from environment variables +2025-04-24 02:10:33,857 INFO MainThread:3231966 [wandb_init.py:setup_run_log_directory():647] Logging user logs to checkpoint/Afteri2i_100kckpt_LLAVAT5_empty_Fixed_just_segment_pretrain_stage3_dreambench_recap_400k_t2i_400k_flux_200k_midjourney_150k_recovery_150k_3e4_epoch2_no_rotated/wandb/run-20250424_021033-arc66v3w/logs/debug.log +2025-04-24 02:10:33,857 INFO MainThread:3231966 [wandb_init.py:setup_run_log_directory():648] Logging internal logs to checkpoint/Afteri2i_100kckpt_LLAVAT5_empty_Fixed_just_segment_pretrain_stage3_dreambench_recap_400k_t2i_400k_flux_200k_midjourney_150k_recovery_150k_3e4_epoch2_no_rotated/wandb/run-20250424_021033-arc66v3w/logs/debug-internal.log +2025-04-24 02:10:33,857 INFO MainThread:3231966 [wandb_init.py:init():761] calling init triggers +2025-04-24 02:10:33,857 INFO MainThread:3231966 [wandb_init.py:init():766] wandb.init called with sweep_config: {} +config: {'data_path': '/tmp/haozhezhao/MLLMG/jsonl_data/merged_train_set_set_subject_400k_recap_t2i_400k_flux_200k_midjourney_150k_recovery_150k_grounding_100fluxseg_50samseg.jsonl', 'cloud_save_path': '/tmp/haozhezhao/checkpoint', 'no_local_save': False, 'vq_model': 'VQ-16', 'vq_ckpt': '/tmp/haozhezhao/model/llamagen_t2i/vq_ds16_t2i.pt', 'codebook_size': 16384, 'codebook_embed_dim': 8, 'gpt_model': 'GPT-XL', 'gpt_ckpt': '/tmp/haozhezhao/MLLMG/MLLMG_ckpts/checkpoints/AfterI2I_llavat5_subject_train_all_T5_subject_t2i_ti2i_120_w_flux_segment_noimageonly_empty_fix_train_all/0100000.pt', 'gpt_type': 't2i', 'vocab_size': 16384, 'cls_token_num': 512, 'dropout_p': 0.1, 'token_dropout_p': 0.1, 'drop_path': 0.0, 'no_compile': False, 'results_dir': 'checkpoint/Afteri2i_100kckpt_LLAVAT5_empty_Fixed_just_segment_pretrain_stage3_dreambench_recap_400k_t2i_400k_flux_200k_midjourney_150k_recovery_150k_3e4_epoch2_no_rotated', 'dataset': 'ti2i', 'image_size': 512, 'downsample_size': 16, 'num_classes': 1000, 'epochs': 2, 'lr': 0.0003, 'weight_decay': 0.05, 'beta1': 0.9, 'beta2': 0.95, 'max_grad_norm': 1.0, 'global_batch_size': 56, 'global_seed': 0, 'num_workers': 8, 'log_every': 25, 'ckpt_every': 2000, 'gradient_accumulation_steps': 4, 'mixed_precision': 'bf16', 'val_data_path': '/tmp/haozhezhao/MLLMG/jsonl_data/dreambench_plus_valid.jsonl', 'use_vision_tower': True, 'model_name_or_path': '/tmp/haozhezhao/model/blip2-flan-t5-xl', 'image_place_holder': '', 'processor_path': None, 'do_eval': True, 'max_eval_samples': 250, 'train_text_encoder': True, 'no_left_padding': False, 'cfg_scale': 7.5, 'top_k': 16384, 'temperature': 0.9, 'top_p': 1.0, 'eval_steps': 2000, 'project_name': 'llamagen_ti2i', 'load_from_checkpoint': '/tmp/haozhezhao/MLLMG/checkpoint/FIXed_3M_1epoch_step32000.pt', 'warmup': 0.05, 'lr_decay_style': 'cosine', 'lr_decay_ratio': 0.1, 'train_iters': 500000, 'class_dropout_prob': 0.1, 'with_image_only': False, 'image_only_rate': 0.1, 'stage2': False, 'subject_driven': True, 'load_subject_embedding': None, 'reference_data_path': '/tmp/haozhezhao/MLLMG/cc12m_reference_tunnel.jsonl', 'multimodal_encoder': 'llava', 'do_recovery': True, 'no_replace': False, 'resume': False, 'dreambench_eval': True, 'find_unused_parameters': True, 'load_visual_encoder': False, 'continue_stage1': False, 'replace_subject': False, 'train_all': True, 'save_total_limit': 1, 'load_language_projection': '/tmp/haozhezhao/MLLMG/llava-v1.5-flant5_fixed-pretrain/mm_projector.bin', 'mm_vision_tower': 'openai/clip-vit-large-patch14', 'load_fixed_llamagen': True, 'unfreeze_output': False, 'fix': 'gpt-empty-fix', 'rank': 0, 'world_size': 8, 'gpu': 0, 'dist_url': 'env://', 'distributed': True, 'dist_backend': 'nccl', '_wandb': {}} +2025-04-24 02:10:33,857 INFO MainThread:3231966 [wandb_init.py:init():784] starting backend +2025-04-24 02:10:33,857 INFO MainThread:3231966 [wandb_init.py:init():788] sending inform_init request +2025-04-24 02:10:33,862 INFO MainThread:3231966 [backend.py:_multiprocessing_setup():101] multiprocessing start_methods=fork,spawn,forkserver, using: spawn +2025-04-24 02:10:33,862 INFO MainThread:3231966 [wandb_init.py:init():798] backend started and connected +2025-04-24 02:10:33,864 INFO MainThread:3231966 [wandb_init.py:init():891] updated telemetry +2025-04-24 02:10:33,864 INFO MainThread:3231966 [wandb_init.py:init():915] communicating run to backend with 90.0 second timeout +2025-04-24 02:10:35,084 INFO MainThread:3231966 [wandb_init.py:init():990] starting run threads in backend +2025-04-24 02:10:35,213 INFO MainThread:3231966 [wandb_run.py:_console_start():2375] atexit reg +2025-04-24 02:10:35,214 INFO MainThread:3231966 [wandb_run.py:_redirect():2227] redirect: wrap_raw +2025-04-24 02:10:35,214 INFO MainThread:3231966 [wandb_run.py:_redirect():2292] Wrapping output streams. +2025-04-24 02:10:35,214 INFO MainThread:3231966 [wandb_run.py:_redirect():2315] Redirects installed. +2025-04-24 02:10:35,216 INFO MainThread:3231966 [wandb_init.py:init():1032] run started, returning control to user process +2025-04-24 15:45:58,588 INFO MsgRouterThr:3231966 [mailbox.py:close():129] Closing mailbox, abandoning 1 handles. diff --git a/CKPTS/Afteri2i_100kckpt_LLAVAT5_empty_Fixed_just_segment_pretrain_stage3_dreambench_recap_400k_t2i_400k_flux_200k_midjourney_150k_recovery_150k_3e4_epoch2_no_rotated/wandb/run-20250424_021033-arc66v3w/run-arc66v3w.wandb b/CKPTS/Afteri2i_100kckpt_LLAVAT5_empty_Fixed_just_segment_pretrain_stage3_dreambench_recap_400k_t2i_400k_flux_200k_midjourney_150k_recovery_150k_3e4_epoch2_no_rotated/wandb/run-20250424_021033-arc66v3w/run-arc66v3w.wandb new file mode 100644 index 0000000000000000000000000000000000000000..25dbdae641162635cf54154f12541eaed20acd42 --- /dev/null +++ b/CKPTS/Afteri2i_100kckpt_LLAVAT5_empty_Fixed_just_segment_pretrain_stage3_dreambench_recap_400k_t2i_400k_flux_200k_midjourney_150k_recovery_150k_3e4_epoch2_no_rotated/wandb/run-20250424_021033-arc66v3w/run-arc66v3w.wandb @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:d9ea93e426f13f8a0089f504abc938bdeca05e83e9abdd1337047990d182f368 +size 29913654 diff --git a/CKPTS/Afteri2i_100kckpt_LLAVAT5_empty_Fixed_just_segment_pretrain_stage3_dreambench_recap_400k_t2i_400k_flux_200k_midjourney_150k_recovery_150k_3e4_epoch2_no_rotated/wandb/run-20250425_060734-amr5cnvk/files/config.yaml b/CKPTS/Afteri2i_100kckpt_LLAVAT5_empty_Fixed_just_segment_pretrain_stage3_dreambench_recap_400k_t2i_400k_flux_200k_midjourney_150k_recovery_150k_3e4_epoch2_no_rotated/wandb/run-20250425_060734-amr5cnvk/files/config.yaml new file mode 100644 index 0000000000000000000000000000000000000000..95440ba3f9739954e4927ea80961008d8a75e6ed --- /dev/null +++ b/CKPTS/Afteri2i_100kckpt_LLAVAT5_empty_Fixed_just_segment_pretrain_stage3_dreambench_recap_400k_t2i_400k_flux_200k_midjourney_150k_recovery_150k_3e4_epoch2_no_rotated/wandb/run-20250425_060734-amr5cnvk/files/config.yaml @@ -0,0 +1,199 @@ +_wandb: + value: + cli_version: 0.19.8 + m: [] + python_version: 3.11.11 + t: + "1": + - 1 + - 11 + - 41 + - 49 + - 51 + - 55 + "2": + - 1 + - 11 + - 41 + - 49 + - 51 + - 55 + "3": + - 13 + - 16 + - 23 + - 55 + - 61 + "4": 3.11.11 + "5": 0.19.8 + "6": 4.49.0 + "8": + - 5 + "12": 0.19.8 + "13": linux-x86_64 +beta1: + value: 0.9 +beta2: + value: 0.95 +cfg_scale: + value: 7.5 +ckpt_every: + value: 2000 +class_dropout_prob: + value: 0.1 +cloud_save_path: + value: /tmp/haozhezhao/checkpoint +cls_token_num: + value: 512 +codebook_embed_dim: + value: 8 +codebook_size: + value: 16384 +continue_stage1: + value: false +data_path: + value: /tmp/haozhezhao/MLLMG/jsonl_data/merged_train_set_set_subject_400k_recap_t2i_400k_flux_200k_midjourney_150k_recovery_150k_grounding_100fluxseg_50samseg.jsonl +dataset: + value: ti2i +dist_backend: + value: nccl +dist_url: + value: env:// +distributed: + value: true +do_eval: + value: true +do_recovery: + value: true +downsample_size: + value: 16 +dreambench_eval: + value: true +drop_path: + value: 0 +dropout_p: + value: 0.1 +epochs: + value: 3 +eval_steps: + value: 2000 +find_unused_parameters: + value: true +fix: + value: gpt-empty-fix +global_batch_size: + value: 56 +global_seed: + value: 0 +gpt_ckpt: + value: /tmp/haozhezhao/MLLMG/checkpoint/Afteri2i_100kckpt_LLAVAT5_empty_Fixed_just_segment_pretrain_stage3_dreambench_recap_400k_t2i_400k_flux_200k_midjourney_150k_recovery_150k_3e4_epoch2_no_rotated/000-GPT-XL/checkpoints/0046000.pt +gpt_model: + value: GPT-XL +gpt_type: + value: t2i +gpu: + value: 0 +gradient_accumulation_steps: + value: 4 +image_only_rate: + value: 0.1 +image_place_holder: + value: +image_size: + value: 512 +load_fixed_llamagen: + value: true +load_from_checkpoint: + value: /tmp/haozhezhao/MLLMG/checkpoint/FIXed_3M_1epoch_step32000.pt +load_language_projection: + value: /tmp/haozhezhao/MLLMG/llava-v1.5-flant5_fixed-pretrain/mm_projector.bin +load_subject_embedding: + value: null +load_visual_encoder: + value: false +log_every: + value: 25 +lr: + value: 0.0003 +lr_decay_ratio: + value: 0.1 +lr_decay_style: + value: cosine +max_eval_samples: + value: 250 +max_grad_norm: + value: 1 +mixed_precision: + value: bf16 +mm_vision_tower: + value: openai/clip-vit-large-patch14 +model_name_or_path: + value: /tmp/haozhezhao/model/blip2-flan-t5-xl +multimodal_encoder: + value: llava +no_compile: + value: false +no_left_padding: + value: false +no_local_save: + value: false +no_replace: + value: false +num_classes: + value: 1000 +num_workers: + value: 8 +processor_path: + value: null +project_name: + value: llamagen_ti2i +rank: + value: 0 +reference_data_path: + value: /tmp/haozhezhao/MLLMG/cc12m_reference_tunnel.jsonl +replace_subject: + value: false +results_dir: + value: checkpoint/Afteri2i_100kckpt_LLAVAT5_empty_Fixed_just_segment_pretrain_stage3_dreambench_recap_400k_t2i_400k_flux_200k_midjourney_150k_recovery_150k_3e4_epoch2_no_rotated +resume: + value: true +save_total_limit: + value: 1 +stage2: + value: false +subject_driven: + value: true +temperature: + value: 0.9 +token_dropout_p: + value: 0.1 +top_k: + value: 16384 +top_p: + value: 1 +train_all: + value: true +train_iters: + value: 500000 +train_text_encoder: + value: true +unfreeze_output: + value: false +use_vision_tower: + value: true +val_data_path: + value: /tmp/haozhezhao/MLLMG/jsonl_data/dreambench_plus_valid.jsonl +vocab_size: + value: 16384 +vq_ckpt: + value: /tmp/haozhezhao/model/llamagen_t2i/vq_ds16_t2i.pt +vq_model: + value: VQ-16 +warmup: + value: 0.05 +weight_decay: + value: 0.05 +with_image_only: + value: false +world_size: + value: 8 diff --git a/CKPTS/Afteri2i_100kckpt_LLAVAT5_empty_Fixed_just_segment_pretrain_stage3_dreambench_recap_400k_t2i_400k_flux_200k_midjourney_150k_recovery_150k_3e4_epoch2_no_rotated/wandb/run-20250425_060734-amr5cnvk/files/output.log b/CKPTS/Afteri2i_100kckpt_LLAVAT5_empty_Fixed_just_segment_pretrain_stage3_dreambench_recap_400k_t2i_400k_flux_200k_midjourney_150k_recovery_150k_3e4_epoch2_no_rotated/wandb/run-20250425_060734-amr5cnvk/files/output.log new file mode 100644 index 0000000000000000000000000000000000000000..36197413f85150785661b98b8db86f1c30d1c4f0 --- /dev/null +++ b/CKPTS/Afteri2i_100kckpt_LLAVAT5_empty_Fixed_just_segment_pretrain_stage3_dreambench_recap_400k_t2i_400k_flux_200k_midjourney_150k_recovery_150k_3e4_epoch2_no_rotated/wandb/run-20250425_060734-amr5cnvk/files/output.log @@ -0,0 +1,1172 @@ +[2025-04-25 06:07:35] Training for 3 epochs... +[2025-04-25 06:07:35] Beginning epoch 1... + 96%|█████████████████████████████████████████████████████████████████████████████████████████████████████████████████▍ | 22539/23458 [22:01<01:04, 14.34it/s]/tmp/haozhezhao/MLLMG/autoregressive/train/train_t2i.py:652: FutureWarning: `torch.cuda.amp.autocast(args...)` is deprecated. Please use `torch.amp.autocast('cuda', args...)` instead. + with torch.cuda.amp.autocast(dtype=ptdtype): + 96%|█████████████████████████████████████████████████████████████████████████████████████████████████████████████████▍ | 22539/23458 [22:20<01:04, 14.34it/s][rank0]:W0425 06:30:18.090000 3330461 site-packages/torch/_dynamo/variables/tensor.py:776] [4/0] Graph break from `Tensor.item()`, consider setting: +[rank0]:W0425 06:30:18.090000 3330461 site-packages/torch/_dynamo/variables/tensor.py:776] [4/0] torch._dynamo.config.capture_scalar_outputs = True +[rank0]:W0425 06:30:18.090000 3330461 site-packages/torch/_dynamo/variables/tensor.py:776] [4/0] or: +[rank0]:W0425 06:30:18.090000 3330461 site-packages/torch/_dynamo/variables/tensor.py:776] [4/0] env TORCHDYNAMO_CAPTURE_SCALAR_OUTPUTS=1 +[rank0]:W0425 06:30:18.090000 3330461 site-packages/torch/_dynamo/variables/tensor.py:776] [4/0] to include these operations in the captured graph. +[rank0]:W0425 06:30:18.090000 3330461 site-packages/torch/_dynamo/variables/tensor.py:776] [4/0] +[rank0]:W0425 06:30:18.090000 3330461 site-packages/torch/_dynamo/variables/tensor.py:776] [4/0] Graph break: from user code at: +[rank0]:W0425 06:30:18.090000 3330461 site-packages/torch/_dynamo/variables/tensor.py:776] [4/0] File "/tmp/haozhezhao/MLLMG/autoregressive/models/empty_fix_gpt.py", line 483, in torch_dynamo_resume_in_forward_at_477 +[rank0]:W0425 06:30:18.090000 3330461 site-packages/torch/_dynamo/variables/tensor.py:776] [4/0] t5_feat_len = t5_attn.sum().item() +[rank0]:W0425 06:30:18.090000 3330461 site-packages/torch/_dynamo/variables/tensor.py:776] [4/0] +[rank0]:W0425 06:30:18.090000 3330461 site-packages/torch/_dynamo/variables/tensor.py:776] [4/0] + 96%|███████████████████████████████████████████████████████████████████████████████████████████████████████████████▍ | 22543/23458 [24:17<2:06:31, 8.30s/it]/tmp/haozhezhao/MLLMG/autoregressive/train/train_t2i.py:652: FutureWarning: `torch.cuda.amp.autocast(args...)` is deprecated. Please use `torch.amp.autocast('cuda', args...)` instead. + with torch.cuda.amp.autocast(dtype=ptdtype): + 96%|███████████████████████████████████████████████████████████████████████████████████████████████████████████████▍ | 22544/23458 [25:05<2:54:57, 11.49s/it]/tmp/haozhezhao/MLLMG/autoregressive/train/train_t2i.py:652: FutureWarning: `torch.cuda.amp.autocast(args...)` is deprecated. Please use `torch.amp.autocast('cuda', args...)` instead. + with torch.cuda.amp.autocast(dtype=ptdtype): + 96%|███████████████████████████████████████████████████████████████████████████████████████████████████████████████▍ | 22545/23458 [25:51<3:48:12, 15.00s/it]/tmp/haozhezhao/MLLMG/autoregressive/train/train_t2i.py:652: FutureWarning: `torch.cuda.amp.autocast(args...)` is deprecated. Please use `torch.amp.autocast('cuda', args...)` instead. + with torch.cuda.amp.autocast(dtype=ptdtype): + 96%|███████████████████████████████████████████████████████████████████████████████████████████████████████████████▌ | 22560/23458 [27:19<2:41:18, 10.78s/it]/tmp/haozhezhao/MLLMG/autoregressive/train/train_t2i.py:652: FutureWarning: `torch.cuda.amp.autocast(args...)` is deprecated. Please use `torch.amp.autocast('cuda', args...)` instead. + with torch.cuda.amp.autocast(dtype=ptdtype): + 96%|███████████████████████████████████████████████████████████████████████████████████████████████████████████████▌ | 22566/23458 [28:31<3:22:52, 13.65s/it][2025-04-25 06:36:07] (step=0046025) Train Loss: 5.2182, Train Steps/Sec: 0.01 + 96%|███████████████████████████████████████████████████████████████████████████████████████████████████████████████▋ | 22591/23458 [29:24<1:50:06, 7.62s/it][2025-04-25 06:37:01] (step=0046050) Train Loss: 5.3363, Train Steps/Sec: 0.47 + 96%|█████████████████████████████████████████████████████████████████████████████████████████████████████████████████▊ | 22616/23458 [30:44<43:59, 3.13s/it][2025-04-25 06:38:20] (step=0046075) Train Loss: 5.2915, Train Steps/Sec: 0.31 + 97%|█████████████████████████████████████████████████████████████████████████████████████████████████████████████████▉ | 22641/23458 [31:05<11:34, 1.18it/s][2025-04-25 06:38:42] (step=0046100) Train Loss: 5.3664, Train Steps/Sec: 1.16 + 97%|██████████████████████████████████████████████████████████████████████████████████████████████████████████████████ | 22666/23458 [31:27<11:26, 1.15it/s][2025-04-25 06:39:03] (step=0046125) Train Loss: 5.3841, Train Steps/Sec: 1.17 + 97%|████████████████████████████████████████████████████████████████████████████████████████████████████████████████▏ | 22691/23458 [33:11<1:51:08, 8.69s/it][2025-04-25 06:40:48] (step=0046150) Train Loss: 5.2847, Train Steps/Sec: 0.24 + 97%|██████████████████████████████████████████████████████████████████████████████████████████████████████████████████▎ | 22716/23458 [33:32<10:25, 1.19it/s][2025-04-25 06:41:09] (step=0046175) Train Loss: 5.3772, Train Steps/Sec: 1.17 + 97%|██████████████████████████████████████████████████████████████████████████████████████████████████████████████████▍ | 22741/23458 [33:54<10:02, 1.19it/s][2025-04-25 06:41:30] (step=0046200) Train Loss: 5.3538, Train Steps/Sec: 1.17 + 97%|██████████████████████████████████████████████████████████████████████████████████████████████████████████████████▌ | 22766/23458 [34:40<40:09, 3.48s/it][2025-04-25 06:42:17] (step=0046225) Train Loss: 5.3947, Train Steps/Sec: 0.54 + 97%|██████████████████████████████████████████████████████████████████████████████████████████████████████████████████▋ | 22791/23458 [35:03<09:52, 1.13it/s][2025-04-25 06:42:39] (step=0046250) Train Loss: 5.2745, Train Steps/Sec: 1.12 + 97%|██████████████████████████████████████████████████████████████████████████████████████████████████████████████████▊ | 22816/23458 [35:25<09:03, 1.18it/s][2025-04-25 06:43:02] (step=0046275) Train Loss: 5.3726, Train Steps/Sec: 1.11 + 97%|████████████████████████████████████████████████████████████████████████████████████████████████████████████████▊ | 22822/23458 [35:55<1:28:21, 8.34s/it]/tmp/haozhezhao/MLLMG/autoregressive/train/train_t2i.py:652: FutureWarning: `torch.cuda.amp.autocast(args...)` is deprecated. Please use `torch.amp.autocast('cuda', args...)` instead. + with torch.cuda.amp.autocast(dtype=ptdtype): + 97%|██████████████████████████████████████████████████████████████████████████████████████████████████████████████████▉ | 22841/23458 [36:11<08:44, 1.18it/s][2025-04-25 06:43:48] (step=0046300) Train Loss: 5.2970, Train Steps/Sec: 0.54 + 97%|███████████████████████████████████████████████████████████████████████████████████████████████████████████████████ | 22866/23458 [36:58<34:12, 3.47s/it][2025-04-25 06:44:34] (step=0046325) Train Loss: 5.3957, Train Steps/Sec: 0.54 + 98%|███████████████████████████████████████████████████████████████████████████████████████████████████████████████████▏ | 22891/23458 [37:19<08:04, 1.17it/s][2025-04-25 06:44:56] (step=0046350) Train Loss: 5.4616, Train Steps/Sec: 1.18 + 98%|███████████████████████████████████████████████████████████████████████████████████████████████████████████████████▎ | 22916/23458 [37:40<07:39, 1.18it/s][2025-04-25 06:45:17] (step=0046375) Train Loss: 5.3499, Train Steps/Sec: 1.17 + 98%|███████████████████████████████████████████████████████████████████████████████████████████████████████████████████▍ | 22941/23458 [38:28<09:12, 1.07s/it][2025-04-25 06:46:04] (step=0046400) Train Loss: 5.3552, Train Steps/Sec: 0.53 + 98%|███████████████████████████████████████████████████████████████████████████████████████████████████████████████████▌ | 22966/23458 [38:49<07:06, 1.15it/s][2025-04-25 06:46:26] (step=0046425) Train Loss: 5.4262, Train Steps/Sec: 1.18 + 98%|███████████████████████████████████████████████████████████████████████████████████████████████████████████████████▋ | 22991/23458 [39:10<06:39, 1.17it/s][2025-04-25 06:46:47] (step=0046450) Train Loss: 5.3122, Train Steps/Sec: 1.17 + 98%|███████████████████████████████████████████████████████████████████████████████████████████████████████████████████▊ | 23016/23458 [39:32<06:15, 1.18it/s][2025-04-25 06:47:08] (step=0046475) Train Loss: 5.3518, Train Steps/Sec: 1.17 + 98%|███████████████████████████████████████████████████████████████████████████████████████████████████████████████████▉ | 23041/23458 [40:19<05:54, 1.18it/s][2025-04-25 06:47:56] (step=0046500) Train Loss: 5.2905, Train Steps/Sec: 0.53 + 98%|████████████████████████████████████████████████████████████████████████████████████████████████████████████████████ | 23066/23458 [40:40<05:41, 1.15it/s][2025-04-25 06:48:17] (step=0046525) Train Loss: 5.3737, Train Steps/Sec: 1.17 + 98%|████████████████████████████████████████████████████████████████████████████████████████████████████████████████████▏ | 23091/23458 [41:03<05:16, 1.16it/s][2025-04-25 06:48:39] (step=0046550) Train Loss: 5.3690, Train Steps/Sec: 1.13 + 99%|████████████████████████████████████████████████████████████████████████████████████████████████████████████████████▎ | 23116/23458 [41:25<04:50, 1.18it/s][2025-04-25 06:49:02] (step=0046575) Train Loss: 5.2869, Train Steps/Sec: 1.11 + 99%|████████████████████████████████████████████████████████████████████████████████████████████████████████████████████▍ | 23141/23458 [41:48<04:34, 1.16it/s][2025-04-25 06:49:24] (step=0046600) Train Loss: 5.3717, Train Steps/Sec: 1.11 + 99%|████████████████████████████████████████████████████████████████████████████████████████████████████████████████████▌ | 23166/23458 [42:12<04:47, 1.01it/s][2025-04-25 06:49:48] (step=0046625) Train Loss: 5.3783, Train Steps/Sec: 1.03 + 99%|████████████████████████████████████████████████████████████████████████████████████████████████████████████████████▋ | 23191/23458 [42:33<03:47, 1.17it/s][2025-04-25 06:50:10] (step=0046650) Train Loss: 5.4032, Train Steps/Sec: 1.17 + 99%|████████████████████████████████████████████████████████████████████████████████████████████████████████████████████▊ | 23216/23458 [42:54<03:25, 1.18it/s][2025-04-25 06:50:31] (step=0046675) Train Loss: 5.3840, Train Steps/Sec: 1.17 + 99%|████████████████████████████████████████████████████████████████████████████████████████████████████████████████████▉ | 23241/23458 [43:16<03:03, 1.19it/s][2025-04-25 06:50:52] (step=0046700) Train Loss: 5.3200, Train Steps/Sec: 1.17 + 99%|█████████████████████████████████████████████████████████████████████████████████████████████████████████████████████ | 23266/23458 [43:37<02:46, 1.15it/s][2025-04-25 06:51:14] (step=0046725) Train Loss: 5.2928, Train Steps/Sec: 1.17 + 99%|█████████████████████████████████████████████████████████████████████████████████████████████████████████████████████▏| 23291/23458 [43:59<02:22, 1.17it/s][2025-04-25 06:51:35] (step=0046750) Train Loss: 5.3795, Train Steps/Sec: 1.17 + 99%|█████████████████████████████████████████████████████████████████████████████████████████████████████████████████████▎| 23316/23458 [44:20<02:00, 1.18it/s][2025-04-25 06:51:56] (step=0046775) Train Loss: 5.4092, Train Steps/Sec: 1.17 +100%|█████████████████████████████████████████████████████████████████████████████████████████████████████████████████████▍| 23341/23458 [44:41<01:38, 1.18it/s][2025-04-25 06:52:18] (step=0046800) Train Loss: 5.2414, Train Steps/Sec: 1.17 +100%|█████████████████████████████████████████████████████████████████████████████████████████████████████████████████████▌| 23366/23458 [45:03<01:19, 1.15it/s][2025-04-25 06:52:39] (step=0046825) Train Loss: 5.3846, Train Steps/Sec: 1.17 +100%|█████████████████████████████████████████████████████████████████████████████████████████████████████████████████████▋| 23391/23458 [45:24<00:57, 1.17it/s][2025-04-25 06:53:01] (step=0046850) Train Loss: 5.3781, Train Steps/Sec: 1.17 +100%|█████████████████████████████████████████████████████████████████████████████████████████████████████████████████████▊| 23416/23458 [45:45<00:35, 1.18it/s][2025-04-25 06:53:22] (step=0046875) Train Loss: 5.3731, Train Steps/Sec: 1.17 +100%|█████████████████████████████████████████████████████████████████████████████████████████████████████████████████████▉| 23441/23458 [46:07<00:14, 1.19it/s][2025-04-25 06:53:43] (step=0046900) Train Loss: 5.4406, Train Steps/Sec: 1.17 +100%|██████████████████████████████████████████████████████████████████████████████████████████████████████████████████████| 23458/23458 [46:22<00:00, 8.43it/s] +[2025-04-25 06:53:58] Beginning epoch 2... + 0%| | 8/23458 [00:10<6:18:57, 1.03it/s][2025-04-25 06:54:09] (step=0046925) Train Loss: 5.3348, Train Steps/Sec: 0.98 + 0%|▏ | 33/23458 [00:31<5:34:25, 1.17it/s][2025-04-25 06:54:30] (step=0046950) Train Loss: 5.3262, Train Steps/Sec: 1.17 + 0%|▎ | 58/23458 [00:52<5:32:02, 1.17it/s][2025-04-25 06:54:52] (step=0046975) Train Loss: 5.3311, Train Steps/Sec: 1.17 + 0%|▍ | 83/23458 [01:16<5:31:07, 1.18it/s][2025-04-25 06:55:15] (step=0047000) Train Loss: 5.2322, Train Steps/Sec: 1.05 + 0%|▌ | 108/23458 [01:38<5:39:23, 1.15it/s][2025-04-25 06:55:37] (step=0047025) Train Loss: 5.2292, Train Steps/Sec: 1.17 + 1%|▋ | 133/23458 [01:59<5:33:48, 1.16it/s][2025-04-25 06:55:58] (step=0047050) Train Loss: 5.2233, Train Steps/Sec: 1.17 + 1%|▊ | 158/23458 [02:20<5:29:52, 1.18it/s][2025-04-25 06:56:20] (step=0047075) Train Loss: 5.2715, Train Steps/Sec: 1.17 + 1%|▉ | 183/23458 [02:42<5:26:41, 1.19it/s][2025-04-25 06:56:41] (step=0047100) Train Loss: 5.2030, Train Steps/Sec: 1.16 + 1%|█ | 208/23458 [03:03<5:37:43, 1.15it/s][2025-04-25 06:57:02] (step=0047125) Train Loss: 5.1688, Train Steps/Sec: 1.17 + 1%|█▏ | 233/23458 [03:25<5:31:01, 1.17it/s][2025-04-25 06:57:24] (step=0047150) Train Loss: 5.2820, Train Steps/Sec: 1.17 + 1%|█▎ | 258/23458 [03:46<5:29:10, 1.17it/s][2025-04-25 06:57:45] (step=0047175) Train Loss: 5.2931, Train Steps/Sec: 1.17 + 1%|█▍ | 283/23458 [04:09<5:33:48, 1.16it/s][2025-04-25 06:58:08] (step=0047200) Train Loss: 5.2749, Train Steps/Sec: 1.11 + 1%|█▌ | 308/23458 [04:30<5:35:57, 1.15it/s][2025-04-25 06:58:29] (step=0047225) Train Loss: 5.2923, Train Steps/Sec: 1.17 + 1%|█▋ | 333/23458 [04:52<5:36:06, 1.15it/s][2025-04-25 06:58:51] (step=0047250) Train Loss: 5.2284, Train Steps/Sec: 1.12 + 2%|█▊ | 358/23458 [05:16<6:33:06, 1.02s/it][2025-04-25 06:59:15] (step=0047275) Train Loss: 5.2331, Train Steps/Sec: 1.06 + 2%|█▉ | 383/23458 [05:38<5:25:26, 1.18it/s][2025-04-25 06:59:38] (step=0047300) Train Loss: 5.2246, Train Steps/Sec: 1.10 + 2%|██ | 408/23458 [06:01<5:34:41, 1.15it/s][2025-04-25 07:00:00] (step=0047325) Train Loss: 5.3062, Train Steps/Sec: 1.11 + 2%|██▏ | 433/23458 [06:23<5:28:04, 1.17it/s][2025-04-25 07:00:22] (step=0047350) Train Loss: 5.2587, Train Steps/Sec: 1.17 + 2%|██▎ | 458/23458 [06:44<5:26:12, 1.18it/s][2025-04-25 07:00:43] (step=0047375) Train Loss: 5.2664, Train Steps/Sec: 1.17 + 2%|██▍ | 483/23458 [07:05<5:22:32, 1.19it/s][2025-04-25 07:01:04] (step=0047400) Train Loss: 5.2718, Train Steps/Sec: 1.16 + 2%|██▌ | 508/23458 [07:27<5:34:27, 1.14it/s][2025-04-25 07:01:26] (step=0047425) Train Loss: 5.2578, Train Steps/Sec: 1.17 + 2%|██▋ | 533/23458 [07:48<5:26:01, 1.17it/s][2025-04-25 07:01:47] (step=0047450) Train Loss: 5.2491, Train Steps/Sec: 1.17 + 2%|██▊ | 558/23458 [08:09<5:25:04, 1.17it/s][2025-04-25 07:02:09] (step=0047475) Train Loss: 5.2689, Train Steps/Sec: 1.17 + 2%|██▉ | 583/23458 [08:31<5:21:03, 1.19it/s][2025-04-25 07:02:30] (step=0047500) Train Loss: 5.2657, Train Steps/Sec: 1.17 + 3%|███ | 608/23458 [08:52<5:31:45, 1.15it/s][2025-04-25 07:02:51] (step=0047525) Train Loss: 5.3215, Train Steps/Sec: 1.17 + 3%|███▏ | 633/23458 [09:14<5:26:21, 1.17it/s][2025-04-25 07:03:13] (step=0047550) Train Loss: 5.2861, Train Steps/Sec: 1.17 + 3%|███▎ | 658/23458 [09:35<5:22:14, 1.18it/s][2025-04-25 07:03:34] (step=0047575) Train Loss: 5.3298, Train Steps/Sec: 1.17 + 3%|███▍ | 683/23458 [09:56<5:20:41, 1.18it/s][2025-04-25 07:03:56] (step=0047600) Train Loss: 5.3455, Train Steps/Sec: 1.16 + 3%|███▌ | 708/23458 [10:18<5:32:15, 1.14it/s][2025-04-25 07:04:17] (step=0047625) Train Loss: 5.2444, Train Steps/Sec: 1.17 + 3%|███▋ | 733/23458 [10:39<5:24:24, 1.17it/s][2025-04-25 07:04:39] (step=0047650) Train Loss: 5.2882, Train Steps/Sec: 1.16 + 3%|███▊ | 758/23458 [11:01<5:23:37, 1.17it/s][2025-04-25 07:05:00] (step=0047675) Train Loss: 5.3281, Train Steps/Sec: 1.17 + 3%|███▉ | 783/23458 [11:22<5:19:44, 1.18it/s][2025-04-25 07:05:22] (step=0047700) Train Loss: 5.2791, Train Steps/Sec: 1.16 + 3%|████ | 808/23458 [11:46<6:04:50, 1.03it/s][2025-04-25 07:05:45] (step=0047725) Train Loss: 5.2672, Train Steps/Sec: 1.08 + 4%|████▏ | 833/23458 [12:07<5:24:01, 1.16it/s][2025-04-25 07:06:06] (step=0047750) Train Loss: 5.2602, Train Steps/Sec: 1.17 + 4%|████▎ | 858/23458 [12:28<5:20:55, 1.17it/s][2025-04-25 07:06:27] (step=0047775) Train Loss: 5.2706, Train Steps/Sec: 1.17 + 4%|████▍ | 883/23458 [12:50<5:18:44, 1.18it/s][2025-04-25 07:06:49] (step=0047800) Train Loss: 5.2113, Train Steps/Sec: 1.16 + 4%|████▌ | 908/23458 [13:11<5:26:53, 1.15it/s][2025-04-25 07:07:10] (step=0047825) Train Loss: 5.2626, Train Steps/Sec: 1.17 + 4%|████▋ | 933/23458 [13:34<5:42:45, 1.10it/s][2025-04-25 07:07:33] (step=0047850) Train Loss: 5.2194, Train Steps/Sec: 1.11 + 4%|████▊ | 958/23458 [13:55<5:20:11, 1.17it/s][2025-04-25 07:07:54] (step=0047875) Train Loss: 5.3253, Train Steps/Sec: 1.17 + 4%|████▉ | 983/23458 [14:17<6:05:01, 1.03it/s][2025-04-25 07:08:17] (step=0047900) Train Loss: 5.3148, Train Steps/Sec: 1.12 + 4%|█████ | 1008/23458 [14:41<7:21:02, 1.18s/it][2025-04-25 07:08:40] (step=0047925) Train Loss: 5.3373, Train Steps/Sec: 1.08 + 4%|█████▏ | 1033/23458 [15:03<5:19:55, 1.17it/s][2025-04-25 07:09:02] (step=0047950) Train Loss: 5.2988, Train Steps/Sec: 1.12 + 5%|█████▎ | 1058/23458 [15:24<5:18:52, 1.17it/s][2025-04-25 07:09:23] (step=0047975) Train Loss: 5.2819, Train Steps/Sec: 1.17 + 5%|█████▍ | 1083/23458 [15:46<5:14:35, 1.19it/s][2025-04-25 07:09:45] (step=0048000) Train Loss: 5.2750, Train Steps/Sec: 1.16 +You are using a model of type blip-2 to instantiate a model of type instructblip. This is not supported for all configurations of models and can yield errors. +[2025-04-25 07:09:45] text_config is None. Initializing the text config with default values (`OPTConfig`). +model_name_or_path:/tmp/haozhezhao/model/blip2-flan-t5-xl +image_place_holder: +tokenizer length after expend 32101 +tokenizer length before expend 32100 + /tmp/haozhezhao/anaconda3/envs/nlp/lib/python3.11/contextlib.py:105: FutureWarning: `torch.backends.cuda.sdp_kernel()` is deprecated. In the future, this context manager will be removed. Please see `torch.nn.attention.sdpa_kernel()` for the new context manager, with updated signature. + self.gen = func(*args, **kwds) | 0/4 [00:00 +tokenizer length after expend 32101 +tokenizer length before expend 32100 +100%|██████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████| 4/4 [03:56<00:00, 59.02s/it] +[2025-04-25 07:49:08] Finish Eval in 50000 steps...███████████████████████████████████████████████████████████████████████| 4/4 [03:55<00:00, 58.71s/it] +processing left-padding... +processing left-padding... +processing left-padding... +processing left-padding... +[2025-04-25 07:49:27] Saved checkpoint to checkpoint/Afteri2i_100kckpt_LLAVAT5_empty_Fixed_just_segment_pretrain_stage3_dreambench_recap_400k_t2i_400k_flux_200k_midjourney_150k_recovery_150k_3e4_epoch2_no_rotated/002-GPT-XL/checkpoints/0050000.pt +[2025-04-25 07:49:30] Removed old checkpoint: checkpoint/Afteri2i_100kckpt_LLAVAT5_empty_Fixed_just_segment_pretrain_stage3_dreambench_recap_400k_t2i_400k_flux_200k_midjourney_150k_recovery_150k_3e4_epoch2_no_rotated/002-GPT-XL/checkpoints/0048000.pt + 13%|███████████████▌ | 3108/23458 [55:52<5:05:05, 1.11it/s][2025-04-25 07:49:51] (step=0050025) Train Loss: 5.3138, Train Steps/Sec: 0.07 + 13%|███████████████▋ | 3133/23458 [56:14<4:52:18, 1.16it/s][2025-04-25 07:50:13] (step=0050050) Train Loss: 5.2595, Train Steps/Sec: 1.16 + 13%|███████████████▊ | 3158/23458 [56:35<4:47:48, 1.18it/s][2025-04-25 07:50:34] (step=0050075) Train Loss: 5.2120, Train Steps/Sec: 1.17 + 14%|███████████████▉ | 3183/23458 [56:57<4:46:38, 1.18it/s][2025-04-25 07:50:56] (step=0050100) Train Loss: 5.2992, Train Steps/Sec: 1.16 + 14%|████████████████ | 3208/23458 [57:18<4:56:54, 1.14it/s][2025-04-25 07:51:17] (step=0050125) Train Loss: 5.3257, Train Steps/Sec: 1.17 + 14%|████████████████▏ | 3233/23458 [57:40<4:49:54, 1.16it/s][2025-04-25 07:51:39] (step=0050150) Train Loss: 5.3392, Train Steps/Sec: 1.16 + 14%|████████████████▏ | 3258/23458 [58:01<4:48:21, 1.17it/s][2025-04-25 07:52:00] (step=0050175) Train Loss: 5.3054, Train Steps/Sec: 1.17 + 14%|████████████████▎ | 3283/23458 [58:23<4:45:42, 1.18it/s][2025-04-25 07:52:22] (step=0050200) Train Loss: 5.2217, Train Steps/Sec: 1.16 + 14%|████████████████▍ | 3308/23458 [58:44<4:54:08, 1.14it/s][2025-04-25 07:52:43] (step=0050225) Train Loss: 5.3270, Train Steps/Sec: 1.16 + 14%|████████████████▌ | 3333/23458 [59:06<4:47:43, 1.17it/s][2025-04-25 07:53:05] (step=0050250) Train Loss: 5.2269, Train Steps/Sec: 1.17 + 14%|████████████████▋ | 3358/23458 [59:27<4:46:03, 1.17it/s][2025-04-25 07:53:26] (step=0050275) Train Loss: 5.2466, Train Steps/Sec: 1.17 + 14%|████████████████▊ | 3383/23458 [59:48<4:42:15, 1.19it/s][2025-04-25 07:53:47] (step=0050300) Train Loss: 5.3594, Train Steps/Sec: 1.16 + 15%|████████████████▋ | 3408/23458 [1:00:10<4:50:46, 1.15it/s][2025-04-25 07:54:09] (step=0050325) Train Loss: 5.3077, Train Steps/Sec: 1.17 + 15%|████████████████▊ | 3433/23458 [1:00:31<4:47:35, 1.16it/s][2025-04-25 07:54:30] (step=0050350) Train Loss: 5.2708, Train Steps/Sec: 1.17 + 15%|████████████████▉ | 3458/23458 [1:00:52<4:43:16, 1.18it/s][2025-04-25 07:54:52] (step=0050375) Train Loss: 5.3305, Train Steps/Sec: 1.17 + 15%|█████████████████ | 3483/23458 [1:01:14<4:41:33, 1.18it/s][2025-04-25 07:55:13] (step=0050400) Train Loss: 5.3107, Train Steps/Sec: 1.15 + 15%|█████████████████▏ | 3508/23458 [1:01:36<4:49:14, 1.15it/s][2025-04-25 07:55:35] (step=0050425) Train Loss: 5.2263, Train Steps/Sec: 1.17 + 15%|█████████████████▎ | 3533/23458 [1:01:58<6:07:59, 1.11s/it][2025-04-25 07:55:57] (step=0050450) Train Loss: 5.2646, Train Steps/Sec: 1.12 + 15%|█████████████████▍ | 3558/23458 [1:02:19<4:41:46, 1.18it/s][2025-04-25 07:56:18] (step=0050475) Train Loss: 5.2875, Train Steps/Sec: 1.17 + 15%|█████████████████▌ | 3583/23458 [1:02:41<4:40:25, 1.18it/s][2025-04-25 07:56:40] (step=0050500) Train Loss: 5.3474, Train Steps/Sec: 1.17 + 15%|█████████████████▋ | 3608/23458 [1:03:03<4:47:44, 1.15it/s][2025-04-25 07:57:02] (step=0050525) Train Loss: 5.1713, Train Steps/Sec: 1.12 + 15%|█████████████████▊ | 3633/23458 [1:03:25<4:44:45, 1.16it/s][2025-04-25 07:57:24] (step=0050550) Train Loss: 5.3388, Train Steps/Sec: 1.12 + 16%|█████████████████▉ | 3658/23458 [1:03:48<4:43:02, 1.17it/s][2025-04-25 07:57:47] (step=0050575) Train Loss: 5.2772, Train Steps/Sec: 1.09 + 16%|██████████████████ | 3683/23458 [1:04:10<4:39:18, 1.18it/s][2025-04-25 07:58:09] (step=0050600) Train Loss: 5.2134, Train Steps/Sec: 1.16 + 16%|██████████████████▏ | 3708/23458 [1:04:32<4:46:17, 1.15it/s][2025-04-25 07:58:31] (step=0050625) Train Loss: 5.3416, Train Steps/Sec: 1.12 + 16%|██████████████████▎ | 3733/23458 [1:04:54<4:59:39, 1.10it/s][2025-04-25 07:58:54] (step=0050650) Train Loss: 5.2410, Train Steps/Sec: 1.12 + 16%|██████████████████▍ | 3758/23458 [1:05:16<4:37:32, 1.18it/s][2025-04-25 07:59:15] (step=0050675) Train Loss: 5.2899, Train Steps/Sec: 1.17 + 16%|██████████████████▌ | 3783/23458 [1:05:37<4:36:42, 1.19it/s][2025-04-25 07:59:36] (step=0050700) Train Loss: 5.3115, Train Steps/Sec: 1.17 + 16%|██████████████████▋ | 3808/23458 [1:05:59<4:44:30, 1.15it/s][2025-04-25 07:59:58] (step=0050725) Train Loss: 5.2461, Train Steps/Sec: 1.17 + 16%|██████████████████▊ | 3833/23458 [1:06:21<4:40:27, 1.17it/s][2025-04-25 08:00:20] (step=0050750) Train Loss: 5.2945, Train Steps/Sec: 1.11 + 16%|██████████████████▉ | 3858/23458 [1:06:42<4:37:04, 1.18it/s][2025-04-25 08:00:41] (step=0050775) Train Loss: 5.3122, Train Steps/Sec: 1.17 + 17%|███████████████████ | 3883/23458 [1:07:04<4:35:07, 1.19it/s][2025-04-25 08:01:03] (step=0050800) Train Loss: 5.2003, Train Steps/Sec: 1.17 + 17%|███████████████████▏ | 3908/23458 [1:07:25<4:43:38, 1.15it/s][2025-04-25 08:01:24] (step=0050825) Train Loss: 5.3067, Train Steps/Sec: 1.17 + 17%|███████████████████▎ | 3933/23458 [1:07:47<4:40:08, 1.16it/s][2025-04-25 08:01:46] (step=0050850) Train Loss: 5.2811, Train Steps/Sec: 1.17 + 17%|███████████████████▍ | 3958/23458 [1:08:08<4:36:39, 1.17it/s][2025-04-25 08:02:07] (step=0050875) Train Loss: 5.2294, Train Steps/Sec: 1.17 + 17%|███████████████████▌ | 3983/23458 [1:08:29<4:34:43, 1.18it/s][2025-04-25 08:02:28] (step=0050900) Train Loss: 5.3407, Train Steps/Sec: 1.17 + 17%|███████████████████▋ | 4008/23458 [1:08:51<4:41:26, 1.15it/s][2025-04-25 08:02:50] (step=0050925) Train Loss: 5.3036, Train Steps/Sec: 1.17 + 17%|███████████████████▊ | 4033/23458 [1:09:12<4:37:56, 1.16it/s][2025-04-25 08:03:11] (step=0050950) Train Loss: 5.3031, Train Steps/Sec: 1.17 + 17%|███████████████████▉ | 4058/23458 [1:09:33<4:36:30, 1.17it/s][2025-04-25 08:03:33] (step=0050975) Train Loss: 5.2697, Train Steps/Sec: 1.17 + 17%|████████████████████ | 4083/23458 [1:09:55<4:32:58, 1.18it/s][2025-04-25 08:03:54] (step=0051000) Train Loss: 5.3382, Train Steps/Sec: 1.16 + 18%|████████████████████▏ | 4108/23458 [1:10:16<4:40:11, 1.15it/s][2025-04-25 08:04:15] (step=0051025) Train Loss: 5.3313, Train Steps/Sec: 1.17 + 18%|████████████████████▎ | 4133/23458 [1:10:38<4:35:32, 1.17it/s][2025-04-25 08:04:37] (step=0051050) Train Loss: 5.2845, Train Steps/Sec: 1.17 + 18%|████████████████████▍ | 4158/23458 [1:10:59<4:33:52, 1.17it/s][2025-04-25 08:04:58] (step=0051075) Train Loss: 5.2751, Train Steps/Sec: 1.17 + 18%|████████████████████▌ | 4183/23458 [1:11:20<4:32:03, 1.18it/s][2025-04-25 08:05:20] (step=0051100) Train Loss: 5.3010, Train Steps/Sec: 1.16 + 18%|████████████████████▋ | 4208/23458 [1:11:43<4:39:17, 1.15it/s][2025-04-25 08:05:42] (step=0051125) Train Loss: 5.2881, Train Steps/Sec: 1.12 + 18%|████████████████████▊ | 4233/23458 [1:12:04<4:34:56, 1.17it/s][2025-04-25 08:06:03] (step=0051150) Train Loss: 5.2731, Train Steps/Sec: 1.17 + 18%|████████████████████▊ | 4258/23458 [1:12:26<4:32:18, 1.18it/s][2025-04-25 08:06:25] (step=0051175) Train Loss: 5.2481, Train Steps/Sec: 1.13 + 18%|████████████████████▉ | 4283/23458 [1:12:49<4:32:49, 1.17it/s][2025-04-25 08:06:48] (step=0051200) Train Loss: 5.2926, Train Steps/Sec: 1.12 + 18%|█████████████████████ | 4308/23458 [1:13:12<5:22:30, 1.01s/it][2025-04-25 08:07:11] (step=0051225) Train Loss: 5.3127, Train Steps/Sec: 1.09 + 18%|█████████████████████▏ | 4333/23458 [1:13:33<4:31:56, 1.17it/s][2025-04-25 08:07:32] (step=0051250) Train Loss: 5.2915, Train Steps/Sec: 1.17 + 19%|█████████████████████▎ | 4358/23458 [1:13:54<4:31:10, 1.17it/s][2025-04-25 08:07:54] (step=0051275) Train Loss: 5.2624, Train Steps/Sec: 1.16 + 19%|█████████████████████▍ | 4383/23458 [1:14:16<4:28:31, 1.18it/s][2025-04-25 08:08:15] (step=0051300) Train Loss: 5.2621, Train Steps/Sec: 1.17 + 19%|█████████████████████▌ | 4408/23458 [1:14:37<4:35:41, 1.15it/s][2025-04-25 08:08:36] (step=0051325) Train Loss: 5.2343, Train Steps/Sec: 1.17 + 19%|█████████████████████▋ | 4433/23458 [1:14:59<4:33:08, 1.16it/s][2025-04-25 08:08:59] (step=0051350) Train Loss: 5.2943, Train Steps/Sec: 1.13 + 19%|█████████████████████▊ | 4458/23458 [1:15:21<4:28:44, 1.18it/s][2025-04-25 08:09:20] (step=0051375) Train Loss: 5.3008, Train Steps/Sec: 1.17 + 19%|█████████████████████▉ | 4483/23458 [1:15:43<4:28:28, 1.18it/s][2025-04-25 08:09:43] (step=0051400) Train Loss: 5.2584, Train Steps/Sec: 1.10 + 19%|██████████████████████ | 4508/23458 [1:16:05<4:35:38, 1.15it/s][2025-04-25 08:10:04] (step=0051425) Train Loss: 5.2868, Train Steps/Sec: 1.17 + 19%|██████████████████████▏ | 4533/23458 [1:16:26<4:30:47, 1.16it/s][2025-04-25 08:10:25] (step=0051450) Train Loss: 5.2585, Train Steps/Sec: 1.17 + 19%|██████████████████████▎ | 4558/23458 [1:16:48<4:28:11, 1.17it/s][2025-04-25 08:10:47] (step=0051475) Train Loss: 5.3129, Train Steps/Sec: 1.17 + 20%|██████████████████████▍ | 4583/23458 [1:17:10<4:27:27, 1.18it/s][2025-04-25 08:11:09] (step=0051500) Train Loss: 5.2160, Train Steps/Sec: 1.10 + 20%|██████████████████████▌ | 4608/23458 [1:17:32<4:33:43, 1.15it/s][2025-04-25 08:11:31] (step=0051525) Train Loss: 5.2878, Train Steps/Sec: 1.17 + 20%|██████████████████████▋ | 4633/23458 [1:17:53<4:28:41, 1.17it/s][2025-04-25 08:11:52] (step=0051550) Train Loss: 5.3765, Train Steps/Sec: 1.17 + 20%|██████████████████████▊ | 4658/23458 [1:18:14<4:26:43, 1.17it/s][2025-04-25 08:12:13] (step=0051575) Train Loss: 5.2994, Train Steps/Sec: 1.17 + 20%|██████████████████████▉ | 4683/23458 [1:18:36<4:24:54, 1.18it/s][2025-04-25 08:12:35] (step=0051600) Train Loss: 5.2954, Train Steps/Sec: 1.16 + 20%|███████████████████████ | 4708/23458 [1:18:57<4:32:26, 1.15it/s][2025-04-25 08:12:57] (step=0051625) Train Loss: 5.2695, Train Steps/Sec: 1.16 + 20%|███████████████████████▏ | 4733/23458 [1:19:19<4:27:13, 1.17it/s][2025-04-25 08:13:18] (step=0051650) Train Loss: 5.3088, Train Steps/Sec: 1.17 + 20%|███████████████████████▎ | 4758/23458 [1:19:40<4:24:01, 1.18it/s][2025-04-25 08:13:39] (step=0051675) Train Loss: 5.2754, Train Steps/Sec: 1.17 + 20%|███████████████████████▍ | 4783/23458 [1:20:01<4:22:50, 1.18it/s][2025-04-25 08:14:01] (step=0051700) Train Loss: 5.2860, Train Steps/Sec: 1.17 + 20%|███████████████████████▌ | 4808/23458 [1:20:23<4:31:04, 1.15it/s][2025-04-25 08:14:22] (step=0051725) Train Loss: 5.2600, Train Steps/Sec: 1.17 + 21%|███████████████████████▋ | 4833/23458 [1:20:44<4:24:46, 1.17it/s][2025-04-25 08:14:43] (step=0051750) Train Loss: 5.2866, Train Steps/Sec: 1.17 + 21%|███████████████████████▊ | 4858/23458 [1:21:07<4:24:12, 1.17it/s][2025-04-25 08:15:06] (step=0051775) Train Loss: 5.3317, Train Steps/Sec: 1.12 + 21%|███████████████████████▉ | 4883/23458 [1:21:28<4:28:32, 1.15it/s][2025-04-25 08:15:27] (step=0051800) Train Loss: 5.3018, Train Steps/Sec: 1.16 + 21%|████████████████████████ | 4908/23458 [1:21:50<4:30:53, 1.14it/s][2025-04-25 08:15:49] (step=0051825) Train Loss: 5.3064, Train Steps/Sec: 1.13 + 21%|████████████████████████▏ | 4933/23458 [1:22:13<4:33:07, 1.13it/s][2025-04-25 08:16:12] (step=0051850) Train Loss: 5.2582, Train Steps/Sec: 1.12 + 21%|████████████████████████▎ | 4958/23458 [1:22:34<4:22:08, 1.18it/s][2025-04-25 08:16:33] (step=0051875) Train Loss: 5.2837, Train Steps/Sec: 1.17 + 21%|████████████████████████▍ | 4983/23458 [1:22:57<4:19:42, 1.19it/s][2025-04-25 08:16:56] (step=0051900) Train Loss: 5.3158, Train Steps/Sec: 1.08 + 21%|████████████████████████▌ | 5008/23458 [1:23:19<4:26:49, 1.15it/s][2025-04-25 08:17:18] (step=0051925) Train Loss: 5.3236, Train Steps/Sec: 1.17 + 21%|████████████████████████▋ | 5033/23458 [1:23:40<4:22:36, 1.17it/s][2025-04-25 08:17:39] (step=0051950) Train Loss: 5.2775, Train Steps/Sec: 1.17 + 22%|████████████████████████▊ | 5058/23458 [1:24:01<4:20:07, 1.18it/s][2025-04-25 08:18:00] (step=0051975) Train Loss: 5.2215, Train Steps/Sec: 1.17 + 22%|████████████████████████▉ | 5083/23458 [1:24:23<4:19:53, 1.18it/s][2025-04-25 08:18:22] (step=0052000) Train Loss: 5.3349, Train Steps/Sec: 1.16 +You are using a model of type blip-2 to instantiate a model of type instructblip. This is not supported for all configurations of models and can yield errors. +[2025-04-25 08:18:22] text_config is None. Initializing the text config with default values (`OPTConfig`). +model_name_or_path:/tmp/haozhezhao/model/blip2-flan-t5-xl +image_place_holder: +tokenizer length after expend 32101 +tokenizer length before expend 32100 +100%|██████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████| 4/4 [03:58<00:00, 59.50s/it] +[2025-04-25 08:23:28] Finish Eval in 52000 steps...███████████████████████████████████████████████████████████████████████| 4/4 [03:57<00:00, 59.17s/it] +processing left-padding... +processing left-padding... +processing left-padding... +processing left-padding... +[2025-04-25 08:23:46] Saved checkpoint to checkpoint/Afteri2i_100kckpt_LLAVAT5_empty_Fixed_just_segment_pretrain_stage3_dreambench_recap_400k_t2i_400k_flux_200k_midjourney_150k_recovery_150k_3e4_epoch2_no_rotated/002-GPT-XL/checkpoints/0052000.pt +[2025-04-25 08:23:48] Removed old checkpoint: checkpoint/Afteri2i_100kckpt_LLAVAT5_empty_Fixed_just_segment_pretrain_stage3_dreambench_recap_400k_t2i_400k_flux_200k_midjourney_150k_recovery_150k_3e4_epoch2_no_rotated/002-GPT-XL/checkpoints/0050000.pt + 22%|█████████████████████████ | 5108/23458 [1:30:11<4:31:41, 1.13it/s][2025-04-25 08:24:10] (step=0052025) Train Loss: 5.3486, Train Steps/Sec: 0.07 + 22%|█████████████████████████▏ | 5133/23458 [1:30:32<4:23:03, 1.16it/s][2025-04-25 08:24:31] (step=0052050) Train Loss: 5.2924, Train Steps/Sec: 1.17 + 22%|█████████████████████████▎ | 5158/23458 [1:30:54<4:23:20, 1.16it/s][2025-04-25 08:24:53] (step=0052075) Train Loss: 5.2657, Train Steps/Sec: 1.13 + 22%|█████████████████████████▍ | 5183/23458 [1:31:15<4:16:36, 1.19it/s][2025-04-25 08:25:15] (step=0052100) Train Loss: 5.3027, Train Steps/Sec: 1.17 + 22%|█████████████████████████▌ | 5208/23458 [1:31:38<4:24:06, 1.15it/s][2025-04-25 08:25:37] (step=0052125) Train Loss: 5.2392, Train Steps/Sec: 1.12 + 22%|█████████████████████████▋ | 5233/23458 [1:31:59<4:21:36, 1.16it/s][2025-04-25 08:25:58] (step=0052150) Train Loss: 5.2757, Train Steps/Sec: 1.17 + 22%|█████████████████████████▊ | 5258/23458 [1:32:21<4:18:51, 1.17it/s][2025-04-25 08:26:20] (step=0052175) Train Loss: 5.2209, Train Steps/Sec: 1.17 + 23%|█████████████████████████▉ | 5283/23458 [1:32:42<4:15:47, 1.18it/s][2025-04-25 08:26:41] (step=0052200) Train Loss: 5.3072, Train Steps/Sec: 1.16 + 23%|██████████████████████████ | 5308/23458 [1:33:04<4:23:43, 1.15it/s][2025-04-25 08:27:03] (step=0052225) Train Loss: 5.3042, Train Steps/Sec: 1.17 + 23%|██████████████████████████▏ | 5333/23458 [1:33:25<4:20:05, 1.16it/s][2025-04-25 08:27:24] (step=0052250) Train Loss: 5.2794, Train Steps/Sec: 1.17 + 23%|██████████████████████████▎ | 5358/23458 [1:33:46<4:16:36, 1.18it/s][2025-04-25 08:27:45] (step=0052275) Train Loss: 5.2672, Train Steps/Sec: 1.17 + 23%|██████████████████████████▍ | 5383/23458 [1:34:08<4:14:03, 1.19it/s][2025-04-25 08:28:07] (step=0052300) Train Loss: 5.2680, Train Steps/Sec: 1.16 + 23%|██████████████████████████▌ | 5408/23458 [1:34:29<4:34:44, 1.09it/s][2025-04-25 08:28:29] (step=0052325) Train Loss: 5.3534, Train Steps/Sec: 1.16 + 23%|██████████████████████████▋ | 5433/23458 [1:34:51<4:17:15, 1.17it/s][2025-04-25 08:28:50] (step=0052350) Train Loss: 5.2623, Train Steps/Sec: 1.17 + 23%|██████████████████████████▊ | 5458/23458 [1:35:12<4:13:41, 1.18it/s][2025-04-25 08:29:11] (step=0052375) Train Loss: 5.2816, Train Steps/Sec: 1.17 + 23%|██████████████████████████▉ | 5483/23458 [1:35:34<4:12:43, 1.19it/s][2025-04-25 08:29:33] (step=0052400) Train Loss: 5.2983, Train Steps/Sec: 1.17 + 23%|███████████████████████████ | 5508/23458 [1:35:56<4:19:48, 1.15it/s][2025-04-25 08:29:55] (step=0052425) Train Loss: 5.3378, Train Steps/Sec: 1.13 + 24%|███████████████████████████ | 5533/23458 [1:36:17<4:15:39, 1.17it/s][2025-04-25 08:30:16] (step=0052450) Train Loss: 5.3607, Train Steps/Sec: 1.17 + 24%|███████████████████████████▏ | 5558/23458 [1:36:39<4:19:44, 1.15it/s][2025-04-25 08:30:38] (step=0052475) Train Loss: 5.3965, Train Steps/Sec: 1.13 + 24%|███████████████████████████▎ | 5583/23458 [1:37:02<4:33:19, 1.09it/s][2025-04-25 08:31:01] (step=0052500) Train Loss: 5.2747, Train Steps/Sec: 1.12 + 24%|███████████████████████████▍ | 5608/23458 [1:37:23<4:18:06, 1.15it/s][2025-04-25 08:31:22] (step=0052525) Train Loss: 5.3493, Train Steps/Sec: 1.17 + 24%|███████████████████████████▌ | 5633/23458 [1:37:46<4:22:23, 1.13it/s][2025-04-25 08:31:45] (step=0052550) Train Loss: 5.2575, Train Steps/Sec: 1.08 + 24%|███████████████████████████▋ | 5658/23458 [1:38:07<4:11:38, 1.18it/s][2025-04-25 08:32:06] (step=0052575) Train Loss: 5.2266, Train Steps/Sec: 1.17 + 24%|███████████████████████████▊ | 5683/23458 [1:38:29<4:09:47, 1.19it/s][2025-04-25 08:32:28] (step=0052600) Train Loss: 5.2579, Train Steps/Sec: 1.17 + 24%|███████████████████████████▉ | 5708/23458 [1:38:50<4:16:59, 1.15it/s][2025-04-25 08:32:49] (step=0052625) Train Loss: 5.2887, Train Steps/Sec: 1.17 + 24%|████████████████████████████ | 5733/23458 [1:39:12<4:12:16, 1.17it/s][2025-04-25 08:33:11] (step=0052650) Train Loss: 5.2585, Train Steps/Sec: 1.17 + 25%|████████████████████████████▏ | 5758/23458 [1:39:33<4:12:04, 1.17it/s][2025-04-25 08:33:32] (step=0052675) Train Loss: 5.2892, Train Steps/Sec: 1.17 + 25%|████████████████████████████▎ | 5783/23458 [1:39:54<4:08:59, 1.18it/s][2025-04-25 08:33:53] (step=0052700) Train Loss: 5.2784, Train Steps/Sec: 1.17 + 25%|████████████████████████████▍ | 5808/23458 [1:40:16<4:15:22, 1.15it/s][2025-04-25 08:34:15] (step=0052725) Train Loss: 5.3006, Train Steps/Sec: 1.17 + 25%|████████████████████████████▌ | 5833/23458 [1:40:38<4:12:05, 1.17it/s][2025-04-25 08:34:37] (step=0052750) Train Loss: 5.2882, Train Steps/Sec: 1.12 + 25%|████████████████████████████▋ | 5858/23458 [1:40:59<4:08:30, 1.18it/s][2025-04-25 08:34:59] (step=0052775) Train Loss: 5.2165, Train Steps/Sec: 1.17 + 25%|████████████████████████████▊ | 5883/23458 [1:41:22<4:35:00, 1.07it/s][2025-04-25 08:35:21] (step=0052800) Train Loss: 5.2068, Train Steps/Sec: 1.12 + 25%|████████████████████████████▉ | 5908/23458 [1:41:43<4:15:25, 1.15it/s][2025-04-25 08:35:42] (step=0052825) Train Loss: 5.2873, Train Steps/Sec: 1.17 + 25%|█████████████████████████████ | 5933/23458 [1:42:05<4:12:16, 1.16it/s][2025-04-25 08:36:05] (step=0052850) Train Loss: 5.3201, Train Steps/Sec: 1.12 + 25%|█████████████████████████████▏ | 5958/23458 [1:42:27<4:07:27, 1.18it/s][2025-04-25 08:36:26] (step=0052875) Train Loss: 5.2943, Train Steps/Sec: 1.17 + 26%|█████████████████████████████▎ | 5983/23458 [1:42:48<4:06:51, 1.18it/s][2025-04-25 08:36:47] (step=0052900) Train Loss: 5.2710, Train Steps/Sec: 1.17 + 26%|█████████████████████████████▍ | 6008/23458 [1:43:10<4:15:27, 1.14it/s][2025-04-25 08:37:09] (step=0052925) Train Loss: 5.3268, Train Steps/Sec: 1.17 + 26%|█████████████████████████████▌ | 6033/23458 [1:43:31<4:09:20, 1.16it/s][2025-04-25 08:37:30] (step=0052950) Train Loss: 5.3488, Train Steps/Sec: 1.17 + 26%|█████████████████████████████▋ | 6058/23458 [1:43:52<4:06:53, 1.17it/s][2025-04-25 08:37:51] (step=0052975) Train Loss: 5.2678, Train Steps/Sec: 1.17 + 26%|█████████████████████████████▊ | 6083/23458 [1:44:14<4:05:32, 1.18it/s][2025-04-25 08:38:13] (step=0053000) Train Loss: 5.2899, Train Steps/Sec: 1.16 + 26%|█████████████████████████████▉ | 6108/23458 [1:44:35<4:11:33, 1.15it/s][2025-04-25 08:38:34] (step=0053025) Train Loss: 5.2765, Train Steps/Sec: 1.17 + 26%|██████████████████████████████ | 6133/23458 [1:44:57<4:07:40, 1.17it/s][2025-04-25 08:38:56] (step=0053050) Train Loss: 5.2875, Train Steps/Sec: 1.17 + 26%|██████████████████████████████▏ | 6158/23458 [1:45:19<4:05:21, 1.18it/s][2025-04-25 08:39:18] (step=0053075) Train Loss: 5.2570, Train Steps/Sec: 1.12 + 26%|██████████████████████████████▎ | 6183/23458 [1:45:40<4:04:10, 1.18it/s][2025-04-25 08:39:39] (step=0053100) Train Loss: 5.2685, Train Steps/Sec: 1.16 + 26%|██████████████████████████████▍ | 6208/23458 [1:46:03<4:27:15, 1.08it/s][2025-04-25 08:40:02] (step=0053125) Train Loss: 5.2305, Train Steps/Sec: 1.13 + 27%|██████████████████████████████▌ | 6233/23458 [1:46:25<4:40:34, 1.02it/s][2025-04-25 08:40:24] (step=0053150) Train Loss: 5.2854, Train Steps/Sec: 1.13 + 27%|██████████████████████████████▋ | 6258/23458 [1:46:46<4:02:45, 1.18it/s][2025-04-25 08:40:45] (step=0053175) Train Loss: 5.2939, Train Steps/Sec: 1.17 + 27%|██████████████████████████████▊ | 6283/23458 [1:47:08<4:06:11, 1.16it/s][2025-04-25 08:41:07] (step=0053200) Train Loss: 5.2526, Train Steps/Sec: 1.12 + 27%|██████████████████████████████▉ | 6308/23458 [1:47:31<4:09:15, 1.15it/s][2025-04-25 08:41:30] (step=0053225) Train Loss: 5.3854, Train Steps/Sec: 1.12 + 27%|███████████████████████████████ | 6333/23458 [1:47:52<4:04:23, 1.17it/s][2025-04-25 08:41:51] (step=0053250) Train Loss: 5.3054, Train Steps/Sec: 1.17 + 27%|███████████████████████████████▏ | 6358/23458 [1:48:13<4:02:21, 1.18it/s][2025-04-25 08:42:12] (step=0053275) Train Loss: 5.3671, Train Steps/Sec: 1.17 + 27%|███████████████████████████████▎ | 6383/23458 [1:48:35<4:00:13, 1.18it/s][2025-04-25 08:42:34] (step=0053300) Train Loss: 5.3695, Train Steps/Sec: 1.17 + 27%|███████████████████████████████▍ | 6408/23458 [1:48:56<4:06:51, 1.15it/s][2025-04-25 08:42:55] (step=0053325) Train Loss: 5.2711, Train Steps/Sec: 1.17 + 27%|███████████████████████████████▌ | 6433/23458 [1:49:18<4:03:07, 1.17it/s][2025-04-25 08:43:17] (step=0053350) Train Loss: 5.2769, Train Steps/Sec: 1.17 + 28%|███████████████████████████████▋ | 6458/23458 [1:49:39<4:00:48, 1.18it/s][2025-04-25 08:43:38] (step=0053375) Train Loss: 5.2386, Train Steps/Sec: 1.17 + 28%|███████████████████████████████▊ | 6483/23458 [1:50:00<3:59:24, 1.18it/s][2025-04-25 08:43:59] (step=0053400) Train Loss: 5.2945, Train Steps/Sec: 1.17 + 28%|███████████████████████████████▉ | 6508/23458 [1:50:22<4:05:25, 1.15it/s][2025-04-25 08:44:21] (step=0053425) Train Loss: 5.2858, Train Steps/Sec: 1.17 + 28%|████████████████████████████████ | 6533/23458 [1:50:43<4:02:16, 1.16it/s][2025-04-25 08:44:42] (step=0053450) Train Loss: 5.2631, Train Steps/Sec: 1.17 + 28%|████████████████████████████████▏ | 6558/23458 [1:51:05<3:59:40, 1.18it/s][2025-04-25 08:45:04] (step=0053475) Train Loss: 5.3600, Train Steps/Sec: 1.17 + 28%|████████████████████████████████▎ | 6583/23458 [1:51:27<3:58:25, 1.18it/s][2025-04-25 08:45:26] (step=0053500) Train Loss: 5.2505, Train Steps/Sec: 1.11 + 28%|████████████████████████████████▍ | 6608/23458 [1:51:49<4:56:33, 1.06s/it][2025-04-25 08:45:49] (step=0053525) Train Loss: 5.2966, Train Steps/Sec: 1.12 + 28%|████████████████████████████████▌ | 6633/23458 [1:52:11<3:59:58, 1.17it/s][2025-04-25 08:46:10] (step=0053550) Train Loss: 5.2848, Train Steps/Sec: 1.17 + 28%|████████████████████████████████▋ | 6658/23458 [1:52:33<4:57:16, 1.06s/it][2025-04-25 08:46:32] (step=0053575) Train Loss: 5.2486, Train Steps/Sec: 1.12 + 28%|████████████████████████████████▊ | 6683/23458 [1:52:55<3:55:59, 1.18it/s][2025-04-25 08:46:54] (step=0053600) Train Loss: 5.2235, Train Steps/Sec: 1.16 + 29%|████████████████████████████████▉ | 6708/23458 [1:53:16<4:02:44, 1.15it/s][2025-04-25 08:47:15] (step=0053625) Train Loss: 5.2916, Train Steps/Sec: 1.17 + 29%|█████████████████████████████████ | 6733/23458 [1:53:37<4:00:07, 1.16it/s][2025-04-25 08:47:36] (step=0053650) Train Loss: 5.3147, Train Steps/Sec: 1.17 + 29%|█████████████████████████████████▏ | 6758/23458 [1:53:59<3:56:34, 1.18it/s][2025-04-25 08:47:58] (step=0053675) Train Loss: 5.2024, Train Steps/Sec: 1.17 + 29%|█████████████████████████████████▎ | 6783/23458 [1:54:20<3:55:05, 1.18it/s][2025-04-25 08:48:19] (step=0053700) Train Loss: 5.3005, Train Steps/Sec: 1.17 + 29%|█████████████████████████████████▍ | 6808/23458 [1:54:43<4:01:59, 1.15it/s][2025-04-25 08:48:42] (step=0053725) Train Loss: 5.3213, Train Steps/Sec: 1.12 + 29%|█████████████████████████████████▍ | 6833/23458 [1:55:04<3:57:49, 1.17it/s][2025-04-25 08:49:03] (step=0053750) Train Loss: 5.2576, Train Steps/Sec: 1.17 + 29%|█████████████████████████████████▌ | 6858/23458 [1:55:26<4:40:59, 1.02s/it][2025-04-25 08:49:25] (step=0053775) Train Loss: 5.3459, Train Steps/Sec: 1.13 + 29%|█████████████████████████████████▋ | 6883/23458 [1:55:47<3:54:17, 1.18it/s][2025-04-25 08:49:47] (step=0053800) Train Loss: 5.2487, Train Steps/Sec: 1.12 + 29%|█████████████████████████████████▊ | 6908/23458 [1:56:10<3:59:40, 1.15it/s][2025-04-25 08:50:09] (step=0053825) Train Loss: 5.2200, Train Steps/Sec: 1.17 + 30%|█████████████████████████████████▉ | 6933/23458 [1:56:31<3:55:44, 1.17it/s][2025-04-25 08:50:30] (step=0053850) Train Loss: 5.3440, Train Steps/Sec: 1.17 + 30%|██████████████████████████████████ | 6958/23458 [1:56:54<3:54:53, 1.17it/s][2025-04-25 08:50:53] (step=0053875) Train Loss: 5.3593, Train Steps/Sec: 1.09 + 30%|██████████████████████████████████▏ | 6983/23458 [1:57:15<3:51:47, 1.18it/s][2025-04-25 08:51:15] (step=0053900) Train Loss: 5.2815, Train Steps/Sec: 1.17 + 30%|██████████████████████████████████▎ | 7008/23458 [1:57:37<3:59:10, 1.15it/s][2025-04-25 08:51:36] (step=0053925) Train Loss: 5.3861, Train Steps/Sec: 1.17 + 30%|██████████████████████████████████▍ | 7033/23458 [1:57:58<3:56:14, 1.16it/s][2025-04-25 08:51:57] (step=0053950) Train Loss: 5.2165, Train Steps/Sec: 1.17 + 30%|██████████████████████████████████▌ | 7058/23458 [1:58:20<3:52:59, 1.17it/s][2025-04-25 08:52:19] (step=0053975) Train Loss: 5.2800, Train Steps/Sec: 1.16 + 30%|██████████████████████████████████▋ | 7083/23458 [1:58:41<3:53:02, 1.17it/s][2025-04-25 08:52:41] (step=0054000) Train Loss: 5.2128, Train Steps/Sec: 1.16 +You are using a model of type blip-2 to instantiate a model of type instructblip. This is not supported for all configurations of models and can yield errors. +[2025-04-25 08:52:41] text_config is None. Initializing the text config with default values (`OPTConfig`). +model_name_or_path:/tmp/haozhezhao/model/blip2-flan-t5-xl +image_place_holder: +tokenizer length after expend 32101 +tokenizer length before expend 32100 +100%|██████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████| 4/4 [03:57<00:00, 59.34s/it] +[2025-04-25 08:57:46] Finish Eval in 54000 steps...███████████████████████████████████████████████████████████████████████| 4/4 [03:56<00:00, 59.01s/it] +processing left-padding... +processing left-padding... +processing left-padding... +processing left-padding... +[2025-04-25 08:58:05] Saved checkpoint to checkpoint/Afteri2i_100kckpt_LLAVAT5_empty_Fixed_just_segment_pretrain_stage3_dreambench_recap_400k_t2i_400k_flux_200k_midjourney_150k_recovery_150k_3e4_epoch2_no_rotated/002-GPT-XL/checkpoints/0054000.pt +[2025-04-25 08:58:08] Removed old checkpoint: checkpoint/Afteri2i_100kckpt_LLAVAT5_empty_Fixed_just_segment_pretrain_stage3_dreambench_recap_400k_t2i_400k_flux_200k_midjourney_150k_recovery_150k_3e4_epoch2_no_rotated/002-GPT-XL/checkpoints/0052000.pt + 30%|██████████████████████████████████▊ | 7108/23458 [2:04:30<4:01:13, 1.13it/s][2025-04-25 08:58:29] (step=0054025) Train Loss: 5.3777, Train Steps/Sec: 0.07 + 30%|██████████████████████████████████▉ | 7133/23458 [2:04:52<3:54:37, 1.16it/s][2025-04-25 08:58:51] (step=0054050) Train Loss: 5.4221, Train Steps/Sec: 1.16 + 31%|███████████████████████████████████ | 7158/23458 [2:05:13<3:52:22, 1.17it/s][2025-04-25 08:59:12] (step=0054075) Train Loss: 5.2302, Train Steps/Sec: 1.16 + 31%|███████████████████████████████████▏ | 7183/23458 [2:05:35<3:50:19, 1.18it/s][2025-04-25 08:59:34] (step=0054100) Train Loss: 5.2255, Train Steps/Sec: 1.16 + 31%|███████████████████████████████████▎ | 7208/23458 [2:05:56<3:56:25, 1.15it/s][2025-04-25 08:59:55] (step=0054125) Train Loss: 5.2972, Train Steps/Sec: 1.17 + 31%|███████████████████████████████████▍ | 7233/23458 [2:06:18<3:51:26, 1.17it/s][2025-04-25 09:00:17] (step=0054150) Train Loss: 5.2248, Train Steps/Sec: 1.17 + 31%|███████████████████████████████████▌ | 7258/23458 [2:06:39<3:49:14, 1.18it/s][2025-04-25 09:00:38] (step=0054175) Train Loss: 5.3726, Train Steps/Sec: 1.17 + 31%|███████████████████████████████████▋ | 7283/23458 [2:07:00<3:48:20, 1.18it/s][2025-04-25 09:00:59] (step=0054200) Train Loss: 5.2850, Train Steps/Sec: 1.16 + 31%|███████████████████████████████████▊ | 7308/23458 [2:07:22<3:54:21, 1.15it/s][2025-04-25 09:01:21] (step=0054225) Train Loss: 5.2925, Train Steps/Sec: 1.17 + 31%|███████████████████████████████████▉ | 7333/23458 [2:07:44<5:16:21, 1.18s/it][2025-04-25 09:01:43] (step=0054250) Train Loss: 5.3419, Train Steps/Sec: 1.10 + 31%|████████████████████████████████████ | 7358/23458 [2:08:06<3:51:20, 1.16it/s][2025-04-25 09:02:05] (step=0054275) Train Loss: 5.2194, Train Steps/Sec: 1.15 + 31%|████████████████████████████████████▏ | 7383/23458 [2:08:28<3:48:57, 1.17it/s][2025-04-25 09:02:27] (step=0054300) Train Loss: 5.3211, Train Steps/Sec: 1.15 + 32%|████████████████████████████████████▎ | 7408/23458 [2:08:51<3:54:36, 1.14it/s][2025-04-25 09:02:50] (step=0054325) Train Loss: 5.3794, Train Steps/Sec: 1.08 + 32%|████████████████████████████████████▍ | 7433/23458 [2:09:12<3:50:57, 1.16it/s][2025-04-25 09:03:12] (step=0054350) Train Loss: 5.2115, Train Steps/Sec: 1.16 + 32%|████████████████████████████████████▌ | 7458/23458 [2:09:35<3:49:29, 1.16it/s][2025-04-25 09:03:34] (step=0054375) Train Loss: 5.1621, Train Steps/Sec: 1.11 + 32%|████████████████████████████████████▋ | 7483/23458 [2:09:56<3:45:09, 1.18it/s][2025-04-25 09:03:56] (step=0054400) Train Loss: 5.3115, Train Steps/Sec: 1.16 + 32%|████████████████████████████████████▊ | 7508/23458 [2:10:18<3:54:39, 1.13it/s][2025-04-25 09:04:17] (step=0054425) Train Loss: 5.2375, Train Steps/Sec: 1.16 + 32%|████████████████████████████████████▉ | 7533/23458 [2:10:40<3:48:07, 1.16it/s][2025-04-25 09:04:39] (step=0054450) Train Loss: 5.2847, Train Steps/Sec: 1.12 + 32%|█████████████████████████████████████ | 7558/23458 [2:11:03<3:47:50, 1.16it/s][2025-04-25 09:05:02] (step=0054475) Train Loss: 5.2918, Train Steps/Sec: 1.12 + 32%|█████████████████████████████████████▏ | 7583/23458 [2:11:24<3:43:37, 1.18it/s][2025-04-25 09:05:23] (step=0054500) Train Loss: 5.3568, Train Steps/Sec: 1.16 + 32%|█████████████████████████████████████▎ | 7608/23458 [2:11:47<4:35:56, 1.04s/it][2025-04-25 09:05:46] (step=0054525) Train Loss: 5.3004, Train Steps/Sec: 1.09 + 33%|█████████████████████████████████████▍ | 7633/23458 [2:12:09<3:47:02, 1.16it/s][2025-04-25 09:06:08] (step=0054550) Train Loss: 5.2616, Train Steps/Sec: 1.17 + 33%|█████████████████████████████████████▌ | 7658/23458 [2:12:30<3:43:43, 1.18it/s][2025-04-25 09:06:29] (step=0054575) Train Loss: 5.3386, Train Steps/Sec: 1.17 + 33%|█████████████████████████████████████▋ | 7683/23458 [2:12:51<3:42:27, 1.18it/s][2025-04-25 09:06:51] (step=0054600) Train Loss: 5.2800, Train Steps/Sec: 1.17 + 33%|█████████████████████████████████████▊ | 7708/23458 [2:13:13<3:48:03, 1.15it/s][2025-04-25 09:07:12] (step=0054625) Train Loss: 5.3447, Train Steps/Sec: 1.17 + 33%|█████████████████████████████████████▉ | 7733/23458 [2:13:34<3:44:19, 1.17it/s][2025-04-25 09:07:33] (step=0054650) Train Loss: 5.2766, Train Steps/Sec: 1.17 + 33%|██████████████████████████████████████ | 7758/23458 [2:13:56<3:41:48, 1.18it/s][2025-04-25 09:07:55] (step=0054675) Train Loss: 5.2347, Train Steps/Sec: 1.17 + 33%|██████████████████████████████████████▏ | 7783/23458 [2:14:17<3:42:11, 1.18it/s][2025-04-25 09:08:16] (step=0054700) Train Loss: 5.2645, Train Steps/Sec: 1.16 + 33%|██████████████████████████████████████▎ | 7808/23458 [2:14:38<3:46:11, 1.15it/s][2025-04-25 09:08:37] (step=0054725) Train Loss: 5.3187, Train Steps/Sec: 1.17 + 33%|██████████████████████████████████████▍ | 7833/23458 [2:15:01<3:43:38, 1.16it/s][2025-04-25 09:09:00] (step=0054750) Train Loss: 5.3027, Train Steps/Sec: 1.12 + 33%|██████████████████████████████████████▌ | 7858/23458 [2:15:22<3:41:22, 1.17it/s][2025-04-25 09:09:21] (step=0054775) Train Loss: 5.2667, Train Steps/Sec: 1.17 + 34%|██████████████████████████████████████▋ | 7883/23458 [2:15:43<3:38:54, 1.19it/s][2025-04-25 09:09:43] (step=0054800) Train Loss: 5.2595, Train Steps/Sec: 1.17 + 34%|██████████████████████████████████████▊ | 7908/23458 [2:16:05<3:45:24, 1.15it/s][2025-04-25 09:10:04] (step=0054825) Train Loss: 5.3338, Train Steps/Sec: 1.17 + 34%|██████████████████████████████████████▉ | 7933/23458 [2:16:26<3:44:05, 1.15it/s][2025-04-25 09:10:25] (step=0054850) Train Loss: 5.2845, Train Steps/Sec: 1.16 + 34%|███████████████████████████████████████ | 7958/23458 [2:16:48<3:39:53, 1.17it/s][2025-04-25 09:10:47] (step=0054875) Train Loss: 5.2682, Train Steps/Sec: 1.16 + 34%|███████████████████████████████████████▏ | 7983/23458 [2:17:09<3:37:11, 1.19it/s][2025-04-25 09:11:08] (step=0054900) Train Loss: 5.2875, Train Steps/Sec: 1.17 + 34%|███████████████████████████████████████▎ | 8008/23458 [2:17:31<3:44:35, 1.15it/s][2025-04-25 09:11:30] (step=0054925) Train Loss: 5.3312, Train Steps/Sec: 1.17 + 34%|███████████████████████████████████████▍ | 8033/23458 [2:17:52<3:41:10, 1.16it/s][2025-04-25 09:11:51] (step=0054950) Train Loss: 5.2811, Train Steps/Sec: 1.17 + 34%|███████████████████████████████████████▌ | 8058/23458 [2:18:14<4:09:34, 1.03it/s][2025-04-25 09:12:13] (step=0054975) Train Loss: 5.3055, Train Steps/Sec: 1.12 + 34%|███████████████████████████████████████▋ | 8083/23458 [2:18:36<3:35:33, 1.19it/s][2025-04-25 09:12:35] (step=0055000) Train Loss: 5.2660, Train Steps/Sec: 1.17 + 35%|███████████████████████████████████████▋ | 8108/23458 [2:18:58<3:43:18, 1.15it/s][2025-04-25 09:12:57] (step=0055025) Train Loss: 5.2717, Train Steps/Sec: 1.13 + 35%|███████████████████████████████████████▊ | 8133/23458 [2:19:20<3:38:30, 1.17it/s][2025-04-25 09:13:19] (step=0055050) Train Loss: 5.2567, Train Steps/Sec: 1.13 + 35%|███████████████████████████████████████▉ | 8158/23458 [2:19:42<3:36:31, 1.18it/s][2025-04-25 09:13:41] (step=0055075) Train Loss: 5.2771, Train Steps/Sec: 1.17 + 35%|████████████████████████████████████████ | 8183/23458 [2:20:04<3:34:50, 1.18it/s][2025-04-25 09:14:03] (step=0055100) Train Loss: 5.3122, Train Steps/Sec: 1.12 + 35%|████████████████████████████████████████▏ | 8208/23458 [2:20:26<3:42:36, 1.14it/s][2025-04-25 09:14:25] (step=0055125) Train Loss: 5.2791, Train Steps/Sec: 1.13 + 35%|████████████████████████████████████████▎ | 8233/23458 [2:20:47<3:38:26, 1.16it/s][2025-04-25 09:14:47] (step=0055150) Train Loss: 5.3104, Train Steps/Sec: 1.17 + 35%|████████████████████████████████████████▍ | 8258/23458 [2:21:10<3:35:38, 1.17it/s][2025-04-25 09:15:09] (step=0055175) Train Loss: 5.2952, Train Steps/Sec: 1.13 + 35%|████████████████████████████████████████▌ | 8283/23458 [2:21:32<3:33:11, 1.19it/s][2025-04-25 09:15:31] (step=0055200) Train Loss: 5.2603, Train Steps/Sec: 1.12 + 35%|████████████████████████████████████████▋ | 8308/23458 [2:21:53<3:39:25, 1.15it/s][2025-04-25 09:15:52] (step=0055225) Train Loss: 5.3251, Train Steps/Sec: 1.17 + 36%|████████████████████████████████████████▊ | 8333/23458 [2:22:15<3:36:38, 1.16it/s][2025-04-25 09:16:14] (step=0055250) Train Loss: 5.2390, Train Steps/Sec: 1.17 + 36%|████████████████████████████████████████▉ | 8358/23458 [2:22:36<3:34:04, 1.18it/s][2025-04-25 09:16:35] (step=0055275) Train Loss: 5.2610, Train Steps/Sec: 1.17 + 36%|█████████████████████████████████████████ | 8383/23458 [2:22:57<3:32:17, 1.18it/s][2025-04-25 09:16:57] (step=0055300) Train Loss: 5.3269, Train Steps/Sec: 1.16 + 36%|█████████████████████████████████████████▏ | 8408/23458 [2:23:19<3:38:43, 1.15it/s][2025-04-25 09:17:18] (step=0055325) Train Loss: 5.2442, Train Steps/Sec: 1.17 + 36%|█████████████████████████████████████████▎ | 8433/23458 [2:23:40<3:34:44, 1.17it/s][2025-04-25 09:17:39] (step=0055350) Train Loss: 5.2754, Train Steps/Sec: 1.17 + 36%|█████████████████████████████████████████▍ | 8458/23458 [2:24:02<3:31:39, 1.18it/s][2025-04-25 09:18:01] (step=0055375) Train Loss: 5.2155, Train Steps/Sec: 1.17 + 36%|█████████████████████████████████████████▌ | 8483/23458 [2:24:23<3:31:14, 1.18it/s][2025-04-25 09:18:22] (step=0055400) Train Loss: 5.2695, Train Steps/Sec: 1.17 + 36%|█████████████████████████████████████████▋ | 8508/23458 [2:24:44<3:35:53, 1.15it/s][2025-04-25 09:18:43] (step=0055425) Train Loss: 5.3190, Train Steps/Sec: 1.17 + 36%|█████████████████████████████████████████▊ | 8533/23458 [2:25:06<3:32:48, 1.17it/s][2025-04-25 09:19:05] (step=0055450) Train Loss: 5.2577, Train Steps/Sec: 1.17 + 36%|█████████████████████████████████████████▉ | 8558/23458 [2:25:27<3:31:09, 1.18it/s][2025-04-25 09:19:26] (step=0055475) Train Loss: 5.3184, Train Steps/Sec: 1.17 + 37%|██████████████████████████████████████████ | 8583/23458 [2:25:49<3:32:05, 1.17it/s][2025-04-25 09:19:49] (step=0055500) Train Loss: 5.3463, Train Steps/Sec: 1.12 + 37%|██████████████████████████████████████████▏ | 8608/23458 [2:26:11<3:35:35, 1.15it/s][2025-04-25 09:20:10] (step=0055525) Train Loss: 5.2282, Train Steps/Sec: 1.17 + 37%|██████████████████████████████████████████▎ | 8633/23458 [2:26:32<3:31:12, 1.17it/s][2025-04-25 09:20:31] (step=0055550) Train Loss: 5.2771, Train Steps/Sec: 1.17 + 37%|██████████████████████████████████████████▍ | 8658/23458 [2:26:54<3:30:27, 1.17it/s][2025-04-25 09:20:53] (step=0055575) Train Loss: 5.3466, Train Steps/Sec: 1.17 + 37%|██████████████████████████████████████████▌ | 8683/23458 [2:27:15<3:28:36, 1.18it/s][2025-04-25 09:21:14] (step=0055600) Train Loss: 5.3142, Train Steps/Sec: 1.16 + 37%|██████████████████████████████████████████▋ | 8708/23458 [2:27:37<3:34:20, 1.15it/s][2025-04-25 09:21:36] (step=0055625) Train Loss: 5.3022, Train Steps/Sec: 1.17 + 37%|██████████████████████████████████████████▊ | 8733/23458 [2:27:58<3:29:58, 1.17it/s][2025-04-25 09:21:57] (step=0055650) Train Loss: 5.2214, Train Steps/Sec: 1.17 + 37%|██████████████████████████████████████████▉ | 8758/23458 [2:28:20<3:29:48, 1.17it/s][2025-04-25 09:22:19] (step=0055675) Train Loss: 5.2672, Train Steps/Sec: 1.12 + 37%|███████████████████████████████████████████ | 8783/23458 [2:28:42<4:32:21, 1.11s/it][2025-04-25 09:22:42] (step=0055700) Train Loss: 5.3390, Train Steps/Sec: 1.12 + 38%|███████████████████████████████████████████▏ | 8808/23458 [2:29:04<3:32:08, 1.15it/s][2025-04-25 09:23:03] (step=0055725) Train Loss: 5.1906, Train Steps/Sec: 1.17 + 38%|███████████████████████████████████████████▎ | 8833/23458 [2:29:26<3:28:53, 1.17it/s][2025-04-25 09:23:25] (step=0055750) Train Loss: 5.2746, Train Steps/Sec: 1.12 + 38%|███████████████████████████████████████████▍ | 8858/23458 [2:29:49<3:38:36, 1.11it/s][2025-04-25 09:23:48] (step=0055775) Train Loss: 5.3198, Train Steps/Sec: 1.08 + 38%|███████████████████████████████████████████▌ | 8883/23458 [2:30:11<3:25:00, 1.18it/s][2025-04-25 09:24:10] (step=0055800) Train Loss: 5.1801, Train Steps/Sec: 1.17 + 38%|███████████████████████████████████████████▋ | 8908/23458 [2:30:33<3:35:28, 1.13it/s][2025-04-25 09:24:32] (step=0055825) Train Loss: 5.1296, Train Steps/Sec: 1.13 + 38%|███████████████████████████████████████████▊ | 8933/23458 [2:30:55<3:31:56, 1.14it/s][2025-04-25 09:24:54] (step=0055850) Train Loss: 5.2243, Train Steps/Sec: 1.12 + 38%|███████████████████████████████████████████▉ | 8958/23458 [2:31:17<3:26:07, 1.17it/s][2025-04-25 09:25:16] (step=0055875) Train Loss: 5.3029, Train Steps/Sec: 1.17 + 38%|████████████████████████████████████████████ | 8983/23458 [2:31:38<3:24:16, 1.18it/s][2025-04-25 09:25:37] (step=0055900) Train Loss: 5.2990, Train Steps/Sec: 1.16 + 38%|████████████████████████████████████████████▏ | 9008/23458 [2:31:59<3:29:13, 1.15it/s][2025-04-25 09:25:58] (step=0055925) Train Loss: 5.1814, Train Steps/Sec: 1.17 + 39%|████████████████████████████████████████████▎ | 9033/23458 [2:32:21<3:26:35, 1.16it/s][2025-04-25 09:26:20] (step=0055950) Train Loss: 5.3687, Train Steps/Sec: 1.17 + 39%|████████████████████████████████████████████▍ | 9058/23458 [2:32:42<3:24:31, 1.17it/s][2025-04-25 09:26:41] (step=0055975) Train Loss: 5.3579, Train Steps/Sec: 1.17 + 39%|████████████████████████████████████████████▌ | 9083/23458 [2:33:04<3:22:36, 1.18it/s][2025-04-25 09:27:03] (step=0056000) Train Loss: 5.3560, Train Steps/Sec: 1.16 +You are using a model of type blip-2 to instantiate a model of type instructblip. This is not supported for all configurations of models and can yield errors. +[2025-04-25 09:27:03] text_config is None. Initializing the text config with default values (`OPTConfig`). +model_name_or_path:/tmp/haozhezhao/model/blip2-flan-t5-xl +image_place_holder: +tokenizer length after expend 32101 +tokenizer length before expend 32100 +100%|██████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████| 4/4 [03:58<00:00, 59.51s/it] +[2025-04-25 09:32:09] Finish Eval in 56000 steps...███████████████████████████████████████████████████████████████████████| 4/4 [03:57<00:00, 59.02s/it] +processing left-padding... +processing left-padding... +processing left-padding... +processing left-padding... +[2025-04-25 09:32:29] Saved checkpoint to checkpoint/Afteri2i_100kckpt_LLAVAT5_empty_Fixed_just_segment_pretrain_stage3_dreambench_recap_400k_t2i_400k_flux_200k_midjourney_150k_recovery_150k_3e4_epoch2_no_rotated/002-GPT-XL/checkpoints/0056000.pt +[2025-04-25 09:32:31] Removed old checkpoint: checkpoint/Afteri2i_100kckpt_LLAVAT5_empty_Fixed_just_segment_pretrain_stage3_dreambench_recap_400k_t2i_400k_flux_200k_midjourney_150k_recovery_150k_3e4_epoch2_no_rotated/002-GPT-XL/checkpoints/0054000.pt + 39%|████████████████████████████████████████████▋ | 9108/23458 [2:38:54<3:32:08, 1.13it/s][2025-04-25 09:32:53] (step=0056025) Train Loss: 5.3169, Train Steps/Sec: 0.07 + 39%|████████████████████████████████████████████▊ | 9133/23458 [2:39:15<3:24:35, 1.17it/s][2025-04-25 09:33:14] (step=0056050) Train Loss: 5.2756, Train Steps/Sec: 1.17 + 39%|████████████████████████████████████████████▉ | 9158/23458 [2:39:36<3:23:07, 1.17it/s][2025-04-25 09:33:35] (step=0056075) Train Loss: 5.3036, Train Steps/Sec: 1.17 + 39%|█████████████████████████████████████████████ | 9183/23458 [2:39:58<3:21:16, 1.18it/s][2025-04-25 09:33:57] (step=0056100) Train Loss: 5.3553, Train Steps/Sec: 1.16 + 39%|█████████████████████████████████████████████▏ | 9208/23458 [2:40:19<3:27:10, 1.15it/s][2025-04-25 09:34:18] (step=0056125) Train Loss: 5.3022, Train Steps/Sec: 1.17 + 39%|█████████████████████████████████████████████▎ | 9233/23458 [2:40:40<3:22:26, 1.17it/s][2025-04-25 09:34:39] (step=0056150) Train Loss: 5.3276, Train Steps/Sec: 1.17 + 39%|█████████████████████████████████████████████▍ | 9258/23458 [2:41:02<3:27:37, 1.14it/s][2025-04-25 09:35:01] (step=0056175) Train Loss: 5.2486, Train Steps/Sec: 1.15 + 40%|█████████████████████████████████████████████▌ | 9283/23458 [2:41:23<3:19:28, 1.18it/s][2025-04-25 09:35:23] (step=0056200) Train Loss: 5.3283, Train Steps/Sec: 1.17 + 40%|█████████████████████████████████████████████▋ | 9308/23458 [2:41:45<3:25:12, 1.15it/s][2025-04-25 09:35:44] (step=0056225) Train Loss: 5.2839, Train Steps/Sec: 1.17 + 40%|█████████████████████████████████████████████▊ | 9333/23458 [2:42:06<3:21:34, 1.17it/s][2025-04-25 09:36:05] (step=0056250) Train Loss: 5.2823, Train Steps/Sec: 1.17 + 40%|█████████████████████████████████████████████▉ | 9358/23458 [2:42:28<3:19:25, 1.18it/s][2025-04-25 09:36:27] (step=0056275) Train Loss: 5.2822, Train Steps/Sec: 1.17 + 40%|█████████████████████████████████████████████▉ | 9383/23458 [2:42:49<3:18:11, 1.18it/s][2025-04-25 09:36:48] (step=0056300) Train Loss: 5.2985, Train Steps/Sec: 1.16 + 40%|██████████████████████████████████████████████ | 9408/23458 [2:43:11<3:25:30, 1.14it/s][2025-04-25 09:37:10] (step=0056325) Train Loss: 5.3006, Train Steps/Sec: 1.13 + 40%|██████████████████████████████████████████████▏ | 9433/23458 [2:43:33<3:18:57, 1.17it/s][2025-04-25 09:37:32] (step=0056350) Train Loss: 5.3926, Train Steps/Sec: 1.17 + 40%|██████████████████████████████████████████████▎ | 9458/23458 [2:43:54<3:17:34, 1.18it/s][2025-04-25 09:37:53] (step=0056375) Train Loss: 5.3270, Train Steps/Sec: 1.17 + 40%|██████████████████████████████████████████████▍ | 9483/23458 [2:44:16<3:17:02, 1.18it/s][2025-04-25 09:38:15] (step=0056400) Train Loss: 5.2389, Train Steps/Sec: 1.12 + 41%|██████████████████████████████████████████████▌ | 9508/23458 [2:44:39<3:23:08, 1.14it/s][2025-04-25 09:38:38] (step=0056425) Train Loss: 5.2750, Train Steps/Sec: 1.12 + 41%|██████████████████████████████████████████████▋ | 9533/23458 [2:45:01<3:18:43, 1.17it/s][2025-04-25 09:39:00] (step=0056450) Train Loss: 5.3009, Train Steps/Sec: 1.12 + 41%|██████████████████████████████████████████████▊ | 9558/23458 [2:45:23<3:29:16, 1.11it/s][2025-04-25 09:39:22] (step=0056475) Train Loss: 5.1975, Train Steps/Sec: 1.13 + 41%|██████████████████████████████████████████████▉ | 9583/23458 [2:45:44<3:15:55, 1.18it/s][2025-04-25 09:39:44] (step=0056500) Train Loss: 5.2619, Train Steps/Sec: 1.16 + 41%|███████████████████████████████████████████████ | 9608/23458 [2:46:08<3:20:49, 1.15it/s][2025-04-25 09:40:07] (step=0056525) Train Loss: 5.2976, Train Steps/Sec: 1.12 + 41%|███████████████████████████████████████████████▏ | 9633/23458 [2:46:29<3:17:13, 1.17it/s][2025-04-25 09:40:28] (step=0056550) Train Loss: 5.2536, Train Steps/Sec: 1.17 + 41%|███████████████████████████████████████████████▎ | 9658/23458 [2:46:50<3:15:34, 1.18it/s][2025-04-25 09:40:50] (step=0056575) Train Loss: 5.2491, Train Steps/Sec: 1.17 + 41%|███████████████████████████████████████████████▍ | 9683/23458 [2:47:12<3:13:44, 1.18it/s][2025-04-25 09:41:11] (step=0056600) Train Loss: 5.3557, Train Steps/Sec: 1.16 + 41%|███████████████████████████████████████████████▌ | 9708/23458 [2:47:33<3:19:00, 1.15it/s][2025-04-25 09:41:32] (step=0056625) Train Loss: 5.3269, Train Steps/Sec: 1.17 + 41%|███████████████████████████████████████████████▋ | 9733/23458 [2:47:55<3:16:11, 1.17it/s][2025-04-25 09:41:54] (step=0056650) Train Loss: 5.3061, Train Steps/Sec: 1.17 + 42%|███████████████████████████████████████████████▊ | 9758/23458 [2:48:16<3:13:41, 1.18it/s][2025-04-25 09:42:15] (step=0056675) Train Loss: 5.2498, Train Steps/Sec: 1.17 + 42%|███████████████████████████████████████████████▉ | 9783/23458 [2:48:37<3:11:47, 1.19it/s][2025-04-25 09:42:37] (step=0056700) Train Loss: 5.2482, Train Steps/Sec: 1.17 + 42%|████████████████████████████████████████████████ | 9808/23458 [2:48:59<3:18:24, 1.15it/s][2025-04-25 09:42:58] (step=0056725) Train Loss: 5.2492, Train Steps/Sec: 1.17 + 42%|████████████████████████████████████████████████▏ | 9833/23458 [2:49:21<3:14:58, 1.16it/s][2025-04-25 09:43:20] (step=0056750) Train Loss: 5.1916, Train Steps/Sec: 1.12 + 42%|████████████████████████████████████████████████▎ | 9858/23458 [2:49:43<3:13:16, 1.17it/s][2025-04-25 09:43:42] (step=0056775) Train Loss: 5.3029, Train Steps/Sec: 1.17 + 42%|████████████████████████████████████████████████▍ | 9883/23458 [2:50:04<3:10:43, 1.19it/s][2025-04-25 09:44:03] (step=0056800) Train Loss: 5.3020, Train Steps/Sec: 1.17 + 42%|████████████████████████████████████████████████▌ | 9908/23458 [2:50:25<3:17:23, 1.14it/s][2025-04-25 09:44:25] (step=0056825) Train Loss: 5.1958, Train Steps/Sec: 1.17 + 42%|████████████████████████████████████████████████▋ | 9933/23458 [2:50:47<3:12:56, 1.17it/s][2025-04-25 09:44:46] (step=0056850) Train Loss: 5.2183, Train Steps/Sec: 1.17 + 42%|████████████████████████████████████████████████▊ | 9958/23458 [2:51:08<3:12:46, 1.17it/s][2025-04-25 09:45:07] (step=0056875) Train Loss: 5.3026, Train Steps/Sec: 1.17 + 43%|████████████████████████████████████████████████▉ | 9983/23458 [2:51:30<3:10:09, 1.18it/s][2025-04-25 09:45:29] (step=0056900) Train Loss: 5.3129, Train Steps/Sec: 1.16 + 43%|████████████████████████████████████████████████▋ | 10008/23458 [2:51:51<3:14:38, 1.15it/s][2025-04-25 09:45:50] (step=0056925) Train Loss: 5.3192, Train Steps/Sec: 1.17 + 43%|████████████████████████████████████████████████▊ | 10033/23458 [2:52:12<3:12:40, 1.16it/s][2025-04-25 09:46:12] (step=0056950) Train Loss: 5.2815, Train Steps/Sec: 1.17 + 43%|████████████████████████████████████████████████▉ | 10058/23458 [2:52:35<3:10:28, 1.17it/s][2025-04-25 09:46:34] (step=0056975) Train Loss: 5.3201, Train Steps/Sec: 1.12 + 43%|█████████████████████████████████████████████████ | 10083/23458 [2:52:56<3:08:33, 1.18it/s][2025-04-25 09:46:55] (step=0057000) Train Loss: 5.2743, Train Steps/Sec: 1.17 + 43%|█████████████████████████████████████████████████ | 10108/23458 [2:53:18<3:12:53, 1.15it/s][2025-04-25 09:47:17] (step=0057025) Train Loss: 5.2622, Train Steps/Sec: 1.17 + 43%|█████████████████████████████████████████████████▏ | 10133/23458 [2:53:40<3:10:52, 1.16it/s][2025-04-25 09:47:39] (step=0057050) Train Loss: 5.2619, Train Steps/Sec: 1.13 + 43%|█████████████████████████████████████████████████▎ | 10158/23458 [2:54:02<3:11:12, 1.16it/s][2025-04-25 09:48:01] (step=0057075) Train Loss: 5.2092, Train Steps/Sec: 1.12 + 43%|█████████████████████████████████████████████████▍ | 10183/23458 [2:54:23<3:06:58, 1.18it/s][2025-04-25 09:48:23] (step=0057100) Train Loss: 5.3599, Train Steps/Sec: 1.17 + 44%|█████████████████████████████████████████████████▌ | 10208/23458 [2:54:45<3:11:43, 1.15it/s][2025-04-25 09:48:44] (step=0057125) Train Loss: 5.2205, Train Steps/Sec: 1.17 + 44%|█████████████████████████████████████████████████▋ | 10233/23458 [2:55:07<3:08:38, 1.17it/s][2025-04-25 09:49:06] (step=0057150) Train Loss: 5.2800, Train Steps/Sec: 1.13 + 44%|█████████████████████████████████████████████████▊ | 10258/23458 [2:55:30<3:06:53, 1.18it/s][2025-04-25 09:49:29] (step=0057175) Train Loss: 5.2760, Train Steps/Sec: 1.08 + 44%|█████████████████████████████████████████████████▉ | 10283/23458 [2:55:51<3:05:21, 1.18it/s][2025-04-25 09:49:51] (step=0057200) Train Loss: 5.3249, Train Steps/Sec: 1.17 + 44%|██████████████████████████████████████████████████ | 10308/23458 [2:56:13<3:10:41, 1.15it/s][2025-04-25 09:50:12] (step=0057225) Train Loss: 5.3148, Train Steps/Sec: 1.17 + 44%|██████████████████████████████████████████████████▏ | 10333/23458 [2:56:35<3:08:30, 1.16it/s][2025-04-25 09:50:34] (step=0057250) Train Loss: 5.2735, Train Steps/Sec: 1.12 + 44%|██████████████████████████████████████████████████▎ | 10358/23458 [2:56:57<3:06:17, 1.17it/s][2025-04-25 09:50:56] (step=0057275) Train Loss: 5.3209, Train Steps/Sec: 1.17 + 44%|██████████████████████████████████████████████████▍ | 10383/23458 [2:57:18<3:03:48, 1.19it/s][2025-04-25 09:51:17] (step=0057300) Train Loss: 5.2391, Train Steps/Sec: 1.16 + 44%|██████████████████████████████████████████████████▌ | 10408/23458 [2:57:39<3:08:34, 1.15it/s][2025-04-25 09:51:38] (step=0057325) Train Loss: 5.3539, Train Steps/Sec: 1.17 + 44%|██████████████████████████████████████████████████▋ | 10433/23458 [2:58:01<3:06:22, 1.16it/s][2025-04-25 09:52:00] (step=0057350) Train Loss: 5.3386, Train Steps/Sec: 1.17 + 45%|██████████████████████████████████████████████████▊ | 10458/23458 [2:58:22<3:04:14, 1.18it/s][2025-04-25 09:52:21] (step=0057375) Train Loss: 5.3473, Train Steps/Sec: 1.17 + 45%|██████████████████████████████████████████████████▉ | 10483/23458 [2:58:43<3:03:00, 1.18it/s][2025-04-25 09:52:43] (step=0057400) Train Loss: 5.2929, Train Steps/Sec: 1.16 + 45%|███████████████████████████████████████████████████ | 10508/23458 [2:59:05<3:07:51, 1.15it/s][2025-04-25 09:53:04] (step=0057425) Train Loss: 5.3097, Train Steps/Sec: 1.17 + 45%|███████████████████████████████████████████████████▏ | 10533/23458 [2:59:26<3:04:16, 1.17it/s][2025-04-25 09:53:25] (step=0057450) Train Loss: 5.3929, Train Steps/Sec: 1.17 + 45%|███████████████████████████████████████████████████▎ | 10558/23458 [2:59:48<3:02:41, 1.18it/s][2025-04-25 09:53:47] (step=0057475) Train Loss: 5.2993, Train Steps/Sec: 1.17 + 45%|███████████████████████████████████████████████████▍ | 10583/23458 [3:00:10<3:01:14, 1.18it/s][2025-04-25 09:54:09] (step=0057500) Train Loss: 5.3005, Train Steps/Sec: 1.11 + 45%|███████████████████████████████████████████████████▌ | 10608/23458 [3:00:31<3:06:10, 1.15it/s][2025-04-25 09:54:31] (step=0057525) Train Loss: 5.3090, Train Steps/Sec: 1.17 + 45%|███████████████████████████████████████████████████▋ | 10633/23458 [3:00:53<3:02:46, 1.17it/s][2025-04-25 09:54:52] (step=0057550) Train Loss: 5.3536, Train Steps/Sec: 1.17 + 45%|███████████████████████████████████████████████████▊ | 10658/23458 [3:01:14<3:01:04, 1.18it/s][2025-04-25 09:55:13] (step=0057575) Train Loss: 5.2989, Train Steps/Sec: 1.17 + 46%|███████████████████████████████████████████████████▉ | 10683/23458 [3:01:36<3:00:16, 1.18it/s][2025-04-25 09:55:35] (step=0057600) Train Loss: 5.2439, Train Steps/Sec: 1.17 + 46%|████████████████████████████████████████████████████ | 10708/23458 [3:01:58<3:04:42, 1.15it/s][2025-04-25 09:55:57] (step=0057625) Train Loss: 5.2547, Train Steps/Sec: 1.13 + 46%|████████████████████████████████████████████████████▏ | 10733/23458 [3:02:19<3:01:30, 1.17it/s][2025-04-25 09:56:18] (step=0057650) Train Loss: 5.2678, Train Steps/Sec: 1.17 + 46%|████████████████████████████████████████████████████▎ | 10758/23458 [3:02:41<2:59:52, 1.18it/s][2025-04-25 09:56:40] (step=0057675) Train Loss: 5.3069, Train Steps/Sec: 1.16 + 46%|████████████████████████████████████████████████████▍ | 10783/23458 [3:03:03<3:00:27, 1.17it/s][2025-04-25 09:57:02] (step=0057700) Train Loss: 5.2753, Train Steps/Sec: 1.12 + 46%|████████████████████████████████████████████████████▌ | 10808/23458 [3:03:25<3:12:06, 1.10it/s][2025-04-25 09:57:24] (step=0057725) Train Loss: 5.2937, Train Steps/Sec: 1.12 + 46%|████████████████████████████████████████████████████▋ | 10833/23458 [3:03:47<3:00:26, 1.17it/s][2025-04-25 09:57:46] (step=0057750) Train Loss: 5.3537, Train Steps/Sec: 1.17 + 46%|████████████████████████████████████████████████████▊ | 10858/23458 [3:04:08<2:58:20, 1.18it/s][2025-04-25 09:58:07] (step=0057775) Train Loss: 5.2501, Train Steps/Sec: 1.17 + 46%|████████████████████████████████████████████████████▉ | 10883/23458 [3:04:30<2:57:21, 1.18it/s][2025-04-25 09:58:29] (step=0057800) Train Loss: 5.3242, Train Steps/Sec: 1.13 + 47%|█████████████████████████████████████████████████████ | 10908/23458 [3:04:53<3:20:58, 1.04it/s][2025-04-25 09:58:52] (step=0057825) Train Loss: 5.2653, Train Steps/Sec: 1.12 + 47%|█████████████████████████████████████████████████████▏ | 10933/23458 [3:05:14<3:01:05, 1.15it/s][2025-04-25 09:59:13] (step=0057850) Train Loss: 5.2615, Train Steps/Sec: 1.17 + 47%|█████████████████████████████████████████████████████▎ | 10958/23458 [3:05:35<2:56:47, 1.18it/s][2025-04-25 09:59:34] (step=0057875) Train Loss: 5.3389, Train Steps/Sec: 1.17 + 47%|█████████████████████████████████████████████████████▎ | 10983/23458 [3:05:58<2:55:52, 1.18it/s][2025-04-25 09:59:57] (step=0057900) Train Loss: 5.2761, Train Steps/Sec: 1.12 + 47%|█████████████████████████████████████████████████████▍ | 11008/23458 [3:06:19<3:00:27, 1.15it/s][2025-04-25 10:00:18] (step=0057925) Train Loss: 5.2504, Train Steps/Sec: 1.17 + 47%|█████████████████████████████████████████████████████▌ | 11033/23458 [3:06:40<2:57:41, 1.17it/s][2025-04-25 10:00:40] (step=0057950) Train Loss: 5.3483, Train Steps/Sec: 1.17 + 47%|█████████████████████████████████████████████████████▋ | 11058/23458 [3:07:03<2:56:45, 1.17it/s][2025-04-25 10:01:02] (step=0057975) Train Loss: 5.2758, Train Steps/Sec: 1.12 + 47%|█████████████████████████████████████████████████████▊ | 11083/23458 [3:07:24<2:54:37, 1.18it/s][2025-04-25 10:01:23] (step=0058000) Train Loss: 5.3085, Train Steps/Sec: 1.17 +You are using a model of type blip-2 to instantiate a model of type instructblip. This is not supported for all configurations of models and can yield errors. +[2025-04-25 10:01:23] text_config is None. Initializing the text config with default values (`OPTConfig`). +model_name_or_path:/tmp/haozhezhao/model/blip2-flan-t5-xl +image_place_holder: +tokenizer length after expend 32101 +tokenizer length before expend 32100 +100%|██████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████| 4/4 [03:58<00:00, 59.60s/it] +[2025-04-25 10:06:29] Finish Eval in 58000 steps...███████████████████████████████████████████████████████████████████████| 4/4 [03:57<00:00, 59.26s/it] +processing left-padding... +processing left-padding... +processing left-padding... +processing left-padding... +[2025-04-25 10:06:49] Saved checkpoint to checkpoint/Afteri2i_100kckpt_LLAVAT5_empty_Fixed_just_segment_pretrain_stage3_dreambench_recap_400k_t2i_400k_flux_200k_midjourney_150k_recovery_150k_3e4_epoch2_no_rotated/002-GPT-XL/checkpoints/0058000.pt +[2025-04-25 10:06:52] Removed old checkpoint: checkpoint/Afteri2i_100kckpt_LLAVAT5_empty_Fixed_just_segment_pretrain_stage3_dreambench_recap_400k_t2i_400k_flux_200k_midjourney_150k_recovery_150k_3e4_epoch2_no_rotated/002-GPT-XL/checkpoints/0056000.pt + 47%|█████████████████████████████████████████████████████▉ | 11108/23458 [3:13:14<3:02:05, 1.13it/s][2025-04-25 10:07:13] (step=0058025) Train Loss: 5.2773, Train Steps/Sec: 0.07 + 47%|██████████████████████████████████████████████████████ | 11133/23458 [3:13:35<2:56:02, 1.17it/s][2025-04-25 10:07:35] (step=0058050) Train Loss: 5.3203, Train Steps/Sec: 1.17 + 48%|██████████████████████████████████████████████████████▏ | 11158/23458 [3:13:57<2:54:12, 1.18it/s][2025-04-25 10:07:56] (step=0058075) Train Loss: 5.2922, Train Steps/Sec: 1.17 + 48%|██████████████████████████████████████████████████████▎ | 11183/23458 [3:14:18<2:56:58, 1.16it/s][2025-04-25 10:08:18] (step=0058100) Train Loss: 5.2621, Train Steps/Sec: 1.15 + 48%|██████████████████████████████████████████████████████▍ | 11208/23458 [3:14:40<2:58:13, 1.15it/s][2025-04-25 10:08:39] (step=0058125) Train Loss: 5.2763, Train Steps/Sec: 1.17 + 48%|██████████████████████████████████████████████████████▌ | 11233/23458 [3:15:01<2:54:01, 1.17it/s][2025-04-25 10:09:00] (step=0058150) Train Loss: 5.3120, Train Steps/Sec: 1.17 + 48%|██████████████████████████████████████████████████████▋ | 11258/23458 [3:15:23<2:53:02, 1.18it/s][2025-04-25 10:09:22] (step=0058175) Train Loss: 5.2789, Train Steps/Sec: 1.17 + 48%|██████████████████████████████████████████████████████▊ | 11283/23458 [3:15:44<2:51:30, 1.18it/s][2025-04-25 10:09:43] (step=0058200) Train Loss: 5.2849, Train Steps/Sec: 1.16 + 48%|██████████████████████████████████████████████████████▉ | 11308/23458 [3:16:05<2:55:58, 1.15it/s][2025-04-25 10:10:05] (step=0058225) Train Loss: 5.2859, Train Steps/Sec: 1.17 + 48%|███████████████████████████████████████████████████████ | 11333/23458 [3:16:27<2:53:22, 1.17it/s][2025-04-25 10:10:26] (step=0058250) Train Loss: 5.2359, Train Steps/Sec: 1.17 + 48%|███████████████████████████████████████████████████████▏ | 11358/23458 [3:16:49<2:52:08, 1.17it/s][2025-04-25 10:10:48] (step=0058275) Train Loss: 5.2938, Train Steps/Sec: 1.12 + 49%|███████████████████████████████████████████████████████▎ | 11383/23458 [3:17:10<2:49:48, 1.19it/s][2025-04-25 10:11:10] (step=0058300) Train Loss: 5.3189, Train Steps/Sec: 1.17 + 49%|███████████████████████████████████████████████████████▍ | 11408/23458 [3:17:32<2:54:58, 1.15it/s][2025-04-25 10:11:31] (step=0058325) Train Loss: 5.3662, Train Steps/Sec: 1.17 + 49%|███████████████████████████████████████████████████████▌ | 11433/23458 [3:17:54<3:00:43, 1.11it/s][2025-04-25 10:11:53] (step=0058350) Train Loss: 5.3598, Train Steps/Sec: 1.12 + 49%|███████████████████████████████████████████████████████▋ | 11458/23458 [3:18:16<3:15:21, 1.02it/s][2025-04-25 10:12:15] (step=0058375) Train Loss: 5.2802, Train Steps/Sec: 1.13 + 49%|███████████████████████████████████████████████████████▊ | 11483/23458 [3:18:38<2:49:10, 1.18it/s][2025-04-25 10:12:37] (step=0058400) Train Loss: 5.1913, Train Steps/Sec: 1.17 + 49%|███████████████████████████████████████████████████████▉ | 11508/23458 [3:18:59<2:53:31, 1.15it/s][2025-04-25 10:12:58] (step=0058425) Train Loss: 5.3371, Train Steps/Sec: 1.17 + 49%|████████████████████████████████████████████████████████ | 11533/23458 [3:19:21<3:02:49, 1.09it/s][2025-04-25 10:13:20] (step=0058450) Train Loss: 5.2303, Train Steps/Sec: 1.13 + 49%|████████████████████████████████████████████████████████▏ | 11558/23458 [3:19:43<2:48:52, 1.17it/s][2025-04-25 10:13:42] (step=0058475) Train Loss: 5.2475, Train Steps/Sec: 1.17 + 49%|████████████████████████████████████████████████████████▎ | 11583/23458 [3:20:05<2:46:28, 1.19it/s][2025-04-25 10:14:04] (step=0058500) Train Loss: 5.3057, Train Steps/Sec: 1.12 + 49%|████████████████████████████████████████████████████████▍ | 11608/23458 [3:20:26<2:51:59, 1.15it/s][2025-04-25 10:14:26] (step=0058525) Train Loss: 5.3227, Train Steps/Sec: 1.17 + 50%|████████████████████████████████████████████████████████▌ | 11633/23458 [3:20:48<2:49:12, 1.16it/s][2025-04-25 10:14:47] (step=0058550) Train Loss: 5.3476, Train Steps/Sec: 1.17 + 50%|████████████████████████████████████████████████████████▋ | 11658/23458 [3:21:09<2:47:12, 1.18it/s][2025-04-25 10:15:08] (step=0058575) Train Loss: 5.3026, Train Steps/Sec: 1.17 + 50%|████████████████████████████████████████████████████████▊ | 11683/23458 [3:21:31<2:46:14, 1.18it/s][2025-04-25 10:15:30] (step=0058600) Train Loss: 5.2403, Train Steps/Sec: 1.16 + 50%|████████████████████████████████████████████████████████▉ | 11708/23458 [3:21:53<2:50:57, 1.15it/s][2025-04-25 10:15:52] (step=0058625) Train Loss: 5.2837, Train Steps/Sec: 1.12 + 50%|█████████████████████████████████████████████████████████ | 11733/23458 [3:22:14<2:47:43, 1.17it/s][2025-04-25 10:16:14] (step=0058650) Train Loss: 5.3110, Train Steps/Sec: 1.17 + 50%|█████████████████████████████████████████████████████████▏ | 11758/23458 [3:22:36<2:46:09, 1.17it/s][2025-04-25 10:16:35] (step=0058675) Train Loss: 5.2776, Train Steps/Sec: 1.17 + 50%|█████████████████████████████████████████████████████████▎ | 11783/23458 [3:22:58<2:57:10, 1.10it/s][2025-04-25 10:16:57] (step=0058700) Train Loss: 5.2267, Train Steps/Sec: 1.12 + 50%|█████████████████████████████████████████████████████████▍ | 11808/23458 [3:23:20<2:48:58, 1.15it/s][2025-04-25 10:17:19] (step=0058725) Train Loss: 5.3002, Train Steps/Sec: 1.17 + 50%|█████████████████████████████████████████████████████████▌ | 11833/23458 [3:23:42<2:46:22, 1.16it/s][2025-04-25 10:17:41] (step=0058750) Train Loss: 5.3081, Train Steps/Sec: 1.12 + 51%|█████████████████████████████████████████████████████████▋ | 11858/23458 [3:24:03<2:44:34, 1.17it/s][2025-04-25 10:18:03] (step=0058775) Train Loss: 5.2659, Train Steps/Sec: 1.17 + 51%|█████████████████████████████████████████████████████████▋ | 11883/23458 [3:24:25<2:42:17, 1.19it/s][2025-04-25 10:18:24] (step=0058800) Train Loss: 5.2432, Train Steps/Sec: 1.17 + 51%|█████████████████████████████████████████████████████████▊ | 11908/23458 [3:24:46<2:47:25, 1.15it/s][2025-04-25 10:18:45] (step=0058825) Train Loss: 5.2899, Train Steps/Sec: 1.17 + 51%|█████████████████████████████████████████████████████████▉ | 11933/23458 [3:25:08<2:44:32, 1.17it/s][2025-04-25 10:19:07] (step=0058850) Train Loss: 5.2921, Train Steps/Sec: 1.17 + 51%|██████████████████████████████████████████████████████████ | 11958/23458 [3:25:29<2:43:12, 1.17it/s][2025-04-25 10:19:28] (step=0058875) Train Loss: 5.3497, Train Steps/Sec: 1.17 + 51%|██████████████████████████████████████████████████████████▏ | 11983/23458 [3:25:50<2:41:03, 1.19it/s][2025-04-25 10:19:49] (step=0058900) Train Loss: 5.2507, Train Steps/Sec: 1.17 + 51%|██████████████████████████████████████████████████████████▎ | 12008/23458 [3:26:13<2:47:40, 1.14it/s][2025-04-25 10:20:12] (step=0058925) Train Loss: 5.3443, Train Steps/Sec: 1.13 + 51%|██████████████████████████████████████████████████████████▍ | 12033/23458 [3:26:34<2:43:14, 1.17it/s][2025-04-25 10:20:33] (step=0058950) Train Loss: 5.2995, Train Steps/Sec: 1.17 + 51%|██████████████████████████████████████████████████████████▌ | 12058/23458 [3:26:55<2:40:56, 1.18it/s][2025-04-25 10:20:54] (step=0058975) Train Loss: 5.2991, Train Steps/Sec: 1.17 + 52%|██████████████████████████████████████████████████████████▋ | 12083/23458 [3:27:17<2:56:22, 1.07it/s][2025-04-25 10:21:17] (step=0059000) Train Loss: 5.3176, Train Steps/Sec: 1.12 + 52%|██████████████████████████████████████████████████████████▊ | 12108/23458 [3:27:39<2:45:07, 1.15it/s][2025-04-25 10:21:38] (step=0059025) Train Loss: 5.1812, Train Steps/Sec: 1.17 + 52%|██████████████████████████████████████████████████████████▉ | 12133/23458 [3:28:01<2:42:12, 1.16it/s][2025-04-25 10:22:00] (step=0059050) Train Loss: 5.2424, Train Steps/Sec: 1.12 + 52%|███████████████████████████████████████████████████████████ | 12158/23458 [3:28:23<2:40:30, 1.17it/s][2025-04-25 10:22:22] (step=0059075) Train Loss: 5.3235, Train Steps/Sec: 1.17 + 52%|███████████████████████████████████████████████████████████▏ | 12183/23458 [3:28:44<2:39:27, 1.18it/s][2025-04-25 10:22:43] (step=0059100) Train Loss: 5.2927, Train Steps/Sec: 1.16 + 52%|███████████████████████████████████████████████████████████▎ | 12208/23458 [3:29:06<2:42:47, 1.15it/s][2025-04-25 10:23:05] (step=0059125) Train Loss: 5.1694, Train Steps/Sec: 1.13 + 52%|███████████████████████████████████████████████████████████▍ | 12233/23458 [3:29:28<2:40:57, 1.16it/s][2025-04-25 10:23:27] (step=0059150) Train Loss: 5.2597, Train Steps/Sec: 1.13 + 52%|███████████████████████████████████████████████████████████▌ | 12258/23458 [3:29:50<2:38:37, 1.18it/s][2025-04-25 10:23:49] (step=0059175) Train Loss: 5.3108, Train Steps/Sec: 1.17 + 52%|███████████████████████████████████████████████████████████▋ | 12283/23458 [3:30:11<2:36:43, 1.19it/s][2025-04-25 10:24:10] (step=0059200) Train Loss: 5.3129, Train Steps/Sec: 1.17 + 52%|███████████████████████████████████████████████████████████▊ | 12308/23458 [3:30:33<2:41:34, 1.15it/s][2025-04-25 10:24:32] (step=0059225) Train Loss: 5.3318, Train Steps/Sec: 1.17 + 53%|███████████████████████████████████████████████████████████▉ | 12333/23458 [3:30:54<2:38:59, 1.17it/s][2025-04-25 10:24:53] (step=0059250) Train Loss: 5.2440, Train Steps/Sec: 1.17 + 53%|████████████████████████████████████████████████████████████ | 12358/23458 [3:31:15<2:37:05, 1.18it/s][2025-04-25 10:25:14] (step=0059275) Train Loss: 5.3401, Train Steps/Sec: 1.17 + 53%|████████████████████████████████████████████████████████████▏ | 12383/23458 [3:31:37<2:36:46, 1.18it/s][2025-04-25 10:25:36] (step=0059300) Train Loss: 5.2764, Train Steps/Sec: 1.17 + 53%|████████████████████████████████████████████████████████████▎ | 12408/23458 [3:31:58<2:40:25, 1.15it/s][2025-04-25 10:25:57] (step=0059325) Train Loss: 5.3568, Train Steps/Sec: 1.17 + 53%|████████████████████████████████████████████████████████████▍ | 12433/23458 [3:32:20<2:37:28, 1.17it/s][2025-04-25 10:26:19] (step=0059350) Train Loss: 5.3294, Train Steps/Sec: 1.13 + 53%|████████████████████████████████████████████████████████████▌ | 12458/23458 [3:32:42<2:35:35, 1.18it/s][2025-04-25 10:26:41] (step=0059375) Train Loss: 5.3026, Train Steps/Sec: 1.17 + 53%|████████████████████████████████████████████████████████████▋ | 12483/23458 [3:33:03<2:34:32, 1.18it/s][2025-04-25 10:27:02] (step=0059400) Train Loss: 5.3288, Train Steps/Sec: 1.17 + 53%|████████████████████████████████████████████████████████████▊ | 12508/23458 [3:33:24<2:39:20, 1.15it/s][2025-04-25 10:27:24] (step=0059425) Train Loss: 5.2847, Train Steps/Sec: 1.17 + 53%|████████████████████████████████████████████████████████████▉ | 12533/23458 [3:33:47<2:35:34, 1.17it/s][2025-04-25 10:27:46] (step=0059450) Train Loss: 5.3240, Train Steps/Sec: 1.12 + 54%|█████████████████████████████████████████████████████████████ | 12558/23458 [3:34:08<2:34:13, 1.18it/s][2025-04-25 10:28:07] (step=0059475) Train Loss: 5.3223, Train Steps/Sec: 1.17 + 54%|█████████████████████████████████████████████████████████████▏ | 12583/23458 [3:34:31<2:33:14, 1.18it/s][2025-04-25 10:28:30] (step=0059500) Train Loss: 5.2546, Train Steps/Sec: 1.11 + 54%|█████████████████████████████████████████████████████████████▎ | 12608/23458 [3:34:52<2:38:05, 1.14it/s][2025-04-25 10:28:51] (step=0059525) Train Loss: 5.2779, Train Steps/Sec: 1.17 + 54%|█████████████████████████████████████████████████████████████▍ | 12633/23458 [3:35:13<2:34:11, 1.17it/s][2025-04-25 10:29:12] (step=0059550) Train Loss: 5.2911, Train Steps/Sec: 1.17 + 54%|█████████████████████████████████████████████████████████████▌ | 12658/23458 [3:35:36<2:38:18, 1.14it/s][2025-04-25 10:29:35] (step=0059575) Train Loss: 5.2197, Train Steps/Sec: 1.12 + 54%|█████████████████████████████████████████████████████████████▋ | 12683/23458 [3:35:57<2:31:02, 1.19it/s][2025-04-25 10:29:56] (step=0059600) Train Loss: 5.3015, Train Steps/Sec: 1.17 + 54%|█████████████████████████████████████████████████████████████▊ | 12708/23458 [3:36:18<2:35:48, 1.15it/s][2025-04-25 10:30:17] (step=0059625) Train Loss: 5.3149, Train Steps/Sec: 1.17 + 54%|█████████████████████████████████████████████████████████████▉ | 12733/23458 [3:36:40<2:32:47, 1.17it/s][2025-04-25 10:30:39] (step=0059650) Train Loss: 5.2136, Train Steps/Sec: 1.17 + 54%|██████████████████████████████████████████████████████████████ | 12758/23458 [3:37:02<2:31:31, 1.18it/s][2025-04-25 10:31:01] (step=0059675) Train Loss: 5.2819, Train Steps/Sec: 1.13 + 54%|██████████████████████████████████████████████████████████████ | 12783/23458 [3:37:24<2:30:15, 1.18it/s][2025-04-25 10:31:23] (step=0059700) Train Loss: 5.3448, Train Steps/Sec: 1.12 + 55%|██████████████████████████████████████████████████████████████▏ | 12808/23458 [3:37:46<2:34:45, 1.15it/s][2025-04-25 10:31:45] (step=0059725) Train Loss: 5.3353, Train Steps/Sec: 1.17 + 55%|██████████████████████████████████████████████████████████████▎ | 12833/23458 [3:38:07<2:31:33, 1.17it/s][2025-04-25 10:32:06] (step=0059750) Train Loss: 5.2608, Train Steps/Sec: 1.17 + 55%|██████████████████████████████████████████████████████████████▍ | 12858/23458 [3:38:29<2:30:13, 1.18it/s][2025-04-25 10:32:28] (step=0059775) Train Loss: 5.2741, Train Steps/Sec: 1.12 + 55%|██████████████████████████████████████████████████████████████▌ | 12883/23458 [3:38:51<3:13:26, 1.10s/it][2025-04-25 10:32:51] (step=0059800) Train Loss: 5.3171, Train Steps/Sec: 1.12 + 55%|██████████████████████████████████████████████████████████████▋ | 12908/23458 [3:39:13<2:32:45, 1.15it/s][2025-04-25 10:33:12] (step=0059825) Train Loss: 5.2556, Train Steps/Sec: 1.17 + 55%|██████████████████████████████████████████████████████████████▊ | 12933/23458 [3:39:34<2:30:27, 1.17it/s][2025-04-25 10:33:33] (step=0059850) Train Loss: 5.2499, Train Steps/Sec: 1.17 + 55%|██████████████████████████████████████████████████████████████▉ | 12958/23458 [3:39:55<2:28:28, 1.18it/s][2025-04-25 10:33:55] (step=0059875) Train Loss: 5.2841, Train Steps/Sec: 1.17 + 55%|███████████████████████████████████████████████████████████████ | 12983/23458 [3:40:17<2:27:51, 1.18it/s][2025-04-25 10:34:16] (step=0059900) Train Loss: 5.1980, Train Steps/Sec: 1.17 + 55%|███████████████████████████████████████████████████████████████▏ | 13008/23458 [3:40:38<2:31:11, 1.15it/s][2025-04-25 10:34:37] (step=0059925) Train Loss: 5.2898, Train Steps/Sec: 1.17 + 56%|███████████████████████████████████████████████████████████████▎ | 13033/23458 [3:41:00<2:28:28, 1.17it/s][2025-04-25 10:34:59] (step=0059950) Train Loss: 5.2988, Train Steps/Sec: 1.17 + 56%|███████████████████████████████████████████████████████████████▍ | 13058/23458 [3:41:21<2:27:06, 1.18it/s][2025-04-25 10:35:20] (step=0059975) Train Loss: 5.3223, Train Steps/Sec: 1.17 + 56%|███████████████████████████████████████████████████████████████▌ | 13083/23458 [3:41:42<2:26:58, 1.18it/s][2025-04-25 10:35:42] (step=0060000) Train Loss: 5.3202, Train Steps/Sec: 1.16 +You are using a model of type blip-2 to instantiate a model of type instructblip. This is not supported for all configurations of models and can yield errors. +[2025-04-25 10:35:42] text_config is None. Initializing the text config with default values (`OPTConfig`). +model_name_or_path:/tmp/haozhezhao/model/blip2-flan-t5-xl +image_place_holder: +tokenizer length after expend 32101 +tokenizer length before expend 32100 +100%|██████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████| 4/4 [03:57<00:00, 59.31s/it] +[2025-04-25 10:40:47] Finish Eval in 60000 steps...███████████████████████████████████████████████████████████████████████| 4/4 [03:56<00:00, 59.01s/it] +processing left-padding... +processing left-padding... +processing left-padding... +processing left-padding... +[2025-04-25 10:41:06] Saved checkpoint to checkpoint/Afteri2i_100kckpt_LLAVAT5_empty_Fixed_just_segment_pretrain_stage3_dreambench_recap_400k_t2i_400k_flux_200k_midjourney_150k_recovery_150k_3e4_epoch2_no_rotated/002-GPT-XL/checkpoints/0060000.pt +[2025-04-25 10:41:09] Removed old checkpoint: checkpoint/Afteri2i_100kckpt_LLAVAT5_empty_Fixed_just_segment_pretrain_stage3_dreambench_recap_400k_t2i_400k_flux_200k_midjourney_150k_recovery_150k_3e4_epoch2_no_rotated/002-GPT-XL/checkpoints/0058000.pt + 56%|███████████████████████████████████████████████████████████████▋ | 13108/23458 [3:47:31<2:36:04, 1.11it/s][2025-04-25 10:41:30] (step=0060025) Train Loss: 5.2903, Train Steps/Sec: 0.07 + 56%|███████████████████████████████████████████████████████████████▊ | 13133/23458 [3:47:53<2:27:49, 1.16it/s][2025-04-25 10:41:52] (step=0060050) Train Loss: 5.2546, Train Steps/Sec: 1.17 + 56%|███████████████████████████████████████████████████████████████▉ | 13158/23458 [3:48:15<2:27:16, 1.17it/s][2025-04-25 10:42:14] (step=0060075) Train Loss: 5.3219, Train Steps/Sec: 1.12 + 56%|████████████████████████████████████████████████████████████████ | 13183/23458 [3:48:36<2:25:03, 1.18it/s][2025-04-25 10:42:35] (step=0060100) Train Loss: 5.2247, Train Steps/Sec: 1.17 + 56%|████████████████████████████████████████████████████████████████▏ | 13208/23458 [3:48:58<2:28:06, 1.15it/s][2025-04-25 10:42:57] (step=0060125) Train Loss: 5.3042, Train Steps/Sec: 1.17 + 56%|████████████████████████████████████████████████████████████████▎ | 13233/23458 [3:49:19<2:26:22, 1.16it/s][2025-04-25 10:43:18] (step=0060150) Train Loss: 5.2465, Train Steps/Sec: 1.17 + 57%|████████████████████████████████████████████████████████████████▍ | 13258/23458 [3:49:41<2:25:22, 1.17it/s][2025-04-25 10:43:40] (step=0060175) Train Loss: 5.2824, Train Steps/Sec: 1.12 + 57%|████████████████████████████████████████████████████████████████▌ | 13283/23458 [3:50:03<2:22:54, 1.19it/s][2025-04-25 10:44:02] (step=0060200) Train Loss: 5.3293, Train Steps/Sec: 1.17 + 57%|████████████████████████████████████████████████████████████████▋ | 13308/23458 [3:50:25<2:32:41, 1.11it/s][2025-04-25 10:44:24] (step=0060225) Train Loss: 5.2412, Train Steps/Sec: 1.12 + 57%|████████████████████████████████████████████████████████████████▊ | 13333/23458 [3:50:46<2:24:03, 1.17it/s][2025-04-25 10:44:45] (step=0060250) Train Loss: 5.2938, Train Steps/Sec: 1.17 + 57%|████████████████████████████████████████████████████████████████▉ | 13358/23458 [3:51:08<2:22:27, 1.18it/s][2025-04-25 10:45:07] (step=0060275) Train Loss: 5.2849, Train Steps/Sec: 1.17 + 57%|█████████████████████████████████████████████████████████████████ | 13383/23458 [3:51:29<2:21:40, 1.19it/s][2025-04-25 10:45:28] (step=0060300) Train Loss: 5.2285, Train Steps/Sec: 1.17 + 57%|█████████████████████████████████████████████████████████████████▏ | 13408/23458 [3:51:51<2:25:54, 1.15it/s][2025-04-25 10:45:50] (step=0060325) Train Loss: 5.2706, Train Steps/Sec: 1.12 + 57%|█████████████████████████████████████████████████████████████████▎ | 13433/23458 [3:52:13<2:23:19, 1.17it/s][2025-04-25 10:46:13] (step=0060350) Train Loss: 5.2950, Train Steps/Sec: 1.13 + 57%|█████████████████████████████████████████████████████████████████▍ | 13458/23458 [3:52:35<2:22:05, 1.17it/s][2025-04-25 10:46:34] (step=0060375) Train Loss: 5.1872, Train Steps/Sec: 1.17 + 57%|█████████████████████████████████████████████████████████████████▌ | 13483/23458 [3:52:56<2:20:08, 1.19it/s][2025-04-25 10:46:55] (step=0060400) Train Loss: 5.3249, Train Steps/Sec: 1.16 + 58%|█████████████████████████████████████████████████████████████████▋ | 13508/23458 [3:53:19<2:31:18, 1.10it/s][2025-04-25 10:47:18] (step=0060425) Train Loss: 5.2726, Train Steps/Sec: 1.13 + 58%|█████████████████████████████████████████████████████████████████▊ | 13533/23458 [3:53:40<2:22:41, 1.16it/s][2025-04-25 10:47:39] (step=0060450) Train Loss: 5.2659, Train Steps/Sec: 1.16 + 58%|█████████████████████████████████████████████████████████████████▉ | 13558/23458 [3:54:02<2:22:23, 1.16it/s][2025-04-25 10:48:02] (step=0060475) Train Loss: 5.2618, Train Steps/Sec: 1.12 + 58%|██████████████████████████████████████████████████████████████████ | 13583/23458 [3:54:24<2:19:36, 1.18it/s][2025-04-25 10:48:23] (step=0060500) Train Loss: 5.2968, Train Steps/Sec: 1.16 + 58%|██████████████████████████████████████████████████████████████████▏ | 13608/23458 [3:54:45<2:22:46, 1.15it/s][2025-04-25 10:48:44] (step=0060525) Train Loss: 5.2961, Train Steps/Sec: 1.17 + 58%|██████████████████████████████████████████████████████████████████▎ | 13633/23458 [3:55:07<2:20:28, 1.17it/s][2025-04-25 10:49:06] (step=0060550) Train Loss: 5.2770, Train Steps/Sec: 1.17 + 58%|██████████████████████████████████████████████████████████████████▎ | 13658/23458 [3:55:28<2:19:22, 1.17it/s][2025-04-25 10:49:27] (step=0060575) Train Loss: 5.3047, Train Steps/Sec: 1.17 + 58%|██████████████████████████████████████████████████████████████████▍ | 13683/23458 [3:55:50<2:17:55, 1.18it/s][2025-04-25 10:49:49] (step=0060600) Train Loss: 5.3593, Train Steps/Sec: 1.16 + 58%|██████████████████████████████████████████████████████████████████▌ | 13708/23458 [3:56:11<2:21:36, 1.15it/s][2025-04-25 10:50:10] (step=0060625) Train Loss: 5.2503, Train Steps/Sec: 1.17 + 59%|██████████████████████████████████████████████████████████████████▋ | 13733/23458 [3:56:32<2:18:46, 1.17it/s][2025-04-25 10:50:31] (step=0060650) Train Loss: 5.2576, Train Steps/Sec: 1.17 + 59%|██████████████████████████████████████████████████████████████████▊ | 13758/23458 [3:56:54<2:17:55, 1.17it/s][2025-04-25 10:50:53] (step=0060675) Train Loss: 5.2836, Train Steps/Sec: 1.17 + 59%|██████████████████████████████████████████████████████████████████▉ | 13783/23458 [3:57:15<2:16:14, 1.18it/s][2025-04-25 10:51:14] (step=0060700) Train Loss: 5.2749, Train Steps/Sec: 1.16 + 59%|███████████████████████████████████████████████████████████████████ | 13808/23458 [3:57:37<2:19:50, 1.15it/s][2025-04-25 10:51:36] (step=0060725) Train Loss: 5.3321, Train Steps/Sec: 1.17 + 59%|███████████████████████████████████████████████████████████████████▏ | 13833/23458 [3:57:59<2:18:32, 1.16it/s][2025-04-25 10:51:58] (step=0060750) Train Loss: 5.2364, Train Steps/Sec: 1.11 + 59%|███████████████████████████████████████████████████████████████████▎ | 13858/23458 [3:58:20<2:15:39, 1.18it/s][2025-04-25 10:52:20] (step=0060775) Train Loss: 5.2285, Train Steps/Sec: 1.17 + 59%|███████████████████████████████████████████████████████████████████▍ | 13883/23458 [3:58:43<2:16:24, 1.17it/s][2025-04-25 10:52:42] (step=0060800) Train Loss: 5.2454, Train Steps/Sec: 1.12 + 59%|███████████████████████████████████████████████████████████████████▌ | 13908/23458 [3:59:04<2:18:58, 1.15it/s][2025-04-25 10:53:03] (step=0060825) Train Loss: 5.3321, Train Steps/Sec: 1.17 + 59%|███████████████████████████████████████████████████████████████████▋ | 13933/23458 [3:59:26<2:15:53, 1.17it/s][2025-04-25 10:53:25] (step=0060850) Train Loss: 5.2449, Train Steps/Sec: 1.17 + 60%|███████████████████████████████████████████████████████████████████▊ | 13958/23458 [3:59:48<2:18:25, 1.14it/s][2025-04-25 10:53:47] (step=0060875) Train Loss: 5.2102, Train Steps/Sec: 1.12 + 60%|███████████████████████████████████████████████████████████████████▉ | 13983/23458 [4:00:10<2:18:36, 1.14it/s][2025-04-25 10:54:09] (step=0060900) Train Loss: 5.2369, Train Steps/Sec: 1.12 + 60%|████████████████████████████████████████████████████████████████████ | 14008/23458 [4:00:32<2:16:40, 1.15it/s][2025-04-25 10:54:31] (step=0060925) Train Loss: 5.3053, Train Steps/Sec: 1.17 + 60%|████████████████████████████████████████████████████████████████████▏ | 14033/23458 [4:00:53<2:15:03, 1.16it/s][2025-04-25 10:54:52] (step=0060950) Train Loss: 5.2585, Train Steps/Sec: 1.17 + 60%|████████████████████████████████████████████████████████████████████▎ | 14058/23458 [4:01:15<2:12:58, 1.18it/s][2025-04-25 10:55:14] (step=0060975) Train Loss: 5.3317, Train Steps/Sec: 1.13 + 60%|████████████████████████████████████████████████████████████████████▍ | 14083/23458 [4:01:37<2:13:26, 1.17it/s][2025-04-25 10:55:37] (step=0061000) Train Loss: 5.2482, Train Steps/Sec: 1.12 + 60%|████████████████████████████████████████████████████████████████████▌ | 14108/23458 [4:01:59<2:15:20, 1.15it/s][2025-04-25 10:55:58] (step=0061025) Train Loss: 5.2760, Train Steps/Sec: 1.17 + 60%|████████████████████████████████████████████████████████████████████▋ | 14133/23458 [4:02:20<2:13:14, 1.17it/s][2025-04-25 10:56:19] (step=0061050) Train Loss: 5.2656, Train Steps/Sec: 1.17 + 60%|████████████████████████████████████████████████████████████████████▊ | 14158/23458 [4:02:42<2:11:51, 1.18it/s][2025-04-25 10:56:41] (step=0061075) Train Loss: 5.2647, Train Steps/Sec: 1.17 + 60%|████████████████████████████████████████████████████████████████████▉ | 14183/23458 [4:03:04<2:10:39, 1.18it/s][2025-04-25 10:57:03] (step=0061100) Train Loss: 5.2231, Train Steps/Sec: 1.12 + 61%|█████████████████████████████████████████████████████████████████████ | 14208/23458 [4:03:26<2:27:57, 1.04it/s][2025-04-25 10:57:25] (step=0061125) Train Loss: 5.2489, Train Steps/Sec: 1.13 + 61%|█████████████████████████████████████████████████████████████████████▏ | 14233/23458 [4:03:48<2:12:15, 1.16it/s][2025-04-25 10:57:47] (step=0061150) Train Loss: 5.3300, Train Steps/Sec: 1.17 + 61%|█████████████████████████████████████████████████████████████████████▎ | 14258/23458 [4:04:09<2:10:20, 1.18it/s][2025-04-25 10:58:08] (step=0061175) Train Loss: 5.1952, Train Steps/Sec: 1.17 + 61%|█████████████████████████████████████████████████████████████████████▍ | 14283/23458 [4:04:30<2:08:53, 1.19it/s][2025-04-25 10:58:29] (step=0061200) Train Loss: 5.2213, Train Steps/Sec: 1.17 + 61%|█████████████████████████████████████████████████████████████████████▌ | 14308/23458 [4:04:52<2:12:58, 1.15it/s][2025-04-25 10:58:51] (step=0061225) Train Loss: 5.2123, Train Steps/Sec: 1.17 + 61%|█████████████████████████████████████████████████████████████████████▋ | 14333/23458 [4:05:13<2:10:27, 1.17it/s][2025-04-25 10:59:12] (step=0061250) Train Loss: 5.3096, Train Steps/Sec: 1.17 + 61%|█████████████████████████████████████████████████████████████████████▊ | 14358/23458 [4:05:34<2:09:10, 1.17it/s][2025-04-25 10:59:34] (step=0061275) Train Loss: 5.2840, Train Steps/Sec: 1.17 + 61%|█████████████████████████████████████████████████████████████████████▉ | 14383/23458 [4:05:56<2:07:35, 1.19it/s][2025-04-25 10:59:55] (step=0061300) Train Loss: 5.3036, Train Steps/Sec: 1.16 + 61%|██████████████████████████████████████████████████████████████████████ | 14408/23458 [4:06:17<2:11:02, 1.15it/s][2025-04-25 11:00:16] (step=0061325) Train Loss: 5.2937, Train Steps/Sec: 1.17 + 62%|██████████████████████████████████████████████████████████████████████▏ | 14433/23458 [4:06:39<2:09:06, 1.17it/s][2025-04-25 11:00:38] (step=0061350) Train Loss: 5.2981, Train Steps/Sec: 1.17 + 62%|██████████████████████████████████████████████████████████████████████▎ | 14458/23458 [4:07:00<2:07:15, 1.18it/s][2025-04-25 11:00:59] (step=0061375) Train Loss: 5.2620, Train Steps/Sec: 1.17 + 62%|██████████████████████████████████████████████████████████████████████▍ | 14483/23458 [4:07:22<2:06:12, 1.19it/s][2025-04-25 11:01:21] (step=0061400) Train Loss: 5.2764, Train Steps/Sec: 1.16 + 62%|██████████████████████████████████████████████████████████████████████▌ | 14508/23458 [4:07:43<2:09:41, 1.15it/s][2025-04-25 11:01:42] (step=0061425) Train Loss: 5.3370, Train Steps/Sec: 1.17 + 62%|██████████████████████████████████████████████████████████████████████▋ | 14533/23458 [4:08:04<2:07:37, 1.17it/s][2025-04-25 11:02:03] (step=0061450) Train Loss: 5.2107, Train Steps/Sec: 1.17 + 62%|██████████████████████████████████████████████████████████████████████▋ | 14558/23458 [4:08:26<2:06:22, 1.17it/s][2025-04-25 11:02:25] (step=0061475) Train Loss: 5.2517, Train Steps/Sec: 1.17 + 62%|██████████████████████████████████████████████████████████████████████▊ | 14583/23458 [4:08:48<2:05:23, 1.18it/s][2025-04-25 11:02:47] (step=0061500) Train Loss: 5.2253, Train Steps/Sec: 1.11 + 62%|██████████████████████████████████████████████████████████████████████▉ | 14608/23458 [4:09:11<2:24:35, 1.02it/s][2025-04-25 11:03:10] (step=0061525) Train Loss: 5.3081, Train Steps/Sec: 1.08 + 62%|███████████████████████████████████████████████████████████████████████ | 14633/23458 [4:09:33<2:06:15, 1.16it/s][2025-04-25 11:03:32] (step=0061550) Train Loss: 5.3916, Train Steps/Sec: 1.17 + 62%|███████████████████████████████████████████████████████████████████████▏ | 14658/23458 [4:09:54<2:04:37, 1.18it/s][2025-04-25 11:03:53] (step=0061575) Train Loss: 5.2753, Train Steps/Sec: 1.17 + 63%|███████████████████████████████████████████████████████████████████████▎ | 14683/23458 [4:10:15<2:03:05, 1.19it/s][2025-04-25 11:04:15] (step=0061600) Train Loss: 5.2418, Train Steps/Sec: 1.17 + 63%|███████████████████████████████████████████████████████████████████████▍ | 14708/23458 [4:10:39<2:47:30, 1.15s/it][2025-04-25 11:04:38] (step=0061625) Train Loss: 5.3001, Train Steps/Sec: 1.08 + 63%|███████████████████████████████████████████████████████████████████████▌ | 14733/23458 [4:11:01<2:05:27, 1.16it/s][2025-04-25 11:05:00] (step=0061650) Train Loss: 5.2804, Train Steps/Sec: 1.13 + 63%|███████████████████████████████████████████████████████████████████████▋ | 14758/23458 [4:11:22<2:03:24, 1.17it/s][2025-04-25 11:05:21] (step=0061675) Train Loss: 5.2134, Train Steps/Sec: 1.17 + 63%|███████████████████████████████████████████████████████████████████████▊ | 14783/23458 [4:11:44<2:02:08, 1.18it/s][2025-04-25 11:05:43] (step=0061700) Train Loss: 5.2887, Train Steps/Sec: 1.16 + 63%|███████████████████████████████████████████████████████████████████████▉ | 14808/23458 [4:12:05<2:05:49, 1.15it/s][2025-04-25 11:06:04] (step=0061725) Train Loss: 5.3041, Train Steps/Sec: 1.16 + 63%|████████████████████████████████████████████████████████████████████████ | 14833/23458 [4:12:27<2:03:55, 1.16it/s][2025-04-25 11:06:27] (step=0061750) Train Loss: 5.2179, Train Steps/Sec: 1.13 + 63%|████████████████████████████████████████████████████████████████████████▏ | 14858/23458 [4:12:49<2:01:50, 1.18it/s][2025-04-25 11:06:48] (step=0061775) Train Loss: 5.2847, Train Steps/Sec: 1.17 + 63%|████████████████████████████████████████████████████████████████████████▎ | 14883/23458 [4:13:11<2:00:49, 1.18it/s][2025-04-25 11:07:10] (step=0061800) Train Loss: 5.2911, Train Steps/Sec: 1.12 + 64%|████████████████████████████████████████████████████████████████████████▍ | 14908/23458 [4:13:33<2:03:51, 1.15it/s][2025-04-25 11:07:32] (step=0061825) Train Loss: 5.3185, Train Steps/Sec: 1.17 + 64%|████████████████████████████████████████████████████████████████████████▌ | 14933/23458 [4:13:54<2:01:38, 1.17it/s][2025-04-25 11:07:53] (step=0061850) Train Loss: 5.2480, Train Steps/Sec: 1.17 + 64%|████████████████████████████████████████████████████████████████████████▋ | 14958/23458 [4:14:15<2:00:53, 1.17it/s][2025-04-25 11:08:14] (step=0061875) Train Loss: 5.2934, Train Steps/Sec: 1.17 + 64%|████████████████████████████████████████████████████████████████████████▊ | 14983/23458 [4:14:37<1:59:25, 1.18it/s][2025-04-25 11:08:36] (step=0061900) Train Loss: 5.3014, Train Steps/Sec: 1.17 + 64%|████████████████████████████████████████████████████████████████████████▉ | 15008/23458 [4:14:58<2:02:20, 1.15it/s][2025-04-25 11:08:57] (step=0061925) Train Loss: 5.3941, Train Steps/Sec: 1.17 + 64%|█████████████████████████████████████████████████████████████████████████ | 15033/23458 [4:15:19<1:59:46, 1.17it/s][2025-04-25 11:09:19] (step=0061950) Train Loss: 5.2803, Train Steps/Sec: 1.17 + 64%|█████████████████████████████████████████████████████████████████████████▏ | 15058/23458 [4:15:41<1:59:27, 1.17it/s][2025-04-25 11:09:40] (step=0061975) Train Loss: 5.3309, Train Steps/Sec: 1.17 + 64%|█████████████████████████████████████████████████████████████████████████▎ | 15083/23458 [4:16:02<1:58:16, 1.18it/s][2025-04-25 11:10:01] (step=0062000) Train Loss: 5.2699, Train Steps/Sec: 1.17 +You are using a model of type blip-2 to instantiate a model of type instructblip. This is not supported for all configurations of models and can yield errors. +[2025-04-25 11:10:01] text_config is None. Initializing the text config with default values (`OPTConfig`). +model_name_or_path:/tmp/haozhezhao/model/blip2-flan-t5-xl +image_place_holder: +tokenizer length after expend 32101 +tokenizer length before expend 32100 +100%|██████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████| 4/4 [03:58<00:00, 59.65s/it] +[2025-04-25 11:15:09] Finish Eval in 62000 steps...███████████████████████████████████████████████████████████████████████| 4/4 [03:57<00:00, 59.25s/it] +processing left-padding... +processing left-padding... +processing left-padding... +processing left-padding... +[2025-04-25 11:15:28] Saved checkpoint to checkpoint/Afteri2i_100kckpt_LLAVAT5_empty_Fixed_just_segment_pretrain_stage3_dreambench_recap_400k_t2i_400k_flux_200k_midjourney_150k_recovery_150k_3e4_epoch2_no_rotated/002-GPT-XL/checkpoints/0062000.pt +[2025-04-25 11:15:31] Removed old checkpoint: checkpoint/Afteri2i_100kckpt_LLAVAT5_empty_Fixed_just_segment_pretrain_stage3_dreambench_recap_400k_t2i_400k_flux_200k_midjourney_150k_recovery_150k_3e4_epoch2_no_rotated/002-GPT-XL/checkpoints/0060000.pt + 64%|█████████████████████████████████████████████████████████████████████████▍ | 15108/23458 [4:21:53<2:04:10, 1.12it/s][2025-04-25 11:15:52] (step=0062025) Train Loss: 5.2381, Train Steps/Sec: 0.07 + 65%|█████████████████████████████████████████████████████████████████████████▌ | 15133/23458 [4:22:14<1:59:15, 1.16it/s][2025-04-25 11:16:13] (step=0062050) Train Loss: 5.2925, Train Steps/Sec: 1.17 + 65%|█████████████████████████████████████████████████████████████████████████▋ | 15158/23458 [4:22:36<1:57:36, 1.18it/s][2025-04-25 11:16:35] (step=0062075) Train Loss: 5.3144, Train Steps/Sec: 1.17 + 65%|█████████████████████████████████████████████████████████████████████████▊ | 15183/23458 [4:22:57<1:56:33, 1.18it/s][2025-04-25 11:16:56] (step=0062100) Train Loss: 5.3473, Train Steps/Sec: 1.16 + 65%|█████████████████████████████████████████████████████████████████████████▉ | 15208/23458 [4:23:19<2:00:12, 1.14it/s][2025-04-25 11:17:18] (step=0062125) Train Loss: 5.2766, Train Steps/Sec: 1.17 + 65%|██████████████████████████████████████████████████████████████████████████ | 15233/23458 [4:23:40<1:57:15, 1.17it/s][2025-04-25 11:17:39] (step=0062150) Train Loss: 5.2953, Train Steps/Sec: 1.17 + 65%|██████████████████████████████████████████████████████████████████████████▏ | 15258/23458 [4:24:02<1:57:44, 1.16it/s][2025-04-25 11:18:01] (step=0062175) Train Loss: 5.2777, Train Steps/Sec: 1.12 + 65%|██████████████████████████████████████████████████████████████████████████▎ | 15283/23458 [4:24:24<1:55:23, 1.18it/s][2025-04-25 11:18:23] (step=0062200) Train Loss: 5.2398, Train Steps/Sec: 1.16 + 65%|██████████████████████████████████████████████████████████████████████████▍ | 15308/23458 [4:24:45<1:58:15, 1.15it/s][2025-04-25 11:18:44] (step=0062225) Train Loss: 5.3130, Train Steps/Sec: 1.17 + 65%|██████████████████████████████████████████████████████████████████████████▌ | 15333/23458 [4:25:07<2:20:38, 1.04s/it][2025-04-25 11:19:06] (step=0062250) Train Loss: 5.2198, Train Steps/Sec: 1.12 + 65%|██████████████████████████████████████████████████████████████████████████▋ | 15358/23458 [4:25:29<1:55:04, 1.17it/s][2025-04-25 11:19:28] (step=0062275) Train Loss: 5.3072, Train Steps/Sec: 1.13 + 66%|██████████████████████████████████████████████████████████████████████████▊ | 15383/23458 [4:25:52<1:59:28, 1.13it/s][2025-04-25 11:19:51] (step=0062300) Train Loss: 5.2469, Train Steps/Sec: 1.12 + 66%|██████████████████████████████████████████████████████████████████████████▉ | 15408/23458 [4:26:13<1:56:51, 1.15it/s][2025-04-25 11:20:12] (step=0062325) Train Loss: 5.3269, Train Steps/Sec: 1.17 + 66%|███████████████████████████████████████████████████████████████████████████ | 15433/23458 [4:26:34<1:54:27, 1.17it/s][2025-04-25 11:20:34] (step=0062350) Train Loss: 5.2984, Train Steps/Sec: 1.17 + 66%|███████████████████████████████████████████████████████████████████████████ | 15458/23458 [4:26:57<1:53:17, 1.18it/s][2025-04-25 11:20:56] (step=0062375) Train Loss: 5.2452, Train Steps/Sec: 1.12 + 66%|███████████████████████████████████████████████████████████████████████████▏ | 15483/23458 [4:27:19<2:24:29, 1.09s/it][2025-04-25 11:21:18] (step=0062400) Train Loss: 5.2130, Train Steps/Sec: 1.12 + 66%|███████████████████████████████████████████████████████████████████████████▎ | 15508/23458 [4:27:40<1:55:45, 1.14it/s][2025-04-25 11:21:39] (step=0062425) Train Loss: 5.3062, Train Steps/Sec: 1.17 + 66%|███████████████████████████████████████████████████████████████████████████▍ | 15533/23458 [4:28:03<1:56:51, 1.13it/s][2025-04-25 11:22:02] (step=0062450) Train Loss: 5.2167, Train Steps/Sec: 1.12 + 66%|███████████████████████████████████████████████████████████████████████████▌ | 15558/23458 [4:28:24<1:51:49, 1.18it/s][2025-04-25 11:22:23] (step=0062475) Train Loss: 5.2977, Train Steps/Sec: 1.17 + 66%|███████████████████████████████████████████████████████████████████████████▋ | 15583/23458 [4:28:45<1:50:32, 1.19it/s][2025-04-25 11:22:45] (step=0062500) Train Loss: 5.2355, Train Steps/Sec: 1.17 + 67%|███████████████████████████████████████████████████████████████████████████▊ | 15608/23458 [4:29:07<1:53:49, 1.15it/s][2025-04-25 11:23:06] (step=0062525) Train Loss: 5.3225, Train Steps/Sec: 1.17 + 67%|███████████████████████████████████████████████████████████████████████████▉ | 15633/23458 [4:29:28<1:51:36, 1.17it/s][2025-04-25 11:23:27] (step=0062550) Train Loss: 5.2079, Train Steps/Sec: 1.17 + 67%|████████████████████████████████████████████████████████████████████████████ | 15658/23458 [4:29:50<1:50:25, 1.18it/s][2025-04-25 11:23:49] (step=0062575) Train Loss: 5.2371, Train Steps/Sec: 1.17 + 67%|████████████████████████████████████████████████████████████████████████████▏ | 15683/23458 [4:30:11<1:49:46, 1.18it/s][2025-04-25 11:24:10] (step=0062600) Train Loss: 5.2944, Train Steps/Sec: 1.16 + 67%|████████████████████████████████████████████████████████████████████████████▎ | 15708/23458 [4:30:33<1:52:18, 1.15it/s][2025-04-25 11:24:32] (step=0062625) Train Loss: 5.3301, Train Steps/Sec: 1.15 + 67%|████████████████████████████████████████████████████████████████████████████▍ | 15733/23458 [4:30:54<1:50:21, 1.17it/s][2025-04-25 11:24:53] (step=0062650) Train Loss: 5.2319, Train Steps/Sec: 1.17 + 67%|████████████████████████████████████████████████████████████████████████████▌ | 15758/23458 [4:31:16<1:49:07, 1.18it/s][2025-04-25 11:25:15] (step=0062675) Train Loss: 5.3203, Train Steps/Sec: 1.17 + 67%|████████████████████████████████████████████████████████████████████████████▋ | 15783/23458 [4:31:37<1:47:54, 1.19it/s][2025-04-25 11:25:36] (step=0062700) Train Loss: 5.2948, Train Steps/Sec: 1.16 + 67%|████████████████████████████████████████████████████████████████████████████▊ | 15808/23458 [4:31:58<1:50:50, 1.15it/s][2025-04-25 11:25:57] (step=0062725) Train Loss: 5.2873, Train Steps/Sec: 1.17 + 67%|████████████████████████████████████████████████████████████████████████████▉ | 15833/23458 [4:32:21<1:49:07, 1.16it/s][2025-04-25 11:26:20] (step=0062750) Train Loss: 5.3012, Train Steps/Sec: 1.12 + 68%|█████████████████████████████████████████████████████████████████████████████ | 15858/23458 [4:32:42<1:47:51, 1.17it/s][2025-04-25 11:26:41] (step=0062775) Train Loss: 5.2741, Train Steps/Sec: 1.17 + 68%|█████████████████████████████████████████████████████████████████████████████▏ | 15883/23458 [4:33:03<1:46:39, 1.18it/s][2025-04-25 11:27:03] (step=0062800) Train Loss: 5.2888, Train Steps/Sec: 1.17 + 68%|█████████████████████████████████████████████████████████████████████████████▎ | 15908/23458 [4:33:26<1:53:58, 1.10it/s][2025-04-25 11:27:25] (step=0062825) Train Loss: 5.2637, Train Steps/Sec: 1.12 + 68%|█████████████████████████████████████████████████████████████████████████████▍ | 15933/23458 [4:33:47<1:47:14, 1.17it/s][2025-04-25 11:27:46] (step=0062850) Train Loss: 5.2891, Train Steps/Sec: 1.17 + 68%|█████████████████████████████████████████████████████████████████████████████▌ | 15958/23458 [4:34:09<1:45:58, 1.18it/s][2025-04-25 11:28:08] (step=0062875) Train Loss: 5.3436, Train Steps/Sec: 1.17 + 68%|█████████████████████████████████████████████████████████████████████████████▋ | 15983/23458 [4:34:30<1:45:17, 1.18it/s][2025-04-25 11:28:29] (step=0062900) Train Loss: 5.2466, Train Steps/Sec: 1.16 + 68%|█████████████████████████████████████████████████████████████████████████████▊ | 16008/23458 [4:34:52<1:55:29, 1.08it/s][2025-04-25 11:28:51] (step=0062925) Train Loss: 5.3076, Train Steps/Sec: 1.13 + 68%|█████████████████████████████████████████████████████████████████████████████▉ | 16033/23458 [4:35:14<2:08:23, 1.04s/it][2025-04-25 11:29:14] (step=0062950) Train Loss: 5.2473, Train Steps/Sec: 1.12 + 68%|██████████████████████████████████████████████████████████████████████████████ | 16058/23458 [4:35:36<1:44:39, 1.18it/s][2025-04-25 11:29:35] (step=0062975) Train Loss: 5.2367, Train Steps/Sec: 1.17 + 69%|██████████████████████████████████████████████████████████████████████████████▏ | 16083/23458 [4:35:58<1:44:44, 1.17it/s][2025-04-25 11:29:57] (step=0063000) Train Loss: 5.2279, Train Steps/Sec: 1.12 + 69%|██████████████████████████████████████████████████████████████████████████████▎ | 16108/23458 [4:36:20<1:46:33, 1.15it/s][2025-04-25 11:30:19] (step=0063025) Train Loss: 5.3075, Train Steps/Sec: 1.17 + 69%|██████████████████████████████████████████████████████████████████████████████▍ | 16133/23458 [4:36:41<1:44:32, 1.17it/s][2025-04-25 11:30:40] (step=0063050) Train Loss: 5.3049, Train Steps/Sec: 1.17 + 69%|██████████████████████████████████████████████████████████████████████████████▌ | 16158/23458 [4:37:03<1:43:31, 1.18it/s][2025-04-25 11:31:02] (step=0063075) Train Loss: 5.2513, Train Steps/Sec: 1.13 + 69%|██████████████████████████████████████████████████████████████████████████████▋ | 16183/23458 [4:37:25<1:42:40, 1.18it/s][2025-04-25 11:31:25] (step=0063100) Train Loss: 5.3263, Train Steps/Sec: 1.11 + 69%|██████████████████████████████████████████████████████████████████████████████▊ | 16208/23458 [4:37:48<1:45:13, 1.15it/s][2025-04-25 11:31:47] (step=0063125) Train Loss: 5.3226, Train Steps/Sec: 1.13 + 69%|██████████████████████████████████████████████████████████████████████████████▉ | 16233/23458 [4:38:09<1:43:10, 1.17it/s][2025-04-25 11:32:08] (step=0063150) Train Loss: 5.3336, Train Steps/Sec: 1.17 + 69%|███████████████████████████████████████████████████████████████████████████████ | 16258/23458 [4:38:30<1:41:59, 1.18it/s][2025-04-25 11:32:30] (step=0063175) Train Loss: 5.3286, Train Steps/Sec: 1.17 + 69%|███████████████████████████████████████████████████████████████████████████████▏ | 16283/23458 [4:38:52<1:41:39, 1.18it/s][2025-04-25 11:32:51] (step=0063200) Train Loss: 5.2025, Train Steps/Sec: 1.16 + 70%|███████████████████████████████████████████████████████████████████████████████▎ | 16308/23458 [4:39:13<1:43:29, 1.15it/s][2025-04-25 11:33:12] (step=0063225) Train Loss: 5.2721, Train Steps/Sec: 1.17 + 70%|███████████████████████████████████████████████████████████████████████████████▎ | 16333/23458 [4:39:35<1:42:18, 1.16it/s][2025-04-25 11:33:34] (step=0063250) Train Loss: 5.2499, Train Steps/Sec: 1.17 + 70%|███████████████████████████████████████████████████████████████████████████████▍ | 16358/23458 [4:39:56<1:40:48, 1.17it/s][2025-04-25 11:33:55] (step=0063275) Train Loss: 5.2251, Train Steps/Sec: 1.16 + 70%|███████████████████████████████████████████████████████████████████████████████▌ | 16383/23458 [4:40:18<1:39:27, 1.19it/s][2025-04-25 11:34:17] (step=0063300) Train Loss: 5.3077, Train Steps/Sec: 1.16 + 70%|███████████████████████████████████████████████████████████████████████████████▋ | 16408/23458 [4:40:39<1:42:40, 1.14it/s][2025-04-25 11:34:38] (step=0063325) Train Loss: 5.3372, Train Steps/Sec: 1.17 + 70%|███████████████████████████████████████████████████████████████████████████████▊ | 16433/23458 [4:41:00<1:40:30, 1.16it/s][2025-04-25 11:34:59] (step=0063350) Train Loss: 5.2926, Train Steps/Sec: 1.17 + 70%|███████████████████████████████████████████████████████████████████████████████▉ | 16458/23458 [4:41:22<1:39:24, 1.17it/s][2025-04-25 11:35:21] (step=0063375) Train Loss: 5.2895, Train Steps/Sec: 1.17 + 70%|████████████████████████████████████████████████████████████████████████████████ | 16483/23458 [4:41:43<1:38:14, 1.18it/s][2025-04-25 11:35:42] (step=0063400) Train Loss: 5.2981, Train Steps/Sec: 1.16 + 70%|████████████████████████████████████████████████████████████████████████████████▏ | 16508/23458 [4:42:05<1:40:40, 1.15it/s][2025-04-25 11:36:04] (step=0063425) Train Loss: 5.2249, Train Steps/Sec: 1.17 + 70%|████████████████████████████████████████████████████████████████████████████████▎ | 16533/23458 [4:42:26<1:38:53, 1.17it/s][2025-04-25 11:36:25] (step=0063450) Train Loss: 5.3208, Train Steps/Sec: 1.17 + 71%|████████████████████████████████████████████████████████████████████████████████▍ | 16558/23458 [4:42:48<1:47:54, 1.07it/s][2025-04-25 11:36:47] (step=0063475) Train Loss: 5.2535, Train Steps/Sec: 1.12 + 71%|████████████████████████████████████████████████████████████████████████████████▌ | 16583/23458 [4:43:11<1:36:52, 1.18it/s][2025-04-25 11:37:10] (step=0063500) Train Loss: 5.1907, Train Steps/Sec: 1.11 + 71%|████████████████████████████████████████████████████████████████████████████████▋ | 16608/23458 [4:43:32<1:39:44, 1.14it/s][2025-04-25 11:37:31] (step=0063525) Train Loss: 5.3159, Train Steps/Sec: 1.17 + 71%|████████████████████████████████████████████████████████████████████████████████▊ | 16633/23458 [4:43:53<1:37:17, 1.17it/s][2025-04-25 11:37:53] (step=0063550) Train Loss: 5.2436, Train Steps/Sec: 1.17 + 71%|████████████████████████████████████████████████████████████████████████████████▉ | 16658/23458 [4:44:16<1:56:43, 1.03s/it][2025-04-25 11:38:15] (step=0063575) Train Loss: 5.2907, Train Steps/Sec: 1.12 + 71%|█████████████████████████████████████████████████████████████████████████████████ | 16683/23458 [4:44:37<1:35:25, 1.18it/s][2025-04-25 11:38:37] (step=0063600) Train Loss: 5.3135, Train Steps/Sec: 1.16 + 71%|█████████████████████████████████████████████████████████████████████████████████▏ | 16708/23458 [4:45:00<1:37:50, 1.15it/s][2025-04-25 11:38:59] (step=0063625) Train Loss: 5.2554, Train Steps/Sec: 1.17 + 71%|█████████████████████████████████████████████████████████████████████████████████▎ | 16733/23458 [4:45:21<1:35:40, 1.17it/s][2025-04-25 11:39:20] (step=0063650) Train Loss: 5.3573, Train Steps/Sec: 1.17 + 71%|█████████████████████████████████████████████████████████████████████████████████▍ | 16758/23458 [4:45:42<1:34:45, 1.18it/s][2025-04-25 11:39:41] (step=0063675) Train Loss: 5.2699, Train Steps/Sec: 1.17 + 72%|█████████████████████████████████████████████████████████████████████████████████▌ | 16783/23458 [4:46:04<1:34:26, 1.18it/s][2025-04-25 11:40:03] (step=0063700) Train Loss: 5.2673, Train Steps/Sec: 1.16 + 72%|█████████████████████████████████████████████████████████████████████████████████▋ | 16808/23458 [4:46:27<1:38:17, 1.13it/s][2025-04-25 11:40:26] (step=0063725) Train Loss: 5.2347, Train Steps/Sec: 1.08 + 72%|█████████████████████████████████████████████████████████████████████████████████▊ | 16833/23458 [4:46:48<1:34:26, 1.17it/s][2025-04-25 11:40:47] (step=0063750) Train Loss: 5.3367, Train Steps/Sec: 1.17 + 72%|█████████████████████████████████████████████████████████████████████████████████▉ | 16858/23458 [4:47:10<1:34:13, 1.17it/s][2025-04-25 11:41:09] (step=0063775) Train Loss: 5.2790, Train Steps/Sec: 1.12 + 72%|██████████████████████████████████████████████████████████████████████████████████ | 16883/23458 [4:47:32<1:32:35, 1.18it/s][2025-04-25 11:41:31] (step=0063800) Train Loss: 5.2655, Train Steps/Sec: 1.16 + 72%|██████████████████████████████████████████████████████████████████████████████████▏ | 16908/23458 [4:47:54<1:35:45, 1.14it/s][2025-04-25 11:41:53] (step=0063825) Train Loss: 5.2647, Train Steps/Sec: 1.12 + 72%|██████████████████████████████████████████████████████████████████████████████████▎ | 16933/23458 [4:48:15<1:33:03, 1.17it/s][2025-04-25 11:42:15] (step=0063850) Train Loss: 5.2725, Train Steps/Sec: 1.17 + 72%|██████████████████████████████████████████████████████████████████████████████████▍ | 16958/23458 [4:48:37<1:32:20, 1.17it/s][2025-04-25 11:42:36] (step=0063875) Train Loss: 5.3431, Train Steps/Sec: 1.17 + 72%|██████████████████████████████████████████████████████████████████████████████████▌ | 16983/23458 [4:48:58<1:31:24, 1.18it/s][2025-04-25 11:42:57] (step=0063900) Train Loss: 5.2987, Train Steps/Sec: 1.16 + 73%|██████████████████████████████████████████████████████████████████████████████████▋ | 17008/23458 [4:49:20<1:33:21, 1.15it/s][2025-04-25 11:43:19] (step=0063925) Train Loss: 5.3018, Train Steps/Sec: 1.17 + 73%|██████████████████████████████████████████████████████████████████████████████████▊ | 17033/23458 [4:49:41<1:32:02, 1.16it/s][2025-04-25 11:43:40] (step=0063950) Train Loss: 5.2648, Train Steps/Sec: 1.16 + 73%|██████████████████████████████████████████████████████████████████████████████████▉ | 17058/23458 [4:50:03<1:30:35, 1.18it/s][2025-04-25 11:44:02] (step=0063975) Train Loss: 5.3315, Train Steps/Sec: 1.17 + 73%|███████████████████████████████████████████████████████████████████████████████████ | 17083/23458 [4:50:24<1:29:59, 1.18it/s][2025-04-25 11:44:23] (step=0064000) Train Loss: 5.2440, Train Steps/Sec: 1.16 +You are using a model of type blip-2 to instantiate a model of type instructblip. This is not supported for all configurations of models and can yield errors. +[2025-04-25 11:44:23] text_config is None. Initializing the text config with default values (`OPTConfig`). +model_name_or_path:/tmp/haozhezhao/model/blip2-flan-t5-xl +image_place_holder: +tokenizer length after expend 32101 +tokenizer length before expend 32100 +100%|██████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████| 4/4 [04:00<00:00, 60.16s/it] +[2025-04-25 11:49:33] Finish Eval in 64000 steps...███████████████████████████████████████████████████████████████████████| 4/4 [03:59<00:00, 59.56s/it] +processing left-padding... +processing left-padding... +processing left-padding... +processing left-padding... +[2025-04-25 11:49:54] Saved checkpoint to checkpoint/Afteri2i_100kckpt_LLAVAT5_empty_Fixed_just_segment_pretrain_stage3_dreambench_recap_400k_t2i_400k_flux_200k_midjourney_150k_recovery_150k_3e4_epoch2_no_rotated/002-GPT-XL/checkpoints/0064000.pt +[2025-04-25 11:49:56] Removed old checkpoint: checkpoint/Afteri2i_100kckpt_LLAVAT5_empty_Fixed_just_segment_pretrain_stage3_dreambench_recap_400k_t2i_400k_flux_200k_midjourney_150k_recovery_150k_3e4_epoch2_no_rotated/002-GPT-XL/checkpoints/0062000.pt + 73%|███████████████████████████████████████████████████████████████████████████████████▏ | 17108/23458 [4:56:18<1:34:05, 1.12it/s][2025-04-25 11:50:18] (step=0064025) Train Loss: 5.2728, Train Steps/Sec: 0.07 + 73%|███████████████████████████████████████████████████████████████████████████████████▎ | 17133/23458 [4:56:40<1:30:21, 1.17it/s][2025-04-25 11:50:39] (step=0064050) Train Loss: 5.2697, Train Steps/Sec: 1.17 + 73%|███████████████████████████████████████████████████████████████████████████████████▍ | 17158/23458 [4:57:01<1:29:01, 1.18it/s][2025-04-25 11:51:00] (step=0064075) Train Loss: 5.3011, Train Steps/Sec: 1.17 + 73%|███████████████████████████████████████████████████████████████████████████████████▌ | 17183/23458 [4:57:23<1:28:31, 1.18it/s][2025-04-25 11:51:22] (step=0064100) Train Loss: 5.2911, Train Steps/Sec: 1.16 + 73%|███████████████████████████████████████████████████████████████████████████████████▋ | 17208/23458 [4:57:45<1:57:58, 1.13s/it][2025-04-25 11:51:44] (step=0064125) Train Loss: 5.2715, Train Steps/Sec: 1.12 + 73%|███████████████████████████████████████████████████████████████████████████████████▋ | 17233/23458 [4:58:06<1:29:28, 1.16it/s][2025-04-25 11:52:05] (step=0064150) Train Loss: 5.2665, Train Steps/Sec: 1.17 + 74%|███████████████████████████████████████████████████████████████████████████████████▊ | 17258/23458 [4:58:28<1:27:34, 1.18it/s][2025-04-25 11:52:27] (step=0064175) Train Loss: 5.3542, Train Steps/Sec: 1.17 + 74%|███████████████████████████████████████████████████████████████████████████████████▉ | 17283/23458 [4:58:49<1:27:34, 1.18it/s][2025-04-25 11:52:48] (step=0064200) Train Loss: 5.2871, Train Steps/Sec: 1.16 + 74%|████████████████████████████████████████████████████████████████████████████████████ | 17308/23458 [4:59:11<1:29:14, 1.15it/s][2025-04-25 11:53:10] (step=0064225) Train Loss: 5.2985, Train Steps/Sec: 1.17 + 74%|████████████████████████████████████████████████████████████████████████████████████▏ | 17333/23458 [4:59:33<1:27:12, 1.17it/s][2025-04-25 11:53:32] (step=0064250) Train Loss: 5.2705, Train Steps/Sec: 1.13 + 74%|████████████████████████████████████████████████████████████████████████████████████▎ | 17358/23458 [4:59:55<1:27:04, 1.17it/s][2025-04-25 11:53:54] (step=0064275) Train Loss: 5.2875, Train Steps/Sec: 1.12 + 74%|████████████████████████████████████████████████████████████████████████████████████▍ | 17383/23458 [5:00:17<1:25:49, 1.18it/s][2025-04-25 11:54:16] (step=0064300) Train Loss: 5.2449, Train Steps/Sec: 1.16 + 74%|████████████████████████████████████████████████████████████████████████████████████▌ | 17408/23458 [5:00:38<1:28:15, 1.14it/s][2025-04-25 11:54:37] (step=0064325) Train Loss: 5.3039, Train Steps/Sec: 1.16 + 74%|████████████████████████████████████████████████████████████████████████████████████▋ | 17433/23458 [5:01:00<1:26:15, 1.16it/s][2025-04-25 11:54:59] (step=0064350) Train Loss: 5.2686, Train Steps/Sec: 1.16 + 74%|████████████████████████████████████████████████████████████████████████████████████▊ | 17458/23458 [5:01:22<1:49:50, 1.10s/it][2025-04-25 11:55:21] (step=0064375) Train Loss: 5.2847, Train Steps/Sec: 1.13 + 75%|████████████████████████████████████████████████████████████████████████████████████▉ | 17483/23458 [5:01:43<1:24:14, 1.18it/s][2025-04-25 11:55:43] (step=0064400) Train Loss: 5.3239, Train Steps/Sec: 1.16 + 75%|█████████████████████████████████████████████████████████████████████████████████████ | 17508/23458 [5:02:06<1:51:12, 1.12s/it][2025-04-25 11:56:05] (step=0064425) Train Loss: 5.2520, Train Steps/Sec: 1.13 + 75%|█████████████████████████████████████████████████████████████████████████████████████▏ | 17533/23458 [5:02:28<1:25:00, 1.16it/s][2025-04-25 11:56:27] (step=0064450) Train Loss: 5.3115, Train Steps/Sec: 1.12 + 75%|█████████████████████████████████████████████████████████████████████████████████████▎ | 17558/23458 [5:02:49<1:23:35, 1.18it/s][2025-04-25 11:56:48] (step=0064475) Train Loss: 5.3204, Train Steps/Sec: 1.17 + 75%|█████████████████████████████████████████████████████████████████████████████████████▍ | 17583/23458 [5:03:11<1:22:46, 1.18it/s][2025-04-25 11:57:10] (step=0064500) Train Loss: 5.2275, Train Steps/Sec: 1.16 + 75%|█████████████████████████████████████████████████████████████████████████████████████▌ | 17608/23458 [5:03:32<1:24:44, 1.15it/s][2025-04-25 11:57:31] (step=0064525) Train Loss: 5.3257, Train Steps/Sec: 1.17 + 75%|█████████████████████████████████████████████████████████████████████████████████████▋ | 17633/23458 [5:03:55<1:50:03, 1.13s/it][2025-04-25 11:57:54] (step=0064550) Train Loss: 5.2728, Train Steps/Sec: 1.09 + 75%|█████████████████████████████████████████████████████████████████████████████████████▊ | 17658/23458 [5:04:17<1:22:02, 1.18it/s][2025-04-25 11:58:16] (step=0064575) Train Loss: 5.3542, Train Steps/Sec: 1.17 + 75%|█████████████████████████████████████████████████████████████████████████████████████▉ | 17683/23458 [5:04:38<1:21:37, 1.18it/s][2025-04-25 11:58:37] (step=0064600) Train Loss: 5.3135, Train Steps/Sec: 1.16 + 75%|██████████████████████████████████████████████████████████████████████████████████████ | 17708/23458 [5:04:59<1:23:35, 1.15it/s][2025-04-25 11:58:58] (step=0064625) Train Loss: 5.3048, Train Steps/Sec: 1.17 + 76%|██████████████████████████████████████████████████████████████████████████████████████▏ | 17733/23458 [5:05:21<1:21:45, 1.17it/s][2025-04-25 11:59:20] (step=0064650) Train Loss: 5.2762, Train Steps/Sec: 1.17 + 76%|██████████████████████████████████████████████████████████████████████████████████████▎ | 17758/23458 [5:05:42<1:20:39, 1.18it/s][2025-04-25 11:59:41] (step=0064675) Train Loss: 5.2831, Train Steps/Sec: 1.17 + 76%|██████████████████████████████████████████████████████████████████████████████████████▍ | 17783/23458 [5:06:04<1:20:15, 1.18it/s][2025-04-25 12:00:03] (step=0064700) Train Loss: 5.3384, Train Steps/Sec: 1.16 + 76%|██████████████████████████████████████████████████████████████████████████████████████▌ | 17808/23458 [5:06:25<1:21:52, 1.15it/s][2025-04-25 12:00:24] (step=0064725) Train Loss: 5.3184, Train Steps/Sec: 1.17 + 76%|██████████████████████████████████████████████████████████████████████████████████████▋ | 17833/23458 [5:06:48<1:20:41, 1.16it/s][2025-04-25 12:00:47] (step=0064750) Train Loss: 5.3196, Train Steps/Sec: 1.11 + 76%|██████████████████████████████████████████████████████████████████████████████████████▊ | 17858/23458 [5:07:09<1:19:22, 1.18it/s][2025-04-25 12:01:08] (step=0064775) Train Loss: 5.2784, Train Steps/Sec: 1.17 + 76%|██████████████████████████████████████████████████████████████████████████████████████▉ | 17883/23458 [5:07:31<1:18:29, 1.18it/s][2025-04-25 12:01:30] (step=0064800) Train Loss: 5.2653, Train Steps/Sec: 1.12 + 76%|███████████████████████████████████████████████████████████████████████████████████████ | 17908/23458 [5:07:53<1:20:33, 1.15it/s][2025-04-25 12:01:52] (step=0064825) Train Loss: 5.3298, Train Steps/Sec: 1.17 + 76%|███████████████████████████████████████████████████████████████████████████████████████▏ | 17933/23458 [5:08:14<1:19:02, 1.16it/s][2025-04-25 12:02:13] (step=0064850) Train Loss: 5.3237, Train Steps/Sec: 1.17 + 77%|███████████████████████████████████████████████████████████████████████████████████████▎ | 17958/23458 [5:08:36<1:18:38, 1.17it/s][2025-04-25 12:02:35] (step=0064875) Train Loss: 5.2528, Train Steps/Sec: 1.16 + 77%|███████████████████████████████████████████████████████████████████████████████████████▍ | 17983/23458 [5:08:58<1:17:53, 1.17it/s][2025-04-25 12:02:57] (step=0064900) Train Loss: 5.3451, Train Steps/Sec: 1.11 + 77%|███████████████████████████████████████████████████████████████████████████████████████▌ | 18008/23458 [5:09:21<1:19:41, 1.14it/s][2025-04-25 12:03:20] (step=0064925) Train Loss: 5.2912, Train Steps/Sec: 1.11 + 77%|███████████████████████████████████████████████████████████████████████████████████████▋ | 18033/23458 [5:09:42<1:18:26, 1.15it/s][2025-04-25 12:03:41] (step=0064950) Train Loss: 5.3011, Train Steps/Sec: 1.16 + 77%|███████████████████████████████████████████████████████████████████████████████████████▊ | 18058/23458 [5:10:04<1:16:44, 1.17it/s][2025-04-25 12:04:03] (step=0064975) Train Loss: 5.3494, Train Steps/Sec: 1.16 + 77%|███████████████████████████████████████████████████████████████████████████████████████▉ | 18083/23458 [5:10:25<1:15:59, 1.18it/s][2025-04-25 12:04:24] (step=0065000) Train Loss: 5.3323, Train Steps/Sec: 1.16 + 77%|████████████████████████████████████████████████████████████████████████████████████████ | 18108/23458 [5:10:47<1:17:33, 1.15it/s][2025-04-25 12:04:46] (step=0065025) Train Loss: 5.2564, Train Steps/Sec: 1.17 + 77%|████████████████████████████████████████████████████████████████████████████████████████ | 18133/23458 [5:11:09<1:16:16, 1.16it/s][2025-04-25 12:05:08] (step=0065050) Train Loss: 5.2641, Train Steps/Sec: 1.12 + 77%|████████████████████████████████████████████████████████████████████████████████████████▏ | 18158/23458 [5:11:30<1:15:45, 1.17it/s][2025-04-25 12:05:30] (step=0065075) Train Loss: 5.2982, Train Steps/Sec: 1.17 + 78%|████████████████████████████████████████████████████████████████████████████████████████▎ | 18183/23458 [5:11:53<1:15:00, 1.17it/s][2025-04-25 12:05:52] (step=0065100) Train Loss: 5.2524, Train Steps/Sec: 1.12 + 78%|████████████████████████████████████████████████████████████████████████████████████████▍ | 18208/23458 [5:12:14<1:16:58, 1.14it/s][2025-04-25 12:06:14] (step=0065125) Train Loss: 5.2975, Train Steps/Sec: 1.16 + 78%|████████████████████████████████████████████████████████████████████████████████████████▌ | 18233/23458 [5:12:36<1:15:44, 1.15it/s][2025-04-25 12:06:35] (step=0065150) Train Loss: 5.3399, Train Steps/Sec: 1.16 + 78%|████████████████████████████████████████████████████████████████████████████████████████▋ | 18258/23458 [5:12:58<1:14:08, 1.17it/s][2025-04-25 12:06:57] (step=0065175) Train Loss: 5.3151, Train Steps/Sec: 1.12 + 78%|████████████████████████████████████████████████████████████████████████████████████████▊ | 18283/23458 [5:13:20<1:14:02, 1.16it/s][2025-04-25 12:07:19] (step=0065200) Train Loss: 5.3030, Train Steps/Sec: 1.16 + 78%|████████████████████████████████████████████████████████████████████████████████████████▉ | 18308/23458 [5:13:41<1:15:01, 1.14it/s][2025-04-25 12:07:40] (step=0065225) Train Loss: 5.3290, Train Steps/Sec: 1.17 + 78%|█████████████████████████████████████████████████████████████████████████████████████████ | 18333/23458 [5:14:03<1:13:08, 1.17it/s][2025-04-25 12:08:02] (step=0065250) Train Loss: 5.1906, Train Steps/Sec: 1.17 + 78%|█████████████████████████████████████████████████████████████████████████████████████████▏ | 18358/23458 [5:14:24<1:12:10, 1.18it/s][2025-04-25 12:08:23] (step=0065275) Train Loss: 5.2619, Train Steps/Sec: 1.17 + 78%|█████████████████████████████████████████████████████████████████████████████████████████▎ | 18383/23458 [5:14:46<1:11:19, 1.19it/s][2025-04-25 12:08:46] (step=0065300) Train Loss: 5.2434, Train Steps/Sec: 1.12 + 78%|█████████████████████████████████████████████████████████████████████████████████████████▍ | 18408/23458 [5:15:08<1:13:08, 1.15it/s][2025-04-25 12:09:07] (step=0065325) Train Loss: 5.2541, Train Steps/Sec: 1.17 + 79%|█████████████████████████████████████████████████████████████████████████████████████████▌ | 18433/23458 [5:15:29<1:12:09, 1.16it/s][2025-04-25 12:09:28] (step=0065350) Train Loss: 5.2584, Train Steps/Sec: 1.17 + 79%|█████████████████████████████████████████████████████████████████████████████████████████▋ | 18458/23458 [5:15:51<1:10:48, 1.18it/s][2025-04-25 12:09:50] (step=0065375) Train Loss: 5.2180, Train Steps/Sec: 1.17 + 79%|█████████████████████████████████████████████████████████████████████████████████████████▊ | 18483/23458 [5:16:12<1:10:16, 1.18it/s][2025-04-25 12:10:11] (step=0065400) Train Loss: 5.3258, Train Steps/Sec: 1.16 + 79%|█████████████████████████████████████████████████████████████████████████████████████████▉ | 18508/23458 [5:16:34<1:11:34, 1.15it/s][2025-04-25 12:10:33] (step=0065425) Train Loss: 5.3915, Train Steps/Sec: 1.17 + 79%|██████████████████████████████████████████████████████████████████████████████████████████ | 18533/23458 [5:16:56<1:10:33, 1.16it/s][2025-04-25 12:10:55] (step=0065450) Train Loss: 5.2688, Train Steps/Sec: 1.13 + 79%|██████████████████████████████████████████████████████████████████████████████████████████▏ | 18558/23458 [5:17:17<1:09:16, 1.18it/s][2025-04-25 12:11:16] (step=0065475) Train Loss: 5.2849, Train Steps/Sec: 1.17 + 79%|██████████████████████████████████████████████████████████████████████████████████████████▎ | 18583/23458 [5:17:40<1:09:02, 1.18it/s][2025-04-25 12:11:39] (step=0065500) Train Loss: 5.2339, Train Steps/Sec: 1.10 + 79%|██████████████████████████████████████████████████████████████████████████████████████████▍ | 18608/23458 [5:18:01<1:10:17, 1.15it/s][2025-04-25 12:12:00] (step=0065525) Train Loss: 5.2441, Train Steps/Sec: 1.17 + 79%|██████████████████████████████████████████████████████████████████████████████████████████▌ | 18633/23458 [5:18:23<1:08:48, 1.17it/s][2025-04-25 12:12:22] (step=0065550) Train Loss: 5.2555, Train Steps/Sec: 1.13 + 80%|██████████████████████████████████████████████████████████████████████████████████████████▋ | 18658/23458 [5:18:45<1:08:18, 1.17it/s][2025-04-25 12:12:44] (step=0065575) Train Loss: 5.2488, Train Steps/Sec: 1.13 + 80%|██████████████████████████████████████████████████████████████████████████████████████████▊ | 18683/23458 [5:19:07<1:07:07, 1.19it/s][2025-04-25 12:13:06] (step=0065600) Train Loss: 5.2734, Train Steps/Sec: 1.16 + 80%|██████████████████████████████████████████████████████████████████████████████████████████▉ | 18708/23458 [5:19:28<1:08:45, 1.15it/s][2025-04-25 12:13:27] (step=0065625) Train Loss: 5.3257, Train Steps/Sec: 1.17 + 80%|███████████████████████████████████████████████████████████████████████████████████████████ | 18733/23458 [5:19:50<1:07:18, 1.17it/s][2025-04-25 12:13:49] (step=0065650) Train Loss: 5.2347, Train Steps/Sec: 1.17 + 80%|███████████████████████████████████████████████████████████████████████████████████████████▏ | 18758/23458 [5:20:11<1:06:27, 1.18it/s][2025-04-25 12:14:10] (step=0065675) Train Loss: 5.2852, Train Steps/Sec: 1.17 + 80%|███████████████████████████████████████████████████████████████████████████████████████████▎ | 18783/23458 [5:20:33<1:06:06, 1.18it/s][2025-04-25 12:14:32] (step=0065700) Train Loss: 5.2503, Train Steps/Sec: 1.12 + 80%|███████████████████████████████████████████████████████████████████████████████████████████▍ | 18808/23458 [5:20:55<1:07:22, 1.15it/s][2025-04-25 12:14:54] (step=0065725) Train Loss: 5.2519, Train Steps/Sec: 1.17 + 80%|███████████████████████████████████████████████████████████████████████████████████████████▌ | 18833/23458 [5:21:17<1:06:58, 1.15it/s][2025-04-25 12:15:16] (step=0065750) Train Loss: 5.2354, Train Steps/Sec: 1.13 + 80%|███████████████████████████████████████████████████████████████████████████████████████████▋ | 18858/23458 [5:21:38<1:05:03, 1.18it/s][2025-04-25 12:15:37] (step=0065775) Train Loss: 5.2931, Train Steps/Sec: 1.17 + 80%|███████████████████████████████████████████████████████████████████████████████████████████▊ | 18883/23458 [5:21:59<1:04:30, 1.18it/s][2025-04-25 12:15:59] (step=0065800) Train Loss: 5.2365, Train Steps/Sec: 1.16 + 81%|███████████████████████████████████████████████████████████████████████████████████████████▉ | 18908/23458 [5:22:21<1:06:06, 1.15it/s][2025-04-25 12:16:20] (step=0065825) Train Loss: 5.2977, Train Steps/Sec: 1.16 + 81%|████████████████████████████████████████████████████████████████████████████████████████████ | 18933/23458 [5:22:42<1:04:41, 1.17it/s][2025-04-25 12:16:42] (step=0065850) Train Loss: 5.2538, Train Steps/Sec: 1.17 + 81%|████████████████████████████████████████████████████████████████████████████████████████████▏ | 18958/23458 [5:23:04<1:03:58, 1.17it/s][2025-04-25 12:17:03] (step=0065875) Train Loss: 5.3097, Train Steps/Sec: 1.17 + 81%|████████████████████████████████████████████████████████████████████████████████████████████▎ | 18983/23458 [5:23:26<1:03:44, 1.17it/s][2025-04-25 12:17:25] (step=0065900) Train Loss: 5.2988, Train Steps/Sec: 1.12 + 81%|████████████████████████████████████████████████████████████████████████████████████████████▎ | 19008/23458 [5:23:48<1:04:26, 1.15it/s][2025-04-25 12:17:47] (step=0065925) Train Loss: 5.2806, Train Steps/Sec: 1.17 + 81%|████████████████████████████████████████████████████████████████████████████████████████████▍ | 19033/23458 [5:24:09<1:03:12, 1.17it/s][2025-04-25 12:18:08] (step=0065950) Train Loss: 5.2496, Train Steps/Sec: 1.17 + 81%|████████████████████████████████████████████████████████████████████████████████████████████▌ | 19058/23458 [5:24:30<1:02:32, 1.17it/s][2025-04-25 12:18:29] (step=0065975) Train Loss: 5.2462, Train Steps/Sec: 1.17 + 81%|████████████████████████████████████████████████████████████████████████████████████████████▋ | 19083/23458 [5:24:52<1:01:34, 1.18it/s][2025-04-25 12:18:51] (step=0066000) Train Loss: 5.3236, Train Steps/Sec: 1.16 +You are using a model of type blip-2 to instantiate a model of type instructblip. This is not supported for all configurations of models and can yield errors. +[2025-04-25 12:18:51] text_config is None. Initializing the text config with default values (`OPTConfig`). +model_name_or_path:/tmp/haozhezhao/model/blip2-flan-t5-xl +image_place_holder: +tokenizer length after expend 32101 +tokenizer length before expend 32100 +100%|██████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████| 4/4 [03:58<00:00, 59.69s/it] +[2025-04-25 12:23:58] Finish Eval in 66000 steps...███████████████████████████████████████████████████████████████████████| 4/4 [03:58<00:00, 59.34s/it] +processing left-padding... +processing left-padding... +processing left-padding... +processing left-padding... +[2025-04-25 12:24:17] Saved checkpoint to checkpoint/Afteri2i_100kckpt_LLAVAT5_empty_Fixed_just_segment_pretrain_stage3_dreambench_recap_400k_t2i_400k_flux_200k_midjourney_150k_recovery_150k_3e4_epoch2_no_rotated/002-GPT-XL/checkpoints/0066000.pt +[2025-04-25 12:24:20] Removed old checkpoint: checkpoint/Afteri2i_100kckpt_LLAVAT5_empty_Fixed_just_segment_pretrain_stage3_dreambench_recap_400k_t2i_400k_flux_200k_midjourney_150k_recovery_150k_3e4_epoch2_no_rotated/002-GPT-XL/checkpoints/0064000.pt + 81%|████████████████████████████████████████████████████████████████████████████████████████████▊ | 19108/23458 [5:30:43<1:04:40, 1.12it/s][2025-04-25 12:24:42] (step=0066025) Train Loss: 5.3122, Train Steps/Sec: 0.07 + 82%|████████████████████████████████████████████████████████████████████████████████████████████▉ | 19133/23458 [5:31:04<1:01:37, 1.17it/s][2025-04-25 12:25:03] (step=0066050) Train Loss: 5.2722, Train Steps/Sec: 1.17 + 82%|█████████████████████████████████████████████████████████████████████████████████████████████ | 19158/23458 [5:31:26<1:01:23, 1.17it/s][2025-04-25 12:25:25] (step=0066075) Train Loss: 5.3333, Train Steps/Sec: 1.17 + 82%|█████████████████████████████████████████████████████████████████████████████████████████████▏ | 19183/23458 [5:31:48<1:00:08, 1.18it/s][2025-04-25 12:25:47] (step=0066100) Train Loss: 5.2389, Train Steps/Sec: 1.12 + 82%|█████████████████████████████████████████████████████████████████████████████████████████████▎ | 19208/23458 [5:32:09<1:01:27, 1.15it/s][2025-04-25 12:26:09] (step=0066125) Train Loss: 5.2791, Train Steps/Sec: 1.17 + 82%|█████████████████████████████████████████████████████████████████████████████████████████████▍ | 19233/23458 [5:32:31<1:00:27, 1.16it/s][2025-04-25 12:26:30] (step=0066150) Train Loss: 5.2844, Train Steps/Sec: 1.17 + 82%|███████████████████████████████████████████████████████████████████████████████████████████████▏ | 19258/23458 [5:32:52<59:29, 1.18it/s][2025-04-25 12:26:51] (step=0066175) Train Loss: 5.2506, Train Steps/Sec: 1.17 + 82%|███████████████████████████████████████████████████████████████████████████████████████████████▎ | 19283/23458 [5:33:14<58:35, 1.19it/s][2025-04-25 12:27:13] (step=0066200) Train Loss: 5.3416, Train Steps/Sec: 1.12 + 82%|█████████████████████████████████████████████████████████████████████████████████████████████▊ | 19308/23458 [5:33:37<1:00:44, 1.14it/s][2025-04-25 12:27:36] (step=0066225) Train Loss: 5.2626, Train Steps/Sec: 1.13 + 82%|███████████████████████████████████████████████████████████████████████████████████████████████▌ | 19333/23458 [5:33:58<58:51, 1.17it/s][2025-04-25 12:27:57] (step=0066250) Train Loss: 5.2669, Train Steps/Sec: 1.17 + 83%|███████████████████████████████████████████████████████████████████████████████████████████████▋ | 19358/23458 [5:34:19<58:10, 1.17it/s][2025-04-25 12:28:18] (step=0066275) Train Loss: 5.2152, Train Steps/Sec: 1.17 + 83%|███████████████████████████████████████████████████████████████████████████████████████████████▊ | 19383/23458 [5:34:41<57:39, 1.18it/s][2025-04-25 12:28:40] (step=0066300) Train Loss: 5.3041, Train Steps/Sec: 1.17 + 83%|███████████████████████████████████████████████████████████████████████████████████████████████▉ | 19408/23458 [5:35:02<58:37, 1.15it/s][2025-04-25 12:29:01] (step=0066325) Train Loss: 5.2513, Train Steps/Sec: 1.17 + 83%|██████████████████████████████████████████████████████████████████████████████████████████████▍ | 19433/23458 [5:35:24<1:05:17, 1.03it/s][2025-04-25 12:29:23] (step=0066350) Train Loss: 5.3461, Train Steps/Sec: 1.13 + 83%|████████████████████████████████████████████████████████████████████████████████████████████████▏ | 19458/23458 [5:35:46<56:33, 1.18it/s][2025-04-25 12:29:45] (step=0066375) Train Loss: 5.2607, Train Steps/Sec: 1.17 + 83%|████████████████████████████████████████████████████████████████████████████████████████████████▎ | 19483/23458 [5:36:07<56:11, 1.18it/s][2025-04-25 12:30:06] (step=0066400) Train Loss: 5.2777, Train Steps/Sec: 1.17 + 83%|████████████████████████████████████████████████████████████████████████████████████████████████▍ | 19508/23458 [5:36:29<57:31, 1.14it/s][2025-04-25 12:30:28] (step=0066425) Train Loss: 5.2717, Train Steps/Sec: 1.13 + 83%|████████████████████████████████████████████████████████████████████████████████████████████████▌ | 19533/23458 [5:36:51<55:58, 1.17it/s][2025-04-25 12:30:50] (step=0066450) Train Loss: 5.3400, Train Steps/Sec: 1.17 + 83%|████████████████████████████████████████████████████████████████████████████████████████████████▋ | 19558/23458 [5:37:12<55:19, 1.17it/s][2025-04-25 12:31:12] (step=0066475) Train Loss: 5.2986, Train Steps/Sec: 1.15 + 83%|████████████████████████████████████████████████████████████████████████████████████████████████▊ | 19583/23458 [5:37:34<54:26, 1.19it/s][2025-04-25 12:31:33] (step=0066500) Train Loss: 5.2846, Train Steps/Sec: 1.17 + 84%|████████████████████████████████████████████████████████████████████████████████████████████████▉ | 19608/23458 [5:37:55<55:50, 1.15it/s][2025-04-25 12:31:54] (step=0066525) Train Loss: 5.3451, Train Steps/Sec: 1.17 + 84%|█████████████████████████████████████████████████████████████████████████████████████████████████ | 19633/23458 [5:38:17<54:32, 1.17it/s][2025-04-25 12:32:16] (step=0066550) Train Loss: 5.1930, Train Steps/Sec: 1.17 + 84%|█████████████████████████████████████████████████████████████████████████████████████████████████▏ | 19658/23458 [5:38:38<54:36, 1.16it/s][2025-04-25 12:32:37] (step=0066575) Train Loss: 5.2591, Train Steps/Sec: 1.17 + 84%|█████████████████████████████████████████████████████████████████████████████████████████████████▎ | 19683/23458 [5:38:59<53:31, 1.18it/s][2025-04-25 12:32:59] (step=0066600) Train Loss: 5.2801, Train Steps/Sec: 1.16 + 84%|█████████████████████████████████████████████████████████████████████████████████████████████████▍ | 19708/23458 [5:39:22<58:29, 1.07it/s][2025-04-25 12:33:21] (step=0066625) Train Loss: 5.2892, Train Steps/Sec: 1.12 + 84%|█████████████████████████████████████████████████████████████████████████████████████████████████▌ | 19733/23458 [5:39:43<53:09, 1.17it/s][2025-04-25 12:33:42] (step=0066650) Train Loss: 5.2893, Train Steps/Sec: 1.17 + 84%|█████████████████████████████████████████████████████████████████████████████████████████████████▋ | 19758/23458 [5:40:05<52:21, 1.18it/s][2025-04-25 12:34:04] (step=0066675) Train Loss: 5.2175, Train Steps/Sec: 1.17 + 84%|█████████████████████████████████████████████████████████████████████████████████████████████████▊ | 19783/23458 [5:40:26<51:44, 1.18it/s][2025-04-25 12:34:25] (step=0066700) Train Loss: 5.2385, Train Steps/Sec: 1.16 + 84%|█████████████████████████████████████████████████████████████████████████████████████████████████▉ | 19808/23458 [5:40:47<52:54, 1.15it/s][2025-04-25 12:34:47] (step=0066725) Train Loss: 5.2278, Train Steps/Sec: 1.17 + 85%|██████████████████████████████████████████████████████████████████████████████████████████████████ | 19833/23458 [5:41:11<52:49, 1.14it/s][2025-04-25 12:35:10] (step=0066750) Train Loss: 5.2007, Train Steps/Sec: 1.07 + 85%|██████████████████████████████████████████████████████████████████████████████████████████████████▏ | 19858/23458 [5:41:32<50:59, 1.18it/s][2025-04-25 12:35:31] (step=0066775) Train Loss: 5.2182, Train Steps/Sec: 1.17 + 85%|██████████████████████████████████████████████████████████████████████████████████████████████████▎ | 19883/23458 [5:41:54<50:12, 1.19it/s][2025-04-25 12:35:53] (step=0066800) Train Loss: 5.2558, Train Steps/Sec: 1.17 + 85%|██████████████████████████████████████████████████████████████████████████████████████████████████▍ | 19908/23458 [5:42:15<51:28, 1.15it/s][2025-04-25 12:36:14] (step=0066825) Train Loss: 5.2631, Train Steps/Sec: 1.17 + 85%|██████████████████████████████████████████████████████████████████████████████████████████████████▌ | 19933/23458 [5:42:37<50:31, 1.16it/s][2025-04-25 12:36:36] (step=0066850) Train Loss: 5.3439, Train Steps/Sec: 1.13 + 85%|██████████████████████████████████████████████████████████████████████████████████████████████████▋ | 19958/23458 [5:42:59<56:56, 1.02it/s][2025-04-25 12:36:59] (step=0066875) Train Loss: 5.3521, Train Steps/Sec: 1.13 + 85%|██████████████████████████████████████████████████████████████████████████████████████████████████▊ | 19983/23458 [5:43:21<48:54, 1.18it/s][2025-04-25 12:37:20] (step=0066900) Train Loss: 5.2771, Train Steps/Sec: 1.16 + 85%|██████████████████████████████████████████████████████████████████████████████████████████████████▉ | 20008/23458 [5:43:42<50:07, 1.15it/s][2025-04-25 12:37:41] (step=0066925) Train Loss: 5.1950, Train Steps/Sec: 1.17 + 85%|███████████████████████████████████████████████████████████████████████████████████████████████████ | 20033/23458 [5:44:04<48:54, 1.17it/s][2025-04-25 12:38:03] (step=0066950) Train Loss: 5.2399, Train Steps/Sec: 1.17 + 86%|███████████████████████████████████████████████████████████████████████████████████████████████████▏ | 20058/23458 [5:44:25<48:01, 1.18it/s][2025-04-25 12:38:24] (step=0066975) Train Loss: 5.2870, Train Steps/Sec: 1.17 + 86%|███████████████████████████████████████████████████████████████████████████████████████████████████▎ | 20083/23458 [5:44:46<47:26, 1.19it/s][2025-04-25 12:38:46] (step=0067000) Train Loss: 5.3785, Train Steps/Sec: 1.17 + 86%|███████████████████████████████████████████████████████████████████████████████████████████████████▍ | 20108/23458 [5:45:09<48:41, 1.15it/s][2025-04-25 12:39:08] (step=0067025) Train Loss: 5.3322, Train Steps/Sec: 1.13 + 86%|███████████████████████████████████████████████████████████████████████████████████████████████████▌ | 20133/23458 [5:45:30<47:35, 1.16it/s][2025-04-25 12:39:29] (step=0067050) Train Loss: 5.2929, Train Steps/Sec: 1.17 + 86%|███████████████████████████████████████████████████████████████████████████████████████████████████▋ | 20158/23458 [5:45:52<46:53, 1.17it/s][2025-04-25 12:39:51] (step=0067075) Train Loss: 5.2622, Train Steps/Sec: 1.13 + 86%|███████████████████████████████████████████████████████████████████████████████████████████████████▊ | 20183/23458 [5:46:14<46:02, 1.19it/s][2025-04-25 12:40:13] (step=0067100) Train Loss: 5.2364, Train Steps/Sec: 1.17 + 86%|███████████████████████████████████████████████████████████████████████████████████████████████████▉ | 20208/23458 [5:46:35<47:30, 1.14it/s][2025-04-25 12:40:34] (step=0067125) Train Loss: 5.3073, Train Steps/Sec: 1.17 + 86%|████████████████████████████████████████████████████████████████████████████████████████████████████ | 20233/23458 [5:46:56<46:05, 1.17it/s][2025-04-25 12:40:56] (step=0067150) Train Loss: 5.2526, Train Steps/Sec: 1.17 + 86%|████████████████████████████████████████████████████████████████████████████████████████████████████▏ | 20258/23458 [5:47:18<45:20, 1.18it/s][2025-04-25 12:41:17] (step=0067175) Train Loss: 5.2587, Train Steps/Sec: 1.17 + 86%|████████████████████████████████████████████████████████████████████████████████████████████████████▎ | 20283/23458 [5:47:39<44:42, 1.18it/s][2025-04-25 12:41:38] (step=0067200) Train Loss: 5.2721, Train Steps/Sec: 1.17 + 87%|████████████████████████████████████████████████████████████████████████████████████████████████████▍ | 20308/23458 [5:48:01<45:43, 1.15it/s][2025-04-25 12:42:00] (step=0067225) Train Loss: 5.2212, Train Steps/Sec: 1.17 + 87%|████████████████████████████████████████████████████████████████████████████████████████████████████▌ | 20333/23458 [5:48:22<44:38, 1.17it/s][2025-04-25 12:42:21] (step=0067250) Train Loss: 5.3060, Train Steps/Sec: 1.17 + 87%|████████████████████████████████████████████████████████████████████████████████████████████████████▋ | 20358/23458 [5:48:43<43:58, 1.17it/s][2025-04-25 12:42:43] (step=0067275) Train Loss: 5.3428, Train Steps/Sec: 1.17 + 87%|████████████████████████████████████████████████████████████████████████████████████████████████████▊ | 20383/23458 [5:49:05<43:22, 1.18it/s][2025-04-25 12:43:04] (step=0067300) Train Loss: 5.2951, Train Steps/Sec: 1.16 + 87%|████████████████████████████████████████████████████████████████████████████████████████████████████▉ | 20408/23458 [5:49:26<45:01, 1.13it/s][2025-04-25 12:43:26] (step=0067325) Train Loss: 5.1986, Train Steps/Sec: 1.16 + 87%|█████████████████████████████████████████████████████████████████████████████████████████████████████ | 20433/23458 [5:49:49<50:04, 1.01it/s][2025-04-25 12:43:48] (step=0067350) Train Loss: 5.3209, Train Steps/Sec: 1.11 + 87%|█████████████████████████████████████████████████████████████████████████████████████████████████████▏ | 20458/23458 [5:50:11<43:29, 1.15it/s][2025-04-25 12:44:10] (step=0067375) Train Loss: 5.2948, Train Steps/Sec: 1.16 + 87%|█████████████████████████████████████████████████████████████████████████████████████████████████████▎ | 20483/23458 [5:50:33<42:15, 1.17it/s][2025-04-25 12:44:32] (step=0067400) Train Loss: 5.2959, Train Steps/Sec: 1.12 + 87%|█████████████████████████████████████████████████████████████████████████████████████████████████████▍ | 20508/23458 [5:50:54<42:54, 1.15it/s][2025-04-25 12:44:53] (step=0067425) Train Loss: 5.2064, Train Steps/Sec: 1.17 + 88%|█████████████████████████████████████████████████████████████████████████████████████████████████████▌ | 20533/23458 [5:51:16<41:49, 1.17it/s][2025-04-25 12:45:15] (step=0067450) Train Loss: 5.3147, Train Steps/Sec: 1.17 + 88%|█████████████████████████████████████████████████████████████████████████████████████████████████████▋ | 20558/23458 [5:51:38<45:46, 1.06it/s][2025-04-25 12:45:37] (step=0067475) Train Loss: 5.2386, Train Steps/Sec: 1.12 + 88%|█████████████████████████████████████████████████████████████████████████████████████████████████████▊ | 20583/23458 [5:52:01<41:41, 1.15it/s][2025-04-25 12:46:00] (step=0067500) Train Loss: 5.2690, Train Steps/Sec: 1.07 + 88%|█████████████████████████████████████████████████████████████████████████████████████████████████████▉ | 20608/23458 [5:52:24<52:53, 1.11s/it][2025-04-25 12:46:23] (step=0067525) Train Loss: 5.2822, Train Steps/Sec: 1.13 + 88%|██████████████████████████████████████████████████████████████████████████████████████████████████████ | 20633/23458 [5:52:45<40:15, 1.17it/s][2025-04-25 12:46:44] (step=0067550) Train Loss: 5.2633, Train Steps/Sec: 1.17 + 88%|██████████████████████████████████████████████████████████████████████████████████████████████████████▏ | 20658/23458 [5:53:06<39:37, 1.18it/s][2025-04-25 12:47:05] (step=0067575) Train Loss: 5.3224, Train Steps/Sec: 1.17 + 88%|██████████████████████████████████████████████████████████████████████████████████████████████████████▎ | 20683/23458 [5:53:28<39:06, 1.18it/s][2025-04-25 12:47:27] (step=0067600) Train Loss: 5.2718, Train Steps/Sec: 1.17 + 88%|██████████████████████████████████████████████████████████████████████████████████████████████████████▍ | 20708/23458 [5:53:49<40:01, 1.15it/s][2025-04-25 12:47:48] (step=0067625) Train Loss: 5.3447, Train Steps/Sec: 1.17 + 88%|██████████████████████████████████████████████████████████████████████████████████████████████████████▌ | 20733/23458 [5:54:10<39:02, 1.16it/s][2025-04-25 12:48:10] (step=0067650) Train Loss: 5.2827, Train Steps/Sec: 1.17 + 88%|██████████████████████████████████████████████████████████████████████████████████████████████████████▋ | 20758/23458 [5:54:33<46:25, 1.03s/it][2025-04-25 12:48:32] (step=0067675) Train Loss: 5.2485, Train Steps/Sec: 1.13 + 89%|██████████████████████████████████████████████████████████████████████████████████████████████████████▊ | 20783/23458 [5:54:54<37:42, 1.18it/s][2025-04-25 12:48:53] (step=0067700) Train Loss: 5.3151, Train Steps/Sec: 1.17 + 89%|██████████████████████████████████████████████████████████████████████████████████████████████████████▉ | 20808/23458 [5:55:16<43:37, 1.01it/s][2025-04-25 12:49:15] (step=0067725) Train Loss: 5.2613, Train Steps/Sec: 1.13 + 89%|███████████████████████████████████████████████████████████████████████████████████████████████████████ | 20833/23458 [5:55:38<37:36, 1.16it/s][2025-04-25 12:49:37] (step=0067750) Train Loss: 5.2742, Train Steps/Sec: 1.17 + 89%|███████████████████████████████████████████████████████████████████████████████████████████████████████▏ | 20858/23458 [5:55:59<36:54, 1.17it/s][2025-04-25 12:49:58] (step=0067775) Train Loss: 5.3248, Train Steps/Sec: 1.17 + 89%|███████████████████████████████████████████████████████████████████████████████████████████████████████▎ | 20883/23458 [5:56:21<36:15, 1.18it/s][2025-04-25 12:50:20] (step=0067800) Train Loss: 5.2314, Train Steps/Sec: 1.16 + 89%|███████████████████████████████████████████████████████████████████████████████████████████████████████▍ | 20908/23458 [5:56:42<37:02, 1.15it/s][2025-04-25 12:50:41] (step=0067825) Train Loss: 5.2049, Train Steps/Sec: 1.17 + 89%|███████████████████████████████████████████████████████████████████████████████████████████████████████▌ | 20933/23458 [5:57:03<35:53, 1.17it/s][2025-04-25 12:51:02] (step=0067850) Train Loss: 5.3367, Train Steps/Sec: 1.17 + 89%|███████████████████████████████████████████████████████████████████████████████████████████████████████▋ | 20958/23458 [5:57:25<35:30, 1.17it/s][2025-04-25 12:51:24] (step=0067875) Train Loss: 5.2525, Train Steps/Sec: 1.17 + 89%|███████████████████████████████████████████████████████████████████████████████████████████████████████▊ | 20983/23458 [5:57:46<34:52, 1.18it/s][2025-04-25 12:51:45] (step=0067900) Train Loss: 5.2579, Train Steps/Sec: 1.17 + 90%|███████████████████████████████████████████████████████████████████████████████████████████████████████▉ | 21008/23458 [5:58:08<35:31, 1.15it/s][2025-04-25 12:52:07] (step=0067925) Train Loss: 5.2871, Train Steps/Sec: 1.17 + 90%|████████████████████████████████████████████████████████████████████████████████████████████████████████ | 21033/23458 [5:58:29<34:41, 1.16it/s][2025-04-25 12:52:28] (step=0067950) Train Loss: 5.2530, Train Steps/Sec: 1.17 + 90%|████████████████████████████████████████████████████████████████████████████████████████████████████████▏ | 21058/23458 [5:58:50<33:58, 1.18it/s][2025-04-25 12:52:49] (step=0067975) Train Loss: 5.2696, Train Steps/Sec: 1.17 + 90%|████████████████████████████████████████████████████████████████████████████████████████████████████████▎ | 21083/23458 [5:59:12<33:30, 1.18it/s][2025-04-25 12:53:11] (step=0068000) Train Loss: 5.3478, Train Steps/Sec: 1.16 +You are using a model of type blip-2 to instantiate a model of type instructblip. This is not supported for all configurations of models and can yield errors. +[2025-04-25 12:53:11] text_config is None. Initializing the text config with default values (`OPTConfig`). +model_name_or_path:/tmp/haozhezhao/model/blip2-flan-t5-xl +image_place_holder: +tokenizer length after expend 32101 +tokenizer length before expend 32100 +100%|██████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████| 4/4 [03:55<00:00, 58.97s/it] +[2025-04-25 12:58:15] Finish Eval in 68000 steps...███████████████████████████████████████████████████████████████████████| 4/4 [03:55<00:00, 58.58s/it] +processing left-padding... +processing left-padding... +processing left-padding... +processing left-padding... +[2025-04-25 12:58:35] Saved checkpoint to checkpoint/Afteri2i_100kckpt_LLAVAT5_empty_Fixed_just_segment_pretrain_stage3_dreambench_recap_400k_t2i_400k_flux_200k_midjourney_150k_recovery_150k_3e4_epoch2_no_rotated/002-GPT-XL/checkpoints/0068000.pt +[2025-04-25 12:58:37] Removed old checkpoint: checkpoint/Afteri2i_100kckpt_LLAVAT5_empty_Fixed_just_segment_pretrain_stage3_dreambench_recap_400k_t2i_400k_flux_200k_midjourney_150k_recovery_150k_3e4_epoch2_no_rotated/002-GPT-XL/checkpoints/0066000.pt + 90%|████████████████████████████████████████████████████████████████████████████████████████████████████████▍ | 21108/23458 [6:05:00<34:49, 1.12it/s][2025-04-25 12:58:59] (step=0068025) Train Loss: 5.3284, Train Steps/Sec: 0.07 + 90%|████████████████████████████████████████████████████████████████████████████████████████████████████████▌ | 21133/23458 [6:05:22<34:55, 1.11it/s][2025-04-25 12:59:21] (step=0068050) Train Loss: 5.2549, Train Steps/Sec: 1.13 + 90%|████████████████████████████████████████████████████████████████████████████████████████████████████████▋ | 21158/23458 [6:05:44<42:46, 1.12s/it][2025-04-25 12:59:43] (step=0068075) Train Loss: 5.3152, Train Steps/Sec: 1.13 + 90%|████████████████████████████████████████████████████████████████████████████████████████████████████████▊ | 21183/23458 [6:06:05<31:55, 1.19it/s][2025-04-25 13:00:04] (step=0068100) Train Loss: 5.2639, Train Steps/Sec: 1.17 + 90%|████████████████████████████████████████████████████████████████████████████████████████████████████████▊ | 21208/23458 [6:06:27<32:36, 1.15it/s][2025-04-25 13:00:26] (step=0068125) Train Loss: 5.3326, Train Steps/Sec: 1.17 + 91%|████████████████████████████████████████████████████████████████████████████████████████████████████████▉ | 21233/23458 [6:06:49<38:25, 1.04s/it][2025-04-25 13:00:48] (step=0068150) Train Loss: 5.2317, Train Steps/Sec: 1.12 + 91%|█████████████████████████████████████████████████████████████████████████████████████████████████████████ | 21258/23458 [6:07:10<31:05, 1.18it/s][2025-04-25 13:01:09] (step=0068175) Train Loss: 5.2620, Train Steps/Sec: 1.17 + 91%|█████████████████████████████████████████████████████████████████████████████████████████████████████████▏ | 21283/23458 [6:07:33<30:37, 1.18it/s][2025-04-25 13:01:32] (step=0068200) Train Loss: 5.2920, Train Steps/Sec: 1.12 + 91%|█████████████████████████████████████████████████████████████████████████████████████████████████████████▎ | 21308/23458 [6:07:55<31:15, 1.15it/s][2025-04-25 13:01:54] (step=0068225) Train Loss: 5.2847, Train Steps/Sec: 1.11 + 91%|█████████████████████████████████████████████████████████████████████████████████████████████████████████▍ | 21333/23458 [6:08:16<30:12, 1.17it/s][2025-04-25 13:02:15] (step=0068250) Train Loss: 5.2895, Train Steps/Sec: 1.17 + 91%|█████████████████████████████████████████████████████████████████████████████████████████████████████████▌ | 21358/23458 [6:08:38<29:43, 1.18it/s][2025-04-25 13:02:37] (step=0068275) Train Loss: 5.2453, Train Steps/Sec: 1.17 + 91%|█████████████████████████████████████████████████████████████████████████████████████████████████████████▋ | 21383/23458 [6:08:59<29:07, 1.19it/s][2025-04-25 13:02:58] (step=0068300) Train Loss: 5.2884, Train Steps/Sec: 1.17 + 91%|█████████████████████████████████████████████████████████████████████████████████████████████████████████▊ | 21408/23458 [6:09:21<29:39, 1.15it/s][2025-04-25 13:03:20] (step=0068325) Train Loss: 5.2455, Train Steps/Sec: 1.17 + 91%|█████████████████████████████████████████████████████████████████████████████████████████████████████████▉ | 21433/23458 [6:09:43<28:57, 1.17it/s][2025-04-25 13:03:42] (step=0068350) Train Loss: 5.2737, Train Steps/Sec: 1.13 + 91%|██████████████████████████████████████████████████████████████████████████████████████████████████████████ | 21458/23458 [6:10:04<28:20, 1.18it/s][2025-04-25 13:04:03] (step=0068375) Train Loss: 5.2648, Train Steps/Sec: 1.17 + 92%|██████████████████████████████████████████████████████████████████████████████████████████████████████████▏ | 21483/23458 [6:10:27<27:51, 1.18it/s][2025-04-25 13:04:26] (step=0068400) Train Loss: 5.2624, Train Steps/Sec: 1.10 + 92%|██████████████████████████████████████████████████████████████████████████████████████████████████████████▎ | 21508/23458 [6:10:48<28:16, 1.15it/s][2025-04-25 13:04:47] (step=0068425) Train Loss: 5.2105, Train Steps/Sec: 1.17 + 92%|██████████████████████████████████████████████████████████████████████████████████████████████████████████▍ | 21533/23458 [6:11:10<27:25, 1.17it/s][2025-04-25 13:05:09] (step=0068450) Train Loss: 5.2519, Train Steps/Sec: 1.17 + 92%|██████████████████████████████████████████████████████████████████████████████████████████████████████████▌ | 21558/23458 [6:11:31<26:50, 1.18it/s][2025-04-25 13:05:30] (step=0068475) Train Loss: 5.2855, Train Steps/Sec: 1.17 + 92%|██████████████████████████████████████████████████████████████████████████████████████████████████████████▋ | 21583/23458 [6:11:52<26:27, 1.18it/s][2025-04-25 13:05:51] (step=0068500) Train Loss: 5.2132, Train Steps/Sec: 1.16 + 92%|██████████████████████████████████████████████████████████████████████████████████████████████████████████▊ | 21608/23458 [6:12:14<26:55, 1.15it/s][2025-04-25 13:06:13] (step=0068525) Train Loss: 5.3203, Train Steps/Sec: 1.17 + 92%|██████████████████████████████████████████████████████████████████████████████████████████████████████████▉ | 21633/23458 [6:12:35<26:08, 1.16it/s][2025-04-25 13:06:34] (step=0068550) Train Loss: 5.2444, Train Steps/Sec: 1.17 + 92%|███████████████████████████████████████████████████████████████████████████████████████████████████████████ | 21658/23458 [6:12:57<25:28, 1.18it/s][2025-04-25 13:06:56] (step=0068575) Train Loss: 5.2654, Train Steps/Sec: 1.17 + 92%|███████████████████████████████████████████████████████████████████████████████████████████████████████████▏ | 21683/23458 [6:13:18<25:04, 1.18it/s][2025-04-25 13:07:17] (step=0068600) Train Loss: 5.2516, Train Steps/Sec: 1.16 + 93%|███████████████████████████████████████████████████████████████████████████████████████████████████████████▎ | 21708/23458 [6:13:39<25:23, 1.15it/s][2025-04-25 13:07:38] (step=0068625) Train Loss: 5.2787, Train Steps/Sec: 1.17 + 93%|███████████████████████████████████████████████████████████████████████████████████████████████████████████▍ | 21733/23458 [6:14:01<24:37, 1.17it/s][2025-04-25 13:08:00] (step=0068650) Train Loss: 5.2895, Train Steps/Sec: 1.17 + 93%|███████████████████████████████████████████████████████████████████████████████████████████████████████████▌ | 21758/23458 [6:14:22<24:03, 1.18it/s][2025-04-25 13:08:21] (step=0068675) Train Loss: 5.2549, Train Steps/Sec: 1.17 + 93%|███████████████████████████████████████████████████████████████████████████████████████████████████████████▋ | 21783/23458 [6:14:44<24:37, 1.13it/s][2025-04-25 13:08:44] (step=0068700) Train Loss: 5.1919, Train Steps/Sec: 1.11 + 93%|███████████████████████████████████████████████████████████████████████████████████████████████████████████▊ | 21808/23458 [6:15:06<23:52, 1.15it/s][2025-04-25 13:09:05] (step=0068725) Train Loss: 5.2938, Train Steps/Sec: 1.17 + 93%|███████████████████████████████████████████████████████████████████████████████████████████████████████████▉ | 21833/23458 [6:15:28<23:15, 1.16it/s][2025-04-25 13:09:27] (step=0068750) Train Loss: 5.2770, Train Steps/Sec: 1.12 + 93%|████████████████████████████████████████████████████████████████████████████████████████████████████████████ | 21858/23458 [6:15:50<22:42, 1.17it/s][2025-04-25 13:09:49] (step=0068775) Train Loss: 5.1915, Train Steps/Sec: 1.17 + 93%|████████████████████████████████████████████████████████████████████████████████████████████████████████████▏ | 21883/23458 [6:16:11<22:09, 1.18it/s][2025-04-25 13:10:11] (step=0068800) Train Loss: 5.2342, Train Steps/Sec: 1.12 + 93%|████████████████████████████████████████████████████████████████████████████████████████████████████████████▎ | 21908/23458 [6:16:34<22:27, 1.15it/s][2025-04-25 13:10:33] (step=0068825) Train Loss: 5.2811, Train Steps/Sec: 1.13 + 93%|████████████████████████████████████████████████████████████████████████████████████████████████████████████▍ | 21933/23458 [6:16:56<21:41, 1.17it/s][2025-04-25 13:10:55] (step=0068850) Train Loss: 5.2294, Train Steps/Sec: 1.13 + 94%|████████████████████████████████████████████████████████████████████████████████████████████████████████████▌ | 21958/23458 [6:17:18<21:12, 1.18it/s][2025-04-25 13:11:17] (step=0068875) Train Loss: 5.3303, Train Steps/Sec: 1.17 + 94%|████████████████████████████████████████████████████████████████████████████████████████████████████████████▋ | 21983/23458 [6:17:39<20:45, 1.18it/s][2025-04-25 13:11:38] (step=0068900) Train Loss: 5.2596, Train Steps/Sec: 1.17 + 94%|████████████████████████████████████████████████████████████████████████████████████████████████████████████▊ | 22008/23458 [6:18:01<21:03, 1.15it/s][2025-04-25 13:12:00] (step=0068925) Train Loss: 5.2830, Train Steps/Sec: 1.17 + 94%|████████████████████████████████████████████████████████████████████████████████████████████████████████████▉ | 22033/23458 [6:18:23<20:27, 1.16it/s][2025-04-25 13:12:22] (step=0068950) Train Loss: 5.2907, Train Steps/Sec: 1.12 + 94%|█████████████████████████████████████████████████████████████████████████████████████████████████████████████ | 22058/23458 [6:18:44<19:52, 1.17it/s][2025-04-25 13:12:43] (step=0068975) Train Loss: 5.3233, Train Steps/Sec: 1.17 + 94%|█████████████████████████████████████████████████████████████████████████████████████████████████████████████▏ | 22083/23458 [6:19:06<19:20, 1.19it/s][2025-04-25 13:13:06] (step=0069000) Train Loss: 5.2842, Train Steps/Sec: 1.12 + 94%|█████████████████████████████████████████████████████████████████████████████████████████████████████████████▎ | 22108/23458 [6:19:28<19:30, 1.15it/s][2025-04-25 13:13:27] (step=0069025) Train Loss: 5.2869, Train Steps/Sec: 1.17 + 94%|█████████████████████████████████████████████████████████████████████████████████████████████████████████████▍ | 22133/23458 [6:19:50<19:59, 1.10it/s][2025-04-25 13:13:49] (step=0069050) Train Loss: 5.2529, Train Steps/Sec: 1.12 + 94%|█████████████████████████████████████████████████████████████████████████████████████████████████████████████▌ | 22158/23458 [6:20:12<18:40, 1.16it/s][2025-04-25 13:14:11] (step=0069075) Train Loss: 5.3417, Train Steps/Sec: 1.17 + 95%|█████████████████████████████████████████████████████████████████████████████████████████████████████████████▋ | 22183/23458 [6:20:33<18:05, 1.17it/s][2025-04-25 13:14:32] (step=0069100) Train Loss: 5.2065, Train Steps/Sec: 1.15 + 95%|█████████████████████████████████████████████████████████████████████████████████████████████████████████████▊ | 22208/23458 [6:20:55<18:08, 1.15it/s][2025-04-25 13:14:54] (step=0069125) Train Loss: 5.3106, Train Steps/Sec: 1.17 + 95%|█████████████████████████████████████████████████████████████████████████████████████████████████████████████▉ | 22233/23458 [6:21:16<17:28, 1.17it/s][2025-04-25 13:15:15] (step=0069150) Train Loss: 5.2046, Train Steps/Sec: 1.17 + 95%|██████████████████████████████████████████████████████████████████████████████████████████████████████████████ | 22258/23458 [6:21:38<16:59, 1.18it/s][2025-04-25 13:15:37] (step=0069175) Train Loss: 5.2311, Train Steps/Sec: 1.17 + 95%|██████████████████████████████████████████████████████████████████████████████████████████████████████████████▏ | 22283/23458 [6:21:59<16:43, 1.17it/s][2025-04-25 13:15:58] (step=0069200) Train Loss: 5.2488, Train Steps/Sec: 1.15 + 95%|██████████████████████████████████████████████████████████████████████████████████████████████████████████████▎ | 22308/23458 [6:22:21<16:42, 1.15it/s][2025-04-25 13:16:20] (step=0069225) Train Loss: 5.2498, Train Steps/Sec: 1.17 + 95%|██████████████████████████████████████████████████████████████████████████████████████████████████████████████▍ | 22333/23458 [6:22:42<16:02, 1.17it/s][2025-04-25 13:16:41] (step=0069250) Train Loss: 5.3456, Train Steps/Sec: 1.17 + 95%|██████████████████████████████████████████████████████████████████████████████████████████████████████████████▌ | 22358/23458 [6:23:03<15:35, 1.18it/s][2025-04-25 13:17:02] (step=0069275) Train Loss: 5.3414, Train Steps/Sec: 1.17 + 95%|██████████████████████████████████████████████████████████████████████████████████████████████████████████████▋ | 22383/23458 [6:23:25<15:07, 1.18it/s][2025-04-25 13:17:24] (step=0069300) Train Loss: 5.2328, Train Steps/Sec: 1.17 + 96%|██████████████████████████████████████████████████████████████████████████████████████████████████████████████▊ | 22408/23458 [6:23:46<15:10, 1.15it/s][2025-04-25 13:17:45] (step=0069325) Train Loss: 5.2631, Train Steps/Sec: 1.17 + 96%|██████████████████████████████████████████████████████████████████████████████████████████████████████████████▉ | 22433/23458 [6:24:09<15:47, 1.08it/s][2025-04-25 13:18:08] (step=0069350) Train Loss: 5.2852, Train Steps/Sec: 1.11 + 96%|███████████████████████████████████████████████████████████████████████████████████████████████████████████████ | 22458/23458 [6:24:30<14:16, 1.17it/s][2025-04-25 13:18:29] (step=0069375) Train Loss: 5.2896, Train Steps/Sec: 1.16 + 96%|███████████████████████████████████████████████████████████████████████████████████████████████████████████████▏ | 22483/23458 [6:24:52<13:49, 1.17it/s][2025-04-25 13:18:51] (step=0069400) Train Loss: 5.2807, Train Steps/Sec: 1.16 + 96%|███████████████████████████████████████████████████████████████████████████████████████████████████████████████▎ | 22508/23458 [6:25:13<13:48, 1.15it/s][2025-04-25 13:19:12] (step=0069425) Train Loss: 5.2748, Train Steps/Sec: 1.17 + 96%|███████████████████████████████████████████████████████████████████████████████████████████████████████████████▍ | 22533/23458 [6:25:35<13:13, 1.16it/s][2025-04-25 13:19:34] (step=0069450) Train Loss: 5.2175, Train Steps/Sec: 1.16 + 96%|███████████████████████████████████████████████████████████████████████████████████████████████████████████████▌ | 22558/23458 [6:25:57<12:59, 1.15it/s][2025-04-25 13:19:56] (step=0069475) Train Loss: 5.2765, Train Steps/Sec: 1.12 + 96%|███████████████████████████████████████████████████████████████████████████████████████████████████████████████▋ | 22583/23458 [6:26:20<12:18, 1.19it/s][2025-04-25 13:20:19] (step=0069500) Train Loss: 5.3443, Train Steps/Sec: 1.07 + 96%|███████████████████████████████████████████████████████████████████████████████████████████████████████████████▊ | 22608/23458 [6:26:42<12:21, 1.15it/s][2025-04-25 13:20:41] (step=0069525) Train Loss: 5.3521, Train Steps/Sec: 1.17 + 96%|███████████████████████████████████████████████████████████████████████████████████████████████████████████████▉ | 22633/23458 [6:27:04<11:47, 1.17it/s][2025-04-25 13:21:03] (step=0069550) Train Loss: 5.2703, Train Steps/Sec: 1.12 + 97%|████████████████████████████████████████████████████████████████████████████████████████████████████████████████ | 22658/23458 [6:27:25<11:18, 1.18it/s][2025-04-25 13:21:24] (step=0069575) Train Loss: 5.3039, Train Steps/Sec: 1.17 + 97%|████████████████████████████████████████████████████████████████████████████████████████████████████████████████▏ | 22683/23458 [6:27:47<10:54, 1.18it/s][2025-04-25 13:21:46] (step=0069600) Train Loss: 5.2823, Train Steps/Sec: 1.17 + 97%|████████████████████████████████████████████████████████████████████████████████████████████████████████████████▎ | 22708/23458 [6:28:08<10:57, 1.14it/s][2025-04-25 13:22:07] (step=0069625) Train Loss: 5.2650, Train Steps/Sec: 1.17 + 97%|████████████████████████████████████████████████████████████████████████████████████████████████████████████████▍ | 22733/23458 [6:28:30<10:36, 1.14it/s][2025-04-25 13:22:29] (step=0069650) Train Loss: 5.3182, Train Steps/Sec: 1.13 + 97%|████████████████████████████████████████████████████████████████████████████████████████████████████████████████▌ | 22758/23458 [6:28:53<12:19, 1.06s/it][2025-04-25 13:22:52] (step=0069675) Train Loss: 5.1984, Train Steps/Sec: 1.12 + 97%|████████████████████████████████████████████████████████████████████████████████████████████████████████████████▋ | 22783/23458 [6:29:14<09:31, 1.18it/s][2025-04-25 13:23:13] (step=0069700) Train Loss: 5.2053, Train Steps/Sec: 1.16 + 97%|████████████████████████████████████████████████████████████████████████████████████████████████████████████████▊ | 22808/23458 [6:29:36<09:26, 1.15it/s][2025-04-25 13:23:35] (step=0069725) Train Loss: 5.2719, Train Steps/Sec: 1.13 + 97%|████████████████████████████████████████████████████████████████████████████████████████████████████████████████▉ | 22833/23458 [6:29:58<08:55, 1.17it/s][2025-04-25 13:23:57] (step=0069750) Train Loss: 5.2552, Train Steps/Sec: 1.17 + 97%|█████████████████████████████████████████████████████████████████████████████████████████████████████████████████ | 22858/23458 [6:30:19<08:30, 1.18it/s][2025-04-25 13:24:18] (step=0069775) Train Loss: 5.3546, Train Steps/Sec: 1.17 + 98%|█████████████████████████████████████████████████████████████████████████████████████████████████████████████████▏ | 22883/23458 [6:30:40<08:06, 1.18it/s][2025-04-25 13:24:40] (step=0069800) Train Loss: 5.2663, Train Steps/Sec: 1.17 + 98%|█████████████████████████████████████████████████████████████████████████████████████████████████████████████████▎ | 22908/23458 [6:31:02<08:02, 1.14it/s][2025-04-25 13:25:01] (step=0069825) Train Loss: 5.3270, Train Steps/Sec: 1.16 + 98%|█████████████████████████████████████████████████████████████████████████████████████████████████████████████████▍ | 22933/23458 [6:31:23<07:29, 1.17it/s][2025-04-25 13:25:22] (step=0069850) Train Loss: 5.3055, Train Steps/Sec: 1.17 + 98%|█████████████████████████████████████████████████████████████████████████████████████████████████████████████████▌ | 22958/23458 [6:31:45<07:04, 1.18it/s][2025-04-25 13:25:44] (step=0069875) Train Loss: 5.2550, Train Steps/Sec: 1.17 + 98%|█████████████████████████████████████████████████████████████████████████████████████████████████████████████████▋ | 22983/23458 [6:32:06<06:42, 1.18it/s][2025-04-25 13:26:05] (step=0069900) Train Loss: 5.2711, Train Steps/Sec: 1.16 + 98%|█████████████████████████████████████████████████████████████████████████████████████████████████████████████████▊ | 23008/23458 [6:32:28<06:33, 1.14it/s][2025-04-25 13:26:27] (step=0069925) Train Loss: 5.2078, Train Steps/Sec: 1.17 + 98%|█████████████████████████████████████████████████████████████████████████████████████████████████████████████████▉ | 23033/23458 [6:32:49<06:03, 1.17it/s][2025-04-25 13:26:48] (step=0069950) Train Loss: 5.2604, Train Steps/Sec: 1.17 + 98%|██████████████████████████████████████████████████████████████████████████████████████████████████████████████████ | 23058/23458 [6:33:10<05:39, 1.18it/s][2025-04-25 13:27:09] (step=0069975) Train Loss: 5.1566, Train Steps/Sec: 1.17 + 98%|██████████████████████████████████████████████████████████████████████████████████████████████████████████████████▏ | 23083/23458 [6:33:33<06:03, 1.03it/s][2025-04-25 13:27:32] (step=0070000) Train Loss: 5.2783, Train Steps/Sec: 1.12 +You are using a model of type blip-2 to instantiate a model of type instructblip. This is not supported for all configurations of models and can yield errors. +[2025-04-25 13:27:32] text_config is None. Initializing the text config with default values (`OPTConfig`). +model_name_or_path:/tmp/haozhezhao/model/blip2-flan-t5-xl +image_place_holder: +tokenizer length after expend 32101 +tokenizer length before expend 32100 +100%|██████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████| 4/4 [03:57<00:00, 59.46s/it] +[2025-04-25 13:32:38] Finish Eval in 70000 steps...███████████████████████████████████████████████████████████████████████| 4/4 [03:57<00:00, 59.10s/it] +processing left-padding... +processing left-padding... +processing left-padding... +processing left-padding... +[2025-04-25 13:32:58] Saved checkpoint to checkpoint/Afteri2i_100kckpt_LLAVAT5_empty_Fixed_just_segment_pretrain_stage3_dreambench_recap_400k_t2i_400k_flux_200k_midjourney_150k_recovery_150k_3e4_epoch2_no_rotated/002-GPT-XL/checkpoints/0070000.pt +[2025-04-25 13:33:00] Removed old checkpoint: checkpoint/Afteri2i_100kckpt_LLAVAT5_empty_Fixed_just_segment_pretrain_stage3_dreambench_recap_400k_t2i_400k_flux_200k_midjourney_150k_recovery_150k_3e4_epoch2_no_rotated/002-GPT-XL/checkpoints/0068000.pt + 99%|██████████████████████████████████████████████████████████████████████████████████████████████████████████████████▎ | 23108/23458 [6:39:22<05:11, 1.12it/s][2025-04-25 13:33:21] (step=0070025) Train Loss: 5.3229, Train Steps/Sec: 0.07 + 99%|██████████████████████████████████████████████████████████████████████████████████████████████████████████████████▍ | 23133/23458 [6:39:43<04:37, 1.17it/s][2025-04-25 13:33:42] (step=0070050) Train Loss: 5.2708, Train Steps/Sec: 1.17 + 99%|██████████████████████████████████████████████████████████████████████████████████████████████████████████████████▌ | 23158/23458 [6:40:05<04:14, 1.18it/s][2025-04-25 13:34:04] (step=0070075) Train Loss: 5.3289, Train Steps/Sec: 1.17 + 99%|██████████████████████████████████████████████████████████████████████████████████████████████████████████████████▋ | 23183/23458 [6:40:26<03:52, 1.18it/s][2025-04-25 13:34:25] (step=0070100) Train Loss: 5.3080, Train Steps/Sec: 1.16 + 99%|██████████████████████████████████████████████████████████████████████████████████████████████████████████████████▊ | 23208/23458 [6:40:48<03:38, 1.14it/s][2025-04-25 13:34:47] (step=0070125) Train Loss: 5.1834, Train Steps/Sec: 1.12 + 99%|██████████████████████████████████████████████████████████████████████████████████████████████████████████████████▉ | 23233/23458 [6:41:11<03:21, 1.12it/s][2025-04-25 13:35:10] (step=0070150) Train Loss: 5.2757, Train Steps/Sec: 1.13 + 99%|███████████████████████████████████████████████████████████████████████████████████████████████████████████████████ | 23258/23458 [6:41:32<02:49, 1.18it/s][2025-04-25 13:35:31] (step=0070175) Train Loss: 5.2015, Train Steps/Sec: 1.17 + 99%|███████████████████████████████████████████████████████████████████████████████████████████████████████████████████▏| 23283/23458 [6:41:53<02:27, 1.18it/s][2025-04-25 13:35:52] (step=0070200) Train Loss: 5.2483, Train Steps/Sec: 1.17 + 99%|███████████████████████████████████████████████████████████████████████████████████████████████████████████████████▎| 23308/23458 [6:42:15<02:10, 1.15it/s][2025-04-25 13:36:14] (step=0070225) Train Loss: 5.1853, Train Steps/Sec: 1.17 + 99%|███████████████████████████████████████████████████████████████████████████████████████████████████████████████████▍| 23333/23458 [6:42:36<01:46, 1.17it/s][2025-04-25 13:36:35] (step=0070250) Train Loss: 5.2452, Train Steps/Sec: 1.17 +100%|███████████████████████████████████████████████████████████████████████████████████████████████████████████████████▌| 23358/23458 [6:42:58<01:24, 1.18it/s][2025-04-25 13:36:57] (step=0070275) Train Loss: 5.2581, Train Steps/Sec: 1.13 +100%|███████████████████████████████████████████████████████████████████████████████████████████████████████████████████▋| 23383/23458 [6:43:20<01:04, 1.17it/s][2025-04-25 13:37:19] (step=0070300) Train Loss: 5.2202, Train Steps/Sec: 1.16 +100%|███████████████████████████████████████████████████████████████████████████████████████████████████████████████████▊| 23408/23458 [6:43:42<00:43, 1.14it/s][2025-04-25 13:37:42] (step=0070325) Train Loss: 5.2565, Train Steps/Sec: 1.10 +100%|███████████████████████████████████████████████████████████████████████████████████████████████████████████████████▉| 23433/23458 [6:44:04<00:21, 1.16it/s][2025-04-25 13:38:03] (step=0070350) Train Loss: 5.3007, Train Steps/Sec: 1.16 +100%|████████████████████████████████████████████████████████████████████████████████████████████████████████████████████| 23458/23458 [6:44:27<00:00, 1.03s/it] +[2025-04-25 13:38:25] Done! diff --git a/CKPTS/Afteri2i_100kckpt_LLAVAT5_empty_Fixed_just_segment_pretrain_stage3_dreambench_recap_400k_t2i_400k_flux_200k_midjourney_150k_recovery_150k_3e4_epoch2_no_rotated/wandb/run-20250425_060734-amr5cnvk/files/requirements.txt b/CKPTS/Afteri2i_100kckpt_LLAVAT5_empty_Fixed_just_segment_pretrain_stage3_dreambench_recap_400k_t2i_400k_flux_200k_midjourney_150k_recovery_150k_3e4_epoch2_no_rotated/wandb/run-20250425_060734-amr5cnvk/files/requirements.txt new file mode 100644 index 0000000000000000000000000000000000000000..06dc78369ffff807b210006a0e79d705ffe2a7d7 --- /dev/null +++ b/CKPTS/Afteri2i_100kckpt_LLAVAT5_empty_Fixed_just_segment_pretrain_stage3_dreambench_recap_400k_t2i_400k_flux_200k_midjourney_150k_recovery_150k_3e4_epoch2_no_rotated/wandb/run-20250425_060734-amr5cnvk/files/requirements.txt @@ -0,0 +1,131 @@ +typing_extensions==4.12.2 +pyzmq==26.3.0 +nvidia-cufft-cu12==11.0.2.54 +triton==3.1.0 +nvidia-cublas-cu12==12.1.3.1 +psutil==7.0.0 +nvidia-cuda-cupti-cu12==12.1.105 +smmap==5.0.2 +nvidia-cuda-runtime-cu12==12.1.105 +aiohappyeyeballs==2.6.1 +asttokens==3.0.0 +huggingface-hub==0.29.3 +pyarrow==19.0.1 +fonttools==4.56.0 +python-dateutil==2.9.0.post0 +GitPython==3.1.44 +aiohttp==3.11.14 +wandb==0.19.8 +setproctitle==1.3.5 +PyYAML==6.0.2 +pydantic_core==2.27.2 +safetensors==0.5.3 +nvidia-nvjitlink-cu12==12.1.105 +aiosignal==1.3.2 +dill==0.3.8 +nvidia-cuda-nvrtc-cu12==12.1.105 +multiprocess==0.70.16 +pure_eval==0.2.3 +stack_data==0.6.3 +pydantic==2.10.6 +MarkupSafe==2.1.5 +tornado==6.4.2 +executing==2.1.0 +executing==2.2.0 +opencv-python==4.11.0.86 +nvitop==1.4.2 +multidict==6.2.0 +Jinja2==3.1.4 +torch==2.5.1+cu121 +nvidia-curand-cu12==10.3.2.106 +platformdirs==4.3.6 +six==1.17.0 +mpmath==1.3.0 +zipp==3.21.0 +packaging==24.2 +requests==2.32.3 +certifi==2025.1.31 +docker-pycreds==0.4.0 +torchvision==0.20.1+cu121 +pandas==2.2.3 +networkx==3.3 +exceptiongroup==1.2.2 +pickleshare==0.7.5 +tokenizers==0.21.1 +charset-normalizer==3.4.1 +jupyter_core==5.7.2 +wcwidth==0.2.13 +nvidia-nvtx-cu12==12.1.105 +prompt_toolkit==3.0.50 +fsspec==2024.12.0 +pillow==11.1.0 +propcache==0.3.0 +regex==2024.11.6 +ptyprocess==0.7.0 +contourpy==1.3.1 +importlib_metadata==8.6.1 +idna==3.10 +comm==0.2.2 +protobuf==5.29.3 +yarl==1.18.3 +ipython_pygments_lexers==1.1.1 +pip==25.0 +parso==0.8.4 +joblib==1.4.2 +nvidia-nccl-cu12==2.21.5 +hf_transfer==0.1.9 +Pygments==2.19.1 +decorator==5.2.1 +filelock==3.18.0 +nvidia-cusparse-cu12==12.1.0.106 +debugpy==1.8.13 +urllib3==2.3.0 +traitlets==5.14.3 +tzdata==2025.1 +matplotlib-inline==0.1.7 +matplotlib==3.10.1 +kiwisolver==1.4.8 +nest_asyncio==1.6.0 +frozenlist==1.5.0 +nvidia-ml-py==12.570.86 +transformers==4.49.0 +nltk==3.9.1 +ipykernel==6.29.5 +click==8.1.8 +gitdb==4.0.12 +pyparsing==3.2.1 +attrs==25.3.0 +jedi==0.19.2 +ipython==9.0.2 +nvidia-cudnn-cu12==9.1.0.70 +pexpect==4.9.0 +nvidia-cusolver-cu12==11.4.5.107 +numpy==2.2.4 +tqdm==4.67.1 +pytz==2025.1 +wheel==0.45.1 +sentry-sdk==2.23.1 +torchaudio==2.5.1+cu121 +jupyter_client==8.6.3 +cycler==0.12.1 +annotated-types==0.7.0 +sympy==1.13.1 +xxhash==3.5.0 +datasets==3.4.1 +setuptools==75.8.0 +typing_extensions==4.12.2 +wheel==0.43.0 +importlib_metadata==8.0.0 +backports.tarfile==1.2.0 +autocommand==2.2.2 +packaging==24.2 +tomli==2.0.1 +typeguard==4.3.0 +zipp==3.19.2 +jaraco.context==5.3.0 +jaraco.functools==4.0.1 +more-itertools==10.3.0 +platformdirs==4.2.2 +jaraco.text==3.12.1 +jaraco.collections==5.1.0 +inflect==7.3.1 diff --git a/CKPTS/Afteri2i_100kckpt_LLAVAT5_empty_Fixed_just_segment_pretrain_stage3_dreambench_recap_400k_t2i_400k_flux_200k_midjourney_150k_recovery_150k_3e4_epoch2_no_rotated/wandb/run-20250425_060734-amr5cnvk/files/wandb-metadata.json b/CKPTS/Afteri2i_100kckpt_LLAVAT5_empty_Fixed_just_segment_pretrain_stage3_dreambench_recap_400k_t2i_400k_flux_200k_midjourney_150k_recovery_150k_3e4_epoch2_no_rotated/wandb/run-20250425_060734-amr5cnvk/files/wandb-metadata.json new file mode 100644 index 0000000000000000000000000000000000000000..205057ca7dce71f045fd1e32b2cc7cd31c1b2536 --- /dev/null +++ b/CKPTS/Afteri2i_100kckpt_LLAVAT5_empty_Fixed_just_segment_pretrain_stage3_dreambench_recap_400k_t2i_400k_flux_200k_midjourney_150k_recovery_150k_3e4_epoch2_no_rotated/wandb/run-20250425_060734-amr5cnvk/files/wandb-metadata.json @@ -0,0 +1,150 @@ +{ + "os": "Linux-5.15.0-1064-azure-x86_64-with-glibc2.31", + "python": "CPython 3.11.11", + "startedAt": "2025-04-25T06:07:34.272624Z", + "args": [ + "--vq-ckpt", + "/tmp/haozhezhao/model/llamagen_t2i/vq_ds16_t2i.pt", + "--data-path", + "/tmp/haozhezhao/MLLMG/jsonl_data/merged_train_set_set_subject_400k_recap_t2i_400k_flux_200k_midjourney_150k_recovery_150k_grounding_100fluxseg_50samseg.jsonl", + "--dataset", + "ti2i", + "--image-size", + "512", + "--results-dir", + "checkpoint/Afteri2i_100kckpt_LLAVAT5_empty_Fixed_just_segment_pretrain_stage3_dreambench_recap_400k_t2i_400k_flux_200k_midjourney_150k_recovery_150k_3e4_epoch2_no_rotated", + "--cloud-save-path", + "/tmp/haozhezhao/checkpoint", + "--lr", + "3e-4", + "--val_data_path", + "/tmp/haozhezhao/MLLMG/jsonl_data/dreambench_plus_valid.jsonl", + "--use_vision_tower", + "--model_name_or_path", + "/tmp/haozhezhao/model/blip2-flan-t5-xl", + "--image_place_holder", + "", + "--do_eval", + "--eval_steps", + "2000", + "--max_eval_samples", + "250", + "--cfg-scale", + "7.5", + "--top-k", + "16384", + "--load_from_checkpoint", + "/tmp/haozhezhao/MLLMG/checkpoint/FIXed_3M_1epoch_step32000.pt", + "--global-batch-size", + "56", + "--num-workers", + "8", + "--warmup", + "0.05", + "--gradient-accumulation-steps", + "4", + "--train_text_encoder", + "--ckpt-every", + "2000", + "--epochs", + "3", + "--subject_driven", + "--reference_data_path", + "/tmp/haozhezhao/MLLMG/cc12m_reference_tunnel.jsonl", + "--multimodal_encoder", + "llava", + "--do_recovery", + "--find_unused_parameters", + "--cls-token-num", + "512", + "--dreambench_eval", + "--save_total_limit", + "1", + "--load_language_projection", + "/tmp/haozhezhao/MLLMG/llava-v1.5-flant5_fixed-pretrain/mm_projector.bin", + "--gpt-ckpt", + "/tmp/haozhezhao/MLLMG/checkpoint/Afteri2i_100kckpt_LLAVAT5_empty_Fixed_just_segment_pretrain_stage3_dreambench_recap_400k_t2i_400k_flux_200k_midjourney_150k_recovery_150k_3e4_epoch2_no_rotated/000-GPT-XL/checkpoints/0046000.pt", + "--mm_vision_tower", + "openai/clip-vit-large-patch14", + "--train_all", + "--load_fixed_llamagen", + "--fix", + "gpt-empty-fix", + "--resume" + ], + "program": "/tmp/haozhezhao/MLLMG/autoregressive/train/train_t2i.py", + "codePath": "autoregressive/train/train_t2i.py", + "email": "mimazhe55360@gmail.com", + "root": "checkpoint/Afteri2i_100kckpt_LLAVAT5_empty_Fixed_just_segment_pretrain_stage3_dreambench_recap_400k_t2i_400k_flux_200k_midjourney_150k_recovery_150k_3e4_epoch2_no_rotated", + "host": "447cc403a8794092814259713c51c1df00001X", + "executable": "/tmp/haozhezhao/anaconda3/envs/nlp/bin/python", + "codePathLocal": "autoregressive/train/train_t2i.py", + "cpu_count": 96, + "cpu_count_logical": 96, + "gpu": "NVIDIA A100-SXM4-80GB", + "gpu_count": 8, + "disk": { + "/": { + "total": "133003395072", + "used": "65652424704" + } + }, + "memory": { + "total": "1902387884032" + }, + "cpu": { + "count": 96, + "countLogical": 96 + }, + "gpu_nvidia": [ + { + "name": "NVIDIA A100-SXM4-80GB", + "memoryTotal": "85899345920", + "cudaCores": 6912, + "architecture": "Ampere" + }, + { + "name": "NVIDIA A100-SXM4-80GB", + "memoryTotal": "85899345920", + "cudaCores": 6912, + "architecture": "Ampere" + }, + { + "name": "NVIDIA A100-SXM4-80GB", + "memoryTotal": "85899345920", + "cudaCores": 6912, + "architecture": "Ampere" + }, + { + "name": "NVIDIA A100-SXM4-80GB", + "memoryTotal": "85899345920", + "cudaCores": 6912, + "architecture": "Ampere" + }, + { + "name": "NVIDIA A100-SXM4-80GB", + "memoryTotal": "85899345920", + "cudaCores": 6912, + "architecture": "Ampere" + }, + { + "name": "NVIDIA A100-SXM4-80GB", + "memoryTotal": "85899345920", + "cudaCores": 6912, + "architecture": "Ampere" + }, + { + "name": "NVIDIA A100-SXM4-80GB", + "memoryTotal": "85899345920", + "cudaCores": 6912, + "architecture": "Ampere" + }, + { + "name": "NVIDIA A100-SXM4-80GB", + "memoryTotal": "85899345920", + "cudaCores": 6912, + "architecture": "Ampere" + } + ], + "cudaVersion": "12.2" +} \ No newline at end of file diff --git a/CKPTS/Afteri2i_100kckpt_LLAVAT5_empty_Fixed_just_segment_pretrain_stage3_dreambench_recap_400k_t2i_400k_flux_200k_midjourney_150k_recovery_150k_3e4_epoch2_no_rotated/wandb/run-20250425_060734-amr5cnvk/files/wandb-summary.json b/CKPTS/Afteri2i_100kckpt_LLAVAT5_empty_Fixed_just_segment_pretrain_stage3_dreambench_recap_400k_t2i_400k_flux_200k_midjourney_150k_recovery_150k_3e4_epoch2_no_rotated/wandb/run-20250425_060734-amr5cnvk/files/wandb-summary.json new file mode 100644 index 0000000000000000000000000000000000000000..27f57705a3cca23ba6f6640da576c64e231c5273 --- /dev/null +++ b/CKPTS/Afteri2i_100kckpt_LLAVAT5_empty_Fixed_just_segment_pretrain_stage3_dreambench_recap_400k_t2i_400k_flux_200k_midjourney_150k_recovery_150k_3e4_epoch2_no_rotated/wandb/run-20250425_060734-amr5cnvk/files/wandb-summary.json @@ -0,0 +1 @@ +{"_timestamp":1.7455882835409706e+09,"_runtime":27054.412106165,"_step":70350,"_wandb":{"runtime":27054},"train lr":0.00026993935262963704,"train loss":5.300661087036133,"Train Steps/Sec":1.1614086891795192} \ No newline at end of file diff --git a/CKPTS/Afteri2i_100kckpt_LLAVAT5_empty_Fixed_just_segment_pretrain_stage3_dreambench_recap_400k_t2i_400k_flux_200k_midjourney_150k_recovery_150k_3e4_epoch2_no_rotated/wandb/run-20250425_060734-amr5cnvk/logs/debug-core.log b/CKPTS/Afteri2i_100kckpt_LLAVAT5_empty_Fixed_just_segment_pretrain_stage3_dreambench_recap_400k_t2i_400k_flux_200k_midjourney_150k_recovery_150k_3e4_epoch2_no_rotated/wandb/run-20250425_060734-amr5cnvk/logs/debug-core.log new file mode 100644 index 0000000000000000000000000000000000000000..0563a416fcfb70007afabf322bfb304c3fa4b72d --- /dev/null +++ b/CKPTS/Afteri2i_100kckpt_LLAVAT5_empty_Fixed_just_segment_pretrain_stage3_dreambench_recap_400k_t2i_400k_flux_200k_midjourney_150k_recovery_150k_3e4_epoch2_no_rotated/wandb/run-20250425_060734-amr5cnvk/logs/debug-core.log @@ -0,0 +1,13 @@ +{"time":"2025-04-25T06:07:33.699731077Z","level":"INFO","msg":"main: starting server","port-filename":"/tmp/tmpw5kkumke/port-3330461.txt","pid":3330461,"log-level":0,"disable-analytics":false,"shutdown-on-parent-exit":false} +{"time":"2025-04-25T06:07:33.70125392Z","level":"INFO","msg":"Will exit if parent process dies.","ppid":3330461} +{"time":"2025-04-25T06:07:33.701242754Z","level":"INFO","msg":"server is running","addr":{"IP":"127.0.0.1","Port":36499,"Zone":""}} +{"time":"2025-04-25T06:07:33.887171233Z","level":"INFO","msg":"connection: ManageConnectionData: new connection created","id":"127.0.0.1:50200"} +{"time":"2025-04-25T06:07:34.274341822Z","level":"INFO","msg":"handleInformInit: received","streamId":"amr5cnvk","id":"127.0.0.1:50200"} +{"time":"2025-04-25T06:07:34.505536656Z","level":"INFO","msg":"handleInformInit: stream started","streamId":"amr5cnvk","id":"127.0.0.1:50200"} +{"time":"2025-04-25T13:38:28.684573893Z","level":"INFO","msg":"handleInformTeardown: server teardown initiated","id":"127.0.0.1:50200"} +{"time":"2025-04-25T13:38:28.684655675Z","level":"INFO","msg":"connection: closing","id":"127.0.0.1:50200"} +{"time":"2025-04-25T13:38:28.684716084Z","level":"INFO","msg":"server is shutting down"} +{"time":"2025-04-25T13:38:28.684785067Z","level":"INFO","msg":"connection: closed successfully","id":"127.0.0.1:50200"} +{"time":"2025-04-25T13:38:29.9081998Z","level":"INFO","msg":"handleInformTeardown: server shutdown complete","id":"127.0.0.1:50200"} +{"time":"2025-04-25T13:38:29.908230357Z","level":"INFO","msg":"connection: ManageConnectionData: connection closed","id":"127.0.0.1:50200"} +{"time":"2025-04-25T13:38:29.908248762Z","level":"INFO","msg":"server is closed"} diff --git a/CKPTS/Afteri2i_100kckpt_LLAVAT5_empty_Fixed_just_segment_pretrain_stage3_dreambench_recap_400k_t2i_400k_flux_200k_midjourney_150k_recovery_150k_3e4_epoch2_no_rotated/wandb/run-20250425_060734-amr5cnvk/logs/debug-internal.log b/CKPTS/Afteri2i_100kckpt_LLAVAT5_empty_Fixed_just_segment_pretrain_stage3_dreambench_recap_400k_t2i_400k_flux_200k_midjourney_150k_recovery_150k_3e4_epoch2_no_rotated/wandb/run-20250425_060734-amr5cnvk/logs/debug-internal.log new file mode 100644 index 0000000000000000000000000000000000000000..8d1450c89eb0e2a06b67a4c2fd4e1776e4d7c8c4 --- /dev/null +++ b/CKPTS/Afteri2i_100kckpt_LLAVAT5_empty_Fixed_just_segment_pretrain_stage3_dreambench_recap_400k_t2i_400k_flux_200k_midjourney_150k_recovery_150k_3e4_epoch2_no_rotated/wandb/run-20250425_060734-amr5cnvk/logs/debug-internal.log @@ -0,0 +1,15 @@ +{"time":"2025-04-25T06:07:34.274581892Z","level":"INFO","msg":"stream: starting","core version":"0.19.8","symlink path":"checkpoint/Afteri2i_100kckpt_LLAVAT5_empty_Fixed_just_segment_pretrain_stage3_dreambench_recap_400k_t2i_400k_flux_200k_midjourney_150k_recovery_150k_3e4_epoch2_no_rotated/wandb/run-20250425_060734-amr5cnvk/logs/debug-core.log"} +{"time":"2025-04-25T06:07:34.505489908Z","level":"INFO","msg":"created new stream","id":"amr5cnvk"} +{"time":"2025-04-25T06:07:34.505530655Z","level":"INFO","msg":"stream: started","id":"amr5cnvk"} +{"time":"2025-04-25T06:07:34.505604914Z","level":"INFO","msg":"writer: Do: started","stream_id":"amr5cnvk"} +{"time":"2025-04-25T06:07:34.50565086Z","level":"INFO","msg":"handler: started","stream_id":"amr5cnvk"} +{"time":"2025-04-25T06:07:34.505611717Z","level":"INFO","msg":"sender: started","stream_id":"amr5cnvk"} +{"time":"2025-04-25T06:07:34.916123289Z","level":"INFO","msg":"Starting system monitor"} +{"time":"2025-04-25T13:38:28.684712933Z","level":"INFO","msg":"stream: closing","id":"amr5cnvk"} +{"time":"2025-04-25T13:38:28.684756174Z","level":"INFO","msg":"Stopping system monitor"} +{"time":"2025-04-25T13:38:28.685570482Z","level":"INFO","msg":"Stopped system monitor"} +{"time":"2025-04-25T13:38:29.677642832Z","level":"INFO","msg":"fileTransfer: Close: file transfer manager closed"} +{"time":"2025-04-25T13:38:29.908023721Z","level":"INFO","msg":"handler: closed","stream_id":"amr5cnvk"} +{"time":"2025-04-25T13:38:29.908051493Z","level":"INFO","msg":"writer: Close: closed","stream_id":"amr5cnvk"} +{"time":"2025-04-25T13:38:29.90809267Z","level":"INFO","msg":"sender: closed","stream_id":"amr5cnvk"} +{"time":"2025-04-25T13:38:29.908102859Z","level":"INFO","msg":"stream: closed","id":"amr5cnvk"} diff --git a/CKPTS/Afteri2i_100kckpt_LLAVAT5_empty_Fixed_just_segment_pretrain_stage3_dreambench_recap_400k_t2i_400k_flux_200k_midjourney_150k_recovery_150k_3e4_epoch2_no_rotated/wandb/run-20250425_060734-amr5cnvk/logs/debug.log b/CKPTS/Afteri2i_100kckpt_LLAVAT5_empty_Fixed_just_segment_pretrain_stage3_dreambench_recap_400k_t2i_400k_flux_200k_midjourney_150k_recovery_150k_3e4_epoch2_no_rotated/wandb/run-20250425_060734-amr5cnvk/logs/debug.log new file mode 100644 index 0000000000000000000000000000000000000000..5fe865c17581f1c2e6ab8d000a1d11b7bb27ce73 --- /dev/null +++ b/CKPTS/Afteri2i_100kckpt_LLAVAT5_empty_Fixed_just_segment_pretrain_stage3_dreambench_recap_400k_t2i_400k_flux_200k_midjourney_150k_recovery_150k_3e4_epoch2_no_rotated/wandb/run-20250425_060734-amr5cnvk/logs/debug.log @@ -0,0 +1,23 @@ +2025-04-25 06:07:34,267 INFO MainThread:3330461 [wandb_setup.py:_flush():67] Current SDK version is 0.19.8 +2025-04-25 06:07:34,267 INFO MainThread:3330461 [wandb_setup.py:_flush():67] Configure stats pid to 3330461 +2025-04-25 06:07:34,267 INFO MainThread:3330461 [wandb_setup.py:_flush():67] Loading settings from /tmp/haozhezhao/.config/wandb/settings +2025-04-25 06:07:34,267 INFO MainThread:3330461 [wandb_setup.py:_flush():67] Loading settings from /tmp/haozhezhao/MLLMG/wandb/settings +2025-04-25 06:07:34,267 INFO MainThread:3330461 [wandb_setup.py:_flush():67] Loading settings from environment variables +2025-04-25 06:07:34,267 INFO MainThread:3330461 [wandb_init.py:setup_run_log_directory():647] Logging user logs to checkpoint/Afteri2i_100kckpt_LLAVAT5_empty_Fixed_just_segment_pretrain_stage3_dreambench_recap_400k_t2i_400k_flux_200k_midjourney_150k_recovery_150k_3e4_epoch2_no_rotated/wandb/run-20250425_060734-amr5cnvk/logs/debug.log +2025-04-25 06:07:34,267 INFO MainThread:3330461 [wandb_init.py:setup_run_log_directory():648] Logging internal logs to checkpoint/Afteri2i_100kckpt_LLAVAT5_empty_Fixed_just_segment_pretrain_stage3_dreambench_recap_400k_t2i_400k_flux_200k_midjourney_150k_recovery_150k_3e4_epoch2_no_rotated/wandb/run-20250425_060734-amr5cnvk/logs/debug-internal.log +2025-04-25 06:07:34,267 INFO MainThread:3330461 [wandb_init.py:init():761] calling init triggers +2025-04-25 06:07:34,267 INFO MainThread:3330461 [wandb_init.py:init():766] wandb.init called with sweep_config: {} +config: {'data_path': '/tmp/haozhezhao/MLLMG/jsonl_data/merged_train_set_set_subject_400k_recap_t2i_400k_flux_200k_midjourney_150k_recovery_150k_grounding_100fluxseg_50samseg.jsonl', 'cloud_save_path': '/tmp/haozhezhao/checkpoint', 'no_local_save': False, 'vq_model': 'VQ-16', 'vq_ckpt': '/tmp/haozhezhao/model/llamagen_t2i/vq_ds16_t2i.pt', 'codebook_size': 16384, 'codebook_embed_dim': 8, 'gpt_model': 'GPT-XL', 'gpt_ckpt': '/tmp/haozhezhao/MLLMG/checkpoint/Afteri2i_100kckpt_LLAVAT5_empty_Fixed_just_segment_pretrain_stage3_dreambench_recap_400k_t2i_400k_flux_200k_midjourney_150k_recovery_150k_3e4_epoch2_no_rotated/000-GPT-XL/checkpoints/0046000.pt', 'gpt_type': 't2i', 'vocab_size': 16384, 'cls_token_num': 512, 'dropout_p': 0.1, 'token_dropout_p': 0.1, 'drop_path': 0.0, 'no_compile': False, 'results_dir': 'checkpoint/Afteri2i_100kckpt_LLAVAT5_empty_Fixed_just_segment_pretrain_stage3_dreambench_recap_400k_t2i_400k_flux_200k_midjourney_150k_recovery_150k_3e4_epoch2_no_rotated', 'dataset': 'ti2i', 'image_size': 512, 'downsample_size': 16, 'num_classes': 1000, 'epochs': 3, 'lr': 0.0003, 'weight_decay': 0.05, 'beta1': 0.9, 'beta2': 0.95, 'max_grad_norm': 1.0, 'global_batch_size': 56, 'global_seed': 0, 'num_workers': 8, 'log_every': 25, 'ckpt_every': 2000, 'gradient_accumulation_steps': 4, 'mixed_precision': 'bf16', 'val_data_path': '/tmp/haozhezhao/MLLMG/jsonl_data/dreambench_plus_valid.jsonl', 'use_vision_tower': True, 'model_name_or_path': '/tmp/haozhezhao/model/blip2-flan-t5-xl', 'image_place_holder': '', 'processor_path': None, 'do_eval': True, 'max_eval_samples': 250, 'train_text_encoder': True, 'no_left_padding': False, 'cfg_scale': 7.5, 'top_k': 16384, 'temperature': 0.9, 'top_p': 1.0, 'eval_steps': 2000, 'project_name': 'llamagen_ti2i', 'load_from_checkpoint': '/tmp/haozhezhao/MLLMG/checkpoint/FIXed_3M_1epoch_step32000.pt', 'warmup': 0.05, 'lr_decay_style': 'cosine', 'lr_decay_ratio': 0.1, 'train_iters': 500000, 'class_dropout_prob': 0.1, 'with_image_only': False, 'image_only_rate': 0.1, 'stage2': False, 'subject_driven': True, 'load_subject_embedding': None, 'reference_data_path': '/tmp/haozhezhao/MLLMG/cc12m_reference_tunnel.jsonl', 'multimodal_encoder': 'llava', 'do_recovery': True, 'no_replace': False, 'resume': True, 'dreambench_eval': True, 'find_unused_parameters': True, 'load_visual_encoder': False, 'continue_stage1': False, 'replace_subject': False, 'train_all': True, 'save_total_limit': 1, 'load_language_projection': '/tmp/haozhezhao/MLLMG/llava-v1.5-flant5_fixed-pretrain/mm_projector.bin', 'mm_vision_tower': 'openai/clip-vit-large-patch14', 'load_fixed_llamagen': True, 'unfreeze_output': False, 'fix': 'gpt-empty-fix', 'rank': 0, 'world_size': 8, 'gpu': 0, 'dist_url': 'env://', 'distributed': True, 'dist_backend': 'nccl', '_wandb': {}} +2025-04-25 06:07:34,267 INFO MainThread:3330461 [wandb_init.py:init():784] starting backend +2025-04-25 06:07:34,267 INFO MainThread:3330461 [wandb_init.py:init():788] sending inform_init request +2025-04-25 06:07:34,272 INFO MainThread:3330461 [backend.py:_multiprocessing_setup():101] multiprocessing start_methods=fork,spawn,forkserver, using: spawn +2025-04-25 06:07:34,272 INFO MainThread:3330461 [wandb_init.py:init():798] backend started and connected +2025-04-25 06:07:34,274 INFO MainThread:3330461 [wandb_init.py:init():891] updated telemetry +2025-04-25 06:07:34,274 INFO MainThread:3330461 [wandb_init.py:init():915] communicating run to backend with 90.0 second timeout +2025-04-25 06:07:34,913 INFO MainThread:3330461 [wandb_init.py:init():990] starting run threads in backend +2025-04-25 06:07:35,044 INFO MainThread:3330461 [wandb_run.py:_console_start():2375] atexit reg +2025-04-25 06:07:35,044 INFO MainThread:3330461 [wandb_run.py:_redirect():2227] redirect: wrap_raw +2025-04-25 06:07:35,045 INFO MainThread:3330461 [wandb_run.py:_redirect():2292] Wrapping output streams. +2025-04-25 06:07:35,045 INFO MainThread:3330461 [wandb_run.py:_redirect():2315] Redirects installed. +2025-04-25 06:07:35,051 INFO MainThread:3330461 [wandb_init.py:init():1032] run started, returning control to user process +2025-04-25 13:38:28,683 INFO MsgRouterThr:3330461 [mailbox.py:close():129] Closing mailbox, abandoning 1 handles. diff --git a/CKPTS/Afteri2i_100kckpt_LLAVAT5_empty_Fixed_just_segment_pretrain_stage3_dreambench_recap_400k_t2i_400k_flux_200k_midjourney_150k_recovery_150k_3e4_epoch2_no_rotated/wandb/run-20250425_060734-amr5cnvk/run-amr5cnvk.wandb b/CKPTS/Afteri2i_100kckpt_LLAVAT5_empty_Fixed_just_segment_pretrain_stage3_dreambench_recap_400k_t2i_400k_flux_200k_midjourney_150k_recovery_150k_3e4_epoch2_no_rotated/wandb/run-20250425_060734-amr5cnvk/run-amr5cnvk.wandb new file mode 100644 index 0000000000000000000000000000000000000000..4d0e000e7483f0027b9f13e8391edfbab2ad7971 --- /dev/null +++ b/CKPTS/Afteri2i_100kckpt_LLAVAT5_empty_Fixed_just_segment_pretrain_stage3_dreambench_recap_400k_t2i_400k_flux_200k_midjourney_150k_recovery_150k_3e4_epoch2_no_rotated/wandb/run-20250425_060734-amr5cnvk/run-amr5cnvk.wandb @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:bf7adfa3b69fc1a1739ae5e01bc8948736d45eaa14dff24956c7fe778d2f9691 +size 16744767 diff --git a/CKPTS/Afteri2i_100kckpt_LLAVAT5_empty_Fixed_just_segment_pretrain_stage3_dreambench_recap_400k_t2i_400k_flux_200k_midjourney_150k_recovery_150k_3e4_epoch2_no_rotated/wandb/run-20250426_080403-2j21z9h0/files/config.yaml b/CKPTS/Afteri2i_100kckpt_LLAVAT5_empty_Fixed_just_segment_pretrain_stage3_dreambench_recap_400k_t2i_400k_flux_200k_midjourney_150k_recovery_150k_3e4_epoch2_no_rotated/wandb/run-20250426_080403-2j21z9h0/files/config.yaml new file mode 100644 index 0000000000000000000000000000000000000000..23066c7bdc7441dec987c4926bcf2193b0a3d1f6 --- /dev/null +++ b/CKPTS/Afteri2i_100kckpt_LLAVAT5_empty_Fixed_just_segment_pretrain_stage3_dreambench_recap_400k_t2i_400k_flux_200k_midjourney_150k_recovery_150k_3e4_epoch2_no_rotated/wandb/run-20250426_080403-2j21z9h0/files/config.yaml @@ -0,0 +1,199 @@ +_wandb: + value: + cli_version: 0.19.8 + m: [] + python_version: 3.11.11 + t: + "1": + - 1 + - 11 + - 41 + - 49 + - 51 + - 55 + "2": + - 1 + - 11 + - 41 + - 49 + - 51 + - 55 + "3": + - 13 + - 16 + - 23 + - 55 + - 61 + "4": 3.11.11 + "5": 0.19.8 + "6": 4.49.0 + "8": + - 5 + "12": 0.19.8 + "13": linux-x86_64 +beta1: + value: 0.9 +beta2: + value: 0.95 +cfg_scale: + value: 7.5 +ckpt_every: + value: 2000 +class_dropout_prob: + value: 0.1 +cloud_save_path: + value: /tmp/haozhezhao/checkpoint +cls_token_num: + value: 512 +codebook_embed_dim: + value: 8 +codebook_size: + value: 16384 +continue_stage1: + value: false +data_path: + value: /tmp/haozhezhao/MLLMG/jsonl_data/merged_train_set_set_subject_400k_recap_t2i_400k_flux_200k_midjourney_150k_recovery_150k_grounding_100fluxseg_50samseg.jsonl +dataset: + value: ti2i +dist_backend: + value: nccl +dist_url: + value: env:// +distributed: + value: true +do_eval: + value: true +do_recovery: + value: true +downsample_size: + value: 16 +dreambench_eval: + value: true +drop_path: + value: 0 +dropout_p: + value: 0.1 +epochs: + value: 4 +eval_steps: + value: 2000 +find_unused_parameters: + value: true +fix: + value: gpt-empty-fix +global_batch_size: + value: 56 +global_seed: + value: 0 +gpt_ckpt: + value: /tmp/haozhezhao/MLLMG/checkpoint/Afteri2i_100kckpt_LLAVAT5_empty_Fixed_just_segment_pretrain_stage3_dreambench_recap_400k_t2i_400k_flux_200k_midjourney_150k_recovery_150k_3e4_epoch2_no_rotated/002-GPT-XL/checkpoints/0070000.pt +gpt_model: + value: GPT-XL +gpt_type: + value: t2i +gpu: + value: 0 +gradient_accumulation_steps: + value: 4 +image_only_rate: + value: 0.1 +image_place_holder: + value: +image_size: + value: 512 +load_fixed_llamagen: + value: true +load_from_checkpoint: + value: /tmp/haozhezhao/MLLMG/checkpoint/FIXed_3M_1epoch_step32000.pt +load_language_projection: + value: /tmp/haozhezhao/MLLMG/llava-v1.5-flant5_fixed-pretrain/mm_projector.bin +load_subject_embedding: + value: null +load_visual_encoder: + value: false +log_every: + value: 25 +lr: + value: 0.0003 +lr_decay_ratio: + value: 0.1 +lr_decay_style: + value: cosine +max_eval_samples: + value: 250 +max_grad_norm: + value: 1 +mixed_precision: + value: bf16 +mm_vision_tower: + value: openai/clip-vit-large-patch14 +model_name_or_path: + value: /tmp/haozhezhao/model/blip2-flan-t5-xl +multimodal_encoder: + value: llava +no_compile: + value: false +no_left_padding: + value: false +no_local_save: + value: false +no_replace: + value: false +num_classes: + value: 1000 +num_workers: + value: 8 +processor_path: + value: null +project_name: + value: llamagen_ti2i +rank: + value: 0 +reference_data_path: + value: /tmp/haozhezhao/MLLMG/cc12m_reference_tunnel.jsonl +replace_subject: + value: false +results_dir: + value: checkpoint/Afteri2i_100kckpt_LLAVAT5_empty_Fixed_just_segment_pretrain_stage3_dreambench_recap_400k_t2i_400k_flux_200k_midjourney_150k_recovery_150k_3e4_epoch2_no_rotated +resume: + value: true +save_total_limit: + value: 1 +stage2: + value: false +subject_driven: + value: true +temperature: + value: 0.9 +token_dropout_p: + value: 0.1 +top_k: + value: 16384 +top_p: + value: 1 +train_all: + value: true +train_iters: + value: 500000 +train_text_encoder: + value: true +unfreeze_output: + value: false +use_vision_tower: + value: true +val_data_path: + value: /tmp/haozhezhao/MLLMG/jsonl_data/dreambench_plus_valid.jsonl +vocab_size: + value: 16384 +vq_ckpt: + value: /tmp/haozhezhao/model/llamagen_t2i/vq_ds16_t2i.pt +vq_model: + value: VQ-16 +warmup: + value: 0.05 +weight_decay: + value: 0.05 +with_image_only: + value: false +world_size: + value: 8 diff --git a/CKPTS/Afteri2i_100kckpt_LLAVAT5_empty_Fixed_just_segment_pretrain_stage3_dreambench_recap_400k_t2i_400k_flux_200k_midjourney_150k_recovery_150k_3e4_epoch2_no_rotated/wandb/run-20250426_080403-2j21z9h0/files/output.log b/CKPTS/Afteri2i_100kckpt_LLAVAT5_empty_Fixed_just_segment_pretrain_stage3_dreambench_recap_400k_t2i_400k_flux_200k_midjourney_150k_recovery_150k_3e4_epoch2_no_rotated/wandb/run-20250426_080403-2j21z9h0/files/output.log new file mode 100644 index 0000000000000000000000000000000000000000..56ce792d34b50de1e159d26179d9aeec1fc67853 --- /dev/null +++ b/CKPTS/Afteri2i_100kckpt_LLAVAT5_empty_Fixed_just_segment_pretrain_stage3_dreambench_recap_400k_t2i_400k_flux_200k_midjourney_150k_recovery_150k_3e4_epoch2_no_rotated/wandb/run-20250426_080403-2j21z9h0/files/output.log @@ -0,0 +1,1135 @@ +[2025-04-26 08:04:04] Training for 4 epochs... +[2025-04-26 08:04:04] Beginning epoch 2... + 98%|████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████▋ | 23084/23458 [22:37<00:21, 17.39it/s]/tmp/haozhezhao/MLLMG/autoregressive/train/train_t2i.py:652: FutureWarning: `torch.cuda.amp.autocast(args...)` is deprecated. Please use `torch.amp.autocast('cuda', args...)` instead. + with torch.cuda.amp.autocast(dtype=ptdtype): + 98%|████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████▋ | 23084/23458 [22:50<00:21, 17.39it/s][rank0]:W0426 08:27:26.610000 3408653 site-packages/torch/_dynamo/variables/tensor.py:776] [4/0] Graph break from `Tensor.item()`, consider setting: +[rank0]:W0426 08:27:26.610000 3408653 site-packages/torch/_dynamo/variables/tensor.py:776] [4/0] torch._dynamo.config.capture_scalar_outputs = True +[rank0]:W0426 08:27:26.610000 3408653 site-packages/torch/_dynamo/variables/tensor.py:776] [4/0] or: +[rank0]:W0426 08:27:26.610000 3408653 site-packages/torch/_dynamo/variables/tensor.py:776] [4/0] env TORCHDYNAMO_CAPTURE_SCALAR_OUTPUTS=1 +[rank0]:W0426 08:27:26.610000 3408653 site-packages/torch/_dynamo/variables/tensor.py:776] [4/0] to include these operations in the captured graph. +[rank0]:W0426 08:27:26.610000 3408653 site-packages/torch/_dynamo/variables/tensor.py:776] [4/0] +[rank0]:W0426 08:27:26.610000 3408653 site-packages/torch/_dynamo/variables/tensor.py:776] [4/0] Graph break: from user code at: +[rank0]:W0426 08:27:26.610000 3408653 site-packages/torch/_dynamo/variables/tensor.py:776] [4/0] File "/tmp/haozhezhao/MLLMG/autoregressive/models/empty_fix_gpt.py", line 483, in torch_dynamo_resume_in_forward_at_477 +[rank0]:W0426 08:27:26.610000 3408653 site-packages/torch/_dynamo/variables/tensor.py:776] [4/0] t5_feat_len = t5_attn.sum().item() +[rank0]:W0426 08:27:26.610000 3408653 site-packages/torch/_dynamo/variables/tensor.py:776] [4/0] +[rank0]:W0426 08:27:26.610000 3408653 site-packages/torch/_dynamo/variables/tensor.py:776] [4/0] + 98%|████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████▋ | 23085/23458 [24:00<33:12, 5.34s/it]/tmp/haozhezhao/MLLMG/autoregressive/train/train_t2i.py:652: FutureWarning: `torch.cuda.amp.autocast(args...)` is deprecated. Please use `torch.amp.autocast('cuda', args...)` instead. + with torch.cuda.amp.autocast(dtype=ptdtype): + 98%|████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████▋ | 23086/23458 [24:47<54:36, 8.81s/it]/tmp/haozhezhao/MLLMG/autoregressive/train/train_t2i.py:652: FutureWarning: `torch.cuda.amp.autocast(args...)` is deprecated. Please use `torch.amp.autocast('cuda', args...)` instead. + with torch.cuda.amp.autocast(dtype=ptdtype): + 99%|████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████▊ | 23108/23458 [26:59<08:16, 1.42s/it][2025-04-26 08:31:04] (step=0070025) Train Loss: 5.3177, Train Steps/Sec: 0.02 + 99%|██████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████▉ | 23125/23458 [29:09<1:31:10, 16.43s/it]/tmp/haozhezhao/MLLMG/autoregressive/train/train_t2i.py:652: FutureWarning: `torch.cuda.amp.autocast(args...)` is deprecated. Please use `torch.amp.autocast('cuda', args...)` instead. + with torch.cuda.amp.autocast(dtype=ptdtype): + 99%|██████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████▉ | 23133/23458 [30:16<1:01:18, 11.32s/it][2025-04-26 08:34:21] (step=0070050) Train Loss: 5.2728, Train Steps/Sec: 0.13 + 99%|█████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████ | 23158/23458 [32:11<40:45, 8.15s/it][2025-04-26 08:36:17] (step=0070075) Train Loss: 5.3276, Train Steps/Sec: 0.22 + 99%|█████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████▎ | 23183/23458 [33:08<03:55, 1.17it/s][2025-04-26 08:37:13] (step=0070100) Train Loss: 5.3134, Train Steps/Sec: 0.44 + 99%|█████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████▍ | 23208/23458 [33:29<03:36, 1.16it/s][2025-04-26 08:37:35] (step=0070125) Train Loss: 5.1849, Train Steps/Sec: 1.18 + 99%|█████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████▌ | 23233/23458 [33:51<03:11, 1.17it/s][2025-04-26 08:37:56] (step=0070150) Train Loss: 5.2783, Train Steps/Sec: 1.17 + 99%|█████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████▋ | 23258/23458 [34:12<02:49, 1.18it/s][2025-04-26 08:38:17] (step=0070175) Train Loss: 5.1995, Train Steps/Sec: 1.18 + 99%|█████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████▉ | 23283/23458 [34:33<02:27, 1.19it/s][2025-04-26 08:38:39] (step=0070200) Train Loss: 5.2465, Train Steps/Sec: 1.17 + 99%|██████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████ | 23308/23458 [34:55<02:09, 1.16it/s][2025-04-26 08:39:00] (step=0070225) Train Loss: 5.1871, Train Steps/Sec: 1.18 + 99%|██████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████▏| 23333/23458 [35:16<01:47, 1.17it/s][2025-04-26 08:39:21] (step=0070250) Train Loss: 5.2435, Train Steps/Sec: 1.17 +100%|██████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████▎| 23358/23458 [35:37<01:24, 1.18it/s][2025-04-26 08:39:42] (step=0070275) Train Loss: 5.2690, Train Steps/Sec: 1.17 +100%|██████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████▌| 23383/23458 [35:58<01:03, 1.19it/s][2025-04-26 08:40:04] (step=0070300) Train Loss: 5.2306, Train Steps/Sec: 1.17 +100%|██████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████▋| 23408/23458 [36:20<00:43, 1.16it/s][2025-04-26 08:40:25] (step=0070325) Train Loss: 5.2591, Train Steps/Sec: 1.17 +100%|██████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████▊| 23433/23458 [36:41<00:21, 1.16it/s][2025-04-26 08:40:46] (step=0070350) Train Loss: 5.3037, Train Steps/Sec: 1.17 +100%|███████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████| 23458/23458 [37:30<00:00, 10.43it/s] +[2025-04-26 08:41:34] Beginning epoch 3... + 0%| | 0/23458 [00:00 +tokenizer length after expend 32101 +tokenizer length before expend 32100 + /tmp/haozhezhao/anaconda3/envs/nlp/lib/python3.11/contextlib.py:105: FutureWarning: `torch.backends.cuda.sdp_kernel()` is deprecated. In the future, this context manager will be removed. Please see `torch.nn.attention.sdpa_kernel()` for the new context manager, with updated signature. + self.gen = func(*args, **kwds) | 0/4 [00:00 +tokenizer length after expend 32101 +tokenizer length before expend 32100 +100%|███████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████| 4/4 [03:51<00:00, 57.80s/it] +[2025-04-26 09:45:38] Finish Eval in 74000 steps...████████████████████████████████████████████████████████████████████████████████████████████████████| 4/4 [03:50<00:00, 57.57s/it] +processing left-padding... +processing left-padding... +processing left-padding... +processing left-padding... +[2025-04-26 09:45:57] Saved checkpoint to checkpoint/Afteri2i_100kckpt_LLAVAT5_empty_Fixed_just_segment_pretrain_stage3_dreambench_recap_400k_t2i_400k_flux_200k_midjourney_150k_recovery_150k_3e4_epoch2_no_rotated/003-GPT-XL/checkpoints/0074000.pt +[2025-04-26 09:45:59] Removed old checkpoint: checkpoint/Afteri2i_100kckpt_LLAVAT5_empty_Fixed_just_segment_pretrain_stage3_dreambench_recap_400k_t2i_400k_flux_200k_midjourney_150k_recovery_150k_3e4_epoch2_no_rotated/003-GPT-XL/checkpoints/0072000.pt + 16%|██████████████████████▍ | 3650/23458 [1:04:45<4:51:58, 1.13it/s][2025-04-26 09:46:21] (step=0074025) Train Loss: 5.1184, Train Steps/Sec: 0.07 + 16%|██████████████████████▌ | 3675/23458 [1:05:07<4:42:30, 1.17it/s][2025-04-26 09:46:42] (step=0074050) Train Loss: 5.3005, Train Steps/Sec: 1.17 + 16%|██████████████████████▋ | 3700/23458 [1:05:29<4:44:47, 1.16it/s][2025-04-26 09:47:04] (step=0074075) Train Loss: 5.1684, Train Steps/Sec: 1.12 + 16%|██████████████████████▊ | 3725/23458 [1:05:51<5:09:58, 1.06it/s][2025-04-26 09:47:27] (step=0074100) Train Loss: 5.2449, Train Steps/Sec: 1.11 + 16%|███████████████████████ | 3750/23458 [1:06:13<4:45:30, 1.15it/s][2025-04-26 09:47:48] (step=0074125) Train Loss: 5.1284, Train Steps/Sec: 1.17 + 16%|███████████████████████▏ | 3775/23458 [1:06:34<4:41:34, 1.17it/s][2025-04-26 09:48:09] (step=0074150) Train Loss: 5.2036, Train Steps/Sec: 1.17 + 16%|███████████████████████▎ | 3800/23458 [1:06:55<4:37:41, 1.18it/s][2025-04-26 09:48:31] (step=0074175) Train Loss: 5.2195, Train Steps/Sec: 1.17 + 16%|███████████████████████▍ | 3825/23458 [1:07:17<4:35:42, 1.19it/s][2025-04-26 09:48:52] (step=0074200) Train Loss: 5.2426, Train Steps/Sec: 1.17 + 16%|███████████████████████▋ | 3850/23458 [1:07:38<4:43:40, 1.15it/s][2025-04-26 09:49:13] (step=0074225) Train Loss: 5.2129, Train Steps/Sec: 1.17 + 17%|███████████████████████▊ | 3875/23458 [1:08:00<4:40:16, 1.16it/s][2025-04-26 09:49:35] (step=0074250) Train Loss: 5.1796, Train Steps/Sec: 1.17 + 17%|███████████████████████▉ | 3900/23458 [1:08:21<4:37:39, 1.17it/s][2025-04-26 09:49:56] (step=0074275) Train Loss: 5.2302, Train Steps/Sec: 1.17 + 17%|████████████████████████ | 3925/23458 [1:08:42<4:36:45, 1.18it/s][2025-04-26 09:50:18] (step=0074300) Train Loss: 5.2911, Train Steps/Sec: 1.16 + 17%|████████████████████████▏ | 3950/23458 [1:09:04<4:42:51, 1.15it/s][2025-04-26 09:50:39] (step=0074325) Train Loss: 5.2462, Train Steps/Sec: 1.17 + 17%|████████████████████████▍ | 3975/23458 [1:09:25<4:37:43, 1.17it/s][2025-04-26 09:51:00] (step=0074350) Train Loss: 5.2051, Train Steps/Sec: 1.17 + 17%|████████████████████████▌ | 4000/23458 [1:09:46<4:36:06, 1.17it/s][2025-04-26 09:51:22] (step=0074375) Train Loss: 5.2195, Train Steps/Sec: 1.17 + 17%|████████████████████████▋ | 4025/23458 [1:10:08<4:33:25, 1.18it/s][2025-04-26 09:51:43] (step=0074400) Train Loss: 5.2866, Train Steps/Sec: 1.16 + 17%|████████████████████████▊ | 4050/23458 [1:10:29<4:40:36, 1.15it/s][2025-04-26 09:52:04] (step=0074425) Train Loss: 5.2964, Train Steps/Sec: 1.17 + 17%|█████████████████████████ | 4075/23458 [1:10:51<4:36:04, 1.17it/s][2025-04-26 09:52:26] (step=0074450) Train Loss: 5.2190, Train Steps/Sec: 1.17 + 17%|█████████████████████████▏ | 4100/23458 [1:11:12<4:33:20, 1.18it/s][2025-04-26 09:52:47] (step=0074475) Train Loss: 5.1818, Train Steps/Sec: 1.17 + 18%|█████████████████████████▎ | 4125/23458 [1:11:33<4:31:09, 1.19it/s][2025-04-26 09:53:09] (step=0074500) Train Loss: 5.2220, Train Steps/Sec: 1.17 + 18%|█████████████████████████▍ | 4150/23458 [1:11:55<4:38:40, 1.15it/s][2025-04-26 09:53:31] (step=0074525) Train Loss: 5.1763, Train Steps/Sec: 1.13 + 18%|█████████████████████████▋ | 4175/23458 [1:12:17<4:35:14, 1.17it/s][2025-04-26 09:53:52] (step=0074550) Train Loss: 5.2442, Train Steps/Sec: 1.17 + 18%|█████████████████████████▊ | 4200/23458 [1:12:38<4:33:51, 1.17it/s][2025-04-26 09:54:13] (step=0074575) Train Loss: 5.2165, Train Steps/Sec: 1.17 + 18%|█████████████████████████▉ | 4225/23458 [1:12:59<4:30:23, 1.19it/s][2025-04-26 09:54:35] (step=0074600) Train Loss: 5.1595, Train Steps/Sec: 1.17 + 18%|██████████████████████████ | 4250/23458 [1:13:23<4:45:00, 1.12it/s][2025-04-26 09:54:58] (step=0074625) Train Loss: 5.2413, Train Steps/Sec: 1.09 + 18%|██████████████████████████▏ | 4275/23458 [1:13:44<4:33:13, 1.17it/s][2025-04-26 09:55:19] (step=0074650) Train Loss: 5.1682, Train Steps/Sec: 1.17 + 18%|██████████████████████████▍ | 4300/23458 [1:14:07<4:53:14, 1.09it/s][2025-04-26 09:55:42] (step=0074675) Train Loss: 5.2214, Train Steps/Sec: 1.08 + 18%|██████████████████████████▌ | 4325/23458 [1:14:28<4:29:07, 1.18it/s][2025-04-26 09:56:04] (step=0074700) Train Loss: 5.2202, Train Steps/Sec: 1.17 + 19%|██████████████████████████▋ | 4350/23458 [1:14:51<4:45:10, 1.12it/s][2025-04-26 09:56:26] (step=0074725) Train Loss: 5.3591, Train Steps/Sec: 1.12 + 19%|██████████████████████████▊ | 4375/23458 [1:15:12<4:32:33, 1.17it/s][2025-04-26 09:56:47] (step=0074750) Train Loss: 5.1127, Train Steps/Sec: 1.17 + 19%|███████████████████████████ | 4400/23458 [1:15:34<4:58:48, 1.06it/s][2025-04-26 09:57:10] (step=0074775) Train Loss: 5.1991, Train Steps/Sec: 1.12 + 19%|███████████████████████████▏ | 4425/23458 [1:15:56<4:28:24, 1.18it/s][2025-04-26 09:57:31] (step=0074800) Train Loss: 5.2166, Train Steps/Sec: 1.16 + 19%|███████████████████████████▎ | 4450/23458 [1:16:17<4:34:48, 1.15it/s][2025-04-26 09:57:52] (step=0074825) Train Loss: 5.1958, Train Steps/Sec: 1.17 + 19%|███████████████████████████▍ | 4475/23458 [1:16:38<4:31:50, 1.16it/s][2025-04-26 09:58:14] (step=0074850) Train Loss: 5.1683, Train Steps/Sec: 1.17 + 19%|███████████████████████████▌ | 4500/23458 [1:17:01<4:36:51, 1.14it/s][2025-04-26 09:58:36] (step=0074875) Train Loss: 5.1631, Train Steps/Sec: 1.11 + 19%|███████████████████████████▊ | 4525/23458 [1:17:22<4:26:01, 1.19it/s][2025-04-26 09:58:58] (step=0074900) Train Loss: 5.2026, Train Steps/Sec: 1.17 + 19%|███████████████████████████▉ | 4550/23458 [1:17:44<4:34:04, 1.15it/s][2025-04-26 09:59:19] (step=0074925) Train Loss: 5.1976, Train Steps/Sec: 1.17 + 20%|████████████████████████████ | 4575/23458 [1:18:05<4:29:01, 1.17it/s][2025-04-26 09:59:40] (step=0074950) Train Loss: 5.1268, Train Steps/Sec: 1.17 + 20%|████████████████████████████▏ | 4600/23458 [1:18:26<4:27:13, 1.18it/s][2025-04-26 10:00:02] (step=0074975) Train Loss: 5.2329, Train Steps/Sec: 1.17 + 20%|████████████████████████████▍ | 4625/23458 [1:18:48<4:25:03, 1.18it/s][2025-04-26 10:00:23] (step=0075000) Train Loss: 5.2005, Train Steps/Sec: 1.17 + 20%|████████████████████████████▌ | 4650/23458 [1:19:09<4:33:06, 1.15it/s][2025-04-26 10:00:44] (step=0075025) Train Loss: 5.2735, Train Steps/Sec: 1.17 + 20%|████████████████████████████▋ | 4675/23458 [1:19:31<4:29:14, 1.16it/s][2025-04-26 10:01:06] (step=0075050) Train Loss: 5.1927, Train Steps/Sec: 1.17 + 20%|████████████████████████████▊ | 4700/23458 [1:19:52<4:25:10, 1.18it/s][2025-04-26 10:01:27] (step=0075075) Train Loss: 5.2109, Train Steps/Sec: 1.17 + 20%|█████████████████████████████ | 4725/23458 [1:20:13<4:23:50, 1.18it/s][2025-04-26 10:01:49] (step=0075100) Train Loss: 5.1940, Train Steps/Sec: 1.17 + 20%|█████████████████████████████▏ | 4750/23458 [1:20:35<4:30:50, 1.15it/s][2025-04-26 10:02:10] (step=0075125) Train Loss: 5.2243, Train Steps/Sec: 1.17 + 20%|█████████████████████████████▎ | 4775/23458 [1:20:56<4:26:50, 1.17it/s][2025-04-26 10:02:31] (step=0075150) Train Loss: 5.1904, Train Steps/Sec: 1.17 + 20%|█████████████████████████████▍ | 4800/23458 [1:21:18<4:28:21, 1.16it/s][2025-04-26 10:02:53] (step=0075175) Train Loss: 5.2919, Train Steps/Sec: 1.13 + 21%|█████████████████████████████▌ | 4825/23458 [1:21:39<4:22:12, 1.18it/s][2025-04-26 10:03:15] (step=0075200) Train Loss: 5.2684, Train Steps/Sec: 1.17 + 21%|█████████████████████████████▊ | 4850/23458 [1:22:01<4:28:59, 1.15it/s][2025-04-26 10:03:36] (step=0075225) Train Loss: 5.2402, Train Steps/Sec: 1.17 + 21%|█████████████████████████████▉ | 4875/23458 [1:22:22<4:25:21, 1.17it/s][2025-04-26 10:03:57] (step=0075250) Train Loss: 5.2662, Train Steps/Sec: 1.17 + 21%|██████████████████████████████ | 4900/23458 [1:22:45<4:43:19, 1.09it/s][2025-04-26 10:04:21] (step=0075275) Train Loss: 5.2182, Train Steps/Sec: 1.08 + 21%|██████████████████████████████▏ | 4925/23458 [1:23:07<4:21:01, 1.18it/s][2025-04-26 10:04:42] (step=0075300) Train Loss: 5.2925, Train Steps/Sec: 1.17 + 21%|██████████████████████████████▍ | 4950/23458 [1:23:28<4:28:47, 1.15it/s][2025-04-26 10:05:03] (step=0075325) Train Loss: 5.3134, Train Steps/Sec: 1.17 + 21%|██████████████████████████████▌ | 4975/23458 [1:23:50<4:28:52, 1.15it/s][2025-04-26 10:05:26] (step=0075350) Train Loss: 5.2629, Train Steps/Sec: 1.13 + 21%|██████████████████████████████▋ | 5000/23458 [1:24:12<4:20:12, 1.18it/s][2025-04-26 10:05:47] (step=0075375) Train Loss: 5.2108, Train Steps/Sec: 1.17 + 21%|██████████████████████████████▊ | 5025/23458 [1:24:34<4:22:44, 1.17it/s][2025-04-26 10:06:09] (step=0075400) Train Loss: 5.2688, Train Steps/Sec: 1.12 + 22%|███████████████████████████████ | 5050/23458 [1:24:55<4:26:29, 1.15it/s][2025-04-26 10:06:31] (step=0075425) Train Loss: 5.1907, Train Steps/Sec: 1.17 + 22%|███████████████████████████████▏ | 5075/23458 [1:25:18<5:04:20, 1.01it/s][2025-04-26 10:06:54] (step=0075450) Train Loss: 5.1866, Train Steps/Sec: 1.08 + 22%|███████████████████████████████▎ | 5100/23458 [1:25:40<4:21:01, 1.17it/s][2025-04-26 10:07:15] (step=0075475) Train Loss: 5.2531, Train Steps/Sec: 1.17 + 22%|███████████████████████████████▍ | 5125/23458 [1:26:01<4:18:32, 1.18it/s][2025-04-26 10:07:36] (step=0075500) Train Loss: 5.2518, Train Steps/Sec: 1.17 + 22%|███████████████████████████████▌ | 5150/23458 [1:26:23<4:25:14, 1.15it/s][2025-04-26 10:07:58] (step=0075525) Train Loss: 5.2192, Train Steps/Sec: 1.17 + 22%|███████████████████████████████▊ | 5175/23458 [1:26:44<4:21:26, 1.17it/s][2025-04-26 10:08:19] (step=0075550) Train Loss: 5.2572, Train Steps/Sec: 1.17 + 22%|███████████████████████████████▉ | 5200/23458 [1:27:05<4:18:16, 1.18it/s][2025-04-26 10:08:41] (step=0075575) Train Loss: 5.1905, Train Steps/Sec: 1.17 + 22%|████████████████████████████████ | 5225/23458 [1:27:27<4:15:57, 1.19it/s][2025-04-26 10:09:02] (step=0075600) Train Loss: 5.1872, Train Steps/Sec: 1.16 + 22%|████████████████████████████████▏ | 5250/23458 [1:27:48<4:23:16, 1.15it/s][2025-04-26 10:09:23] (step=0075625) Train Loss: 5.2429, Train Steps/Sec: 1.17 + 22%|████████████████████████████████▍ | 5275/23458 [1:28:09<4:20:32, 1.16it/s][2025-04-26 10:09:45] (step=0075650) Train Loss: 5.2182, Train Steps/Sec: 1.17 + 23%|████████████████████████████████▌ | 5300/23458 [1:28:32<4:28:23, 1.13it/s][2025-04-26 10:10:07] (step=0075675) Train Loss: 5.2292, Train Steps/Sec: 1.12 + 23%|████████████████████████████████▋ | 5325/23458 [1:28:53<4:14:44, 1.19it/s][2025-04-26 10:10:29] (step=0075700) Train Loss: 5.1356, Train Steps/Sec: 1.17 + 23%|████████████████████████████████▊ | 5350/23458 [1:29:15<4:21:32, 1.15it/s][2025-04-26 10:10:50] (step=0075725) Train Loss: 5.2368, Train Steps/Sec: 1.17 + 23%|████████████████████████████████▉ | 5375/23458 [1:29:36<4:17:15, 1.17it/s][2025-04-26 10:11:11] (step=0075750) Train Loss: 5.1997, Train Steps/Sec: 1.17 + 23%|█████████████████████████████████▏ | 5400/23458 [1:29:57<4:15:51, 1.18it/s][2025-04-26 10:11:32] (step=0075775) Train Loss: 5.2296, Train Steps/Sec: 1.17 + 23%|█████████████████████████████████▎ | 5425/23458 [1:30:19<4:13:51, 1.18it/s][2025-04-26 10:11:54] (step=0075800) Train Loss: 5.2874, Train Steps/Sec: 1.17 + 23%|█████████████████████████████████▍ | 5450/23458 [1:30:41<4:29:12, 1.11it/s][2025-04-26 10:12:16] (step=0075825) Train Loss: 5.2431, Train Steps/Sec: 1.13 + 23%|█████████████████████████████████▌ | 5475/23458 [1:31:02<4:15:53, 1.17it/s][2025-04-26 10:12:37] (step=0075850) Train Loss: 5.3591, Train Steps/Sec: 1.17 + 23%|█████████████████████████████████▊ | 5500/23458 [1:31:23<4:15:09, 1.17it/s][2025-04-26 10:12:59] (step=0075875) Train Loss: 5.3043, Train Steps/Sec: 1.17 + 24%|█████████████████████████████████▉ | 5525/23458 [1:31:45<4:12:15, 1.18it/s][2025-04-26 10:13:20] (step=0075900) Train Loss: 5.1970, Train Steps/Sec: 1.17 + 24%|██████████████████████████████████ | 5550/23458 [1:32:07<4:25:00, 1.13it/s][2025-04-26 10:13:42] (step=0075925) Train Loss: 5.2334, Train Steps/Sec: 1.13 + 24%|██████████████████████████████████▏ | 5575/23458 [1:32:29<4:14:48, 1.17it/s][2025-04-26 10:14:05] (step=0075950) Train Loss: 5.1530, Train Steps/Sec: 1.13 + 24%|██████████████████████████████████▍ | 5600/23458 [1:32:51<4:11:26, 1.18it/s][2025-04-26 10:14:26] (step=0075975) Train Loss: 5.2604, Train Steps/Sec: 1.17 + 24%|██████████████████████████████████▌ | 5625/23458 [1:33:12<4:10:22, 1.19it/s][2025-04-26 10:14:47] (step=0076000) Train Loss: 5.1788, Train Steps/Sec: 1.17 +You are using a model of type blip-2 to instantiate a model of type instructblip. This is not supported for all configurations of models and can yield errors. +[2025-04-26 10:14:47] text_config is None. Initializing the text config with default values (`OPTConfig`). +model_name_or_path:/tmp/haozhezhao/model/blip2-flan-t5-xl +image_place_holder: +tokenizer length after expend 32101 +tokenizer length before expend 32100 +100%|███████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████| 4/4 [03:54<00:00, 58.71s/it] +[2025-04-26 10:19:51] Finish Eval in 76000 steps...████████████████████████████████████████████████████████████████████████████████████████████████████| 4/4 [03:54<00:00, 58.44s/it] +processing left-padding... +processing left-padding... +processing left-padding... +processing left-padding... +[2025-04-26 10:20:10] Saved checkpoint to checkpoint/Afteri2i_100kckpt_LLAVAT5_empty_Fixed_just_segment_pretrain_stage3_dreambench_recap_400k_t2i_400k_flux_200k_midjourney_150k_recovery_150k_3e4_epoch2_no_rotated/003-GPT-XL/checkpoints/0076000.pt +[2025-04-26 10:20:12] Removed old checkpoint: checkpoint/Afteri2i_100kckpt_LLAVAT5_empty_Fixed_just_segment_pretrain_stage3_dreambench_recap_400k_t2i_400k_flux_200k_midjourney_150k_recovery_150k_3e4_epoch2_no_rotated/003-GPT-XL/checkpoints/0074000.pt + 24%|██████████████████████████████████▋ | 5650/23458 [1:38:59<4:25:57, 1.12it/s][2025-04-26 10:20:35] (step=0076025) Train Loss: 5.2282, Train Steps/Sec: 0.07 + 24%|██████████████████████████████████▊ | 5675/23458 [1:39:21<4:13:08, 1.17it/s][2025-04-26 10:20:56] (step=0076050) Train Loss: 5.1730, Train Steps/Sec: 1.17 + 24%|██████████████████████████████████▉ | 5700/23458 [1:39:42<4:12:36, 1.17it/s][2025-04-26 10:21:17] (step=0076075) Train Loss: 5.2761, Train Steps/Sec: 1.17 + 24%|███████████████████████████████████▏ | 5725/23458 [1:40:03<4:09:08, 1.19it/s][2025-04-26 10:21:39] (step=0076100) Train Loss: 5.2569, Train Steps/Sec: 1.17 + 25%|███████████████████████████████████▎ | 5750/23458 [1:40:27<4:50:43, 1.02it/s][2025-04-26 10:22:02] (step=0076125) Train Loss: 5.3060, Train Steps/Sec: 1.08 + 25%|███████████████████████████████████▍ | 5775/23458 [1:40:48<4:13:00, 1.16it/s][2025-04-26 10:22:23] (step=0076150) Train Loss: 5.2034, Train Steps/Sec: 1.17 + 25%|███████████████████████████████████▌ | 5800/23458 [1:41:09<4:09:22, 1.18it/s][2025-04-26 10:22:44] (step=0076175) Train Loss: 5.2180, Train Steps/Sec: 1.17 + 25%|███████████████████████████████████▊ | 5825/23458 [1:41:30<4:07:27, 1.19it/s][2025-04-26 10:23:06] (step=0076200) Train Loss: 5.2756, Train Steps/Sec: 1.17 + 25%|███████████████████████████████████▉ | 5850/23458 [1:41:52<4:15:05, 1.15it/s][2025-04-26 10:23:27] (step=0076225) Train Loss: 5.2242, Train Steps/Sec: 1.17 + 25%|████████████████████████████████████ | 5875/23458 [1:42:13<4:09:38, 1.17it/s][2025-04-26 10:23:48] (step=0076250) Train Loss: 5.2432, Train Steps/Sec: 1.17 + 25%|████████████████████████████████████▏ | 5900/23458 [1:42:34<4:08:12, 1.18it/s][2025-04-26 10:24:10] (step=0076275) Train Loss: 5.2239, Train Steps/Sec: 1.17 + 25%|████████████████████████████████████▎ | 5925/23458 [1:42:56<4:06:42, 1.18it/s][2025-04-26 10:24:31] (step=0076300) Train Loss: 5.2556, Train Steps/Sec: 1.17 + 25%|████████████████████████████████████▌ | 5950/23458 [1:43:17<4:13:12, 1.15it/s][2025-04-26 10:24:53] (step=0076325) Train Loss: 5.2450, Train Steps/Sec: 1.17 + 25%|████████████████████████████████████▋ | 5975/23458 [1:43:39<4:10:01, 1.17it/s][2025-04-26 10:25:14] (step=0076350) Train Loss: 5.2379, Train Steps/Sec: 1.17 + 26%|████████████████████████████████████▊ | 6000/23458 [1:44:00<4:07:18, 1.18it/s][2025-04-26 10:25:35] (step=0076375) Train Loss: 5.2819, Train Steps/Sec: 1.17 + 26%|████████████████████████████████████▉ | 6025/23458 [1:44:21<4:05:46, 1.18it/s][2025-04-26 10:25:57] (step=0076400) Train Loss: 5.1849, Train Steps/Sec: 1.17 + 26%|█████████████████████████████████████▏ | 6050/23458 [1:44:43<4:11:40, 1.15it/s][2025-04-26 10:26:18] (step=0076425) Train Loss: 5.1960, Train Steps/Sec: 1.17 + 26%|█████████████████████████████████████▎ | 6075/23458 [1:45:04<4:07:21, 1.17it/s][2025-04-26 10:26:39] (step=0076450) Train Loss: 5.1921, Train Steps/Sec: 1.17 + 26%|█████████████████████████████████████▍ | 6100/23458 [1:45:26<4:17:02, 1.13it/s][2025-04-26 10:27:02] (step=0076475) Train Loss: 5.1592, Train Steps/Sec: 1.08 + 26%|█████████████████████████████████████▌ | 6125/23458 [1:45:48<4:03:52, 1.18it/s][2025-04-26 10:27:24] (step=0076500) Train Loss: 5.1632, Train Steps/Sec: 1.17 + 26%|█████████████████████████████████████▊ | 6150/23458 [1:46:10<4:10:51, 1.15it/s][2025-04-26 10:27:45] (step=0076525) Train Loss: 5.3221, Train Steps/Sec: 1.17 + 26%|█████████████████████████████████████▉ | 6175/23458 [1:46:31<4:06:22, 1.17it/s][2025-04-26 10:28:06] (step=0076550) Train Loss: 5.2393, Train Steps/Sec: 1.18 + 26%|██████████████████████████████████████ | 6200/23458 [1:46:53<5:16:42, 1.10s/it][2025-04-26 10:28:29] (step=0076575) Train Loss: 5.1996, Train Steps/Sec: 1.13 + 27%|██████████████████████████████████████▏ | 6225/23458 [1:47:16<4:01:43, 1.19it/s][2025-04-26 10:28:51] (step=0076600) Train Loss: 5.2227, Train Steps/Sec: 1.12 + 27%|██████████████████████████████████████▎ | 6250/23458 [1:47:37<4:09:36, 1.15it/s][2025-04-26 10:29:12] (step=0076625) Train Loss: 5.2115, Train Steps/Sec: 1.17 + 27%|██████████████████████████████████████▌ | 6275/23458 [1:47:58<4:04:39, 1.17it/s][2025-04-26 10:29:34] (step=0076650) Train Loss: 5.1501, Train Steps/Sec: 1.17 + 27%|██████████████████████████████████████▋ | 6300/23458 [1:48:20<4:03:29, 1.17it/s][2025-04-26 10:29:55] (step=0076675) Train Loss: 5.1772, Train Steps/Sec: 1.17 + 27%|██████████████████████████████████████▊ | 6325/23458 [1:48:42<4:01:08, 1.18it/s][2025-04-26 10:30:17] (step=0076700) Train Loss: 5.1620, Train Steps/Sec: 1.12 + 27%|██████████████████████████████████████▉ | 6350/23458 [1:49:04<4:10:09, 1.14it/s][2025-04-26 10:30:39] (step=0076725) Train Loss: 5.1974, Train Steps/Sec: 1.12 + 27%|███████████████████████████████████████▏ | 6375/23458 [1:49:25<4:03:54, 1.17it/s][2025-04-26 10:31:01] (step=0076750) Train Loss: 5.2154, Train Steps/Sec: 1.17 + 27%|███████████████████████████████████████▎ | 6400/23458 [1:49:47<4:01:05, 1.18it/s][2025-04-26 10:31:22] (step=0076775) Train Loss: 5.1880, Train Steps/Sec: 1.17 + 27%|███████████████████████████████████████▍ | 6425/23458 [1:50:08<3:59:15, 1.19it/s][2025-04-26 10:31:44] (step=0076800) Train Loss: 5.2167, Train Steps/Sec: 1.12 + 27%|███████████████████████████████████████▌ | 6450/23458 [1:50:30<4:05:39, 1.15it/s][2025-04-26 10:32:06] (step=0076825) Train Loss: 5.2630, Train Steps/Sec: 1.17 + 28%|███████████████████████████████████████▋ | 6475/23458 [1:50:53<4:11:36, 1.12it/s][2025-04-26 10:32:28] (step=0076850) Train Loss: 5.2477, Train Steps/Sec: 1.13 + 28%|███████████████████████████████████████▉ | 6500/23458 [1:51:14<3:59:38, 1.18it/s][2025-04-26 10:32:49] (step=0076875) Train Loss: 5.3000, Train Steps/Sec: 1.17 + 28%|████████████████████████████████████████ | 6525/23458 [1:51:35<3:57:46, 1.19it/s][2025-04-26 10:33:11] (step=0076900) Train Loss: 5.2153, Train Steps/Sec: 1.17 + 28%|████████████████████████████████████████▏ | 6550/23458 [1:51:57<4:04:19, 1.15it/s][2025-04-26 10:33:32] (step=0076925) Train Loss: 5.2121, Train Steps/Sec: 1.17 + 28%|████████████████████████████████████████▎ | 6575/23458 [1:52:18<4:01:30, 1.17it/s][2025-04-26 10:33:53] (step=0076950) Train Loss: 5.1852, Train Steps/Sec: 1.17 + 28%|████████████████████████████████████████▌ | 6600/23458 [1:52:39<3:58:09, 1.18it/s][2025-04-26 10:34:15] (step=0076975) Train Loss: 5.2703, Train Steps/Sec: 1.17 + 28%|████████████████████████████████████████▋ | 6625/23458 [1:53:01<3:56:09, 1.19it/s][2025-04-26 10:34:36] (step=0077000) Train Loss: 5.1002, Train Steps/Sec: 1.17 + 28%|████████████████████████████████████████▊ | 6650/23458 [1:53:22<4:02:32, 1.16it/s][2025-04-26 10:34:57] (step=0077025) Train Loss: 5.2017, Train Steps/Sec: 1.18 + 28%|████████████████████████████████████████▉ | 6675/23458 [1:53:43<3:59:44, 1.17it/s][2025-04-26 10:35:19] (step=0077050) Train Loss: 5.2576, Train Steps/Sec: 1.17 + 29%|█████████████████████████████████████████▏ | 6700/23458 [1:54:05<3:57:05, 1.18it/s][2025-04-26 10:35:40] (step=0077075) Train Loss: 5.2865, Train Steps/Sec: 1.17 + 29%|█████████████████████████████████████████▎ | 6725/23458 [1:54:26<3:55:57, 1.18it/s][2025-04-26 10:36:01] (step=0077100) Train Loss: 5.2455, Train Steps/Sec: 1.16 + 29%|█████████████████████████████████████████▍ | 6750/23458 [1:54:47<4:02:01, 1.15it/s][2025-04-26 10:36:23] (step=0077125) Train Loss: 5.2395, Train Steps/Sec: 1.17 + 29%|█████████████████████████████████████████▌ | 6775/23458 [1:55:10<3:57:43, 1.17it/s][2025-04-26 10:36:45] (step=0077150) Train Loss: 5.2484, Train Steps/Sec: 1.13 + 29%|█████████████████████████████████████████▋ | 6800/23458 [1:55:31<3:55:39, 1.18it/s][2025-04-26 10:37:06] (step=0077175) Train Loss: 5.2104, Train Steps/Sec: 1.17 + 29%|█████████████████████████████████████████▉ | 6825/23458 [1:55:52<3:54:26, 1.18it/s][2025-04-26 10:37:28] (step=0077200) Train Loss: 5.2271, Train Steps/Sec: 1.17 + 29%|██████████████████████████████████████████ | 6850/23458 [1:56:14<4:00:54, 1.15it/s][2025-04-26 10:37:49] (step=0077225) Train Loss: 5.1997, Train Steps/Sec: 1.18 + 29%|██████████████████████████████████████████▏ | 6875/23458 [1:56:37<4:00:23, 1.15it/s][2025-04-26 10:38:12] (step=0077250) Train Loss: 5.2665, Train Steps/Sec: 1.09 + 29%|██████████████████████████████████████████▎ | 6900/23458 [1:56:59<4:09:20, 1.11it/s][2025-04-26 10:38:34] (step=0077275) Train Loss: 5.2167, Train Steps/Sec: 1.12 + 30%|██████████████████████████████████████████▌ | 6925/23458 [1:57:20<3:51:38, 1.19it/s][2025-04-26 10:38:56] (step=0077300) Train Loss: 5.2560, Train Steps/Sec: 1.17 + 30%|██████████████████████████████████████████▋ | 6950/23458 [1:57:42<3:59:02, 1.15it/s][2025-04-26 10:39:17] (step=0077325) Train Loss: 5.2550, Train Steps/Sec: 1.18 + 30%|██████████████████████████████████████████▊ | 6975/23458 [1:58:03<3:55:41, 1.17it/s][2025-04-26 10:39:38] (step=0077350) Train Loss: 5.2165, Train Steps/Sec: 1.17 + 30%|██████████████████████████████████████████▉ | 7000/23458 [1:58:25<3:52:04, 1.18it/s][2025-04-26 10:40:00] (step=0077375) Train Loss: 5.2506, Train Steps/Sec: 1.13 + 30%|███████████████████████████████████████████ | 7025/23458 [1:58:46<3:51:18, 1.18it/s][2025-04-26 10:40:22] (step=0077400) Train Loss: 5.2076, Train Steps/Sec: 1.17 + 30%|███████████████████████████████████████████▎ | 7050/23458 [1:59:08<3:57:47, 1.15it/s][2025-04-26 10:40:43] (step=0077425) Train Loss: 5.2469, Train Steps/Sec: 1.17 + 30%|███████████████████████████████████████████▍ | 7075/23458 [1:59:30<3:57:56, 1.15it/s][2025-04-26 10:41:05] (step=0077450) Train Loss: 5.1338, Train Steps/Sec: 1.13 + 30%|███████████████████████████████████████████▌ | 7100/23458 [1:59:51<3:50:57, 1.18it/s][2025-04-26 10:41:27] (step=0077475) Train Loss: 5.1505, Train Steps/Sec: 1.17 + 30%|███████████████████████████████████████████▋ | 7125/23458 [2:00:14<3:49:48, 1.18it/s][2025-04-26 10:41:49] (step=0077500) Train Loss: 5.1970, Train Steps/Sec: 1.12 + 30%|███████████████████████████████████████████▉ | 7150/23458 [2:00:35<3:55:34, 1.15it/s][2025-04-26 10:42:10] (step=0077525) Train Loss: 5.1828, Train Steps/Sec: 1.17 + 31%|████████████████████████████████████████████ | 7175/23458 [2:00:56<3:51:43, 1.17it/s][2025-04-26 10:42:32] (step=0077550) Train Loss: 5.3007, Train Steps/Sec: 1.17 + 31%|████████████████████████████████████████████▏ | 7200/23458 [2:01:19<3:53:36, 1.16it/s][2025-04-26 10:42:54] (step=0077575) Train Loss: 5.3253, Train Steps/Sec: 1.13 + 31%|████████████████████████████████████████████▎ | 7225/23458 [2:01:40<3:47:12, 1.19it/s][2025-04-26 10:43:15] (step=0077600) Train Loss: 5.1145, Train Steps/Sec: 1.17 + 31%|████████████████████████████████████████████▌ | 7250/23458 [2:02:01<3:53:27, 1.16it/s][2025-04-26 10:43:36] (step=0077625) Train Loss: 5.1850, Train Steps/Sec: 1.18 + 31%|████████████████████████████████████████████▋ | 7275/23458 [2:02:22<3:50:07, 1.17it/s][2025-04-26 10:43:58] (step=0077650) Train Loss: 5.2001, Train Steps/Sec: 1.17 + 31%|████████████████████████████████████████████▊ | 7300/23458 [2:02:44<3:48:16, 1.18it/s][2025-04-26 10:44:19] (step=0077675) Train Loss: 5.1618, Train Steps/Sec: 1.17 + 31%|████████████████████████████████████████████▉ | 7325/23458 [2:03:05<3:46:54, 1.19it/s][2025-04-26 10:44:40] (step=0077700) Train Loss: 5.2704, Train Steps/Sec: 1.17 + 31%|█████████████████████████████████████████████ | 7350/23458 [2:03:27<3:52:45, 1.15it/s][2025-04-26 10:45:02] (step=0077725) Train Loss: 5.2656, Train Steps/Sec: 1.17 + 31%|█████████████████████████████████████████████▎ | 7375/23458 [2:03:48<3:48:59, 1.17it/s][2025-04-26 10:45:23] (step=0077750) Train Loss: 5.2015, Train Steps/Sec: 1.17 + 32%|█████████████████████████████████████████████▍ | 7400/23458 [2:04:09<3:46:45, 1.18it/s][2025-04-26 10:45:44] (step=0077775) Train Loss: 5.2372, Train Steps/Sec: 1.17 + 32%|█████████████████████████████████████████████▌ | 7425/23458 [2:04:31<3:45:16, 1.19it/s][2025-04-26 10:46:07] (step=0077800) Train Loss: 5.1600, Train Steps/Sec: 1.12 + 32%|█████████████████████████████████████████████▋ | 7450/23458 [2:04:53<3:51:46, 1.15it/s][2025-04-26 10:46:28] (step=0077825) Train Loss: 5.2210, Train Steps/Sec: 1.17 + 32%|█████████████████████████████████████████████▉ | 7475/23458 [2:05:14<3:47:31, 1.17it/s][2025-04-26 10:46:49] (step=0077850) Train Loss: 5.2890, Train Steps/Sec: 1.18 + 32%|██████████████████████████████████████████████ | 7500/23458 [2:05:35<3:45:11, 1.18it/s][2025-04-26 10:47:11] (step=0077875) Train Loss: 5.1744, Train Steps/Sec: 1.17 + 32%|██████████████████████████████████████████████▏ | 7525/23458 [2:05:58<3:57:25, 1.12it/s][2025-04-26 10:47:34] (step=0077900) Train Loss: 5.1005, Train Steps/Sec: 1.08 + 32%|██████████████████████████████████████████████▎ | 7550/23458 [2:06:20<3:49:33, 1.15it/s][2025-04-26 10:47:55] (step=0077925) Train Loss: 5.2191, Train Steps/Sec: 1.17 + 32%|██████████████████████████████████████████████▌ | 7575/23458 [2:06:41<3:46:59, 1.17it/s][2025-04-26 10:48:16] (step=0077950) Train Loss: 5.2554, Train Steps/Sec: 1.17 + 32%|██████████████████████████████████████████████▋ | 7600/23458 [2:07:02<3:44:29, 1.18it/s][2025-04-26 10:48:38] (step=0077975) Train Loss: 5.2580, Train Steps/Sec: 1.17 + 33%|██████████████████████████████████████████████▊ | 7625/23458 [2:07:24<3:42:41, 1.19it/s][2025-04-26 10:48:59] (step=0078000) Train Loss: 5.2949, Train Steps/Sec: 1.17 +You are using a model of type blip-2 to instantiate a model of type instructblip. This is not supported for all configurations of models and can yield errors. +[2025-04-26 10:48:59] text_config is None. Initializing the text config with default values (`OPTConfig`). +model_name_or_path:/tmp/haozhezhao/model/blip2-flan-t5-xl +image_place_holder: +tokenizer length after expend 32101 +tokenizer length before expend 32100 +100%|███████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████| 4/4 [03:55<00:00, 58.91s/it] +[2025-04-26 10:54:03] Finish Eval in 78000 steps...████████████████████████████████████████████████████████████████████████████████████████████████████| 4/4 [03:55<00:00, 58.66s/it] +processing left-padding... +processing left-padding... +processing left-padding... +processing left-padding... +[2025-04-26 10:54:21] Saved checkpoint to checkpoint/Afteri2i_100kckpt_LLAVAT5_empty_Fixed_just_segment_pretrain_stage3_dreambench_recap_400k_t2i_400k_flux_200k_midjourney_150k_recovery_150k_3e4_epoch2_no_rotated/003-GPT-XL/checkpoints/0078000.pt +[2025-04-26 10:54:23] Removed old checkpoint: checkpoint/Afteri2i_100kckpt_LLAVAT5_empty_Fixed_just_segment_pretrain_stage3_dreambench_recap_400k_t2i_400k_flux_200k_midjourney_150k_recovery_150k_3e4_epoch2_no_rotated/003-GPT-XL/checkpoints/0076000.pt + 33%|██████████████████████████████████████████████▉ | 7650/23458 [2:13:10<4:39:14, 1.06s/it][2025-04-26 10:54:45] (step=0078025) Train Loss: 5.0942, Train Steps/Sec: 0.07 + 33%|███████████████████████████████████████████████ | 7675/23458 [2:13:31<3:45:13, 1.17it/s][2025-04-26 10:55:06] (step=0078050) Train Loss: 5.2269, Train Steps/Sec: 1.17 + 33%|███████████████████████████████████████████████▎ | 7700/23458 [2:13:54<4:03:17, 1.08it/s][2025-04-26 10:55:29] (step=0078075) Train Loss: 5.2480, Train Steps/Sec: 1.12 + 33%|███████████████████████████████████████████████▍ | 7725/23458 [2:14:15<3:41:32, 1.18it/s][2025-04-26 10:55:50] (step=0078100) Train Loss: 5.1732, Train Steps/Sec: 1.17 + 33%|███████████████████████████████████████████████▌ | 7750/23458 [2:14:36<3:47:10, 1.15it/s][2025-04-26 10:56:11] (step=0078125) Train Loss: 5.2350, Train Steps/Sec: 1.17 + 33%|███████████████████████████████████████████████▋ | 7775/23458 [2:14:58<3:43:27, 1.17it/s][2025-04-26 10:56:33] (step=0078150) Train Loss: 5.2776, Train Steps/Sec: 1.17 + 33%|███████████████████████████████████████████████▉ | 7800/23458 [2:15:20<3:40:54, 1.18it/s][2025-04-26 10:56:55] (step=0078175) Train Loss: 5.2423, Train Steps/Sec: 1.13 + 33%|████████████████████████████████████████████████ | 7825/23458 [2:15:41<3:40:06, 1.18it/s][2025-04-26 10:57:16] (step=0078200) Train Loss: 5.2205, Train Steps/Sec: 1.17 + 33%|████████████████████████████████████████████████▏ | 7850/23458 [2:16:02<3:45:48, 1.15it/s][2025-04-26 10:57:38] (step=0078225) Train Loss: 5.2980, Train Steps/Sec: 1.17 + 34%|████████████████████████████████████████████████▎ | 7875/23458 [2:16:24<3:42:16, 1.17it/s][2025-04-26 10:57:59] (step=0078250) Train Loss: 5.2624, Train Steps/Sec: 1.17 + 34%|████████████████████████████████████████████████▍ | 7900/23458 [2:16:45<3:39:46, 1.18it/s][2025-04-26 10:58:20] (step=0078275) Train Loss: 5.2611, Train Steps/Sec: 1.17 + 34%|████████████████████████████████████████████████▋ | 7925/23458 [2:17:07<3:46:13, 1.14it/s][2025-04-26 10:58:43] (step=0078300) Train Loss: 5.2976, Train Steps/Sec: 1.12 + 34%|████████████████████████████████████████████████▊ | 7950/23458 [2:17:29<3:44:06, 1.15it/s][2025-04-26 10:59:04] (step=0078325) Train Loss: 5.2013, Train Steps/Sec: 1.17 + 34%|████████████████████████████████████████████████▉ | 7975/23458 [2:17:50<3:40:36, 1.17it/s][2025-04-26 10:59:25] (step=0078350) Train Loss: 5.1889, Train Steps/Sec: 1.17 + 34%|█████████████████████████████████████████████████ | 8000/23458 [2:18:11<3:38:26, 1.18it/s][2025-04-26 10:59:46] (step=0078375) Train Loss: 5.2154, Train Steps/Sec: 1.17 + 34%|█████████████████████████████████████████████████▎ | 8025/23458 [2:18:32<3:36:14, 1.19it/s][2025-04-26 11:00:08] (step=0078400) Train Loss: 5.2249, Train Steps/Sec: 1.17 + 34%|█████████████████████████████████████████████████▍ | 8050/23458 [2:18:54<3:42:39, 1.15it/s][2025-04-26 11:00:29] (step=0078425) Train Loss: 5.1799, Train Steps/Sec: 1.17 + 34%|█████████████████████████████████████████████████▌ | 8075/23458 [2:19:16<3:40:12, 1.16it/s][2025-04-26 11:00:51] (step=0078450) Train Loss: 5.2537, Train Steps/Sec: 1.13 + 35%|█████████████████████████████████████████████████▋ | 8100/23458 [2:19:37<3:36:57, 1.18it/s][2025-04-26 11:01:13] (step=0078475) Train Loss: 5.1908, Train Steps/Sec: 1.17 + 35%|█████████████████████████████████████████████████▉ | 8125/23458 [2:19:59<3:35:28, 1.19it/s][2025-04-26 11:01:34] (step=0078500) Train Loss: 5.2150, Train Steps/Sec: 1.17 + 35%|██████████████████████████████████████████████████ | 8150/23458 [2:20:20<3:41:48, 1.15it/s][2025-04-26 11:01:55] (step=0078525) Train Loss: 5.2139, Train Steps/Sec: 1.17 + 35%|██████████████████████████████████████████████████▏ | 8175/23458 [2:20:42<4:42:41, 1.11s/it][2025-04-26 11:02:19] (step=0078550) Train Loss: 5.2097, Train Steps/Sec: 1.08 + 35%|██████████████████████████████████████████████████▎ | 8200/23458 [2:21:05<3:35:55, 1.18it/s][2025-04-26 11:02:40] (step=0078575) Train Loss: 5.1725, Train Steps/Sec: 1.17 + 35%|██████████████████████████████████████████████████▍ | 8225/23458 [2:21:26<3:34:46, 1.18it/s][2025-04-26 11:03:01] (step=0078600) Train Loss: 5.1654, Train Steps/Sec: 1.17 + 35%|██████████████████████████████████████████████████▋ | 8250/23458 [2:21:47<3:39:44, 1.15it/s][2025-04-26 11:03:23] (step=0078625) Train Loss: 5.2390, Train Steps/Sec: 1.17 + 35%|██████████████████████████████████████████████████▊ | 8275/23458 [2:22:09<3:35:17, 1.18it/s][2025-04-26 11:03:44] (step=0078650) Train Loss: 5.2043, Train Steps/Sec: 1.18 + 35%|██████████████████████████████████████████████████▉ | 8300/23458 [2:22:30<3:35:13, 1.17it/s][2025-04-26 11:04:05] (step=0078675) Train Loss: 5.2323, Train Steps/Sec: 1.17 + 35%|███████████████████████████████████████████████████ | 8325/23458 [2:22:52<3:48:31, 1.10it/s][2025-04-26 11:04:28] (step=0078700) Train Loss: 5.1712, Train Steps/Sec: 1.12 + 36%|███████████████████████████████████████████████████▎ | 8350/23458 [2:23:15<3:47:16, 1.11it/s][2025-04-26 11:04:50] (step=0078725) Train Loss: 5.3020, Train Steps/Sec: 1.12 + 36%|███████████████████████████████████████████████████▍ | 8375/23458 [2:23:36<3:35:31, 1.17it/s][2025-04-26 11:05:11] (step=0078750) Train Loss: 5.2100, Train Steps/Sec: 1.17 + 36%|███████████████████████████████████████████████████▌ | 8400/23458 [2:23:57<3:32:38, 1.18it/s][2025-04-26 11:05:32] (step=0078775) Train Loss: 5.1714, Train Steps/Sec: 1.17 + 36%|███████████████████████████████████████████████████▋ | 8425/23458 [2:24:19<3:31:24, 1.19it/s][2025-04-26 11:05:54] (step=0078800) Train Loss: 5.2345, Train Steps/Sec: 1.17 + 36%|███████████████████████████████████████████████████▊ | 8450/23458 [2:24:40<3:36:45, 1.15it/s][2025-04-26 11:06:15] (step=0078825) Train Loss: 5.1673, Train Steps/Sec: 1.17 + 36%|████████████████████████████████████████████████████ | 8475/23458 [2:25:02<3:34:06, 1.17it/s][2025-04-26 11:06:38] (step=0078850) Train Loss: 5.2388, Train Steps/Sec: 1.12 + 36%|████████████████████████████████████████████████████▏ | 8500/23458 [2:25:25<3:59:58, 1.04it/s][2025-04-26 11:07:00] (step=0078875) Train Loss: 5.1118, Train Steps/Sec: 1.11 + 36%|████████████████████████████████████████████████████▎ | 8525/23458 [2:25:46<3:29:46, 1.19it/s][2025-04-26 11:07:21] (step=0078900) Train Loss: 5.1860, Train Steps/Sec: 1.17 + 36%|████████████████████████████████████████████████████▍ | 8550/23458 [2:26:07<3:36:16, 1.15it/s][2025-04-26 11:07:43] (step=0078925) Train Loss: 5.3031, Train Steps/Sec: 1.17 + 37%|████████████████████████████████████████████████████▋ | 8575/23458 [2:26:29<3:32:39, 1.17it/s][2025-04-26 11:08:04] (step=0078950) Train Loss: 5.2052, Train Steps/Sec: 1.17 + 37%|████████████████████████████████████████████████████▊ | 8600/23458 [2:26:50<3:29:36, 1.18it/s][2025-04-26 11:08:25] (step=0078975) Train Loss: 5.3103, Train Steps/Sec: 1.17 + 37%|████████████████████████████████████████████████████▉ | 8625/23458 [2:27:11<3:28:11, 1.19it/s][2025-04-26 11:08:47] (step=0079000) Train Loss: 5.2245, Train Steps/Sec: 1.17 + 37%|█████████████████████████████████████████████████████ | 8650/23458 [2:27:34<3:44:43, 1.10it/s][2025-04-26 11:09:09] (step=0079025) Train Loss: 5.2455, Train Steps/Sec: 1.13 + 37%|█████████████████████████████████████████████████████▎ | 8675/23458 [2:27:55<3:31:13, 1.17it/s][2025-04-26 11:09:30] (step=0079050) Train Loss: 5.1736, Train Steps/Sec: 1.17 + 37%|█████████████████████████████████████████████████████▍ | 8700/23458 [2:28:16<3:28:58, 1.18it/s][2025-04-26 11:09:52] (step=0079075) Train Loss: 5.2344, Train Steps/Sec: 1.17 + 37%|█████████████████████████████████████████████████████▌ | 8725/23458 [2:28:38<3:41:57, 1.11it/s][2025-04-26 11:10:14] (step=0079100) Train Loss: 5.1818, Train Steps/Sec: 1.12 + 37%|█████████████████████████████████████████████████████▋ | 8750/23458 [2:29:00<3:33:12, 1.15it/s][2025-04-26 11:10:35] (step=0079125) Train Loss: 5.1617, Train Steps/Sec: 1.17 + 37%|█████████████████████████████████████████████████████▊ | 8775/23458 [2:29:21<3:29:14, 1.17it/s][2025-04-26 11:10:56] (step=0079150) Train Loss: 5.2646, Train Steps/Sec: 1.17 + 38%|██████████████████████████████████████████████████████ | 8800/23458 [2:29:42<3:26:28, 1.18it/s][2025-04-26 11:11:18] (step=0079175) Train Loss: 5.2288, Train Steps/Sec: 1.18 + 38%|██████████████████████████████████████████████████████▏ | 8825/23458 [2:30:04<3:25:44, 1.19it/s][2025-04-26 11:11:39] (step=0079200) Train Loss: 5.1792, Train Steps/Sec: 1.17 + 38%|██████████████████████████████████████████████████████▎ | 8850/23458 [2:30:27<3:31:26, 1.15it/s][2025-04-26 11:12:02] (step=0079225) Train Loss: 5.1763, Train Steps/Sec: 1.09 + 38%|██████████████████████████████████████████████████████▍ | 8875/23458 [2:30:48<3:27:30, 1.17it/s][2025-04-26 11:12:23] (step=0079250) Train Loss: 5.1914, Train Steps/Sec: 1.17 + 38%|██████████████████████████████████████████████████████▋ | 8900/23458 [2:31:09<3:26:22, 1.18it/s][2025-04-26 11:12:45] (step=0079275) Train Loss: 5.2450, Train Steps/Sec: 1.17 + 38%|██████████████████████████████████████████████████████▊ | 8925/23458 [2:31:31<3:24:07, 1.19it/s][2025-04-26 11:13:06] (step=0079300) Train Loss: 5.2746, Train Steps/Sec: 1.17 + 38%|██████████████████████████████████████████████████████▉ | 8950/23458 [2:31:52<3:29:53, 1.15it/s][2025-04-26 11:13:27] (step=0079325) Train Loss: 5.1077, Train Steps/Sec: 1.17 + 38%|███████████████████████████████████████████████████████ | 8975/23458 [2:32:14<3:27:32, 1.16it/s][2025-04-26 11:13:49] (step=0079350) Train Loss: 5.1906, Train Steps/Sec: 1.17 + 38%|███████████████████████████████████████████████████████▏ | 9000/23458 [2:32:36<3:29:18, 1.15it/s][2025-04-26 11:14:11] (step=0079375) Train Loss: 5.2699, Train Steps/Sec: 1.13 + 38%|███████████████████████████████████████████████████████▍ | 9025/23458 [2:32:57<3:23:01, 1.18it/s][2025-04-26 11:14:32] (step=0079400) Train Loss: 5.2240, Train Steps/Sec: 1.17 + 39%|███████████████████████████████████████████████████████▌ | 9050/23458 [2:33:18<3:28:17, 1.15it/s][2025-04-26 11:14:54] (step=0079425) Train Loss: 5.2064, Train Steps/Sec: 1.18 + 39%|███████████████████████████████████████████████████████▋ | 9075/23458 [2:33:41<3:36:52, 1.11it/s][2025-04-26 11:15:16] (step=0079450) Train Loss: 5.2317, Train Steps/Sec: 1.12 + 39%|███████████████████████████████████████████████████████▊ | 9100/23458 [2:34:02<3:23:28, 1.18it/s][2025-04-26 11:15:37] (step=0079475) Train Loss: 5.2154, Train Steps/Sec: 1.17 + 39%|████████████████████████████████████████████████████████ | 9125/23458 [2:34:23<3:20:44, 1.19it/s][2025-04-26 11:15:59] (step=0079500) Train Loss: 5.2910, Train Steps/Sec: 1.17 + 39%|████████████████████████████████████████████████████████▏ | 9150/23458 [2:34:46<3:26:17, 1.16it/s][2025-04-26 11:16:21] (step=0079525) Train Loss: 5.1139, Train Steps/Sec: 1.13 + 39%|████████████████████████████████████████████████████████▎ | 9175/23458 [2:35:07<3:23:43, 1.17it/s][2025-04-26 11:16:42] (step=0079550) Train Loss: 5.2217, Train Steps/Sec: 1.17 + 39%|████████████████████████████████████████████████████████▍ | 9200/23458 [2:35:28<3:21:19, 1.18it/s][2025-04-26 11:17:03] (step=0079575) Train Loss: 5.1969, Train Steps/Sec: 1.18 + 39%|████████████████████████████████████████████████████████▋ | 9225/23458 [2:35:50<3:20:15, 1.18it/s][2025-04-26 11:17:25] (step=0079600) Train Loss: 5.1557, Train Steps/Sec: 1.17 + 39%|████████████████████████████████████████████████████████▊ | 9250/23458 [2:36:11<3:24:43, 1.16it/s][2025-04-26 11:17:46] (step=0079625) Train Loss: 5.1815, Train Steps/Sec: 1.17 + 40%|████████████████████████████████████████████████████████▉ | 9275/23458 [2:36:32<3:22:06, 1.17it/s][2025-04-26 11:18:07] (step=0079650) Train Loss: 5.2411, Train Steps/Sec: 1.17 + 40%|█████████████████████████████████████████████████████████ | 9300/23458 [2:36:55<3:46:19, 1.04it/s][2025-04-26 11:18:30] (step=0079675) Train Loss: 5.1448, Train Steps/Sec: 1.12 + 40%|█████████████████████████████████████████████████████████▏ | 9325/23458 [2:37:16<3:18:53, 1.18it/s][2025-04-26 11:18:51] (step=0079700) Train Loss: 5.2429, Train Steps/Sec: 1.17 + 40%|█████████████████████████████████████████████████████████▍ | 9350/23458 [2:37:37<3:25:59, 1.14it/s][2025-04-26 11:19:13] (step=0079725) Train Loss: 5.1949, Train Steps/Sec: 1.17 + 40%|█████████████████████████████████████████████████████████▌ | 9375/23458 [2:38:00<3:29:22, 1.12it/s][2025-04-26 11:19:35] (step=0079750) Train Loss: 5.1465, Train Steps/Sec: 1.12 + 40%|█████████████████████████████████████████████████████████▋ | 9400/23458 [2:38:22<3:18:39, 1.18it/s][2025-04-26 11:19:57] (step=0079775) Train Loss: 5.2275, Train Steps/Sec: 1.13 + 40%|█████████████████████████████████████████████████████████▊ | 9425/23458 [2:38:43<3:17:39, 1.18it/s][2025-04-26 11:20:18] (step=0079800) Train Loss: 5.2449, Train Steps/Sec: 1.17 + 40%|██████████████████████████████████████████████████████████ | 9450/23458 [2:39:04<3:22:05, 1.16it/s][2025-04-26 11:20:40] (step=0079825) Train Loss: 5.1761, Train Steps/Sec: 1.17 + 40%|██████████████████████████████████████████████████████████▏ | 9475/23458 [2:39:26<3:19:17, 1.17it/s][2025-04-26 11:21:01] (step=0079850) Train Loss: 5.2577, Train Steps/Sec: 1.17 + 40%|██████████████████████████████████████████████████████████▎ | 9500/23458 [2:39:49<3:29:08, 1.11it/s][2025-04-26 11:21:24] (step=0079875) Train Loss: 5.1251, Train Steps/Sec: 1.08 + 41%|██████████████████████████████████████████████████████████▍ | 9525/23458 [2:40:10<3:15:27, 1.19it/s][2025-04-26 11:21:46] (step=0079900) Train Loss: 5.1968, Train Steps/Sec: 1.17 + 41%|██████████████████████████████████████████████████████████▌ | 9550/23458 [2:40:32<3:21:51, 1.15it/s][2025-04-26 11:22:07] (step=0079925) Train Loss: 5.2369, Train Steps/Sec: 1.17 + 41%|██████████████████████████████████████████████████████████▊ | 9575/23458 [2:40:53<3:17:35, 1.17it/s][2025-04-26 11:22:28] (step=0079950) Train Loss: 5.2255, Train Steps/Sec: 1.17 + 41%|██████████████████████████████████████████████████████████▉ | 9600/23458 [2:41:14<3:15:39, 1.18it/s][2025-04-26 11:22:49] (step=0079975) Train Loss: 5.2699, Train Steps/Sec: 1.17 + 41%|███████████████████████████████████████████████████████████ | 9625/23458 [2:41:36<3:15:38, 1.18it/s][2025-04-26 11:23:11] (step=0080000) Train Loss: 5.1260, Train Steps/Sec: 1.17 +You are using a model of type blip-2 to instantiate a model of type instructblip. This is not supported for all configurations of models and can yield errors. +[2025-04-26 11:23:11] text_config is None. Initializing the text config with default values (`OPTConfig`). +model_name_or_path:/tmp/haozhezhao/model/blip2-flan-t5-xl +image_place_holder: +tokenizer length after expend 32101 +tokenizer length before expend 32100 +100%|███████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████| 4/4 [03:55<00:00, 58.94s/it] +[2025-04-26 11:28:14] Finish Eval in 80000 steps...████████████████████████████████████████████████████████████████████████████████████████████████████| 4/4 [03:55<00:00, 58.66s/it] +processing left-padding... +processing left-padding... +processing left-padding... +processing left-padding... +[2025-04-26 11:28:33] Saved checkpoint to checkpoint/Afteri2i_100kckpt_LLAVAT5_empty_Fixed_just_segment_pretrain_stage3_dreambench_recap_400k_t2i_400k_flux_200k_midjourney_150k_recovery_150k_3e4_epoch2_no_rotated/003-GPT-XL/checkpoints/0080000.pt +[2025-04-26 11:28:35] Removed old checkpoint: checkpoint/Afteri2i_100kckpt_LLAVAT5_empty_Fixed_just_segment_pretrain_stage3_dreambench_recap_400k_t2i_400k_flux_200k_midjourney_150k_recovery_150k_3e4_epoch2_no_rotated/003-GPT-XL/checkpoints/0078000.pt + 41%|███████████████████████████████████████████████████████████▏ | 9650/23458 [2:47:21<3:25:41, 1.12it/s][2025-04-26 11:28:57] (step=0080025) Train Loss: 5.2299, Train Steps/Sec: 0.07 + 41%|███████████████████████████████████████████████████████████▍ | 9675/23458 [2:47:44<3:17:26, 1.16it/s][2025-04-26 11:29:19] (step=0080050) Train Loss: 5.2434, Train Steps/Sec: 1.13 + 41%|███████████████████████████████████████████████████████████▌ | 9700/23458 [2:48:05<3:14:16, 1.18it/s][2025-04-26 11:29:40] (step=0080075) Train Loss: 5.2021, Train Steps/Sec: 1.18 + 41%|███████████████████████████████████████████████████████████▋ | 9725/23458 [2:48:26<3:13:21, 1.18it/s][2025-04-26 11:30:02] (step=0080100) Train Loss: 5.2045, Train Steps/Sec: 1.17 + 42%|███████████████████████████████████████████████████████████▊ | 9750/23458 [2:48:48<3:17:23, 1.16it/s][2025-04-26 11:30:23] (step=0080125) Train Loss: 5.1867, Train Steps/Sec: 1.18 + 42%|████████████████████████████████████████████████████████████ | 9775/23458 [2:49:09<3:15:00, 1.17it/s][2025-04-26 11:30:44] (step=0080150) Train Loss: 5.2372, Train Steps/Sec: 1.17 + 42%|████████████████████████████████████████████████████████████▏ | 9800/23458 [2:49:30<3:12:36, 1.18it/s][2025-04-26 11:31:05] (step=0080175) Train Loss: 5.1763, Train Steps/Sec: 1.17 + 42%|████████████████████████████████████████████████████████████▎ | 9825/23458 [2:49:52<3:12:21, 1.18it/s][2025-04-26 11:31:28] (step=0080200) Train Loss: 5.2027, Train Steps/Sec: 1.12 + 42%|████████████████████████████████████████████████████████████▍ | 9850/23458 [2:50:14<3:16:13, 1.16it/s][2025-04-26 11:31:49] (step=0080225) Train Loss: 5.2090, Train Steps/Sec: 1.17 + 42%|████████████████████████████████████████████████████████████▌ | 9875/23458 [2:50:35<3:13:42, 1.17it/s][2025-04-26 11:32:10] (step=0080250) Train Loss: 5.1535, Train Steps/Sec: 1.17 + 42%|████████████████████████████████████████████████████████████▊ | 9900/23458 [2:50:56<3:11:05, 1.18it/s][2025-04-26 11:32:32] (step=0080275) Train Loss: 5.2156, Train Steps/Sec: 1.17 + 42%|████████████████████████████████████████████████████████████▉ | 9925/23458 [2:51:18<3:09:50, 1.19it/s][2025-04-26 11:32:53] (step=0080300) Train Loss: 5.2084, Train Steps/Sec: 1.17 + 42%|█████████████████████████████████████████████████████████████ | 9950/23458 [2:51:39<3:16:08, 1.15it/s][2025-04-26 11:33:14] (step=0080325) Train Loss: 5.1890, Train Steps/Sec: 1.17 + 43%|█████████████████████████████████████████████████████████████▏ | 9975/23458 [2:52:00<3:12:20, 1.17it/s][2025-04-26 11:33:36] (step=0080350) Train Loss: 5.2679, Train Steps/Sec: 1.17 + 43%|████████████████████████████████████████████████████████████▉ | 10000/23458 [2:52:22<3:10:59, 1.17it/s][2025-04-26 11:33:57] (step=0080375) Train Loss: 5.1822, Train Steps/Sec: 1.17 + 43%|█████████████████████████████████████████████████████████████ | 10025/23458 [2:52:43<3:09:09, 1.18it/s][2025-04-26 11:34:18] (step=0080400) Train Loss: 5.2923, Train Steps/Sec: 1.17 + 43%|█████████████████████████████████████████████████████████████▎ | 10050/23458 [2:53:05<3:13:41, 1.15it/s][2025-04-26 11:34:41] (step=0080425) Train Loss: 5.2111, Train Steps/Sec: 1.12 + 43%|█████████████████████████████████████████████████████████████▍ | 10075/23458 [2:53:27<3:10:35, 1.17it/s][2025-04-26 11:35:02] (step=0080450) Train Loss: 5.2618, Train Steps/Sec: 1.17 + 43%|█████████████████████████████████████████████████████████████▌ | 10100/23458 [2:53:50<3:47:46, 1.02s/it][2025-04-26 11:35:25] (step=0080475) Train Loss: 5.2126, Train Steps/Sec: 1.07 + 43%|█████████████████████████████████████████████████████████████▋ | 10125/23458 [2:54:11<3:06:37, 1.19it/s][2025-04-26 11:35:47] (step=0080500) Train Loss: 5.2458, Train Steps/Sec: 1.17 + 43%|█████████████████████████████████████████████████████████████▊ | 10150/23458 [2:54:33<3:13:03, 1.15it/s][2025-04-26 11:36:10] (step=0080525) Train Loss: 5.2094, Train Steps/Sec: 1.08 + 43%|██████████████████████████████████████████████████████████████ | 10175/23458 [2:54:56<3:08:59, 1.17it/s][2025-04-26 11:36:31] (step=0080550) Train Loss: 5.1727, Train Steps/Sec: 1.16 + 43%|██████████████████████████████████████████████████████████████▏ | 10200/23458 [2:55:17<3:06:56, 1.18it/s][2025-04-26 11:36:53] (step=0080575) Train Loss: 5.2490, Train Steps/Sec: 1.18 + 44%|██████████████████████████████████████████████████████████████▎ | 10225/23458 [2:55:39<3:06:01, 1.19it/s][2025-04-26 11:37:14] (step=0080600) Train Loss: 5.1792, Train Steps/Sec: 1.17 + 44%|██████████████████████████████████████████████████████████████▍ | 10250/23458 [2:56:00<3:10:26, 1.16it/s][2025-04-26 11:37:35] (step=0080625) Train Loss: 5.2775, Train Steps/Sec: 1.18 + 44%|██████████████████████████████████████████████████████████████▋ | 10275/23458 [2:56:21<3:08:05, 1.17it/s][2025-04-26 11:37:57] (step=0080650) Train Loss: 5.2371, Train Steps/Sec: 1.17 + 44%|██████████████████████████████████████████████████████████████▊ | 10300/23458 [2:56:43<3:06:10, 1.18it/s][2025-04-26 11:38:18] (step=0080675) Train Loss: 5.2461, Train Steps/Sec: 1.17 + 44%|██████████████████████████████████████████████████████████████▉ | 10325/23458 [2:57:04<3:04:35, 1.19it/s][2025-04-26 11:38:39] (step=0080700) Train Loss: 5.2001, Train Steps/Sec: 1.17 + 44%|███████████████████████████████████████████████████████████████ | 10350/23458 [2:57:27<3:14:36, 1.12it/s][2025-04-26 11:39:02] (step=0080725) Train Loss: 5.2294, Train Steps/Sec: 1.08 + 44%|███████████████████████████████████████████████████████████████▏ | 10375/23458 [2:57:49<3:05:55, 1.17it/s][2025-04-26 11:39:24] (step=0080750) Train Loss: 5.1969, Train Steps/Sec: 1.17 + 44%|███████████████████████████████████████████████████████████████▍ | 10400/23458 [2:58:10<3:04:30, 1.18it/s][2025-04-26 11:39:45] (step=0080775) Train Loss: 5.2025, Train Steps/Sec: 1.17 + 44%|███████████████████████████████████████████████████████████████▌ | 10425/23458 [2:58:31<3:02:49, 1.19it/s][2025-04-26 11:40:06] (step=0080800) Train Loss: 5.2036, Train Steps/Sec: 1.17 + 45%|███████████████████████████████████████████████████████████████▋ | 10450/23458 [2:58:52<3:07:33, 1.16it/s][2025-04-26 11:40:28] (step=0080825) Train Loss: 5.2235, Train Steps/Sec: 1.18 + 45%|███████████████████████████████████████████████████████████████▊ | 10475/23458 [2:59:14<3:04:28, 1.17it/s][2025-04-26 11:40:49] (step=0080850) Train Loss: 5.2662, Train Steps/Sec: 1.17 + 45%|████████████████████████████████████████████████████████████████ | 10500/23458 [2:59:36<3:04:26, 1.17it/s][2025-04-26 11:41:11] (step=0080875) Train Loss: 5.2085, Train Steps/Sec: 1.13 + 45%|████████████████████████████████████████████████████████████████▏ | 10525/23458 [2:59:57<3:01:41, 1.19it/s][2025-04-26 11:41:33] (step=0080900) Train Loss: 5.2199, Train Steps/Sec: 1.17 + 45%|████████████████████████████████████████████████████████████████▎ | 10550/23458 [3:00:19<3:07:51, 1.15it/s][2025-04-26 11:41:54] (step=0080925) Train Loss: 5.2501, Train Steps/Sec: 1.17 + 45%|████████████████████████████████████████████████████████████████▍ | 10575/23458 [3:00:40<3:03:48, 1.17it/s][2025-04-26 11:42:15] (step=0080950) Train Loss: 5.2358, Train Steps/Sec: 1.17 + 45%|████████████████████████████████████████████████████████████████▌ | 10600/23458 [3:01:01<3:01:39, 1.18it/s][2025-04-26 11:42:37] (step=0080975) Train Loss: 5.1623, Train Steps/Sec: 1.17 + 45%|████████████████████████████████████████████████████████████████▊ | 10625/23458 [3:01:23<3:00:03, 1.19it/s][2025-04-26 11:42:58] (step=0081000) Train Loss: 5.1754, Train Steps/Sec: 1.17 + 45%|████████████████████████████████████████████████████████████████▉ | 10650/23458 [3:01:44<3:05:26, 1.15it/s][2025-04-26 11:43:19] (step=0081025) Train Loss: 5.2735, Train Steps/Sec: 1.17 + 46%|█████████████████████████████████████████████████████████████████ | 10675/23458 [3:02:05<3:02:54, 1.16it/s][2025-04-26 11:43:41] (step=0081050) Train Loss: 5.1815, Train Steps/Sec: 1.17 + 46%|█████████████████████████████████████████████████████████████████▏ | 10700/23458 [3:02:28<3:00:34, 1.18it/s][2025-04-26 11:44:03] (step=0081075) Train Loss: 5.2468, Train Steps/Sec: 1.13 + 46%|█████████████████████████████████████████████████████████████████▍ | 10725/23458 [3:02:49<2:58:43, 1.19it/s][2025-04-26 11:44:24] (step=0081100) Train Loss: 5.2042, Train Steps/Sec: 1.17 + 46%|█████████████████████████████████████████████████████████████████▌ | 10750/23458 [3:03:10<3:03:21, 1.16it/s][2025-04-26 11:44:46] (step=0081125) Train Loss: 5.2870, Train Steps/Sec: 1.17 + 46%|█████████████████████████████████████████████████████████████████▋ | 10775/23458 [3:03:32<3:00:24, 1.17it/s][2025-04-26 11:45:07] (step=0081150) Train Loss: 5.2561, Train Steps/Sec: 1.17 + 46%|█████████████████████████████████████████████████████████████████▊ | 10800/23458 [3:03:53<2:59:12, 1.18it/s][2025-04-26 11:45:28] (step=0081175) Train Loss: 5.2443, Train Steps/Sec: 1.17 + 46%|█████████████████████████████████████████████████████████████████▉ | 10825/23458 [3:04:17<3:01:38, 1.16it/s][2025-04-26 11:45:52] (step=0081200) Train Loss: 5.2681, Train Steps/Sec: 1.04 + 46%|██████████████████████████████████████████████████████████████████▏ | 10850/23458 [3:04:38<3:02:01, 1.15it/s][2025-04-26 11:46:13] (step=0081225) Train Loss: 5.2193, Train Steps/Sec: 1.17 + 46%|██████████████████████████████████████████████████████████████████▎ | 10875/23458 [3:04:59<2:59:15, 1.17it/s][2025-04-26 11:46:35] (step=0081250) Train Loss: 5.2152, Train Steps/Sec: 1.17 + 46%|██████████████████████████████████████████████████████████████████▍ | 10900/23458 [3:05:22<3:40:30, 1.05s/it][2025-04-26 11:46:57] (step=0081275) Train Loss: 5.2812, Train Steps/Sec: 1.12 + 47%|██████████████████████████████████████████████████████████████████▌ | 10925/23458 [3:05:43<2:56:12, 1.19it/s][2025-04-26 11:47:18] (step=0081300) Train Loss: 5.2006, Train Steps/Sec: 1.17 + 47%|██████████████████████████████████████████████████████████████████▊ | 10950/23458 [3:06:04<3:00:46, 1.15it/s][2025-04-26 11:47:40] (step=0081325) Train Loss: 5.2496, Train Steps/Sec: 1.17 + 47%|██████████████████████████████████████████████████████████████████▉ | 10975/23458 [3:06:26<2:58:22, 1.17it/s][2025-04-26 11:48:01] (step=0081350) Train Loss: 5.1429, Train Steps/Sec: 1.17 + 47%|███████████████████████████████████████████████████████████████████ | 11000/23458 [3:06:47<2:56:48, 1.17it/s][2025-04-26 11:48:22] (step=0081375) Train Loss: 5.2397, Train Steps/Sec: 1.17 + 47%|███████████████████████████████████████████████████████████████████▏ | 11025/23458 [3:07:09<2:55:06, 1.18it/s][2025-04-26 11:48:45] (step=0081400) Train Loss: 5.2700, Train Steps/Sec: 1.12 + 47%|███████████████████████████████████████████████████████████████████▎ | 11050/23458 [3:07:31<2:59:28, 1.15it/s][2025-04-26 11:49:06] (step=0081425) Train Loss: 5.2294, Train Steps/Sec: 1.17 + 47%|███████████████████████████████████████████████████████████████████▌ | 11075/23458 [3:07:53<2:58:38, 1.16it/s][2025-04-26 11:49:28] (step=0081450) Train Loss: 5.1557, Train Steps/Sec: 1.12 + 47%|███████████████████████████████████████████████████████████████████▋ | 11100/23458 [3:08:14<2:55:36, 1.17it/s][2025-04-26 11:49:50] (step=0081475) Train Loss: 5.2303, Train Steps/Sec: 1.17 + 47%|███████████████████████████████████████████████████████████████████▊ | 11125/23458 [3:08:36<2:52:51, 1.19it/s][2025-04-26 11:50:11] (step=0081500) Train Loss: 5.2469, Train Steps/Sec: 1.17 + 48%|███████████████████████████████████████████████████████████████████▉ | 11150/23458 [3:08:57<2:58:00, 1.15it/s][2025-04-26 11:50:32] (step=0081525) Train Loss: 5.2575, Train Steps/Sec: 1.17 + 48%|████████████████████████████████████████████████████████████████████ | 11175/23458 [3:09:19<2:58:27, 1.15it/s][2025-04-26 11:50:55] (step=0081550) Train Loss: 5.1735, Train Steps/Sec: 1.12 + 48%|████████████████████████████████████████████████████████████████████▎ | 11200/23458 [3:09:41<2:52:21, 1.19it/s][2025-04-26 11:51:16] (step=0081575) Train Loss: 5.1104, Train Steps/Sec: 1.17 + 48%|████████████████████████████████████████████████████████████████████▍ | 11225/23458 [3:10:02<2:51:25, 1.19it/s][2025-04-26 11:51:37] (step=0081600) Train Loss: 5.2384, Train Steps/Sec: 1.17 + 48%|████████████████████████████████████████████████████████████████████▌ | 11250/23458 [3:10:23<2:56:00, 1.16it/s][2025-04-26 11:51:59] (step=0081625) Train Loss: 5.1480, Train Steps/Sec: 1.18 + 48%|████████████████████████████████████████████████████████████████████▋ | 11275/23458 [3:10:45<2:54:29, 1.16it/s][2025-04-26 11:52:20] (step=0081650) Train Loss: 5.2557, Train Steps/Sec: 1.18 + 48%|████████████████████████████████████████████████████████████████████▉ | 11300/23458 [3:11:06<2:51:35, 1.18it/s][2025-04-26 11:52:41] (step=0081675) Train Loss: 5.1780, Train Steps/Sec: 1.17 + 48%|█████████████████████████████████████████████████████████████████████ | 11325/23458 [3:11:27<2:50:28, 1.19it/s][2025-04-26 11:53:03] (step=0081700) Train Loss: 5.2164, Train Steps/Sec: 1.17 + 48%|█████████████████████████████████████████████████████████████████████▏ | 11350/23458 [3:11:49<2:56:08, 1.15it/s][2025-04-26 11:53:25] (step=0081725) Train Loss: 5.1688, Train Steps/Sec: 1.13 + 48%|█████████████████████████████████████████████████████████████████████▎ | 11375/23458 [3:12:11<2:51:48, 1.17it/s][2025-04-26 11:53:46] (step=0081750) Train Loss: 5.2312, Train Steps/Sec: 1.18 + 49%|█████████████████████████████████████████████████████████████████████▍ | 11400/23458 [3:12:32<2:50:30, 1.18it/s][2025-04-26 11:54:07] (step=0081775) Train Loss: 5.1833, Train Steps/Sec: 1.17 + 49%|█████████████████████████████████████████████████████████████████████▋ | 11425/23458 [3:12:53<2:49:34, 1.18it/s][2025-04-26 11:54:29] (step=0081800) Train Loss: 5.2069, Train Steps/Sec: 1.17 + 49%|█████████████████████████████████████████████████████████████████████▊ | 11450/23458 [3:13:15<2:54:01, 1.15it/s][2025-04-26 11:54:50] (step=0081825) Train Loss: 5.1078, Train Steps/Sec: 1.17 + 49%|█████████████████████████████████████████████████████████████████████▉ | 11475/23458 [3:13:38<2:57:14, 1.13it/s][2025-04-26 11:55:13] (step=0081850) Train Loss: 5.2393, Train Steps/Sec: 1.09 + 49%|██████████████████████████████████████████████████████████████████████ | 11500/23458 [3:13:59<2:48:24, 1.18it/s][2025-04-26 11:55:34] (step=0081875) Train Loss: 5.2020, Train Steps/Sec: 1.17 + 49%|██████████████████████████████████████████████████████████████████████▎ | 11525/23458 [3:14:20<2:48:27, 1.18it/s][2025-04-26 11:55:56] (step=0081900) Train Loss: 5.2522, Train Steps/Sec: 1.17 + 49%|██████████████████████████████████████████████████████████████████████▍ | 11550/23458 [3:14:43<3:00:56, 1.10it/s][2025-04-26 11:56:18] (step=0081925) Train Loss: 5.1425, Train Steps/Sec: 1.13 + 49%|██████████████████████████████████████████████████████████████████████▌ | 11575/23458 [3:15:04<2:48:56, 1.17it/s][2025-04-26 11:56:39] (step=0081950) Train Loss: 5.1746, Train Steps/Sec: 1.17 + 49%|██████████████████████████████████████████████████████████████████████▋ | 11600/23458 [3:15:25<2:47:28, 1.18it/s][2025-04-26 11:57:00] (step=0081975) Train Loss: 5.2081, Train Steps/Sec: 1.17 + 50%|██████████████████████████████████████████████████████████████████████▊ | 11625/23458 [3:15:46<2:45:34, 1.19it/s][2025-04-26 11:57:22] (step=0082000) Train Loss: 5.1475, Train Steps/Sec: 1.17 +You are using a model of type blip-2 to instantiate a model of type instructblip. This is not supported for all configurations of models and can yield errors. +[2025-04-26 11:57:22] text_config is None. Initializing the text config with default values (`OPTConfig`). +model_name_or_path:/tmp/haozhezhao/model/blip2-flan-t5-xl +image_place_holder: +tokenizer length after expend 32101 +tokenizer length before expend 32100 +100%|███████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████| 4/4 [03:56<00:00, 59.07s/it] +[2025-04-26 12:02:25] Finish Eval in 82000 steps...████████████████████████████████████████████████████████████████████████████████████████████████████| 4/4 [03:55<00:00, 58.60s/it] +processing left-padding... +processing left-padding... +processing left-padding... +processing left-padding... +[2025-04-26 12:02:44] Saved checkpoint to checkpoint/Afteri2i_100kckpt_LLAVAT5_empty_Fixed_just_segment_pretrain_stage3_dreambench_recap_400k_t2i_400k_flux_200k_midjourney_150k_recovery_150k_3e4_epoch2_no_rotated/003-GPT-XL/checkpoints/0082000.pt +[2025-04-26 12:02:46] Removed old checkpoint: checkpoint/Afteri2i_100kckpt_LLAVAT5_empty_Fixed_just_segment_pretrain_stage3_dreambench_recap_400k_t2i_400k_flux_200k_midjourney_150k_recovery_150k_3e4_epoch2_no_rotated/003-GPT-XL/checkpoints/0080000.pt + 50%|███████████████████████████████████████████████████████████████████████ | 11650/23458 [3:21:33<2:54:29, 1.13it/s][2025-04-26 12:03:08] (step=0082025) Train Loss: 5.2146, Train Steps/Sec: 0.07 + 50%|███████████████████████████████████████████████████████████████████████▏ | 11675/23458 [3:21:54<2:47:10, 1.17it/s][2025-04-26 12:03:29] (step=0082050) Train Loss: 5.2572, Train Steps/Sec: 1.18 + 50%|███████████████████████████████████████████████████████████████████████▎ | 11700/23458 [3:22:17<3:27:00, 1.06s/it][2025-04-26 12:03:52] (step=0082075) Train Loss: 5.2018, Train Steps/Sec: 1.08 + 50%|███████████████████████████████████████████████████████████████████████▍ | 11725/23458 [3:22:38<2:44:35, 1.19it/s][2025-04-26 12:04:14] (step=0082100) Train Loss: 5.1821, Train Steps/Sec: 1.17 + 50%|███████████████████████████████████████████████████████████████████████▋ | 11750/23458 [3:23:00<2:49:06, 1.15it/s][2025-04-26 12:04:35] (step=0082125) Train Loss: 5.2398, Train Steps/Sec: 1.17 + 50%|███████████████████████████████████████████████████████████████████████▊ | 11775/23458 [3:23:21<2:46:35, 1.17it/s][2025-04-26 12:04:56] (step=0082150) Train Loss: 5.1624, Train Steps/Sec: 1.17 + 50%|███████████████████████████████████████████████████████████████████████▉ | 11800/23458 [3:23:42<2:45:41, 1.17it/s][2025-04-26 12:05:18] (step=0082175) Train Loss: 5.1693, Train Steps/Sec: 1.17 + 50%|████████████████████████████████████████████████████████████████████████ | 11825/23458 [3:24:04<2:43:05, 1.19it/s][2025-04-26 12:05:39] (step=0082200) Train Loss: 5.1855, Train Steps/Sec: 1.17 + 51%|████████████████████████████████████████████████████████████████████████▏ | 11850/23458 [3:24:26<2:50:30, 1.13it/s][2025-04-26 12:06:01] (step=0082225) Train Loss: 5.2456, Train Steps/Sec: 1.13 + 51%|████████████████████████████████████████████████████████████████████████▍ | 11875/23458 [3:24:47<2:45:04, 1.17it/s][2025-04-26 12:06:23] (step=0082250) Train Loss: 5.3109, Train Steps/Sec: 1.17 + 51%|████████████████████████████████████████████████████████████████████████▌ | 11900/23458 [3:25:09<2:43:47, 1.18it/s][2025-04-26 12:06:44] (step=0082275) Train Loss: 5.2153, Train Steps/Sec: 1.17 + 51%|████████████████████████████████████████████████████████████████████████▋ | 11925/23458 [3:25:30<2:42:14, 1.18it/s][2025-04-26 12:07:05] (step=0082300) Train Loss: 5.1786, Train Steps/Sec: 1.17 + 51%|████████████████████████████████████████████████████████████████████████▊ | 11950/23458 [3:25:51<2:47:29, 1.15it/s][2025-04-26 12:07:27] (step=0082325) Train Loss: 5.2174, Train Steps/Sec: 1.17 + 51%|████████████████████████████████████████████████████████████████████████▉ | 11975/23458 [3:26:13<2:43:08, 1.17it/s][2025-04-26 12:07:48] (step=0082350) Train Loss: 5.1947, Train Steps/Sec: 1.17 + 51%|█████████████████████████████████████████████████████████████████████████▏ | 12000/23458 [3:26:35<2:42:43, 1.17it/s][2025-04-26 12:08:10] (step=0082375) Train Loss: 5.2321, Train Steps/Sec: 1.13 + 51%|█████████████████████████████████████████████████████████████████████████▎ | 12025/23458 [3:26:56<2:39:43, 1.19it/s][2025-04-26 12:08:31] (step=0082400) Train Loss: 5.2587, Train Steps/Sec: 1.17 + 51%|█████████████████████████████████████████████████████████████████████████▍ | 12050/23458 [3:27:17<2:44:49, 1.15it/s][2025-04-26 12:08:53] (step=0082425) Train Loss: 5.3106, Train Steps/Sec: 1.17 + 51%|█████████████████████████████████████████████████████████████████████████▌ | 12075/23458 [3:27:39<2:42:02, 1.17it/s][2025-04-26 12:09:14] (step=0082450) Train Loss: 5.2623, Train Steps/Sec: 1.17 + 52%|█████████████████████████████████████████████████████████████████████████▊ | 12100/23458 [3:28:00<2:40:18, 1.18it/s][2025-04-26 12:09:35] (step=0082475) Train Loss: 5.2453, Train Steps/Sec: 1.17 + 52%|█████████████████████████████████████████████████████████████████████████▉ | 12125/23458 [3:28:22<2:43:09, 1.16it/s][2025-04-26 12:09:58] (step=0082500) Train Loss: 5.2369, Train Steps/Sec: 1.12 + 52%|██████████████████████████████████████████████████████████████████████████ | 12150/23458 [3:28:45<2:43:53, 1.15it/s][2025-04-26 12:10:20] (step=0082525) Train Loss: 5.1854, Train Steps/Sec: 1.13 + 52%|██████████████████████████████████████████████████████████████████████████▏ | 12175/23458 [3:29:06<2:40:43, 1.17it/s][2025-04-26 12:10:41] (step=0082550) Train Loss: 5.2246, Train Steps/Sec: 1.17 + 52%|██████████████████████████████████████████████████████████████████████████▎ | 12200/23458 [3:29:27<2:39:06, 1.18it/s][2025-04-26 12:11:03] (step=0082575) Train Loss: 5.2645, Train Steps/Sec: 1.17 + 52%|██████████████████████████████████████████████████████████████████████████▌ | 12225/23458 [3:29:49<2:37:28, 1.19it/s][2025-04-26 12:11:24] (step=0082600) Train Loss: 5.2601, Train Steps/Sec: 1.17 + 52%|██████████████████████████████████████████████████████████████████████████▋ | 12250/23458 [3:30:10<2:41:44, 1.15it/s][2025-04-26 12:11:45] (step=0082625) Train Loss: 5.2164, Train Steps/Sec: 1.17 + 52%|██████████████████████████████████████████████████████████████████████████▊ | 12275/23458 [3:30:32<3:03:54, 1.01it/s][2025-04-26 12:12:07] (step=0082650) Train Loss: 5.2003, Train Steps/Sec: 1.13 + 52%|██████████████████████████████████████████████████████████████████████████▉ | 12300/23458 [3:30:53<2:37:14, 1.18it/s][2025-04-26 12:12:29] (step=0082675) Train Loss: 5.2575, Train Steps/Sec: 1.17 + 53%|███████████████████████████████████████████████████████████████████████████▏ | 12325/23458 [3:31:15<2:36:05, 1.19it/s][2025-04-26 12:12:50] (step=0082700) Train Loss: 5.1848, Train Steps/Sec: 1.17 + 53%|███████████████████████████████████████████████████████████████████████████▎ | 12350/23458 [3:31:38<3:28:04, 1.12s/it][2025-04-26 12:13:13] (step=0082725) Train Loss: 5.2034, Train Steps/Sec: 1.08 + 53%|███████████████████████████████████████████████████████████████████████████▍ | 12375/23458 [3:31:59<2:38:09, 1.17it/s][2025-04-26 12:13:34] (step=0082750) Train Loss: 5.1924, Train Steps/Sec: 1.17 + 53%|███████████████████████████████████████████████████████████████████████████▌ | 12400/23458 [3:32:20<2:35:57, 1.18it/s][2025-04-26 12:13:56] (step=0082775) Train Loss: 5.1899, Train Steps/Sec: 1.17 + 53%|███████████████████████████████████████████████████████████████████████████▋ | 12425/23458 [3:32:42<2:34:44, 1.19it/s][2025-04-26 12:14:17] (step=0082800) Train Loss: 5.2436, Train Steps/Sec: 1.17 + 53%|███████████████████████████████████████████████████████████████████████████▉ | 12450/23458 [3:33:03<2:39:02, 1.15it/s][2025-04-26 12:14:38] (step=0082825) Train Loss: 5.2469, Train Steps/Sec: 1.17 + 53%|████████████████████████████████████████████████████████████████████████████ | 12475/23458 [3:33:24<2:36:22, 1.17it/s][2025-04-26 12:15:00] (step=0082850) Train Loss: 5.1976, Train Steps/Sec: 1.17 + 53%|████████████████████████████████████████████████████████████████████████████▏ | 12500/23458 [3:33:47<3:28:24, 1.14s/it][2025-04-26 12:15:22] (step=0082875) Train Loss: 5.2191, Train Steps/Sec: 1.12 + 53%|████████████████████████████████████████████████████████████████████████████▎ | 12525/23458 [3:34:09<2:35:13, 1.17it/s][2025-04-26 12:15:44] (step=0082900) Train Loss: 5.1782, Train Steps/Sec: 1.12 + 53%|████████████████████████████████████████████████████████████████████████████▌ | 12550/23458 [3:34:30<2:37:41, 1.15it/s][2025-04-26 12:16:06] (step=0082925) Train Loss: 5.2027, Train Steps/Sec: 1.17 + 54%|████████████████████████████████████████████████████████████████████████████▋ | 12575/23458 [3:34:52<2:34:17, 1.18it/s][2025-04-26 12:16:27] (step=0082950) Train Loss: 5.2438, Train Steps/Sec: 1.18 + 54%|████████████████████████████████████████████████████████████████████████████▊ | 12600/23458 [3:35:13<2:33:37, 1.18it/s][2025-04-26 12:16:48] (step=0082975) Train Loss: 5.2097, Train Steps/Sec: 1.17 + 54%|████████████████████████████████████████████████████████████████████████████▉ | 12625/23458 [3:35:34<2:32:15, 1.19it/s][2025-04-26 12:17:10] (step=0083000) Train Loss: 5.2168, Train Steps/Sec: 1.17 + 54%|█████████████████████████████████████████████████████████████████████████████ | 12650/23458 [3:35:56<2:40:22, 1.12it/s][2025-04-26 12:17:32] (step=0083025) Train Loss: 5.1443, Train Steps/Sec: 1.13 + 54%|█████████████████████████████████████████████████████████████████████████████▎ | 12675/23458 [3:36:18<2:33:44, 1.17it/s][2025-04-26 12:17:53] (step=0083050) Train Loss: 5.1646, Train Steps/Sec: 1.17 + 54%|█████████████████████████████████████████████████████████████████████████████▍ | 12700/23458 [3:36:39<2:31:25, 1.18it/s][2025-04-26 12:18:14] (step=0083075) Train Loss: 5.1859, Train Steps/Sec: 1.17 + 54%|█████████████████████████████████████████████████████████████████████████████▌ | 12725/23458 [3:37:00<2:30:19, 1.19it/s][2025-04-26 12:18:36] (step=0083100) Train Loss: 5.2762, Train Steps/Sec: 1.17 + 54%|█████████████████████████████████████████████████████████████████████████████▋ | 12750/23458 [3:37:22<2:34:49, 1.15it/s][2025-04-26 12:18:57] (step=0083125) Train Loss: 5.2262, Train Steps/Sec: 1.17 + 54%|█████████████████████████████████████████████████████████████████████████████▉ | 12775/23458 [3:37:44<3:18:56, 1.12s/it][2025-04-26 12:19:19] (step=0083150) Train Loss: 5.2372, Train Steps/Sec: 1.12 + 55%|██████████████████████████████████████████████████████████████████████████████ | 12800/23458 [3:38:06<2:36:10, 1.14it/s][2025-04-26 12:19:41] (step=0083175) Train Loss: 5.2105, Train Steps/Sec: 1.13 + 55%|██████████████████████████████████████████████████████████████████████████████▏ | 12825/23458 [3:38:27<2:29:50, 1.18it/s][2025-04-26 12:20:03] (step=0083200) Train Loss: 5.2879, Train Steps/Sec: 1.17 + 55%|██████████████████████████████████████████████████████████████████████████████▎ | 12850/23458 [3:38:49<2:33:37, 1.15it/s][2025-04-26 12:20:24] (step=0083225) Train Loss: 5.2919, Train Steps/Sec: 1.18 + 55%|██████████████████████████████████████████████████████████████████████████████▍ | 12875/23458 [3:39:10<2:30:10, 1.17it/s][2025-04-26 12:20:45] (step=0083250) Train Loss: 5.2661, Train Steps/Sec: 1.18 + 55%|██████████████████████████████████████████████████████████████████████████████▋ | 12900/23458 [3:39:31<2:29:17, 1.18it/s][2025-04-26 12:21:07] (step=0083275) Train Loss: 5.2156, Train Steps/Sec: 1.18 + 55%|██████████████████████████████████████████████████████████████████████████████▊ | 12925/23458 [3:39:53<2:27:46, 1.19it/s][2025-04-26 12:21:28] (step=0083300) Train Loss: 5.1600, Train Steps/Sec: 1.17 + 55%|██████████████████████████████████████████████████████████████████████████████▉ | 12950/23458 [3:40:14<2:31:46, 1.15it/s][2025-04-26 12:21:49] (step=0083325) Train Loss: 5.2731, Train Steps/Sec: 1.18 + 55%|███████████████████████████████████████████████████████████████████████████████ | 12975/23458 [3:40:35<2:29:10, 1.17it/s][2025-04-26 12:22:11] (step=0083350) Train Loss: 5.1684, Train Steps/Sec: 1.17 + 55%|███████████████████████████████████████████████████████████████████████████████▏ | 13000/23458 [3:40:57<3:12:23, 1.10s/it][2025-04-26 12:22:33] (step=0083375) Train Loss: 5.2394, Train Steps/Sec: 1.13 + 56%|███████████████████████████████████████████████████████████████████████████████▍ | 13025/23458 [3:41:19<2:26:24, 1.19it/s][2025-04-26 12:22:55] (step=0083400) Train Loss: 5.3234, Train Steps/Sec: 1.12 + 56%|███████████████████████████████████████████████████████████████████████████████▌ | 13050/23458 [3:41:41<2:30:54, 1.15it/s][2025-04-26 12:23:16] (step=0083425) Train Loss: 5.1660, Train Steps/Sec: 1.17 + 56%|███████████████████████████████████████████████████████████████████████████████▋ | 13075/23458 [3:42:03<2:39:51, 1.08it/s][2025-04-26 12:23:39] (step=0083450) Train Loss: 5.2341, Train Steps/Sec: 1.12 + 56%|███████████████████████████████████████████████████████████████████████████████▊ | 13100/23458 [3:42:25<2:26:36, 1.18it/s][2025-04-26 12:24:00] (step=0083475) Train Loss: 5.3115, Train Steps/Sec: 1.18 + 56%|████████████████████████████████████████████████████████████████████████████████ | 13125/23458 [3:42:46<2:24:46, 1.19it/s][2025-04-26 12:24:21] (step=0083500) Train Loss: 5.1411, Train Steps/Sec: 1.17 + 56%|████████████████████████████████████████████████████████████████████████████████▏ | 13150/23458 [3:43:07<2:28:55, 1.15it/s][2025-04-26 12:24:43] (step=0083525) Train Loss: 5.2177, Train Steps/Sec: 1.17 + 56%|████████████████████████████████████████████████████████████████████████████████▎ | 13175/23458 [3:43:29<2:25:56, 1.17it/s][2025-04-26 12:25:04] (step=0083550) Train Loss: 5.2463, Train Steps/Sec: 1.17 + 56%|████████████████████████████████████████████████████████████████████████████████▍ | 13200/23458 [3:43:51<2:28:00, 1.16it/s][2025-04-26 12:25:26] (step=0083575) Train Loss: 5.2573, Train Steps/Sec: 1.13 + 56%|████████████████████████████████████████████████████████████████████████████████▌ | 13225/23458 [3:44:12<2:23:05, 1.19it/s][2025-04-26 12:25:47] (step=0083600) Train Loss: 5.1722, Train Steps/Sec: 1.17 + 56%|████████████████████████████████████████████████████████████████████████████████▊ | 13250/23458 [3:44:33<2:27:23, 1.15it/s][2025-04-26 12:26:09] (step=0083625) Train Loss: 5.2237, Train Steps/Sec: 1.17 + 57%|████████████████████████████████████████████████████████████████████████████████▉ | 13275/23458 [3:44:55<2:25:03, 1.17it/s][2025-04-26 12:26:30] (step=0083650) Train Loss: 5.2694, Train Steps/Sec: 1.17 + 57%|█████████████████████████████████████████████████████████████████████████████████ | 13300/23458 [3:45:17<2:33:13, 1.10it/s][2025-04-26 12:26:52] (step=0083675) Train Loss: 5.2693, Train Steps/Sec: 1.13 + 57%|█████████████████████████████████████████████████████████████████████████████████▏ | 13325/23458 [3:45:39<2:22:32, 1.18it/s][2025-04-26 12:27:15] (step=0083700) Train Loss: 5.2825, Train Steps/Sec: 1.12 + 57%|█████████████████████████████████████████████████████████████████████████████████▍ | 13350/23458 [3:46:01<2:26:36, 1.15it/s][2025-04-26 12:27:36] (step=0083725) Train Loss: 5.2050, Train Steps/Sec: 1.17 + 57%|█████████████████████████████████████████████████████████████████████████████████▌ | 13375/23458 [3:46:22<2:23:59, 1.17it/s][2025-04-26 12:27:57] (step=0083750) Train Loss: 5.1875, Train Steps/Sec: 1.17 + 57%|█████████████████████████████████████████████████████████████████████████████████▋ | 13400/23458 [3:46:43<2:21:52, 1.18it/s][2025-04-26 12:28:19] (step=0083775) Train Loss: 5.2583, Train Steps/Sec: 1.17 + 57%|█████████████████████████████████████████████████████████████████████████████████▊ | 13425/23458 [3:47:05<2:21:35, 1.18it/s][2025-04-26 12:28:40] (step=0083800) Train Loss: 5.2006, Train Steps/Sec: 1.16 + 57%|█████████████████████████████████████████████████████████████████████████████████▉ | 13450/23458 [3:47:27<2:25:07, 1.15it/s][2025-04-26 12:29:02] (step=0083825) Train Loss: 5.2362, Train Steps/Sec: 1.13 + 57%|██████████████████████████████████████████████████████████████████████████████████▏ | 13475/23458 [3:47:49<2:21:56, 1.17it/s][2025-04-26 12:29:24] (step=0083850) Train Loss: 5.2831, Train Steps/Sec: 1.13 + 58%|██████████████████████████████████████████████████████████████████████████████████▎ | 13500/23458 [3:48:10<2:21:00, 1.18it/s][2025-04-26 12:29:46] (step=0083875) Train Loss: 5.2392, Train Steps/Sec: 1.17 + 58%|██████████████████████████████████████████████████████████████████████████████████▍ | 13525/23458 [3:48:32<2:19:27, 1.19it/s][2025-04-26 12:30:07] (step=0083900) Train Loss: 5.1922, Train Steps/Sec: 1.17 + 58%|██████████████████████████████████████████████████████████████████████████████████▌ | 13550/23458 [3:48:53<2:23:13, 1.15it/s][2025-04-26 12:30:28] (step=0083925) Train Loss: 5.1389, Train Steps/Sec: 1.17 + 58%|██████████████████████████████████████████████████████████████████████████████████▊ | 13575/23458 [3:49:14<2:20:38, 1.17it/s][2025-04-26 12:30:50] (step=0083950) Train Loss: 5.2510, Train Steps/Sec: 1.17 + 58%|██████████████████████████████████████████████████████████████████████████████████▉ | 13600/23458 [3:49:36<2:19:15, 1.18it/s][2025-04-26 12:31:11] (step=0083975) Train Loss: 5.2281, Train Steps/Sec: 1.17 + 58%|███████████████████████████████████████████████████████████████████████████████████ | 13625/23458 [3:49:57<2:18:14, 1.19it/s][2025-04-26 12:31:32] (step=0084000) Train Loss: 5.1968, Train Steps/Sec: 1.17 +You are using a model of type blip-2 to instantiate a model of type instructblip. This is not supported for all configurations of models and can yield errors. +[2025-04-26 12:31:32] text_config is None. Initializing the text config with default values (`OPTConfig`). +model_name_or_path:/tmp/haozhezhao/model/blip2-flan-t5-xl +image_place_holder: +tokenizer length after expend 32101 +tokenizer length before expend 32100 +100%|███████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████| 4/4 [03:55<00:00, 58.83s/it] +[2025-04-26 12:36:36] Finish Eval in 84000 steps...████████████████████████████████████████████████████████████████████████████████████████████████████| 4/4 [03:54<00:00, 58.55s/it] +processing left-padding... +processing left-padding... +processing left-padding... +processing left-padding... +[2025-04-26 12:36:57] Saved checkpoint to checkpoint/Afteri2i_100kckpt_LLAVAT5_empty_Fixed_just_segment_pretrain_stage3_dreambench_recap_400k_t2i_400k_flux_200k_midjourney_150k_recovery_150k_3e4_epoch2_no_rotated/003-GPT-XL/checkpoints/0084000.pt +[2025-04-26 12:36:59] Removed old checkpoint: checkpoint/Afteri2i_100kckpt_LLAVAT5_empty_Fixed_just_segment_pretrain_stage3_dreambench_recap_400k_t2i_400k_flux_200k_midjourney_150k_recovery_150k_3e4_epoch2_no_rotated/003-GPT-XL/checkpoints/0082000.pt + 58%|███████████████████████████████████████████████████████████████████████████████████▏ | 13650/23458 [3:55:45<2:25:21, 1.12it/s][2025-04-26 12:37:20] (step=0084025) Train Loss: 5.2378, Train Steps/Sec: 0.07 + 58%|███████████████████████████████████████████████████████████████████████████████████▎ | 13675/23458 [3:56:06<2:19:13, 1.17it/s][2025-04-26 12:37:42] (step=0084050) Train Loss: 5.1547, Train Steps/Sec: 1.17 + 58%|███████████████████████████████████████████████████████████████████████████████████▌ | 13700/23458 [3:56:28<2:17:27, 1.18it/s][2025-04-26 12:38:03] (step=0084075) Train Loss: 5.2813, Train Steps/Sec: 1.17 + 59%|███████████████████████████████████████████████████████████████████████████████████▋ | 13725/23458 [3:56:50<2:17:17, 1.18it/s][2025-04-26 12:38:25] (step=0084100) Train Loss: 5.1336, Train Steps/Sec: 1.12 + 59%|███████████████████████████████████████████████████████████████████████████████████▊ | 13750/23458 [3:57:12<2:20:34, 1.15it/s][2025-04-26 12:38:47] (step=0084125) Train Loss: 5.3587, Train Steps/Sec: 1.13 + 59%|███████████████████████████████████████████████████████████████████████████████████▉ | 13775/23458 [3:57:33<2:18:18, 1.17it/s][2025-04-26 12:39:09] (step=0084150) Train Loss: 5.1854, Train Steps/Sec: 1.17 + 59%|████████████████████████████████████████████████████████████████████████████████████ | 13800/23458 [3:57:55<2:17:19, 1.17it/s][2025-04-26 12:39:30] (step=0084175) Train Loss: 5.2343, Train Steps/Sec: 1.17 + 59%|████████████████████████████████████████████████████████████████████████████████████▎ | 13825/23458 [3:58:16<2:14:35, 1.19it/s][2025-04-26 12:39:51] (step=0084200) Train Loss: 5.2323, Train Steps/Sec: 1.17 + 59%|████████████████████████████████████████████████████████████████████████████████████▍ | 13850/23458 [3:58:37<2:19:01, 1.15it/s][2025-04-26 12:40:12] (step=0084225) Train Loss: 5.2228, Train Steps/Sec: 1.17 + 59%|████████████████████████████████████████████████████████████████████████████████████▌ | 13875/23458 [3:58:59<2:18:12, 1.16it/s][2025-04-26 12:40:35] (step=0084250) Train Loss: 5.1654, Train Steps/Sec: 1.13 + 59%|████████████████████████████████████████████████████████████████████████████████████▋ | 13900/23458 [3:59:21<2:15:14, 1.18it/s][2025-04-26 12:40:56] (step=0084275) Train Loss: 5.3421, Train Steps/Sec: 1.17 + 59%|████████████████████████████████████████████████████████████████████████████████████▉ | 13925/23458 [3:59:42<2:13:45, 1.19it/s][2025-04-26 12:41:17] (step=0084300) Train Loss: 5.2484, Train Steps/Sec: 1.17 + 59%|█████████████████████████████████████████████████████████████████████████████████████ | 13950/23458 [4:00:04<2:55:08, 1.11s/it][2025-04-26 12:41:39] (step=0084325) Train Loss: 5.1976, Train Steps/Sec: 1.13 + 60%|█████████████████████████████████████████████████████████████████████████████████████▏ | 13975/23458 [4:00:25<2:15:12, 1.17it/s][2025-04-26 12:42:01] (step=0084350) Train Loss: 5.2352, Train Steps/Sec: 1.17 + 60%|█████████████████████████████████████████████████████████████████████████████████████▎ | 14000/23458 [4:00:47<2:14:05, 1.18it/s][2025-04-26 12:42:22] (step=0084375) Train Loss: 5.2759, Train Steps/Sec: 1.17 + 60%|█████████████████████████████████████████████████████████████████████████████████████▍ | 14025/23458 [4:01:08<2:12:34, 1.19it/s][2025-04-26 12:42:44] (step=0084400) Train Loss: 5.2078, Train Steps/Sec: 1.16 + 60%|█████████████████████████████████████████████████████████████████████████████████████▋ | 14050/23458 [4:01:30<2:16:13, 1.15it/s][2025-04-26 12:43:05] (step=0084425) Train Loss: 5.2184, Train Steps/Sec: 1.17 + 60%|█████████████████████████████████████████████████████████████████████████████████████▊ | 14075/23458 [4:01:51<2:13:48, 1.17it/s][2025-04-26 12:43:26] (step=0084450) Train Loss: 5.2634, Train Steps/Sec: 1.17 + 60%|█████████████████████████████████████████████████████████████████████████████████████▉ | 14100/23458 [4:02:13<2:26:27, 1.06it/s][2025-04-26 12:43:49] (step=0084475) Train Loss: 5.1496, Train Steps/Sec: 1.12 + 60%|██████████████████████████████████████████████████████████████████████████████████████ | 14125/23458 [4:02:36<2:11:09, 1.19it/s][2025-04-26 12:44:12] (step=0084500) Train Loss: 5.2304, Train Steps/Sec: 1.08 + 60%|██████████████████████████████████████████████████████████████████████████████████████▎ | 14150/23458 [4:02:58<2:15:13, 1.15it/s][2025-04-26 12:44:33] (step=0084525) Train Loss: 5.2540, Train Steps/Sec: 1.17 + 60%|██████████████████████████████████████████████████████████████████████████████████████▍ | 14175/23458 [4:03:19<2:11:43, 1.17it/s][2025-04-26 12:44:54] (step=0084550) Train Loss: 5.2330, Train Steps/Sec: 1.18 + 61%|██████████████████████████████████████████████████████████████████████████████████████▌ | 14200/23458 [4:03:40<2:10:24, 1.18it/s][2025-04-26 12:45:16] (step=0084575) Train Loss: 5.1535, Train Steps/Sec: 1.18 + 61%|██████████████████████████████████████████████████████████████████████████████████████▋ | 14225/23458 [4:04:02<2:09:38, 1.19it/s][2025-04-26 12:45:37] (step=0084600) Train Loss: 5.1684, Train Steps/Sec: 1.17 + 61%|██████████████████████████████████████████████████████████████████████████████████████▊ | 14250/23458 [4:04:23<2:13:31, 1.15it/s][2025-04-26 12:45:58] (step=0084625) Train Loss: 5.1829, Train Steps/Sec: 1.17 + 61%|███████████████████████████████████████████████████████████████████████████████████████ | 14275/23458 [4:04:44<2:11:01, 1.17it/s][2025-04-26 12:46:20] (step=0084650) Train Loss: 5.2168, Train Steps/Sec: 1.17 + 61%|███████████████████████████████████████████████████████████████████████████████████████▏ | 14300/23458 [4:05:06<2:09:59, 1.17it/s][2025-04-26 12:46:41] (step=0084675) Train Loss: 5.2803, Train Steps/Sec: 1.17 + 61%|███████████████████████████████████████████████████████████████████████████████████████▎ | 14325/23458 [4:05:27<2:07:45, 1.19it/s][2025-04-26 12:47:02] (step=0084700) Train Loss: 5.2972, Train Steps/Sec: 1.17 + 61%|███████████████████████████████████████████████████████████████████████████████████████▍ | 14350/23458 [4:05:50<2:13:31, 1.14it/s][2025-04-26 12:47:25] (step=0084725) Train Loss: 5.1646, Train Steps/Sec: 1.11 + 61%|███████████████████████████████████████████████████████████████████████████████████████▋ | 14375/23458 [4:06:12<2:29:10, 1.01it/s][2025-04-26 12:47:47] (step=0084750) Train Loss: 5.2072, Train Steps/Sec: 1.12 + 61%|███████████████████████████████████████████████████████████████████████████████████████▊ | 14400/23458 [4:06:33<2:07:59, 1.18it/s][2025-04-26 12:48:08] (step=0084775) Train Loss: 5.1674, Train Steps/Sec: 1.17 + 61%|███████████████████████████████████████████████████████████████████████████████████████▉ | 14425/23458 [4:06:54<2:06:49, 1.19it/s][2025-04-26 12:48:30] (step=0084800) Train Loss: 5.3115, Train Steps/Sec: 1.17 + 62%|████████████████████████████████████████████████████████████████████████████████████████ | 14450/23458 [4:07:16<2:10:06, 1.15it/s][2025-04-26 12:48:51] (step=0084825) Train Loss: 5.2484, Train Steps/Sec: 1.17 + 62%|████████████████████████████████████████████████████████████████████████████████████████▏ | 14475/23458 [4:07:38<2:08:11, 1.17it/s][2025-04-26 12:49:13] (step=0084850) Train Loss: 5.2287, Train Steps/Sec: 1.13 + 62%|████████████████████████████████████████████████████████████████████████████████████████▍ | 14500/23458 [4:07:59<2:06:15, 1.18it/s][2025-04-26 12:49:35] (step=0084875) Train Loss: 5.1467, Train Steps/Sec: 1.18 + 62%|████████████████████████████████████████████████████████████████████████████████████████▌ | 14525/23458 [4:08:21<2:05:51, 1.18it/s][2025-04-26 12:49:56] (step=0084900) Train Loss: 5.2955, Train Steps/Sec: 1.16 + 62%|████████████████████████████████████████████████████████████████████████████████████████▋ | 14550/23458 [4:08:43<2:11:30, 1.13it/s][2025-04-26 12:50:18] (step=0084925) Train Loss: 5.2105, Train Steps/Sec: 1.13 + 62%|████████████████████████████████████████████████████████████████████████████████████████▊ | 14575/23458 [4:09:04<2:06:21, 1.17it/s][2025-04-26 12:50:39] (step=0084950) Train Loss: 5.1113, Train Steps/Sec: 1.17 + 62%|█████████████████████████████████████████████████████████████████████████████████████████ | 14600/23458 [4:09:25<2:05:21, 1.18it/s][2025-04-26 12:51:01] (step=0084975) Train Loss: 5.1397, Train Steps/Sec: 1.17 + 62%|█████████████████████████████████████████████████████████████████████████████████████████▏ | 14625/23458 [4:09:48<2:04:20, 1.18it/s][2025-04-26 12:51:23] (step=0085000) Train Loss: 5.1740, Train Steps/Sec: 1.12 + 62%|█████████████████████████████████████████████████████████████████████████████████████████▎ | 14650/23458 [4:10:09<2:07:26, 1.15it/s][2025-04-26 12:51:44] (step=0085025) Train Loss: 5.2634, Train Steps/Sec: 1.17 + 63%|█████████████████████████████████████████████████████████████████████████████████████████▍ | 14675/23458 [4:10:30<2:05:03, 1.17it/s][2025-04-26 12:52:06] (step=0085050) Train Loss: 5.2193, Train Steps/Sec: 1.17 + 63%|█████████████████████████████████████████████████████████████████████████████████████████▌ | 14700/23458 [4:10:52<2:03:44, 1.18it/s][2025-04-26 12:52:27] (step=0085075) Train Loss: 5.2725, Train Steps/Sec: 1.17 + 63%|█████████████████████████████████████████████████████████████████████████████████████████▊ | 14725/23458 [4:11:13<2:02:37, 1.19it/s][2025-04-26 12:52:48] (step=0085100) Train Loss: 5.2572, Train Steps/Sec: 1.17 + 63%|█████████████████████████████████████████████████████████████████████████████████████████▉ | 14750/23458 [4:11:34<2:05:54, 1.15it/s][2025-04-26 12:53:10] (step=0085125) Train Loss: 5.1184, Train Steps/Sec: 1.17 + 63%|██████████████████████████████████████████████████████████████████████████████████████████ | 14775/23458 [4:11:57<2:09:09, 1.12it/s][2025-04-26 12:53:33] (step=0085150) Train Loss: 5.2053, Train Steps/Sec: 1.09 + 63%|██████████████████████████████████████████████████████████████████████████████████████████▏ | 14800/23458 [4:12:19<2:02:07, 1.18it/s][2025-04-26 12:53:54] (step=0085175) Train Loss: 5.2049, Train Steps/Sec: 1.18 + 63%|██████████████████████████████████████████████████████████████████████████████████████████▎ | 14825/23458 [4:12:40<2:01:19, 1.19it/s][2025-04-26 12:54:15] (step=0085200) Train Loss: 5.2862, Train Steps/Sec: 1.17 + 63%|██████████████████████████████████████████████████████████████████████████████████████████▌ | 14850/23458 [4:13:01<2:04:19, 1.15it/s][2025-04-26 12:54:37] (step=0085225) Train Loss: 5.2122, Train Steps/Sec: 1.18 + 63%|██████████████████████████████████████████████████████████████████████████████████████████▋ | 14875/23458 [4:13:23<2:02:12, 1.17it/s][2025-04-26 12:54:58] (step=0085250) Train Loss: 5.2593, Train Steps/Sec: 1.17 + 64%|██████████████████████████████████████████████████████████████████████████████████████████▊ | 14900/23458 [4:13:44<2:00:50, 1.18it/s][2025-04-26 12:55:19] (step=0085275) Train Loss: 5.2075, Train Steps/Sec: 1.17 + 64%|██████████████████████████████████████████████████████████████████████████████████████████▉ | 14925/23458 [4:14:06<2:00:28, 1.18it/s][2025-04-26 12:55:42] (step=0085300) Train Loss: 5.1088, Train Steps/Sec: 1.11 + 64%|███████████████████████████████████████████████████████████████████████████████████████████▏ | 14950/23458 [4:14:28<2:02:51, 1.15it/s][2025-04-26 12:56:03] (step=0085325) Train Loss: 5.2390, Train Steps/Sec: 1.17 + 64%|███████████████████████████████████████████████████████████████████████████████████████████▎ | 14975/23458 [4:14:49<2:00:59, 1.17it/s][2025-04-26 12:56:24] (step=0085350) Train Loss: 5.2014, Train Steps/Sec: 1.17 + 64%|███████████████████████████████████████████████████████████████████████████████████████████▍ | 15000/23458 [4:15:10<1:59:01, 1.18it/s][2025-04-26 12:56:46] (step=0085375) Train Loss: 5.2358, Train Steps/Sec: 1.17 + 64%|███████████████████████████████████████████████████████████████████████████████████████████▌ | 15025/23458 [4:15:32<1:58:46, 1.18it/s][2025-04-26 12:57:07] (step=0085400) Train Loss: 5.2248, Train Steps/Sec: 1.17 + 64%|███████████████████████████████████████████████████████████████████████████████████████████▋ | 15050/23458 [4:15:54<2:05:20, 1.12it/s][2025-04-26 12:57:29] (step=0085425) Train Loss: 5.2480, Train Steps/Sec: 1.13 + 64%|███████████████████████████████████████████████████████████████████████████████████████████▉ | 15075/23458 [4:16:16<2:08:14, 1.09it/s][2025-04-26 12:57:52] (step=0085450) Train Loss: 5.2202, Train Steps/Sec: 1.11 + 64%|████████████████████████████████████████████████████████████████████████████████████████████ | 15100/23458 [4:16:38<1:57:50, 1.18it/s][2025-04-26 12:58:13] (step=0085475) Train Loss: 5.2568, Train Steps/Sec: 1.18 + 64%|████████████████████████████████████████████████████████████████████████████████████████████▏ | 15125/23458 [4:16:59<1:56:39, 1.19it/s][2025-04-26 12:58:34] (step=0085500) Train Loss: 5.1838, Train Steps/Sec: 1.17 + 65%|████████████████████████████████████████████████████████████████████████████████████████████▎ | 15150/23458 [4:17:20<1:59:40, 1.16it/s][2025-04-26 12:58:56] (step=0085525) Train Loss: 5.2175, Train Steps/Sec: 1.17 + 65%|████████████████████████████████████████████████████████████████████████████████████████████▌ | 15175/23458 [4:17:42<1:58:25, 1.17it/s][2025-04-26 12:59:17] (step=0085550) Train Loss: 5.2222, Train Steps/Sec: 1.17 + 65%|████████████████████████████████████████████████████████████████████████████████████████████▋ | 15200/23458 [4:18:04<1:56:06, 1.19it/s][2025-04-26 12:59:39] (step=0085575) Train Loss: 5.2632, Train Steps/Sec: 1.13 + 65%|████████████████████████████████████████████████████████████████████████████████████████████▊ | 15225/23458 [4:18:26<2:02:05, 1.12it/s][2025-04-26 13:00:01] (step=0085600) Train Loss: 5.2269, Train Steps/Sec: 1.12 + 65%|████████████████████████████████████████████████████████████████████████████████████████████▉ | 15250/23458 [4:18:47<1:58:38, 1.15it/s][2025-04-26 13:00:23] (step=0085625) Train Loss: 5.2224, Train Steps/Sec: 1.17 + 65%|█████████████████████████████████████████████████████████████████████████████████████████████ | 15275/23458 [4:19:10<1:57:10, 1.16it/s][2025-04-26 13:00:45] (step=0085650) Train Loss: 5.1772, Train Steps/Sec: 1.13 + 65%|█████████████████████████████████████████████████████████████████████████████████████████████▎ | 15300/23458 [4:19:31<1:54:57, 1.18it/s][2025-04-26 13:01:06] (step=0085675) Train Loss: 5.1836, Train Steps/Sec: 1.18 + 65%|█████████████████████████████████████████████████████████████████████████████████████████████▍ | 15325/23458 [4:19:52<1:54:40, 1.18it/s][2025-04-26 13:01:27] (step=0085700) Train Loss: 5.3223, Train Steps/Sec: 1.17 + 65%|█████████████████████████████████████████████████████████████████████████████████████████████▌ | 15350/23458 [4:20:13<1:56:45, 1.16it/s][2025-04-26 13:01:49] (step=0085725) Train Loss: 5.2029, Train Steps/Sec: 1.17 + 66%|█████████████████████████████████████████████████████████████████████████████████████████████▋ | 15375/23458 [4:20:35<1:55:12, 1.17it/s][2025-04-26 13:02:10] (step=0085750) Train Loss: 5.1804, Train Steps/Sec: 1.17 + 66%|█████████████████████████████████████████████████████████████████████████████████████████████▉ | 15400/23458 [4:20:56<1:53:43, 1.18it/s][2025-04-26 13:02:31] (step=0085775) Train Loss: 5.2954, Train Steps/Sec: 1.17 + 66%|██████████████████████████████████████████████████████████████████████████████████████████████ | 15425/23458 [4:21:18<1:53:07, 1.18it/s][2025-04-26 13:02:54] (step=0085800) Train Loss: 5.2202, Train Steps/Sec: 1.12 + 66%|██████████████████████████████████████████████████████████████████████████████████████████████▏ | 15450/23458 [4:21:41<1:56:00, 1.15it/s][2025-04-26 13:03:16] (step=0085825) Train Loss: 5.2971, Train Steps/Sec: 1.13 + 66%|██████████████████████████████████████████████████████████████████████████████████████████████▎ | 15475/23458 [4:22:02<1:54:04, 1.17it/s][2025-04-26 13:03:37] (step=0085850) Train Loss: 5.2220, Train Steps/Sec: 1.17 + 66%|██████████████████████████████████████████████████████████████████████████████████████████████▍ | 15500/23458 [4:22:23<1:52:15, 1.18it/s][2025-04-26 13:03:59] (step=0085875) Train Loss: 5.2304, Train Steps/Sec: 1.17 + 66%|██████████████████████████████████████████████████████████████████████████████████████████████▋ | 15525/23458 [4:22:45<1:51:21, 1.19it/s][2025-04-26 13:04:20] (step=0085900) Train Loss: 5.2367, Train Steps/Sec: 1.17 + 66%|██████████████████████████████████████████████████████████████████████████████████████████████▊ | 15550/23458 [4:23:06<1:54:37, 1.15it/s][2025-04-26 13:04:41] (step=0085925) Train Loss: 5.2397, Train Steps/Sec: 1.17 + 66%|██████████████████████████████████████████████████████████████████████████████████████████████▉ | 15575/23458 [4:23:27<1:52:31, 1.17it/s][2025-04-26 13:05:03] (step=0085950) Train Loss: 5.2447, Train Steps/Sec: 1.17 + 67%|███████████████████████████████████████████████████████████████████████████████████████████████ | 15600/23458 [4:23:49<1:50:35, 1.18it/s][2025-04-26 13:05:24] (step=0085975) Train Loss: 5.2599, Train Steps/Sec: 1.17 + 67%|███████████████████████████████████████████████████████████████████████████████████████████████▎ | 15625/23458 [4:24:10<1:50:03, 1.19it/s][2025-04-26 13:05:45] (step=0086000) Train Loss: 5.2693, Train Steps/Sec: 1.17 +You are using a model of type blip-2 to instantiate a model of type instructblip. This is not supported for all configurations of models and can yield errors. +[2025-04-26 13:05:45] text_config is None. Initializing the text config with default values (`OPTConfig`). +model_name_or_path:/tmp/haozhezhao/model/blip2-flan-t5-xl +image_place_holder: +tokenizer length after expend 32101 +tokenizer length before expend 32100 +100%|███████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████| 4/4 [03:54<00:00, 58.57s/it] +[2025-04-26 13:10:49] Finish Eval in 86000 steps...████████████████████████████████████████████████████████████████████████████████████████████████████| 4/4 [03:53<00:00, 58.22s/it] +processing left-padding... +processing left-padding... +processing left-padding... +processing left-padding... +[2025-04-26 13:11:08] Saved checkpoint to checkpoint/Afteri2i_100kckpt_LLAVAT5_empty_Fixed_just_segment_pretrain_stage3_dreambench_recap_400k_t2i_400k_flux_200k_midjourney_150k_recovery_150k_3e4_epoch2_no_rotated/003-GPT-XL/checkpoints/0086000.pt +[2025-04-26 13:11:10] Removed old checkpoint: checkpoint/Afteri2i_100kckpt_LLAVAT5_empty_Fixed_just_segment_pretrain_stage3_dreambench_recap_400k_t2i_400k_flux_200k_midjourney_150k_recovery_150k_3e4_epoch2_no_rotated/003-GPT-XL/checkpoints/0084000.pt + 67%|███████████████████████████████████████████████████████████████████████████████████████████████▍ | 15650/23458 [4:29:56<1:55:35, 1.13it/s][2025-04-26 13:11:32] (step=0086025) Train Loss: 5.2534, Train Steps/Sec: 0.07 + 67%|███████████████████████████████████████████████████████████████████████████████████████████████▌ | 15675/23458 [4:30:18<1:51:33, 1.16it/s][2025-04-26 13:11:53] (step=0086050) Train Loss: 5.1996, Train Steps/Sec: 1.17 + 67%|███████████████████████████████████████████████████████████████████████████████████████████████▋ | 15700/23458 [4:30:39<1:50:09, 1.17it/s][2025-04-26 13:12:14] (step=0086075) Train Loss: 5.2011, Train Steps/Sec: 1.17 + 67%|███████████████████████████████████████████████████████████████████████████████████████████████▊ | 15725/23458 [4:31:02<1:50:28, 1.17it/s][2025-04-26 13:12:38] (step=0086100) Train Loss: 5.2901, Train Steps/Sec: 1.07 + 67%|████████████████████████████████████████████████████████████████████████████████████████████████ | 15750/23458 [4:31:24<1:51:46, 1.15it/s][2025-04-26 13:12:59] (step=0086125) Train Loss: 5.1875, Train Steps/Sec: 1.17 + 67%|████████████████████████████████████████████████████████████████████████████████████████████████▏ | 15775/23458 [4:31:45<1:49:47, 1.17it/s][2025-04-26 13:13:20] (step=0086150) Train Loss: 5.1956, Train Steps/Sec: 1.17 + 67%|████████████████████████████████████████████████████████████████████████████████████████████████▎ | 15800/23458 [4:32:07<1:48:37, 1.17it/s][2025-04-26 13:13:42] (step=0086175) Train Loss: 5.2781, Train Steps/Sec: 1.17 + 67%|████████████████████████████████████████████████████████████████████████████████████████████████▍ | 15825/23458 [4:32:28<1:47:21, 1.18it/s][2025-04-26 13:14:03] (step=0086200) Train Loss: 5.2062, Train Steps/Sec: 1.17 + 68%|████████████████████████████████████████████████████████████████████████████████████████████████▌ | 15850/23458 [4:32:49<1:50:52, 1.14it/s][2025-04-26 13:14:25] (step=0086225) Train Loss: 5.2333, Train Steps/Sec: 1.17 + 68%|████████████████████████████████████████████████████████████████████████████████████████████████▊ | 15875/23458 [4:33:11<1:48:18, 1.17it/s][2025-04-26 13:14:46] (step=0086250) Train Loss: 5.2621, Train Steps/Sec: 1.17 + 68%|████████████████████████████████████████████████████████████████████████████████████████████████▉ | 15900/23458 [4:33:33<2:12:00, 1.05s/it][2025-04-26 13:15:09] (step=0086275) Train Loss: 5.2492, Train Steps/Sec: 1.11 + 68%|█████████████████████████████████████████████████████████████████████████████████████████████████ | 15925/23458 [4:33:56<1:46:14, 1.18it/s][2025-04-26 13:15:32] (step=0086300) Train Loss: 5.1617, Train Steps/Sec: 1.08 + 68%|█████████████████████████████████████████████████████████████████████████████████████████████████▏ | 15950/23458 [4:34:18<1:49:32, 1.14it/s][2025-04-26 13:15:53] (step=0086325) Train Loss: 5.2847, Train Steps/Sec: 1.16 + 68%|█████████████████████████████████████████████████████████████████████████████████████████████████▍ | 15975/23458 [4:34:40<1:48:32, 1.15it/s][2025-04-26 13:16:15] (step=0086350) Train Loss: 5.2623, Train Steps/Sec: 1.16 + 68%|█████████████████████████████████████████████████████████████████████████████████████████████████▌ | 16000/23458 [4:35:01<1:45:27, 1.18it/s][2025-04-26 13:16:36] (step=0086375) Train Loss: 5.1861, Train Steps/Sec: 1.17 + 68%|█████████████████████████████████████████████████████████████████████████████████████████████████▋ | 16025/23458 [4:35:22<1:44:40, 1.18it/s][2025-04-26 13:16:58] (step=0086400) Train Loss: 5.2379, Train Steps/Sec: 1.17 + 68%|█████████████████████████████████████████████████████████████████████████████████████████████████▊ | 16050/23458 [4:35:44<1:47:09, 1.15it/s][2025-04-26 13:17:19] (step=0086425) Train Loss: 5.1802, Train Steps/Sec: 1.18 + 69%|█████████████████████████████████████████████████████████████████████████████████████████████████▉ | 16075/23458 [4:36:06<1:47:51, 1.14it/s][2025-04-26 13:17:41] (step=0086450) Train Loss: 5.2266, Train Steps/Sec: 1.13 + 69%|██████████████████████████████████████████████████████████████████████████████████████████████████▏ | 16100/23458 [4:36:28<1:44:18, 1.18it/s][2025-04-26 13:18:03] (step=0086475) Train Loss: 5.2178, Train Steps/Sec: 1.13 + 69%|██████████████████████████████████████████████████████████████████████████████████████████████████▎ | 16125/23458 [4:36:49<1:42:38, 1.19it/s][2025-04-26 13:18:25] (step=0086500) Train Loss: 5.2630, Train Steps/Sec: 1.17 + 69%|██████████████████████████████████████████████████████████████████████████████████████████████████▍ | 16150/23458 [4:37:11<1:45:41, 1.15it/s][2025-04-26 13:18:46] (step=0086525) Train Loss: 5.1679, Train Steps/Sec: 1.17 + 69%|██████████████████████████████████████████████████████████████████████████████████████████████████▌ | 16175/23458 [4:37:32<1:44:00, 1.17it/s][2025-04-26 13:19:07] (step=0086550) Train Loss: 5.2826, Train Steps/Sec: 1.17 + 69%|██████████████████████████████████████████████████████████████████████████████████████████████████▊ | 16200/23458 [4:37:53<1:42:47, 1.18it/s][2025-04-26 13:19:29] (step=0086575) Train Loss: 5.2457, Train Steps/Sec: 1.17 + 69%|██████████████████████████████████████████████████████████████████████████████████████████████████▉ | 16225/23458 [4:38:15<1:41:37, 1.19it/s][2025-04-26 13:19:50] (step=0086600) Train Loss: 5.1706, Train Steps/Sec: 1.17 + 69%|███████████████████████████████████████████████████████████████████████████████████████████████████ | 16250/23458 [4:38:36<1:44:41, 1.15it/s][2025-04-26 13:20:11] (step=0086625) Train Loss: 5.2522, Train Steps/Sec: 1.17 + 69%|███████████████████████████████████████████████████████████████████████████████████████████████████▏ | 16275/23458 [4:38:58<1:42:12, 1.17it/s][2025-04-26 13:20:33] (step=0086650) Train Loss: 5.2711, Train Steps/Sec: 1.17 + 69%|███████████████████████████████████████████████████████████████████████████████████████████████████▎ | 16300/23458 [4:39:19<1:41:14, 1.18it/s][2025-04-26 13:20:54] (step=0086675) Train Loss: 5.2326, Train Steps/Sec: 1.17 + 70%|███████████████████████████████████████████████████████████████████████████████████████████████████▌ | 16325/23458 [4:39:40<1:40:21, 1.18it/s][2025-04-26 13:21:16] (step=0086700) Train Loss: 5.2066, Train Steps/Sec: 1.17 + 70%|███████████████████████████████████████████████████████████████████████████████████████████████████▋ | 16350/23458 [4:40:03<1:43:33, 1.14it/s][2025-04-26 13:21:38] (step=0086725) Train Loss: 5.2541, Train Steps/Sec: 1.12 + 70%|███████████████████████████████████████████████████████████████████████████████████████████████████▊ | 16375/23458 [4:40:24<1:40:52, 1.17it/s][2025-04-26 13:21:59] (step=0086750) Train Loss: 5.2955, Train Steps/Sec: 1.17 + 70%|███████████████████████████████████████████████████████████████████████████████████████████████████▉ | 16400/23458 [4:40:46<1:39:41, 1.18it/s][2025-04-26 13:22:21] (step=0086775) Train Loss: 5.2549, Train Steps/Sec: 1.12 + 70%|████████████████████████████████████████████████████████████████████████████████████████████████████▏ | 16425/23458 [4:41:07<1:38:30, 1.19it/s][2025-04-26 13:22:43] (step=0086800) Train Loss: 5.1885, Train Steps/Sec: 1.17 + 70%|████████████████████████████████████████████████████████████████████████████████████████████████████▎ | 16450/23458 [4:41:29<1:41:22, 1.15it/s][2025-04-26 13:23:04] (step=0086825) Train Loss: 5.1905, Train Steps/Sec: 1.17 + 70%|████████████████████████████████████████████████████████████████████████████████████████████████████▍ | 16475/23458 [4:41:50<1:39:17, 1.17it/s][2025-04-26 13:23:25] (step=0086850) Train Loss: 5.2981, Train Steps/Sec: 1.18 + 70%|████████████████████████████████████████████████████████████████████████████████████████████████████▌ | 16500/23458 [4:42:11<1:38:25, 1.18it/s][2025-04-26 13:23:47] (step=0086875) Train Loss: 5.2581, Train Steps/Sec: 1.17 + 70%|████████████████████████████████████████████████████████████████████████████████████████████████████▋ | 16525/23458 [4:42:34<1:37:12, 1.19it/s][2025-04-26 13:24:09] (step=0086900) Train Loss: 5.2495, Train Steps/Sec: 1.12 + 71%|████████████████████████████████████████████████████████████████████████████████████████████████████▉ | 16550/23458 [4:42:55<1:39:55, 1.15it/s][2025-04-26 13:24:30] (step=0086925) Train Loss: 5.2624, Train Steps/Sec: 1.17 + 71%|█████████████████████████████████████████████████████████████████████████████████████████████████████ | 16575/23458 [4:43:18<1:54:00, 1.01it/s][2025-04-26 13:24:53] (step=0086950) Train Loss: 5.2537, Train Steps/Sec: 1.08 + 71%|█████████████████████████████████████████████████████████████████████████████████████████████████████▏ | 16600/23458 [4:43:39<1:36:32, 1.18it/s][2025-04-26 13:25:15] (step=0086975) Train Loss: 5.1669, Train Steps/Sec: 1.18 + 71%|█████████████████████████████████████████████████████████████████████████████████████████████████████▎ | 16625/23458 [4:44:01<1:36:16, 1.18it/s][2025-04-26 13:25:36] (step=0087000) Train Loss: 5.2642, Train Steps/Sec: 1.17 + 71%|█████████████████████████████████████████████████████████████████████████████████████████████████████▍ | 16650/23458 [4:44:23<1:38:42, 1.15it/s][2025-04-26 13:25:58] (step=0087025) Train Loss: 5.2317, Train Steps/Sec: 1.13 + 71%|█████████████████████████████████████████████████████████████████████████████████████████████████████▋ | 16675/23458 [4:44:44<1:36:43, 1.17it/s][2025-04-26 13:26:19] (step=0087050) Train Loss: 5.2547, Train Steps/Sec: 1.17 + 71%|█████████████████████████████████████████████████████████████████████████████████████████████████████▊ | 16700/23458 [4:45:05<1:35:04, 1.18it/s][2025-04-26 13:26:41] (step=0087075) Train Loss: 5.2482, Train Steps/Sec: 1.18 + 71%|█████████████████████████████████████████████████████████████████████████████████████████████████████▉ | 16725/23458 [4:45:28<1:55:26, 1.03s/it][2025-04-26 13:27:03] (step=0087100) Train Loss: 5.2513, Train Steps/Sec: 1.12 + 71%|██████████████████████████████████████████████████████████████████████████████████████████████████████ | 16750/23458 [4:45:50<1:38:33, 1.13it/s][2025-04-26 13:27:25] (step=0087125) Train Loss: 5.2012, Train Steps/Sec: 1.13 + 72%|██████████████████████████████████████████████████████████████████████████████████████████████████████▎ | 16775/23458 [4:46:11<1:35:19, 1.17it/s][2025-04-26 13:27:47] (step=0087150) Train Loss: 5.3794, Train Steps/Sec: 1.17 + 72%|██████████████████████████████████████████████████████████████████████████████████████████████████████▍ | 16800/23458 [4:46:32<1:33:36, 1.19it/s][2025-04-26 13:28:08] (step=0087175) Train Loss: 5.2513, Train Steps/Sec: 1.18 + 72%|██████████████████████████████████████████████████████████████████████████████████████████████████████▌ | 16825/23458 [4:46:54<1:33:06, 1.19it/s][2025-04-26 13:28:29] (step=0087200) Train Loss: 5.2792, Train Steps/Sec: 1.17 + 72%|██████████████████████████████████████████████████████████████████████████████████████████████████████▋ | 16850/23458 [4:47:15<1:35:38, 1.15it/s][2025-04-26 13:28:50] (step=0087225) Train Loss: 5.2440, Train Steps/Sec: 1.17 + 72%|██████████████████████████████████████████████████████████████████████████████████████████████████████▊ | 16875/23458 [4:47:37<1:33:44, 1.17it/s][2025-04-26 13:29:12] (step=0087250) Train Loss: 5.2771, Train Steps/Sec: 1.17 + 72%|███████████████████████████████████████████████████████████████████████████████████████████████████████ | 16900/23458 [4:47:58<1:32:42, 1.18it/s][2025-04-26 13:29:33] (step=0087275) Train Loss: 5.2096, Train Steps/Sec: 1.17 + 72%|███████████████████████████████████████████████████████████████████████████████████████████████████████▏ | 16925/23458 [4:48:23<1:32:02, 1.18it/s][2025-04-26 13:29:58] (step=0087300) Train Loss: 5.1818, Train Steps/Sec: 1.01 + 72%|███████████████████████████████████████████████████████████████████████████████████████████████████████▎ | 16950/23458 [4:48:44<1:34:27, 1.15it/s][2025-04-26 13:30:19] (step=0087325) Train Loss: 5.2443, Train Steps/Sec: 1.17 + 72%|███████████████████████████████████████████████████████████████████████████████████████████████████████▍ | 16975/23458 [4:49:05<1:32:33, 1.17it/s][2025-04-26 13:30:41] (step=0087350) Train Loss: 5.2058, Train Steps/Sec: 1.17 + 72%|███████████████████████████████████████████████████████████████████████████████████████████████████████▋ | 17000/23458 [4:49:27<1:31:22, 1.18it/s][2025-04-26 13:31:02] (step=0087375) Train Loss: 5.2147, Train Steps/Sec: 1.17 + 73%|███████████████████████████████████████████████████████████████████████████████████████████████████████▊ | 17025/23458 [4:49:48<1:30:17, 1.19it/s][2025-04-26 13:31:23] (step=0087400) Train Loss: 5.2235, Train Steps/Sec: 1.17 + 73%|███████████████████████████████████████████████████████████████████████████████████████████████████████▉ | 17050/23458 [4:50:09<1:32:36, 1.15it/s][2025-04-26 13:31:45] (step=0087425) Train Loss: 5.2480, Train Steps/Sec: 1.18 + 73%|████████████████████████████████████████████████████████████████████████████████████████████████████████ | 17075/23458 [4:50:33<1:33:07, 1.14it/s][2025-04-26 13:32:08] (step=0087450) Train Loss: 5.2324, Train Steps/Sec: 1.07 + 73%|████████████████████████████████████████████████████████████████████████████████████████████████████████▏ | 17100/23458 [4:50:54<1:29:23, 1.19it/s][2025-04-26 13:32:29] (step=0087475) Train Loss: 5.2938, Train Steps/Sec: 1.18 + 73%|████████████████████████████████████████████████████████████████████████████████████████████████████████▍ | 17125/23458 [4:51:15<1:28:52, 1.19it/s][2025-04-26 13:32:50] (step=0087500) Train Loss: 5.3411, Train Steps/Sec: 1.17 + 73%|████████████████████████████████████████████████████████████████████████████████████████████████████████▌ | 17150/23458 [4:51:36<1:31:21, 1.15it/s][2025-04-26 13:33:12] (step=0087525) Train Loss: 5.2652, Train Steps/Sec: 1.17 + 73%|████████████████████████████████████████████████████████████████████████████████████████████████████████▋ | 17175/23458 [4:51:58<1:29:08, 1.17it/s][2025-04-26 13:33:33] (step=0087550) Train Loss: 5.2339, Train Steps/Sec: 1.18 + 73%|████████████████████████████████████████████████████████████████████████████████████████████████████████▊ | 17200/23458 [4:52:19<1:28:17, 1.18it/s][2025-04-26 13:33:54] (step=0087575) Train Loss: 5.1793, Train Steps/Sec: 1.17 + 73%|█████████████████████████████████████████████████████████████████████████████████████████████████████████ | 17225/23458 [4:52:41<1:31:45, 1.13it/s][2025-04-26 13:34:17] (step=0087600) Train Loss: 5.2982, Train Steps/Sec: 1.12 + 74%|█████████████████████████████████████████████████████████████████████████████████████████████████████████▏ | 17250/23458 [4:53:04<1:49:08, 1.05s/it][2025-04-26 13:34:39] (step=0087625) Train Loss: 5.1681, Train Steps/Sec: 1.13 + 74%|█████████████████████████████████████████████████████████████████████████████████████████████████████████▎ | 17275/23458 [4:53:25<1:28:05, 1.17it/s][2025-04-26 13:35:00] (step=0087650) Train Loss: 5.1411, Train Steps/Sec: 1.17 + 74%|█████████████████████████████████████████████████████████████████████████████████████████████████████████▍ | 17300/23458 [4:53:46<1:26:47, 1.18it/s][2025-04-26 13:35:21] (step=0087675) Train Loss: 5.1968, Train Steps/Sec: 1.17 + 74%|█████████████████████████████████████████████████████████████████████████████████████████████████████████▌ | 17325/23458 [4:54:09<1:26:50, 1.18it/s][2025-04-26 13:35:44] (step=0087700) Train Loss: 5.2241, Train Steps/Sec: 1.11 + 74%|█████████████████████████████████████████████████████████████████████████████████████████████████████████▊ | 17350/23458 [4:54:30<1:28:22, 1.15it/s][2025-04-26 13:36:05] (step=0087725) Train Loss: 5.1682, Train Steps/Sec: 1.17 + 74%|█████████████████████████████████████████████████████████████████████████████████████████████████████████▉ | 17375/23458 [4:54:52<1:26:43, 1.17it/s][2025-04-26 13:36:27] (step=0087750) Train Loss: 5.2304, Train Steps/Sec: 1.13 + 74%|██████████████████████████████████████████████████████████████████████████████████████████████████████████ | 17400/23458 [4:55:15<1:26:11, 1.17it/s][2025-04-26 13:36:50] (step=0087775) Train Loss: 5.2376, Train Steps/Sec: 1.12 + 74%|██████████████████████████████████████████████████████████████████████████████████████████████████████████▏ | 17425/23458 [4:55:37<1:24:53, 1.18it/s][2025-04-26 13:37:12] (step=0087800) Train Loss: 5.3087, Train Steps/Sec: 1.12 + 74%|██████████████████████████████████████████████████████████████████████████████████████████████████████████▍ | 17450/23458 [4:55:58<1:26:56, 1.15it/s][2025-04-26 13:37:33] (step=0087825) Train Loss: 5.2120, Train Steps/Sec: 1.17 + 74%|██████████████████████████████████████████████████████████████████████████████████████████████████████████▌ | 17475/23458 [4:56:20<1:24:52, 1.17it/s][2025-04-26 13:37:55] (step=0087850) Train Loss: 5.2760, Train Steps/Sec: 1.17 + 75%|██████████████████████████████████████████████████████████████████████████████████████████████████████████▋ | 17500/23458 [4:56:41<1:24:26, 1.18it/s][2025-04-26 13:38:16] (step=0087875) Train Loss: 5.2433, Train Steps/Sec: 1.17 + 75%|██████████████████████████████████████████████████████████████████████████████████████████████████████████▊ | 17525/23458 [4:57:02<1:23:12, 1.19it/s][2025-04-26 13:38:38] (step=0087900) Train Loss: 5.1919, Train Steps/Sec: 1.17 + 75%|██████████████████████████████████████████████████████████████████████████████████████████████████████████▉ | 17550/23458 [4:57:24<1:25:16, 1.15it/s][2025-04-26 13:38:59] (step=0087925) Train Loss: 5.3212, Train Steps/Sec: 1.17 + 75%|███████████████████████████████████████████████████████████████████████████████████████████████████████████▏ | 17575/23458 [4:57:45<1:23:57, 1.17it/s][2025-04-26 13:39:20] (step=0087950) Train Loss: 5.2428, Train Steps/Sec: 1.17 + 75%|███████████████████████████████████████████████████████████████████████████████████████████████████████████▎ | 17600/23458 [4:58:06<1:22:37, 1.18it/s][2025-04-26 13:39:41] (step=0087975) Train Loss: 5.1635, Train Steps/Sec: 1.17 + 75%|███████████████████████████████████████████████████████████████████████████████████████████████████████████▍ | 17625/23458 [4:58:27<1:22:06, 1.18it/s][2025-04-26 13:40:03] (step=0088000) Train Loss: 5.1698, Train Steps/Sec: 1.17 +You are using a model of type blip-2 to instantiate a model of type instructblip. This is not supported for all configurations of models and can yield errors. +[2025-04-26 13:40:03] text_config is None. Initializing the text config with default values (`OPTConfig`). +model_name_or_path:/tmp/haozhezhao/model/blip2-flan-t5-xl +image_place_holder: +tokenizer length after expend 32101 +tokenizer length before expend 32100 +100%|███████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████| 4/4 [03:55<00:00, 58.99s/it] +[2025-04-26 13:45:09] Finish Eval in 88000 steps...████████████████████████████████████████████████████████████████████████████████████████████████████████████████████| 4/4 [03:55<00:00, 58.69s/it] +processing left-padding... +processing left-padding... +processing left-padding... +processing left-padding... +[2025-04-26 13:45:28] Saved checkpoint to checkpoint/Afteri2i_100kckpt_LLAVAT5_empty_Fixed_just_segment_pretrain_stage3_dreambench_recap_400k_t2i_400k_flux_200k_midjourney_150k_recovery_150k_3e4_epoch2_no_rotated/003-GPT-XL/checkpoints/0088000.pt +[2025-04-26 13:45:30] Removed old checkpoint: checkpoint/Afteri2i_100kckpt_LLAVAT5_empty_Fixed_just_segment_pretrain_stage3_dreambench_recap_400k_t2i_400k_flux_200k_midjourney_150k_recovery_150k_3e4_epoch2_no_rotated/003-GPT-XL/checkpoints/0086000.pt + 75%|███████████████████████████████████████████████████████████████████████████████████████████████████████████▌ | 17650/23458 [5:04:16<1:25:26, 1.13it/s][2025-04-26 13:45:52] (step=0088025) Train Loss: 5.1892, Train Steps/Sec: 0.07 + 75%|███████████████████████████████████████████████████████████████████████████████████████████████████████████▋ | 17675/23458 [5:04:38<1:22:09, 1.17it/s][2025-04-26 13:46:13] (step=0088050) Train Loss: 5.2951, Train Steps/Sec: 1.18 + 75%|███████████████████████████████████████████████████████████████████████████████████████████████████████████▉ | 17700/23458 [5:04:59<1:21:26, 1.18it/s][2025-04-26 13:46:34] (step=0088075) Train Loss: 5.2612, Train Steps/Sec: 1.17 + 76%|████████████████████████████████████████████████████████████████████████████████████████████████████████████ | 17725/23458 [5:05:20<1:20:36, 1.19it/s][2025-04-26 13:46:57] (step=0088100) Train Loss: 5.2667, Train Steps/Sec: 1.12 + 76%|████████████████████████████████████████████████████████████████████████████████████████████████████████████▏ | 17750/23458 [5:05:43<1:22:20, 1.16it/s][2025-04-26 13:47:18] (step=0088125) Train Loss: 5.2525, Train Steps/Sec: 1.17 + 76%|████████████████████████████████████████████████████████████████████████████████████████████████████████████▎ | 17775/23458 [5:06:04<1:20:54, 1.17it/s][2025-04-26 13:47:39] (step=0088150) Train Loss: 5.2326, Train Steps/Sec: 1.17 + 76%|████████████████████████████████████████████████████████████████████████████████████████████████████████████▌ | 17800/23458 [5:06:25<1:20:15, 1.17it/s][2025-04-26 13:48:00] (step=0088175) Train Loss: 5.2339, Train Steps/Sec: 1.17 + 76%|████████████████████████████████████████████████████████████████████████████████████████████████████████████▋ | 17825/23458 [5:06:47<1:19:05, 1.19it/s][2025-04-26 13:48:22] (step=0088200) Train Loss: 5.2267, Train Steps/Sec: 1.17 + 76%|████████████████████████████████████████████████████████████████████████████████████████████████████████████▊ | 17850/23458 [5:07:08<1:20:47, 1.16it/s][2025-04-26 13:48:43] (step=0088225) Train Loss: 5.3199, Train Steps/Sec: 1.17 + 76%|████████████████████████████████████████████████████████████████████████████████████████████████████████████▉ | 17875/23458 [5:07:29<1:19:46, 1.17it/s][2025-04-26 13:49:05] (step=0088250) Train Loss: 5.2083, Train Steps/Sec: 1.16 + 76%|█████████████████████████████████████████████████████████████████████████████████████████████████████████████ | 17900/23458 [5:07:52<1:18:37, 1.18it/s][2025-04-26 13:49:27] (step=0088275) Train Loss: 5.2533, Train Steps/Sec: 1.13 + 76%|█████████████████████████████████████████████████████████████████████████████████████████████████████████████▎ | 17925/23458 [5:08:13<1:18:00, 1.18it/s][2025-04-26 13:49:48] (step=0088300) Train Loss: 5.1575, Train Steps/Sec: 1.17 + 77%|█████████████████████████████████████████████████████████████████████████████████████████████████████████████▍ | 17950/23458 [5:08:35<1:19:42, 1.15it/s][2025-04-26 13:50:10] (step=0088325) Train Loss: 5.3107, Train Steps/Sec: 1.13 + 77%|█████████████████████████████████████████████████████████████████████████████████████████████████████████████▌ | 17975/23458 [5:08:57<1:18:00, 1.17it/s][2025-04-26 13:50:32] (step=0088350) Train Loss: 5.2492, Train Steps/Sec: 1.17 + 77%|█████████████████████████████████████████████████████████████████████████████████████████████████████████████▋ | 18000/23458 [5:09:18<1:16:59, 1.18it/s][2025-04-26 13:50:53] (step=0088375) Train Loss: 5.2424, Train Steps/Sec: 1.17 + 77%|█████████████████████████████████████████████████████████████████████████████████████████████████████████████▉ | 18025/23458 [5:09:39<1:16:13, 1.19it/s][2025-04-26 13:51:15] (step=0088400) Train Loss: 5.2286, Train Steps/Sec: 1.17 + 77%|██████████████████████████████████████████████████████████████████████████████████████████████████████████████ | 18050/23458 [5:10:01<1:19:35, 1.13it/s][2025-04-26 13:51:37] (step=0088425) Train Loss: 5.1782, Train Steps/Sec: 1.13 + 77%|██████████████████████████████████████████████████████████████████████████████████████████████████████████████▏ | 18075/23458 [5:10:24<1:16:30, 1.17it/s][2025-04-26 13:51:59] (step=0088450) Train Loss: 5.1716, Train Steps/Sec: 1.13 + 77%|██████████████████████████████████████████████████████████████████████████████████████████████████████████████▎ | 18100/23458 [5:10:46<1:15:34, 1.18it/s][2025-04-26 13:52:21] (step=0088475) Train Loss: 5.1746, Train Steps/Sec: 1.13 + 77%|██████████████████████████████████████████████████████████████████████████████████████████████████████████████▍ | 18125/23458 [5:11:08<1:14:46, 1.19it/s][2025-04-26 13:52:43] (step=0088500) Train Loss: 5.1612, Train Steps/Sec: 1.12 + 77%|██████████████████████████████████████████████████████████████████████████████████████████████████████████████▋ | 18150/23458 [5:11:29<1:16:58, 1.15it/s][2025-04-26 13:53:05] (step=0088525) Train Loss: 5.1721, Train Steps/Sec: 1.17 + 77%|██████████████████████████████████████████████████████████████████████████████████████████████████████████████▊ | 18175/23458 [5:11:51<1:15:23, 1.17it/s][2025-04-26 13:53:26] (step=0088550) Train Loss: 5.2572, Train Steps/Sec: 1.17 + 78%|██████████████████████████████████████████████████████████████████████████████████████████████████████████████▉ | 18200/23458 [5:12:12<1:14:14, 1.18it/s][2025-04-26 13:53:47] (step=0088575) Train Loss: 5.2578, Train Steps/Sec: 1.17 + 78%|███████████████████████████████████████████████████████████████████████████████████████████████████████████████ | 18225/23458 [5:12:33<1:13:25, 1.19it/s][2025-04-26 13:54:09] (step=0088600) Train Loss: 5.2059, Train Steps/Sec: 1.17 + 78%|███████████████████████████████████████████████████████████████████████████████████████████████████████████████▎ | 18250/23458 [5:12:55<1:15:06, 1.16it/s][2025-04-26 13:54:30] (step=0088625) Train Loss: 5.1815, Train Steps/Sec: 1.17 + 78%|███████████████████████████████████████████████████████████████████████████████████████████████████████████████▍ | 18275/23458 [5:13:16<1:13:55, 1.17it/s][2025-04-26 13:54:51] (step=0088650) Train Loss: 5.1503, Train Steps/Sec: 1.17 + 78%|███████████████████████████████████████████████████████████████████████████████████████████████████████████████▌ | 18300/23458 [5:13:37<1:13:01, 1.18it/s][2025-04-26 13:55:13] (step=0088675) Train Loss: 5.2386, Train Steps/Sec: 1.17 + 78%|███████████████████████████████████████████████████████████████████████████████████████████████████████████████▋ | 18325/23458 [5:13:59<1:12:09, 1.19it/s][2025-04-26 13:55:34] (step=0088700) Train Loss: 5.1779, Train Steps/Sec: 1.17 + 78%|███████████████████████████████████████████████████████████████████████████████████████████████████████████████▊ | 18350/23458 [5:14:21<1:16:30, 1.11it/s][2025-04-26 13:55:56] (step=0088725) Train Loss: 5.2298, Train Steps/Sec: 1.12 + 78%|████████████████████████████████████████████████████████████████████████████████████████████████████████████████ | 18375/23458 [5:14:42<1:12:25, 1.17it/s][2025-04-26 13:56:18] (step=0088750) Train Loss: 5.1594, Train Steps/Sec: 1.17 + 78%|████████████████████████████████████████████████████████████████████████████████████████████████████████████████▏ | 18400/23458 [5:15:04<1:18:49, 1.07it/s][2025-04-26 13:56:40] (step=0088775) Train Loss: 5.2273, Train Steps/Sec: 1.13 + 79%|████████████████████████████████████████████████████████████████████████████████████████████████████████████████▎ | 18425/23458 [5:15:26<1:11:20, 1.18it/s][2025-04-26 13:57:01] (step=0088800) Train Loss: 5.1788, Train Steps/Sec: 1.16 + 79%|████████████████████████████████████████████████████████████████████████████████████████████████████████████████▍ | 18450/23458 [5:15:47<1:12:52, 1.15it/s][2025-04-26 13:57:23] (step=0088825) Train Loss: 5.2487, Train Steps/Sec: 1.17 + 79%|████████████████████████████████████████████████████████████████████████████████████████████████████████████████▌ | 18475/23458 [5:16:09<1:11:07, 1.17it/s][2025-04-26 13:57:44] (step=0088850) Train Loss: 5.2479, Train Steps/Sec: 1.17 + 79%|████████████████████████████████████████████████████████████████████████████████████████████████████████████████▊ | 18500/23458 [5:16:30<1:11:48, 1.15it/s][2025-04-26 13:58:06] (step=0088875) Train Loss: 5.2078, Train Steps/Sec: 1.16 + 79%|████████████████████████████████████████████████████████████████████████████████████████████████████████████████▉ | 18525/23458 [5:16:52<1:10:47, 1.16it/s][2025-04-26 13:58:27] (step=0088900) Train Loss: 5.2527, Train Steps/Sec: 1.15 + 79%|█████████████████████████████████████████████████████████████████████████████████████████████████████████████████ | 18550/23458 [5:17:15<1:11:40, 1.14it/s][2025-04-26 13:58:50] (step=0088925) Train Loss: 5.2223, Train Steps/Sec: 1.11 + 79%|█████████████████████████████████████████████████████████████████████████████████████████████████████████████████▏ | 18575/23458 [5:17:36<1:09:40, 1.17it/s][2025-04-26 13:59:11] (step=0088950) Train Loss: 5.2307, Train Steps/Sec: 1.16 + 79%|█████████████████████████████████████████████████████████████████████████████████████████████████████████████████▍ | 18600/23458 [5:17:57<1:08:42, 1.18it/s][2025-04-26 13:59:33] (step=0088975) Train Loss: 5.2801, Train Steps/Sec: 1.17 + 79%|█████████████████████████████████████████████████████████████████████████████████████████████████████████████████▌ | 18625/23458 [5:18:20<1:07:51, 1.19it/s][2025-04-26 13:59:55] (step=0089000) Train Loss: 5.2540, Train Steps/Sec: 1.12 + 80%|█████████████████████████████████████████████████████████████████████████████████████████████████████████████████▋ | 18650/23458 [5:18:41<1:09:57, 1.15it/s][2025-04-26 14:00:17] (step=0089025) Train Loss: 5.1873, Train Steps/Sec: 1.16 + 80%|█████████████████████████████████████████████████████████████████████████████████████████████████████████████████▊ | 18675/23458 [5:19:03<1:08:13, 1.17it/s][2025-04-26 14:00:38] (step=0089050) Train Loss: 5.2626, Train Steps/Sec: 1.17 + 80%|█████████████████████████████████████████████████████████████████████████████████████████████████████████████████▉ | 18700/23458 [5:19:25<1:14:34, 1.06it/s][2025-04-26 14:01:00] (step=0089075) Train Loss: 5.2666, Train Steps/Sec: 1.12 + 80%|██████████████████████████████████████████████████████████████████████████████████████████████████████████████████▏ | 18725/23458 [5:19:47<1:06:52, 1.18it/s][2025-04-26 14:01:23] (step=0089100) Train Loss: 5.1995, Train Steps/Sec: 1.12 + 80%|██████████████████████████████████████████████████████████████████████████████████████████████████████████████████▎ | 18750/23458 [5:20:09<1:08:12, 1.15it/s][2025-04-26 14:01:44] (step=0089125) Train Loss: 5.2048, Train Steps/Sec: 1.17 + 80%|██████████████████████████████████████████████████████████████████████████████████████████████████████████████████▍ | 18775/23458 [5:20:30<1:06:28, 1.17it/s][2025-04-26 14:02:05] (step=0089150) Train Loss: 5.2233, Train Steps/Sec: 1.17 + 80%|██████████████████████████████████████████████████████████████████████████████████████████████████████████████████▌ | 18800/23458 [5:20:51<1:06:31, 1.17it/s][2025-04-26 14:02:27] (step=0089175) Train Loss: 5.2017, Train Steps/Sec: 1.16 + 80%|██████████████████████████████████████████████████████████████████████████████████████████████████████████████████▊ | 18825/23458 [5:21:14<1:05:15, 1.18it/s][2025-04-26 14:02:49] (step=0089200) Train Loss: 5.2886, Train Steps/Sec: 1.12 + 80%|██████████████████████████████████████████████████████████████████████████████████████████████████████████████████▉ | 18850/23458 [5:21:35<1:06:55, 1.15it/s][2025-04-26 14:03:10] (step=0089225) Train Loss: 5.3054, Train Steps/Sec: 1.17 + 80%|███████████████████████████████████████████████████████████████████████████████████████████████████████████████████ | 18875/23458 [5:21:57<1:06:01, 1.16it/s][2025-04-26 14:03:32] (step=0089250) Train Loss: 5.2428, Train Steps/Sec: 1.17 + 81%|███████████████████████████████████████████████████████████████████████████████████████████████████████████████████▏ | 18900/23458 [5:22:18<1:04:51, 1.17it/s][2025-04-26 14:03:53] (step=0089275) Train Loss: 5.1640, Train Steps/Sec: 1.16 + 81%|███████████████████████████████████████████████████████████████████████████████████████████████████████████████████▎ | 18925/23458 [5:22:41<1:04:10, 1.18it/s][2025-04-26 14:04:16] (step=0089300) Train Loss: 5.1989, Train Steps/Sec: 1.10 + 81%|███████████████████████████████████████████████████████████████████████████████████████████████████████████████████▌ | 18950/23458 [5:23:02<1:05:10, 1.15it/s][2025-04-26 14:04:38] (step=0089325) Train Loss: 5.1712, Train Steps/Sec: 1.17 + 81%|███████████████████████████████████████████████████████████████████████████████████████████████████████████████████▋ | 18975/23458 [5:23:24<1:04:15, 1.16it/s][2025-04-26 14:04:59] (step=0089350) Train Loss: 5.1662, Train Steps/Sec: 1.17 + 81%|███████████████████████████████████████████████████████████████████████████████████████████████████████████████████▊ | 19000/23458 [5:23:45<1:03:08, 1.18it/s][2025-04-26 14:05:20] (step=0089375) Train Loss: 5.2355, Train Steps/Sec: 1.17 + 81%|███████████████████████████████████████████████████████████████████████████████████████████████████████████████████▉ | 19025/23458 [5:24:06<1:02:29, 1.18it/s][2025-04-26 14:05:42] (step=0089400) Train Loss: 5.3000, Train Steps/Sec: 1.17 + 81%|████████████████████████████████████████████████████████████████████████████████████████████████████████████████████▏ | 19050/23458 [5:24:28<1:03:58, 1.15it/s][2025-04-26 14:06:03] (step=0089425) Train Loss: 5.2197, Train Steps/Sec: 1.16 + 81%|████████████████████████████████████████████████████████████████████████████████████████████████████████████████████▎ | 19075/23458 [5:24:52<1:30:36, 1.24s/it][2025-04-26 14:06:27] (step=0089450) Train Loss: 5.2155, Train Steps/Sec: 1.06 + 81%|████████████████████████████████████████████████████████████████████████████████████████████████████████████████████▍ | 19100/23458 [5:25:13<1:01:49, 1.17it/s][2025-04-26 14:06:48] (step=0089475) Train Loss: 5.2430, Train Steps/Sec: 1.17 + 82%|████████████████████████████████████████████████████████████████████████████████████████████████████████████████████▌ | 19125/23458 [5:25:34<1:02:00, 1.16it/s][2025-04-26 14:07:10] (step=0089500) Train Loss: 5.3112, Train Steps/Sec: 1.15 + 82%|████████████████████████████████████████████████████████████████████████████████████████████████████████████████████▋ | 19150/23458 [5:25:56<1:02:36, 1.15it/s][2025-04-26 14:07:31] (step=0089525) Train Loss: 5.2604, Train Steps/Sec: 1.16 + 82%|████████████████████████████████████████████████████████████████████████████████████████████████████████████████████▉ | 19175/23458 [5:26:18<1:01:06, 1.17it/s][2025-04-26 14:07:53] (step=0089550) Train Loss: 5.2270, Train Steps/Sec: 1.17 + 82%|█████████████████████████████████████████████████████████████████████████████████████████████████████████████████████ | 19200/23458 [5:26:40<1:00:39, 1.17it/s][2025-04-26 14:08:15] (step=0089575) Train Loss: 5.2362, Train Steps/Sec: 1.13 + 82%|█████████████████████████████████████████████████████████████████████████████████████████████████████████████████████▏ | 19225/23458 [5:27:01<1:00:27, 1.17it/s][2025-04-26 14:08:37] (step=0089600) Train Loss: 5.2615, Train Steps/Sec: 1.15 + 82%|█████████████████████████████████████████████████████████████████████████████████████████████████████████████████████▎ | 19250/23458 [5:27:23<1:01:20, 1.14it/s][2025-04-26 14:08:58] (step=0089625) Train Loss: 5.1774, Train Steps/Sec: 1.16 + 82%|███████████████████████████████████████████████████████████████████████████████████████████████████████████████████████▏ | 19275/23458 [5:27:44<59:58, 1.16it/s][2025-04-26 14:09:20] (step=0089650) Train Loss: 5.2355, Train Steps/Sec: 1.16 + 82%|███████████████████████████████████████████████████████████████████████████████████████████████████████████████████████▎ | 19300/23458 [5:28:07<59:15, 1.17it/s][2025-04-26 14:09:42] (step=0089675) Train Loss: 5.2369, Train Steps/Sec: 1.12 + 82%|███████████████████████████████████████████████████████████████████████████████████████████████████████████████████████▍ | 19325/23458 [5:28:28<58:06, 1.19it/s][2025-04-26 14:10:03] (step=0089700) Train Loss: 5.2447, Train Steps/Sec: 1.17 + 82%|███████████████████████████████████████████████████████████████████████████████████████████████████████████████████████▌ | 19350/23458 [5:28:50<59:26, 1.15it/s][2025-04-26 14:10:25] (step=0089725) Train Loss: 5.1787, Train Steps/Sec: 1.16 + 83%|██████████████████████████████████████████████████████████████████████████████████████████████████████████████████████ | 19375/23458 [5:29:13<1:01:44, 1.10it/s][2025-04-26 14:10:48] (step=0089750) Train Loss: 5.2261, Train Steps/Sec: 1.07 + 83%|███████████████████████████████████████████████████████████████████████████████████████████████████████████████████████▉ | 19400/23458 [5:29:35<57:31, 1.18it/s][2025-04-26 14:11:10] (step=0089775) Train Loss: 5.2511, Train Steps/Sec: 1.17 + 83%|████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████ | 19425/23458 [5:29:56<57:03, 1.18it/s][2025-04-26 14:11:31] (step=0089800) Train Loss: 5.2347, Train Steps/Sec: 1.16 + 83%|████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████▏ | 19450/23458 [5:30:17<57:56, 1.15it/s][2025-04-26 14:11:53] (step=0089825) Train Loss: 5.1934, Train Steps/Sec: 1.17 + 83%|████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████▍ | 19475/23458 [5:30:39<56:44, 1.17it/s][2025-04-26 14:12:14] (step=0089850) Train Loss: 5.2450, Train Steps/Sec: 1.17 + 83%|████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████▌ | 19500/23458 [5:31:00<56:32, 1.17it/s][2025-04-26 14:12:35] (step=0089875) Train Loss: 5.1600, Train Steps/Sec: 1.17 + 83%|████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████▋ | 19525/23458 [5:31:21<55:09, 1.19it/s][2025-04-26 14:12:57] (step=0089900) Train Loss: 5.2461, Train Steps/Sec: 1.17 + 83%|████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████▊ | 19550/23458 [5:31:44<56:45, 1.15it/s][2025-04-26 14:13:19] (step=0089925) Train Loss: 5.2053, Train Steps/Sec: 1.12 + 83%|████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████▉ | 19575/23458 [5:32:05<55:24, 1.17it/s][2025-04-26 14:13:41] (step=0089950) Train Loss: 5.2325, Train Steps/Sec: 1.17 + 84%|█████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████▏ | 19600/23458 [5:32:27<54:43, 1.18it/s][2025-04-26 14:14:02] (step=0089975) Train Loss: 5.1925, Train Steps/Sec: 1.17 + 84%|█████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████▎ | 19625/23458 [5:32:48<53:57, 1.18it/s][2025-04-26 14:14:23] (step=0090000) Train Loss: 5.1632, Train Steps/Sec: 1.16 +You are using a model of type blip-2 to instantiate a model of type instructblip. This is not supported for all configurations of models and can yield errors. +[2025-04-26 14:14:23] text_config is None. Initializing the text config with default values (`OPTConfig`). +model_name_or_path:/tmp/haozhezhao/model/blip2-flan-t5-xl +image_place_holder: +tokenizer length after expend 32101 +tokenizer length before expend 32100 +100%|███████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████| 4/4 [03:53<00:00, 58.40s/it] +[2025-04-26 14:19:25] Finish Eval in 90000 steps...████████████████████████████████████████████████████████████████████████████████████████████████████████████████████| 4/4 [03:53<00:00, 58.14s/it] +processing left-padding... +processing left-padding... +processing left-padding... +processing left-padding... +[2025-04-26 14:19:53] Saved checkpoint to checkpoint/Afteri2i_100kckpt_LLAVAT5_empty_Fixed_just_segment_pretrain_stage3_dreambench_recap_400k_t2i_400k_flux_200k_midjourney_150k_recovery_150k_3e4_epoch2_no_rotated/003-GPT-XL/checkpoints/0090000.pt +[2025-04-26 14:19:56] Removed old checkpoint: checkpoint/Afteri2i_100kckpt_LLAVAT5_empty_Fixed_just_segment_pretrain_stage3_dreambench_recap_400k_t2i_400k_flux_200k_midjourney_150k_recovery_150k_3e4_epoch2_no_rotated/003-GPT-XL/checkpoints/0088000.pt + 84%|█████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████▍ | 19650/23458 [5:38:42<56:41, 1.12it/s][2025-04-26 14:20:17] (step=0090025) Train Loss: 5.2262, Train Steps/Sec: 0.07 + 84%|█████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████▌ | 19675/23458 [5:39:04<54:41, 1.15it/s][2025-04-26 14:20:39] (step=0090050) Train Loss: 5.2542, Train Steps/Sec: 1.15 + 84%|█████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████▊ | 19700/23458 [5:39:25<53:04, 1.18it/s][2025-04-26 14:21:00] (step=0090075) Train Loss: 5.1689, Train Steps/Sec: 1.17 + 84%|█████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████▉ | 19725/23458 [5:39:47<52:48, 1.18it/s][2025-04-26 14:21:23] (step=0090100) Train Loss: 5.2298, Train Steps/Sec: 1.12 + 84%|██████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████ | 19750/23458 [5:40:10<54:12, 1.14it/s][2025-04-26 14:21:45] (step=0090125) Train Loss: 5.2348, Train Steps/Sec: 1.11 + 84%|██████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████▏ | 19775/23458 [5:40:31<52:35, 1.17it/s][2025-04-26 14:22:06] (step=0090150) Train Loss: 5.3167, Train Steps/Sec: 1.16 + 84%|██████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████▍ | 19800/23458 [5:40:53<51:44, 1.18it/s][2025-04-26 14:22:28] (step=0090175) Train Loss: 5.2560, Train Steps/Sec: 1.16 + 85%|██████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████▌ | 19825/23458 [5:41:14<51:08, 1.18it/s][2025-04-26 14:22:49] (step=0090200) Train Loss: 5.2767, Train Steps/Sec: 1.17 + 85%|██████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████▋ | 19850/23458 [5:41:36<54:01, 1.11it/s][2025-04-26 14:23:12] (step=0090225) Train Loss: 5.2540, Train Steps/Sec: 1.12 + 85%|██████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████▊ | 19875/23458 [5:41:58<51:34, 1.16it/s][2025-04-26 14:23:33] (step=0090250) Train Loss: 5.2135, Train Steps/Sec: 1.17 + 85%|███████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████ | 19900/23458 [5:42:19<50:24, 1.18it/s][2025-04-26 14:23:55] (step=0090275) Train Loss: 5.1393, Train Steps/Sec: 1.16 + 85%|███████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████▏ | 19925/23458 [5:42:41<50:06, 1.18it/s][2025-04-26 14:24:16] (step=0090300) Train Loss: 5.2205, Train Steps/Sec: 1.16 + 85%|███████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████▎ | 19950/23458 [5:43:02<50:42, 1.15it/s][2025-04-26 14:24:38] (step=0090325) Train Loss: 5.2361, Train Steps/Sec: 1.17 + 85%|███████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████▍ | 19975/23458 [5:43:24<50:00, 1.16it/s][2025-04-26 14:25:00] (step=0090350) Train Loss: 5.2362, Train Steps/Sec: 1.12 + 85%|███████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████▋ | 20000/23458 [5:43:46<49:13, 1.17it/s][2025-04-26 14:25:21] (step=0090375) Train Loss: 5.2021, Train Steps/Sec: 1.16 + 85%|███████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████▊ | 20025/23458 [5:44:08<49:04, 1.17it/s][2025-04-26 14:25:44] (step=0090400) Train Loss: 5.2192, Train Steps/Sec: 1.11 + 85%|███████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████▉ | 20050/23458 [5:44:31<49:22, 1.15it/s][2025-04-26 14:26:06] (step=0090425) Train Loss: 5.2019, Train Steps/Sec: 1.12 + 86%|████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████ | 20075/23458 [5:44:52<48:23, 1.17it/s][2025-04-26 14:26:27] (step=0090450) Train Loss: 5.2561, Train Steps/Sec: 1.17 + 86%|████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████▏ | 20100/23458 [5:45:14<47:43, 1.17it/s][2025-04-26 14:26:49] (step=0090475) Train Loss: 5.2621, Train Steps/Sec: 1.16 + 86%|████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████▍ | 20125/23458 [5:45:35<46:51, 1.19it/s][2025-04-26 14:27:11] (step=0090500) Train Loss: 5.1460, Train Steps/Sec: 1.16 + 86%|████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████▌ | 20150/23458 [5:45:57<48:09, 1.14it/s][2025-04-26 14:27:32] (step=0090525) Train Loss: 5.1856, Train Steps/Sec: 1.17 + 86%|████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████▋ | 20175/23458 [5:46:18<46:49, 1.17it/s][2025-04-26 14:27:53] (step=0090550) Train Loss: 5.2238, Train Steps/Sec: 1.17 + 86%|████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████▊ | 20200/23458 [5:46:39<46:00, 1.18it/s][2025-04-26 14:28:15] (step=0090575) Train Loss: 5.2297, Train Steps/Sec: 1.17 + 86%|█████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████ | 20225/23458 [5:47:01<45:18, 1.19it/s][2025-04-26 14:28:36] (step=0090600) Train Loss: 5.2220, Train Steps/Sec: 1.17 + 86%|█████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████▏ | 20250/23458 [5:47:22<46:19, 1.15it/s][2025-04-26 14:28:57] (step=0090625) Train Loss: 5.2372, Train Steps/Sec: 1.18 + 86%|█████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████▎ | 20275/23458 [5:47:44<46:11, 1.15it/s][2025-04-26 14:29:19] (step=0090650) Train Loss: 5.2688, Train Steps/Sec: 1.12 + 87%|█████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████▍ | 20300/23458 [5:48:05<44:30, 1.18it/s][2025-04-26 14:29:41] (step=0090675) Train Loss: 5.1891, Train Steps/Sec: 1.18 + 87%|█████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████▋ | 20325/23458 [5:48:27<44:02, 1.19it/s][2025-04-26 14:30:02] (step=0090700) Train Loss: 5.3241, Train Steps/Sec: 1.17 + 87%|█████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████▊ | 20350/23458 [5:48:49<46:50, 1.11it/s][2025-04-26 14:30:24] (step=0090725) Train Loss: 5.2136, Train Steps/Sec: 1.12 + 87%|█████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████▉ | 20375/23458 [5:49:10<43:49, 1.17it/s][2025-04-26 14:30:46] (step=0090750) Train Loss: 5.2470, Train Steps/Sec: 1.18 + 87%|██████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████ | 20400/23458 [5:49:32<43:12, 1.18it/s][2025-04-26 14:31:07] (step=0090775) Train Loss: 5.1749, Train Steps/Sec: 1.17 + 87%|██████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████▎ | 20425/23458 [5:49:54<42:41, 1.18it/s][2025-04-26 14:31:29] (step=0090800) Train Loss: 5.2014, Train Steps/Sec: 1.12 + 87%|██████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████▍ | 20450/23458 [5:50:15<43:37, 1.15it/s][2025-04-26 14:31:51] (step=0090825) Train Loss: 5.2019, Train Steps/Sec: 1.17 + 87%|██████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████▌ | 20475/23458 [5:50:37<42:50, 1.16it/s][2025-04-26 14:32:12] (step=0090850) Train Loss: 5.3517, Train Steps/Sec: 1.17 + 87%|██████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████▋ | 20500/23458 [5:50:59<46:03, 1.07it/s][2025-04-26 14:32:34] (step=0090875) Train Loss: 5.1937, Train Steps/Sec: 1.13 + 87%|██████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████▊ | 20525/23458 [5:51:21<41:15, 1.18it/s][2025-04-26 14:32:57] (step=0090900) Train Loss: 5.1731, Train Steps/Sec: 1.11 + 88%|███████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████ | 20550/23458 [5:51:43<42:09, 1.15it/s][2025-04-26 14:33:18] (step=0090925) Train Loss: 5.2446, Train Steps/Sec: 1.17 + 88%|███████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████▏ | 20575/23458 [5:52:04<41:05, 1.17it/s][2025-04-26 14:33:39] (step=0090950) Train Loss: 5.1775, Train Steps/Sec: 1.17 + 88%|███████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████▎ | 20600/23458 [5:52:25<40:21, 1.18it/s][2025-04-26 14:34:00] (step=0090975) Train Loss: 5.2638, Train Steps/Sec: 1.18 + 88%|███████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████▍ | 20625/23458 [5:52:46<39:51, 1.18it/s][2025-04-26 14:34:22] (step=0091000) Train Loss: 5.3206, Train Steps/Sec: 1.17 + 88%|███████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████▋ | 20650/23458 [5:53:09<40:55, 1.14it/s][2025-04-26 14:34:44] (step=0091025) Train Loss: 5.2054, Train Steps/Sec: 1.13 + 88%|███████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████▊ | 20675/23458 [5:53:31<52:31, 1.13s/it][2025-04-26 14:35:06] (step=0091050) Train Loss: 5.2451, Train Steps/Sec: 1.13 + 88%|███████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████▉ | 20700/23458 [5:53:53<39:04, 1.18it/s][2025-04-26 14:35:28] (step=0091075) Train Loss: 5.1870, Train Steps/Sec: 1.13 + 88%|████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████ | 20725/23458 [5:54:14<38:32, 1.18it/s][2025-04-26 14:35:50] (step=0091100) Train Loss: 5.2540, Train Steps/Sec: 1.17 + 88%|████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████▎ | 20750/23458 [5:54:36<39:06, 1.15it/s][2025-04-26 14:36:11] (step=0091125) Train Loss: 5.2366, Train Steps/Sec: 1.17 + 89%|████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████▍ | 20775/23458 [5:54:57<38:09, 1.17it/s][2025-04-26 14:36:32] (step=0091150) Train Loss: 5.1328, Train Steps/Sec: 1.17 + 89%|████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████▌ | 20800/23458 [5:55:18<37:29, 1.18it/s][2025-04-26 14:36:54] (step=0091175) Train Loss: 5.1533, Train Steps/Sec: 1.17 + 89%|████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████▋ | 20825/23458 [5:55:40<36:54, 1.19it/s][2025-04-26 14:37:15] (step=0091200) Train Loss: 5.2468, Train Steps/Sec: 1.17 + 89%|████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████▉ | 20850/23458 [5:56:01<37:41, 1.15it/s][2025-04-26 14:37:36] (step=0091225) Train Loss: 5.1544, Train Steps/Sec: 1.17 + 89%|█████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████ | 20875/23458 [5:56:22<36:44, 1.17it/s][2025-04-26 14:37:58] (step=0091250) Train Loss: 5.2958, Train Steps/Sec: 1.17 + 89%|█████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████▏ | 20900/23458 [5:56:44<36:10, 1.18it/s][2025-04-26 14:38:19] (step=0091275) Train Loss: 5.2441, Train Steps/Sec: 1.17 + 89%|█████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████▎ | 20925/23458 [5:57:05<35:31, 1.19it/s][2025-04-26 14:38:40] (step=0091300) Train Loss: 5.2316, Train Steps/Sec: 1.17 + 89%|█████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████▍ | 20950/23458 [5:57:26<36:15, 1.15it/s][2025-04-26 14:39:02] (step=0091325) Train Loss: 5.2198, Train Steps/Sec: 1.17 + 89%|█████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████▋ | 20975/23458 [5:57:48<35:17, 1.17it/s][2025-04-26 14:39:23] (step=0091350) Train Loss: 5.2055, Train Steps/Sec: 1.18 + 90%|█████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████▊ | 21000/23458 [5:58:10<35:47, 1.14it/s][2025-04-26 14:39:45] (step=0091375) Train Loss: 5.2436, Train Steps/Sec: 1.13 + 90%|█████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████▉ | 21025/23458 [5:58:31<34:12, 1.19it/s][2025-04-26 14:40:07] (step=0091400) Train Loss: 5.1907, Train Steps/Sec: 1.17 + 90%|██████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████ | 21050/23458 [5:58:53<34:42, 1.16it/s][2025-04-26 14:40:28] (step=0091425) Train Loss: 5.1758, Train Steps/Sec: 1.18 + 90%|██████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████▎ | 21075/23458 [5:59:15<43:23, 1.09s/it][2025-04-26 14:40:50] (step=0091450) Train Loss: 5.2669, Train Steps/Sec: 1.12 + 90%|██████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████▍ | 21100/23458 [5:59:37<33:20, 1.18it/s][2025-04-26 14:41:12] (step=0091475) Train Loss: 5.1017, Train Steps/Sec: 1.13 + 90%|██████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████▌ | 21125/23458 [5:59:58<32:40, 1.19it/s][2025-04-26 14:41:34] (step=0091500) Train Loss: 5.1643, Train Steps/Sec: 1.17 + 90%|██████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████▋ | 21150/23458 [6:00:21<42:43, 1.11s/it][2025-04-26 14:41:56] (step=0091525) Train Loss: 5.2491, Train Steps/Sec: 1.13 + 90%|██████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████▉ | 21175/23458 [6:00:42<32:26, 1.17it/s][2025-04-26 14:42:17] (step=0091550) Train Loss: 5.2693, Train Steps/Sec: 1.18 + 90%|███████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████ | 21200/23458 [6:01:03<32:02, 1.17it/s][2025-04-26 14:42:38] (step=0091575) Train Loss: 5.2570, Train Steps/Sec: 1.17 + 90%|███████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████▏ | 21225/23458 [6:01:25<31:17, 1.19it/s][2025-04-26 14:43:00] (step=0091600) Train Loss: 5.2349, Train Steps/Sec: 1.17 + 91%|███████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████▎ | 21250/23458 [6:01:46<31:54, 1.15it/s][2025-04-26 14:43:21] (step=0091625) Train Loss: 5.2476, Train Steps/Sec: 1.17 + 91%|███████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████▌ | 21275/23458 [6:02:07<31:04, 1.17it/s][2025-04-26 14:43:42] (step=0091650) Train Loss: 5.2776, Train Steps/Sec: 1.17 + 91%|███████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████▋ | 21300/23458 [6:02:28<30:35, 1.18it/s][2025-04-26 14:44:04] (step=0091675) Train Loss: 5.1855, Train Steps/Sec: 1.17 + 91%|███████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████▊ | 21325/23458 [6:02:51<29:56, 1.19it/s][2025-04-26 14:44:26] (step=0091700) Train Loss: 5.1685, Train Steps/Sec: 1.12 + 91%|███████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████▉ | 21350/23458 [6:03:14<32:05, 1.09it/s][2025-04-26 14:44:49] (step=0091725) Train Loss: 5.2990, Train Steps/Sec: 1.08 + 91%|████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████ | 21375/23458 [6:03:35<29:37, 1.17it/s][2025-04-26 14:45:10] (step=0091750) Train Loss: 5.3128, Train Steps/Sec: 1.18 + 91%|████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████▎ | 21400/23458 [6:03:57<29:08, 1.18it/s][2025-04-26 14:45:32] (step=0091775) Train Loss: 5.1800, Train Steps/Sec: 1.17 + 91%|████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████▍ | 21425/23458 [6:04:18<28:29, 1.19it/s][2025-04-26 14:45:53] (step=0091800) Train Loss: 5.1306, Train Steps/Sec: 1.17 + 91%|████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████▌ | 21450/23458 [6:04:39<29:00, 1.15it/s][2025-04-26 14:46:14] (step=0091825) Train Loss: 5.3233, Train Steps/Sec: 1.17 + 92%|████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████▋ | 21475/23458 [6:05:01<28:15, 1.17it/s][2025-04-26 14:46:36] (step=0091850) Train Loss: 5.2241, Train Steps/Sec: 1.17 + 92%|████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████▉ | 21500/23458 [6:05:22<27:35, 1.18it/s][2025-04-26 14:46:57] (step=0091875) Train Loss: 5.2547, Train Steps/Sec: 1.17 + 92%|█████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████ | 21525/23458 [6:05:43<27:09, 1.19it/s][2025-04-26 14:47:19] (step=0091900) Train Loss: 5.2830, Train Steps/Sec: 1.17 + 92%|█████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████▏ | 21550/23458 [6:06:05<27:47, 1.14it/s][2025-04-26 14:47:40] (step=0091925) Train Loss: 5.1818, Train Steps/Sec: 1.17 + 92%|█████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████▎ | 21575/23458 [6:06:26<26:42, 1.18it/s][2025-04-26 14:48:01] (step=0091950) Train Loss: 5.2696, Train Steps/Sec: 1.17 + 92%|█████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████▌ | 21600/23458 [6:06:47<26:12, 1.18it/s][2025-04-26 14:48:22] (step=0091975) Train Loss: 5.1632, Train Steps/Sec: 1.18 + 92%|█████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████▋ | 21625/23458 [6:07:08<25:45, 1.19it/s][2025-04-26 14:48:44] (step=0092000) Train Loss: 5.2357, Train Steps/Sec: 1.17 +You are using a model of type blip-2 to instantiate a model of type instructblip. This is not supported for all configurations of models and can yield errors. +[2025-04-26 14:48:44] text_config is None. Initializing the text config with default values (`OPTConfig`). +model_name_or_path:/tmp/haozhezhao/model/blip2-flan-t5-xl +image_place_holder: +tokenizer length after expend 32101 +tokenizer length before expend 32100 +100%|███████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████| 4/4 [03:54<00:00, 58.57s/it] +[2025-04-26 14:53:47] Finish Eval in 92000 steps...████████████████████████████████████████████████████████████████████████████████████████████████████████████████████| 4/4 [03:53<00:00, 58.09s/it] +processing left-padding... +processing left-padding... +processing left-padding... +processing left-padding... +[2025-04-26 14:54:06] Saved checkpoint to checkpoint/Afteri2i_100kckpt_LLAVAT5_empty_Fixed_just_segment_pretrain_stage3_dreambench_recap_400k_t2i_400k_flux_200k_midjourney_150k_recovery_150k_3e4_epoch2_no_rotated/003-GPT-XL/checkpoints/0092000.pt +[2025-04-26 14:54:08] Removed old checkpoint: checkpoint/Afteri2i_100kckpt_LLAVAT5_empty_Fixed_just_segment_pretrain_stage3_dreambench_recap_400k_t2i_400k_flux_200k_midjourney_150k_recovery_150k_3e4_epoch2_no_rotated/003-GPT-XL/checkpoints/0090000.pt + 92%|█████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████▊ | 21650/23458 [6:12:54<26:36, 1.13it/s][2025-04-26 14:54:30] (step=0092025) Train Loss: 5.2550, Train Steps/Sec: 0.07 + 92%|█████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████▉ | 21675/23458 [6:13:15<25:16, 1.18it/s][2025-04-26 14:54:51] (step=0092050) Train Loss: 5.2718, Train Steps/Sec: 1.18 + 93%|██████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████▏ | 21700/23458 [6:13:37<24:49, 1.18it/s][2025-04-26 14:55:12] (step=0092075) Train Loss: 5.1775, Train Steps/Sec: 1.17 + 93%|██████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████▎ | 21725/23458 [6:13:59<24:25, 1.18it/s][2025-04-26 14:55:34] (step=0092100) Train Loss: 5.2944, Train Steps/Sec: 1.11 + 93%|██████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████▍ | 21750/23458 [6:14:20<24:37, 1.16it/s][2025-04-26 14:55:56] (step=0092125) Train Loss: 5.2479, Train Steps/Sec: 1.18 + 93%|██████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████▌ | 21775/23458 [6:14:43<23:53, 1.17it/s][2025-04-26 14:56:18] (step=0092150) Train Loss: 5.2769, Train Steps/Sec: 1.13 + 93%|██████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████▊ | 21800/23458 [6:15:04<23:22, 1.18it/s][2025-04-26 14:56:39] (step=0092175) Train Loss: 5.2369, Train Steps/Sec: 1.18 + 93%|██████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████▉ | 21825/23458 [6:15:26<22:53, 1.19it/s][2025-04-26 14:57:01] (step=0092200) Train Loss: 5.2095, Train Steps/Sec: 1.13 + 93%|███████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████ | 21850/23458 [6:15:47<23:19, 1.15it/s][2025-04-26 14:57:23] (step=0092225) Train Loss: 5.2055, Train Steps/Sec: 1.17 + 93%|███████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████▏ | 21875/23458 [6:16:09<22:31, 1.17it/s][2025-04-26 14:57:44] (step=0092250) Train Loss: 5.2420, Train Steps/Sec: 1.17 + 93%|███████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████▎ | 21900/23458 [6:16:30<21:58, 1.18it/s][2025-04-26 14:58:05] (step=0092275) Train Loss: 5.2455, Train Steps/Sec: 1.17 + 93%|███████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████▌ | 21925/23458 [6:16:51<21:31, 1.19it/s][2025-04-26 14:58:27] (step=0092300) Train Loss: 5.2560, Train Steps/Sec: 1.17 + 94%|███████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████▋ | 21950/23458 [6:17:13<21:47, 1.15it/s][2025-04-26 14:58:48] (step=0092325) Train Loss: 5.2372, Train Steps/Sec: 1.18 + 94%|███████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████▊ | 21975/23458 [6:17:34<21:03, 1.17it/s][2025-04-26 14:59:09] (step=0092350) Train Loss: 5.2496, Train Steps/Sec: 1.17 + 94%|███████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████▉ | 22000/23458 [6:17:57<21:10, 1.15it/s][2025-04-26 14:59:32] (step=0092375) Train Loss: 5.1715, Train Steps/Sec: 1.08 + 94%|████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████▏ | 22025/23458 [6:18:19<20:11, 1.18it/s][2025-04-26 14:59:55] (step=0092400) Train Loss: 5.2935, Train Steps/Sec: 1.12 + 94%|████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████▎ | 22050/23458 [6:18:41<20:21, 1.15it/s][2025-04-26 15:00:16] (step=0092425) Train Loss: 5.2222, Train Steps/Sec: 1.17 + 94%|████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████▍ | 22075/23458 [6:19:02<19:45, 1.17it/s][2025-04-26 15:00:37] (step=0092450) Train Loss: 5.1902, Train Steps/Sec: 1.17 + 94%|████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████▌ | 22100/23458 [6:19:23<19:07, 1.18it/s][2025-04-26 15:00:58] (step=0092475) Train Loss: 5.1754, Train Steps/Sec: 1.17 + 94%|████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████▊ | 22125/23458 [6:19:45<18:47, 1.18it/s][2025-04-26 15:01:21] (step=0092500) Train Loss: 5.2052, Train Steps/Sec: 1.12 + 94%|████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████▉ | 22150/23458 [6:20:07<18:54, 1.15it/s][2025-04-26 15:01:42] (step=0092525) Train Loss: 5.1529, Train Steps/Sec: 1.17 + 95%|█████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████ | 22175/23458 [6:20:28<18:15, 1.17it/s][2025-04-26 15:02:03] (step=0092550) Train Loss: 5.3493, Train Steps/Sec: 1.17 + 95%|█████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████▏ | 22200/23458 [6:20:49<17:46, 1.18it/s][2025-04-26 15:02:25] (step=0092575) Train Loss: 5.1634, Train Steps/Sec: 1.17 + 95%|█████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████▍ | 22225/23458 [6:21:11<17:19, 1.19it/s][2025-04-26 15:02:46] (step=0092600) Train Loss: 5.2780, Train Steps/Sec: 1.17 + 95%|█████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████▌ | 22250/23458 [6:21:32<17:27, 1.15it/s][2025-04-26 15:03:07] (step=0092625) Train Loss: 5.1629, Train Steps/Sec: 1.17 + 95%|█████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████▋ | 22275/23458 [6:21:53<16:49, 1.17it/s][2025-04-26 15:03:29] (step=0092650) Train Loss: 5.1931, Train Steps/Sec: 1.17 + 95%|█████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████▊ | 22300/23458 [6:22:15<16:20, 1.18it/s][2025-04-26 15:03:50] (step=0092675) Train Loss: 5.1880, Train Steps/Sec: 1.17 + 95%|█████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████▉ | 22325/23458 [6:22:36<15:54, 1.19it/s][2025-04-26 15:04:11] (step=0092700) Train Loss: 5.2338, Train Steps/Sec: 1.17 + 95%|██████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████▏ | 22350/23458 [6:22:58<16:22, 1.13it/s][2025-04-26 15:04:34] (step=0092725) Train Loss: 5.2412, Train Steps/Sec: 1.13 + 95%|██████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████▎ | 22375/23458 [6:23:20<15:26, 1.17it/s][2025-04-26 15:04:55] (step=0092750) Train Loss: 5.2120, Train Steps/Sec: 1.17 + 95%|██████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████▍ | 22400/23458 [6:23:41<14:54, 1.18it/s][2025-04-26 15:05:16] (step=0092775) Train Loss: 5.3538, Train Steps/Sec: 1.17 + 96%|██████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████▌ | 22425/23458 [6:24:02<14:30, 1.19it/s][2025-04-26 15:05:38] (step=0092800) Train Loss: 5.2353, Train Steps/Sec: 1.17 + 96%|██████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████▊ | 22450/23458 [6:24:25<14:52, 1.13it/s][2025-04-26 15:06:01] (step=0092825) Train Loss: 5.1691, Train Steps/Sec: 1.08 + 96%|██████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████▉ | 22475/23458 [6:24:47<14:00, 1.17it/s][2025-04-26 15:06:23] (step=0092850) Train Loss: 5.2859, Train Steps/Sec: 1.13 + 96%|███████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████ | 22500/23458 [6:25:09<13:31, 1.18it/s][2025-04-26 15:06:44] (step=0092875) Train Loss: 5.2087, Train Steps/Sec: 1.17 + 96%|███████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████▏ | 22525/23458 [6:25:30<13:08, 1.18it/s][2025-04-26 15:07:06] (step=0092900) Train Loss: 5.1838, Train Steps/Sec: 1.17 + 96%|███████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████▍ | 22550/23458 [6:25:52<13:07, 1.15it/s][2025-04-26 15:07:27] (step=0092925) Train Loss: 5.2958, Train Steps/Sec: 1.17 + 96%|███████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████▌ | 22575/23458 [6:26:13<12:33, 1.17it/s][2025-04-26 15:07:48] (step=0092950) Train Loss: 5.2527, Train Steps/Sec: 1.17 + 96%|███████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████▋ | 22600/23458 [6:26:34<12:09, 1.18it/s][2025-04-26 15:08:09] (step=0092975) Train Loss: 5.2661, Train Steps/Sec: 1.17 + 96%|███████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████▊ | 22625/23458 [6:26:55<11:41, 1.19it/s][2025-04-26 15:08:31] (step=0093000) Train Loss: 5.1949, Train Steps/Sec: 1.17 + 97%|████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████ | 22650/23458 [6:27:17<11:40, 1.15it/s][2025-04-26 15:08:52] (step=0093025) Train Loss: 5.2165, Train Steps/Sec: 1.17 + 97%|████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████▏ | 22675/23458 [6:27:41<11:09, 1.17it/s][2025-04-26 15:09:16] (step=0093050) Train Loss: 5.1622, Train Steps/Sec: 1.05 + 97%|████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████▎ | 22700/23458 [6:28:02<10:42, 1.18it/s][2025-04-26 15:09:37] (step=0093075) Train Loss: 5.2077, Train Steps/Sec: 1.17 + 97%|████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████▍ | 22725/23458 [6:28:23<10:17, 1.19it/s][2025-04-26 15:09:59] (step=0093100) Train Loss: 5.2386, Train Steps/Sec: 1.17 + 97%|████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████▌ | 22750/23458 [6:28:45<10:12, 1.16it/s][2025-04-26 15:10:20] (step=0093125) Train Loss: 5.2036, Train Steps/Sec: 1.17 + 97%|████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████▊ | 22775/23458 [6:29:06<09:46, 1.17it/s][2025-04-26 15:10:41] (step=0093150) Train Loss: 5.2691, Train Steps/Sec: 1.17 + 97%|████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████▉ | 22800/23458 [6:29:27<09:17, 1.18it/s][2025-04-26 15:11:03] (step=0093175) Train Loss: 5.1493, Train Steps/Sec: 1.17 + 97%|█████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████ | 22825/23458 [6:29:49<08:53, 1.19it/s][2025-04-26 15:11:24] (step=0093200) Train Loss: 5.2619, Train Steps/Sec: 1.17 + 97%|█████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████▏ | 22850/23458 [6:30:10<08:48, 1.15it/s][2025-04-26 15:11:45] (step=0093225) Train Loss: 5.2152, Train Steps/Sec: 1.17 + 98%|█████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████▍ | 22875/23458 [6:30:31<08:19, 1.17it/s][2025-04-26 15:12:07] (step=0093250) Train Loss: 5.2636, Train Steps/Sec: 1.17 + 98%|█████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████▌ | 22900/23458 [6:30:53<07:54, 1.18it/s][2025-04-26 15:12:28] (step=0093275) Train Loss: 5.2517, Train Steps/Sec: 1.17 + 98%|█████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████▋ | 22925/23458 [6:31:15<07:29, 1.18it/s][2025-04-26 15:12:50] (step=0093300) Train Loss: 5.2671, Train Steps/Sec: 1.12 + 98%|█████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████▊ | 22950/23458 [6:31:36<07:20, 1.15it/s][2025-04-26 15:13:12] (step=0093325) Train Loss: 5.2703, Train Steps/Sec: 1.17 + 98%|██████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████ | 22975/23458 [6:31:58<06:53, 1.17it/s][2025-04-26 15:13:33] (step=0093350) Train Loss: 5.1847, Train Steps/Sec: 1.17 + 98%|██████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████▏ | 23000/23458 [6:32:19<06:29, 1.18it/s][2025-04-26 15:13:54] (step=0093375) Train Loss: 5.2008, Train Steps/Sec: 1.17 + 98%|██████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████▎ | 23025/23458 [6:32:40<06:03, 1.19it/s][2025-04-26 15:14:16] (step=0093400) Train Loss: 5.2751, Train Steps/Sec: 1.17 + 98%|██████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████▍ | 23050/23458 [6:33:02<05:53, 1.15it/s][2025-04-26 15:14:37] (step=0093425) Train Loss: 5.2426, Train Steps/Sec: 1.17 + 98%|██████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████▋ | 23075/23458 [6:33:24<06:42, 1.05s/it][2025-04-26 15:14:59] (step=0093450) Train Loss: 5.2634, Train Steps/Sec: 1.13 + 98%|██████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████▊ | 23100/23458 [6:33:46<05:33, 1.07it/s][2025-04-26 15:15:21] (step=0093475) Train Loss: 5.2814, Train Steps/Sec: 1.13 + 99%|██████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████▉ | 23125/23458 [6:34:08<04:44, 1.17it/s][2025-04-26 15:15:43] (step=0093500) Train Loss: 5.2686, Train Steps/Sec: 1.13 + 99%|███████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████ | 23150/23458 [6:34:29<04:26, 1.16it/s][2025-04-26 15:16:05] (step=0093525) Train Loss: 5.2574, Train Steps/Sec: 1.18 + 99%|███████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████▎ | 23175/23458 [6:34:52<04:08, 1.14it/s][2025-04-26 15:16:27] (step=0093550) Train Loss: 5.1546, Train Steps/Sec: 1.13 + 99%|███████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████▍ | 23200/23458 [6:35:13<03:39, 1.17it/s][2025-04-26 15:16:48] (step=0093575) Train Loss: 5.2106, Train Steps/Sec: 1.17 + 99%|███████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████▌ | 23225/23458 [6:35:34<03:16, 1.19it/s][2025-04-26 15:17:10] (step=0093600) Train Loss: 5.2060, Train Steps/Sec: 1.17 + 99%|███████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████▋ | 23250/23458 [6:35:56<02:59, 1.16it/s][2025-04-26 15:17:31] (step=0093625) Train Loss: 5.2009, Train Steps/Sec: 1.18 + 99%|███████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████▊ | 23275/23458 [6:36:17<02:35, 1.17it/s][2025-04-26 15:17:52] (step=0093650) Train Loss: 5.2195, Train Steps/Sec: 1.17 + 99%|████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████ | 23300/23458 [6:36:38<02:13, 1.18it/s][2025-04-26 15:18:14] (step=0093675) Train Loss: 5.2121, Train Steps/Sec: 1.17 + 99%|████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████▏| 23325/23458 [6:37:00<01:52, 1.18it/s][2025-04-26 15:18:36] (step=0093700) Train Loss: 5.1729, Train Steps/Sec: 1.12 +100%|████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████▎| 23350/23458 [6:37:23<01:35, 1.14it/s][2025-04-26 15:18:58] (step=0093725) Train Loss: 5.2559, Train Steps/Sec: 1.13 +100%|████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████▍| 23375/23458 [6:37:44<01:10, 1.17it/s][2025-04-26 15:19:19] (step=0093750) Train Loss: 5.2123, Train Steps/Sec: 1.18 +100%|████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████▋| 23400/23458 [6:38:05<00:49, 1.18it/s][2025-04-26 15:19:41] (step=0093775) Train Loss: 5.1502, Train Steps/Sec: 1.18 +100%|████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████▊| 23425/23458 [6:38:27<00:27, 1.19it/s][2025-04-26 15:20:02] (step=0093800) Train Loss: 5.1662, Train Steps/Sec: 1.17 +100%|████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████▉| 23450/23458 [6:38:48<00:06, 1.15it/s][2025-04-26 15:20:23] (step=0093825) Train Loss: 5.1947, Train Steps/Sec: 1.18 +100%|█████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████| 23458/23458 [6:38:55<00:00, 1.02s/it] +[2025-04-26 15:20:30] Done! diff --git a/CKPTS/Afteri2i_100kckpt_LLAVAT5_empty_Fixed_just_segment_pretrain_stage3_dreambench_recap_400k_t2i_400k_flux_200k_midjourney_150k_recovery_150k_3e4_epoch2_no_rotated/wandb/run-20250426_080403-2j21z9h0/files/requirements.txt b/CKPTS/Afteri2i_100kckpt_LLAVAT5_empty_Fixed_just_segment_pretrain_stage3_dreambench_recap_400k_t2i_400k_flux_200k_midjourney_150k_recovery_150k_3e4_epoch2_no_rotated/wandb/run-20250426_080403-2j21z9h0/files/requirements.txt new file mode 100644 index 0000000000000000000000000000000000000000..06dc78369ffff807b210006a0e79d705ffe2a7d7 --- /dev/null +++ b/CKPTS/Afteri2i_100kckpt_LLAVAT5_empty_Fixed_just_segment_pretrain_stage3_dreambench_recap_400k_t2i_400k_flux_200k_midjourney_150k_recovery_150k_3e4_epoch2_no_rotated/wandb/run-20250426_080403-2j21z9h0/files/requirements.txt @@ -0,0 +1,131 @@ +typing_extensions==4.12.2 +pyzmq==26.3.0 +nvidia-cufft-cu12==11.0.2.54 +triton==3.1.0 +nvidia-cublas-cu12==12.1.3.1 +psutil==7.0.0 +nvidia-cuda-cupti-cu12==12.1.105 +smmap==5.0.2 +nvidia-cuda-runtime-cu12==12.1.105 +aiohappyeyeballs==2.6.1 +asttokens==3.0.0 +huggingface-hub==0.29.3 +pyarrow==19.0.1 +fonttools==4.56.0 +python-dateutil==2.9.0.post0 +GitPython==3.1.44 +aiohttp==3.11.14 +wandb==0.19.8 +setproctitle==1.3.5 +PyYAML==6.0.2 +pydantic_core==2.27.2 +safetensors==0.5.3 +nvidia-nvjitlink-cu12==12.1.105 +aiosignal==1.3.2 +dill==0.3.8 +nvidia-cuda-nvrtc-cu12==12.1.105 +multiprocess==0.70.16 +pure_eval==0.2.3 +stack_data==0.6.3 +pydantic==2.10.6 +MarkupSafe==2.1.5 +tornado==6.4.2 +executing==2.1.0 +executing==2.2.0 +opencv-python==4.11.0.86 +nvitop==1.4.2 +multidict==6.2.0 +Jinja2==3.1.4 +torch==2.5.1+cu121 +nvidia-curand-cu12==10.3.2.106 +platformdirs==4.3.6 +six==1.17.0 +mpmath==1.3.0 +zipp==3.21.0 +packaging==24.2 +requests==2.32.3 +certifi==2025.1.31 +docker-pycreds==0.4.0 +torchvision==0.20.1+cu121 +pandas==2.2.3 +networkx==3.3 +exceptiongroup==1.2.2 +pickleshare==0.7.5 +tokenizers==0.21.1 +charset-normalizer==3.4.1 +jupyter_core==5.7.2 +wcwidth==0.2.13 +nvidia-nvtx-cu12==12.1.105 +prompt_toolkit==3.0.50 +fsspec==2024.12.0 +pillow==11.1.0 +propcache==0.3.0 +regex==2024.11.6 +ptyprocess==0.7.0 +contourpy==1.3.1 +importlib_metadata==8.6.1 +idna==3.10 +comm==0.2.2 +protobuf==5.29.3 +yarl==1.18.3 +ipython_pygments_lexers==1.1.1 +pip==25.0 +parso==0.8.4 +joblib==1.4.2 +nvidia-nccl-cu12==2.21.5 +hf_transfer==0.1.9 +Pygments==2.19.1 +decorator==5.2.1 +filelock==3.18.0 +nvidia-cusparse-cu12==12.1.0.106 +debugpy==1.8.13 +urllib3==2.3.0 +traitlets==5.14.3 +tzdata==2025.1 +matplotlib-inline==0.1.7 +matplotlib==3.10.1 +kiwisolver==1.4.8 +nest_asyncio==1.6.0 +frozenlist==1.5.0 +nvidia-ml-py==12.570.86 +transformers==4.49.0 +nltk==3.9.1 +ipykernel==6.29.5 +click==8.1.8 +gitdb==4.0.12 +pyparsing==3.2.1 +attrs==25.3.0 +jedi==0.19.2 +ipython==9.0.2 +nvidia-cudnn-cu12==9.1.0.70 +pexpect==4.9.0 +nvidia-cusolver-cu12==11.4.5.107 +numpy==2.2.4 +tqdm==4.67.1 +pytz==2025.1 +wheel==0.45.1 +sentry-sdk==2.23.1 +torchaudio==2.5.1+cu121 +jupyter_client==8.6.3 +cycler==0.12.1 +annotated-types==0.7.0 +sympy==1.13.1 +xxhash==3.5.0 +datasets==3.4.1 +setuptools==75.8.0 +typing_extensions==4.12.2 +wheel==0.43.0 +importlib_metadata==8.0.0 +backports.tarfile==1.2.0 +autocommand==2.2.2 +packaging==24.2 +tomli==2.0.1 +typeguard==4.3.0 +zipp==3.19.2 +jaraco.context==5.3.0 +jaraco.functools==4.0.1 +more-itertools==10.3.0 +platformdirs==4.2.2 +jaraco.text==3.12.1 +jaraco.collections==5.1.0 +inflect==7.3.1 diff --git a/CKPTS/Afteri2i_100kckpt_LLAVAT5_empty_Fixed_just_segment_pretrain_stage3_dreambench_recap_400k_t2i_400k_flux_200k_midjourney_150k_recovery_150k_3e4_epoch2_no_rotated/wandb/run-20250426_080403-2j21z9h0/files/wandb-metadata.json b/CKPTS/Afteri2i_100kckpt_LLAVAT5_empty_Fixed_just_segment_pretrain_stage3_dreambench_recap_400k_t2i_400k_flux_200k_midjourney_150k_recovery_150k_3e4_epoch2_no_rotated/wandb/run-20250426_080403-2j21z9h0/files/wandb-metadata.json new file mode 100644 index 0000000000000000000000000000000000000000..036f5ed22426cf238f2486e898a5bfdde06442b1 --- /dev/null +++ b/CKPTS/Afteri2i_100kckpt_LLAVAT5_empty_Fixed_just_segment_pretrain_stage3_dreambench_recap_400k_t2i_400k_flux_200k_midjourney_150k_recovery_150k_3e4_epoch2_no_rotated/wandb/run-20250426_080403-2j21z9h0/files/wandb-metadata.json @@ -0,0 +1,150 @@ +{ + "os": "Linux-5.15.0-1064-azure-x86_64-with-glibc2.31", + "python": "CPython 3.11.11", + "startedAt": "2025-04-26T08:04:03.465300Z", + "args": [ + "--vq-ckpt", + "/tmp/haozhezhao/model/llamagen_t2i/vq_ds16_t2i.pt", + "--data-path", + "/tmp/haozhezhao/MLLMG/jsonl_data/merged_train_set_set_subject_400k_recap_t2i_400k_flux_200k_midjourney_150k_recovery_150k_grounding_100fluxseg_50samseg.jsonl", + "--dataset", + "ti2i", + "--image-size", + "512", + "--results-dir", + "checkpoint/Afteri2i_100kckpt_LLAVAT5_empty_Fixed_just_segment_pretrain_stage3_dreambench_recap_400k_t2i_400k_flux_200k_midjourney_150k_recovery_150k_3e4_epoch2_no_rotated", + "--cloud-save-path", + "/tmp/haozhezhao/checkpoint", + "--lr", + "3e-4", + "--val_data_path", + "/tmp/haozhezhao/MLLMG/jsonl_data/dreambench_plus_valid.jsonl", + "--use_vision_tower", + "--model_name_or_path", + "/tmp/haozhezhao/model/blip2-flan-t5-xl", + "--image_place_holder", + "", + "--do_eval", + "--eval_steps", + "2000", + "--max_eval_samples", + "250", + "--cfg-scale", + "7.5", + "--top-k", + "16384", + "--load_from_checkpoint", + "/tmp/haozhezhao/MLLMG/checkpoint/FIXed_3M_1epoch_step32000.pt", + "--global-batch-size", + "56", + "--num-workers", + "8", + "--warmup", + "0.05", + "--gradient-accumulation-steps", + "4", + "--train_text_encoder", + "--ckpt-every", + "2000", + "--epochs", + "4", + "--subject_driven", + "--reference_data_path", + "/tmp/haozhezhao/MLLMG/cc12m_reference_tunnel.jsonl", + "--multimodal_encoder", + "llava", + "--do_recovery", + "--find_unused_parameters", + "--cls-token-num", + "512", + "--dreambench_eval", + "--save_total_limit", + "1", + "--load_language_projection", + "/tmp/haozhezhao/MLLMG/llava-v1.5-flant5_fixed-pretrain/mm_projector.bin", + "--gpt-ckpt", + "/tmp/haozhezhao/MLLMG/checkpoint/Afteri2i_100kckpt_LLAVAT5_empty_Fixed_just_segment_pretrain_stage3_dreambench_recap_400k_t2i_400k_flux_200k_midjourney_150k_recovery_150k_3e4_epoch2_no_rotated/002-GPT-XL/checkpoints/0070000.pt", + "--mm_vision_tower", + "openai/clip-vit-large-patch14", + "--train_all", + "--load_fixed_llamagen", + "--fix", + "gpt-empty-fix", + "--resume" + ], + "program": "/tmp/haozhezhao/MLLMG/autoregressive/train/train_t2i.py", + "codePath": "autoregressive/train/train_t2i.py", + "email": "mimazhe55360@gmail.com", + "root": "checkpoint/Afteri2i_100kckpt_LLAVAT5_empty_Fixed_just_segment_pretrain_stage3_dreambench_recap_400k_t2i_400k_flux_200k_midjourney_150k_recovery_150k_3e4_epoch2_no_rotated", + "host": "447cc403a8794092814259713c51c1df00001X", + "executable": "/tmp/haozhezhao/anaconda3/envs/nlp/bin/python", + "codePathLocal": "autoregressive/train/train_t2i.py", + "cpu_count": 96, + "cpu_count_logical": 96, + "gpu": "NVIDIA A100-SXM4-80GB", + "gpu_count": 8, + "disk": { + "/": { + "total": "133003395072", + "used": "65722880000" + } + }, + "memory": { + "total": "1902387884032" + }, + "cpu": { + "count": 96, + "countLogical": 96 + }, + "gpu_nvidia": [ + { + "name": "NVIDIA A100-SXM4-80GB", + "memoryTotal": "85899345920", + "cudaCores": 6912, + "architecture": "Ampere" + }, + { + "name": "NVIDIA A100-SXM4-80GB", + "memoryTotal": "85899345920", + "cudaCores": 6912, + "architecture": "Ampere" + }, + { + "name": "NVIDIA A100-SXM4-80GB", + "memoryTotal": "85899345920", + "cudaCores": 6912, + "architecture": "Ampere" + }, + { + "name": "NVIDIA A100-SXM4-80GB", + "memoryTotal": "85899345920", + "cudaCores": 6912, + "architecture": "Ampere" + }, + { + "name": "NVIDIA A100-SXM4-80GB", + "memoryTotal": "85899345920", + "cudaCores": 6912, + "architecture": "Ampere" + }, + { + "name": "NVIDIA A100-SXM4-80GB", + "memoryTotal": "85899345920", + "cudaCores": 6912, + "architecture": "Ampere" + }, + { + "name": "NVIDIA A100-SXM4-80GB", + "memoryTotal": "85899345920", + "cudaCores": 6912, + "architecture": "Ampere" + }, + { + "name": "NVIDIA A100-SXM4-80GB", + "memoryTotal": "85899345920", + "cudaCores": 6912, + "architecture": "Ampere" + } + ], + "cudaVersion": "12.2" +} \ No newline at end of file diff --git a/CKPTS/Afteri2i_100kckpt_LLAVAT5_empty_Fixed_just_segment_pretrain_stage3_dreambench_recap_400k_t2i_400k_flux_200k_midjourney_150k_recovery_150k_3e4_epoch2_no_rotated/wandb/run-20250426_080403-2j21z9h0/files/wandb-summary.json b/CKPTS/Afteri2i_100kckpt_LLAVAT5_empty_Fixed_just_segment_pretrain_stage3_dreambench_recap_400k_t2i_400k_flux_200k_midjourney_150k_recovery_150k_3e4_epoch2_no_rotated/wandb/run-20250426_080403-2j21z9h0/files/wandb-summary.json new file mode 100644 index 0000000000000000000000000000000000000000..7bc14289ca9699108165b6cf4f9a82bf4b302789 --- /dev/null +++ b/CKPTS/Afteri2i_100kckpt_LLAVAT5_empty_Fixed_just_segment_pretrain_stage3_dreambench_recap_400k_t2i_400k_flux_200k_midjourney_150k_recovery_150k_3e4_epoch2_no_rotated/wandb/run-20250426_080403-2j21z9h0/files/wandb-summary.json @@ -0,0 +1 @@ +{"_timestamp":1.7456808237094688e+09,"_runtime":26189.722360655,"_step":93825,"_wandb":{"runtime":26189},"train lr":0.00026766067605824944,"train loss":5.194711685180664,"Train Steps/Sec":1.1758481584872367} \ No newline at end of file diff --git a/CKPTS/Afteri2i_100kckpt_LLAVAT5_empty_Fixed_just_segment_pretrain_stage3_dreambench_recap_400k_t2i_400k_flux_200k_midjourney_150k_recovery_150k_3e4_epoch2_no_rotated/wandb/run-20250426_080403-2j21z9h0/logs/debug-core.log b/CKPTS/Afteri2i_100kckpt_LLAVAT5_empty_Fixed_just_segment_pretrain_stage3_dreambench_recap_400k_t2i_400k_flux_200k_midjourney_150k_recovery_150k_3e4_epoch2_no_rotated/wandb/run-20250426_080403-2j21z9h0/logs/debug-core.log new file mode 100644 index 0000000000000000000000000000000000000000..13021a36659ee3629bec5896bd018286df557ddd --- /dev/null +++ b/CKPTS/Afteri2i_100kckpt_LLAVAT5_empty_Fixed_just_segment_pretrain_stage3_dreambench_recap_400k_t2i_400k_flux_200k_midjourney_150k_recovery_150k_3e4_epoch2_no_rotated/wandb/run-20250426_080403-2j21z9h0/logs/debug-core.log @@ -0,0 +1,13 @@ +{"time":"2025-04-26T08:04:02.712852311Z","level":"INFO","msg":"main: starting server","port-filename":"/tmp/tmp_56jfbkb/port-3408653.txt","pid":3408653,"log-level":0,"disable-analytics":false,"shutdown-on-parent-exit":false} +{"time":"2025-04-26T08:04:02.714800007Z","level":"INFO","msg":"Will exit if parent process dies.","ppid":3408653} +{"time":"2025-04-26T08:04:02.714791255Z","level":"INFO","msg":"server is running","addr":{"IP":"127.0.0.1","Port":35429,"Zone":""}} +{"time":"2025-04-26T08:04:02.900372958Z","level":"INFO","msg":"connection: ManageConnectionData: new connection created","id":"127.0.0.1:35746"} +{"time":"2025-04-26T08:04:03.466532523Z","level":"INFO","msg":"handleInformInit: received","streamId":"2j21z9h0","id":"127.0.0.1:35746"} +{"time":"2025-04-26T08:04:03.673057516Z","level":"INFO","msg":"handleInformInit: stream started","streamId":"2j21z9h0","id":"127.0.0.1:35746"} +{"time":"2025-04-26T15:20:33.187520002Z","level":"INFO","msg":"handleInformTeardown: server teardown initiated","id":"127.0.0.1:35746"} +{"time":"2025-04-26T15:20:33.187629602Z","level":"INFO","msg":"connection: closing","id":"127.0.0.1:35746"} +{"time":"2025-04-26T15:20:33.187661683Z","level":"INFO","msg":"server is shutting down"} +{"time":"2025-04-26T15:20:33.187783039Z","level":"INFO","msg":"connection: closed successfully","id":"127.0.0.1:35746"} +{"time":"2025-04-26T15:20:34.4941129Z","level":"INFO","msg":"handleInformTeardown: server shutdown complete","id":"127.0.0.1:35746"} +{"time":"2025-04-26T15:20:34.494138307Z","level":"INFO","msg":"connection: ManageConnectionData: connection closed","id":"127.0.0.1:35746"} +{"time":"2025-04-26T15:20:34.494154748Z","level":"INFO","msg":"server is closed"} diff --git a/CKPTS/Afteri2i_100kckpt_LLAVAT5_empty_Fixed_just_segment_pretrain_stage3_dreambench_recap_400k_t2i_400k_flux_200k_midjourney_150k_recovery_150k_3e4_epoch2_no_rotated/wandb/run-20250426_080403-2j21z9h0/logs/debug-internal.log b/CKPTS/Afteri2i_100kckpt_LLAVAT5_empty_Fixed_just_segment_pretrain_stage3_dreambench_recap_400k_t2i_400k_flux_200k_midjourney_150k_recovery_150k_3e4_epoch2_no_rotated/wandb/run-20250426_080403-2j21z9h0/logs/debug-internal.log new file mode 100644 index 0000000000000000000000000000000000000000..512261d9109fab3b8772044697823bc9f6cbcaf9 --- /dev/null +++ b/CKPTS/Afteri2i_100kckpt_LLAVAT5_empty_Fixed_just_segment_pretrain_stage3_dreambench_recap_400k_t2i_400k_flux_200k_midjourney_150k_recovery_150k_3e4_epoch2_no_rotated/wandb/run-20250426_080403-2j21z9h0/logs/debug-internal.log @@ -0,0 +1,15 @@ +{"time":"2025-04-26T08:04:03.466771029Z","level":"INFO","msg":"stream: starting","core version":"0.19.8","symlink path":"checkpoint/Afteri2i_100kckpt_LLAVAT5_empty_Fixed_just_segment_pretrain_stage3_dreambench_recap_400k_t2i_400k_flux_200k_midjourney_150k_recovery_150k_3e4_epoch2_no_rotated/wandb/run-20250426_080403-2j21z9h0/logs/debug-core.log"} +{"time":"2025-04-26T08:04:03.67302256Z","level":"INFO","msg":"created new stream","id":"2j21z9h0"} +{"time":"2025-04-26T08:04:03.673052226Z","level":"INFO","msg":"stream: started","id":"2j21z9h0"} +{"time":"2025-04-26T08:04:03.673070455Z","level":"INFO","msg":"writer: Do: started","stream_id":"2j21z9h0"} +{"time":"2025-04-26T08:04:03.673092306Z","level":"INFO","msg":"sender: started","stream_id":"2j21z9h0"} +{"time":"2025-04-26T08:04:03.673315729Z","level":"INFO","msg":"handler: started","stream_id":"2j21z9h0"} +{"time":"2025-04-26T08:04:04.102489046Z","level":"INFO","msg":"Starting system monitor"} +{"time":"2025-04-26T15:20:33.187645817Z","level":"INFO","msg":"stream: closing","id":"2j21z9h0"} +{"time":"2025-04-26T15:20:33.187688377Z","level":"INFO","msg":"Stopping system monitor"} +{"time":"2025-04-26T15:20:33.188384875Z","level":"INFO","msg":"Stopped system monitor"} +{"time":"2025-04-26T15:20:34.293026283Z","level":"INFO","msg":"fileTransfer: Close: file transfer manager closed"} +{"time":"2025-04-26T15:20:34.493877028Z","level":"INFO","msg":"handler: closed","stream_id":"2j21z9h0"} +{"time":"2025-04-26T15:20:34.49391269Z","level":"INFO","msg":"sender: closed","stream_id":"2j21z9h0"} +{"time":"2025-04-26T15:20:34.493906784Z","level":"INFO","msg":"writer: Close: closed","stream_id":"2j21z9h0"} +{"time":"2025-04-26T15:20:34.494012131Z","level":"INFO","msg":"stream: closed","id":"2j21z9h0"} diff --git a/CKPTS/Afteri2i_100kckpt_LLAVAT5_empty_Fixed_just_segment_pretrain_stage3_dreambench_recap_400k_t2i_400k_flux_200k_midjourney_150k_recovery_150k_3e4_epoch2_no_rotated/wandb/run-20250426_080403-2j21z9h0/logs/debug.log b/CKPTS/Afteri2i_100kckpt_LLAVAT5_empty_Fixed_just_segment_pretrain_stage3_dreambench_recap_400k_t2i_400k_flux_200k_midjourney_150k_recovery_150k_3e4_epoch2_no_rotated/wandb/run-20250426_080403-2j21z9h0/logs/debug.log new file mode 100644 index 0000000000000000000000000000000000000000..67daf6350cdc194f1aba2211885c4876c254f0e6 --- /dev/null +++ b/CKPTS/Afteri2i_100kckpt_LLAVAT5_empty_Fixed_just_segment_pretrain_stage3_dreambench_recap_400k_t2i_400k_flux_200k_midjourney_150k_recovery_150k_3e4_epoch2_no_rotated/wandb/run-20250426_080403-2j21z9h0/logs/debug.log @@ -0,0 +1,23 @@ +2025-04-26 08:04:03,459 INFO MainThread:3408653 [wandb_setup.py:_flush():67] Current SDK version is 0.19.8 +2025-04-26 08:04:03,460 INFO MainThread:3408653 [wandb_setup.py:_flush():67] Configure stats pid to 3408653 +2025-04-26 08:04:03,460 INFO MainThread:3408653 [wandb_setup.py:_flush():67] Loading settings from /tmp/haozhezhao/.config/wandb/settings +2025-04-26 08:04:03,460 INFO MainThread:3408653 [wandb_setup.py:_flush():67] Loading settings from /tmp/haozhezhao/MLLMG/wandb/settings +2025-04-26 08:04:03,460 INFO MainThread:3408653 [wandb_setup.py:_flush():67] Loading settings from environment variables +2025-04-26 08:04:03,460 INFO MainThread:3408653 [wandb_init.py:setup_run_log_directory():647] Logging user logs to checkpoint/Afteri2i_100kckpt_LLAVAT5_empty_Fixed_just_segment_pretrain_stage3_dreambench_recap_400k_t2i_400k_flux_200k_midjourney_150k_recovery_150k_3e4_epoch2_no_rotated/wandb/run-20250426_080403-2j21z9h0/logs/debug.log +2025-04-26 08:04:03,460 INFO MainThread:3408653 [wandb_init.py:setup_run_log_directory():648] Logging internal logs to checkpoint/Afteri2i_100kckpt_LLAVAT5_empty_Fixed_just_segment_pretrain_stage3_dreambench_recap_400k_t2i_400k_flux_200k_midjourney_150k_recovery_150k_3e4_epoch2_no_rotated/wandb/run-20250426_080403-2j21z9h0/logs/debug-internal.log +2025-04-26 08:04:03,460 INFO MainThread:3408653 [wandb_init.py:init():761] calling init triggers +2025-04-26 08:04:03,460 INFO MainThread:3408653 [wandb_init.py:init():766] wandb.init called with sweep_config: {} +config: {'data_path': '/tmp/haozhezhao/MLLMG/jsonl_data/merged_train_set_set_subject_400k_recap_t2i_400k_flux_200k_midjourney_150k_recovery_150k_grounding_100fluxseg_50samseg.jsonl', 'cloud_save_path': '/tmp/haozhezhao/checkpoint', 'no_local_save': False, 'vq_model': 'VQ-16', 'vq_ckpt': '/tmp/haozhezhao/model/llamagen_t2i/vq_ds16_t2i.pt', 'codebook_size': 16384, 'codebook_embed_dim': 8, 'gpt_model': 'GPT-XL', 'gpt_ckpt': '/tmp/haozhezhao/MLLMG/checkpoint/Afteri2i_100kckpt_LLAVAT5_empty_Fixed_just_segment_pretrain_stage3_dreambench_recap_400k_t2i_400k_flux_200k_midjourney_150k_recovery_150k_3e4_epoch2_no_rotated/002-GPT-XL/checkpoints/0070000.pt', 'gpt_type': 't2i', 'vocab_size': 16384, 'cls_token_num': 512, 'dropout_p': 0.1, 'token_dropout_p': 0.1, 'drop_path': 0.0, 'no_compile': False, 'results_dir': 'checkpoint/Afteri2i_100kckpt_LLAVAT5_empty_Fixed_just_segment_pretrain_stage3_dreambench_recap_400k_t2i_400k_flux_200k_midjourney_150k_recovery_150k_3e4_epoch2_no_rotated', 'dataset': 'ti2i', 'image_size': 512, 'downsample_size': 16, 'num_classes': 1000, 'epochs': 4, 'lr': 0.0003, 'weight_decay': 0.05, 'beta1': 0.9, 'beta2': 0.95, 'max_grad_norm': 1.0, 'global_batch_size': 56, 'global_seed': 0, 'num_workers': 8, 'log_every': 25, 'ckpt_every': 2000, 'gradient_accumulation_steps': 4, 'mixed_precision': 'bf16', 'val_data_path': '/tmp/haozhezhao/MLLMG/jsonl_data/dreambench_plus_valid.jsonl', 'use_vision_tower': True, 'model_name_or_path': '/tmp/haozhezhao/model/blip2-flan-t5-xl', 'image_place_holder': '', 'processor_path': None, 'do_eval': True, 'max_eval_samples': 250, 'train_text_encoder': True, 'no_left_padding': False, 'cfg_scale': 7.5, 'top_k': 16384, 'temperature': 0.9, 'top_p': 1.0, 'eval_steps': 2000, 'project_name': 'llamagen_ti2i', 'load_from_checkpoint': '/tmp/haozhezhao/MLLMG/checkpoint/FIXed_3M_1epoch_step32000.pt', 'warmup': 0.05, 'lr_decay_style': 'cosine', 'lr_decay_ratio': 0.1, 'train_iters': 500000, 'class_dropout_prob': 0.1, 'with_image_only': False, 'image_only_rate': 0.1, 'stage2': False, 'subject_driven': True, 'load_subject_embedding': None, 'reference_data_path': '/tmp/haozhezhao/MLLMG/cc12m_reference_tunnel.jsonl', 'multimodal_encoder': 'llava', 'do_recovery': True, 'no_replace': False, 'resume': True, 'dreambench_eval': True, 'find_unused_parameters': True, 'load_visual_encoder': False, 'continue_stage1': False, 'replace_subject': False, 'train_all': True, 'save_total_limit': 1, 'load_language_projection': '/tmp/haozhezhao/MLLMG/llava-v1.5-flant5_fixed-pretrain/mm_projector.bin', 'mm_vision_tower': 'openai/clip-vit-large-patch14', 'load_fixed_llamagen': True, 'unfreeze_output': False, 'fix': 'gpt-empty-fix', 'rank': 0, 'world_size': 8, 'gpu': 0, 'dist_url': 'env://', 'distributed': True, 'dist_backend': 'nccl', '_wandb': {}} +2025-04-26 08:04:03,460 INFO MainThread:3408653 [wandb_init.py:init():784] starting backend +2025-04-26 08:04:03,460 INFO MainThread:3408653 [wandb_init.py:init():788] sending inform_init request +2025-04-26 08:04:03,465 INFO MainThread:3408653 [backend.py:_multiprocessing_setup():101] multiprocessing start_methods=fork,spawn,forkserver, using: spawn +2025-04-26 08:04:03,465 INFO MainThread:3408653 [wandb_init.py:init():798] backend started and connected +2025-04-26 08:04:03,467 INFO MainThread:3408653 [wandb_init.py:init():891] updated telemetry +2025-04-26 08:04:03,467 INFO MainThread:3408653 [wandb_init.py:init():915] communicating run to backend with 90.0 second timeout +2025-04-26 08:04:04,100 INFO MainThread:3408653 [wandb_init.py:init():990] starting run threads in backend +2025-04-26 08:04:04,235 INFO MainThread:3408653 [wandb_run.py:_console_start():2375] atexit reg +2025-04-26 08:04:04,235 INFO MainThread:3408653 [wandb_run.py:_redirect():2227] redirect: wrap_raw +2025-04-26 08:04:04,236 INFO MainThread:3408653 [wandb_run.py:_redirect():2292] Wrapping output streams. +2025-04-26 08:04:04,236 INFO MainThread:3408653 [wandb_run.py:_redirect():2315] Redirects installed. +2025-04-26 08:04:04,237 INFO MainThread:3408653 [wandb_init.py:init():1032] run started, returning control to user process +2025-04-26 15:20:33,186 INFO MsgRouterThr:3408653 [mailbox.py:close():129] Closing mailbox, abandoning 1 handles. diff --git a/CKPTS/Afteri2i_100kckpt_LLAVAT5_empty_Fixed_just_segment_pretrain_stage3_dreambench_recap_400k_t2i_400k_flux_200k_midjourney_150k_recovery_150k_3e4_epoch2_no_rotated/wandb/run-20250426_080403-2j21z9h0/run-2j21z9h0.wandb b/CKPTS/Afteri2i_100kckpt_LLAVAT5_empty_Fixed_just_segment_pretrain_stage3_dreambench_recap_400k_t2i_400k_flux_200k_midjourney_150k_recovery_150k_3e4_epoch2_no_rotated/wandb/run-20250426_080403-2j21z9h0/run-2j21z9h0.wandb new file mode 100644 index 0000000000000000000000000000000000000000..d90a180b9a1820a66f9e483521aca0cc0a8e80af --- /dev/null +++ b/CKPTS/Afteri2i_100kckpt_LLAVAT5_empty_Fixed_just_segment_pretrain_stage3_dreambench_recap_400k_t2i_400k_flux_200k_midjourney_150k_recovery_150k_3e4_epoch2_no_rotated/wandb/run-20250426_080403-2j21z9h0/run-2j21z9h0.wandb @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:786bf19bb9ce61c19194cdd854659d0620eb823c167cb7e81d1c0a653dfbee84 +size 17983517 diff --git a/CKPTS/BeforeFix_Blip_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_objects/000-GPT-XL/log.txt b/CKPTS/BeforeFix_Blip_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_objects/000-GPT-XL/log.txt new file mode 100644 index 0000000000000000000000000000000000000000..f48d18906bbec351044d1b7b6e88385cf8a2cd7f --- /dev/null +++ b/CKPTS/BeforeFix_Blip_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_objects/000-GPT-XL/log.txt @@ -0,0 +1,9 @@ +[2025-04-21 16:50:26] Experiment directory created at checkpoint/BeforeFix_Blip_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_objects/000-GPT-XL +[2025-04-21 16:50:26] Namespace(data_path='/tmp/haozhezhao/MLLMG/jsonl_data/multiobjects_molom_imagenet_flux_qwen_midsource_gen_2_2m_trained.jsonl', cloud_save_path='/tmp/haozhezhao/MLLMG/checkpoint', no_local_save=False, vq_model='VQ-16', vq_ckpt='/tmp/haozhezhao/model/llamagen_t2i/vq_ds16_t2i.pt', codebook_size=16384, codebook_embed_dim=8, gpt_model='GPT-XL', gpt_ckpt='/tmp/haozhezhao/MLLMG/checkpoint/BLIP2Trainall_just_segment_pretrain_stage3_dreambench_recap_Subject200k_filtered_t2i_flux400k_200kmid_recovery_150k_extract_150k_100_fluxseg_50samseg_trainall_1e4/000-GPT-XL/checkpoints/0010000.pt', gpt_type='t2i', vocab_size=16384, cls_token_num=512, dropout_p=0.1, token_dropout_p=0.1, drop_path=0.0, no_compile=False, results_dir='checkpoint/BeforeFix_Blip_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_objects', dataset='ti2i', image_size=512, downsample_size=16, num_classes=1000, epochs=2, lr=5e-05, weight_decay=0.05, beta1=0.9, beta2=0.95, max_grad_norm=1.0, global_batch_size=56, global_seed=0, num_workers=4, log_every=25, ckpt_every=2000, gradient_accumulation_steps=4, mixed_precision='bf16', val_data_path='/tmp/haozhezhao/MLLMG/jsonl_data/multiobjects_molom_imagenet_flux_qwen_midsource_gen_val.jsonl', use_vision_tower=True, model_name_or_path='/tmp/haozhezhao/model/instructblip-flan-t5-xl', image_place_holder='', processor_path=None, do_eval=True, max_eval_samples=200, train_text_encoder=True, no_left_padding=False, cfg_scale=7.5, top_k=16384, temperature=0.9, top_p=1.0, eval_steps=2000, project_name='llamagen_ti2i', load_from_checkpoint='/tmp/haozhezhao/model/llamagen_t2i/t2i_XL_stage2_512.pt', warmup=0.05, lr_decay_style='cosine', lr_decay_ratio=0.1, train_iters=500000, class_dropout_prob=0.1, with_image_only=False, image_only_rate=0.1, stage2=False, subject_driven=True, load_subject_embedding=None, reference_data_path='/tmp/haozhezhao/MLLMG/cc12m_reference_tunnel.jsonl', multimodal_encoder='blip', do_recovery=True, no_replace=False, resume=False, dreambench_eval=False, find_unused_parameters=True, load_visual_encoder=False, continue_stage1=False, replace_subject=False, train_all=True, save_total_limit=2, load_language_projection=None, mm_vision_tower='openai/clip-vit-large-patch14', load_fixed_llamagen=False, unfreeze_output=False, fix='gpt', rank=0, world_size=8, gpu=0, dist_url='env://', distributed=True, dist_backend='nccl') +[2025-04-21 16:50:26] Starting rank=0, seed=0, world_size=8. +[2025-04-21 16:52:06] GPT Parameters: 3,174,417,408 +[2025-04-21 16:52:06] num decayed parameter tensors: 637, with 3,173,270,016 parameters +[2025-04-21 16:52:06] num non-decayed parameter tensors: 644, with 1,147,392 parameters +[2025-04-21 16:52:06] using fused AdamW: True +[2025-04-21 16:52:36] Dataset contains 2,290,583 images +[2025-04-21 16:52:36] Train iters 81806 , warmup 4090.3, len of loader 40903 diff --git a/CKPTS/BeforeFix_Blip_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_objects/001-GPT-XL/log.txt b/CKPTS/BeforeFix_Blip_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_objects/001-GPT-XL/log.txt new file mode 100644 index 0000000000000000000000000000000000000000..68fde95bf4adec9fcaee78fcbd2280c6acf0f376 --- /dev/null +++ b/CKPTS/BeforeFix_Blip_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_objects/001-GPT-XL/log.txt @@ -0,0 +1,9 @@ +[2025-04-21 16:57:04] Experiment directory created at checkpoint/BeforeFix_Blip_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_objects/001-GPT-XL +[2025-04-21 16:57:04] Namespace(data_path='/tmp/haozhezhao/MLLMG/jsonl_data/multiobjects_molom_imagenet_flux_qwen_midsource_gen_2_2m_trained.jsonl', cloud_save_path='/tmp/haozhezhao/MLLMG/checkpoint', no_local_save=False, vq_model='VQ-16', vq_ckpt='/tmp/haozhezhao/model/llamagen_t2i/vq_ds16_t2i.pt', codebook_size=16384, codebook_embed_dim=8, gpt_model='GPT-XL', gpt_ckpt='/tmp/haozhezhao/MLLMG/checkpoint/BLIP2Trainall_just_segment_pretrain_stage3_dreambench_recap_Subject200k_filtered_t2i_flux400k_200kmid_recovery_150k_extract_150k_100_fluxseg_50samseg_trainall_1e4/000-GPT-XL/checkpoints/0010000.pt', gpt_type='t2i', vocab_size=16384, cls_token_num=512, dropout_p=0.1, token_dropout_p=0.1, drop_path=0.0, no_compile=False, results_dir='checkpoint/BeforeFix_Blip_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_objects', dataset='ti2i', image_size=512, downsample_size=16, num_classes=1000, epochs=2, lr=5e-05, weight_decay=0.05, beta1=0.9, beta2=0.95, max_grad_norm=1.0, global_batch_size=56, global_seed=0, num_workers=4, log_every=25, ckpt_every=2000, gradient_accumulation_steps=4, mixed_precision='bf16', val_data_path='/tmp/haozhezhao/MLLMG/jsonl_data/multiobjects_molom_imagenet_flux_qwen_midsource_gen_val.jsonl', use_vision_tower=True, model_name_or_path='/tmp/haozhezhao/model/instructblip-flan-t5-xl', image_place_holder='', processor_path=None, do_eval=True, max_eval_samples=200, train_text_encoder=True, no_left_padding=False, cfg_scale=7.5, top_k=16384, temperature=0.9, top_p=1.0, eval_steps=2000, project_name='llamagen_ti2i', load_from_checkpoint='/tmp/haozhezhao/model/llamagen_t2i/t2i_XL_stage2_512.pt', warmup=0.05, lr_decay_style='cosine', lr_decay_ratio=0.1, train_iters=500000, class_dropout_prob=0.1, with_image_only=False, image_only_rate=0.1, stage2=False, subject_driven=True, load_subject_embedding='/tmp/haozhezhao/MLLMG/subject_embedding.bin', reference_data_path='/tmp/haozhezhao/MLLMG/cc12m_reference_tunnel.jsonl', multimodal_encoder='blip', do_recovery=True, no_replace=False, resume=False, dreambench_eval=False, find_unused_parameters=True, load_visual_encoder=False, continue_stage1=False, replace_subject=False, train_all=True, save_total_limit=2, load_language_projection=None, mm_vision_tower='openai/clip-vit-large-patch14', load_fixed_llamagen=False, unfreeze_output=False, fix='gpt', rank=0, world_size=8, gpu=0, dist_url='env://', distributed=True, dist_backend='nccl') +[2025-04-21 16:57:04] Starting rank=0, seed=0, world_size=8. +[2025-04-21 16:58:44] GPT Parameters: 3,174,417,408 +[2025-04-21 16:58:44] num decayed parameter tensors: 637, with 3,173,270,016 parameters +[2025-04-21 16:58:44] num non-decayed parameter tensors: 644, with 1,147,392 parameters +[2025-04-21 16:58:44] using fused AdamW: True +[2025-04-21 16:59:16] Dataset contains 2,290,583 images +[2025-04-21 16:59:16] Train iters 81806 , warmup 4090.3, len of loader 40903 diff --git a/CKPTS/BeforeFix_Blip_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_objects/002-GPT-XL/checkpoints/0080000.pt b/CKPTS/BeforeFix_Blip_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_objects/002-GPT-XL/checkpoints/0080000.pt new file mode 100644 index 0000000000000000000000000000000000000000..3b419f31e0c0f77c873904f17c817f3ff299f38e --- /dev/null +++ b/CKPTS/BeforeFix_Blip_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_objects/002-GPT-XL/checkpoints/0080000.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:3c91d3c6dc2dd307b580efbea5e5db5fd7a5b4c868ab1e2855b5fffd7d4ebc29 +size 19774281278 diff --git a/CKPTS/BeforeFix_Blip_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_objects/002-GPT-XL/checkpoints/eval_step_10000/batch_0_cfg_7.5_topk_16384.jpg b/CKPTS/BeforeFix_Blip_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_objects/002-GPT-XL/checkpoints/eval_step_10000/batch_0_cfg_7.5_topk_16384.jpg new file mode 100644 index 0000000000000000000000000000000000000000..d5e4c59a5687a0a66a3dc0bfe5f1e5dd07f2c739 --- /dev/null +++ b/CKPTS/BeforeFix_Blip_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_objects/002-GPT-XL/checkpoints/eval_step_10000/batch_0_cfg_7.5_topk_16384.jpg @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:f47a87c11217bf80a18a76569368e53728cd9d1a2f0da0d1ccda7072b72a64c5 +size 1091570 diff --git a/CKPTS/BeforeFix_Blip_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_objects/002-GPT-XL/checkpoints/eval_step_10000/batch_1_cfg_7.5_topk_16384.jpg b/CKPTS/BeforeFix_Blip_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_objects/002-GPT-XL/checkpoints/eval_step_10000/batch_1_cfg_7.5_topk_16384.jpg new file mode 100644 index 0000000000000000000000000000000000000000..33d0344491188afa8d804d7f08311076e56b1a52 --- /dev/null +++ b/CKPTS/BeforeFix_Blip_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_objects/002-GPT-XL/checkpoints/eval_step_10000/batch_1_cfg_7.5_topk_16384.jpg @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:d8c6fa03cf03a0d6b0ac0a77f7eceed1c67771fd24d3bf610b38411e0dc10ed8 +size 976778 diff --git a/CKPTS/BeforeFix_Blip_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_objects/002-GPT-XL/checkpoints/eval_step_10000/batch_2_cfg_7.5_topk_16384.jpg b/CKPTS/BeforeFix_Blip_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_objects/002-GPT-XL/checkpoints/eval_step_10000/batch_2_cfg_7.5_topk_16384.jpg new file mode 100644 index 0000000000000000000000000000000000000000..f6b57f9bf9235d1f6cd9356dda4113e4cbdb0fd5 --- /dev/null +++ b/CKPTS/BeforeFix_Blip_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_objects/002-GPT-XL/checkpoints/eval_step_10000/batch_2_cfg_7.5_topk_16384.jpg @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:c93a4c6189cddd3e69638def34b84223b67b89e3dc1ff47e49b1c67ea5f25a34 +size 989631 diff --git a/CKPTS/BeforeFix_Blip_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_objects/002-GPT-XL/checkpoints/eval_step_12000/batch_0_cfg_7.5_topk_16384.jpg b/CKPTS/BeforeFix_Blip_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_objects/002-GPT-XL/checkpoints/eval_step_12000/batch_0_cfg_7.5_topk_16384.jpg new file mode 100644 index 0000000000000000000000000000000000000000..7840b9176bd198cf29c17b887d33dd67cdaf7017 --- /dev/null +++ b/CKPTS/BeforeFix_Blip_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_objects/002-GPT-XL/checkpoints/eval_step_12000/batch_0_cfg_7.5_topk_16384.jpg @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:141d5e2c39aae8759206d25739d762e68e658a1d5f9d716aa4c3dfb3b650e286 +size 1102172 diff --git a/CKPTS/BeforeFix_Blip_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_objects/002-GPT-XL/checkpoints/eval_step_12000/batch_1_cfg_7.5_topk_16384.jpg b/CKPTS/BeforeFix_Blip_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_objects/002-GPT-XL/checkpoints/eval_step_12000/batch_1_cfg_7.5_topk_16384.jpg new file mode 100644 index 0000000000000000000000000000000000000000..1f05d1e6fddec6984a281d5db54ca2ba9f87665f --- /dev/null +++ b/CKPTS/BeforeFix_Blip_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_objects/002-GPT-XL/checkpoints/eval_step_12000/batch_1_cfg_7.5_topk_16384.jpg @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:fbf2c6f048ac75405f7eb952851356bb4fc3dde5d65358e3c7f1c3be0426a685 +size 969101 diff --git a/CKPTS/BeforeFix_Blip_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_objects/002-GPT-XL/checkpoints/eval_step_12000/batch_2_cfg_7.5_topk_16384.jpg b/CKPTS/BeforeFix_Blip_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_objects/002-GPT-XL/checkpoints/eval_step_12000/batch_2_cfg_7.5_topk_16384.jpg new file mode 100644 index 0000000000000000000000000000000000000000..15b8e8dec18e913b2abc878476a9c53095caec54 --- /dev/null +++ b/CKPTS/BeforeFix_Blip_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_objects/002-GPT-XL/checkpoints/eval_step_12000/batch_2_cfg_7.5_topk_16384.jpg @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:3716dd1de8b95f2e96cee581f6ffc24dfe3401967ed7be4af9778e9fa8e5bb70 +size 999310 diff --git a/CKPTS/BeforeFix_Blip_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_objects/002-GPT-XL/checkpoints/eval_step_14000/batch_0_cfg_7.5_topk_16384.jpg b/CKPTS/BeforeFix_Blip_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_objects/002-GPT-XL/checkpoints/eval_step_14000/batch_0_cfg_7.5_topk_16384.jpg new file mode 100644 index 0000000000000000000000000000000000000000..8966c8e458fa07e9892489d08f132ca412a75b24 --- /dev/null +++ b/CKPTS/BeforeFix_Blip_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_objects/002-GPT-XL/checkpoints/eval_step_14000/batch_0_cfg_7.5_topk_16384.jpg @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:88184cf3b867c10d9bf3afa10d6e9f6c9e7f53bfa332000af877eb58dc6e7b83 +size 1103749 diff --git a/CKPTS/BeforeFix_Blip_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_objects/002-GPT-XL/checkpoints/eval_step_14000/batch_1_cfg_7.5_topk_16384.jpg b/CKPTS/BeforeFix_Blip_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_objects/002-GPT-XL/checkpoints/eval_step_14000/batch_1_cfg_7.5_topk_16384.jpg new file mode 100644 index 0000000000000000000000000000000000000000..808ff1a627837f14270a6889be21e3dd3ebf546f --- /dev/null +++ b/CKPTS/BeforeFix_Blip_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_objects/002-GPT-XL/checkpoints/eval_step_14000/batch_1_cfg_7.5_topk_16384.jpg @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:678b1ef965efc587f239539ffca09dc1c73d488718b199234c3f9b59e45b96ed +size 959956 diff --git a/CKPTS/BeforeFix_Blip_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_objects/002-GPT-XL/checkpoints/eval_step_14000/batch_2_cfg_7.5_topk_16384.jpg b/CKPTS/BeforeFix_Blip_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_objects/002-GPT-XL/checkpoints/eval_step_14000/batch_2_cfg_7.5_topk_16384.jpg new file mode 100644 index 0000000000000000000000000000000000000000..d09efee91061496fbe4a382277144d6722ba3232 --- /dev/null +++ b/CKPTS/BeforeFix_Blip_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_objects/002-GPT-XL/checkpoints/eval_step_14000/batch_2_cfg_7.5_topk_16384.jpg @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:7ca577d9a4d06e325592c1c889b5f0c5928f65146935d62b083ec23e940b63be +size 982059 diff --git a/CKPTS/BeforeFix_Blip_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_objects/002-GPT-XL/checkpoints/eval_step_16000/batch_0_cfg_7.5_topk_16384.jpg b/CKPTS/BeforeFix_Blip_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_objects/002-GPT-XL/checkpoints/eval_step_16000/batch_0_cfg_7.5_topk_16384.jpg new file mode 100644 index 0000000000000000000000000000000000000000..277562d14a4ff797419e3bd3e78b8a6af2d60724 --- /dev/null +++ b/CKPTS/BeforeFix_Blip_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_objects/002-GPT-XL/checkpoints/eval_step_16000/batch_0_cfg_7.5_topk_16384.jpg @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:f016b9a95c68a7068e21c463f4ba09a916c2c5439ae36f74425ed9ada777d8df +size 1098441 diff --git a/CKPTS/BeforeFix_Blip_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_objects/002-GPT-XL/checkpoints/eval_step_16000/batch_1_cfg_7.5_topk_16384.jpg b/CKPTS/BeforeFix_Blip_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_objects/002-GPT-XL/checkpoints/eval_step_16000/batch_1_cfg_7.5_topk_16384.jpg new file mode 100644 index 0000000000000000000000000000000000000000..20ebd8317d7afef68610796b7897c90d5da00785 --- /dev/null +++ b/CKPTS/BeforeFix_Blip_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_objects/002-GPT-XL/checkpoints/eval_step_16000/batch_1_cfg_7.5_topk_16384.jpg @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:bdf981b4fac478f231530243ddb10d34d3ac6a527c9607f844c7f51ba89fd706 +size 967922 diff --git a/CKPTS/BeforeFix_Blip_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_objects/002-GPT-XL/checkpoints/eval_step_16000/batch_2_cfg_7.5_topk_16384.jpg b/CKPTS/BeforeFix_Blip_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_objects/002-GPT-XL/checkpoints/eval_step_16000/batch_2_cfg_7.5_topk_16384.jpg new file mode 100644 index 0000000000000000000000000000000000000000..3e8f3b865b775b51d38b1590b9024c660bfe6cda --- /dev/null +++ b/CKPTS/BeforeFix_Blip_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_objects/002-GPT-XL/checkpoints/eval_step_16000/batch_2_cfg_7.5_topk_16384.jpg @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:869707a9c65238490c4dc06105397e3faed5546f2f0d0fb9b8cb0e2f2b7b7045 +size 978529 diff --git a/CKPTS/BeforeFix_Blip_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_objects/002-GPT-XL/checkpoints/eval_step_18000/batch_0_cfg_7.5_topk_16384.jpg b/CKPTS/BeforeFix_Blip_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_objects/002-GPT-XL/checkpoints/eval_step_18000/batch_0_cfg_7.5_topk_16384.jpg new file mode 100644 index 0000000000000000000000000000000000000000..db66120592f6af257e4195558257f48e68f1601d --- /dev/null +++ b/CKPTS/BeforeFix_Blip_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_objects/002-GPT-XL/checkpoints/eval_step_18000/batch_0_cfg_7.5_topk_16384.jpg @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:ae47b326605a7bf51101cb2fec1bfd91d0187fdcd95ff63574833ee8ec559a80 +size 1119397 diff --git a/CKPTS/BeforeFix_Blip_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_objects/002-GPT-XL/checkpoints/eval_step_18000/batch_1_cfg_7.5_topk_16384.jpg b/CKPTS/BeforeFix_Blip_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_objects/002-GPT-XL/checkpoints/eval_step_18000/batch_1_cfg_7.5_topk_16384.jpg new file mode 100644 index 0000000000000000000000000000000000000000..e517d2c5385b985d7fcec77e1d21a8fa32bfb946 --- /dev/null +++ b/CKPTS/BeforeFix_Blip_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_objects/002-GPT-XL/checkpoints/eval_step_18000/batch_1_cfg_7.5_topk_16384.jpg @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:7999a34197c218f11146adb59a2f2666d5383f1847a5d38c741ccdb446c0294e +size 979075 diff --git a/CKPTS/BeforeFix_Blip_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_objects/002-GPT-XL/checkpoints/eval_step_18000/batch_2_cfg_7.5_topk_16384.jpg b/CKPTS/BeforeFix_Blip_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_objects/002-GPT-XL/checkpoints/eval_step_18000/batch_2_cfg_7.5_topk_16384.jpg new file mode 100644 index 0000000000000000000000000000000000000000..f263b3502a3411a05a81c3934b05b666fe6745e9 --- /dev/null +++ b/CKPTS/BeforeFix_Blip_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_objects/002-GPT-XL/checkpoints/eval_step_18000/batch_2_cfg_7.5_topk_16384.jpg @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:ebc04e3be5a9a0a6ecfb73a2cbca0355acb8b75907d9d703aa2346bdf542b615 +size 975277 diff --git a/CKPTS/BeforeFix_Blip_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_objects/002-GPT-XL/checkpoints/eval_step_2000/batch_0_cfg_7.5_topk_16384.jpg b/CKPTS/BeforeFix_Blip_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_objects/002-GPT-XL/checkpoints/eval_step_2000/batch_0_cfg_7.5_topk_16384.jpg new file mode 100644 index 0000000000000000000000000000000000000000..0910475fff45db8b1ad26aa21cb2023abc995e53 --- /dev/null +++ b/CKPTS/BeforeFix_Blip_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_objects/002-GPT-XL/checkpoints/eval_step_2000/batch_0_cfg_7.5_topk_16384.jpg @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:b24c9eb92330b65aa8ab77fd88dad7342c6111614aa24e4fabd158dac9ce1ff1 +size 1096261 diff --git a/CKPTS/BeforeFix_Blip_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_objects/002-GPT-XL/checkpoints/eval_step_2000/batch_1_cfg_7.5_topk_16384.jpg b/CKPTS/BeforeFix_Blip_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_objects/002-GPT-XL/checkpoints/eval_step_2000/batch_1_cfg_7.5_topk_16384.jpg new file mode 100644 index 0000000000000000000000000000000000000000..5db42699bcddfeac18210ccbf9c3ecb396f884dd --- /dev/null +++ b/CKPTS/BeforeFix_Blip_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_objects/002-GPT-XL/checkpoints/eval_step_2000/batch_1_cfg_7.5_topk_16384.jpg @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:1ff2c6a20265cfb16140db989519f5082afa842a2cb25939da1f1b3d636ab6b0 +size 969643 diff --git a/CKPTS/BeforeFix_Blip_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_objects/002-GPT-XL/checkpoints/eval_step_2000/batch_2_cfg_7.5_topk_16384.jpg b/CKPTS/BeforeFix_Blip_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_objects/002-GPT-XL/checkpoints/eval_step_2000/batch_2_cfg_7.5_topk_16384.jpg new file mode 100644 index 0000000000000000000000000000000000000000..f640e913dbc87fe7c81e36b8e3e909ee20d631df --- /dev/null +++ b/CKPTS/BeforeFix_Blip_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_objects/002-GPT-XL/checkpoints/eval_step_2000/batch_2_cfg_7.5_topk_16384.jpg @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:dc374631ad8bc43d1e4899aa23ec2d0762c4545ca2e453fbe239b06f874b3fc0 +size 987255 diff --git a/CKPTS/BeforeFix_Blip_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_objects/002-GPT-XL/checkpoints/eval_step_20000/batch_0_cfg_7.5_topk_16384.jpg b/CKPTS/BeforeFix_Blip_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_objects/002-GPT-XL/checkpoints/eval_step_20000/batch_0_cfg_7.5_topk_16384.jpg new file mode 100644 index 0000000000000000000000000000000000000000..dd83037647615503304b0726fb4c8162e13f51d1 --- /dev/null +++ b/CKPTS/BeforeFix_Blip_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_objects/002-GPT-XL/checkpoints/eval_step_20000/batch_0_cfg_7.5_topk_16384.jpg @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:f0884a373935abc52539dde9e3fdef85ee33b31ba74ba25235a7715b2b511701 +size 1104553 diff --git a/CKPTS/BeforeFix_Blip_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_objects/002-GPT-XL/checkpoints/eval_step_20000/batch_1_cfg_7.5_topk_16384.jpg b/CKPTS/BeforeFix_Blip_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_objects/002-GPT-XL/checkpoints/eval_step_20000/batch_1_cfg_7.5_topk_16384.jpg new file mode 100644 index 0000000000000000000000000000000000000000..3b65190c01d8a2ce02116a3454c11242550e7593 --- /dev/null +++ b/CKPTS/BeforeFix_Blip_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_objects/002-GPT-XL/checkpoints/eval_step_20000/batch_1_cfg_7.5_topk_16384.jpg @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:4761058e4c19ad3c5fbc0e84df62a96f4fc747a39c735f9caa2061b490d3b2cb +size 960680 diff --git a/CKPTS/BeforeFix_Blip_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_objects/002-GPT-XL/checkpoints/eval_step_20000/batch_2_cfg_7.5_topk_16384.jpg b/CKPTS/BeforeFix_Blip_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_objects/002-GPT-XL/checkpoints/eval_step_20000/batch_2_cfg_7.5_topk_16384.jpg new file mode 100644 index 0000000000000000000000000000000000000000..c0534f4e3601b5e37324a9496ce62a1efd7cd2f0 --- /dev/null +++ b/CKPTS/BeforeFix_Blip_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_objects/002-GPT-XL/checkpoints/eval_step_20000/batch_2_cfg_7.5_topk_16384.jpg @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:6b41f1f9c04e9134b7d9aff22597fb7887204097d6c8940d80bf50f45b99053f +size 975261 diff --git a/CKPTS/BeforeFix_Blip_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_objects/002-GPT-XL/checkpoints/eval_step_22000/batch_0_cfg_7.5_topk_16384.jpg b/CKPTS/BeforeFix_Blip_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_objects/002-GPT-XL/checkpoints/eval_step_22000/batch_0_cfg_7.5_topk_16384.jpg new file mode 100644 index 0000000000000000000000000000000000000000..400906c7d5b768f91da4b117690ca86a79377274 --- /dev/null +++ b/CKPTS/BeforeFix_Blip_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_objects/002-GPT-XL/checkpoints/eval_step_22000/batch_0_cfg_7.5_topk_16384.jpg @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:b4621ac6adfa4ed9d2675caeb90145138679b0a89de9455c18c062abce9994d9 +size 1110609 diff --git a/CKPTS/BeforeFix_Blip_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_objects/002-GPT-XL/checkpoints/eval_step_22000/batch_1_cfg_7.5_topk_16384.jpg b/CKPTS/BeforeFix_Blip_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_objects/002-GPT-XL/checkpoints/eval_step_22000/batch_1_cfg_7.5_topk_16384.jpg new file mode 100644 index 0000000000000000000000000000000000000000..142309a6de412cca95656723fcff2084b7322230 --- /dev/null +++ b/CKPTS/BeforeFix_Blip_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_objects/002-GPT-XL/checkpoints/eval_step_22000/batch_1_cfg_7.5_topk_16384.jpg @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:9ce5fecea03bc7ea2863eb40eb116d01da3dbfe30f295e7e83c78933d47c36d1 +size 978996 diff --git a/CKPTS/BeforeFix_Blip_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_objects/002-GPT-XL/checkpoints/eval_step_22000/batch_2_cfg_7.5_topk_16384.jpg b/CKPTS/BeforeFix_Blip_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_objects/002-GPT-XL/checkpoints/eval_step_22000/batch_2_cfg_7.5_topk_16384.jpg new file mode 100644 index 0000000000000000000000000000000000000000..46d18f1d1909e0b38ede94dd87c07be2cb0b121a --- /dev/null +++ b/CKPTS/BeforeFix_Blip_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_objects/002-GPT-XL/checkpoints/eval_step_22000/batch_2_cfg_7.5_topk_16384.jpg @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:d86caa1fc99d495a3381debe549516992c94e10204a8592036353a163aa376b4 +size 952549 diff --git a/CKPTS/BeforeFix_Blip_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_objects/002-GPT-XL/checkpoints/eval_step_24000/batch_0_cfg_7.5_topk_16384.jpg b/CKPTS/BeforeFix_Blip_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_objects/002-GPT-XL/checkpoints/eval_step_24000/batch_0_cfg_7.5_topk_16384.jpg new file mode 100644 index 0000000000000000000000000000000000000000..7cdf61d12a933698a81b1cad09a595f9ee2dc3d6 --- /dev/null +++ b/CKPTS/BeforeFix_Blip_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_objects/002-GPT-XL/checkpoints/eval_step_24000/batch_0_cfg_7.5_topk_16384.jpg @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:942e1ee36a1352cf59f66bf92d0edb9ca4aeab5ae0ef21af9c6b95a09a2e3140 +size 1112011 diff --git a/CKPTS/BeforeFix_Blip_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_objects/002-GPT-XL/checkpoints/eval_step_24000/batch_1_cfg_7.5_topk_16384.jpg b/CKPTS/BeforeFix_Blip_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_objects/002-GPT-XL/checkpoints/eval_step_24000/batch_1_cfg_7.5_topk_16384.jpg new file mode 100644 index 0000000000000000000000000000000000000000..1264262df2130e30c32f09465131bf600c27052b --- /dev/null +++ b/CKPTS/BeforeFix_Blip_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_objects/002-GPT-XL/checkpoints/eval_step_24000/batch_1_cfg_7.5_topk_16384.jpg @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:74207f82cabf04896a139f1bc243003e3d5c5d00e8f5e1210d8ce3347c3c7513 +size 970319 diff --git a/CKPTS/BeforeFix_Blip_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_objects/002-GPT-XL/checkpoints/eval_step_24000/batch_2_cfg_7.5_topk_16384.jpg b/CKPTS/BeforeFix_Blip_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_objects/002-GPT-XL/checkpoints/eval_step_24000/batch_2_cfg_7.5_topk_16384.jpg new file mode 100644 index 0000000000000000000000000000000000000000..11c31eed8de6f5a3cb227708012c4f61ffcbc413 --- /dev/null +++ b/CKPTS/BeforeFix_Blip_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_objects/002-GPT-XL/checkpoints/eval_step_24000/batch_2_cfg_7.5_topk_16384.jpg @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:9c509aea4441134fd8289ee9cb3fd9fb67fb57defd0821c1884ae8b843fe8951 +size 970185 diff --git a/CKPTS/BeforeFix_Blip_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_objects/002-GPT-XL/checkpoints/eval_step_26000/batch_0_cfg_7.5_topk_16384.jpg b/CKPTS/BeforeFix_Blip_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_objects/002-GPT-XL/checkpoints/eval_step_26000/batch_0_cfg_7.5_topk_16384.jpg new file mode 100644 index 0000000000000000000000000000000000000000..3ec5e36d0a8ff5ac7a55c6eb897278b02c767e73 --- /dev/null +++ b/CKPTS/BeforeFix_Blip_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_objects/002-GPT-XL/checkpoints/eval_step_26000/batch_0_cfg_7.5_topk_16384.jpg @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:279747b2625e366bbdf327973b8574ce6728997ead639b5f1da7eb090736befc +size 1127949 diff --git a/CKPTS/BeforeFix_Blip_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_objects/002-GPT-XL/checkpoints/eval_step_26000/batch_1_cfg_7.5_topk_16384.jpg b/CKPTS/BeforeFix_Blip_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_objects/002-GPT-XL/checkpoints/eval_step_26000/batch_1_cfg_7.5_topk_16384.jpg new file mode 100644 index 0000000000000000000000000000000000000000..68d6fc5cec7be919baa555d31fc427bb0b1f54fc --- /dev/null +++ b/CKPTS/BeforeFix_Blip_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_objects/002-GPT-XL/checkpoints/eval_step_26000/batch_1_cfg_7.5_topk_16384.jpg @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:bb63200c1933c242fcccd471b80757a317dee4fd2321fe31630799656c4756bd +size 977275 diff --git a/CKPTS/BeforeFix_Blip_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_objects/002-GPT-XL/checkpoints/eval_step_26000/batch_2_cfg_7.5_topk_16384.jpg b/CKPTS/BeforeFix_Blip_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_objects/002-GPT-XL/checkpoints/eval_step_26000/batch_2_cfg_7.5_topk_16384.jpg new file mode 100644 index 0000000000000000000000000000000000000000..f1139adee68a9b9c10fc3f1a13b243df2012dff8 --- /dev/null +++ b/CKPTS/BeforeFix_Blip_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_objects/002-GPT-XL/checkpoints/eval_step_26000/batch_2_cfg_7.5_topk_16384.jpg @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:c10715484c2ae739e119dcb845a7b2c306b58f0939d237fc32ab86cf22ccebe0 +size 1013021 diff --git a/CKPTS/BeforeFix_Blip_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_objects/002-GPT-XL/checkpoints/eval_step_28000/batch_0_cfg_7.5_topk_16384.jpg b/CKPTS/BeforeFix_Blip_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_objects/002-GPT-XL/checkpoints/eval_step_28000/batch_0_cfg_7.5_topk_16384.jpg new file mode 100644 index 0000000000000000000000000000000000000000..66e06c3e1bf4030f1200a675f9d0ad8c192d2634 --- /dev/null +++ b/CKPTS/BeforeFix_Blip_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_objects/002-GPT-XL/checkpoints/eval_step_28000/batch_0_cfg_7.5_topk_16384.jpg @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:57972dfd9afb97a153df5932617aced9176f134a1c5b0d8f6a51d7bb1cf21fcc +size 1110219 diff --git a/CKPTS/BeforeFix_Blip_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_objects/002-GPT-XL/checkpoints/eval_step_28000/batch_1_cfg_7.5_topk_16384.jpg b/CKPTS/BeforeFix_Blip_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_objects/002-GPT-XL/checkpoints/eval_step_28000/batch_1_cfg_7.5_topk_16384.jpg new file mode 100644 index 0000000000000000000000000000000000000000..0f9438deb64556cc0afb7dd0e07bb7d7e691c251 --- /dev/null +++ b/CKPTS/BeforeFix_Blip_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_objects/002-GPT-XL/checkpoints/eval_step_28000/batch_1_cfg_7.5_topk_16384.jpg @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:8057e2cf8d3551df0f2047d560caad0f6fadfcb3ebccde5a223d0594d76f76b8 +size 1004847 diff --git a/CKPTS/BeforeFix_Blip_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_objects/002-GPT-XL/checkpoints/eval_step_28000/batch_2_cfg_7.5_topk_16384.jpg b/CKPTS/BeforeFix_Blip_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_objects/002-GPT-XL/checkpoints/eval_step_28000/batch_2_cfg_7.5_topk_16384.jpg new file mode 100644 index 0000000000000000000000000000000000000000..3caf6c0058a85135bf84a354d13e79cd2cbda226 --- /dev/null +++ b/CKPTS/BeforeFix_Blip_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_objects/002-GPT-XL/checkpoints/eval_step_28000/batch_2_cfg_7.5_topk_16384.jpg @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:126dc399ac641096fec6f22adde97ebe647b7c8b1b761639ac01887cc84a60e7 +size 974229 diff --git a/CKPTS/BeforeFix_Blip_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_objects/002-GPT-XL/checkpoints/eval_step_30000/batch_0_cfg_7.5_topk_16384.jpg b/CKPTS/BeforeFix_Blip_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_objects/002-GPT-XL/checkpoints/eval_step_30000/batch_0_cfg_7.5_topk_16384.jpg new file mode 100644 index 0000000000000000000000000000000000000000..9d6967360e7dc3181ceae7510538f4ad7ba29a18 --- /dev/null +++ b/CKPTS/BeforeFix_Blip_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_objects/002-GPT-XL/checkpoints/eval_step_30000/batch_0_cfg_7.5_topk_16384.jpg @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:33c8d2d02e3b595ff72f46da7727c45f61545b6ec8a27dfacb3a1c827351668b +size 1119949 diff --git a/CKPTS/BeforeFix_Blip_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_objects/002-GPT-XL/checkpoints/eval_step_30000/batch_1_cfg_7.5_topk_16384.jpg b/CKPTS/BeforeFix_Blip_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_objects/002-GPT-XL/checkpoints/eval_step_30000/batch_1_cfg_7.5_topk_16384.jpg new file mode 100644 index 0000000000000000000000000000000000000000..c072ad7bb1438be34f465ba7ef4f0b919309e3cf --- /dev/null +++ b/CKPTS/BeforeFix_Blip_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_objects/002-GPT-XL/checkpoints/eval_step_30000/batch_1_cfg_7.5_topk_16384.jpg @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:3361a2c5635b813cc910e5825fc5a70bce313ead3d5fdb1789a4497f1de1c8e9 +size 965826 diff --git a/CKPTS/BeforeFix_Blip_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_objects/002-GPT-XL/checkpoints/eval_step_30000/batch_2_cfg_7.5_topk_16384.jpg b/CKPTS/BeforeFix_Blip_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_objects/002-GPT-XL/checkpoints/eval_step_30000/batch_2_cfg_7.5_topk_16384.jpg new file mode 100644 index 0000000000000000000000000000000000000000..7d525e1cc4fca1bd4e9375890ef1548c915a71a3 --- /dev/null +++ b/CKPTS/BeforeFix_Blip_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_objects/002-GPT-XL/checkpoints/eval_step_30000/batch_2_cfg_7.5_topk_16384.jpg @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:f971b109d9ce0f7c26722bdfa54919202c4f535f150a8be18f5b06b6d39e0ecd +size 992944 diff --git a/CKPTS/BeforeFix_Blip_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_objects/002-GPT-XL/checkpoints/eval_step_32000/batch_0_cfg_7.5_topk_16384.jpg b/CKPTS/BeforeFix_Blip_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_objects/002-GPT-XL/checkpoints/eval_step_32000/batch_0_cfg_7.5_topk_16384.jpg new file mode 100644 index 0000000000000000000000000000000000000000..995263765bcfe15f3d47b0f9b7579e6d9f18f429 --- /dev/null +++ b/CKPTS/BeforeFix_Blip_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_objects/002-GPT-XL/checkpoints/eval_step_32000/batch_0_cfg_7.5_topk_16384.jpg @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:a23f086f795cbe809987fd8dc94ac8528143ee5fd3bf200b8d055f2b0ea15abe +size 1141905 diff --git a/CKPTS/BeforeFix_Blip_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_objects/002-GPT-XL/checkpoints/eval_step_32000/batch_1_cfg_7.5_topk_16384.jpg b/CKPTS/BeforeFix_Blip_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_objects/002-GPT-XL/checkpoints/eval_step_32000/batch_1_cfg_7.5_topk_16384.jpg new file mode 100644 index 0000000000000000000000000000000000000000..40fd46dd6ce6d108ee38d2e330e43e0296129aa3 --- /dev/null +++ b/CKPTS/BeforeFix_Blip_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_objects/002-GPT-XL/checkpoints/eval_step_32000/batch_1_cfg_7.5_topk_16384.jpg @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:4e6c1e84fbd782d3ea8f19871abbc1b94a53921ce0c23708563741e6dca30bdd +size 983161 diff --git a/CKPTS/BeforeFix_Blip_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_objects/002-GPT-XL/checkpoints/eval_step_32000/batch_2_cfg_7.5_topk_16384.jpg b/CKPTS/BeforeFix_Blip_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_objects/002-GPT-XL/checkpoints/eval_step_32000/batch_2_cfg_7.5_topk_16384.jpg new file mode 100644 index 0000000000000000000000000000000000000000..c8dc00ee1caca4fb9d363ce39924c4aca30f7098 --- /dev/null +++ b/CKPTS/BeforeFix_Blip_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_objects/002-GPT-XL/checkpoints/eval_step_32000/batch_2_cfg_7.5_topk_16384.jpg @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:ec11b15af64fd14df02bffe8e4aeab7ee7deb929f1fec36ae42557ceb3f1a061 +size 1012239 diff --git a/CKPTS/BeforeFix_Blip_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_objects/002-GPT-XL/checkpoints/eval_step_34000/batch_0_cfg_7.5_topk_16384.jpg b/CKPTS/BeforeFix_Blip_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_objects/002-GPT-XL/checkpoints/eval_step_34000/batch_0_cfg_7.5_topk_16384.jpg new file mode 100644 index 0000000000000000000000000000000000000000..b42f1c18bea0428252f1c9b45683f7d8b0d30279 --- /dev/null +++ b/CKPTS/BeforeFix_Blip_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_objects/002-GPT-XL/checkpoints/eval_step_34000/batch_0_cfg_7.5_topk_16384.jpg @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:12896fab098e31cec4e73043906cda97cac937306118fdb4d593a206ffe83dde +size 1111113 diff --git a/CKPTS/BeforeFix_Blip_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_objects/002-GPT-XL/checkpoints/eval_step_34000/batch_1_cfg_7.5_topk_16384.jpg b/CKPTS/BeforeFix_Blip_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_objects/002-GPT-XL/checkpoints/eval_step_34000/batch_1_cfg_7.5_topk_16384.jpg new file mode 100644 index 0000000000000000000000000000000000000000..084c1811f86bff7f261bd79424af15bbd9f1e074 --- /dev/null +++ b/CKPTS/BeforeFix_Blip_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_objects/002-GPT-XL/checkpoints/eval_step_34000/batch_1_cfg_7.5_topk_16384.jpg @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:5d812b120464db898a259b7bdd5c2d524ec959060944a30e2b6b7dfc563d5dcf +size 972458 diff --git a/CKPTS/BeforeFix_Blip_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_objects/002-GPT-XL/checkpoints/eval_step_34000/batch_2_cfg_7.5_topk_16384.jpg b/CKPTS/BeforeFix_Blip_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_objects/002-GPT-XL/checkpoints/eval_step_34000/batch_2_cfg_7.5_topk_16384.jpg new file mode 100644 index 0000000000000000000000000000000000000000..01fd6ae0ccaabeba89bfb75178ac869e67bdbbb9 --- /dev/null +++ b/CKPTS/BeforeFix_Blip_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_objects/002-GPT-XL/checkpoints/eval_step_34000/batch_2_cfg_7.5_topk_16384.jpg @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:2c935959f2c26d9454bdce916f49e020e27209a952adb04b64ce8bad192e1ae3 +size 1013214 diff --git a/CKPTS/BeforeFix_Blip_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_objects/002-GPT-XL/checkpoints/eval_step_36000/batch_0_cfg_7.5_topk_16384.jpg b/CKPTS/BeforeFix_Blip_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_objects/002-GPT-XL/checkpoints/eval_step_36000/batch_0_cfg_7.5_topk_16384.jpg new file mode 100644 index 0000000000000000000000000000000000000000..03e7c27bc0c5c3d7faf9e887cc6b09100c9f624b --- /dev/null +++ b/CKPTS/BeforeFix_Blip_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_objects/002-GPT-XL/checkpoints/eval_step_36000/batch_0_cfg_7.5_topk_16384.jpg @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:3f503a829ffdf21ecf854abfd0554aae8cadc974aea520d5e5ffe21350be349b +size 1141887 diff --git a/CKPTS/BeforeFix_Blip_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_objects/002-GPT-XL/checkpoints/eval_step_36000/batch_1_cfg_7.5_topk_16384.jpg b/CKPTS/BeforeFix_Blip_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_objects/002-GPT-XL/checkpoints/eval_step_36000/batch_1_cfg_7.5_topk_16384.jpg new file mode 100644 index 0000000000000000000000000000000000000000..2fc7e68b905b23cb6b41652a44b4fdebb3669f70 --- /dev/null +++ b/CKPTS/BeforeFix_Blip_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_objects/002-GPT-XL/checkpoints/eval_step_36000/batch_1_cfg_7.5_topk_16384.jpg @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:32f4f4aafc159b3b38b13ec611f7d7a545fbe05588846e42441772dd10db8275 +size 978880 diff --git a/CKPTS/BeforeFix_Blip_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_objects/002-GPT-XL/checkpoints/eval_step_36000/batch_2_cfg_7.5_topk_16384.jpg b/CKPTS/BeforeFix_Blip_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_objects/002-GPT-XL/checkpoints/eval_step_36000/batch_2_cfg_7.5_topk_16384.jpg new file mode 100644 index 0000000000000000000000000000000000000000..57fb50a52040bc631b09cb7750673e6945dce956 --- /dev/null +++ b/CKPTS/BeforeFix_Blip_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_objects/002-GPT-XL/checkpoints/eval_step_36000/batch_2_cfg_7.5_topk_16384.jpg @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:cbbe333b6a6e05d73103f3307a2c0d4a5a71d965a2a7a3f545ccd78972245fe3 +size 979909 diff --git a/CKPTS/BeforeFix_Blip_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_objects/002-GPT-XL/checkpoints/eval_step_38000/batch_0_cfg_7.5_topk_16384.jpg b/CKPTS/BeforeFix_Blip_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_objects/002-GPT-XL/checkpoints/eval_step_38000/batch_0_cfg_7.5_topk_16384.jpg new file mode 100644 index 0000000000000000000000000000000000000000..f76c37df285ec6fba258031b0b1913f470decf77 --- /dev/null +++ b/CKPTS/BeforeFix_Blip_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_objects/002-GPT-XL/checkpoints/eval_step_38000/batch_0_cfg_7.5_topk_16384.jpg @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:e845848ed8fe5ca86c88126117e18dad3c19b611067267872de474f241230b45 +size 1112912 diff --git a/CKPTS/BeforeFix_Blip_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_objects/002-GPT-XL/checkpoints/eval_step_38000/batch_1_cfg_7.5_topk_16384.jpg b/CKPTS/BeforeFix_Blip_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_objects/002-GPT-XL/checkpoints/eval_step_38000/batch_1_cfg_7.5_topk_16384.jpg new file mode 100644 index 0000000000000000000000000000000000000000..f15c388d6429ebb6d2b133d1360ae83e35272c17 --- /dev/null +++ b/CKPTS/BeforeFix_Blip_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_objects/002-GPT-XL/checkpoints/eval_step_38000/batch_1_cfg_7.5_topk_16384.jpg @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:6c5a502733f6351a86d31b1e51961f4196f3453bdb9b421856882f8bbb9c3154 +size 990288 diff --git a/CKPTS/BeforeFix_Blip_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_objects/002-GPT-XL/checkpoints/eval_step_38000/batch_2_cfg_7.5_topk_16384.jpg b/CKPTS/BeforeFix_Blip_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_objects/002-GPT-XL/checkpoints/eval_step_38000/batch_2_cfg_7.5_topk_16384.jpg new file mode 100644 index 0000000000000000000000000000000000000000..e23152271794513e6e0bf4d84f20c53fa8e265f4 --- /dev/null +++ b/CKPTS/BeforeFix_Blip_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_objects/002-GPT-XL/checkpoints/eval_step_38000/batch_2_cfg_7.5_topk_16384.jpg @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:34960cdf2e1a585aa35fdc266abd7a51d253230ed740c52c4bc81db68d1f88f4 +size 1014064 diff --git a/CKPTS/BeforeFix_Blip_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_objects/002-GPT-XL/checkpoints/eval_step_4000/batch_0_cfg_7.5_topk_16384.jpg b/CKPTS/BeforeFix_Blip_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_objects/002-GPT-XL/checkpoints/eval_step_4000/batch_0_cfg_7.5_topk_16384.jpg new file mode 100644 index 0000000000000000000000000000000000000000..de27f541a7b925414ed767beb22989fd7282e172 --- /dev/null +++ b/CKPTS/BeforeFix_Blip_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_objects/002-GPT-XL/checkpoints/eval_step_4000/batch_0_cfg_7.5_topk_16384.jpg @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:a1bbc89b061f6de242c282613b58f7ab817bc9d68bf90e7a77d2da63fc3c2eb6 +size 1095551 diff --git a/CKPTS/BeforeFix_Blip_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_objects/002-GPT-XL/checkpoints/eval_step_4000/batch_1_cfg_7.5_topk_16384.jpg b/CKPTS/BeforeFix_Blip_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_objects/002-GPT-XL/checkpoints/eval_step_4000/batch_1_cfg_7.5_topk_16384.jpg new file mode 100644 index 0000000000000000000000000000000000000000..7a1cd02f604133040cb151b8cad91d0adbc1d523 --- /dev/null +++ b/CKPTS/BeforeFix_Blip_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_objects/002-GPT-XL/checkpoints/eval_step_4000/batch_1_cfg_7.5_topk_16384.jpg @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:890a6579606190ad7300a91d544bb7229b51fc71160a7724722e5d59c5d63c3f +size 987172 diff --git a/CKPTS/BeforeFix_Blip_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_objects/002-GPT-XL/checkpoints/eval_step_4000/batch_2_cfg_7.5_topk_16384.jpg b/CKPTS/BeforeFix_Blip_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_objects/002-GPT-XL/checkpoints/eval_step_4000/batch_2_cfg_7.5_topk_16384.jpg new file mode 100644 index 0000000000000000000000000000000000000000..f4274f3de97c9d8411a813fcbd2251d92ff59d82 --- /dev/null +++ b/CKPTS/BeforeFix_Blip_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_objects/002-GPT-XL/checkpoints/eval_step_4000/batch_2_cfg_7.5_topk_16384.jpg @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:63c04186e5e87e7947f530fc8ac77ffaa21f3fcb2183516c297273aa85ea4295 +size 966627 diff --git a/CKPTS/BeforeFix_Blip_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_objects/002-GPT-XL/checkpoints/eval_step_40000/batch_0_cfg_7.5_topk_16384.jpg b/CKPTS/BeforeFix_Blip_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_objects/002-GPT-XL/checkpoints/eval_step_40000/batch_0_cfg_7.5_topk_16384.jpg new file mode 100644 index 0000000000000000000000000000000000000000..e5751f8482a68bc27cd411f28166db6692271426 --- /dev/null +++ b/CKPTS/BeforeFix_Blip_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_objects/002-GPT-XL/checkpoints/eval_step_40000/batch_0_cfg_7.5_topk_16384.jpg @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:f826b7233414535993b3abc7b536a57a6d1e6752375f500aab86997b0afc2394 +size 1104679 diff --git a/CKPTS/BeforeFix_Blip_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_objects/002-GPT-XL/checkpoints/eval_step_40000/batch_1_cfg_7.5_topk_16384.jpg b/CKPTS/BeforeFix_Blip_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_objects/002-GPT-XL/checkpoints/eval_step_40000/batch_1_cfg_7.5_topk_16384.jpg new file mode 100644 index 0000000000000000000000000000000000000000..4bdfb2373497c2381efe220b655dc531e1049d79 --- /dev/null +++ b/CKPTS/BeforeFix_Blip_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_objects/002-GPT-XL/checkpoints/eval_step_40000/batch_1_cfg_7.5_topk_16384.jpg @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:34777e661cb7e3ca0d181ca3cf1c52f4c866d47884e13f217634286dd21e89ea +size 976021 diff --git a/CKPTS/BeforeFix_Blip_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_objects/002-GPT-XL/checkpoints/eval_step_40000/batch_2_cfg_7.5_topk_16384.jpg b/CKPTS/BeforeFix_Blip_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_objects/002-GPT-XL/checkpoints/eval_step_40000/batch_2_cfg_7.5_topk_16384.jpg new file mode 100644 index 0000000000000000000000000000000000000000..f7922e2882921155023cfc4c4ab912004373f0bd --- /dev/null +++ b/CKPTS/BeforeFix_Blip_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_objects/002-GPT-XL/checkpoints/eval_step_40000/batch_2_cfg_7.5_topk_16384.jpg @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:9fae8a24da9bb8ab0c31899ab4fb0aa0deaf6594c6354adb64e188df6652995d +size 988973 diff --git a/CKPTS/BeforeFix_Blip_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_objects/002-GPT-XL/checkpoints/eval_step_42000/batch_0_cfg_7.5_topk_16384.jpg b/CKPTS/BeforeFix_Blip_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_objects/002-GPT-XL/checkpoints/eval_step_42000/batch_0_cfg_7.5_topk_16384.jpg new file mode 100644 index 0000000000000000000000000000000000000000..9e9fb1704bf38e0b5e453b0aa31d790f666d2b5d --- /dev/null +++ b/CKPTS/BeforeFix_Blip_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_objects/002-GPT-XL/checkpoints/eval_step_42000/batch_0_cfg_7.5_topk_16384.jpg @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:0e257fde40d250d25e9f250c30732ae58b457371872d134439c7360ce3dae28a +size 1123349 diff --git a/CKPTS/BeforeFix_Blip_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_objects/002-GPT-XL/checkpoints/eval_step_42000/batch_1_cfg_7.5_topk_16384.jpg b/CKPTS/BeforeFix_Blip_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_objects/002-GPT-XL/checkpoints/eval_step_42000/batch_1_cfg_7.5_topk_16384.jpg new file mode 100644 index 0000000000000000000000000000000000000000..d4118e890e02946ad6931933e045b90c515989f2 --- /dev/null +++ b/CKPTS/BeforeFix_Blip_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_objects/002-GPT-XL/checkpoints/eval_step_42000/batch_1_cfg_7.5_topk_16384.jpg @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:d29bb167eec2391f6a31198e40e77669ebe59f55ed1226d52984ba9fb7301e92 +size 947308 diff --git a/CKPTS/BeforeFix_Blip_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_objects/002-GPT-XL/checkpoints/eval_step_42000/batch_2_cfg_7.5_topk_16384.jpg b/CKPTS/BeforeFix_Blip_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_objects/002-GPT-XL/checkpoints/eval_step_42000/batch_2_cfg_7.5_topk_16384.jpg new file mode 100644 index 0000000000000000000000000000000000000000..a20d975cf95b0da147e8c75cb28ca429fd90d296 --- /dev/null +++ b/CKPTS/BeforeFix_Blip_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_objects/002-GPT-XL/checkpoints/eval_step_42000/batch_2_cfg_7.5_topk_16384.jpg @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:f8e3b05db85facb16ca3c63a9206965a5a5fc5424d6d29ce52a774f6c7d8aa65 +size 1009420 diff --git a/CKPTS/BeforeFix_Blip_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_objects/002-GPT-XL/checkpoints/eval_step_44000/batch_0_cfg_7.5_topk_16384.jpg b/CKPTS/BeforeFix_Blip_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_objects/002-GPT-XL/checkpoints/eval_step_44000/batch_0_cfg_7.5_topk_16384.jpg new file mode 100644 index 0000000000000000000000000000000000000000..73152b0de365ebac1f06bff18eca8dffdb8f18a2 --- /dev/null +++ b/CKPTS/BeforeFix_Blip_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_objects/002-GPT-XL/checkpoints/eval_step_44000/batch_0_cfg_7.5_topk_16384.jpg @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:fefa106ef7de6b4fb96b785436e1eec5a08598b1d6c27b6670448214fc677a09 +size 1138084 diff --git a/CKPTS/BeforeFix_Blip_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_objects/002-GPT-XL/checkpoints/eval_step_44000/batch_1_cfg_7.5_topk_16384.jpg b/CKPTS/BeforeFix_Blip_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_objects/002-GPT-XL/checkpoints/eval_step_44000/batch_1_cfg_7.5_topk_16384.jpg new file mode 100644 index 0000000000000000000000000000000000000000..2424be881ab1b134026a1eebfd953e21753feb3e --- /dev/null +++ b/CKPTS/BeforeFix_Blip_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_objects/002-GPT-XL/checkpoints/eval_step_44000/batch_1_cfg_7.5_topk_16384.jpg @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:f6b1257380588185b35d923cdfa01d20aebfc7222821b5d79e0ee9117d57f8ae +size 973008 diff --git a/CKPTS/BeforeFix_Blip_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_objects/002-GPT-XL/checkpoints/eval_step_44000/batch_2_cfg_7.5_topk_16384.jpg b/CKPTS/BeforeFix_Blip_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_objects/002-GPT-XL/checkpoints/eval_step_44000/batch_2_cfg_7.5_topk_16384.jpg new file mode 100644 index 0000000000000000000000000000000000000000..34fceb294bf6ccf780ef180a3bf4c33575f26b85 --- /dev/null +++ b/CKPTS/BeforeFix_Blip_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_objects/002-GPT-XL/checkpoints/eval_step_44000/batch_2_cfg_7.5_topk_16384.jpg @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:411fcb2bab7fb538fa0d853caf611292a2f3801ce0c3f68ffeedf10a78ddfc70 +size 984058 diff --git a/CKPTS/BeforeFix_Blip_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_objects/002-GPT-XL/checkpoints/eval_step_46000/batch_0_cfg_7.5_topk_16384.jpg b/CKPTS/BeforeFix_Blip_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_objects/002-GPT-XL/checkpoints/eval_step_46000/batch_0_cfg_7.5_topk_16384.jpg new file mode 100644 index 0000000000000000000000000000000000000000..bfc4adc7676fda460d5f800412051feee9580bbb --- /dev/null +++ b/CKPTS/BeforeFix_Blip_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_objects/002-GPT-XL/checkpoints/eval_step_46000/batch_0_cfg_7.5_topk_16384.jpg @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:511b20b306cb88680d825d5be99a7d0ce6b81a798a61f30d4e6f3c8e418cbcae +size 1111713 diff --git a/CKPTS/BeforeFix_Blip_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_objects/002-GPT-XL/checkpoints/eval_step_46000/batch_1_cfg_7.5_topk_16384.jpg b/CKPTS/BeforeFix_Blip_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_objects/002-GPT-XL/checkpoints/eval_step_46000/batch_1_cfg_7.5_topk_16384.jpg new file mode 100644 index 0000000000000000000000000000000000000000..85eae03b6586f4332f01db27469721bba248e997 --- /dev/null +++ b/CKPTS/BeforeFix_Blip_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_objects/002-GPT-XL/checkpoints/eval_step_46000/batch_1_cfg_7.5_topk_16384.jpg @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:6fd0c4766aa71068e82ed06f42298f147c9023b6daec9cc1eeeec59edfe8856a +size 971772 diff --git a/CKPTS/BeforeFix_Blip_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_objects/002-GPT-XL/checkpoints/eval_step_46000/batch_2_cfg_7.5_topk_16384.jpg b/CKPTS/BeforeFix_Blip_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_objects/002-GPT-XL/checkpoints/eval_step_46000/batch_2_cfg_7.5_topk_16384.jpg new file mode 100644 index 0000000000000000000000000000000000000000..5e52f237c6413a9d9d93e18a1d9add41b9416769 --- /dev/null +++ b/CKPTS/BeforeFix_Blip_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_objects/002-GPT-XL/checkpoints/eval_step_46000/batch_2_cfg_7.5_topk_16384.jpg @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:8c89a8f7fc084847d0484e64cd35d28d77d74d8c49d42fb526e5017704e5e278 +size 986955 diff --git a/CKPTS/BeforeFix_Blip_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_objects/002-GPT-XL/checkpoints/eval_step_48000/batch_0_cfg_7.5_topk_16384.jpg b/CKPTS/BeforeFix_Blip_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_objects/002-GPT-XL/checkpoints/eval_step_48000/batch_0_cfg_7.5_topk_16384.jpg new file mode 100644 index 0000000000000000000000000000000000000000..1b5deae6814547b51863a1f1958f80ea202d32c1 --- /dev/null +++ b/CKPTS/BeforeFix_Blip_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_objects/002-GPT-XL/checkpoints/eval_step_48000/batch_0_cfg_7.5_topk_16384.jpg @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:6f8878845fa22f67e796153ec0c249d87b8179870b4d3d7df5b77a2c66aa0111 +size 1119214 diff --git a/CKPTS/BeforeFix_Blip_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_objects/002-GPT-XL/checkpoints/eval_step_48000/batch_1_cfg_7.5_topk_16384.jpg b/CKPTS/BeforeFix_Blip_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_objects/002-GPT-XL/checkpoints/eval_step_48000/batch_1_cfg_7.5_topk_16384.jpg new file mode 100644 index 0000000000000000000000000000000000000000..19907aec092ef696a0452f1b0b03d932025194c0 --- /dev/null +++ b/CKPTS/BeforeFix_Blip_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_objects/002-GPT-XL/checkpoints/eval_step_48000/batch_1_cfg_7.5_topk_16384.jpg @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:340e6ba6a0e7e35549d2f6ae1fe155d13a44ede723b7ee6d3194994f0591bf98 +size 976451 diff --git a/CKPTS/BeforeFix_Blip_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_objects/002-GPT-XL/checkpoints/eval_step_48000/batch_2_cfg_7.5_topk_16384.jpg b/CKPTS/BeforeFix_Blip_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_objects/002-GPT-XL/checkpoints/eval_step_48000/batch_2_cfg_7.5_topk_16384.jpg new file mode 100644 index 0000000000000000000000000000000000000000..5454188d7cab28a3761f10fc0f8c5adfcc83722f --- /dev/null +++ b/CKPTS/BeforeFix_Blip_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_objects/002-GPT-XL/checkpoints/eval_step_48000/batch_2_cfg_7.5_topk_16384.jpg @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:44d8f93125ba5a642a6480eead1bc5e61d2918e052c1af320b056ced3f3c0423 +size 1008627 diff --git a/CKPTS/BeforeFix_Blip_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_objects/002-GPT-XL/checkpoints/eval_step_50000/batch_0_cfg_7.5_topk_16384.jpg b/CKPTS/BeforeFix_Blip_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_objects/002-GPT-XL/checkpoints/eval_step_50000/batch_0_cfg_7.5_topk_16384.jpg new file mode 100644 index 0000000000000000000000000000000000000000..be955de8f47e2034f1ad682ba9efe19891772cdd --- /dev/null +++ b/CKPTS/BeforeFix_Blip_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_objects/002-GPT-XL/checkpoints/eval_step_50000/batch_0_cfg_7.5_topk_16384.jpg @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:f588241740a437145d56e6303bd31aaa4975880c6516a9fc022cc7bd6605818e +size 1117189 diff --git a/CKPTS/BeforeFix_Blip_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_objects/002-GPT-XL/checkpoints/eval_step_50000/batch_1_cfg_7.5_topk_16384.jpg b/CKPTS/BeforeFix_Blip_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_objects/002-GPT-XL/checkpoints/eval_step_50000/batch_1_cfg_7.5_topk_16384.jpg new file mode 100644 index 0000000000000000000000000000000000000000..0d149bc264363abc4969635f99e8f020ada14f21 --- /dev/null +++ b/CKPTS/BeforeFix_Blip_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_objects/002-GPT-XL/checkpoints/eval_step_50000/batch_1_cfg_7.5_topk_16384.jpg @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:725118285a5481a6028f63d18377fe6c9752ee199b9694e56dcd183816b342f6 +size 965330 diff --git a/CKPTS/BeforeFix_Blip_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_objects/002-GPT-XL/checkpoints/eval_step_50000/batch_2_cfg_7.5_topk_16384.jpg b/CKPTS/BeforeFix_Blip_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_objects/002-GPT-XL/checkpoints/eval_step_50000/batch_2_cfg_7.5_topk_16384.jpg new file mode 100644 index 0000000000000000000000000000000000000000..2df9af05556895c0515ba74337633aed9560d1ea --- /dev/null +++ b/CKPTS/BeforeFix_Blip_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_objects/002-GPT-XL/checkpoints/eval_step_50000/batch_2_cfg_7.5_topk_16384.jpg @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:0c20a8daa2b8e2036594cec912270daf04bd70c13859df065029cf3cbe1c14dd +size 992235 diff --git a/CKPTS/BeforeFix_Blip_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_objects/002-GPT-XL/checkpoints/eval_step_52000/batch_0_cfg_7.5_topk_16384.jpg b/CKPTS/BeforeFix_Blip_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_objects/002-GPT-XL/checkpoints/eval_step_52000/batch_0_cfg_7.5_topk_16384.jpg new file mode 100644 index 0000000000000000000000000000000000000000..f9c73af14d769c0480ef37e2bc70e9e23fc330d2 --- /dev/null +++ b/CKPTS/BeforeFix_Blip_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_objects/002-GPT-XL/checkpoints/eval_step_52000/batch_0_cfg_7.5_topk_16384.jpg @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:416770a20ab58dd5cf1bcd6b866aa954d607e6becf7251f2bf4dcba8ad1dab2a +size 1110543 diff --git a/CKPTS/BeforeFix_Blip_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_objects/002-GPT-XL/checkpoints/eval_step_52000/batch_1_cfg_7.5_topk_16384.jpg b/CKPTS/BeforeFix_Blip_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_objects/002-GPT-XL/checkpoints/eval_step_52000/batch_1_cfg_7.5_topk_16384.jpg new file mode 100644 index 0000000000000000000000000000000000000000..9e81bbd7e15209b23c7d260dc091eed5110c00d7 --- /dev/null +++ b/CKPTS/BeforeFix_Blip_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_objects/002-GPT-XL/checkpoints/eval_step_52000/batch_1_cfg_7.5_topk_16384.jpg @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:fb712da8f0b0296e7c3ddf47ce4284895590634d2b50505785b97c26235f5973 +size 960732 diff --git a/CKPTS/BeforeFix_Blip_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_objects/002-GPT-XL/checkpoints/eval_step_52000/batch_2_cfg_7.5_topk_16384.jpg b/CKPTS/BeforeFix_Blip_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_objects/002-GPT-XL/checkpoints/eval_step_52000/batch_2_cfg_7.5_topk_16384.jpg new file mode 100644 index 0000000000000000000000000000000000000000..a21c1c7a31d844dbaf83a7cc1e472a8366055d56 --- /dev/null +++ b/CKPTS/BeforeFix_Blip_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_objects/002-GPT-XL/checkpoints/eval_step_52000/batch_2_cfg_7.5_topk_16384.jpg @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:d32c2101857859b6b347512d3c092811baea0f8be0eb512788809010abfad37f +size 1023770 diff --git a/CKPTS/BeforeFix_Blip_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_objects/002-GPT-XL/checkpoints/eval_step_54000/batch_0_cfg_7.5_topk_16384.jpg b/CKPTS/BeforeFix_Blip_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_objects/002-GPT-XL/checkpoints/eval_step_54000/batch_0_cfg_7.5_topk_16384.jpg new file mode 100644 index 0000000000000000000000000000000000000000..d452e1864a5c7fc7308d6d988dcba278d828b441 --- /dev/null +++ b/CKPTS/BeforeFix_Blip_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_objects/002-GPT-XL/checkpoints/eval_step_54000/batch_0_cfg_7.5_topk_16384.jpg @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:3d39100724053665233f048849ba31c3f334ea6e1e554e11b2aaf1f953e59b1b +size 1126715 diff --git a/CKPTS/BeforeFix_Blip_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_objects/002-GPT-XL/checkpoints/eval_step_54000/batch_1_cfg_7.5_topk_16384.jpg b/CKPTS/BeforeFix_Blip_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_objects/002-GPT-XL/checkpoints/eval_step_54000/batch_1_cfg_7.5_topk_16384.jpg new file mode 100644 index 0000000000000000000000000000000000000000..55efbfd452bc1b1a7d46c76fcc25ea64403fc5ea --- /dev/null +++ b/CKPTS/BeforeFix_Blip_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_objects/002-GPT-XL/checkpoints/eval_step_54000/batch_1_cfg_7.5_topk_16384.jpg @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:0f33c02e1835e1bce95d5720663851277db881bd5670a1c0734d7f179f43395a +size 961758 diff --git a/CKPTS/BeforeFix_Blip_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_objects/002-GPT-XL/checkpoints/eval_step_54000/batch_2_cfg_7.5_topk_16384.jpg b/CKPTS/BeforeFix_Blip_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_objects/002-GPT-XL/checkpoints/eval_step_54000/batch_2_cfg_7.5_topk_16384.jpg new file mode 100644 index 0000000000000000000000000000000000000000..dc6236457ef0c385b2d0b41373ceea8192d2b0dd --- /dev/null +++ b/CKPTS/BeforeFix_Blip_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_objects/002-GPT-XL/checkpoints/eval_step_54000/batch_2_cfg_7.5_topk_16384.jpg @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:f558fa947e9d6a18542e680be3be44055d1e7d2d2d9f85e1851ee4533569c92e +size 1005909 diff --git a/CKPTS/BeforeFix_Blip_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_objects/002-GPT-XL/checkpoints/eval_step_56000/batch_0_cfg_7.5_topk_16384.jpg b/CKPTS/BeforeFix_Blip_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_objects/002-GPT-XL/checkpoints/eval_step_56000/batch_0_cfg_7.5_topk_16384.jpg new file mode 100644 index 0000000000000000000000000000000000000000..e2eb257b1ed6fc0891a001545a6b3673a1d6ff71 --- /dev/null +++ b/CKPTS/BeforeFix_Blip_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_objects/002-GPT-XL/checkpoints/eval_step_56000/batch_0_cfg_7.5_topk_16384.jpg @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:4780d4bc284c47822d3720ef3ef3b12e496b5362847cbb0edc4ff053c6e5e883 +size 1131780 diff --git a/CKPTS/BeforeFix_Blip_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_objects/002-GPT-XL/checkpoints/eval_step_56000/batch_1_cfg_7.5_topk_16384.jpg b/CKPTS/BeforeFix_Blip_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_objects/002-GPT-XL/checkpoints/eval_step_56000/batch_1_cfg_7.5_topk_16384.jpg new file mode 100644 index 0000000000000000000000000000000000000000..f06d6d34ff88573c2955c408f15743afe3aca7f2 --- /dev/null +++ b/CKPTS/BeforeFix_Blip_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_objects/002-GPT-XL/checkpoints/eval_step_56000/batch_1_cfg_7.5_topk_16384.jpg @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:588883202892ec2c22a39e0ec722157507d889ec7b43d26abec2c257d24e37da +size 972228 diff --git a/CKPTS/BeforeFix_Blip_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_objects/002-GPT-XL/checkpoints/eval_step_56000/batch_2_cfg_7.5_topk_16384.jpg b/CKPTS/BeforeFix_Blip_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_objects/002-GPT-XL/checkpoints/eval_step_56000/batch_2_cfg_7.5_topk_16384.jpg new file mode 100644 index 0000000000000000000000000000000000000000..4ba71ebea921e4a51a132c2dbd3cabd732108aaf --- /dev/null +++ b/CKPTS/BeforeFix_Blip_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_objects/002-GPT-XL/checkpoints/eval_step_56000/batch_2_cfg_7.5_topk_16384.jpg @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:513b780c62f5258fe719fde3ec41841eac0fb9b7e85ed59405e2bf10e3d7ed98 +size 1015817 diff --git a/CKPTS/BeforeFix_Blip_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_objects/002-GPT-XL/checkpoints/eval_step_58000/batch_0_cfg_7.5_topk_16384.jpg b/CKPTS/BeforeFix_Blip_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_objects/002-GPT-XL/checkpoints/eval_step_58000/batch_0_cfg_7.5_topk_16384.jpg new file mode 100644 index 0000000000000000000000000000000000000000..19e15759349ae9ea492a85797081f4b54db8c006 --- /dev/null +++ b/CKPTS/BeforeFix_Blip_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_objects/002-GPT-XL/checkpoints/eval_step_58000/batch_0_cfg_7.5_topk_16384.jpg @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:1eb49920a8fbc66fa8220843e505b27b626eeb0404d4b0bebcee5f212947e975 +size 1111578 diff --git a/CKPTS/BeforeFix_Blip_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_objects/002-GPT-XL/checkpoints/eval_step_58000/batch_1_cfg_7.5_topk_16384.jpg b/CKPTS/BeforeFix_Blip_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_objects/002-GPT-XL/checkpoints/eval_step_58000/batch_1_cfg_7.5_topk_16384.jpg new file mode 100644 index 0000000000000000000000000000000000000000..1db0b110f72e54f9187784fda788b83d7bbfa328 --- /dev/null +++ b/CKPTS/BeforeFix_Blip_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_objects/002-GPT-XL/checkpoints/eval_step_58000/batch_1_cfg_7.5_topk_16384.jpg @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:6c176833f7b5e0dd30a531e49a4d1a1b8f62f22607095ba5542ac5cf6274b440 +size 982440 diff --git a/CKPTS/BeforeFix_Blip_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_objects/002-GPT-XL/checkpoints/eval_step_58000/batch_2_cfg_7.5_topk_16384.jpg b/CKPTS/BeforeFix_Blip_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_objects/002-GPT-XL/checkpoints/eval_step_58000/batch_2_cfg_7.5_topk_16384.jpg new file mode 100644 index 0000000000000000000000000000000000000000..2f154543f7870a0477b757f0480a3bb702f42214 --- /dev/null +++ b/CKPTS/BeforeFix_Blip_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_objects/002-GPT-XL/checkpoints/eval_step_58000/batch_2_cfg_7.5_topk_16384.jpg @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:9d6a2a1da0dddc5c340f3fafe30ec92c9b49adeb353054b8b6deee15e1ad7a98 +size 991552 diff --git a/CKPTS/BeforeFix_Blip_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_objects/002-GPT-XL/checkpoints/eval_step_6000/batch_0_cfg_7.5_topk_16384.jpg b/CKPTS/BeforeFix_Blip_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_objects/002-GPT-XL/checkpoints/eval_step_6000/batch_0_cfg_7.5_topk_16384.jpg new file mode 100644 index 0000000000000000000000000000000000000000..d1e105791e19b5a2b02fc0a012f087b6d80bdfff --- /dev/null +++ b/CKPTS/BeforeFix_Blip_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_objects/002-GPT-XL/checkpoints/eval_step_6000/batch_0_cfg_7.5_topk_16384.jpg @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:09310b2903350c54d417a59e399dc2561d82ca544580c71ad53dd6e5d2789613 +size 1090333 diff --git a/CKPTS/BeforeFix_Blip_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_objects/002-GPT-XL/checkpoints/eval_step_6000/batch_1_cfg_7.5_topk_16384.jpg b/CKPTS/BeforeFix_Blip_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_objects/002-GPT-XL/checkpoints/eval_step_6000/batch_1_cfg_7.5_topk_16384.jpg new file mode 100644 index 0000000000000000000000000000000000000000..a1acfc03eaee06e3def02360b8020831a3d7b0e4 --- /dev/null +++ b/CKPTS/BeforeFix_Blip_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_objects/002-GPT-XL/checkpoints/eval_step_6000/batch_1_cfg_7.5_topk_16384.jpg @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:44c6ca13b995b09bd8d8d2ed53ef24cc73fbdd62685d43e52013894829613f54 +size 955774 diff --git a/CKPTS/BeforeFix_Blip_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_objects/002-GPT-XL/checkpoints/eval_step_6000/batch_2_cfg_7.5_topk_16384.jpg b/CKPTS/BeforeFix_Blip_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_objects/002-GPT-XL/checkpoints/eval_step_6000/batch_2_cfg_7.5_topk_16384.jpg new file mode 100644 index 0000000000000000000000000000000000000000..870aeb984ea82ef5e86b4c3b51d6b164126a97d0 --- /dev/null +++ b/CKPTS/BeforeFix_Blip_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_objects/002-GPT-XL/checkpoints/eval_step_6000/batch_2_cfg_7.5_topk_16384.jpg @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:c1a7fd3c97d39b3ffa13862649e8756ed20e498094a892ea8801561ea00ac584 +size 967216 diff --git a/CKPTS/BeforeFix_Blip_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_objects/002-GPT-XL/checkpoints/eval_step_60000/batch_0_cfg_7.5_topk_16384.jpg b/CKPTS/BeforeFix_Blip_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_objects/002-GPT-XL/checkpoints/eval_step_60000/batch_0_cfg_7.5_topk_16384.jpg new file mode 100644 index 0000000000000000000000000000000000000000..8502f796f4ddf92e3c7881e76851a6a2c459109b --- /dev/null +++ b/CKPTS/BeforeFix_Blip_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_objects/002-GPT-XL/checkpoints/eval_step_60000/batch_0_cfg_7.5_topk_16384.jpg @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:efcc9b1ebf0f3d6ae1d4be2320b53f44933fd86dab937c7d72fa383dfaf82062 +size 1132592 diff --git a/CKPTS/BeforeFix_Blip_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_objects/002-GPT-XL/checkpoints/eval_step_60000/batch_1_cfg_7.5_topk_16384.jpg b/CKPTS/BeforeFix_Blip_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_objects/002-GPT-XL/checkpoints/eval_step_60000/batch_1_cfg_7.5_topk_16384.jpg new file mode 100644 index 0000000000000000000000000000000000000000..bac646181ca25f403c9d57ae69b458951595fa18 --- /dev/null +++ b/CKPTS/BeforeFix_Blip_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_objects/002-GPT-XL/checkpoints/eval_step_60000/batch_1_cfg_7.5_topk_16384.jpg @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:9cb83d4fe57d554b1b1ab2c232c602f28ad4f93181ef4c297969b360ec8bee00 +size 1000422 diff --git a/CKPTS/BeforeFix_Blip_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_objects/002-GPT-XL/checkpoints/eval_step_60000/batch_2_cfg_7.5_topk_16384.jpg b/CKPTS/BeforeFix_Blip_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_objects/002-GPT-XL/checkpoints/eval_step_60000/batch_2_cfg_7.5_topk_16384.jpg new file mode 100644 index 0000000000000000000000000000000000000000..1eef88b7dafd6485561771a57814334d693c6b12 --- /dev/null +++ b/CKPTS/BeforeFix_Blip_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_objects/002-GPT-XL/checkpoints/eval_step_60000/batch_2_cfg_7.5_topk_16384.jpg @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:2e358ba31f45d3e3bc88235345e227fc54c1bdfe6abbd50a7123dc888f491bdb +size 1025350 diff --git a/CKPTS/BeforeFix_Blip_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_objects/002-GPT-XL/checkpoints/eval_step_62000/batch_0_cfg_7.5_topk_16384.jpg b/CKPTS/BeforeFix_Blip_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_objects/002-GPT-XL/checkpoints/eval_step_62000/batch_0_cfg_7.5_topk_16384.jpg new file mode 100644 index 0000000000000000000000000000000000000000..d45f41c4d32c3da16cc56e4cbce342873a61937a --- /dev/null +++ b/CKPTS/BeforeFix_Blip_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_objects/002-GPT-XL/checkpoints/eval_step_62000/batch_0_cfg_7.5_topk_16384.jpg @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:1d446c6c34a93c57ac8d4b55db1f1ba697ce672c9cadbad4385b1f5cdea84381 +size 1122361 diff --git a/CKPTS/BeforeFix_Blip_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_objects/002-GPT-XL/checkpoints/eval_step_62000/batch_1_cfg_7.5_topk_16384.jpg b/CKPTS/BeforeFix_Blip_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_objects/002-GPT-XL/checkpoints/eval_step_62000/batch_1_cfg_7.5_topk_16384.jpg new file mode 100644 index 0000000000000000000000000000000000000000..38cb93129dca83c8ab0de2f612e46363a3a247d0 --- /dev/null +++ b/CKPTS/BeforeFix_Blip_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_objects/002-GPT-XL/checkpoints/eval_step_62000/batch_1_cfg_7.5_topk_16384.jpg @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:7fd84bede373258b5ba2cb426cdcf23e254b232cc9b62c44be6c61e1c8af235e +size 948622 diff --git a/CKPTS/BeforeFix_Blip_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_objects/002-GPT-XL/checkpoints/eval_step_62000/batch_2_cfg_7.5_topk_16384.jpg b/CKPTS/BeforeFix_Blip_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_objects/002-GPT-XL/checkpoints/eval_step_62000/batch_2_cfg_7.5_topk_16384.jpg new file mode 100644 index 0000000000000000000000000000000000000000..18dcbe8208b7b8c510ed1388eb728f221272e09d --- /dev/null +++ b/CKPTS/BeforeFix_Blip_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_objects/002-GPT-XL/checkpoints/eval_step_62000/batch_2_cfg_7.5_topk_16384.jpg @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:99ca65ceb97bc5c114051832a94e0e1de935baf872629083a47aa13d6f283ff3 +size 994659 diff --git a/CKPTS/BeforeFix_Blip_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_objects/002-GPT-XL/checkpoints/eval_step_64000/batch_0_cfg_7.5_topk_16384.jpg b/CKPTS/BeforeFix_Blip_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_objects/002-GPT-XL/checkpoints/eval_step_64000/batch_0_cfg_7.5_topk_16384.jpg new file mode 100644 index 0000000000000000000000000000000000000000..fa88bb58692ae1ca9ec48f647fbe74f9f7d1e1bb --- /dev/null +++ b/CKPTS/BeforeFix_Blip_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_objects/002-GPT-XL/checkpoints/eval_step_64000/batch_0_cfg_7.5_topk_16384.jpg @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:bd7eba9986933053582016d40f5d79d5db1623d4292489232cb75e72b48eafd2 +size 1125549 diff --git a/CKPTS/BeforeFix_Blip_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_objects/002-GPT-XL/checkpoints/eval_step_64000/batch_1_cfg_7.5_topk_16384.jpg b/CKPTS/BeforeFix_Blip_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_objects/002-GPT-XL/checkpoints/eval_step_64000/batch_1_cfg_7.5_topk_16384.jpg new file mode 100644 index 0000000000000000000000000000000000000000..7f2782ed178cbbc86e66e5684b633737cd12cffc --- /dev/null +++ b/CKPTS/BeforeFix_Blip_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_objects/002-GPT-XL/checkpoints/eval_step_64000/batch_1_cfg_7.5_topk_16384.jpg @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:ecbb3320950dda255fe93593a9860b28606d38cdf199ff8a56c7a242090ffc33 +size 968098 diff --git a/CKPTS/BeforeFix_Blip_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_objects/002-GPT-XL/checkpoints/eval_step_64000/batch_2_cfg_7.5_topk_16384.jpg b/CKPTS/BeforeFix_Blip_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_objects/002-GPT-XL/checkpoints/eval_step_64000/batch_2_cfg_7.5_topk_16384.jpg new file mode 100644 index 0000000000000000000000000000000000000000..4d254d414778539215899ed7eca2592bc0ba6203 --- /dev/null +++ b/CKPTS/BeforeFix_Blip_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_objects/002-GPT-XL/checkpoints/eval_step_64000/batch_2_cfg_7.5_topk_16384.jpg @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:1b282bebfcda18631f3c8b7c0b1d45642f2a5da5e925f5f1d080dc3cf57a3053 +size 1006292 diff --git a/CKPTS/BeforeFix_Blip_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_objects/002-GPT-XL/checkpoints/eval_step_66000/batch_0_cfg_7.5_topk_16384.jpg b/CKPTS/BeforeFix_Blip_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_objects/002-GPT-XL/checkpoints/eval_step_66000/batch_0_cfg_7.5_topk_16384.jpg new file mode 100644 index 0000000000000000000000000000000000000000..484ce0d53b3d2baf44b5a1a095d9b261b1a32ce1 --- /dev/null +++ b/CKPTS/BeforeFix_Blip_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_objects/002-GPT-XL/checkpoints/eval_step_66000/batch_0_cfg_7.5_topk_16384.jpg @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:0c6e18ca9bcb10515da35dc2096545ac79ab5ef6b1eb26712ed84c77c0d1e79e +size 1124817 diff --git a/CKPTS/BeforeFix_Blip_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_objects/002-GPT-XL/checkpoints/eval_step_66000/batch_1_cfg_7.5_topk_16384.jpg b/CKPTS/BeforeFix_Blip_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_objects/002-GPT-XL/checkpoints/eval_step_66000/batch_1_cfg_7.5_topk_16384.jpg new file mode 100644 index 0000000000000000000000000000000000000000..842f9a89acdd2161ecca0c99091e2c74588ce3c4 --- /dev/null +++ b/CKPTS/BeforeFix_Blip_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_objects/002-GPT-XL/checkpoints/eval_step_66000/batch_1_cfg_7.5_topk_16384.jpg @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:4d2f93e9bfee95a50062c6f63af5964a6bca31ec55f00c47034755d795a45987 +size 995615 diff --git a/CKPTS/BeforeFix_Blip_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_objects/002-GPT-XL/checkpoints/eval_step_66000/batch_2_cfg_7.5_topk_16384.jpg b/CKPTS/BeforeFix_Blip_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_objects/002-GPT-XL/checkpoints/eval_step_66000/batch_2_cfg_7.5_topk_16384.jpg new file mode 100644 index 0000000000000000000000000000000000000000..1f4afeba1221623d97c77c3e14a27a03a19449b2 --- /dev/null +++ b/CKPTS/BeforeFix_Blip_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_objects/002-GPT-XL/checkpoints/eval_step_66000/batch_2_cfg_7.5_topk_16384.jpg @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:af475883b020814cf0cc0909ea0c2ff3f81ddcac869a32faa09634e0b66d6bdc +size 1003043 diff --git a/CKPTS/BeforeFix_Blip_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_objects/002-GPT-XL/checkpoints/eval_step_68000/batch_0_cfg_7.5_topk_16384.jpg b/CKPTS/BeforeFix_Blip_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_objects/002-GPT-XL/checkpoints/eval_step_68000/batch_0_cfg_7.5_topk_16384.jpg new file mode 100644 index 0000000000000000000000000000000000000000..d7d39eba1034bab08dbc8f0f5460268c9c832863 --- /dev/null +++ b/CKPTS/BeforeFix_Blip_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_objects/002-GPT-XL/checkpoints/eval_step_68000/batch_0_cfg_7.5_topk_16384.jpg @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:3467c26e9ac093d6e0f3d7aed9b89add8353f0580abb37878451364b34adfe63 +size 1123164 diff --git a/CKPTS/BeforeFix_Blip_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_objects/002-GPT-XL/checkpoints/eval_step_68000/batch_1_cfg_7.5_topk_16384.jpg b/CKPTS/BeforeFix_Blip_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_objects/002-GPT-XL/checkpoints/eval_step_68000/batch_1_cfg_7.5_topk_16384.jpg new file mode 100644 index 0000000000000000000000000000000000000000..7f62c535ce055ca7a07e6e240c1ad9b085aab561 --- /dev/null +++ b/CKPTS/BeforeFix_Blip_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_objects/002-GPT-XL/checkpoints/eval_step_68000/batch_1_cfg_7.5_topk_16384.jpg @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:b992dd0934d77536863d0b403eb25321b635c7367ea570358af41df3d0f4b5f0 +size 965217 diff --git a/CKPTS/BeforeFix_Blip_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_objects/002-GPT-XL/checkpoints/eval_step_68000/batch_2_cfg_7.5_topk_16384.jpg b/CKPTS/BeforeFix_Blip_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_objects/002-GPT-XL/checkpoints/eval_step_68000/batch_2_cfg_7.5_topk_16384.jpg new file mode 100644 index 0000000000000000000000000000000000000000..e353cdf2b2a092d002c6fcd91c4ba5191ad2f76c --- /dev/null +++ b/CKPTS/BeforeFix_Blip_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_objects/002-GPT-XL/checkpoints/eval_step_68000/batch_2_cfg_7.5_topk_16384.jpg @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:4ef571e11220d7b7c424a8b190997ad35725e33a7ced6dcb27e62c2d59f63819 +size 1003304 diff --git a/CKPTS/BeforeFix_Blip_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_objects/002-GPT-XL/checkpoints/eval_step_70000/batch_0_cfg_7.5_topk_16384.jpg b/CKPTS/BeforeFix_Blip_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_objects/002-GPT-XL/checkpoints/eval_step_70000/batch_0_cfg_7.5_topk_16384.jpg new file mode 100644 index 0000000000000000000000000000000000000000..da177151860bb0e8214d5addd172d113b47f26dd --- /dev/null +++ b/CKPTS/BeforeFix_Blip_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_objects/002-GPT-XL/checkpoints/eval_step_70000/batch_0_cfg_7.5_topk_16384.jpg @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:066972d0871b7db3a9e6bc55affb79d34e7d27d921ed42858b773aca0dd3c7a6 +size 1120867 diff --git a/CKPTS/BeforeFix_Blip_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_objects/002-GPT-XL/checkpoints/eval_step_70000/batch_1_cfg_7.5_topk_16384.jpg b/CKPTS/BeforeFix_Blip_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_objects/002-GPT-XL/checkpoints/eval_step_70000/batch_1_cfg_7.5_topk_16384.jpg new file mode 100644 index 0000000000000000000000000000000000000000..4160f7dacee283e23efe9c21f2c9bc8c9288afb8 --- /dev/null +++ b/CKPTS/BeforeFix_Blip_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_objects/002-GPT-XL/checkpoints/eval_step_70000/batch_1_cfg_7.5_topk_16384.jpg @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:e87ed0d82e87471d282d359dc2a8599df3f2fe98001c379ca037d78c0556ca86 +size 969321 diff --git a/CKPTS/BeforeFix_Blip_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_objects/002-GPT-XL/checkpoints/eval_step_70000/batch_2_cfg_7.5_topk_16384.jpg b/CKPTS/BeforeFix_Blip_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_objects/002-GPT-XL/checkpoints/eval_step_70000/batch_2_cfg_7.5_topk_16384.jpg new file mode 100644 index 0000000000000000000000000000000000000000..ccb16fd0ded52367ca402cf314fccbe1c8508412 --- /dev/null +++ b/CKPTS/BeforeFix_Blip_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_objects/002-GPT-XL/checkpoints/eval_step_70000/batch_2_cfg_7.5_topk_16384.jpg @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:726aea9cd3b3b5b81a94ce961d2d781b3f475348e11409218c96fa531f46c436 +size 1001996 diff --git a/CKPTS/BeforeFix_Blip_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_objects/002-GPT-XL/checkpoints/eval_step_72000/batch_0_cfg_7.5_topk_16384.jpg b/CKPTS/BeforeFix_Blip_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_objects/002-GPT-XL/checkpoints/eval_step_72000/batch_0_cfg_7.5_topk_16384.jpg new file mode 100644 index 0000000000000000000000000000000000000000..f5420724d236b307d9e21a6e00542f4a5f264362 --- /dev/null +++ b/CKPTS/BeforeFix_Blip_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_objects/002-GPT-XL/checkpoints/eval_step_72000/batch_0_cfg_7.5_topk_16384.jpg @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:0eacb3e6639dcb014a4559445175a7f65da1f72af6e92a73e011f8f689b196f4 +size 1101405 diff --git a/CKPTS/BeforeFix_Blip_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_objects/002-GPT-XL/checkpoints/eval_step_72000/batch_1_cfg_7.5_topk_16384.jpg b/CKPTS/BeforeFix_Blip_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_objects/002-GPT-XL/checkpoints/eval_step_72000/batch_1_cfg_7.5_topk_16384.jpg new file mode 100644 index 0000000000000000000000000000000000000000..4c62596becea3ccf5ae8bded4fff05f2fef61c31 --- /dev/null +++ b/CKPTS/BeforeFix_Blip_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_objects/002-GPT-XL/checkpoints/eval_step_72000/batch_1_cfg_7.5_topk_16384.jpg @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:1f0417c92f9384c6041c73bb17dc47e4e0358f890a4d61da998bb2f5c0eb36e8 +size 965611 diff --git a/CKPTS/BeforeFix_Blip_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_objects/002-GPT-XL/checkpoints/eval_step_72000/batch_2_cfg_7.5_topk_16384.jpg b/CKPTS/BeforeFix_Blip_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_objects/002-GPT-XL/checkpoints/eval_step_72000/batch_2_cfg_7.5_topk_16384.jpg new file mode 100644 index 0000000000000000000000000000000000000000..eeec885126627baacf4d576c235ce96a394a9614 --- /dev/null +++ b/CKPTS/BeforeFix_Blip_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_objects/002-GPT-XL/checkpoints/eval_step_72000/batch_2_cfg_7.5_topk_16384.jpg @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:50f132c9c5ca422f8a9dd9c1167a3277ce72ed84b695c532a5be2ec6c65a437a +size 979044 diff --git a/CKPTS/BeforeFix_Blip_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_objects/002-GPT-XL/checkpoints/eval_step_74000/batch_0_cfg_7.5_topk_16384.jpg b/CKPTS/BeforeFix_Blip_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_objects/002-GPT-XL/checkpoints/eval_step_74000/batch_0_cfg_7.5_topk_16384.jpg new file mode 100644 index 0000000000000000000000000000000000000000..4895012fb24260e388f94def7871052a742bd30c --- /dev/null +++ b/CKPTS/BeforeFix_Blip_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_objects/002-GPT-XL/checkpoints/eval_step_74000/batch_0_cfg_7.5_topk_16384.jpg @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:55956ea5258c25c3ef468fca125e148ec52e65e1728be81584ec116d126a5ef0 +size 1135172 diff --git a/CKPTS/BeforeFix_Blip_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_objects/002-GPT-XL/checkpoints/eval_step_74000/batch_1_cfg_7.5_topk_16384.jpg b/CKPTS/BeforeFix_Blip_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_objects/002-GPT-XL/checkpoints/eval_step_74000/batch_1_cfg_7.5_topk_16384.jpg new file mode 100644 index 0000000000000000000000000000000000000000..bcdf21ee6d26969bccf130c1b5d7a946f43d32b4 --- /dev/null +++ b/CKPTS/BeforeFix_Blip_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_objects/002-GPT-XL/checkpoints/eval_step_74000/batch_1_cfg_7.5_topk_16384.jpg @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:bbb19a4f391a5dadd40738628662f28803b790ba486b0da569e8d4d752edc1c9 +size 965439 diff --git a/CKPTS/BeforeFix_Blip_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_objects/002-GPT-XL/checkpoints/eval_step_74000/batch_2_cfg_7.5_topk_16384.jpg b/CKPTS/BeforeFix_Blip_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_objects/002-GPT-XL/checkpoints/eval_step_74000/batch_2_cfg_7.5_topk_16384.jpg new file mode 100644 index 0000000000000000000000000000000000000000..1b7527ab137657b18ab21fde2ae7e02744618b2a --- /dev/null +++ b/CKPTS/BeforeFix_Blip_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_objects/002-GPT-XL/checkpoints/eval_step_74000/batch_2_cfg_7.5_topk_16384.jpg @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:a225c621eac3c499dd63c84aa2ca120cc8685c063f499f904b4a1ddb6c03f9ef +size 1020461 diff --git a/CKPTS/BeforeFix_Blip_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_objects/002-GPT-XL/checkpoints/eval_step_76000/batch_0_cfg_7.5_topk_16384.jpg b/CKPTS/BeforeFix_Blip_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_objects/002-GPT-XL/checkpoints/eval_step_76000/batch_0_cfg_7.5_topk_16384.jpg new file mode 100644 index 0000000000000000000000000000000000000000..83ebf974a69a8200e69eb54a14544d412b9021c2 --- /dev/null +++ b/CKPTS/BeforeFix_Blip_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_objects/002-GPT-XL/checkpoints/eval_step_76000/batch_0_cfg_7.5_topk_16384.jpg @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:c8acb6c2f29b66bcb106b2a77e3a954cda98ccbe486bab663020fedb9765f6f5 +size 1135067 diff --git a/CKPTS/BeforeFix_Blip_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_objects/002-GPT-XL/checkpoints/eval_step_76000/batch_1_cfg_7.5_topk_16384.jpg b/CKPTS/BeforeFix_Blip_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_objects/002-GPT-XL/checkpoints/eval_step_76000/batch_1_cfg_7.5_topk_16384.jpg new file mode 100644 index 0000000000000000000000000000000000000000..5bc64871c94441537037f563a238ee517c897f2f --- /dev/null +++ b/CKPTS/BeforeFix_Blip_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_objects/002-GPT-XL/checkpoints/eval_step_76000/batch_1_cfg_7.5_topk_16384.jpg @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:5a10ad1ef3e3052ae54ad261e7567357f4208f3e75e8cbd11b5aa71955765b72 +size 987428 diff --git a/CKPTS/BeforeFix_Blip_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_objects/002-GPT-XL/checkpoints/eval_step_76000/batch_2_cfg_7.5_topk_16384.jpg b/CKPTS/BeforeFix_Blip_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_objects/002-GPT-XL/checkpoints/eval_step_76000/batch_2_cfg_7.5_topk_16384.jpg new file mode 100644 index 0000000000000000000000000000000000000000..0f88207dc3afc76b6fac0637ef13e5ee0eb65427 --- /dev/null +++ b/CKPTS/BeforeFix_Blip_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_objects/002-GPT-XL/checkpoints/eval_step_76000/batch_2_cfg_7.5_topk_16384.jpg @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:c84f084b25ab88a3f406e943c571e8c9d3cda1322d7e4e3d1f0f3e6970f383e3 +size 970262 diff --git a/CKPTS/BeforeFix_Blip_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_objects/002-GPT-XL/checkpoints/eval_step_78000/batch_0_cfg_7.5_topk_16384.jpg b/CKPTS/BeforeFix_Blip_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_objects/002-GPT-XL/checkpoints/eval_step_78000/batch_0_cfg_7.5_topk_16384.jpg new file mode 100644 index 0000000000000000000000000000000000000000..a7a224739e6cbd5ed87861c719bcc4414bd5d154 --- /dev/null +++ b/CKPTS/BeforeFix_Blip_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_objects/002-GPT-XL/checkpoints/eval_step_78000/batch_0_cfg_7.5_topk_16384.jpg @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:9e30014144c6dced337b347b2b690f5afa491b0874c0dfd89999f99784a8d2f2 +size 1134307 diff --git a/CKPTS/BeforeFix_Blip_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_objects/002-GPT-XL/checkpoints/eval_step_78000/batch_1_cfg_7.5_topk_16384.jpg b/CKPTS/BeforeFix_Blip_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_objects/002-GPT-XL/checkpoints/eval_step_78000/batch_1_cfg_7.5_topk_16384.jpg new file mode 100644 index 0000000000000000000000000000000000000000..9ff72832a501baf2cbac3b72e6f1a2c43d50c92b --- /dev/null +++ b/CKPTS/BeforeFix_Blip_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_objects/002-GPT-XL/checkpoints/eval_step_78000/batch_1_cfg_7.5_topk_16384.jpg @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:196560ac4eb03e447124cafff0e7f6c7a8a8211306aa9b8bcf53e63fbc05cdf6 +size 988808 diff --git a/CKPTS/BeforeFix_Blip_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_objects/002-GPT-XL/checkpoints/eval_step_78000/batch_2_cfg_7.5_topk_16384.jpg b/CKPTS/BeforeFix_Blip_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_objects/002-GPT-XL/checkpoints/eval_step_78000/batch_2_cfg_7.5_topk_16384.jpg new file mode 100644 index 0000000000000000000000000000000000000000..256dfbeea09867989e66bf4d8b18ab47108cd611 --- /dev/null +++ b/CKPTS/BeforeFix_Blip_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_objects/002-GPT-XL/checkpoints/eval_step_78000/batch_2_cfg_7.5_topk_16384.jpg @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:157a7c237d1a76307d469fa5a3fa95c07802b2f63f01df4a9b578a81f1505013 +size 998006 diff --git a/CKPTS/BeforeFix_Blip_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_objects/002-GPT-XL/checkpoints/eval_step_8000/batch_0_cfg_7.5_topk_16384.jpg b/CKPTS/BeforeFix_Blip_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_objects/002-GPT-XL/checkpoints/eval_step_8000/batch_0_cfg_7.5_topk_16384.jpg new file mode 100644 index 0000000000000000000000000000000000000000..ed9c4fcbec37dbda9eaa6afa09795007c2956836 --- /dev/null +++ b/CKPTS/BeforeFix_Blip_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_objects/002-GPT-XL/checkpoints/eval_step_8000/batch_0_cfg_7.5_topk_16384.jpg @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:5c6916704df8c6bdc7eb8eb117e3a2aff2177a5e6b40ba60705e3bd969ae1954 +size 1097095 diff --git a/CKPTS/BeforeFix_Blip_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_objects/002-GPT-XL/checkpoints/eval_step_8000/batch_1_cfg_7.5_topk_16384.jpg b/CKPTS/BeforeFix_Blip_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_objects/002-GPT-XL/checkpoints/eval_step_8000/batch_1_cfg_7.5_topk_16384.jpg new file mode 100644 index 0000000000000000000000000000000000000000..7951d4308e80076c94dceaceb951bc3b7c5e2685 --- /dev/null +++ b/CKPTS/BeforeFix_Blip_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_objects/002-GPT-XL/checkpoints/eval_step_8000/batch_1_cfg_7.5_topk_16384.jpg @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:d786365313d74df19161c32eaf627f0e5ff4f4cf88c4699e096967eedf21a21d +size 985356 diff --git a/CKPTS/BeforeFix_Blip_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_objects/002-GPT-XL/checkpoints/eval_step_8000/batch_2_cfg_7.5_topk_16384.jpg b/CKPTS/BeforeFix_Blip_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_objects/002-GPT-XL/checkpoints/eval_step_8000/batch_2_cfg_7.5_topk_16384.jpg new file mode 100644 index 0000000000000000000000000000000000000000..bb7915592f7d3518824be12b423dcb332f8acb92 --- /dev/null +++ b/CKPTS/BeforeFix_Blip_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_objects/002-GPT-XL/checkpoints/eval_step_8000/batch_2_cfg_7.5_topk_16384.jpg @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:66405b0ff83b52dcae7a706414a00fd5440964a10976ce9b4dd6ef4cf02dcf56 +size 991947 diff --git a/CKPTS/BeforeFix_Blip_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_objects/002-GPT-XL/checkpoints/eval_step_80000/batch_0_cfg_7.5_topk_16384.jpg b/CKPTS/BeforeFix_Blip_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_objects/002-GPT-XL/checkpoints/eval_step_80000/batch_0_cfg_7.5_topk_16384.jpg new file mode 100644 index 0000000000000000000000000000000000000000..5eb0689e37a67a74fd4182d11116d6c75665d058 --- /dev/null +++ b/CKPTS/BeforeFix_Blip_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_objects/002-GPT-XL/checkpoints/eval_step_80000/batch_0_cfg_7.5_topk_16384.jpg @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:0c60da50dad1fd274ac94bb44c3a2bf1c8c9d50a3b33b166e6c9c756b913b3a4 +size 1110062 diff --git a/CKPTS/BeforeFix_Blip_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_objects/002-GPT-XL/checkpoints/eval_step_80000/batch_1_cfg_7.5_topk_16384.jpg b/CKPTS/BeforeFix_Blip_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_objects/002-GPT-XL/checkpoints/eval_step_80000/batch_1_cfg_7.5_topk_16384.jpg new file mode 100644 index 0000000000000000000000000000000000000000..b1a4e0d4c389810c49cfdd43b2656846b87fc5cb --- /dev/null +++ b/CKPTS/BeforeFix_Blip_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_objects/002-GPT-XL/checkpoints/eval_step_80000/batch_1_cfg_7.5_topk_16384.jpg @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:d2943484c9509dca7af4826081df700939f0dedf824b3b7666ca60bb1bddd907 +size 993480 diff --git a/CKPTS/BeforeFix_Blip_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_objects/002-GPT-XL/checkpoints/eval_step_80000/batch_2_cfg_7.5_topk_16384.jpg b/CKPTS/BeforeFix_Blip_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_objects/002-GPT-XL/checkpoints/eval_step_80000/batch_2_cfg_7.5_topk_16384.jpg new file mode 100644 index 0000000000000000000000000000000000000000..bdeb3238b06d470cff4096227411aa1d629d4efa --- /dev/null +++ b/CKPTS/BeforeFix_Blip_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_objects/002-GPT-XL/checkpoints/eval_step_80000/batch_2_cfg_7.5_topk_16384.jpg @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:4d632ac5662743df013f48288f0e1c682e090a5bb693806ea674f46a15f55d86 +size 997002 diff --git a/CKPTS/BeforeFix_Blip_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_objects/002-GPT-XL/log.txt b/CKPTS/BeforeFix_Blip_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_objects/002-GPT-XL/log.txt new file mode 100644 index 0000000000000000000000000000000000000000..4476ee7867ea13cbdb26fc5eee6639fef8d43e95 --- /dev/null +++ b/CKPTS/BeforeFix_Blip_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_objects/002-GPT-XL/log.txt @@ -0,0 +1,3354 @@ +[2025-04-22 01:44:54] Experiment directory created at checkpoint/BeforeFix_Blip_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_objects/002-GPT-XL +[2025-04-22 01:44:54] Namespace(data_path='/tmp/haozhezhao/MLLMG/jsonl_data/multiobjects_molom_imagenet_flux_qwen_midsource_gen_2_2m_trained.jsonl', cloud_save_path='/tmp/haozhezhao/MLLMG/checkpoint', no_local_save=False, vq_model='VQ-16', vq_ckpt='/tmp/haozhezhao/model/llamagen_t2i/vq_ds16_t2i.pt', codebook_size=16384, codebook_embed_dim=8, gpt_model='GPT-XL', gpt_ckpt='/tmp/haozhezhao/MLLMG/checkpoint/BLIP2Trainall_just_segment_pretrain_stage3_dreambench_recap_Subject200k_filtered_t2i_flux400k_200kmid_recovery_150k_extract_150k_100_fluxseg_50samseg_trainall_1e4/000-GPT-XL/checkpoints/0010000.pt', gpt_type='t2i', vocab_size=16384, cls_token_num=512, dropout_p=0.1, token_dropout_p=0.1, drop_path=0.0, no_compile=False, results_dir='checkpoint/BeforeFix_Blip_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_objects', dataset='ti2i', image_size=512, downsample_size=16, num_classes=1000, epochs=2, lr=5e-05, weight_decay=0.05, beta1=0.9, beta2=0.95, max_grad_norm=1.0, global_batch_size=56, global_seed=0, num_workers=4, log_every=25, ckpt_every=2000, gradient_accumulation_steps=4, mixed_precision='bf16', val_data_path='/tmp/haozhezhao/MLLMG/jsonl_data/multiobjects_molom_imagenet_flux_qwen_midsource_gen_val.jsonl', use_vision_tower=True, model_name_or_path='/tmp/haozhezhao/model/instructblip-flan-t5-xl', image_place_holder='', processor_path=None, do_eval=True, max_eval_samples=200, train_text_encoder=True, no_left_padding=False, cfg_scale=7.5, top_k=16384, temperature=0.9, top_p=1.0, eval_steps=2000, project_name='llamagen_ti2i', load_from_checkpoint='/tmp/haozhezhao/model/llamagen_t2i/t2i_XL_stage2_512.pt', warmup=0.05, lr_decay_style='cosine', lr_decay_ratio=0.1, train_iters=500000, class_dropout_prob=0.1, with_image_only=False, image_only_rate=0.1, stage2=False, subject_driven=True, load_subject_embedding='/tmp/haozhezhao/MLLMG/subject_embedding.bin', reference_data_path='/tmp/haozhezhao/MLLMG/cc12m_reference_tunnel.jsonl', multimodal_encoder='blip', do_recovery=True, no_replace=False, resume=False, dreambench_eval=False, find_unused_parameters=True, load_visual_encoder=False, continue_stage1=False, replace_subject=False, train_all=True, save_total_limit=2, load_language_projection=None, mm_vision_tower='openai/clip-vit-large-patch14', load_fixed_llamagen=False, unfreeze_output=False, fix='gpt', rank=0, world_size=8, gpu=0, dist_url='env://', distributed=True, dist_backend='nccl') +[2025-04-22 01:44:54] Starting rank=0, seed=0, world_size=8. +[2025-04-22 01:46:35] GPT Parameters: 3,174,417,408 +[2025-04-22 01:46:35] num decayed parameter tensors: 637, with 3,173,270,016 parameters +[2025-04-22 01:46:35] num non-decayed parameter tensors: 644, with 1,147,392 parameters +[2025-04-22 01:46:35] using fused AdamW: True +[2025-04-22 01:47:04] Dataset contains 2,290,583 images +[2025-04-22 01:47:04] Train iters 81806 , warmup 4090.3, len of loader 40903 +[2025-04-22 01:47:25] ### LOAD pretraining weights from checkpoint: /tmp/haozhezhao/MLLMG/checkpoint/BLIP2Trainall_just_segment_pretrain_stage3_dreambench_recap_Subject200k_filtered_t2i_flux400k_200kmid_recovery_150k_extract_150k_100_fluxseg_50samseg_trainall_1e4/000-GPT-XL/checkpoints/0010000.pt +[2025-04-22 01:47:25] Initial state: steps=0, epochs=0 +[2025-04-22 01:47:25] compiling the model... (may take several minutes) +[2025-04-22 01:47:26] freeze the vit +[2025-04-22 01:47:26] ***** total param is 3174417408 ***** +[2025-04-22 01:47:26] ***** total trained param is 2188465152 ***** +[2025-04-22 01:47:30] Training for 2 epochs... +[2025-04-22 01:47:30] Beginning epoch 0... +[2025-04-22 02:18:26] (step=0000025) Train Loss: 6.5206, Train Steps/Sec: 0.01 +[2025-04-22 02:27:25] (step=0000050) Train Loss: 6.4822, Train Steps/Sec: 0.05 +[2025-04-22 02:28:17] (step=0000075) Train Loss: 6.5165, Train Steps/Sec: 0.48 +[2025-04-22 02:30:22] (step=0000100) Train Loss: 6.5071, Train Steps/Sec: 0.20 +[2025-04-22 02:30:51] (step=0000125) Train Loss: 6.4721, Train Steps/Sec: 0.85 +[2025-04-22 02:31:21] (step=0000150) Train Loss: 6.4755, Train Steps/Sec: 0.85 +[2025-04-22 02:31:50] (step=0000175) Train Loss: 6.5462, Train Steps/Sec: 0.85 +[2025-04-22 02:32:20] (step=0000200) Train Loss: 6.5129, Train Steps/Sec: 0.84 +[2025-04-22 02:32:50] (step=0000225) Train Loss: 6.4623, Train Steps/Sec: 0.85 +[2025-04-22 02:33:19] (step=0000250) Train Loss: 6.4735, Train Steps/Sec: 0.85 +[2025-04-22 02:33:48] (step=0000275) Train Loss: 6.4566, Train Steps/Sec: 0.85 +[2025-04-22 02:34:18] (step=0000300) Train Loss: 6.5080, Train Steps/Sec: 0.84 +[2025-04-22 02:34:48] (step=0000325) Train Loss: 6.4814, Train Steps/Sec: 0.85 +[2025-04-22 02:35:17] (step=0000350) Train Loss: 6.4743, Train Steps/Sec: 0.84 +[2025-04-22 02:35:46] (step=0000375) Train Loss: 6.4496, Train Steps/Sec: 0.86 +[2025-04-22 02:36:16] (step=0000400) Train Loss: 6.5116, Train Steps/Sec: 0.85 +[2025-04-22 02:36:46] (step=0000425) Train Loss: 6.4402, Train Steps/Sec: 0.85 +[2025-04-22 02:37:15] (step=0000450) Train Loss: 6.4814, Train Steps/Sec: 0.85 +[2025-04-22 02:37:44] (step=0000475) Train Loss: 6.5230, Train Steps/Sec: 0.85 +[2025-04-22 02:38:14] (step=0000500) Train Loss: 6.4677, Train Steps/Sec: 0.84 +[2025-04-22 02:38:43] (step=0000525) Train Loss: 6.4826, Train Steps/Sec: 0.85 +[2025-04-22 02:39:13] (step=0000550) Train Loss: 6.4752, Train Steps/Sec: 0.85 +[2025-04-22 02:39:42] (step=0000575) Train Loss: 6.4816, Train Steps/Sec: 0.85 +[2025-04-22 02:40:12] (step=0000600) Train Loss: 6.5100, Train Steps/Sec: 0.84 +[2025-04-22 02:40:42] (step=0000625) Train Loss: 6.4690, Train Steps/Sec: 0.85 +[2025-04-22 02:41:11] (step=0000650) Train Loss: 6.4683, Train Steps/Sec: 0.85 +[2025-04-22 02:41:41] (step=0000675) Train Loss: 6.4567, Train Steps/Sec: 0.85 +[2025-04-22 02:42:10] (step=0000700) Train Loss: 6.4508, Train Steps/Sec: 0.85 +[2025-04-22 02:42:40] (step=0000725) Train Loss: 6.4758, Train Steps/Sec: 0.85 +[2025-04-22 02:43:09] (step=0000750) Train Loss: 6.4601, Train Steps/Sec: 0.85 +[2025-04-22 02:43:38] (step=0000775) Train Loss: 6.4447, Train Steps/Sec: 0.85 +[2025-04-22 02:44:08] (step=0000800) Train Loss: 6.4656, Train Steps/Sec: 0.84 +[2025-04-22 02:44:38] (step=0000825) Train Loss: 6.4648, Train Steps/Sec: 0.85 +[2025-04-22 02:45:07] (step=0000850) Train Loss: 6.4878, Train Steps/Sec: 0.85 +[2025-04-22 02:45:37] (step=0000875) Train Loss: 6.4463, Train Steps/Sec: 0.84 +[2025-04-22 02:46:07] (step=0000900) Train Loss: 6.5097, Train Steps/Sec: 0.84 +[2025-04-22 02:46:36] (step=0000925) Train Loss: 6.4664, Train Steps/Sec: 0.85 +[2025-04-22 02:47:06] (step=0000950) Train Loss: 6.4590, Train Steps/Sec: 0.84 +[2025-04-22 02:47:35] (step=0000975) Train Loss: 6.4521, Train Steps/Sec: 0.85 +[2025-04-22 02:48:05] (step=0001000) Train Loss: 6.4971, Train Steps/Sec: 0.84 +[2025-04-22 02:48:35] (step=0001025) Train Loss: 6.4865, Train Steps/Sec: 0.84 +[2025-04-22 02:49:04] (step=0001050) Train Loss: 6.4732, Train Steps/Sec: 0.85 +[2025-04-22 02:49:34] (step=0001075) Train Loss: 6.4757, Train Steps/Sec: 0.85 +[2025-04-22 02:50:03] (step=0001100) Train Loss: 6.4918, Train Steps/Sec: 0.85 +[2025-04-22 02:50:33] (step=0001125) Train Loss: 6.4610, Train Steps/Sec: 0.85 +[2025-04-22 02:51:02] (step=0001150) Train Loss: 6.4348, Train Steps/Sec: 0.84 +[2025-04-22 02:51:32] (step=0001175) Train Loss: 6.4795, Train Steps/Sec: 0.84 +[2025-04-22 02:52:02] (step=0001200) Train Loss: 6.4603, Train Steps/Sec: 0.85 +[2025-04-22 02:52:31] (step=0001225) Train Loss: 6.4859, Train Steps/Sec: 0.84 +[2025-04-22 02:53:01] (step=0001250) Train Loss: 6.4572, Train Steps/Sec: 0.84 +[2025-04-22 02:53:30] (step=0001275) Train Loss: 6.4279, Train Steps/Sec: 0.85 +[2025-04-22 02:54:00] (step=0001300) Train Loss: 6.5168, Train Steps/Sec: 0.85 +[2025-04-22 02:54:29] (step=0001325) Train Loss: 6.4507, Train Steps/Sec: 0.85 +[2025-04-22 02:54:59] (step=0001350) Train Loss: 6.4357, Train Steps/Sec: 0.84 +[2025-04-22 02:55:29] (step=0001375) Train Loss: 6.4666, Train Steps/Sec: 0.85 +[2025-04-22 02:55:59] (step=0001400) Train Loss: 6.4676, Train Steps/Sec: 0.83 +[2025-04-22 02:56:28] (step=0001425) Train Loss: 6.4845, Train Steps/Sec: 0.84 +[2025-04-22 02:56:58] (step=0001450) Train Loss: 6.4824, Train Steps/Sec: 0.85 +[2025-04-22 02:57:28] (step=0001475) Train Loss: 6.4820, Train Steps/Sec: 0.84 +[2025-04-22 02:57:57] (step=0001500) Train Loss: 6.4697, Train Steps/Sec: 0.84 +[2025-04-22 02:58:27] (step=0001525) Train Loss: 6.4817, Train Steps/Sec: 0.85 +[2025-04-22 02:58:56] (step=0001550) Train Loss: 6.4635, Train Steps/Sec: 0.85 +[2025-04-22 02:59:26] (step=0001575) Train Loss: 6.4990, Train Steps/Sec: 0.85 +[2025-04-22 02:59:55] (step=0001600) Train Loss: 6.3966, Train Steps/Sec: 0.84 +[2025-04-22 03:00:25] (step=0001625) Train Loss: 6.4715, Train Steps/Sec: 0.85 +[2025-04-22 03:00:54] (step=0001650) Train Loss: 6.4825, Train Steps/Sec: 0.85 +[2025-04-22 03:01:24] (step=0001675) Train Loss: 6.4907, Train Steps/Sec: 0.85 +[2025-04-22 03:01:53] (step=0001700) Train Loss: 6.4317, Train Steps/Sec: 0.85 +[2025-04-22 03:02:23] (step=0001725) Train Loss: 6.4481, Train Steps/Sec: 0.85 +[2025-04-22 03:02:52] (step=0001750) Train Loss: 6.4687, Train Steps/Sec: 0.85 +[2025-04-22 03:03:22] (step=0001775) Train Loss: 6.4345, Train Steps/Sec: 0.84 +[2025-04-22 03:03:51] (step=0001800) Train Loss: 6.5014, Train Steps/Sec: 0.85 +[2025-04-22 03:04:21] (step=0001825) Train Loss: 6.4882, Train Steps/Sec: 0.85 +[2025-04-22 03:04:51] (step=0001850) Train Loss: 6.5171, Train Steps/Sec: 0.85 +[2025-04-22 03:05:20] (step=0001875) Train Loss: 6.4768, Train Steps/Sec: 0.86 +[2025-04-22 03:05:49] (step=0001900) Train Loss: 6.4388, Train Steps/Sec: 0.84 +[2025-04-22 03:06:19] (step=0001925) Train Loss: 6.4312, Train Steps/Sec: 0.85 +[2025-04-22 03:06:48] (step=0001950) Train Loss: 6.4428, Train Steps/Sec: 0.85 +[2025-04-22 03:07:18] (step=0001975) Train Loss: 6.4400, Train Steps/Sec: 0.85 +[2025-04-22 03:07:48] (step=0002000) Train Loss: 6.4159, Train Steps/Sec: 0.84 +[2025-04-22 03:12:26] Finish Eval in 2000 steps... +[2025-04-22 03:12:48] Saved checkpoint to checkpoint/BeforeFix_Blip_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_objects/002-GPT-XL/checkpoints/0002000.pt +[2025-04-22 03:13:18] (step=0002025) Train Loss: 6.4650, Train Steps/Sec: 0.08 +[2025-04-22 03:13:47] (step=0002050) Train Loss: 6.4623, Train Steps/Sec: 0.84 +[2025-04-22 03:14:17] (step=0002075) Train Loss: 6.4914, Train Steps/Sec: 0.85 +[2025-04-22 03:14:47] (step=0002100) Train Loss: 6.4438, Train Steps/Sec: 0.84 +[2025-04-22 03:15:16] (step=0002125) Train Loss: 6.4642, Train Steps/Sec: 0.85 +[2025-04-22 03:15:46] (step=0002150) Train Loss: 6.4573, Train Steps/Sec: 0.85 +[2025-04-22 03:16:15] (step=0002175) Train Loss: 6.4868, Train Steps/Sec: 0.85 +[2025-04-22 03:16:45] (step=0002200) Train Loss: 6.4718, Train Steps/Sec: 0.84 +[2025-04-22 03:17:14] (step=0002225) Train Loss: 6.4090, Train Steps/Sec: 0.84 +[2025-04-22 03:17:44] (step=0002250) Train Loss: 6.4706, Train Steps/Sec: 0.85 +[2025-04-22 03:18:13] (step=0002275) Train Loss: 6.4507, Train Steps/Sec: 0.85 +[2025-04-22 03:18:43] (step=0002300) Train Loss: 6.4793, Train Steps/Sec: 0.84 +[2025-04-22 03:19:12] (step=0002325) Train Loss: 6.4751, Train Steps/Sec: 0.85 +[2025-04-22 03:19:42] (step=0002350) Train Loss: 6.4208, Train Steps/Sec: 0.85 +[2025-04-22 03:20:11] (step=0002375) Train Loss: 6.4282, Train Steps/Sec: 0.85 +[2025-04-22 03:20:41] (step=0002400) Train Loss: 6.4536, Train Steps/Sec: 0.84 +[2025-04-22 03:21:10] (step=0002425) Train Loss: 6.4360, Train Steps/Sec: 0.85 +[2025-04-22 03:21:40] (step=0002450) Train Loss: 6.4791, Train Steps/Sec: 0.85 +[2025-04-22 03:22:09] (step=0002475) Train Loss: 6.4036, Train Steps/Sec: 0.84 +[2025-04-22 03:22:39] (step=0002500) Train Loss: 6.4224, Train Steps/Sec: 0.84 +[2025-04-22 03:23:09] (step=0002525) Train Loss: 6.4418, Train Steps/Sec: 0.85 +[2025-04-22 03:23:38] (step=0002550) Train Loss: 6.3998, Train Steps/Sec: 0.85 +[2025-04-22 03:24:08] (step=0002575) Train Loss: 6.4549, Train Steps/Sec: 0.85 +[2025-04-22 03:24:37] (step=0002600) Train Loss: 6.4412, Train Steps/Sec: 0.85 +[2025-04-22 03:25:07] (step=0002625) Train Loss: 6.4758, Train Steps/Sec: 0.85 +[2025-04-22 03:25:36] (step=0002650) Train Loss: 6.4642, Train Steps/Sec: 0.85 +[2025-04-22 03:26:06] (step=0002675) Train Loss: 6.4970, Train Steps/Sec: 0.84 +[2025-04-22 03:26:36] (step=0002700) Train Loss: 6.4515, Train Steps/Sec: 0.84 +[2025-04-22 03:27:05] (step=0002725) Train Loss: 6.4545, Train Steps/Sec: 0.84 +[2025-04-22 03:27:35] (step=0002750) Train Loss: 6.4753, Train Steps/Sec: 0.85 +[2025-04-22 03:28:04] (step=0002775) Train Loss: 6.4840, Train Steps/Sec: 0.85 +[2025-04-22 03:28:34] (step=0002800) Train Loss: 6.4432, Train Steps/Sec: 0.84 +[2025-04-22 03:29:03] (step=0002825) Train Loss: 6.4319, Train Steps/Sec: 0.85 +[2025-04-22 03:29:33] (step=0002850) Train Loss: 6.4684, Train Steps/Sec: 0.85 +[2025-04-22 03:30:02] (step=0002875) Train Loss: 6.4390, Train Steps/Sec: 0.85 +[2025-04-22 03:30:32] (step=0002900) Train Loss: 6.4367, Train Steps/Sec: 0.84 +[2025-04-22 03:31:01] (step=0002925) Train Loss: 6.4061, Train Steps/Sec: 0.85 +[2025-04-22 03:31:31] (step=0002950) Train Loss: 6.4160, Train Steps/Sec: 0.85 +[2025-04-22 03:32:00] (step=0002975) Train Loss: 6.4719, Train Steps/Sec: 0.84 +[2025-04-22 03:32:30] (step=0003000) Train Loss: 6.4495, Train Steps/Sec: 0.84 +[2025-04-22 03:32:59] (step=0003025) Train Loss: 6.4220, Train Steps/Sec: 0.85 +[2025-04-22 03:33:29] (step=0003050) Train Loss: 6.4659, Train Steps/Sec: 0.85 +[2025-04-22 03:33:58] (step=0003075) Train Loss: 6.4782, Train Steps/Sec: 0.85 +[2025-04-22 03:34:28] (step=0003100) Train Loss: 6.4383, Train Steps/Sec: 0.85 +[2025-04-22 03:34:57] (step=0003125) Train Loss: 6.4614, Train Steps/Sec: 0.85 +[2025-04-22 03:35:26] (step=0003150) Train Loss: 6.4281, Train Steps/Sec: 0.85 +[2025-04-22 03:35:56] (step=0003175) Train Loss: 6.4810, Train Steps/Sec: 0.85 +[2025-04-22 03:36:25] (step=0003200) Train Loss: 6.4326, Train Steps/Sec: 0.85 +[2025-04-22 03:36:55] (step=0003225) Train Loss: 6.4532, Train Steps/Sec: 0.84 +[2025-04-22 03:37:25] (step=0003250) Train Loss: 6.4537, Train Steps/Sec: 0.85 +[2025-04-22 03:37:54] (step=0003275) Train Loss: 6.4474, Train Steps/Sec: 0.85 +[2025-04-22 03:38:24] (step=0003300) Train Loss: 6.4523, Train Steps/Sec: 0.84 +[2025-04-22 03:38:53] (step=0003325) Train Loss: 6.4793, Train Steps/Sec: 0.85 +[2025-04-22 03:39:23] (step=0003350) Train Loss: 6.4544, Train Steps/Sec: 0.85 +[2025-04-22 03:39:52] (step=0003375) Train Loss: 6.4563, Train Steps/Sec: 0.85 +[2025-04-22 03:40:22] (step=0003400) Train Loss: 6.4514, Train Steps/Sec: 0.84 +[2025-04-22 03:40:51] (step=0003425) Train Loss: 6.4821, Train Steps/Sec: 0.85 +[2025-04-22 03:41:21] (step=0003450) Train Loss: 6.4673, Train Steps/Sec: 0.84 +[2025-04-22 03:41:58] (step=0003475) Train Loss: 6.4579, Train Steps/Sec: 0.67 +[2025-04-22 03:42:28] (step=0003500) Train Loss: 6.4123, Train Steps/Sec: 0.85 +[2025-04-22 03:42:57] (step=0003525) Train Loss: 6.4260, Train Steps/Sec: 0.85 +[2025-04-22 03:43:27] (step=0003550) Train Loss: 6.4271, Train Steps/Sec: 0.84 +[2025-04-22 03:43:56] (step=0003575) Train Loss: 6.4251, Train Steps/Sec: 0.85 +[2025-04-22 03:44:26] (step=0003600) Train Loss: 6.4470, Train Steps/Sec: 0.84 +[2025-04-22 03:45:04] (step=0003625) Train Loss: 6.4310, Train Steps/Sec: 0.66 +[2025-04-22 03:45:33] (step=0003650) Train Loss: 6.4070, Train Steps/Sec: 0.85 +[2025-04-22 03:46:03] (step=0003675) Train Loss: 6.4868, Train Steps/Sec: 0.85 +[2025-04-22 03:46:32] (step=0003700) Train Loss: 6.4700, Train Steps/Sec: 0.84 +[2025-04-22 03:47:02] (step=0003725) Train Loss: 6.4264, Train Steps/Sec: 0.85 +[2025-04-22 03:47:39] (step=0003750) Train Loss: 6.4323, Train Steps/Sec: 0.68 +[2025-04-22 03:48:08] (step=0003775) Train Loss: 6.4508, Train Steps/Sec: 0.85 +[2025-04-22 03:48:38] (step=0003800) Train Loss: 6.4366, Train Steps/Sec: 0.84 +[2025-04-22 03:49:07] (step=0003825) Train Loss: 6.3827, Train Steps/Sec: 0.85 +[2025-04-22 03:49:37] (step=0003850) Train Loss: 6.4437, Train Steps/Sec: 0.85 +[2025-04-22 03:50:06] (step=0003875) Train Loss: 6.4224, Train Steps/Sec: 0.84 +[2025-04-22 03:50:36] (step=0003900) Train Loss: 6.4422, Train Steps/Sec: 0.85 +[2025-04-22 03:51:05] (step=0003925) Train Loss: 6.4724, Train Steps/Sec: 0.85 +[2025-04-22 03:51:35] (step=0003950) Train Loss: 6.4841, Train Steps/Sec: 0.84 +[2025-04-22 03:52:05] (step=0003975) Train Loss: 6.4559, Train Steps/Sec: 0.85 +[2025-04-22 03:52:34] (step=0004000) Train Loss: 6.4333, Train Steps/Sec: 0.84 +[2025-04-22 03:57:15] Finish Eval in 4000 steps... +[2025-04-22 03:57:35] Saved checkpoint to checkpoint/BeforeFix_Blip_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_objects/002-GPT-XL/checkpoints/0004000.pt +[2025-04-22 03:58:05] (step=0004025) Train Loss: 6.4191, Train Steps/Sec: 0.08 +[2025-04-22 03:58:45] (step=0004050) Train Loss: 6.4513, Train Steps/Sec: 0.63 +[2025-04-22 03:59:14] (step=0004075) Train Loss: 6.4515, Train Steps/Sec: 0.85 +[2025-04-22 03:59:44] (step=0004100) Train Loss: 6.4459, Train Steps/Sec: 0.84 +[2025-04-22 04:00:13] (step=0004125) Train Loss: 6.5043, Train Steps/Sec: 0.85 +[2025-04-22 04:00:50] (step=0004150) Train Loss: 6.4140, Train Steps/Sec: 0.68 +[2025-04-22 04:01:29] (step=0004175) Train Loss: 6.4911, Train Steps/Sec: 0.65 +[2025-04-22 04:01:58] (step=0004200) Train Loss: 6.4677, Train Steps/Sec: 0.84 +[2025-04-22 04:02:28] (step=0004225) Train Loss: 6.4655, Train Steps/Sec: 0.84 +[2025-04-22 04:02:58] (step=0004250) Train Loss: 6.3943, Train Steps/Sec: 0.84 +[2025-04-22 04:03:27] (step=0004275) Train Loss: 6.4599, Train Steps/Sec: 0.85 +[2025-04-22 04:03:57] (step=0004300) Train Loss: 6.4727, Train Steps/Sec: 0.84 +[2025-04-22 04:04:26] (step=0004325) Train Loss: 6.4495, Train Steps/Sec: 0.85 +[2025-04-22 04:04:56] (step=0004350) Train Loss: 6.4427, Train Steps/Sec: 0.85 +[2025-04-22 04:05:25] (step=0004375) Train Loss: 6.4397, Train Steps/Sec: 0.85 +[2025-04-22 04:05:55] (step=0004400) Train Loss: 6.4273, Train Steps/Sec: 0.85 +[2025-04-22 04:06:24] (step=0004425) Train Loss: 6.4375, Train Steps/Sec: 0.84 +[2025-04-22 04:06:54] (step=0004450) Train Loss: 6.4140, Train Steps/Sec: 0.85 +[2025-04-22 04:07:23] (step=0004475) Train Loss: 6.4419, Train Steps/Sec: 0.85 +[2025-04-22 04:07:53] (step=0004500) Train Loss: 6.4107, Train Steps/Sec: 0.84 +[2025-04-22 04:08:22] (step=0004525) Train Loss: 6.4434, Train Steps/Sec: 0.85 +[2025-04-22 04:08:52] (step=0004550) Train Loss: 6.4925, Train Steps/Sec: 0.84 +[2025-04-22 04:09:21] (step=0004575) Train Loss: 6.3957, Train Steps/Sec: 0.85 +[2025-04-22 04:09:51] (step=0004600) Train Loss: 6.4853, Train Steps/Sec: 0.84 +[2025-04-22 04:10:21] (step=0004625) Train Loss: 6.4346, Train Steps/Sec: 0.85 +[2025-04-22 04:10:50] (step=0004650) Train Loss: 6.4601, Train Steps/Sec: 0.84 +[2025-04-22 04:11:20] (step=0004675) Train Loss: 6.4670, Train Steps/Sec: 0.85 +[2025-04-22 04:11:50] (step=0004700) Train Loss: 6.4542, Train Steps/Sec: 0.84 +[2025-04-22 04:12:19] (step=0004725) Train Loss: 6.4189, Train Steps/Sec: 0.85 +[2025-04-22 04:12:48] (step=0004750) Train Loss: 6.4563, Train Steps/Sec: 0.85 +[2025-04-22 04:13:18] (step=0004775) Train Loss: 6.4308, Train Steps/Sec: 0.85 +[2025-04-22 04:13:48] (step=0004800) Train Loss: 6.4267, Train Steps/Sec: 0.83 +[2025-04-22 04:14:18] (step=0004825) Train Loss: 6.4279, Train Steps/Sec: 0.85 +[2025-04-22 04:14:47] (step=0004850) Train Loss: 6.4653, Train Steps/Sec: 0.84 +[2025-04-22 04:15:17] (step=0004875) Train Loss: 6.4824, Train Steps/Sec: 0.85 +[2025-04-22 04:15:46] (step=0004900) Train Loss: 6.4320, Train Steps/Sec: 0.84 +[2025-04-22 04:16:16] (step=0004925) Train Loss: 6.4517, Train Steps/Sec: 0.84 +[2025-04-22 04:16:45] (step=0004950) Train Loss: 6.4620, Train Steps/Sec: 0.85 +[2025-04-22 04:17:15] (step=0004975) Train Loss: 6.3910, Train Steps/Sec: 0.84 +[2025-04-22 04:17:45] (step=0005000) Train Loss: 6.4438, Train Steps/Sec: 0.84 +[2025-04-22 04:18:14] (step=0005025) Train Loss: 6.5328, Train Steps/Sec: 0.84 +[2025-04-22 04:18:44] (step=0005050) Train Loss: 6.4355, Train Steps/Sec: 0.84 +[2025-04-22 04:19:14] (step=0005075) Train Loss: 6.4835, Train Steps/Sec: 0.85 +[2025-04-22 04:19:43] (step=0005100) Train Loss: 6.4328, Train Steps/Sec: 0.84 +[2025-04-22 04:20:13] (step=0005125) Train Loss: 6.4565, Train Steps/Sec: 0.85 +[2025-04-22 04:20:43] (step=0005150) Train Loss: 6.4073, Train Steps/Sec: 0.85 +[2025-04-22 04:21:12] (step=0005175) Train Loss: 6.4802, Train Steps/Sec: 0.84 +[2025-04-22 04:21:42] (step=0005200) Train Loss: 6.4722, Train Steps/Sec: 0.84 +[2025-04-22 04:22:11] (step=0005225) Train Loss: 6.4488, Train Steps/Sec: 0.85 +[2025-04-22 04:22:41] (step=0005250) Train Loss: 6.4325, Train Steps/Sec: 0.85 +[2025-04-22 04:23:10] (step=0005275) Train Loss: 6.4444, Train Steps/Sec: 0.85 +[2025-04-22 04:23:40] (step=0005300) Train Loss: 6.4000, Train Steps/Sec: 0.85 +[2025-04-22 04:24:09] (step=0005325) Train Loss: 6.4475, Train Steps/Sec: 0.85 +[2025-04-22 04:24:39] (step=0005350) Train Loss: 6.4529, Train Steps/Sec: 0.85 +[2025-04-22 04:25:09] (step=0005375) Train Loss: 6.4400, Train Steps/Sec: 0.85 +[2025-04-22 04:25:38] (step=0005400) Train Loss: 6.4569, Train Steps/Sec: 0.84 +[2025-04-22 04:26:08] (step=0005425) Train Loss: 6.4167, Train Steps/Sec: 0.85 +[2025-04-22 04:26:44] (step=0005450) Train Loss: 6.4174, Train Steps/Sec: 0.68 +[2025-04-22 04:27:14] (step=0005475) Train Loss: 6.4183, Train Steps/Sec: 0.84 +[2025-04-22 04:27:44] (step=0005500) Train Loss: 6.4338, Train Steps/Sec: 0.85 +[2025-04-22 04:28:13] (step=0005525) Train Loss: 6.4067, Train Steps/Sec: 0.84 +[2025-04-22 04:28:43] (step=0005550) Train Loss: 6.4390, Train Steps/Sec: 0.84 +[2025-04-22 04:29:13] (step=0005575) Train Loss: 6.4595, Train Steps/Sec: 0.84 +[2025-04-22 04:29:42] (step=0005600) Train Loss: 6.4366, Train Steps/Sec: 0.84 +[2025-04-22 04:30:12] (step=0005625) Train Loss: 6.4249, Train Steps/Sec: 0.84 +[2025-04-22 04:30:41] (step=0005650) Train Loss: 6.4477, Train Steps/Sec: 0.85 +[2025-04-22 04:31:11] (step=0005675) Train Loss: 6.4250, Train Steps/Sec: 0.84 +[2025-04-22 04:31:41] (step=0005700) Train Loss: 6.4348, Train Steps/Sec: 0.85 +[2025-04-22 04:32:10] (step=0005725) Train Loss: 6.4586, Train Steps/Sec: 0.84 +[2025-04-22 04:32:40] (step=0005750) Train Loss: 6.4007, Train Steps/Sec: 0.85 +[2025-04-22 04:33:09] (step=0005775) Train Loss: 6.4411, Train Steps/Sec: 0.85 +[2025-04-22 04:33:39] (step=0005800) Train Loss: 6.3803, Train Steps/Sec: 0.84 +[2025-04-22 04:34:08] (step=0005825) Train Loss: 6.4405, Train Steps/Sec: 0.85 +[2025-04-22 04:34:38] (step=0005850) Train Loss: 6.4655, Train Steps/Sec: 0.85 +[2025-04-22 04:35:07] (step=0005875) Train Loss: 6.3995, Train Steps/Sec: 0.85 +[2025-04-22 04:35:37] (step=0005900) Train Loss: 6.4594, Train Steps/Sec: 0.84 +[2025-04-22 04:36:07] (step=0005925) Train Loss: 6.4942, Train Steps/Sec: 0.84 +[2025-04-22 04:36:36] (step=0005950) Train Loss: 6.4297, Train Steps/Sec: 0.86 +[2025-04-22 04:37:06] (step=0005975) Train Loss: 6.4199, Train Steps/Sec: 0.84 +[2025-04-22 04:37:35] (step=0006000) Train Loss: 6.4320, Train Steps/Sec: 0.84 +[2025-04-22 04:42:17] Finish Eval in 6000 steps... +[2025-04-22 04:42:37] Saved checkpoint to checkpoint/BeforeFix_Blip_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_objects/002-GPT-XL/checkpoints/0006000.pt +[2025-04-22 04:42:40] Removed old checkpoint: checkpoint/BeforeFix_Blip_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_objects/002-GPT-XL/checkpoints/0002000.pt +[2025-04-22 04:43:09] (step=0006025) Train Loss: 6.4042, Train Steps/Sec: 0.07 +[2025-04-22 04:43:38] (step=0006050) Train Loss: 6.4061, Train Steps/Sec: 0.85 +[2025-04-22 04:44:08] (step=0006075) Train Loss: 6.4441, Train Steps/Sec: 0.84 +[2025-04-22 04:44:38] (step=0006100) Train Loss: 6.4696, Train Steps/Sec: 0.84 +[2025-04-22 04:45:07] (step=0006125) Train Loss: 6.4503, Train Steps/Sec: 0.85 +[2025-04-22 04:45:37] (step=0006150) Train Loss: 6.4648, Train Steps/Sec: 0.85 +[2025-04-22 04:46:06] (step=0006175) Train Loss: 6.3984, Train Steps/Sec: 0.85 +[2025-04-22 04:46:36] (step=0006200) Train Loss: 6.4704, Train Steps/Sec: 0.85 +[2025-04-22 04:47:06] (step=0006225) Train Loss: 6.4571, Train Steps/Sec: 0.84 +[2025-04-22 04:47:35] (step=0006250) Train Loss: 6.4737, Train Steps/Sec: 0.84 +[2025-04-22 04:48:05] (step=0006275) Train Loss: 6.4091, Train Steps/Sec: 0.85 +[2025-04-22 04:48:34] (step=0006300) Train Loss: 6.4578, Train Steps/Sec: 0.85 +[2025-04-22 04:49:04] (step=0006325) Train Loss: 6.4020, Train Steps/Sec: 0.85 +[2025-04-22 04:49:33] (step=0006350) Train Loss: 6.4319, Train Steps/Sec: 0.85 +[2025-04-22 04:50:03] (step=0006375) Train Loss: 6.4251, Train Steps/Sec: 0.85 +[2025-04-22 04:50:32] (step=0006400) Train Loss: 6.4388, Train Steps/Sec: 0.84 +[2025-04-22 04:51:02] (step=0006425) Train Loss: 6.4613, Train Steps/Sec: 0.84 +[2025-04-22 04:51:32] (step=0006450) Train Loss: 6.4108, Train Steps/Sec: 0.85 +[2025-04-22 04:52:01] (step=0006475) Train Loss: 6.4138, Train Steps/Sec: 0.85 +[2025-04-22 04:52:31] (step=0006500) Train Loss: 6.4392, Train Steps/Sec: 0.84 +[2025-04-22 04:53:00] (step=0006525) Train Loss: 6.4641, Train Steps/Sec: 0.85 +[2025-04-22 04:53:30] (step=0006550) Train Loss: 6.4135, Train Steps/Sec: 0.85 +[2025-04-22 04:53:59] (step=0006575) Train Loss: 6.4267, Train Steps/Sec: 0.84 +[2025-04-22 04:54:29] (step=0006600) Train Loss: 6.4612, Train Steps/Sec: 0.84 +[2025-04-22 04:54:59] (step=0006625) Train Loss: 6.5013, Train Steps/Sec: 0.84 +[2025-04-22 04:55:28] (step=0006650) Train Loss: 6.4246, Train Steps/Sec: 0.85 +[2025-04-22 04:55:58] (step=0006675) Train Loss: 6.4340, Train Steps/Sec: 0.84 +[2025-04-22 04:56:28] (step=0006700) Train Loss: 6.4292, Train Steps/Sec: 0.84 +[2025-04-22 04:56:57] (step=0006725) Train Loss: 6.4324, Train Steps/Sec: 0.85 +[2025-04-22 04:57:27] (step=0006750) Train Loss: 6.4006, Train Steps/Sec: 0.85 +[2025-04-22 04:57:56] (step=0006775) Train Loss: 6.4824, Train Steps/Sec: 0.85 +[2025-04-22 04:58:26] (step=0006800) Train Loss: 6.4344, Train Steps/Sec: 0.85 +[2025-04-22 04:58:55] (step=0006825) Train Loss: 6.4001, Train Steps/Sec: 0.85 +[2025-04-22 04:59:25] (step=0006850) Train Loss: 6.4550, Train Steps/Sec: 0.85 +[2025-04-22 04:59:54] (step=0006875) Train Loss: 6.4316, Train Steps/Sec: 0.85 +[2025-04-22 05:00:24] (step=0006900) Train Loss: 6.4438, Train Steps/Sec: 0.84 +[2025-04-22 05:00:53] (step=0006925) Train Loss: 6.3701, Train Steps/Sec: 0.85 +[2025-04-22 05:01:23] (step=0006950) Train Loss: 6.4235, Train Steps/Sec: 0.85 +[2025-04-22 05:01:52] (step=0006975) Train Loss: 6.4011, Train Steps/Sec: 0.85 +[2025-04-22 05:02:22] (step=0007000) Train Loss: 6.4178, Train Steps/Sec: 0.84 +[2025-04-22 05:02:51] (step=0007025) Train Loss: 6.4231, Train Steps/Sec: 0.85 +[2025-04-22 05:03:21] (step=0007050) Train Loss: 6.4614, Train Steps/Sec: 0.85 +[2025-04-22 05:03:50] (step=0007075) Train Loss: 6.4004, Train Steps/Sec: 0.85 +[2025-04-22 05:04:20] (step=0007100) Train Loss: 6.4195, Train Steps/Sec: 0.85 +[2025-04-22 05:04:49] (step=0007125) Train Loss: 6.4786, Train Steps/Sec: 0.85 +[2025-04-22 05:05:19] (step=0007150) Train Loss: 6.4608, Train Steps/Sec: 0.85 +[2025-04-22 05:05:48] (step=0007175) Train Loss: 6.4142, Train Steps/Sec: 0.85 +[2025-04-22 05:06:18] (step=0007200) Train Loss: 6.4416, Train Steps/Sec: 0.85 +[2025-04-22 05:06:47] (step=0007225) Train Loss: 6.4275, Train Steps/Sec: 0.85 +[2025-04-22 05:07:17] (step=0007250) Train Loss: 6.4670, Train Steps/Sec: 0.84 +[2025-04-22 05:07:46] (step=0007275) Train Loss: 6.3819, Train Steps/Sec: 0.84 +[2025-04-22 05:08:16] (step=0007300) Train Loss: 6.3999, Train Steps/Sec: 0.84 +[2025-04-22 05:08:46] (step=0007325) Train Loss: 6.4158, Train Steps/Sec: 0.85 +[2025-04-22 05:09:15] (step=0007350) Train Loss: 6.4016, Train Steps/Sec: 0.85 +[2025-04-22 05:09:45] (step=0007375) Train Loss: 6.4345, Train Steps/Sec: 0.84 +[2025-04-22 05:10:14] (step=0007400) Train Loss: 6.4054, Train Steps/Sec: 0.84 +[2025-04-22 05:10:44] (step=0007425) Train Loss: 6.4060, Train Steps/Sec: 0.85 +[2025-04-22 05:11:13] (step=0007450) Train Loss: 6.4697, Train Steps/Sec: 0.85 +[2025-04-22 05:11:43] (step=0007475) Train Loss: 6.4399, Train Steps/Sec: 0.84 +[2025-04-22 05:12:13] (step=0007500) Train Loss: 6.3996, Train Steps/Sec: 0.84 +[2025-04-22 05:12:42] (step=0007525) Train Loss: 6.4004, Train Steps/Sec: 0.85 +[2025-04-22 05:13:12] (step=0007550) Train Loss: 6.4325, Train Steps/Sec: 0.85 +[2025-04-22 05:13:41] (step=0007575) Train Loss: 6.4190, Train Steps/Sec: 0.85 +[2025-04-22 05:14:11] (step=0007600) Train Loss: 6.4305, Train Steps/Sec: 0.84 +[2025-04-22 05:14:40] (step=0007625) Train Loss: 6.4460, Train Steps/Sec: 0.85 +[2025-04-22 05:15:10] (step=0007650) Train Loss: 6.4404, Train Steps/Sec: 0.84 +[2025-04-22 05:15:39] (step=0007675) Train Loss: 6.4574, Train Steps/Sec: 0.85 +[2025-04-22 05:16:09] (step=0007700) Train Loss: 6.4477, Train Steps/Sec: 0.84 +[2025-04-22 05:16:39] (step=0007725) Train Loss: 6.4142, Train Steps/Sec: 0.85 +[2025-04-22 05:17:08] (step=0007750) Train Loss: 6.4439, Train Steps/Sec: 0.85 +[2025-04-22 05:17:37] (step=0007775) Train Loss: 6.4280, Train Steps/Sec: 0.85 +[2025-04-22 05:18:07] (step=0007800) Train Loss: 6.4113, Train Steps/Sec: 0.84 +[2025-04-22 05:18:37] (step=0007825) Train Loss: 6.4562, Train Steps/Sec: 0.84 +[2025-04-22 05:19:06] (step=0007850) Train Loss: 6.4149, Train Steps/Sec: 0.85 +[2025-04-22 05:19:36] (step=0007875) Train Loss: 6.4219, Train Steps/Sec: 0.85 +[2025-04-22 05:20:06] (step=0007900) Train Loss: 6.4223, Train Steps/Sec: 0.84 +[2025-04-22 05:20:35] (step=0007925) Train Loss: 6.4512, Train Steps/Sec: 0.85 +[2025-04-22 05:21:05] (step=0007950) Train Loss: 6.4431, Train Steps/Sec: 0.84 +[2025-04-22 05:21:34] (step=0007975) Train Loss: 6.4071, Train Steps/Sec: 0.85 +[2025-04-22 05:22:04] (step=0008000) Train Loss: 6.4358, Train Steps/Sec: 0.84 +[2025-04-22 05:26:46] Finish Eval in 8000 steps... +[2025-04-22 05:27:07] Saved checkpoint to checkpoint/BeforeFix_Blip_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_objects/002-GPT-XL/checkpoints/0008000.pt +[2025-04-22 05:27:09] Removed old checkpoint: checkpoint/BeforeFix_Blip_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_objects/002-GPT-XL/checkpoints/0004000.pt +[2025-04-22 05:27:39] (step=0008025) Train Loss: 6.4118, Train Steps/Sec: 0.07 +[2025-04-22 05:28:09] (step=0008050) Train Loss: 6.4051, Train Steps/Sec: 0.85 +[2025-04-22 05:28:38] (step=0008075) Train Loss: 6.4516, Train Steps/Sec: 0.84 +[2025-04-22 05:29:08] (step=0008100) Train Loss: 6.4593, Train Steps/Sec: 0.85 +[2025-04-22 05:29:38] (step=0008125) Train Loss: 6.4414, Train Steps/Sec: 0.85 +[2025-04-22 05:30:07] (step=0008150) Train Loss: 6.4606, Train Steps/Sec: 0.85 +[2025-04-22 05:30:37] (step=0008175) Train Loss: 6.4421, Train Steps/Sec: 0.84 +[2025-04-22 05:31:06] (step=0008200) Train Loss: 6.4469, Train Steps/Sec: 0.84 +[2025-04-22 05:31:36] (step=0008225) Train Loss: 6.4415, Train Steps/Sec: 0.85 +[2025-04-22 05:32:05] (step=0008250) Train Loss: 6.4863, Train Steps/Sec: 0.85 +[2025-04-22 05:32:35] (step=0008275) Train Loss: 6.4025, Train Steps/Sec: 0.84 +[2025-04-22 05:33:05] (step=0008300) Train Loss: 6.4570, Train Steps/Sec: 0.84 +[2025-04-22 05:33:34] (step=0008325) Train Loss: 6.3967, Train Steps/Sec: 0.85 +[2025-04-22 05:34:03] (step=0008350) Train Loss: 6.4488, Train Steps/Sec: 0.85 +[2025-04-22 05:34:33] (step=0008375) Train Loss: 6.4477, Train Steps/Sec: 0.85 +[2025-04-22 05:35:03] (step=0008400) Train Loss: 6.4579, Train Steps/Sec: 0.84 +[2025-04-22 05:35:32] (step=0008425) Train Loss: 6.4367, Train Steps/Sec: 0.85 +[2025-04-22 05:36:02] (step=0008450) Train Loss: 6.4137, Train Steps/Sec: 0.84 +[2025-04-22 05:36:31] (step=0008475) Train Loss: 6.4394, Train Steps/Sec: 0.85 +[2025-04-22 05:37:01] (step=0008500) Train Loss: 6.4251, Train Steps/Sec: 0.84 +[2025-04-22 05:37:30] (step=0008525) Train Loss: 6.4112, Train Steps/Sec: 0.85 +[2025-04-22 05:38:00] (step=0008550) Train Loss: 6.4107, Train Steps/Sec: 0.85 +[2025-04-22 05:38:29] (step=0008575) Train Loss: 6.4093, Train Steps/Sec: 0.84 +[2025-04-22 05:38:59] (step=0008600) Train Loss: 6.4293, Train Steps/Sec: 0.84 +[2025-04-22 05:39:29] (step=0008625) Train Loss: 6.4433, Train Steps/Sec: 0.84 +[2025-04-22 05:39:59] (step=0008650) Train Loss: 6.4185, Train Steps/Sec: 0.84 +[2025-04-22 05:40:28] (step=0008675) Train Loss: 6.3939, Train Steps/Sec: 0.85 +[2025-04-22 05:40:58] (step=0008700) Train Loss: 6.4490, Train Steps/Sec: 0.84 +[2025-04-22 05:41:28] (step=0008725) Train Loss: 6.4531, Train Steps/Sec: 0.84 +[2025-04-22 05:41:57] (step=0008750) Train Loss: 6.4063, Train Steps/Sec: 0.84 +[2025-04-22 05:42:27] (step=0008775) Train Loss: 6.4224, Train Steps/Sec: 0.84 +[2025-04-22 05:42:57] (step=0008800) Train Loss: 6.3669, Train Steps/Sec: 0.85 +[2025-04-22 05:43:26] (step=0008825) Train Loss: 6.4033, Train Steps/Sec: 0.84 +[2025-04-22 05:43:56] (step=0008850) Train Loss: 6.4218, Train Steps/Sec: 0.84 +[2025-04-22 05:44:25] (step=0008875) Train Loss: 6.3912, Train Steps/Sec: 0.85 +[2025-04-22 05:44:55] (step=0008900) Train Loss: 6.4342, Train Steps/Sec: 0.84 +[2025-04-22 05:45:25] (step=0008925) Train Loss: 6.3981, Train Steps/Sec: 0.84 +[2025-04-22 05:46:03] (step=0008950) Train Loss: 6.4344, Train Steps/Sec: 0.66 +[2025-04-22 05:46:33] (step=0008975) Train Loss: 6.4286, Train Steps/Sec: 0.84 +[2025-04-22 05:47:02] (step=0009000) Train Loss: 6.4256, Train Steps/Sec: 0.84 +[2025-04-22 05:47:32] (step=0009025) Train Loss: 6.4468, Train Steps/Sec: 0.84 +[2025-04-22 05:48:02] (step=0009050) Train Loss: 6.4331, Train Steps/Sec: 0.84 +[2025-04-22 05:48:43] (step=0009075) Train Loss: 6.4097, Train Steps/Sec: 0.60 +[2025-04-22 05:49:13] (step=0009100) Train Loss: 6.4240, Train Steps/Sec: 0.84 +[2025-04-22 05:49:43] (step=0009125) Train Loss: 6.4378, Train Steps/Sec: 0.84 +[2025-04-22 05:50:19] (step=0009150) Train Loss: 6.4367, Train Steps/Sec: 0.69 +[2025-04-22 05:50:49] (step=0009175) Train Loss: 6.4272, Train Steps/Sec: 0.84 +[2025-04-22 05:51:19] (step=0009200) Train Loss: 6.4581, Train Steps/Sec: 0.84 +[2025-04-22 05:51:48] (step=0009225) Train Loss: 6.4317, Train Steps/Sec: 0.85 +[2025-04-22 05:52:17] (step=0009250) Train Loss: 6.4104, Train Steps/Sec: 0.85 +[2025-04-22 05:52:54] (step=0009275) Train Loss: 6.4137, Train Steps/Sec: 0.68 +[2025-04-22 05:53:30] (step=0009300) Train Loss: 6.4222, Train Steps/Sec: 0.70 +[2025-04-22 05:53:59] (step=0009325) Train Loss: 6.4217, Train Steps/Sec: 0.85 +[2025-04-22 05:54:29] (step=0009350) Train Loss: 6.4479, Train Steps/Sec: 0.84 +[2025-04-22 05:55:05] (step=0009375) Train Loss: 6.3927, Train Steps/Sec: 0.70 +[2025-04-22 05:55:34] (step=0009400) Train Loss: 6.4228, Train Steps/Sec: 0.84 +[2025-04-22 05:56:04] (step=0009425) Train Loss: 6.4270, Train Steps/Sec: 0.85 +[2025-04-22 05:56:33] (step=0009450) Train Loss: 6.4358, Train Steps/Sec: 0.85 +[2025-04-22 05:57:03] (step=0009475) Train Loss: 6.4113, Train Steps/Sec: 0.85 +[2025-04-22 05:57:32] (step=0009500) Train Loss: 6.3655, Train Steps/Sec: 0.84 +[2025-04-22 05:58:02] (step=0009525) Train Loss: 6.4416, Train Steps/Sec: 0.84 +[2025-04-22 05:58:32] (step=0009550) Train Loss: 6.4586, Train Steps/Sec: 0.84 +[2025-04-22 05:59:01] (step=0009575) Train Loss: 6.4246, Train Steps/Sec: 0.84 +[2025-04-22 05:59:31] (step=0009600) Train Loss: 6.3927, Train Steps/Sec: 0.85 +[2025-04-22 06:00:00] (step=0009625) Train Loss: 6.4047, Train Steps/Sec: 0.85 +[2025-04-22 06:00:30] (step=0009650) Train Loss: 6.4218, Train Steps/Sec: 0.84 +[2025-04-22 06:01:00] (step=0009675) Train Loss: 6.4506, Train Steps/Sec: 0.84 +[2025-04-22 06:01:29] (step=0009700) Train Loss: 6.4236, Train Steps/Sec: 0.84 +[2025-04-22 06:01:59] (step=0009725) Train Loss: 6.4273, Train Steps/Sec: 0.85 +[2025-04-22 06:02:28] (step=0009750) Train Loss: 6.3873, Train Steps/Sec: 0.85 +[2025-04-22 06:02:58] (step=0009775) Train Loss: 6.4205, Train Steps/Sec: 0.85 +[2025-04-22 06:03:28] (step=0009800) Train Loss: 6.4406, Train Steps/Sec: 0.84 +[2025-04-22 06:03:57] (step=0009825) Train Loss: 6.4026, Train Steps/Sec: 0.85 +[2025-04-22 06:04:27] (step=0009850) Train Loss: 6.4222, Train Steps/Sec: 0.85 +[2025-04-22 06:04:56] (step=0009875) Train Loss: 6.4258, Train Steps/Sec: 0.84 +[2025-04-22 06:05:26] (step=0009900) Train Loss: 6.4088, Train Steps/Sec: 0.84 +[2025-04-22 06:05:56] (step=0009925) Train Loss: 6.4207, Train Steps/Sec: 0.85 +[2025-04-22 06:06:25] (step=0009950) Train Loss: 6.4157, Train Steps/Sec: 0.85 +[2025-04-22 06:06:54] (step=0009975) Train Loss: 6.4127, Train Steps/Sec: 0.85 +[2025-04-22 06:07:24] (step=0010000) Train Loss: 6.3861, Train Steps/Sec: 0.85 +[2025-04-22 06:12:05] Finish Eval in 10000 steps... +[2025-04-22 06:12:26] Saved checkpoint to checkpoint/BeforeFix_Blip_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_objects/002-GPT-XL/checkpoints/0010000.pt +[2025-04-22 06:12:29] Removed old checkpoint: checkpoint/BeforeFix_Blip_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_objects/002-GPT-XL/checkpoints/0006000.pt +[2025-04-22 06:12:58] (step=0010025) Train Loss: 6.3891, Train Steps/Sec: 0.07 +[2025-04-22 06:13:28] (step=0010050) Train Loss: 6.4807, Train Steps/Sec: 0.84 +[2025-04-22 06:13:58] (step=0010075) Train Loss: 6.4386, Train Steps/Sec: 0.85 +[2025-04-22 06:14:27] (step=0010100) Train Loss: 6.4287, Train Steps/Sec: 0.84 +[2025-04-22 06:14:57] (step=0010125) Train Loss: 6.4193, Train Steps/Sec: 0.84 +[2025-04-22 06:15:27] (step=0010150) Train Loss: 6.4057, Train Steps/Sec: 0.85 +[2025-04-22 06:15:56] (step=0010175) Train Loss: 6.4081, Train Steps/Sec: 0.85 +[2025-04-22 06:16:26] (step=0010200) Train Loss: 6.4423, Train Steps/Sec: 0.84 +[2025-04-22 06:16:56] (step=0010225) Train Loss: 6.3946, Train Steps/Sec: 0.84 +[2025-04-22 06:17:25] (step=0010250) Train Loss: 6.3643, Train Steps/Sec: 0.85 +[2025-04-22 06:17:55] (step=0010275) Train Loss: 6.4058, Train Steps/Sec: 0.85 +[2025-04-22 06:18:25] (step=0010300) Train Loss: 6.4206, Train Steps/Sec: 0.84 +[2025-04-22 06:18:54] (step=0010325) Train Loss: 6.3924, Train Steps/Sec: 0.85 +[2025-04-22 06:19:24] (step=0010350) Train Loss: 6.4308, Train Steps/Sec: 0.85 +[2025-04-22 06:19:53] (step=0010375) Train Loss: 6.4186, Train Steps/Sec: 0.85 +[2025-04-22 06:20:23] (step=0010400) Train Loss: 6.4604, Train Steps/Sec: 0.85 +[2025-04-22 06:20:52] (step=0010425) Train Loss: 6.4354, Train Steps/Sec: 0.85 +[2025-04-22 06:21:22] (step=0010450) Train Loss: 6.4488, Train Steps/Sec: 0.84 +[2025-04-22 06:21:52] (step=0010475) Train Loss: 6.4478, Train Steps/Sec: 0.84 +[2025-04-22 06:22:21] (step=0010500) Train Loss: 6.4293, Train Steps/Sec: 0.84 +[2025-04-22 06:22:51] (step=0010525) Train Loss: 6.4261, Train Steps/Sec: 0.84 +[2025-04-22 06:23:21] (step=0010550) Train Loss: 6.4337, Train Steps/Sec: 0.84 +[2025-04-22 06:23:50] (step=0010575) Train Loss: 6.4539, Train Steps/Sec: 0.85 +[2025-04-22 06:24:20] (step=0010600) Train Loss: 6.4147, Train Steps/Sec: 0.84 +[2025-04-22 06:24:50] (step=0010625) Train Loss: 6.4386, Train Steps/Sec: 0.85 +[2025-04-22 06:25:19] (step=0010650) Train Loss: 6.4373, Train Steps/Sec: 0.85 +[2025-04-22 06:25:49] (step=0010675) Train Loss: 6.4343, Train Steps/Sec: 0.85 +[2025-04-22 06:26:19] (step=0010700) Train Loss: 6.4245, Train Steps/Sec: 0.84 +[2025-04-22 06:26:48] (step=0010725) Train Loss: 6.3789, Train Steps/Sec: 0.85 +[2025-04-22 06:27:18] (step=0010750) Train Loss: 6.4271, Train Steps/Sec: 0.84 +[2025-04-22 06:27:48] (step=0010775) Train Loss: 6.4291, Train Steps/Sec: 0.84 +[2025-04-22 06:28:17] (step=0010800) Train Loss: 6.4098, Train Steps/Sec: 0.84 +[2025-04-22 06:28:47] (step=0010825) Train Loss: 6.4728, Train Steps/Sec: 0.84 +[2025-04-22 06:29:16] (step=0010850) Train Loss: 6.4141, Train Steps/Sec: 0.85 +[2025-04-22 06:29:52] (step=0010875) Train Loss: 6.3910, Train Steps/Sec: 0.70 +[2025-04-22 06:30:22] (step=0010900) Train Loss: 6.4552, Train Steps/Sec: 0.84 +[2025-04-22 06:30:52] (step=0010925) Train Loss: 6.4424, Train Steps/Sec: 0.84 +[2025-04-22 06:31:21] (step=0010950) Train Loss: 6.4464, Train Steps/Sec: 0.84 +[2025-04-22 06:31:51] (step=0010975) Train Loss: 6.3941, Train Steps/Sec: 0.85 +[2025-04-22 06:32:21] (step=0011000) Train Loss: 6.4556, Train Steps/Sec: 0.84 +[2025-04-22 06:32:50] (step=0011025) Train Loss: 6.4135, Train Steps/Sec: 0.84 +[2025-04-22 06:33:20] (step=0011050) Train Loss: 6.3777, Train Steps/Sec: 0.84 +[2025-04-22 06:33:50] (step=0011075) Train Loss: 6.4229, Train Steps/Sec: 0.85 +[2025-04-22 06:34:19] (step=0011100) Train Loss: 6.4202, Train Steps/Sec: 0.84 +[2025-04-22 06:34:49] (step=0011125) Train Loss: 6.4361, Train Steps/Sec: 0.84 +[2025-04-22 06:35:19] (step=0011150) Train Loss: 6.4084, Train Steps/Sec: 0.85 +[2025-04-22 06:35:48] (step=0011175) Train Loss: 6.4532, Train Steps/Sec: 0.85 +[2025-04-22 06:36:18] (step=0011200) Train Loss: 6.4345, Train Steps/Sec: 0.84 +[2025-04-22 06:36:47] (step=0011225) Train Loss: 6.3897, Train Steps/Sec: 0.85 +[2025-04-22 06:37:17] (step=0011250) Train Loss: 6.4212, Train Steps/Sec: 0.84 +[2025-04-22 06:37:47] (step=0011275) Train Loss: 6.4085, Train Steps/Sec: 0.85 +[2025-04-22 06:38:16] (step=0011300) Train Loss: 6.3998, Train Steps/Sec: 0.84 +[2025-04-22 06:38:46] (step=0011325) Train Loss: 6.4308, Train Steps/Sec: 0.84 +[2025-04-22 06:39:15] (step=0011350) Train Loss: 6.4311, Train Steps/Sec: 0.84 +[2025-04-22 06:39:45] (step=0011375) Train Loss: 6.4183, Train Steps/Sec: 0.85 +[2025-04-22 06:40:15] (step=0011400) Train Loss: 6.4378, Train Steps/Sec: 0.84 +[2025-04-22 06:40:44] (step=0011425) Train Loss: 6.4317, Train Steps/Sec: 0.84 +[2025-04-22 06:41:14] (step=0011450) Train Loss: 6.4018, Train Steps/Sec: 0.84 +[2025-04-22 06:41:44] (step=0011475) Train Loss: 6.4352, Train Steps/Sec: 0.85 +[2025-04-22 06:42:13] (step=0011500) Train Loss: 6.3977, Train Steps/Sec: 0.85 +[2025-04-22 06:42:43] (step=0011525) Train Loss: 6.4193, Train Steps/Sec: 0.85 +[2025-04-22 06:43:12] (step=0011550) Train Loss: 6.4396, Train Steps/Sec: 0.84 +[2025-04-22 06:43:42] (step=0011575) Train Loss: 6.4451, Train Steps/Sec: 0.84 +[2025-04-22 06:44:12] (step=0011600) Train Loss: 6.3761, Train Steps/Sec: 0.84 +[2025-04-22 06:44:41] (step=0011625) Train Loss: 6.3951, Train Steps/Sec: 0.85 +[2025-04-22 06:45:11] (step=0011650) Train Loss: 6.4014, Train Steps/Sec: 0.85 +[2025-04-22 06:45:40] (step=0011675) Train Loss: 6.4283, Train Steps/Sec: 0.85 +[2025-04-22 06:46:10] (step=0011700) Train Loss: 6.4254, Train Steps/Sec: 0.84 +[2025-04-22 06:46:40] (step=0011725) Train Loss: 6.4144, Train Steps/Sec: 0.84 +[2025-04-22 06:47:09] (step=0011750) Train Loss: 6.4357, Train Steps/Sec: 0.84 +[2025-04-22 06:47:39] (step=0011775) Train Loss: 6.4305, Train Steps/Sec: 0.85 +[2025-04-22 06:48:09] (step=0011800) Train Loss: 6.4306, Train Steps/Sec: 0.84 +[2025-04-22 06:48:38] (step=0011825) Train Loss: 6.4221, Train Steps/Sec: 0.85 +[2025-04-22 06:49:08] (step=0011850) Train Loss: 6.4638, Train Steps/Sec: 0.85 +[2025-04-22 06:49:37] (step=0011875) Train Loss: 6.4034, Train Steps/Sec: 0.85 +[2025-04-22 06:50:07] (step=0011900) Train Loss: 6.4011, Train Steps/Sec: 0.84 +[2025-04-22 06:50:36] (step=0011925) Train Loss: 6.4348, Train Steps/Sec: 0.85 +[2025-04-22 06:51:06] (step=0011950) Train Loss: 6.4666, Train Steps/Sec: 0.84 +[2025-04-22 06:51:36] (step=0011975) Train Loss: 6.4195, Train Steps/Sec: 0.85 +[2025-04-22 06:52:05] (step=0012000) Train Loss: 6.4212, Train Steps/Sec: 0.84 +[2025-04-22 06:56:46] Finish Eval in 12000 steps... +[2025-04-22 06:57:07] Saved checkpoint to checkpoint/BeforeFix_Blip_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_objects/002-GPT-XL/checkpoints/0012000.pt +[2025-04-22 06:57:09] Removed old checkpoint: checkpoint/BeforeFix_Blip_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_objects/002-GPT-XL/checkpoints/0008000.pt +[2025-04-22 06:57:39] (step=0012025) Train Loss: 6.4100, Train Steps/Sec: 0.07 +[2025-04-22 06:58:09] (step=0012050) Train Loss: 6.4286, Train Steps/Sec: 0.84 +[2025-04-22 06:58:38] (step=0012075) Train Loss: 6.4226, Train Steps/Sec: 0.84 +[2025-04-22 06:59:08] (step=0012100) Train Loss: 6.3808, Train Steps/Sec: 0.84 +[2025-04-22 06:59:37] (step=0012125) Train Loss: 6.4030, Train Steps/Sec: 0.85 +[2025-04-22 07:00:07] (step=0012150) Train Loss: 6.4008, Train Steps/Sec: 0.85 +[2025-04-22 07:00:37] (step=0012175) Train Loss: 6.4065, Train Steps/Sec: 0.84 +[2025-04-22 07:01:06] (step=0012200) Train Loss: 6.4182, Train Steps/Sec: 0.84 +[2025-04-22 07:01:36] (step=0012225) Train Loss: 6.4563, Train Steps/Sec: 0.84 +[2025-04-22 07:02:05] (step=0012250) Train Loss: 6.4354, Train Steps/Sec: 0.85 +[2025-04-22 07:02:35] (step=0012275) Train Loss: 6.4330, Train Steps/Sec: 0.85 +[2025-04-22 07:03:05] (step=0012300) Train Loss: 6.4426, Train Steps/Sec: 0.85 +[2025-04-22 07:03:34] (step=0012325) Train Loss: 6.4052, Train Steps/Sec: 0.84 +[2025-04-22 07:04:04] (step=0012350) Train Loss: 6.3799, Train Steps/Sec: 0.85 +[2025-04-22 07:04:33] (step=0012375) Train Loss: 6.4193, Train Steps/Sec: 0.85 +[2025-04-22 07:05:03] (step=0012400) Train Loss: 6.4232, Train Steps/Sec: 0.84 +[2025-04-22 07:05:33] (step=0012425) Train Loss: 6.4487, Train Steps/Sec: 0.84 +[2025-04-22 07:06:02] (step=0012450) Train Loss: 6.4128, Train Steps/Sec: 0.85 +[2025-04-22 07:06:32] (step=0012475) Train Loss: 6.4063, Train Steps/Sec: 0.84 +[2025-04-22 07:07:01] (step=0012500) Train Loss: 6.4220, Train Steps/Sec: 0.84 +[2025-04-22 07:07:31] (step=0012525) Train Loss: 6.4428, Train Steps/Sec: 0.85 +[2025-04-22 07:08:01] (step=0012550) Train Loss: 6.4673, Train Steps/Sec: 0.84 +[2025-04-22 07:08:30] (step=0012575) Train Loss: 6.4416, Train Steps/Sec: 0.84 +[2025-04-22 07:09:00] (step=0012600) Train Loss: 6.4433, Train Steps/Sec: 0.84 +[2025-04-22 07:09:29] (step=0012625) Train Loss: 6.4512, Train Steps/Sec: 0.85 +[2025-04-22 07:09:59] (step=0012650) Train Loss: 6.3996, Train Steps/Sec: 0.84 +[2025-04-22 07:10:29] (step=0012675) Train Loss: 6.4061, Train Steps/Sec: 0.85 +[2025-04-22 07:10:58] (step=0012700) Train Loss: 6.4391, Train Steps/Sec: 0.84 +[2025-04-22 07:11:28] (step=0012725) Train Loss: 6.3917, Train Steps/Sec: 0.85 +[2025-04-22 07:11:57] (step=0012750) Train Loss: 6.4380, Train Steps/Sec: 0.85 +[2025-04-22 07:12:27] (step=0012775) Train Loss: 6.4031, Train Steps/Sec: 0.85 +[2025-04-22 07:12:57] (step=0012800) Train Loss: 6.4686, Train Steps/Sec: 0.84 +[2025-04-22 07:13:26] (step=0012825) Train Loss: 6.4426, Train Steps/Sec: 0.85 +[2025-04-22 07:13:56] (step=0012850) Train Loss: 6.4520, Train Steps/Sec: 0.85 +[2025-04-22 07:14:25] (step=0012875) Train Loss: 6.4792, Train Steps/Sec: 0.85 +[2025-04-22 07:14:55] (step=0012900) Train Loss: 6.4546, Train Steps/Sec: 0.84 +[2025-04-22 07:15:25] (step=0012925) Train Loss: 6.4188, Train Steps/Sec: 0.84 +[2025-04-22 07:15:54] (step=0012950) Train Loss: 6.4538, Train Steps/Sec: 0.85 +[2025-04-22 07:16:24] (step=0012975) Train Loss: 6.4548, Train Steps/Sec: 0.84 +[2025-04-22 07:16:53] (step=0013000) Train Loss: 6.4355, Train Steps/Sec: 0.85 +[2025-04-22 07:17:23] (step=0013025) Train Loss: 6.4222, Train Steps/Sec: 0.85 +[2025-04-22 07:17:52] (step=0013050) Train Loss: 6.4359, Train Steps/Sec: 0.84 +[2025-04-22 07:18:22] (step=0013075) Train Loss: 6.3431, Train Steps/Sec: 0.85 +[2025-04-22 07:18:52] (step=0013100) Train Loss: 6.4442, Train Steps/Sec: 0.85 +[2025-04-22 07:19:21] (step=0013125) Train Loss: 6.4116, Train Steps/Sec: 0.85 +[2025-04-22 07:19:51] (step=0013150) Train Loss: 6.4290, Train Steps/Sec: 0.85 +[2025-04-22 07:20:20] (step=0013175) Train Loss: 6.3950, Train Steps/Sec: 0.85 +[2025-04-22 07:20:50] (step=0013200) Train Loss: 6.4166, Train Steps/Sec: 0.84 +[2025-04-22 07:21:19] (step=0013225) Train Loss: 6.3974, Train Steps/Sec: 0.85 +[2025-04-22 07:21:49] (step=0013250) Train Loss: 6.4241, Train Steps/Sec: 0.85 +[2025-04-22 07:22:18] (step=0013275) Train Loss: 6.4108, Train Steps/Sec: 0.85 +[2025-04-22 07:22:48] (step=0013300) Train Loss: 6.4200, Train Steps/Sec: 0.84 +[2025-04-22 07:23:18] (step=0013325) Train Loss: 6.4424, Train Steps/Sec: 0.85 +[2025-04-22 07:23:47] (step=0013350) Train Loss: 6.4253, Train Steps/Sec: 0.84 +[2025-04-22 07:24:17] (step=0013375) Train Loss: 6.4466, Train Steps/Sec: 0.85 +[2025-04-22 07:24:46] (step=0013400) Train Loss: 6.4001, Train Steps/Sec: 0.84 +[2025-04-22 07:25:16] (step=0013425) Train Loss: 6.4324, Train Steps/Sec: 0.85 +[2025-04-22 07:25:46] (step=0013450) Train Loss: 6.4272, Train Steps/Sec: 0.85 +[2025-04-22 07:26:15] (step=0013475) Train Loss: 6.4492, Train Steps/Sec: 0.84 +[2025-04-22 07:26:45] (step=0013500) Train Loss: 6.4020, Train Steps/Sec: 0.84 +[2025-04-22 07:27:14] (step=0013525) Train Loss: 6.3503, Train Steps/Sec: 0.85 +[2025-04-22 07:27:44] (step=0013550) Train Loss: 6.4107, Train Steps/Sec: 0.84 +[2025-04-22 07:28:14] (step=0013575) Train Loss: 6.3923, Train Steps/Sec: 0.85 +[2025-04-22 07:28:43] (step=0013600) Train Loss: 6.4483, Train Steps/Sec: 0.84 +[2025-04-22 07:29:13] (step=0013625) Train Loss: 6.4241, Train Steps/Sec: 0.84 +[2025-04-22 07:29:43] (step=0013650) Train Loss: 6.4074, Train Steps/Sec: 0.84 +[2025-04-22 07:30:12] (step=0013675) Train Loss: 6.4610, Train Steps/Sec: 0.85 +[2025-04-22 07:30:42] (step=0013700) Train Loss: 6.4352, Train Steps/Sec: 0.84 +[2025-04-22 07:31:11] (step=0013725) Train Loss: 6.4173, Train Steps/Sec: 0.85 +[2025-04-22 07:31:41] (step=0013750) Train Loss: 6.4301, Train Steps/Sec: 0.84 +[2025-04-22 07:32:11] (step=0013775) Train Loss: 6.4418, Train Steps/Sec: 0.84 +[2025-04-22 07:32:41] (step=0013800) Train Loss: 6.4124, Train Steps/Sec: 0.84 +[2025-04-22 07:33:10] (step=0013825) Train Loss: 6.4055, Train Steps/Sec: 0.85 +[2025-04-22 07:33:40] (step=0013850) Train Loss: 6.4105, Train Steps/Sec: 0.84 +[2025-04-22 07:34:09] (step=0013875) Train Loss: 6.3948, Train Steps/Sec: 0.85 +[2025-04-22 07:34:39] (step=0013900) Train Loss: 6.4106, Train Steps/Sec: 0.85 +[2025-04-22 07:35:08] (step=0013925) Train Loss: 6.4258, Train Steps/Sec: 0.85 +[2025-04-22 07:35:38] (step=0013950) Train Loss: 6.3876, Train Steps/Sec: 0.85 +[2025-04-22 07:36:08] (step=0013975) Train Loss: 6.4621, Train Steps/Sec: 0.84 +[2025-04-22 07:36:37] (step=0014000) Train Loss: 6.3985, Train Steps/Sec: 0.84 +[2025-04-22 07:41:18] Finish Eval in 14000 steps... +[2025-04-22 07:41:39] Saved checkpoint to checkpoint/BeforeFix_Blip_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_objects/002-GPT-XL/checkpoints/0014000.pt +[2025-04-22 07:41:41] Removed old checkpoint: checkpoint/BeforeFix_Blip_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_objects/002-GPT-XL/checkpoints/0010000.pt +[2025-04-22 07:42:11] (step=0014025) Train Loss: 6.4693, Train Steps/Sec: 0.07 +[2025-04-22 07:42:47] (step=0014050) Train Loss: 6.4313, Train Steps/Sec: 0.69 +[2025-04-22 07:43:17] (step=0014075) Train Loss: 6.4180, Train Steps/Sec: 0.85 +[2025-04-22 07:43:47] (step=0014100) Train Loss: 6.4193, Train Steps/Sec: 0.84 +[2025-04-22 07:44:16] (step=0014125) Train Loss: 6.4037, Train Steps/Sec: 0.84 +[2025-04-22 07:44:46] (step=0014150) Train Loss: 6.4190, Train Steps/Sec: 0.85 +[2025-04-22 07:45:15] (step=0014175) Train Loss: 6.4553, Train Steps/Sec: 0.85 +[2025-04-22 07:45:45] (step=0014200) Train Loss: 6.4538, Train Steps/Sec: 0.84 +[2025-04-22 07:46:15] (step=0014225) Train Loss: 6.4411, Train Steps/Sec: 0.85 +[2025-04-22 07:46:44] (step=0014250) Train Loss: 6.4285, Train Steps/Sec: 0.85 +[2025-04-22 07:47:14] (step=0014275) Train Loss: 6.4339, Train Steps/Sec: 0.85 +[2025-04-22 07:47:43] (step=0014300) Train Loss: 6.3886, Train Steps/Sec: 0.84 +[2025-04-22 07:48:19] (step=0014325) Train Loss: 6.4338, Train Steps/Sec: 0.70 +[2025-04-22 07:48:49] (step=0014350) Train Loss: 6.4192, Train Steps/Sec: 0.84 +[2025-04-22 07:49:18] (step=0014375) Train Loss: 6.4338, Train Steps/Sec: 0.85 +[2025-04-22 07:49:55] (step=0014400) Train Loss: 6.4285, Train Steps/Sec: 0.69 +[2025-04-22 07:50:24] (step=0014425) Train Loss: 6.4423, Train Steps/Sec: 0.84 +[2025-04-22 07:50:54] (step=0014450) Train Loss: 6.4259, Train Steps/Sec: 0.84 +[2025-04-22 07:51:34] (step=0014475) Train Loss: 6.4280, Train Steps/Sec: 0.62 +[2025-04-22 07:52:04] (step=0014500) Train Loss: 6.4153, Train Steps/Sec: 0.84 +[2025-04-22 07:52:34] (step=0014525) Train Loss: 6.4609, Train Steps/Sec: 0.84 +[2025-04-22 07:53:03] (step=0014550) Train Loss: 6.4032, Train Steps/Sec: 0.85 +[2025-04-22 07:53:45] (step=0014575) Train Loss: 6.4299, Train Steps/Sec: 0.60 +[2025-04-22 07:54:21] (step=0014600) Train Loss: 6.3866, Train Steps/Sec: 0.69 +[2025-04-22 07:54:51] (step=0014625) Train Loss: 6.4117, Train Steps/Sec: 0.85 +[2025-04-22 07:55:20] (step=0014650) Train Loss: 6.4182, Train Steps/Sec: 0.84 +[2025-04-22 07:55:50] (step=0014675) Train Loss: 6.4219, Train Steps/Sec: 0.85 +[2025-04-22 07:56:19] (step=0014700) Train Loss: 6.3981, Train Steps/Sec: 0.84 +[2025-04-22 07:56:49] (step=0014725) Train Loss: 6.4415, Train Steps/Sec: 0.85 +[2025-04-22 07:57:19] (step=0014750) Train Loss: 6.4521, Train Steps/Sec: 0.85 +[2025-04-22 07:57:48] (step=0014775) Train Loss: 6.4300, Train Steps/Sec: 0.85 +[2025-04-22 07:58:18] (step=0014800) Train Loss: 6.4256, Train Steps/Sec: 0.85 +[2025-04-22 07:58:47] (step=0014825) Train Loss: 6.4434, Train Steps/Sec: 0.85 +[2025-04-22 07:59:17] (step=0014850) Train Loss: 6.4050, Train Steps/Sec: 0.85 +[2025-04-22 07:59:46] (step=0014875) Train Loss: 6.4632, Train Steps/Sec: 0.85 +[2025-04-22 08:00:16] (step=0014900) Train Loss: 6.4066, Train Steps/Sec: 0.84 +[2025-04-22 08:00:45] (step=0014925) Train Loss: 6.4634, Train Steps/Sec: 0.85 +[2025-04-22 08:01:15] (step=0014950) Train Loss: 6.3919, Train Steps/Sec: 0.85 +[2025-04-22 08:01:44] (step=0014975) Train Loss: 6.4298, Train Steps/Sec: 0.85 +[2025-04-22 08:02:14] (step=0015000) Train Loss: 6.4106, Train Steps/Sec: 0.84 +[2025-04-22 08:02:44] (step=0015025) Train Loss: 6.4176, Train Steps/Sec: 0.85 +[2025-04-22 08:03:13] (step=0015050) Train Loss: 6.4434, Train Steps/Sec: 0.85 +[2025-04-22 08:03:43] (step=0015075) Train Loss: 6.4042, Train Steps/Sec: 0.85 +[2025-04-22 08:04:12] (step=0015100) Train Loss: 6.4115, Train Steps/Sec: 0.84 +[2025-04-22 08:04:42] (step=0015125) Train Loss: 6.4401, Train Steps/Sec: 0.85 +[2025-04-22 08:05:11] (step=0015150) Train Loss: 6.4093, Train Steps/Sec: 0.85 +[2025-04-22 08:05:41] (step=0015175) Train Loss: 6.4221, Train Steps/Sec: 0.85 +[2025-04-22 08:06:10] (step=0015200) Train Loss: 6.4515, Train Steps/Sec: 0.85 +[2025-04-22 08:06:40] (step=0015225) Train Loss: 6.3716, Train Steps/Sec: 0.85 +[2025-04-22 08:07:09] (step=0015250) Train Loss: 6.4346, Train Steps/Sec: 0.85 +[2025-04-22 08:07:39] (step=0015275) Train Loss: 6.3893, Train Steps/Sec: 0.85 +[2025-04-22 08:08:08] (step=0015300) Train Loss: 6.4593, Train Steps/Sec: 0.84 +[2025-04-22 08:08:38] (step=0015325) Train Loss: 6.4252, Train Steps/Sec: 0.85 +[2025-04-22 08:09:07] (step=0015350) Train Loss: 6.3844, Train Steps/Sec: 0.85 +[2025-04-22 08:09:37] (step=0015375) Train Loss: 6.4388, Train Steps/Sec: 0.85 +[2025-04-22 08:10:06] (step=0015400) Train Loss: 6.4044, Train Steps/Sec: 0.84 +[2025-04-22 08:10:36] (step=0015425) Train Loss: 6.4009, Train Steps/Sec: 0.85 +[2025-04-22 08:11:05] (step=0015450) Train Loss: 6.4127, Train Steps/Sec: 0.85 +[2025-04-22 08:11:35] (step=0015475) Train Loss: 6.4388, Train Steps/Sec: 0.85 +[2025-04-22 08:12:05] (step=0015500) Train Loss: 6.4290, Train Steps/Sec: 0.84 +[2025-04-22 08:12:34] (step=0015525) Train Loss: 6.4105, Train Steps/Sec: 0.85 +[2025-04-22 08:13:04] (step=0015550) Train Loss: 6.3995, Train Steps/Sec: 0.85 +[2025-04-22 08:13:33] (step=0015575) Train Loss: 6.3927, Train Steps/Sec: 0.85 +[2025-04-22 08:14:03] (step=0015600) Train Loss: 6.4137, Train Steps/Sec: 0.84 +[2025-04-22 08:14:32] (step=0015625) Train Loss: 6.4166, Train Steps/Sec: 0.84 +[2025-04-22 08:15:02] (step=0015650) Train Loss: 6.4215, Train Steps/Sec: 0.84 +[2025-04-22 08:15:32] (step=0015675) Train Loss: 6.4388, Train Steps/Sec: 0.84 +[2025-04-22 08:16:01] (step=0015700) Train Loss: 6.4517, Train Steps/Sec: 0.84 +[2025-04-22 08:16:31] (step=0015725) Train Loss: 6.4121, Train Steps/Sec: 0.84 +[2025-04-22 08:17:01] (step=0015750) Train Loss: 6.4222, Train Steps/Sec: 0.84 +[2025-04-22 08:17:30] (step=0015775) Train Loss: 6.4053, Train Steps/Sec: 0.84 +[2025-04-22 08:18:00] (step=0015800) Train Loss: 6.3975, Train Steps/Sec: 0.84 +[2025-04-22 08:18:30] (step=0015825) Train Loss: 6.4017, Train Steps/Sec: 0.84 +[2025-04-22 08:18:59] (step=0015850) Train Loss: 6.4214, Train Steps/Sec: 0.84 +[2025-04-22 08:19:29] (step=0015875) Train Loss: 6.4691, Train Steps/Sec: 0.84 +[2025-04-22 08:19:59] (step=0015900) Train Loss: 6.4200, Train Steps/Sec: 0.84 +[2025-04-22 08:20:29] (step=0015925) Train Loss: 6.3781, Train Steps/Sec: 0.85 +[2025-04-22 08:20:58] (step=0015950) Train Loss: 6.4423, Train Steps/Sec: 0.84 +[2025-04-22 08:21:28] (step=0015975) Train Loss: 6.4409, Train Steps/Sec: 0.85 +[2025-04-22 08:21:58] (step=0016000) Train Loss: 6.4469, Train Steps/Sec: 0.84 +[2025-04-22 08:26:38] Finish Eval in 16000 steps... +[2025-04-22 08:26:59] Saved checkpoint to checkpoint/BeforeFix_Blip_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_objects/002-GPT-XL/checkpoints/0016000.pt +[2025-04-22 08:27:01] Removed old checkpoint: checkpoint/BeforeFix_Blip_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_objects/002-GPT-XL/checkpoints/0012000.pt +[2025-04-22 08:27:31] (step=0016025) Train Loss: 6.3975, Train Steps/Sec: 0.08 +[2025-04-22 08:28:00] (step=0016050) Train Loss: 6.4103, Train Steps/Sec: 0.84 +[2025-04-22 08:28:30] (step=0016075) Train Loss: 6.3994, Train Steps/Sec: 0.85 +[2025-04-22 08:29:00] (step=0016100) Train Loss: 6.4197, Train Steps/Sec: 0.84 +[2025-04-22 08:29:29] (step=0016125) Train Loss: 6.4137, Train Steps/Sec: 0.85 +[2025-04-22 08:29:59] (step=0016150) Train Loss: 6.4180, Train Steps/Sec: 0.84 +[2025-04-22 08:30:29] (step=0016175) Train Loss: 6.4072, Train Steps/Sec: 0.85 +[2025-04-22 08:30:58] (step=0016200) Train Loss: 6.4198, Train Steps/Sec: 0.85 +[2025-04-22 08:31:34] (step=0016225) Train Loss: 6.4463, Train Steps/Sec: 0.70 +[2025-04-22 08:32:04] (step=0016250) Train Loss: 6.4450, Train Steps/Sec: 0.84 +[2025-04-22 08:32:33] (step=0016275) Train Loss: 6.3947, Train Steps/Sec: 0.85 +[2025-04-22 08:33:03] (step=0016300) Train Loss: 6.4278, Train Steps/Sec: 0.84 +[2025-04-22 08:33:33] (step=0016325) Train Loss: 6.4104, Train Steps/Sec: 0.85 +[2025-04-22 08:34:02] (step=0016350) Train Loss: 6.4286, Train Steps/Sec: 0.85 +[2025-04-22 08:34:32] (step=0016375) Train Loss: 6.3971, Train Steps/Sec: 0.85 +[2025-04-22 08:35:01] (step=0016400) Train Loss: 6.4306, Train Steps/Sec: 0.84 +[2025-04-22 08:35:31] (step=0016425) Train Loss: 6.4125, Train Steps/Sec: 0.85 +[2025-04-22 08:36:00] (step=0016450) Train Loss: 6.4407, Train Steps/Sec: 0.84 +[2025-04-22 08:36:30] (step=0016475) Train Loss: 6.4459, Train Steps/Sec: 0.85 +[2025-04-22 08:37:00] (step=0016500) Train Loss: 6.4376, Train Steps/Sec: 0.84 +[2025-04-22 08:37:29] (step=0016525) Train Loss: 6.4227, Train Steps/Sec: 0.85 +[2025-04-22 08:37:59] (step=0016550) Train Loss: 6.3922, Train Steps/Sec: 0.84 +[2025-04-22 08:38:28] (step=0016575) Train Loss: 6.4006, Train Steps/Sec: 0.85 +[2025-04-22 08:38:58] (step=0016600) Train Loss: 6.4488, Train Steps/Sec: 0.85 +[2025-04-22 08:39:28] (step=0016625) Train Loss: 6.4136, Train Steps/Sec: 0.84 +[2025-04-22 08:39:57] (step=0016650) Train Loss: 6.4326, Train Steps/Sec: 0.84 +[2025-04-22 08:40:27] (step=0016675) Train Loss: 6.4264, Train Steps/Sec: 0.84 +[2025-04-22 08:40:57] (step=0016700) Train Loss: 6.4254, Train Steps/Sec: 0.84 +[2025-04-22 08:41:26] (step=0016725) Train Loss: 6.4271, Train Steps/Sec: 0.85 +[2025-04-22 08:41:56] (step=0016750) Train Loss: 6.4501, Train Steps/Sec: 0.85 +[2025-04-22 08:42:25] (step=0016775) Train Loss: 6.4277, Train Steps/Sec: 0.85 +[2025-04-22 08:42:55] (step=0016800) Train Loss: 6.4276, Train Steps/Sec: 0.84 +[2025-04-22 08:43:24] (step=0016825) Train Loss: 6.4302, Train Steps/Sec: 0.85 +[2025-04-22 08:43:54] (step=0016850) Train Loss: 6.3940, Train Steps/Sec: 0.85 +[2025-04-22 08:44:23] (step=0016875) Train Loss: 6.4270, Train Steps/Sec: 0.85 +[2025-04-22 08:44:53] (step=0016900) Train Loss: 6.4067, Train Steps/Sec: 0.84 +[2025-04-22 08:45:23] (step=0016925) Train Loss: 6.3878, Train Steps/Sec: 0.85 +[2025-04-22 08:45:53] (step=0016950) Train Loss: 6.4024, Train Steps/Sec: 0.84 +[2025-04-22 08:46:22] (step=0016975) Train Loss: 6.3771, Train Steps/Sec: 0.84 +[2025-04-22 08:46:52] (step=0017000) Train Loss: 6.4073, Train Steps/Sec: 0.85 +[2025-04-22 08:47:21] (step=0017025) Train Loss: 6.4490, Train Steps/Sec: 0.84 +[2025-04-22 08:47:51] (step=0017050) Train Loss: 6.4306, Train Steps/Sec: 0.85 +[2025-04-22 08:48:20] (step=0017075) Train Loss: 6.4536, Train Steps/Sec: 0.85 +[2025-04-22 08:48:50] (step=0017100) Train Loss: 6.4575, Train Steps/Sec: 0.84 +[2025-04-22 08:49:19] (step=0017125) Train Loss: 6.4303, Train Steps/Sec: 0.85 +[2025-04-22 08:49:49] (step=0017150) Train Loss: 6.4120, Train Steps/Sec: 0.84 +[2025-04-22 08:50:19] (step=0017175) Train Loss: 6.3969, Train Steps/Sec: 0.84 +[2025-04-22 08:50:49] (step=0017200) Train Loss: 6.4631, Train Steps/Sec: 0.84 +[2025-04-22 08:51:18] (step=0017225) Train Loss: 6.3727, Train Steps/Sec: 0.85 +[2025-04-22 08:51:48] (step=0017250) Train Loss: 6.3900, Train Steps/Sec: 0.85 +[2025-04-22 08:52:17] (step=0017275) Train Loss: 6.4548, Train Steps/Sec: 0.85 +[2025-04-22 08:52:47] (step=0017300) Train Loss: 6.4583, Train Steps/Sec: 0.84 +[2025-04-22 08:53:16] (step=0017325) Train Loss: 6.4130, Train Steps/Sec: 0.85 +[2025-04-22 08:53:46] (step=0017350) Train Loss: 6.4143, Train Steps/Sec: 0.85 +[2025-04-22 08:54:15] (step=0017375) Train Loss: 6.4463, Train Steps/Sec: 0.84 +[2025-04-22 08:54:45] (step=0017400) Train Loss: 6.3682, Train Steps/Sec: 0.84 +[2025-04-22 08:55:15] (step=0017425) Train Loss: 6.4001, Train Steps/Sec: 0.85 +[2025-04-22 08:55:44] (step=0017450) Train Loss: 6.4239, Train Steps/Sec: 0.85 +[2025-04-22 08:56:14] (step=0017475) Train Loss: 6.4068, Train Steps/Sec: 0.85 +[2025-04-22 08:56:44] (step=0017500) Train Loss: 6.4232, Train Steps/Sec: 0.83 +[2025-04-22 08:57:13] (step=0017525) Train Loss: 6.4551, Train Steps/Sec: 0.85 +[2025-04-22 08:57:43] (step=0017550) Train Loss: 6.4530, Train Steps/Sec: 0.85 +[2025-04-22 08:58:12] (step=0017575) Train Loss: 6.4207, Train Steps/Sec: 0.85 +[2025-04-22 08:58:42] (step=0017600) Train Loss: 6.4458, Train Steps/Sec: 0.85 +[2025-04-22 08:59:11] (step=0017625) Train Loss: 6.4037, Train Steps/Sec: 0.85 +[2025-04-22 08:59:41] (step=0017650) Train Loss: 6.4181, Train Steps/Sec: 0.85 +[2025-04-22 09:00:10] (step=0017675) Train Loss: 6.3989, Train Steps/Sec: 0.85 +[2025-04-22 09:00:40] (step=0017700) Train Loss: 6.4196, Train Steps/Sec: 0.84 +[2025-04-22 09:01:10] (step=0017725) Train Loss: 6.4124, Train Steps/Sec: 0.85 +[2025-04-22 09:01:39] (step=0017750) Train Loss: 6.4118, Train Steps/Sec: 0.84 +[2025-04-22 09:02:09] (step=0017775) Train Loss: 6.3962, Train Steps/Sec: 0.85 +[2025-04-22 09:02:38] (step=0017800) Train Loss: 6.4456, Train Steps/Sec: 0.84 +[2025-04-22 09:03:08] (step=0017825) Train Loss: 6.4216, Train Steps/Sec: 0.85 +[2025-04-22 09:03:37] (step=0017850) Train Loss: 6.4062, Train Steps/Sec: 0.85 +[2025-04-22 09:04:07] (step=0017875) Train Loss: 6.3928, Train Steps/Sec: 0.84 +[2025-04-22 09:04:37] (step=0017900) Train Loss: 6.4266, Train Steps/Sec: 0.84 +[2025-04-22 09:05:06] (step=0017925) Train Loss: 6.4278, Train Steps/Sec: 0.85 +[2025-04-22 09:05:36] (step=0017950) Train Loss: 6.4863, Train Steps/Sec: 0.84 +[2025-04-22 09:06:05] (step=0017975) Train Loss: 6.4138, Train Steps/Sec: 0.85 +[2025-04-22 09:06:35] (step=0018000) Train Loss: 6.3507, Train Steps/Sec: 0.85 +[2025-04-22 09:11:17] Finish Eval in 18000 steps... +[2025-04-22 09:11:39] Saved checkpoint to checkpoint/BeforeFix_Blip_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_objects/002-GPT-XL/checkpoints/0018000.pt +[2025-04-22 09:11:41] Removed old checkpoint: checkpoint/BeforeFix_Blip_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_objects/002-GPT-XL/checkpoints/0014000.pt +[2025-04-22 09:12:10] (step=0018025) Train Loss: 6.4199, Train Steps/Sec: 0.07 +[2025-04-22 09:12:40] (step=0018050) Train Loss: 6.3951, Train Steps/Sec: 0.85 +[2025-04-22 09:13:09] (step=0018075) Train Loss: 6.4203, Train Steps/Sec: 0.85 +[2025-04-22 09:13:39] (step=0018100) Train Loss: 6.4566, Train Steps/Sec: 0.85 +[2025-04-22 09:14:09] (step=0018125) Train Loss: 6.4027, Train Steps/Sec: 0.84 +[2025-04-22 09:14:38] (step=0018150) Train Loss: 6.4142, Train Steps/Sec: 0.85 +[2025-04-22 09:15:08] (step=0018175) Train Loss: 6.4036, Train Steps/Sec: 0.85 +[2025-04-22 09:15:37] (step=0018200) Train Loss: 6.3972, Train Steps/Sec: 0.85 +[2025-04-22 09:16:07] (step=0018225) Train Loss: 6.3905, Train Steps/Sec: 0.85 +[2025-04-22 09:16:36] (step=0018250) Train Loss: 6.4210, Train Steps/Sec: 0.85 +[2025-04-22 09:17:06] (step=0018275) Train Loss: 6.4290, Train Steps/Sec: 0.85 +[2025-04-22 09:17:35] (step=0018300) Train Loss: 6.3897, Train Steps/Sec: 0.85 +[2025-04-22 09:18:05] (step=0018325) Train Loss: 6.3823, Train Steps/Sec: 0.84 +[2025-04-22 09:18:34] (step=0018350) Train Loss: 6.4251, Train Steps/Sec: 0.85 +[2025-04-22 09:19:04] (step=0018375) Train Loss: 6.4132, Train Steps/Sec: 0.85 +[2025-04-22 09:19:33] (step=0018400) Train Loss: 6.4282, Train Steps/Sec: 0.84 +[2025-04-22 09:20:03] (step=0018425) Train Loss: 6.4270, Train Steps/Sec: 0.85 +[2025-04-22 09:20:32] (step=0018450) Train Loss: 6.4425, Train Steps/Sec: 0.84 +[2025-04-22 09:21:02] (step=0018475) Train Loss: 6.4016, Train Steps/Sec: 0.85 +[2025-04-22 09:21:32] (step=0018500) Train Loss: 6.3882, Train Steps/Sec: 0.84 +[2025-04-22 09:22:01] (step=0018525) Train Loss: 6.3979, Train Steps/Sec: 0.85 +[2025-04-22 09:22:31] (step=0018550) Train Loss: 6.4016, Train Steps/Sec: 0.84 +[2025-04-22 09:23:00] (step=0018575) Train Loss: 6.4785, Train Steps/Sec: 0.84 +[2025-04-22 09:23:30] (step=0018600) Train Loss: 6.4318, Train Steps/Sec: 0.85 +[2025-04-22 09:24:00] (step=0018625) Train Loss: 6.3953, Train Steps/Sec: 0.84 +[2025-04-22 09:24:29] (step=0018650) Train Loss: 6.4094, Train Steps/Sec: 0.84 +[2025-04-22 09:24:59] (step=0018675) Train Loss: 6.4348, Train Steps/Sec: 0.84 +[2025-04-22 09:25:28] (step=0018700) Train Loss: 6.3841, Train Steps/Sec: 0.85 +[2025-04-22 09:25:58] (step=0018725) Train Loss: 6.3804, Train Steps/Sec: 0.85 +[2025-04-22 09:26:28] (step=0018750) Train Loss: 6.4320, Train Steps/Sec: 0.84 +[2025-04-22 09:26:57] (step=0018775) Train Loss: 6.3998, Train Steps/Sec: 0.84 +[2025-04-22 09:27:27] (step=0018800) Train Loss: 6.4052, Train Steps/Sec: 0.84 +[2025-04-22 09:27:57] (step=0018825) Train Loss: 6.4713, Train Steps/Sec: 0.84 +[2025-04-22 09:28:26] (step=0018850) Train Loss: 6.4556, Train Steps/Sec: 0.84 +[2025-04-22 09:28:56] (step=0018875) Train Loss: 6.4426, Train Steps/Sec: 0.84 +[2025-04-22 09:29:26] (step=0018900) Train Loss: 6.4001, Train Steps/Sec: 0.85 +[2025-04-22 09:29:55] (step=0018925) Train Loss: 6.4218, Train Steps/Sec: 0.85 +[2025-04-22 09:30:25] (step=0018950) Train Loss: 6.4184, Train Steps/Sec: 0.84 +[2025-04-22 09:30:55] (step=0018975) Train Loss: 6.4011, Train Steps/Sec: 0.85 +[2025-04-22 09:31:24] (step=0019000) Train Loss: 6.3984, Train Steps/Sec: 0.85 +[2025-04-22 09:31:54] (step=0019025) Train Loss: 6.3947, Train Steps/Sec: 0.84 +[2025-04-22 09:32:23] (step=0019050) Train Loss: 6.4068, Train Steps/Sec: 0.85 +[2025-04-22 09:32:53] (step=0019075) Train Loss: 6.4159, Train Steps/Sec: 0.84 +[2025-04-22 09:33:23] (step=0019100) Train Loss: 6.4152, Train Steps/Sec: 0.84 +[2025-04-22 09:33:59] (step=0019125) Train Loss: 6.4129, Train Steps/Sec: 0.68 +[2025-04-22 09:34:29] (step=0019150) Train Loss: 6.3933, Train Steps/Sec: 0.85 +[2025-04-22 09:34:58] (step=0019175) Train Loss: 6.3753, Train Steps/Sec: 0.85 +[2025-04-22 09:35:28] (step=0019200) Train Loss: 6.4270, Train Steps/Sec: 0.83 +[2025-04-22 09:35:58] (step=0019225) Train Loss: 6.4468, Train Steps/Sec: 0.84 +[2025-04-22 09:36:27] (step=0019250) Train Loss: 6.4205, Train Steps/Sec: 0.85 +[2025-04-22 09:36:57] (step=0019275) Train Loss: 6.4459, Train Steps/Sec: 0.84 +[2025-04-22 09:37:27] (step=0019300) Train Loss: 6.3945, Train Steps/Sec: 0.84 +[2025-04-22 09:38:02] (step=0019325) Train Loss: 6.4342, Train Steps/Sec: 0.70 +[2025-04-22 09:38:32] (step=0019350) Train Loss: 6.4025, Train Steps/Sec: 0.84 +[2025-04-22 09:39:02] (step=0019375) Train Loss: 6.3957, Train Steps/Sec: 0.84 +[2025-04-22 09:39:31] (step=0019400) Train Loss: 6.4348, Train Steps/Sec: 0.84 +[2025-04-22 09:40:01] (step=0019425) Train Loss: 6.4035, Train Steps/Sec: 0.85 +[2025-04-22 09:40:30] (step=0019450) Train Loss: 6.4133, Train Steps/Sec: 0.85 +[2025-04-22 09:41:00] (step=0019475) Train Loss: 6.4025, Train Steps/Sec: 0.84 +[2025-04-22 09:41:30] (step=0019500) Train Loss: 6.4234, Train Steps/Sec: 0.84 +[2025-04-22 09:42:06] (step=0019525) Train Loss: 6.3994, Train Steps/Sec: 0.69 +[2025-04-22 09:42:36] (step=0019550) Train Loss: 6.4021, Train Steps/Sec: 0.85 +[2025-04-22 09:43:11] (step=0019575) Train Loss: 6.4701, Train Steps/Sec: 0.70 +[2025-04-22 09:43:41] (step=0019600) Train Loss: 6.3778, Train Steps/Sec: 0.85 +[2025-04-22 09:44:10] (step=0019625) Train Loss: 6.4504, Train Steps/Sec: 0.85 +[2025-04-22 09:44:40] (step=0019650) Train Loss: 6.4237, Train Steps/Sec: 0.84 +[2025-04-22 09:45:10] (step=0019675) Train Loss: 6.4298, Train Steps/Sec: 0.85 +[2025-04-22 09:45:39] (step=0019700) Train Loss: 6.3818, Train Steps/Sec: 0.85 +[2025-04-22 09:46:08] (step=0019725) Train Loss: 6.4118, Train Steps/Sec: 0.85 +[2025-04-22 09:46:38] (step=0019750) Train Loss: 6.4552, Train Steps/Sec: 0.85 +[2025-04-22 09:47:07] (step=0019775) Train Loss: 6.4310, Train Steps/Sec: 0.85 +[2025-04-22 09:47:37] (step=0019800) Train Loss: 6.3683, Train Steps/Sec: 0.84 +[2025-04-22 09:48:07] (step=0019825) Train Loss: 6.3848, Train Steps/Sec: 0.85 +[2025-04-22 09:48:36] (step=0019850) Train Loss: 6.4013, Train Steps/Sec: 0.85 +[2025-04-22 09:49:06] (step=0019875) Train Loss: 6.3684, Train Steps/Sec: 0.84 +[2025-04-22 09:49:35] (step=0019900) Train Loss: 6.4187, Train Steps/Sec: 0.85 +[2025-04-22 09:50:05] (step=0019925) Train Loss: 6.4052, Train Steps/Sec: 0.85 +[2025-04-22 09:50:34] (step=0019950) Train Loss: 6.4102, Train Steps/Sec: 0.84 +[2025-04-22 09:51:04] (step=0019975) Train Loss: 6.4265, Train Steps/Sec: 0.85 +[2025-04-22 09:51:34] (step=0020000) Train Loss: 6.4077, Train Steps/Sec: 0.84 +[2025-04-22 09:56:15] Finish Eval in 20000 steps... +[2025-04-22 09:56:36] Saved checkpoint to checkpoint/BeforeFix_Blip_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_objects/002-GPT-XL/checkpoints/0020000.pt +[2025-04-22 09:56:38] Removed old checkpoint: checkpoint/BeforeFix_Blip_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_objects/002-GPT-XL/checkpoints/0016000.pt +[2025-04-22 09:57:08] (step=0020025) Train Loss: 6.4494, Train Steps/Sec: 0.07 +[2025-04-22 09:57:46] (step=0020050) Train Loss: 6.4058, Train Steps/Sec: 0.65 +[2025-04-22 09:58:22] (step=0020075) Train Loss: 6.4320, Train Steps/Sec: 0.69 +[2025-04-22 09:58:52] (step=0020100) Train Loss: 6.4149, Train Steps/Sec: 0.84 +[2025-04-22 09:59:22] (step=0020125) Train Loss: 6.4273, Train Steps/Sec: 0.84 +[2025-04-22 09:59:51] (step=0020150) Train Loss: 6.4527, Train Steps/Sec: 0.84 +[2025-04-22 10:00:21] (step=0020175) Train Loss: 6.4179, Train Steps/Sec: 0.84 +[2025-04-22 10:00:51] (step=0020200) Train Loss: 6.4532, Train Steps/Sec: 0.84 +[2025-04-22 10:01:21] (step=0020225) Train Loss: 6.4045, Train Steps/Sec: 0.85 +[2025-04-22 10:01:50] (step=0020250) Train Loss: 6.3759, Train Steps/Sec: 0.85 +[2025-04-22 10:02:20] (step=0020275) Train Loss: 6.4772, Train Steps/Sec: 0.84 +[2025-04-22 10:02:50] (step=0020300) Train Loss: 6.4062, Train Steps/Sec: 0.84 +[2025-04-22 10:03:19] (step=0020325) Train Loss: 6.4402, Train Steps/Sec: 0.85 +[2025-04-22 10:03:49] (step=0020350) Train Loss: 6.4386, Train Steps/Sec: 0.84 +[2025-04-22 10:04:18] (step=0020375) Train Loss: 6.3914, Train Steps/Sec: 0.84 +[2025-04-22 10:04:48] (step=0020400) Train Loss: 6.3818, Train Steps/Sec: 0.84 +[2025-04-22 10:05:17] (step=0020425) Train Loss: 6.4102, Train Steps/Sec: 0.85 +[2025-04-22 10:05:47] (step=0020450) Train Loss: 6.4058, Train Steps/Sec: 0.85 +[2025-04-22 10:06:16] (step=0020475) Train Loss: 6.4087, Train Steps/Sec: 0.85 +[2025-04-22 10:06:46] (step=0020500) Train Loss: 6.3993, Train Steps/Sec: 0.84 +[2025-04-22 10:07:16] (step=0020525) Train Loss: 6.4536, Train Steps/Sec: 0.85 +[2025-04-22 10:07:45] (step=0020550) Train Loss: 6.3868, Train Steps/Sec: 0.84 +[2025-04-22 10:08:15] (step=0020575) Train Loss: 6.4093, Train Steps/Sec: 0.85 +[2025-04-22 10:08:44] (step=0020600) Train Loss: 6.3997, Train Steps/Sec: 0.85 +[2025-04-22 10:09:14] (step=0020625) Train Loss: 6.4593, Train Steps/Sec: 0.85 +[2025-04-22 10:09:43] (step=0020650) Train Loss: 6.3720, Train Steps/Sec: 0.85 +[2025-04-22 10:10:13] (step=0020675) Train Loss: 6.3769, Train Steps/Sec: 0.85 +[2025-04-22 10:10:42] (step=0020700) Train Loss: 6.3980, Train Steps/Sec: 0.84 +[2025-04-22 10:11:12] (step=0020725) Train Loss: 6.4174, Train Steps/Sec: 0.85 +[2025-04-22 10:11:42] (step=0020750) Train Loss: 6.4076, Train Steps/Sec: 0.84 +[2025-04-22 10:12:11] (step=0020775) Train Loss: 6.4132, Train Steps/Sec: 0.85 +[2025-04-22 10:12:41] (step=0020800) Train Loss: 6.4226, Train Steps/Sec: 0.84 +[2025-04-22 10:13:10] (step=0020825) Train Loss: 6.3607, Train Steps/Sec: 0.85 +[2025-04-22 10:13:40] (step=0020850) Train Loss: 6.3776, Train Steps/Sec: 0.85 +[2025-04-22 10:14:09] (step=0020875) Train Loss: 6.3997, Train Steps/Sec: 0.84 +[2025-04-22 10:14:39] (step=0020900) Train Loss: 6.4146, Train Steps/Sec: 0.85 +[2025-04-22 10:15:09] (step=0020925) Train Loss: 6.3945, Train Steps/Sec: 0.85 +[2025-04-22 10:15:38] (step=0020950) Train Loss: 6.4100, Train Steps/Sec: 0.85 +[2025-04-22 10:16:08] (step=0020975) Train Loss: 6.4670, Train Steps/Sec: 0.84 +[2025-04-22 10:16:37] (step=0021000) Train Loss: 6.3881, Train Steps/Sec: 0.84 +[2025-04-22 10:17:07] (step=0021025) Train Loss: 6.4519, Train Steps/Sec: 0.84 +[2025-04-22 10:17:36] (step=0021050) Train Loss: 6.4771, Train Steps/Sec: 0.85 +[2025-04-22 10:18:06] (step=0021075) Train Loss: 6.3956, Train Steps/Sec: 0.85 +[2025-04-22 10:18:36] (step=0021100) Train Loss: 6.4039, Train Steps/Sec: 0.84 +[2025-04-22 10:19:05] (step=0021125) Train Loss: 6.4277, Train Steps/Sec: 0.84 +[2025-04-22 10:19:35] (step=0021150) Train Loss: 6.3767, Train Steps/Sec: 0.84 +[2025-04-22 10:20:04] (step=0021175) Train Loss: 6.3923, Train Steps/Sec: 0.85 +[2025-04-22 10:20:34] (step=0021200) Train Loss: 6.3749, Train Steps/Sec: 0.84 +[2025-04-22 10:21:03] (step=0021225) Train Loss: 6.3918, Train Steps/Sec: 0.85 +[2025-04-22 10:21:33] (step=0021250) Train Loss: 6.4189, Train Steps/Sec: 0.84 +[2025-04-22 10:22:03] (step=0021275) Train Loss: 6.3996, Train Steps/Sec: 0.85 +[2025-04-22 10:22:32] (step=0021300) Train Loss: 6.4304, Train Steps/Sec: 0.85 +[2025-04-22 10:23:01] (step=0021325) Train Loss: 6.4015, Train Steps/Sec: 0.85 +[2025-04-22 10:23:31] (step=0021350) Train Loss: 6.4090, Train Steps/Sec: 0.85 +[2025-04-22 10:24:00] (step=0021375) Train Loss: 6.3574, Train Steps/Sec: 0.85 +[2025-04-22 10:24:30] (step=0021400) Train Loss: 6.4192, Train Steps/Sec: 0.84 +[2025-04-22 10:25:00] (step=0021425) Train Loss: 6.4071, Train Steps/Sec: 0.85 +[2025-04-22 10:25:29] (step=0021450) Train Loss: 6.4565, Train Steps/Sec: 0.85 +[2025-04-22 10:25:59] (step=0021475) Train Loss: 6.4412, Train Steps/Sec: 0.84 +[2025-04-22 10:26:28] (step=0021500) Train Loss: 6.4242, Train Steps/Sec: 0.84 +[2025-04-22 10:26:58] (step=0021525) Train Loss: 6.4115, Train Steps/Sec: 0.85 +[2025-04-22 10:27:34] (step=0021550) Train Loss: 6.3872, Train Steps/Sec: 0.70 +[2025-04-22 10:28:03] (step=0021575) Train Loss: 6.4205, Train Steps/Sec: 0.85 +[2025-04-22 10:28:33] (step=0021600) Train Loss: 6.4034, Train Steps/Sec: 0.84 +[2025-04-22 10:29:02] (step=0021625) Train Loss: 6.4141, Train Steps/Sec: 0.84 +[2025-04-22 10:29:32] (step=0021650) Train Loss: 6.4256, Train Steps/Sec: 0.84 +[2025-04-22 10:30:02] (step=0021675) Train Loss: 6.3826, Train Steps/Sec: 0.84 +[2025-04-22 10:30:31] (step=0021700) Train Loss: 6.3784, Train Steps/Sec: 0.84 +[2025-04-22 10:31:01] (step=0021725) Train Loss: 6.3879, Train Steps/Sec: 0.85 +[2025-04-22 10:31:30] (step=0021750) Train Loss: 6.3946, Train Steps/Sec: 0.85 +[2025-04-22 10:32:00] (step=0021775) Train Loss: 6.4270, Train Steps/Sec: 0.85 +[2025-04-22 10:32:29] (step=0021800) Train Loss: 6.3793, Train Steps/Sec: 0.85 +[2025-04-22 10:32:59] (step=0021825) Train Loss: 6.4283, Train Steps/Sec: 0.85 +[2025-04-22 10:33:29] (step=0021850) Train Loss: 6.4026, Train Steps/Sec: 0.85 +[2025-04-22 10:33:58] (step=0021875) Train Loss: 6.4121, Train Steps/Sec: 0.84 +[2025-04-22 10:34:28] (step=0021900) Train Loss: 6.4080, Train Steps/Sec: 0.84 +[2025-04-22 10:34:57] (step=0021925) Train Loss: 6.4465, Train Steps/Sec: 0.84 +[2025-04-22 10:35:27] (step=0021950) Train Loss: 6.3663, Train Steps/Sec: 0.84 +[2025-04-22 10:35:57] (step=0021975) Train Loss: 6.4225, Train Steps/Sec: 0.85 +[2025-04-22 10:36:26] (step=0022000) Train Loss: 6.4150, Train Steps/Sec: 0.84 +[2025-04-22 10:41:08] Finish Eval in 22000 steps... +[2025-04-22 10:41:29] Saved checkpoint to checkpoint/BeforeFix_Blip_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_objects/002-GPT-XL/checkpoints/0022000.pt +[2025-04-22 10:41:31] Removed old checkpoint: checkpoint/BeforeFix_Blip_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_objects/002-GPT-XL/checkpoints/0018000.pt +[2025-04-22 10:42:01] (step=0022025) Train Loss: 6.4093, Train Steps/Sec: 0.07 +[2025-04-22 10:42:30] (step=0022050) Train Loss: 6.4118, Train Steps/Sec: 0.85 +[2025-04-22 10:43:00] (step=0022075) Train Loss: 6.3897, Train Steps/Sec: 0.85 +[2025-04-22 10:43:29] (step=0022100) Train Loss: 6.3836, Train Steps/Sec: 0.84 +[2025-04-22 10:43:59] (step=0022125) Train Loss: 6.3467, Train Steps/Sec: 0.85 +[2025-04-22 10:44:29] (step=0022150) Train Loss: 6.4110, Train Steps/Sec: 0.84 +[2025-04-22 10:44:58] (step=0022175) Train Loss: 6.4934, Train Steps/Sec: 0.84 +[2025-04-22 10:45:28] (step=0022200) Train Loss: 6.3614, Train Steps/Sec: 0.84 +[2025-04-22 10:45:57] (step=0022225) Train Loss: 6.4127, Train Steps/Sec: 0.85 +[2025-04-22 10:46:27] (step=0022250) Train Loss: 6.3873, Train Steps/Sec: 0.85 +[2025-04-22 10:46:56] (step=0022275) Train Loss: 6.4001, Train Steps/Sec: 0.85 +[2025-04-22 10:47:26] (step=0022300) Train Loss: 6.4150, Train Steps/Sec: 0.84 +[2025-04-22 10:47:56] (step=0022325) Train Loss: 6.3677, Train Steps/Sec: 0.85 +[2025-04-22 10:48:25] (step=0022350) Train Loss: 6.3881, Train Steps/Sec: 0.85 +[2025-04-22 10:48:55] (step=0022375) Train Loss: 6.3541, Train Steps/Sec: 0.84 +[2025-04-22 10:49:24] (step=0022400) Train Loss: 6.4313, Train Steps/Sec: 0.84 +[2025-04-22 10:49:54] (step=0022425) Train Loss: 6.4068, Train Steps/Sec: 0.85 +[2025-04-22 10:50:24] (step=0022450) Train Loss: 6.3662, Train Steps/Sec: 0.84 +[2025-04-22 10:50:53] (step=0022475) Train Loss: 6.4649, Train Steps/Sec: 0.85 +[2025-04-22 10:51:23] (step=0022500) Train Loss: 6.3986, Train Steps/Sec: 0.84 +[2025-04-22 10:51:52] (step=0022525) Train Loss: 6.3748, Train Steps/Sec: 0.85 +[2025-04-22 10:52:22] (step=0022550) Train Loss: 6.4197, Train Steps/Sec: 0.85 +[2025-04-22 10:52:51] (step=0022575) Train Loss: 6.4001, Train Steps/Sec: 0.85 +[2025-04-22 10:53:21] (step=0022600) Train Loss: 6.4200, Train Steps/Sec: 0.84 +[2025-04-22 10:53:51] (step=0022625) Train Loss: 6.4181, Train Steps/Sec: 0.85 +[2025-04-22 10:54:20] (step=0022650) Train Loss: 6.4113, Train Steps/Sec: 0.85 +[2025-04-22 10:54:50] (step=0022675) Train Loss: 6.3763, Train Steps/Sec: 0.85 +[2025-04-22 10:55:19] (step=0022700) Train Loss: 6.4188, Train Steps/Sec: 0.85 +[2025-04-22 10:55:49] (step=0022725) Train Loss: 6.4556, Train Steps/Sec: 0.85 +[2025-04-22 10:56:18] (step=0022750) Train Loss: 6.4334, Train Steps/Sec: 0.85 +[2025-04-22 10:56:48] (step=0022775) Train Loss: 6.4214, Train Steps/Sec: 0.85 +[2025-04-22 10:57:17] (step=0022800) Train Loss: 6.3976, Train Steps/Sec: 0.85 +[2025-04-22 10:57:47] (step=0022825) Train Loss: 6.4513, Train Steps/Sec: 0.85 +[2025-04-22 10:58:16] (step=0022850) Train Loss: 6.4055, Train Steps/Sec: 0.85 +[2025-04-22 10:58:45] (step=0022875) Train Loss: 6.3751, Train Steps/Sec: 0.85 +[2025-04-22 10:59:15] (step=0022900) Train Loss: 6.4050, Train Steps/Sec: 0.85 +[2025-04-22 10:59:44] (step=0022925) Train Loss: 6.4095, Train Steps/Sec: 0.85 +[2025-04-22 11:00:14] (step=0022950) Train Loss: 6.3938, Train Steps/Sec: 0.85 +[2025-04-22 11:00:44] (step=0022975) Train Loss: 6.3840, Train Steps/Sec: 0.85 +[2025-04-22 11:01:13] (step=0023000) Train Loss: 6.4164, Train Steps/Sec: 0.84 +[2025-04-22 11:01:43] (step=0023025) Train Loss: 6.3797, Train Steps/Sec: 0.85 +[2025-04-22 11:02:12] (step=0023050) Train Loss: 6.4362, Train Steps/Sec: 0.84 +[2025-04-22 11:02:42] (step=0023075) Train Loss: 6.3932, Train Steps/Sec: 0.85 +[2025-04-22 11:03:11] (step=0023100) Train Loss: 6.3749, Train Steps/Sec: 0.85 +[2025-04-22 11:03:41] (step=0023125) Train Loss: 6.4282, Train Steps/Sec: 0.84 +[2025-04-22 11:04:10] (step=0023150) Train Loss: 6.3907, Train Steps/Sec: 0.85 +[2025-04-22 11:04:39] (step=0023175) Train Loss: 6.4076, Train Steps/Sec: 0.85 +[2025-04-22 11:05:09] (step=0023200) Train Loss: 6.4536, Train Steps/Sec: 0.84 +[2025-04-22 11:05:39] (step=0023225) Train Loss: 6.4061, Train Steps/Sec: 0.85 +[2025-04-22 11:06:08] (step=0023250) Train Loss: 6.4185, Train Steps/Sec: 0.85 +[2025-04-22 11:06:38] (step=0023275) Train Loss: 6.3948, Train Steps/Sec: 0.85 +[2025-04-22 11:07:07] (step=0023300) Train Loss: 6.3716, Train Steps/Sec: 0.84 +[2025-04-22 11:07:37] (step=0023325) Train Loss: 6.3989, Train Steps/Sec: 0.84 +[2025-04-22 11:08:06] (step=0023350) Train Loss: 6.3834, Train Steps/Sec: 0.85 +[2025-04-22 11:08:36] (step=0023375) Train Loss: 6.3918, Train Steps/Sec: 0.85 +[2025-04-22 11:09:06] (step=0023400) Train Loss: 6.3834, Train Steps/Sec: 0.84 +[2025-04-22 11:09:35] (step=0023425) Train Loss: 6.3861, Train Steps/Sec: 0.85 +[2025-04-22 11:10:05] (step=0023450) Train Loss: 6.3716, Train Steps/Sec: 0.84 +[2025-04-22 11:10:35] (step=0023475) Train Loss: 6.4182, Train Steps/Sec: 0.84 +[2025-04-22 11:11:04] (step=0023500) Train Loss: 6.3609, Train Steps/Sec: 0.84 +[2025-04-22 11:11:34] (step=0023525) Train Loss: 6.3931, Train Steps/Sec: 0.84 +[2025-04-22 11:12:04] (step=0023550) Train Loss: 6.4157, Train Steps/Sec: 0.85 +[2025-04-22 11:12:33] (step=0023575) Train Loss: 6.4151, Train Steps/Sec: 0.84 +[2025-04-22 11:13:03] (step=0023600) Train Loss: 6.4163, Train Steps/Sec: 0.84 +[2025-04-22 11:13:33] (step=0023625) Train Loss: 6.4068, Train Steps/Sec: 0.85 +[2025-04-22 11:14:02] (step=0023650) Train Loss: 6.3960, Train Steps/Sec: 0.85 +[2025-04-22 11:14:32] (step=0023675) Train Loss: 6.4643, Train Steps/Sec: 0.85 +[2025-04-22 11:15:01] (step=0023700) Train Loss: 6.3948, Train Steps/Sec: 0.85 +[2025-04-22 11:15:31] (step=0023725) Train Loss: 6.4343, Train Steps/Sec: 0.85 +[2025-04-22 11:16:00] (step=0023750) Train Loss: 6.4421, Train Steps/Sec: 0.84 +[2025-04-22 11:16:30] (step=0023775) Train Loss: 6.4026, Train Steps/Sec: 0.85 +[2025-04-22 11:16:59] (step=0023800) Train Loss: 6.4282, Train Steps/Sec: 0.85 +[2025-04-22 11:17:29] (step=0023825) Train Loss: 6.4000, Train Steps/Sec: 0.85 +[2025-04-22 11:17:58] (step=0023850) Train Loss: 6.3930, Train Steps/Sec: 0.85 +[2025-04-22 11:18:28] (step=0023875) Train Loss: 6.3725, Train Steps/Sec: 0.85 +[2025-04-22 11:18:58] (step=0023900) Train Loss: 6.4104, Train Steps/Sec: 0.84 +[2025-04-22 11:19:27] (step=0023925) Train Loss: 6.4127, Train Steps/Sec: 0.84 +[2025-04-22 11:19:57] (step=0023950) Train Loss: 6.3933, Train Steps/Sec: 0.85 +[2025-04-22 11:20:26] (step=0023975) Train Loss: 6.3925, Train Steps/Sec: 0.85 +[2025-04-22 11:20:56] (step=0024000) Train Loss: 6.4506, Train Steps/Sec: 0.85 +[2025-04-22 11:25:37] Finish Eval in 24000 steps... +[2025-04-22 11:25:58] Saved checkpoint to checkpoint/BeforeFix_Blip_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_objects/002-GPT-XL/checkpoints/0024000.pt +[2025-04-22 11:26:01] Removed old checkpoint: checkpoint/BeforeFix_Blip_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_objects/002-GPT-XL/checkpoints/0020000.pt +[2025-04-22 11:26:30] (step=0024025) Train Loss: 6.3986, Train Steps/Sec: 0.07 +[2025-04-22 11:26:59] (step=0024050) Train Loss: 6.3979, Train Steps/Sec: 0.85 +[2025-04-22 11:27:29] (step=0024075) Train Loss: 6.4077, Train Steps/Sec: 0.85 +[2025-04-22 11:27:59] (step=0024100) Train Loss: 6.4447, Train Steps/Sec: 0.84 +[2025-04-22 11:28:28] (step=0024125) Train Loss: 6.4194, Train Steps/Sec: 0.85 +[2025-04-22 11:28:58] (step=0024150) Train Loss: 6.3994, Train Steps/Sec: 0.85 +[2025-04-22 11:29:27] (step=0024175) Train Loss: 6.4066, Train Steps/Sec: 0.85 +[2025-04-22 11:29:57] (step=0024200) Train Loss: 6.4283, Train Steps/Sec: 0.84 +[2025-04-22 11:30:32] (step=0024225) Train Loss: 6.4176, Train Steps/Sec: 0.70 +[2025-04-22 11:31:02] (step=0024250) Train Loss: 6.4253, Train Steps/Sec: 0.85 +[2025-04-22 11:31:32] (step=0024275) Train Loss: 6.4249, Train Steps/Sec: 0.84 +[2025-04-22 11:32:01] (step=0024300) Train Loss: 6.3794, Train Steps/Sec: 0.84 +[2025-04-22 11:32:31] (step=0024325) Train Loss: 6.3870, Train Steps/Sec: 0.84 +[2025-04-22 11:33:01] (step=0024350) Train Loss: 6.3934, Train Steps/Sec: 0.84 +[2025-04-22 11:33:36] (step=0024375) Train Loss: 6.4186, Train Steps/Sec: 0.71 +[2025-04-22 11:34:06] (step=0024400) Train Loss: 6.4027, Train Steps/Sec: 0.84 +[2025-04-22 11:34:35] (step=0024425) Train Loss: 6.4176, Train Steps/Sec: 0.85 +[2025-04-22 11:35:05] (step=0024450) Train Loss: 6.3909, Train Steps/Sec: 0.84 +[2025-04-22 11:35:34] (step=0024475) Train Loss: 6.4035, Train Steps/Sec: 0.85 +[2025-04-22 11:36:04] (step=0024500) Train Loss: 6.3992, Train Steps/Sec: 0.84 +[2025-04-22 11:36:33] (step=0024525) Train Loss: 6.4112, Train Steps/Sec: 0.85 +[2025-04-22 11:37:03] (step=0024550) Train Loss: 6.4146, Train Steps/Sec: 0.85 +[2025-04-22 11:37:33] (step=0024575) Train Loss: 6.4123, Train Steps/Sec: 0.84 +[2025-04-22 11:38:02] (step=0024600) Train Loss: 6.4168, Train Steps/Sec: 0.84 +[2025-04-22 11:38:43] (step=0024625) Train Loss: 6.3941, Train Steps/Sec: 0.62 +[2025-04-22 11:39:12] (step=0024650) Train Loss: 6.3901, Train Steps/Sec: 0.84 +[2025-04-22 11:39:42] (step=0024675) Train Loss: 6.4258, Train Steps/Sec: 0.85 +[2025-04-22 11:40:17] (step=0024700) Train Loss: 6.3942, Train Steps/Sec: 0.71 +[2025-04-22 11:40:47] (step=0024725) Train Loss: 6.4318, Train Steps/Sec: 0.85 +[2025-04-22 11:41:16] (step=0024750) Train Loss: 6.4038, Train Steps/Sec: 0.85 +[2025-04-22 11:41:46] (step=0024775) Train Loss: 6.3511, Train Steps/Sec: 0.84 +[2025-04-22 11:42:16] (step=0024800) Train Loss: 6.4495, Train Steps/Sec: 0.84 +[2025-04-22 11:42:45] (step=0024825) Train Loss: 6.4081, Train Steps/Sec: 0.84 +[2025-04-22 11:43:15] (step=0024850) Train Loss: 6.4054, Train Steps/Sec: 0.84 +[2025-04-22 11:43:45] (step=0024875) Train Loss: 6.4040, Train Steps/Sec: 0.84 +[2025-04-22 11:44:14] (step=0024900) Train Loss: 6.3701, Train Steps/Sec: 0.85 +[2025-04-22 11:44:44] (step=0024925) Train Loss: 6.3957, Train Steps/Sec: 0.84 +[2025-04-22 11:45:14] (step=0024950) Train Loss: 6.3890, Train Steps/Sec: 0.84 +[2025-04-22 11:45:44] (step=0024975) Train Loss: 6.4153, Train Steps/Sec: 0.84 +[2025-04-22 11:46:13] (step=0025000) Train Loss: 6.3882, Train Steps/Sec: 0.84 +[2025-04-22 11:46:43] (step=0025025) Train Loss: 6.4043, Train Steps/Sec: 0.84 +[2025-04-22 11:47:12] (step=0025050) Train Loss: 6.4070, Train Steps/Sec: 0.85 +[2025-04-22 11:47:42] (step=0025075) Train Loss: 6.3898, Train Steps/Sec: 0.84 +[2025-04-22 11:48:11] (step=0025100) Train Loss: 6.4247, Train Steps/Sec: 0.84 +[2025-04-22 11:48:41] (step=0025125) Train Loss: 6.4277, Train Steps/Sec: 0.85 +[2025-04-22 11:49:10] (step=0025150) Train Loss: 6.4075, Train Steps/Sec: 0.85 +[2025-04-22 11:49:40] (step=0025175) Train Loss: 6.3708, Train Steps/Sec: 0.84 +[2025-04-22 11:50:10] (step=0025200) Train Loss: 6.4080, Train Steps/Sec: 0.84 +[2025-04-22 11:50:39] (step=0025225) Train Loss: 6.4235, Train Steps/Sec: 0.84 +[2025-04-22 11:51:09] (step=0025250) Train Loss: 6.4453, Train Steps/Sec: 0.85 +[2025-04-22 11:51:38] (step=0025275) Train Loss: 6.3733, Train Steps/Sec: 0.85 +[2025-04-22 11:52:08] (step=0025300) Train Loss: 6.3965, Train Steps/Sec: 0.84 +[2025-04-22 11:52:38] (step=0025325) Train Loss: 6.3751, Train Steps/Sec: 0.84 +[2025-04-22 11:53:07] (step=0025350) Train Loss: 6.3948, Train Steps/Sec: 0.84 +[2025-04-22 11:53:37] (step=0025375) Train Loss: 6.4073, Train Steps/Sec: 0.84 +[2025-04-22 11:54:07] (step=0025400) Train Loss: 6.3859, Train Steps/Sec: 0.85 +[2025-04-22 11:54:36] (step=0025425) Train Loss: 6.4200, Train Steps/Sec: 0.85 +[2025-04-22 11:55:05] (step=0025450) Train Loss: 6.3703, Train Steps/Sec: 0.85 +[2025-04-22 11:55:35] (step=0025475) Train Loss: 6.3969, Train Steps/Sec: 0.85 +[2025-04-22 11:56:04] (step=0025500) Train Loss: 6.3837, Train Steps/Sec: 0.85 +[2025-04-22 11:56:41] (step=0025525) Train Loss: 6.4233, Train Steps/Sec: 0.69 +[2025-04-22 11:57:18] (step=0025550) Train Loss: 6.3681, Train Steps/Sec: 0.68 +[2025-04-22 11:57:47] (step=0025575) Train Loss: 6.3770, Train Steps/Sec: 0.85 +[2025-04-22 11:58:17] (step=0025600) Train Loss: 6.4189, Train Steps/Sec: 0.84 +[2025-04-22 11:58:46] (step=0025625) Train Loss: 6.3783, Train Steps/Sec: 0.85 +[2025-04-22 11:59:16] (step=0025650) Train Loss: 6.3787, Train Steps/Sec: 0.85 +[2025-04-22 11:59:45] (step=0025675) Train Loss: 6.4154, Train Steps/Sec: 0.85 +[2025-04-22 12:00:15] (step=0025700) Train Loss: 6.4135, Train Steps/Sec: 0.84 +[2025-04-22 12:00:45] (step=0025725) Train Loss: 6.3870, Train Steps/Sec: 0.84 +[2025-04-22 12:01:14] (step=0025750) Train Loss: 6.3618, Train Steps/Sec: 0.85 +[2025-04-22 12:01:44] (step=0025775) Train Loss: 6.4361, Train Steps/Sec: 0.85 +[2025-04-22 12:02:13] (step=0025800) Train Loss: 6.4192, Train Steps/Sec: 0.84 +[2025-04-22 12:02:43] (step=0025825) Train Loss: 6.4103, Train Steps/Sec: 0.85 +[2025-04-22 12:03:12] (step=0025850) Train Loss: 6.4460, Train Steps/Sec: 0.85 +[2025-04-22 12:03:42] (step=0025875) Train Loss: 6.4229, Train Steps/Sec: 0.85 +[2025-04-22 12:04:11] (step=0025900) Train Loss: 6.3940, Train Steps/Sec: 0.85 +[2025-04-22 12:04:41] (step=0025925) Train Loss: 6.4186, Train Steps/Sec: 0.84 +[2025-04-22 12:05:11] (step=0025950) Train Loss: 6.4397, Train Steps/Sec: 0.84 +[2025-04-22 12:05:40] (step=0025975) Train Loss: 6.4216, Train Steps/Sec: 0.85 +[2025-04-22 12:06:10] (step=0026000) Train Loss: 6.4130, Train Steps/Sec: 0.84 +[2025-04-22 12:10:52] Finish Eval in 26000 steps... +[2025-04-22 12:11:13] Saved checkpoint to checkpoint/BeforeFix_Blip_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_objects/002-GPT-XL/checkpoints/0026000.pt +[2025-04-22 12:11:15] Removed old checkpoint: checkpoint/BeforeFix_Blip_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_objects/002-GPT-XL/checkpoints/0022000.pt +[2025-04-22 12:11:45] (step=0026025) Train Loss: 6.3918, Train Steps/Sec: 0.07 +[2025-04-22 12:12:14] (step=0026050) Train Loss: 6.4077, Train Steps/Sec: 0.84 +[2025-04-22 12:12:44] (step=0026075) Train Loss: 6.3913, Train Steps/Sec: 0.85 +[2025-04-22 12:13:14] (step=0026100) Train Loss: 6.4359, Train Steps/Sec: 0.84 +[2025-04-22 12:13:43] (step=0026125) Train Loss: 6.4196, Train Steps/Sec: 0.85 +[2025-04-22 12:14:13] (step=0026150) Train Loss: 6.3612, Train Steps/Sec: 0.85 +[2025-04-22 12:14:42] (step=0026175) Train Loss: 6.4022, Train Steps/Sec: 0.84 +[2025-04-22 12:15:12] (step=0026200) Train Loss: 6.4137, Train Steps/Sec: 0.84 +[2025-04-22 12:15:42] (step=0026225) Train Loss: 6.4210, Train Steps/Sec: 0.85 +[2025-04-22 12:16:11] (step=0026250) Train Loss: 6.4120, Train Steps/Sec: 0.84 +[2025-04-22 12:16:41] (step=0026275) Train Loss: 6.3913, Train Steps/Sec: 0.84 +[2025-04-22 12:17:10] (step=0026300) Train Loss: 6.3595, Train Steps/Sec: 0.85 +[2025-04-22 12:17:40] (step=0026325) Train Loss: 6.4031, Train Steps/Sec: 0.85 +[2025-04-22 12:18:10] (step=0026350) Train Loss: 6.3751, Train Steps/Sec: 0.84 +[2025-04-22 12:18:40] (step=0026375) Train Loss: 6.3862, Train Steps/Sec: 0.84 +[2025-04-22 12:19:09] (step=0026400) Train Loss: 6.4219, Train Steps/Sec: 0.84 +[2025-04-22 12:19:39] (step=0026425) Train Loss: 6.3951, Train Steps/Sec: 0.85 +[2025-04-22 12:20:09] (step=0026450) Train Loss: 6.3858, Train Steps/Sec: 0.84 +[2025-04-22 12:20:38] (step=0026475) Train Loss: 6.4283, Train Steps/Sec: 0.84 +[2025-04-22 12:21:08] (step=0026500) Train Loss: 6.3632, Train Steps/Sec: 0.84 +[2025-04-22 12:21:38] (step=0026525) Train Loss: 6.3975, Train Steps/Sec: 0.84 +[2025-04-22 12:22:07] (step=0026550) Train Loss: 6.4019, Train Steps/Sec: 0.85 +[2025-04-22 12:22:36] (step=0026575) Train Loss: 6.3803, Train Steps/Sec: 0.85 +[2025-04-22 12:23:06] (step=0026600) Train Loss: 6.3680, Train Steps/Sec: 0.85 +[2025-04-22 12:23:35] (step=0026625) Train Loss: 6.4116, Train Steps/Sec: 0.85 +[2025-04-22 12:24:05] (step=0026650) Train Loss: 6.4093, Train Steps/Sec: 0.85 +[2025-04-22 12:24:34] (step=0026675) Train Loss: 6.3766, Train Steps/Sec: 0.85 +[2025-04-22 12:25:04] (step=0026700) Train Loss: 6.3754, Train Steps/Sec: 0.84 +[2025-04-22 12:25:34] (step=0026725) Train Loss: 6.4305, Train Steps/Sec: 0.84 +[2025-04-22 12:26:03] (step=0026750) Train Loss: 6.3848, Train Steps/Sec: 0.85 +[2025-04-22 12:26:33] (step=0026775) Train Loss: 6.3941, Train Steps/Sec: 0.85 +[2025-04-22 12:27:02] (step=0026800) Train Loss: 6.4522, Train Steps/Sec: 0.84 +[2025-04-22 12:27:32] (step=0026825) Train Loss: 6.4150, Train Steps/Sec: 0.85 +[2025-04-22 12:28:01] (step=0026850) Train Loss: 6.3981, Train Steps/Sec: 0.85 +[2025-04-22 12:28:31] (step=0026875) Train Loss: 6.3971, Train Steps/Sec: 0.85 +[2025-04-22 12:29:06] (step=0026900) Train Loss: 6.4253, Train Steps/Sec: 0.70 +[2025-04-22 12:29:36] (step=0026925) Train Loss: 6.3941, Train Steps/Sec: 0.85 +[2025-04-22 12:30:05] (step=0026950) Train Loss: 6.3735, Train Steps/Sec: 0.85 +[2025-04-22 12:30:35] (step=0026975) Train Loss: 6.4401, Train Steps/Sec: 0.85 +[2025-04-22 12:31:05] (step=0027000) Train Loss: 6.3474, Train Steps/Sec: 0.84 +[2025-04-22 12:31:34] (step=0027025) Train Loss: 6.4269, Train Steps/Sec: 0.85 +[2025-04-22 12:32:03] (step=0027050) Train Loss: 6.4030, Train Steps/Sec: 0.85 +[2025-04-22 12:32:33] (step=0027075) Train Loss: 6.3911, Train Steps/Sec: 0.85 +[2025-04-22 12:33:02] (step=0027100) Train Loss: 6.4088, Train Steps/Sec: 0.85 +[2025-04-22 12:33:32] (step=0027125) Train Loss: 6.3737, Train Steps/Sec: 0.85 +[2025-04-22 12:34:02] (step=0027150) Train Loss: 6.4344, Train Steps/Sec: 0.85 +[2025-04-22 12:34:31] (step=0027175) Train Loss: 6.3792, Train Steps/Sec: 0.85 +[2025-04-22 12:35:01] (step=0027200) Train Loss: 6.3996, Train Steps/Sec: 0.84 +[2025-04-22 12:35:30] (step=0027225) Train Loss: 6.3779, Train Steps/Sec: 0.84 +[2025-04-22 12:36:00] (step=0027250) Train Loss: 6.3479, Train Steps/Sec: 0.85 +[2025-04-22 12:36:29] (step=0027275) Train Loss: 6.4369, Train Steps/Sec: 0.85 +[2025-04-22 12:36:59] (step=0027300) Train Loss: 6.4114, Train Steps/Sec: 0.84 +[2025-04-22 12:37:28] (step=0027325) Train Loss: 6.4126, Train Steps/Sec: 0.85 +[2025-04-22 12:37:58] (step=0027350) Train Loss: 6.4077, Train Steps/Sec: 0.85 +[2025-04-22 12:38:28] (step=0027375) Train Loss: 6.4189, Train Steps/Sec: 0.85 +[2025-04-22 12:38:57] (step=0027400) Train Loss: 6.4515, Train Steps/Sec: 0.84 +[2025-04-22 12:39:27] (step=0027425) Train Loss: 6.3869, Train Steps/Sec: 0.85 +[2025-04-22 12:39:56] (step=0027450) Train Loss: 6.3828, Train Steps/Sec: 0.85 +[2025-04-22 12:40:26] (step=0027475) Train Loss: 6.4216, Train Steps/Sec: 0.85 +[2025-04-22 12:40:55] (step=0027500) Train Loss: 6.4534, Train Steps/Sec: 0.85 +[2025-04-22 12:41:25] (step=0027525) Train Loss: 6.4001, Train Steps/Sec: 0.84 +[2025-04-22 12:41:54] (step=0027550) Train Loss: 6.4073, Train Steps/Sec: 0.85 +[2025-04-22 12:42:24] (step=0027575) Train Loss: 6.3658, Train Steps/Sec: 0.84 +[2025-04-22 12:42:54] (step=0027600) Train Loss: 6.3839, Train Steps/Sec: 0.84 +[2025-04-22 12:43:23] (step=0027625) Train Loss: 6.3747, Train Steps/Sec: 0.85 +[2025-04-22 12:43:53] (step=0027650) Train Loss: 6.4032, Train Steps/Sec: 0.85 +[2025-04-22 12:44:23] (step=0027675) Train Loss: 6.4510, Train Steps/Sec: 0.84 +[2025-04-22 12:44:52] (step=0027700) Train Loss: 6.3655, Train Steps/Sec: 0.85 +[2025-04-22 12:45:21] (step=0027725) Train Loss: 6.3668, Train Steps/Sec: 0.85 +[2025-04-22 12:45:51] (step=0027750) Train Loss: 6.3430, Train Steps/Sec: 0.85 +[2025-04-22 12:46:20] (step=0027775) Train Loss: 6.4219, Train Steps/Sec: 0.85 +[2025-04-22 12:46:50] (step=0027800) Train Loss: 6.3666, Train Steps/Sec: 0.84 +[2025-04-22 12:47:19] (step=0027825) Train Loss: 6.3937, Train Steps/Sec: 0.85 +[2025-04-22 12:47:49] (step=0027850) Train Loss: 6.4389, Train Steps/Sec: 0.85 +[2025-04-22 12:48:18] (step=0027875) Train Loss: 6.3838, Train Steps/Sec: 0.85 +[2025-04-22 12:48:48] (step=0027900) Train Loss: 6.4014, Train Steps/Sec: 0.85 +[2025-04-22 12:49:17] (step=0027925) Train Loss: 6.3744, Train Steps/Sec: 0.85 +[2025-04-22 12:49:47] (step=0027950) Train Loss: 6.3857, Train Steps/Sec: 0.84 +[2025-04-22 12:50:17] (step=0027975) Train Loss: 6.4262, Train Steps/Sec: 0.85 +[2025-04-22 12:50:46] (step=0028000) Train Loss: 6.4002, Train Steps/Sec: 0.84 +[2025-04-22 12:55:28] Finish Eval in 28000 steps... +[2025-04-22 12:55:50] Saved checkpoint to checkpoint/BeforeFix_Blip_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_objects/002-GPT-XL/checkpoints/0028000.pt +[2025-04-22 12:55:52] Removed old checkpoint: checkpoint/BeforeFix_Blip_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_objects/002-GPT-XL/checkpoints/0024000.pt +[2025-04-22 12:56:22] (step=0028025) Train Loss: 6.4437, Train Steps/Sec: 0.07 +[2025-04-22 12:56:52] (step=0028050) Train Loss: 6.3659, Train Steps/Sec: 0.85 +[2025-04-22 12:57:21] (step=0028075) Train Loss: 6.3779, Train Steps/Sec: 0.85 +[2025-04-22 12:57:51] (step=0028100) Train Loss: 6.4194, Train Steps/Sec: 0.84 +[2025-04-22 12:58:20] (step=0028125) Train Loss: 6.3810, Train Steps/Sec: 0.85 +[2025-04-22 12:58:50] (step=0028150) Train Loss: 6.4045, Train Steps/Sec: 0.85 +[2025-04-22 12:59:19] (step=0028175) Train Loss: 6.3962, Train Steps/Sec: 0.85 +[2025-04-22 12:59:49] (step=0028200) Train Loss: 6.4088, Train Steps/Sec: 0.84 +[2025-04-22 13:00:18] (step=0028225) Train Loss: 6.3994, Train Steps/Sec: 0.85 +[2025-04-22 13:00:48] (step=0028250) Train Loss: 6.4017, Train Steps/Sec: 0.85 +[2025-04-22 13:01:17] (step=0028275) Train Loss: 6.4390, Train Steps/Sec: 0.85 +[2025-04-22 13:01:46] (step=0028300) Train Loss: 6.3844, Train Steps/Sec: 0.85 +[2025-04-22 13:02:16] (step=0028325) Train Loss: 6.4054, Train Steps/Sec: 0.84 +[2025-04-22 13:02:46] (step=0028350) Train Loss: 6.3903, Train Steps/Sec: 0.85 +[2025-04-22 13:03:15] (step=0028375) Train Loss: 6.4019, Train Steps/Sec: 0.85 +[2025-04-22 13:03:45] (step=0028400) Train Loss: 6.4117, Train Steps/Sec: 0.84 +[2025-04-22 13:04:14] (step=0028425) Train Loss: 6.3957, Train Steps/Sec: 0.84 +[2025-04-22 13:04:44] (step=0028450) Train Loss: 6.4182, Train Steps/Sec: 0.84 +[2025-04-22 13:05:13] (step=0028475) Train Loss: 6.3964, Train Steps/Sec: 0.85 +[2025-04-22 13:05:43] (step=0028500) Train Loss: 6.4400, Train Steps/Sec: 0.84 +[2025-04-22 13:06:13] (step=0028525) Train Loss: 6.4294, Train Steps/Sec: 0.85 +[2025-04-22 13:06:42] (step=0028550) Train Loss: 6.3880, Train Steps/Sec: 0.85 +[2025-04-22 13:07:12] (step=0028575) Train Loss: 6.3893, Train Steps/Sec: 0.85 +[2025-04-22 13:07:42] (step=0028600) Train Loss: 6.3978, Train Steps/Sec: 0.84 +[2025-04-22 13:08:11] (step=0028625) Train Loss: 6.3750, Train Steps/Sec: 0.85 +[2025-04-22 13:08:41] (step=0028650) Train Loss: 6.4400, Train Steps/Sec: 0.84 +[2025-04-22 13:09:10] (step=0028675) Train Loss: 6.4160, Train Steps/Sec: 0.85 +[2025-04-22 13:09:40] (step=0028700) Train Loss: 6.4393, Train Steps/Sec: 0.85 +[2025-04-22 13:10:09] (step=0028725) Train Loss: 6.3707, Train Steps/Sec: 0.84 +[2025-04-22 13:10:39] (step=0028750) Train Loss: 6.4175, Train Steps/Sec: 0.85 +[2025-04-22 13:11:08] (step=0028775) Train Loss: 6.3651, Train Steps/Sec: 0.85 +[2025-04-22 13:11:38] (step=0028800) Train Loss: 6.3965, Train Steps/Sec: 0.85 +[2025-04-22 13:12:08] (step=0028825) Train Loss: 6.3930, Train Steps/Sec: 0.84 +[2025-04-22 13:12:37] (step=0028850) Train Loss: 6.4324, Train Steps/Sec: 0.85 +[2025-04-22 13:13:07] (step=0028875) Train Loss: 6.4128, Train Steps/Sec: 0.85 +[2025-04-22 13:13:36] (step=0028900) Train Loss: 6.4241, Train Steps/Sec: 0.84 +[2025-04-22 13:14:06] (step=0028925) Train Loss: 6.4085, Train Steps/Sec: 0.84 +[2025-04-22 13:14:35] (step=0028950) Train Loss: 6.4338, Train Steps/Sec: 0.85 +[2025-04-22 13:15:05] (step=0028975) Train Loss: 6.3879, Train Steps/Sec: 0.85 +[2025-04-22 13:15:35] (step=0029000) Train Loss: 6.3726, Train Steps/Sec: 0.84 +[2025-04-22 13:16:04] (step=0029025) Train Loss: 6.3460, Train Steps/Sec: 0.85 +[2025-04-22 13:16:34] (step=0029050) Train Loss: 6.3905, Train Steps/Sec: 0.85 +[2025-04-22 13:17:03] (step=0029075) Train Loss: 6.3903, Train Steps/Sec: 0.84 +[2025-04-22 13:17:33] (step=0029100) Train Loss: 6.3932, Train Steps/Sec: 0.84 +[2025-04-22 13:18:03] (step=0029125) Train Loss: 6.3988, Train Steps/Sec: 0.84 +[2025-04-22 13:18:32] (step=0029150) Train Loss: 6.4091, Train Steps/Sec: 0.85 +[2025-04-22 13:19:01] (step=0029175) Train Loss: 6.4031, Train Steps/Sec: 0.85 +[2025-04-22 13:19:31] (step=0029200) Train Loss: 6.3994, Train Steps/Sec: 0.84 +[2025-04-22 13:20:01] (step=0029225) Train Loss: 6.3877, Train Steps/Sec: 0.85 +[2025-04-22 13:20:30] (step=0029250) Train Loss: 6.4080, Train Steps/Sec: 0.85 +[2025-04-22 13:21:00] (step=0029275) Train Loss: 6.4453, Train Steps/Sec: 0.84 +[2025-04-22 13:21:30] (step=0029300) Train Loss: 6.3873, Train Steps/Sec: 0.84 +[2025-04-22 13:22:05] (step=0029325) Train Loss: 6.4123, Train Steps/Sec: 0.70 +[2025-04-22 13:22:35] (step=0029350) Train Loss: 6.4068, Train Steps/Sec: 0.85 +[2025-04-22 13:23:04] (step=0029375) Train Loss: 6.4015, Train Steps/Sec: 0.86 +[2025-04-22 13:23:40] (step=0029400) Train Loss: 6.4255, Train Steps/Sec: 0.70 +[2025-04-22 13:24:09] (step=0029425) Train Loss: 6.4455, Train Steps/Sec: 0.84 +[2025-04-22 13:24:39] (step=0029450) Train Loss: 6.4274, Train Steps/Sec: 0.85 +[2025-04-22 13:25:08] (step=0029475) Train Loss: 6.3957, Train Steps/Sec: 0.85 +[2025-04-22 13:25:38] (step=0029500) Train Loss: 6.3859, Train Steps/Sec: 0.84 +[2025-04-22 13:26:08] (step=0029525) Train Loss: 6.3734, Train Steps/Sec: 0.85 +[2025-04-22 13:26:37] (step=0029550) Train Loss: 6.3737, Train Steps/Sec: 0.85 +[2025-04-22 13:27:06] (step=0029575) Train Loss: 6.4160, Train Steps/Sec: 0.85 +[2025-04-22 13:27:36] (step=0029600) Train Loss: 6.4547, Train Steps/Sec: 0.84 +[2025-04-22 13:28:05] (step=0029625) Train Loss: 6.3919, Train Steps/Sec: 0.85 +[2025-04-22 13:28:35] (step=0029650) Train Loss: 6.3767, Train Steps/Sec: 0.85 +[2025-04-22 13:29:04] (step=0029675) Train Loss: 6.3649, Train Steps/Sec: 0.85 +[2025-04-22 13:29:34] (step=0029700) Train Loss: 6.3547, Train Steps/Sec: 0.84 +[2025-04-22 13:30:03] (step=0029725) Train Loss: 6.3731, Train Steps/Sec: 0.85 +[2025-04-22 13:30:43] (step=0029750) Train Loss: 6.4060, Train Steps/Sec: 0.64 +[2025-04-22 13:31:12] (step=0029775) Train Loss: 6.4311, Train Steps/Sec: 0.85 +[2025-04-22 13:31:48] (step=0029800) Train Loss: 6.4255, Train Steps/Sec: 0.71 +[2025-04-22 13:32:17] (step=0029825) Train Loss: 6.4325, Train Steps/Sec: 0.85 +[2025-04-22 13:32:46] (step=0029850) Train Loss: 6.4237, Train Steps/Sec: 0.85 +[2025-04-22 13:33:16] (step=0029875) Train Loss: 6.3685, Train Steps/Sec: 0.85 +[2025-04-22 13:33:46] (step=0029900) Train Loss: 6.3918, Train Steps/Sec: 0.85 +[2025-04-22 13:34:15] (step=0029925) Train Loss: 6.3566, Train Steps/Sec: 0.84 +[2025-04-22 13:34:45] (step=0029950) Train Loss: 6.4130, Train Steps/Sec: 0.84 +[2025-04-22 13:35:15] (step=0029975) Train Loss: 6.4261, Train Steps/Sec: 0.85 +[2025-04-22 13:35:44] (step=0030000) Train Loss: 6.4229, Train Steps/Sec: 0.85 +[2025-04-22 13:40:25] Finish Eval in 30000 steps... +[2025-04-22 13:40:46] Saved checkpoint to checkpoint/BeforeFix_Blip_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_objects/002-GPT-XL/checkpoints/0030000.pt +[2025-04-22 13:40:48] Removed old checkpoint: checkpoint/BeforeFix_Blip_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_objects/002-GPT-XL/checkpoints/0026000.pt +[2025-04-22 13:41:18] (step=0030025) Train Loss: 6.3751, Train Steps/Sec: 0.07 +[2025-04-22 13:41:47] (step=0030050) Train Loss: 6.3903, Train Steps/Sec: 0.84 +[2025-04-22 13:42:17] (step=0030075) Train Loss: 6.4075, Train Steps/Sec: 0.85 +[2025-04-22 13:42:47] (step=0030100) Train Loss: 6.3988, Train Steps/Sec: 0.84 +[2025-04-22 13:43:17] (step=0030125) Train Loss: 6.4054, Train Steps/Sec: 0.84 +[2025-04-22 13:43:46] (step=0030150) Train Loss: 6.4228, Train Steps/Sec: 0.85 +[2025-04-22 13:44:16] (step=0030175) Train Loss: 6.3685, Train Steps/Sec: 0.84 +[2025-04-22 13:44:45] (step=0030200) Train Loss: 6.3826, Train Steps/Sec: 0.84 +[2025-04-22 13:45:15] (step=0030225) Train Loss: 6.4093, Train Steps/Sec: 0.84 +[2025-04-22 13:45:45] (step=0030250) Train Loss: 6.3820, Train Steps/Sec: 0.85 +[2025-04-22 13:46:14] (step=0030275) Train Loss: 6.4184, Train Steps/Sec: 0.84 +[2025-04-22 13:46:44] (step=0030300) Train Loss: 6.4223, Train Steps/Sec: 0.84 +[2025-04-22 13:47:14] (step=0030325) Train Loss: 6.3975, Train Steps/Sec: 0.84 +[2025-04-22 13:47:43] (step=0030350) Train Loss: 6.4051, Train Steps/Sec: 0.85 +[2025-04-22 13:48:13] (step=0030375) Train Loss: 6.3583, Train Steps/Sec: 0.84 +[2025-04-22 13:48:43] (step=0030400) Train Loss: 6.3914, Train Steps/Sec: 0.84 +[2025-04-22 13:49:12] (step=0030425) Train Loss: 6.3940, Train Steps/Sec: 0.85 +[2025-04-22 13:49:42] (step=0030450) Train Loss: 6.3859, Train Steps/Sec: 0.84 +[2025-04-22 13:50:12] (step=0030475) Train Loss: 6.4088, Train Steps/Sec: 0.84 +[2025-04-22 13:50:42] (step=0030500) Train Loss: 6.3790, Train Steps/Sec: 0.84 +[2025-04-22 13:51:11] (step=0030525) Train Loss: 6.4163, Train Steps/Sec: 0.84 +[2025-04-22 13:51:41] (step=0030550) Train Loss: 6.3798, Train Steps/Sec: 0.84 +[2025-04-22 13:52:11] (step=0030575) Train Loss: 6.3725, Train Steps/Sec: 0.84 +[2025-04-22 13:52:40] (step=0030600) Train Loss: 6.3918, Train Steps/Sec: 0.84 +[2025-04-22 13:53:10] (step=0030625) Train Loss: 6.4050, Train Steps/Sec: 0.84 +[2025-04-22 13:53:40] (step=0030650) Train Loss: 6.3989, Train Steps/Sec: 0.84 +[2025-04-22 13:54:09] (step=0030675) Train Loss: 6.4142, Train Steps/Sec: 0.85 +[2025-04-22 13:54:39] (step=0030700) Train Loss: 6.4262, Train Steps/Sec: 0.84 +[2025-04-22 13:55:09] (step=0030725) Train Loss: 6.4070, Train Steps/Sec: 0.85 +[2025-04-22 13:55:38] (step=0030750) Train Loss: 6.3846, Train Steps/Sec: 0.84 +[2025-04-22 13:56:08] (step=0030775) Train Loss: 6.3911, Train Steps/Sec: 0.85 +[2025-04-22 13:56:38] (step=0030800) Train Loss: 6.3652, Train Steps/Sec: 0.84 +[2025-04-22 13:57:07] (step=0030825) Train Loss: 6.3735, Train Steps/Sec: 0.85 +[2025-04-22 13:57:37] (step=0030850) Train Loss: 6.4404, Train Steps/Sec: 0.84 +[2025-04-22 13:58:06] (step=0030875) Train Loss: 6.4038, Train Steps/Sec: 0.85 +[2025-04-22 13:58:36] (step=0030900) Train Loss: 6.3841, Train Steps/Sec: 0.84 +[2025-04-22 13:59:06] (step=0030925) Train Loss: 6.3982, Train Steps/Sec: 0.84 +[2025-04-22 13:59:36] (step=0030950) Train Loss: 6.3823, Train Steps/Sec: 0.85 +[2025-04-22 14:00:12] (step=0030975) Train Loss: 6.4065, Train Steps/Sec: 0.69 +[2025-04-22 14:00:42] (step=0031000) Train Loss: 6.3958, Train Steps/Sec: 0.84 +[2025-04-22 14:01:11] (step=0031025) Train Loss: 6.3820, Train Steps/Sec: 0.85 +[2025-04-22 14:01:48] (step=0031050) Train Loss: 6.4150, Train Steps/Sec: 0.68 +[2025-04-22 14:02:17] (step=0031075) Train Loss: 6.3857, Train Steps/Sec: 0.85 +[2025-04-22 14:02:48] (step=0031100) Train Loss: 6.4152, Train Steps/Sec: 0.83 +[2025-04-22 14:03:17] (step=0031125) Train Loss: 6.3958, Train Steps/Sec: 0.84 +[2025-04-22 14:03:47] (step=0031150) Train Loss: 6.4017, Train Steps/Sec: 0.85 +[2025-04-22 14:04:16] (step=0031175) Train Loss: 6.4036, Train Steps/Sec: 0.84 +[2025-04-22 14:04:46] (step=0031200) Train Loss: 6.3896, Train Steps/Sec: 0.84 +[2025-04-22 14:05:15] (step=0031225) Train Loss: 6.3908, Train Steps/Sec: 0.85 +[2025-04-22 14:05:45] (step=0031250) Train Loss: 6.3930, Train Steps/Sec: 0.84 +[2025-04-22 14:06:15] (step=0031275) Train Loss: 6.4126, Train Steps/Sec: 0.84 +[2025-04-22 14:06:45] (step=0031300) Train Loss: 6.4411, Train Steps/Sec: 0.84 +[2025-04-22 14:07:14] (step=0031325) Train Loss: 6.4084, Train Steps/Sec: 0.85 +[2025-04-22 14:07:44] (step=0031350) Train Loss: 6.4094, Train Steps/Sec: 0.84 +[2025-04-22 14:08:13] (step=0031375) Train Loss: 6.3818, Train Steps/Sec: 0.85 +[2025-04-22 14:08:43] (step=0031400) Train Loss: 6.3985, Train Steps/Sec: 0.83 +[2025-04-22 14:09:13] (step=0031425) Train Loss: 6.4125, Train Steps/Sec: 0.85 +[2025-04-22 14:09:42] (step=0031450) Train Loss: 6.3783, Train Steps/Sec: 0.84 +[2025-04-22 14:10:12] (step=0031475) Train Loss: 6.3778, Train Steps/Sec: 0.84 +[2025-04-22 14:10:42] (step=0031500) Train Loss: 6.4124, Train Steps/Sec: 0.85 +[2025-04-22 14:11:12] (step=0031525) Train Loss: 6.3700, Train Steps/Sec: 0.84 +[2025-04-22 14:11:41] (step=0031550) Train Loss: 6.3801, Train Steps/Sec: 0.84 +[2025-04-22 14:12:11] (step=0031575) Train Loss: 6.3888, Train Steps/Sec: 0.85 +[2025-04-22 14:12:40] (step=0031600) Train Loss: 6.3697, Train Steps/Sec: 0.84 +[2025-04-22 14:13:10] (step=0031625) Train Loss: 6.3769, Train Steps/Sec: 0.84 +[2025-04-22 14:13:40] (step=0031650) Train Loss: 6.4454, Train Steps/Sec: 0.84 +[2025-04-22 14:14:10] (step=0031675) Train Loss: 6.4133, Train Steps/Sec: 0.84 +[2025-04-22 14:14:39] (step=0031700) Train Loss: 6.4214, Train Steps/Sec: 0.84 +[2025-04-22 14:15:09] (step=0031725) Train Loss: 6.4169, Train Steps/Sec: 0.84 +[2025-04-22 14:15:39] (step=0031750) Train Loss: 6.3698, Train Steps/Sec: 0.83 +[2025-04-22 14:16:08] (step=0031775) Train Loss: 6.4091, Train Steps/Sec: 0.85 +[2025-04-22 14:16:38] (step=0031800) Train Loss: 6.3889, Train Steps/Sec: 0.84 +[2025-04-22 14:17:08] (step=0031825) Train Loss: 6.3842, Train Steps/Sec: 0.84 +[2025-04-22 14:17:38] (step=0031850) Train Loss: 6.3588, Train Steps/Sec: 0.84 +[2025-04-22 14:18:07] (step=0031875) Train Loss: 6.3965, Train Steps/Sec: 0.85 +[2025-04-22 14:18:37] (step=0031900) Train Loss: 6.4055, Train Steps/Sec: 0.84 +[2025-04-22 14:19:06] (step=0031925) Train Loss: 6.4002, Train Steps/Sec: 0.85 +[2025-04-22 14:19:36] (step=0031950) Train Loss: 6.4114, Train Steps/Sec: 0.84 +[2025-04-22 14:20:05] (step=0031975) Train Loss: 6.4141, Train Steps/Sec: 0.85 +[2025-04-22 14:20:35] (step=0032000) Train Loss: 6.3538, Train Steps/Sec: 0.84 +[2025-04-22 14:25:17] Finish Eval in 32000 steps... +[2025-04-22 14:25:38] Saved checkpoint to checkpoint/BeforeFix_Blip_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_objects/002-GPT-XL/checkpoints/0032000.pt +[2025-04-22 14:25:40] Removed old checkpoint: checkpoint/BeforeFix_Blip_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_objects/002-GPT-XL/checkpoints/0028000.pt +[2025-04-22 14:26:10] (step=0032025) Train Loss: 6.4145, Train Steps/Sec: 0.07 +[2025-04-22 14:26:40] (step=0032050) Train Loss: 6.4145, Train Steps/Sec: 0.84 +[2025-04-22 14:27:10] (step=0032075) Train Loss: 6.3676, Train Steps/Sec: 0.84 +[2025-04-22 14:27:39] (step=0032100) Train Loss: 6.3747, Train Steps/Sec: 0.84 +[2025-04-22 14:28:09] (step=0032125) Train Loss: 6.3598, Train Steps/Sec: 0.85 +[2025-04-22 14:28:39] (step=0032150) Train Loss: 6.3942, Train Steps/Sec: 0.84 +[2025-04-22 14:29:08] (step=0032175) Train Loss: 6.3990, Train Steps/Sec: 0.84 +[2025-04-22 14:29:38] (step=0032200) Train Loss: 6.3713, Train Steps/Sec: 0.84 +[2025-04-22 14:30:14] (step=0032225) Train Loss: 6.3659, Train Steps/Sec: 0.70 +[2025-04-22 14:30:44] (step=0032250) Train Loss: 6.3866, Train Steps/Sec: 0.85 +[2025-04-22 14:31:13] (step=0032275) Train Loss: 6.3922, Train Steps/Sec: 0.85 +[2025-04-22 14:31:43] (step=0032300) Train Loss: 6.4095, Train Steps/Sec: 0.84 +[2025-04-22 14:32:12] (step=0032325) Train Loss: 6.3996, Train Steps/Sec: 0.85 +[2025-04-22 14:32:42] (step=0032350) Train Loss: 6.4176, Train Steps/Sec: 0.85 +[2025-04-22 14:33:12] (step=0032375) Train Loss: 6.3950, Train Steps/Sec: 0.84 +[2025-04-22 14:33:42] (step=0032400) Train Loss: 6.3684, Train Steps/Sec: 0.84 +[2025-04-22 14:34:11] (step=0032425) Train Loss: 6.4232, Train Steps/Sec: 0.85 +[2025-04-22 14:34:41] (step=0032450) Train Loss: 6.4014, Train Steps/Sec: 0.84 +[2025-04-22 14:35:11] (step=0032475) Train Loss: 6.4381, Train Steps/Sec: 0.84 +[2025-04-22 14:35:40] (step=0032500) Train Loss: 6.4431, Train Steps/Sec: 0.85 +[2025-04-22 14:36:10] (step=0032525) Train Loss: 6.3402, Train Steps/Sec: 0.85 +[2025-04-22 14:36:39] (step=0032550) Train Loss: 6.4396, Train Steps/Sec: 0.84 +[2025-04-22 14:37:09] (step=0032575) Train Loss: 6.4032, Train Steps/Sec: 0.85 +[2025-04-22 14:37:39] (step=0032600) Train Loss: 6.3539, Train Steps/Sec: 0.84 +[2025-04-22 14:38:08] (step=0032625) Train Loss: 6.3748, Train Steps/Sec: 0.85 +[2025-04-22 14:38:38] (step=0032650) Train Loss: 6.3854, Train Steps/Sec: 0.84 +[2025-04-22 14:39:08] (step=0032675) Train Loss: 6.4116, Train Steps/Sec: 0.84 +[2025-04-22 14:39:37] (step=0032700) Train Loss: 6.4144, Train Steps/Sec: 0.84 +[2025-04-22 14:40:07] (step=0032725) Train Loss: 6.3929, Train Steps/Sec: 0.84 +[2025-04-22 14:40:37] (step=0032750) Train Loss: 6.4378, Train Steps/Sec: 0.84 +[2025-04-22 14:41:06] (step=0032775) Train Loss: 6.3950, Train Steps/Sec: 0.84 +[2025-04-22 14:41:36] (step=0032800) Train Loss: 6.4499, Train Steps/Sec: 0.84 +[2025-04-22 14:42:06] (step=0032825) Train Loss: 6.3963, Train Steps/Sec: 0.85 +[2025-04-22 14:42:35] (step=0032850) Train Loss: 6.3673, Train Steps/Sec: 0.84 +[2025-04-22 14:43:05] (step=0032875) Train Loss: 6.4241, Train Steps/Sec: 0.85 +[2025-04-22 14:43:35] (step=0032900) Train Loss: 6.4000, Train Steps/Sec: 0.84 +[2025-04-22 14:44:04] (step=0032925) Train Loss: 6.3881, Train Steps/Sec: 0.84 +[2025-04-22 14:44:34] (step=0032950) Train Loss: 6.3986, Train Steps/Sec: 0.85 +[2025-04-22 14:45:04] (step=0032975) Train Loss: 6.4081, Train Steps/Sec: 0.84 +[2025-04-22 14:45:33] (step=0033000) Train Loss: 6.3944, Train Steps/Sec: 0.84 +[2025-04-22 14:46:03] (step=0033025) Train Loss: 6.3976, Train Steps/Sec: 0.84 +[2025-04-22 14:46:33] (step=0033050) Train Loss: 6.4161, Train Steps/Sec: 0.84 +[2025-04-22 14:47:03] (step=0033075) Train Loss: 6.3876, Train Steps/Sec: 0.84 +[2025-04-22 14:47:32] (step=0033100) Train Loss: 6.3639, Train Steps/Sec: 0.84 +[2025-04-22 14:48:02] (step=0033125) Train Loss: 6.3470, Train Steps/Sec: 0.85 +[2025-04-22 14:48:31] (step=0033150) Train Loss: 6.3703, Train Steps/Sec: 0.85 +[2025-04-22 14:49:01] (step=0033175) Train Loss: 6.3811, Train Steps/Sec: 0.84 +[2025-04-22 14:49:31] (step=0033200) Train Loss: 6.3931, Train Steps/Sec: 0.84 +[2025-04-22 14:50:00] (step=0033225) Train Loss: 6.4002, Train Steps/Sec: 0.85 +[2025-04-22 14:50:30] (step=0033250) Train Loss: 6.4262, Train Steps/Sec: 0.85 +[2025-04-22 14:51:00] (step=0033275) Train Loss: 6.4029, Train Steps/Sec: 0.84 +[2025-04-22 14:51:29] (step=0033300) Train Loss: 6.3929, Train Steps/Sec: 0.84 +[2025-04-22 14:51:59] (step=0033325) Train Loss: 6.3903, Train Steps/Sec: 0.85 +[2025-04-22 14:52:28] (step=0033350) Train Loss: 6.4253, Train Steps/Sec: 0.85 +[2025-04-22 14:52:58] (step=0033375) Train Loss: 6.4408, Train Steps/Sec: 0.84 +[2025-04-22 14:53:28] (step=0033400) Train Loss: 6.3884, Train Steps/Sec: 0.84 +[2025-04-22 14:53:57] (step=0033425) Train Loss: 6.4281, Train Steps/Sec: 0.84 +[2025-04-22 14:54:27] (step=0033450) Train Loss: 6.4011, Train Steps/Sec: 0.84 +[2025-04-22 14:54:57] (step=0033475) Train Loss: 6.3645, Train Steps/Sec: 0.85 +[2025-04-22 14:55:26] (step=0033500) Train Loss: 6.4060, Train Steps/Sec: 0.84 +[2025-04-22 14:55:56] (step=0033525) Train Loss: 6.4187, Train Steps/Sec: 0.84 +[2025-04-22 14:56:25] (step=0033550) Train Loss: 6.3640, Train Steps/Sec: 0.85 +[2025-04-22 14:56:55] (step=0033575) Train Loss: 6.3624, Train Steps/Sec: 0.84 +[2025-04-22 14:57:25] (step=0033600) Train Loss: 6.3820, Train Steps/Sec: 0.84 +[2025-04-22 14:57:54] (step=0033625) Train Loss: 6.4253, Train Steps/Sec: 0.85 +[2025-04-22 14:58:24] (step=0033650) Train Loss: 6.3957, Train Steps/Sec: 0.85 +[2025-04-22 14:58:53] (step=0033675) Train Loss: 6.4018, Train Steps/Sec: 0.85 +[2025-04-22 14:59:23] (step=0033700) Train Loss: 6.4039, Train Steps/Sec: 0.84 +[2025-04-22 14:59:52] (step=0033725) Train Loss: 6.3845, Train Steps/Sec: 0.85 +[2025-04-22 15:00:22] (step=0033750) Train Loss: 6.4232, Train Steps/Sec: 0.84 +[2025-04-22 15:00:52] (step=0033775) Train Loss: 6.4244, Train Steps/Sec: 0.84 +[2025-04-22 15:01:22] (step=0033800) Train Loss: 6.4078, Train Steps/Sec: 0.84 +[2025-04-22 15:01:51] (step=0033825) Train Loss: 6.4455, Train Steps/Sec: 0.85 +[2025-04-22 15:02:21] (step=0033850) Train Loss: 6.3647, Train Steps/Sec: 0.84 +[2025-04-22 15:02:50] (step=0033875) Train Loss: 6.4026, Train Steps/Sec: 0.84 +[2025-04-22 15:03:20] (step=0033900) Train Loss: 6.3868, Train Steps/Sec: 0.85 +[2025-04-22 15:03:50] (step=0033925) Train Loss: 6.4215, Train Steps/Sec: 0.85 +[2025-04-22 15:04:19] (step=0033950) Train Loss: 6.4025, Train Steps/Sec: 0.84 +[2025-04-22 15:04:49] (step=0033975) Train Loss: 6.3735, Train Steps/Sec: 0.85 +[2025-04-22 15:05:18] (step=0034000) Train Loss: 6.4061, Train Steps/Sec: 0.84 +[2025-04-22 15:10:01] Finish Eval in 34000 steps... +[2025-04-22 15:10:22] Saved checkpoint to checkpoint/BeforeFix_Blip_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_objects/002-GPT-XL/checkpoints/0034000.pt +[2025-04-22 15:10:24] Removed old checkpoint: checkpoint/BeforeFix_Blip_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_objects/002-GPT-XL/checkpoints/0030000.pt +[2025-04-22 15:10:54] (step=0034025) Train Loss: 6.4240, Train Steps/Sec: 0.07 +[2025-04-22 15:11:23] (step=0034050) Train Loss: 6.4115, Train Steps/Sec: 0.85 +[2025-04-22 15:11:53] (step=0034075) Train Loss: 6.3818, Train Steps/Sec: 0.85 +[2025-04-22 15:12:22] (step=0034100) Train Loss: 6.4001, Train Steps/Sec: 0.85 +[2025-04-22 15:12:52] (step=0034125) Train Loss: 6.3942, Train Steps/Sec: 0.84 +[2025-04-22 15:13:21] (step=0034150) Train Loss: 6.4018, Train Steps/Sec: 0.84 +[2025-04-22 15:13:51] (step=0034175) Train Loss: 6.3549, Train Steps/Sec: 0.85 +[2025-04-22 15:14:20] (step=0034200) Train Loss: 6.3248, Train Steps/Sec: 0.85 +[2025-04-22 15:14:50] (step=0034225) Train Loss: 6.4017, Train Steps/Sec: 0.85 +[2025-04-22 15:15:20] (step=0034250) Train Loss: 6.4316, Train Steps/Sec: 0.84 +[2025-04-22 15:15:49] (step=0034275) Train Loss: 6.3115, Train Steps/Sec: 0.85 +[2025-04-22 15:16:19] (step=0034300) Train Loss: 6.3861, Train Steps/Sec: 0.84 +[2025-04-22 15:16:48] (step=0034325) Train Loss: 6.3744, Train Steps/Sec: 0.84 +[2025-04-22 15:17:18] (step=0034350) Train Loss: 6.3944, Train Steps/Sec: 0.85 +[2025-04-22 15:17:47] (step=0034375) Train Loss: 6.4000, Train Steps/Sec: 0.84 +[2025-04-22 15:18:25] (step=0034400) Train Loss: 6.4114, Train Steps/Sec: 0.67 +[2025-04-22 15:18:54] (step=0034425) Train Loss: 6.3903, Train Steps/Sec: 0.84 +[2025-04-22 15:19:30] (step=0034450) Train Loss: 6.4425, Train Steps/Sec: 0.70 +[2025-04-22 15:19:59] (step=0034475) Train Loss: 6.3950, Train Steps/Sec: 0.85 +[2025-04-22 15:20:29] (step=0034500) Train Loss: 6.4144, Train Steps/Sec: 0.83 +[2025-04-22 15:20:59] (step=0034525) Train Loss: 6.3963, Train Steps/Sec: 0.85 +[2025-04-22 15:21:28] (step=0034550) Train Loss: 6.3794, Train Steps/Sec: 0.85 +[2025-04-22 15:21:58] (step=0034575) Train Loss: 6.3995, Train Steps/Sec: 0.84 +[2025-04-22 15:22:28] (step=0034600) Train Loss: 6.4065, Train Steps/Sec: 0.85 +[2025-04-22 15:22:57] (step=0034625) Train Loss: 6.3700, Train Steps/Sec: 0.85 +[2025-04-22 15:23:27] (step=0034650) Train Loss: 6.3474, Train Steps/Sec: 0.84 +[2025-04-22 15:23:56] (step=0034675) Train Loss: 6.4095, Train Steps/Sec: 0.85 +[2025-04-22 15:24:26] (step=0034700) Train Loss: 6.4319, Train Steps/Sec: 0.84 +[2025-04-22 15:24:56] (step=0034725) Train Loss: 6.3898, Train Steps/Sec: 0.84 +[2025-04-22 15:25:25] (step=0034750) Train Loss: 6.4153, Train Steps/Sec: 0.85 +[2025-04-22 15:25:55] (step=0034775) Train Loss: 6.3761, Train Steps/Sec: 0.84 +[2025-04-22 15:26:25] (step=0034800) Train Loss: 6.3956, Train Steps/Sec: 0.84 +[2025-04-22 15:26:54] (step=0034825) Train Loss: 6.3757, Train Steps/Sec: 0.84 +[2025-04-22 15:27:24] (step=0034850) Train Loss: 6.4345, Train Steps/Sec: 0.84 +[2025-04-22 15:27:53] (step=0034875) Train Loss: 6.3694, Train Steps/Sec: 0.85 +[2025-04-22 15:28:38] (step=0034900) Train Loss: 6.4109, Train Steps/Sec: 0.56 +[2025-04-22 15:29:08] (step=0034925) Train Loss: 6.3543, Train Steps/Sec: 0.84 +[2025-04-22 15:29:37] (step=0034950) Train Loss: 6.3924, Train Steps/Sec: 0.84 +[2025-04-22 15:30:07] (step=0034975) Train Loss: 6.4063, Train Steps/Sec: 0.85 +[2025-04-22 15:30:37] (step=0035000) Train Loss: 6.3812, Train Steps/Sec: 0.84 +[2025-04-22 15:31:06] (step=0035025) Train Loss: 6.4077, Train Steps/Sec: 0.85 +[2025-04-22 15:31:36] (step=0035050) Train Loss: 6.4396, Train Steps/Sec: 0.84 +[2025-04-22 15:32:05] (step=0035075) Train Loss: 6.3567, Train Steps/Sec: 0.85 +[2025-04-22 15:32:35] (step=0035100) Train Loss: 6.4010, Train Steps/Sec: 0.84 +[2025-04-22 15:33:04] (step=0035125) Train Loss: 6.3978, Train Steps/Sec: 0.85 +[2025-04-22 15:33:34] (step=0035150) Train Loss: 6.3360, Train Steps/Sec: 0.85 +[2025-04-22 15:34:03] (step=0035175) Train Loss: 6.3980, Train Steps/Sec: 0.85 +[2025-04-22 15:34:33] (step=0035200) Train Loss: 6.3619, Train Steps/Sec: 0.85 +[2025-04-22 15:35:02] (step=0035225) Train Loss: 6.3978, Train Steps/Sec: 0.85 +[2025-04-22 15:35:32] (step=0035250) Train Loss: 6.4172, Train Steps/Sec: 0.85 +[2025-04-22 15:36:02] (step=0035275) Train Loss: 6.4069, Train Steps/Sec: 0.85 +[2025-04-22 15:36:31] (step=0035300) Train Loss: 6.4067, Train Steps/Sec: 0.84 +[2025-04-22 15:37:01] (step=0035325) Train Loss: 6.3459, Train Steps/Sec: 0.85 +[2025-04-22 15:37:30] (step=0035350) Train Loss: 6.4221, Train Steps/Sec: 0.84 +[2025-04-22 15:38:00] (step=0035375) Train Loss: 6.3724, Train Steps/Sec: 0.85 +[2025-04-22 15:38:30] (step=0035400) Train Loss: 6.3626, Train Steps/Sec: 0.84 +[2025-04-22 15:38:59] (step=0035425) Train Loss: 6.3963, Train Steps/Sec: 0.85 +[2025-04-22 15:39:29] (step=0035450) Train Loss: 6.4150, Train Steps/Sec: 0.84 +[2025-04-22 15:39:58] (step=0035475) Train Loss: 6.4221, Train Steps/Sec: 0.85 +[2025-04-22 15:40:28] (step=0035500) Train Loss: 6.4129, Train Steps/Sec: 0.84 +[2025-04-22 15:40:57] (step=0035525) Train Loss: 6.3829, Train Steps/Sec: 0.85 +[2025-04-22 15:41:27] (step=0035550) Train Loss: 6.3895, Train Steps/Sec: 0.85 +[2025-04-22 15:41:56] (step=0035575) Train Loss: 6.3887, Train Steps/Sec: 0.85 +[2025-04-22 15:42:26] (step=0035600) Train Loss: 6.3739, Train Steps/Sec: 0.85 +[2025-04-22 15:42:55] (step=0035625) Train Loss: 6.4013, Train Steps/Sec: 0.84 +[2025-04-22 15:43:25] (step=0035650) Train Loss: 6.3944, Train Steps/Sec: 0.85 +[2025-04-22 15:43:55] (step=0035675) Train Loss: 6.4297, Train Steps/Sec: 0.84 +[2025-04-22 15:44:25] (step=0035700) Train Loss: 6.3936, Train Steps/Sec: 0.84 +[2025-04-22 15:44:54] (step=0035725) Train Loss: 6.3762, Train Steps/Sec: 0.84 +[2025-04-22 15:45:24] (step=0035750) Train Loss: 6.3790, Train Steps/Sec: 0.84 +[2025-04-22 15:45:53] (step=0035775) Train Loss: 6.4074, Train Steps/Sec: 0.85 +[2025-04-22 15:46:23] (step=0035800) Train Loss: 6.3839, Train Steps/Sec: 0.84 +[2025-04-22 15:46:52] (step=0035825) Train Loss: 6.3894, Train Steps/Sec: 0.85 +[2025-04-22 15:47:22] (step=0035850) Train Loss: 6.3680, Train Steps/Sec: 0.85 +[2025-04-22 15:47:51] (step=0035875) Train Loss: 6.3920, Train Steps/Sec: 0.85 +[2025-04-22 15:48:21] (step=0035900) Train Loss: 6.4001, Train Steps/Sec: 0.85 +[2025-04-22 15:48:50] (step=0035925) Train Loss: 6.3946, Train Steps/Sec: 0.85 +[2025-04-22 15:49:20] (step=0035950) Train Loss: 6.3736, Train Steps/Sec: 0.85 +[2025-04-22 15:49:49] (step=0035975) Train Loss: 6.3566, Train Steps/Sec: 0.84 +[2025-04-22 15:50:19] (step=0036000) Train Loss: 6.3866, Train Steps/Sec: 0.84 +[2025-04-22 15:55:00] Finish Eval in 36000 steps... +[2025-04-22 15:55:22] Saved checkpoint to checkpoint/BeforeFix_Blip_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_objects/002-GPT-XL/checkpoints/0036000.pt +[2025-04-22 15:55:24] Removed old checkpoint: checkpoint/BeforeFix_Blip_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_objects/002-GPT-XL/checkpoints/0032000.pt +[2025-04-22 15:55:53] (step=0036025) Train Loss: 6.3926, Train Steps/Sec: 0.07 +[2025-04-22 15:56:23] (step=0036050) Train Loss: 6.3844, Train Steps/Sec: 0.84 +[2025-04-22 15:56:52] (step=0036075) Train Loss: 6.3889, Train Steps/Sec: 0.84 +[2025-04-22 15:57:22] (step=0036100) Train Loss: 6.3621, Train Steps/Sec: 0.85 +[2025-04-22 15:57:51] (step=0036125) Train Loss: 6.3975, Train Steps/Sec: 0.85 +[2025-04-22 15:58:21] (step=0036150) Train Loss: 6.4166, Train Steps/Sec: 0.85 +[2025-04-22 15:58:50] (step=0036175) Train Loss: 6.3883, Train Steps/Sec: 0.85 +[2025-04-22 15:59:20] (step=0036200) Train Loss: 6.4191, Train Steps/Sec: 0.84 +[2025-04-22 15:59:49] (step=0036225) Train Loss: 6.4028, Train Steps/Sec: 0.85 +[2025-04-22 16:00:19] (step=0036250) Train Loss: 6.4002, Train Steps/Sec: 0.85 +[2025-04-22 16:00:48] (step=0036275) Train Loss: 6.3773, Train Steps/Sec: 0.85 +[2025-04-22 16:01:18] (step=0036300) Train Loss: 6.3965, Train Steps/Sec: 0.85 +[2025-04-22 16:01:47] (step=0036325) Train Loss: 6.3721, Train Steps/Sec: 0.85 +[2025-04-22 16:02:17] (step=0036350) Train Loss: 6.3787, Train Steps/Sec: 0.85 +[2025-04-22 16:02:46] (step=0036375) Train Loss: 6.3801, Train Steps/Sec: 0.85 +[2025-04-22 16:03:16] (step=0036400) Train Loss: 6.3571, Train Steps/Sec: 0.85 +[2025-04-22 16:03:45] (step=0036425) Train Loss: 6.3542, Train Steps/Sec: 0.85 +[2025-04-22 16:04:21] (step=0036450) Train Loss: 6.3737, Train Steps/Sec: 0.70 +[2025-04-22 16:04:51] (step=0036475) Train Loss: 6.4427, Train Steps/Sec: 0.84 +[2025-04-22 16:05:21] (step=0036500) Train Loss: 6.3957, Train Steps/Sec: 0.84 +[2025-04-22 16:05:50] (step=0036525) Train Loss: 6.4257, Train Steps/Sec: 0.85 +[2025-04-22 16:06:27] (step=0036550) Train Loss: 6.3909, Train Steps/Sec: 0.68 +[2025-04-22 16:06:57] (step=0036575) Train Loss: 6.3853, Train Steps/Sec: 0.84 +[2025-04-22 16:07:26] (step=0036600) Train Loss: 6.4033, Train Steps/Sec: 0.84 +[2025-04-22 16:07:56] (step=0036625) Train Loss: 6.3511, Train Steps/Sec: 0.85 +[2025-04-22 16:08:25] (step=0036650) Train Loss: 6.3725, Train Steps/Sec: 0.84 +[2025-04-22 16:08:55] (step=0036675) Train Loss: 6.4274, Train Steps/Sec: 0.84 +[2025-04-22 16:09:25] (step=0036700) Train Loss: 6.3893, Train Steps/Sec: 0.83 +[2025-04-22 16:09:55] (step=0036725) Train Loss: 6.3850, Train Steps/Sec: 0.84 +[2025-04-22 16:10:25] (step=0036750) Train Loss: 6.3646, Train Steps/Sec: 0.84 +[2025-04-22 16:10:55] (step=0036775) Train Loss: 6.3610, Train Steps/Sec: 0.84 +[2025-04-22 16:11:24] (step=0036800) Train Loss: 6.4365, Train Steps/Sec: 0.84 +[2025-04-22 16:11:54] (step=0036825) Train Loss: 6.3569, Train Steps/Sec: 0.85 +[2025-04-22 16:12:23] (step=0036850) Train Loss: 6.3901, Train Steps/Sec: 0.85 +[2025-04-22 16:12:53] (step=0036875) Train Loss: 6.3364, Train Steps/Sec: 0.85 +[2025-04-22 16:13:22] (step=0036900) Train Loss: 6.3706, Train Steps/Sec: 0.84 +[2025-04-22 16:13:52] (step=0036925) Train Loss: 6.3937, Train Steps/Sec: 0.85 +[2025-04-22 16:14:21] (step=0036950) Train Loss: 6.4491, Train Steps/Sec: 0.85 +[2025-04-22 16:14:51] (step=0036975) Train Loss: 6.3584, Train Steps/Sec: 0.85 +[2025-04-22 16:15:21] (step=0037000) Train Loss: 6.4146, Train Steps/Sec: 0.84 +[2025-04-22 16:15:50] (step=0037025) Train Loss: 6.4016, Train Steps/Sec: 0.85 +[2025-04-22 16:16:20] (step=0037050) Train Loss: 6.4073, Train Steps/Sec: 0.85 +[2025-04-22 16:16:49] (step=0037075) Train Loss: 6.3936, Train Steps/Sec: 0.85 +[2025-04-22 16:17:19] (step=0037100) Train Loss: 6.3821, Train Steps/Sec: 0.84 +[2025-04-22 16:17:48] (step=0037125) Train Loss: 6.4026, Train Steps/Sec: 0.85 +[2025-04-22 16:18:18] (step=0037150) Train Loss: 6.3746, Train Steps/Sec: 0.85 +[2025-04-22 16:18:47] (step=0037175) Train Loss: 6.3622, Train Steps/Sec: 0.85 +[2025-04-22 16:19:17] (step=0037200) Train Loss: 6.3726, Train Steps/Sec: 0.84 +[2025-04-22 16:19:46] (step=0037225) Train Loss: 6.3852, Train Steps/Sec: 0.85 +[2025-04-22 16:20:16] (step=0037250) Train Loss: 6.3942, Train Steps/Sec: 0.85 +[2025-04-22 16:20:45] (step=0037275) Train Loss: 6.4194, Train Steps/Sec: 0.85 +[2025-04-22 16:21:14] (step=0037300) Train Loss: 6.3860, Train Steps/Sec: 0.84 +[2025-04-22 16:21:44] (step=0037325) Train Loss: 6.3733, Train Steps/Sec: 0.85 +[2025-04-22 16:22:13] (step=0037350) Train Loss: 6.4012, Train Steps/Sec: 0.85 +[2025-04-22 16:22:43] (step=0037375) Train Loss: 6.3772, Train Steps/Sec: 0.84 +[2025-04-22 16:23:13] (step=0037400) Train Loss: 6.3920, Train Steps/Sec: 0.84 +[2025-04-22 16:23:42] (step=0037425) Train Loss: 6.3933, Train Steps/Sec: 0.85 +[2025-04-22 16:24:12] (step=0037450) Train Loss: 6.4193, Train Steps/Sec: 0.85 +[2025-04-22 16:24:41] (step=0037475) Train Loss: 6.3799, Train Steps/Sec: 0.84 +[2025-04-22 16:25:11] (step=0037500) Train Loss: 6.3580, Train Steps/Sec: 0.85 +[2025-04-22 16:25:40] (step=0037525) Train Loss: 6.3917, Train Steps/Sec: 0.84 +[2025-04-22 16:26:10] (step=0037550) Train Loss: 6.3670, Train Steps/Sec: 0.84 +[2025-04-22 16:26:46] (step=0037575) Train Loss: 6.4076, Train Steps/Sec: 0.70 +[2025-04-22 16:27:16] (step=0037600) Train Loss: 6.3415, Train Steps/Sec: 0.84 +[2025-04-22 16:27:45] (step=0037625) Train Loss: 6.3662, Train Steps/Sec: 0.85 +[2025-04-22 16:28:15] (step=0037650) Train Loss: 6.3984, Train Steps/Sec: 0.84 +[2025-04-22 16:28:44] (step=0037675) Train Loss: 6.3988, Train Steps/Sec: 0.84 +[2025-04-22 16:29:14] (step=0037700) Train Loss: 6.3682, Train Steps/Sec: 0.84 +[2025-04-22 16:29:44] (step=0037725) Train Loss: 6.4348, Train Steps/Sec: 0.84 +[2025-04-22 16:30:13] (step=0037750) Train Loss: 6.3407, Train Steps/Sec: 0.84 +[2025-04-22 16:30:43] (step=0037775) Train Loss: 6.3601, Train Steps/Sec: 0.86 +[2025-04-22 16:31:12] (step=0037800) Train Loss: 6.3757, Train Steps/Sec: 0.85 +[2025-04-22 16:31:42] (step=0037825) Train Loss: 6.3906, Train Steps/Sec: 0.84 +[2025-04-22 16:32:11] (step=0037850) Train Loss: 6.3762, Train Steps/Sec: 0.85 +[2025-04-22 16:32:41] (step=0037875) Train Loss: 6.3912, Train Steps/Sec: 0.85 +[2025-04-22 16:33:10] (step=0037900) Train Loss: 6.3727, Train Steps/Sec: 0.84 +[2025-04-22 16:33:40] (step=0037925) Train Loss: 6.3845, Train Steps/Sec: 0.85 +[2025-04-22 16:34:09] (step=0037950) Train Loss: 6.3648, Train Steps/Sec: 0.85 +[2025-04-22 16:34:39] (step=0037975) Train Loss: 6.4365, Train Steps/Sec: 0.85 +[2025-04-22 16:35:08] (step=0038000) Train Loss: 6.4148, Train Steps/Sec: 0.85 +[2025-04-22 16:39:53] Finish Eval in 38000 steps... +[2025-04-22 16:40:16] Saved checkpoint to checkpoint/BeforeFix_Blip_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_objects/002-GPT-XL/checkpoints/0038000.pt +[2025-04-22 16:40:18] Removed old checkpoint: checkpoint/BeforeFix_Blip_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_objects/002-GPT-XL/checkpoints/0034000.pt +[2025-04-22 16:40:48] (step=0038025) Train Loss: 6.3879, Train Steps/Sec: 0.07 +[2025-04-22 16:41:17] (step=0038050) Train Loss: 6.3910, Train Steps/Sec: 0.85 +[2025-04-22 16:41:47] (step=0038075) Train Loss: 6.4278, Train Steps/Sec: 0.84 +[2025-04-22 16:42:16] (step=0038100) Train Loss: 6.3830, Train Steps/Sec: 0.84 +[2025-04-22 16:42:46] (step=0038125) Train Loss: 6.4145, Train Steps/Sec: 0.84 +[2025-04-22 16:43:16] (step=0038150) Train Loss: 6.3584, Train Steps/Sec: 0.85 +[2025-04-22 16:43:45] (step=0038175) Train Loss: 6.3594, Train Steps/Sec: 0.85 +[2025-04-22 16:44:15] (step=0038200) Train Loss: 6.3688, Train Steps/Sec: 0.83 +[2025-04-22 16:44:45] (step=0038225) Train Loss: 6.3466, Train Steps/Sec: 0.85 +[2025-04-22 16:45:14] (step=0038250) Train Loss: 6.3710, Train Steps/Sec: 0.85 +[2025-04-22 16:45:44] (step=0038275) Train Loss: 6.3718, Train Steps/Sec: 0.84 +[2025-04-22 16:46:14] (step=0038300) Train Loss: 6.4114, Train Steps/Sec: 0.84 +[2025-04-22 16:46:43] (step=0038325) Train Loss: 6.4093, Train Steps/Sec: 0.85 +[2025-04-22 16:47:13] (step=0038350) Train Loss: 6.3862, Train Steps/Sec: 0.85 +[2025-04-22 16:47:42] (step=0038375) Train Loss: 6.4022, Train Steps/Sec: 0.85 +[2025-04-22 16:48:12] (step=0038400) Train Loss: 6.4000, Train Steps/Sec: 0.84 +[2025-04-22 16:48:41] (step=0038425) Train Loss: 6.4421, Train Steps/Sec: 0.85 +[2025-04-22 16:49:11] (step=0038450) Train Loss: 6.3572, Train Steps/Sec: 0.85 +[2025-04-22 16:49:40] (step=0038475) Train Loss: 6.3880, Train Steps/Sec: 0.85 +[2025-04-22 16:50:10] (step=0038500) Train Loss: 6.4173, Train Steps/Sec: 0.84 +[2025-04-22 16:50:40] (step=0038525) Train Loss: 6.3651, Train Steps/Sec: 0.85 +[2025-04-22 16:51:09] (step=0038550) Train Loss: 6.3772, Train Steps/Sec: 0.85 +[2025-04-22 16:51:39] (step=0038575) Train Loss: 6.3964, Train Steps/Sec: 0.85 +[2025-04-22 16:52:08] (step=0038600) Train Loss: 6.3644, Train Steps/Sec: 0.85 +[2025-04-22 16:52:37] (step=0038625) Train Loss: 6.3885, Train Steps/Sec: 0.85 +[2025-04-22 16:53:07] (step=0038650) Train Loss: 6.3826, Train Steps/Sec: 0.85 +[2025-04-22 16:53:37] (step=0038675) Train Loss: 6.4106, Train Steps/Sec: 0.85 +[2025-04-22 16:54:06] (step=0038700) Train Loss: 6.4017, Train Steps/Sec: 0.84 +[2025-04-22 16:54:36] (step=0038725) Train Loss: 6.4089, Train Steps/Sec: 0.84 +[2025-04-22 16:55:05] (step=0038750) Train Loss: 6.4190, Train Steps/Sec: 0.85 +[2025-04-22 16:55:35] (step=0038775) Train Loss: 6.3880, Train Steps/Sec: 0.84 +[2025-04-22 16:56:05] (step=0038800) Train Loss: 6.4394, Train Steps/Sec: 0.85 +[2025-04-22 16:56:34] (step=0038825) Train Loss: 6.3848, Train Steps/Sec: 0.84 +[2025-04-22 16:57:04] (step=0038850) Train Loss: 6.3746, Train Steps/Sec: 0.85 +[2025-04-22 16:57:33] (step=0038875) Train Loss: 6.3936, Train Steps/Sec: 0.85 +[2025-04-22 16:58:03] (step=0038900) Train Loss: 6.3625, Train Steps/Sec: 0.84 +[2025-04-22 16:58:32] (step=0038925) Train Loss: 6.3931, Train Steps/Sec: 0.85 +[2025-04-22 16:59:02] (step=0038950) Train Loss: 6.3660, Train Steps/Sec: 0.84 +[2025-04-22 16:59:31] (step=0038975) Train Loss: 6.3736, Train Steps/Sec: 0.85 +[2025-04-22 17:00:01] (step=0039000) Train Loss: 6.3818, Train Steps/Sec: 0.84 +[2025-04-22 17:00:30] (step=0039025) Train Loss: 6.3904, Train Steps/Sec: 0.85 +[2025-04-22 17:01:00] (step=0039050) Train Loss: 6.3967, Train Steps/Sec: 0.85 +[2025-04-22 17:01:29] (step=0039075) Train Loss: 6.3369, Train Steps/Sec: 0.85 +[2025-04-22 17:01:59] (step=0039100) Train Loss: 6.4115, Train Steps/Sec: 0.84 +[2025-04-22 17:02:29] (step=0039125) Train Loss: 6.4077, Train Steps/Sec: 0.84 +[2025-04-22 17:02:58] (step=0039150) Train Loss: 6.3622, Train Steps/Sec: 0.85 +[2025-04-22 17:03:28] (step=0039175) Train Loss: 6.3941, Train Steps/Sec: 0.85 +[2025-04-22 17:03:57] (step=0039200) Train Loss: 6.3965, Train Steps/Sec: 0.84 +[2025-04-22 17:04:27] (step=0039225) Train Loss: 6.3501, Train Steps/Sec: 0.85 +[2025-04-22 17:04:57] (step=0039250) Train Loss: 6.3854, Train Steps/Sec: 0.84 +[2025-04-22 17:05:26] (step=0039275) Train Loss: 6.3476, Train Steps/Sec: 0.85 +[2025-04-22 17:05:56] (step=0039300) Train Loss: 6.4154, Train Steps/Sec: 0.85 +[2025-04-22 17:06:25] (step=0039325) Train Loss: 6.3741, Train Steps/Sec: 0.85 +[2025-04-22 17:06:55] (step=0039350) Train Loss: 6.4183, Train Steps/Sec: 0.84 +[2025-04-22 17:07:24] (step=0039375) Train Loss: 6.3601, Train Steps/Sec: 0.85 +[2025-04-22 17:07:54] (step=0039400) Train Loss: 6.3854, Train Steps/Sec: 0.85 +[2025-04-22 17:08:23] (step=0039425) Train Loss: 6.3627, Train Steps/Sec: 0.84 +[2025-04-22 17:08:53] (step=0039450) Train Loss: 6.3506, Train Steps/Sec: 0.84 +[2025-04-22 17:09:29] (step=0039475) Train Loss: 6.3800, Train Steps/Sec: 0.70 +[2025-04-22 17:10:05] (step=0039500) Train Loss: 6.3657, Train Steps/Sec: 0.69 +[2025-04-22 17:10:34] (step=0039525) Train Loss: 6.3602, Train Steps/Sec: 0.85 +[2025-04-22 17:11:04] (step=0039550) Train Loss: 6.3837, Train Steps/Sec: 0.85 +[2025-04-22 17:11:33] (step=0039575) Train Loss: 6.4554, Train Steps/Sec: 0.84 +[2025-04-22 17:12:03] (step=0039600) Train Loss: 6.3356, Train Steps/Sec: 0.84 +[2025-04-22 17:12:33] (step=0039625) Train Loss: 6.4024, Train Steps/Sec: 0.85 +[2025-04-22 17:13:02] (step=0039650) Train Loss: 6.3968, Train Steps/Sec: 0.85 +[2025-04-22 17:13:32] (step=0039675) Train Loss: 6.4121, Train Steps/Sec: 0.85 +[2025-04-22 17:14:01] (step=0039700) Train Loss: 6.4003, Train Steps/Sec: 0.85 +[2025-04-22 17:14:31] (step=0039725) Train Loss: 6.3771, Train Steps/Sec: 0.85 +[2025-04-22 17:15:00] (step=0039750) Train Loss: 6.3769, Train Steps/Sec: 0.85 +[2025-04-22 17:15:30] (step=0039775) Train Loss: 6.3925, Train Steps/Sec: 0.84 +[2025-04-22 17:15:59] (step=0039800) Train Loss: 6.3614, Train Steps/Sec: 0.85 +[2025-04-22 17:16:29] (step=0039825) Train Loss: 6.4094, Train Steps/Sec: 0.84 +[2025-04-22 17:16:58] (step=0039850) Train Loss: 6.3623, Train Steps/Sec: 0.85 +[2025-04-22 17:17:28] (step=0039875) Train Loss: 6.3943, Train Steps/Sec: 0.85 +[2025-04-22 17:17:58] (step=0039900) Train Loss: 6.3972, Train Steps/Sec: 0.84 +[2025-04-22 17:18:27] (step=0039925) Train Loss: 6.3980, Train Steps/Sec: 0.85 +[2025-04-22 17:18:57] (step=0039950) Train Loss: 6.4260, Train Steps/Sec: 0.85 +[2025-04-22 17:19:31] (step=0039975) Train Loss: 6.3641, Train Steps/Sec: 0.72 +[2025-04-22 17:20:01] (step=0040000) Train Loss: 6.4167, Train Steps/Sec: 0.84 +[2025-04-22 17:24:42] Finish Eval in 40000 steps... +[2025-04-22 17:25:03] Saved checkpoint to checkpoint/BeforeFix_Blip_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_objects/002-GPT-XL/checkpoints/0040000.pt +[2025-04-22 17:25:06] Removed old checkpoint: checkpoint/BeforeFix_Blip_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_objects/002-GPT-XL/checkpoints/0036000.pt +[2025-04-22 17:25:44] (step=0040025) Train Loss: 6.4418, Train Steps/Sec: 0.07 +[2025-04-22 17:26:13] (step=0040050) Train Loss: 6.3717, Train Steps/Sec: 0.85 +[2025-04-22 17:26:43] (step=0040075) Train Loss: 6.3587, Train Steps/Sec: 0.85 +[2025-04-22 17:27:12] (step=0040100) Train Loss: 6.4024, Train Steps/Sec: 0.85 +[2025-04-22 17:27:42] (step=0040125) Train Loss: 6.3943, Train Steps/Sec: 0.84 +[2025-04-22 17:28:11] (step=0040150) Train Loss: 6.3823, Train Steps/Sec: 0.85 +[2025-04-22 17:28:41] (step=0040175) Train Loss: 6.3927, Train Steps/Sec: 0.85 +[2025-04-22 17:29:11] (step=0040200) Train Loss: 6.4010, Train Steps/Sec: 0.84 +[2025-04-22 17:29:40] (step=0040225) Train Loss: 6.4033, Train Steps/Sec: 0.85 +[2025-04-22 17:30:10] (step=0040250) Train Loss: 6.3539, Train Steps/Sec: 0.85 +[2025-04-22 17:30:39] (step=0040275) Train Loss: 6.3940, Train Steps/Sec: 0.85 +[2025-04-22 17:31:09] (step=0040300) Train Loss: 6.4047, Train Steps/Sec: 0.84 +[2025-04-22 17:31:38] (step=0040325) Train Loss: 6.4025, Train Steps/Sec: 0.85 +[2025-04-22 17:32:08] (step=0040350) Train Loss: 6.3988, Train Steps/Sec: 0.84 +[2025-04-22 17:32:37] (step=0040375) Train Loss: 6.3818, Train Steps/Sec: 0.85 +[2025-04-22 17:33:07] (step=0040400) Train Loss: 6.4376, Train Steps/Sec: 0.84 +[2025-04-22 17:33:36] (step=0040425) Train Loss: 6.3867, Train Steps/Sec: 0.85 +[2025-04-22 17:34:06] (step=0040450) Train Loss: 6.3956, Train Steps/Sec: 0.85 +[2025-04-22 17:34:35] (step=0040475) Train Loss: 6.3705, Train Steps/Sec: 0.85 +[2025-04-22 17:35:05] (step=0040500) Train Loss: 6.3735, Train Steps/Sec: 0.85 +[2025-04-22 17:35:35] (step=0040525) Train Loss: 6.3882, Train Steps/Sec: 0.84 +[2025-04-22 17:36:04] (step=0040550) Train Loss: 6.4112, Train Steps/Sec: 0.85 +[2025-04-22 17:36:33] (step=0040575) Train Loss: 6.3647, Train Steps/Sec: 0.85 +[2025-04-22 17:37:03] (step=0040600) Train Loss: 6.4277, Train Steps/Sec: 0.84 +[2025-04-22 17:37:32] (step=0040625) Train Loss: 6.3959, Train Steps/Sec: 0.85 +[2025-04-22 17:38:02] (step=0040650) Train Loss: 6.4219, Train Steps/Sec: 0.84 +[2025-04-22 17:38:32] (step=0040675) Train Loss: 6.3737, Train Steps/Sec: 0.85 +[2025-04-22 17:39:01] (step=0040700) Train Loss: 6.4057, Train Steps/Sec: 0.85 +[2025-04-22 17:39:31] (step=0040725) Train Loss: 6.3926, Train Steps/Sec: 0.85 +[2025-04-22 17:40:00] (step=0040750) Train Loss: 6.3837, Train Steps/Sec: 0.85 +[2025-04-22 17:40:30] (step=0040775) Train Loss: 6.3801, Train Steps/Sec: 0.84 +[2025-04-22 17:40:59] (step=0040800) Train Loss: 6.3722, Train Steps/Sec: 0.84 +[2025-04-22 17:41:29] (step=0040825) Train Loss: 6.4028, Train Steps/Sec: 0.85 +[2025-04-22 17:41:59] (step=0040850) Train Loss: 6.3711, Train Steps/Sec: 0.84 +[2025-04-22 17:42:28] (step=0040875) Train Loss: 6.3848, Train Steps/Sec: 0.85 +[2025-04-22 17:42:58] (step=0040900) Train Loss: 6.3917, Train Steps/Sec: 0.84 +[2025-04-22 17:43:03] Beginning epoch 1... +[2025-04-22 17:43:32] (step=0040925) Train Loss: 6.3901, Train Steps/Sec: 0.73 +[2025-04-22 17:44:01] (step=0040950) Train Loss: 6.3812, Train Steps/Sec: 0.84 +[2025-04-22 17:44:31] (step=0040975) Train Loss: 6.3427, Train Steps/Sec: 0.85 +[2025-04-22 17:45:01] (step=0041000) Train Loss: 6.3212, Train Steps/Sec: 0.84 +[2025-04-22 17:45:30] (step=0041025) Train Loss: 6.3258, Train Steps/Sec: 0.84 +[2025-04-22 17:46:00] (step=0041050) Train Loss: 6.3368, Train Steps/Sec: 0.84 +[2025-04-22 17:46:30] (step=0041075) Train Loss: 6.3127, Train Steps/Sec: 0.84 +[2025-04-22 17:47:00] (step=0041100) Train Loss: 6.3534, Train Steps/Sec: 0.84 +[2025-04-22 17:47:29] (step=0041125) Train Loss: 6.3645, Train Steps/Sec: 0.85 +[2025-04-22 17:47:59] (step=0041150) Train Loss: 6.3807, Train Steps/Sec: 0.84 +[2025-04-22 17:48:29] (step=0041175) Train Loss: 6.3954, Train Steps/Sec: 0.84 +[2025-04-22 17:48:59] (step=0041200) Train Loss: 6.3611, Train Steps/Sec: 0.84 +[2025-04-22 17:49:28] (step=0041225) Train Loss: 6.3475, Train Steps/Sec: 0.84 +[2025-04-22 17:49:58] (step=0041250) Train Loss: 6.3249, Train Steps/Sec: 0.84 +[2025-04-22 17:50:28] (step=0041275) Train Loss: 6.3919, Train Steps/Sec: 0.84 +[2025-04-22 17:50:58] (step=0041300) Train Loss: 6.3520, Train Steps/Sec: 0.84 +[2025-04-22 17:51:27] (step=0041325) Train Loss: 6.3128, Train Steps/Sec: 0.84 +[2025-04-22 17:51:57] (step=0041350) Train Loss: 6.3927, Train Steps/Sec: 0.84 +[2025-04-22 17:52:27] (step=0041375) Train Loss: 6.3694, Train Steps/Sec: 0.84 +[2025-04-22 17:52:57] (step=0041400) Train Loss: 6.3853, Train Steps/Sec: 0.84 +[2025-04-22 17:53:26] (step=0041425) Train Loss: 6.3679, Train Steps/Sec: 0.84 +[2025-04-22 17:53:56] (step=0041450) Train Loss: 6.3664, Train Steps/Sec: 0.85 +[2025-04-22 17:54:25] (step=0041475) Train Loss: 6.3803, Train Steps/Sec: 0.84 +[2025-04-22 17:54:55] (step=0041500) Train Loss: 6.3494, Train Steps/Sec: 0.84 +[2025-04-22 17:55:25] (step=0041525) Train Loss: 6.3470, Train Steps/Sec: 0.85 +[2025-04-22 17:55:54] (step=0041550) Train Loss: 6.3975, Train Steps/Sec: 0.84 +[2025-04-22 17:56:24] (step=0041575) Train Loss: 6.3639, Train Steps/Sec: 0.85 +[2025-04-22 17:56:54] (step=0041600) Train Loss: 6.3478, Train Steps/Sec: 0.84 +[2025-04-22 17:57:23] (step=0041625) Train Loss: 6.3309, Train Steps/Sec: 0.85 +[2025-04-22 17:57:53] (step=0041650) Train Loss: 6.3345, Train Steps/Sec: 0.85 +[2025-04-22 17:58:22] (step=0041675) Train Loss: 6.3486, Train Steps/Sec: 0.85 +[2025-04-22 17:58:52] (step=0041700) Train Loss: 6.3645, Train Steps/Sec: 0.84 +[2025-04-22 17:59:22] (step=0041725) Train Loss: 6.3475, Train Steps/Sec: 0.84 +[2025-04-22 17:59:51] (step=0041750) Train Loss: 6.3181, Train Steps/Sec: 0.84 +[2025-04-22 18:00:21] (step=0041775) Train Loss: 6.3406, Train Steps/Sec: 0.85 +[2025-04-22 18:00:50] (step=0041800) Train Loss: 6.3266, Train Steps/Sec: 0.84 +[2025-04-22 18:01:20] (step=0041825) Train Loss: 6.3481, Train Steps/Sec: 0.85 +[2025-04-22 18:01:50] (step=0041850) Train Loss: 6.3935, Train Steps/Sec: 0.83 +[2025-04-22 18:02:20] (step=0041875) Train Loss: 6.3809, Train Steps/Sec: 0.84 +[2025-04-22 18:02:59] (step=0041900) Train Loss: 6.3549, Train Steps/Sec: 0.64 +[2025-04-22 18:03:28] (step=0041925) Train Loss: 6.3549, Train Steps/Sec: 0.85 +[2025-04-22 18:03:58] (step=0041950) Train Loss: 6.3666, Train Steps/Sec: 0.84 +[2025-04-22 18:04:28] (step=0041975) Train Loss: 6.3433, Train Steps/Sec: 0.84 +[2025-04-22 18:04:58] (step=0042000) Train Loss: 6.3577, Train Steps/Sec: 0.83 +[2025-04-22 18:09:43] Finish Eval in 42000 steps... +[2025-04-22 18:10:04] Saved checkpoint to checkpoint/BeforeFix_Blip_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_objects/002-GPT-XL/checkpoints/0042000.pt +[2025-04-22 18:10:07] Removed old checkpoint: checkpoint/BeforeFix_Blip_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_objects/002-GPT-XL/checkpoints/0038000.pt +[2025-04-22 18:10:37] (step=0042025) Train Loss: 6.3218, Train Steps/Sec: 0.07 +[2025-04-22 18:11:16] (step=0042050) Train Loss: 6.3900, Train Steps/Sec: 0.64 +[2025-04-22 18:11:45] (step=0042075) Train Loss: 6.3320, Train Steps/Sec: 0.85 +[2025-04-22 18:12:15] (step=0042100) Train Loss: 6.3505, Train Steps/Sec: 0.84 +[2025-04-22 18:12:45] (step=0042125) Train Loss: 6.3899, Train Steps/Sec: 0.85 +[2025-04-22 18:13:14] (step=0042150) Train Loss: 6.3559, Train Steps/Sec: 0.84 +[2025-04-22 18:13:44] (step=0042175) Train Loss: 6.3898, Train Steps/Sec: 0.84 +[2025-04-22 18:14:14] (step=0042200) Train Loss: 6.3210, Train Steps/Sec: 0.84 +[2025-04-22 18:14:43] (step=0042225) Train Loss: 6.3530, Train Steps/Sec: 0.85 +[2025-04-22 18:15:13] (step=0042250) Train Loss: 6.3239, Train Steps/Sec: 0.84 +[2025-04-22 18:15:42] (step=0042275) Train Loss: 6.3371, Train Steps/Sec: 0.85 +[2025-04-22 18:16:12] (step=0042300) Train Loss: 6.3724, Train Steps/Sec: 0.84 +[2025-04-22 18:16:41] (step=0042325) Train Loss: 6.3414, Train Steps/Sec: 0.85 +[2025-04-22 18:17:11] (step=0042350) Train Loss: 6.3119, Train Steps/Sec: 0.84 +[2025-04-22 18:17:41] (step=0042375) Train Loss: 6.3188, Train Steps/Sec: 0.84 +[2025-04-22 18:18:11] (step=0042400) Train Loss: 6.3432, Train Steps/Sec: 0.84 +[2025-04-22 18:18:40] (step=0042425) Train Loss: 6.3846, Train Steps/Sec: 0.84 +[2025-04-22 18:19:10] (step=0042450) Train Loss: 6.3741, Train Steps/Sec: 0.85 +[2025-04-22 18:19:39] (step=0042475) Train Loss: 6.3802, Train Steps/Sec: 0.84 +[2025-04-22 18:20:09] (step=0042500) Train Loss: 6.3461, Train Steps/Sec: 0.84 +[2025-04-22 18:20:39] (step=0042525) Train Loss: 6.3232, Train Steps/Sec: 0.83 +[2025-04-22 18:21:09] (step=0042550) Train Loss: 6.3675, Train Steps/Sec: 0.84 +[2025-04-22 18:21:39] (step=0042575) Train Loss: 6.3698, Train Steps/Sec: 0.84 +[2025-04-22 18:22:09] (step=0042600) Train Loss: 6.3545, Train Steps/Sec: 0.83 +[2025-04-22 18:22:38] (step=0042625) Train Loss: 6.3332, Train Steps/Sec: 0.84 +[2025-04-22 18:23:08] (step=0042650) Train Loss: 6.3095, Train Steps/Sec: 0.85 +[2025-04-22 18:23:37] (step=0042675) Train Loss: 6.3860, Train Steps/Sec: 0.84 +[2025-04-22 18:24:07] (step=0042700) Train Loss: 6.3606, Train Steps/Sec: 0.84 +[2025-04-22 18:24:37] (step=0042725) Train Loss: 6.3430, Train Steps/Sec: 0.84 +[2025-04-22 18:25:06] (step=0042750) Train Loss: 6.3738, Train Steps/Sec: 0.85 +[2025-04-22 18:25:36] (step=0042775) Train Loss: 6.3368, Train Steps/Sec: 0.85 +[2025-04-22 18:26:06] (step=0042800) Train Loss: 6.3132, Train Steps/Sec: 0.84 +[2025-04-22 18:26:35] (step=0042825) Train Loss: 6.3487, Train Steps/Sec: 0.84 +[2025-04-22 18:27:05] (step=0042850) Train Loss: 6.3209, Train Steps/Sec: 0.84 +[2025-04-22 18:27:34] (step=0042875) Train Loss: 6.3489, Train Steps/Sec: 0.85 +[2025-04-22 18:28:04] (step=0042900) Train Loss: 6.3756, Train Steps/Sec: 0.84 +[2025-04-22 18:28:42] (step=0042925) Train Loss: 6.3487, Train Steps/Sec: 0.66 +[2025-04-22 18:29:12] (step=0042950) Train Loss: 6.3300, Train Steps/Sec: 0.85 +[2025-04-22 18:29:41] (step=0042975) Train Loss: 6.3248, Train Steps/Sec: 0.85 +[2025-04-22 18:30:11] (step=0043000) Train Loss: 6.3420, Train Steps/Sec: 0.85 +[2025-04-22 18:30:40] (step=0043025) Train Loss: 6.3190, Train Steps/Sec: 0.84 +[2025-04-22 18:31:10] (step=0043050) Train Loss: 6.3629, Train Steps/Sec: 0.84 +[2025-04-22 18:31:40] (step=0043075) Train Loss: 6.3728, Train Steps/Sec: 0.84 +[2025-04-22 18:32:10] (step=0043100) Train Loss: 6.3548, Train Steps/Sec: 0.84 +[2025-04-22 18:32:39] (step=0043125) Train Loss: 6.3330, Train Steps/Sec: 0.84 +[2025-04-22 18:33:09] (step=0043150) Train Loss: 6.3627, Train Steps/Sec: 0.84 +[2025-04-22 18:33:39] (step=0043175) Train Loss: 6.3392, Train Steps/Sec: 0.84 +[2025-04-22 18:34:08] (step=0043200) Train Loss: 6.3610, Train Steps/Sec: 0.84 +[2025-04-22 18:34:38] (step=0043225) Train Loss: 6.3559, Train Steps/Sec: 0.84 +[2025-04-22 18:35:08] (step=0043250) Train Loss: 6.3404, Train Steps/Sec: 0.85 +[2025-04-22 18:35:37] (step=0043275) Train Loss: 6.3359, Train Steps/Sec: 0.85 +[2025-04-22 18:36:07] (step=0043300) Train Loss: 6.3615, Train Steps/Sec: 0.84 +[2025-04-22 18:36:36] (step=0043325) Train Loss: 6.3419, Train Steps/Sec: 0.85 +[2025-04-22 18:37:06] (step=0043350) Train Loss: 6.3686, Train Steps/Sec: 0.84 +[2025-04-22 18:37:36] (step=0043375) Train Loss: 6.3784, Train Steps/Sec: 0.85 +[2025-04-22 18:38:05] (step=0043400) Train Loss: 6.3240, Train Steps/Sec: 0.84 +[2025-04-22 18:38:35] (step=0043425) Train Loss: 6.3404, Train Steps/Sec: 0.84 +[2025-04-22 18:39:05] (step=0043450) Train Loss: 6.3422, Train Steps/Sec: 0.85 +[2025-04-22 18:39:34] (step=0043475) Train Loss: 6.3744, Train Steps/Sec: 0.84 +[2025-04-22 18:40:04] (step=0043500) Train Loss: 6.3525, Train Steps/Sec: 0.84 +[2025-04-22 18:40:34] (step=0043525) Train Loss: 6.3850, Train Steps/Sec: 0.84 +[2025-04-22 18:41:03] (step=0043550) Train Loss: 6.3318, Train Steps/Sec: 0.85 +[2025-04-22 18:41:33] (step=0043575) Train Loss: 6.3523, Train Steps/Sec: 0.85 +[2025-04-22 18:42:03] (step=0043600) Train Loss: 6.4006, Train Steps/Sec: 0.84 +[2025-04-22 18:42:32] (step=0043625) Train Loss: 6.3559, Train Steps/Sec: 0.85 +[2025-04-22 18:43:02] (step=0043650) Train Loss: 6.3132, Train Steps/Sec: 0.85 +[2025-04-22 18:43:31] (step=0043675) Train Loss: 6.3564, Train Steps/Sec: 0.85 +[2025-04-22 18:44:01] (step=0043700) Train Loss: 6.3709, Train Steps/Sec: 0.84 +[2025-04-22 18:44:31] (step=0043725) Train Loss: 6.3397, Train Steps/Sec: 0.85 +[2025-04-22 18:45:00] (step=0043750) Train Loss: 6.3607, Train Steps/Sec: 0.84 +[2025-04-22 18:45:30] (step=0043775) Train Loss: 6.3373, Train Steps/Sec: 0.85 +[2025-04-22 18:45:59] (step=0043800) Train Loss: 6.3215, Train Steps/Sec: 0.84 +[2025-04-22 18:46:29] (step=0043825) Train Loss: 6.3239, Train Steps/Sec: 0.84 +[2025-04-22 18:46:59] (step=0043850) Train Loss: 6.3552, Train Steps/Sec: 0.85 +[2025-04-22 18:47:28] (step=0043875) Train Loss: 6.3714, Train Steps/Sec: 0.84 +[2025-04-22 18:47:58] (step=0043900) Train Loss: 6.3881, Train Steps/Sec: 0.84 +[2025-04-22 18:48:28] (step=0043925) Train Loss: 6.3555, Train Steps/Sec: 0.84 +[2025-04-22 18:48:57] (step=0043950) Train Loss: 6.3570, Train Steps/Sec: 0.84 +[2025-04-22 18:49:27] (step=0043975) Train Loss: 6.3241, Train Steps/Sec: 0.85 +[2025-04-22 18:49:57] (step=0044000) Train Loss: 6.3785, Train Steps/Sec: 0.85 +[2025-04-22 18:54:40] Finish Eval in 44000 steps... +[2025-04-22 18:55:05] Saved checkpoint to checkpoint/BeforeFix_Blip_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_objects/002-GPT-XL/checkpoints/0044000.pt +[2025-04-22 18:55:07] Removed old checkpoint: checkpoint/BeforeFix_Blip_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_objects/002-GPT-XL/checkpoints/0040000.pt +[2025-04-22 18:55:37] (step=0044025) Train Loss: 6.3719, Train Steps/Sec: 0.07 +[2025-04-22 18:56:06] (step=0044050) Train Loss: 6.3331, Train Steps/Sec: 0.84 +[2025-04-22 18:56:36] (step=0044075) Train Loss: 6.3558, Train Steps/Sec: 0.84 +[2025-04-22 18:57:06] (step=0044100) Train Loss: 6.3622, Train Steps/Sec: 0.84 +[2025-04-22 18:57:35] (step=0044125) Train Loss: 6.3373, Train Steps/Sec: 0.85 +[2025-04-22 18:58:05] (step=0044150) Train Loss: 6.3595, Train Steps/Sec: 0.85 +[2025-04-22 18:58:34] (step=0044175) Train Loss: 6.3452, Train Steps/Sec: 0.85 +[2025-04-22 18:59:04] (step=0044200) Train Loss: 6.3636, Train Steps/Sec: 0.84 +[2025-04-22 18:59:34] (step=0044225) Train Loss: 6.3166, Train Steps/Sec: 0.85 +[2025-04-22 19:00:03] (step=0044250) Train Loss: 6.3685, Train Steps/Sec: 0.84 +[2025-04-22 19:00:33] (step=0044275) Train Loss: 6.3529, Train Steps/Sec: 0.84 +[2025-04-22 19:01:03] (step=0044300) Train Loss: 6.3422, Train Steps/Sec: 0.84 +[2025-04-22 19:01:32] (step=0044325) Train Loss: 6.3758, Train Steps/Sec: 0.85 +[2025-04-22 19:02:02] (step=0044350) Train Loss: 6.3364, Train Steps/Sec: 0.85 +[2025-04-22 19:02:32] (step=0044375) Train Loss: 6.3635, Train Steps/Sec: 0.85 +[2025-04-22 19:03:02] (step=0044400) Train Loss: 6.3804, Train Steps/Sec: 0.84 +[2025-04-22 19:03:31] (step=0044425) Train Loss: 6.3543, Train Steps/Sec: 0.84 +[2025-04-22 19:04:01] (step=0044450) Train Loss: 6.3399, Train Steps/Sec: 0.85 +[2025-04-22 19:04:30] (step=0044475) Train Loss: 6.3622, Train Steps/Sec: 0.85 +[2025-04-22 19:05:09] (step=0044500) Train Loss: 6.3949, Train Steps/Sec: 0.65 +[2025-04-22 19:05:38] (step=0044525) Train Loss: 6.3622, Train Steps/Sec: 0.85 +[2025-04-22 19:06:08] (step=0044550) Train Loss: 6.3979, Train Steps/Sec: 0.85 +[2025-04-22 19:06:48] (step=0044575) Train Loss: 6.3725, Train Steps/Sec: 0.62 +[2025-04-22 19:07:17] (step=0044600) Train Loss: 6.3788, Train Steps/Sec: 0.84 +[2025-04-22 19:07:47] (step=0044625) Train Loss: 6.3605, Train Steps/Sec: 0.85 +[2025-04-22 19:08:17] (step=0044650) Train Loss: 6.3315, Train Steps/Sec: 0.85 +[2025-04-22 19:08:46] (step=0044675) Train Loss: 6.3693, Train Steps/Sec: 0.84 +[2025-04-22 19:09:16] (step=0044700) Train Loss: 6.3458, Train Steps/Sec: 0.84 +[2025-04-22 19:09:45] (step=0044725) Train Loss: 6.3447, Train Steps/Sec: 0.85 +[2025-04-22 19:10:15] (step=0044750) Train Loss: 6.3592, Train Steps/Sec: 0.84 +[2025-04-22 19:10:45] (step=0044775) Train Loss: 6.3514, Train Steps/Sec: 0.84 +[2025-04-22 19:11:15] (step=0044800) Train Loss: 6.3412, Train Steps/Sec: 0.84 +[2025-04-22 19:11:44] (step=0044825) Train Loss: 6.3542, Train Steps/Sec: 0.85 +[2025-04-22 19:12:14] (step=0044850) Train Loss: 6.3429, Train Steps/Sec: 0.85 +[2025-04-22 19:12:43] (step=0044875) Train Loss: 6.3160, Train Steps/Sec: 0.84 +[2025-04-22 19:13:13] (step=0044900) Train Loss: 6.3419, Train Steps/Sec: 0.84 +[2025-04-22 19:13:42] (step=0044925) Train Loss: 6.3403, Train Steps/Sec: 0.85 +[2025-04-22 19:14:12] (step=0044950) Train Loss: 6.3624, Train Steps/Sec: 0.84 +[2025-04-22 19:14:42] (step=0044975) Train Loss: 6.3636, Train Steps/Sec: 0.85 +[2025-04-22 19:15:12] (step=0045000) Train Loss: 6.3875, Train Steps/Sec: 0.84 +[2025-04-22 19:15:41] (step=0045025) Train Loss: 6.4013, Train Steps/Sec: 0.84 +[2025-04-22 19:16:11] (step=0045050) Train Loss: 6.3277, Train Steps/Sec: 0.85 +[2025-04-22 19:16:41] (step=0045075) Train Loss: 6.3806, Train Steps/Sec: 0.84 +[2025-04-22 19:17:17] (step=0045100) Train Loss: 6.3445, Train Steps/Sec: 0.68 +[2025-04-22 19:17:47] (step=0045125) Train Loss: 6.3604, Train Steps/Sec: 0.85 +[2025-04-22 19:18:25] (step=0045150) Train Loss: 6.3661, Train Steps/Sec: 0.66 +[2025-04-22 19:18:55] (step=0045175) Train Loss: 6.3557, Train Steps/Sec: 0.85 +[2025-04-22 19:19:24] (step=0045200) Train Loss: 6.3451, Train Steps/Sec: 0.84 +[2025-04-22 19:19:54] (step=0045225) Train Loss: 6.3767, Train Steps/Sec: 0.85 +[2025-04-22 19:20:24] (step=0045250) Train Loss: 6.3571, Train Steps/Sec: 0.84 +[2025-04-22 19:20:53] (step=0045275) Train Loss: 6.3243, Train Steps/Sec: 0.84 +[2025-04-22 19:21:23] (step=0045300) Train Loss: 6.3537, Train Steps/Sec: 0.84 +[2025-04-22 19:21:53] (step=0045325) Train Loss: 6.3403, Train Steps/Sec: 0.84 +[2025-04-22 19:22:22] (step=0045350) Train Loss: 6.3813, Train Steps/Sec: 0.84 +[2025-04-22 19:22:52] (step=0045375) Train Loss: 6.3539, Train Steps/Sec: 0.85 +[2025-04-22 19:23:21] (step=0045400) Train Loss: 6.3089, Train Steps/Sec: 0.85 +[2025-04-22 19:23:51] (step=0045425) Train Loss: 6.3437, Train Steps/Sec: 0.85 +[2025-04-22 19:24:20] (step=0045450) Train Loss: 6.3604, Train Steps/Sec: 0.84 +[2025-04-22 19:24:50] (step=0045475) Train Loss: 6.3767, Train Steps/Sec: 0.84 +[2025-04-22 19:25:20] (step=0045500) Train Loss: 6.3741, Train Steps/Sec: 0.84 +[2025-04-22 19:25:49] (step=0045525) Train Loss: 6.3454, Train Steps/Sec: 0.85 +[2025-04-22 19:26:19] (step=0045550) Train Loss: 6.3384, Train Steps/Sec: 0.84 +[2025-04-22 19:26:49] (step=0045575) Train Loss: 6.3654, Train Steps/Sec: 0.84 +[2025-04-22 19:27:18] (step=0045600) Train Loss: 6.3539, Train Steps/Sec: 0.84 +[2025-04-22 19:27:48] (step=0045625) Train Loss: 6.3372, Train Steps/Sec: 0.85 +[2025-04-22 19:28:18] (step=0045650) Train Loss: 6.3221, Train Steps/Sec: 0.84 +[2025-04-22 19:28:47] (step=0045675) Train Loss: 6.3890, Train Steps/Sec: 0.84 +[2025-04-22 19:29:17] (step=0045700) Train Loss: 6.3835, Train Steps/Sec: 0.84 +[2025-04-22 19:29:46] (step=0045725) Train Loss: 6.3830, Train Steps/Sec: 0.85 +[2025-04-22 19:30:16] (step=0045750) Train Loss: 6.3830, Train Steps/Sec: 0.85 +[2025-04-22 19:30:45] (step=0045775) Train Loss: 6.3653, Train Steps/Sec: 0.84 +[2025-04-22 19:31:15] (step=0045800) Train Loss: 6.3748, Train Steps/Sec: 0.84 +[2025-04-22 19:31:45] (step=0045825) Train Loss: 6.3517, Train Steps/Sec: 0.84 +[2025-04-22 19:32:14] (step=0045850) Train Loss: 6.3520, Train Steps/Sec: 0.84 +[2025-04-22 19:32:44] (step=0045875) Train Loss: 6.3585, Train Steps/Sec: 0.84 +[2025-04-22 19:33:14] (step=0045900) Train Loss: 6.3187, Train Steps/Sec: 0.84 +[2025-04-22 19:33:43] (step=0045925) Train Loss: 6.3279, Train Steps/Sec: 0.85 +[2025-04-22 19:34:13] (step=0045950) Train Loss: 6.3870, Train Steps/Sec: 0.84 +[2025-04-22 19:34:43] (step=0045975) Train Loss: 6.3266, Train Steps/Sec: 0.85 +[2025-04-22 19:35:12] (step=0046000) Train Loss: 6.3457, Train Steps/Sec: 0.85 +[2025-04-22 19:39:56] Finish Eval in 46000 steps... +[2025-04-22 19:40:18] Saved checkpoint to checkpoint/BeforeFix_Blip_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_objects/002-GPT-XL/checkpoints/0046000.pt +[2025-04-22 19:40:20] Removed old checkpoint: checkpoint/BeforeFix_Blip_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_objects/002-GPT-XL/checkpoints/0042000.pt +[2025-04-22 19:40:50] (step=0046025) Train Loss: 6.3824, Train Steps/Sec: 0.07 +[2025-04-22 19:41:20] (step=0046050) Train Loss: 6.3466, Train Steps/Sec: 0.84 +[2025-04-22 19:41:49] (step=0046075) Train Loss: 6.3299, Train Steps/Sec: 0.84 +[2025-04-22 19:42:19] (step=0046100) Train Loss: 6.3974, Train Steps/Sec: 0.84 +[2025-04-22 19:42:49] (step=0046125) Train Loss: 6.3639, Train Steps/Sec: 0.84 +[2025-04-22 19:43:18] (step=0046150) Train Loss: 6.3204, Train Steps/Sec: 0.85 +[2025-04-22 19:43:48] (step=0046175) Train Loss: 6.3494, Train Steps/Sec: 0.85 +[2025-04-22 19:44:18] (step=0046200) Train Loss: 6.3853, Train Steps/Sec: 0.83 +[2025-04-22 19:44:47] (step=0046225) Train Loss: 6.2802, Train Steps/Sec: 0.85 +[2025-04-22 19:45:17] (step=0046250) Train Loss: 6.3295, Train Steps/Sec: 0.85 +[2025-04-22 19:45:47] (step=0046275) Train Loss: 6.3649, Train Steps/Sec: 0.84 +[2025-04-22 19:46:16] (step=0046300) Train Loss: 6.3629, Train Steps/Sec: 0.85 +[2025-04-22 19:46:46] (step=0046325) Train Loss: 6.3137, Train Steps/Sec: 0.85 +[2025-04-22 19:47:16] (step=0046350) Train Loss: 6.3457, Train Steps/Sec: 0.85 +[2025-04-22 19:47:45] (step=0046375) Train Loss: 6.4115, Train Steps/Sec: 0.84 +[2025-04-22 19:48:15] (step=0046400) Train Loss: 6.3853, Train Steps/Sec: 0.84 +[2025-04-22 19:48:45] (step=0046425) Train Loss: 6.3232, Train Steps/Sec: 0.84 +[2025-04-22 19:49:14] (step=0046450) Train Loss: 6.3830, Train Steps/Sec: 0.85 +[2025-04-22 19:49:44] (step=0046475) Train Loss: 6.3672, Train Steps/Sec: 0.84 +[2025-04-22 19:50:14] (step=0046500) Train Loss: 6.3896, Train Steps/Sec: 0.84 +[2025-04-22 19:50:43] (step=0046525) Train Loss: 6.3865, Train Steps/Sec: 0.84 +[2025-04-22 19:51:13] (step=0046550) Train Loss: 6.3044, Train Steps/Sec: 0.85 +[2025-04-22 19:51:43] (step=0046575) Train Loss: 6.4009, Train Steps/Sec: 0.84 +[2025-04-22 19:52:12] (step=0046600) Train Loss: 6.3826, Train Steps/Sec: 0.84 +[2025-04-22 19:52:42] (step=0046625) Train Loss: 6.3683, Train Steps/Sec: 0.84 +[2025-04-22 19:53:12] (step=0046650) Train Loss: 6.3966, Train Steps/Sec: 0.84 +[2025-04-22 19:53:41] (step=0046675) Train Loss: 6.3603, Train Steps/Sec: 0.85 +[2025-04-22 19:54:11] (step=0046700) Train Loss: 6.3428, Train Steps/Sec: 0.84 +[2025-04-22 19:54:41] (step=0046725) Train Loss: 6.3170, Train Steps/Sec: 0.85 +[2025-04-22 19:55:10] (step=0046750) Train Loss: 6.3661, Train Steps/Sec: 0.84 +[2025-04-22 19:55:40] (step=0046775) Train Loss: 6.3155, Train Steps/Sec: 0.85 +[2025-04-22 19:56:10] (step=0046800) Train Loss: 6.3424, Train Steps/Sec: 0.84 +[2025-04-22 19:56:39] (step=0046825) Train Loss: 6.3535, Train Steps/Sec: 0.85 +[2025-04-22 19:57:09] (step=0046850) Train Loss: 6.3466, Train Steps/Sec: 0.84 +[2025-04-22 19:57:38] (step=0046875) Train Loss: 6.3634, Train Steps/Sec: 0.84 +[2025-04-22 19:58:08] (step=0046900) Train Loss: 6.3500, Train Steps/Sec: 0.84 +[2025-04-22 19:58:37] (step=0046925) Train Loss: 6.3254, Train Steps/Sec: 0.85 +[2025-04-22 19:59:07] (step=0046950) Train Loss: 6.3102, Train Steps/Sec: 0.84 +[2025-04-22 19:59:37] (step=0046975) Train Loss: 6.3462, Train Steps/Sec: 0.84 +[2025-04-22 20:00:07] (step=0047000) Train Loss: 6.3647, Train Steps/Sec: 0.84 +[2025-04-22 20:00:36] (step=0047025) Train Loss: 6.3989, Train Steps/Sec: 0.84 +[2025-04-22 20:01:06] (step=0047050) Train Loss: 6.3557, Train Steps/Sec: 0.85 +[2025-04-22 20:01:35] (step=0047075) Train Loss: 6.3641, Train Steps/Sec: 0.85 +[2025-04-22 20:02:05] (step=0047100) Train Loss: 6.3248, Train Steps/Sec: 0.84 +[2025-04-22 20:02:35] (step=0047125) Train Loss: 6.3831, Train Steps/Sec: 0.84 +[2025-04-22 20:03:04] (step=0047150) Train Loss: 6.3530, Train Steps/Sec: 0.85 +[2025-04-22 20:03:34] (step=0047175) Train Loss: 6.3859, Train Steps/Sec: 0.84 +[2025-04-22 20:04:04] (step=0047200) Train Loss: 6.3375, Train Steps/Sec: 0.84 +[2025-04-22 20:04:33] (step=0047225) Train Loss: 6.3409, Train Steps/Sec: 0.85 +[2025-04-22 20:05:03] (step=0047250) Train Loss: 6.3653, Train Steps/Sec: 0.85 +[2025-04-22 20:05:32] (step=0047275) Train Loss: 6.3482, Train Steps/Sec: 0.85 +[2025-04-22 20:06:02] (step=0047300) Train Loss: 6.3749, Train Steps/Sec: 0.84 +[2025-04-22 20:06:39] (step=0047325) Train Loss: 6.3663, Train Steps/Sec: 0.68 +[2025-04-22 20:07:08] (step=0047350) Train Loss: 6.3200, Train Steps/Sec: 0.84 +[2025-04-22 20:07:38] (step=0047375) Train Loss: 6.3687, Train Steps/Sec: 0.84 +[2025-04-22 20:08:08] (step=0047400) Train Loss: 6.3628, Train Steps/Sec: 0.84 +[2025-04-22 20:08:37] (step=0047425) Train Loss: 6.3957, Train Steps/Sec: 0.84 +[2025-04-22 20:09:07] (step=0047450) Train Loss: 6.3659, Train Steps/Sec: 0.85 +[2025-04-22 20:09:36] (step=0047475) Train Loss: 6.3441, Train Steps/Sec: 0.85 +[2025-04-22 20:10:06] (step=0047500) Train Loss: 6.3852, Train Steps/Sec: 0.84 +[2025-04-22 20:10:43] (step=0047525) Train Loss: 6.3659, Train Steps/Sec: 0.67 +[2025-04-22 20:11:13] (step=0047550) Train Loss: 6.3765, Train Steps/Sec: 0.84 +[2025-04-22 20:11:42] (step=0047575) Train Loss: 6.3769, Train Steps/Sec: 0.85 +[2025-04-22 20:12:12] (step=0047600) Train Loss: 6.3858, Train Steps/Sec: 0.83 +[2025-04-22 20:12:42] (step=0047625) Train Loss: 6.3407, Train Steps/Sec: 0.85 +[2025-04-22 20:13:12] (step=0047650) Train Loss: 6.3726, Train Steps/Sec: 0.84 +[2025-04-22 20:13:41] (step=0047675) Train Loss: 6.3700, Train Steps/Sec: 0.84 +[2025-04-22 20:14:11] (step=0047700) Train Loss: 6.3594, Train Steps/Sec: 0.84 +[2025-04-22 20:14:41] (step=0047725) Train Loss: 6.3703, Train Steps/Sec: 0.84 +[2025-04-22 20:15:10] (step=0047750) Train Loss: 6.3487, Train Steps/Sec: 0.84 +[2025-04-22 20:15:40] (step=0047775) Train Loss: 6.3124, Train Steps/Sec: 0.85 +[2025-04-22 20:16:10] (step=0047800) Train Loss: 6.3493, Train Steps/Sec: 0.84 +[2025-04-22 20:16:39] (step=0047825) Train Loss: 6.3561, Train Steps/Sec: 0.85 +[2025-04-22 20:17:09] (step=0047850) Train Loss: 6.3511, Train Steps/Sec: 0.84 +[2025-04-22 20:17:38] (step=0047875) Train Loss: 6.3614, Train Steps/Sec: 0.84 +[2025-04-22 20:18:08] (step=0047900) Train Loss: 6.3660, Train Steps/Sec: 0.84 +[2025-04-22 20:18:38] (step=0047925) Train Loss: 6.3380, Train Steps/Sec: 0.85 +[2025-04-22 20:19:07] (step=0047950) Train Loss: 6.3669, Train Steps/Sec: 0.84 +[2025-04-22 20:19:37] (step=0047975) Train Loss: 6.3681, Train Steps/Sec: 0.84 +[2025-04-22 20:20:07] (step=0048000) Train Loss: 6.3427, Train Steps/Sec: 0.84 +[2025-04-22 20:24:50] Finish Eval in 48000 steps... +[2025-04-22 20:25:12] Saved checkpoint to checkpoint/BeforeFix_Blip_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_objects/002-GPT-XL/checkpoints/0048000.pt +[2025-04-22 20:25:14] Removed old checkpoint: checkpoint/BeforeFix_Blip_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_objects/002-GPT-XL/checkpoints/0044000.pt +[2025-04-22 20:25:44] (step=0048025) Train Loss: 6.3675, Train Steps/Sec: 0.07 +[2025-04-22 20:26:14] (step=0048050) Train Loss: 6.4020, Train Steps/Sec: 0.84 +[2025-04-22 20:26:43] (step=0048075) Train Loss: 6.3341, Train Steps/Sec: 0.84 +[2025-04-22 20:27:13] (step=0048100) Train Loss: 6.3328, Train Steps/Sec: 0.85 +[2025-04-22 20:27:43] (step=0048125) Train Loss: 6.3608, Train Steps/Sec: 0.85 +[2025-04-22 20:28:12] (step=0048150) Train Loss: 6.3852, Train Steps/Sec: 0.84 +[2025-04-22 20:28:42] (step=0048175) Train Loss: 6.4194, Train Steps/Sec: 0.84 +[2025-04-22 20:29:12] (step=0048200) Train Loss: 6.3544, Train Steps/Sec: 0.85 +[2025-04-22 20:29:41] (step=0048225) Train Loss: 6.3848, Train Steps/Sec: 0.85 +[2025-04-22 20:30:17] (step=0048250) Train Loss: 6.3638, Train Steps/Sec: 0.70 +[2025-04-22 20:30:47] (step=0048275) Train Loss: 6.3962, Train Steps/Sec: 0.84 +[2025-04-22 20:31:16] (step=0048300) Train Loss: 6.3620, Train Steps/Sec: 0.84 +[2025-04-22 20:31:46] (step=0048325) Train Loss: 6.3205, Train Steps/Sec: 0.84 +[2025-04-22 20:32:16] (step=0048350) Train Loss: 6.3509, Train Steps/Sec: 0.84 +[2025-04-22 20:32:45] (step=0048375) Train Loss: 6.3693, Train Steps/Sec: 0.85 +[2025-04-22 20:33:15] (step=0048400) Train Loss: 6.3715, Train Steps/Sec: 0.84 +[2025-04-22 20:33:44] (step=0048425) Train Loss: 6.3452, Train Steps/Sec: 0.85 +[2025-04-22 20:34:14] (step=0048450) Train Loss: 6.3810, Train Steps/Sec: 0.84 +[2025-04-22 20:34:44] (step=0048475) Train Loss: 6.3666, Train Steps/Sec: 0.84 +[2025-04-22 20:35:13] (step=0048500) Train Loss: 6.3337, Train Steps/Sec: 0.84 +[2025-04-22 20:35:43] (step=0048525) Train Loss: 6.3847, Train Steps/Sec: 0.85 +[2025-04-22 20:36:12] (step=0048550) Train Loss: 6.3342, Train Steps/Sec: 0.85 +[2025-04-22 20:36:42] (step=0048575) Train Loss: 6.3634, Train Steps/Sec: 0.85 +[2025-04-22 20:37:11] (step=0048600) Train Loss: 6.3193, Train Steps/Sec: 0.84 +[2025-04-22 20:37:41] (step=0048625) Train Loss: 6.3557, Train Steps/Sec: 0.84 +[2025-04-22 20:38:11] (step=0048650) Train Loss: 6.3800, Train Steps/Sec: 0.84 +[2025-04-22 20:38:40] (step=0048675) Train Loss: 6.3499, Train Steps/Sec: 0.85 +[2025-04-22 20:39:10] (step=0048700) Train Loss: 6.3624, Train Steps/Sec: 0.84 +[2025-04-22 20:39:40] (step=0048725) Train Loss: 6.3269, Train Steps/Sec: 0.84 +[2025-04-22 20:40:09] (step=0048750) Train Loss: 6.3889, Train Steps/Sec: 0.85 +[2025-04-22 20:40:39] (step=0048775) Train Loss: 6.3376, Train Steps/Sec: 0.84 +[2025-04-22 20:41:09] (step=0048800) Train Loss: 6.3652, Train Steps/Sec: 0.84 +[2025-04-22 20:41:38] (step=0048825) Train Loss: 6.3058, Train Steps/Sec: 0.85 +[2025-04-22 20:42:08] (step=0048850) Train Loss: 6.3460, Train Steps/Sec: 0.85 +[2025-04-22 20:42:37] (step=0048875) Train Loss: 6.3809, Train Steps/Sec: 0.84 +[2025-04-22 20:43:07] (step=0048900) Train Loss: 6.3554, Train Steps/Sec: 0.84 +[2025-04-22 20:43:36] (step=0048925) Train Loss: 6.3605, Train Steps/Sec: 0.85 +[2025-04-22 20:44:06] (step=0048950) Train Loss: 6.3683, Train Steps/Sec: 0.84 +[2025-04-22 20:44:36] (step=0048975) Train Loss: 6.3110, Train Steps/Sec: 0.84 +[2025-04-22 20:45:05] (step=0049000) Train Loss: 6.3722, Train Steps/Sec: 0.84 +[2025-04-22 20:45:35] (step=0049025) Train Loss: 6.3578, Train Steps/Sec: 0.85 +[2025-04-22 20:46:05] (step=0049050) Train Loss: 6.3975, Train Steps/Sec: 0.84 +[2025-04-22 20:46:34] (step=0049075) Train Loss: 6.3417, Train Steps/Sec: 0.85 +[2025-04-22 20:47:04] (step=0049100) Train Loss: 6.3858, Train Steps/Sec: 0.84 +[2025-04-22 20:47:33] (step=0049125) Train Loss: 6.3428, Train Steps/Sec: 0.85 +[2025-04-22 20:48:03] (step=0049150) Train Loss: 6.3872, Train Steps/Sec: 0.85 +[2025-04-22 20:48:33] (step=0049175) Train Loss: 6.3822, Train Steps/Sec: 0.84 +[2025-04-22 20:49:02] (step=0049200) Train Loss: 6.3595, Train Steps/Sec: 0.84 +[2025-04-22 20:49:32] (step=0049225) Train Loss: 6.3950, Train Steps/Sec: 0.84 +[2025-04-22 20:50:02] (step=0049250) Train Loss: 6.3908, Train Steps/Sec: 0.85 +[2025-04-22 20:50:31] (step=0049275) Train Loss: 6.3626, Train Steps/Sec: 0.84 +[2025-04-22 20:51:01] (step=0049300) Train Loss: 6.3815, Train Steps/Sec: 0.84 +[2025-04-22 20:51:30] (step=0049325) Train Loss: 6.3680, Train Steps/Sec: 0.85 +[2025-04-22 20:52:00] (step=0049350) Train Loss: 6.3428, Train Steps/Sec: 0.85 +[2025-04-22 20:52:30] (step=0049375) Train Loss: 6.3665, Train Steps/Sec: 0.84 +[2025-04-22 20:52:59] (step=0049400) Train Loss: 6.3675, Train Steps/Sec: 0.84 +[2025-04-22 20:53:29] (step=0049425) Train Loss: 6.3356, Train Steps/Sec: 0.84 +[2025-04-22 20:53:59] (step=0049450) Train Loss: 6.3215, Train Steps/Sec: 0.85 +[2025-04-22 20:54:28] (step=0049475) Train Loss: 6.3407, Train Steps/Sec: 0.85 +[2025-04-22 20:54:58] (step=0049500) Train Loss: 6.3615, Train Steps/Sec: 0.84 +[2025-04-22 20:55:34] (step=0049525) Train Loss: 6.3650, Train Steps/Sec: 0.69 +[2025-04-22 20:56:04] (step=0049550) Train Loss: 6.3607, Train Steps/Sec: 0.84 +[2025-04-22 20:56:33] (step=0049575) Train Loss: 6.3545, Train Steps/Sec: 0.84 +[2025-04-22 20:57:03] (step=0049600) Train Loss: 6.3357, Train Steps/Sec: 0.84 +[2025-04-22 20:57:33] (step=0049625) Train Loss: 6.3352, Train Steps/Sec: 0.85 +[2025-04-22 20:58:02] (step=0049650) Train Loss: 6.3838, Train Steps/Sec: 0.84 +[2025-04-22 20:58:39] (step=0049675) Train Loss: 6.3212, Train Steps/Sec: 0.69 +[2025-04-22 20:59:09] (step=0049700) Train Loss: 6.3437, Train Steps/Sec: 0.84 +[2025-04-22 20:59:38] (step=0049725) Train Loss: 6.4050, Train Steps/Sec: 0.84 +[2025-04-22 21:00:08] (step=0049750) Train Loss: 6.4146, Train Steps/Sec: 0.84 +[2025-04-22 21:00:37] (step=0049775) Train Loss: 6.3308, Train Steps/Sec: 0.85 +[2025-04-22 21:01:07] (step=0049800) Train Loss: 6.3435, Train Steps/Sec: 0.84 +[2025-04-22 21:01:37] (step=0049825) Train Loss: 6.3690, Train Steps/Sec: 0.84 +[2025-04-22 21:02:07] (step=0049850) Train Loss: 6.3600, Train Steps/Sec: 0.84 +[2025-04-22 21:02:36] (step=0049875) Train Loss: 6.3841, Train Steps/Sec: 0.85 +[2025-04-22 21:03:06] (step=0049900) Train Loss: 6.3517, Train Steps/Sec: 0.84 +[2025-04-22 21:03:35] (step=0049925) Train Loss: 6.3643, Train Steps/Sec: 0.84 +[2025-04-22 21:04:05] (step=0049950) Train Loss: 6.3613, Train Steps/Sec: 0.84 +[2025-04-22 21:04:35] (step=0049975) Train Loss: 6.3952, Train Steps/Sec: 0.85 +[2025-04-22 21:05:04] (step=0050000) Train Loss: 6.3936, Train Steps/Sec: 0.84 +[2025-04-22 21:09:48] Finish Eval in 50000 steps... +[2025-04-22 21:10:09] Saved checkpoint to checkpoint/BeforeFix_Blip_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_objects/002-GPT-XL/checkpoints/0050000.pt +[2025-04-22 21:10:11] Removed old checkpoint: checkpoint/BeforeFix_Blip_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_objects/002-GPT-XL/checkpoints/0046000.pt +[2025-04-22 21:10:41] (step=0050025) Train Loss: 6.3514, Train Steps/Sec: 0.07 +[2025-04-22 21:11:10] (step=0050050) Train Loss: 6.3385, Train Steps/Sec: 0.85 +[2025-04-22 21:11:40] (step=0050075) Train Loss: 6.3067, Train Steps/Sec: 0.85 +[2025-04-22 21:12:09] (step=0050100) Train Loss: 6.3481, Train Steps/Sec: 0.84 +[2025-04-22 21:12:39] (step=0050125) Train Loss: 6.4039, Train Steps/Sec: 0.84 +[2025-04-22 21:13:08] (step=0050150) Train Loss: 6.3658, Train Steps/Sec: 0.85 +[2025-04-22 21:13:38] (step=0050175) Train Loss: 6.3445, Train Steps/Sec: 0.85 +[2025-04-22 21:14:14] (step=0050200) Train Loss: 6.3639, Train Steps/Sec: 0.70 +[2025-04-22 21:14:43] (step=0050225) Train Loss: 6.3848, Train Steps/Sec: 0.85 +[2025-04-22 21:15:13] (step=0050250) Train Loss: 6.3718, Train Steps/Sec: 0.85 +[2025-04-22 21:15:49] (step=0050275) Train Loss: 6.3758, Train Steps/Sec: 0.69 +[2025-04-22 21:16:19] (step=0050300) Train Loss: 6.3821, Train Steps/Sec: 0.84 +[2025-04-22 21:16:48] (step=0050325) Train Loss: 6.3327, Train Steps/Sec: 0.84 +[2025-04-22 21:17:18] (step=0050350) Train Loss: 6.3675, Train Steps/Sec: 0.85 +[2025-04-22 21:17:48] (step=0050375) Train Loss: 6.3805, Train Steps/Sec: 0.84 +[2025-04-22 21:18:17] (step=0050400) Train Loss: 6.3980, Train Steps/Sec: 0.84 +[2025-04-22 21:18:47] (step=0050425) Train Loss: 6.3152, Train Steps/Sec: 0.85 +[2025-04-22 21:19:16] (step=0050450) Train Loss: 6.3156, Train Steps/Sec: 0.84 +[2025-04-22 21:19:46] (step=0050475) Train Loss: 6.3838, Train Steps/Sec: 0.85 +[2025-04-22 21:20:16] (step=0050500) Train Loss: 6.4255, Train Steps/Sec: 0.84 +[2025-04-22 21:20:46] (step=0050525) Train Loss: 6.3317, Train Steps/Sec: 0.83 +[2025-04-22 21:21:15] (step=0050550) Train Loss: 6.3871, Train Steps/Sec: 0.84 +[2025-04-22 21:21:45] (step=0050575) Train Loss: 6.3330, Train Steps/Sec: 0.85 +[2025-04-22 21:22:15] (step=0050600) Train Loss: 6.3627, Train Steps/Sec: 0.84 +[2025-04-22 21:22:44] (step=0050625) Train Loss: 6.3439, Train Steps/Sec: 0.85 +[2025-04-22 21:23:14] (step=0050650) Train Loss: 6.3643, Train Steps/Sec: 0.84 +[2025-04-22 21:23:43] (step=0050675) Train Loss: 6.3381, Train Steps/Sec: 0.85 +[2025-04-22 21:24:13] (step=0050700) Train Loss: 6.3712, Train Steps/Sec: 0.84 +[2025-04-22 21:24:42] (step=0050725) Train Loss: 6.3539, Train Steps/Sec: 0.85 +[2025-04-22 21:25:12] (step=0050750) Train Loss: 6.3167, Train Steps/Sec: 0.85 +[2025-04-22 21:25:41] (step=0050775) Train Loss: 6.3693, Train Steps/Sec: 0.85 +[2025-04-22 21:26:11] (step=0050800) Train Loss: 6.4028, Train Steps/Sec: 0.85 +[2025-04-22 21:26:40] (step=0050825) Train Loss: 6.3905, Train Steps/Sec: 0.84 +[2025-04-22 21:27:10] (step=0050850) Train Loss: 6.3747, Train Steps/Sec: 0.84 +[2025-04-22 21:27:40] (step=0050875) Train Loss: 6.3899, Train Steps/Sec: 0.85 +[2025-04-22 21:28:10] (step=0050900) Train Loss: 6.3754, Train Steps/Sec: 0.84 +[2025-04-22 21:28:39] (step=0050925) Train Loss: 6.3260, Train Steps/Sec: 0.85 +[2025-04-22 21:29:09] (step=0050950) Train Loss: 6.3489, Train Steps/Sec: 0.84 +[2025-04-22 21:29:38] (step=0050975) Train Loss: 6.3390, Train Steps/Sec: 0.85 +[2025-04-22 21:30:08] (step=0051000) Train Loss: 6.3635, Train Steps/Sec: 0.84 +[2025-04-22 21:30:37] (step=0051025) Train Loss: 6.3321, Train Steps/Sec: 0.85 +[2025-04-22 21:31:07] (step=0051050) Train Loss: 6.3071, Train Steps/Sec: 0.84 +[2025-04-22 21:31:37] (step=0051075) Train Loss: 6.3728, Train Steps/Sec: 0.85 +[2025-04-22 21:32:06] (step=0051100) Train Loss: 6.3545, Train Steps/Sec: 0.84 +[2025-04-22 21:32:36] (step=0051125) Train Loss: 6.3818, Train Steps/Sec: 0.84 +[2025-04-22 21:33:05] (step=0051150) Train Loss: 6.3304, Train Steps/Sec: 0.85 +[2025-04-22 21:33:35] (step=0051175) Train Loss: 6.3977, Train Steps/Sec: 0.85 +[2025-04-22 21:34:05] (step=0051200) Train Loss: 6.3587, Train Steps/Sec: 0.84 +[2025-04-22 21:34:34] (step=0051225) Train Loss: 6.3230, Train Steps/Sec: 0.85 +[2025-04-22 21:35:04] (step=0051250) Train Loss: 6.3800, Train Steps/Sec: 0.85 +[2025-04-22 21:35:34] (step=0051275) Train Loss: 6.3531, Train Steps/Sec: 0.84 +[2025-04-22 21:36:03] (step=0051300) Train Loss: 6.3749, Train Steps/Sec: 0.84 +[2025-04-22 21:36:33] (step=0051325) Train Loss: 6.3458, Train Steps/Sec: 0.85 +[2025-04-22 21:37:03] (step=0051350) Train Loss: 6.3734, Train Steps/Sec: 0.84 +[2025-04-22 21:37:32] (step=0051375) Train Loss: 6.3654, Train Steps/Sec: 0.84 +[2025-04-22 21:38:02] (step=0051400) Train Loss: 6.3498, Train Steps/Sec: 0.85 +[2025-04-22 21:38:31] (step=0051425) Train Loss: 6.3604, Train Steps/Sec: 0.85 +[2025-04-22 21:39:01] (step=0051450) Train Loss: 6.3374, Train Steps/Sec: 0.85 +[2025-04-22 21:39:30] (step=0051475) Train Loss: 6.3158, Train Steps/Sec: 0.85 +[2025-04-22 21:40:00] (step=0051500) Train Loss: 6.3431, Train Steps/Sec: 0.84 +[2025-04-22 21:40:29] (step=0051525) Train Loss: 6.3082, Train Steps/Sec: 0.84 +[2025-04-22 21:40:59] (step=0051550) Train Loss: 6.4122, Train Steps/Sec: 0.84 +[2025-04-22 21:41:29] (step=0051575) Train Loss: 6.3284, Train Steps/Sec: 0.84 +[2025-04-22 21:41:58] (step=0051600) Train Loss: 6.3458, Train Steps/Sec: 0.85 +[2025-04-22 21:42:28] (step=0051625) Train Loss: 6.3633, Train Steps/Sec: 0.85 +[2025-04-22 21:42:57] (step=0051650) Train Loss: 6.3330, Train Steps/Sec: 0.85 +[2025-04-22 21:43:27] (step=0051675) Train Loss: 6.3701, Train Steps/Sec: 0.85 +[2025-04-22 21:43:57] (step=0051700) Train Loss: 6.3758, Train Steps/Sec: 0.84 +[2025-04-22 21:44:26] (step=0051725) Train Loss: 6.3409, Train Steps/Sec: 0.85 +[2025-04-22 21:44:56] (step=0051750) Train Loss: 6.3523, Train Steps/Sec: 0.84 +[2025-04-22 21:45:25] (step=0051775) Train Loss: 6.3365, Train Steps/Sec: 0.85 +[2025-04-22 21:45:55] (step=0051800) Train Loss: 6.3439, Train Steps/Sec: 0.84 +[2025-04-22 21:46:25] (step=0051825) Train Loss: 6.3647, Train Steps/Sec: 0.85 +[2025-04-22 21:46:54] (step=0051850) Train Loss: 6.3590, Train Steps/Sec: 0.85 +[2025-04-22 21:47:24] (step=0051875) Train Loss: 6.4129, Train Steps/Sec: 0.84 +[2025-04-22 21:47:54] (step=0051900) Train Loss: 6.3705, Train Steps/Sec: 0.85 +[2025-04-22 21:48:23] (step=0051925) Train Loss: 6.3339, Train Steps/Sec: 0.84 +[2025-04-22 21:48:53] (step=0051950) Train Loss: 6.3688, Train Steps/Sec: 0.85 +[2025-04-22 21:49:22] (step=0051975) Train Loss: 6.3199, Train Steps/Sec: 0.85 +[2025-04-22 21:49:52] (step=0052000) Train Loss: 6.2975, Train Steps/Sec: 0.84 +[2025-04-22 21:54:34] Finish Eval in 52000 steps... +[2025-04-22 21:54:55] Saved checkpoint to checkpoint/BeforeFix_Blip_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_objects/002-GPT-XL/checkpoints/0052000.pt +[2025-04-22 21:54:57] Removed old checkpoint: checkpoint/BeforeFix_Blip_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_objects/002-GPT-XL/checkpoints/0048000.pt +[2025-04-22 21:55:27] (step=0052025) Train Loss: 6.3551, Train Steps/Sec: 0.07 +[2025-04-22 21:55:56] (step=0052050) Train Loss: 6.3397, Train Steps/Sec: 0.84 +[2025-04-22 21:56:26] (step=0052075) Train Loss: 6.3306, Train Steps/Sec: 0.84 +[2025-04-22 21:56:56] (step=0052100) Train Loss: 6.3617, Train Steps/Sec: 0.84 +[2025-04-22 21:57:25] (step=0052125) Train Loss: 6.3185, Train Steps/Sec: 0.85 +[2025-04-22 21:57:55] (step=0052150) Train Loss: 6.3297, Train Steps/Sec: 0.84 +[2025-04-22 21:58:24] (step=0052175) Train Loss: 6.3426, Train Steps/Sec: 0.85 +[2025-04-22 21:58:54] (step=0052200) Train Loss: 6.3452, Train Steps/Sec: 0.84 +[2025-04-22 21:59:24] (step=0052225) Train Loss: 6.3689, Train Steps/Sec: 0.85 +[2025-04-22 21:59:53] (step=0052250) Train Loss: 6.3658, Train Steps/Sec: 0.85 +[2025-04-22 22:00:23] (step=0052275) Train Loss: 6.3020, Train Steps/Sec: 0.84 +[2025-04-22 22:00:52] (step=0052300) Train Loss: 6.3408, Train Steps/Sec: 0.84 +[2025-04-22 22:01:22] (step=0052325) Train Loss: 6.3629, Train Steps/Sec: 0.84 +[2025-04-22 22:01:52] (step=0052350) Train Loss: 6.3136, Train Steps/Sec: 0.85 +[2025-04-22 22:02:22] (step=0052375) Train Loss: 6.3242, Train Steps/Sec: 0.84 +[2025-04-22 22:02:51] (step=0052400) Train Loss: 6.3592, Train Steps/Sec: 0.85 +[2025-04-22 22:03:21] (step=0052425) Train Loss: 6.3891, Train Steps/Sec: 0.84 +[2025-04-22 22:03:50] (step=0052450) Train Loss: 6.3568, Train Steps/Sec: 0.84 +[2025-04-22 22:04:20] (step=0052475) Train Loss: 6.3862, Train Steps/Sec: 0.84 +[2025-04-22 22:04:50] (step=0052500) Train Loss: 6.3509, Train Steps/Sec: 0.85 +[2025-04-22 22:05:19] (step=0052525) Train Loss: 6.3229, Train Steps/Sec: 0.85 +[2025-04-22 22:05:49] (step=0052550) Train Loss: 6.3608, Train Steps/Sec: 0.85 +[2025-04-22 22:06:18] (step=0052575) Train Loss: 6.3128, Train Steps/Sec: 0.85 +[2025-04-22 22:06:48] (step=0052600) Train Loss: 6.3342, Train Steps/Sec: 0.84 +[2025-04-22 22:07:17] (step=0052625) Train Loss: 6.2980, Train Steps/Sec: 0.85 +[2025-04-22 22:07:47] (step=0052650) Train Loss: 6.3503, Train Steps/Sec: 0.85 +[2025-04-22 22:08:17] (step=0052675) Train Loss: 6.3839, Train Steps/Sec: 0.84 +[2025-04-22 22:08:46] (step=0052700) Train Loss: 6.3336, Train Steps/Sec: 0.84 +[2025-04-22 22:09:16] (step=0052725) Train Loss: 6.3114, Train Steps/Sec: 0.85 +[2025-04-22 22:09:52] (step=0052750) Train Loss: 6.3173, Train Steps/Sec: 0.70 +[2025-04-22 22:10:21] (step=0052775) Train Loss: 6.3613, Train Steps/Sec: 0.85 +[2025-04-22 22:10:51] (step=0052800) Train Loss: 6.3439, Train Steps/Sec: 0.85 +[2025-04-22 22:11:20] (step=0052825) Train Loss: 6.3260, Train Steps/Sec: 0.85 +[2025-04-22 22:11:50] (step=0052850) Train Loss: 6.3555, Train Steps/Sec: 0.85 +[2025-04-22 22:12:19] (step=0052875) Train Loss: 6.2772, Train Steps/Sec: 0.85 +[2025-04-22 22:12:49] (step=0052900) Train Loss: 6.3157, Train Steps/Sec: 0.84 +[2025-04-22 22:13:19] (step=0052925) Train Loss: 6.3365, Train Steps/Sec: 0.85 +[2025-04-22 22:13:48] (step=0052950) Train Loss: 6.3617, Train Steps/Sec: 0.85 +[2025-04-22 22:14:18] (step=0052975) Train Loss: 6.3668, Train Steps/Sec: 0.84 +[2025-04-22 22:14:47] (step=0053000) Train Loss: 6.3447, Train Steps/Sec: 0.85 +[2025-04-22 22:15:25] (step=0053025) Train Loss: 6.3301, Train Steps/Sec: 0.67 +[2025-04-22 22:15:54] (step=0053050) Train Loss: 6.3492, Train Steps/Sec: 0.84 +[2025-04-22 22:16:24] (step=0053075) Train Loss: 6.3220, Train Steps/Sec: 0.85 +[2025-04-22 22:16:53] (step=0053100) Train Loss: 6.3168, Train Steps/Sec: 0.84 +[2025-04-22 22:17:23] (step=0053125) Train Loss: 6.3370, Train Steps/Sec: 0.85 +[2025-04-22 22:17:52] (step=0053150) Train Loss: 6.3471, Train Steps/Sec: 0.85 +[2025-04-22 22:18:22] (step=0053175) Train Loss: 6.3386, Train Steps/Sec: 0.84 +[2025-04-22 22:18:52] (step=0053200) Train Loss: 6.3188, Train Steps/Sec: 0.84 +[2025-04-22 22:19:21] (step=0053225) Train Loss: 6.3524, Train Steps/Sec: 0.85 +[2025-04-22 22:19:51] (step=0053250) Train Loss: 6.3657, Train Steps/Sec: 0.85 +[2025-04-22 22:20:20] (step=0053275) Train Loss: 6.3745, Train Steps/Sec: 0.85 +[2025-04-22 22:20:50] (step=0053300) Train Loss: 6.3740, Train Steps/Sec: 0.84 +[2025-04-22 22:21:19] (step=0053325) Train Loss: 6.3402, Train Steps/Sec: 0.85 +[2025-04-22 22:21:49] (step=0053350) Train Loss: 6.3562, Train Steps/Sec: 0.85 +[2025-04-22 22:22:18] (step=0053375) Train Loss: 6.3436, Train Steps/Sec: 0.85 +[2025-04-22 22:22:48] (step=0053400) Train Loss: 6.3244, Train Steps/Sec: 0.84 +[2025-04-22 22:23:18] (step=0053425) Train Loss: 6.3225, Train Steps/Sec: 0.85 +[2025-04-22 22:23:47] (step=0053450) Train Loss: 6.3449, Train Steps/Sec: 0.84 +[2025-04-22 22:24:17] (step=0053475) Train Loss: 6.3376, Train Steps/Sec: 0.84 +[2025-04-22 22:24:47] (step=0053500) Train Loss: 6.3672, Train Steps/Sec: 0.84 +[2025-04-22 22:25:16] (step=0053525) Train Loss: 6.3623, Train Steps/Sec: 0.85 +[2025-04-22 22:25:46] (step=0053550) Train Loss: 6.3641, Train Steps/Sec: 0.85 +[2025-04-22 22:26:15] (step=0053575) Train Loss: 6.3872, Train Steps/Sec: 0.85 +[2025-04-22 22:26:51] (step=0053600) Train Loss: 6.3453, Train Steps/Sec: 0.70 +[2025-04-22 22:27:21] (step=0053625) Train Loss: 6.3656, Train Steps/Sec: 0.85 +[2025-04-22 22:27:50] (step=0053650) Train Loss: 6.3811, Train Steps/Sec: 0.85 +[2025-04-22 22:28:20] (step=0053675) Train Loss: 6.3335, Train Steps/Sec: 0.85 +[2025-04-22 22:28:49] (step=0053700) Train Loss: 6.3338, Train Steps/Sec: 0.84 +[2025-04-22 22:29:19] (step=0053725) Train Loss: 6.3603, Train Steps/Sec: 0.85 +[2025-04-22 22:29:49] (step=0053750) Train Loss: 6.3645, Train Steps/Sec: 0.84 +[2025-04-22 22:30:18] (step=0053775) Train Loss: 6.3482, Train Steps/Sec: 0.84 +[2025-04-22 22:30:48] (step=0053800) Train Loss: 6.3494, Train Steps/Sec: 0.84 +[2025-04-22 22:31:17] (step=0053825) Train Loss: 6.3488, Train Steps/Sec: 0.85 +[2025-04-22 22:31:47] (step=0053850) Train Loss: 6.3424, Train Steps/Sec: 0.85 +[2025-04-22 22:32:16] (step=0053875) Train Loss: 6.3602, Train Steps/Sec: 0.85 +[2025-04-22 22:32:46] (step=0053900) Train Loss: 6.3368, Train Steps/Sec: 0.84 +[2025-04-22 22:33:16] (step=0053925) Train Loss: 6.3475, Train Steps/Sec: 0.85 +[2025-04-22 22:33:45] (step=0053950) Train Loss: 6.3465, Train Steps/Sec: 0.85 +[2025-04-22 22:34:15] (step=0053975) Train Loss: 6.4114, Train Steps/Sec: 0.84 +[2025-04-22 22:34:44] (step=0054000) Train Loss: 6.3212, Train Steps/Sec: 0.85 +[2025-04-22 22:39:26] Finish Eval in 54000 steps... +[2025-04-22 22:39:47] Saved checkpoint to checkpoint/BeforeFix_Blip_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_objects/002-GPT-XL/checkpoints/0054000.pt +[2025-04-22 22:39:49] Removed old checkpoint: checkpoint/BeforeFix_Blip_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_objects/002-GPT-XL/checkpoints/0050000.pt +[2025-04-22 22:40:19] (step=0054025) Train Loss: 6.3558, Train Steps/Sec: 0.07 +[2025-04-22 22:40:49] (step=0054050) Train Loss: 6.3873, Train Steps/Sec: 0.84 +[2025-04-22 22:41:18] (step=0054075) Train Loss: 6.3549, Train Steps/Sec: 0.85 +[2025-04-22 22:41:48] (step=0054100) Train Loss: 6.2926, Train Steps/Sec: 0.85 +[2025-04-22 22:42:17] (step=0054125) Train Loss: 6.3421, Train Steps/Sec: 0.84 +[2025-04-22 22:42:47] (step=0054150) Train Loss: 6.3640, Train Steps/Sec: 0.85 +[2025-04-22 22:43:16] (step=0054175) Train Loss: 6.3310, Train Steps/Sec: 0.85 +[2025-04-22 22:43:46] (step=0054200) Train Loss: 6.3536, Train Steps/Sec: 0.84 +[2025-04-22 22:44:16] (step=0054225) Train Loss: 6.3540, Train Steps/Sec: 0.83 +[2025-04-22 22:44:45] (step=0054250) Train Loss: 6.3510, Train Steps/Sec: 0.84 +[2025-04-22 22:45:15] (step=0054275) Train Loss: 6.3500, Train Steps/Sec: 0.85 +[2025-04-22 22:45:45] (step=0054300) Train Loss: 6.3317, Train Steps/Sec: 0.85 +[2025-04-22 22:46:14] (step=0054325) Train Loss: 6.4095, Train Steps/Sec: 0.85 +[2025-04-22 22:46:44] (step=0054350) Train Loss: 6.3225, Train Steps/Sec: 0.84 +[2025-04-22 22:47:13] (step=0054375) Train Loss: 6.3592, Train Steps/Sec: 0.85 +[2025-04-22 22:47:43] (step=0054400) Train Loss: 6.3660, Train Steps/Sec: 0.84 +[2025-04-22 22:48:12] (step=0054425) Train Loss: 6.3191, Train Steps/Sec: 0.85 +[2025-04-22 22:48:42] (step=0054450) Train Loss: 6.3248, Train Steps/Sec: 0.84 +[2025-04-22 22:49:11] (step=0054475) Train Loss: 6.3690, Train Steps/Sec: 0.84 +[2025-04-22 22:49:41] (step=0054500) Train Loss: 6.3310, Train Steps/Sec: 0.84 +[2025-04-22 22:50:11] (step=0054525) Train Loss: 6.2947, Train Steps/Sec: 0.84 +[2025-04-22 22:50:40] (step=0054550) Train Loss: 6.3424, Train Steps/Sec: 0.85 +[2025-04-22 22:51:17] (step=0054575) Train Loss: 6.3365, Train Steps/Sec: 0.69 +[2025-04-22 22:51:46] (step=0054600) Train Loss: 6.3928, Train Steps/Sec: 0.84 +[2025-04-22 22:52:16] (step=0054625) Train Loss: 6.3631, Train Steps/Sec: 0.85 +[2025-04-22 22:52:45] (step=0054650) Train Loss: 6.3537, Train Steps/Sec: 0.84 +[2025-04-22 22:53:15] (step=0054675) Train Loss: 6.3417, Train Steps/Sec: 0.84 +[2025-04-22 22:53:45] (step=0054700) Train Loss: 6.3901, Train Steps/Sec: 0.85 +[2025-04-22 22:54:14] (step=0054725) Train Loss: 6.3204, Train Steps/Sec: 0.85 +[2025-04-22 22:54:44] (step=0054750) Train Loss: 6.3846, Train Steps/Sec: 0.84 +[2025-04-22 22:55:20] (step=0054775) Train Loss: 6.3784, Train Steps/Sec: 0.69 +[2025-04-22 22:55:50] (step=0054800) Train Loss: 6.3303, Train Steps/Sec: 0.84 +[2025-04-22 22:56:19] (step=0054825) Train Loss: 6.3437, Train Steps/Sec: 0.84 +[2025-04-22 22:56:49] (step=0054850) Train Loss: 6.3703, Train Steps/Sec: 0.85 +[2025-04-22 22:57:19] (step=0054875) Train Loss: 6.3478, Train Steps/Sec: 0.84 +[2025-04-22 22:57:48] (step=0054900) Train Loss: 6.3611, Train Steps/Sec: 0.84 +[2025-04-22 22:58:18] (step=0054925) Train Loss: 6.3167, Train Steps/Sec: 0.85 +[2025-04-22 22:58:47] (step=0054950) Train Loss: 6.3351, Train Steps/Sec: 0.85 +[2025-04-22 22:59:17] (step=0054975) Train Loss: 6.3810, Train Steps/Sec: 0.85 +[2025-04-22 22:59:46] (step=0055000) Train Loss: 6.3776, Train Steps/Sec: 0.85 +[2025-04-22 23:00:16] (step=0055025) Train Loss: 6.3506, Train Steps/Sec: 0.85 +[2025-04-22 23:00:46] (step=0055050) Train Loss: 6.3278, Train Steps/Sec: 0.84 +[2025-04-22 23:01:15] (step=0055075) Train Loss: 6.3490, Train Steps/Sec: 0.85 +[2025-04-22 23:01:45] (step=0055100) Train Loss: 6.3454, Train Steps/Sec: 0.85 +[2025-04-22 23:02:14] (step=0055125) Train Loss: 6.3790, Train Steps/Sec: 0.84 +[2025-04-22 23:02:44] (step=0055150) Train Loss: 6.4077, Train Steps/Sec: 0.85 +[2025-04-22 23:03:14] (step=0055175) Train Loss: 6.3145, Train Steps/Sec: 0.84 +[2025-04-22 23:03:43] (step=0055200) Train Loss: 6.3769, Train Steps/Sec: 0.84 +[2025-04-22 23:04:13] (step=0055225) Train Loss: 6.3337, Train Steps/Sec: 0.85 +[2025-04-22 23:04:42] (step=0055250) Train Loss: 6.4037, Train Steps/Sec: 0.84 +[2025-04-22 23:05:18] (step=0055275) Train Loss: 6.3586, Train Steps/Sec: 0.71 +[2025-04-22 23:05:47] (step=0055300) Train Loss: 6.3134, Train Steps/Sec: 0.84 +[2025-04-22 23:06:17] (step=0055325) Train Loss: 6.3562, Train Steps/Sec: 0.85 +[2025-04-22 23:06:47] (step=0055350) Train Loss: 6.3906, Train Steps/Sec: 0.85 +[2025-04-22 23:07:23] (step=0055375) Train Loss: 6.3679, Train Steps/Sec: 0.69 +[2025-04-22 23:07:52] (step=0055400) Train Loss: 6.3516, Train Steps/Sec: 0.84 +[2025-04-22 23:08:22] (step=0055425) Train Loss: 6.3204, Train Steps/Sec: 0.85 +[2025-04-22 23:08:51] (step=0055450) Train Loss: 6.3659, Train Steps/Sec: 0.84 +[2025-04-22 23:09:21] (step=0055475) Train Loss: 6.3890, Train Steps/Sec: 0.84 +[2025-04-22 23:09:51] (step=0055500) Train Loss: 6.3438, Train Steps/Sec: 0.84 +[2025-04-22 23:10:20] (step=0055525) Train Loss: 6.3565, Train Steps/Sec: 0.85 +[2025-04-22 23:10:50] (step=0055550) Train Loss: 6.3329, Train Steps/Sec: 0.85 +[2025-04-22 23:11:19] (step=0055575) Train Loss: 6.3456, Train Steps/Sec: 0.85 +[2025-04-22 23:11:49] (step=0055600) Train Loss: 6.3777, Train Steps/Sec: 0.84 +[2025-04-22 23:12:19] (step=0055625) Train Loss: 6.3734, Train Steps/Sec: 0.85 +[2025-04-22 23:12:48] (step=0055650) Train Loss: 6.3530, Train Steps/Sec: 0.85 +[2025-04-22 23:13:18] (step=0055675) Train Loss: 6.3411, Train Steps/Sec: 0.84 +[2025-04-22 23:13:48] (step=0055700) Train Loss: 6.3365, Train Steps/Sec: 0.84 +[2025-04-22 23:14:17] (step=0055725) Train Loss: 6.3414, Train Steps/Sec: 0.85 +[2025-04-22 23:14:46] (step=0055750) Train Loss: 6.3173, Train Steps/Sec: 0.85 +[2025-04-22 23:15:16] (step=0055775) Train Loss: 6.3624, Train Steps/Sec: 0.85 +[2025-04-22 23:15:45] (step=0055800) Train Loss: 6.3532, Train Steps/Sec: 0.85 +[2025-04-22 23:16:15] (step=0055825) Train Loss: 6.3347, Train Steps/Sec: 0.85 +[2025-04-22 23:16:45] (step=0055850) Train Loss: 6.3455, Train Steps/Sec: 0.85 +[2025-04-22 23:17:14] (step=0055875) Train Loss: 6.3213, Train Steps/Sec: 0.85 +[2025-04-22 23:17:44] (step=0055900) Train Loss: 6.3595, Train Steps/Sec: 0.84 +[2025-04-22 23:18:13] (step=0055925) Train Loss: 6.4034, Train Steps/Sec: 0.84 +[2025-04-22 23:18:43] (step=0055950) Train Loss: 6.3050, Train Steps/Sec: 0.85 +[2025-04-22 23:19:12] (step=0055975) Train Loss: 6.3751, Train Steps/Sec: 0.84 +[2025-04-22 23:19:42] (step=0056000) Train Loss: 6.4245, Train Steps/Sec: 0.84 +[2025-04-22 23:24:26] Finish Eval in 56000 steps... +[2025-04-22 23:24:47] Saved checkpoint to checkpoint/BeforeFix_Blip_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_objects/002-GPT-XL/checkpoints/0056000.pt +[2025-04-22 23:24:49] Removed old checkpoint: checkpoint/BeforeFix_Blip_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_objects/002-GPT-XL/checkpoints/0052000.pt +[2025-04-22 23:25:19] (step=0056025) Train Loss: 6.3377, Train Steps/Sec: 0.07 +[2025-04-22 23:25:48] (step=0056050) Train Loss: 6.3397, Train Steps/Sec: 0.85 +[2025-04-22 23:26:18] (step=0056075) Train Loss: 6.3903, Train Steps/Sec: 0.84 +[2025-04-22 23:26:47] (step=0056100) Train Loss: 6.3491, Train Steps/Sec: 0.84 +[2025-04-22 23:27:17] (step=0056125) Train Loss: 6.3250, Train Steps/Sec: 0.85 +[2025-04-22 23:27:46] (step=0056150) Train Loss: 6.3073, Train Steps/Sec: 0.85 +[2025-04-22 23:28:16] (step=0056175) Train Loss: 6.3217, Train Steps/Sec: 0.85 +[2025-04-22 23:28:45] (step=0056200) Train Loss: 6.3457, Train Steps/Sec: 0.85 +[2025-04-22 23:29:15] (step=0056225) Train Loss: 6.3093, Train Steps/Sec: 0.85 +[2025-04-22 23:29:44] (step=0056250) Train Loss: 6.3495, Train Steps/Sec: 0.85 +[2025-04-22 23:30:14] (step=0056275) Train Loss: 6.3593, Train Steps/Sec: 0.84 +[2025-04-22 23:30:44] (step=0056300) Train Loss: 6.3292, Train Steps/Sec: 0.84 +[2025-04-22 23:31:13] (step=0056325) Train Loss: 6.3496, Train Steps/Sec: 0.85 +[2025-04-22 23:31:43] (step=0056350) Train Loss: 6.3584, Train Steps/Sec: 0.85 +[2025-04-22 23:32:12] (step=0056375) Train Loss: 6.3291, Train Steps/Sec: 0.84 +[2025-04-22 23:32:42] (step=0056400) Train Loss: 6.3568, Train Steps/Sec: 0.84 +[2025-04-22 23:33:12] (step=0056425) Train Loss: 6.3449, Train Steps/Sec: 0.84 +[2025-04-22 23:33:41] (step=0056450) Train Loss: 6.2899, Train Steps/Sec: 0.84 +[2025-04-22 23:34:11] (step=0056475) Train Loss: 6.3712, Train Steps/Sec: 0.85 +[2025-04-22 23:34:41] (step=0056500) Train Loss: 6.3364, Train Steps/Sec: 0.85 +[2025-04-22 23:35:10] (step=0056525) Train Loss: 6.3889, Train Steps/Sec: 0.84 +[2025-04-22 23:35:40] (step=0056550) Train Loss: 6.3981, Train Steps/Sec: 0.85 +[2025-04-22 23:36:09] (step=0056575) Train Loss: 6.3067, Train Steps/Sec: 0.84 +[2025-04-22 23:36:39] (step=0056600) Train Loss: 6.3754, Train Steps/Sec: 0.85 +[2025-04-22 23:37:08] (step=0056625) Train Loss: 6.3566, Train Steps/Sec: 0.85 +[2025-04-22 23:37:38] (step=0056650) Train Loss: 6.3214, Train Steps/Sec: 0.85 +[2025-04-22 23:38:07] (step=0056675) Train Loss: 6.3450, Train Steps/Sec: 0.85 +[2025-04-22 23:38:37] (step=0056700) Train Loss: 6.3693, Train Steps/Sec: 0.84 +[2025-04-22 23:39:07] (step=0056725) Train Loss: 6.3612, Train Steps/Sec: 0.85 +[2025-04-22 23:39:36] (step=0056750) Train Loss: 6.3388, Train Steps/Sec: 0.85 +[2025-04-22 23:40:06] (step=0056775) Train Loss: 6.3310, Train Steps/Sec: 0.84 +[2025-04-22 23:40:36] (step=0056800) Train Loss: 6.3601, Train Steps/Sec: 0.83 +[2025-04-22 23:41:05] (step=0056825) Train Loss: 6.3451, Train Steps/Sec: 0.84 +[2025-04-22 23:41:35] (step=0056850) Train Loss: 6.3513, Train Steps/Sec: 0.85 +[2025-04-22 23:42:04] (step=0056875) Train Loss: 6.3635, Train Steps/Sec: 0.85 +[2025-04-22 23:42:34] (step=0056900) Train Loss: 6.3390, Train Steps/Sec: 0.84 +[2025-04-22 23:43:04] (step=0056925) Train Loss: 6.3425, Train Steps/Sec: 0.85 +[2025-04-22 23:43:33] (step=0056950) Train Loss: 6.3505, Train Steps/Sec: 0.85 +[2025-04-22 23:44:03] (step=0056975) Train Loss: 6.3425, Train Steps/Sec: 0.84 +[2025-04-22 23:44:32] (step=0057000) Train Loss: 6.3702, Train Steps/Sec: 0.84 +[2025-04-22 23:45:02] (step=0057025) Train Loss: 6.3826, Train Steps/Sec: 0.85 +[2025-04-22 23:45:31] (step=0057050) Train Loss: 6.3433, Train Steps/Sec: 0.85 +[2025-04-22 23:46:01] (step=0057075) Train Loss: 6.3489, Train Steps/Sec: 0.85 +[2025-04-22 23:46:31] (step=0057100) Train Loss: 6.3365, Train Steps/Sec: 0.84 +[2025-04-22 23:47:00] (step=0057125) Train Loss: 6.3767, Train Steps/Sec: 0.85 +[2025-04-22 23:47:30] (step=0057150) Train Loss: 6.3572, Train Steps/Sec: 0.85 +[2025-04-22 23:47:59] (step=0057175) Train Loss: 6.3373, Train Steps/Sec: 0.85 +[2025-04-22 23:48:29] (step=0057200) Train Loss: 6.3375, Train Steps/Sec: 0.85 +[2025-04-22 23:48:59] (step=0057225) Train Loss: 6.3276, Train Steps/Sec: 0.84 +[2025-04-22 23:49:28] (step=0057250) Train Loss: 6.3482, Train Steps/Sec: 0.84 +[2025-04-22 23:49:58] (step=0057275) Train Loss: 6.3370, Train Steps/Sec: 0.85 +[2025-04-22 23:50:27] (step=0057300) Train Loss: 6.3217, Train Steps/Sec: 0.84 +[2025-04-22 23:50:57] (step=0057325) Train Loss: 6.3290, Train Steps/Sec: 0.85 +[2025-04-22 23:51:26] (step=0057350) Train Loss: 6.3970, Train Steps/Sec: 0.85 +[2025-04-22 23:51:56] (step=0057375) Train Loss: 6.3492, Train Steps/Sec: 0.85 +[2025-04-22 23:52:26] (step=0057400) Train Loss: 6.3638, Train Steps/Sec: 0.84 +[2025-04-22 23:52:55] (step=0057425) Train Loss: 6.3525, Train Steps/Sec: 0.85 +[2025-04-22 23:53:25] (step=0057450) Train Loss: 6.3624, Train Steps/Sec: 0.84 +[2025-04-22 23:53:54] (step=0057475) Train Loss: 6.3671, Train Steps/Sec: 0.85 +[2025-04-22 23:54:24] (step=0057500) Train Loss: 6.3372, Train Steps/Sec: 0.84 +[2025-04-22 23:54:54] (step=0057525) Train Loss: 6.3775, Train Steps/Sec: 0.84 +[2025-04-22 23:55:23] (step=0057550) Train Loss: 6.3511, Train Steps/Sec: 0.84 +[2025-04-22 23:55:53] (step=0057575) Train Loss: 6.3765, Train Steps/Sec: 0.85 +[2025-04-22 23:56:22] (step=0057600) Train Loss: 6.3573, Train Steps/Sec: 0.85 +[2025-04-22 23:56:52] (step=0057625) Train Loss: 6.3730, Train Steps/Sec: 0.85 +[2025-04-22 23:57:21] (step=0057650) Train Loss: 6.3445, Train Steps/Sec: 0.85 +[2025-04-22 23:57:51] (step=0057675) Train Loss: 6.3342, Train Steps/Sec: 0.84 +[2025-04-22 23:58:20] (step=0057700) Train Loss: 6.3634, Train Steps/Sec: 0.85 +[2025-04-22 23:58:50] (step=0057725) Train Loss: 6.3528, Train Steps/Sec: 0.84 +[2025-04-22 23:59:20] (step=0057750) Train Loss: 6.3264, Train Steps/Sec: 0.85 +[2025-04-22 23:59:49] (step=0057775) Train Loss: 6.3578, Train Steps/Sec: 0.84 +[2025-04-23 00:00:19] (step=0057800) Train Loss: 6.3556, Train Steps/Sec: 0.84 +[2025-04-23 00:00:49] (step=0057825) Train Loss: 6.3528, Train Steps/Sec: 0.84 +[2025-04-23 00:01:18] (step=0057850) Train Loss: 6.3667, Train Steps/Sec: 0.85 +[2025-04-23 00:01:48] (step=0057875) Train Loss: 6.2504, Train Steps/Sec: 0.85 +[2025-04-23 00:02:17] (step=0057900) Train Loss: 6.3355, Train Steps/Sec: 0.84 +[2025-04-23 00:02:47] (step=0057925) Train Loss: 6.3522, Train Steps/Sec: 0.84 +[2025-04-23 00:03:16] (step=0057950) Train Loss: 6.3422, Train Steps/Sec: 0.85 +[2025-04-23 00:03:46] (step=0057975) Train Loss: 6.3737, Train Steps/Sec: 0.85 +[2025-04-23 00:04:16] (step=0058000) Train Loss: 6.3704, Train Steps/Sec: 0.84 +[2025-04-23 00:08:58] Finish Eval in 58000 steps... +[2025-04-23 00:09:19] Saved checkpoint to checkpoint/BeforeFix_Blip_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_objects/002-GPT-XL/checkpoints/0058000.pt +[2025-04-23 00:09:21] Removed old checkpoint: checkpoint/BeforeFix_Blip_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_objects/002-GPT-XL/checkpoints/0054000.pt +[2025-04-23 00:09:51] (step=0058025) Train Loss: 6.3290, Train Steps/Sec: 0.07 +[2025-04-23 00:10:21] (step=0058050) Train Loss: 6.3338, Train Steps/Sec: 0.84 +[2025-04-23 00:10:50] (step=0058075) Train Loss: 6.3431, Train Steps/Sec: 0.85 +[2025-04-23 00:11:20] (step=0058100) Train Loss: 6.3163, Train Steps/Sec: 0.85 +[2025-04-23 00:11:49] (step=0058125) Train Loss: 6.3915, Train Steps/Sec: 0.84 +[2025-04-23 00:12:19] (step=0058150) Train Loss: 6.3559, Train Steps/Sec: 0.85 +[2025-04-23 00:12:48] (step=0058175) Train Loss: 6.3612, Train Steps/Sec: 0.84 +[2025-04-23 00:13:24] (step=0058200) Train Loss: 6.3006, Train Steps/Sec: 0.69 +[2025-04-23 00:13:54] (step=0058225) Train Loss: 6.3712, Train Steps/Sec: 0.84 +[2025-04-23 00:14:24] (step=0058250) Train Loss: 6.3507, Train Steps/Sec: 0.84 +[2025-04-23 00:14:53] (step=0058275) Train Loss: 6.3322, Train Steps/Sec: 0.85 +[2025-04-23 00:15:23] (step=0058300) Train Loss: 6.3266, Train Steps/Sec: 0.85 +[2025-04-23 00:15:52] (step=0058325) Train Loss: 6.3670, Train Steps/Sec: 0.84 +[2025-04-23 00:16:22] (step=0058350) Train Loss: 6.3547, Train Steps/Sec: 0.85 +[2025-04-23 00:16:52] (step=0058375) Train Loss: 6.3786, Train Steps/Sec: 0.84 +[2025-04-23 00:17:21] (step=0058400) Train Loss: 6.3510, Train Steps/Sec: 0.85 +[2025-04-23 00:17:51] (step=0058425) Train Loss: 6.3942, Train Steps/Sec: 0.84 +[2025-04-23 00:18:21] (step=0058450) Train Loss: 6.3586, Train Steps/Sec: 0.85 +[2025-04-23 00:18:50] (step=0058475) Train Loss: 6.3456, Train Steps/Sec: 0.85 +[2025-04-23 00:19:27] (step=0058500) Train Loss: 6.3676, Train Steps/Sec: 0.67 +[2025-04-23 00:19:57] (step=0058525) Train Loss: 6.3509, Train Steps/Sec: 0.85 +[2025-04-23 00:20:27] (step=0058550) Train Loss: 6.3403, Train Steps/Sec: 0.83 +[2025-04-23 00:20:56] (step=0058575) Train Loss: 6.3165, Train Steps/Sec: 0.85 +[2025-04-23 00:21:26] (step=0058600) Train Loss: 6.3961, Train Steps/Sec: 0.84 +[2025-04-23 00:21:56] (step=0058625) Train Loss: 6.3610, Train Steps/Sec: 0.84 +[2025-04-23 00:22:25] (step=0058650) Train Loss: 6.3470, Train Steps/Sec: 0.85 +[2025-04-23 00:22:55] (step=0058675) Train Loss: 6.3559, Train Steps/Sec: 0.85 +[2025-04-23 00:23:25] (step=0058700) Train Loss: 6.3629, Train Steps/Sec: 0.84 +[2025-04-23 00:23:54] (step=0058725) Train Loss: 6.3521, Train Steps/Sec: 0.85 +[2025-04-23 00:24:24] (step=0058750) Train Loss: 6.3274, Train Steps/Sec: 0.85 +[2025-04-23 00:24:53] (step=0058775) Train Loss: 6.3484, Train Steps/Sec: 0.84 +[2025-04-23 00:25:23] (step=0058800) Train Loss: 6.3098, Train Steps/Sec: 0.85 +[2025-04-23 00:25:52] (step=0058825) Train Loss: 6.3793, Train Steps/Sec: 0.85 +[2025-04-23 00:26:22] (step=0058850) Train Loss: 6.3275, Train Steps/Sec: 0.85 +[2025-04-23 00:26:51] (step=0058875) Train Loss: 6.3640, Train Steps/Sec: 0.85 +[2025-04-23 00:27:21] (step=0058900) Train Loss: 6.3613, Train Steps/Sec: 0.84 +[2025-04-23 00:27:57] (step=0058925) Train Loss: 6.3642, Train Steps/Sec: 0.70 +[2025-04-23 00:28:27] (step=0058950) Train Loss: 6.4144, Train Steps/Sec: 0.85 +[2025-04-23 00:28:56] (step=0058975) Train Loss: 6.3442, Train Steps/Sec: 0.84 +[2025-04-23 00:29:26] (step=0059000) Train Loss: 6.3802, Train Steps/Sec: 0.84 +[2025-04-23 00:29:56] (step=0059025) Train Loss: 6.3840, Train Steps/Sec: 0.84 +[2025-04-23 00:30:25] (step=0059050) Train Loss: 6.3740, Train Steps/Sec: 0.85 +[2025-04-23 00:30:55] (step=0059075) Train Loss: 6.3302, Train Steps/Sec: 0.84 +[2025-04-23 00:31:25] (step=0059100) Train Loss: 6.3647, Train Steps/Sec: 0.84 +[2025-04-23 00:31:54] (step=0059125) Train Loss: 6.3592, Train Steps/Sec: 0.85 +[2025-04-23 00:32:24] (step=0059150) Train Loss: 6.3390, Train Steps/Sec: 0.84 +[2025-04-23 00:32:53] (step=0059175) Train Loss: 6.3479, Train Steps/Sec: 0.85 +[2025-04-23 00:33:23] (step=0059200) Train Loss: 6.3417, Train Steps/Sec: 0.84 +[2025-04-23 00:33:52] (step=0059225) Train Loss: 6.3618, Train Steps/Sec: 0.85 +[2025-04-23 00:34:22] (step=0059250) Train Loss: 6.3447, Train Steps/Sec: 0.85 +[2025-04-23 00:34:52] (step=0059275) Train Loss: 6.3677, Train Steps/Sec: 0.85 +[2025-04-23 00:35:21] (step=0059300) Train Loss: 6.3314, Train Steps/Sec: 0.84 +[2025-04-23 00:35:51] (step=0059325) Train Loss: 6.3300, Train Steps/Sec: 0.84 +[2025-04-23 00:36:20] (step=0059350) Train Loss: 6.3084, Train Steps/Sec: 0.85 +[2025-04-23 00:36:50] (step=0059375) Train Loss: 6.3901, Train Steps/Sec: 0.85 +[2025-04-23 00:37:20] (step=0059400) Train Loss: 6.4052, Train Steps/Sec: 0.84 +[2025-04-23 00:37:49] (step=0059425) Train Loss: 6.3712, Train Steps/Sec: 0.85 +[2025-04-23 00:38:19] (step=0059450) Train Loss: 6.3404, Train Steps/Sec: 0.85 +[2025-04-23 00:38:48] (step=0059475) Train Loss: 6.3754, Train Steps/Sec: 0.85 +[2025-04-23 00:39:18] (step=0059500) Train Loss: 6.3944, Train Steps/Sec: 0.84 +[2025-04-23 00:39:48] (step=0059525) Train Loss: 6.3459, Train Steps/Sec: 0.85 +[2025-04-23 00:40:17] (step=0059550) Train Loss: 6.4229, Train Steps/Sec: 0.84 +[2025-04-23 00:40:47] (step=0059575) Train Loss: 6.3403, Train Steps/Sec: 0.85 +[2025-04-23 00:41:23] (step=0059600) Train Loss: 6.4009, Train Steps/Sec: 0.69 +[2025-04-23 00:41:53] (step=0059625) Train Loss: 6.3561, Train Steps/Sec: 0.84 +[2025-04-23 00:42:22] (step=0059650) Train Loss: 6.3420, Train Steps/Sec: 0.84 +[2025-04-23 00:42:52] (step=0059675) Train Loss: 6.3317, Train Steps/Sec: 0.85 +[2025-04-23 00:43:22] (step=0059700) Train Loss: 6.3246, Train Steps/Sec: 0.84 +[2025-04-23 00:43:51] (step=0059725) Train Loss: 6.3087, Train Steps/Sec: 0.85 +[2025-04-23 00:44:21] (step=0059750) Train Loss: 6.3451, Train Steps/Sec: 0.85 +[2025-04-23 00:44:50] (step=0059775) Train Loss: 6.3745, Train Steps/Sec: 0.85 +[2025-04-23 00:45:20] (step=0059800) Train Loss: 6.3218, Train Steps/Sec: 0.84 +[2025-04-23 00:45:49] (step=0059825) Train Loss: 6.3161, Train Steps/Sec: 0.84 +[2025-04-23 00:46:19] (step=0059850) Train Loss: 6.3508, Train Steps/Sec: 0.84 +[2025-04-23 00:46:48] (step=0059875) Train Loss: 6.3573, Train Steps/Sec: 0.85 +[2025-04-23 00:47:25] (step=0059900) Train Loss: 6.3272, Train Steps/Sec: 0.69 +[2025-04-23 00:47:54] (step=0059925) Train Loss: 6.3600, Train Steps/Sec: 0.85 +[2025-04-23 00:48:24] (step=0059950) Train Loss: 6.3728, Train Steps/Sec: 0.84 +[2025-04-23 00:48:54] (step=0059975) Train Loss: 6.3718, Train Steps/Sec: 0.84 +[2025-04-23 00:49:24] (step=0060000) Train Loss: 6.3603, Train Steps/Sec: 0.84 +[2025-04-23 00:54:06] Finish Eval in 60000 steps... +[2025-04-23 00:54:28] Saved checkpoint to checkpoint/BeforeFix_Blip_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_objects/002-GPT-XL/checkpoints/0060000.pt +[2025-04-23 00:54:30] Removed old checkpoint: checkpoint/BeforeFix_Blip_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_objects/002-GPT-XL/checkpoints/0056000.pt +[2025-04-23 00:55:00] (step=0060025) Train Loss: 6.3512, Train Steps/Sec: 0.07 +[2025-04-23 00:55:29] (step=0060050) Train Loss: 6.3060, Train Steps/Sec: 0.84 +[2025-04-23 00:55:59] (step=0060075) Train Loss: 6.3510, Train Steps/Sec: 0.85 +[2025-04-23 00:56:28] (step=0060100) Train Loss: 6.3394, Train Steps/Sec: 0.85 +[2025-04-23 00:56:58] (step=0060125) Train Loss: 6.3834, Train Steps/Sec: 0.84 +[2025-04-23 00:57:28] (step=0060150) Train Loss: 6.3614, Train Steps/Sec: 0.85 +[2025-04-23 00:57:57] (step=0060175) Train Loss: 6.3555, Train Steps/Sec: 0.84 +[2025-04-23 00:58:27] (step=0060200) Train Loss: 6.3691, Train Steps/Sec: 0.85 +[2025-04-23 00:58:56] (step=0060225) Train Loss: 6.3437, Train Steps/Sec: 0.85 +[2025-04-23 00:59:26] (step=0060250) Train Loss: 6.4016, Train Steps/Sec: 0.85 +[2025-04-23 00:59:55] (step=0060275) Train Loss: 6.3458, Train Steps/Sec: 0.85 +[2025-04-23 01:00:25] (step=0060300) Train Loss: 6.3585, Train Steps/Sec: 0.84 +[2025-04-23 01:00:55] (step=0060325) Train Loss: 6.3774, Train Steps/Sec: 0.84 +[2025-04-23 01:01:24] (step=0060350) Train Loss: 6.3316, Train Steps/Sec: 0.85 +[2025-04-23 01:01:54] (step=0060375) Train Loss: 6.3835, Train Steps/Sec: 0.84 +[2025-04-23 01:02:30] (step=0060400) Train Loss: 6.3950, Train Steps/Sec: 0.69 +[2025-04-23 01:03:00] (step=0060425) Train Loss: 6.3499, Train Steps/Sec: 0.84 +[2025-04-23 01:03:29] (step=0060450) Train Loss: 6.3143, Train Steps/Sec: 0.85 +[2025-04-23 01:03:59] (step=0060475) Train Loss: 6.3718, Train Steps/Sec: 0.85 +[2025-04-23 01:04:35] (step=0060500) Train Loss: 6.3172, Train Steps/Sec: 0.69 +[2025-04-23 01:05:05] (step=0060525) Train Loss: 6.3282, Train Steps/Sec: 0.85 +[2025-04-23 01:05:34] (step=0060550) Train Loss: 6.3741, Train Steps/Sec: 0.85 +[2025-04-23 01:06:03] (step=0060575) Train Loss: 6.3486, Train Steps/Sec: 0.85 +[2025-04-23 01:06:33] (step=0060600) Train Loss: 6.3623, Train Steps/Sec: 0.85 +[2025-04-23 01:07:03] (step=0060625) Train Loss: 6.2981, Train Steps/Sec: 0.85 +[2025-04-23 01:07:32] (step=0060650) Train Loss: 6.3824, Train Steps/Sec: 0.84 +[2025-04-23 01:08:02] (step=0060675) Train Loss: 6.3720, Train Steps/Sec: 0.84 +[2025-04-23 01:08:32] (step=0060700) Train Loss: 6.3654, Train Steps/Sec: 0.84 +[2025-04-23 01:09:01] (step=0060725) Train Loss: 6.3973, Train Steps/Sec: 0.85 +[2025-04-23 01:09:31] (step=0060750) Train Loss: 6.3510, Train Steps/Sec: 0.84 +[2025-04-23 01:10:00] (step=0060775) Train Loss: 6.3503, Train Steps/Sec: 0.85 +[2025-04-23 01:10:30] (step=0060800) Train Loss: 6.3566, Train Steps/Sec: 0.84 +[2025-04-23 01:11:00] (step=0060825) Train Loss: 6.3621, Train Steps/Sec: 0.84 +[2025-04-23 01:11:29] (step=0060850) Train Loss: 6.3518, Train Steps/Sec: 0.85 +[2025-04-23 01:11:59] (step=0060875) Train Loss: 6.3714, Train Steps/Sec: 0.85 +[2025-04-23 01:12:29] (step=0060900) Train Loss: 6.3538, Train Steps/Sec: 0.84 +[2025-04-23 01:12:58] (step=0060925) Train Loss: 6.3483, Train Steps/Sec: 0.85 +[2025-04-23 01:13:28] (step=0060950) Train Loss: 6.3831, Train Steps/Sec: 0.85 +[2025-04-23 01:13:57] (step=0060975) Train Loss: 6.3270, Train Steps/Sec: 0.84 +[2025-04-23 01:14:27] (step=0061000) Train Loss: 6.3770, Train Steps/Sec: 0.84 +[2025-04-23 01:14:57] (step=0061025) Train Loss: 6.3351, Train Steps/Sec: 0.84 +[2025-04-23 01:15:26] (step=0061050) Train Loss: 6.3727, Train Steps/Sec: 0.85 +[2025-04-23 01:15:56] (step=0061075) Train Loss: 6.3652, Train Steps/Sec: 0.84 +[2025-04-23 01:16:26] (step=0061100) Train Loss: 6.3468, Train Steps/Sec: 0.84 +[2025-04-23 01:16:55] (step=0061125) Train Loss: 6.3403, Train Steps/Sec: 0.85 +[2025-04-23 01:17:25] (step=0061150) Train Loss: 6.3650, Train Steps/Sec: 0.85 +[2025-04-23 01:17:54] (step=0061175) Train Loss: 6.3871, Train Steps/Sec: 0.85 +[2025-04-23 01:18:24] (step=0061200) Train Loss: 6.3918, Train Steps/Sec: 0.84 +[2025-04-23 01:18:53] (step=0061225) Train Loss: 6.3595, Train Steps/Sec: 0.84 +[2025-04-23 01:19:23] (step=0061250) Train Loss: 6.3611, Train Steps/Sec: 0.84 +[2025-04-23 01:19:52] (step=0061275) Train Loss: 6.3463, Train Steps/Sec: 0.85 +[2025-04-23 01:20:22] (step=0061300) Train Loss: 6.3469, Train Steps/Sec: 0.84 +[2025-04-23 01:20:52] (step=0061325) Train Loss: 6.3372, Train Steps/Sec: 0.85 +[2025-04-23 01:21:21] (step=0061350) Train Loss: 6.3679, Train Steps/Sec: 0.85 +[2025-04-23 01:21:51] (step=0061375) Train Loss: 6.3137, Train Steps/Sec: 0.84 +[2025-04-23 01:22:20] (step=0061400) Train Loss: 6.3569, Train Steps/Sec: 0.84 +[2025-04-23 01:22:50] (step=0061425) Train Loss: 6.3639, Train Steps/Sec: 0.85 +[2025-04-23 01:23:19] (step=0061450) Train Loss: 6.3690, Train Steps/Sec: 0.85 +[2025-04-23 01:23:49] (step=0061475) Train Loss: 6.3763, Train Steps/Sec: 0.85 +[2025-04-23 01:24:19] (step=0061500) Train Loss: 6.3721, Train Steps/Sec: 0.84 +[2025-04-23 01:24:48] (step=0061525) Train Loss: 6.3231, Train Steps/Sec: 0.85 +[2025-04-23 01:25:18] (step=0061550) Train Loss: 6.3792, Train Steps/Sec: 0.85 +[2025-04-23 01:25:47] (step=0061575) Train Loss: 6.4044, Train Steps/Sec: 0.85 +[2025-04-23 01:26:17] (step=0061600) Train Loss: 6.3546, Train Steps/Sec: 0.84 +[2025-04-23 01:26:47] (step=0061625) Train Loss: 6.3706, Train Steps/Sec: 0.84 +[2025-04-23 01:27:16] (step=0061650) Train Loss: 6.3131, Train Steps/Sec: 0.85 +[2025-04-23 01:27:46] (step=0061675) Train Loss: 6.3882, Train Steps/Sec: 0.85 +[2025-04-23 01:28:15] (step=0061700) Train Loss: 6.3468, Train Steps/Sec: 0.84 +[2025-04-23 01:28:45] (step=0061725) Train Loss: 6.3929, Train Steps/Sec: 0.84 +[2025-04-23 01:29:15] (step=0061750) Train Loss: 6.3556, Train Steps/Sec: 0.85 +[2025-04-23 01:29:44] (step=0061775) Train Loss: 6.3595, Train Steps/Sec: 0.85 +[2025-04-23 01:30:14] (step=0061800) Train Loss: 6.4402, Train Steps/Sec: 0.84 +[2025-04-23 01:30:43] (step=0061825) Train Loss: 6.3372, Train Steps/Sec: 0.85 +[2025-04-23 01:31:13] (step=0061850) Train Loss: 6.3577, Train Steps/Sec: 0.85 +[2025-04-23 01:31:42] (step=0061875) Train Loss: 6.3141, Train Steps/Sec: 0.85 +[2025-04-23 01:32:12] (step=0061900) Train Loss: 6.3724, Train Steps/Sec: 0.84 +[2025-04-23 01:32:41] (step=0061925) Train Loss: 6.3366, Train Steps/Sec: 0.85 +[2025-04-23 01:33:11] (step=0061950) Train Loss: 6.3379, Train Steps/Sec: 0.85 +[2025-04-23 01:33:40] (step=0061975) Train Loss: 6.3504, Train Steps/Sec: 0.85 +[2025-04-23 01:34:10] (step=0062000) Train Loss: 6.3501, Train Steps/Sec: 0.84 +[2025-04-23 01:38:53] Finish Eval in 62000 steps... +[2025-04-23 01:39:14] Saved checkpoint to checkpoint/BeforeFix_Blip_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_objects/002-GPT-XL/checkpoints/0062000.pt +[2025-04-23 01:39:17] Removed old checkpoint: checkpoint/BeforeFix_Blip_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_objects/002-GPT-XL/checkpoints/0058000.pt +[2025-04-23 01:39:46] (step=0062025) Train Loss: 6.3557, Train Steps/Sec: 0.07 +[2025-04-23 01:40:16] (step=0062050) Train Loss: 6.3626, Train Steps/Sec: 0.85 +[2025-04-23 01:40:45] (step=0062075) Train Loss: 6.3252, Train Steps/Sec: 0.85 +[2025-04-23 01:41:15] (step=0062100) Train Loss: 6.3430, Train Steps/Sec: 0.84 +[2025-04-23 01:41:44] (step=0062125) Train Loss: 6.3151, Train Steps/Sec: 0.84 +[2025-04-23 01:42:14] (step=0062150) Train Loss: 6.3603, Train Steps/Sec: 0.84 +[2025-04-23 01:42:44] (step=0062175) Train Loss: 6.3918, Train Steps/Sec: 0.85 +[2025-04-23 01:43:14] (step=0062200) Train Loss: 6.4090, Train Steps/Sec: 0.84 +[2025-04-23 01:43:43] (step=0062225) Train Loss: 6.3646, Train Steps/Sec: 0.85 +[2025-04-23 01:44:13] (step=0062250) Train Loss: 6.3674, Train Steps/Sec: 0.83 +[2025-04-23 01:44:43] (step=0062275) Train Loss: 6.3514, Train Steps/Sec: 0.84 +[2025-04-23 01:45:13] (step=0062300) Train Loss: 6.3793, Train Steps/Sec: 0.84 +[2025-04-23 01:45:42] (step=0062325) Train Loss: 6.3635, Train Steps/Sec: 0.85 +[2025-04-23 01:46:12] (step=0062350) Train Loss: 6.3360, Train Steps/Sec: 0.85 +[2025-04-23 01:46:41] (step=0062375) Train Loss: 6.3926, Train Steps/Sec: 0.85 +[2025-04-23 01:47:11] (step=0062400) Train Loss: 6.3639, Train Steps/Sec: 0.84 +[2025-04-23 01:47:40] (step=0062425) Train Loss: 6.2681, Train Steps/Sec: 0.85 +[2025-04-23 01:48:10] (step=0062450) Train Loss: 6.3176, Train Steps/Sec: 0.84 +[2025-04-23 01:48:40] (step=0062475) Train Loss: 6.3695, Train Steps/Sec: 0.85 +[2025-04-23 01:49:09] (step=0062500) Train Loss: 6.3309, Train Steps/Sec: 0.84 +[2025-04-23 01:49:39] (step=0062525) Train Loss: 6.3456, Train Steps/Sec: 0.85 +[2025-04-23 01:50:09] (step=0062550) Train Loss: 6.3922, Train Steps/Sec: 0.84 +[2025-04-23 01:50:38] (step=0062575) Train Loss: 6.3579, Train Steps/Sec: 0.84 +[2025-04-23 01:51:08] (step=0062600) Train Loss: 6.3286, Train Steps/Sec: 0.84 +[2025-04-23 01:51:37] (step=0062625) Train Loss: 6.3469, Train Steps/Sec: 0.85 +[2025-04-23 01:52:07] (step=0062650) Train Loss: 6.3562, Train Steps/Sec: 0.85 +[2025-04-23 01:52:37] (step=0062675) Train Loss: 6.3648, Train Steps/Sec: 0.84 +[2025-04-23 01:53:06] (step=0062700) Train Loss: 6.3379, Train Steps/Sec: 0.85 +[2025-04-23 01:53:36] (step=0062725) Train Loss: 6.3610, Train Steps/Sec: 0.85 +[2025-04-23 01:54:05] (step=0062750) Train Loss: 6.3883, Train Steps/Sec: 0.84 +[2025-04-23 01:54:35] (step=0062775) Train Loss: 6.3681, Train Steps/Sec: 0.85 +[2025-04-23 01:55:05] (step=0062800) Train Loss: 6.3585, Train Steps/Sec: 0.84 +[2025-04-23 01:55:34] (step=0062825) Train Loss: 6.3875, Train Steps/Sec: 0.84 +[2025-04-23 01:56:04] (step=0062850) Train Loss: 6.3796, Train Steps/Sec: 0.84 +[2025-04-23 01:56:33] (step=0062875) Train Loss: 6.3456, Train Steps/Sec: 0.85 +[2025-04-23 01:57:03] (step=0062900) Train Loss: 6.3158, Train Steps/Sec: 0.84 +[2025-04-23 01:57:33] (step=0062925) Train Loss: 6.3458, Train Steps/Sec: 0.84 +[2025-04-23 01:58:02] (step=0062950) Train Loss: 6.3445, Train Steps/Sec: 0.84 +[2025-04-23 01:58:32] (step=0062975) Train Loss: 6.3467, Train Steps/Sec: 0.85 +[2025-04-23 01:59:02] (step=0063000) Train Loss: 6.3569, Train Steps/Sec: 0.84 +[2025-04-23 01:59:31] (step=0063025) Train Loss: 6.3482, Train Steps/Sec: 0.84 +[2025-04-23 02:00:01] (step=0063050) Train Loss: 6.3249, Train Steps/Sec: 0.85 +[2025-04-23 02:00:30] (step=0063075) Train Loss: 6.3804, Train Steps/Sec: 0.85 +[2025-04-23 02:01:00] (step=0063100) Train Loss: 6.3363, Train Steps/Sec: 0.84 +[2025-04-23 02:01:29] (step=0063125) Train Loss: 6.2975, Train Steps/Sec: 0.85 +[2025-04-23 02:01:59] (step=0063150) Train Loss: 6.3370, Train Steps/Sec: 0.85 +[2025-04-23 02:02:29] (step=0063175) Train Loss: 6.3386, Train Steps/Sec: 0.84 +[2025-04-23 02:02:58] (step=0063200) Train Loss: 6.3425, Train Steps/Sec: 0.85 +[2025-04-23 02:03:28] (step=0063225) Train Loss: 6.3789, Train Steps/Sec: 0.85 +[2025-04-23 02:03:57] (step=0063250) Train Loss: 6.3952, Train Steps/Sec: 0.84 +[2025-04-23 02:04:27] (step=0063275) Train Loss: 6.3712, Train Steps/Sec: 0.84 +[2025-04-23 02:04:56] (step=0063300) Train Loss: 6.3246, Train Steps/Sec: 0.84 +[2025-04-23 02:05:26] (step=0063325) Train Loss: 6.3427, Train Steps/Sec: 0.85 +[2025-04-23 02:05:56] (step=0063350) Train Loss: 6.3416, Train Steps/Sec: 0.84 +[2025-04-23 02:06:25] (step=0063375) Train Loss: 6.3771, Train Steps/Sec: 0.84 +[2025-04-23 02:06:55] (step=0063400) Train Loss: 6.3277, Train Steps/Sec: 0.84 +[2025-04-23 02:07:25] (step=0063425) Train Loss: 6.2981, Train Steps/Sec: 0.85 +[2025-04-23 02:07:54] (step=0063450) Train Loss: 6.4009, Train Steps/Sec: 0.85 +[2025-04-23 02:08:24] (step=0063475) Train Loss: 6.3537, Train Steps/Sec: 0.85 +[2025-04-23 02:08:53] (step=0063500) Train Loss: 6.3791, Train Steps/Sec: 0.85 +[2025-04-23 02:09:23] (step=0063525) Train Loss: 6.3215, Train Steps/Sec: 0.84 +[2025-04-23 02:09:53] (step=0063550) Train Loss: 6.3749, Train Steps/Sec: 0.84 +[2025-04-23 02:10:22] (step=0063575) Train Loss: 6.3508, Train Steps/Sec: 0.84 +[2025-04-23 02:10:58] (step=0063600) Train Loss: 6.3413, Train Steps/Sec: 0.69 +[2025-04-23 02:11:28] (step=0063625) Train Loss: 6.3008, Train Steps/Sec: 0.85 +[2025-04-23 02:11:57] (step=0063650) Train Loss: 6.3721, Train Steps/Sec: 0.85 +[2025-04-23 02:12:27] (step=0063675) Train Loss: 6.3174, Train Steps/Sec: 0.84 +[2025-04-23 02:12:57] (step=0063700) Train Loss: 6.3715, Train Steps/Sec: 0.84 +[2025-04-23 02:13:26] (step=0063725) Train Loss: 6.3516, Train Steps/Sec: 0.85 +[2025-04-23 02:13:56] (step=0063750) Train Loss: 6.3485, Train Steps/Sec: 0.84 +[2025-04-23 02:14:26] (step=0063775) Train Loss: 6.3504, Train Steps/Sec: 0.85 +[2025-04-23 02:14:55] (step=0063800) Train Loss: 6.3364, Train Steps/Sec: 0.84 +[2025-04-23 02:15:25] (step=0063825) Train Loss: 6.3256, Train Steps/Sec: 0.85 +[2025-04-23 02:15:54] (step=0063850) Train Loss: 6.4014, Train Steps/Sec: 0.85 +[2025-04-23 02:16:24] (step=0063875) Train Loss: 6.3323, Train Steps/Sec: 0.84 +[2025-04-23 02:16:53] (step=0063900) Train Loss: 6.3499, Train Steps/Sec: 0.85 +[2025-04-23 02:17:23] (step=0063925) Train Loss: 6.3401, Train Steps/Sec: 0.85 +[2025-04-23 02:17:52] (step=0063950) Train Loss: 6.3843, Train Steps/Sec: 0.85 +[2025-04-23 02:18:22] (step=0063975) Train Loss: 6.3725, Train Steps/Sec: 0.85 +[2025-04-23 02:18:59] (step=0064000) Train Loss: 6.3980, Train Steps/Sec: 0.68 +[2025-04-23 02:23:42] Finish Eval in 64000 steps... +[2025-04-23 02:24:04] Saved checkpoint to checkpoint/BeforeFix_Blip_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_objects/002-GPT-XL/checkpoints/0064000.pt +[2025-04-23 02:24:06] Removed old checkpoint: checkpoint/BeforeFix_Blip_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_objects/002-GPT-XL/checkpoints/0060000.pt +[2025-04-23 02:24:36] (step=0064025) Train Loss: 6.3392, Train Steps/Sec: 0.07 +[2025-04-23 02:25:05] (step=0064050) Train Loss: 6.3861, Train Steps/Sec: 0.84 +[2025-04-23 02:25:35] (step=0064075) Train Loss: 6.3302, Train Steps/Sec: 0.84 +[2025-04-23 02:26:05] (step=0064100) Train Loss: 6.3661, Train Steps/Sec: 0.83 +[2025-04-23 02:26:35] (step=0064125) Train Loss: 6.3796, Train Steps/Sec: 0.84 +[2025-04-23 02:27:04] (step=0064150) Train Loss: 6.3818, Train Steps/Sec: 0.84 +[2025-04-23 02:27:34] (step=0064175) Train Loss: 6.3574, Train Steps/Sec: 0.84 +[2025-04-23 02:28:04] (step=0064200) Train Loss: 6.3455, Train Steps/Sec: 0.85 +[2025-04-23 02:28:33] (step=0064225) Train Loss: 6.3539, Train Steps/Sec: 0.85 +[2025-04-23 02:29:08] (step=0064250) Train Loss: 6.3644, Train Steps/Sec: 0.71 +[2025-04-23 02:29:38] (step=0064275) Train Loss: 6.3313, Train Steps/Sec: 0.85 +[2025-04-23 02:30:08] (step=0064300) Train Loss: 6.3388, Train Steps/Sec: 0.84 +[2025-04-23 02:30:37] (step=0064325) Train Loss: 6.3166, Train Steps/Sec: 0.85 +[2025-04-23 02:31:06] (step=0064350) Train Loss: 6.3685, Train Steps/Sec: 0.86 +[2025-04-23 02:31:36] (step=0064375) Train Loss: 6.3484, Train Steps/Sec: 0.85 +[2025-04-23 02:32:05] (step=0064400) Train Loss: 6.3136, Train Steps/Sec: 0.85 +[2025-04-23 02:32:35] (step=0064425) Train Loss: 6.3520, Train Steps/Sec: 0.85 +[2025-04-23 02:33:04] (step=0064450) Train Loss: 6.3871, Train Steps/Sec: 0.85 +[2025-04-23 02:33:34] (step=0064475) Train Loss: 6.3369, Train Steps/Sec: 0.84 +[2025-04-23 02:34:04] (step=0064500) Train Loss: 6.3527, Train Steps/Sec: 0.84 +[2025-04-23 02:34:33] (step=0064525) Train Loss: 6.3190, Train Steps/Sec: 0.85 +[2025-04-23 02:35:03] (step=0064550) Train Loss: 6.2953, Train Steps/Sec: 0.85 +[2025-04-23 02:35:32] (step=0064575) Train Loss: 6.4045, Train Steps/Sec: 0.84 +[2025-04-23 02:36:02] (step=0064600) Train Loss: 6.3595, Train Steps/Sec: 0.84 +[2025-04-23 02:36:38] (step=0064625) Train Loss: 6.3060, Train Steps/Sec: 0.70 +[2025-04-23 02:37:07] (step=0064650) Train Loss: 6.3351, Train Steps/Sec: 0.85 +[2025-04-23 02:37:37] (step=0064675) Train Loss: 6.4252, Train Steps/Sec: 0.84 +[2025-04-23 02:38:06] (step=0064700) Train Loss: 6.3582, Train Steps/Sec: 0.85 +[2025-04-23 02:38:36] (step=0064725) Train Loss: 6.3378, Train Steps/Sec: 0.85 +[2025-04-23 02:39:05] (step=0064750) Train Loss: 6.3467, Train Steps/Sec: 0.84 +[2025-04-23 02:39:35] (step=0064775) Train Loss: 6.4106, Train Steps/Sec: 0.85 +[2025-04-23 02:40:05] (step=0064800) Train Loss: 6.3491, Train Steps/Sec: 0.84 +[2025-04-23 02:40:34] (step=0064825) Train Loss: 6.3646, Train Steps/Sec: 0.84 +[2025-04-23 02:41:04] (step=0064850) Train Loss: 6.3323, Train Steps/Sec: 0.84 +[2025-04-23 02:41:34] (step=0064875) Train Loss: 6.3399, Train Steps/Sec: 0.85 +[2025-04-23 02:42:03] (step=0064900) Train Loss: 6.3531, Train Steps/Sec: 0.84 +[2025-04-23 02:42:33] (step=0064925) Train Loss: 6.3273, Train Steps/Sec: 0.85 +[2025-04-23 02:43:02] (step=0064950) Train Loss: 6.3822, Train Steps/Sec: 0.85 +[2025-04-23 02:43:39] (step=0064975) Train Loss: 6.3431, Train Steps/Sec: 0.69 +[2025-04-23 02:44:08] (step=0065000) Train Loss: 6.3736, Train Steps/Sec: 0.84 +[2025-04-23 02:44:38] (step=0065025) Train Loss: 6.3617, Train Steps/Sec: 0.85 +[2025-04-23 02:45:07] (step=0065050) Train Loss: 6.3250, Train Steps/Sec: 0.84 +[2025-04-23 02:45:37] (step=0065075) Train Loss: 6.3283, Train Steps/Sec: 0.85 +[2025-04-23 02:46:07] (step=0065100) Train Loss: 6.3877, Train Steps/Sec: 0.84 +[2025-04-23 02:46:36] (step=0065125) Train Loss: 6.3965, Train Steps/Sec: 0.84 +[2025-04-23 02:47:06] (step=0065150) Train Loss: 6.3641, Train Steps/Sec: 0.84 +[2025-04-23 02:47:36] (step=0065175) Train Loss: 6.3372, Train Steps/Sec: 0.85 +[2025-04-23 02:48:05] (step=0065200) Train Loss: 6.3477, Train Steps/Sec: 0.84 +[2025-04-23 02:48:35] (step=0065225) Train Loss: 6.3493, Train Steps/Sec: 0.84 +[2025-04-23 02:49:04] (step=0065250) Train Loss: 6.3417, Train Steps/Sec: 0.84 +[2025-04-23 02:49:34] (step=0065275) Train Loss: 6.3010, Train Steps/Sec: 0.85 +[2025-04-23 02:50:04] (step=0065300) Train Loss: 6.3587, Train Steps/Sec: 0.84 +[2025-04-23 02:50:33] (step=0065325) Train Loss: 6.3424, Train Steps/Sec: 0.85 +[2025-04-23 02:51:03] (step=0065350) Train Loss: 6.3805, Train Steps/Sec: 0.84 +[2025-04-23 02:51:32] (step=0065375) Train Loss: 6.2927, Train Steps/Sec: 0.84 +[2025-04-23 02:52:02] (step=0065400) Train Loss: 6.3656, Train Steps/Sec: 0.85 +[2025-04-23 02:52:31] (step=0065425) Train Loss: 6.3667, Train Steps/Sec: 0.85 +[2025-04-23 02:53:01] (step=0065450) Train Loss: 6.3276, Train Steps/Sec: 0.85 +[2025-04-23 02:53:36] (step=0065475) Train Loss: 6.3375, Train Steps/Sec: 0.70 +[2025-04-23 02:54:06] (step=0065500) Train Loss: 6.3213, Train Steps/Sec: 0.84 +[2025-04-23 02:54:36] (step=0065525) Train Loss: 6.3455, Train Steps/Sec: 0.84 +[2025-04-23 02:55:05] (step=0065550) Train Loss: 6.3407, Train Steps/Sec: 0.85 +[2025-04-23 02:55:35] (step=0065575) Train Loss: 6.3742, Train Steps/Sec: 0.85 +[2025-04-23 02:56:11] (step=0065600) Train Loss: 6.3811, Train Steps/Sec: 0.69 +[2025-04-23 02:56:41] (step=0065625) Train Loss: 6.3286, Train Steps/Sec: 0.85 +[2025-04-23 02:57:10] (step=0065650) Train Loss: 6.3204, Train Steps/Sec: 0.84 +[2025-04-23 02:57:40] (step=0065675) Train Loss: 6.3540, Train Steps/Sec: 0.84 +[2025-04-23 02:58:10] (step=0065700) Train Loss: 6.3491, Train Steps/Sec: 0.84 +[2025-04-23 02:58:39] (step=0065725) Train Loss: 6.3510, Train Steps/Sec: 0.85 +[2025-04-23 02:59:08] (step=0065750) Train Loss: 6.3026, Train Steps/Sec: 0.85 +[2025-04-23 02:59:38] (step=0065775) Train Loss: 6.3619, Train Steps/Sec: 0.85 +[2025-04-23 03:00:08] (step=0065800) Train Loss: 6.3727, Train Steps/Sec: 0.84 +[2025-04-23 03:00:37] (step=0065825) Train Loss: 6.3767, Train Steps/Sec: 0.85 +[2025-04-23 03:01:07] (step=0065850) Train Loss: 6.3487, Train Steps/Sec: 0.85 +[2025-04-23 03:01:37] (step=0065875) Train Loss: 6.3888, Train Steps/Sec: 0.84 +[2025-04-23 03:02:06] (step=0065900) Train Loss: 6.3431, Train Steps/Sec: 0.85 +[2025-04-23 03:02:36] (step=0065925) Train Loss: 6.3807, Train Steps/Sec: 0.84 +[2025-04-23 03:03:05] (step=0065950) Train Loss: 6.3260, Train Steps/Sec: 0.85 +[2025-04-23 03:03:35] (step=0065975) Train Loss: 6.3659, Train Steps/Sec: 0.85 +[2025-04-23 03:04:04] (step=0066000) Train Loss: 6.3234, Train Steps/Sec: 0.85 +[2025-04-23 03:08:46] Finish Eval in 66000 steps... +[2025-04-23 03:09:07] Saved checkpoint to checkpoint/BeforeFix_Blip_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_objects/002-GPT-XL/checkpoints/0066000.pt +[2025-04-23 03:09:10] Removed old checkpoint: checkpoint/BeforeFix_Blip_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_objects/002-GPT-XL/checkpoints/0062000.pt +[2025-04-23 03:09:39] (step=0066025) Train Loss: 6.3385, Train Steps/Sec: 0.07 +[2025-04-23 03:10:09] (step=0066050) Train Loss: 6.3768, Train Steps/Sec: 0.85 +[2025-04-23 03:10:38] (step=0066075) Train Loss: 6.3173, Train Steps/Sec: 0.84 +[2025-04-23 03:11:08] (step=0066100) Train Loss: 6.3832, Train Steps/Sec: 0.84 +[2025-04-23 03:11:38] (step=0066125) Train Loss: 6.3364, Train Steps/Sec: 0.85 +[2025-04-23 03:12:07] (step=0066150) Train Loss: 6.3700, Train Steps/Sec: 0.85 +[2025-04-23 03:12:36] (step=0066175) Train Loss: 6.3740, Train Steps/Sec: 0.85 +[2025-04-23 03:13:06] (step=0066200) Train Loss: 6.3260, Train Steps/Sec: 0.85 +[2025-04-23 03:13:36] (step=0066225) Train Loss: 6.3687, Train Steps/Sec: 0.84 +[2025-04-23 03:14:05] (step=0066250) Train Loss: 6.3342, Train Steps/Sec: 0.85 +[2025-04-23 03:14:35] (step=0066275) Train Loss: 6.3542, Train Steps/Sec: 0.85 +[2025-04-23 03:15:04] (step=0066300) Train Loss: 6.3369, Train Steps/Sec: 0.84 +[2025-04-23 03:15:34] (step=0066325) Train Loss: 6.3529, Train Steps/Sec: 0.85 +[2025-04-23 03:16:03] (step=0066350) Train Loss: 6.3651, Train Steps/Sec: 0.84 +[2025-04-23 03:16:33] (step=0066375) Train Loss: 6.3005, Train Steps/Sec: 0.85 +[2025-04-23 03:17:02] (step=0066400) Train Loss: 6.3621, Train Steps/Sec: 0.85 +[2025-04-23 03:17:32] (step=0066425) Train Loss: 6.3078, Train Steps/Sec: 0.85 +[2025-04-23 03:18:02] (step=0066450) Train Loss: 6.3655, Train Steps/Sec: 0.84 +[2025-04-23 03:18:31] (step=0066475) Train Loss: 6.3260, Train Steps/Sec: 0.84 +[2025-04-23 03:19:01] (step=0066500) Train Loss: 6.3164, Train Steps/Sec: 0.85 +[2025-04-23 03:19:30] (step=0066525) Train Loss: 6.3345, Train Steps/Sec: 0.85 +[2025-04-23 03:19:59] (step=0066550) Train Loss: 6.3707, Train Steps/Sec: 0.85 +[2025-04-23 03:20:29] (step=0066575) Train Loss: 6.3026, Train Steps/Sec: 0.84 +[2025-04-23 03:20:59] (step=0066600) Train Loss: 6.3408, Train Steps/Sec: 0.85 +[2025-04-23 03:21:29] (step=0066625) Train Loss: 6.3819, Train Steps/Sec: 0.84 +[2025-04-23 03:21:58] (step=0066650) Train Loss: 6.3302, Train Steps/Sec: 0.85 +[2025-04-23 03:22:27] (step=0066675) Train Loss: 6.3462, Train Steps/Sec: 0.85 +[2025-04-23 03:22:57] (step=0066700) Train Loss: 6.3153, Train Steps/Sec: 0.85 +[2025-04-23 03:23:27] (step=0066725) Train Loss: 6.3556, Train Steps/Sec: 0.85 +[2025-04-23 03:23:56] (step=0066750) Train Loss: 6.3270, Train Steps/Sec: 0.84 +[2025-04-23 03:24:26] (step=0066775) Train Loss: 6.3833, Train Steps/Sec: 0.85 +[2025-04-23 03:24:55] (step=0066800) Train Loss: 6.3228, Train Steps/Sec: 0.85 +[2025-04-23 03:25:25] (step=0066825) Train Loss: 6.3594, Train Steps/Sec: 0.84 +[2025-04-23 03:25:54] (step=0066850) Train Loss: 6.3680, Train Steps/Sec: 0.84 +[2025-04-23 03:26:24] (step=0066875) Train Loss: 6.3347, Train Steps/Sec: 0.84 +[2025-04-23 03:26:54] (step=0066900) Train Loss: 6.3713, Train Steps/Sec: 0.84 +[2025-04-23 03:27:24] (step=0066925) Train Loss: 6.3526, Train Steps/Sec: 0.85 +[2025-04-23 03:27:53] (step=0066950) Train Loss: 6.3577, Train Steps/Sec: 0.84 +[2025-04-23 03:28:23] (step=0066975) Train Loss: 6.3842, Train Steps/Sec: 0.84 +[2025-04-23 03:28:52] (step=0067000) Train Loss: 6.3351, Train Steps/Sec: 0.85 +[2025-04-23 03:29:22] (step=0067025) Train Loss: 6.3368, Train Steps/Sec: 0.84 +[2025-04-23 03:29:52] (step=0067050) Train Loss: 6.3699, Train Steps/Sec: 0.84 +[2025-04-23 03:30:22] (step=0067075) Train Loss: 6.3649, Train Steps/Sec: 0.84 +[2025-04-23 03:30:51] (step=0067100) Train Loss: 6.3492, Train Steps/Sec: 0.84 +[2025-04-23 03:31:21] (step=0067125) Train Loss: 6.3555, Train Steps/Sec: 0.85 +[2025-04-23 03:31:50] (step=0067150) Train Loss: 6.3542, Train Steps/Sec: 0.85 +[2025-04-23 03:32:20] (step=0067175) Train Loss: 6.3286, Train Steps/Sec: 0.84 +[2025-04-23 03:32:50] (step=0067200) Train Loss: 6.2953, Train Steps/Sec: 0.84 +[2025-04-23 03:33:19] (step=0067225) Train Loss: 6.3171, Train Steps/Sec: 0.84 +[2025-04-23 03:33:49] (step=0067250) Train Loss: 6.3496, Train Steps/Sec: 0.85 +[2025-04-23 03:34:18] (step=0067275) Train Loss: 6.3219, Train Steps/Sec: 0.85 +[2025-04-23 03:34:48] (step=0067300) Train Loss: 6.3881, Train Steps/Sec: 0.85 +[2025-04-23 03:35:17] (step=0067325) Train Loss: 6.3436, Train Steps/Sec: 0.84 +[2025-04-23 03:35:47] (step=0067350) Train Loss: 6.3732, Train Steps/Sec: 0.85 +[2025-04-23 03:36:17] (step=0067375) Train Loss: 6.3529, Train Steps/Sec: 0.85 +[2025-04-23 03:36:46] (step=0067400) Train Loss: 6.3418, Train Steps/Sec: 0.84 +[2025-04-23 03:37:16] (step=0067425) Train Loss: 6.3578, Train Steps/Sec: 0.84 +[2025-04-23 03:37:45] (step=0067450) Train Loss: 6.3558, Train Steps/Sec: 0.85 +[2025-04-23 03:38:15] (step=0067475) Train Loss: 6.3481, Train Steps/Sec: 0.85 +[2025-04-23 03:38:45] (step=0067500) Train Loss: 6.3243, Train Steps/Sec: 0.84 +[2025-04-23 03:39:14] (step=0067525) Train Loss: 6.3968, Train Steps/Sec: 0.85 +[2025-04-23 03:39:43] (step=0067550) Train Loss: 6.3015, Train Steps/Sec: 0.85 +[2025-04-23 03:40:13] (step=0067575) Train Loss: 6.3854, Train Steps/Sec: 0.85 +[2025-04-23 03:40:43] (step=0067600) Train Loss: 6.3681, Train Steps/Sec: 0.84 +[2025-04-23 03:41:12] (step=0067625) Train Loss: 6.3718, Train Steps/Sec: 0.85 +[2025-04-23 03:41:42] (step=0067650) Train Loss: 6.3531, Train Steps/Sec: 0.84 +[2025-04-23 03:42:12] (step=0067675) Train Loss: 6.4035, Train Steps/Sec: 0.84 +[2025-04-23 03:42:41] (step=0067700) Train Loss: 6.3745, Train Steps/Sec: 0.84 +[2025-04-23 03:43:11] (step=0067725) Train Loss: 6.3522, Train Steps/Sec: 0.85 +[2025-04-23 03:43:41] (step=0067750) Train Loss: 6.3945, Train Steps/Sec: 0.84 +[2025-04-23 03:44:10] (step=0067775) Train Loss: 6.3681, Train Steps/Sec: 0.85 +[2025-04-23 03:44:40] (step=0067800) Train Loss: 6.3421, Train Steps/Sec: 0.85 +[2025-04-23 03:45:09] (step=0067825) Train Loss: 6.3226, Train Steps/Sec: 0.85 +[2025-04-23 03:45:39] (step=0067850) Train Loss: 6.3771, Train Steps/Sec: 0.84 +[2025-04-23 03:46:08] (step=0067875) Train Loss: 6.3529, Train Steps/Sec: 0.85 +[2025-04-23 03:46:38] (step=0067900) Train Loss: 6.3673, Train Steps/Sec: 0.85 +[2025-04-23 03:47:07] (step=0067925) Train Loss: 6.3418, Train Steps/Sec: 0.84 +[2025-04-23 03:47:37] (step=0067950) Train Loss: 6.3299, Train Steps/Sec: 0.85 +[2025-04-23 03:48:06] (step=0067975) Train Loss: 6.3490, Train Steps/Sec: 0.85 +[2025-04-23 03:48:36] (step=0068000) Train Loss: 6.3817, Train Steps/Sec: 0.84 +[2025-04-23 03:53:19] Finish Eval in 68000 steps... +[2025-04-23 03:53:40] Saved checkpoint to checkpoint/BeforeFix_Blip_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_objects/002-GPT-XL/checkpoints/0068000.pt +[2025-04-23 03:53:42] Removed old checkpoint: checkpoint/BeforeFix_Blip_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_objects/002-GPT-XL/checkpoints/0064000.pt +[2025-04-23 03:54:12] (step=0068025) Train Loss: 6.3556, Train Steps/Sec: 0.07 +[2025-04-23 03:54:41] (step=0068050) Train Loss: 6.3482, Train Steps/Sec: 0.85 +[2025-04-23 03:55:11] (step=0068075) Train Loss: 6.3781, Train Steps/Sec: 0.84 +[2025-04-23 03:55:41] (step=0068100) Train Loss: 6.3700, Train Steps/Sec: 0.84 +[2025-04-23 03:56:10] (step=0068125) Train Loss: 6.3353, Train Steps/Sec: 0.85 +[2025-04-23 03:56:40] (step=0068150) Train Loss: 6.3502, Train Steps/Sec: 0.84 +[2025-04-23 03:57:10] (step=0068175) Train Loss: 6.3670, Train Steps/Sec: 0.85 +[2025-04-23 03:57:39] (step=0068200) Train Loss: 6.3444, Train Steps/Sec: 0.85 +[2025-04-23 03:58:09] (step=0068225) Train Loss: 6.3998, Train Steps/Sec: 0.85 +[2025-04-23 03:58:38] (step=0068250) Train Loss: 6.3752, Train Steps/Sec: 0.84 +[2025-04-23 03:59:08] (step=0068275) Train Loss: 6.3178, Train Steps/Sec: 0.85 +[2025-04-23 03:59:38] (step=0068300) Train Loss: 6.3597, Train Steps/Sec: 0.84 +[2025-04-23 04:00:07] (step=0068325) Train Loss: 6.3677, Train Steps/Sec: 0.85 +[2025-04-23 04:00:37] (step=0068350) Train Loss: 6.3767, Train Steps/Sec: 0.84 +[2025-04-23 04:01:06] (step=0068375) Train Loss: 6.3317, Train Steps/Sec: 0.84 +[2025-04-23 04:01:36] (step=0068400) Train Loss: 6.3709, Train Steps/Sec: 0.84 +[2025-04-23 04:02:06] (step=0068425) Train Loss: 6.3203, Train Steps/Sec: 0.83 +[2025-04-23 04:02:35] (step=0068450) Train Loss: 6.3157, Train Steps/Sec: 0.85 +[2025-04-23 04:03:05] (step=0068475) Train Loss: 6.3468, Train Steps/Sec: 0.84 +[2025-04-23 04:03:35] (step=0068500) Train Loss: 6.2917, Train Steps/Sec: 0.84 +[2025-04-23 04:04:04] (step=0068525) Train Loss: 6.3486, Train Steps/Sec: 0.84 +[2025-04-23 04:04:34] (step=0068550) Train Loss: 6.3864, Train Steps/Sec: 0.84 +[2025-04-23 04:05:04] (step=0068575) Train Loss: 6.3577, Train Steps/Sec: 0.85 +[2025-04-23 04:05:33] (step=0068600) Train Loss: 6.3270, Train Steps/Sec: 0.85 +[2025-04-23 04:06:03] (step=0068625) Train Loss: 6.3440, Train Steps/Sec: 0.85 +[2025-04-23 04:06:32] (step=0068650) Train Loss: 6.3424, Train Steps/Sec: 0.84 +[2025-04-23 04:07:02] (step=0068675) Train Loss: 6.3229, Train Steps/Sec: 0.85 +[2025-04-23 04:07:31] (step=0068700) Train Loss: 6.3194, Train Steps/Sec: 0.84 +[2025-04-23 04:08:01] (step=0068725) Train Loss: 6.3478, Train Steps/Sec: 0.84 +[2025-04-23 04:08:31] (step=0068750) Train Loss: 6.3208, Train Steps/Sec: 0.84 +[2025-04-23 04:09:00] (step=0068775) Train Loss: 6.3846, Train Steps/Sec: 0.84 +[2025-04-23 04:09:30] (step=0068800) Train Loss: 6.3346, Train Steps/Sec: 0.84 +[2025-04-23 04:10:00] (step=0068825) Train Loss: 6.3480, Train Steps/Sec: 0.84 +[2025-04-23 04:10:29] (step=0068850) Train Loss: 6.3485, Train Steps/Sec: 0.85 +[2025-04-23 04:10:59] (step=0068875) Train Loss: 6.3587, Train Steps/Sec: 0.85 +[2025-04-23 04:11:28] (step=0068900) Train Loss: 6.3659, Train Steps/Sec: 0.84 +[2025-04-23 04:11:58] (step=0068925) Train Loss: 6.3085, Train Steps/Sec: 0.84 +[2025-04-23 04:12:28] (step=0068950) Train Loss: 6.3548, Train Steps/Sec: 0.85 +[2025-04-23 04:12:57] (step=0068975) Train Loss: 6.3191, Train Steps/Sec: 0.84 +[2025-04-23 04:13:27] (step=0069000) Train Loss: 6.3589, Train Steps/Sec: 0.84 +[2025-04-23 04:14:03] (step=0069025) Train Loss: 6.3698, Train Steps/Sec: 0.70 +[2025-04-23 04:14:32] (step=0069050) Train Loss: 6.3421, Train Steps/Sec: 0.85 +[2025-04-23 04:15:02] (step=0069075) Train Loss: 6.3555, Train Steps/Sec: 0.84 +[2025-04-23 04:15:32] (step=0069100) Train Loss: 6.3277, Train Steps/Sec: 0.84 +[2025-04-23 04:16:01] (step=0069125) Train Loss: 6.3516, Train Steps/Sec: 0.85 +[2025-04-23 04:16:31] (step=0069150) Train Loss: 6.3877, Train Steps/Sec: 0.85 +[2025-04-23 04:17:00] (step=0069175) Train Loss: 6.3725, Train Steps/Sec: 0.85 +[2025-04-23 04:17:30] (step=0069200) Train Loss: 6.3530, Train Steps/Sec: 0.84 +[2025-04-23 04:18:00] (step=0069225) Train Loss: 6.2957, Train Steps/Sec: 0.85 +[2025-04-23 04:18:29] (step=0069250) Train Loss: 6.3530, Train Steps/Sec: 0.84 +[2025-04-23 04:18:59] (step=0069275) Train Loss: 6.3147, Train Steps/Sec: 0.85 +[2025-04-23 04:19:29] (step=0069300) Train Loss: 6.3508, Train Steps/Sec: 0.84 +[2025-04-23 04:19:58] (step=0069325) Train Loss: 6.3524, Train Steps/Sec: 0.85 +[2025-04-23 04:20:28] (step=0069350) Train Loss: 6.3527, Train Steps/Sec: 0.84 +[2025-04-23 04:20:57] (step=0069375) Train Loss: 6.3539, Train Steps/Sec: 0.84 +[2025-04-23 04:21:27] (step=0069400) Train Loss: 6.3681, Train Steps/Sec: 0.85 +[2025-04-23 04:21:56] (step=0069425) Train Loss: 6.3468, Train Steps/Sec: 0.85 +[2025-04-23 04:22:26] (step=0069450) Train Loss: 6.3334, Train Steps/Sec: 0.84 +[2025-04-23 04:22:56] (step=0069475) Train Loss: 6.3455, Train Steps/Sec: 0.84 +[2025-04-23 04:23:33] (step=0069500) Train Loss: 6.3640, Train Steps/Sec: 0.67 +[2025-04-23 04:24:03] (step=0069525) Train Loss: 6.3660, Train Steps/Sec: 0.84 +[2025-04-23 04:24:32] (step=0069550) Train Loss: 6.3431, Train Steps/Sec: 0.85 +[2025-04-23 04:25:02] (step=0069575) Train Loss: 6.3408, Train Steps/Sec: 0.84 +[2025-04-23 04:25:37] (step=0069600) Train Loss: 6.3426, Train Steps/Sec: 0.71 +[2025-04-23 04:26:07] (step=0069625) Train Loss: 6.2847, Train Steps/Sec: 0.85 +[2025-04-23 04:26:36] (step=0069650) Train Loss: 6.3360, Train Steps/Sec: 0.84 +[2025-04-23 04:27:12] (step=0069675) Train Loss: 6.3424, Train Steps/Sec: 0.69 +[2025-04-23 04:27:42] (step=0069700) Train Loss: 6.3369, Train Steps/Sec: 0.84 +[2025-04-23 04:28:12] (step=0069725) Train Loss: 6.2735, Train Steps/Sec: 0.85 +[2025-04-23 04:28:41] (step=0069750) Train Loss: 6.3332, Train Steps/Sec: 0.85 +[2025-04-23 04:29:11] (step=0069775) Train Loss: 6.3216, Train Steps/Sec: 0.85 +[2025-04-23 04:29:40] (step=0069800) Train Loss: 6.3615, Train Steps/Sec: 0.84 +[2025-04-23 04:30:10] (step=0069825) Train Loss: 6.3897, Train Steps/Sec: 0.85 +[2025-04-23 04:30:39] (step=0069850) Train Loss: 6.3381, Train Steps/Sec: 0.85 +[2025-04-23 04:31:09] (step=0069875) Train Loss: 6.3087, Train Steps/Sec: 0.85 +[2025-04-23 04:31:38] (step=0069900) Train Loss: 6.3758, Train Steps/Sec: 0.85 +[2025-04-23 04:32:08] (step=0069925) Train Loss: 6.3304, Train Steps/Sec: 0.84 +[2025-04-23 04:32:38] (step=0069950) Train Loss: 6.3813, Train Steps/Sec: 0.84 +[2025-04-23 04:33:07] (step=0069975) Train Loss: 6.3422, Train Steps/Sec: 0.85 +[2025-04-23 04:33:37] (step=0070000) Train Loss: 6.3426, Train Steps/Sec: 0.84 +[2025-04-23 04:38:21] Finish Eval in 70000 steps... +[2025-04-23 04:38:43] Saved checkpoint to checkpoint/BeforeFix_Blip_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_objects/002-GPT-XL/checkpoints/0070000.pt +[2025-04-23 04:38:45] Removed old checkpoint: checkpoint/BeforeFix_Blip_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_objects/002-GPT-XL/checkpoints/0066000.pt +[2025-04-23 04:39:14] (step=0070025) Train Loss: 6.3465, Train Steps/Sec: 0.07 +[2025-04-23 04:39:44] (step=0070050) Train Loss: 6.3475, Train Steps/Sec: 0.85 +[2025-04-23 04:40:20] (step=0070075) Train Loss: 6.3551, Train Steps/Sec: 0.69 +[2025-04-23 04:40:50] (step=0070100) Train Loss: 6.3826, Train Steps/Sec: 0.84 +[2025-04-23 04:41:20] (step=0070125) Train Loss: 6.3390, Train Steps/Sec: 0.84 +[2025-04-23 04:41:50] (step=0070150) Train Loss: 6.3039, Train Steps/Sec: 0.84 +[2025-04-23 04:42:19] (step=0070175) Train Loss: 6.3818, Train Steps/Sec: 0.85 +[2025-04-23 04:42:49] (step=0070200) Train Loss: 6.2835, Train Steps/Sec: 0.84 +[2025-04-23 04:43:19] (step=0070225) Train Loss: 6.3692, Train Steps/Sec: 0.84 +[2025-04-23 04:43:48] (step=0070250) Train Loss: 6.3930, Train Steps/Sec: 0.84 +[2025-04-23 04:44:18] (step=0070275) Train Loss: 6.3942, Train Steps/Sec: 0.83 +[2025-04-23 04:44:48] (step=0070300) Train Loss: 6.3494, Train Steps/Sec: 0.84 +[2025-04-23 04:45:18] (step=0070325) Train Loss: 6.3470, Train Steps/Sec: 0.84 +[2025-04-23 04:45:47] (step=0070350) Train Loss: 6.3546, Train Steps/Sec: 0.85 +[2025-04-23 04:46:17] (step=0070375) Train Loss: 6.3257, Train Steps/Sec: 0.85 +[2025-04-23 04:46:46] (step=0070400) Train Loss: 6.3703, Train Steps/Sec: 0.84 +[2025-04-23 04:47:16] (step=0070425) Train Loss: 6.3362, Train Steps/Sec: 0.85 +[2025-04-23 04:47:46] (step=0070450) Train Loss: 6.3916, Train Steps/Sec: 0.84 +[2025-04-23 04:48:15] (step=0070475) Train Loss: 6.3729, Train Steps/Sec: 0.84 +[2025-04-23 04:48:45] (step=0070500) Train Loss: 6.3430, Train Steps/Sec: 0.85 +[2025-04-23 04:49:14] (step=0070525) Train Loss: 6.3696, Train Steps/Sec: 0.84 +[2025-04-23 04:49:44] (step=0070550) Train Loss: 6.3212, Train Steps/Sec: 0.85 +[2025-04-23 04:50:13] (step=0070575) Train Loss: 6.3366, Train Steps/Sec: 0.85 +[2025-04-23 04:50:49] (step=0070600) Train Loss: 6.3677, Train Steps/Sec: 0.70 +[2025-04-23 04:51:19] (step=0070625) Train Loss: 6.3892, Train Steps/Sec: 0.84 +[2025-04-23 04:51:48] (step=0070650) Train Loss: 6.3326, Train Steps/Sec: 0.85 +[2025-04-23 04:52:18] (step=0070675) Train Loss: 6.3681, Train Steps/Sec: 0.85 +[2025-04-23 04:52:54] (step=0070700) Train Loss: 6.3604, Train Steps/Sec: 0.68 +[2025-04-23 04:53:24] (step=0070725) Train Loss: 6.2997, Train Steps/Sec: 0.84 +[2025-04-23 04:53:53] (step=0070750) Train Loss: 6.3118, Train Steps/Sec: 0.85 +[2025-04-23 04:54:23] (step=0070775) Train Loss: 6.3532, Train Steps/Sec: 0.84 +[2025-04-23 04:54:53] (step=0070800) Train Loss: 6.3230, Train Steps/Sec: 0.85 +[2025-04-23 04:55:22] (step=0070825) Train Loss: 6.3404, Train Steps/Sec: 0.85 +[2025-04-23 04:55:51] (step=0070850) Train Loss: 6.3396, Train Steps/Sec: 0.85 +[2025-04-23 04:56:21] (step=0070875) Train Loss: 6.3589, Train Steps/Sec: 0.85 +[2025-04-23 04:56:51] (step=0070900) Train Loss: 6.3163, Train Steps/Sec: 0.84 +[2025-04-23 04:57:20] (step=0070925) Train Loss: 6.2975, Train Steps/Sec: 0.85 +[2025-04-23 04:57:50] (step=0070950) Train Loss: 6.3587, Train Steps/Sec: 0.85 +[2025-04-23 04:58:19] (step=0070975) Train Loss: 6.3797, Train Steps/Sec: 0.85 +[2025-04-23 04:58:49] (step=0071000) Train Loss: 6.3171, Train Steps/Sec: 0.84 +[2025-04-23 04:59:19] (step=0071025) Train Loss: 6.3808, Train Steps/Sec: 0.84 +[2025-04-23 04:59:48] (step=0071050) Train Loss: 6.3630, Train Steps/Sec: 0.85 +[2025-04-23 05:00:18] (step=0071075) Train Loss: 6.3412, Train Steps/Sec: 0.84 +[2025-04-23 05:00:48] (step=0071100) Train Loss: 6.3603, Train Steps/Sec: 0.84 +[2025-04-23 05:01:17] (step=0071125) Train Loss: 6.3729, Train Steps/Sec: 0.84 +[2025-04-23 05:01:47] (step=0071150) Train Loss: 6.3781, Train Steps/Sec: 0.84 +[2025-04-23 05:02:17] (step=0071175) Train Loss: 6.3162, Train Steps/Sec: 0.84 +[2025-04-23 05:02:47] (step=0071200) Train Loss: 6.3495, Train Steps/Sec: 0.84 +[2025-04-23 05:03:16] (step=0071225) Train Loss: 6.3507, Train Steps/Sec: 0.85 +[2025-04-23 05:03:46] (step=0071250) Train Loss: 6.3671, Train Steps/Sec: 0.84 +[2025-04-23 05:04:15] (step=0071275) Train Loss: 6.3732, Train Steps/Sec: 0.85 +[2025-04-23 05:04:45] (step=0071300) Train Loss: 6.4040, Train Steps/Sec: 0.84 +[2025-04-23 05:05:15] (step=0071325) Train Loss: 6.3138, Train Steps/Sec: 0.84 +[2025-04-23 05:05:45] (step=0071350) Train Loss: 6.3619, Train Steps/Sec: 0.84 +[2025-04-23 05:06:15] (step=0071375) Train Loss: 6.3423, Train Steps/Sec: 0.84 +[2025-04-23 05:06:44] (step=0071400) Train Loss: 6.3287, Train Steps/Sec: 0.84 +[2025-04-23 05:07:14] (step=0071425) Train Loss: 6.3940, Train Steps/Sec: 0.85 +[2025-04-23 05:07:43] (step=0071450) Train Loss: 6.3545, Train Steps/Sec: 0.85 +[2025-04-23 05:08:13] (step=0071475) Train Loss: 6.3348, Train Steps/Sec: 0.85 +[2025-04-23 05:08:42] (step=0071500) Train Loss: 6.3816, Train Steps/Sec: 0.84 +[2025-04-23 05:09:12] (step=0071525) Train Loss: 6.3782, Train Steps/Sec: 0.85 +[2025-04-23 05:09:41] (step=0071550) Train Loss: 6.3726, Train Steps/Sec: 0.85 +[2025-04-23 05:10:11] (step=0071575) Train Loss: 6.2966, Train Steps/Sec: 0.85 +[2025-04-23 05:10:41] (step=0071600) Train Loss: 6.3758, Train Steps/Sec: 0.84 +[2025-04-23 05:11:10] (step=0071625) Train Loss: 6.3487, Train Steps/Sec: 0.84 +[2025-04-23 05:11:40] (step=0071650) Train Loss: 6.3379, Train Steps/Sec: 0.85 +[2025-04-23 05:12:09] (step=0071675) Train Loss: 6.3350, Train Steps/Sec: 0.85 +[2025-04-23 05:12:39] (step=0071700) Train Loss: 6.3971, Train Steps/Sec: 0.84 +[2025-04-23 05:13:09] (step=0071725) Train Loss: 6.3351, Train Steps/Sec: 0.85 +[2025-04-23 05:13:38] (step=0071750) Train Loss: 6.3553, Train Steps/Sec: 0.85 +[2025-04-23 05:14:08] (step=0071775) Train Loss: 6.3214, Train Steps/Sec: 0.84 +[2025-04-23 05:14:37] (step=0071800) Train Loss: 6.3676, Train Steps/Sec: 0.84 +[2025-04-23 05:15:07] (step=0071825) Train Loss: 6.3562, Train Steps/Sec: 0.85 +[2025-04-23 05:15:36] (step=0071850) Train Loss: 6.3276, Train Steps/Sec: 0.84 +[2025-04-23 05:16:06] (step=0071875) Train Loss: 6.3389, Train Steps/Sec: 0.85 +[2025-04-23 05:16:35] (step=0071900) Train Loss: 6.3059, Train Steps/Sec: 0.84 +[2025-04-23 05:17:05] (step=0071925) Train Loss: 6.3134, Train Steps/Sec: 0.84 +[2025-04-23 05:17:35] (step=0071950) Train Loss: 6.3801, Train Steps/Sec: 0.85 +[2025-04-23 05:18:04] (step=0071975) Train Loss: 6.3715, Train Steps/Sec: 0.85 +[2025-04-23 05:18:34] (step=0072000) Train Loss: 6.3830, Train Steps/Sec: 0.84 +[2025-04-23 05:23:17] Finish Eval in 72000 steps... +[2025-04-23 05:23:39] Saved checkpoint to checkpoint/BeforeFix_Blip_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_objects/002-GPT-XL/checkpoints/0072000.pt +[2025-04-23 05:23:41] Removed old checkpoint: checkpoint/BeforeFix_Blip_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_objects/002-GPT-XL/checkpoints/0068000.pt +[2025-04-23 05:24:10] (step=0072025) Train Loss: 6.3182, Train Steps/Sec: 0.07 +[2025-04-23 05:24:40] (step=0072050) Train Loss: 6.3494, Train Steps/Sec: 0.85 +[2025-04-23 05:25:10] (step=0072075) Train Loss: 6.3617, Train Steps/Sec: 0.84 +[2025-04-23 05:25:40] (step=0072100) Train Loss: 6.3418, Train Steps/Sec: 0.83 +[2025-04-23 05:26:09] (step=0072125) Train Loss: 6.3159, Train Steps/Sec: 0.85 +[2025-04-23 05:26:39] (step=0072150) Train Loss: 6.3703, Train Steps/Sec: 0.85 +[2025-04-23 05:27:08] (step=0072175) Train Loss: 6.3450, Train Steps/Sec: 0.85 +[2025-04-23 05:27:38] (step=0072200) Train Loss: 6.3307, Train Steps/Sec: 0.85 +[2025-04-23 05:28:07] (step=0072225) Train Loss: 6.3357, Train Steps/Sec: 0.85 +[2025-04-23 05:28:37] (step=0072250) Train Loss: 6.3656, Train Steps/Sec: 0.85 +[2025-04-23 05:29:06] (step=0072275) Train Loss: 6.3411, Train Steps/Sec: 0.85 +[2025-04-23 05:29:36] (step=0072300) Train Loss: 6.3224, Train Steps/Sec: 0.84 +[2025-04-23 05:30:06] (step=0072325) Train Loss: 6.3501, Train Steps/Sec: 0.85 +[2025-04-23 05:30:35] (step=0072350) Train Loss: 6.3434, Train Steps/Sec: 0.85 +[2025-04-23 05:31:05] (step=0072375) Train Loss: 6.3803, Train Steps/Sec: 0.85 +[2025-04-23 05:31:34] (step=0072400) Train Loss: 6.3769, Train Steps/Sec: 0.84 +[2025-04-23 05:32:04] (step=0072425) Train Loss: 6.3336, Train Steps/Sec: 0.84 +[2025-04-23 05:32:34] (step=0072450) Train Loss: 6.3762, Train Steps/Sec: 0.85 +[2025-04-23 05:33:03] (step=0072475) Train Loss: 6.3761, Train Steps/Sec: 0.85 +[2025-04-23 05:33:33] (step=0072500) Train Loss: 6.3843, Train Steps/Sec: 0.85 +[2025-04-23 05:34:02] (step=0072525) Train Loss: 6.2855, Train Steps/Sec: 0.85 +[2025-04-23 05:34:32] (step=0072550) Train Loss: 6.3537, Train Steps/Sec: 0.84 +[2025-04-23 05:35:01] (step=0072575) Train Loss: 6.3624, Train Steps/Sec: 0.85 +[2025-04-23 05:35:31] (step=0072600) Train Loss: 6.3623, Train Steps/Sec: 0.85 +[2025-04-23 05:36:00] (step=0072625) Train Loss: 6.3507, Train Steps/Sec: 0.85 +[2025-04-23 05:36:30] (step=0072650) Train Loss: 6.3444, Train Steps/Sec: 0.84 +[2025-04-23 05:37:00] (step=0072675) Train Loss: 6.3028, Train Steps/Sec: 0.85 +[2025-04-23 05:37:29] (step=0072700) Train Loss: 6.3677, Train Steps/Sec: 0.85 +[2025-04-23 05:37:59] (step=0072725) Train Loss: 6.2953, Train Steps/Sec: 0.85 +[2025-04-23 05:38:28] (step=0072750) Train Loss: 6.3283, Train Steps/Sec: 0.84 +[2025-04-23 05:38:58] (step=0072775) Train Loss: 6.3087, Train Steps/Sec: 0.85 +[2025-04-23 05:39:28] (step=0072800) Train Loss: 6.3651, Train Steps/Sec: 0.84 +[2025-04-23 05:39:57] (step=0072825) Train Loss: 6.3535, Train Steps/Sec: 0.85 +[2025-04-23 05:40:27] (step=0072850) Train Loss: 6.3297, Train Steps/Sec: 0.85 +[2025-04-23 05:40:57] (step=0072875) Train Loss: 6.3485, Train Steps/Sec: 0.84 +[2025-04-23 05:41:26] (step=0072900) Train Loss: 6.3218, Train Steps/Sec: 0.84 +[2025-04-23 05:41:56] (step=0072925) Train Loss: 6.3673, Train Steps/Sec: 0.85 +[2025-04-23 05:42:25] (step=0072950) Train Loss: 6.3042, Train Steps/Sec: 0.85 +[2025-04-23 05:42:55] (step=0072975) Train Loss: 6.3133, Train Steps/Sec: 0.85 +[2025-04-23 05:43:24] (step=0073000) Train Loss: 6.3545, Train Steps/Sec: 0.84 +[2025-04-23 05:43:54] (step=0073025) Train Loss: 6.3624, Train Steps/Sec: 0.84 +[2025-04-23 05:44:24] (step=0073050) Train Loss: 6.3356, Train Steps/Sec: 0.85 +[2025-04-23 05:44:53] (step=0073075) Train Loss: 6.3983, Train Steps/Sec: 0.84 +[2025-04-23 05:45:23] (step=0073100) Train Loss: 6.3874, Train Steps/Sec: 0.84 +[2025-04-23 05:45:53] (step=0073125) Train Loss: 6.3402, Train Steps/Sec: 0.85 +[2025-04-23 05:46:22] (step=0073150) Train Loss: 6.2843, Train Steps/Sec: 0.85 +[2025-04-23 05:46:52] (step=0073175) Train Loss: 6.2901, Train Steps/Sec: 0.85 +[2025-04-23 05:47:21] (step=0073200) Train Loss: 6.3595, Train Steps/Sec: 0.84 +[2025-04-23 05:47:51] (step=0073225) Train Loss: 6.3140, Train Steps/Sec: 0.85 +[2025-04-23 05:48:20] (step=0073250) Train Loss: 6.3659, Train Steps/Sec: 0.85 +[2025-04-23 05:48:50] (step=0073275) Train Loss: 6.3957, Train Steps/Sec: 0.85 +[2025-04-23 05:49:19] (step=0073300) Train Loss: 6.3874, Train Steps/Sec: 0.84 +[2025-04-23 05:49:49] (step=0073325) Train Loss: 6.3419, Train Steps/Sec: 0.84 +[2025-04-23 05:50:19] (step=0073350) Train Loss: 6.3496, Train Steps/Sec: 0.84 +[2025-04-23 05:50:49] (step=0073375) Train Loss: 6.3650, Train Steps/Sec: 0.84 +[2025-04-23 05:51:18] (step=0073400) Train Loss: 6.3479, Train Steps/Sec: 0.84 +[2025-04-23 05:51:48] (step=0073425) Train Loss: 6.3033, Train Steps/Sec: 0.85 +[2025-04-23 05:52:17] (step=0073450) Train Loss: 6.2895, Train Steps/Sec: 0.85 +[2025-04-23 05:52:46] (step=0073475) Train Loss: 6.3464, Train Steps/Sec: 0.85 +[2025-04-23 05:53:16] (step=0073500) Train Loss: 6.3206, Train Steps/Sec: 0.84 +[2025-04-23 05:53:46] (step=0073525) Train Loss: 6.3357, Train Steps/Sec: 0.84 +[2025-04-23 05:54:16] (step=0073550) Train Loss: 6.3293, Train Steps/Sec: 0.84 +[2025-04-23 05:54:45] (step=0073575) Train Loss: 6.3375, Train Steps/Sec: 0.85 +[2025-04-23 05:55:15] (step=0073600) Train Loss: 6.3500, Train Steps/Sec: 0.84 +[2025-04-23 05:55:45] (step=0073625) Train Loss: 6.3740, Train Steps/Sec: 0.84 +[2025-04-23 05:56:15] (step=0073650) Train Loss: 6.3535, Train Steps/Sec: 0.84 +[2025-04-23 05:56:44] (step=0073675) Train Loss: 6.3706, Train Steps/Sec: 0.85 +[2025-04-23 05:57:14] (step=0073700) Train Loss: 6.3625, Train Steps/Sec: 0.84 +[2025-04-23 05:57:43] (step=0073725) Train Loss: 6.3679, Train Steps/Sec: 0.84 +[2025-04-23 05:58:13] (step=0073750) Train Loss: 6.3581, Train Steps/Sec: 0.85 +[2025-04-23 05:58:43] (step=0073775) Train Loss: 6.3789, Train Steps/Sec: 0.84 +[2025-04-23 05:59:12] (step=0073800) Train Loss: 6.3911, Train Steps/Sec: 0.84 +[2025-04-23 05:59:42] (step=0073825) Train Loss: 6.3711, Train Steps/Sec: 0.85 +[2025-04-23 06:00:11] (step=0073850) Train Loss: 6.3173, Train Steps/Sec: 0.85 +[2025-04-23 06:00:41] (step=0073875) Train Loss: 6.3534, Train Steps/Sec: 0.85 +[2025-04-23 06:01:11] (step=0073900) Train Loss: 6.3414, Train Steps/Sec: 0.84 +[2025-04-23 06:01:40] (step=0073925) Train Loss: 6.2894, Train Steps/Sec: 0.85 +[2025-04-23 06:02:10] (step=0073950) Train Loss: 6.3567, Train Steps/Sec: 0.84 +[2025-04-23 06:02:40] (step=0073975) Train Loss: 6.3291, Train Steps/Sec: 0.84 +[2025-04-23 06:03:09] (step=0074000) Train Loss: 6.3349, Train Steps/Sec: 0.84 +[2025-04-23 06:07:52] Finish Eval in 74000 steps... +[2025-04-23 06:08:13] Saved checkpoint to checkpoint/BeforeFix_Blip_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_objects/002-GPT-XL/checkpoints/0074000.pt +[2025-04-23 06:08:16] Removed old checkpoint: checkpoint/BeforeFix_Blip_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_objects/002-GPT-XL/checkpoints/0070000.pt +[2025-04-23 06:08:46] (step=0074025) Train Loss: 6.3452, Train Steps/Sec: 0.07 +[2025-04-23 06:09:16] (step=0074050) Train Loss: 6.3687, Train Steps/Sec: 0.84 +[2025-04-23 06:09:45] (step=0074075) Train Loss: 6.3676, Train Steps/Sec: 0.84 +[2025-04-23 06:10:15] (step=0074100) Train Loss: 6.3437, Train Steps/Sec: 0.84 +[2025-04-23 06:10:45] (step=0074125) Train Loss: 6.4079, Train Steps/Sec: 0.85 +[2025-04-23 06:11:14] (step=0074150) Train Loss: 6.3216, Train Steps/Sec: 0.85 +[2025-04-23 06:11:44] (step=0074175) Train Loss: 6.3633, Train Steps/Sec: 0.85 +[2025-04-23 06:12:13] (step=0074200) Train Loss: 6.3356, Train Steps/Sec: 0.84 +[2025-04-23 06:12:43] (step=0074225) Train Loss: 6.3070, Train Steps/Sec: 0.85 +[2025-04-23 06:13:12] (step=0074250) Train Loss: 6.3273, Train Steps/Sec: 0.84 +[2025-04-23 06:13:42] (step=0074275) Train Loss: 6.3341, Train Steps/Sec: 0.85 +[2025-04-23 06:14:11] (step=0074300) Train Loss: 6.3587, Train Steps/Sec: 0.85 +[2025-04-23 06:14:42] (step=0074325) Train Loss: 6.3354, Train Steps/Sec: 0.83 +[2025-04-23 06:15:11] (step=0074350) Train Loss: 6.3160, Train Steps/Sec: 0.85 +[2025-04-23 06:15:41] (step=0074375) Train Loss: 6.3403, Train Steps/Sec: 0.84 +[2025-04-23 06:16:10] (step=0074400) Train Loss: 6.3625, Train Steps/Sec: 0.84 +[2025-04-23 06:16:40] (step=0074425) Train Loss: 6.3563, Train Steps/Sec: 0.84 +[2025-04-23 06:17:16] (step=0074450) Train Loss: 6.3485, Train Steps/Sec: 0.69 +[2025-04-23 06:17:46] (step=0074475) Train Loss: 6.3460, Train Steps/Sec: 0.84 +[2025-04-23 06:18:16] (step=0074500) Train Loss: 6.3663, Train Steps/Sec: 0.84 +[2025-04-23 06:18:45] (step=0074525) Train Loss: 6.3606, Train Steps/Sec: 0.85 +[2025-04-23 06:19:15] (step=0074550) Train Loss: 6.3881, Train Steps/Sec: 0.84 +[2025-04-23 06:19:44] (step=0074575) Train Loss: 6.3630, Train Steps/Sec: 0.85 +[2025-04-23 06:20:14] (step=0074600) Train Loss: 6.3816, Train Steps/Sec: 0.84 +[2025-04-23 06:20:43] (step=0074625) Train Loss: 6.3447, Train Steps/Sec: 0.85 +[2025-04-23 06:21:13] (step=0074650) Train Loss: 6.3542, Train Steps/Sec: 0.84 +[2025-04-23 06:21:43] (step=0074675) Train Loss: 6.3629, Train Steps/Sec: 0.84 +[2025-04-23 06:22:19] (step=0074700) Train Loss: 6.3541, Train Steps/Sec: 0.70 +[2025-04-23 06:22:48] (step=0074725) Train Loss: 6.3391, Train Steps/Sec: 0.85 +[2025-04-23 06:23:18] (step=0074750) Train Loss: 6.3226, Train Steps/Sec: 0.84 +[2025-04-23 06:23:48] (step=0074775) Train Loss: 6.3538, Train Steps/Sec: 0.85 +[2025-04-23 06:24:17] (step=0074800) Train Loss: 6.3620, Train Steps/Sec: 0.84 +[2025-04-23 06:24:47] (step=0074825) Train Loss: 6.3569, Train Steps/Sec: 0.85 +[2025-04-23 06:25:16] (step=0074850) Train Loss: 6.3591, Train Steps/Sec: 0.85 +[2025-04-23 06:25:46] (step=0074875) Train Loss: 6.3874, Train Steps/Sec: 0.85 +[2025-04-23 06:26:15] (step=0074900) Train Loss: 6.3073, Train Steps/Sec: 0.84 +[2025-04-23 06:26:51] (step=0074925) Train Loss: 6.3462, Train Steps/Sec: 0.71 +[2025-04-23 06:27:20] (step=0074950) Train Loss: 6.3276, Train Steps/Sec: 0.84 +[2025-04-23 06:27:50] (step=0074975) Train Loss: 6.3238, Train Steps/Sec: 0.85 +[2025-04-23 06:28:27] (step=0075000) Train Loss: 6.3976, Train Steps/Sec: 0.67 +[2025-04-23 06:28:57] (step=0075025) Train Loss: 6.3376, Train Steps/Sec: 0.84 +[2025-04-23 06:29:26] (step=0075050) Train Loss: 6.3846, Train Steps/Sec: 0.85 +[2025-04-23 06:29:56] (step=0075075) Train Loss: 6.3756, Train Steps/Sec: 0.85 +[2025-04-23 06:30:25] (step=0075100) Train Loss: 6.3248, Train Steps/Sec: 0.84 +[2025-04-23 06:30:55] (step=0075125) Train Loss: 6.3368, Train Steps/Sec: 0.84 +[2025-04-23 06:31:25] (step=0075150) Train Loss: 6.3337, Train Steps/Sec: 0.85 +[2025-04-23 06:32:01] (step=0075175) Train Loss: 6.3733, Train Steps/Sec: 0.68 +[2025-04-23 06:32:31] (step=0075200) Train Loss: 6.3562, Train Steps/Sec: 0.84 +[2025-04-23 06:33:00] (step=0075225) Train Loss: 6.3279, Train Steps/Sec: 0.85 +[2025-04-23 06:33:30] (step=0075250) Train Loss: 6.3229, Train Steps/Sec: 0.85 +[2025-04-23 06:33:59] (step=0075275) Train Loss: 6.3146, Train Steps/Sec: 0.85 +[2025-04-23 06:34:29] (step=0075300) Train Loss: 6.3336, Train Steps/Sec: 0.85 +[2025-04-23 06:34:58] (step=0075325) Train Loss: 6.3629, Train Steps/Sec: 0.85 +[2025-04-23 06:35:28] (step=0075350) Train Loss: 6.3658, Train Steps/Sec: 0.84 +[2025-04-23 06:35:57] (step=0075375) Train Loss: 6.3848, Train Steps/Sec: 0.85 +[2025-04-23 06:36:27] (step=0075400) Train Loss: 6.3446, Train Steps/Sec: 0.84 +[2025-04-23 06:36:56] (step=0075425) Train Loss: 6.3783, Train Steps/Sec: 0.85 +[2025-04-23 06:37:26] (step=0075450) Train Loss: 6.3332, Train Steps/Sec: 0.85 +[2025-04-23 06:37:55] (step=0075475) Train Loss: 6.3575, Train Steps/Sec: 0.85 +[2025-04-23 06:38:25] (step=0075500) Train Loss: 6.3664, Train Steps/Sec: 0.85 +[2025-04-23 06:38:55] (step=0075525) Train Loss: 6.3626, Train Steps/Sec: 0.84 +[2025-04-23 06:39:24] (step=0075550) Train Loss: 6.3466, Train Steps/Sec: 0.84 +[2025-04-23 06:39:54] (step=0075575) Train Loss: 6.3398, Train Steps/Sec: 0.84 +[2025-04-23 06:40:24] (step=0075600) Train Loss: 6.3229, Train Steps/Sec: 0.84 +[2025-04-23 06:40:53] (step=0075625) Train Loss: 6.3517, Train Steps/Sec: 0.85 +[2025-04-23 06:41:23] (step=0075650) Train Loss: 6.3400, Train Steps/Sec: 0.85 +[2025-04-23 06:41:52] (step=0075675) Train Loss: 6.3695, Train Steps/Sec: 0.85 +[2025-04-23 06:42:28] (step=0075700) Train Loss: 6.3605, Train Steps/Sec: 0.71 +[2025-04-23 06:42:57] (step=0075725) Train Loss: 6.2944, Train Steps/Sec: 0.85 +[2025-04-23 06:43:27] (step=0075750) Train Loss: 6.3800, Train Steps/Sec: 0.84 +[2025-04-23 06:43:56] (step=0075775) Train Loss: 6.3421, Train Steps/Sec: 0.85 +[2025-04-23 06:44:26] (step=0075800) Train Loss: 6.3526, Train Steps/Sec: 0.84 +[2025-04-23 06:45:02] (step=0075825) Train Loss: 6.3566, Train Steps/Sec: 0.70 +[2025-04-23 06:45:31] (step=0075850) Train Loss: 6.3599, Train Steps/Sec: 0.84 +[2025-04-23 06:46:01] (step=0075875) Train Loss: 6.3399, Train Steps/Sec: 0.84 +[2025-04-23 06:46:31] (step=0075900) Train Loss: 6.3467, Train Steps/Sec: 0.84 +[2025-04-23 06:47:00] (step=0075925) Train Loss: 6.3376, Train Steps/Sec: 0.85 +[2025-04-23 06:47:30] (step=0075950) Train Loss: 6.3298, Train Steps/Sec: 0.84 +[2025-04-23 06:47:59] (step=0075975) Train Loss: 6.3420, Train Steps/Sec: 0.85 +[2025-04-23 06:48:29] (step=0076000) Train Loss: 6.3395, Train Steps/Sec: 0.84 +[2025-04-23 06:53:13] Finish Eval in 76000 steps... +[2025-04-23 06:53:34] Saved checkpoint to checkpoint/BeforeFix_Blip_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_objects/002-GPT-XL/checkpoints/0076000.pt +[2025-04-23 06:53:36] Removed old checkpoint: checkpoint/BeforeFix_Blip_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_objects/002-GPT-XL/checkpoints/0072000.pt +[2025-04-23 06:54:06] (step=0076025) Train Loss: 6.3710, Train Steps/Sec: 0.07 +[2025-04-23 06:54:35] (step=0076050) Train Loss: 6.3300, Train Steps/Sec: 0.85 +[2025-04-23 06:55:05] (step=0076075) Train Loss: 6.3622, Train Steps/Sec: 0.84 +[2025-04-23 06:55:35] (step=0076100) Train Loss: 6.3648, Train Steps/Sec: 0.84 +[2025-04-23 06:56:04] (step=0076125) Train Loss: 6.3580, Train Steps/Sec: 0.85 +[2025-04-23 06:56:34] (step=0076150) Train Loss: 6.3582, Train Steps/Sec: 0.85 +[2025-04-23 06:57:04] (step=0076175) Train Loss: 6.3651, Train Steps/Sec: 0.84 +[2025-04-23 06:57:33] (step=0076200) Train Loss: 6.3338, Train Steps/Sec: 0.84 +[2025-04-23 06:58:03] (step=0076225) Train Loss: 6.3642, Train Steps/Sec: 0.84 +[2025-04-23 06:58:32] (step=0076250) Train Loss: 6.3692, Train Steps/Sec: 0.85 +[2025-04-23 06:59:02] (step=0076275) Train Loss: 6.3740, Train Steps/Sec: 0.84 +[2025-04-23 06:59:32] (step=0076300) Train Loss: 6.3358, Train Steps/Sec: 0.84 +[2025-04-23 07:00:01] (step=0076325) Train Loss: 6.3583, Train Steps/Sec: 0.84 +[2025-04-23 07:00:31] (step=0076350) Train Loss: 6.3391, Train Steps/Sec: 0.85 +[2025-04-23 07:01:00] (step=0076375) Train Loss: 6.3396, Train Steps/Sec: 0.85 +[2025-04-23 07:01:30] (step=0076400) Train Loss: 6.3404, Train Steps/Sec: 0.84 +[2025-04-23 07:02:00] (step=0076425) Train Loss: 6.3365, Train Steps/Sec: 0.83 +[2025-04-23 07:02:30] (step=0076450) Train Loss: 6.3239, Train Steps/Sec: 0.84 +[2025-04-23 07:02:59] (step=0076475) Train Loss: 6.3530, Train Steps/Sec: 0.85 +[2025-04-23 07:03:29] (step=0076500) Train Loss: 6.3640, Train Steps/Sec: 0.84 +[2025-04-23 07:03:58] (step=0076525) Train Loss: 6.3210, Train Steps/Sec: 0.85 +[2025-04-23 07:04:28] (step=0076550) Train Loss: 6.3785, Train Steps/Sec: 0.84 +[2025-04-23 07:04:58] (step=0076575) Train Loss: 6.3596, Train Steps/Sec: 0.84 +[2025-04-23 07:05:27] (step=0076600) Train Loss: 6.3501, Train Steps/Sec: 0.84 +[2025-04-23 07:05:57] (step=0076625) Train Loss: 6.3432, Train Steps/Sec: 0.85 +[2025-04-23 07:06:26] (step=0076650) Train Loss: 6.3264, Train Steps/Sec: 0.85 +[2025-04-23 07:06:56] (step=0076675) Train Loss: 6.3597, Train Steps/Sec: 0.84 +[2025-04-23 07:07:25] (step=0076700) Train Loss: 6.3682, Train Steps/Sec: 0.84 +[2025-04-23 07:07:55] (step=0076725) Train Loss: 6.3179, Train Steps/Sec: 0.85 +[2025-04-23 07:08:25] (step=0076750) Train Loss: 6.3677, Train Steps/Sec: 0.85 +[2025-04-23 07:08:54] (step=0076775) Train Loss: 6.3427, Train Steps/Sec: 0.85 +[2025-04-23 07:09:24] (step=0076800) Train Loss: 6.3266, Train Steps/Sec: 0.84 +[2025-04-23 07:09:53] (step=0076825) Train Loss: 6.3784, Train Steps/Sec: 0.84 +[2025-04-23 07:10:23] (step=0076850) Train Loss: 6.3407, Train Steps/Sec: 0.84 +[2025-04-23 07:10:53] (step=0076875) Train Loss: 6.3178, Train Steps/Sec: 0.84 +[2025-04-23 07:11:23] (step=0076900) Train Loss: 6.3645, Train Steps/Sec: 0.84 +[2025-04-23 07:11:52] (step=0076925) Train Loss: 6.3645, Train Steps/Sec: 0.84 +[2025-04-23 07:12:22] (step=0076950) Train Loss: 6.3682, Train Steps/Sec: 0.84 +[2025-04-23 07:12:52] (step=0076975) Train Loss: 6.3090, Train Steps/Sec: 0.84 +[2025-04-23 07:13:21] (step=0077000) Train Loss: 6.3298, Train Steps/Sec: 0.85 +[2025-04-23 07:13:51] (step=0077025) Train Loss: 6.3303, Train Steps/Sec: 0.85 +[2025-04-23 07:14:20] (step=0077050) Train Loss: 6.3166, Train Steps/Sec: 0.85 +[2025-04-23 07:14:50] (step=0077075) Train Loss: 6.3358, Train Steps/Sec: 0.84 +[2025-04-23 07:15:20] (step=0077100) Train Loss: 6.3414, Train Steps/Sec: 0.84 +[2025-04-23 07:15:49] (step=0077125) Train Loss: 6.3573, Train Steps/Sec: 0.85 +[2025-04-23 07:16:19] (step=0077150) Train Loss: 6.3979, Train Steps/Sec: 0.85 +[2025-04-23 07:16:48] (step=0077175) Train Loss: 6.3928, Train Steps/Sec: 0.85 +[2025-04-23 07:17:18] (step=0077200) Train Loss: 6.3438, Train Steps/Sec: 0.85 +[2025-04-23 07:17:47] (step=0077225) Train Loss: 6.3227, Train Steps/Sec: 0.85 +[2025-04-23 07:18:17] (step=0077250) Train Loss: 6.3356, Train Steps/Sec: 0.84 +[2025-04-23 07:18:46] (step=0077275) Train Loss: 6.3373, Train Steps/Sec: 0.85 +[2025-04-23 07:19:16] (step=0077300) Train Loss: 6.3692, Train Steps/Sec: 0.85 +[2025-04-23 07:19:45] (step=0077325) Train Loss: 6.3664, Train Steps/Sec: 0.84 +[2025-04-23 07:20:15] (step=0077350) Train Loss: 6.3117, Train Steps/Sec: 0.84 +[2025-04-23 07:20:45] (step=0077375) Train Loss: 6.3560, Train Steps/Sec: 0.84 +[2025-04-23 07:21:14] (step=0077400) Train Loss: 6.3381, Train Steps/Sec: 0.84 +[2025-04-23 07:21:44] (step=0077425) Train Loss: 6.3556, Train Steps/Sec: 0.84 +[2025-04-23 07:22:14] (step=0077450) Train Loss: 6.3438, Train Steps/Sec: 0.84 +[2025-04-23 07:22:43] (step=0077475) Train Loss: 6.3435, Train Steps/Sec: 0.85 +[2025-04-23 07:23:13] (step=0077500) Train Loss: 6.3177, Train Steps/Sec: 0.84 +[2025-04-23 07:23:42] (step=0077525) Train Loss: 6.3505, Train Steps/Sec: 0.85 +[2025-04-23 07:24:12] (step=0077550) Train Loss: 6.3179, Train Steps/Sec: 0.84 +[2025-04-23 07:24:42] (step=0077575) Train Loss: 6.3607, Train Steps/Sec: 0.85 +[2025-04-23 07:25:11] (step=0077600) Train Loss: 6.3339, Train Steps/Sec: 0.85 +[2025-04-23 07:25:41] (step=0077625) Train Loss: 6.3192, Train Steps/Sec: 0.84 +[2025-04-23 07:26:10] (step=0077650) Train Loss: 6.3599, Train Steps/Sec: 0.84 +[2025-04-23 07:26:40] (step=0077675) Train Loss: 6.3239, Train Steps/Sec: 0.85 +[2025-04-23 07:27:10] (step=0077700) Train Loss: 6.3361, Train Steps/Sec: 0.84 +[2025-04-23 07:27:39] (step=0077725) Train Loss: 6.3436, Train Steps/Sec: 0.85 +[2025-04-23 07:28:09] (step=0077750) Train Loss: 6.3777, Train Steps/Sec: 0.84 +[2025-04-23 07:28:38] (step=0077775) Train Loss: 6.3170, Train Steps/Sec: 0.85 +[2025-04-23 07:29:08] (step=0077800) Train Loss: 6.3717, Train Steps/Sec: 0.85 +[2025-04-23 07:29:37] (step=0077825) Train Loss: 6.3351, Train Steps/Sec: 0.84 +[2025-04-23 07:30:07] (step=0077850) Train Loss: 6.3691, Train Steps/Sec: 0.85 +[2025-04-23 07:30:36] (step=0077875) Train Loss: 6.3989, Train Steps/Sec: 0.85 +[2025-04-23 07:31:06] (step=0077900) Train Loss: 6.3504, Train Steps/Sec: 0.84 +[2025-04-23 07:31:36] (step=0077925) Train Loss: 6.3630, Train Steps/Sec: 0.84 +[2025-04-23 07:32:05] (step=0077950) Train Loss: 6.2973, Train Steps/Sec: 0.84 +[2025-04-23 07:32:35] (step=0077975) Train Loss: 6.3474, Train Steps/Sec: 0.84 +[2025-04-23 07:33:05] (step=0078000) Train Loss: 6.2938, Train Steps/Sec: 0.84 +[2025-04-23 07:37:48] Finish Eval in 78000 steps... +[2025-04-23 07:38:11] Saved checkpoint to checkpoint/BeforeFix_Blip_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_objects/002-GPT-XL/checkpoints/0078000.pt +[2025-04-23 07:38:13] Removed old checkpoint: checkpoint/BeforeFix_Blip_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_objects/002-GPT-XL/checkpoints/0074000.pt +[2025-04-23 07:38:42] (step=0078025) Train Loss: 6.3359, Train Steps/Sec: 0.07 +[2025-04-23 07:39:12] (step=0078050) Train Loss: 6.3523, Train Steps/Sec: 0.85 +[2025-04-23 07:39:41] (step=0078075) Train Loss: 6.4101, Train Steps/Sec: 0.84 +[2025-04-23 07:40:11] (step=0078100) Train Loss: 6.3501, Train Steps/Sec: 0.84 +[2025-04-23 07:40:41] (step=0078125) Train Loss: 6.3519, Train Steps/Sec: 0.84 +[2025-04-23 07:41:10] (step=0078150) Train Loss: 6.3507, Train Steps/Sec: 0.85 +[2025-04-23 07:41:40] (step=0078175) Train Loss: 6.3646, Train Steps/Sec: 0.84 +[2025-04-23 07:42:09] (step=0078200) Train Loss: 6.3302, Train Steps/Sec: 0.85 +[2025-04-23 07:42:39] (step=0078225) Train Loss: 6.3469, Train Steps/Sec: 0.85 +[2025-04-23 07:43:08] (step=0078250) Train Loss: 6.3299, Train Steps/Sec: 0.85 +[2025-04-23 07:43:38] (step=0078275) Train Loss: 6.3728, Train Steps/Sec: 0.85 +[2025-04-23 07:44:08] (step=0078300) Train Loss: 6.3820, Train Steps/Sec: 0.84 +[2025-04-23 07:44:37] (step=0078325) Train Loss: 6.3756, Train Steps/Sec: 0.85 +[2025-04-23 07:45:07] (step=0078350) Train Loss: 6.3242, Train Steps/Sec: 0.84 +[2025-04-23 07:45:36] (step=0078375) Train Loss: 6.3703, Train Steps/Sec: 0.85 +[2025-04-23 07:46:06] (step=0078400) Train Loss: 6.4080, Train Steps/Sec: 0.84 +[2025-04-23 07:46:36] (step=0078425) Train Loss: 6.3824, Train Steps/Sec: 0.85 +[2025-04-23 07:47:05] (step=0078450) Train Loss: 6.3740, Train Steps/Sec: 0.84 +[2025-04-23 07:47:35] (step=0078475) Train Loss: 6.3671, Train Steps/Sec: 0.85 +[2025-04-23 07:48:05] (step=0078500) Train Loss: 6.4051, Train Steps/Sec: 0.84 +[2025-04-23 07:48:34] (step=0078525) Train Loss: 6.3586, Train Steps/Sec: 0.85 +[2025-04-23 07:49:04] (step=0078550) Train Loss: 6.3614, Train Steps/Sec: 0.84 +[2025-04-23 07:49:33] (step=0078575) Train Loss: 6.3361, Train Steps/Sec: 0.85 +[2025-04-23 07:50:03] (step=0078600) Train Loss: 6.3664, Train Steps/Sec: 0.84 +[2025-04-23 07:50:32] (step=0078625) Train Loss: 6.3641, Train Steps/Sec: 0.85 +[2025-04-23 07:51:02] (step=0078650) Train Loss: 6.3663, Train Steps/Sec: 0.85 +[2025-04-23 07:51:31] (step=0078675) Train Loss: 6.3432, Train Steps/Sec: 0.84 +[2025-04-23 07:52:01] (step=0078700) Train Loss: 6.3370, Train Steps/Sec: 0.85 +[2025-04-23 07:52:30] (step=0078725) Train Loss: 6.3041, Train Steps/Sec: 0.85 +[2025-04-23 07:53:00] (step=0078750) Train Loss: 6.3254, Train Steps/Sec: 0.85 +[2025-04-23 07:53:29] (step=0078775) Train Loss: 6.3348, Train Steps/Sec: 0.85 +[2025-04-23 07:53:59] (step=0078800) Train Loss: 6.3398, Train Steps/Sec: 0.84 +[2025-04-23 07:54:29] (step=0078825) Train Loss: 6.3608, Train Steps/Sec: 0.85 +[2025-04-23 07:54:58] (step=0078850) Train Loss: 6.3802, Train Steps/Sec: 0.85 +[2025-04-23 07:55:27] (step=0078875) Train Loss: 6.3466, Train Steps/Sec: 0.85 +[2025-04-23 07:55:57] (step=0078900) Train Loss: 6.3609, Train Steps/Sec: 0.84 +[2025-04-23 07:56:27] (step=0078925) Train Loss: 6.3449, Train Steps/Sec: 0.85 +[2025-04-23 07:56:56] (step=0078950) Train Loss: 6.3370, Train Steps/Sec: 0.85 +[2025-04-23 07:57:26] (step=0078975) Train Loss: 6.3076, Train Steps/Sec: 0.85 +[2025-04-23 07:57:55] (step=0079000) Train Loss: 6.3324, Train Steps/Sec: 0.84 +[2025-04-23 07:58:25] (step=0079025) Train Loss: 6.2824, Train Steps/Sec: 0.85 +[2025-04-23 07:58:55] (step=0079050) Train Loss: 6.3335, Train Steps/Sec: 0.84 +[2025-04-23 07:59:24] (step=0079075) Train Loss: 6.3374, Train Steps/Sec: 0.85 +[2025-04-23 07:59:54] (step=0079100) Train Loss: 6.3786, Train Steps/Sec: 0.85 +[2025-04-23 08:00:23] (step=0079125) Train Loss: 6.3115, Train Steps/Sec: 0.85 +[2025-04-23 08:00:53] (step=0079150) Train Loss: 6.3658, Train Steps/Sec: 0.85 +[2025-04-23 08:01:22] (step=0079175) Train Loss: 6.3507, Train Steps/Sec: 0.85 +[2025-04-23 08:01:52] (step=0079200) Train Loss: 6.3581, Train Steps/Sec: 0.84 +[2025-04-23 08:02:21] (step=0079225) Train Loss: 6.3107, Train Steps/Sec: 0.85 +[2025-04-23 08:02:51] (step=0079250) Train Loss: 6.3426, Train Steps/Sec: 0.85 +[2025-04-23 08:03:21] (step=0079275) Train Loss: 6.3595, Train Steps/Sec: 0.84 +[2025-04-23 08:03:50] (step=0079300) Train Loss: 6.3722, Train Steps/Sec: 0.85 +[2025-04-23 08:04:20] (step=0079325) Train Loss: 6.3736, Train Steps/Sec: 0.85 +[2025-04-23 08:04:49] (step=0079350) Train Loss: 6.3806, Train Steps/Sec: 0.85 +[2025-04-23 08:05:19] (step=0079375) Train Loss: 6.3198, Train Steps/Sec: 0.85 +[2025-04-23 08:05:48] (step=0079400) Train Loss: 6.3414, Train Steps/Sec: 0.84 +[2025-04-23 08:06:18] (step=0079425) Train Loss: 6.3912, Train Steps/Sec: 0.84 +[2025-04-23 08:06:48] (step=0079450) Train Loss: 6.4017, Train Steps/Sec: 0.85 +[2025-04-23 08:07:17] (step=0079475) Train Loss: 6.3202, Train Steps/Sec: 0.85 +[2025-04-23 08:07:47] (step=0079500) Train Loss: 6.3548, Train Steps/Sec: 0.85 +[2025-04-23 08:08:16] (step=0079525) Train Loss: 6.3089, Train Steps/Sec: 0.85 +[2025-04-23 08:08:46] (step=0079550) Train Loss: 6.3707, Train Steps/Sec: 0.85 +[2025-04-23 08:09:15] (step=0079575) Train Loss: 6.3111, Train Steps/Sec: 0.84 +[2025-04-23 08:09:45] (step=0079600) Train Loss: 6.3688, Train Steps/Sec: 0.84 +[2025-04-23 08:10:15] (step=0079625) Train Loss: 6.3791, Train Steps/Sec: 0.84 +[2025-04-23 08:10:44] (step=0079650) Train Loss: 6.3147, Train Steps/Sec: 0.85 +[2025-04-23 08:11:14] (step=0079675) Train Loss: 6.3600, Train Steps/Sec: 0.84 +[2025-04-23 08:11:43] (step=0079700) Train Loss: 6.3574, Train Steps/Sec: 0.84 +[2025-04-23 08:12:13] (step=0079725) Train Loss: 6.3236, Train Steps/Sec: 0.85 +[2025-04-23 08:12:49] (step=0079750) Train Loss: 6.3563, Train Steps/Sec: 0.69 +[2025-04-23 08:13:18] (step=0079775) Train Loss: 6.3497, Train Steps/Sec: 0.85 +[2025-04-23 08:13:48] (step=0079800) Train Loss: 6.3228, Train Steps/Sec: 0.84 +[2025-04-23 08:14:18] (step=0079825) Train Loss: 6.3464, Train Steps/Sec: 0.85 +[2025-04-23 08:14:47] (step=0079850) Train Loss: 6.3154, Train Steps/Sec: 0.85 +[2025-04-23 08:15:17] (step=0079875) Train Loss: 6.3322, Train Steps/Sec: 0.85 +[2025-04-23 08:15:46] (step=0079900) Train Loss: 6.3097, Train Steps/Sec: 0.84 +[2025-04-23 08:16:22] (step=0079925) Train Loss: 6.3435, Train Steps/Sec: 0.70 +[2025-04-23 08:16:52] (step=0079950) Train Loss: 6.3956, Train Steps/Sec: 0.85 +[2025-04-23 08:17:21] (step=0079975) Train Loss: 6.3386, Train Steps/Sec: 0.85 +[2025-04-23 08:17:51] (step=0080000) Train Loss: 6.3754, Train Steps/Sec: 0.85 +[2025-04-23 08:22:32] Finish Eval in 80000 steps... +[2025-04-23 08:22:54] Saved checkpoint to checkpoint/BeforeFix_Blip_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_objects/002-GPT-XL/checkpoints/0080000.pt +[2025-04-23 08:22:56] Removed old checkpoint: checkpoint/BeforeFix_Blip_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_objects/002-GPT-XL/checkpoints/0076000.pt +[2025-04-23 08:23:26] (step=0080025) Train Loss: 6.3246, Train Steps/Sec: 0.07 +[2025-04-23 08:23:56] (step=0080050) Train Loss: 6.3600, Train Steps/Sec: 0.85 +[2025-04-23 08:24:26] (step=0080075) Train Loss: 6.3799, Train Steps/Sec: 0.83 +[2025-04-23 08:24:55] (step=0080100) Train Loss: 6.3017, Train Steps/Sec: 0.85 +[2025-04-23 08:25:25] (step=0080125) Train Loss: 6.3257, Train Steps/Sec: 0.84 +[2025-04-23 08:25:54] (step=0080150) Train Loss: 6.3831, Train Steps/Sec: 0.85 +[2025-04-23 08:26:24] (step=0080175) Train Loss: 6.3443, Train Steps/Sec: 0.85 +[2025-04-23 08:26:53] (step=0080200) Train Loss: 6.3237, Train Steps/Sec: 0.84 +[2025-04-23 08:27:23] (step=0080225) Train Loss: 6.4110, Train Steps/Sec: 0.84 +[2025-04-23 08:27:53] (step=0080250) Train Loss: 6.3286, Train Steps/Sec: 0.84 +[2025-04-23 08:28:35] (step=0080275) Train Loss: 6.3791, Train Steps/Sec: 0.59 +[2025-04-23 08:29:05] (step=0080300) Train Loss: 6.3523, Train Steps/Sec: 0.84 +[2025-04-23 08:29:34] (step=0080325) Train Loss: 6.3277, Train Steps/Sec: 0.85 +[2025-04-23 08:30:03] (step=0080350) Train Loss: 6.3378, Train Steps/Sec: 0.85 +[2025-04-23 08:30:33] (step=0080375) Train Loss: 6.3302, Train Steps/Sec: 0.85 +[2025-04-23 08:31:03] (step=0080400) Train Loss: 6.3459, Train Steps/Sec: 0.84 +[2025-04-23 08:31:32] (step=0080425) Train Loss: 6.3570, Train Steps/Sec: 0.85 +[2025-04-23 08:32:01] (step=0080450) Train Loss: 6.3320, Train Steps/Sec: 0.85 diff --git a/CKPTS/BeforeFix_Blip_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_objects/wandb/debug-internal.log b/CKPTS/BeforeFix_Blip_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_objects/wandb/debug-internal.log new file mode 100644 index 0000000000000000000000000000000000000000..eb9cc17862e5b3621c9639b15dd2c9d774e83cc0 --- /dev/null +++ b/CKPTS/BeforeFix_Blip_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_objects/wandb/debug-internal.log @@ -0,0 +1,9 @@ +{"time":"2025-04-22T01:47:29.361870194Z","level":"INFO","msg":"stream: starting","core version":"0.19.8","symlink path":"checkpoint/BeforeFix_Blip_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_objects/wandb/run-20250422_014729-ni4jp6ul/logs/debug-core.log"} +{"time":"2025-04-22T01:47:29.467936381Z","level":"INFO","msg":"created new stream","id":"ni4jp6ul"} +{"time":"2025-04-22T01:47:29.467981215Z","level":"INFO","msg":"stream: started","id":"ni4jp6ul"} +{"time":"2025-04-22T01:47:29.46801618Z","level":"INFO","msg":"writer: Do: started","stream_id":"ni4jp6ul"} +{"time":"2025-04-22T01:47:29.468021706Z","level":"INFO","msg":"sender: started","stream_id":"ni4jp6ul"} +{"time":"2025-04-22T01:47:29.468072912Z","level":"INFO","msg":"handler: started","stream_id":"ni4jp6ul"} +{"time":"2025-04-22T01:47:29.833858878Z","level":"INFO","msg":"Starting system monitor"} +{"time":"2025-04-22T18:15:28.577913461Z","level":"INFO","msg":"api: retrying error","error":"Post \"https://api.wandb.ai/graphql\": net/http: request canceled (Client.Timeout exceeded while awaiting headers)"} +{"time":"2025-04-22T21:04:58.809243773Z","level":"INFO","msg":"api: retrying error","error":"Post \"https://api.wandb.ai/graphql\": context deadline exceeded (Client.Timeout exceeded while awaiting headers)"} diff --git a/CKPTS/BeforeFix_Blip_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_objects/wandb/debug.log b/CKPTS/BeforeFix_Blip_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_objects/wandb/debug.log new file mode 100644 index 0000000000000000000000000000000000000000..528f8c3ee609095481f5914449e01f2ac296d6e6 --- /dev/null +++ b/CKPTS/BeforeFix_Blip_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_objects/wandb/debug.log @@ -0,0 +1,22 @@ +2025-04-22 01:47:29,354 INFO MainThread:2951284 [wandb_setup.py:_flush():67] Current SDK version is 0.19.8 +2025-04-22 01:47:29,354 INFO MainThread:2951284 [wandb_setup.py:_flush():67] Configure stats pid to 2951284 +2025-04-22 01:47:29,354 INFO MainThread:2951284 [wandb_setup.py:_flush():67] Loading settings from /tmp/haozhezhao/.config/wandb/settings +2025-04-22 01:47:29,354 INFO MainThread:2951284 [wandb_setup.py:_flush():67] Loading settings from /tmp/haozhezhao/MLLMG/wandb/settings +2025-04-22 01:47:29,355 INFO MainThread:2951284 [wandb_setup.py:_flush():67] Loading settings from environment variables +2025-04-22 01:47:29,355 INFO MainThread:2951284 [wandb_init.py:setup_run_log_directory():647] Logging user logs to checkpoint/BeforeFix_Blip_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_objects/wandb/run-20250422_014729-ni4jp6ul/logs/debug.log +2025-04-22 01:47:29,355 INFO MainThread:2951284 [wandb_init.py:setup_run_log_directory():648] Logging internal logs to checkpoint/BeforeFix_Blip_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_objects/wandb/run-20250422_014729-ni4jp6ul/logs/debug-internal.log +2025-04-22 01:47:29,355 INFO MainThread:2951284 [wandb_init.py:init():761] calling init triggers +2025-04-22 01:47:29,355 INFO MainThread:2951284 [wandb_init.py:init():766] wandb.init called with sweep_config: {} +config: {'data_path': '/tmp/haozhezhao/MLLMG/jsonl_data/multiobjects_molom_imagenet_flux_qwen_midsource_gen_2_2m_trained.jsonl', 'cloud_save_path': '/tmp/haozhezhao/MLLMG/checkpoint', 'no_local_save': False, 'vq_model': 'VQ-16', 'vq_ckpt': '/tmp/haozhezhao/model/llamagen_t2i/vq_ds16_t2i.pt', 'codebook_size': 16384, 'codebook_embed_dim': 8, 'gpt_model': 'GPT-XL', 'gpt_ckpt': '/tmp/haozhezhao/MLLMG/checkpoint/BLIP2Trainall_just_segment_pretrain_stage3_dreambench_recap_Subject200k_filtered_t2i_flux400k_200kmid_recovery_150k_extract_150k_100_fluxseg_50samseg_trainall_1e4/000-GPT-XL/checkpoints/0010000.pt', 'gpt_type': 't2i', 'vocab_size': 16384, 'cls_token_num': 512, 'dropout_p': 0.1, 'token_dropout_p': 0.1, 'drop_path': 0.0, 'no_compile': False, 'results_dir': 'checkpoint/BeforeFix_Blip_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_objects', 'dataset': 'ti2i', 'image_size': 512, 'downsample_size': 16, 'num_classes': 1000, 'epochs': 2, 'lr': 5e-05, 'weight_decay': 0.05, 'beta1': 0.9, 'beta2': 0.95, 'max_grad_norm': 1.0, 'global_batch_size': 56, 'global_seed': 0, 'num_workers': 4, 'log_every': 25, 'ckpt_every': 2000, 'gradient_accumulation_steps': 4, 'mixed_precision': 'bf16', 'val_data_path': '/tmp/haozhezhao/MLLMG/jsonl_data/multiobjects_molom_imagenet_flux_qwen_midsource_gen_val.jsonl', 'use_vision_tower': True, 'model_name_or_path': '/tmp/haozhezhao/model/instructblip-flan-t5-xl', 'image_place_holder': '', 'processor_path': None, 'do_eval': True, 'max_eval_samples': 200, 'train_text_encoder': True, 'no_left_padding': False, 'cfg_scale': 7.5, 'top_k': 16384, 'temperature': 0.9, 'top_p': 1.0, 'eval_steps': 2000, 'project_name': 'llamagen_ti2i', 'load_from_checkpoint': '/tmp/haozhezhao/model/llamagen_t2i/t2i_XL_stage2_512.pt', 'warmup': 0.05, 'lr_decay_style': 'cosine', 'lr_decay_ratio': 0.1, 'train_iters': 500000, 'class_dropout_prob': 0.1, 'with_image_only': False, 'image_only_rate': 0.1, 'stage2': False, 'subject_driven': True, 'load_subject_embedding': '/tmp/haozhezhao/MLLMG/subject_embedding.bin', 'reference_data_path': '/tmp/haozhezhao/MLLMG/cc12m_reference_tunnel.jsonl', 'multimodal_encoder': 'blip', 'do_recovery': True, 'no_replace': False, 'resume': False, 'dreambench_eval': False, 'find_unused_parameters': True, 'load_visual_encoder': False, 'continue_stage1': False, 'replace_subject': False, 'train_all': True, 'save_total_limit': 2, 'load_language_projection': None, 'mm_vision_tower': 'openai/clip-vit-large-patch14', 'load_fixed_llamagen': False, 'unfreeze_output': False, 'fix': 'gpt', 'rank': 0, 'world_size': 8, 'gpu': 0, 'dist_url': 'env://', 'distributed': True, 'dist_backend': 'nccl', '_wandb': {}} +2025-04-22 01:47:29,355 INFO MainThread:2951284 [wandb_init.py:init():784] starting backend +2025-04-22 01:47:29,355 INFO MainThread:2951284 [wandb_init.py:init():788] sending inform_init request +2025-04-22 01:47:29,361 INFO MainThread:2951284 [backend.py:_multiprocessing_setup():101] multiprocessing start_methods=fork,spawn,forkserver, using: spawn +2025-04-22 01:47:29,361 INFO MainThread:2951284 [wandb_init.py:init():798] backend started and connected +2025-04-22 01:47:29,363 INFO MainThread:2951284 [wandb_init.py:init():891] updated telemetry +2025-04-22 01:47:29,364 INFO MainThread:2951284 [wandb_init.py:init():915] communicating run to backend with 90.0 second timeout +2025-04-22 01:47:29,831 INFO MainThread:2951284 [wandb_init.py:init():990] starting run threads in backend +2025-04-22 01:47:29,917 INFO MainThread:2951284 [wandb_run.py:_console_start():2375] atexit reg +2025-04-22 01:47:29,917 INFO MainThread:2951284 [wandb_run.py:_redirect():2227] redirect: wrap_raw +2025-04-22 01:47:29,918 INFO MainThread:2951284 [wandb_run.py:_redirect():2292] Wrapping output streams. +2025-04-22 01:47:29,918 INFO MainThread:2951284 [wandb_run.py:_redirect():2315] Redirects installed. +2025-04-22 01:47:29,919 INFO MainThread:2951284 [wandb_init.py:init():1032] run started, returning control to user process diff --git a/CKPTS/BeforeFix_Blip_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_objects/wandb/run-20250422_014729-ni4jp6ul/files/output.log b/CKPTS/BeforeFix_Blip_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_objects/wandb/run-20250422_014729-ni4jp6ul/files/output.log new file mode 100644 index 0000000000000000000000000000000000000000..8c9e2d8129f48114ae9d7448b501403b2c0d776f --- /dev/null +++ b/CKPTS/BeforeFix_Blip_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_objects/wandb/run-20250422_014729-ni4jp6ul/files/output.log @@ -0,0 +1,3781 @@ +[2025-04-22 01:47:30] Training for 2 epochs... +[2025-04-22 01:47:30] Beginning epoch 0... + 0%| | 0/40903 [00:00 +Some kwargs in processor config are unused and will not have any effect: num_query_tokens. +tokenizer length after expend 32102 +tokenizer length before expend 32102 + /tmp/haozhezhao/anaconda3/envs/nlp/lib/python3.11/contextlib.py:105: FutureWarning: `torch.backends.cuda.sdp_kernel()` is deprecated. In the future, this context manager will be removed. Please see `torch.nn.attention.sdpa_kernel()` for the new context manager, with updated signature. + self.gen = func(*args, **kwds) | 0/3 [00:00 +Some kwargs in processor config are unused and will not have any effect: num_query_tokens. +tokenizer length after expend 32102 +tokenizer length before expend 32102 +100%|████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████| 3/3 [02:53<00:00, 57.72s/it] +[2025-04-22 03:57:15] Finish Eval in 4000 steps...██████████████████████████████████████████████████████████████████████| 3/3 [02:52<00:00, 57.31s/it] +processing left-padding... +processing left-padding... +processing left-padding... +[2025-04-22 03:57:35] Saved checkpoint to checkpoint/BeforeFix_Blip_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_objects/002-GPT-XL/checkpoints/0004000.pt + 10%|███████████ | 4024/40903 [2:10:34<12:29:46, 1.22s/it][2025-04-22 03:58:05] (step=0004025) Train Loss: 6.4191, Train Steps/Sec: 0.08 + 10%|███████████ | 4049/40903 [2:11:13<22:38:03, 2.21s/it][2025-04-22 03:58:45] (step=0004050) Train Loss: 6.4513, Train Steps/Sec: 0.63 + 10%|███████████▏ | 4074/40903 [2:11:43<11:54:27, 1.16s/it][2025-04-22 03:59:14] (step=0004075) Train Loss: 6.4515, Train Steps/Sec: 0.85 + 10%|███████████▏ | 4099/40903 [2:12:12<11:53:15, 1.16s/it][2025-04-22 03:59:44] (step=0004100) Train Loss: 6.4459, Train Steps/Sec: 0.84 + 10%|███████████▎ | 4124/40903 [2:12:42<12:10:17, 1.19s/it][2025-04-22 04:00:13] (step=0004125) Train Loss: 6.5043, Train Steps/Sec: 0.85 + 10%|███████████▎ | 4149/40903 [2:13:12<12:03:25, 1.18s/it][2025-04-22 04:00:50] (step=0004150) Train Loss: 6.4140, Train Steps/Sec: 0.68 + 10%|███████████▍ | 4174/40903 [2:13:57<12:12:54, 1.20s/it][2025-04-22 04:01:29] (step=0004175) Train Loss: 6.4911, Train Steps/Sec: 0.65 + 10%|███████████▍ | 4199/40903 [2:14:27<11:59:04, 1.18s/it][2025-04-22 04:01:58] (step=0004200) Train Loss: 6.4677, Train Steps/Sec: 0.84 + 10%|███████████▌ | 4224/40903 [2:14:57<12:26:20, 1.22s/it][2025-04-22 04:02:28] (step=0004225) Train Loss: 6.4655, Train Steps/Sec: 0.84 + 10%|███████████▋ | 4249/40903 [2:15:26<12:06:27, 1.19s/it][2025-04-22 04:02:58] (step=0004250) Train Loss: 6.3943, Train Steps/Sec: 0.84 + 10%|███████████▋ | 4274/40903 [2:15:56<11:58:16, 1.18s/it][2025-04-22 04:03:27] (step=0004275) Train Loss: 6.4599, Train Steps/Sec: 0.85 + 11%|███████████▊ | 4299/40903 [2:16:25<11:53:21, 1.17s/it][2025-04-22 04:03:57] (step=0004300) Train Loss: 6.4727, Train Steps/Sec: 0.84 + 11%|███████████▊ | 4324/40903 [2:16:55<12:10:38, 1.20s/it][2025-04-22 04:04:26] (step=0004325) Train Loss: 6.4495, Train Steps/Sec: 0.85 + 11%|███████████▉ | 4349/40903 [2:17:24<11:51:35, 1.17s/it][2025-04-22 04:04:56] (step=0004350) Train Loss: 6.4427, Train Steps/Sec: 0.85 + 11%|███████████▉ | 4374/40903 [2:17:54<12:01:45, 1.19s/it][2025-04-22 04:05:25] (step=0004375) Train Loss: 6.4397, Train Steps/Sec: 0.85 + 11%|████████████ | 4399/40903 [2:18:23<11:49:49, 1.17s/it][2025-04-22 04:05:55] (step=0004400) Train Loss: 6.4273, Train Steps/Sec: 0.85 + 11%|████████████ | 4424/40903 [2:18:53<12:04:58, 1.19s/it][2025-04-22 04:06:24] (step=0004425) Train Loss: 6.4375, Train Steps/Sec: 0.84 + 11%|████████████▏ | 4449/40903 [2:19:22<11:56:35, 1.18s/it][2025-04-22 04:06:54] (step=0004450) Train Loss: 6.4140, Train Steps/Sec: 0.85 + 11%|████████████▎ | 4474/40903 [2:19:52<11:46:16, 1.16s/it][2025-04-22 04:07:23] (step=0004475) Train Loss: 6.4419, Train Steps/Sec: 0.85 + 11%|████████████▎ | 4499/40903 [2:20:21<11:52:02, 1.17s/it][2025-04-22 04:07:53] (step=0004500) Train Loss: 6.4107, Train Steps/Sec: 0.84 + 11%|████████████▍ | 4524/40903 [2:20:51<12:09:57, 1.20s/it][2025-04-22 04:08:22] (step=0004525) Train Loss: 6.4434, Train Steps/Sec: 0.85 + 11%|████████████▍ | 4549/40903 [2:21:21<11:55:23, 1.18s/it][2025-04-22 04:08:52] (step=0004550) Train Loss: 6.4925, Train Steps/Sec: 0.84 + 11%|████████████▌ | 4574/40903 [2:21:50<11:52:16, 1.18s/it][2025-04-22 04:09:21] (step=0004575) Train Loss: 6.3957, Train Steps/Sec: 0.85 + 11%|████████████▌ | 4599/40903 [2:22:20<11:57:10, 1.19s/it][2025-04-22 04:09:51] (step=0004600) Train Loss: 6.4853, Train Steps/Sec: 0.84 + 11%|████████████▋ | 4624/40903 [2:22:49<12:03:29, 1.20s/it][2025-04-22 04:10:21] (step=0004625) Train Loss: 6.4346, Train Steps/Sec: 0.85 + 11%|████████████▋ | 4649/40903 [2:23:19<11:50:47, 1.18s/it][2025-04-22 04:10:50] (step=0004650) Train Loss: 6.4601, Train Steps/Sec: 0.84 + 11%|████████████▊ | 4674/40903 [2:23:49<11:47:26, 1.17s/it][2025-04-22 04:11:20] (step=0004675) Train Loss: 6.4670, Train Steps/Sec: 0.85 + 11%|████████████▊ | 4699/40903 [2:24:18<11:52:39, 1.18s/it][2025-04-22 04:11:50] (step=0004700) Train Loss: 6.4542, Train Steps/Sec: 0.84 + 12%|████████████▉ | 4724/40903 [2:24:48<11:58:47, 1.19s/it][2025-04-22 04:12:19] (step=0004725) Train Loss: 6.4189, Train Steps/Sec: 0.85 + 12%|█████████████ | 4749/40903 [2:25:17<11:46:43, 1.17s/it][2025-04-22 04:12:48] (step=0004750) Train Loss: 6.4563, Train Steps/Sec: 0.85 + 12%|█████████████ | 4774/40903 [2:25:47<11:54:40, 1.19s/it][2025-04-22 04:13:18] (step=0004775) Train Loss: 6.4308, Train Steps/Sec: 0.85 + 12%|█████████████▏ | 4799/40903 [2:26:17<11:49:14, 1.18s/it][2025-04-22 04:13:48] (step=0004800) Train Loss: 6.4267, Train Steps/Sec: 0.83 + 12%|█████████████▏ | 4824/40903 [2:26:46<11:57:46, 1.19s/it][2025-04-22 04:14:18] (step=0004825) Train Loss: 6.4279, Train Steps/Sec: 0.85 + 12%|█████████████▎ | 4849/40903 [2:27:16<11:40:13, 1.17s/it][2025-04-22 04:14:47] (step=0004850) Train Loss: 6.4653, Train Steps/Sec: 0.84 + 12%|█████████████▎ | 4874/40903 [2:27:45<11:47:04, 1.18s/it][2025-04-22 04:15:17] (step=0004875) Train Loss: 6.4824, Train Steps/Sec: 0.85 + 12%|█████████████▍ | 4899/40903 [2:28:15<11:42:45, 1.17s/it][2025-04-22 04:15:46] (step=0004900) Train Loss: 6.4320, Train Steps/Sec: 0.84 + 12%|█████████████▍ | 4924/40903 [2:28:45<12:00:37, 1.20s/it][2025-04-22 04:16:16] (step=0004925) Train Loss: 6.4517, Train Steps/Sec: 0.84 + 12%|█████████████▌ | 4949/40903 [2:29:14<11:45:00, 1.18s/it][2025-04-22 04:16:45] (step=0004950) Train Loss: 6.4620, Train Steps/Sec: 0.85 + 12%|█████████████▌ | 4974/40903 [2:29:44<11:46:02, 1.18s/it][2025-04-22 04:17:15] (step=0004975) Train Loss: 6.3910, Train Steps/Sec: 0.84 + 12%|█████████████▋ | 4999/40903 [2:30:13<11:39:28, 1.17s/it][2025-04-22 04:17:45] (step=0005000) Train Loss: 6.4438, Train Steps/Sec: 0.84 + 12%|█████████████▊ | 5024/40903 [2:30:43<12:00:15, 1.20s/it][2025-04-22 04:18:14] (step=0005025) Train Loss: 6.5328, Train Steps/Sec: 0.84 + 12%|█████████████▊ | 5049/40903 [2:31:13<11:44:50, 1.18s/it][2025-04-22 04:18:44] (step=0005050) Train Loss: 6.4355, Train Steps/Sec: 0.84 + 12%|█████████████▉ | 5074/40903 [2:31:42<11:45:17, 1.18s/it][2025-04-22 04:19:14] (step=0005075) Train Loss: 6.4835, Train Steps/Sec: 0.85 + 12%|█████████████▉ | 5099/40903 [2:32:12<11:28:05, 1.15s/it][2025-04-22 04:19:43] (step=0005100) Train Loss: 6.4328, Train Steps/Sec: 0.84 + 13%|██████████████ | 5124/40903 [2:32:42<11:47:24, 1.19s/it][2025-04-22 04:20:13] (step=0005125) Train Loss: 6.4565, Train Steps/Sec: 0.85 + 13%|██████████████ | 5149/40903 [2:33:11<11:41:52, 1.18s/it][2025-04-22 04:20:43] (step=0005150) Train Loss: 6.4073, Train Steps/Sec: 0.85 + 13%|██████████████▏ | 5174/40903 [2:33:41<11:41:10, 1.18s/it][2025-04-22 04:21:12] (step=0005175) Train Loss: 6.4802, Train Steps/Sec: 0.84 + 13%|██████████████▏ | 5199/40903 [2:34:11<11:42:04, 1.18s/it][2025-04-22 04:21:42] (step=0005200) Train Loss: 6.4722, Train Steps/Sec: 0.84 + 13%|██████████████▎ | 5224/40903 [2:34:40<12:00:12, 1.21s/it][2025-04-22 04:22:11] (step=0005225) Train Loss: 6.4488, Train Steps/Sec: 0.85 + 13%|██████████████▎ | 5249/40903 [2:35:10<11:46:07, 1.19s/it][2025-04-22 04:22:41] (step=0005250) Train Loss: 6.4325, Train Steps/Sec: 0.85 + 13%|██████████████▍ | 5274/40903 [2:35:39<11:43:28, 1.18s/it][2025-04-22 04:23:10] (step=0005275) Train Loss: 6.4444, Train Steps/Sec: 0.85 + 13%|██████████████▌ | 5299/40903 [2:36:09<11:25:51, 1.16s/it][2025-04-22 04:23:40] (step=0005300) Train Loss: 6.4000, Train Steps/Sec: 0.85 + 13%|██████████████▌ | 5324/40903 [2:36:38<11:55:19, 1.21s/it][2025-04-22 04:24:09] (step=0005325) Train Loss: 6.4475, Train Steps/Sec: 0.85 + 13%|██████████████▋ | 5349/40903 [2:37:08<11:43:22, 1.19s/it][2025-04-22 04:24:39] (step=0005350) Train Loss: 6.4529, Train Steps/Sec: 0.85 + 13%|██████████████▋ | 5374/40903 [2:37:37<11:33:00, 1.17s/it][2025-04-22 04:25:09] (step=0005375) Train Loss: 6.4400, Train Steps/Sec: 0.85 + 13%|██████████████▊ | 5399/40903 [2:38:07<11:28:43, 1.16s/it][2025-04-22 04:25:38] (step=0005400) Train Loss: 6.4569, Train Steps/Sec: 0.84 + 13%|██████████████▊ | 5424/40903 [2:38:37<11:51:35, 1.20s/it][2025-04-22 04:26:08] (step=0005425) Train Loss: 6.4167, Train Steps/Sec: 0.85 + 13%|██████████████▉ | 5449/40903 [2:39:13<26:26:02, 2.68s/it][2025-04-22 04:26:44] (step=0005450) Train Loss: 6.4174, Train Steps/Sec: 0.68 + 13%|██████████████▉ | 5474/40903 [2:39:43<11:39:17, 1.18s/it][2025-04-22 04:27:14] (step=0005475) Train Loss: 6.4183, Train Steps/Sec: 0.84 + 13%|███████████████ | 5499/40903 [2:40:12<11:31:42, 1.17s/it][2025-04-22 04:27:44] (step=0005500) Train Loss: 6.4338, Train Steps/Sec: 0.85 + 14%|███████████████▏ | 5524/40903 [2:40:42<11:46:20, 1.20s/it][2025-04-22 04:28:13] (step=0005525) Train Loss: 6.4067, Train Steps/Sec: 0.84 + 14%|███████████████▏ | 5549/40903 [2:41:12<11:47:17, 1.20s/it][2025-04-22 04:28:43] (step=0005550) Train Loss: 6.4390, Train Steps/Sec: 0.84 + 14%|███████████████▎ | 5574/40903 [2:41:41<11:32:13, 1.18s/it][2025-04-22 04:29:13] (step=0005575) Train Loss: 6.4595, Train Steps/Sec: 0.84 + 14%|███████████████▎ | 5599/40903 [2:42:11<11:33:32, 1.18s/it][2025-04-22 04:29:42] (step=0005600) Train Loss: 6.4366, Train Steps/Sec: 0.84 + 14%|███████████████▍ | 5624/40903 [2:42:41<11:57:39, 1.22s/it][2025-04-22 04:30:12] (step=0005625) Train Loss: 6.4249, Train Steps/Sec: 0.84 + 14%|███████████████▍ | 5649/40903 [2:43:10<11:39:10, 1.19s/it][2025-04-22 04:30:41] (step=0005650) Train Loss: 6.4477, Train Steps/Sec: 0.85 + 14%|███████████████▌ | 5674/40903 [2:43:40<11:25:17, 1.17s/it][2025-04-22 04:31:11] (step=0005675) Train Loss: 6.4250, Train Steps/Sec: 0.84 + 14%|███████████████▌ | 5699/40903 [2:44:09<11:25:51, 1.17s/it][2025-04-22 04:31:41] (step=0005700) Train Loss: 6.4348, Train Steps/Sec: 0.85 + 14%|███████████████▋ | 5724/40903 [2:44:39<11:54:57, 1.22s/it][2025-04-22 04:32:10] (step=0005725) Train Loss: 6.4586, Train Steps/Sec: 0.84 + 14%|███████████████▋ | 5749/40903 [2:45:09<11:38:08, 1.19s/it][2025-04-22 04:32:40] (step=0005750) Train Loss: 6.4007, Train Steps/Sec: 0.85 + 14%|███████████████▊ | 5774/40903 [2:45:38<11:25:18, 1.17s/it][2025-04-22 04:33:09] (step=0005775) Train Loss: 6.4411, Train Steps/Sec: 0.85 + 14%|███████████████▉ | 5799/40903 [2:46:08<11:22:33, 1.17s/it][2025-04-22 04:33:39] (step=0005800) Train Loss: 6.3803, Train Steps/Sec: 0.84 + 14%|███████████████▉ | 5824/40903 [2:46:37<11:44:45, 1.21s/it][2025-04-22 04:34:08] (step=0005825) Train Loss: 6.4405, Train Steps/Sec: 0.85 + 14%|████████████████ | 5849/40903 [2:47:07<11:32:43, 1.19s/it][2025-04-22 04:34:38] (step=0005850) Train Loss: 6.4655, Train Steps/Sec: 0.85 + 14%|████████████████ | 5874/40903 [2:47:36<11:27:46, 1.18s/it][2025-04-22 04:35:07] (step=0005875) Train Loss: 6.3995, Train Steps/Sec: 0.85 + 14%|████████████████▏ | 5899/40903 [2:48:06<11:14:24, 1.16s/it][2025-04-22 04:35:37] (step=0005900) Train Loss: 6.4594, Train Steps/Sec: 0.84 + 14%|████████████████▏ | 5924/40903 [2:48:35<11:42:19, 1.20s/it][2025-04-22 04:36:07] (step=0005925) Train Loss: 6.4942, Train Steps/Sec: 0.84 + 15%|████████████████▎ | 5949/40903 [2:49:05<11:18:09, 1.16s/it][2025-04-22 04:36:36] (step=0005950) Train Loss: 6.4297, Train Steps/Sec: 0.86 + 15%|████████████████▎ | 5974/40903 [2:49:34<11:25:23, 1.18s/it][2025-04-22 04:37:06] (step=0005975) Train Loss: 6.4199, Train Steps/Sec: 0.84 + 15%|████████████████▍ | 5999/40903 [2:50:04<11:25:29, 1.18s/it][2025-04-22 04:37:35] (step=0006000) Train Loss: 6.4320, Train Steps/Sec: 0.84 +You are using a model of type instructblip to instantiate a model of type blip-2. This is not supported for all configurations of models and can yield errors. +model_name_or_path:/tmp/haozhezhao/model/instructblip-flan-t5-xl +image_place_holder: +Some kwargs in processor config are unused and will not have any effect: num_query_tokens. +tokenizer length after expend 32102 +tokenizer length before expend 32102 +100%|████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████| 3/3 [02:54<00:00, 58.07s/it] +[2025-04-22 04:42:17] Finish Eval in 6000 steps...██████████████████████████████████████████████████████████████████████| 3/3 [02:53<00:00, 57.61s/it] +processing left-padding... +processing left-padding... +processing left-padding... +[2025-04-22 04:42:37] Saved checkpoint to checkpoint/BeforeFix_Blip_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_objects/002-GPT-XL/checkpoints/0006000.pt +[2025-04-22 04:42:40] Removed old checkpoint: checkpoint/BeforeFix_Blip_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_objects/002-GPT-XL/checkpoints/0002000.pt + 15%|████████████████▍ | 6024/40903 [2:55:38<11:50:49, 1.22s/it][2025-04-22 04:43:09] (step=0006025) Train Loss: 6.4042, Train Steps/Sec: 0.07 + 15%|████████████████▌ | 6049/40903 [2:56:07<11:26:19, 1.18s/it][2025-04-22 04:43:38] (step=0006050) Train Loss: 6.4061, Train Steps/Sec: 0.85 + 15%|████████████████▋ | 6074/40903 [2:56:37<11:18:13, 1.17s/it][2025-04-22 04:44:08] (step=0006075) Train Loss: 6.4441, Train Steps/Sec: 0.84 + 15%|████████████████▋ | 6099/40903 [2:57:06<11:24:01, 1.18s/it][2025-04-22 04:44:38] (step=0006100) Train Loss: 6.4696, Train Steps/Sec: 0.84 + 15%|████████████████▊ | 6124/40903 [2:57:36<11:36:07, 1.20s/it][2025-04-22 04:45:07] (step=0006125) Train Loss: 6.4503, Train Steps/Sec: 0.85 + 15%|████████████████▊ | 6149/40903 [2:58:06<11:23:48, 1.18s/it][2025-04-22 04:45:37] (step=0006150) Train Loss: 6.4648, Train Steps/Sec: 0.85 + 15%|████████████████▉ | 6174/40903 [2:58:35<11:16:59, 1.17s/it][2025-04-22 04:46:06] (step=0006175) Train Loss: 6.3984, Train Steps/Sec: 0.85 + 15%|████████████████▉ | 6199/40903 [2:59:05<11:15:48, 1.17s/it][2025-04-22 04:46:36] (step=0006200) Train Loss: 6.4704, Train Steps/Sec: 0.85 + 15%|█████████████████ | 6224/40903 [2:59:34<11:32:06, 1.20s/it][2025-04-22 04:47:06] (step=0006225) Train Loss: 6.4571, Train Steps/Sec: 0.84 + 15%|█████████████████ | 6249/40903 [3:00:04<11:27:14, 1.19s/it][2025-04-22 04:47:35] (step=0006250) Train Loss: 6.4737, Train Steps/Sec: 0.84 + 15%|█████████████████▏ | 6274/40903 [3:00:34<11:16:48, 1.17s/it][2025-04-22 04:48:05] (step=0006275) Train Loss: 6.4091, Train Steps/Sec: 0.85 + 15%|█████████████████▏ | 6299/40903 [3:01:03<11:21:55, 1.18s/it][2025-04-22 04:48:34] (step=0006300) Train Loss: 6.4578, Train Steps/Sec: 0.85 + 15%|█████████████████▎ | 6324/40903 [3:01:32<11:30:12, 1.20s/it][2025-04-22 04:49:04] (step=0006325) Train Loss: 6.4020, Train Steps/Sec: 0.85 + 16%|█████████████████▍ | 6349/40903 [3:02:02<11:22:17, 1.18s/it][2025-04-22 04:49:33] (step=0006350) Train Loss: 6.4319, Train Steps/Sec: 0.85 + 16%|█████████████████▍ | 6374/40903 [3:02:31<11:17:26, 1.18s/it][2025-04-22 04:50:03] (step=0006375) Train Loss: 6.4251, Train Steps/Sec: 0.85 + 16%|█████████████████▌ | 6399/40903 [3:03:01<11:14:25, 1.17s/it][2025-04-22 04:50:32] (step=0006400) Train Loss: 6.4388, Train Steps/Sec: 0.84 + 16%|█████████████████▌ | 6424/40903 [3:03:31<11:33:14, 1.21s/it][2025-04-22 04:51:02] (step=0006425) Train Loss: 6.4613, Train Steps/Sec: 0.84 + 16%|█████████████████▋ | 6449/40903 [3:04:00<11:16:33, 1.18s/it][2025-04-22 04:51:32] (step=0006450) Train Loss: 6.4108, Train Steps/Sec: 0.85 + 16%|█████████████████▋ | 6474/40903 [3:04:30<11:09:42, 1.17s/it][2025-04-22 04:52:01] (step=0006475) Train Loss: 6.4138, Train Steps/Sec: 0.85 + 16%|█████████████████▊ | 6499/40903 [3:04:59<11:09:38, 1.17s/it][2025-04-22 04:52:31] (step=0006500) Train Loss: 6.4392, Train Steps/Sec: 0.84 + 16%|█████████████████▊ | 6524/40903 [3:05:29<11:25:12, 1.20s/it][2025-04-22 04:53:00] (step=0006525) Train Loss: 6.4641, Train Steps/Sec: 0.85 + 16%|█████████████████▉ | 6549/40903 [3:05:59<11:22:48, 1.19s/it][2025-04-22 04:53:30] (step=0006550) Train Loss: 6.4135, Train Steps/Sec: 0.85 + 16%|██████████████████ | 6574/40903 [3:06:28<11:15:14, 1.18s/it][2025-04-22 04:53:59] (step=0006575) Train Loss: 6.4267, Train Steps/Sec: 0.84 + 16%|██████████████████ | 6599/40903 [3:06:58<11:10:00, 1.17s/it][2025-04-22 04:54:29] (step=0006600) Train Loss: 6.4612, Train Steps/Sec: 0.84 + 16%|██████████████████▏ | 6624/40903 [3:07:28<11:30:22, 1.21s/it][2025-04-22 04:54:59] (step=0006625) Train Loss: 6.5013, Train Steps/Sec: 0.84 + 16%|██████████████████▏ | 6649/40903 [3:07:57<11:20:20, 1.19s/it][2025-04-22 04:55:28] (step=0006650) Train Loss: 6.4246, Train Steps/Sec: 0.85 + 16%|██████████████████▎ | 6674/40903 [3:08:27<11:09:40, 1.17s/it][2025-04-22 04:55:58] (step=0006675) Train Loss: 6.4340, Train Steps/Sec: 0.84 + 16%|██████████████████▎ | 6699/40903 [3:08:56<11:16:46, 1.19s/it][2025-04-22 04:56:28] (step=0006700) Train Loss: 6.4292, Train Steps/Sec: 0.84 + 16%|██████████████████▍ | 6724/40903 [3:09:26<11:24:45, 1.20s/it][2025-04-22 04:56:57] (step=0006725) Train Loss: 6.4324, Train Steps/Sec: 0.85 + 17%|██████████████████▍ | 6749/40903 [3:09:55<11:08:05, 1.17s/it][2025-04-22 04:57:27] (step=0006750) Train Loss: 6.4006, Train Steps/Sec: 0.85 + 17%|██████████████████▌ | 6774/40903 [3:10:25<11:11:05, 1.18s/it][2025-04-22 04:57:56] (step=0006775) Train Loss: 6.4824, Train Steps/Sec: 0.85 + 17%|██████████████████▌ | 6799/40903 [3:10:54<10:54:34, 1.15s/it][2025-04-22 04:58:26] (step=0006800) Train Loss: 6.4344, Train Steps/Sec: 0.85 + 17%|██████████████████▋ | 6824/40903 [3:11:24<11:19:15, 1.20s/it][2025-04-22 04:58:55] (step=0006825) Train Loss: 6.4001, Train Steps/Sec: 0.85 + 17%|██████████████████▊ | 6849/40903 [3:11:54<11:17:16, 1.19s/it][2025-04-22 04:59:25] (step=0006850) Train Loss: 6.4550, Train Steps/Sec: 0.85 + 17%|██████████████████▊ | 6874/40903 [3:12:23<11:05:12, 1.17s/it][2025-04-22 04:59:54] (step=0006875) Train Loss: 6.4316, Train Steps/Sec: 0.85 + 17%|██████████████████▉ | 6899/40903 [3:12:52<11:06:00, 1.18s/it][2025-04-22 05:00:24] (step=0006900) Train Loss: 6.4438, Train Steps/Sec: 0.84 + 17%|██████████████████▉ | 6924/40903 [3:13:22<11:14:46, 1.19s/it][2025-04-22 05:00:53] (step=0006925) Train Loss: 6.3701, Train Steps/Sec: 0.85 + 17%|███████████████████ | 6949/40903 [3:13:52<11:14:13, 1.19s/it][2025-04-22 05:01:23] (step=0006950) Train Loss: 6.4235, Train Steps/Sec: 0.85 + 17%|███████████████████ | 6974/40903 [3:14:21<11:09:11, 1.18s/it][2025-04-22 05:01:52] (step=0006975) Train Loss: 6.4011, Train Steps/Sec: 0.85 + 17%|███████████████████▏ | 6999/40903 [3:14:51<10:59:27, 1.17s/it][2025-04-22 05:02:22] (step=0007000) Train Loss: 6.4178, Train Steps/Sec: 0.84 + 17%|███████████████████▏ | 7024/40903 [3:15:20<11:12:05, 1.19s/it][2025-04-22 05:02:51] (step=0007025) Train Loss: 6.4231, Train Steps/Sec: 0.85 + 17%|███████████████████▎ | 7049/40903 [3:15:50<11:08:26, 1.18s/it][2025-04-22 05:03:21] (step=0007050) Train Loss: 6.4614, Train Steps/Sec: 0.85 + 17%|███████████████████▎ | 7074/40903 [3:16:19<10:59:37, 1.17s/it][2025-04-22 05:03:50] (step=0007075) Train Loss: 6.4004, Train Steps/Sec: 0.85 + 17%|███████████████████▍ | 7099/40903 [3:16:49<10:50:25, 1.15s/it][2025-04-22 05:04:20] (step=0007100) Train Loss: 6.4195, Train Steps/Sec: 0.85 + 17%|███████████████████▌ | 7124/40903 [3:17:18<11:22:39, 1.21s/it][2025-04-22 05:04:49] (step=0007125) Train Loss: 6.4786, Train Steps/Sec: 0.85 + 17%|███████████████████▌ | 7149/40903 [3:17:48<11:09:01, 1.19s/it][2025-04-22 05:05:19] (step=0007150) Train Loss: 6.4608, Train Steps/Sec: 0.85 + 18%|███████████████████▋ | 7174/40903 [3:18:17<10:57:15, 1.17s/it][2025-04-22 05:05:48] (step=0007175) Train Loss: 6.4142, Train Steps/Sec: 0.85 + 18%|███████████████████▋ | 7199/40903 [3:18:46<10:59:34, 1.17s/it][2025-04-22 05:06:18] (step=0007200) Train Loss: 6.4416, Train Steps/Sec: 0.85 + 18%|███████████████████▊ | 7224/40903 [3:19:16<11:22:45, 1.22s/it][2025-04-22 05:06:47] (step=0007225) Train Loss: 6.4275, Train Steps/Sec: 0.85 + 18%|███████████████████▊ | 7249/40903 [3:19:46<11:02:07, 1.18s/it][2025-04-22 05:07:17] (step=0007250) Train Loss: 6.4670, Train Steps/Sec: 0.84 + 18%|███████████████████▉ | 7274/40903 [3:20:15<10:59:53, 1.18s/it][2025-04-22 05:07:46] (step=0007275) Train Loss: 6.3819, Train Steps/Sec: 0.84 + 18%|███████████████████▉ | 7299/40903 [3:20:45<11:00:29, 1.18s/it][2025-04-22 05:08:16] (step=0007300) Train Loss: 6.3999, Train Steps/Sec: 0.84 + 18%|████████████████████ | 7324/40903 [3:21:14<11:01:31, 1.18s/it][2025-04-22 05:08:46] (step=0007325) Train Loss: 6.4158, Train Steps/Sec: 0.85 + 18%|████████████████████ | 7349/40903 [3:21:44<10:57:25, 1.18s/it][2025-04-22 05:09:15] (step=0007350) Train Loss: 6.4016, Train Steps/Sec: 0.85 + 18%|████████████████████▏ | 7374/40903 [3:22:14<10:54:44, 1.17s/it][2025-04-22 05:09:45] (step=0007375) Train Loss: 6.4345, Train Steps/Sec: 0.84 + 18%|████████████████████▎ | 7399/40903 [3:22:43<10:54:33, 1.17s/it][2025-04-22 05:10:14] (step=0007400) Train Loss: 6.4054, Train Steps/Sec: 0.84 + 18%|████████████████████▎ | 7424/40903 [3:23:13<11:02:09, 1.19s/it][2025-04-22 05:10:44] (step=0007425) Train Loss: 6.4060, Train Steps/Sec: 0.85 + 18%|████████████████████▍ | 7449/40903 [3:23:42<11:02:18, 1.19s/it][2025-04-22 05:11:13] (step=0007450) Train Loss: 6.4697, Train Steps/Sec: 0.85 + 18%|████████████████████▍ | 7474/40903 [3:24:12<10:59:33, 1.18s/it][2025-04-22 05:11:43] (step=0007475) Train Loss: 6.4399, Train Steps/Sec: 0.84 + 18%|████████████████████▌ | 7499/40903 [3:24:41<10:51:12, 1.17s/it][2025-04-22 05:12:13] (step=0007500) Train Loss: 6.3996, Train Steps/Sec: 0.84 + 18%|████████████████████▌ | 7524/40903 [3:25:11<10:59:19, 1.19s/it][2025-04-22 05:12:42] (step=0007525) Train Loss: 6.4004, Train Steps/Sec: 0.85 + 18%|████████████████████▋ | 7549/40903 [3:25:41<10:55:04, 1.18s/it][2025-04-22 05:13:12] (step=0007550) Train Loss: 6.4325, Train Steps/Sec: 0.85 + 19%|████████████████████▋ | 7574/40903 [3:26:10<10:43:51, 1.16s/it][2025-04-22 05:13:41] (step=0007575) Train Loss: 6.4190, Train Steps/Sec: 0.85 + 19%|████████████████████▊ | 7599/40903 [3:26:39<11:00:35, 1.19s/it][2025-04-22 05:14:11] (step=0007600) Train Loss: 6.4305, Train Steps/Sec: 0.84 + 19%|████████████████████▉ | 7624/40903 [3:27:09<11:01:44, 1.19s/it][2025-04-22 05:14:40] (step=0007625) Train Loss: 6.4460, Train Steps/Sec: 0.85 + 19%|████████████████████▉ | 7649/40903 [3:27:39<11:03:04, 1.20s/it][2025-04-22 05:15:10] (step=0007650) Train Loss: 6.4404, Train Steps/Sec: 0.84 + 19%|█████████████████████ | 7674/40903 [3:28:08<10:49:02, 1.17s/it][2025-04-22 05:15:39] (step=0007675) Train Loss: 6.4574, Train Steps/Sec: 0.85 + 19%|█████████████████████ | 7699/40903 [3:28:38<10:48:06, 1.17s/it][2025-04-22 05:16:09] (step=0007700) Train Loss: 6.4477, Train Steps/Sec: 0.84 + 19%|█████████████████████▏ | 7724/40903 [3:29:07<10:59:47, 1.19s/it][2025-04-22 05:16:39] (step=0007725) Train Loss: 6.4142, Train Steps/Sec: 0.85 + 19%|█████████████████████▏ | 7749/40903 [3:29:37<10:53:55, 1.18s/it][2025-04-22 05:17:08] (step=0007750) Train Loss: 6.4439, Train Steps/Sec: 0.85 + 19%|█████████████████████▎ | 7774/40903 [3:30:06<10:46:47, 1.17s/it][2025-04-22 05:17:37] (step=0007775) Train Loss: 6.4280, Train Steps/Sec: 0.85 + 19%|█████████████████████▎ | 7799/40903 [3:30:36<10:44:17, 1.17s/it][2025-04-22 05:18:07] (step=0007800) Train Loss: 6.4113, Train Steps/Sec: 0.84 + 19%|█████████████████████▍ | 7824/40903 [3:31:06<11:16:40, 1.23s/it][2025-04-22 05:18:37] (step=0007825) Train Loss: 6.4562, Train Steps/Sec: 0.84 + 19%|█████████████████████▍ | 7849/40903 [3:31:35<10:50:51, 1.18s/it][2025-04-22 05:19:06] (step=0007850) Train Loss: 6.4149, Train Steps/Sec: 0.85 + 19%|█████████████████████▌ | 7874/40903 [3:32:05<10:40:27, 1.16s/it][2025-04-22 05:19:36] (step=0007875) Train Loss: 6.4219, Train Steps/Sec: 0.85 + 19%|█████████████████████▋ | 7899/40903 [3:32:34<10:48:35, 1.18s/it][2025-04-22 05:20:06] (step=0007900) Train Loss: 6.4223, Train Steps/Sec: 0.84 + 19%|█████████████████████▋ | 7924/40903 [3:33:04<10:55:45, 1.19s/it][2025-04-22 05:20:35] (step=0007925) Train Loss: 6.4512, Train Steps/Sec: 0.85 + 19%|█████████████████████▊ | 7949/40903 [3:33:33<10:46:05, 1.18s/it][2025-04-22 05:21:05] (step=0007950) Train Loss: 6.4431, Train Steps/Sec: 0.84 + 19%|█████████████████████▊ | 7974/40903 [3:34:03<10:47:23, 1.18s/it][2025-04-22 05:21:34] (step=0007975) Train Loss: 6.4071, Train Steps/Sec: 0.85 + 20%|█████████████████████▉ | 7999/40903 [3:34:33<10:54:02, 1.19s/it][2025-04-22 05:22:04] (step=0008000) Train Loss: 6.4358, Train Steps/Sec: 0.84 +You are using a model of type instructblip to instantiate a model of type blip-2. This is not supported for all configurations of models and can yield errors. +model_name_or_path:/tmp/haozhezhao/model/instructblip-flan-t5-xl +image_place_holder: +Some kwargs in processor config are unused and will not have any effect: num_query_tokens. +tokenizer length after expend 32102 +tokenizer length before expend 32102 +100%|████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████| 3/3 [02:54<00:00, 58.01s/it] +[2025-04-22 05:26:46] Finish Eval in 8000 steps...██████████████████████████████████████████████████████████████████████| 3/3 [02:53<00:00, 57.47s/it] +processing left-padding... +processing left-padding... +processing left-padding... +[2025-04-22 05:27:07] Saved checkpoint to checkpoint/BeforeFix_Blip_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_objects/002-GPT-XL/checkpoints/0008000.pt +[2025-04-22 05:27:09] Removed old checkpoint: checkpoint/BeforeFix_Blip_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_objects/002-GPT-XL/checkpoints/0004000.pt + 20%|█████████████████████▉ | 8024/40903 [3:40:08<11:10:43, 1.22s/it][2025-04-22 05:27:39] (step=0008025) Train Loss: 6.4118, Train Steps/Sec: 0.07 + 20%|██████████████████████ | 8049/40903 [3:40:37<10:41:37, 1.17s/it][2025-04-22 05:28:09] (step=0008050) Train Loss: 6.4051, Train Steps/Sec: 0.85 + 20%|██████████████████████ | 8074/40903 [3:41:07<10:49:49, 1.19s/it][2025-04-22 05:28:38] (step=0008075) Train Loss: 6.4516, Train Steps/Sec: 0.84 + 20%|██████████████████████▏ | 8099/40903 [3:41:37<10:39:08, 1.17s/it][2025-04-22 05:29:08] (step=0008100) Train Loss: 6.4593, Train Steps/Sec: 0.85 + 20%|██████████████████████▏ | 8124/40903 [3:42:06<10:51:21, 1.19s/it][2025-04-22 05:29:38] (step=0008125) Train Loss: 6.4414, Train Steps/Sec: 0.85 + 20%|██████████████████████▎ | 8149/40903 [3:42:36<10:49:46, 1.19s/it][2025-04-22 05:30:07] (step=0008150) Train Loss: 6.4606, Train Steps/Sec: 0.85 + 20%|██████████████████████▍ | 8174/40903 [3:43:06<10:46:49, 1.19s/it][2025-04-22 05:30:37] (step=0008175) Train Loss: 6.4421, Train Steps/Sec: 0.84 + 20%|██████████████████████▍ | 8199/40903 [3:43:35<10:36:55, 1.17s/it][2025-04-22 05:31:06] (step=0008200) Train Loss: 6.4469, Train Steps/Sec: 0.84 + 20%|██████████████████████▌ | 8224/40903 [3:44:05<10:46:26, 1.19s/it][2025-04-22 05:31:36] (step=0008225) Train Loss: 6.4415, Train Steps/Sec: 0.85 + 20%|██████████████████████▌ | 8249/40903 [3:44:34<10:44:31, 1.18s/it][2025-04-22 05:32:05] (step=0008250) Train Loss: 6.4863, Train Steps/Sec: 0.85 + 20%|██████████████████████▋ | 8274/40903 [3:45:04<10:41:17, 1.18s/it][2025-04-22 05:32:35] (step=0008275) Train Loss: 6.4025, Train Steps/Sec: 0.84 + 20%|██████████████████████▋ | 8299/40903 [3:45:33<10:32:41, 1.16s/it][2025-04-22 05:33:05] (step=0008300) Train Loss: 6.4570, Train Steps/Sec: 0.84 + 20%|██████████████████████▊ | 8324/40903 [3:46:03<10:51:50, 1.20s/it][2025-04-22 05:33:34] (step=0008325) Train Loss: 6.3967, Train Steps/Sec: 0.85 + 20%|██████████████████████▊ | 8349/40903 [3:46:32<10:42:13, 1.18s/it][2025-04-22 05:34:03] (step=0008350) Train Loss: 6.4488, Train Steps/Sec: 0.85 + 20%|██████████████████████▉ | 8374/40903 [3:47:02<10:42:48, 1.19s/it][2025-04-22 05:34:33] (step=0008375) Train Loss: 6.4477, Train Steps/Sec: 0.85 + 21%|██████████████████████▉ | 8399/40903 [3:47:31<10:32:41, 1.17s/it][2025-04-22 05:35:03] (step=0008400) Train Loss: 6.4579, Train Steps/Sec: 0.84 + 21%|███████████████████████ | 8424/40903 [3:48:01<10:47:05, 1.20s/it][2025-04-22 05:35:32] (step=0008425) Train Loss: 6.4367, Train Steps/Sec: 0.85 + 21%|███████████████████████▏ | 8449/40903 [3:48:30<10:34:28, 1.17s/it][2025-04-22 05:36:02] (step=0008450) Train Loss: 6.4137, Train Steps/Sec: 0.84 + 21%|███████████████████████▏ | 8474/40903 [3:49:00<10:30:03, 1.17s/it][2025-04-22 05:36:31] (step=0008475) Train Loss: 6.4394, Train Steps/Sec: 0.85 + 21%|███████████████████████▎ | 8499/40903 [3:49:29<10:22:22, 1.15s/it][2025-04-22 05:37:01] (step=0008500) Train Loss: 6.4251, Train Steps/Sec: 0.84 + 21%|███████████████████████▎ | 8524/40903 [3:49:59<10:48:58, 1.20s/it][2025-04-22 05:37:30] (step=0008525) Train Loss: 6.4112, Train Steps/Sec: 0.85 + 21%|███████████████████████▍ | 8549/40903 [3:50:28<10:37:43, 1.18s/it][2025-04-22 05:38:00] (step=0008550) Train Loss: 6.4107, Train Steps/Sec: 0.85 + 21%|███████████████████████▍ | 8574/40903 [3:50:58<10:40:54, 1.19s/it][2025-04-22 05:38:29] (step=0008575) Train Loss: 6.4093, Train Steps/Sec: 0.84 + 21%|███████████████████████▌ | 8599/40903 [3:51:28<10:34:38, 1.18s/it][2025-04-22 05:38:59] (step=0008600) Train Loss: 6.4293, Train Steps/Sec: 0.84 + 21%|███████████████████████▌ | 8624/40903 [3:51:58<10:50:29, 1.21s/it][2025-04-22 05:39:29] (step=0008625) Train Loss: 6.4433, Train Steps/Sec: 0.84 + 21%|███████████████████████▋ | 8649/40903 [3:52:27<10:34:27, 1.18s/it][2025-04-22 05:39:59] (step=0008650) Train Loss: 6.4185, Train Steps/Sec: 0.84 + 21%|███████████████████████▊ | 8674/40903 [3:52:57<10:32:27, 1.18s/it][2025-04-22 05:40:28] (step=0008675) Train Loss: 6.3939, Train Steps/Sec: 0.85 + 21%|███████████████████████▊ | 8699/40903 [3:53:27<10:29:16, 1.17s/it][2025-04-22 05:40:58] (step=0008700) Train Loss: 6.4490, Train Steps/Sec: 0.84 + 21%|███████████████████████▉ | 8724/40903 [3:53:56<10:42:37, 1.20s/it][2025-04-22 05:41:28] (step=0008725) Train Loss: 6.4531, Train Steps/Sec: 0.84 + 21%|███████████████████████▉ | 8749/40903 [3:54:26<10:42:22, 1.20s/it][2025-04-22 05:41:57] (step=0008750) Train Loss: 6.4063, Train Steps/Sec: 0.84 + 21%|████████████████████████ | 8774/40903 [3:54:56<10:35:35, 1.19s/it][2025-04-22 05:42:27] (step=0008775) Train Loss: 6.4224, Train Steps/Sec: 0.84 + 22%|████████████████████████ | 8799/40903 [3:55:25<10:22:58, 1.16s/it][2025-04-22 05:42:57] (step=0008800) Train Loss: 6.3669, Train Steps/Sec: 0.85 + 22%|████████████████████████▏ | 8824/40903 [3:55:55<10:43:09, 1.20s/it][2025-04-22 05:43:26] (step=0008825) Train Loss: 6.4033, Train Steps/Sec: 0.84 + 22%|████████████████████████▏ | 8849/40903 [3:56:25<10:34:31, 1.19s/it][2025-04-22 05:43:56] (step=0008850) Train Loss: 6.4218, Train Steps/Sec: 0.84 + 22%|████████████████████████▎ | 8874/40903 [3:56:54<10:28:35, 1.18s/it][2025-04-22 05:44:25] (step=0008875) Train Loss: 6.3912, Train Steps/Sec: 0.85 + 22%|████████████████████████▎ | 8899/40903 [3:57:24<10:21:09, 1.16s/it][2025-04-22 05:44:55] (step=0008900) Train Loss: 6.4342, Train Steps/Sec: 0.84 + 22%|████████████████████████▍ | 8924/40903 [3:57:54<10:37:36, 1.20s/it][2025-04-22 05:45:25] (step=0008925) Train Loss: 6.3981, Train Steps/Sec: 0.84 + 22%|████████████████████████▌ | 8949/40903 [3:58:32<10:41:58, 1.21s/it][2025-04-22 05:46:03] (step=0008950) Train Loss: 6.4344, Train Steps/Sec: 0.66 + 22%|████████████████████████▌ | 8974/40903 [3:59:01<10:30:57, 1.19s/it][2025-04-22 05:46:33] (step=0008975) Train Loss: 6.4286, Train Steps/Sec: 0.84 + 22%|████████████████████████▋ | 8999/40903 [3:59:31<10:32:45, 1.19s/it][2025-04-22 05:47:02] (step=0009000) Train Loss: 6.4256, Train Steps/Sec: 0.84 + 22%|████████████████████████▋ | 9024/40903 [4:00:01<10:34:55, 1.20s/it][2025-04-22 05:47:32] (step=0009025) Train Loss: 6.4468, Train Steps/Sec: 0.84 + 22%|████████████████████████▊ | 9049/40903 [4:00:31<10:38:20, 1.20s/it][2025-04-22 05:48:02] (step=0009050) Train Loss: 6.4331, Train Steps/Sec: 0.84 + 22%|████████████████████████▊ | 9074/40903 [4:01:12<15:44:14, 1.78s/it][2025-04-22 05:48:43] (step=0009075) Train Loss: 6.4097, Train Steps/Sec: 0.60 + 22%|████████████████████████▉ | 9099/40903 [4:01:42<10:25:22, 1.18s/it][2025-04-22 05:49:13] (step=0009100) Train Loss: 6.4240, Train Steps/Sec: 0.84 + 22%|████████████████████████▉ | 9124/40903 [4:02:12<10:41:24, 1.21s/it][2025-04-22 05:49:43] (step=0009125) Train Loss: 6.4378, Train Steps/Sec: 0.84 + 22%|█████████████████████████ | 9149/40903 [4:02:48<10:51:48, 1.23s/it][2025-04-22 05:50:19] (step=0009150) Train Loss: 6.4367, Train Steps/Sec: 0.69 + 22%|█████████████████████████ | 9174/40903 [4:03:18<10:25:02, 1.18s/it][2025-04-22 05:50:49] (step=0009175) Train Loss: 6.4272, Train Steps/Sec: 0.84 + 22%|█████████████████████████▏ | 9199/40903 [4:03:47<10:12:41, 1.16s/it][2025-04-22 05:51:19] (step=0009200) Train Loss: 6.4581, Train Steps/Sec: 0.84 + 23%|█████████████████████████▎ | 9224/40903 [4:04:17<10:29:03, 1.19s/it][2025-04-22 05:51:48] (step=0009225) Train Loss: 6.4317, Train Steps/Sec: 0.85 + 23%|█████████████████████████▎ | 9249/40903 [4:04:46<10:20:57, 1.18s/it][2025-04-22 05:52:17] (step=0009250) Train Loss: 6.4104, Train Steps/Sec: 0.85 + 23%|█████████████████████████▍ | 9274/40903 [4:05:23<10:40:30, 1.22s/it][2025-04-22 05:52:54] (step=0009275) Train Loss: 6.4137, Train Steps/Sec: 0.68 + 23%|█████████████████████████▍ | 9299/40903 [4:05:58<10:16:58, 1.17s/it][2025-04-22 05:53:30] (step=0009300) Train Loss: 6.4222, Train Steps/Sec: 0.70 + 23%|█████████████████████████▌ | 9324/40903 [4:06:28<10:37:44, 1.21s/it][2025-04-22 05:53:59] (step=0009325) Train Loss: 6.4217, Train Steps/Sec: 0.85 + 23%|█████████████████████████▌ | 9349/40903 [4:06:58<10:28:35, 1.20s/it][2025-04-22 05:54:29] (step=0009350) Train Loss: 6.4479, Train Steps/Sec: 0.84 + 23%|█████████████████████████▋ | 9374/40903 [4:07:33<10:28:49, 1.20s/it][2025-04-22 05:55:05] (step=0009375) Train Loss: 6.3927, Train Steps/Sec: 0.70 + 23%|█████████████████████████▋ | 9399/40903 [4:08:03<10:11:17, 1.16s/it][2025-04-22 05:55:34] (step=0009400) Train Loss: 6.4228, Train Steps/Sec: 0.84 + 23%|█████████████████████████▊ | 9424/40903 [4:08:33<10:31:57, 1.20s/it][2025-04-22 05:56:04] (step=0009425) Train Loss: 6.4270, Train Steps/Sec: 0.85 + 23%|█████████████████████████▊ | 9449/40903 [4:09:02<10:14:37, 1.17s/it][2025-04-22 05:56:33] (step=0009450) Train Loss: 6.4358, Train Steps/Sec: 0.85 + 23%|█████████████████████████▉ | 9474/40903 [4:09:31<10:14:55, 1.17s/it][2025-04-22 05:57:03] (step=0009475) Train Loss: 6.4113, Train Steps/Sec: 0.85 + 23%|██████████████████████████ | 9499/40903 [4:10:01<10:15:55, 1.18s/it][2025-04-22 05:57:32] (step=0009500) Train Loss: 6.3655, Train Steps/Sec: 0.84 + 23%|██████████████████████████ | 9524/40903 [4:10:31<10:23:52, 1.19s/it][2025-04-22 05:58:02] (step=0009525) Train Loss: 6.4416, Train Steps/Sec: 0.84 + 23%|██████████████████████████▏ | 9549/40903 [4:11:00<10:20:13, 1.19s/it][2025-04-22 05:58:32] (step=0009550) Train Loss: 6.4586, Train Steps/Sec: 0.84 + 23%|██████████████████████████▏ | 9574/40903 [4:11:30<10:21:00, 1.19s/it][2025-04-22 05:59:01] (step=0009575) Train Loss: 6.4246, Train Steps/Sec: 0.84 + 23%|██████████████████████████▎ | 9599/40903 [4:12:00<10:12:04, 1.17s/it][2025-04-22 05:59:31] (step=0009600) Train Loss: 6.3927, Train Steps/Sec: 0.85 + 24%|██████████████████████████▎ | 9624/40903 [4:12:29<10:21:49, 1.19s/it][2025-04-22 06:00:00] (step=0009625) Train Loss: 6.4047, Train Steps/Sec: 0.85 + 24%|██████████████████████████▍ | 9649/40903 [4:12:59<10:17:59, 1.19s/it][2025-04-22 06:00:30] (step=0009650) Train Loss: 6.4218, Train Steps/Sec: 0.84 + 24%|██████████████████████████▍ | 9674/40903 [4:13:28<10:12:31, 1.18s/it][2025-04-22 06:01:00] (step=0009675) Train Loss: 6.4506, Train Steps/Sec: 0.84 + 24%|██████████████████████████▌ | 9699/40903 [4:13:58<10:03:32, 1.16s/it][2025-04-22 06:01:29] (step=0009700) Train Loss: 6.4236, Train Steps/Sec: 0.84 + 24%|██████████████████████████▋ | 9724/40903 [4:14:28<10:24:00, 1.20s/it][2025-04-22 06:01:59] (step=0009725) Train Loss: 6.4273, Train Steps/Sec: 0.85 + 24%|██████████████████████████▋ | 9749/40903 [4:14:57<10:11:55, 1.18s/it][2025-04-22 06:02:28] (step=0009750) Train Loss: 6.3873, Train Steps/Sec: 0.85 + 24%|██████████████████████████▊ | 9774/40903 [4:15:27<10:10:17, 1.18s/it][2025-04-22 06:02:58] (step=0009775) Train Loss: 6.4205, Train Steps/Sec: 0.85 + 24%|██████████████████████████▊ | 9799/40903 [4:15:56<10:04:04, 1.17s/it][2025-04-22 06:03:28] (step=0009800) Train Loss: 6.4406, Train Steps/Sec: 0.84 + 24%|██████████████████████████▉ | 9824/40903 [4:16:26<10:15:47, 1.19s/it][2025-04-22 06:03:57] (step=0009825) Train Loss: 6.4026, Train Steps/Sec: 0.85 + 24%|██████████████████████████▉ | 9849/40903 [4:16:55<10:12:19, 1.18s/it][2025-04-22 06:04:27] (step=0009850) Train Loss: 6.4222, Train Steps/Sec: 0.85 + 24%|███████████████████████████ | 9874/40903 [4:17:25<10:07:57, 1.18s/it][2025-04-22 06:04:56] (step=0009875) Train Loss: 6.4258, Train Steps/Sec: 0.84 + 24%|███████████████████████████ | 9899/40903 [4:17:55<10:03:04, 1.17s/it][2025-04-22 06:05:26] (step=0009900) Train Loss: 6.4088, Train Steps/Sec: 0.84 + 24%|███████████████████████████▏ | 9924/40903 [4:18:24<10:14:25, 1.19s/it][2025-04-22 06:05:56] (step=0009925) Train Loss: 6.4207, Train Steps/Sec: 0.85 + 24%|███████████████████████████▏ | 9949/40903 [4:18:54<10:12:23, 1.19s/it][2025-04-22 06:06:25] (step=0009950) Train Loss: 6.4157, Train Steps/Sec: 0.85 + 24%|███████████████████████████▎ | 9974/40903 [4:19:23<10:04:16, 1.17s/it][2025-04-22 06:06:54] (step=0009975) Train Loss: 6.4127, Train Steps/Sec: 0.85 + 24%|███████████████████████████▌ | 9999/40903 [4:19:53<9:56:56, 1.16s/it][2025-04-22 06:07:24] (step=0010000) Train Loss: 6.3861, Train Steps/Sec: 0.85 +You are using a model of type instructblip to instantiate a model of type blip-2. This is not supported for all configurations of models and can yield errors. +model_name_or_path:/tmp/haozhezhao/model/instructblip-flan-t5-xl +image_place_holder: +Some kwargs in processor config are unused and will not have any effect: num_query_tokens. +tokenizer length after expend 32102 +tokenizer length before expend 32102 +100%|████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████| 3/3 [02:54<00:00, 58.02s/it] +[2025-04-22 06:12:05] Finish Eval in 10000 steps...█████████████████████████████████████████████████████████████████████| 3/3 [02:53<00:00, 57.51s/it] +processing left-padding... +processing left-padding... +processing left-padding... +[2025-04-22 06:12:26] Saved checkpoint to checkpoint/BeforeFix_Blip_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_objects/002-GPT-XL/checkpoints/0010000.pt +[2025-04-22 06:12:29] Removed old checkpoint: checkpoint/BeforeFix_Blip_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_objects/002-GPT-XL/checkpoints/0006000.pt + 25%|███████████████████████████▏ | 10024/40903 [4:25:27<10:38:27, 1.24s/it][2025-04-22 06:12:58] (step=0010025) Train Loss: 6.3891, Train Steps/Sec: 0.07 + 25%|███████████████████████████▎ | 10049/40903 [4:25:57<10:16:16, 1.20s/it][2025-04-22 06:13:28] (step=0010050) Train Loss: 6.4807, Train Steps/Sec: 0.84 + 25%|███████████████████████████▎ | 10074/40903 [4:26:26<10:00:04, 1.17s/it][2025-04-22 06:13:58] (step=0010075) Train Loss: 6.4386, Train Steps/Sec: 0.85 + 25%|███████████████████████████▍ | 10099/40903 [4:26:56<10:01:50, 1.17s/it][2025-04-22 06:14:27] (step=0010100) Train Loss: 6.4287, Train Steps/Sec: 0.84 + 25%|███████████████████████████▍ | 10124/40903 [4:27:26<10:13:59, 1.20s/it][2025-04-22 06:14:57] (step=0010125) Train Loss: 6.4193, Train Steps/Sec: 0.84 + 25%|███████████████████████████▌ | 10149/40903 [4:27:55<10:03:55, 1.18s/it][2025-04-22 06:15:27] (step=0010150) Train Loss: 6.4057, Train Steps/Sec: 0.85 + 25%|███████████████████████████▌ | 10174/40903 [4:28:25<10:08:05, 1.19s/it][2025-04-22 06:15:56] (step=0010175) Train Loss: 6.4081, Train Steps/Sec: 0.85 + 25%|███████████████████████████▋ | 10199/40903 [4:28:55<10:04:05, 1.18s/it][2025-04-22 06:16:26] (step=0010200) Train Loss: 6.4423, Train Steps/Sec: 0.84 + 25%|███████████████████████████▋ | 10224/40903 [4:29:24<10:20:37, 1.21s/it][2025-04-22 06:16:56] (step=0010225) Train Loss: 6.3946, Train Steps/Sec: 0.84 + 25%|███████████████████████████▊ | 10249/40903 [4:29:54<10:08:18, 1.19s/it][2025-04-22 06:17:25] (step=0010250) Train Loss: 6.3643, Train Steps/Sec: 0.85 + 25%|████████████████████████████▏ | 10274/40903 [4:30:24<9:58:07, 1.17s/it][2025-04-22 06:17:55] (step=0010275) Train Loss: 6.4058, Train Steps/Sec: 0.85 + 25%|████████████████████████████▏ | 10299/40903 [4:30:53<9:59:17, 1.17s/it][2025-04-22 06:18:25] (step=0010300) Train Loss: 6.4206, Train Steps/Sec: 0.84 + 25%|████████████████████████████ | 10324/40903 [4:31:23<10:15:06, 1.21s/it][2025-04-22 06:18:54] (step=0010325) Train Loss: 6.3924, Train Steps/Sec: 0.85 + 25%|████████████████████████████▎ | 10349/40903 [4:31:52<9:59:16, 1.18s/it][2025-04-22 06:19:24] (step=0010350) Train Loss: 6.4308, Train Steps/Sec: 0.85 + 25%|████████████████████████████▏ | 10374/40903 [4:32:22<10:07:02, 1.19s/it][2025-04-22 06:19:53] (step=0010375) Train Loss: 6.4186, Train Steps/Sec: 0.85 + 25%|████████████████████████████▍ | 10399/40903 [4:32:51<9:52:12, 1.16s/it][2025-04-22 06:20:23] (step=0010400) Train Loss: 6.4604, Train Steps/Sec: 0.85 + 25%|████████████████████████████▎ | 10424/40903 [4:33:21<10:03:43, 1.19s/it][2025-04-22 06:20:52] (step=0010425) Train Loss: 6.4354, Train Steps/Sec: 0.85 + 26%|████████████████████████████▎ | 10449/40903 [4:33:51<10:08:04, 1.20s/it][2025-04-22 06:21:22] (step=0010450) Train Loss: 6.4488, Train Steps/Sec: 0.84 + 26%|████████████████████████████▋ | 10474/40903 [4:34:20<9:59:57, 1.18s/it][2025-04-22 06:21:52] (step=0010475) Train Loss: 6.4478, Train Steps/Sec: 0.84 + 26%|████████████████████████████▋ | 10499/40903 [4:34:50<9:52:53, 1.17s/it][2025-04-22 06:22:21] (step=0010500) Train Loss: 6.4293, Train Steps/Sec: 0.84 + 26%|████████████████████████████▌ | 10524/40903 [4:35:20<10:04:12, 1.19s/it][2025-04-22 06:22:51] (step=0010525) Train Loss: 6.4261, Train Steps/Sec: 0.84 + 26%|████████████████████████████▋ | 10549/40903 [4:35:49<10:04:24, 1.19s/it][2025-04-22 06:23:21] (step=0010550) Train Loss: 6.4337, Train Steps/Sec: 0.84 + 26%|████████████████████████████▉ | 10574/40903 [4:36:19<9:53:48, 1.17s/it][2025-04-22 06:23:50] (step=0010575) Train Loss: 6.4539, Train Steps/Sec: 0.85 + 26%|█████████████████████████████ | 10599/40903 [4:36:49<9:54:58, 1.18s/it][2025-04-22 06:24:20] (step=0010600) Train Loss: 6.4147, Train Steps/Sec: 0.84 + 26%|████████████████████████████▊ | 10624/40903 [4:37:18<10:12:20, 1.21s/it][2025-04-22 06:24:50] (step=0010625) Train Loss: 6.4386, Train Steps/Sec: 0.85 + 26%|█████████████████████████████▏ | 10649/40903 [4:37:48<9:59:26, 1.19s/it][2025-04-22 06:25:19] (step=0010650) Train Loss: 6.4373, Train Steps/Sec: 0.85 + 26%|█████████████████████████████▏ | 10674/40903 [4:38:18<9:50:25, 1.17s/it][2025-04-22 06:25:49] (step=0010675) Train Loss: 6.4343, Train Steps/Sec: 0.85 + 26%|█████████████████████████████▎ | 10699/40903 [4:38:47<9:53:40, 1.18s/it][2025-04-22 06:26:19] (step=0010700) Train Loss: 6.4245, Train Steps/Sec: 0.84 + 26%|█████████████████████████████ | 10724/40903 [4:39:17<10:03:36, 1.20s/it][2025-04-22 06:26:48] (step=0010725) Train Loss: 6.3789, Train Steps/Sec: 0.85 + 26%|█████████████████████████████▏ | 10749/40903 [4:39:47<10:02:01, 1.20s/it][2025-04-22 06:27:18] (step=0010750) Train Loss: 6.4271, Train Steps/Sec: 0.84 + 26%|█████████████████████████████▌ | 10774/40903 [4:40:16<9:47:57, 1.17s/it][2025-04-22 06:27:48] (step=0010775) Train Loss: 6.4291, Train Steps/Sec: 0.84 + 26%|█████████████████████████████▌ | 10799/40903 [4:40:46<9:51:50, 1.18s/it][2025-04-22 06:28:17] (step=0010800) Train Loss: 6.4098, Train Steps/Sec: 0.84 + 26%|█████████████████████████████▎ | 10824/40903 [4:41:16<10:04:13, 1.21s/it][2025-04-22 06:28:47] (step=0010825) Train Loss: 6.4728, Train Steps/Sec: 0.84 + 27%|█████████████████████████████▋ | 10849/40903 [4:41:45<9:52:15, 1.18s/it][2025-04-22 06:29:16] (step=0010850) Train Loss: 6.4141, Train Steps/Sec: 0.85 + 27%|█████████████████████████████▌ | 10874/40903 [4:42:21<10:22:50, 1.24s/it][2025-04-22 06:29:52] (step=0010875) Train Loss: 6.3910, Train Steps/Sec: 0.70 + 27%|█████████████████████████████▊ | 10899/40903 [4:42:50<9:43:15, 1.17s/it][2025-04-22 06:30:22] (step=0010900) Train Loss: 6.4552, Train Steps/Sec: 0.84 + 27%|█████████████████████████████▋ | 10924/40903 [4:43:20<10:01:16, 1.20s/it][2025-04-22 06:30:52] (step=0010925) Train Loss: 6.4424, Train Steps/Sec: 0.84 + 27%|█████████████████████████████▉ | 10949/40903 [4:43:50<9:53:38, 1.19s/it][2025-04-22 06:31:21] (step=0010950) Train Loss: 6.4464, Train Steps/Sec: 0.84 + 27%|██████████████████████████████ | 10974/40903 [4:44:20<9:49:57, 1.18s/it][2025-04-22 06:31:51] (step=0010975) Train Loss: 6.3941, Train Steps/Sec: 0.85 + 27%|██████████████████████████████ | 10999/40903 [4:44:49<9:51:03, 1.19s/it][2025-04-22 06:32:21] (step=0011000) Train Loss: 6.4556, Train Steps/Sec: 0.84 + 27%|█████████████████████████████▉ | 11024/40903 [4:45:19<10:07:18, 1.22s/it][2025-04-22 06:32:50] (step=0011025) Train Loss: 6.4135, Train Steps/Sec: 0.84 + 27%|██████████████████████████████▎ | 11049/40903 [4:45:49<9:52:41, 1.19s/it][2025-04-22 06:33:20] (step=0011050) Train Loss: 6.3777, Train Steps/Sec: 0.84 + 27%|██████████████████████████████▎ | 11074/40903 [4:46:18<9:48:11, 1.18s/it][2025-04-22 06:33:50] (step=0011075) Train Loss: 6.4229, Train Steps/Sec: 0.85 + 27%|██████████████████████████████▍ | 11099/40903 [4:46:48<9:45:06, 1.18s/it][2025-04-22 06:34:19] (step=0011100) Train Loss: 6.4202, Train Steps/Sec: 0.84 + 27%|██████████████████████████████▏ | 11124/40903 [4:47:18<10:04:34, 1.22s/it][2025-04-22 06:34:49] (step=0011125) Train Loss: 6.4361, Train Steps/Sec: 0.84 + 27%|██████████████████████████████▌ | 11149/40903 [4:47:47<9:44:17, 1.18s/it][2025-04-22 06:35:19] (step=0011150) Train Loss: 6.4084, Train Steps/Sec: 0.85 + 27%|██████████████████████████████▌ | 11174/40903 [4:48:17<9:50:50, 1.19s/it][2025-04-22 06:35:48] (step=0011175) Train Loss: 6.4532, Train Steps/Sec: 0.85 + 27%|██████████████████████████████▋ | 11199/40903 [4:48:47<9:36:57, 1.17s/it][2025-04-22 06:36:18] (step=0011200) Train Loss: 6.4345, Train Steps/Sec: 0.84 + 27%|██████████████████████████████▋ | 11224/40903 [4:49:16<9:49:39, 1.19s/it][2025-04-22 06:36:47] (step=0011225) Train Loss: 6.3897, Train Steps/Sec: 0.85 + 28%|██████████████████████████████▊ | 11249/40903 [4:49:46<9:51:54, 1.20s/it][2025-04-22 06:37:17] (step=0011250) Train Loss: 6.4212, Train Steps/Sec: 0.84 + 28%|██████████████████████████████▊ | 11274/40903 [4:50:15<9:34:34, 1.16s/it][2025-04-22 06:37:47] (step=0011275) Train Loss: 6.4085, Train Steps/Sec: 0.85 + 28%|██████████████████████████████▉ | 11299/40903 [4:50:45<9:40:54, 1.18s/it][2025-04-22 06:38:16] (step=0011300) Train Loss: 6.3998, Train Steps/Sec: 0.84 + 28%|███████████████████████████████ | 11324/40903 [4:51:15<9:51:45, 1.20s/it][2025-04-22 06:38:46] (step=0011325) Train Loss: 6.4308, Train Steps/Sec: 0.84 + 28%|███████████████████████████████ | 11349/40903 [4:51:44<9:42:14, 1.18s/it][2025-04-22 06:39:15] (step=0011350) Train Loss: 6.4311, Train Steps/Sec: 0.84 + 28%|███████████████████████████████▏ | 11374/40903 [4:52:14<9:34:53, 1.17s/it][2025-04-22 06:39:45] (step=0011375) Train Loss: 6.4183, Train Steps/Sec: 0.85 + 28%|███████████████████████████████▏ | 11399/40903 [4:52:43<9:33:31, 1.17s/it][2025-04-22 06:40:15] (step=0011400) Train Loss: 6.4378, Train Steps/Sec: 0.84 + 28%|███████████████████████████████▎ | 11424/40903 [4:53:13<9:56:25, 1.21s/it][2025-04-22 06:40:44] (step=0011425) Train Loss: 6.4317, Train Steps/Sec: 0.84 + 28%|███████████████████████████████▎ | 11449/40903 [4:53:43<9:44:15, 1.19s/it][2025-04-22 06:41:14] (step=0011450) Train Loss: 6.4018, Train Steps/Sec: 0.84 + 28%|███████████████████████████████▍ | 11474/40903 [4:54:12<9:32:27, 1.17s/it][2025-04-22 06:41:44] (step=0011475) Train Loss: 6.4352, Train Steps/Sec: 0.85 + 28%|███████████████████████████████▍ | 11499/40903 [4:54:42<9:32:14, 1.17s/it][2025-04-22 06:42:13] (step=0011500) Train Loss: 6.3977, Train Steps/Sec: 0.85 + 28%|███████████████████████████████▌ | 11524/40903 [4:55:11<9:46:01, 1.20s/it][2025-04-22 06:42:43] (step=0011525) Train Loss: 6.4193, Train Steps/Sec: 0.85 + 28%|███████████████████████████████▌ | 11549/40903 [4:55:41<9:39:21, 1.18s/it][2025-04-22 06:43:12] (step=0011550) Train Loss: 6.4396, Train Steps/Sec: 0.84 + 28%|███████████████████████████████▋ | 11574/40903 [4:56:11<9:40:28, 1.19s/it][2025-04-22 06:43:42] (step=0011575) Train Loss: 6.4451, Train Steps/Sec: 0.84 + 28%|███████████████████████████████▊ | 11599/40903 [4:56:40<9:27:13, 1.16s/it][2025-04-22 06:44:12] (step=0011600) Train Loss: 6.3761, Train Steps/Sec: 0.84 + 28%|███████████████████████████████▊ | 11624/40903 [4:57:10<9:49:49, 1.21s/it][2025-04-22 06:44:41] (step=0011625) Train Loss: 6.3951, Train Steps/Sec: 0.85 + 28%|███████████████████████████████▉ | 11649/40903 [4:57:39<9:39:11, 1.19s/it][2025-04-22 06:45:11] (step=0011650) Train Loss: 6.4014, Train Steps/Sec: 0.85 + 29%|███████████████████████████████▉ | 11674/40903 [4:58:09<9:27:40, 1.17s/it][2025-04-22 06:45:40] (step=0011675) Train Loss: 6.4283, Train Steps/Sec: 0.85 + 29%|████████████████████████████████ | 11699/40903 [4:58:39<9:27:05, 1.17s/it][2025-04-22 06:46:10] (step=0011700) Train Loss: 6.4254, Train Steps/Sec: 0.84 + 29%|████████████████████████████████ | 11724/40903 [4:59:08<9:44:26, 1.20s/it][2025-04-22 06:46:40] (step=0011725) Train Loss: 6.4144, Train Steps/Sec: 0.84 + 29%|████████████████████████████████▏ | 11749/40903 [4:59:38<9:42:05, 1.20s/it][2025-04-22 06:47:09] (step=0011750) Train Loss: 6.4357, Train Steps/Sec: 0.84 + 29%|████████████████████████████████▏ | 11774/40903 [5:00:08<9:35:51, 1.19s/it][2025-04-22 06:47:39] (step=0011775) Train Loss: 6.4305, Train Steps/Sec: 0.85 + 29%|████████████████████████████████▎ | 11799/40903 [5:00:37<9:22:53, 1.16s/it][2025-04-22 06:48:09] (step=0011800) Train Loss: 6.4306, Train Steps/Sec: 0.84 + 29%|████████████████████████████████▍ | 11824/40903 [5:01:07<9:35:22, 1.19s/it][2025-04-22 06:48:38] (step=0011825) Train Loss: 6.4221, Train Steps/Sec: 0.85 + 29%|████████████████████████████████▍ | 11849/40903 [5:01:36<9:39:27, 1.20s/it][2025-04-22 06:49:08] (step=0011850) Train Loss: 6.4638, Train Steps/Sec: 0.85 + 29%|████████████████████████████████▌ | 11874/40903 [5:02:06<9:24:17, 1.17s/it][2025-04-22 06:49:37] (step=0011875) Train Loss: 6.4034, Train Steps/Sec: 0.85 + 29%|████████████████████████████████▌ | 11899/40903 [5:02:36<9:31:33, 1.18s/it][2025-04-22 06:50:07] (step=0011900) Train Loss: 6.4011, Train Steps/Sec: 0.84 + 29%|████████████████████████████████▋ | 11924/40903 [5:03:05<9:41:55, 1.20s/it][2025-04-22 06:50:36] (step=0011925) Train Loss: 6.4348, Train Steps/Sec: 0.85 + 29%|████████████████████████████████▋ | 11949/40903 [5:03:35<9:23:16, 1.17s/it][2025-04-22 06:51:06] (step=0011950) Train Loss: 6.4666, Train Steps/Sec: 0.84 + 29%|████████████████████████████████▊ | 11974/40903 [5:04:04<9:27:23, 1.18s/it][2025-04-22 06:51:36] (step=0011975) Train Loss: 6.4195, Train Steps/Sec: 0.85 + 29%|████████████████████████████████▊ | 11999/40903 [5:04:34<9:35:48, 1.20s/it][2025-04-22 06:52:05] (step=0012000) Train Loss: 6.4212, Train Steps/Sec: 0.84 +You are using a model of type instructblip to instantiate a model of type blip-2. This is not supported for all configurations of models and can yield errors. +model_name_or_path:/tmp/haozhezhao/model/instructblip-flan-t5-xl +image_place_holder: +Some kwargs in processor config are unused and will not have any effect: num_query_tokens. +tokenizer length after expend 32102 +tokenizer length before expend 32102 +100%|████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████| 3/3 [02:53<00:00, 57.90s/it] +[2025-04-22 06:56:46] Finish Eval in 12000 steps...█████████████████████████████████████████████████████████████████████| 3/3 [02:53<00:00, 57.46s/it] +processing left-padding... +processing left-padding... +processing left-padding... +[2025-04-22 06:57:07] Saved checkpoint to checkpoint/BeforeFix_Blip_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_objects/002-GPT-XL/checkpoints/0012000.pt +[2025-04-22 06:57:09] Removed old checkpoint: checkpoint/BeforeFix_Blip_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_objects/002-GPT-XL/checkpoints/0008000.pt + 29%|████████████████████████████████▉ | 12024/40903 [5:10:08<9:51:57, 1.23s/it][2025-04-22 06:57:39] (step=0012025) Train Loss: 6.4100, Train Steps/Sec: 0.07 + 29%|████████████████████████████████▉ | 12049/40903 [5:10:37<9:25:02, 1.17s/it][2025-04-22 06:58:09] (step=0012050) Train Loss: 6.4286, Train Steps/Sec: 0.84 + 30%|█████████████████████████████████ | 12074/40903 [5:11:07<9:21:01, 1.17s/it][2025-04-22 06:58:38] (step=0012075) Train Loss: 6.4226, Train Steps/Sec: 0.84 + 30%|█████████████████████████████████▏ | 12099/40903 [5:11:37<9:17:55, 1.16s/it][2025-04-22 06:59:08] (step=0012100) Train Loss: 6.3808, Train Steps/Sec: 0.84 + 30%|█████████████████████████████████▏ | 12124/40903 [5:12:06<9:33:55, 1.20s/it][2025-04-22 06:59:37] (step=0012125) Train Loss: 6.4030, Train Steps/Sec: 0.85 + 30%|█████████████████████████████████▎ | 12149/40903 [5:12:36<9:31:58, 1.19s/it][2025-04-22 07:00:07] (step=0012150) Train Loss: 6.4008, Train Steps/Sec: 0.85 + 30%|█████████████████████████████████▎ | 12174/40903 [5:13:05<9:25:26, 1.18s/it][2025-04-22 07:00:37] (step=0012175) Train Loss: 6.4065, Train Steps/Sec: 0.84 + 30%|█████████████████████████████████▍ | 12199/40903 [5:13:35<9:20:22, 1.17s/it][2025-04-22 07:01:06] (step=0012200) Train Loss: 6.4182, Train Steps/Sec: 0.84 + 30%|█████████████████████████████████▍ | 12224/40903 [5:14:05<9:35:02, 1.20s/it][2025-04-22 07:01:36] (step=0012225) Train Loss: 6.4563, Train Steps/Sec: 0.84 + 30%|█████████████████████████████████▌ | 12249/40903 [5:14:34<9:31:50, 1.20s/it][2025-04-22 07:02:05] (step=0012250) Train Loss: 6.4354, Train Steps/Sec: 0.85 + 30%|█████████████████████████████████▌ | 12274/40903 [5:15:04<9:28:06, 1.19s/it][2025-04-22 07:02:35] (step=0012275) Train Loss: 6.4330, Train Steps/Sec: 0.85 + 30%|█████████████████████████████████▋ | 12299/40903 [5:15:33<9:20:31, 1.18s/it][2025-04-22 07:03:05] (step=0012300) Train Loss: 6.4426, Train Steps/Sec: 0.85 + 30%|█████████████████████████████████▋ | 12324/40903 [5:16:03<9:29:22, 1.20s/it][2025-04-22 07:03:34] (step=0012325) Train Loss: 6.4052, Train Steps/Sec: 0.84 + 30%|█████████████████████████████████▊ | 12349/40903 [5:16:32<9:24:32, 1.19s/it][2025-04-22 07:04:04] (step=0012350) Train Loss: 6.3799, Train Steps/Sec: 0.85 + 30%|█████████████████████████████████▉ | 12374/40903 [5:17:02<9:16:33, 1.17s/it][2025-04-22 07:04:33] (step=0012375) Train Loss: 6.4193, Train Steps/Sec: 0.85 + 30%|█████████████████████████████████▉ | 12399/40903 [5:17:32<9:21:22, 1.18s/it][2025-04-22 07:05:03] (step=0012400) Train Loss: 6.4232, Train Steps/Sec: 0.84 + 30%|██████████████████████████████████ | 12424/40903 [5:18:01<9:33:01, 1.21s/it][2025-04-22 07:05:33] (step=0012425) Train Loss: 6.4487, Train Steps/Sec: 0.84 + 30%|██████████████████████████████████ | 12449/40903 [5:18:31<9:19:18, 1.18s/it][2025-04-22 07:06:02] (step=0012450) Train Loss: 6.4128, Train Steps/Sec: 0.85 + 30%|██████████████████████████████████▏ | 12474/40903 [5:19:00<9:14:04, 1.17s/it][2025-04-22 07:06:32] (step=0012475) Train Loss: 6.4063, Train Steps/Sec: 0.84 + 31%|██████████████████████████████████▏ | 12499/40903 [5:19:30<9:15:08, 1.17s/it][2025-04-22 07:07:01] (step=0012500) Train Loss: 6.4220, Train Steps/Sec: 0.84 + 31%|██████████████████████████████████▎ | 12524/40903 [5:20:00<9:27:03, 1.20s/it][2025-04-22 07:07:31] (step=0012525) Train Loss: 6.4428, Train Steps/Sec: 0.85 + 31%|██████████████████████████████████▎ | 12549/40903 [5:20:29<9:26:58, 1.20s/it][2025-04-22 07:08:01] (step=0012550) Train Loss: 6.4673, Train Steps/Sec: 0.84 + 31%|██████████████████████████████████▍ | 12574/40903 [5:20:59<9:16:24, 1.18s/it][2025-04-22 07:08:30] (step=0012575) Train Loss: 6.4416, Train Steps/Sec: 0.84 + 31%|██████████████████████████████████▍ | 12599/40903 [5:21:29<9:15:19, 1.18s/it][2025-04-22 07:09:00] (step=0012600) Train Loss: 6.4433, Train Steps/Sec: 0.84 + 31%|██████████████████████████████████▌ | 12624/40903 [5:21:58<9:25:27, 1.20s/it][2025-04-22 07:09:29] (step=0012625) Train Loss: 6.4512, Train Steps/Sec: 0.85 + 31%|██████████████████████████████████▋ | 12649/40903 [5:22:28<9:19:28, 1.19s/it][2025-04-22 07:09:59] (step=0012650) Train Loss: 6.3996, Train Steps/Sec: 0.84 + 31%|██████████████████████████████████▋ | 12674/40903 [5:22:57<9:12:01, 1.17s/it][2025-04-22 07:10:29] (step=0012675) Train Loss: 6.4061, Train Steps/Sec: 0.85 + 31%|██████████████████████████████████▊ | 12699/40903 [5:23:27<9:08:58, 1.17s/it][2025-04-22 07:10:58] (step=0012700) Train Loss: 6.4391, Train Steps/Sec: 0.84 + 31%|██████████████████████████████████▊ | 12724/40903 [5:23:57<9:19:32, 1.19s/it][2025-04-22 07:11:28] (step=0012725) Train Loss: 6.3917, Train Steps/Sec: 0.85 + 31%|██████████████████████████████████▉ | 12749/40903 [5:24:26<9:15:21, 1.18s/it][2025-04-22 07:11:57] (step=0012750) Train Loss: 6.4380, Train Steps/Sec: 0.85 + 31%|██████████████████████████████████▉ | 12774/40903 [5:24:56<9:12:51, 1.18s/it][2025-04-22 07:12:27] (step=0012775) Train Loss: 6.4031, Train Steps/Sec: 0.85 + 31%|███████████████████████████████████ | 12799/40903 [5:25:25<9:12:37, 1.18s/it][2025-04-22 07:12:57] (step=0012800) Train Loss: 6.4686, Train Steps/Sec: 0.84 + 31%|███████████████████████████████████ | 12824/40903 [5:25:55<9:18:15, 1.19s/it][2025-04-22 07:13:26] (step=0012825) Train Loss: 6.4426, Train Steps/Sec: 0.85 + 31%|███████████████████████████████████▏ | 12849/40903 [5:26:24<9:17:01, 1.19s/it][2025-04-22 07:13:56] (step=0012850) Train Loss: 6.4520, Train Steps/Sec: 0.85 + 31%|███████████████████████████████████▎ | 12874/40903 [5:26:54<9:06:36, 1.17s/it][2025-04-22 07:14:25] (step=0012875) Train Loss: 6.4792, Train Steps/Sec: 0.85 + 32%|███████████████████████████████████▎ | 12899/40903 [5:27:24<9:08:58, 1.18s/it][2025-04-22 07:14:55] (step=0012900) Train Loss: 6.4546, Train Steps/Sec: 0.84 + 32%|███████████████████████████████████▍ | 12924/40903 [5:27:53<9:22:18, 1.21s/it][2025-04-22 07:15:25] (step=0012925) Train Loss: 6.4188, Train Steps/Sec: 0.84 + 32%|███████████████████████████████████▍ | 12949/40903 [5:28:23<9:10:49, 1.18s/it][2025-04-22 07:15:54] (step=0012950) Train Loss: 6.4538, Train Steps/Sec: 0.85 + 32%|███████████████████████████████████▌ | 12974/40903 [5:28:52<9:15:18, 1.19s/it][2025-04-22 07:16:24] (step=0012975) Train Loss: 6.4548, Train Steps/Sec: 0.84 + 32%|███████████████████████████████████▌ | 12999/40903 [5:29:22<9:02:57, 1.17s/it][2025-04-22 07:16:53] (step=0013000) Train Loss: 6.4355, Train Steps/Sec: 0.85 + 32%|███████████████████████████████████▋ | 13024/40903 [5:29:51<9:19:11, 1.20s/it][2025-04-22 07:17:23] (step=0013025) Train Loss: 6.4222, Train Steps/Sec: 0.85 + 32%|███████████████████████████████████▋ | 13049/40903 [5:30:21<9:11:11, 1.19s/it][2025-04-22 07:17:52] (step=0013050) Train Loss: 6.4359, Train Steps/Sec: 0.84 + 32%|███████████████████████████████████▊ | 13074/40903 [5:30:51<9:05:26, 1.18s/it][2025-04-22 07:18:22] (step=0013075) Train Loss: 6.3431, Train Steps/Sec: 0.85 + 32%|███████████████████████████████████▊ | 13099/40903 [5:31:20<8:58:53, 1.16s/it][2025-04-22 07:18:52] (step=0013100) Train Loss: 6.4442, Train Steps/Sec: 0.85 + 32%|███████████████████████████████████▉ | 13124/40903 [5:31:50<9:13:05, 1.19s/it][2025-04-22 07:19:21] (step=0013125) Train Loss: 6.4116, Train Steps/Sec: 0.85 + 32%|████████████████████████████████████ | 13149/40903 [5:32:19<9:04:33, 1.18s/it][2025-04-22 07:19:51] (step=0013150) Train Loss: 6.4290, Train Steps/Sec: 0.85 + 32%|████████████████████████████████████ | 13174/40903 [5:32:49<9:04:36, 1.18s/it][2025-04-22 07:20:20] (step=0013175) Train Loss: 6.3950, Train Steps/Sec: 0.85 + 32%|████████████████████████████████████▏ | 13199/40903 [5:33:18<9:02:54, 1.18s/it][2025-04-22 07:20:50] (step=0013200) Train Loss: 6.4166, Train Steps/Sec: 0.84 + 32%|████████████████████████████████████▏ | 13224/40903 [5:33:48<9:09:04, 1.19s/it][2025-04-22 07:21:19] (step=0013225) Train Loss: 6.3974, Train Steps/Sec: 0.85 + 32%|████████████████████████████████████▎ | 13249/40903 [5:34:18<9:11:12, 1.20s/it][2025-04-22 07:21:49] (step=0013250) Train Loss: 6.4241, Train Steps/Sec: 0.85 + 32%|████████████████████████████████████▎ | 13274/40903 [5:34:47<8:53:57, 1.16s/it][2025-04-22 07:22:18] (step=0013275) Train Loss: 6.4108, Train Steps/Sec: 0.85 + 33%|████████████████████████████████████▍ | 13299/40903 [5:35:17<8:59:20, 1.17s/it][2025-04-22 07:22:48] (step=0013300) Train Loss: 6.4200, Train Steps/Sec: 0.84 + 33%|████████████████████████████████████▍ | 13324/40903 [5:35:46<9:09:07, 1.19s/it][2025-04-22 07:23:18] (step=0013325) Train Loss: 6.4424, Train Steps/Sec: 0.85 + 33%|████████████████████████████████████▌ | 13349/40903 [5:36:16<9:07:46, 1.19s/it][2025-04-22 07:23:47] (step=0013350) Train Loss: 6.4253, Train Steps/Sec: 0.84 + 33%|████████████████████████████████████▌ | 13374/40903 [5:36:45<9:01:26, 1.18s/it][2025-04-22 07:24:17] (step=0013375) Train Loss: 6.4466, Train Steps/Sec: 0.85 + 33%|████████████████████████████████████▋ | 13399/40903 [5:37:15<9:01:08, 1.18s/it][2025-04-22 07:24:46] (step=0013400) Train Loss: 6.4001, Train Steps/Sec: 0.84 + 33%|████████████████████████████████████▊ | 13424/40903 [5:37:45<9:09:37, 1.20s/it][2025-04-22 07:25:16] (step=0013425) Train Loss: 6.4324, Train Steps/Sec: 0.85 + 33%|████████████████████████████████████▊ | 13449/40903 [5:38:14<9:06:00, 1.19s/it][2025-04-22 07:25:46] (step=0013450) Train Loss: 6.4272, Train Steps/Sec: 0.85 + 33%|████████████████████████████████████▉ | 13474/40903 [5:38:44<8:59:59, 1.18s/it][2025-04-22 07:26:15] (step=0013475) Train Loss: 6.4492, Train Steps/Sec: 0.84 + 33%|████████████████████████████████████▉ | 13499/40903 [5:39:13<8:56:07, 1.17s/it][2025-04-22 07:26:45] (step=0013500) Train Loss: 6.4020, Train Steps/Sec: 0.84 + 33%|█████████████████████████████████████ | 13524/40903 [5:39:43<9:09:40, 1.20s/it][2025-04-22 07:27:14] (step=0013525) Train Loss: 6.3503, Train Steps/Sec: 0.85 + 33%|█████████████████████████████████████ | 13549/40903 [5:40:13<9:08:51, 1.20s/it][2025-04-22 07:27:44] (step=0013550) Train Loss: 6.4107, Train Steps/Sec: 0.84 + 33%|█████████████████████████████████████▏ | 13574/40903 [5:40:42<8:53:20, 1.17s/it][2025-04-22 07:28:14] (step=0013575) Train Loss: 6.3923, Train Steps/Sec: 0.85 + 33%|█████████████████████████████████████▏ | 13599/40903 [5:41:12<8:54:03, 1.17s/it][2025-04-22 07:28:43] (step=0013600) Train Loss: 6.4483, Train Steps/Sec: 0.84 + 33%|█████████████████████████████████████▎ | 13624/40903 [5:41:42<9:05:05, 1.20s/it][2025-04-22 07:29:13] (step=0013625) Train Loss: 6.4241, Train Steps/Sec: 0.84 + 33%|█████████████████████████████████████▎ | 13649/40903 [5:42:12<9:03:56, 1.20s/it][2025-04-22 07:29:43] (step=0013650) Train Loss: 6.4074, Train Steps/Sec: 0.84 + 33%|█████████████████████████████████████▍ | 13674/40903 [5:42:41<9:00:18, 1.19s/it][2025-04-22 07:30:12] (step=0013675) Train Loss: 6.4610, Train Steps/Sec: 0.85 + 33%|█████████████████████████████████████▌ | 13699/40903 [5:43:11<8:51:38, 1.17s/it][2025-04-22 07:30:42] (step=0013700) Train Loss: 6.4352, Train Steps/Sec: 0.84 + 34%|█████████████████████████████████████▌ | 13724/40903 [5:43:40<8:59:49, 1.19s/it][2025-04-22 07:31:11] (step=0013725) Train Loss: 6.4173, Train Steps/Sec: 0.85 + 34%|█████████████████████████████████████▋ | 13749/40903 [5:44:10<8:52:52, 1.18s/it][2025-04-22 07:31:41] (step=0013750) Train Loss: 6.4301, Train Steps/Sec: 0.84 + 34%|█████████████████████████████████████▋ | 13774/40903 [5:44:40<9:01:07, 1.20s/it][2025-04-22 07:32:11] (step=0013775) Train Loss: 6.4418, Train Steps/Sec: 0.84 + 34%|█████████████████████████████████████▊ | 13799/40903 [5:45:09<8:51:48, 1.18s/it][2025-04-22 07:32:41] (step=0013800) Train Loss: 6.4124, Train Steps/Sec: 0.84 + 34%|█████████████████████████████████████▊ | 13824/40903 [5:45:39<8:58:58, 1.19s/it][2025-04-22 07:33:10] (step=0013825) Train Loss: 6.4055, Train Steps/Sec: 0.85 + 34%|█████████████████████████████████████▉ | 13849/40903 [5:46:09<8:57:26, 1.19s/it][2025-04-22 07:33:40] (step=0013850) Train Loss: 6.4105, Train Steps/Sec: 0.84 + 34%|█████████████████████████████████████▉ | 13874/40903 [5:46:38<8:56:15, 1.19s/it][2025-04-22 07:34:09] (step=0013875) Train Loss: 6.3948, Train Steps/Sec: 0.85 + 34%|██████████████████████████████████████ | 13899/40903 [5:47:08<8:44:25, 1.17s/it][2025-04-22 07:34:39] (step=0013900) Train Loss: 6.4106, Train Steps/Sec: 0.85 + 34%|██████████████████████████████████████▏ | 13924/40903 [5:47:37<8:52:45, 1.18s/it][2025-04-22 07:35:08] (step=0013925) Train Loss: 6.4258, Train Steps/Sec: 0.85 + 34%|██████████████████████████████████████▏ | 13949/40903 [5:48:07<8:47:03, 1.17s/it][2025-04-22 07:35:38] (step=0013950) Train Loss: 6.3876, Train Steps/Sec: 0.85 + 34%|██████████████████████████████████████▎ | 13974/40903 [5:48:36<8:49:47, 1.18s/it][2025-04-22 07:36:08] (step=0013975) Train Loss: 6.4621, Train Steps/Sec: 0.84 + 34%|██████████████████████████████████████▎ | 13999/40903 [5:49:06<8:47:32, 1.18s/it][2025-04-22 07:36:37] (step=0014000) Train Loss: 6.3985, Train Steps/Sec: 0.84 +You are using a model of type instructblip to instantiate a model of type blip-2. This is not supported for all configurations of models and can yield errors. +model_name_or_path:/tmp/haozhezhao/model/instructblip-flan-t5-xl +image_place_holder: +Some kwargs in processor config are unused and will not have any effect: num_query_tokens. +tokenizer length after expend 32102 +tokenizer length before expend 32102 +100%|████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████| 3/3 [02:54<00:00, 58.01s/it] +[2025-04-22 07:41:18] Finish Eval in 14000 steps...█████████████████████████████████████████████████████████████████████| 3/3 [02:53<00:00, 57.58s/it] +processing left-padding... +processing left-padding... +processing left-padding... +[2025-04-22 07:41:39] Saved checkpoint to checkpoint/BeforeFix_Blip_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_objects/002-GPT-XL/checkpoints/0014000.pt +[2025-04-22 07:41:41] Removed old checkpoint: checkpoint/BeforeFix_Blip_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_objects/002-GPT-XL/checkpoints/0010000.pt + 34%|██████████████████████████████████████▍ | 14024/40903 [5:54:40<9:02:06, 1.21s/it][2025-04-22 07:42:11] (step=0014025) Train Loss: 6.4693, Train Steps/Sec: 0.07 + 34%|██████████████████████████████████████▍ | 14049/40903 [5:55:16<8:51:41, 1.19s/it][2025-04-22 07:42:47] (step=0014050) Train Loss: 6.4313, Train Steps/Sec: 0.69 + 34%|██████████████████████████████████████▌ | 14074/40903 [5:55:46<8:48:40, 1.18s/it][2025-04-22 07:43:17] (step=0014075) Train Loss: 6.4180, Train Steps/Sec: 0.85 + 34%|██████████████████████████████████████▌ | 14099/40903 [5:56:15<8:41:42, 1.17s/it][2025-04-22 07:43:47] (step=0014100) Train Loss: 6.4193, Train Steps/Sec: 0.84 + 35%|██████████████████████████████████████▋ | 14124/40903 [5:56:45<9:08:12, 1.23s/it][2025-04-22 07:44:16] (step=0014125) Train Loss: 6.4037, Train Steps/Sec: 0.84 + 35%|██████████████████████████████████████▋ | 14149/40903 [5:57:15<8:45:13, 1.18s/it][2025-04-22 07:44:46] (step=0014150) Train Loss: 6.4190, Train Steps/Sec: 0.85 + 35%|██████████████████████████████████████▊ | 14174/40903 [5:57:44<8:47:48, 1.18s/it][2025-04-22 07:45:15] (step=0014175) Train Loss: 6.4553, Train Steps/Sec: 0.85 + 35%|██████████████████████████████████████▉ | 14199/40903 [5:58:14<8:45:38, 1.18s/it][2025-04-22 07:45:45] (step=0014200) Train Loss: 6.4538, Train Steps/Sec: 0.84 + 35%|██████████████████████████████████████▉ | 14224/40903 [5:58:43<8:54:59, 1.20s/it][2025-04-22 07:46:15] (step=0014225) Train Loss: 6.4411, Train Steps/Sec: 0.85 + 35%|███████████████████████████████████████ | 14249/40903 [5:59:13<8:47:31, 1.19s/it][2025-04-22 07:46:44] (step=0014250) Train Loss: 6.4285, Train Steps/Sec: 0.85 + 35%|███████████████████████████████████████ | 14274/40903 [5:59:42<8:41:05, 1.17s/it][2025-04-22 07:47:14] (step=0014275) Train Loss: 6.4339, Train Steps/Sec: 0.85 + 35%|███████████████████████████████████████▏ | 14299/40903 [6:00:12<8:36:08, 1.16s/it][2025-04-22 07:47:43] (step=0014300) Train Loss: 6.3886, Train Steps/Sec: 0.84 + 35%|███████████████████████████████████████▏ | 14324/40903 [6:00:48<9:03:19, 1.23s/it][2025-04-22 07:48:19] (step=0014325) Train Loss: 6.4338, Train Steps/Sec: 0.70 + 35%|███████████████████████████████████████▎ | 14349/40903 [6:01:18<8:49:04, 1.20s/it][2025-04-22 07:48:49] (step=0014350) Train Loss: 6.4192, Train Steps/Sec: 0.84 + 35%|███████████████████████████████████████▎ | 14374/40903 [6:01:47<8:33:56, 1.16s/it][2025-04-22 07:49:18] (step=0014375) Train Loss: 6.4338, Train Steps/Sec: 0.85 + 35%|███████████████████████████████████████▍ | 14399/40903 [6:02:23<8:37:09, 1.17s/it][2025-04-22 07:49:55] (step=0014400) Train Loss: 6.4285, Train Steps/Sec: 0.69 + 35%|███████████████████████████████████████▍ | 14424/40903 [6:02:53<9:02:48, 1.23s/it][2025-04-22 07:50:24] (step=0014425) Train Loss: 6.4423, Train Steps/Sec: 0.84 + 35%|███████████████████████████████████████▌ | 14449/40903 [6:03:23<8:49:32, 1.20s/it][2025-04-22 07:50:54] (step=0014450) Train Loss: 6.4259, Train Steps/Sec: 0.84 + 35%|███████████████████████████████████████▋ | 14474/40903 [6:04:03<8:40:19, 1.18s/it][2025-04-22 07:51:34] (step=0014475) Train Loss: 6.4280, Train Steps/Sec: 0.62 + 35%|███████████████████████████████████████▋ | 14499/40903 [6:04:33<8:35:07, 1.17s/it][2025-04-22 07:52:04] (step=0014500) Train Loss: 6.4153, Train Steps/Sec: 0.84 + 36%|███████████████████████████████████████▊ | 14524/40903 [6:05:03<8:51:09, 1.21s/it][2025-04-22 07:52:34] (step=0014525) Train Loss: 6.4609, Train Steps/Sec: 0.84 + 36%|███████████████████████████████████████▊ | 14549/40903 [6:05:32<8:37:52, 1.18s/it][2025-04-22 07:53:03] (step=0014550) Train Loss: 6.4032, Train Steps/Sec: 0.85 + 36%|███████████████████████████████████████▉ | 14574/40903 [6:06:13<8:33:55, 1.17s/it][2025-04-22 07:53:45] (step=0014575) Train Loss: 6.4299, Train Steps/Sec: 0.60 + 36%|███████████████████████████████████████▌ | 14599/40903 [6:06:50<11:06:30, 1.52s/it][2025-04-22 07:54:21] (step=0014600) Train Loss: 6.3866, Train Steps/Sec: 0.69 + 36%|████████████████████████████████████████ | 14624/40903 [6:07:19<8:47:09, 1.20s/it][2025-04-22 07:54:51] (step=0014625) Train Loss: 6.4117, Train Steps/Sec: 0.85 + 36%|████████████████████████████████████████ | 14649/40903 [6:07:49<8:40:06, 1.19s/it][2025-04-22 07:55:20] (step=0014650) Train Loss: 6.4182, Train Steps/Sec: 0.84 + 36%|████████████████████████████████████████▏ | 14674/40903 [6:08:19<8:32:52, 1.17s/it][2025-04-22 07:55:50] (step=0014675) Train Loss: 6.4219, Train Steps/Sec: 0.85 + 36%|████████████████████████████████████████▏ | 14699/40903 [6:08:48<8:32:08, 1.17s/it][2025-04-22 07:56:19] (step=0014700) Train Loss: 6.3981, Train Steps/Sec: 0.84 + 36%|████████████████████████████████████████▎ | 14724/40903 [6:09:18<8:43:51, 1.20s/it][2025-04-22 07:56:49] (step=0014725) Train Loss: 6.4415, Train Steps/Sec: 0.85 + 36%|████████████████████████████████████████▍ | 14749/40903 [6:09:47<8:37:35, 1.19s/it][2025-04-22 07:57:19] (step=0014750) Train Loss: 6.4521, Train Steps/Sec: 0.85 + 36%|████████████████████████████████████████▍ | 14774/40903 [6:10:17<8:36:01, 1.18s/it][2025-04-22 07:57:48] (step=0014775) Train Loss: 6.4300, Train Steps/Sec: 0.85 + 36%|████████████████████████████████████████▌ | 14799/40903 [6:10:46<8:26:44, 1.16s/it][2025-04-22 07:58:18] (step=0014800) Train Loss: 6.4256, Train Steps/Sec: 0.85 + 36%|████████████████████████████████████████▌ | 14824/40903 [6:11:16<8:40:42, 1.20s/it][2025-04-22 07:58:47] (step=0014825) Train Loss: 6.4434, Train Steps/Sec: 0.85 + 36%|████████████████████████████████████████▋ | 14849/40903 [6:11:45<8:40:40, 1.20s/it][2025-04-22 07:59:17] (step=0014850) Train Loss: 6.4050, Train Steps/Sec: 0.85 + 36%|████████████████████████████████████████▋ | 14874/40903 [6:12:15<8:33:18, 1.18s/it][2025-04-22 07:59:46] (step=0014875) Train Loss: 6.4632, Train Steps/Sec: 0.85 + 36%|████████████████████████████████████████▊ | 14899/40903 [6:12:45<8:34:27, 1.19s/it][2025-04-22 08:00:16] (step=0014900) Train Loss: 6.4066, Train Steps/Sec: 0.84 + 36%|████████████████████████████████████████▊ | 14924/40903 [6:13:14<8:36:03, 1.19s/it][2025-04-22 08:00:45] (step=0014925) Train Loss: 6.4634, Train Steps/Sec: 0.85 + 37%|████████████████████████████████████████▉ | 14949/40903 [6:13:44<8:29:36, 1.18s/it][2025-04-22 08:01:15] (step=0014950) Train Loss: 6.3919, Train Steps/Sec: 0.85 + 37%|█████████████████████████████████████████ | 14974/40903 [6:14:13<8:24:23, 1.17s/it][2025-04-22 08:01:44] (step=0014975) Train Loss: 6.4298, Train Steps/Sec: 0.85 + 37%|█████████████████████████████████████████ | 14999/40903 [6:14:43<8:28:57, 1.18s/it][2025-04-22 08:02:14] (step=0015000) Train Loss: 6.4106, Train Steps/Sec: 0.84 + 37%|█████████████████████████████████████████▏ | 15024/40903 [6:15:12<8:42:26, 1.21s/it][2025-04-22 08:02:44] (step=0015025) Train Loss: 6.4176, Train Steps/Sec: 0.85 + 37%|█████████████████████████████████████████▏ | 15049/40903 [6:15:42<8:29:38, 1.18s/it][2025-04-22 08:03:13] (step=0015050) Train Loss: 6.4434, Train Steps/Sec: 0.85 + 37%|█████████████████████████████████████████▎ | 15074/40903 [6:16:11<8:31:03, 1.19s/it][2025-04-22 08:03:43] (step=0015075) Train Loss: 6.4042, Train Steps/Sec: 0.85 + 37%|█████████████████████████████████████████▎ | 15099/40903 [6:16:41<8:26:12, 1.18s/it][2025-04-22 08:04:12] (step=0015100) Train Loss: 6.4115, Train Steps/Sec: 0.84 + 37%|█████████████████████████████████████████▍ | 15124/40903 [6:17:10<8:37:21, 1.20s/it][2025-04-22 08:04:42] (step=0015125) Train Loss: 6.4401, Train Steps/Sec: 0.85 + 37%|█████████████████████████████████████████▍ | 15149/40903 [6:17:40<8:28:23, 1.18s/it][2025-04-22 08:05:11] (step=0015150) Train Loss: 6.4093, Train Steps/Sec: 0.85 + 37%|█████████████████████████████████████████▌ | 15174/40903 [6:18:09<8:28:22, 1.19s/it][2025-04-22 08:05:41] (step=0015175) Train Loss: 6.4221, Train Steps/Sec: 0.85 + 37%|█████████████████████████████████████████▌ | 15199/40903 [6:18:39<8:20:14, 1.17s/it][2025-04-22 08:06:10] (step=0015200) Train Loss: 6.4515, Train Steps/Sec: 0.85 + 37%|█████████████████████████████████████████▋ | 15224/40903 [6:19:08<8:29:07, 1.19s/it][2025-04-22 08:06:40] (step=0015225) Train Loss: 6.3716, Train Steps/Sec: 0.85 + 37%|█████████████████████████████████████████▊ | 15249/40903 [6:19:38<8:30:49, 1.19s/it][2025-04-22 08:07:09] (step=0015250) Train Loss: 6.4346, Train Steps/Sec: 0.85 + 37%|█████████████████████████████████████████▊ | 15274/40903 [6:20:07<8:20:11, 1.17s/it][2025-04-22 08:07:39] (step=0015275) Train Loss: 6.3893, Train Steps/Sec: 0.85 + 37%|█████████████████████████████████████████▉ | 15299/40903 [6:20:37<8:19:43, 1.17s/it][2025-04-22 08:08:08] (step=0015300) Train Loss: 6.4593, Train Steps/Sec: 0.84 + 37%|█████████████████████████████████████████▉ | 15324/40903 [6:21:07<8:31:21, 1.20s/it][2025-04-22 08:08:38] (step=0015325) Train Loss: 6.4252, Train Steps/Sec: 0.85 + 38%|██████████████████████████████████████████ | 15349/40903 [6:21:36<8:21:32, 1.18s/it][2025-04-22 08:09:07] (step=0015350) Train Loss: 6.3844, Train Steps/Sec: 0.85 + 38%|██████████████████████████████████████████ | 15374/40903 [6:22:05<8:11:19, 1.15s/it][2025-04-22 08:09:37] (step=0015375) Train Loss: 6.4388, Train Steps/Sec: 0.85 + 38%|██████████████████████████████████████████▏ | 15399/40903 [6:22:35<8:18:36, 1.17s/it][2025-04-22 08:10:06] (step=0015400) Train Loss: 6.4044, Train Steps/Sec: 0.84 + 38%|██████████████████████████████████████████▏ | 15424/40903 [6:23:05<8:31:27, 1.20s/it][2025-04-22 08:10:36] (step=0015425) Train Loss: 6.4009, Train Steps/Sec: 0.85 + 38%|██████████████████████████████████████████▎ | 15449/40903 [6:23:34<8:18:20, 1.17s/it][2025-04-22 08:11:05] (step=0015450) Train Loss: 6.4127, Train Steps/Sec: 0.85 + 38%|██████████████████████████████████████████▎ | 15474/40903 [6:24:04<8:13:54, 1.17s/it][2025-04-22 08:11:35] (step=0015475) Train Loss: 6.4388, Train Steps/Sec: 0.85 + 38%|██████████████████████████████████████████▍ | 15499/40903 [6:24:33<8:17:00, 1.17s/it][2025-04-22 08:12:05] (step=0015500) Train Loss: 6.4290, Train Steps/Sec: 0.84 + 38%|██████████████████████████████████████████▌ | 15524/40903 [6:25:03<8:27:49, 1.20s/it][2025-04-22 08:12:34] (step=0015525) Train Loss: 6.4105, Train Steps/Sec: 0.85 + 38%|██████████████████████████████████████████▌ | 15549/40903 [6:25:33<8:16:26, 1.17s/it][2025-04-22 08:13:04] (step=0015550) Train Loss: 6.3995, Train Steps/Sec: 0.85 + 38%|██████████████████████████████████████████▋ | 15574/40903 [6:26:02<8:15:57, 1.17s/it][2025-04-22 08:13:33] (step=0015575) Train Loss: 6.3927, Train Steps/Sec: 0.85 + 38%|██████████████████████████████████████████▋ | 15599/40903 [6:26:31<8:10:26, 1.16s/it][2025-04-22 08:14:03] (step=0015600) Train Loss: 6.4137, Train Steps/Sec: 0.84 + 38%|██████████████████████████████████████████▊ | 15624/40903 [6:27:01<8:25:43, 1.20s/it][2025-04-22 08:14:32] (step=0015625) Train Loss: 6.4166, Train Steps/Sec: 0.84 + 38%|██████████████████████████████████████████▊ | 15649/40903 [6:27:31<8:25:11, 1.20s/it][2025-04-22 08:15:02] (step=0015650) Train Loss: 6.4215, Train Steps/Sec: 0.84 + 38%|██████████████████████████████████████████▉ | 15674/40903 [6:28:00<8:11:26, 1.17s/it][2025-04-22 08:15:32] (step=0015675) Train Loss: 6.4388, Train Steps/Sec: 0.84 + 38%|██████████████████████████████████████████▉ | 15699/40903 [6:28:30<8:12:50, 1.17s/it][2025-04-22 08:16:01] (step=0015700) Train Loss: 6.4517, Train Steps/Sec: 0.84 + 38%|███████████████████████████████████████████ | 15724/40903 [6:29:00<8:26:02, 1.21s/it][2025-04-22 08:16:31] (step=0015725) Train Loss: 6.4121, Train Steps/Sec: 0.84 + 39%|███████████████████████████████████████████ | 15749/40903 [6:29:29<8:13:21, 1.18s/it][2025-04-22 08:17:01] (step=0015750) Train Loss: 6.4222, Train Steps/Sec: 0.84 + 39%|███████████████████████████████████████████▏ | 15774/40903 [6:29:59<8:18:26, 1.19s/it][2025-04-22 08:17:30] (step=0015775) Train Loss: 6.4053, Train Steps/Sec: 0.84 + 39%|███████████████████████████████████████████▎ | 15799/40903 [6:30:29<8:10:21, 1.17s/it][2025-04-22 08:18:00] (step=0015800) Train Loss: 6.3975, Train Steps/Sec: 0.84 + 39%|███████████████████████████████████████████▎ | 15824/40903 [6:30:58<8:26:41, 1.21s/it][2025-04-22 08:18:30] (step=0015825) Train Loss: 6.4017, Train Steps/Sec: 0.84 + 39%|███████████████████████████████████████████▍ | 15849/40903 [6:31:28<8:20:06, 1.20s/it][2025-04-22 08:18:59] (step=0015850) Train Loss: 6.4214, Train Steps/Sec: 0.84 + 39%|███████████████████████████████████████████▍ | 15874/40903 [6:31:58<8:13:02, 1.18s/it][2025-04-22 08:19:29] (step=0015875) Train Loss: 6.4691, Train Steps/Sec: 0.84 + 39%|███████████████████████████████████████████▌ | 15899/40903 [6:32:28<8:03:42, 1.16s/it][2025-04-22 08:19:59] (step=0015900) Train Loss: 6.4200, Train Steps/Sec: 0.84 + 39%|███████████████████████████████████████████▌ | 15924/40903 [6:32:57<8:15:25, 1.19s/it][2025-04-22 08:20:29] (step=0015925) Train Loss: 6.3781, Train Steps/Sec: 0.85 + 39%|███████████████████████████████████████████▋ | 15949/40903 [6:33:27<8:17:51, 1.20s/it][2025-04-22 08:20:58] (step=0015950) Train Loss: 6.4423, Train Steps/Sec: 0.84 + 39%|███████████████████████████████████████████▋ | 15974/40903 [6:33:57<8:08:28, 1.18s/it][2025-04-22 08:21:28] (step=0015975) Train Loss: 6.4409, Train Steps/Sec: 0.85 + 39%|███████████████████████████████████████████▊ | 15999/40903 [6:34:26<8:05:57, 1.17s/it][2025-04-22 08:21:58] (step=0016000) Train Loss: 6.4469, Train Steps/Sec: 0.84 +You are using a model of type instructblip to instantiate a model of type blip-2. This is not supported for all configurations of models and can yield errors. +model_name_or_path:/tmp/haozhezhao/model/instructblip-flan-t5-xl +image_place_holder: +Some kwargs in processor config are unused and will not have any effect: num_query_tokens. +tokenizer length after expend 32102 +tokenizer length before expend 32102 +100%|████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████| 3/3 [02:53<00:00, 57.76s/it] +[2025-04-22 08:26:38] Finish Eval in 16000 steps...█████████████████████████████████████████████████████████████████████| 3/3 [02:52<00:00, 57.33s/it] +processing left-padding... +processing left-padding... +processing left-padding... +[2025-04-22 08:26:59] Saved checkpoint to checkpoint/BeforeFix_Blip_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_objects/002-GPT-XL/checkpoints/0016000.pt +[2025-04-22 08:27:01] Removed old checkpoint: checkpoint/BeforeFix_Blip_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_objects/002-GPT-XL/checkpoints/0012000.pt + 39%|███████████████████████████████████████████▉ | 16024/40903 [6:40:00<8:30:04, 1.23s/it][2025-04-22 08:27:31] (step=0016025) Train Loss: 6.3975, Train Steps/Sec: 0.08 + 39%|███████████████████████████████████████████▉ | 16049/40903 [6:40:29<8:06:19, 1.17s/it][2025-04-22 08:28:00] (step=0016050) Train Loss: 6.4103, Train Steps/Sec: 0.84 + 39%|████████████████████████████████████████████ | 16074/40903 [6:40:59<8:01:31, 1.16s/it][2025-04-22 08:28:30] (step=0016075) Train Loss: 6.3994, Train Steps/Sec: 0.85 + 39%|████████████████████████████████████████████ | 16099/40903 [6:41:29<8:07:07, 1.18s/it][2025-04-22 08:29:00] (step=0016100) Train Loss: 6.4197, Train Steps/Sec: 0.84 + 39%|████████████████████████████████████████████▏ | 16124/40903 [6:41:58<8:17:53, 1.21s/it][2025-04-22 08:29:29] (step=0016125) Train Loss: 6.4137, Train Steps/Sec: 0.85 + 39%|████████████████████████████████████████████▏ | 16149/40903 [6:42:28<8:12:40, 1.19s/it][2025-04-22 08:29:59] (step=0016150) Train Loss: 6.4180, Train Steps/Sec: 0.84 + 40%|████████████████████████████████████████████▎ | 16174/40903 [6:42:57<7:58:29, 1.16s/it][2025-04-22 08:30:29] (step=0016175) Train Loss: 6.4072, Train Steps/Sec: 0.85 + 40%|████████████████████████████████████████████▎ | 16199/40903 [6:43:27<8:01:00, 1.17s/it][2025-04-22 08:30:58] (step=0016200) Train Loss: 6.4198, Train Steps/Sec: 0.85 + 40%|████████████████████████████████████████████▍ | 16224/40903 [6:44:03<8:13:14, 1.20s/it][2025-04-22 08:31:34] (step=0016225) Train Loss: 6.4463, Train Steps/Sec: 0.70 + 40%|████████████████████████████████████████████▍ | 16249/40903 [6:44:32<8:09:22, 1.19s/it][2025-04-22 08:32:04] (step=0016250) Train Loss: 6.4450, Train Steps/Sec: 0.84 + 40%|████████████████████████████████████████████▌ | 16274/40903 [6:45:02<8:01:38, 1.17s/it][2025-04-22 08:32:33] (step=0016275) Train Loss: 6.3947, Train Steps/Sec: 0.85 + 40%|████████████████████████████████████████████▋ | 16299/40903 [6:45:32<8:06:49, 1.19s/it][2025-04-22 08:33:03] (step=0016300) Train Loss: 6.4278, Train Steps/Sec: 0.84 + 40%|████████████████████████████████████████████▋ | 16324/40903 [6:46:01<8:08:33, 1.19s/it][2025-04-22 08:33:33] (step=0016325) Train Loss: 6.4104, Train Steps/Sec: 0.85 + 40%|████████████████████████████████████████████▊ | 16349/40903 [6:46:31<7:59:57, 1.17s/it][2025-04-22 08:34:02] (step=0016350) Train Loss: 6.4286, Train Steps/Sec: 0.85 + 40%|████████████████████████████████████████████▊ | 16374/40903 [6:47:00<7:59:14, 1.17s/it][2025-04-22 08:34:32] (step=0016375) Train Loss: 6.3971, Train Steps/Sec: 0.85 + 40%|████████████████████████████████████████████▉ | 16399/40903 [6:47:30<7:54:57, 1.16s/it][2025-04-22 08:35:01] (step=0016400) Train Loss: 6.4306, Train Steps/Sec: 0.84 + 40%|████████████████████████████████████████████▉ | 16424/40903 [6:47:59<8:11:59, 1.21s/it][2025-04-22 08:35:31] (step=0016425) Train Loss: 6.4125, Train Steps/Sec: 0.85 + 40%|█████████████████████████████████████████████ | 16449/40903 [6:48:29<8:14:13, 1.21s/it][2025-04-22 08:36:00] (step=0016450) Train Loss: 6.4407, Train Steps/Sec: 0.84 + 40%|█████████████████████████████████████████████ | 16474/40903 [6:48:59<7:54:22, 1.17s/it][2025-04-22 08:36:30] (step=0016475) Train Loss: 6.4459, Train Steps/Sec: 0.85 + 40%|█████████████████████████████████████████████▏ | 16499/40903 [6:49:28<7:55:05, 1.17s/it][2025-04-22 08:37:00] (step=0016500) Train Loss: 6.4376, Train Steps/Sec: 0.84 + 40%|█████████████████████████████████████████████▏ | 16524/40903 [6:49:58<8:03:02, 1.19s/it][2025-04-22 08:37:29] (step=0016525) Train Loss: 6.4227, Train Steps/Sec: 0.85 + 40%|█████████████████████████████████████████████▎ | 16549/40903 [6:50:28<8:09:38, 1.21s/it][2025-04-22 08:37:59] (step=0016550) Train Loss: 6.3922, Train Steps/Sec: 0.84 + 41%|█████████████████████████████████████████████▍ | 16574/40903 [6:50:57<8:02:46, 1.19s/it][2025-04-22 08:38:28] (step=0016575) Train Loss: 6.4006, Train Steps/Sec: 0.85 + 41%|█████████████████████████████████████████████▍ | 16599/40903 [6:51:27<7:54:56, 1.17s/it][2025-04-22 08:38:58] (step=0016600) Train Loss: 6.4488, Train Steps/Sec: 0.85 + 41%|█████████████████████████████████████████████▌ | 16624/40903 [6:51:56<8:04:41, 1.20s/it][2025-04-22 08:39:28] (step=0016625) Train Loss: 6.4136, Train Steps/Sec: 0.84 + 41%|█████████████████████████████████████████████▌ | 16649/40903 [6:52:26<8:00:21, 1.19s/it][2025-04-22 08:39:57] (step=0016650) Train Loss: 6.4326, Train Steps/Sec: 0.84 + 41%|█████████████████████████████████████████████▋ | 16674/40903 [6:52:56<7:59:27, 1.19s/it][2025-04-22 08:40:27] (step=0016675) Train Loss: 6.4264, Train Steps/Sec: 0.84 + 41%|█████████████████████████████████████████████▋ | 16699/40903 [6:53:25<7:54:36, 1.18s/it][2025-04-22 08:40:57] (step=0016700) Train Loss: 6.4254, Train Steps/Sec: 0.84 + 41%|█████████████████████████████████████████████▊ | 16724/40903 [6:53:55<8:05:45, 1.21s/it][2025-04-22 08:41:26] (step=0016725) Train Loss: 6.4271, Train Steps/Sec: 0.85 + 41%|█████████████████████████████████████████████▊ | 16749/40903 [6:54:24<7:57:20, 1.19s/it][2025-04-22 08:41:56] (step=0016750) Train Loss: 6.4501, Train Steps/Sec: 0.85 + 41%|█████████████████████████████████████████████▉ | 16774/40903 [6:54:54<7:49:05, 1.17s/it][2025-04-22 08:42:25] (step=0016775) Train Loss: 6.4277, Train Steps/Sec: 0.85 + 41%|█████████████████████████████████████████████▉ | 16799/40903 [6:55:23<7:48:52, 1.17s/it][2025-04-22 08:42:55] (step=0016800) Train Loss: 6.4276, Train Steps/Sec: 0.84 + 41%|██████████████████████████████████████████████ | 16824/40903 [6:55:53<8:03:03, 1.20s/it][2025-04-22 08:43:24] (step=0016825) Train Loss: 6.4302, Train Steps/Sec: 0.85 + 41%|██████████████████████████████████████████████▏ | 16849/40903 [6:56:23<7:54:03, 1.18s/it][2025-04-22 08:43:54] (step=0016850) Train Loss: 6.3940, Train Steps/Sec: 0.85 + 41%|██████████████████████████████████████████████▏ | 16874/40903 [6:56:52<7:49:32, 1.17s/it][2025-04-22 08:44:23] (step=0016875) Train Loss: 6.4270, Train Steps/Sec: 0.85 + 41%|██████████████████████████████████████████████▎ | 16899/40903 [6:57:22<7:51:53, 1.18s/it][2025-04-22 08:44:53] (step=0016900) Train Loss: 6.4067, Train Steps/Sec: 0.84 + 41%|██████████████████████████████████████████████▎ | 16924/40903 [6:57:52<7:55:25, 1.19s/it][2025-04-22 08:45:23] (step=0016925) Train Loss: 6.3878, Train Steps/Sec: 0.85 + 41%|██████████████████████████████████████████████▍ | 16949/40903 [6:58:21<7:55:10, 1.19s/it][2025-04-22 08:45:53] (step=0016950) Train Loss: 6.4024, Train Steps/Sec: 0.84 + 41%|██████████████████████████████████████████████▍ | 16974/40903 [6:58:51<7:50:35, 1.18s/it][2025-04-22 08:46:22] (step=0016975) Train Loss: 6.3771, Train Steps/Sec: 0.84 + 42%|██████████████████████████████████████████████▌ | 16999/40903 [6:59:20<7:48:42, 1.18s/it][2025-04-22 08:46:52] (step=0017000) Train Loss: 6.4073, Train Steps/Sec: 0.85 + 42%|██████████████████████████████████████████████▌ | 17024/40903 [6:59:50<7:58:40, 1.20s/it][2025-04-22 08:47:21] (step=0017025) Train Loss: 6.4490, Train Steps/Sec: 0.84 + 42%|██████████████████████████████████████████████▋ | 17049/40903 [7:00:20<7:54:56, 1.19s/it][2025-04-22 08:47:51] (step=0017050) Train Loss: 6.4306, Train Steps/Sec: 0.85 + 42%|██████████████████████████████████████████████▊ | 17074/40903 [7:00:49<7:41:59, 1.16s/it][2025-04-22 08:48:20] (step=0017075) Train Loss: 6.4536, Train Steps/Sec: 0.85 + 42%|██████████████████████████████████████████████▊ | 17099/40903 [7:01:19<7:40:58, 1.16s/it][2025-04-22 08:48:50] (step=0017100) Train Loss: 6.4575, Train Steps/Sec: 0.84 + 42%|██████████████████████████████████████████████▉ | 17124/40903 [7:01:48<7:57:23, 1.20s/it][2025-04-22 08:49:19] (step=0017125) Train Loss: 6.4303, Train Steps/Sec: 0.85 + 42%|██████████████████████████████████████████████▉ | 17149/40903 [7:02:18<7:49:25, 1.19s/it][2025-04-22 08:49:49] (step=0017150) Train Loss: 6.4120, Train Steps/Sec: 0.84 + 42%|███████████████████████████████████████████████ | 17174/40903 [7:02:47<7:41:38, 1.17s/it][2025-04-22 08:50:19] (step=0017175) Train Loss: 6.3969, Train Steps/Sec: 0.84 + 42%|███████████████████████████████████████████████ | 17199/40903 [7:03:17<7:43:56, 1.17s/it][2025-04-22 08:50:49] (step=0017200) Train Loss: 6.4631, Train Steps/Sec: 0.84 + 42%|███████████████████████████████████████████████▏ | 17224/40903 [7:03:47<7:52:48, 1.20s/it][2025-04-22 08:51:18] (step=0017225) Train Loss: 6.3727, Train Steps/Sec: 0.85 + 42%|███████████████████████████████████████████████▏ | 17249/40903 [7:04:16<7:48:41, 1.19s/it][2025-04-22 08:51:48] (step=0017250) Train Loss: 6.3900, Train Steps/Sec: 0.85 + 42%|███████████████████████████████████████████████▎ | 17274/40903 [7:04:46<7:43:37, 1.18s/it][2025-04-22 08:52:17] (step=0017275) Train Loss: 6.4548, Train Steps/Sec: 0.85 + 42%|███████████████████████████████████████████████▎ | 17299/40903 [7:05:15<7:39:24, 1.17s/it][2025-04-22 08:52:47] (step=0017300) Train Loss: 6.4583, Train Steps/Sec: 0.84 + 42%|███████████████████████████████████████████████▍ | 17324/40903 [7:05:45<7:46:47, 1.19s/it][2025-04-22 08:53:16] (step=0017325) Train Loss: 6.4130, Train Steps/Sec: 0.85 + 42%|███████████████████████████████████████████████▌ | 17349/40903 [7:06:15<7:41:04, 1.17s/it][2025-04-22 08:53:46] (step=0017350) Train Loss: 6.4143, Train Steps/Sec: 0.85 + 42%|███████████████████████████████████████████████▌ | 17374/40903 [7:06:44<7:40:13, 1.17s/it][2025-04-22 08:54:15] (step=0017375) Train Loss: 6.4463, Train Steps/Sec: 0.84 + 43%|███████████████████████████████████████████████▋ | 17399/40903 [7:07:14<7:39:26, 1.17s/it][2025-04-22 08:54:45] (step=0017400) Train Loss: 6.3682, Train Steps/Sec: 0.84 + 43%|███████████████████████████████████████████████▋ | 17424/40903 [7:07:43<7:50:11, 1.20s/it][2025-04-22 08:55:15] (step=0017425) Train Loss: 6.4001, Train Steps/Sec: 0.85 + 43%|███████████████████████████████████████████████▊ | 17449/40903 [7:08:13<7:46:49, 1.19s/it][2025-04-22 08:55:44] (step=0017450) Train Loss: 6.4239, Train Steps/Sec: 0.85 + 43%|███████████████████████████████████████████████▊ | 17474/40903 [7:08:42<7:36:31, 1.17s/it][2025-04-22 08:56:14] (step=0017475) Train Loss: 6.4068, Train Steps/Sec: 0.85 + 43%|███████████████████████████████████████████████▉ | 17499/40903 [7:09:12<7:36:28, 1.17s/it][2025-04-22 08:56:44] (step=0017500) Train Loss: 6.4232, Train Steps/Sec: 0.83 + 43%|███████████████████████████████████████████████▉ | 17524/40903 [7:09:42<7:47:48, 1.20s/it][2025-04-22 08:57:13] (step=0017525) Train Loss: 6.4551, Train Steps/Sec: 0.85 + 43%|████████████████████████████████████████████████ | 17549/40903 [7:10:12<7:45:38, 1.20s/it][2025-04-22 08:57:43] (step=0017550) Train Loss: 6.4530, Train Steps/Sec: 0.85 + 43%|████████████████████████████████████████████████ | 17574/40903 [7:10:41<7:35:04, 1.17s/it][2025-04-22 08:58:12] (step=0017575) Train Loss: 6.4207, Train Steps/Sec: 0.85 + 43%|████████████████████████████████████████████████▏ | 17599/40903 [7:11:11<7:34:48, 1.17s/it][2025-04-22 08:58:42] (step=0017600) Train Loss: 6.4458, Train Steps/Sec: 0.85 + 43%|████████████████████████████████████████████████▎ | 17624/40903 [7:11:40<7:44:10, 1.20s/it][2025-04-22 08:59:11] (step=0017625) Train Loss: 6.4037, Train Steps/Sec: 0.85 + 43%|████████████████████████████████████████████████▎ | 17649/40903 [7:12:10<7:37:21, 1.18s/it][2025-04-22 08:59:41] (step=0017650) Train Loss: 6.4181, Train Steps/Sec: 0.85 + 43%|████████████████████████████████████████████████▍ | 17674/40903 [7:12:39<7:34:28, 1.17s/it][2025-04-22 09:00:10] (step=0017675) Train Loss: 6.3989, Train Steps/Sec: 0.85 + 43%|████████████████████████████████████████████████▍ | 17699/40903 [7:13:09<7:36:28, 1.18s/it][2025-04-22 09:00:40] (step=0017700) Train Loss: 6.4196, Train Steps/Sec: 0.84 + 43%|████████████████████████████████████████████████▌ | 17724/40903 [7:13:38<7:43:41, 1.20s/it][2025-04-22 09:01:10] (step=0017725) Train Loss: 6.4124, Train Steps/Sec: 0.85 + 43%|████████████████████████████████████████████████▌ | 17749/40903 [7:14:08<7:40:59, 1.19s/it][2025-04-22 09:01:39] (step=0017750) Train Loss: 6.4118, Train Steps/Sec: 0.84 + 43%|████████████████████████████████████████████████▋ | 17774/40903 [7:14:38<7:31:06, 1.17s/it][2025-04-22 09:02:09] (step=0017775) Train Loss: 6.3962, Train Steps/Sec: 0.85 + 44%|████████████████████████████████████████████████▋ | 17799/40903 [7:15:07<7:33:17, 1.18s/it][2025-04-22 09:02:38] (step=0017800) Train Loss: 6.4456, Train Steps/Sec: 0.84 + 44%|████████████████████████████████████████████████▊ | 17824/40903 [7:15:37<7:38:22, 1.19s/it][2025-04-22 09:03:08] (step=0017825) Train Loss: 6.4216, Train Steps/Sec: 0.85 + 44%|████████████████████████████████████████████████▊ | 17849/40903 [7:16:06<7:33:05, 1.18s/it][2025-04-22 09:03:37] (step=0017850) Train Loss: 6.4062, Train Steps/Sec: 0.85 + 44%|████████████████████████████████████████████████▉ | 17874/40903 [7:16:36<7:30:46, 1.17s/it][2025-04-22 09:04:07] (step=0017875) Train Loss: 6.3928, Train Steps/Sec: 0.84 + 44%|█████████████████████████████████████████████████ | 17899/40903 [7:17:05<7:31:02, 1.18s/it][2025-04-22 09:04:37] (step=0017900) Train Loss: 6.4266, Train Steps/Sec: 0.84 + 44%|█████████████████████████████████████████████████ | 17924/40903 [7:17:35<7:40:47, 1.20s/it][2025-04-22 09:05:06] (step=0017925) Train Loss: 6.4278, Train Steps/Sec: 0.85 + 44%|█████████████████████████████████████████████████▏ | 17949/40903 [7:18:05<7:37:52, 1.20s/it][2025-04-22 09:05:36] (step=0017950) Train Loss: 6.4863, Train Steps/Sec: 0.84 + 44%|█████████████████████████████████████████████████▏ | 17974/40903 [7:18:34<7:30:09, 1.18s/it][2025-04-22 09:06:05] (step=0017975) Train Loss: 6.4138, Train Steps/Sec: 0.85 + 44%|█████████████████████████████████████████████████▎ | 17999/40903 [7:19:03<7:21:20, 1.16s/it][2025-04-22 09:06:35] (step=0018000) Train Loss: 6.3507, Train Steps/Sec: 0.85 +You are using a model of type instructblip to instantiate a model of type blip-2. This is not supported for all configurations of models and can yield errors. +model_name_or_path:/tmp/haozhezhao/model/instructblip-flan-t5-xl +image_place_holder: +Some kwargs in processor config are unused and will not have any effect: num_query_tokens. +tokenizer length after expend 32102 +tokenizer length before expend 32102 +100%|████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████| 3/3 [02:54<00:00, 58.26s/it] +[2025-04-22 09:11:17] Finish Eval in 18000 steps...█████████████████████████████████████████████████████████████████████| 3/3 [02:54<00:00, 57.88s/it] +processing left-padding... +processing left-padding... +processing left-padding... +[2025-04-22 09:11:39] Saved checkpoint to checkpoint/BeforeFix_Blip_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_objects/002-GPT-XL/checkpoints/0018000.pt +[2025-04-22 09:11:41] Removed old checkpoint: checkpoint/BeforeFix_Blip_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_objects/002-GPT-XL/checkpoints/0014000.pt + 44%|█████████████████████████████████████████████████▎ | 18024/40903 [7:24:39<7:47:54, 1.23s/it][2025-04-22 09:12:10] (step=0018025) Train Loss: 6.4199, Train Steps/Sec: 0.07 + 44%|█████████████████████████████████████████████████▍ | 18049/40903 [7:25:09<7:32:23, 1.19s/it][2025-04-22 09:12:40] (step=0018050) Train Loss: 6.3951, Train Steps/Sec: 0.85 + 44%|█████████████████████████████████████████████████▍ | 18074/40903 [7:25:38<7:29:43, 1.18s/it][2025-04-22 09:13:09] (step=0018075) Train Loss: 6.4203, Train Steps/Sec: 0.85 + 44%|█████████████████████████████████████████████████▌ | 18099/40903 [7:26:07<7:24:33, 1.17s/it][2025-04-22 09:13:39] (step=0018100) Train Loss: 6.4566, Train Steps/Sec: 0.85 + 44%|█████████████████████████████████████████████████▋ | 18124/40903 [7:26:37<7:39:19, 1.21s/it][2025-04-22 09:14:09] (step=0018125) Train Loss: 6.4027, Train Steps/Sec: 0.84 + 44%|█████████████████████████████████████████████████▋ | 18149/40903 [7:27:07<7:33:29, 1.20s/it][2025-04-22 09:14:38] (step=0018150) Train Loss: 6.4142, Train Steps/Sec: 0.85 + 44%|█████████████████████████████████████████████████▊ | 18174/40903 [7:27:36<7:27:34, 1.18s/it][2025-04-22 09:15:08] (step=0018175) Train Loss: 6.4036, Train Steps/Sec: 0.85 + 44%|█████████████████████████████████████████████████▊ | 18199/40903 [7:28:06<7:22:48, 1.17s/it][2025-04-22 09:15:37] (step=0018200) Train Loss: 6.3972, Train Steps/Sec: 0.85 + 45%|█████████████████████████████████████████████████▉ | 18224/40903 [7:28:35<7:35:23, 1.20s/it][2025-04-22 09:16:07] (step=0018225) Train Loss: 6.3905, Train Steps/Sec: 0.85 + 45%|█████████████████████████████████████████████████▉ | 18249/40903 [7:29:05<7:26:29, 1.18s/it][2025-04-22 09:16:36] (step=0018250) Train Loss: 6.4210, Train Steps/Sec: 0.85 + 45%|██████████████████████████████████████████████████ | 18274/40903 [7:29:34<7:27:43, 1.19s/it][2025-04-22 09:17:06] (step=0018275) Train Loss: 6.4290, Train Steps/Sec: 0.85 + 45%|██████████████████████████████████████████████████ | 18299/40903 [7:30:04<7:17:49, 1.16s/it][2025-04-22 09:17:35] (step=0018300) Train Loss: 6.3897, Train Steps/Sec: 0.85 + 45%|██████████████████████████████████████████████████▏ | 18324/40903 [7:30:33<7:34:49, 1.21s/it][2025-04-22 09:18:05] (step=0018325) Train Loss: 6.3823, Train Steps/Sec: 0.84 + 45%|██████████████████████████████████████████████████▏ | 18349/40903 [7:31:03<7:20:49, 1.17s/it][2025-04-22 09:18:34] (step=0018350) Train Loss: 6.4251, Train Steps/Sec: 0.85 + 45%|██████████████████████████████████████████████████▎ | 18374/40903 [7:31:32<7:21:08, 1.17s/it][2025-04-22 09:19:04] (step=0018375) Train Loss: 6.4132, Train Steps/Sec: 0.85 + 45%|██████████████████████████████████████████████████▍ | 18399/40903 [7:32:02<7:19:01, 1.17s/it][2025-04-22 09:19:33] (step=0018400) Train Loss: 6.4282, Train Steps/Sec: 0.84 + 45%|██████████████████████████████████████████████████▍ | 18424/40903 [7:32:32<7:22:33, 1.18s/it][2025-04-22 09:20:03] (step=0018425) Train Loss: 6.4270, Train Steps/Sec: 0.85 + 45%|██████████████████████████████████████████████████▌ | 18449/40903 [7:33:01<7:30:29, 1.20s/it][2025-04-22 09:20:32] (step=0018450) Train Loss: 6.4425, Train Steps/Sec: 0.84 + 45%|██████████████████████████████████████████████████▌ | 18474/40903 [7:33:31<7:18:10, 1.17s/it][2025-04-22 09:21:02] (step=0018475) Train Loss: 6.4016, Train Steps/Sec: 0.85 + 45%|██████████████████████████████████████████████████▋ | 18499/40903 [7:34:00<7:16:13, 1.17s/it][2025-04-22 09:21:32] (step=0018500) Train Loss: 6.3882, Train Steps/Sec: 0.84 + 45%|██████████████████████████████████████████████████▋ | 18524/40903 [7:34:30<7:25:33, 1.19s/it][2025-04-22 09:22:01] (step=0018525) Train Loss: 6.3979, Train Steps/Sec: 0.85 + 45%|██████████████████████████████████████████████████▊ | 18549/40903 [7:35:00<7:26:37, 1.20s/it][2025-04-22 09:22:31] (step=0018550) Train Loss: 6.4016, Train Steps/Sec: 0.84 + 45%|██████████████████████████████████████████████████▊ | 18574/40903 [7:35:29<7:19:14, 1.18s/it][2025-04-22 09:23:00] (step=0018575) Train Loss: 6.4785, Train Steps/Sec: 0.84 + 45%|██████████████████████████████████████████████████▉ | 18599/40903 [7:35:59<7:12:06, 1.16s/it][2025-04-22 09:23:30] (step=0018600) Train Loss: 6.4318, Train Steps/Sec: 0.85 + 46%|██████████████████████████████████████████████████▉ | 18624/40903 [7:36:28<7:31:00, 1.21s/it][2025-04-22 09:24:00] (step=0018625) Train Loss: 6.3953, Train Steps/Sec: 0.84 + 46%|███████████████████████████████████████████████████ | 18649/40903 [7:36:58<7:19:57, 1.19s/it][2025-04-22 09:24:29] (step=0018650) Train Loss: 6.4094, Train Steps/Sec: 0.84 + 46%|███████████████████████████████████████████████████▏ | 18674/40903 [7:37:28<7:22:25, 1.19s/it][2025-04-22 09:24:59] (step=0018675) Train Loss: 6.4348, Train Steps/Sec: 0.84 + 46%|███████████████████████████████████████████████████▏ | 18699/40903 [7:37:57<7:11:31, 1.17s/it][2025-04-22 09:25:28] (step=0018700) Train Loss: 6.3841, Train Steps/Sec: 0.85 + 46%|███████████████████████████████████████████████████▎ | 18724/40903 [7:38:27<7:23:55, 1.20s/it][2025-04-22 09:25:58] (step=0018725) Train Loss: 6.3804, Train Steps/Sec: 0.85 + 46%|███████████████████████████████████████████████████▎ | 18749/40903 [7:38:57<7:21:13, 1.19s/it][2025-04-22 09:26:28] (step=0018750) Train Loss: 6.4320, Train Steps/Sec: 0.84 + 46%|███████████████████████████████████████████████████▍ | 18774/40903 [7:39:26<7:12:48, 1.17s/it][2025-04-22 09:26:57] (step=0018775) Train Loss: 6.3998, Train Steps/Sec: 0.84 + 46%|███████████████████████████████████████████████████▍ | 18799/40903 [7:39:56<7:03:12, 1.15s/it][2025-04-22 09:27:27] (step=0018800) Train Loss: 6.4052, Train Steps/Sec: 0.84 + 46%|███████████████████████████████████████████████████▌ | 18824/40903 [7:40:25<7:25:40, 1.21s/it][2025-04-22 09:27:57] (step=0018825) Train Loss: 6.4713, Train Steps/Sec: 0.84 + 46%|███████████████████████████████████████████████████▌ | 18849/40903 [7:40:55<7:13:59, 1.18s/it][2025-04-22 09:28:26] (step=0018850) Train Loss: 6.4556, Train Steps/Sec: 0.84 + 46%|███████████████████████████████████████████████████▋ | 18874/40903 [7:41:25<7:13:23, 1.18s/it][2025-04-22 09:28:56] (step=0018875) Train Loss: 6.4426, Train Steps/Sec: 0.84 + 46%|███████████████████████████████████████████████████▋ | 18899/40903 [7:41:54<7:06:32, 1.16s/it][2025-04-22 09:29:26] (step=0018900) Train Loss: 6.4001, Train Steps/Sec: 0.85 + 46%|███████████████████████████████████████████████████▊ | 18924/40903 [7:42:24<7:16:50, 1.19s/it][2025-04-22 09:29:55] (step=0018925) Train Loss: 6.4218, Train Steps/Sec: 0.85 + 46%|███████████████████████████████████████████████████▉ | 18949/40903 [7:42:54<7:15:33, 1.19s/it][2025-04-22 09:30:25] (step=0018950) Train Loss: 6.4184, Train Steps/Sec: 0.84 + 46%|███████████████████████████████████████████████████▉ | 18974/40903 [7:43:23<7:09:01, 1.17s/it][2025-04-22 09:30:55] (step=0018975) Train Loss: 6.4011, Train Steps/Sec: 0.85 + 46%|████████████████████████████████████████████████████ | 18999/40903 [7:43:53<7:08:41, 1.17s/it][2025-04-22 09:31:24] (step=0019000) Train Loss: 6.3984, Train Steps/Sec: 0.85 + 47%|████████████████████████████████████████████████████ | 19024/40903 [7:44:22<7:16:49, 1.20s/it][2025-04-22 09:31:54] (step=0019025) Train Loss: 6.3947, Train Steps/Sec: 0.84 + 47%|████████████████████████████████████████████████████▏ | 19049/40903 [7:44:52<7:13:36, 1.19s/it][2025-04-22 09:32:23] (step=0019050) Train Loss: 6.4068, Train Steps/Sec: 0.85 + 47%|████████████████████████████████████████████████████▏ | 19074/40903 [7:45:22<7:08:55, 1.18s/it][2025-04-22 09:32:53] (step=0019075) Train Loss: 6.4159, Train Steps/Sec: 0.84 + 47%|████████████████████████████████████████████████████▎ | 19099/40903 [7:45:51<7:02:38, 1.16s/it][2025-04-22 09:33:23] (step=0019100) Train Loss: 6.4152, Train Steps/Sec: 0.84 + 47%|███████████████████████████████████████████████████▉ | 19124/40903 [7:46:28<13:38:33, 2.26s/it][2025-04-22 09:33:59] (step=0019125) Train Loss: 6.4129, Train Steps/Sec: 0.68 + 47%|████████████████████████████████████████████████████▍ | 19149/40903 [7:46:58<7:08:36, 1.18s/it][2025-04-22 09:34:29] (step=0019150) Train Loss: 6.3933, Train Steps/Sec: 0.85 + 47%|████████████████████████████████████████████████████▌ | 19174/40903 [7:47:27<7:05:23, 1.17s/it][2025-04-22 09:34:58] (step=0019175) Train Loss: 6.3753, Train Steps/Sec: 0.85 + 47%|████████████████████████████████████████████████████▌ | 19199/40903 [7:47:57<7:05:55, 1.18s/it][2025-04-22 09:35:28] (step=0019200) Train Loss: 6.4270, Train Steps/Sec: 0.83 + 47%|████████████████████████████████████████████████████▋ | 19224/40903 [7:48:27<7:13:39, 1.20s/it][2025-04-22 09:35:58] (step=0019225) Train Loss: 6.4468, Train Steps/Sec: 0.84 + 47%|████████████████████████████████████████████████████▋ | 19249/40903 [7:48:56<7:02:51, 1.17s/it][2025-04-22 09:36:27] (step=0019250) Train Loss: 6.4205, Train Steps/Sec: 0.85 + 47%|████████████████████████████████████████████████████▊ | 19274/40903 [7:49:26<7:03:17, 1.17s/it][2025-04-22 09:36:57] (step=0019275) Train Loss: 6.4459, Train Steps/Sec: 0.84 + 47%|████████████████████████████████████████████████████▊ | 19299/40903 [7:49:55<6:57:47, 1.16s/it][2025-04-22 09:37:27] (step=0019300) Train Loss: 6.3945, Train Steps/Sec: 0.84 + 47%|████████████████████████████████████████████████████▍ | 19324/40903 [7:50:31<14:50:23, 2.48s/it][2025-04-22 09:38:02] (step=0019325) Train Loss: 6.4342, Train Steps/Sec: 0.70 + 47%|████████████████████████████████████████████████████▉ | 19349/40903 [7:51:01<7:07:40, 1.19s/it][2025-04-22 09:38:32] (step=0019350) Train Loss: 6.4025, Train Steps/Sec: 0.84 + 47%|█████████████████████████████████████████████████████ | 19374/40903 [7:51:30<7:02:20, 1.18s/it][2025-04-22 09:39:02] (step=0019375) Train Loss: 6.3957, Train Steps/Sec: 0.84 + 47%|█████████████████████████████████████████████████████ | 19399/40903 [7:52:00<7:04:30, 1.18s/it][2025-04-22 09:39:31] (step=0019400) Train Loss: 6.4348, Train Steps/Sec: 0.84 + 47%|█████████████████████████████████████████████████████▏ | 19424/40903 [7:52:30<7:14:03, 1.21s/it][2025-04-22 09:40:01] (step=0019425) Train Loss: 6.4035, Train Steps/Sec: 0.85 + 48%|█████████████████████████████████████████████████████▎ | 19449/40903 [7:52:59<6:59:57, 1.17s/it][2025-04-22 09:40:30] (step=0019450) Train Loss: 6.4133, Train Steps/Sec: 0.85 + 48%|█████████████████████████████████████████████████████▎ | 19474/40903 [7:53:29<7:01:47, 1.18s/it][2025-04-22 09:41:00] (step=0019475) Train Loss: 6.4025, Train Steps/Sec: 0.84 + 48%|█████████████████████████████████████████████████████▍ | 19499/40903 [7:53:58<6:58:37, 1.17s/it][2025-04-22 09:41:30] (step=0019500) Train Loss: 6.4234, Train Steps/Sec: 0.84 + 48%|█████████████████████████████████████████████████████▍ | 19524/40903 [7:54:35<7:04:19, 1.19s/it][2025-04-22 09:42:06] (step=0019525) Train Loss: 6.3994, Train Steps/Sec: 0.69 + 48%|█████████████████████████████████████████████████████▌ | 19549/40903 [7:55:04<7:00:23, 1.18s/it][2025-04-22 09:42:36] (step=0019550) Train Loss: 6.4021, Train Steps/Sec: 0.85 + 48%|█████████████████████████████████████████████████████▌ | 19574/40903 [7:55:40<7:05:29, 1.20s/it][2025-04-22 09:43:11] (step=0019575) Train Loss: 6.4701, Train Steps/Sec: 0.70 + 48%|█████████████████████████████████████████████████████▋ | 19599/40903 [7:56:10<6:54:04, 1.17s/it][2025-04-22 09:43:41] (step=0019600) Train Loss: 6.3778, Train Steps/Sec: 0.85 + 48%|█████████████████████████████████████████████████████▋ | 19624/40903 [7:56:39<7:08:26, 1.21s/it][2025-04-22 09:44:10] (step=0019625) Train Loss: 6.4504, Train Steps/Sec: 0.85 + 48%|█████████████████████████████████████████████████████▊ | 19649/40903 [7:57:09<6:53:31, 1.17s/it][2025-04-22 09:44:40] (step=0019650) Train Loss: 6.4237, Train Steps/Sec: 0.84 + 48%|█████████████████████████████████████████████████████▊ | 19674/40903 [7:57:38<6:55:55, 1.18s/it][2025-04-22 09:45:10] (step=0019675) Train Loss: 6.4298, Train Steps/Sec: 0.85 + 48%|█████████████████████████████████████████████████████▉ | 19699/40903 [7:58:08<6:49:59, 1.16s/it][2025-04-22 09:45:39] (step=0019700) Train Loss: 6.3818, Train Steps/Sec: 0.85 + 48%|██████████████████████████████████████████████████████ | 19724/40903 [7:58:37<7:07:48, 1.21s/it][2025-04-22 09:46:08] (step=0019725) Train Loss: 6.4118, Train Steps/Sec: 0.85 + 48%|██████████████████████████████████████████████████████ | 19749/40903 [7:59:07<6:56:29, 1.18s/it][2025-04-22 09:46:38] (step=0019750) Train Loss: 6.4552, Train Steps/Sec: 0.85 + 48%|██████████████████████████████████████████████████████▏ | 19774/40903 [7:59:36<6:51:52, 1.17s/it][2025-04-22 09:47:07] (step=0019775) Train Loss: 6.4310, Train Steps/Sec: 0.85 + 48%|██████████████████████████████████████████████████████▏ | 19799/40903 [8:00:06<6:51:51, 1.17s/it][2025-04-22 09:47:37] (step=0019800) Train Loss: 6.3683, Train Steps/Sec: 0.84 + 48%|██████████████████████████████████████████████████████▎ | 19824/40903 [8:00:35<6:58:41, 1.19s/it][2025-04-22 09:48:07] (step=0019825) Train Loss: 6.3848, Train Steps/Sec: 0.85 + 49%|██████████████████████████████████████████████████████▎ | 19849/40903 [8:01:05<6:55:36, 1.18s/it][2025-04-22 09:48:36] (step=0019850) Train Loss: 6.4013, Train Steps/Sec: 0.85 + 49%|██████████████████████████████████████████████████████▍ | 19874/40903 [8:01:34<6:56:09, 1.19s/it][2025-04-22 09:49:06] (step=0019875) Train Loss: 6.3684, Train Steps/Sec: 0.84 + 49%|██████████████████████████████████████████████████████▍ | 19899/40903 [8:02:04<6:48:25, 1.17s/it][2025-04-22 09:49:35] (step=0019900) Train Loss: 6.4187, Train Steps/Sec: 0.85 + 49%|██████████████████████████████████████████████████████▌ | 19924/40903 [8:02:33<6:55:39, 1.19s/it][2025-04-22 09:50:05] (step=0019925) Train Loss: 6.4052, Train Steps/Sec: 0.85 + 49%|██████████████████████████████████████████████████████▌ | 19949/40903 [8:03:03<6:51:50, 1.18s/it][2025-04-22 09:50:34] (step=0019950) Train Loss: 6.4102, Train Steps/Sec: 0.84 + 49%|██████████████████████████████████████████████████████▋ | 19974/40903 [8:03:33<6:47:30, 1.17s/it][2025-04-22 09:51:04] (step=0019975) Train Loss: 6.4265, Train Steps/Sec: 0.85 + 49%|██████████████████████████████████████████████████████▊ | 19999/40903 [8:04:02<6:48:21, 1.17s/it][2025-04-22 09:51:34] (step=0020000) Train Loss: 6.4077, Train Steps/Sec: 0.84 +You are using a model of type instructblip to instantiate a model of type blip-2. This is not supported for all configurations of models and can yield errors. +model_name_or_path:/tmp/haozhezhao/model/instructblip-flan-t5-xl +image_place_holder: +Some kwargs in processor config are unused and will not have any effect: num_query_tokens. +tokenizer length after expend 32102 +tokenizer length before expend 32102 +100%|████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████| 3/3 [02:54<00:00, 58.04s/it] +[2025-04-22 09:56:15] Finish Eval in 20000 steps...█████████████████████████████████████████████████████████████████████| 3/3 [02:53<00:00, 57.67s/it] +processing left-padding... +processing left-padding... +processing left-padding... +[2025-04-22 09:56:36] Saved checkpoint to checkpoint/BeforeFix_Blip_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_objects/002-GPT-XL/checkpoints/0020000.pt +[2025-04-22 09:56:38] Removed old checkpoint: checkpoint/BeforeFix_Blip_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_objects/002-GPT-XL/checkpoints/0016000.pt + 49%|██████████████████████████████████████████████████████▊ | 20024/40903 [8:09:37<7:05:29, 1.22s/it][2025-04-22 09:57:08] (step=0020025) Train Loss: 6.4494, Train Steps/Sec: 0.07 + 49%|██████████████████████████████████████████████████████▉ | 20049/40903 [8:10:15<7:00:21, 1.21s/it][2025-04-22 09:57:46] (step=0020050) Train Loss: 6.4058, Train Steps/Sec: 0.65 + 49%|██████████████████████████████████████████████████████▉ | 20074/40903 [8:10:51<6:53:17, 1.19s/it][2025-04-22 09:58:22] (step=0020075) Train Loss: 6.4320, Train Steps/Sec: 0.69 + 49%|███████████████████████████████████████████████████████ | 20099/40903 [8:11:21<6:45:52, 1.17s/it][2025-04-22 09:58:52] (step=0020100) Train Loss: 6.4149, Train Steps/Sec: 0.84 + 49%|███████████████████████████████████████████████████████ | 20124/40903 [8:11:50<6:52:12, 1.19s/it][2025-04-22 09:59:22] (step=0020125) Train Loss: 6.4273, Train Steps/Sec: 0.84 + 49%|███████████████████████████████████████████████████████▏ | 20149/40903 [8:12:20<6:49:05, 1.18s/it][2025-04-22 09:59:51] (step=0020150) Train Loss: 6.4527, Train Steps/Sec: 0.84 + 49%|███████████████████████████████████████████████████████▏ | 20174/40903 [8:12:50<6:49:57, 1.19s/it][2025-04-22 10:00:21] (step=0020175) Train Loss: 6.4179, Train Steps/Sec: 0.84 + 49%|███████████████████████████████████████████████████████▎ | 20199/40903 [8:13:20<6:49:04, 1.19s/it][2025-04-22 10:00:51] (step=0020200) Train Loss: 6.4532, Train Steps/Sec: 0.84 + 49%|███████████████████████████████████████████████████████▍ | 20224/40903 [8:13:49<6:50:45, 1.19s/it][2025-04-22 10:01:21] (step=0020225) Train Loss: 6.4045, Train Steps/Sec: 0.85 + 50%|███████████████████████████████████████████████████████▍ | 20249/40903 [8:14:19<6:41:22, 1.17s/it][2025-04-22 10:01:50] (step=0020250) Train Loss: 6.3759, Train Steps/Sec: 0.85 + 50%|███████████████████████████████████████████████████████▌ | 20274/40903 [8:14:48<6:46:25, 1.18s/it][2025-04-22 10:02:20] (step=0020275) Train Loss: 6.4772, Train Steps/Sec: 0.84 + 50%|███████████████████████████████████████████████████████▌ | 20299/40903 [8:15:18<6:42:02, 1.17s/it][2025-04-22 10:02:50] (step=0020300) Train Loss: 6.4062, Train Steps/Sec: 0.84 + 50%|███████████████████████████████████████████████████████▋ | 20324/40903 [8:15:48<6:57:07, 1.22s/it][2025-04-22 10:03:19] (step=0020325) Train Loss: 6.4402, Train Steps/Sec: 0.85 + 50%|███████████████████████████████████████████████████████▋ | 20349/40903 [8:16:17<6:42:46, 1.18s/it][2025-04-22 10:03:49] (step=0020350) Train Loss: 6.4386, Train Steps/Sec: 0.84 + 50%|███████████████████████████████████████████████████████▊ | 20374/40903 [8:16:47<6:49:12, 1.20s/it][2025-04-22 10:04:18] (step=0020375) Train Loss: 6.3914, Train Steps/Sec: 0.84 + 50%|███████████████████████████████████████████████████████▊ | 20399/40903 [8:17:17<6:41:49, 1.18s/it][2025-04-22 10:04:48] (step=0020400) Train Loss: 6.3818, Train Steps/Sec: 0.84 + 50%|███████████████████████████████████████████████████████▉ | 20424/40903 [8:17:46<6:50:00, 1.20s/it][2025-04-22 10:05:17] (step=0020425) Train Loss: 6.4102, Train Steps/Sec: 0.85 + 50%|███████████████████████████████████████████████████████▉ | 20449/40903 [8:18:16<6:41:27, 1.18s/it][2025-04-22 10:05:47] (step=0020450) Train Loss: 6.4058, Train Steps/Sec: 0.85 + 50%|████████████████████████████████████████████████████████ | 20474/40903 [8:18:45<6:37:09, 1.17s/it][2025-04-22 10:06:16] (step=0020475) Train Loss: 6.4087, Train Steps/Sec: 0.85 + 50%|████████████████████████████████████████████████████████▏ | 20499/40903 [8:19:15<6:43:02, 1.19s/it][2025-04-22 10:06:46] (step=0020500) Train Loss: 6.3993, Train Steps/Sec: 0.84 + 50%|████████████████████████████████████████████████████████▏ | 20524/40903 [8:19:44<6:51:18, 1.21s/it][2025-04-22 10:07:16] (step=0020525) Train Loss: 6.4536, Train Steps/Sec: 0.85 + 50%|████████████████████████████████████████████████████████▎ | 20549/40903 [8:20:14<6:42:46, 1.19s/it][2025-04-22 10:07:45] (step=0020550) Train Loss: 6.3868, Train Steps/Sec: 0.84 + 50%|████████████████████████████████████████████████████████▎ | 20574/40903 [8:20:43<6:34:07, 1.16s/it][2025-04-22 10:08:15] (step=0020575) Train Loss: 6.4093, Train Steps/Sec: 0.85 + 50%|████████████████████████████████████████████████████████▍ | 20599/40903 [8:21:13<6:32:55, 1.16s/it][2025-04-22 10:08:44] (step=0020600) Train Loss: 6.3997, Train Steps/Sec: 0.85 + 50%|████████████████████████████████████████████████████████▍ | 20624/40903 [8:21:42<6:43:18, 1.19s/it][2025-04-22 10:09:14] (step=0020625) Train Loss: 6.4593, Train Steps/Sec: 0.85 + 50%|████████████████████████████████████████████████████████▌ | 20649/40903 [8:22:12<6:36:28, 1.17s/it][2025-04-22 10:09:43] (step=0020650) Train Loss: 6.3720, Train Steps/Sec: 0.85 + 51%|████████████████████████████████████████████████████████▌ | 20674/40903 [8:22:42<6:32:14, 1.16s/it][2025-04-22 10:10:13] (step=0020675) Train Loss: 6.3769, Train Steps/Sec: 0.85 + 51%|████████████████████████████████████████████████████████▋ | 20699/40903 [8:23:11<6:33:23, 1.17s/it][2025-04-22 10:10:42] (step=0020700) Train Loss: 6.3980, Train Steps/Sec: 0.84 + 51%|████████████████████████████████████████████████████████▋ | 20724/40903 [8:23:41<6:46:33, 1.21s/it][2025-04-22 10:11:12] (step=0020725) Train Loss: 6.4174, Train Steps/Sec: 0.85 + 51%|████████████████████████████████████████████████████████▊ | 20749/40903 [8:24:10<6:39:25, 1.19s/it][2025-04-22 10:11:42] (step=0020750) Train Loss: 6.4076, Train Steps/Sec: 0.84 + 51%|████████████████████████████████████████████████████████▉ | 20774/40903 [8:24:40<6:36:00, 1.18s/it][2025-04-22 10:12:11] (step=0020775) Train Loss: 6.4132, Train Steps/Sec: 0.85 + 51%|████████████████████████████████████████████████████████▉ | 20799/40903 [8:25:10<6:29:42, 1.16s/it][2025-04-22 10:12:41] (step=0020800) Train Loss: 6.4226, Train Steps/Sec: 0.84 + 51%|█████████████████████████████████████████████████████████ | 20824/40903 [8:25:39<6:42:24, 1.20s/it][2025-04-22 10:13:10] (step=0020825) Train Loss: 6.3607, Train Steps/Sec: 0.85 + 51%|█████████████████████████████████████████████████████████ | 20849/40903 [8:26:09<6:34:58, 1.18s/it][2025-04-22 10:13:40] (step=0020850) Train Loss: 6.3776, Train Steps/Sec: 0.85 + 51%|█████████████████████████████████████████████████████████▏ | 20874/40903 [8:26:38<6:30:19, 1.17s/it][2025-04-22 10:14:09] (step=0020875) Train Loss: 6.3997, Train Steps/Sec: 0.84 + 51%|█████████████████████████████████████████████████████████▏ | 20899/40903 [8:27:08<6:30:07, 1.17s/it][2025-04-22 10:14:39] (step=0020900) Train Loss: 6.4146, Train Steps/Sec: 0.85 + 51%|█████████████████████████████████████████████████████████▎ | 20924/40903 [8:27:37<6:38:56, 1.20s/it][2025-04-22 10:15:09] (step=0020925) Train Loss: 6.3945, Train Steps/Sec: 0.85 + 51%|█████████████████████████████████████████████████████████▎ | 20949/40903 [8:28:07<6:34:12, 1.19s/it][2025-04-22 10:15:38] (step=0020950) Train Loss: 6.4100, Train Steps/Sec: 0.85 + 51%|█████████████████████████████████████████████████████████▍ | 20974/40903 [8:28:36<6:31:04, 1.18s/it][2025-04-22 10:16:08] (step=0020975) Train Loss: 6.4670, Train Steps/Sec: 0.84 + 51%|█████████████████████████████████████████████████████████▍ | 20999/40903 [8:29:06<6:29:28, 1.17s/it][2025-04-22 10:16:37] (step=0021000) Train Loss: 6.3881, Train Steps/Sec: 0.84 + 51%|█████████████████████████████████████████████████████████▌ | 21024/40903 [8:29:36<6:40:36, 1.21s/it][2025-04-22 10:17:07] (step=0021025) Train Loss: 6.4519, Train Steps/Sec: 0.84 + 51%|█████████████████████████████████████████████████████████▋ | 21049/40903 [8:30:05<6:32:59, 1.19s/it][2025-04-22 10:17:36] (step=0021050) Train Loss: 6.4771, Train Steps/Sec: 0.85 + 52%|█████████████████████████████████████████████████████████▋ | 21074/40903 [8:30:35<6:27:14, 1.17s/it][2025-04-22 10:18:06] (step=0021075) Train Loss: 6.3956, Train Steps/Sec: 0.85 + 52%|█████████████████████████████████████████████████████████▊ | 21099/40903 [8:31:04<6:28:52, 1.18s/it][2025-04-22 10:18:36] (step=0021100) Train Loss: 6.4039, Train Steps/Sec: 0.84 + 52%|█████████████████████████████████████████████████████████▊ | 21124/40903 [8:31:34<6:38:32, 1.21s/it][2025-04-22 10:19:05] (step=0021125) Train Loss: 6.4277, Train Steps/Sec: 0.84 + 52%|█████████████████████████████████████████████████████████▉ | 21149/40903 [8:32:04<6:32:40, 1.19s/it][2025-04-22 10:19:35] (step=0021150) Train Loss: 6.3767, Train Steps/Sec: 0.84 + 52%|█████████████████████████████████████████████████████████▉ | 21174/40903 [8:32:33<6:22:56, 1.16s/it][2025-04-22 10:20:04] (step=0021175) Train Loss: 6.3923, Train Steps/Sec: 0.85 + 52%|██████████████████████████████████████████████████████████ | 21199/40903 [8:33:03<6:20:53, 1.16s/it][2025-04-22 10:20:34] (step=0021200) Train Loss: 6.3749, Train Steps/Sec: 0.84 + 52%|██████████████████████████████████████████████████████████ | 21224/40903 [8:33:32<6:30:47, 1.19s/it][2025-04-22 10:21:03] (step=0021225) Train Loss: 6.3918, Train Steps/Sec: 0.85 + 52%|██████████████████████████████████████████████████████████▏ | 21249/40903 [8:34:02<6:30:09, 1.19s/it][2025-04-22 10:21:33] (step=0021250) Train Loss: 6.4189, Train Steps/Sec: 0.84 + 52%|██████████████████████████████████████████████████████████▎ | 21274/40903 [8:34:31<6:22:54, 1.17s/it][2025-04-22 10:22:03] (step=0021275) Train Loss: 6.3996, Train Steps/Sec: 0.85 + 52%|██████████████████████████████████████████████████████████▎ | 21299/40903 [8:35:01<6:22:37, 1.17s/it][2025-04-22 10:22:32] (step=0021300) Train Loss: 6.4304, Train Steps/Sec: 0.85 + 52%|██████████████████████████████████████████████████████████▍ | 21324/40903 [8:35:30<6:26:42, 1.19s/it][2025-04-22 10:23:01] (step=0021325) Train Loss: 6.4015, Train Steps/Sec: 0.85 + 52%|██████████████████████████████████████████████████████████▍ | 21349/40903 [8:36:00<6:21:05, 1.17s/it][2025-04-22 10:23:31] (step=0021350) Train Loss: 6.4090, Train Steps/Sec: 0.85 + 52%|██████████████████████████████████████████████████████████▌ | 21374/40903 [8:36:29<6:23:01, 1.18s/it][2025-04-22 10:24:00] (step=0021375) Train Loss: 6.3574, Train Steps/Sec: 0.85 + 52%|██████████████████████████████████████████████████████████▌ | 21399/40903 [8:36:59<6:17:49, 1.16s/it][2025-04-22 10:24:30] (step=0021400) Train Loss: 6.4192, Train Steps/Sec: 0.84 + 52%|██████████████████████████████████████████████████████████▋ | 21424/40903 [8:37:28<6:23:05, 1.18s/it][2025-04-22 10:25:00] (step=0021425) Train Loss: 6.4071, Train Steps/Sec: 0.85 + 52%|██████████████████████████████████████████████████████████▋ | 21449/40903 [8:37:58<6:22:23, 1.18s/it][2025-04-22 10:25:29] (step=0021450) Train Loss: 6.4565, Train Steps/Sec: 0.85 + 52%|██████████████████████████████████████████████████████████▊ | 21474/40903 [8:38:27<6:15:15, 1.16s/it][2025-04-22 10:25:59] (step=0021475) Train Loss: 6.4412, Train Steps/Sec: 0.84 + 53%|██████████████████████████████████████████████████████████▊ | 21499/40903 [8:38:57<6:20:41, 1.18s/it][2025-04-22 10:26:28] (step=0021500) Train Loss: 6.4242, Train Steps/Sec: 0.84 + 53%|██████████████████████████████████████████████████████████▉ | 21524/40903 [8:39:26<6:24:51, 1.19s/it][2025-04-22 10:26:58] (step=0021525) Train Loss: 6.4115, Train Steps/Sec: 0.85 + 53%|██████████████████████████████████████████████████████████▍ | 21549/40903 [8:40:02<13:33:09, 2.52s/it][2025-04-22 10:27:34] (step=0021550) Train Loss: 6.3872, Train Steps/Sec: 0.70 + 53%|███████████████████████████████████████████████████████████ | 21574/40903 [8:40:32<6:17:33, 1.17s/it][2025-04-22 10:28:03] (step=0021575) Train Loss: 6.4205, Train Steps/Sec: 0.85 + 53%|███████████████████████████████████████████████████████████▏ | 21599/40903 [8:41:01<6:17:58, 1.17s/it][2025-04-22 10:28:33] (step=0021600) Train Loss: 6.4034, Train Steps/Sec: 0.84 + 53%|███████████████████████████████████████████████████████████▏ | 21624/40903 [8:41:31<6:27:59, 1.21s/it][2025-04-22 10:29:02] (step=0021625) Train Loss: 6.4141, Train Steps/Sec: 0.84 + 53%|███████████████████████████████████████████████████████████▎ | 21649/40903 [8:42:01<6:20:24, 1.19s/it][2025-04-22 10:29:32] (step=0021650) Train Loss: 6.4256, Train Steps/Sec: 0.84 + 53%|███████████████████████████████████████████████████████████▎ | 21674/40903 [8:42:30<6:18:09, 1.18s/it][2025-04-22 10:30:02] (step=0021675) Train Loss: 6.3826, Train Steps/Sec: 0.84 + 53%|███████████████████████████████████████████████████████████▍ | 21699/40903 [8:43:00<6:12:41, 1.16s/it][2025-04-22 10:30:31] (step=0021700) Train Loss: 6.3784, Train Steps/Sec: 0.84 + 53%|███████████████████████████████████████████████████████████▍ | 21724/40903 [8:43:30<6:21:17, 1.19s/it][2025-04-22 10:31:01] (step=0021725) Train Loss: 6.3879, Train Steps/Sec: 0.85 + 53%|███████████████████████████████████████████████████████████▌ | 21749/40903 [8:43:59<6:16:42, 1.18s/it][2025-04-22 10:31:30] (step=0021750) Train Loss: 6.3946, Train Steps/Sec: 0.85 + 53%|███████████████████████████████████████████████████████████▌ | 21774/40903 [8:44:29<6:13:02, 1.17s/it][2025-04-22 10:32:00] (step=0021775) Train Loss: 6.4270, Train Steps/Sec: 0.85 + 53%|███████████████████████████████████████████████████████████▋ | 21799/40903 [8:44:58<6:13:08, 1.17s/it][2025-04-22 10:32:29] (step=0021800) Train Loss: 6.3793, Train Steps/Sec: 0.85 + 53%|███████████████████████████████████████████████████████████▊ | 21824/40903 [8:45:28<6:22:47, 1.20s/it][2025-04-22 10:32:59] (step=0021825) Train Loss: 6.4283, Train Steps/Sec: 0.85 + 53%|███████████████████████████████████████████████████████████▊ | 21849/40903 [8:45:57<6:18:21, 1.19s/it][2025-04-22 10:33:29] (step=0021850) Train Loss: 6.4026, Train Steps/Sec: 0.85 + 53%|███████████████████████████████████████████████████████████▉ | 21874/40903 [8:46:27<6:10:07, 1.17s/it][2025-04-22 10:33:58] (step=0021875) Train Loss: 6.4121, Train Steps/Sec: 0.84 + 54%|███████████████████████████████████████████████████████████▉ | 21899/40903 [8:46:56<6:14:24, 1.18s/it][2025-04-22 10:34:28] (step=0021900) Train Loss: 6.4080, Train Steps/Sec: 0.84 + 54%|████████████████████████████████████████████████████████████ | 21924/40903 [8:47:26<6:20:53, 1.20s/it][2025-04-22 10:34:57] (step=0021925) Train Loss: 6.4465, Train Steps/Sec: 0.84 + 54%|████████████████████████████████████████████████████████████ | 21949/40903 [8:47:56<6:15:19, 1.19s/it][2025-04-22 10:35:27] (step=0021950) Train Loss: 6.3663, Train Steps/Sec: 0.84 + 54%|████████████████████████████████████████████████████████████▏ | 21974/40903 [8:48:25<6:08:43, 1.17s/it][2025-04-22 10:35:57] (step=0021975) Train Loss: 6.4225, Train Steps/Sec: 0.85 + 54%|████████████████████████████████████████████████████████████▏ | 21999/40903 [8:48:55<6:06:51, 1.16s/it][2025-04-22 10:36:26] (step=0022000) Train Loss: 6.4150, Train Steps/Sec: 0.84 +You are using a model of type instructblip to instantiate a model of type blip-2. This is not supported for all configurations of models and can yield errors. +model_name_or_path:/tmp/haozhezhao/model/instructblip-flan-t5-xl +image_place_holder: +Some kwargs in processor config are unused and will not have any effect: num_query_tokens. +tokenizer length after expend 32102 +tokenizer length before expend 32102 +100%|██████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████| 3/3 [02:53<00:00, 57.99s/it] +[2025-04-22 10:41:08] Finish Eval in 22000 steps...███████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████| 3/3 [02:53<00:00, 57.61s/it] +processing left-padding... +processing left-padding... +processing left-padding... +[2025-04-22 10:41:29] Saved checkpoint to checkpoint/BeforeFix_Blip_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_objects/002-GPT-XL/checkpoints/0022000.pt +[2025-04-22 10:41:31] Removed old checkpoint: checkpoint/BeforeFix_Blip_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_objects/002-GPT-XL/checkpoints/0018000.pt + 54%|████████████████████████████████████████████████████████████▎ | 22024/40903 [8:54:29<6:22:12, 1.21s/it][2025-04-22 10:42:01] (step=0022025) Train Loss: 6.4093, Train Steps/Sec: 0.07 + 54%|████████████████████████████████████████████████████████████▎ | 22049/40903 [8:54:59<6:10:55, 1.18s/it][2025-04-22 10:42:30] (step=0022050) Train Loss: 6.4118, Train Steps/Sec: 0.85 + 54%|████████████████████████████████████████████████████████████▍ | 22074/40903 [8:55:28<6:09:00, 1.18s/it][2025-04-22 10:43:00] (step=0022075) Train Loss: 6.3897, Train Steps/Sec: 0.85 + 54%|████████████████████████████████████████████████████████████▌ | 22099/40903 [8:55:58<6:11:03, 1.18s/it][2025-04-22 10:43:29] (step=0022100) Train Loss: 6.3836, Train Steps/Sec: 0.84 + 54%|████████████████████████████████████████████████████████████▌ | 22124/40903 [8:56:28<6:17:57, 1.21s/it][2025-04-22 10:43:59] (step=0022125) Train Loss: 6.3467, Train Steps/Sec: 0.85 + 54%|████████████████████████████████████████████████████████████▋ | 22149/40903 [8:56:57<6:14:32, 1.20s/it][2025-04-22 10:44:29] (step=0022150) Train Loss: 6.4110, Train Steps/Sec: 0.84 + 54%|████████████████████████████████████████████████████████████▋ | 22174/40903 [8:57:27<6:06:03, 1.17s/it][2025-04-22 10:44:58] (step=0022175) Train Loss: 6.4934, Train Steps/Sec: 0.84 + 54%|████████████████████████████████████████████████████████████▊ | 22199/40903 [8:57:56<6:03:05, 1.16s/it][2025-04-22 10:45:28] (step=0022200) Train Loss: 6.3614, Train Steps/Sec: 0.84 + 54%|████████████████████████████████████████████████████████████▊ | 22224/40903 [8:58:26<6:12:46, 1.20s/it][2025-04-22 10:45:57] (step=0022225) Train Loss: 6.4127, Train Steps/Sec: 0.85 + 54%|████████████████████████████████████████████████████████████▉ | 22249/40903 [8:58:56<6:10:19, 1.19s/it][2025-04-22 10:46:27] (step=0022250) Train Loss: 6.3873, Train Steps/Sec: 0.85 + 54%|████████████████████████████████████████████████████████████▉ | 22274/40903 [8:59:25<6:03:02, 1.17s/it][2025-04-22 10:46:56] (step=0022275) Train Loss: 6.4001, Train Steps/Sec: 0.85 + 55%|█████████████████████████████████████████████████████████████ | 22299/40903 [8:59:55<6:06:15, 1.18s/it][2025-04-22 10:47:26] (step=0022300) Train Loss: 6.4150, Train Steps/Sec: 0.84 + 55%|█████████████████████████████████████████████████████████████▏ | 22324/40903 [9:00:24<6:11:21, 1.20s/it][2025-04-22 10:47:56] (step=0022325) Train Loss: 6.3677, Train Steps/Sec: 0.85 + 55%|█████████████████████████████████████████████████████████████▏ | 22349/40903 [9:00:54<6:08:01, 1.19s/it][2025-04-22 10:48:25] (step=0022350) Train Loss: 6.3881, Train Steps/Sec: 0.85 + 55%|█████████████████████████████████████████████████████████████▎ | 22374/40903 [9:01:23<6:02:08, 1.17s/it][2025-04-22 10:48:55] (step=0022375) Train Loss: 6.3541, Train Steps/Sec: 0.84 + 55%|█████████████████████████████████████████████████████████████▎ | 22399/40903 [9:01:53<5:58:23, 1.16s/it][2025-04-22 10:49:24] (step=0022400) Train Loss: 6.4313, Train Steps/Sec: 0.84 + 55%|█████████████████████████████████████████████████████████████▍ | 22424/40903 [9:02:23<6:10:47, 1.20s/it][2025-04-22 10:49:54] (step=0022425) Train Loss: 6.4068, Train Steps/Sec: 0.85 + 55%|█████████████████████████████████████████████████████████████▍ | 22449/40903 [9:02:52<6:07:14, 1.19s/it][2025-04-22 10:50:24] (step=0022450) Train Loss: 6.3662, Train Steps/Sec: 0.84 + 55%|█████████████████████████████████████████████████████████████▌ | 22474/40903 [9:03:22<6:04:43, 1.19s/it][2025-04-22 10:50:53] (step=0022475) Train Loss: 6.4649, Train Steps/Sec: 0.85 + 55%|█████████████████████████████████████████████████████████████▌ | 22499/40903 [9:03:51<6:03:02, 1.18s/it][2025-04-22 10:51:23] (step=0022500) Train Loss: 6.3986, Train Steps/Sec: 0.84 + 55%|█████████████████████████████████████████████████████████████▋ | 22524/40903 [9:04:21<6:03:51, 1.19s/it][2025-04-22 10:51:52] (step=0022525) Train Loss: 6.3748, Train Steps/Sec: 0.85 + 55%|█████████████████████████████████████████████████████████████▋ | 22549/40903 [9:04:51<6:00:07, 1.18s/it][2025-04-22 10:52:22] (step=0022550) Train Loss: 6.4197, Train Steps/Sec: 0.85 + 55%|█████████████████████████████████████████████████████████████▊ | 22574/40903 [9:05:20<6:00:21, 1.18s/it][2025-04-22 10:52:51] (step=0022575) Train Loss: 6.4001, Train Steps/Sec: 0.85 + 55%|█████████████████████████████████████████████████████████████▉ | 22599/40903 [9:05:50<6:02:14, 1.19s/it][2025-04-22 10:53:21] (step=0022600) Train Loss: 6.4200, Train Steps/Sec: 0.84 + 55%|█████████████████████████████████████████████████████████████▉ | 22624/40903 [9:06:19<6:05:38, 1.20s/it][2025-04-22 10:53:51] (step=0022625) Train Loss: 6.4181, Train Steps/Sec: 0.85 + 55%|██████████████████████████████████████████████████████████████ | 22649/40903 [9:06:49<5:59:12, 1.18s/it][2025-04-22 10:54:20] (step=0022650) Train Loss: 6.4113, Train Steps/Sec: 0.85 + 55%|██████████████████████████████████████████████████████████████ | 22674/40903 [9:07:18<6:00:50, 1.19s/it][2025-04-22 10:54:50] (step=0022675) Train Loss: 6.3763, Train Steps/Sec: 0.85 + 55%|██████████████████████████████████████████████████████████████▏ | 22699/40903 [9:07:48<5:52:46, 1.16s/it][2025-04-22 10:55:19] (step=0022700) Train Loss: 6.4188, Train Steps/Sec: 0.85 + 56%|██████████████████████████████████████████████████████████████▏ | 22724/40903 [9:08:17<6:03:26, 1.20s/it][2025-04-22 10:55:49] (step=0022725) Train Loss: 6.4556, Train Steps/Sec: 0.85 + 56%|██████████████████████████████████████████████████████████████▎ | 22749/40903 [9:08:47<5:57:25, 1.18s/it][2025-04-22 10:56:18] (step=0022750) Train Loss: 6.4334, Train Steps/Sec: 0.85 + 56%|██████████████████████████████████████████████████████████████▎ | 22774/40903 [9:09:16<5:55:42, 1.18s/it][2025-04-22 10:56:48] (step=0022775) Train Loss: 6.4214, Train Steps/Sec: 0.85 + 56%|██████████████████████████████████████████████████████████████▍ | 22799/40903 [9:09:46<5:53:21, 1.17s/it][2025-04-22 10:57:17] (step=0022800) Train Loss: 6.3976, Train Steps/Sec: 0.85 + 56%|██████████████████████████████████████████████████████████████▍ | 22824/40903 [9:10:15<6:05:07, 1.21s/it][2025-04-22 10:57:47] (step=0022825) Train Loss: 6.4513, Train Steps/Sec: 0.85 + 56%|██████████████████████████████████████████████████████████████▌ | 22849/40903 [9:10:45<5:57:47, 1.19s/it][2025-04-22 10:58:16] (step=0022850) Train Loss: 6.4055, Train Steps/Sec: 0.85 + 56%|██████████████████████████████████████████████████████████████▋ | 22874/40903 [9:11:14<5:50:20, 1.17s/it][2025-04-22 10:58:45] (step=0022875) Train Loss: 6.3751, Train Steps/Sec: 0.85 + 56%|██████████████████████████████████████████████████████████████▋ | 22899/40903 [9:11:44<5:47:44, 1.16s/it][2025-04-22 10:59:15] (step=0022900) Train Loss: 6.4050, Train Steps/Sec: 0.85 + 56%|██████████████████████████████████████████████████████████████▊ | 22924/40903 [9:12:13<5:58:04, 1.19s/it][2025-04-22 10:59:44] (step=0022925) Train Loss: 6.4095, Train Steps/Sec: 0.85 + 56%|██████████████████████████████████████████████████████████████▊ | 22949/40903 [9:12:43<5:52:40, 1.18s/it][2025-04-22 11:00:14] (step=0022950) Train Loss: 6.3938, Train Steps/Sec: 0.85 + 56%|██████████████████████████████████████████████████████████████▉ | 22974/40903 [9:13:12<5:52:01, 1.18s/it][2025-04-22 11:00:44] (step=0022975) Train Loss: 6.3840, Train Steps/Sec: 0.85 + 56%|██████████████████████████████████████████████████████████████▉ | 22999/40903 [9:13:42<5:46:28, 1.16s/it][2025-04-22 11:01:13] (step=0023000) Train Loss: 6.4164, Train Steps/Sec: 0.84 + 56%|███████████████████████████████████████████████████████████████ | 23024/40903 [9:14:11<5:52:16, 1.18s/it][2025-04-22 11:01:43] (step=0023025) Train Loss: 6.3797, Train Steps/Sec: 0.85 + 56%|███████████████████████████████████████████████████████████████ | 23049/40903 [9:14:41<5:53:06, 1.19s/it][2025-04-22 11:02:12] (step=0023050) Train Loss: 6.4362, Train Steps/Sec: 0.84 + 56%|███████████████████████████████████████████████████████████████▏ | 23074/40903 [9:15:10<5:44:29, 1.16s/it][2025-04-22 11:02:42] (step=0023075) Train Loss: 6.3932, Train Steps/Sec: 0.85 + 56%|███████████████████████████████████████████████████████████████▏ | 23099/40903 [9:15:40<5:44:10, 1.16s/it][2025-04-22 11:03:11] (step=0023100) Train Loss: 6.3749, Train Steps/Sec: 0.85 + 57%|███████████████████████████████████████████████████████████████▎ | 23124/40903 [9:16:09<5:55:37, 1.20s/it][2025-04-22 11:03:41] (step=0023125) Train Loss: 6.4282, Train Steps/Sec: 0.84 + 57%|███████████████████████████████████████████████████████████████▍ | 23149/40903 [9:16:39<5:49:09, 1.18s/it][2025-04-22 11:04:10] (step=0023150) Train Loss: 6.3907, Train Steps/Sec: 0.85 + 57%|███████████████████████████████████████████████████████████████▍ | 23174/40903 [9:17:08<5:45:10, 1.17s/it][2025-04-22 11:04:39] (step=0023175) Train Loss: 6.4076, Train Steps/Sec: 0.85 + 57%|███████████████████████████████████████████████████████████████▌ | 23199/40903 [9:17:38<5:46:28, 1.17s/it][2025-04-22 11:05:09] (step=0023200) Train Loss: 6.4536, Train Steps/Sec: 0.84 + 57%|███████████████████████████████████████████████████████████████▌ | 23224/40903 [9:18:07<5:52:57, 1.20s/it][2025-04-22 11:05:39] (step=0023225) Train Loss: 6.4061, Train Steps/Sec: 0.85 + 57%|███████████████████████████████████████████████████████████████▋ | 23249/40903 [9:18:37<5:48:38, 1.18s/it][2025-04-22 11:06:08] (step=0023250) Train Loss: 6.4185, Train Steps/Sec: 0.85 + 57%|███████████████████████████████████████████████████████████████▋ | 23274/40903 [9:19:06<5:47:02, 1.18s/it][2025-04-22 11:06:38] (step=0023275) Train Loss: 6.3948, Train Steps/Sec: 0.85 + 57%|███████████████████████████████████████████████████████████████▊ | 23299/40903 [9:19:36<5:44:31, 1.17s/it][2025-04-22 11:07:07] (step=0023300) Train Loss: 6.3716, Train Steps/Sec: 0.84 + 57%|███████████████████████████████████████████████████████████████▊ | 23324/40903 [9:20:06<5:51:54, 1.20s/it][2025-04-22 11:07:37] (step=0023325) Train Loss: 6.3989, Train Steps/Sec: 0.84 + 57%|███████████████████████████████████████████████████████████████▉ | 23349/40903 [9:20:35<5:43:46, 1.18s/it][2025-04-22 11:08:06] (step=0023350) Train Loss: 6.3834, Train Steps/Sec: 0.85 + 57%|████████████████████████████████████████████████████████████████ | 23374/40903 [9:21:05<5:41:59, 1.17s/it][2025-04-22 11:08:36] (step=0023375) Train Loss: 6.3918, Train Steps/Sec: 0.85 + 57%|████████████████████████████████████████████████████████████████ | 23399/40903 [9:21:34<5:44:21, 1.18s/it][2025-04-22 11:09:06] (step=0023400) Train Loss: 6.3834, Train Steps/Sec: 0.84 + 57%|████████████████████████████████████████████████████████████████▏ | 23424/40903 [9:22:04<5:47:56, 1.19s/it][2025-04-22 11:09:35] (step=0023425) Train Loss: 6.3861, Train Steps/Sec: 0.85 + 57%|████████████████████████████████████████████████████████████████▏ | 23449/40903 [9:22:34<5:42:25, 1.18s/it][2025-04-22 11:10:05] (step=0023450) Train Loss: 6.3716, Train Steps/Sec: 0.84 + 57%|████████████████████████████████████████████████████████████████▎ | 23474/40903 [9:23:03<5:46:45, 1.19s/it][2025-04-22 11:10:35] (step=0023475) Train Loss: 6.4182, Train Steps/Sec: 0.84 + 57%|████████████████████████████████████████████████████████████████▎ | 23499/40903 [9:23:33<5:39:05, 1.17s/it][2025-04-22 11:11:04] (step=0023500) Train Loss: 6.3609, Train Steps/Sec: 0.84 + 58%|████████████████████████████████████████████████████████████████▍ | 23524/40903 [9:24:03<5:51:19, 1.21s/it][2025-04-22 11:11:34] (step=0023525) Train Loss: 6.3931, Train Steps/Sec: 0.84 + 58%|████████████████████████████████████████████████████████████████▍ | 23549/40903 [9:24:32<5:45:06, 1.19s/it][2025-04-22 11:12:04] (step=0023550) Train Loss: 6.4157, Train Steps/Sec: 0.85 + 58%|████████████████████████████████████████████████████████████████▌ | 23574/40903 [9:25:02<5:39:27, 1.18s/it][2025-04-22 11:12:33] (step=0023575) Train Loss: 6.4151, Train Steps/Sec: 0.84 + 58%|████████████████████████████████████████████████████████████████▌ | 23599/40903 [9:25:32<5:38:30, 1.17s/it][2025-04-22 11:13:03] (step=0023600) Train Loss: 6.4163, Train Steps/Sec: 0.84 + 58%|████████████████████████████████████████████████████████████████▋ | 23624/40903 [9:26:01<5:46:09, 1.20s/it][2025-04-22 11:13:33] (step=0023625) Train Loss: 6.4068, Train Steps/Sec: 0.85 + 58%|████████████████████████████████████████████████████████████████▊ | 23649/40903 [9:26:31<5:39:23, 1.18s/it][2025-04-22 11:14:02] (step=0023650) Train Loss: 6.3960, Train Steps/Sec: 0.85 + 58%|████████████████████████████████████████████████████████████████▊ | 23674/40903 [9:27:00<5:39:27, 1.18s/it][2025-04-22 11:14:32] (step=0023675) Train Loss: 6.4643, Train Steps/Sec: 0.85 + 58%|████████████████████████████████████████████████████████████████▉ | 23699/40903 [9:27:30<5:33:38, 1.16s/it][2025-04-22 11:15:01] (step=0023700) Train Loss: 6.3948, Train Steps/Sec: 0.85 + 58%|████████████████████████████████████████████████████████████████▉ | 23724/40903 [9:27:59<5:45:00, 1.20s/it][2025-04-22 11:15:31] (step=0023725) Train Loss: 6.4343, Train Steps/Sec: 0.85 + 58%|█████████████████████████████████████████████████████████████████ | 23749/40903 [9:28:29<5:42:19, 1.20s/it][2025-04-22 11:16:00] (step=0023750) Train Loss: 6.4421, Train Steps/Sec: 0.84 + 58%|█████████████████████████████████████████████████████████████████ | 23774/40903 [9:28:58<5:32:24, 1.16s/it][2025-04-22 11:16:30] (step=0023775) Train Loss: 6.4026, Train Steps/Sec: 0.85 + 58%|█████████████████████████████████████████████████████████████████▏ | 23799/40903 [9:29:28<5:32:29, 1.17s/it][2025-04-22 11:16:59] (step=0023800) Train Loss: 6.4282, Train Steps/Sec: 0.85 + 58%|█████████████████████████████████████████████████████████████████▏ | 23824/40903 [9:29:57<5:41:31, 1.20s/it][2025-04-22 11:17:29] (step=0023825) Train Loss: 6.4000, Train Steps/Sec: 0.85 + 58%|█████████████████████████████████████████████████████████████████▎ | 23849/40903 [9:30:27<5:36:30, 1.18s/it][2025-04-22 11:17:58] (step=0023850) Train Loss: 6.3930, Train Steps/Sec: 0.85 + 58%|█████████████████████████████████████████████████████████████████▎ | 23874/40903 [9:30:57<5:32:04, 1.17s/it][2025-04-22 11:18:28] (step=0023875) Train Loss: 6.3725, Train Steps/Sec: 0.85 + 58%|█████████████████████████████████████████████████████████████████▍ | 23899/40903 [9:31:26<5:33:29, 1.18s/it][2025-04-22 11:18:58] (step=0023900) Train Loss: 6.4104, Train Steps/Sec: 0.84 + 58%|█████████████████████████████████████████████████████████████████▌ | 23924/40903 [9:31:56<5:45:05, 1.22s/it][2025-04-22 11:19:27] (step=0023925) Train Loss: 6.4127, Train Steps/Sec: 0.84 + 59%|█████████████████████████████████████████████████████████████████▌ | 23949/40903 [9:32:26<5:36:46, 1.19s/it][2025-04-22 11:19:57] (step=0023950) Train Loss: 6.3933, Train Steps/Sec: 0.85 + 59%|█████████████████████████████████████████████████████████████████▋ | 23974/40903 [9:32:55<5:34:36, 1.19s/it][2025-04-22 11:20:26] (step=0023975) Train Loss: 6.3925, Train Steps/Sec: 0.85 + 59%|█████████████████████████████████████████████████████████████████▋ | 23999/40903 [9:33:24<5:24:53, 1.15s/it][2025-04-22 11:20:56] (step=0024000) Train Loss: 6.4506, Train Steps/Sec: 0.85 +You are using a model of type instructblip to instantiate a model of type blip-2. This is not supported for all configurations of models and can yield errors. +model_name_or_path:/tmp/haozhezhao/model/instructblip-flan-t5-xl +image_place_holder: +Some kwargs in processor config are unused and will not have any effect: num_query_tokens. +tokenizer length after expend 32102 +tokenizer length before expend 32102 +100%|██████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████| 3/3 [02:54<00:00, 58.08s/it] +[2025-04-22 11:25:37] Finish Eval in 24000 steps...███████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████| 3/3 [02:53<00:00, 57.68s/it] +processing left-padding... +processing left-padding... +processing left-padding... +[2025-04-22 11:25:58] Saved checkpoint to checkpoint/BeforeFix_Blip_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_objects/002-GPT-XL/checkpoints/0024000.pt +[2025-04-22 11:26:01] Removed old checkpoint: checkpoint/BeforeFix_Blip_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_objects/002-GPT-XL/checkpoints/0020000.pt + 59%|█████████████████████████████████████████████████████████████████▊ | 24024/40903 [9:38:59<5:41:17, 1.21s/it][2025-04-22 11:26:30] (step=0024025) Train Loss: 6.3986, Train Steps/Sec: 0.07 + 59%|█████████████████████████████████████████████████████████████████▊ | 24049/40903 [9:39:28<5:33:28, 1.19s/it][2025-04-22 11:26:59] (step=0024050) Train Loss: 6.3979, Train Steps/Sec: 0.85 + 59%|█████████████████████████████████████████████████████████████████▉ | 24074/40903 [9:39:58<5:28:24, 1.17s/it][2025-04-22 11:27:29] (step=0024075) Train Loss: 6.4077, Train Steps/Sec: 0.85 + 59%|█████████████████████████████████████████████████████████████████▉ | 24099/40903 [9:40:27<5:29:12, 1.18s/it][2025-04-22 11:27:59] (step=0024100) Train Loss: 6.4447, Train Steps/Sec: 0.84 + 59%|██████████████████████████████████████████████████████████████████ | 24124/40903 [9:40:57<5:32:40, 1.19s/it][2025-04-22 11:28:28] (step=0024125) Train Loss: 6.4194, Train Steps/Sec: 0.85 + 59%|██████████████████████████████████████████████████████████████████ | 24149/40903 [9:41:27<5:31:49, 1.19s/it][2025-04-22 11:28:58] (step=0024150) Train Loss: 6.3994, Train Steps/Sec: 0.85 + 59%|██████████████████████████████████████████████████████████████████▏ | 24174/40903 [9:41:56<5:25:56, 1.17s/it][2025-04-22 11:29:27] (step=0024175) Train Loss: 6.4066, Train Steps/Sec: 0.85 + 59%|██████████████████████████████████████████████████████████████████▎ | 24199/40903 [9:42:26<5:27:25, 1.18s/it][2025-04-22 11:29:57] (step=0024200) Train Loss: 6.4283, Train Steps/Sec: 0.84 + 59%|██████████████████████████████████████████████████████████████████▎ | 24224/40903 [9:43:01<5:52:14, 1.27s/it][2025-04-22 11:30:32] (step=0024225) Train Loss: 6.4176, Train Steps/Sec: 0.70 + 59%|██████████████████████████████████████████████████████████████████▍ | 24249/40903 [9:43:31<5:32:22, 1.20s/it][2025-04-22 11:31:02] (step=0024250) Train Loss: 6.4253, Train Steps/Sec: 0.85 + 59%|██████████████████████████████████████████████████████████████████▍ | 24274/40903 [9:44:00<5:25:28, 1.17s/it][2025-04-22 11:31:32] (step=0024275) Train Loss: 6.4249, Train Steps/Sec: 0.84 + 59%|██████████████████████████████████████████████████████████████████▌ | 24299/40903 [9:44:30<5:24:47, 1.17s/it][2025-04-22 11:32:01] (step=0024300) Train Loss: 6.3794, Train Steps/Sec: 0.84 + 59%|██████████████████████████████████████████████████████████████████▌ | 24324/40903 [9:45:00<5:34:29, 1.21s/it][2025-04-22 11:32:31] (step=0024325) Train Loss: 6.3870, Train Steps/Sec: 0.84 + 60%|██████████████████████████████████████████████████████████████████▋ | 24349/40903 [9:45:30<5:33:10, 1.21s/it][2025-04-22 11:33:01] (step=0024350) Train Loss: 6.3934, Train Steps/Sec: 0.84 + 60%|██████████████████████████████████████████████████████████████████▋ | 24374/40903 [9:46:05<5:21:34, 1.17s/it][2025-04-22 11:33:36] (step=0024375) Train Loss: 6.4186, Train Steps/Sec: 0.71 + 60%|██████████████████████████████████████████████████████████████████▊ | 24399/40903 [9:46:34<5:20:17, 1.16s/it][2025-04-22 11:34:06] (step=0024400) Train Loss: 6.4027, Train Steps/Sec: 0.84 + 60%|██████████████████████████████████████████████████████████████████▉ | 24424/40903 [9:47:04<5:26:37, 1.19s/it][2025-04-22 11:34:35] (step=0024425) Train Loss: 6.4176, Train Steps/Sec: 0.85 + 60%|██████████████████████████████████████████████████████████████████▉ | 24449/40903 [9:47:34<5:25:53, 1.19s/it][2025-04-22 11:35:05] (step=0024450) Train Loss: 6.3909, Train Steps/Sec: 0.84 + 60%|███████████████████████████████████████████████████████████████████ | 24474/40903 [9:48:03<5:19:19, 1.17s/it][2025-04-22 11:35:34] (step=0024475) Train Loss: 6.4035, Train Steps/Sec: 0.85 + 60%|███████████████████████████████████████████████████████████████████ | 24499/40903 [9:48:33<5:16:51, 1.16s/it][2025-04-22 11:36:04] (step=0024500) Train Loss: 6.3992, Train Steps/Sec: 0.84 + 60%|███████████████████████████████████████████████████████████████████▏ | 24524/40903 [9:49:02<5:27:40, 1.20s/it][2025-04-22 11:36:33] (step=0024525) Train Loss: 6.4112, Train Steps/Sec: 0.85 + 60%|███████████████████████████████████████████████████████████████████▏ | 24549/40903 [9:49:32<5:22:40, 1.18s/it][2025-04-22 11:37:03] (step=0024550) Train Loss: 6.4146, Train Steps/Sec: 0.85 + 60%|███████████████████████████████████████████████████████████████████▎ | 24574/40903 [9:50:01<5:20:53, 1.18s/it][2025-04-22 11:37:33] (step=0024575) Train Loss: 6.4123, Train Steps/Sec: 0.84 + 60%|███████████████████████████████████████████████████████████████████▎ | 24599/40903 [9:50:31<5:17:11, 1.17s/it][2025-04-22 11:38:02] (step=0024600) Train Loss: 6.4168, Train Steps/Sec: 0.84 + 60%|███████████████████████████████████████████████████████████████████▍ | 24624/40903 [9:51:01<5:26:43, 1.20s/it][2025-04-22 11:38:43] (step=0024625) Train Loss: 6.3941, Train Steps/Sec: 0.62 + 60%|███████████████████████████████████████████████████████████████████▍ | 24649/40903 [9:51:41<5:25:24, 1.20s/it][2025-04-22 11:39:12] (step=0024650) Train Loss: 6.3901, Train Steps/Sec: 0.84 + 60%|███████████████████████████████████████████████████████████████████▌ | 24674/40903 [9:52:11<5:16:12, 1.17s/it][2025-04-22 11:39:42] (step=0024675) Train Loss: 6.4258, Train Steps/Sec: 0.85 + 60%|███████████████████████████████████████████████████████████████████▋ | 24699/40903 [9:52:46<5:14:52, 1.17s/it][2025-04-22 11:40:17] (step=0024700) Train Loss: 6.3942, Train Steps/Sec: 0.71 + 60%|███████████████████████████████████████████████████████████████████▋ | 24724/40903 [9:53:16<5:24:39, 1.20s/it][2025-04-22 11:40:47] (step=0024725) Train Loss: 6.4318, Train Steps/Sec: 0.85 + 61%|███████████████████████████████████████████████████████████████████▊ | 24749/40903 [9:53:45<5:21:38, 1.19s/it][2025-04-22 11:41:16] (step=0024750) Train Loss: 6.4038, Train Steps/Sec: 0.85 + 61%|███████████████████████████████████████████████████████████████████▊ | 24774/40903 [9:54:15<5:22:50, 1.20s/it][2025-04-22 11:41:46] (step=0024775) Train Loss: 6.3511, Train Steps/Sec: 0.84 + 61%|███████████████████████████████████████████████████████████████████▉ | 24799/40903 [9:54:44<5:14:52, 1.17s/it][2025-04-22 11:42:16] (step=0024800) Train Loss: 6.4495, Train Steps/Sec: 0.84 + 61%|███████████████████████████████████████████████████████████████████▉ | 24824/40903 [9:55:14<5:22:45, 1.20s/it][2025-04-22 11:42:45] (step=0024825) Train Loss: 6.4081, Train Steps/Sec: 0.84 + 61%|████████████████████████████████████████████████████████████████████ | 24849/40903 [9:55:44<5:18:35, 1.19s/it][2025-04-22 11:43:15] (step=0024850) Train Loss: 6.4054, Train Steps/Sec: 0.84 + 61%|████████████████████████████████████████████████████████████████████ | 24874/40903 [9:56:14<5:15:20, 1.18s/it][2025-04-22 11:43:45] (step=0024875) Train Loss: 6.4040, Train Steps/Sec: 0.84 + 61%|████████████████████████████████████████████████████████████████████▏ | 24899/40903 [9:56:43<5:13:03, 1.17s/it][2025-04-22 11:44:14] (step=0024900) Train Loss: 6.3701, Train Steps/Sec: 0.85 + 61%|████████████████████████████████████████████████████████████████████▏ | 24924/40903 [9:57:13<5:26:09, 1.22s/it][2025-04-22 11:44:44] (step=0024925) Train Loss: 6.3957, Train Steps/Sec: 0.84 + 61%|████████████████████████████████████████████████████████████████████▎ | 24949/40903 [9:57:43<5:16:41, 1.19s/it][2025-04-22 11:45:14] (step=0024950) Train Loss: 6.3890, Train Steps/Sec: 0.84 + 61%|████████████████████████████████████████████████████████████████████▍ | 24974/40903 [9:58:12<5:10:27, 1.17s/it][2025-04-22 11:45:44] (step=0024975) Train Loss: 6.4153, Train Steps/Sec: 0.84 + 61%|████████████████████████████████████████████████████████████████████▍ | 24999/40903 [9:58:42<5:12:12, 1.18s/it][2025-04-22 11:46:13] (step=0025000) Train Loss: 6.3882, Train Steps/Sec: 0.84 + 61%|████████████████████████████████████████████████████████████████████▌ | 25024/40903 [9:59:12<5:18:34, 1.20s/it][2025-04-22 11:46:43] (step=0025025) Train Loss: 6.4043, Train Steps/Sec: 0.84 + 61%|████████████████████████████████████████████████████████████████████▌ | 25049/40903 [9:59:41<5:08:41, 1.17s/it][2025-04-22 11:47:12] (step=0025050) Train Loss: 6.4070, Train Steps/Sec: 0.85 + 61%|████████████████████████████████████████████████████████████████████ | 25074/40903 [10:00:10<5:13:19, 1.19s/it][2025-04-22 11:47:42] (step=0025075) Train Loss: 6.3898, Train Steps/Sec: 0.84 + 61%|████████████████████████████████████████████████████████████████████ | 25099/40903 [10:00:40<5:08:13, 1.17s/it][2025-04-22 11:48:11] (step=0025100) Train Loss: 6.4247, Train Steps/Sec: 0.84 + 61%|████████████████████████████████████████████████████████████████████▏ | 25124/40903 [10:01:10<5:16:39, 1.20s/it][2025-04-22 11:48:41] (step=0025125) Train Loss: 6.4277, Train Steps/Sec: 0.85 + 61%|████████████████████████████████████████████████████████████████████▏ | 25149/40903 [10:01:39<5:09:53, 1.18s/it][2025-04-22 11:49:10] (step=0025150) Train Loss: 6.4075, Train Steps/Sec: 0.85 + 62%|████████████████████████████████████████████████████████████████████▎ | 25174/40903 [10:02:09<5:08:17, 1.18s/it][2025-04-22 11:49:40] (step=0025175) Train Loss: 6.3708, Train Steps/Sec: 0.84 + 62%|████████████████████████████████████████████████████████████████████▍ | 25199/40903 [10:02:38<5:05:00, 1.17s/it][2025-04-22 11:50:10] (step=0025200) Train Loss: 6.4080, Train Steps/Sec: 0.84 + 62%|████████████████████████████████████████████████████████████████████▍ | 25224/40903 [10:03:08<5:17:08, 1.21s/it][2025-04-22 11:50:39] (step=0025225) Train Loss: 6.4235, Train Steps/Sec: 0.84 + 62%|████████████████████████████████████████████████████████████████████▌ | 25249/40903 [10:03:38<5:06:43, 1.18s/it][2025-04-22 11:51:09] (step=0025250) Train Loss: 6.4453, Train Steps/Sec: 0.85 + 62%|████████████████████████████████████████████████████████████████████▌ | 25274/40903 [10:04:07<5:05:07, 1.17s/it][2025-04-22 11:51:38] (step=0025275) Train Loss: 6.3733, Train Steps/Sec: 0.85 + 62%|████████████████████████████████████████████████████████████████████▋ | 25299/40903 [10:04:37<5:02:15, 1.16s/it][2025-04-22 11:52:08] (step=0025300) Train Loss: 6.3965, Train Steps/Sec: 0.84 + 62%|████████████████████████████████████████████████████████████████████▋ | 25324/40903 [10:05:06<5:11:09, 1.20s/it][2025-04-22 11:52:38] (step=0025325) Train Loss: 6.3751, Train Steps/Sec: 0.84 + 62%|████████████████████████████████████████████████████████████████████▊ | 25349/40903 [10:05:36<5:03:58, 1.17s/it][2025-04-22 11:53:07] (step=0025350) Train Loss: 6.3948, Train Steps/Sec: 0.84 + 62%|████████████████████████████████████████████████████████████████████▊ | 25374/40903 [10:06:06<5:07:40, 1.19s/it][2025-04-22 11:53:37] (step=0025375) Train Loss: 6.4073, Train Steps/Sec: 0.84 + 62%|████████████████████████████████████████████████████████████████████▉ | 25399/40903 [10:06:35<5:05:09, 1.18s/it][2025-04-22 11:54:07] (step=0025400) Train Loss: 6.3859, Train Steps/Sec: 0.85 + 62%|████████████████████████████████████████████████████████████████████▉ | 25424/40903 [10:07:05<5:08:17, 1.19s/it][2025-04-22 11:54:36] (step=0025425) Train Loss: 6.4200, Train Steps/Sec: 0.85 + 62%|█████████████████████████████████████████████████████████████████████ | 25449/40903 [10:07:34<5:03:15, 1.18s/it][2025-04-22 11:55:05] (step=0025450) Train Loss: 6.3703, Train Steps/Sec: 0.85 + 62%|█████████████████████████████████████████████████████████████████████▏ | 25474/40903 [10:08:04<5:00:29, 1.17s/it][2025-04-22 11:55:35] (step=0025475) Train Loss: 6.3969, Train Steps/Sec: 0.85 + 62%|█████████████████████████████████████████████████████████████████████▏ | 25499/40903 [10:08:33<4:59:58, 1.17s/it][2025-04-22 11:56:04] (step=0025500) Train Loss: 6.3837, Train Steps/Sec: 0.85 + 62%|█████████████████████████████████████████████████████████████████████▎ | 25524/40903 [10:09:09<5:06:25, 1.20s/it][2025-04-22 11:56:41] (step=0025525) Train Loss: 6.4233, Train Steps/Sec: 0.69 + 62%|█████████████████████████████████████████████████████████████████████▎ | 25549/40903 [10:09:46<6:34:38, 1.54s/it][2025-04-22 11:57:18] (step=0025550) Train Loss: 6.3681, Train Steps/Sec: 0.68 + 63%|█████████████████████████████████████████████████████████████████████▍ | 25574/40903 [10:10:16<5:01:56, 1.18s/it][2025-04-22 11:57:47] (step=0025575) Train Loss: 6.3770, Train Steps/Sec: 0.85 + 63%|█████████████████████████████████████████████████████████████████████▍ | 25599/40903 [10:10:45<4:58:30, 1.17s/it][2025-04-22 11:58:17] (step=0025600) Train Loss: 6.4189, Train Steps/Sec: 0.84 + 63%|█████████████████████████████████████████████████████████████████████▌ | 25624/40903 [10:11:15<5:06:17, 1.20s/it][2025-04-22 11:58:46] (step=0025625) Train Loss: 6.3783, Train Steps/Sec: 0.85 + 63%|█████████████████████████████████████████████████████████████████████▌ | 25649/40903 [10:11:44<4:57:58, 1.17s/it][2025-04-22 11:59:16] (step=0025650) Train Loss: 6.3787, Train Steps/Sec: 0.85 + 63%|█████████████████████████████████████████████████████████████████████▋ | 25674/40903 [10:12:14<4:57:22, 1.17s/it][2025-04-22 11:59:45] (step=0025675) Train Loss: 6.4154, Train Steps/Sec: 0.85 + 63%|█████████████████████████████████████████████████████████████████████▋ | 25699/40903 [10:12:44<5:00:14, 1.18s/it][2025-04-22 12:00:15] (step=0025700) Train Loss: 6.4135, Train Steps/Sec: 0.84 + 63%|█████████████████████████████████████████████████████████████████████▊ | 25724/40903 [10:13:13<5:05:30, 1.21s/it][2025-04-22 12:00:45] (step=0025725) Train Loss: 6.3870, Train Steps/Sec: 0.84 + 63%|█████████████████████████████████████████████████████████████████████▉ | 25749/40903 [10:13:43<5:01:44, 1.19s/it][2025-04-22 12:01:14] (step=0025750) Train Loss: 6.3618, Train Steps/Sec: 0.85 + 63%|█████████████████████████████████████████████████████████████████████▉ | 25774/40903 [10:14:12<4:57:25, 1.18s/it][2025-04-22 12:01:44] (step=0025775) Train Loss: 6.4361, Train Steps/Sec: 0.85 + 63%|██████████████████████████████████████████████████████████████████████ | 25799/40903 [10:14:42<4:49:04, 1.15s/it][2025-04-22 12:02:13] (step=0025800) Train Loss: 6.4192, Train Steps/Sec: 0.84 + 63%|██████████████████████████████████████████████████████████████████████ | 25824/40903 [10:15:11<5:01:10, 1.20s/it][2025-04-22 12:02:43] (step=0025825) Train Loss: 6.4103, Train Steps/Sec: 0.85 + 63%|██████████████████████████████████████████████████████████████████████▏ | 25849/40903 [10:15:41<4:55:58, 1.18s/it][2025-04-22 12:03:12] (step=0025850) Train Loss: 6.4460, Train Steps/Sec: 0.85 + 63%|██████████████████████████████████████████████████████████████████████▏ | 25874/40903 [10:16:10<4:55:45, 1.18s/it][2025-04-22 12:03:42] (step=0025875) Train Loss: 6.4229, Train Steps/Sec: 0.85 + 63%|██████████████████████████████████████████████████████████████████████▎ | 25899/40903 [10:16:40<4:51:23, 1.17s/it][2025-04-22 12:04:11] (step=0025900) Train Loss: 6.3940, Train Steps/Sec: 0.85 + 63%|██████████████████████████████████████████████████████████████████████▎ | 25924/40903 [10:17:10<5:02:04, 1.21s/it][2025-04-22 12:04:41] (step=0025925) Train Loss: 6.4186, Train Steps/Sec: 0.84 + 63%|██████████████████████████████████████████████████████████████████████▍ | 25949/40903 [10:17:39<4:54:09, 1.18s/it][2025-04-22 12:05:11] (step=0025950) Train Loss: 6.4397, Train Steps/Sec: 0.84 + 64%|██████████████████████████████████████████████████████████████████████▍ | 25974/40903 [10:18:09<4:51:52, 1.17s/it][2025-04-22 12:05:40] (step=0025975) Train Loss: 6.4216, Train Steps/Sec: 0.85 + 64%|██████████████████████████████████████████████████████████████████████▌ | 25999/40903 [10:18:38<4:51:22, 1.17s/it][2025-04-22 12:06:10] (step=0026000) Train Loss: 6.4130, Train Steps/Sec: 0.84 +You are using a model of type instructblip to instantiate a model of type blip-2. This is not supported for all configurations of models and can yield errors. +model_name_or_path:/tmp/haozhezhao/model/instructblip-flan-t5-xl +image_place_holder: +Some kwargs in processor config are unused and will not have any effect: num_query_tokens. +tokenizer length after expend 32102 +tokenizer length before expend 32102 +100%|██████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████| 3/3 [02:54<00:00, 58.16s/it] +[2025-04-22 12:10:52] Finish Eval in 26000 steps...███████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████| 3/3 [02:53<00:00, 57.66s/it] +processing left-padding... +processing left-padding... +processing left-padding... +[2025-04-22 12:11:13] Saved checkpoint to checkpoint/BeforeFix_Blip_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_objects/002-GPT-XL/checkpoints/0026000.pt +[2025-04-22 12:11:15] Removed old checkpoint: checkpoint/BeforeFix_Blip_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_objects/002-GPT-XL/checkpoints/0022000.pt + 64%|██████████████████████████████████████████████████████████████████████▌ | 26024/40903 [10:24:13<5:03:52, 1.23s/it][2025-04-22 12:11:45] (step=0026025) Train Loss: 6.3918, Train Steps/Sec: 0.07 + 64%|██████████████████████████████████████████████████████████████████████▋ | 26049/40903 [10:24:43<4:55:00, 1.19s/it][2025-04-22 12:12:14] (step=0026050) Train Loss: 6.4077, Train Steps/Sec: 0.84 + 64%|██████████████████████████████████████████████████████████████████████▊ | 26074/40903 [10:25:13<4:52:36, 1.18s/it][2025-04-22 12:12:44] (step=0026075) Train Loss: 6.3913, Train Steps/Sec: 0.85 + 64%|██████████████████████████████████████████████████████████████████████▊ | 26099/40903 [10:25:42<4:48:27, 1.17s/it][2025-04-22 12:13:14] (step=0026100) Train Loss: 6.4359, Train Steps/Sec: 0.84 + 64%|██████████████████████████████████████████████████████████████████████▉ | 26124/40903 [10:26:12<4:58:19, 1.21s/it][2025-04-22 12:13:43] (step=0026125) Train Loss: 6.4196, Train Steps/Sec: 0.85 + 64%|██████████████████████████████████████████████████████████████████████▉ | 26149/40903 [10:26:41<4:49:49, 1.18s/it][2025-04-22 12:14:13] (step=0026150) Train Loss: 6.3612, Train Steps/Sec: 0.85 + 64%|███████████████████████████████████████████████████████████████████████ | 26174/40903 [10:27:11<4:48:49, 1.18s/it][2025-04-22 12:14:42] (step=0026175) Train Loss: 6.4022, Train Steps/Sec: 0.84 + 64%|███████████████████████████████████████████████████████████████████████ | 26199/40903 [10:27:41<4:48:42, 1.18s/it][2025-04-22 12:15:12] (step=0026200) Train Loss: 6.4137, Train Steps/Sec: 0.84 + 64%|███████████████████████████████████████████████████████████████████████▏ | 26224/40903 [10:28:10<4:53:44, 1.20s/it][2025-04-22 12:15:42] (step=0026225) Train Loss: 6.4210, Train Steps/Sec: 0.85 + 64%|███████████████████████████████████████████████████████████████████████▏ | 26249/40903 [10:28:40<4:48:16, 1.18s/it][2025-04-22 12:16:11] (step=0026250) Train Loss: 6.4120, Train Steps/Sec: 0.84 + 64%|███████████████████████████████████████████████████████████████████████▎ | 26274/40903 [10:29:10<4:50:50, 1.19s/it][2025-04-22 12:16:41] (step=0026275) Train Loss: 6.3913, Train Steps/Sec: 0.84 + 64%|███████████████████████████████████████████████████████████████████████▎ | 26299/40903 [10:29:39<4:41:57, 1.16s/it][2025-04-22 12:17:10] (step=0026300) Train Loss: 6.3595, Train Steps/Sec: 0.85 + 64%|███████████████████████████████████████████████████████████████████████▍ | 26324/40903 [10:30:09<4:48:15, 1.19s/it][2025-04-22 12:17:40] (step=0026325) Train Loss: 6.4031, Train Steps/Sec: 0.85 + 64%|███████████████████████████████████████████████████████████████████████▌ | 26349/40903 [10:30:39<4:45:21, 1.18s/it][2025-04-22 12:18:10] (step=0026350) Train Loss: 6.3751, Train Steps/Sec: 0.84 + 64%|███████████████████████████████████████████████████████████████████████▌ | 26374/40903 [10:31:08<4:44:54, 1.18s/it][2025-04-22 12:18:40] (step=0026375) Train Loss: 6.3862, Train Steps/Sec: 0.84 + 65%|███████████████████████████████████████████████████████████████████████▋ | 26399/40903 [10:31:38<4:42:24, 1.17s/it][2025-04-22 12:19:09] (step=0026400) Train Loss: 6.4219, Train Steps/Sec: 0.84 + 65%|███████████████████████████████████████████████████████████████████████▋ | 26424/40903 [10:32:08<4:49:11, 1.20s/it][2025-04-22 12:19:39] (step=0026425) Train Loss: 6.3951, Train Steps/Sec: 0.85 + 65%|███████████████████████████████████████████████████████████████████████▊ | 26449/40903 [10:32:37<4:46:52, 1.19s/it][2025-04-22 12:20:09] (step=0026450) Train Loss: 6.3858, Train Steps/Sec: 0.84 + 65%|███████████████████████████████████████████████████████████████████████▊ | 26474/40903 [10:33:07<4:40:58, 1.17s/it][2025-04-22 12:20:38] (step=0026475) Train Loss: 6.4283, Train Steps/Sec: 0.84 + 65%|███████████████████████████████████████████████████████████████████████▉ | 26499/40903 [10:33:37<4:43:32, 1.18s/it][2025-04-22 12:21:08] (step=0026500) Train Loss: 6.3632, Train Steps/Sec: 0.84 + 65%|███████████████████████████████████████████████████████████████████████▉ | 26524/40903 [10:34:06<4:47:04, 1.20s/it][2025-04-22 12:21:38] (step=0026525) Train Loss: 6.3975, Train Steps/Sec: 0.84 + 65%|████████████████████████████████████████████████████████████████████████ | 26549/40903 [10:34:36<4:42:58, 1.18s/it][2025-04-22 12:22:07] (step=0026550) Train Loss: 6.4019, Train Steps/Sec: 0.85 + 65%|████████████████████████████████████████████████████████████████████████ | 26574/40903 [10:35:05<4:39:58, 1.17s/it][2025-04-22 12:22:36] (step=0026575) Train Loss: 6.3803, Train Steps/Sec: 0.85 + 65%|████████████████████████████████████████████████████████████████████████▏ | 26599/40903 [10:35:35<4:38:38, 1.17s/it][2025-04-22 12:23:06] (step=0026600) Train Loss: 6.3680, Train Steps/Sec: 0.85 + 65%|████████████████████████████████████████████████████████████████████████▎ | 26624/40903 [10:36:04<4:48:08, 1.21s/it][2025-04-22 12:23:35] (step=0026625) Train Loss: 6.4116, Train Steps/Sec: 0.85 + 65%|████████████████████████████████████████████████████████████████████████▎ | 26649/40903 [10:36:33<4:41:03, 1.18s/it][2025-04-22 12:24:05] (step=0026650) Train Loss: 6.4093, Train Steps/Sec: 0.85 + 65%|████████████████████████████████████████████████████████████████████████▍ | 26674/40903 [10:37:03<4:38:59, 1.18s/it][2025-04-22 12:24:34] (step=0026675) Train Loss: 6.3766, Train Steps/Sec: 0.85 + 65%|████████████████████████████████████████████████████████████████████████▍ | 26699/40903 [10:37:32<4:37:30, 1.17s/it][2025-04-22 12:25:04] (step=0026700) Train Loss: 6.3754, Train Steps/Sec: 0.84 + 65%|████████████████████████████████████████████████████████████████████████▌ | 26724/40903 [10:38:02<4:45:29, 1.21s/it][2025-04-22 12:25:34] (step=0026725) Train Loss: 6.4305, Train Steps/Sec: 0.84 + 65%|████████████████████████████████████████████████████████████████████████▌ | 26749/40903 [10:38:32<4:37:43, 1.18s/it][2025-04-22 12:26:03] (step=0026750) Train Loss: 6.3848, Train Steps/Sec: 0.85 + 65%|████████████████████████████████████████████████████████████████████████▋ | 26774/40903 [10:39:01<4:39:11, 1.19s/it][2025-04-22 12:26:33] (step=0026775) Train Loss: 6.3941, Train Steps/Sec: 0.85 + 66%|████████████████████████████████████████████████████████████████████████▋ | 26799/40903 [10:39:31<4:35:48, 1.17s/it][2025-04-22 12:27:02] (step=0026800) Train Loss: 6.4522, Train Steps/Sec: 0.84 + 66%|████████████████████████████████████████████████████████████████████████▊ | 26824/40903 [10:40:00<4:42:23, 1.20s/it][2025-04-22 12:27:32] (step=0026825) Train Loss: 6.4150, Train Steps/Sec: 0.85 + 66%|████████████████████████████████████████████████████████████████████████▊ | 26849/40903 [10:40:30<4:39:29, 1.19s/it][2025-04-22 12:28:01] (step=0026850) Train Loss: 6.3981, Train Steps/Sec: 0.85 + 66%|████████████████████████████████████████████████████████████████████████▉ | 26874/40903 [10:40:59<4:32:26, 1.17s/it][2025-04-22 12:28:31] (step=0026875) Train Loss: 6.3971, Train Steps/Sec: 0.85 + 66%|████████████████████████████████████████████████████████████████████████▉ | 26899/40903 [10:41:35<4:33:32, 1.17s/it][2025-04-22 12:29:06] (step=0026900) Train Loss: 6.4253, Train Steps/Sec: 0.70 + 66%|█████████████████████████████████████████████████████████████████████████ | 26924/40903 [10:42:05<4:40:33, 1.20s/it][2025-04-22 12:29:36] (step=0026925) Train Loss: 6.3941, Train Steps/Sec: 0.85 + 66%|█████████████████████████████████████████████████████████████████████████▏ | 26949/40903 [10:42:34<4:31:13, 1.17s/it][2025-04-22 12:30:05] (step=0026950) Train Loss: 6.3735, Train Steps/Sec: 0.85 + 66%|█████████████████████████████████████████████████████████████████████████▏ | 26974/40903 [10:43:04<4:31:42, 1.17s/it][2025-04-22 12:30:35] (step=0026975) Train Loss: 6.4401, Train Steps/Sec: 0.85 + 66%|█████████████████████████████████████████████████████████████████████████▎ | 26999/40903 [10:43:33<4:33:31, 1.18s/it][2025-04-22 12:31:05] (step=0027000) Train Loss: 6.3474, Train Steps/Sec: 0.84 + 66%|█████████████████████████████████████████████████████████████████████████▎ | 27024/40903 [10:44:03<4:39:08, 1.21s/it][2025-04-22 12:31:34] (step=0027025) Train Loss: 6.4269, Train Steps/Sec: 0.85 + 66%|█████████████████████████████████████████████████████████████████████████▍ | 27049/40903 [10:44:32<4:34:04, 1.19s/it][2025-04-22 12:32:03] (step=0027050) Train Loss: 6.4030, Train Steps/Sec: 0.85 + 66%|█████████████████████████████████████████████████████████████████████████▍ | 27074/40903 [10:45:02<4:32:49, 1.18s/it][2025-04-22 12:32:33] (step=0027075) Train Loss: 6.3911, Train Steps/Sec: 0.85 + 66%|█████████████████████████████████████████████████████████████████████████▌ | 27099/40903 [10:45:31<4:29:05, 1.17s/it][2025-04-22 12:33:02] (step=0027100) Train Loss: 6.4088, Train Steps/Sec: 0.85 + 66%|█████████████████████████████████████████████████████████████████████████▌ | 27124/40903 [10:46:01<4:34:21, 1.19s/it][2025-04-22 12:33:32] (step=0027125) Train Loss: 6.3737, Train Steps/Sec: 0.85 + 66%|█████████████████████████████████████████████████████████████████████████▋ | 27149/40903 [10:46:30<4:30:45, 1.18s/it][2025-04-22 12:34:02] (step=0027150) Train Loss: 6.4344, Train Steps/Sec: 0.85 + 66%|█████████████████████████████████████████████████████████████████████████▋ | 27174/40903 [10:47:00<4:29:54, 1.18s/it][2025-04-22 12:34:31] (step=0027175) Train Loss: 6.3792, Train Steps/Sec: 0.85 + 66%|█████████████████████████████████████████████████████████████████████████▊ | 27199/40903 [10:47:29<4:28:10, 1.17s/it][2025-04-22 12:35:01] (step=0027200) Train Loss: 6.3996, Train Steps/Sec: 0.84 + 67%|█████████████████████████████████████████████████████████████████████████▉ | 27224/40903 [10:47:59<4:37:03, 1.22s/it][2025-04-22 12:35:30] (step=0027225) Train Loss: 6.3779, Train Steps/Sec: 0.84 + 67%|█████████████████████████████████████████████████████████████████████████▉ | 27249/40903 [10:48:29<4:27:25, 1.18s/it][2025-04-22 12:36:00] (step=0027250) Train Loss: 6.3479, Train Steps/Sec: 0.85 + 67%|██████████████████████████████████████████████████████████████████████████ | 27274/40903 [10:48:58<4:26:07, 1.17s/it][2025-04-22 12:36:29] (step=0027275) Train Loss: 6.4369, Train Steps/Sec: 0.85 + 67%|██████████████████████████████████████████████████████████████████████████ | 27299/40903 [10:49:28<4:28:00, 1.18s/it][2025-04-22 12:36:59] (step=0027300) Train Loss: 6.4114, Train Steps/Sec: 0.84 + 67%|██████████████████████████████████████████████████████████████████████████▏ | 27324/40903 [10:49:57<4:31:53, 1.20s/it][2025-04-22 12:37:28] (step=0027325) Train Loss: 6.4126, Train Steps/Sec: 0.85 + 67%|██████████████████████████████████████████████████████████████████████████▏ | 27349/40903 [10:50:27<4:27:34, 1.18s/it][2025-04-22 12:37:58] (step=0027350) Train Loss: 6.4077, Train Steps/Sec: 0.85 + 67%|██████████████████████████████████████████████████████████████████████████▎ | 27374/40903 [10:50:56<4:25:24, 1.18s/it][2025-04-22 12:38:28] (step=0027375) Train Loss: 6.4189, Train Steps/Sec: 0.85 + 67%|██████████████████████████████████████████████████████████████████████████▎ | 27399/40903 [10:51:26<4:21:31, 1.16s/it][2025-04-22 12:38:57] (step=0027400) Train Loss: 6.4515, Train Steps/Sec: 0.84 + 67%|██████████████████████████████████████████████████████████████████████████▍ | 27424/40903 [10:51:55<4:27:49, 1.19s/it][2025-04-22 12:39:27] (step=0027425) Train Loss: 6.3869, Train Steps/Sec: 0.85 + 67%|██████████████████████████████████████████████████████████████████████████▍ | 27449/40903 [10:52:25<4:25:45, 1.19s/it][2025-04-22 12:39:56] (step=0027450) Train Loss: 6.3828, Train Steps/Sec: 0.85 + 67%|██████████████████████████████████████████████████████████████████████████▌ | 27474/40903 [10:52:54<4:20:49, 1.17s/it][2025-04-22 12:40:26] (step=0027475) Train Loss: 6.4216, Train Steps/Sec: 0.85 + 67%|██████████████████████████████████████████████████████████████████████████▋ | 27499/40903 [10:53:24<4:20:22, 1.17s/it][2025-04-22 12:40:55] (step=0027500) Train Loss: 6.4534, Train Steps/Sec: 0.85 + 67%|██████████████████████████████████████████████████████████████████████████▋ | 27524/40903 [10:53:54<4:29:04, 1.21s/it][2025-04-22 12:41:25] (step=0027525) Train Loss: 6.4001, Train Steps/Sec: 0.84 + 67%|██████████████████████████████████████████████████████████████████████████▊ | 27549/40903 [10:54:23<4:19:39, 1.17s/it][2025-04-22 12:41:54] (step=0027550) Train Loss: 6.4073, Train Steps/Sec: 0.85 + 67%|██████████████████████████████████████████████████████████████████████████▊ | 27574/40903 [10:54:53<4:19:42, 1.17s/it][2025-04-22 12:42:24] (step=0027575) Train Loss: 6.3658, Train Steps/Sec: 0.84 + 67%|██████████████████████████████████████████████████████████████████████████▉ | 27599/40903 [10:55:22<4:19:59, 1.17s/it][2025-04-22 12:42:54] (step=0027600) Train Loss: 6.3839, Train Steps/Sec: 0.84 + 68%|██████████████████████████████████████████████████████████████████████████▉ | 27624/40903 [10:55:52<4:24:09, 1.19s/it][2025-04-22 12:43:23] (step=0027625) Train Loss: 6.3747, Train Steps/Sec: 0.85 + 68%|███████████████████████████████████████████████████████████████████████████ | 27649/40903 [10:56:22<4:21:19, 1.18s/it][2025-04-22 12:43:53] (step=0027650) Train Loss: 6.4032, Train Steps/Sec: 0.85 + 68%|███████████████████████████████████████████████████████████████████████████ | 27674/40903 [10:56:51<4:25:56, 1.21s/it][2025-04-22 12:44:23] (step=0027675) Train Loss: 6.4510, Train Steps/Sec: 0.84 + 68%|███████████████████████████████████████████████████████████████████████████▏ | 27699/40903 [10:57:21<4:16:45, 1.17s/it][2025-04-22 12:44:52] (step=0027700) Train Loss: 6.3655, Train Steps/Sec: 0.85 + 68%|███████████████████████████████████████████████████████████████████████████▏ | 27724/40903 [10:57:50<4:20:10, 1.18s/it][2025-04-22 12:45:21] (step=0027725) Train Loss: 6.3668, Train Steps/Sec: 0.85 + 68%|███████████████████████████████████████████████████████████████████████████▎ | 27749/40903 [10:58:20<4:18:18, 1.18s/it][2025-04-22 12:45:51] (step=0027750) Train Loss: 6.3430, Train Steps/Sec: 0.85 + 68%|███████████████████████████████████████████████████████████████████████████▎ | 27774/40903 [10:58:49<4:15:26, 1.17s/it][2025-04-22 12:46:20] (step=0027775) Train Loss: 6.4219, Train Steps/Sec: 0.85 + 68%|███████████████████████████████████████████████████████████████████████████▍ | 27799/40903 [10:59:19<4:16:24, 1.17s/it][2025-04-22 12:46:50] (step=0027800) Train Loss: 6.3666, Train Steps/Sec: 0.84 + 68%|███████████████████████████████████████████████████████████████████████████▌ | 27824/40903 [10:59:48<4:20:57, 1.20s/it][2025-04-22 12:47:19] (step=0027825) Train Loss: 6.3937, Train Steps/Sec: 0.85 + 68%|███████████████████████████████████████████████████████████████████████████▌ | 27849/40903 [11:00:18<4:17:54, 1.19s/it][2025-04-22 12:47:49] (step=0027850) Train Loss: 6.4389, Train Steps/Sec: 0.85 + 68%|███████████████████████████████████████████████████████████████████████████▋ | 27874/40903 [11:00:47<4:14:04, 1.17s/it][2025-04-22 12:48:18] (step=0027875) Train Loss: 6.3838, Train Steps/Sec: 0.85 + 68%|███████████████████████████████████████████████████████████████████████████▋ | 27899/40903 [11:01:17<4:14:15, 1.17s/it][2025-04-22 12:48:48] (step=0027900) Train Loss: 6.4014, Train Steps/Sec: 0.85 + 68%|███████████████████████████████████████████████████████████████████████████▊ | 27924/40903 [11:01:46<4:19:23, 1.20s/it][2025-04-22 12:49:17] (step=0027925) Train Loss: 6.3744, Train Steps/Sec: 0.85 + 68%|███████████████████████████████████████████████████████████████████████████▊ | 27949/40903 [11:02:16<4:16:09, 1.19s/it][2025-04-22 12:49:47] (step=0027950) Train Loss: 6.3857, Train Steps/Sec: 0.84 + 68%|███████████████████████████████████████████████████████████████████████████▉ | 27974/40903 [11:02:45<4:12:47, 1.17s/it][2025-04-22 12:50:17] (step=0027975) Train Loss: 6.4262, Train Steps/Sec: 0.85 + 68%|███████████████████████████████████████████████████████████████████████████▉ | 27999/40903 [11:03:15<4:12:25, 1.17s/it][2025-04-22 12:50:46] (step=0028000) Train Loss: 6.4002, Train Steps/Sec: 0.84 +You are using a model of type instructblip to instantiate a model of type blip-2. This is not supported for all configurations of models and can yield errors. +model_name_or_path:/tmp/haozhezhao/model/instructblip-flan-t5-xl +image_place_holder: +Some kwargs in processor config are unused and will not have any effect: num_query_tokens. +tokenizer length after expend 32102 +tokenizer length before expend 32102 +100%|██████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████| 3/3 [02:54<00:00, 58.11s/it] +[2025-04-22 12:55:28] Finish Eval in 28000 steps...███████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████| 3/3 [02:53<00:00, 57.74s/it] +processing left-padding... +processing left-padding... +processing left-padding... +[2025-04-22 12:55:50] Saved checkpoint to checkpoint/BeforeFix_Blip_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_objects/002-GPT-XL/checkpoints/0028000.pt +[2025-04-22 12:55:52] Removed old checkpoint: checkpoint/BeforeFix_Blip_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_objects/002-GPT-XL/checkpoints/0024000.pt + 69%|████████████████████████████████████████████████████████████████████████████ | 28024/40903 [11:08:51<4:24:47, 1.23s/it][2025-04-22 12:56:22] (step=0028025) Train Loss: 6.4437, Train Steps/Sec: 0.07 + 69%|████████████████████████████████████████████████████████████████████████████ | 28049/40903 [11:09:20<4:11:41, 1.17s/it][2025-04-22 12:56:52] (step=0028050) Train Loss: 6.3659, Train Steps/Sec: 0.85 + 69%|████████████████████████████████████████████████████████████████████████████▏ | 28074/40903 [11:09:50<4:10:50, 1.17s/it][2025-04-22 12:57:21] (step=0028075) Train Loss: 6.3779, Train Steps/Sec: 0.85 + 69%|████████████████████████████████████████████████████████████████████████████▎ | 28099/40903 [11:10:19<4:09:59, 1.17s/it][2025-04-22 12:57:51] (step=0028100) Train Loss: 6.4194, Train Steps/Sec: 0.84 + 69%|████████████████████████████████████████████████████████████████████████████▎ | 28124/40903 [11:10:49<4:17:32, 1.21s/it][2025-04-22 12:58:20] (step=0028125) Train Loss: 6.3810, Train Steps/Sec: 0.85 + 69%|████████████████████████████████████████████████████████████████████████████▍ | 28149/40903 [11:11:19<4:11:59, 1.19s/it][2025-04-22 12:58:50] (step=0028150) Train Loss: 6.4045, Train Steps/Sec: 0.85 + 69%|████████████████████████████████████████████████████████████████████████████▍ | 28174/40903 [11:11:48<4:07:37, 1.17s/it][2025-04-22 12:59:19] (step=0028175) Train Loss: 6.3962, Train Steps/Sec: 0.85 + 69%|████████████████████████████████████████████████████████████████████████████▌ | 28199/40903 [11:12:17<4:07:29, 1.17s/it][2025-04-22 12:59:49] (step=0028200) Train Loss: 6.4088, Train Steps/Sec: 0.84 + 69%|████████████████████████████████████████████████████████████████████████████▌ | 28224/40903 [11:12:47<4:13:55, 1.20s/it][2025-04-22 13:00:18] (step=0028225) Train Loss: 6.3994, Train Steps/Sec: 0.85 + 69%|████████████████████████████████████████████████████████████████████████████▋ | 28249/40903 [11:13:16<4:06:35, 1.17s/it][2025-04-22 13:00:48] (step=0028250) Train Loss: 6.4017, Train Steps/Sec: 0.85 + 69%|████████████████████████████████████████████████████████████████████████████▋ | 28274/40903 [11:13:46<4:07:23, 1.18s/it][2025-04-22 13:01:17] (step=0028275) Train Loss: 6.4390, Train Steps/Sec: 0.85 + 69%|████████████████████████████████████████████████████████████████████████████▊ | 28299/40903 [11:14:15<4:07:47, 1.18s/it][2025-04-22 13:01:46] (step=0028300) Train Loss: 6.3844, Train Steps/Sec: 0.85 + 69%|████████████████████████████████████████████████████████████████████████████▊ | 28324/40903 [11:14:45<4:11:12, 1.20s/it][2025-04-22 13:02:16] (step=0028325) Train Loss: 6.4054, Train Steps/Sec: 0.84 + 69%|████████████████████████████████████████████████████████████████████████████▉ | 28349/40903 [11:15:14<4:07:06, 1.18s/it][2025-04-22 13:02:46] (step=0028350) Train Loss: 6.3903, Train Steps/Sec: 0.85 + 69%|████████████████████████████████████████████████████████████████████████████▉ | 28374/40903 [11:15:44<4:08:09, 1.19s/it][2025-04-22 13:03:15] (step=0028375) Train Loss: 6.4019, Train Steps/Sec: 0.85 + 69%|█████████████████████████████████████████████████████████████████████████████ | 28399/40903 [11:16:13<4:02:02, 1.16s/it][2025-04-22 13:03:45] (step=0028400) Train Loss: 6.4117, Train Steps/Sec: 0.84 + 69%|█████████████████████████████████████████████████████████████████████████████▏ | 28424/40903 [11:16:43<4:10:38, 1.21s/it][2025-04-22 13:04:14] (step=0028425) Train Loss: 6.3957, Train Steps/Sec: 0.84 + 70%|█████████████████████████████████████████████████████████████████████████████▏ | 28449/40903 [11:17:13<4:04:43, 1.18s/it][2025-04-22 13:04:44] (step=0028450) Train Loss: 6.4182, Train Steps/Sec: 0.84 + 70%|█████████████████████████████████████████████████████████████████████████████▎ | 28474/40903 [11:17:42<4:03:08, 1.17s/it][2025-04-22 13:05:13] (step=0028475) Train Loss: 6.3964, Train Steps/Sec: 0.85 + 70%|█████████████████████████████████████████████████████████████████████████████▎ | 28499/40903 [11:18:12<4:01:10, 1.17s/it][2025-04-22 13:05:43] (step=0028500) Train Loss: 6.4400, Train Steps/Sec: 0.84 + 70%|█████████████████████████████████████████████████████████████████████████████▍ | 28524/40903 [11:18:41<4:07:44, 1.20s/it][2025-04-22 13:06:13] (step=0028525) Train Loss: 6.4294, Train Steps/Sec: 0.85 + 70%|█████████████████████████████████████████████████████████████████████████████▍ | 28549/40903 [11:19:11<4:00:59, 1.17s/it][2025-04-22 13:06:42] (step=0028550) Train Loss: 6.3880, Train Steps/Sec: 0.85 + 70%|█████████████████████████████████████████████████████████████████████████████▌ | 28574/40903 [11:19:40<4:02:23, 1.18s/it][2025-04-22 13:07:12] (step=0028575) Train Loss: 6.3893, Train Steps/Sec: 0.85 + 70%|█████████████████████████████████████████████████████████████████████████████▌ | 28599/40903 [11:20:10<4:03:58, 1.19s/it][2025-04-22 13:07:42] (step=0028600) Train Loss: 6.3978, Train Steps/Sec: 0.84 + 70%|█████████████████████████████████████████████████████████████████████████████▋ | 28624/40903 [11:20:40<4:07:37, 1.21s/it][2025-04-22 13:08:11] (step=0028625) Train Loss: 6.3750, Train Steps/Sec: 0.85 + 70%|█████████████████████████████████████████████████████████████████████████████▋ | 28649/40903 [11:21:09<4:02:48, 1.19s/it][2025-04-22 13:08:41] (step=0028650) Train Loss: 6.4400, Train Steps/Sec: 0.84 + 70%|█████████████████████████████████████████████████████████████████████████████▊ | 28674/40903 [11:21:39<3:59:04, 1.17s/it][2025-04-22 13:09:10] (step=0028675) Train Loss: 6.4160, Train Steps/Sec: 0.85 + 70%|█████████████████████████████████████████████████████████████████████████████▉ | 28699/40903 [11:22:08<3:56:48, 1.16s/it][2025-04-22 13:09:40] (step=0028700) Train Loss: 6.4393, Train Steps/Sec: 0.85 + 70%|█████████████████████████████████████████████████████████████████████████████▉ | 28724/40903 [11:22:38<4:04:56, 1.21s/it][2025-04-22 13:10:09] (step=0028725) Train Loss: 6.3707, Train Steps/Sec: 0.84 + 70%|██████████████████████████████████████████████████████████████████████████████ | 28749/40903 [11:23:08<3:59:01, 1.18s/it][2025-04-22 13:10:39] (step=0028750) Train Loss: 6.4175, Train Steps/Sec: 0.85 + 70%|██████████████████████████████████████████████████████████████████████████████ | 28774/40903 [11:23:37<3:55:34, 1.17s/it][2025-04-22 13:11:08] (step=0028775) Train Loss: 6.3651, Train Steps/Sec: 0.85 + 70%|██████████████████████████████████████████████████████████████████████████████▏ | 28799/40903 [11:24:06<3:55:07, 1.17s/it][2025-04-22 13:11:38] (step=0028800) Train Loss: 6.3965, Train Steps/Sec: 0.85 + 70%|██████████████████████████████████████████████████████████████████████████████▏ | 28824/40903 [11:24:36<4:03:30, 1.21s/it][2025-04-22 13:12:08] (step=0028825) Train Loss: 6.3930, Train Steps/Sec: 0.84 + 71%|██████████████████████████████████████████████████████████████████████████████▎ | 28849/40903 [11:25:06<3:57:06, 1.18s/it][2025-04-22 13:12:37] (step=0028850) Train Loss: 6.4324, Train Steps/Sec: 0.85 + 71%|██████████████████████████████████████████████████████████████████████████████▎ | 28874/40903 [11:25:35<3:56:06, 1.18s/it][2025-04-22 13:13:07] (step=0028875) Train Loss: 6.4128, Train Steps/Sec: 0.85 + 71%|██████████████████████████████████████████████████████████████████████████████▍ | 28899/40903 [11:26:05<3:52:47, 1.16s/it][2025-04-22 13:13:36] (step=0028900) Train Loss: 6.4241, Train Steps/Sec: 0.84 + 71%|██████████████████████████████████████████████████████████████████████████████▍ | 28924/40903 [11:26:35<4:02:18, 1.21s/it][2025-04-22 13:14:06] (step=0028925) Train Loss: 6.4085, Train Steps/Sec: 0.84 + 71%|██████████████████████████████████████████████████████████████████████████████▌ | 28949/40903 [11:27:04<3:56:14, 1.19s/it][2025-04-22 13:14:35] (step=0028950) Train Loss: 6.4338, Train Steps/Sec: 0.85 + 71%|██████████████████████████████████████████████████████████████████████████████▋ | 28974/40903 [11:27:34<3:53:19, 1.17s/it][2025-04-22 13:15:05] (step=0028975) Train Loss: 6.3879, Train Steps/Sec: 0.85 + 71%|██████████████████████████████████████████████████████████████████████████████▋ | 28999/40903 [11:28:03<3:53:38, 1.18s/it][2025-04-22 13:15:35] (step=0029000) Train Loss: 6.3726, Train Steps/Sec: 0.84 + 71%|██████████████████████████████████████████████████████████████████████████████▊ | 29024/40903 [11:28:33<3:58:20, 1.20s/it][2025-04-22 13:16:04] (step=0029025) Train Loss: 6.3460, Train Steps/Sec: 0.85 + 71%|██████████████████████████████████████████████████████████████████████████████▊ | 29049/40903 [11:29:02<3:53:36, 1.18s/it][2025-04-22 13:16:34] (step=0029050) Train Loss: 6.3905, Train Steps/Sec: 0.85 + 71%|██████████████████████████████████████████████████████████████████████████████▉ | 29074/40903 [11:29:32<3:51:49, 1.18s/it][2025-04-22 13:17:03] (step=0029075) Train Loss: 6.3903, Train Steps/Sec: 0.84 + 71%|██████████████████████████████████████████████████████████████████████████████▉ | 29099/40903 [11:30:02<3:51:56, 1.18s/it][2025-04-22 13:17:33] (step=0029100) Train Loss: 6.3932, Train Steps/Sec: 0.84 + 71%|███████████████████████████████████████████████████████████████████████████████ | 29124/40903 [11:30:31<3:57:02, 1.21s/it][2025-04-22 13:18:03] (step=0029125) Train Loss: 6.3988, Train Steps/Sec: 0.84 + 71%|███████████████████████████████████████████████████████████████████████████████ | 29149/40903 [11:31:01<3:50:46, 1.18s/it][2025-04-22 13:18:32] (step=0029150) Train Loss: 6.4091, Train Steps/Sec: 0.85 + 71%|███████████████████████████████████████████████████████████████████████████████▏ | 29174/40903 [11:31:30<3:47:46, 1.17s/it][2025-04-22 13:19:01] (step=0029175) Train Loss: 6.4031, Train Steps/Sec: 0.85 + 71%|███████████████████████████████████████████████████████████████████████████████▏ | 29199/40903 [11:32:00<3:48:35, 1.17s/it][2025-04-22 13:19:31] (step=0029200) Train Loss: 6.3994, Train Steps/Sec: 0.84 + 71%|███████████████████████████████████████████████████████████████████████████████▎ | 29224/40903 [11:32:29<3:51:05, 1.19s/it][2025-04-22 13:20:01] (step=0029225) Train Loss: 6.3877, Train Steps/Sec: 0.85 + 72%|███████████████████████████████████████████████████████████████████████████████▎ | 29249/40903 [11:32:59<3:51:11, 1.19s/it][2025-04-22 13:20:30] (step=0029250) Train Loss: 6.4080, Train Steps/Sec: 0.85 + 72%|███████████████████████████████████████████████████████████████████████████████▍ | 29274/40903 [11:33:29<3:49:16, 1.18s/it][2025-04-22 13:21:00] (step=0029275) Train Loss: 6.4453, Train Steps/Sec: 0.84 + 72%|███████████████████████████████████████████████████████████████████████████████▌ | 29299/40903 [11:33:58<3:46:44, 1.17s/it][2025-04-22 13:21:30] (step=0029300) Train Loss: 6.3873, Train Steps/Sec: 0.84 + 72%|███████████████████████████████████████████████████████████████████████████████▌ | 29324/40903 [11:34:34<3:53:24, 1.21s/it][2025-04-22 13:22:05] (step=0029325) Train Loss: 6.4123, Train Steps/Sec: 0.70 + 72%|███████████████████████████████████████████████████████████████████████████████▋ | 29349/40903 [11:35:04<3:46:36, 1.18s/it][2025-04-22 13:22:35] (step=0029350) Train Loss: 6.4068, Train Steps/Sec: 0.85 + 72%|███████████████████████████████████████████████████████████████████████████████▋ | 29374/40903 [11:35:33<3:43:20, 1.16s/it][2025-04-22 13:23:04] (step=0029375) Train Loss: 6.4015, Train Steps/Sec: 0.86 + 72%|███████████████████████████████████████████████████████████████████████████████▊ | 29399/40903 [11:36:08<3:50:19, 1.20s/it][2025-04-22 13:23:40] (step=0029400) Train Loss: 6.4255, Train Steps/Sec: 0.70 + 72%|███████████████████████████████████████████████████████████████████████████████▊ | 29424/40903 [11:36:38<3:49:42, 1.20s/it][2025-04-22 13:24:09] (step=0029425) Train Loss: 6.4455, Train Steps/Sec: 0.84 + 72%|███████████████████████████████████████████████████████████████████████████████▉ | 29449/40903 [11:37:08<3:45:03, 1.18s/it][2025-04-22 13:24:39] (step=0029450) Train Loss: 6.4274, Train Steps/Sec: 0.85 + 72%|███████████████████████████████████████████████████████████████████████████████▉ | 29474/40903 [11:37:37<3:40:52, 1.16s/it][2025-04-22 13:25:08] (step=0029475) Train Loss: 6.3957, Train Steps/Sec: 0.85 + 72%|████████████████████████████████████████████████████████████████████████████████ | 29499/40903 [11:38:07<3:42:05, 1.17s/it][2025-04-22 13:25:38] (step=0029500) Train Loss: 6.3859, Train Steps/Sec: 0.84 + 72%|████████████████████████████████████████████████████████████████████████████████ | 29524/40903 [11:38:36<3:49:08, 1.21s/it][2025-04-22 13:26:08] (step=0029525) Train Loss: 6.3734, Train Steps/Sec: 0.85 + 72%|████████████████████████████████████████████████████████████████████████████████▏ | 29549/40903 [11:39:06<3:39:52, 1.16s/it][2025-04-22 13:26:37] (step=0029550) Train Loss: 6.3737, Train Steps/Sec: 0.85 + 72%|████████████████████████████████████████████████████████████████████████████████▎ | 29574/40903 [11:39:35<3:41:26, 1.17s/it][2025-04-22 13:27:06] (step=0029575) Train Loss: 6.4160, Train Steps/Sec: 0.85 + 72%|████████████████████████████████████████████████████████████████████████████████▎ | 29599/40903 [11:40:04<3:41:30, 1.18s/it][2025-04-22 13:27:36] (step=0029600) Train Loss: 6.4547, Train Steps/Sec: 0.84 + 72%|████████████████████████████████████████████████████████████████████████████████▍ | 29624/40903 [11:40:34<3:45:14, 1.20s/it][2025-04-22 13:28:05] (step=0029625) Train Loss: 6.3919, Train Steps/Sec: 0.85 + 72%|████████████████████████████████████████████████████████████████████████████████▍ | 29649/40903 [11:41:04<3:40:44, 1.18s/it][2025-04-22 13:28:35] (step=0029650) Train Loss: 6.3767, Train Steps/Sec: 0.85 + 73%|████████████████████████████████████████████████████████████████████████████████▌ | 29674/40903 [11:41:33<3:37:07, 1.16s/it][2025-04-22 13:29:04] (step=0029675) Train Loss: 6.3649, Train Steps/Sec: 0.85 + 73%|████████████████████████████████████████████████████████████████████████████████▌ | 29699/40903 [11:42:03<3:37:50, 1.17s/it][2025-04-22 13:29:34] (step=0029700) Train Loss: 6.3547, Train Steps/Sec: 0.84 + 73%|████████████████████████████████████████████████████████████████████████████████▋ | 29724/40903 [11:42:32<3:41:43, 1.19s/it][2025-04-22 13:30:03] (step=0029725) Train Loss: 6.3731, Train Steps/Sec: 0.85 + 73%|████████████████████████████████████████████████████████████████████████████████ | 29749/40903 [11:43:12<10:07:49, 3.27s/it][2025-04-22 13:30:43] (step=0029750) Train Loss: 6.4060, Train Steps/Sec: 0.64 + 73%|████████████████████████████████████████████████████████████████████████████████▊ | 29774/40903 [11:43:41<3:38:25, 1.18s/it][2025-04-22 13:31:12] (step=0029775) Train Loss: 6.4311, Train Steps/Sec: 0.85 + 73%|████████████████████████████████████████████████████████████████████████████████▊ | 29799/40903 [11:44:16<3:41:06, 1.19s/it][2025-04-22 13:31:48] (step=0029800) Train Loss: 6.4255, Train Steps/Sec: 0.71 + 73%|████████████████████████████████████████████████████████████████████████████████▉ | 29824/40903 [11:44:46<3:40:15, 1.19s/it][2025-04-22 13:32:17] (step=0029825) Train Loss: 6.4325, Train Steps/Sec: 0.85 + 73%|█████████████████████████████████████████████████████████████████████████████████ | 29849/40903 [11:45:15<3:37:22, 1.18s/it][2025-04-22 13:32:46] (step=0029850) Train Loss: 6.4237, Train Steps/Sec: 0.85 + 73%|█████████████████████████████████████████████████████████████████████████████████ | 29874/40903 [11:45:45<3:34:43, 1.17s/it][2025-04-22 13:33:16] (step=0029875) Train Loss: 6.3685, Train Steps/Sec: 0.85 + 73%|█████████████████████████████████████████████████████████████████████████████████▏ | 29899/40903 [11:46:14<3:36:33, 1.18s/it][2025-04-22 13:33:46] (step=0029900) Train Loss: 6.3918, Train Steps/Sec: 0.85 + 73%|█████████████████████████████████████████████████████████████████████████████████▏ | 29924/40903 [11:46:44<3:43:34, 1.22s/it][2025-04-22 13:34:15] (step=0029925) Train Loss: 6.3566, Train Steps/Sec: 0.84 + 73%|█████████████████████████████████████████████████████████████████████████████████▎ | 29949/40903 [11:47:14<3:38:18, 1.20s/it][2025-04-22 13:34:45] (step=0029950) Train Loss: 6.4130, Train Steps/Sec: 0.84 + 73%|█████████████████████████████████████████████████████████████████████████████████▎ | 29974/40903 [11:47:43<3:32:27, 1.17s/it][2025-04-22 13:35:15] (step=0029975) Train Loss: 6.4261, Train Steps/Sec: 0.85 + 73%|█████████████████████████████████████████████████████████████████████████████████▍ | 29999/40903 [11:48:13<3:32:20, 1.17s/it][2025-04-22 13:35:44] (step=0030000) Train Loss: 6.4229, Train Steps/Sec: 0.85 +You are using a model of type instructblip to instantiate a model of type blip-2. This is not supported for all configurations of models and can yield errors. +model_name_or_path:/tmp/haozhezhao/model/instructblip-flan-t5-xl +image_place_holder: +Some kwargs in processor config are unused and will not have any effect: num_query_tokens. +tokenizer length after expend 32102 +tokenizer length before expend 32102 +100%|██████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████| 3/3 [02:53<00:00, 57.87s/it] +[2025-04-22 13:40:25] Finish Eval in 30000 steps...███████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████| 3/3 [02:53<00:00, 57.51s/it] +processing left-padding... +processing left-padding... +processing left-padding... +[2025-04-22 13:40:46] Saved checkpoint to checkpoint/BeforeFix_Blip_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_objects/002-GPT-XL/checkpoints/0030000.pt +[2025-04-22 13:40:48] Removed old checkpoint: checkpoint/BeforeFix_Blip_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_objects/002-GPT-XL/checkpoints/0026000.pt + 73%|█████████████████████████████████████████████████████████████████████████████████▍ | 30024/40903 [11:53:46<3:40:53, 1.22s/it][2025-04-22 13:41:18] (step=0030025) Train Loss: 6.3751, Train Steps/Sec: 0.07 + 73%|█████████████████████████████████████████████████████████████████████████████████▌ | 30049/40903 [11:54:16<3:36:09, 1.19s/it][2025-04-22 13:41:47] (step=0030050) Train Loss: 6.3903, Train Steps/Sec: 0.84 + 74%|█████████████████████████████████████████████████████████████████████████████████▌ | 30074/40903 [11:54:46<3:30:56, 1.17s/it][2025-04-22 13:42:17] (step=0030075) Train Loss: 6.4075, Train Steps/Sec: 0.85 + 74%|█████████████████████████████████████████████████████████████████████████████████▋ | 30099/40903 [11:55:15<3:32:12, 1.18s/it][2025-04-22 13:42:47] (step=0030100) Train Loss: 6.3988, Train Steps/Sec: 0.84 + 74%|█████████████████████████████████████████████████████████████████████████████████▋ | 30124/40903 [11:55:45<3:37:15, 1.21s/it][2025-04-22 13:43:17] (step=0030125) Train Loss: 6.4054, Train Steps/Sec: 0.84 + 74%|█████████████████████████████████████████████████████████████████████████████████▊ | 30149/40903 [11:56:15<3:31:54, 1.18s/it][2025-04-22 13:43:46] (step=0030150) Train Loss: 6.4228, Train Steps/Sec: 0.85 + 74%|█████████████████████████████████████████████████████████████████████████████████▉ | 30174/40903 [11:56:45<3:37:49, 1.22s/it][2025-04-22 13:44:16] (step=0030175) Train Loss: 6.3685, Train Steps/Sec: 0.84 + 74%|█████████████████████████████████████████████████████████████████████████████████▉ | 30199/40903 [11:57:14<3:27:24, 1.16s/it][2025-04-22 13:44:45] (step=0030200) Train Loss: 6.3826, Train Steps/Sec: 0.84 + 74%|██████████████████████████████████████████████████████████████████████████████████ | 30224/40903 [11:57:44<3:34:07, 1.20s/it][2025-04-22 13:45:15] (step=0030225) Train Loss: 6.4093, Train Steps/Sec: 0.84 + 74%|██████████████████████████████████████████████████████████████████████████████████ | 30249/40903 [11:58:13<3:29:21, 1.18s/it][2025-04-22 13:45:45] (step=0030250) Train Loss: 6.3820, Train Steps/Sec: 0.85 + 74%|██████████████████████████████████████████████████████████████████████████████████▏ | 30274/40903 [11:58:43<3:30:02, 1.19s/it][2025-04-22 13:46:14] (step=0030275) Train Loss: 6.4184, Train Steps/Sec: 0.84 + 74%|██████████████████████████████████████████████████████████████████████████████████▏ | 30299/40903 [11:59:13<3:27:13, 1.17s/it][2025-04-22 13:46:44] (step=0030300) Train Loss: 6.4223, Train Steps/Sec: 0.84 + 74%|██████████████████████████████████████████████████████████████████████████████████▎ | 30324/40903 [11:59:43<3:30:46, 1.20s/it][2025-04-22 13:47:14] (step=0030325) Train Loss: 6.3975, Train Steps/Sec: 0.84 + 74%|██████████████████████████████████████████████████████████████████████████████████▎ | 30349/40903 [12:00:12<3:28:03, 1.18s/it][2025-04-22 13:47:43] (step=0030350) Train Loss: 6.4051, Train Steps/Sec: 0.85 + 74%|██████████████████████████████████████████████████████████████████████████████████▍ | 30374/40903 [12:00:42<3:24:14, 1.16s/it][2025-04-22 13:48:13] (step=0030375) Train Loss: 6.3583, Train Steps/Sec: 0.84 + 74%|██████████████████████████████████████████████████████████████████████████████████▍ | 30399/40903 [12:01:11<3:23:46, 1.16s/it][2025-04-22 13:48:43] (step=0030400) Train Loss: 6.3914, Train Steps/Sec: 0.84 + 74%|██████████████████████████████████████████████████████████████████████████████████▌ | 30424/40903 [12:01:41<3:28:12, 1.19s/it][2025-04-22 13:49:12] (step=0030425) Train Loss: 6.3940, Train Steps/Sec: 0.85 + 74%|██████████████████████████████████████████████████████████████████████████████████▋ | 30449/40903 [12:02:11<3:24:32, 1.17s/it][2025-04-22 13:49:42] (step=0030450) Train Loss: 6.3859, Train Steps/Sec: 0.84 + 75%|██████████████████████████████████████████████████████████████████████████████████▋ | 30474/40903 [12:02:40<3:23:37, 1.17s/it][2025-04-22 13:50:12] (step=0030475) Train Loss: 6.4088, Train Steps/Sec: 0.84 + 75%|██████████████████████████████████████████████████████████████████████████████████▊ | 30499/40903 [12:03:10<3:27:49, 1.20s/it][2025-04-22 13:50:42] (step=0030500) Train Loss: 6.3790, Train Steps/Sec: 0.84 + 75%|██████████████████████████████████████████████████████████████████████████████████▊ | 30524/40903 [12:03:40<3:29:45, 1.21s/it][2025-04-22 13:51:11] (step=0030525) Train Loss: 6.4163, Train Steps/Sec: 0.84 + 75%|██████████████████████████████████████████████████████████████████████████████████▉ | 30549/40903 [12:04:10<3:22:26, 1.17s/it][2025-04-22 13:51:41] (step=0030550) Train Loss: 6.3798, Train Steps/Sec: 0.84 + 75%|██████████████████████████████████████████████████████████████████████████████████▉ | 30574/40903 [12:04:39<3:24:22, 1.19s/it][2025-04-22 13:52:11] (step=0030575) Train Loss: 6.3725, Train Steps/Sec: 0.84 + 75%|███████████████████████████████████████████████████████████████████████████████████ | 30599/40903 [12:05:09<3:19:58, 1.16s/it][2025-04-22 13:52:40] (step=0030600) Train Loss: 6.3918, Train Steps/Sec: 0.84 + 75%|███████████████████████████████████████████████████████████████████████████████████ | 30624/40903 [12:05:39<3:25:33, 1.20s/it][2025-04-22 13:53:10] (step=0030625) Train Loss: 6.4050, Train Steps/Sec: 0.84 + 75%|███████████████████████████████████████████████████████████████████████████████████▏ | 30649/40903 [12:06:09<3:23:39, 1.19s/it][2025-04-22 13:53:40] (step=0030650) Train Loss: 6.3989, Train Steps/Sec: 0.84 + 75%|███████████████████████████████████████████████████████████████████████████████████▏ | 30674/40903 [12:06:38<3:20:44, 1.18s/it][2025-04-22 13:54:09] (step=0030675) Train Loss: 6.4142, Train Steps/Sec: 0.85 + 75%|███████████████████████████████████████████████████████████████████████████████████▎ | 30699/40903 [12:07:08<3:23:36, 1.20s/it][2025-04-22 13:54:39] (step=0030700) Train Loss: 6.4262, Train Steps/Sec: 0.84 + 75%|███████████████████████████████████████████████████████████████████████████████████▍ | 30724/40903 [12:07:37<3:24:33, 1.21s/it][2025-04-22 13:55:09] (step=0030725) Train Loss: 6.4070, Train Steps/Sec: 0.85 + 75%|███████████████████████████████████████████████████████████████████████████████████▍ | 30749/40903 [12:08:07<3:19:29, 1.18s/it][2025-04-22 13:55:38] (step=0030750) Train Loss: 6.3846, Train Steps/Sec: 0.84 + 75%|███████████████████████████████████████████████████████████████████████████████████▌ | 30774/40903 [12:08:37<3:17:44, 1.17s/it][2025-04-22 13:56:08] (step=0030775) Train Loss: 6.3911, Train Steps/Sec: 0.85 + 75%|███████████████████████████████████████████████████████████████████████████████████▌ | 30799/40903 [12:09:06<3:17:30, 1.17s/it][2025-04-22 13:56:38] (step=0030800) Train Loss: 6.3652, Train Steps/Sec: 0.84 + 75%|███████████████████████████████████████████████████████████████████████████████████▋ | 30824/40903 [12:09:36<3:25:14, 1.22s/it][2025-04-22 13:57:07] (step=0030825) Train Loss: 6.3735, Train Steps/Sec: 0.85 + 75%|███████████████████████████████████████████████████████████████████████████████████▋ | 30849/40903 [12:10:05<3:19:24, 1.19s/it][2025-04-22 13:57:37] (step=0030850) Train Loss: 6.4404, Train Steps/Sec: 0.84 + 75%|███████████████████████████████████████████████████████████████████████████████████▊ | 30874/40903 [12:10:35<3:15:25, 1.17s/it][2025-04-22 13:58:06] (step=0030875) Train Loss: 6.4038, Train Steps/Sec: 0.85 + 76%|███████████████████████████████████████████████████████████████████████████████████▊ | 30899/40903 [12:11:05<3:15:36, 1.17s/it][2025-04-22 13:58:36] (step=0030900) Train Loss: 6.3841, Train Steps/Sec: 0.84 + 76%|███████████████████████████████████████████████████████████████████████████████████▉ | 30924/40903 [12:11:35<3:24:12, 1.23s/it][2025-04-22 13:59:06] (step=0030925) Train Loss: 6.3982, Train Steps/Sec: 0.84 + 76%|███████████████████████████████████████████████████████████████████████████████████▉ | 30949/40903 [12:12:04<3:16:01, 1.18s/it][2025-04-22 13:59:36] (step=0030950) Train Loss: 6.3823, Train Steps/Sec: 0.85 + 76%|████████████████████████████████████████████████████████████████████████████████████ | 30974/40903 [12:12:41<7:06:13, 2.58s/it][2025-04-22 14:00:12] (step=0030975) Train Loss: 6.4065, Train Steps/Sec: 0.69 + 76%|████████████████████████████████████████████████████████████████████████████████████ | 30999/40903 [12:13:10<3:12:24, 1.17s/it][2025-04-22 14:00:42] (step=0031000) Train Loss: 6.3958, Train Steps/Sec: 0.84 + 76%|████████████████████████████████████████████████████████████████████████████████████▏ | 31024/40903 [12:13:40<3:19:45, 1.21s/it][2025-04-22 14:01:11] (step=0031025) Train Loss: 6.3820, Train Steps/Sec: 0.85 + 76%|████████████████████████████████████████████████████████████████████████████████████▎ | 31049/40903 [12:14:17<3:20:17, 1.22s/it][2025-04-22 14:01:48] (step=0031050) Train Loss: 6.4150, Train Steps/Sec: 0.68 + 76%|████████████████████████████████████████████████████████████████████████████████████▎ | 31074/40903 [12:14:46<3:14:04, 1.18s/it][2025-04-22 14:02:17] (step=0031075) Train Loss: 6.3857, Train Steps/Sec: 0.85 + 76%|████████████████████████████████████████████████████████████████████████████████████▍ | 31099/40903 [12:15:16<3:13:59, 1.19s/it][2025-04-22 14:02:48] (step=0031100) Train Loss: 6.4152, Train Steps/Sec: 0.83 + 76%|████████████████████████████████████████████████████████████████████████████████████▍ | 31124/40903 [12:15:46<3:14:26, 1.19s/it][2025-04-22 14:03:17] (step=0031125) Train Loss: 6.3958, Train Steps/Sec: 0.84 + 76%|████████████████████████████████████████████████████████████████████████████████████▌ | 31149/40903 [12:16:15<3:12:44, 1.19s/it][2025-04-22 14:03:47] (step=0031150) Train Loss: 6.4017, Train Steps/Sec: 0.85 + 76%|████████████████████████████████████████████████████████████████████████████████████▌ | 31174/40903 [12:16:45<3:11:38, 1.18s/it][2025-04-22 14:04:16] (step=0031175) Train Loss: 6.4036, Train Steps/Sec: 0.84 + 76%|████████████████████████████████████████████████████████████████████████████████████▋ | 31199/40903 [12:17:15<3:09:03, 1.17s/it][2025-04-22 14:04:46] (step=0031200) Train Loss: 6.3896, Train Steps/Sec: 0.84 + 76%|████████████████████████████████████████████████████████████████████████████████████▋ | 31224/40903 [12:17:44<3:10:41, 1.18s/it][2025-04-22 14:05:15] (step=0031225) Train Loss: 6.3908, Train Steps/Sec: 0.85 + 76%|████████████████████████████████████████████████████████████████████████████████████▊ | 31249/40903 [12:18:14<3:09:03, 1.17s/it][2025-04-22 14:05:45] (step=0031250) Train Loss: 6.3930, Train Steps/Sec: 0.84 + 76%|████████████████████████████████████████████████████████████████████████████████████▊ | 31274/40903 [12:18:43<3:08:36, 1.18s/it][2025-04-22 14:06:15] (step=0031275) Train Loss: 6.4126, Train Steps/Sec: 0.84 + 77%|████████████████████████████████████████████████████████████████████████████████████▉ | 31299/40903 [12:19:13<3:09:10, 1.18s/it][2025-04-22 14:06:45] (step=0031300) Train Loss: 6.4411, Train Steps/Sec: 0.84 + 77%|█████████████████████████████████████████████████████████████████████████████████████ | 31324/40903 [12:19:43<3:10:24, 1.19s/it][2025-04-22 14:07:14] (step=0031325) Train Loss: 6.4084, Train Steps/Sec: 0.85 + 77%|█████████████████████████████████████████████████████████████████████████████████████ | 31349/40903 [12:20:13<3:08:06, 1.18s/it][2025-04-22 14:07:44] (step=0031350) Train Loss: 6.4094, Train Steps/Sec: 0.84 + 77%|█████████████████████████████████████████████████████████████████████████████████████▏ | 31374/40903 [12:20:42<3:07:07, 1.18s/it][2025-04-22 14:08:13] (step=0031375) Train Loss: 6.3818, Train Steps/Sec: 0.85 + 77%|█████████████████████████████████████████████████████████████████████████████████████▏ | 31399/40903 [12:21:12<3:10:44, 1.20s/it][2025-04-22 14:08:43] (step=0031400) Train Loss: 6.3985, Train Steps/Sec: 0.83 + 77%|█████████████████████████████████████████████████████████████████████████████████████▎ | 31424/40903 [12:21:41<3:08:17, 1.19s/it][2025-04-22 14:09:13] (step=0031425) Train Loss: 6.4125, Train Steps/Sec: 0.85 + 77%|█████████████████████████████████████████████████████████████████████████████████████▎ | 31449/40903 [12:22:11<3:04:27, 1.17s/it][2025-04-22 14:09:42] (step=0031450) Train Loss: 6.3783, Train Steps/Sec: 0.84 + 77%|█████████████████████████████████████████████████████████████████████████████████████▍ | 31474/40903 [12:22:41<3:07:07, 1.19s/it][2025-04-22 14:10:12] (step=0031475) Train Loss: 6.3778, Train Steps/Sec: 0.84 + 77%|█████████████████████████████████████████████████████████████████████████████████████▍ | 31499/40903 [12:23:10<3:00:34, 1.15s/it][2025-04-22 14:10:42] (step=0031500) Train Loss: 6.4124, Train Steps/Sec: 0.85 + 77%|█████████████████████████████████████████████████████████████████████████████████████▌ | 31524/40903 [12:23:40<3:09:28, 1.21s/it][2025-04-22 14:11:12] (step=0031525) Train Loss: 6.3700, Train Steps/Sec: 0.84 + 77%|█████████████████████████████████████████████████████████████████████████████████████▌ | 31549/40903 [12:24:10<3:06:39, 1.20s/it][2025-04-22 14:11:41] (step=0031550) Train Loss: 6.3801, Train Steps/Sec: 0.84 + 77%|█████████████████████████████████████████████████████████████████████████████████████▋ | 31574/40903 [12:24:40<3:01:18, 1.17s/it][2025-04-22 14:12:11] (step=0031575) Train Loss: 6.3888, Train Steps/Sec: 0.85 + 77%|█████████████████████████████████████████████████████████████████████████████████████▊ | 31599/40903 [12:25:09<3:01:04, 1.17s/it][2025-04-22 14:12:40] (step=0031600) Train Loss: 6.3697, Train Steps/Sec: 0.84 + 77%|█████████████████████████████████████████████████████████████████████████████████████▊ | 31624/40903 [12:25:39<3:06:02, 1.20s/it][2025-04-22 14:13:10] (step=0031625) Train Loss: 6.3769, Train Steps/Sec: 0.84 + 77%|█████████████████████████████████████████████████████████████████████████████████████▉ | 31649/40903 [12:26:09<3:03:53, 1.19s/it][2025-04-22 14:13:40] (step=0031650) Train Loss: 6.4454, Train Steps/Sec: 0.84 + 77%|█████████████████████████████████████████████████████████████████████████████████████▉ | 31674/40903 [12:26:38<3:03:36, 1.19s/it][2025-04-22 14:14:10] (step=0031675) Train Loss: 6.4133, Train Steps/Sec: 0.84 + 77%|██████████████████████████████████████████████████████████████████████████████████████ | 31699/40903 [12:27:08<2:57:51, 1.16s/it][2025-04-22 14:14:39] (step=0031700) Train Loss: 6.4214, Train Steps/Sec: 0.84 + 78%|██████████████████████████████████████████████████████████████████████████████████████ | 31724/40903 [12:27:38<3:04:35, 1.21s/it][2025-04-22 14:15:09] (step=0031725) Train Loss: 6.4169, Train Steps/Sec: 0.84 + 78%|██████████████████████████████████████████████████████████████████████████████████████▏ | 31749/40903 [12:28:08<3:06:30, 1.22s/it][2025-04-22 14:15:39] (step=0031750) Train Loss: 6.3698, Train Steps/Sec: 0.83 + 78%|██████████████████████████████████████████████████████████████████████████████████████▏ | 31774/40903 [12:28:37<2:58:03, 1.17s/it][2025-04-22 14:16:08] (step=0031775) Train Loss: 6.4091, Train Steps/Sec: 0.85 + 78%|██████████████████████████████████████████████████████████████████████████████████████▎ | 31799/40903 [12:29:07<2:57:08, 1.17s/it][2025-04-22 14:16:38] (step=0031800) Train Loss: 6.3889, Train Steps/Sec: 0.84 + 78%|██████████████████████████████████████████████████████████████████████████████████████▎ | 31824/40903 [12:29:37<3:04:26, 1.22s/it][2025-04-22 14:17:08] (step=0031825) Train Loss: 6.3842, Train Steps/Sec: 0.84 + 78%|██████████████████████████████████████████████████████████████████████████████████████▍ | 31849/40903 [12:30:06<3:01:28, 1.20s/it][2025-04-22 14:17:38] (step=0031850) Train Loss: 6.3588, Train Steps/Sec: 0.84 + 78%|██████████████████████████████████████████████████████████████████████████████████████▍ | 31874/40903 [12:30:36<2:56:44, 1.17s/it][2025-04-22 14:18:07] (step=0031875) Train Loss: 6.3965, Train Steps/Sec: 0.85 + 78%|██████████████████████████████████████████████████████████████████████████████████████▌ | 31899/40903 [12:31:05<2:55:57, 1.17s/it][2025-04-22 14:18:37] (step=0031900) Train Loss: 6.4055, Train Steps/Sec: 0.84 + 78%|██████████████████████████████████████████████████████████████████████████████████████▋ | 31924/40903 [12:31:35<2:58:57, 1.20s/it][2025-04-22 14:19:06] (step=0031925) Train Loss: 6.4002, Train Steps/Sec: 0.85 + 78%|██████████████████████████████████████████████████████████████████████████████████████▋ | 31949/40903 [12:32:05<2:57:35, 1.19s/it][2025-04-22 14:19:36] (step=0031950) Train Loss: 6.4114, Train Steps/Sec: 0.84 + 78%|██████████████████████████████████████████████████████████████████████████████████████▊ | 31974/40903 [12:32:34<2:52:35, 1.16s/it][2025-04-22 14:20:05] (step=0031975) Train Loss: 6.4141, Train Steps/Sec: 0.85 + 78%|██████████████████████████████████████████████████████████████████████████████████████▊ | 31999/40903 [12:33:04<2:58:10, 1.20s/it][2025-04-22 14:20:35] (step=0032000) Train Loss: 6.3538, Train Steps/Sec: 0.84 +You are using a model of type instructblip to instantiate a model of type blip-2. This is not supported for all configurations of models and can yield errors. +model_name_or_path:/tmp/haozhezhao/model/instructblip-flan-t5-xl +image_place_holder: +Some kwargs in processor config are unused and will not have any effect: num_query_tokens. +tokenizer length after expend 32102 +tokenizer length before expend 32102 +100%|██████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████| 3/3 [02:54<00:00, 58.16s/it] +[2025-04-22 14:25:17] Finish Eval in 32000 steps...███████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████| 3/3 [02:54<00:00, 57.82s/it] +processing left-padding... +processing left-padding... +processing left-padding... +[2025-04-22 14:25:38] Saved checkpoint to checkpoint/BeforeFix_Blip_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_objects/002-GPT-XL/checkpoints/0032000.pt +[2025-04-22 14:25:40] Removed old checkpoint: checkpoint/BeforeFix_Blip_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_objects/002-GPT-XL/checkpoints/0028000.pt + 78%|██████████████████████████████████████████████████████████████████████████████████████▉ | 32024/40903 [12:38:39<3:00:58, 1.22s/it][2025-04-22 14:26:10] (step=0032025) Train Loss: 6.4145, Train Steps/Sec: 0.07 + 78%|██████████████████████████████████████████████████████████████████████████████████████▉ | 32049/40903 [12:39:09<2:54:07, 1.18s/it][2025-04-22 14:26:40] (step=0032050) Train Loss: 6.4145, Train Steps/Sec: 0.84 + 78%|███████████████████████████████████████████████████████████████████████████████████████ | 32074/40903 [12:39:38<2:52:13, 1.17s/it][2025-04-22 14:27:10] (step=0032075) Train Loss: 6.3676, Train Steps/Sec: 0.84 + 78%|███████████████████████████████████████████████████████████████████████████████████████ | 32099/40903 [12:40:08<2:51:59, 1.17s/it][2025-04-22 14:27:39] (step=0032100) Train Loss: 6.3747, Train Steps/Sec: 0.84 + 79%|███████████████████████████████████████████████████████████████████████████████████████▏ | 32124/40903 [12:40:38<2:53:37, 1.19s/it][2025-04-22 14:28:09] (step=0032125) Train Loss: 6.3598, Train Steps/Sec: 0.85 + 79%|███████████████████████████████████████████████████████████████████████████████████████▏ | 32149/40903 [12:41:07<2:56:08, 1.21s/it][2025-04-22 14:28:39] (step=0032150) Train Loss: 6.3942, Train Steps/Sec: 0.84 + 79%|███████████████████████████████████████████████████████████████████████████████████████▎ | 32174/40903 [12:41:37<2:50:31, 1.17s/it][2025-04-22 14:29:08] (step=0032175) Train Loss: 6.3990, Train Steps/Sec: 0.84 + 79%|███████████████████████████████████████████████████████████████████████████████████████▍ | 32199/40903 [12:42:07<2:49:59, 1.17s/it][2025-04-22 14:29:38] (step=0032200) Train Loss: 6.3713, Train Steps/Sec: 0.84 + 79%|███████████████████████████████████████████████████████████████████████████████████████▍ | 32224/40903 [12:42:37<2:54:20, 1.21s/it][2025-04-22 14:30:14] (step=0032225) Train Loss: 6.3659, Train Steps/Sec: 0.70 + 79%|███████████████████████████████████████████████████████████████████████████████████████▌ | 32249/40903 [12:43:12<2:51:17, 1.19s/it][2025-04-22 14:30:44] (step=0032250) Train Loss: 6.3866, Train Steps/Sec: 0.85 + 79%|███████████████████████████████████████████████████████████████████████████████████████▌ | 32274/40903 [12:43:42<2:50:06, 1.18s/it][2025-04-22 14:31:13] (step=0032275) Train Loss: 6.3922, Train Steps/Sec: 0.85 + 79%|███████████████████████████████████████████████████████████████████████████████████████▋ | 32299/40903 [12:44:12<2:49:24, 1.18s/it][2025-04-22 14:31:43] (step=0032300) Train Loss: 6.4095, Train Steps/Sec: 0.84 + 79%|███████████████████████████████████████████████████████████████████████████████████████▋ | 32324/40903 [12:44:41<2:51:36, 1.20s/it][2025-04-22 14:32:12] (step=0032325) Train Loss: 6.3996, Train Steps/Sec: 0.85 + 79%|███████████████████████████████████████████████████████████████████████████████████████▊ | 32349/40903 [12:45:11<2:48:01, 1.18s/it][2025-04-22 14:32:42] (step=0032350) Train Loss: 6.4176, Train Steps/Sec: 0.85 + 79%|███████████████████████████████████████████████████████████████████████████████████████▊ | 32374/40903 [12:45:41<2:48:29, 1.19s/it][2025-04-22 14:33:12] (step=0032375) Train Loss: 6.3950, Train Steps/Sec: 0.84 + 79%|███████████████████████████████████████████████████████████████████████████████████████▉ | 32399/40903 [12:46:10<2:49:05, 1.19s/it][2025-04-22 14:33:42] (step=0032400) Train Loss: 6.3684, Train Steps/Sec: 0.84 + 79%|███████████████████████████████████████████████████████████████████████████████████████▉ | 32424/40903 [12:46:40<2:49:10, 1.20s/it][2025-04-22 14:34:11] (step=0032425) Train Loss: 6.4232, Train Steps/Sec: 0.85 + 79%|████████████████████████████████████████████████████████████████████████████████████████ | 32449/40903 [12:47:10<2:46:29, 1.18s/it][2025-04-22 14:34:41] (step=0032450) Train Loss: 6.4014, Train Steps/Sec: 0.84 + 79%|████████████████████████████████████████████████████████████████████████████████████████▏ | 32474/40903 [12:47:39<2:45:33, 1.18s/it][2025-04-22 14:35:11] (step=0032475) Train Loss: 6.4381, Train Steps/Sec: 0.84 + 79%|████████████████████████████████████████████████████████████████████████████████████████▏ | 32499/40903 [12:48:09<2:45:44, 1.18s/it][2025-04-22 14:35:40] (step=0032500) Train Loss: 6.4431, Train Steps/Sec: 0.85 + 80%|████████████████████████████████████████████████████████████████████████████████████████▎ | 32524/40903 [12:48:38<2:48:19, 1.21s/it][2025-04-22 14:36:10] (step=0032525) Train Loss: 6.3402, Train Steps/Sec: 0.85 + 80%|████████████████████████████████████████████████████████████████████████████████████████▎ | 32549/40903 [12:49:08<2:44:27, 1.18s/it][2025-04-22 14:36:39] (step=0032550) Train Loss: 6.4396, Train Steps/Sec: 0.84 + 80%|████████████████████████████████████████████████████████████████████████████████████████▍ | 32574/40903 [12:49:38<2:43:14, 1.18s/it][2025-04-22 14:37:09] (step=0032575) Train Loss: 6.4032, Train Steps/Sec: 0.85 + 80%|████████████████████████████████████████████████████████████████████████████████████████▍ | 32599/40903 [12:50:07<2:42:50, 1.18s/it][2025-04-22 14:37:39] (step=0032600) Train Loss: 6.3539, Train Steps/Sec: 0.84 + 80%|████████████████████████████████████████████████████████████████████████████████████████▌ | 32624/40903 [12:50:37<2:46:24, 1.21s/it][2025-04-22 14:38:08] (step=0032625) Train Loss: 6.3748, Train Steps/Sec: 0.85 + 80%|████████████████████████████████████████████████████████████████████████████████████████▌ | 32649/40903 [12:51:07<2:44:39, 1.20s/it][2025-04-22 14:38:38] (step=0032650) Train Loss: 6.3854, Train Steps/Sec: 0.84 + 80%|████████████████████████████████████████████████████████████████████████████████████████▋ | 32674/40903 [12:51:36<2:38:36, 1.16s/it][2025-04-22 14:39:08] (step=0032675) Train Loss: 6.4116, Train Steps/Sec: 0.84 + 80%|████████████████████████████████████████████████████████████████████████████████████████▋ | 32699/40903 [12:52:06<2:42:29, 1.19s/it][2025-04-22 14:39:37] (step=0032700) Train Loss: 6.4144, Train Steps/Sec: 0.84 + 80%|████████████████████████████████████████████████████████████████████████████████████████▊ | 32724/40903 [12:52:36<2:43:00, 1.20s/it][2025-04-22 14:40:07] (step=0032725) Train Loss: 6.3929, Train Steps/Sec: 0.84 + 80%|████████████████████████████████████████████████████████████████████████████████████████▊ | 32749/40903 [12:53:06<2:40:43, 1.18s/it][2025-04-22 14:40:37] (step=0032750) Train Loss: 6.4378, Train Steps/Sec: 0.84 + 80%|████████████████████████████████████████████████████████████████████████████████████████▉ | 32774/40903 [12:53:35<2:39:54, 1.18s/it][2025-04-22 14:41:06] (step=0032775) Train Loss: 6.3950, Train Steps/Sec: 0.84 + 80%|█████████████████████████████████████████████████████████████████████████████████████████ | 32799/40903 [12:54:05<2:36:41, 1.16s/it][2025-04-22 14:41:36] (step=0032800) Train Loss: 6.4499, Train Steps/Sec: 0.84 + 80%|█████████████████████████████████████████████████████████████████████████████████████████ | 32824/40903 [12:54:34<2:41:44, 1.20s/it][2025-04-22 14:42:06] (step=0032825) Train Loss: 6.3963, Train Steps/Sec: 0.85 + 80%|█████████████████████████████████████████████████████████████████████████████████████████▏ | 32849/40903 [12:55:04<2:42:04, 1.21s/it][2025-04-22 14:42:35] (step=0032850) Train Loss: 6.3673, Train Steps/Sec: 0.84 + 80%|█████████████████████████████████████████████████████████████████████████████████████████▏ | 32874/40903 [12:55:34<2:36:50, 1.17s/it][2025-04-22 14:43:05] (step=0032875) Train Loss: 6.4241, Train Steps/Sec: 0.85 + 80%|█████████████████████████████████████████████████████████████████████████████████████████▎ | 32899/40903 [12:56:03<2:35:27, 1.17s/it][2025-04-22 14:43:35] (step=0032900) Train Loss: 6.4000, Train Steps/Sec: 0.84 + 80%|█████████████████████████████████████████████████████████████████████████████████████████▎ | 32924/40903 [12:56:33<2:39:50, 1.20s/it][2025-04-22 14:44:04] (step=0032925) Train Loss: 6.3881, Train Steps/Sec: 0.84 + 81%|█████████████████████████████████████████████████████████████████████████████████████████▍ | 32949/40903 [12:57:03<2:38:49, 1.20s/it][2025-04-22 14:44:34] (step=0032950) Train Loss: 6.3986, Train Steps/Sec: 0.85 + 81%|█████████████████████████████████████████████████████████████████████████████████████████▍ | 32974/40903 [12:57:32<2:33:41, 1.16s/it][2025-04-22 14:45:04] (step=0032975) Train Loss: 6.4081, Train Steps/Sec: 0.84 + 81%|█████████████████████████████████████████████████████████████████████████████████████████▌ | 32999/40903 [12:58:02<2:37:09, 1.19s/it][2025-04-22 14:45:33] (step=0033000) Train Loss: 6.3944, Train Steps/Sec: 0.84 + 81%|█████████████████████████████████████████████████████████████████████████████████████████▌ | 33024/40903 [12:58:32<2:37:27, 1.20s/it][2025-04-22 14:46:03] (step=0033025) Train Loss: 6.3976, Train Steps/Sec: 0.84 + 81%|█████████████████████████████████████████████████████████████████████████████████████████▋ | 33049/40903 [12:59:02<2:34:05, 1.18s/it][2025-04-22 14:46:33] (step=0033050) Train Loss: 6.4161, Train Steps/Sec: 0.84 + 81%|█████████████████████████████████████████████████████████████████████████████████████████▊ | 33074/40903 [12:59:31<2:33:37, 1.18s/it][2025-04-22 14:47:03] (step=0033075) Train Loss: 6.3876, Train Steps/Sec: 0.84 + 81%|█████████████████████████████████████████████████████████████████████████████████████████▊ | 33099/40903 [13:00:01<2:33:03, 1.18s/it][2025-04-22 14:47:32] (step=0033100) Train Loss: 6.3639, Train Steps/Sec: 0.84 + 81%|█████████████████████████████████████████████████████████████████████████████████████████▉ | 33124/40903 [13:00:31<2:36:40, 1.21s/it][2025-04-22 14:48:02] (step=0033125) Train Loss: 6.3470, Train Steps/Sec: 0.85 + 81%|█████████████████████████████████████████████████████████████████████████████████████████▉ | 33149/40903 [13:01:00<2:31:53, 1.18s/it][2025-04-22 14:48:31] (step=0033150) Train Loss: 6.3703, Train Steps/Sec: 0.85 + 81%|██████████████████████████████████████████████████████████████████████████████████████████ | 33174/40903 [13:01:30<2:32:17, 1.18s/it][2025-04-22 14:49:01] (step=0033175) Train Loss: 6.3811, Train Steps/Sec: 0.84 + 81%|██████████████████████████████████████████████████████████████████████████████████████████ | 33199/40903 [13:01:59<2:29:56, 1.17s/it][2025-04-22 14:49:31] (step=0033200) Train Loss: 6.3931, Train Steps/Sec: 0.84 + 81%|██████████████████████████████████████████████████████████████████████████████████████████▏ | 33224/40903 [13:02:29<2:34:52, 1.21s/it][2025-04-22 14:50:00] (step=0033225) Train Loss: 6.4002, Train Steps/Sec: 0.85 + 81%|██████████████████████████████████████████████████████████████████████████████████████████▏ | 33249/40903 [13:02:59<2:31:11, 1.19s/it][2025-04-22 14:50:30] (step=0033250) Train Loss: 6.4262, Train Steps/Sec: 0.85 + 81%|██████████████████████████████████████████████████████████████████████████████████████████▎ | 33274/40903 [13:03:28<2:29:56, 1.18s/it][2025-04-22 14:51:00] (step=0033275) Train Loss: 6.4029, Train Steps/Sec: 0.84 + 81%|██████████████████████████████████████████████████████████████████████████████████████████▎ | 33299/40903 [13:03:58<2:31:24, 1.19s/it][2025-04-22 14:51:29] (step=0033300) Train Loss: 6.3929, Train Steps/Sec: 0.84 + 81%|██████████████████████████████████████████████████████████████████████████████████████████▍ | 33324/40903 [13:04:27<2:31:31, 1.20s/it][2025-04-22 14:51:59] (step=0033325) Train Loss: 6.3903, Train Steps/Sec: 0.85 + 82%|██████████████████████████████████████████████████████████████████████████████████████████▌ | 33349/40903 [13:04:57<2:30:14, 1.19s/it][2025-04-22 14:52:28] (step=0033350) Train Loss: 6.4253, Train Steps/Sec: 0.85 + 82%|██████████████████████████████████████████████████████████████████████████████████████████▌ | 33374/40903 [13:05:27<2:29:04, 1.19s/it][2025-04-22 14:52:58] (step=0033375) Train Loss: 6.4408, Train Steps/Sec: 0.84 + 82%|██████████████████████████████████████████████████████████████████████████████████████████▋ | 33399/40903 [13:05:56<2:27:13, 1.18s/it][2025-04-22 14:53:28] (step=0033400) Train Loss: 6.3884, Train Steps/Sec: 0.84 + 82%|██████████████████████████████████████████████████████████████████████████████████████████▋ | 33424/40903 [13:06:26<2:29:15, 1.20s/it][2025-04-22 14:53:57] (step=0033425) Train Loss: 6.4281, Train Steps/Sec: 0.84 + 82%|██████████████████████████████████████████████████████████████████████████████████████████▊ | 33449/40903 [13:06:56<2:29:45, 1.21s/it][2025-04-22 14:54:27] (step=0033450) Train Loss: 6.4011, Train Steps/Sec: 0.84 + 82%|██████████████████████████████████████████████████████████████████████████████████████████▊ | 33474/40903 [13:07:25<2:23:19, 1.16s/it][2025-04-22 14:54:57] (step=0033475) Train Loss: 6.3645, Train Steps/Sec: 0.85 + 82%|██████████████████████████████████████████████████████████████████████████████████████████▉ | 33499/40903 [13:07:55<2:23:21, 1.16s/it][2025-04-22 14:55:26] (step=0033500) Train Loss: 6.4060, Train Steps/Sec: 0.84 + 82%|██████████████████████████████████████████████████████████████████████████████████████████▉ | 33524/40903 [13:08:25<2:28:08, 1.20s/it][2025-04-22 14:55:56] (step=0033525) Train Loss: 6.4187, Train Steps/Sec: 0.84 + 82%|███████████████████████████████████████████████████████████████████████████████████████████ | 33549/40903 [13:08:54<2:24:28, 1.18s/it][2025-04-22 14:56:25] (step=0033550) Train Loss: 6.3640, Train Steps/Sec: 0.85 + 82%|███████████████████████████████████████████████████████████████████████████████████████████ | 33574/40903 [13:09:24<2:24:10, 1.18s/it][2025-04-22 14:56:55] (step=0033575) Train Loss: 6.3624, Train Steps/Sec: 0.84 + 82%|███████████████████████████████████████████████████████████████████████████████████████████▏ | 33599/40903 [13:09:53<2:22:10, 1.17s/it][2025-04-22 14:57:25] (step=0033600) Train Loss: 6.3820, Train Steps/Sec: 0.84 + 82%|███████████████████████████████████████████████████████████████████████████████████████████▏ | 33624/40903 [13:10:23<2:27:26, 1.22s/it][2025-04-22 14:57:54] (step=0033625) Train Loss: 6.4253, Train Steps/Sec: 0.85 + 82%|███████████████████████████████████████████████████████████████████████████████████████████▎ | 33649/40903 [13:10:52<2:23:55, 1.19s/it][2025-04-22 14:58:24] (step=0033650) Train Loss: 6.3957, Train Steps/Sec: 0.85 + 82%|███████████████████████████████████████████████████████████████████████████████████████████▍ | 33674/40903 [13:11:22<2:20:38, 1.17s/it][2025-04-22 14:58:53] (step=0033675) Train Loss: 6.4018, Train Steps/Sec: 0.85 + 82%|███████████████████████████████████████████████████████████████████████████████████████████▍ | 33699/40903 [13:11:51<2:21:24, 1.18s/it][2025-04-22 14:59:23] (step=0033700) Train Loss: 6.4039, Train Steps/Sec: 0.84 + 82%|███████████████████████████████████████████████████████████████████████████████████████████▌ | 33724/40903 [13:12:21<2:22:58, 1.19s/it][2025-04-22 14:59:52] (step=0033725) Train Loss: 6.3845, Train Steps/Sec: 0.85 + 83%|███████████████████████████████████████████████████████████████████████████████████████████▌ | 33749/40903 [13:12:51<2:23:42, 1.21s/it][2025-04-22 15:00:22] (step=0033750) Train Loss: 6.4232, Train Steps/Sec: 0.84 + 83%|███████████████████████████████████████████████████████████████████████████████████████████▋ | 33774/40903 [13:13:20<2:21:48, 1.19s/it][2025-04-22 15:00:52] (step=0033775) Train Loss: 6.4244, Train Steps/Sec: 0.84 + 83%|███████████████████████████████████████████████████████████████████████████████████████████▋ | 33799/40903 [13:13:50<2:20:12, 1.18s/it][2025-04-22 15:01:22] (step=0033800) Train Loss: 6.4078, Train Steps/Sec: 0.84 + 83%|███████████████████████████████████████████████████████████████████████████████████████████▊ | 33824/40903 [13:14:20<2:20:21, 1.19s/it][2025-04-22 15:01:51] (step=0033825) Train Loss: 6.4455, Train Steps/Sec: 0.85 + 83%|███████████████████████████████████████████████████████████████████████████████████████████▊ | 33849/40903 [13:14:49<2:19:43, 1.19s/it][2025-04-22 15:02:21] (step=0033850) Train Loss: 6.3647, Train Steps/Sec: 0.84 + 83%|███████████████████████████████████████████████████████████████████████████████████████████▉ | 33874/40903 [13:15:19<2:18:10, 1.18s/it][2025-04-22 15:02:50] (step=0033875) Train Loss: 6.4026, Train Steps/Sec: 0.84 + 83%|███████████████████████████████████████████████████████████████████████████████████████████▉ | 33899/40903 [13:15:49<2:15:55, 1.16s/it][2025-04-22 15:03:20] (step=0033900) Train Loss: 6.3868, Train Steps/Sec: 0.85 + 83%|████████████████████████████████████████████████████████████████████████████████████████████ | 33924/40903 [13:16:18<2:19:26, 1.20s/it][2025-04-22 15:03:50] (step=0033925) Train Loss: 6.4215, Train Steps/Sec: 0.85 + 83%|████████████████████████████████████████████████████████████████████████████████████████████▏ | 33949/40903 [13:16:48<2:16:17, 1.18s/it][2025-04-22 15:04:19] (step=0033950) Train Loss: 6.4025, Train Steps/Sec: 0.84 + 83%|████████████████████████████████████████████████████████████████████████████████████████████▏ | 33974/40903 [13:17:18<2:17:45, 1.19s/it][2025-04-22 15:04:49] (step=0033975) Train Loss: 6.3735, Train Steps/Sec: 0.85 + 83%|████████████████████████████████████████████████████████████████████████████████████████████▎ | 33999/40903 [13:17:47<2:13:56, 1.16s/it][2025-04-22 15:05:18] (step=0034000) Train Loss: 6.4061, Train Steps/Sec: 0.84 +You are using a model of type instructblip to instantiate a model of type blip-2. This is not supported for all configurations of models and can yield errors. +model_name_or_path:/tmp/haozhezhao/model/instructblip-flan-t5-xl +image_place_holder: +Some kwargs in processor config are unused and will not have any effect: num_query_tokens. +tokenizer length after expend 32102 +tokenizer length before expend 32102 +100%|██████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████| 3/3 [02:54<00:00, 58.29s/it] +[2025-04-22 15:10:01] Finish Eval in 34000 steps...███████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████| 3/3 [02:54<00:00, 57.82s/it] +processing left-padding... +processing left-padding... +processing left-padding... +[2025-04-22 15:10:22] Saved checkpoint to checkpoint/BeforeFix_Blip_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_objects/002-GPT-XL/checkpoints/0034000.pt +[2025-04-22 15:10:24] Removed old checkpoint: checkpoint/BeforeFix_Blip_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_objects/002-GPT-XL/checkpoints/0030000.pt + 83%|████████████████████████████████████████████████████████████████████████████████████████████▎ | 34024/40903 [13:23:22<2:19:36, 1.22s/it][2025-04-22 15:10:54] (step=0034025) Train Loss: 6.4240, Train Steps/Sec: 0.07 + 83%|████████████████████████████████████████████████████████████████████████████████████████████▍ | 34049/40903 [13:23:52<2:14:21, 1.18s/it][2025-04-22 15:11:23] (step=0034050) Train Loss: 6.4115, Train Steps/Sec: 0.85 + 83%|████████████████████████████████████████████████████████████████████████████████████████████▍ | 34074/40903 [13:24:21<2:14:15, 1.18s/it][2025-04-22 15:11:53] (step=0034075) Train Loss: 6.3818, Train Steps/Sec: 0.85 + 83%|████████████████████████████████████████████████████████████████████████████████████████████▌ | 34099/40903 [13:24:51<2:11:52, 1.16s/it][2025-04-22 15:12:22] (step=0034100) Train Loss: 6.4001, Train Steps/Sec: 0.85 + 83%|████████████████████████████████████████████████████████████████████████████████████████████▌ | 34124/40903 [13:25:20<2:16:11, 1.21s/it][2025-04-22 15:12:52] (step=0034125) Train Loss: 6.3942, Train Steps/Sec: 0.84 + 83%|████████████████████████████████████████████████████████████████████████████████████████████▋ | 34149/40903 [13:25:50<2:12:34, 1.18s/it][2025-04-22 15:13:21] (step=0034150) Train Loss: 6.4018, Train Steps/Sec: 0.84 + 84%|████████████████████████████████████████████████████████████████████████████████████████████▋ | 34174/40903 [13:26:20<2:12:46, 1.18s/it][2025-04-22 15:13:51] (step=0034175) Train Loss: 6.3549, Train Steps/Sec: 0.85 + 84%|████████████████████████████████████████████████████████████████████████████████████████████▊ | 34199/40903 [13:26:49<2:09:30, 1.16s/it][2025-04-22 15:14:20] (step=0034200) Train Loss: 6.3248, Train Steps/Sec: 0.85 + 84%|████████████████████████████████████████████████████████████████████████████████████████████▊ | 34224/40903 [13:27:19<2:13:13, 1.20s/it][2025-04-22 15:14:50] (step=0034225) Train Loss: 6.4017, Train Steps/Sec: 0.85 + 84%|████████████████████████████████████████████████████████████████████████████████████████████▉ | 34249/40903 [13:27:48<2:12:13, 1.19s/it][2025-04-22 15:15:20] (step=0034250) Train Loss: 6.4316, Train Steps/Sec: 0.84 + 84%|█████████████████████████████████████████████████████████████████████████████████████████████ | 34274/40903 [13:28:18<2:08:48, 1.17s/it][2025-04-22 15:15:49] (step=0034275) Train Loss: 6.3115, Train Steps/Sec: 0.85 + 84%|█████████████████████████████████████████████████████████████████████████████████████████████ | 34299/40903 [13:28:47<2:08:40, 1.17s/it][2025-04-22 15:16:19] (step=0034300) Train Loss: 6.3861, Train Steps/Sec: 0.84 + 84%|█████████████████████████████████████████████████████████████████████████████████████████████▏ | 34324/40903 [13:29:17<2:10:40, 1.19s/it][2025-04-22 15:16:48] (step=0034325) Train Loss: 6.3744, Train Steps/Sec: 0.84 + 84%|█████████████████████████████████████████████████████████████████████████████████████████████▏ | 34349/40903 [13:29:47<2:08:56, 1.18s/it][2025-04-22 15:17:18] (step=0034350) Train Loss: 6.3944, Train Steps/Sec: 0.85 + 84%|█████████████████████████████████████████████████████████████████████████████████████████████▎ | 34374/40903 [13:30:16<2:07:54, 1.18s/it][2025-04-22 15:17:47] (step=0034375) Train Loss: 6.4000, Train Steps/Sec: 0.84 + 84%|█████████████████████████████████████████████████████████████████████████████████████████████▎ | 34399/40903 [13:30:46<2:05:56, 1.16s/it][2025-04-22 15:18:25] (step=0034400) Train Loss: 6.4114, Train Steps/Sec: 0.67 + 84%|█████████████████████████████████████████████████████████████████████████████████████████████▍ | 34424/40903 [13:31:23<2:13:02, 1.23s/it][2025-04-22 15:18:54] (step=0034425) Train Loss: 6.3903, Train Steps/Sec: 0.84 + 84%|█████████████████████████████████████████████████████████████████████████████████████████████▍ | 34449/40903 [13:31:59<2:08:59, 1.20s/it][2025-04-22 15:19:30] (step=0034450) Train Loss: 6.4425, Train Steps/Sec: 0.70 + 84%|█████████████████████████████████████████████████████████████████████████████████████████████▌ | 34474/40903 [13:32:28<2:06:12, 1.18s/it][2025-04-22 15:19:59] (step=0034475) Train Loss: 6.3950, Train Steps/Sec: 0.85 + 84%|█████████████████████████████████████████████████████████████████████████████████████████████▌ | 34499/40903 [13:32:58<2:05:31, 1.18s/it][2025-04-22 15:20:29] (step=0034500) Train Loss: 6.4144, Train Steps/Sec: 0.83 + 84%|█████████████████████████████████████████████████████████████████████████████████████████████▋ | 34524/40903 [13:33:28<2:08:47, 1.21s/it][2025-04-22 15:20:59] (step=0034525) Train Loss: 6.3963, Train Steps/Sec: 0.85 + 84%|█████████████████████████████████████████████████████████████████████████████████████████████▊ | 34549/40903 [13:33:57<2:04:25, 1.17s/it][2025-04-22 15:21:28] (step=0034550) Train Loss: 6.3794, Train Steps/Sec: 0.85 + 85%|█████████████████████████████████████████████████████████████████████████████████████████████▊ | 34574/40903 [13:34:27<2:06:45, 1.20s/it][2025-04-22 15:21:58] (step=0034575) Train Loss: 6.3995, Train Steps/Sec: 0.84 + 85%|█████████████████████████████████████████████████████████████████████████████████████████████▉ | 34599/40903 [13:34:56<2:03:04, 1.17s/it][2025-04-22 15:22:28] (step=0034600) Train Loss: 6.4065, Train Steps/Sec: 0.85 + 85%|█████████████████████████████████████████████████████████████████████████████████████████████▉ | 34624/40903 [13:35:26<2:05:08, 1.20s/it][2025-04-22 15:22:57] (step=0034625) Train Loss: 6.3700, Train Steps/Sec: 0.85 + 85%|██████████████████████████████████████████████████████████████████████████████████████████████ | 34649/40903 [13:35:56<2:03:07, 1.18s/it][2025-04-22 15:23:27] (step=0034650) Train Loss: 6.3474, Train Steps/Sec: 0.84 + 85%|██████████████████████████████████████████████████████████████████████████████████████████████ | 34674/40903 [13:36:25<2:02:21, 1.18s/it][2025-04-22 15:23:56] (step=0034675) Train Loss: 6.4095, Train Steps/Sec: 0.85 + 85%|██████████████████████████████████████████████████████████████████████████████████████████████▏ | 34699/40903 [13:36:55<2:01:19, 1.17s/it][2025-04-22 15:24:26] (step=0034700) Train Loss: 6.4319, Train Steps/Sec: 0.84 + 85%|██████████████████████████████████████████████████████████████████████████████████████████████▏ | 34724/40903 [13:37:24<2:03:02, 1.19s/it][2025-04-22 15:24:56] (step=0034725) Train Loss: 6.3898, Train Steps/Sec: 0.84 + 85%|██████████████████████████████████████████████████████████████████████████████████████████████▎ | 34749/40903 [13:37:54<2:01:52, 1.19s/it][2025-04-22 15:25:25] (step=0034750) Train Loss: 6.4153, Train Steps/Sec: 0.85 + 85%|██████████████████████████████████████████████████████████████████████████████████████████████▎ | 34774/40903 [13:38:24<2:01:10, 1.19s/it][2025-04-22 15:25:55] (step=0034775) Train Loss: 6.3761, Train Steps/Sec: 0.84 + 85%|██████████████████████████████████████████████████████████████████████████████████████████████▍ | 34799/40903 [13:38:53<1:59:20, 1.17s/it][2025-04-22 15:26:25] (step=0034800) Train Loss: 6.3956, Train Steps/Sec: 0.84 + 85%|██████████████████████████████████████████████████████████████████████████████████████████████▌ | 34824/40903 [13:39:23<2:02:51, 1.21s/it][2025-04-22 15:26:54] (step=0034825) Train Loss: 6.3757, Train Steps/Sec: 0.84 + 85%|██████████████████████████████████████████████████████████████████████████████████████████████▌ | 34849/40903 [13:39:53<2:00:15, 1.19s/it][2025-04-22 15:27:24] (step=0034850) Train Loss: 6.4345, Train Steps/Sec: 0.84 + 85%|██████████████████████████████████████████████████████████████████████████████████████████████▋ | 34874/40903 [13:40:22<1:58:54, 1.18s/it][2025-04-22 15:27:53] (step=0034875) Train Loss: 6.3694, Train Steps/Sec: 0.85 + 85%|██████████████████████████████████████████████████████████████████████████████████████████████▋ | 34899/40903 [13:41:06<2:23:43, 1.44s/it][2025-04-22 15:28:38] (step=0034900) Train Loss: 6.4109, Train Steps/Sec: 0.56 + 85%|██████████████████████████████████████████████████████████████████████████████████████████████▊ | 34924/40903 [13:41:36<2:00:49, 1.21s/it][2025-04-22 15:29:08] (step=0034925) Train Loss: 6.3543, Train Steps/Sec: 0.84 + 85%|██████████████████████████████████████████████████████████████████████████████████████████████▊ | 34949/40903 [13:42:06<1:58:30, 1.19s/it][2025-04-22 15:29:37] (step=0034950) Train Loss: 6.3924, Train Steps/Sec: 0.84 + 86%|██████████████████████████████████████████████████████████████████████████████████████████████▉ | 34974/40903 [13:42:35<1:56:03, 1.17s/it][2025-04-22 15:30:07] (step=0034975) Train Loss: 6.4063, Train Steps/Sec: 0.85 + 86%|██████████████████████████████████████████████████████████████████████████████████████████████▉ | 34999/40903 [13:43:05<1:55:39, 1.18s/it][2025-04-22 15:30:37] (step=0035000) Train Loss: 6.3812, Train Steps/Sec: 0.84 + 86%|███████████████████████████████████████████████████████████████████████████████████████████████ | 35024/40903 [13:43:35<1:56:26, 1.19s/it][2025-04-22 15:31:06] (step=0035025) Train Loss: 6.4077, Train Steps/Sec: 0.85 + 86%|███████████████████████████████████████████████████████████████████████████████████████████████ | 35049/40903 [13:44:04<1:56:06, 1.19s/it][2025-04-22 15:31:36] (step=0035050) Train Loss: 6.4396, Train Steps/Sec: 0.84 + 86%|███████████████████████████████████████████████████████████████████████████████████████████████▏ | 35074/40903 [13:44:34<1:54:45, 1.18s/it][2025-04-22 15:32:05] (step=0035075) Train Loss: 6.3567, Train Steps/Sec: 0.85 + 86%|███████████████████████████████████████████████████████████████████████████████████████████████▏ | 35099/40903 [13:45:04<1:55:37, 1.20s/it][2025-04-22 15:32:35] (step=0035100) Train Loss: 6.4010, Train Steps/Sec: 0.84 + 86%|███████████████████████████████████████████████████████████████████████████████████████████████▎ | 35124/40903 [13:45:33<1:55:51, 1.20s/it][2025-04-22 15:33:04] (step=0035125) Train Loss: 6.3978, Train Steps/Sec: 0.85 + 86%|███████████████████████████████████████████████████████████████████████████████████████████████▍ | 35149/40903 [13:46:03<1:52:42, 1.18s/it][2025-04-22 15:33:34] (step=0035150) Train Loss: 6.3360, Train Steps/Sec: 0.85 + 86%|███████████████████████████████████████████████████████████████████████████████████████████████▍ | 35174/40903 [13:46:32<1:52:50, 1.18s/it][2025-04-22 15:34:03] (step=0035175) Train Loss: 6.3980, Train Steps/Sec: 0.85 + 86%|███████████████████████████████████████████████████████████████████████████████████████████████▌ | 35199/40903 [13:47:02<1:51:42, 1.18s/it][2025-04-22 15:34:33] (step=0035200) Train Loss: 6.3619, Train Steps/Sec: 0.85 + 86%|███████████████████████████████████████████████████████████████████████████████████████████████▌ | 35224/40903 [13:47:31<1:53:15, 1.20s/it][2025-04-22 15:35:02] (step=0035225) Train Loss: 6.3978, Train Steps/Sec: 0.85 + 86%|███████████████████████████████████████████████████████████████████████████████████████████████▋ | 35249/40903 [13:48:01<1:51:50, 1.19s/it][2025-04-22 15:35:32] (step=0035250) Train Loss: 6.4172, Train Steps/Sec: 0.85 + 86%|███████████████████████████████████████████████████████████████████████████████████████████████▋ | 35274/40903 [13:48:30<1:51:16, 1.19s/it][2025-04-22 15:36:02] (step=0035275) Train Loss: 6.4069, Train Steps/Sec: 0.85 + 86%|███████████████████████████████████████████████████████████████████████████████████████████████▊ | 35299/40903 [13:49:00<1:51:35, 1.19s/it][2025-04-22 15:36:31] (step=0035300) Train Loss: 6.4067, Train Steps/Sec: 0.84 + 86%|███████████████████████████████████████████████████████████████████████████████████████████████▊ | 35324/40903 [13:49:30<1:50:31, 1.19s/it][2025-04-22 15:37:01] (step=0035325) Train Loss: 6.3459, Train Steps/Sec: 0.85 + 86%|███████████████████████████████████████████████████████████████████████████████████████████████▉ | 35349/40903 [13:49:59<1:49:19, 1.18s/it][2025-04-22 15:37:30] (step=0035350) Train Loss: 6.4221, Train Steps/Sec: 0.84 + 86%|███████████████████████████████████████████████████████████████████████████████████████████████▉ | 35374/40903 [13:50:29<1:48:09, 1.17s/it][2025-04-22 15:38:00] (step=0035375) Train Loss: 6.3724, Train Steps/Sec: 0.85 + 87%|████████████████████████████████████████████████████████████████████████████████████████████████ | 35399/40903 [13:50:58<1:47:05, 1.17s/it][2025-04-22 15:38:30] (step=0035400) Train Loss: 6.3626, Train Steps/Sec: 0.84 + 87%|████████████████████████████████████████████████████████████████████████████████████████████████▏ | 35424/40903 [13:51:28<1:49:02, 1.19s/it][2025-04-22 15:38:59] (step=0035425) Train Loss: 6.3963, Train Steps/Sec: 0.85 + 87%|████████████████████████████████████████████████████████████████████████████████████████████████▏ | 35449/40903 [13:51:57<1:47:18, 1.18s/it][2025-04-22 15:39:29] (step=0035450) Train Loss: 6.4150, Train Steps/Sec: 0.84 + 87%|████████████████████████████████████████████████████████████████████████████████████████████████▎ | 35474/40903 [13:52:27<1:46:39, 1.18s/it][2025-04-22 15:39:58] (step=0035475) Train Loss: 6.4221, Train Steps/Sec: 0.85 + 87%|████████████████████████████████████████████████████████████████████████████████████████████████▎ | 35499/40903 [13:52:56<1:45:52, 1.18s/it][2025-04-22 15:40:28] (step=0035500) Train Loss: 6.4129, Train Steps/Sec: 0.84 + 87%|████████████████████████████████████████████████████████████████████████████████████████████████▍ | 35524/40903 [13:53:26<1:47:28, 1.20s/it][2025-04-22 15:40:57] (step=0035525) Train Loss: 6.3829, Train Steps/Sec: 0.85 + 87%|████████████████████████████████████████████████████████████████████████████████████████████████▍ | 35549/40903 [13:53:55<1:44:09, 1.17s/it][2025-04-22 15:41:27] (step=0035550) Train Loss: 6.3895, Train Steps/Sec: 0.85 + 87%|████████████████████████████████████████████████████████████████████████████████████████████████▌ | 35574/40903 [13:54:25<1:44:13, 1.17s/it][2025-04-22 15:41:56] (step=0035575) Train Loss: 6.3887, Train Steps/Sec: 0.85 + 87%|████████████████████████████████████████████████████████████████████████████████████████████████▌ | 35599/40903 [13:54:55<1:43:57, 1.18s/it][2025-04-22 15:42:26] (step=0035600) Train Loss: 6.3739, Train Steps/Sec: 0.85 + 87%|████████████████████████████████████████████████████████████████████████████████████████████████▋ | 35624/40903 [13:55:24<1:45:08, 1.19s/it][2025-04-22 15:42:55] (step=0035625) Train Loss: 6.4013, Train Steps/Sec: 0.84 + 87%|████████████████████████████████████████████████████████████████████████████████████████████████▋ | 35649/40903 [13:55:54<1:44:22, 1.19s/it][2025-04-22 15:43:25] (step=0035650) Train Loss: 6.3944, Train Steps/Sec: 0.85 + 87%|████████████████████████████████████████████████████████████████████████████████████████████████▊ | 35674/40903 [13:56:23<1:42:12, 1.17s/it][2025-04-22 15:43:55] (step=0035675) Train Loss: 6.4297, Train Steps/Sec: 0.84 + 87%|████████████████████████████████████████████████████████████████████████████████████████████████▉ | 35699/40903 [13:56:53<1:42:38, 1.18s/it][2025-04-22 15:44:25] (step=0035700) Train Loss: 6.3936, Train Steps/Sec: 0.84 + 87%|████████████████████████████████████████████████████████████████████████████████████████████████▉ | 35724/40903 [13:57:23<1:43:24, 1.20s/it][2025-04-22 15:44:54] (step=0035725) Train Loss: 6.3762, Train Steps/Sec: 0.84 + 87%|█████████████████████████████████████████████████████████████████████████████████████████████████ | 35749/40903 [13:57:52<1:41:57, 1.19s/it][2025-04-22 15:45:24] (step=0035750) Train Loss: 6.3790, Train Steps/Sec: 0.84 + 87%|█████████████████████████████████████████████████████████████████████████████████████████████████ | 35774/40903 [13:58:22<1:41:31, 1.19s/it][2025-04-22 15:45:53] (step=0035775) Train Loss: 6.4074, Train Steps/Sec: 0.85 + 88%|█████████████████████████████████████████████████████████████████████████████████████████████████▏ | 35799/40903 [13:58:52<1:39:04, 1.16s/it][2025-04-22 15:46:23] (step=0035800) Train Loss: 6.3839, Train Steps/Sec: 0.84 + 88%|█████████████████████████████████████████████████████████████████████████████████████████████████▏ | 35824/40903 [13:59:21<1:41:23, 1.20s/it][2025-04-22 15:46:52] (step=0035825) Train Loss: 6.3894, Train Steps/Sec: 0.85 + 88%|█████████████████████████████████████████████████████████████████████████████████████████████████▎ | 35849/40903 [13:59:50<1:39:49, 1.19s/it][2025-04-22 15:47:22] (step=0035850) Train Loss: 6.3680, Train Steps/Sec: 0.85 + 88%|█████████████████████████████████████████████████████████████████████████████████████████████████▎ | 35874/40903 [14:00:20<1:38:05, 1.17s/it][2025-04-22 15:47:51] (step=0035875) Train Loss: 6.3920, Train Steps/Sec: 0.85 + 88%|█████████████████████████████████████████████████████████████████████████████████████████████████▍ | 35899/40903 [14:00:49<1:38:53, 1.19s/it][2025-04-22 15:48:21] (step=0035900) Train Loss: 6.4001, Train Steps/Sec: 0.85 + 88%|█████████████████████████████████████████████████████████████████████████████████████████████████▍ | 35924/40903 [14:01:19<1:39:42, 1.20s/it][2025-04-22 15:48:50] (step=0035925) Train Loss: 6.3946, Train Steps/Sec: 0.85 + 88%|█████████████████████████████████████████████████████████████████████████████████████████████████▌ | 35949/40903 [14:01:49<1:37:01, 1.18s/it][2025-04-22 15:49:20] (step=0035950) Train Loss: 6.3736, Train Steps/Sec: 0.85 + 88%|█████████████████████████████████████████████████████████████████████████████████████████████████▌ | 35974/40903 [14:02:18<1:35:54, 1.17s/it][2025-04-22 15:49:49] (step=0035975) Train Loss: 6.3566, Train Steps/Sec: 0.84 + 88%|█████████████████████████████████████████████████████████████████████████████████████████████████▋ | 35999/40903 [14:02:48<1:35:48, 1.17s/it][2025-04-22 15:50:19] (step=0036000) Train Loss: 6.3866, Train Steps/Sec: 0.84 +You are using a model of type instructblip to instantiate a model of type blip-2. This is not supported for all configurations of models and can yield errors. +model_name_or_path:/tmp/haozhezhao/model/instructblip-flan-t5-xl +image_place_holder: +Some kwargs in processor config are unused and will not have any effect: num_query_tokens. +tokenizer length after expend 32102 +tokenizer length before expend 32102 +100%|██████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████| 3/3 [02:53<00:00, 57.92s/it] +[2025-04-22 15:55:00] Finish Eval in 36000 steps...███████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████| 3/3 [02:53<00:00, 57.56s/it] +processing left-padding... +processing left-padding... +processing left-padding... +[2025-04-22 15:55:22] Saved checkpoint to checkpoint/BeforeFix_Blip_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_objects/002-GPT-XL/checkpoints/0036000.pt +[2025-04-22 15:55:24] Removed old checkpoint: checkpoint/BeforeFix_Blip_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_objects/002-GPT-XL/checkpoints/0032000.pt + 88%|█████████████████████████████████████████████████████████████████████████████████████████████████▊ | 36024/40903 [14:08:22<1:38:36, 1.21s/it][2025-04-22 15:55:53] (step=0036025) Train Loss: 6.3926, Train Steps/Sec: 0.07 + 88%|█████████████████████████████████████████████████████████████████████████████████████████████████▊ | 36049/40903 [14:08:51<1:34:41, 1.17s/it][2025-04-22 15:56:23] (step=0036050) Train Loss: 6.3844, Train Steps/Sec: 0.84 + 88%|█████████████████████████████████████████████████████████████████████████████████████████████████▉ | 36074/40903 [14:09:21<1:34:49, 1.18s/it][2025-04-22 15:56:52] (step=0036075) Train Loss: 6.3889, Train Steps/Sec: 0.84 + 88%|█████████████████████████████████████████████████████████████████████████████████████████████████▉ | 36099/40903 [14:09:51<1:33:12, 1.16s/it][2025-04-22 15:57:22] (step=0036100) Train Loss: 6.3621, Train Steps/Sec: 0.85 + 88%|██████████████████████████████████████████████████████████████████████████████████████████████████ | 36124/40903 [14:10:20<1:36:04, 1.21s/it][2025-04-22 15:57:51] (step=0036125) Train Loss: 6.3975, Train Steps/Sec: 0.85 + 88%|██████████████████████████████████████████████████████████████████████████████████████████████████ | 36149/40903 [14:10:50<1:33:24, 1.18s/it][2025-04-22 15:58:21] (step=0036150) Train Loss: 6.4166, Train Steps/Sec: 0.85 + 88%|██████████████████████████████████████████████████████████████████████████████████████████████████▏ | 36174/40903 [14:11:19<1:32:10, 1.17s/it][2025-04-22 15:58:50] (step=0036175) Train Loss: 6.3883, Train Steps/Sec: 0.85 + 88%|██████████████████████████████████████████████████████████████████████████████████████████████████▏ | 36199/40903 [14:11:49<1:33:18, 1.19s/it][2025-04-22 15:59:20] (step=0036200) Train Loss: 6.4191, Train Steps/Sec: 0.84 + 89%|██████████████████████████████████████████████████████████████████████████████████████████████████▎ | 36224/40903 [14:12:18<1:31:55, 1.18s/it][2025-04-22 15:59:49] (step=0036225) Train Loss: 6.4028, Train Steps/Sec: 0.85 + 89%|██████████████████████████████████████████████████████████████████████████████████████████████████▎ | 36249/40903 [14:12:48<1:31:50, 1.18s/it][2025-04-22 16:00:19] (step=0036250) Train Loss: 6.4002, Train Steps/Sec: 0.85 + 89%|██████████████████████████████████████████████████████████████████████████████████████████████████▍ | 36274/40903 [14:13:17<1:30:00, 1.17s/it][2025-04-22 16:00:48] (step=0036275) Train Loss: 6.3773, Train Steps/Sec: 0.85 + 89%|██████████████████████████████████████████████████████████████████████████████████████████████████▌ | 36299/40903 [14:13:46<1:30:38, 1.18s/it][2025-04-22 16:01:18] (step=0036300) Train Loss: 6.3965, Train Steps/Sec: 0.85 + 89%|██████████████████████████████████████████████████████████████████████████████████████████████████▌ | 36324/40903 [14:14:16<1:31:05, 1.19s/it][2025-04-22 16:01:47] (step=0036325) Train Loss: 6.3721, Train Steps/Sec: 0.85 + 89%|██████████████████████████████████████████████████████████████████████████████████████████████████▋ | 36349/40903 [14:14:46<1:29:00, 1.17s/it][2025-04-22 16:02:17] (step=0036350) Train Loss: 6.3787, Train Steps/Sec: 0.85 + 89%|██████████████████████████████████████████████████████████████████████████████████████████████████▋ | 36374/40903 [14:15:15<1:27:40, 1.16s/it][2025-04-22 16:02:46] (step=0036375) Train Loss: 6.3801, Train Steps/Sec: 0.85 + 89%|██████████████████████████████████████████████████████████████████████████████████████████████████▊ | 36399/40903 [14:15:44<1:27:37, 1.17s/it][2025-04-22 16:03:16] (step=0036400) Train Loss: 6.3571, Train Steps/Sec: 0.85 + 89%|██████████████████████████████████████████████████████████████████████████████████████████████████▊ | 36424/40903 [14:16:14<1:29:34, 1.20s/it][2025-04-22 16:03:45] (step=0036425) Train Loss: 6.3542, Train Steps/Sec: 0.85 + 89%|██████████████████████████████████████████████████████████████████████████████████████████████████▉ | 36449/40903 [14:16:50<2:39:04, 2.14s/it][2025-04-22 16:04:21] (step=0036450) Train Loss: 6.3737, Train Steps/Sec: 0.70 + 89%|██████████████████████████████████████████████████████████████████████████████████████████████████▉ | 36474/40903 [14:17:20<1:27:33, 1.19s/it][2025-04-22 16:04:51] (step=0036475) Train Loss: 6.4427, Train Steps/Sec: 0.84 + 89%|███████████████████████████████████████████████████████████████████████████████████████████████████ | 36499/40903 [14:17:49<1:25:37, 1.17s/it][2025-04-22 16:05:21] (step=0036500) Train Loss: 6.3957, Train Steps/Sec: 0.84 + 89%|███████████████████████████████████████████████████████████████████████████████████████████████████ | 36524/40903 [14:18:19<1:28:28, 1.21s/it][2025-04-22 16:05:50] (step=0036525) Train Loss: 6.4257, Train Steps/Sec: 0.85 + 89%|███████████████████████████████████████████████████████████████████████████████████████████████████▏ | 36549/40903 [14:18:56<1:34:55, 1.31s/it][2025-04-22 16:06:27] (step=0036550) Train Loss: 6.3909, Train Steps/Sec: 0.68 + 89%|███████████████████████████████████████████████████████████████████████████████████████████████████▎ | 36574/40903 [14:19:25<1:26:16, 1.20s/it][2025-04-22 16:06:57] (step=0036575) Train Loss: 6.3853, Train Steps/Sec: 0.84 + 89%|███████████████████████████████████████████████████████████████████████████████████████████████████▎ | 36599/40903 [14:19:55<1:24:34, 1.18s/it][2025-04-22 16:07:26] (step=0036600) Train Loss: 6.4033, Train Steps/Sec: 0.84 + 90%|███████████████████████████████████████████████████████████████████████████████████████████████████▍ | 36624/40903 [14:20:25<1:25:21, 1.20s/it][2025-04-22 16:07:56] (step=0036625) Train Loss: 6.3511, Train Steps/Sec: 0.85 + 90%|███████████████████████████████████████████████████████████████████████████████████████████████████▍ | 36649/40903 [14:20:54<1:25:49, 1.21s/it][2025-04-22 16:08:25] (step=0036650) Train Loss: 6.3725, Train Steps/Sec: 0.84 + 90%|███████████████████████████████████████████████████████████████████████████████████████████████████▌ | 36674/40903 [14:21:24<1:23:30, 1.18s/it][2025-04-22 16:08:55] (step=0036675) Train Loss: 6.4274, Train Steps/Sec: 0.84 + 90%|███████████████████████████████████████████████████████████████████████████████████████████████████▌ | 36699/40903 [14:21:54<1:22:44, 1.18s/it][2025-04-22 16:09:25] (step=0036700) Train Loss: 6.3893, Train Steps/Sec: 0.83 + 90%|███████████████████████████████████████████████████████████████████████████████████████████████████▋ | 36724/40903 [14:22:24<1:24:51, 1.22s/it][2025-04-22 16:09:55] (step=0036725) Train Loss: 6.3850, Train Steps/Sec: 0.84 + 90%|███████████████████████████████████████████████████████████████████████████████████████████████████▋ | 36749/40903 [14:22:53<1:22:39, 1.19s/it][2025-04-22 16:10:25] (step=0036750) Train Loss: 6.3646, Train Steps/Sec: 0.84 + 90%|███████████████████████████████████████████████████████████████████████████████████████████████████▊ | 36774/40903 [14:23:23<1:21:05, 1.18s/it][2025-04-22 16:10:55] (step=0036775) Train Loss: 6.3610, Train Steps/Sec: 0.84 + 90%|███████████████████████████████████████████████████████████████████████████████████████████████████▊ | 36799/40903 [14:23:53<1:19:47, 1.17s/it][2025-04-22 16:11:24] (step=0036800) Train Loss: 6.4365, Train Steps/Sec: 0.84 + 90%|███████████████████████████████████████████████████████████████████████████████████████████████████▉ | 36824/40903 [14:24:23<1:21:11, 1.19s/it][2025-04-22 16:11:54] (step=0036825) Train Loss: 6.3569, Train Steps/Sec: 0.85 + 90%|███████████████████████████████████████████████████████████████████████████████████████████████████▉ | 36849/40903 [14:24:52<1:19:41, 1.18s/it][2025-04-22 16:12:23] (step=0036850) Train Loss: 6.3901, Train Steps/Sec: 0.85 + 90%|████████████████████████████████████████████████████████████████████████████████████████████████████ | 36874/40903 [14:25:21<1:18:47, 1.17s/it][2025-04-22 16:12:53] (step=0036875) Train Loss: 6.3364, Train Steps/Sec: 0.85 + 90%|████████████████████████████████████████████████████████████████████████████████████████████████████▏ | 36899/40903 [14:25:51<1:18:13, 1.17s/it][2025-04-22 16:13:22] (step=0036900) Train Loss: 6.3706, Train Steps/Sec: 0.84 + 90%|████████████████████████████████████████████████████████████████████████████████████████████████████▏ | 36924/40903 [14:26:21<1:18:52, 1.19s/it][2025-04-22 16:13:52] (step=0036925) Train Loss: 6.3937, Train Steps/Sec: 0.85 + 90%|████████████████████████████████████████████████████████████████████████████████████████████████████▎ | 36949/40903 [14:26:50<1:17:47, 1.18s/it][2025-04-22 16:14:21] (step=0036950) Train Loss: 6.4491, Train Steps/Sec: 0.85 + 90%|████████████████████████████████████████████████████████████████████████████████████████████████████▎ | 36974/40903 [14:27:20<1:16:39, 1.17s/it][2025-04-22 16:14:51] (step=0036975) Train Loss: 6.3584, Train Steps/Sec: 0.85 + 90%|████████████████████████████████████████████████████████████████████████████████████████████████████▍ | 36999/40903 [14:27:49<1:16:30, 1.18s/it][2025-04-22 16:15:21] (step=0037000) Train Loss: 6.4146, Train Steps/Sec: 0.84 + 91%|████████████████████████████████████████████████████████████████████████████████████████████████████▍ | 37024/40903 [14:28:19<1:17:39, 1.20s/it][2025-04-22 16:15:50] (step=0037025) Train Loss: 6.4016, Train Steps/Sec: 0.85 + 91%|████████████████████████████████████████████████████████████████████████████████████████████████████▌ | 37049/40903 [14:28:48<1:15:36, 1.18s/it][2025-04-22 16:16:20] (step=0037050) Train Loss: 6.4073, Train Steps/Sec: 0.85 + 91%|████████████████████████████████████████████████████████████████████████████████████████████████████▌ | 37074/40903 [14:29:18<1:15:28, 1.18s/it][2025-04-22 16:16:49] (step=0037075) Train Loss: 6.3936, Train Steps/Sec: 0.85 + 91%|████████████████████████████████████████████████████████████████████████████████████████████████████▋ | 37099/40903 [14:29:47<1:15:03, 1.18s/it][2025-04-22 16:17:19] (step=0037100) Train Loss: 6.3821, Train Steps/Sec: 0.84 + 91%|████████████████████████████████████████████████████████████████████████████████████████████████████▋ | 37124/40903 [14:30:17<1:14:58, 1.19s/it][2025-04-22 16:17:48] (step=0037125) Train Loss: 6.4026, Train Steps/Sec: 0.85 + 91%|████████████████████████████████████████████████████████████████████████████████████████████████████▊ | 37149/40903 [14:30:46<1:13:57, 1.18s/it][2025-04-22 16:18:18] (step=0037150) Train Loss: 6.3746, Train Steps/Sec: 0.85 + 91%|████████████████████████████████████████████████████████████████████████████████████████████████████▉ | 37174/40903 [14:31:16<1:12:34, 1.17s/it][2025-04-22 16:18:47] (step=0037175) Train Loss: 6.3622, Train Steps/Sec: 0.85 + 91%|████████████████████████████████████████████████████████████████████████████████████████████████████▉ | 37199/40903 [14:31:45<1:12:04, 1.17s/it][2025-04-22 16:19:17] (step=0037200) Train Loss: 6.3726, Train Steps/Sec: 0.84 + 91%|█████████████████████████████████████████████████████████████████████████████████████████████████████ | 37224/40903 [14:32:15<1:13:40, 1.20s/it][2025-04-22 16:19:46] (step=0037225) Train Loss: 6.3852, Train Steps/Sec: 0.85 + 91%|█████████████████████████████████████████████████████████████████████████████████████████████████████ | 37249/40903 [14:32:44<1:11:26, 1.17s/it][2025-04-22 16:20:16] (step=0037250) Train Loss: 6.3942, Train Steps/Sec: 0.85 + 91%|█████████████████████████████████████████████████████████████████████████████████████████████████████▏ | 37274/40903 [14:33:14<1:11:04, 1.17s/it][2025-04-22 16:20:45] (step=0037275) Train Loss: 6.4194, Train Steps/Sec: 0.85 + 91%|█████████████████████████████████████████████████████████████████████████████████████████████████████▏ | 37299/40903 [14:33:43<1:10:07, 1.17s/it][2025-04-22 16:21:14] (step=0037300) Train Loss: 6.3860, Train Steps/Sec: 0.84 + 91%|█████████████████████████████████████████████████████████████████████████████████████████████████████▎ | 37324/40903 [14:34:13<1:11:47, 1.20s/it][2025-04-22 16:21:44] (step=0037325) Train Loss: 6.3733, Train Steps/Sec: 0.85 + 91%|█████████████████████████████████████████████████████████████████████████████████████████████████████▎ | 37349/40903 [14:34:42<1:09:34, 1.17s/it][2025-04-22 16:22:13] (step=0037350) Train Loss: 6.4012, Train Steps/Sec: 0.85 + 91%|█████████████████████████████████████████████████████████████████████████████████████████████████████▍ | 37374/40903 [14:35:12<1:10:13, 1.19s/it][2025-04-22 16:22:43] (step=0037375) Train Loss: 6.3772, Train Steps/Sec: 0.84 + 91%|█████████████████████████████████████████████████████████████████████████████████████████████████████▍ | 37399/40903 [14:35:41<1:08:02, 1.17s/it][2025-04-22 16:23:13] (step=0037400) Train Loss: 6.3920, Train Steps/Sec: 0.84 + 91%|█████████████████████████████████████████████████████████████████████████████████████████████████████▌ | 37424/40903 [14:36:11<1:08:46, 1.19s/it][2025-04-22 16:23:42] (step=0037425) Train Loss: 6.3933, Train Steps/Sec: 0.85 + 92%|█████████████████████████████████████████████████████████████████████████████████████████████████████▋ | 37449/40903 [14:36:40<1:09:15, 1.20s/it][2025-04-22 16:24:12] (step=0037450) Train Loss: 6.4193, Train Steps/Sec: 0.85 + 92%|█████████████████████████████████████████████████████████████████████████████████████████████████████▋ | 37474/40903 [14:37:10<1:07:36, 1.18s/it][2025-04-22 16:24:41] (step=0037475) Train Loss: 6.3799, Train Steps/Sec: 0.84 + 92%|█████████████████████████████████████████████████████████████████████████████████████████████████████▊ | 37499/40903 [14:37:39<1:06:27, 1.17s/it][2025-04-22 16:25:11] (step=0037500) Train Loss: 6.3580, Train Steps/Sec: 0.85 + 92%|█████████████████████████████████████████████████████████████████████████████████████████████████████▊ | 37524/40903 [14:38:09<1:07:49, 1.20s/it][2025-04-22 16:25:40] (step=0037525) Train Loss: 6.3917, Train Steps/Sec: 0.84 + 92%|█████████████████████████████████████████████████████████████████████████████████████████████████████▉ | 37549/40903 [14:38:39<1:06:41, 1.19s/it][2025-04-22 16:26:10] (step=0037550) Train Loss: 6.3670, Train Steps/Sec: 0.84 + 92%|█████████████████████████████████████████████████████████████████████████████████████████████████████▉ | 37574/40903 [14:39:15<1:05:24, 1.18s/it][2025-04-22 16:26:46] (step=0037575) Train Loss: 6.4076, Train Steps/Sec: 0.70 + 92%|██████████████████████████████████████████████████████████████████████████████████████████████████████ | 37599/40903 [14:39:44<1:05:05, 1.18s/it][2025-04-22 16:27:16] (step=0037600) Train Loss: 6.3415, Train Steps/Sec: 0.84 + 92%|██████████████████████████████████████████████████████████████████████████████████████████████████████ | 37624/40903 [14:40:14<1:06:03, 1.21s/it][2025-04-22 16:27:45] (step=0037625) Train Loss: 6.3662, Train Steps/Sec: 0.85 + 92%|██████████████████████████████████████████████████████████████████████████████████████████████████████▏ | 37649/40903 [14:40:43<1:04:21, 1.19s/it][2025-04-22 16:28:15] (step=0037650) Train Loss: 6.3984, Train Steps/Sec: 0.84 + 92%|██████████████████████████████████████████████████████████████████████████████████████████████████████▏ | 37674/40903 [14:41:13<1:03:32, 1.18s/it][2025-04-22 16:28:44] (step=0037675) Train Loss: 6.3988, Train Steps/Sec: 0.84 + 92%|██████████████████████████████████████████████████████████████████████████████████████████████████████▎ | 37699/40903 [14:41:42<1:03:04, 1.18s/it][2025-04-22 16:29:14] (step=0037700) Train Loss: 6.3682, Train Steps/Sec: 0.84 + 92%|██████████████████████████████████████████████████████████████████████████████████████████████████████▎ | 37724/40903 [14:42:12<1:05:03, 1.23s/it][2025-04-22 16:29:44] (step=0037725) Train Loss: 6.4348, Train Steps/Sec: 0.84 + 92%|██████████████████████████████████████████████████████████████████████████████████████████████████████▍ | 37749/40903 [14:42:42<1:03:09, 1.20s/it][2025-04-22 16:30:13] (step=0037750) Train Loss: 6.3407, Train Steps/Sec: 0.84 + 92%|██████████████████████████████████████████████████████████████████████████████████████████████████████▌ | 37774/40903 [14:43:11<1:00:19, 1.16s/it][2025-04-22 16:30:43] (step=0037775) Train Loss: 6.3601, Train Steps/Sec: 0.86 + 92%|████████████████████████████████████████████████████████████████████████████████████████████████████████▍ | 37799/40903 [14:43:41<59:53, 1.16s/it][2025-04-22 16:31:12] (step=0037800) Train Loss: 6.3757, Train Steps/Sec: 0.85 + 92%|██████████████████████████████████████████████████████████████████████████████████████████████████████▋ | 37824/40903 [14:44:10<1:02:07, 1.21s/it][2025-04-22 16:31:42] (step=0037825) Train Loss: 6.3906, Train Steps/Sec: 0.84 + 93%|██████████████████████████████████████████████████████████████████████████████████████████████████████▋ | 37849/40903 [14:44:40<1:00:03, 1.18s/it][2025-04-22 16:32:11] (step=0037850) Train Loss: 6.3762, Train Steps/Sec: 0.85 + 93%|████████████████████████████████████████████████████████████████████████████████████████████████████████▋ | 37874/40903 [14:45:09<58:57, 1.17s/it][2025-04-22 16:32:41] (step=0037875) Train Loss: 6.3912, Train Steps/Sec: 0.85 + 93%|████████████████████████████████████████████████████████████████████████████████████████████████████████▋ | 37899/40903 [14:45:39<58:45, 1.17s/it][2025-04-22 16:33:10] (step=0037900) Train Loss: 6.3727, Train Steps/Sec: 0.84 + 93%|██████████████████████████████████████████████████████████████████████████████████████████████████████▉ | 37924/40903 [14:46:09<1:00:03, 1.21s/it][2025-04-22 16:33:40] (step=0037925) Train Loss: 6.3845, Train Steps/Sec: 0.85 + 93%|████████████████████████████████████████████████████████████████████████████████████████████████████████▊ | 37949/40903 [14:46:38<58:06, 1.18s/it][2025-04-22 16:34:09] (step=0037950) Train Loss: 6.3648, Train Steps/Sec: 0.85 + 93%|████████████████████████████████████████████████████████████████████████████████████████████████████████▉ | 37974/40903 [14:47:08<56:45, 1.16s/it][2025-04-22 16:34:39] (step=0037975) Train Loss: 6.4365, Train Steps/Sec: 0.85 + 93%|████████████████████████████████████████████████████████████████████████████████████████████████████████▉ | 37999/40903 [14:47:37<56:24, 1.17s/it][2025-04-22 16:35:08] (step=0038000) Train Loss: 6.4148, Train Steps/Sec: 0.85 +You are using a model of type instructblip to instantiate a model of type blip-2. This is not supported for all configurations of models and can yield errors. +model_name_or_path:/tmp/haozhezhao/model/instructblip-flan-t5-xl +image_place_holder: +Some kwargs in processor config are unused and will not have any effect: num_query_tokens. +tokenizer length after expend 32102 +tokenizer length before expend 32102 +100%|██████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████| 3/3 [02:54<00:00, 58.07s/it] +[2025-04-22 16:39:53] Finish Eval in 38000 steps...███████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████| 3/3 [02:53<00:00, 57.51s/it] +processing left-padding... +processing left-padding... +processing left-padding... +[2025-04-22 16:40:16] Saved checkpoint to checkpoint/BeforeFix_Blip_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_objects/002-GPT-XL/checkpoints/0038000.pt +[2025-04-22 16:40:18] Removed old checkpoint: checkpoint/BeforeFix_Blip_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_objects/002-GPT-XL/checkpoints/0034000.pt + 93%|█████████████████████████████████████████████████████████████████████████████████████████████████████████ | 38024/40903 [14:53:17<59:10, 1.23s/it][2025-04-22 16:40:48] (step=0038025) Train Loss: 6.3879, Train Steps/Sec: 0.07 + 93%|█████████████████████████████████████████████████████████████████████████████████████████████████████████ | 38049/40903 [14:53:46<56:35, 1.19s/it][2025-04-22 16:41:17] (step=0038050) Train Loss: 6.3910, Train Steps/Sec: 0.85 + 93%|█████████████████████████████████████████████████████████████████████████████████████████████████████████▏ | 38074/40903 [14:54:16<55:34, 1.18s/it][2025-04-22 16:41:47] (step=0038075) Train Loss: 6.4278, Train Steps/Sec: 0.84 + 93%|█████████████████████████████████████████████████████████████████████████████████████████████████████████▎ | 38099/40903 [14:54:45<54:05, 1.16s/it][2025-04-22 16:42:16] (step=0038100) Train Loss: 6.3830, Train Steps/Sec: 0.84 + 93%|█████████████████████████████████████████████████████████████████████████████████████████████████████████▎ | 38124/40903 [14:55:15<55:30, 1.20s/it][2025-04-22 16:42:46] (step=0038125) Train Loss: 6.4145, Train Steps/Sec: 0.84 + 93%|█████████████████████████████████████████████████████████████████████████████████████████████████████████▍ | 38149/40903 [14:55:44<54:09, 1.18s/it][2025-04-22 16:43:16] (step=0038150) Train Loss: 6.3584, Train Steps/Sec: 0.85 + 93%|█████████████████████████████████████████████████████████████████████████████████████████████████████████▍ | 38174/40903 [14:56:14<54:22, 1.20s/it][2025-04-22 16:43:45] (step=0038175) Train Loss: 6.3594, Train Steps/Sec: 0.85 + 93%|█████████████████████████████████████████████████████████████████████████████████████████████████████████▌ | 38199/40903 [14:56:44<52:16, 1.16s/it][2025-04-22 16:44:15] (step=0038200) Train Loss: 6.3688, Train Steps/Sec: 0.83 + 93%|█████████████████████████████████████████████████████████████████████████████████████████████████████████▌ | 38224/40903 [14:57:13<53:54, 1.21s/it][2025-04-22 16:44:45] (step=0038225) Train Loss: 6.3466, Train Steps/Sec: 0.85 + 94%|█████████████████████████████████████████████████████████████████████████████████████████████████████████▋ | 38249/40903 [14:57:43<51:47, 1.17s/it][2025-04-22 16:45:14] (step=0038250) Train Loss: 6.3710, Train Steps/Sec: 0.85 + 94%|█████████████████████████████████████████████████████████████████████████████████████████████████████████▋ | 38274/40903 [14:58:13<51:43, 1.18s/it][2025-04-22 16:45:44] (step=0038275) Train Loss: 6.3718, Train Steps/Sec: 0.84 + 94%|█████████████████████████████████████████████████████████████████████████████████████████████████████████▊ | 38299/40903 [14:58:42<50:44, 1.17s/it][2025-04-22 16:46:14] (step=0038300) Train Loss: 6.4114, Train Steps/Sec: 0.84 + 94%|█████████████████████████████████████████████████████████████████████████████████████████████████████████▉ | 38324/40903 [14:59:12<51:40, 1.20s/it][2025-04-22 16:46:43] (step=0038325) Train Loss: 6.4093, Train Steps/Sec: 0.85 + 94%|█████████████████████████████████████████████████████████████████████████████████████████████████████████▉ | 38349/40903 [14:59:41<50:38, 1.19s/it][2025-04-22 16:47:13] (step=0038350) Train Loss: 6.3862, Train Steps/Sec: 0.85 + 94%|██████████████████████████████████████████████████████████████████████████████████████████████████████████ | 38374/40903 [15:00:11<49:17, 1.17s/it][2025-04-22 16:47:42] (step=0038375) Train Loss: 6.4022, Train Steps/Sec: 0.85 + 94%|██████████████████████████████████████████████████████████████████████████████████████████████████████████ | 38399/40903 [15:00:41<49:00, 1.17s/it][2025-04-22 16:48:12] (step=0038400) Train Loss: 6.4000, Train Steps/Sec: 0.84 + 94%|██████████████████████████████████████████████████████████████████████████████████████████████████████████▏ | 38424/40903 [15:01:10<48:57, 1.18s/it][2025-04-22 16:48:41] (step=0038425) Train Loss: 6.4421, Train Steps/Sec: 0.85 + 94%|██████████████████████████████████████████████████████████████████████████████████████████████████████████▏ | 38449/40903 [15:01:40<48:22, 1.18s/it][2025-04-22 16:49:11] (step=0038450) Train Loss: 6.3572, Train Steps/Sec: 0.85 + 94%|██████████████████████████████████████████████████████████████████████████████████████████████████████████▎ | 38474/40903 [15:02:09<47:30, 1.17s/it][2025-04-22 16:49:40] (step=0038475) Train Loss: 6.3880, Train Steps/Sec: 0.85 + 94%|██████████████████████████████████████████████████████████████████████████████████████████████████████████▎ | 38499/40903 [15:02:39<46:58, 1.17s/it][2025-04-22 16:50:10] (step=0038500) Train Loss: 6.4173, Train Steps/Sec: 0.84 + 94%|██████████████████████████████████████████████████████████████████████████████████████████████████████████▍ | 38524/40903 [15:03:08<47:11, 1.19s/it][2025-04-22 16:50:40] (step=0038525) Train Loss: 6.3651, Train Steps/Sec: 0.85 + 94%|██████████████████████████████████████████████████████████████████████████████████████████████████████████▍ | 38549/40903 [15:03:38<46:22, 1.18s/it][2025-04-22 16:51:09] (step=0038550) Train Loss: 6.3772, Train Steps/Sec: 0.85 + 94%|██████████████████████████████████████████████████████████████████████████████████████████████████████████▌ | 38574/40903 [15:04:07<45:19, 1.17s/it][2025-04-22 16:51:39] (step=0038575) Train Loss: 6.3964, Train Steps/Sec: 0.85 + 94%|██████████████████████████████████████████████████████████████████████████████████████████████████████████▋ | 38599/40903 [15:04:37<44:44, 1.17s/it][2025-04-22 16:52:08] (step=0038600) Train Loss: 6.3644, Train Steps/Sec: 0.85 + 94%|██████████████████████████████████████████████████████████████████████████████████████████████████████████▋ | 38624/40903 [15:05:06<45:30, 1.20s/it][2025-04-22 16:52:37] (step=0038625) Train Loss: 6.3885, Train Steps/Sec: 0.85 + 94%|██████████████████████████████████████████████████████████████████████████████████████████████████████████▊ | 38649/40903 [15:05:36<44:52, 1.19s/it][2025-04-22 16:53:07] (step=0038650) Train Loss: 6.3826, Train Steps/Sec: 0.85 + 95%|██████████████████████████████████████████████████████████████████████████████████████████████████████████▊ | 38674/40903 [15:06:05<43:33, 1.17s/it][2025-04-22 16:53:37] (step=0038675) Train Loss: 6.4106, Train Steps/Sec: 0.85 + 95%|██████████████████████████████████████████████████████████████████████████████████████████████████████████▉ | 38699/40903 [15:06:35<43:03, 1.17s/it][2025-04-22 16:54:06] (step=0038700) Train Loss: 6.4017, Train Steps/Sec: 0.84 + 95%|██████████████████████████████████████████████████████████████████████████████████████████████████████████▉ | 38724/40903 [15:07:05<43:48, 1.21s/it][2025-04-22 16:54:36] (step=0038725) Train Loss: 6.4089, Train Steps/Sec: 0.84 + 95%|███████████████████████████████████████████████████████████████████████████████████████████████████████████ | 38749/40903 [15:07:34<42:17, 1.18s/it][2025-04-22 16:55:05] (step=0038750) Train Loss: 6.4190, Train Steps/Sec: 0.85 + 95%|███████████████████████████████████████████████████████████████████████████████████████████████████████████ | 38774/40903 [15:08:04<42:49, 1.21s/it][2025-04-22 16:55:35] (step=0038775) Train Loss: 6.3880, Train Steps/Sec: 0.84 + 95%|███████████████████████████████████████████████████████████████████████████████████████████████████████████▏ | 38799/40903 [15:08:33<40:54, 1.17s/it][2025-04-22 16:56:05] (step=0038800) Train Loss: 6.4394, Train Steps/Sec: 0.85 + 95%|███████████████████████████████████████████████████████████████████████████████████████████████████████████▎ | 38824/40903 [15:09:03<42:08, 1.22s/it][2025-04-22 16:56:34] (step=0038825) Train Loss: 6.3848, Train Steps/Sec: 0.84 + 95%|███████████████████████████████████████████████████████████████████████████████████████████████████████████▎ | 38849/40903 [15:09:32<40:12, 1.17s/it][2025-04-22 16:57:04] (step=0038850) Train Loss: 6.3746, Train Steps/Sec: 0.85 + 95%|███████████████████████████████████████████████████████████████████████████████████████████████████████████▍ | 38874/40903 [15:10:02<39:34, 1.17s/it][2025-04-22 16:57:33] (step=0038875) Train Loss: 6.3936, Train Steps/Sec: 0.85 + 95%|███████████████████████████████████████████████████████████████████████████████████████████████████████████▍ | 38899/40903 [15:10:31<39:28, 1.18s/it][2025-04-22 16:58:03] (step=0038900) Train Loss: 6.3625, Train Steps/Sec: 0.84 + 95%|███████████████████████████████████████████████████████████████████████████████████████████████████████████▌ | 38924/40903 [15:11:01<39:34, 1.20s/it][2025-04-22 16:58:32] (step=0038925) Train Loss: 6.3931, Train Steps/Sec: 0.85 + 95%|███████████████████████████████████████████████████████████████████████████████████████████████████████████▌ | 38949/40903 [15:11:30<38:41, 1.19s/it][2025-04-22 16:59:02] (step=0038950) Train Loss: 6.3660, Train Steps/Sec: 0.84 + 95%|███████████████████████████████████████████████████████████████████████████████████████████████████████████▋ | 38974/40903 [15:12:00<37:51, 1.18s/it][2025-04-22 16:59:31] (step=0038975) Train Loss: 6.3736, Train Steps/Sec: 0.85 + 95%|███████████████████████████████████████████████████████████████████████████████████████████████████████████▋ | 38999/40903 [15:12:30<36:55, 1.16s/it][2025-04-22 17:00:01] (step=0039000) Train Loss: 6.3818, Train Steps/Sec: 0.84 + 95%|███████████████████████████████████████████████████████████████████████████████████████████████████████████▊ | 39024/40903 [15:12:59<37:41, 1.20s/it][2025-04-22 17:00:30] (step=0039025) Train Loss: 6.3904, Train Steps/Sec: 0.85 + 95%|███████████████████████████████████████████████████████████████████████████████████████████████████████████▉ | 39049/40903 [15:13:29<36:14, 1.17s/it][2025-04-22 17:01:00] (step=0039050) Train Loss: 6.3967, Train Steps/Sec: 0.85 + 96%|███████████████████████████████████████████████████████████████████████████████████████████████████████████▉ | 39074/40903 [15:13:58<35:37, 1.17s/it][2025-04-22 17:01:29] (step=0039075) Train Loss: 6.3369, Train Steps/Sec: 0.85 + 96%|████████████████████████████████████████████████████████████████████████████████████████████████████████████ | 39099/40903 [15:14:28<35:03, 1.17s/it][2025-04-22 17:01:59] (step=0039100) Train Loss: 6.4115, Train Steps/Sec: 0.84 + 96%|████████████████████████████████████████████████████████████████████████████████████████████████████████████ | 39124/40903 [15:14:57<35:35, 1.20s/it][2025-04-22 17:02:29] (step=0039125) Train Loss: 6.4077, Train Steps/Sec: 0.84 + 96%|████████████████████████████████████████████████████████████████████████████████████████████████████████████▏ | 39149/40903 [15:15:27<34:23, 1.18s/it][2025-04-22 17:02:58] (step=0039150) Train Loss: 6.3622, Train Steps/Sec: 0.85 + 96%|████████████████████████████████████████████████████████████████████████████████████████████████████████████▏ | 39174/40903 [15:15:56<33:33, 1.16s/it][2025-04-22 17:03:28] (step=0039175) Train Loss: 6.3941, Train Steps/Sec: 0.85 + 96%|████████████████████████████████████████████████████████████████████████████████████████████████████████████▎ | 39199/40903 [15:16:26<33:16, 1.17s/it][2025-04-22 17:03:57] (step=0039200) Train Loss: 6.3965, Train Steps/Sec: 0.84 + 96%|████████████████████████████████████████████████████████████████████████████████████████████████████████████▎ | 39224/40903 [15:16:56<33:39, 1.20s/it][2025-04-22 17:04:27] (step=0039225) Train Loss: 6.3501, Train Steps/Sec: 0.85 + 96%|████████████████████████████████████████████████████████████████████████████████████████████████████████████▍ | 39249/40903 [15:17:25<32:33, 1.18s/it][2025-04-22 17:04:57] (step=0039250) Train Loss: 6.3854, Train Steps/Sec: 0.84 + 96%|████████████████████████████████████████████████████████████████████████████████████████████████████████████▍ | 39274/40903 [15:17:55<31:38, 1.17s/it][2025-04-22 17:05:26] (step=0039275) Train Loss: 6.3476, Train Steps/Sec: 0.85 + 96%|████████████████████████████████████████████████████████████████████████████████████████████████████████████▌ | 39299/40903 [15:18:24<31:08, 1.16s/it][2025-04-22 17:05:56] (step=0039300) Train Loss: 6.4154, Train Steps/Sec: 0.85 + 96%|████████████████████████████████████████████████████████████████████████████████████████████████████████████▋ | 39324/40903 [15:18:54<31:22, 1.19s/it][2025-04-22 17:06:25] (step=0039325) Train Loss: 6.3741, Train Steps/Sec: 0.85 + 96%|████████████████████████████████████████████████████████████████████████████████████████████████████████████▋ | 39349/40903 [15:19:23<30:19, 1.17s/it][2025-04-22 17:06:55] (step=0039350) Train Loss: 6.4183, Train Steps/Sec: 0.84 + 96%|████████████████████████████████████████████████████████████████████████████████████████████████████████████▊ | 39374/40903 [15:19:53<29:41, 1.17s/it][2025-04-22 17:07:24] (step=0039375) Train Loss: 6.3601, Train Steps/Sec: 0.85 + 96%|████████████████████████████████████████████████████████████████████████████████████████████████████████████▊ | 39399/40903 [15:20:22<29:12, 1.17s/it][2025-04-22 17:07:54] (step=0039400) Train Loss: 6.3854, Train Steps/Sec: 0.85 + 96%|████████████████████████████████████████████████████████████████████████████████████████████████████████████▉ | 39424/40903 [15:20:52<30:07, 1.22s/it][2025-04-22 17:08:23] (step=0039425) Train Loss: 6.3627, Train Steps/Sec: 0.84 + 96%|████████████████████████████████████████████████████████████████████████████████████████████████████████████▉ | 39449/40903 [15:21:22<28:50, 1.19s/it][2025-04-22 17:08:53] (step=0039450) Train Loss: 6.3506, Train Steps/Sec: 0.84 + 97%|█████████████████████████████████████████████████████████████████████████████████████████████████████████████ | 39474/40903 [15:21:57<27:54, 1.17s/it][2025-04-22 17:09:29] (step=0039475) Train Loss: 6.3800, Train Steps/Sec: 0.70 + 97%|█████████████████████████████████████████████████████████████████████████████████████████████████████████████ | 39499/40903 [15:22:33<27:24, 1.17s/it][2025-04-22 17:10:05] (step=0039500) Train Loss: 6.3657, Train Steps/Sec: 0.69 + 97%|█████████████████████████████████████████████████████████████████████████████████████████████████████████████▏ | 39524/40903 [15:23:03<27:17, 1.19s/it][2025-04-22 17:10:34] (step=0039525) Train Loss: 6.3602, Train Steps/Sec: 0.85 + 97%|█████████████████████████████████████████████████████████████████████████████████████████████████████████████▎ | 39549/40903 [15:23:32<26:43, 1.18s/it][2025-04-22 17:11:04] (step=0039550) Train Loss: 6.3837, Train Steps/Sec: 0.85 + 97%|█████████████████████████████████████████████████████████████████████████████████████████████████████████████▎ | 39574/40903 [15:24:02<26:20, 1.19s/it][2025-04-22 17:11:33] (step=0039575) Train Loss: 6.4554, Train Steps/Sec: 0.84 + 97%|█████████████████████████████████████████████████████████████████████████████████████████████████████████████▍ | 39599/40903 [15:24:32<25:27, 1.17s/it][2025-04-22 17:12:03] (step=0039600) Train Loss: 6.3356, Train Steps/Sec: 0.84 + 97%|█████████████████████████████████████████████████████████████████████████████████████████████████████████████▍ | 39624/40903 [15:25:01<25:46, 1.21s/it][2025-04-22 17:12:33] (step=0039625) Train Loss: 6.4024, Train Steps/Sec: 0.85 + 97%|█████████████████████████████████████████████████████████████████████████████████████████████████████████████▌ | 39649/40903 [15:25:31<24:46, 1.19s/it][2025-04-22 17:13:02] (step=0039650) Train Loss: 6.3968, Train Steps/Sec: 0.85 + 97%|█████████████████████████████████████████████████████████████████████████████████████████████████████████████▌ | 39674/40903 [15:26:00<24:08, 1.18s/it][2025-04-22 17:13:32] (step=0039675) Train Loss: 6.4121, Train Steps/Sec: 0.85 + 97%|█████████████████████████████████████████████████████████████████████████████████████████████████████████████▋ | 39699/40903 [15:26:30<23:30, 1.17s/it][2025-04-22 17:14:01] (step=0039700) Train Loss: 6.4003, Train Steps/Sec: 0.85 + 97%|█████████████████████████████████████████████████████████████████████████████████████████████████████████████▋ | 39724/40903 [15:26:59<23:34, 1.20s/it][2025-04-22 17:14:31] (step=0039725) Train Loss: 6.3771, Train Steps/Sec: 0.85 + 97%|█████████████████████████████████████████████████████████████████████████████████████████████████████████████▊ | 39749/40903 [15:27:29<22:56, 1.19s/it][2025-04-22 17:15:00] (step=0039750) Train Loss: 6.3769, Train Steps/Sec: 0.85 + 97%|█████████████████████████████████████████████████████████████████████████████████████████████████████████████▉ | 39774/40903 [15:27:59<22:05, 1.17s/it][2025-04-22 17:15:30] (step=0039775) Train Loss: 6.3925, Train Steps/Sec: 0.84 + 97%|█████████████████████████████████████████████████████████████████████████████████████████████████████████████▉ | 39799/40903 [15:28:28<21:34, 1.17s/it][2025-04-22 17:15:59] (step=0039800) Train Loss: 6.3614, Train Steps/Sec: 0.85 + 97%|██████████████████████████████████████████████████████████████████████████████████████████████████████████████ | 39824/40903 [15:28:58<21:44, 1.21s/it][2025-04-22 17:16:29] (step=0039825) Train Loss: 6.4094, Train Steps/Sec: 0.84 + 97%|██████████████████████████████████████████████████████████████████████████████████████████████████████████████ | 39849/40903 [15:29:27<20:59, 1.19s/it][2025-04-22 17:16:58] (step=0039850) Train Loss: 6.3623, Train Steps/Sec: 0.85 + 97%|██████████████████████████████████████████████████████████████████████████████████████████████████████████████▏ | 39874/40903 [15:29:57<20:17, 1.18s/it][2025-04-22 17:17:28] (step=0039875) Train Loss: 6.3943, Train Steps/Sec: 0.85 + 98%|██████████████████████████████████████████████████████████████████████████████████████████████████████████████▏ | 39899/40903 [15:30:26<19:44, 1.18s/it][2025-04-22 17:17:58] (step=0039900) Train Loss: 6.3972, Train Steps/Sec: 0.84 + 98%|██████████████████████████████████████████████████████████████████████████████████████████████████████████████▎ | 39924/40903 [15:30:56<19:34, 1.20s/it][2025-04-22 17:18:27] (step=0039925) Train Loss: 6.3980, Train Steps/Sec: 0.85 + 98%|██████████████████████████████████████████████████████████████████████████████████████████████████████████████▎ | 39949/40903 [15:31:25<18:43, 1.18s/it][2025-04-22 17:18:57] (step=0039950) Train Loss: 6.4260, Train Steps/Sec: 0.85 + 98%|██████████████████████████████████████████████████████████████████████████████████████████████████████████████▍ | 39974/40903 [15:32:00<26:25, 1.71s/it][2025-04-22 17:19:31] (step=0039975) Train Loss: 6.3641, Train Steps/Sec: 0.72 + 98%|██████████████████████████████████████████████████████████████████████████████████████████████████████████████▌ | 39999/40903 [15:32:30<17:38, 1.17s/it][2025-04-22 17:20:01] (step=0040000) Train Loss: 6.4167, Train Steps/Sec: 0.84 +You are using a model of type instructblip to instantiate a model of type blip-2. This is not supported for all configurations of models and can yield errors. +model_name_or_path:/tmp/haozhezhao/model/instructblip-flan-t5-xl +image_place_holder: +Some kwargs in processor config are unused and will not have any effect: num_query_tokens. +tokenizer length after expend 32102 +tokenizer length before expend 32102 +100%|██████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████| 3/3 [02:53<00:00, 57.67s/it] +[2025-04-22 17:24:42] Finish Eval in 40000 steps...███████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████| 3/3 [02:52<00:00, 57.24s/it] +processing left-padding... +processing left-padding... +processing left-padding... +[2025-04-22 17:25:03] Saved checkpoint to checkpoint/BeforeFix_Blip_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_objects/002-GPT-XL/checkpoints/0040000.pt +[2025-04-22 17:25:06] Removed old checkpoint: checkpoint/BeforeFix_Blip_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_objects/002-GPT-XL/checkpoints/0036000.pt + 98%|██████████████████████████████████████████████████████████████████████████████████████████████████████████████▌ | 40024/40903 [15:38:12<42:36, 2.91s/it][2025-04-22 17:25:44] (step=0040025) Train Loss: 6.4418, Train Steps/Sec: 0.07 + 98%|██████████████████████████████████████████████████████████████████████████████████████████████████████████████▋ | 40049/40903 [15:38:42<16:45, 1.18s/it][2025-04-22 17:26:13] (step=0040050) Train Loss: 6.3717, Train Steps/Sec: 0.85 + 98%|██████████████████████████████████████████████████████████████████████████████████████████████████████████████▋ | 40074/40903 [15:39:11<16:17, 1.18s/it][2025-04-22 17:26:43] (step=0040075) Train Loss: 6.3587, Train Steps/Sec: 0.85 + 98%|██████████████████████████████████████████████████████████████████████████████████████████████████████████████▊ | 40099/40903 [15:39:41<15:41, 1.17s/it][2025-04-22 17:27:12] (step=0040100) Train Loss: 6.4024, Train Steps/Sec: 0.85 + 98%|██████████████████████████████████████████████████████████████████████████████████████████████████████████████▊ | 40124/40903 [15:40:11<15:42, 1.21s/it][2025-04-22 17:27:42] (step=0040125) Train Loss: 6.3943, Train Steps/Sec: 0.84 + 98%|██████████████████████████████████████████████████████████████████████████████████████████████████████████████▉ | 40149/40903 [15:40:40<14:55, 1.19s/it][2025-04-22 17:28:11] (step=0040150) Train Loss: 6.3823, Train Steps/Sec: 0.85 + 98%|██████████████████████████████████████████████████████████████████████████████████████████████████████████████▉ | 40174/40903 [15:41:10<14:28, 1.19s/it][2025-04-22 17:28:41] (step=0040175) Train Loss: 6.3927, Train Steps/Sec: 0.85 + 98%|███████████████████████████████████████████████████████████████████████████████████████████████████████████████ | 40199/40903 [15:41:39<13:41, 1.17s/it][2025-04-22 17:29:11] (step=0040200) Train Loss: 6.4010, Train Steps/Sec: 0.84 + 98%|███████████████████████████████████████████████████████████████████████████████████████████████████████████████ | 40224/40903 [15:42:09<13:32, 1.20s/it][2025-04-22 17:29:40] (step=0040225) Train Loss: 6.4033, Train Steps/Sec: 0.85 + 98%|███████████████████████████████████████████████████████████████████████████████████████████████████████████████▏ | 40249/40903 [15:42:38<13:01, 1.20s/it][2025-04-22 17:30:10] (step=0040250) Train Loss: 6.3539, Train Steps/Sec: 0.85 + 98%|███████████████████████████████████████████████████████████████████████████████████████████████████████████████▎ | 40274/40903 [15:43:08<12:12, 1.17s/it][2025-04-22 17:30:39] (step=0040275) Train Loss: 6.3940, Train Steps/Sec: 0.85 + 99%|███████████████████████████████████████████████████████████████████████████████████████████████████████████████▎ | 40299/40903 [15:43:37<11:41, 1.16s/it][2025-04-22 17:31:09] (step=0040300) Train Loss: 6.4047, Train Steps/Sec: 0.84 + 99%|███████████████████████████████████████████████████████████████████████████████████████████████████████████████▍ | 40324/40903 [15:44:07<11:32, 1.20s/it][2025-04-22 17:31:38] (step=0040325) Train Loss: 6.4025, Train Steps/Sec: 0.85 + 99%|███████████████████████████████████████████████████████████████████████████████████████████████████████████████▍ | 40349/40903 [15:44:37<10:58, 1.19s/it][2025-04-22 17:32:08] (step=0040350) Train Loss: 6.3988, Train Steps/Sec: 0.84 + 99%|███████████████████████████████████████████████████████████████████████████████████████████████████████████████▌ | 40374/40903 [15:45:06<10:14, 1.16s/it][2025-04-22 17:32:37] (step=0040375) Train Loss: 6.3818, Train Steps/Sec: 0.85 + 99%|███████████████████████████████████████████████████████████████████████████████████████████████████████████████▌ | 40399/40903 [15:45:36<09:49, 1.17s/it][2025-04-22 17:33:07] (step=0040400) Train Loss: 6.4376, Train Steps/Sec: 0.84 + 99%|███████████████████████████████████████████████████████████████████████████████████████████████████████████████▋ | 40424/40903 [15:46:05<09:31, 1.19s/it][2025-04-22 17:33:36] (step=0040425) Train Loss: 6.3867, Train Steps/Sec: 0.85 + 99%|███████████████████████████████████████████████████████████████████████████████████████████████████████████████▋ | 40449/40903 [15:46:34<08:53, 1.17s/it][2025-04-22 17:34:06] (step=0040450) Train Loss: 6.3956, Train Steps/Sec: 0.85 + 99%|███████████████████████████████████████████████████████████████████████████████████████████████████████████████▊ | 40474/40903 [15:47:04<08:23, 1.17s/it][2025-04-22 17:34:35] (step=0040475) Train Loss: 6.3705, Train Steps/Sec: 0.85 + 99%|███████████████████████████████████████████████████████████████████████████████████████████████████████████████▉ | 40499/40903 [15:47:33<07:53, 1.17s/it][2025-04-22 17:35:05] (step=0040500) Train Loss: 6.3735, Train Steps/Sec: 0.85 + 99%|███████████████████████████████████████████████████████████████████████████████████████████████████████████████▉ | 40524/40903 [15:48:03<07:39, 1.21s/it][2025-04-22 17:35:35] (step=0040525) Train Loss: 6.3882, Train Steps/Sec: 0.84 + 99%|████████████████████████████████████████████████████████████████████████████████████████████████████████████████ | 40549/40903 [15:48:33<06:56, 1.18s/it][2025-04-22 17:36:04] (step=0040550) Train Loss: 6.4112, Train Steps/Sec: 0.85 + 99%|████████████████████████████████████████████████████████████████████████████████████████████████████████████████ | 40574/40903 [15:49:02<06:23, 1.17s/it][2025-04-22 17:36:33] (step=0040575) Train Loss: 6.3647, Train Steps/Sec: 0.85 + 99%|████████████████████████████████████████████████████████████████████████████████████████████████████████████████▏| 40599/40903 [15:49:32<05:56, 1.17s/it][2025-04-22 17:37:03] (step=0040600) Train Loss: 6.4277, Train Steps/Sec: 0.84 + 99%|████████████████████████████████████████████████████████████████████████████████████████████████████████████████▏| 40624/40903 [15:50:01<05:34, 1.20s/it][2025-04-22 17:37:32] (step=0040625) Train Loss: 6.3959, Train Steps/Sec: 0.85 + 99%|████████████████████████████████████████████████████████████████████████████████████████████████████████████████▎| 40649/40903 [15:50:31<05:00, 1.18s/it][2025-04-22 17:38:02] (step=0040650) Train Loss: 6.4219, Train Steps/Sec: 0.84 + 99%|████████████████████████████████████████████████████████████████████████████████████████████████████████████████▎| 40674/40903 [15:51:00<04:30, 1.18s/it][2025-04-22 17:38:32] (step=0040675) Train Loss: 6.3737, Train Steps/Sec: 0.85 +100%|████████████████████████████████████████████████████████████████████████████████████████████████████████████████▍| 40699/40903 [15:51:30<03:54, 1.15s/it][2025-04-22 17:39:01] (step=0040700) Train Loss: 6.4057, Train Steps/Sec: 0.85 +100%|████████████████████████████████████████████████████████████████████████████████████████████████████████████████▌| 40724/40903 [15:51:59<03:34, 1.20s/it][2025-04-22 17:39:31] (step=0040725) Train Loss: 6.3926, Train Steps/Sec: 0.85 +100%|████████████████████████████████████████████████████████████████████████████████████████████████████████████████▌| 40749/40903 [15:52:29<03:00, 1.17s/it][2025-04-22 17:40:00] (step=0040750) Train Loss: 6.3837, Train Steps/Sec: 0.85 +100%|████████████████████████████████████████████████████████████████████████████████████████████████████████████████▋| 40774/40903 [15:52:58<02:31, 1.18s/it][2025-04-22 17:40:30] (step=0040775) Train Loss: 6.3801, Train Steps/Sec: 0.84 +100%|████████████████████████████████████████████████████████████████████████████████████████████████████████████████▋| 40799/40903 [15:53:28<02:01, 1.17s/it][2025-04-22 17:40:59] (step=0040800) Train Loss: 6.3722, Train Steps/Sec: 0.84 +100%|████████████████████████████████████████████████████████████████████████████████████████████████████████████████▊| 40824/40903 [15:53:58<01:34, 1.20s/it][2025-04-22 17:41:29] (step=0040825) Train Loss: 6.4028, Train Steps/Sec: 0.85 +100%|████████████████████████████████████████████████████████████████████████████████████████████████████████████████▊| 40849/40903 [15:54:27<01:04, 1.19s/it][2025-04-22 17:41:59] (step=0040850) Train Loss: 6.3711, Train Steps/Sec: 0.84 +100%|████████████████████████████████████████████████████████████████████████████████████████████████████████████████▉| 40874/40903 [15:54:57<00:34, 1.18s/it][2025-04-22 17:42:28] (step=0040875) Train Loss: 6.3848, Train Steps/Sec: 0.85 +100%|████████████████████████████████████████████████████████████████████████████████████████████████████████████████▉| 40899/40903 [15:55:26<00:04, 1.18s/it][2025-04-22 17:42:58] (step=0040900) Train Loss: 6.3917, Train Steps/Sec: 0.84 +100%|█████████████████████████████████████████████████████████████████████████████████████████████████████████████████| 40903/40903 [15:55:33<00:00, 1.40s/it] +[2025-04-22 17:43:03] Beginning epoch 1... + 0%| | 21/40903 [00:27<14:02:52, 1.24s/it][2025-04-22 17:43:32] (step=0040925) Train Loss: 6.3901, Train Steps/Sec: 0.73 + 0%|▏ | 46/40903 [00:57<13:30:36, 1.19s/it][2025-04-22 17:44:01] (step=0040950) Train Loss: 6.3812, Train Steps/Sec: 0.84 + 0%|▎ | 71/40903 [01:26<13:23:03, 1.18s/it][2025-04-22 17:44:31] (step=0040975) Train Loss: 6.3427, Train Steps/Sec: 0.85 + 0%|▍ | 96/40903 [01:56<13:12:51, 1.17s/it][2025-04-22 17:45:01] (step=0041000) Train Loss: 6.3212, Train Steps/Sec: 0.84 + 0%|▌ | 121/40903 [02:26<13:41:46, 1.21s/it][2025-04-22 17:45:30] (step=0041025) Train Loss: 6.3258, Train Steps/Sec: 0.84 + 0%|▋ | 146/40903 [02:55<13:24:10, 1.18s/it][2025-04-22 17:46:00] (step=0041050) Train Loss: 6.3368, Train Steps/Sec: 0.84 + 0%|▊ | 171/40903 [03:25<13:31:43, 1.20s/it][2025-04-22 17:46:30] (step=0041075) Train Loss: 6.3127, Train Steps/Sec: 0.84 + 0%|▊ | 196/40903 [03:55<13:27:02, 1.19s/it][2025-04-22 17:47:00] (step=0041100) Train Loss: 6.3534, Train Steps/Sec: 0.84 + 1%|▉ | 221/40903 [04:25<13:27:39, 1.19s/it][2025-04-22 17:47:29] (step=0041125) Train Loss: 6.3645, Train Steps/Sec: 0.85 + 1%|█ | 246/40903 [04:54<13:24:26, 1.19s/it][2025-04-22 17:47:59] (step=0041150) Train Loss: 6.3807, Train Steps/Sec: 0.84 + 1%|█▏ | 271/40903 [05:24<13:17:43, 1.18s/it][2025-04-22 17:48:29] (step=0041175) Train Loss: 6.3954, Train Steps/Sec: 0.84 + 1%|█▎ | 296/40903 [05:54<13:29:15, 1.20s/it][2025-04-22 17:48:59] (step=0041200) Train Loss: 6.3611, Train Steps/Sec: 0.84 + 1%|█▍ | 321/40903 [06:24<13:36:09, 1.21s/it][2025-04-22 17:49:28] (step=0041225) Train Loss: 6.3475, Train Steps/Sec: 0.84 + 1%|█▌ | 346/40903 [06:53<13:20:21, 1.18s/it][2025-04-22 17:49:58] (step=0041250) Train Loss: 6.3249, Train Steps/Sec: 0.84 + 1%|█▋ | 371/40903 [07:23<13:15:43, 1.18s/it][2025-04-22 17:50:28] (step=0041275) Train Loss: 6.3919, Train Steps/Sec: 0.84 + 1%|█▊ | 396/40903 [07:53<13:17:14, 1.18s/it][2025-04-22 17:50:58] (step=0041300) Train Loss: 6.3520, Train Steps/Sec: 0.84 + 1%|█▊ | 421/40903 [08:22<13:29:00, 1.20s/it][2025-04-22 17:51:27] (step=0041325) Train Loss: 6.3128, Train Steps/Sec: 0.84 + 1%|█▉ | 446/40903 [08:52<13:23:09, 1.19s/it][2025-04-22 17:51:57] (step=0041350) Train Loss: 6.3927, Train Steps/Sec: 0.84 + 1%|██ | 471/40903 [09:22<13:15:38, 1.18s/it][2025-04-22 17:52:27] (step=0041375) Train Loss: 6.3694, Train Steps/Sec: 0.84 + 1%|██▏ | 496/40903 [09:52<13:12:18, 1.18s/it][2025-04-22 17:52:57] (step=0041400) Train Loss: 6.3853, Train Steps/Sec: 0.84 + 1%|██▎ | 521/40903 [10:21<13:33:52, 1.21s/it][2025-04-22 17:53:26] (step=0041425) Train Loss: 6.3679, Train Steps/Sec: 0.84 + 1%|██▍ | 546/40903 [10:51<13:17:04, 1.19s/it][2025-04-22 17:53:56] (step=0041450) Train Loss: 6.3664, Train Steps/Sec: 0.85 + 1%|██▌ | 571/40903 [11:21<13:14:55, 1.18s/it][2025-04-22 17:54:25] (step=0041475) Train Loss: 6.3803, Train Steps/Sec: 0.84 + 1%|██▋ | 596/40903 [11:50<13:10:48, 1.18s/it][2025-04-22 17:54:55] (step=0041500) Train Loss: 6.3494, Train Steps/Sec: 0.84 + 2%|██▋ | 621/40903 [12:20<13:37:32, 1.22s/it][2025-04-22 17:55:25] (step=0041525) Train Loss: 6.3470, Train Steps/Sec: 0.85 + 2%|██▊ | 646/40903 [12:49<13:13:24, 1.18s/it][2025-04-22 17:55:54] (step=0041550) Train Loss: 6.3975, Train Steps/Sec: 0.84 + 2%|██▉ | 671/40903 [13:19<13:15:48, 1.19s/it][2025-04-22 17:56:24] (step=0041575) Train Loss: 6.3639, Train Steps/Sec: 0.85 + 2%|███ | 696/40903 [13:49<13:02:40, 1.17s/it][2025-04-22 17:56:54] (step=0041600) Train Loss: 6.3478, Train Steps/Sec: 0.84 + 2%|███▏ | 721/40903 [14:18<13:25:20, 1.20s/it][2025-04-22 17:57:23] (step=0041625) Train Loss: 6.3309, Train Steps/Sec: 0.85 + 2%|███▎ | 746/40903 [14:48<13:01:42, 1.17s/it][2025-04-22 17:57:53] (step=0041650) Train Loss: 6.3345, Train Steps/Sec: 0.85 + 2%|███▍ | 771/40903 [15:17<13:02:54, 1.17s/it][2025-04-22 17:58:22] (step=0041675) Train Loss: 6.3486, Train Steps/Sec: 0.85 + 2%|███▌ | 796/40903 [15:47<13:05:35, 1.18s/it][2025-04-22 17:58:52] (step=0041700) Train Loss: 6.3645, Train Steps/Sec: 0.84 + 2%|███▋ | 821/40903 [16:17<13:21:01, 1.20s/it][2025-04-22 17:59:22] (step=0041725) Train Loss: 6.3475, Train Steps/Sec: 0.84 + 2%|███▋ | 846/40903 [16:46<13:20:58, 1.20s/it][2025-04-22 17:59:51] (step=0041750) Train Loss: 6.3181, Train Steps/Sec: 0.84 + 2%|███▊ | 871/40903 [17:16<13:09:05, 1.18s/it][2025-04-22 18:00:21] (step=0041775) Train Loss: 6.3406, Train Steps/Sec: 0.85 + 2%|███▉ | 896/40903 [17:45<12:58:02, 1.17s/it][2025-04-22 18:00:50] (step=0041800) Train Loss: 6.3266, Train Steps/Sec: 0.84 + 2%|████ | 921/40903 [18:15<13:17:16, 1.20s/it][2025-04-22 18:01:20] (step=0041825) Train Loss: 6.3481, Train Steps/Sec: 0.85 + 2%|████▏ | 946/40903 [18:45<13:10:14, 1.19s/it][2025-04-22 18:01:50] (step=0041850) Train Loss: 6.3935, Train Steps/Sec: 0.83 + 2%|████▎ | 971/40903 [19:15<13:08:22, 1.18s/it][2025-04-22 18:02:20] (step=0041875) Train Loss: 6.3809, Train Steps/Sec: 0.84 + 2%|████▍ | 996/40903 [19:54<12:49:17, 1.16s/it][2025-04-22 18:02:59] (step=0041900) Train Loss: 6.3549, Train Steps/Sec: 0.64 + 2%|████▍ | 1021/40903 [20:23<13:12:31, 1.19s/it][2025-04-22 18:03:28] (step=0041925) Train Loss: 6.3549, Train Steps/Sec: 0.85 + 3%|████▌ | 1046/40903 [20:53<13:04:19, 1.18s/it][2025-04-22 18:03:58] (step=0041950) Train Loss: 6.3666, Train Steps/Sec: 0.84 + 3%|████▋ | 1071/40903 [21:23<13:06:11, 1.18s/it][2025-04-22 18:04:28] (step=0041975) Train Loss: 6.3433, Train Steps/Sec: 0.84 + 3%|████▊ | 1096/40903 [21:53<13:14:45, 1.20s/it][2025-04-22 18:04:58] (step=0042000) Train Loss: 6.3577, Train Steps/Sec: 0.83 +You are using a model of type instructblip to instantiate a model of type blip-2. This is not supported for all configurations of models and can yield errors. +model_name_or_path:/tmp/haozhezhao/model/instructblip-flan-t5-xl +image_place_holder: +Some kwargs in processor config are unused and will not have any effect: num_query_tokens. +tokenizer length after expend 32102 +tokenizer length before expend 32102 +100%|██████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████| 3/3 [02:54<00:00, 58.07s/it] +[2025-04-22 18:09:43] Finish Eval in 42000 steps...███████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████| 3/3 [02:53<00:00, 57.55s/it] +processing left-padding... +processing left-padding... +processing left-padding... +[2025-04-22 18:10:04] Saved checkpoint to checkpoint/BeforeFix_Blip_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_objects/002-GPT-XL/checkpoints/0042000.pt +[2025-04-22 18:10:07] Removed old checkpoint: checkpoint/BeforeFix_Blip_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_objects/002-GPT-XL/checkpoints/0038000.pt + 3%|████▉ | 1121/40903 [27:32<13:27:53, 1.22s/it][2025-04-22 18:10:37] (step=0042025) Train Loss: 6.3218, Train Steps/Sec: 0.07 + 3%|█████ | 1146/40903 [28:11<24:12:55, 2.19s/it][2025-04-22 18:11:16] (step=0042050) Train Loss: 6.3900, Train Steps/Sec: 0.64 + 3%|█████▏ | 1171/40903 [28:41<12:52:21, 1.17s/it][2025-04-22 18:11:45] (step=0042075) Train Loss: 6.3320, Train Steps/Sec: 0.85 + 3%|█████▎ | 1196/40903 [29:10<13:08:08, 1.19s/it][2025-04-22 18:12:15] (step=0042100) Train Loss: 6.3505, Train Steps/Sec: 0.84 + 3%|█████▎ | 1221/40903 [29:40<13:06:31, 1.19s/it][2025-04-22 18:12:45] (step=0042125) Train Loss: 6.3899, Train Steps/Sec: 0.85 + 3%|█████▍ | 1246/40903 [30:09<13:09:54, 1.20s/it][2025-04-22 18:13:14] (step=0042150) Train Loss: 6.3559, Train Steps/Sec: 0.84 + 3%|█████▌ | 1271/40903 [30:39<13:00:45, 1.18s/it][2025-04-22 18:13:44] (step=0042175) Train Loss: 6.3898, Train Steps/Sec: 0.84 + 3%|█████▋ | 1296/40903 [31:09<12:48:18, 1.16s/it][2025-04-22 18:14:14] (step=0042200) Train Loss: 6.3210, Train Steps/Sec: 0.84 + 3%|█████▊ | 1321/40903 [31:38<13:05:27, 1.19s/it][2025-04-22 18:14:43] (step=0042225) Train Loss: 6.3530, Train Steps/Sec: 0.85 + 3%|█████▉ | 1346/40903 [32:08<13:18:31, 1.21s/it][2025-04-22 18:15:13] (step=0042250) Train Loss: 6.3239, Train Steps/Sec: 0.84 + 3%|██████ | 1371/40903 [32:37<12:52:23, 1.17s/it][2025-04-22 18:15:42] (step=0042275) Train Loss: 6.3371, Train Steps/Sec: 0.85 + 3%|██████▏ | 1396/40903 [33:07<12:57:49, 1.18s/it][2025-04-22 18:16:12] (step=0042300) Train Loss: 6.3724, Train Steps/Sec: 0.84 + 3%|██████▎ | 1421/40903 [33:37<13:01:03, 1.19s/it][2025-04-22 18:16:41] (step=0042325) Train Loss: 6.3414, Train Steps/Sec: 0.85 + 4%|██████▎ | 1446/40903 [34:06<13:03:26, 1.19s/it][2025-04-22 18:17:11] (step=0042350) Train Loss: 6.3119, Train Steps/Sec: 0.84 + 4%|██████▍ | 1471/40903 [34:36<13:04:21, 1.19s/it][2025-04-22 18:17:41] (step=0042375) Train Loss: 6.3188, Train Steps/Sec: 0.84 + 4%|██████▌ | 1496/40903 [35:06<12:53:32, 1.18s/it][2025-04-22 18:18:11] (step=0042400) Train Loss: 6.3432, Train Steps/Sec: 0.84 + 4%|██████▋ | 1521/40903 [35:35<13:06:53, 1.20s/it][2025-04-22 18:18:40] (step=0042425) Train Loss: 6.3846, Train Steps/Sec: 0.84 + 4%|██████▊ | 1546/40903 [36:05<12:51:10, 1.18s/it][2025-04-22 18:19:10] (step=0042450) Train Loss: 6.3741, Train Steps/Sec: 0.85 + 4%|██████▉ | 1571/40903 [36:35<12:57:50, 1.19s/it][2025-04-22 18:19:39] (step=0042475) Train Loss: 6.3802, Train Steps/Sec: 0.84 + 4%|███████ | 1596/40903 [37:04<12:48:56, 1.17s/it][2025-04-22 18:20:09] (step=0042500) Train Loss: 6.3461, Train Steps/Sec: 0.84 + 4%|███████▏ | 1621/40903 [37:35<13:10:55, 1.21s/it][2025-04-22 18:20:39] (step=0042525) Train Loss: 6.3232, Train Steps/Sec: 0.83 + 4%|███████▏ | 1646/40903 [38:04<13:07:06, 1.20s/it][2025-04-22 18:21:09] (step=0042550) Train Loss: 6.3675, Train Steps/Sec: 0.84 + 4%|███████▎ | 1671/40903 [38:34<12:51:37, 1.18s/it][2025-04-22 18:21:39] (step=0042575) Train Loss: 6.3698, Train Steps/Sec: 0.84 + 4%|███████▍ | 1696/40903 [39:04<12:42:19, 1.17s/it][2025-04-22 18:22:09] (step=0042600) Train Loss: 6.3545, Train Steps/Sec: 0.83 + 4%|███████▌ | 1721/40903 [39:33<13:07:21, 1.21s/it][2025-04-22 18:22:38] (step=0042625) Train Loss: 6.3332, Train Steps/Sec: 0.84 + 4%|███████▋ | 1746/40903 [40:03<12:58:40, 1.19s/it][2025-04-22 18:23:08] (step=0042650) Train Loss: 6.3095, Train Steps/Sec: 0.85 + 4%|███████▊ | 1771/40903 [40:33<12:51:39, 1.18s/it][2025-04-22 18:23:37] (step=0042675) Train Loss: 6.3860, Train Steps/Sec: 0.84 + 4%|███████▉ | 1796/40903 [41:02<12:48:24, 1.18s/it][2025-04-22 18:24:07] (step=0042700) Train Loss: 6.3606, Train Steps/Sec: 0.84 + 4%|████████ | 1821/40903 [41:32<13:02:27, 1.20s/it][2025-04-22 18:24:37] (step=0042725) Train Loss: 6.3430, Train Steps/Sec: 0.84 + 5%|████████ | 1846/40903 [42:02<12:55:14, 1.19s/it][2025-04-22 18:25:06] (step=0042750) Train Loss: 6.3738, Train Steps/Sec: 0.85 + 5%|████████▏ | 1871/40903 [42:31<12:42:43, 1.17s/it][2025-04-22 18:25:36] (step=0042775) Train Loss: 6.3368, Train Steps/Sec: 0.85 + 5%|████████▎ | 1896/40903 [43:01<12:38:41, 1.17s/it][2025-04-22 18:26:06] (step=0042800) Train Loss: 6.3132, Train Steps/Sec: 0.84 + 5%|████████▍ | 1921/40903 [43:30<13:02:10, 1.20s/it][2025-04-22 18:26:35] (step=0042825) Train Loss: 6.3487, Train Steps/Sec: 0.84 + 5%|████████▌ | 1946/40903 [44:00<12:50:24, 1.19s/it][2025-04-22 18:27:05] (step=0042850) Train Loss: 6.3209, Train Steps/Sec: 0.84 + 5%|████████▋ | 1971/40903 [44:30<12:39:23, 1.17s/it][2025-04-22 18:27:34] (step=0042875) Train Loss: 6.3489, Train Steps/Sec: 0.85 + 5%|████████▊ | 1996/40903 [44:59<12:38:52, 1.17s/it][2025-04-22 18:28:04] (step=0042900) Train Loss: 6.3756, Train Steps/Sec: 0.84 + 5%|████████▉ | 2021/40903 [45:37<13:03:15, 1.21s/it][2025-04-22 18:28:42] (step=0042925) Train Loss: 6.3487, Train Steps/Sec: 0.66 + 5%|█████████ | 2046/40903 [46:07<12:43:23, 1.18s/it][2025-04-22 18:29:12] (step=0042950) Train Loss: 6.3300, Train Steps/Sec: 0.85 + 5%|█████████ | 2071/40903 [46:37<12:43:51, 1.18s/it][2025-04-22 18:29:41] (step=0042975) Train Loss: 6.3248, Train Steps/Sec: 0.85 + 5%|█████████▏ | 2096/40903 [47:06<12:29:55, 1.16s/it][2025-04-22 18:30:11] (step=0043000) Train Loss: 6.3420, Train Steps/Sec: 0.85 + 5%|█████████▎ | 2121/40903 [47:36<13:02:22, 1.21s/it][2025-04-22 18:30:40] (step=0043025) Train Loss: 6.3190, Train Steps/Sec: 0.84 + 5%|█████████▍ | 2146/40903 [48:05<12:48:56, 1.19s/it][2025-04-22 18:31:10] (step=0043050) Train Loss: 6.3629, Train Steps/Sec: 0.84 + 5%|█████████▌ | 2171/40903 [48:35<12:47:24, 1.19s/it][2025-04-22 18:31:40] (step=0043075) Train Loss: 6.3728, Train Steps/Sec: 0.84 + 5%|█████████▋ | 2196/40903 [49:05<12:30:31, 1.16s/it][2025-04-22 18:32:10] (step=0043100) Train Loss: 6.3548, Train Steps/Sec: 0.84 + 5%|█████████▊ | 2221/40903 [49:34<12:48:22, 1.19s/it][2025-04-22 18:32:39] (step=0043125) Train Loss: 6.3330, Train Steps/Sec: 0.84 + 5%|█████████▉ | 2246/40903 [50:04<12:47:49, 1.19s/it][2025-04-22 18:33:09] (step=0043150) Train Loss: 6.3627, Train Steps/Sec: 0.84 + 6%|█████████▉ | 2271/40903 [50:34<12:45:28, 1.19s/it][2025-04-22 18:33:39] (step=0043175) Train Loss: 6.3392, Train Steps/Sec: 0.84 + 6%|██████████ | 2296/40903 [51:03<12:23:14, 1.16s/it][2025-04-22 18:34:08] (step=0043200) Train Loss: 6.3610, Train Steps/Sec: 0.84 + 6%|██████████▏ | 2321/40903 [51:33<12:46:30, 1.19s/it][2025-04-22 18:34:38] (step=0043225) Train Loss: 6.3559, Train Steps/Sec: 0.84 + 6%|██████████▎ | 2346/40903 [52:03<12:43:56, 1.19s/it][2025-04-22 18:35:08] (step=0043250) Train Loss: 6.3404, Train Steps/Sec: 0.85 + 6%|██████████▍ | 2371/40903 [52:32<12:32:44, 1.17s/it][2025-04-22 18:35:37] (step=0043275) Train Loss: 6.3359, Train Steps/Sec: 0.85 + 6%|██████████▌ | 2396/40903 [53:02<12:30:43, 1.17s/it][2025-04-22 18:36:07] (step=0043300) Train Loss: 6.3615, Train Steps/Sec: 0.84 + 6%|██████████▋ | 2421/40903 [53:32<12:46:38, 1.20s/it][2025-04-22 18:36:36] (step=0043325) Train Loss: 6.3419, Train Steps/Sec: 0.85 + 6%|██████████▊ | 2446/40903 [54:01<12:33:47, 1.18s/it][2025-04-22 18:37:06] (step=0043350) Train Loss: 6.3686, Train Steps/Sec: 0.84 + 6%|██████████▊ | 2471/40903 [54:31<12:30:22, 1.17s/it][2025-04-22 18:37:36] (step=0043375) Train Loss: 6.3784, Train Steps/Sec: 0.85 + 6%|██████████▉ | 2496/40903 [55:00<12:26:53, 1.17s/it][2025-04-22 18:38:05] (step=0043400) Train Loss: 6.3240, Train Steps/Sec: 0.84 + 6%|███████████ | 2521/40903 [55:30<12:51:06, 1.21s/it][2025-04-22 18:38:35] (step=0043425) Train Loss: 6.3404, Train Steps/Sec: 0.84 + 6%|███████████▏ | 2546/40903 [56:00<12:41:18, 1.19s/it][2025-04-22 18:39:05] (step=0043450) Train Loss: 6.3422, Train Steps/Sec: 0.85 + 6%|███████████▎ | 2571/40903 [56:29<12:32:00, 1.18s/it][2025-04-22 18:39:34] (step=0043475) Train Loss: 6.3744, Train Steps/Sec: 0.84 + 6%|███████████▍ | 2596/40903 [56:59<12:28:31, 1.17s/it][2025-04-22 18:40:04] (step=0043500) Train Loss: 6.3525, Train Steps/Sec: 0.84 + 6%|███████████▌ | 2621/40903 [57:29<12:49:29, 1.21s/it][2025-04-22 18:40:34] (step=0043525) Train Loss: 6.3850, Train Steps/Sec: 0.84 + 6%|███████████▋ | 2646/40903 [57:58<12:39:11, 1.19s/it][2025-04-22 18:41:03] (step=0043550) Train Loss: 6.3318, Train Steps/Sec: 0.85 + 7%|███████████▊ | 2671/40903 [58:28<12:23:54, 1.17s/it][2025-04-22 18:41:33] (step=0043575) Train Loss: 6.3523, Train Steps/Sec: 0.85 + 7%|███████████▊ | 2696/40903 [58:58<12:28:48, 1.18s/it][2025-04-22 18:42:03] (step=0043600) Train Loss: 6.4006, Train Steps/Sec: 0.84 + 7%|███████████▉ | 2721/40903 [59:27<12:39:23, 1.19s/it][2025-04-22 18:42:32] (step=0043625) Train Loss: 6.3559, Train Steps/Sec: 0.85 + 7%|████████████ | 2746/40903 [59:57<12:25:47, 1.17s/it][2025-04-22 18:43:02] (step=0043650) Train Loss: 6.3132, Train Steps/Sec: 0.85 + 7%|████████████ | 2771/40903 [1:00:26<12:32:37, 1.18s/it][2025-04-22 18:43:31] (step=0043675) Train Loss: 6.3564, Train Steps/Sec: 0.85 + 7%|████████████▏ | 2796/40903 [1:00:56<12:16:24, 1.16s/it][2025-04-22 18:44:01] (step=0043700) Train Loss: 6.3709, Train Steps/Sec: 0.84 + 7%|████████████▎ | 2821/40903 [1:01:26<12:45:10, 1.21s/it][2025-04-22 18:44:31] (step=0043725) Train Loss: 6.3397, Train Steps/Sec: 0.85 + 7%|████████████▍ | 2846/40903 [1:01:55<12:34:10, 1.19s/it][2025-04-22 18:45:00] (step=0043750) Train Loss: 6.3607, Train Steps/Sec: 0.84 + 7%|████████████▍ | 2871/40903 [1:02:25<12:26:10, 1.18s/it][2025-04-22 18:45:30] (step=0043775) Train Loss: 6.3373, Train Steps/Sec: 0.85 + 7%|████████████▌ | 2896/40903 [1:02:54<12:20:18, 1.17s/it][2025-04-22 18:45:59] (step=0043800) Train Loss: 6.3215, Train Steps/Sec: 0.84 + 7%|████████████▋ | 2921/40903 [1:03:24<12:40:35, 1.20s/it][2025-04-22 18:46:29] (step=0043825) Train Loss: 6.3239, Train Steps/Sec: 0.84 + 7%|████████████▊ | 2946/40903 [1:03:54<12:32:59, 1.19s/it][2025-04-22 18:46:59] (step=0043850) Train Loss: 6.3552, Train Steps/Sec: 0.85 + 7%|████████████▉ | 2971/40903 [1:04:24<12:38:16, 1.20s/it][2025-04-22 18:47:28] (step=0043875) Train Loss: 6.3714, Train Steps/Sec: 0.84 + 7%|█████████████ | 2996/40903 [1:04:53<12:22:50, 1.18s/it][2025-04-22 18:47:58] (step=0043900) Train Loss: 6.3881, Train Steps/Sec: 0.84 + 7%|█████████████▏ | 3021/40903 [1:05:23<12:51:02, 1.22s/it][2025-04-22 18:48:28] (step=0043925) Train Loss: 6.3555, Train Steps/Sec: 0.84 + 7%|█████████████▎ | 3046/40903 [1:05:53<12:28:34, 1.19s/it][2025-04-22 18:48:57] (step=0043950) Train Loss: 6.3570, Train Steps/Sec: 0.84 + 8%|█████████████▎ | 3071/40903 [1:06:22<12:17:31, 1.17s/it][2025-04-22 18:49:27] (step=0043975) Train Loss: 6.3241, Train Steps/Sec: 0.85 + 8%|█████████████▍ | 3096/40903 [1:06:52<12:13:26, 1.16s/it][2025-04-22 18:49:57] (step=0044000) Train Loss: 6.3785, Train Steps/Sec: 0.85 +You are using a model of type instructblip to instantiate a model of type blip-2. This is not supported for all configurations of models and can yield errors. +model_name_or_path:/tmp/haozhezhao/model/instructblip-flan-t5-xl +image_place_holder: +Some kwargs in processor config are unused and will not have any effect: num_query_tokens. +tokenizer length after expend 32102 +tokenizer length before expend 32102 +100%|██████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████| 3/3 [02:53<00:00, 57.92s/it] +[2025-04-22 18:54:40] Finish Eval in 44000 steps...███████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████| 3/3 [02:53<00:00, 57.43s/it] +processing left-padding... +processing left-padding... +processing left-padding... +[2025-04-22 18:55:05] Saved checkpoint to checkpoint/BeforeFix_Blip_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_objects/002-GPT-XL/checkpoints/0044000.pt +[2025-04-22 18:55:07] Removed old checkpoint: checkpoint/BeforeFix_Blip_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_objects/002-GPT-XL/checkpoints/0040000.pt + 8%|█████████████▌ | 3121/40903 [1:12:32<12:47:52, 1.22s/it][2025-04-22 18:55:37] (step=0044025) Train Loss: 6.3719, Train Steps/Sec: 0.07 + 8%|█████████████▋ | 3146/40903 [1:13:02<12:27:45, 1.19s/it][2025-04-22 18:56:06] (step=0044050) Train Loss: 6.3331, Train Steps/Sec: 0.84 + 8%|█████████████▊ | 3171/40903 [1:13:31<12:24:09, 1.18s/it][2025-04-22 18:56:36] (step=0044075) Train Loss: 6.3558, Train Steps/Sec: 0.84 + 8%|█████████████▉ | 3196/40903 [1:14:01<12:19:31, 1.18s/it][2025-04-22 18:57:06] (step=0044100) Train Loss: 6.3622, Train Steps/Sec: 0.84 + 8%|██████████████ | 3221/40903 [1:14:31<12:29:47, 1.19s/it][2025-04-22 18:57:35] (step=0044125) Train Loss: 6.3373, Train Steps/Sec: 0.85 + 8%|██████████████▏ | 3246/40903 [1:15:00<12:20:10, 1.18s/it][2025-04-22 18:58:05] (step=0044150) Train Loss: 6.3595, Train Steps/Sec: 0.85 + 8%|██████████████▏ | 3271/40903 [1:15:30<12:08:46, 1.16s/it][2025-04-22 18:58:34] (step=0044175) Train Loss: 6.3452, Train Steps/Sec: 0.85 + 8%|██████████████▎ | 3296/40903 [1:15:59<12:21:09, 1.18s/it][2025-04-22 18:59:04] (step=0044200) Train Loss: 6.3636, Train Steps/Sec: 0.84 + 8%|██████████████▍ | 3321/40903 [1:16:29<12:21:47, 1.18s/it][2025-04-22 18:59:34] (step=0044225) Train Loss: 6.3166, Train Steps/Sec: 0.85 + 8%|██████████████▌ | 3346/40903 [1:16:59<12:22:54, 1.19s/it][2025-04-22 19:00:03] (step=0044250) Train Loss: 6.3685, Train Steps/Sec: 0.84 + 8%|██████████████▋ | 3371/40903 [1:17:28<12:31:25, 1.20s/it][2025-04-22 19:00:33] (step=0044275) Train Loss: 6.3529, Train Steps/Sec: 0.84 + 8%|██████████████▊ | 3396/40903 [1:17:58<12:10:52, 1.17s/it][2025-04-22 19:01:03] (step=0044300) Train Loss: 6.3422, Train Steps/Sec: 0.84 + 8%|██████████████▉ | 3421/40903 [1:18:28<12:27:11, 1.20s/it][2025-04-22 19:01:32] (step=0044325) Train Loss: 6.3758, Train Steps/Sec: 0.85 + 8%|██████████████▉ | 3446/40903 [1:18:57<12:14:08, 1.18s/it][2025-04-22 19:02:02] (step=0044350) Train Loss: 6.3364, Train Steps/Sec: 0.85 + 8%|███████████████ | 3471/40903 [1:19:27<12:12:52, 1.17s/it][2025-04-22 19:02:32] (step=0044375) Train Loss: 6.3635, Train Steps/Sec: 0.85 + 9%|███████████████▏ | 3496/40903 [1:19:57<12:16:45, 1.18s/it][2025-04-22 19:03:02] (step=0044400) Train Loss: 6.3804, Train Steps/Sec: 0.84 + 9%|███████████████▎ | 3521/40903 [1:20:26<12:41:14, 1.22s/it][2025-04-22 19:03:31] (step=0044425) Train Loss: 6.3543, Train Steps/Sec: 0.84 + 9%|███████████████▍ | 3546/40903 [1:20:56<12:15:09, 1.18s/it][2025-04-22 19:04:01] (step=0044450) Train Loss: 6.3399, Train Steps/Sec: 0.85 + 9%|███████████████▌ | 3571/40903 [1:21:26<11:59:26, 1.16s/it][2025-04-22 19:04:30] (step=0044475) Train Loss: 6.3622, Train Steps/Sec: 0.85 + 9%|███████████████▋ | 3596/40903 [1:22:04<12:26:41, 1.20s/it][2025-04-22 19:05:09] (step=0044500) Train Loss: 6.3949, Train Steps/Sec: 0.65 + 9%|███████████████▊ | 3621/40903 [1:22:33<12:30:08, 1.21s/it][2025-04-22 19:05:38] (step=0044525) Train Loss: 6.3622, Train Steps/Sec: 0.85 + 9%|███████████████▊ | 3646/40903 [1:23:03<12:08:39, 1.17s/it][2025-04-22 19:06:08] (step=0044550) Train Loss: 6.3979, Train Steps/Sec: 0.85 + 9%|███████████████▉ | 3671/40903 [1:23:43<12:42:42, 1.23s/it][2025-04-22 19:06:48] (step=0044575) Train Loss: 6.3725, Train Steps/Sec: 0.62 + 9%|████████████████ | 3696/40903 [1:24:12<12:05:39, 1.17s/it][2025-04-22 19:07:17] (step=0044600) Train Loss: 6.3788, Train Steps/Sec: 0.84 + 9%|████████████████▏ | 3721/40903 [1:24:42<12:21:48, 1.20s/it][2025-04-22 19:07:47] (step=0044625) Train Loss: 6.3605, Train Steps/Sec: 0.85 + 9%|████████████████▎ | 3746/40903 [1:25:12<12:08:16, 1.18s/it][2025-04-22 19:08:17] (step=0044650) Train Loss: 6.3315, Train Steps/Sec: 0.85 + 9%|████████████████▍ | 3771/40903 [1:25:41<12:10:31, 1.18s/it][2025-04-22 19:08:46] (step=0044675) Train Loss: 6.3693, Train Steps/Sec: 0.84 + 9%|████████████████▌ | 3796/40903 [1:26:11<12:03:05, 1.17s/it][2025-04-22 19:09:16] (step=0044700) Train Loss: 6.3458, Train Steps/Sec: 0.84 + 9%|████████████████▋ | 3821/40903 [1:26:41<12:31:07, 1.22s/it][2025-04-22 19:09:45] (step=0044725) Train Loss: 6.3447, Train Steps/Sec: 0.85 + 9%|████████████████▋ | 3846/40903 [1:27:10<12:17:51, 1.19s/it][2025-04-22 19:10:15] (step=0044750) Train Loss: 6.3592, Train Steps/Sec: 0.84 + 9%|████████████████▊ | 3871/40903 [1:27:40<11:59:30, 1.17s/it][2025-04-22 19:10:45] (step=0044775) Train Loss: 6.3514, Train Steps/Sec: 0.84 + 10%|████████████████▉ | 3896/40903 [1:28:10<12:03:27, 1.17s/it][2025-04-22 19:11:15] (step=0044800) Train Loss: 6.3412, Train Steps/Sec: 0.84 + 10%|█████████████████ | 3921/40903 [1:28:39<12:19:42, 1.20s/it][2025-04-22 19:11:44] (step=0044825) Train Loss: 6.3542, Train Steps/Sec: 0.85 + 10%|█████████████████▏ | 3946/40903 [1:29:09<12:08:40, 1.18s/it][2025-04-22 19:12:14] (step=0044850) Train Loss: 6.3429, Train Steps/Sec: 0.85 + 10%|█████████████████▎ | 3971/40903 [1:29:39<12:06:50, 1.18s/it][2025-04-22 19:12:43] (step=0044875) Train Loss: 6.3160, Train Steps/Sec: 0.84 + 10%|█████████████████▍ | 3996/40903 [1:30:08<12:10:33, 1.19s/it][2025-04-22 19:13:13] (step=0044900) Train Loss: 6.3419, Train Steps/Sec: 0.84 + 10%|█████████████████▍ | 4021/40903 [1:30:38<12:07:11, 1.18s/it][2025-04-22 19:13:42] (step=0044925) Train Loss: 6.3403, Train Steps/Sec: 0.85 + 10%|█████████████████▌ | 4046/40903 [1:31:07<12:16:12, 1.20s/it][2025-04-22 19:14:12] (step=0044950) Train Loss: 6.3624, Train Steps/Sec: 0.84 + 10%|█████████████████▋ | 4071/40903 [1:31:37<11:53:08, 1.16s/it][2025-04-22 19:14:42] (step=0044975) Train Loss: 6.3636, Train Steps/Sec: 0.85 + 10%|█████████████████▊ | 4096/40903 [1:32:07<11:48:53, 1.16s/it][2025-04-22 19:15:12] (step=0045000) Train Loss: 6.3875, Train Steps/Sec: 0.84 + 10%|█████████████████▉ | 4121/40903 [1:32:36<12:19:29, 1.21s/it][2025-04-22 19:15:41] (step=0045025) Train Loss: 6.4013, Train Steps/Sec: 0.84 + 10%|██████████████████ | 4146/40903 [1:33:06<12:07:00, 1.19s/it][2025-04-22 19:16:11] (step=0045050) Train Loss: 6.3277, Train Steps/Sec: 0.85 + 10%|██████████████████▏ | 4171/40903 [1:33:36<12:12:22, 1.20s/it][2025-04-22 19:16:41] (step=0045075) Train Loss: 6.3806, Train Steps/Sec: 0.84 + 10%|██████████████████▎ | 4196/40903 [1:34:13<12:12:12, 1.20s/it][2025-04-22 19:17:17] (step=0045100) Train Loss: 6.3445, Train Steps/Sec: 0.68 + 10%|██████████████████▎ | 4221/40903 [1:34:42<12:14:01, 1.20s/it][2025-04-22 19:17:47] (step=0045125) Train Loss: 6.3604, Train Steps/Sec: 0.85 + 10%|██████████████████▍ | 4246/40903 [1:35:20<13:14:45, 1.30s/it][2025-04-22 19:18:25] (step=0045150) Train Loss: 6.3661, Train Steps/Sec: 0.66 + 10%|██████████████████▌ | 4271/40903 [1:35:50<11:57:05, 1.17s/it][2025-04-22 19:18:55] (step=0045175) Train Loss: 6.3557, Train Steps/Sec: 0.85 + 11%|██████████████████▋ | 4296/40903 [1:36:19<12:03:43, 1.19s/it][2025-04-22 19:19:24] (step=0045200) Train Loss: 6.3451, Train Steps/Sec: 0.84 + 11%|██████████████████▊ | 4321/40903 [1:36:49<12:11:33, 1.20s/it][2025-04-22 19:19:54] (step=0045225) Train Loss: 6.3767, Train Steps/Sec: 0.85 + 11%|██████████████████▉ | 4346/40903 [1:37:19<11:57:08, 1.18s/it][2025-04-22 19:20:24] (step=0045250) Train Loss: 6.3571, Train Steps/Sec: 0.84 + 11%|███████████████████ | 4371/40903 [1:37:48<11:57:06, 1.18s/it][2025-04-22 19:20:53] (step=0045275) Train Loss: 6.3243, Train Steps/Sec: 0.84 + 11%|███████████████████▏ | 4396/40903 [1:38:18<11:54:04, 1.17s/it][2025-04-22 19:21:23] (step=0045300) Train Loss: 6.3537, Train Steps/Sec: 0.84 + 11%|███████████████████▏ | 4421/40903 [1:38:48<12:18:53, 1.22s/it][2025-04-22 19:21:53] (step=0045325) Train Loss: 6.3403, Train Steps/Sec: 0.84 + 11%|███████████████████▎ | 4446/40903 [1:39:17<12:09:18, 1.20s/it][2025-04-22 19:22:22] (step=0045350) Train Loss: 6.3813, Train Steps/Sec: 0.84 + 11%|███████████████████▍ | 4471/40903 [1:39:47<11:59:15, 1.18s/it][2025-04-22 19:22:52] (step=0045375) Train Loss: 6.3539, Train Steps/Sec: 0.85 + 11%|███████████████████▌ | 4496/40903 [1:40:16<11:53:55, 1.18s/it][2025-04-22 19:23:21] (step=0045400) Train Loss: 6.3089, Train Steps/Sec: 0.85 + 11%|███████████████████▋ | 4521/40903 [1:40:46<11:58:36, 1.19s/it][2025-04-22 19:23:51] (step=0045425) Train Loss: 6.3437, Train Steps/Sec: 0.85 + 11%|███████████████████▊ | 4546/40903 [1:41:16<11:56:05, 1.18s/it][2025-04-22 19:24:20] (step=0045450) Train Loss: 6.3604, Train Steps/Sec: 0.84 + 11%|███████████████████▉ | 4571/40903 [1:41:45<11:58:35, 1.19s/it][2025-04-22 19:24:50] (step=0045475) Train Loss: 6.3767, Train Steps/Sec: 0.84 + 11%|████████████████████ | 4596/40903 [1:42:15<11:45:24, 1.17s/it][2025-04-22 19:25:20] (step=0045500) Train Loss: 6.3741, Train Steps/Sec: 0.84 + 11%|████████████████████ | 4621/40903 [1:42:44<12:07:01, 1.20s/it][2025-04-22 19:25:49] (step=0045525) Train Loss: 6.3454, Train Steps/Sec: 0.85 + 11%|████████████████████▏ | 4646/40903 [1:43:14<12:00:50, 1.19s/it][2025-04-22 19:26:19] (step=0045550) Train Loss: 6.3384, Train Steps/Sec: 0.84 + 11%|████████████████████▎ | 4671/40903 [1:43:44<11:56:53, 1.19s/it][2025-04-22 19:26:49] (step=0045575) Train Loss: 6.3654, Train Steps/Sec: 0.84 + 11%|████████████████████▍ | 4696/40903 [1:44:13<11:48:49, 1.17s/it][2025-04-22 19:27:18] (step=0045600) Train Loss: 6.3539, Train Steps/Sec: 0.84 + 12%|████████████████████▌ | 4721/40903 [1:44:43<12:09:27, 1.21s/it][2025-04-22 19:27:48] (step=0045625) Train Loss: 6.3372, Train Steps/Sec: 0.85 + 12%|████████████████████▋ | 4746/40903 [1:45:13<11:58:34, 1.19s/it][2025-04-22 19:28:18] (step=0045650) Train Loss: 6.3221, Train Steps/Sec: 0.84 + 12%|████████████████████▊ | 4771/40903 [1:45:42<11:43:11, 1.17s/it][2025-04-22 19:28:47] (step=0045675) Train Loss: 6.3890, Train Steps/Sec: 0.84 + 12%|████████████████████▊ | 4796/40903 [1:46:12<11:40:45, 1.16s/it][2025-04-22 19:29:17] (step=0045700) Train Loss: 6.3835, Train Steps/Sec: 0.84 + 12%|████████████████████▉ | 4821/40903 [1:46:41<12:04:08, 1.20s/it][2025-04-22 19:29:46] (step=0045725) Train Loss: 6.3830, Train Steps/Sec: 0.85 + 12%|█████████████████████ | 4846/40903 [1:47:11<11:49:25, 1.18s/it][2025-04-22 19:30:16] (step=0045750) Train Loss: 6.3830, Train Steps/Sec: 0.85 + 12%|█████████████████████▏ | 4871/40903 [1:47:40<11:51:05, 1.18s/it][2025-04-22 19:30:45] (step=0045775) Train Loss: 6.3653, Train Steps/Sec: 0.84 + 12%|█████████████████████▎ | 4896/40903 [1:48:10<11:47:40, 1.18s/it][2025-04-22 19:31:15] (step=0045800) Train Loss: 6.3748, Train Steps/Sec: 0.84 + 12%|█████████████████████▍ | 4921/40903 [1:48:40<11:57:28, 1.20s/it][2025-04-22 19:31:45] (step=0045825) Train Loss: 6.3517, Train Steps/Sec: 0.84 + 12%|█████████████████████▌ | 4946/40903 [1:49:10<11:41:30, 1.17s/it][2025-04-22 19:32:14] (step=0045850) Train Loss: 6.3520, Train Steps/Sec: 0.84 + 12%|█████████████████████▋ | 4971/40903 [1:49:39<11:50:27, 1.19s/it][2025-04-22 19:32:44] (step=0045875) Train Loss: 6.3585, Train Steps/Sec: 0.84 + 12%|█████████████████████▋ | 4996/40903 [1:50:09<11:49:09, 1.18s/it][2025-04-22 19:33:14] (step=0045900) Train Loss: 6.3187, Train Steps/Sec: 0.84 + 12%|█████████████████████▊ | 5021/40903 [1:50:39<11:57:07, 1.20s/it][2025-04-22 19:33:43] (step=0045925) Train Loss: 6.3279, Train Steps/Sec: 0.85 + 12%|█████████████████████▉ | 5046/40903 [1:51:08<11:53:10, 1.19s/it][2025-04-22 19:34:13] (step=0045950) Train Loss: 6.3870, Train Steps/Sec: 0.84 + 12%|██████████████████████ | 5071/40903 [1:51:38<11:46:44, 1.18s/it][2025-04-22 19:34:43] (step=0045975) Train Loss: 6.3266, Train Steps/Sec: 0.85 + 12%|██████████████████████▏ | 5096/40903 [1:52:07<11:39:00, 1.17s/it][2025-04-22 19:35:12] (step=0046000) Train Loss: 6.3457, Train Steps/Sec: 0.85 +You are using a model of type instructblip to instantiate a model of type blip-2. This is not supported for all configurations of models and can yield errors. +model_name_or_path:/tmp/haozhezhao/model/instructblip-flan-t5-xl +image_place_holder: +Some kwargs in processor config are unused and will not have any effect: num_query_tokens. +tokenizer length after expend 32102 +tokenizer length before expend 32102 +100%|██████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████| 3/3 [02:54<00:00, 58.29s/it] +[2025-04-22 19:39:56] Finish Eval in 46000 steps...███████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████| 3/3 [02:54<00:00, 57.74s/it] +processing left-padding... +processing left-padding... +processing left-padding... +[2025-04-22 19:40:18] Saved checkpoint to checkpoint/BeforeFix_Blip_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_objects/002-GPT-XL/checkpoints/0046000.pt +[2025-04-22 19:40:20] Removed old checkpoint: checkpoint/BeforeFix_Blip_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_objects/002-GPT-XL/checkpoints/0042000.pt + 13%|██████████████████████▎ | 5121/40903 [1:57:45<12:13:41, 1.23s/it][2025-04-22 19:40:50] (step=0046025) Train Loss: 6.3824, Train Steps/Sec: 0.07 + 13%|██████████████████████▍ | 5146/40903 [1:58:15<11:45:47, 1.18s/it][2025-04-22 19:41:20] (step=0046050) Train Loss: 6.3466, Train Steps/Sec: 0.84 + 13%|██████████████████████▌ | 5171/40903 [1:58:44<11:37:54, 1.17s/it][2025-04-22 19:41:49] (step=0046075) Train Loss: 6.3299, Train Steps/Sec: 0.84 + 13%|██████████████████████▌ | 5196/40903 [1:59:14<11:40:16, 1.18s/it][2025-04-22 19:42:19] (step=0046100) Train Loss: 6.3974, Train Steps/Sec: 0.84 + 13%|██████████████████████▋ | 5221/40903 [1:59:44<11:55:22, 1.20s/it][2025-04-22 19:42:49] (step=0046125) Train Loss: 6.3639, Train Steps/Sec: 0.84 + 13%|██████████████████████▊ | 5246/40903 [2:00:13<11:45:43, 1.19s/it][2025-04-22 19:43:18] (step=0046150) Train Loss: 6.3204, Train Steps/Sec: 0.85 + 13%|██████████████████████▉ | 5271/40903 [2:00:43<11:39:10, 1.18s/it][2025-04-22 19:43:48] (step=0046175) Train Loss: 6.3494, Train Steps/Sec: 0.85 + 13%|███████████████████████ | 5296/40903 [2:01:13<11:43:16, 1.19s/it][2025-04-22 19:44:18] (step=0046200) Train Loss: 6.3853, Train Steps/Sec: 0.83 + 13%|███████████████████████▏ | 5321/40903 [2:01:43<11:48:24, 1.19s/it][2025-04-22 19:44:47] (step=0046225) Train Loss: 6.2802, Train Steps/Sec: 0.85 + 13%|███████████████████████▎ | 5346/40903 [2:02:12<11:38:23, 1.18s/it][2025-04-22 19:45:17] (step=0046250) Train Loss: 6.3295, Train Steps/Sec: 0.85 + 13%|███████████████████████▎ | 5371/40903 [2:02:42<11:37:52, 1.18s/it][2025-04-22 19:45:47] (step=0046275) Train Loss: 6.3649, Train Steps/Sec: 0.84 + 13%|███████████████████████▍ | 5396/40903 [2:03:11<11:31:31, 1.17s/it][2025-04-22 19:46:16] (step=0046300) Train Loss: 6.3629, Train Steps/Sec: 0.85 + 13%|███████████████████████▌ | 5421/40903 [2:03:41<11:47:00, 1.20s/it][2025-04-22 19:46:46] (step=0046325) Train Loss: 6.3137, Train Steps/Sec: 0.85 + 13%|███████████████████████▋ | 5446/40903 [2:04:11<11:39:35, 1.18s/it][2025-04-22 19:47:16] (step=0046350) Train Loss: 6.3457, Train Steps/Sec: 0.85 + 13%|███████████████████████▊ | 5471/40903 [2:04:40<11:34:31, 1.18s/it][2025-04-22 19:47:45] (step=0046375) Train Loss: 6.4115, Train Steps/Sec: 0.84 + 13%|███████████████████████▉ | 5496/40903 [2:05:10<11:32:57, 1.17s/it][2025-04-22 19:48:15] (step=0046400) Train Loss: 6.3853, Train Steps/Sec: 0.84 + 13%|████████████████████████ | 5521/40903 [2:05:40<11:55:08, 1.21s/it][2025-04-22 19:48:45] (step=0046425) Train Loss: 6.3232, Train Steps/Sec: 0.84 + 14%|████████████████████████▏ | 5546/40903 [2:06:09<11:42:40, 1.19s/it][2025-04-22 19:49:14] (step=0046450) Train Loss: 6.3830, Train Steps/Sec: 0.85 + 14%|████████████████████████▏ | 5571/40903 [2:06:39<11:37:02, 1.18s/it][2025-04-22 19:49:44] (step=0046475) Train Loss: 6.3672, Train Steps/Sec: 0.84 + 14%|████████████████████████▎ | 5596/40903 [2:07:09<11:32:11, 1.18s/it][2025-04-22 19:50:14] (step=0046500) Train Loss: 6.3896, Train Steps/Sec: 0.84 + 14%|████████████████████████▍ | 5621/40903 [2:07:39<11:55:29, 1.22s/it][2025-04-22 19:50:43] (step=0046525) Train Loss: 6.3865, Train Steps/Sec: 0.84 + 14%|████████████████████████▌ | 5646/40903 [2:08:08<11:41:31, 1.19s/it][2025-04-22 19:51:13] (step=0046550) Train Loss: 6.3044, Train Steps/Sec: 0.85 + 14%|████████████████████████▋ | 5671/40903 [2:08:38<11:35:29, 1.18s/it][2025-04-22 19:51:43] (step=0046575) Train Loss: 6.4009, Train Steps/Sec: 0.84 + 14%|████████████████████████▊ | 5696/40903 [2:09:07<11:35:30, 1.19s/it][2025-04-22 19:52:12] (step=0046600) Train Loss: 6.3826, Train Steps/Sec: 0.84 + 14%|████████████████████████▉ | 5721/40903 [2:09:37<11:50:00, 1.21s/it][2025-04-22 19:52:42] (step=0046625) Train Loss: 6.3683, Train Steps/Sec: 0.84 + 14%|█████████████████████████ | 5746/40903 [2:10:07<11:42:59, 1.20s/it][2025-04-22 19:53:12] (step=0046650) Train Loss: 6.3966, Train Steps/Sec: 0.84 + 14%|█████████████████████████ | 5771/40903 [2:10:37<11:31:48, 1.18s/it][2025-04-22 19:53:41] (step=0046675) Train Loss: 6.3603, Train Steps/Sec: 0.85 + 14%|█████████████████████████▏ | 5796/40903 [2:11:06<11:30:58, 1.18s/it][2025-04-22 19:54:11] (step=0046700) Train Loss: 6.3428, Train Steps/Sec: 0.84 + 14%|█████████████████████████▎ | 5821/40903 [2:11:36<11:39:13, 1.20s/it][2025-04-22 19:54:41] (step=0046725) Train Loss: 6.3170, Train Steps/Sec: 0.85 + 14%|█████████████████████████▍ | 5846/40903 [2:12:06<11:42:32, 1.20s/it][2025-04-22 19:55:10] (step=0046750) Train Loss: 6.3661, Train Steps/Sec: 0.84 + 14%|█████████████████████████▌ | 5871/40903 [2:12:35<11:27:34, 1.18s/it][2025-04-22 19:55:40] (step=0046775) Train Loss: 6.3155, Train Steps/Sec: 0.85 + 14%|█████████████████████████▋ | 5896/40903 [2:13:05<11:18:03, 1.16s/it][2025-04-22 19:56:10] (step=0046800) Train Loss: 6.3424, Train Steps/Sec: 0.84 + 14%|█████████████████████████▊ | 5921/40903 [2:13:34<11:46:51, 1.21s/it][2025-04-22 19:56:39] (step=0046825) Train Loss: 6.3535, Train Steps/Sec: 0.85 + 15%|█████████████████████████▉ | 5946/40903 [2:14:04<11:32:04, 1.19s/it][2025-04-22 19:57:09] (step=0046850) Train Loss: 6.3466, Train Steps/Sec: 0.84 + 15%|█████████████████████████▉ | 5971/40903 [2:14:34<11:30:58, 1.19s/it][2025-04-22 19:57:38] (step=0046875) Train Loss: 6.3634, Train Steps/Sec: 0.84 + 15%|██████████████████████████ | 5996/40903 [2:15:03<11:22:02, 1.17s/it][2025-04-22 19:58:08] (step=0046900) Train Loss: 6.3500, Train Steps/Sec: 0.84 + 15%|██████████████████████████▏ | 6021/40903 [2:15:33<11:29:19, 1.19s/it][2025-04-22 19:58:37] (step=0046925) Train Loss: 6.3254, Train Steps/Sec: 0.85 + 15%|██████████████████████████▎ | 6046/40903 [2:16:02<11:32:11, 1.19s/it][2025-04-22 19:59:07] (step=0046950) Train Loss: 6.3102, Train Steps/Sec: 0.84 + 15%|██████████████████████████▍ | 6071/40903 [2:16:32<11:20:57, 1.17s/it][2025-04-22 19:59:37] (step=0046975) Train Loss: 6.3462, Train Steps/Sec: 0.84 + 15%|██████████████████████████▌ | 6096/40903 [2:17:02<11:22:04, 1.18s/it][2025-04-22 20:00:07] (step=0047000) Train Loss: 6.3647, Train Steps/Sec: 0.84 + 15%|██████████████████████████▋ | 6121/40903 [2:17:31<11:39:26, 1.21s/it][2025-04-22 20:00:36] (step=0047025) Train Loss: 6.3989, Train Steps/Sec: 0.84 + 15%|██████████████████████████▋ | 6146/40903 [2:18:01<11:32:01, 1.19s/it][2025-04-22 20:01:06] (step=0047050) Train Loss: 6.3557, Train Steps/Sec: 0.85 + 15%|██████████████████████████▊ | 6171/40903 [2:18:30<11:26:13, 1.19s/it][2025-04-22 20:01:35] (step=0047075) Train Loss: 6.3641, Train Steps/Sec: 0.85 + 15%|██████████████████████████▉ | 6196/40903 [2:19:00<11:20:34, 1.18s/it][2025-04-22 20:02:05] (step=0047100) Train Loss: 6.3248, Train Steps/Sec: 0.84 + 15%|███████████████████████████ | 6221/40903 [2:19:30<11:34:14, 1.20s/it][2025-04-22 20:02:35] (step=0047125) Train Loss: 6.3831, Train Steps/Sec: 0.84 + 15%|███████████████████████████▏ | 6246/40903 [2:19:59<11:20:39, 1.18s/it][2025-04-22 20:03:04] (step=0047150) Train Loss: 6.3530, Train Steps/Sec: 0.85 + 15%|███████████████████████████▎ | 6271/40903 [2:20:29<11:26:15, 1.19s/it][2025-04-22 20:03:34] (step=0047175) Train Loss: 6.3859, Train Steps/Sec: 0.84 + 15%|███████████████████████████▍ | 6296/40903 [2:20:59<11:20:13, 1.18s/it][2025-04-22 20:04:04] (step=0047200) Train Loss: 6.3375, Train Steps/Sec: 0.84 + 15%|███████████████████████████▌ | 6321/40903 [2:21:28<11:26:39, 1.19s/it][2025-04-22 20:04:33] (step=0047225) Train Loss: 6.3409, Train Steps/Sec: 0.85 + 16%|███████████████████████████▌ | 6346/40903 [2:21:58<11:24:39, 1.19s/it][2025-04-22 20:05:03] (step=0047250) Train Loss: 6.3653, Train Steps/Sec: 0.85 + 16%|███████████████████████████▋ | 6371/40903 [2:22:27<11:09:44, 1.16s/it][2025-04-22 20:05:32] (step=0047275) Train Loss: 6.3482, Train Steps/Sec: 0.85 + 16%|███████████████████████████▊ | 6396/40903 [2:22:57<11:10:15, 1.17s/it][2025-04-22 20:06:02] (step=0047300) Train Loss: 6.3749, Train Steps/Sec: 0.84 + 16%|███████████████████████████▉ | 6421/40903 [2:23:34<11:44:00, 1.22s/it][2025-04-22 20:06:39] (step=0047325) Train Loss: 6.3663, Train Steps/Sec: 0.68 + 16%|████████████████████████████ | 6446/40903 [2:24:03<11:20:01, 1.18s/it][2025-04-22 20:07:08] (step=0047350) Train Loss: 6.3200, Train Steps/Sec: 0.84 + 16%|████████████████████████████▏ | 6471/40903 [2:24:33<11:20:53, 1.19s/it][2025-04-22 20:07:38] (step=0047375) Train Loss: 6.3687, Train Steps/Sec: 0.84 + 16%|████████████████████████████▎ | 6496/40903 [2:25:03<11:12:42, 1.17s/it][2025-04-22 20:08:08] (step=0047400) Train Loss: 6.3628, Train Steps/Sec: 0.84 + 16%|████████████████████████████▍ | 6521/40903 [2:25:32<11:25:27, 1.20s/it][2025-04-22 20:08:37] (step=0047425) Train Loss: 6.3957, Train Steps/Sec: 0.84 + 16%|████████████████████████████▍ | 6546/40903 [2:26:02<11:09:48, 1.17s/it][2025-04-22 20:09:07] (step=0047450) Train Loss: 6.3659, Train Steps/Sec: 0.85 + 16%|████████████████████████████▌ | 6571/40903 [2:26:31<11:13:04, 1.18s/it][2025-04-22 20:09:36] (step=0047475) Train Loss: 6.3441, Train Steps/Sec: 0.85 + 16%|████████████████████████████▋ | 6596/40903 [2:27:01<11:11:20, 1.17s/it][2025-04-22 20:10:06] (step=0047500) Train Loss: 6.3852, Train Steps/Sec: 0.84 + 16%|████████████████████████████▊ | 6621/40903 [2:27:39<11:29:16, 1.21s/it][2025-04-22 20:10:43] (step=0047525) Train Loss: 6.3659, Train Steps/Sec: 0.67 + 16%|████████████████████████████▉ | 6646/40903 [2:28:08<11:27:18, 1.20s/it][2025-04-22 20:11:13] (step=0047550) Train Loss: 6.3765, Train Steps/Sec: 0.84 + 16%|█████████████████████████████ | 6671/40903 [2:28:38<11:09:21, 1.17s/it][2025-04-22 20:11:42] (step=0047575) Train Loss: 6.3769, Train Steps/Sec: 0.85 + 16%|█████████████████████████████▏ | 6696/40903 [2:29:07<11:13:32, 1.18s/it][2025-04-22 20:12:12] (step=0047600) Train Loss: 6.3858, Train Steps/Sec: 0.83 + 16%|█████████████████████████████▏ | 6721/40903 [2:29:37<11:23:09, 1.20s/it][2025-04-22 20:12:42] (step=0047625) Train Loss: 6.3407, Train Steps/Sec: 0.85 + 16%|█████████████████████████████▎ | 6746/40903 [2:30:07<11:13:14, 1.18s/it][2025-04-22 20:13:12] (step=0047650) Train Loss: 6.3726, Train Steps/Sec: 0.84 + 17%|█████████████████████████████▍ | 6771/40903 [2:30:36<11:12:49, 1.18s/it][2025-04-22 20:13:41] (step=0047675) Train Loss: 6.3700, Train Steps/Sec: 0.84 + 17%|█████████████████████████████▌ | 6796/40903 [2:31:06<11:03:10, 1.17s/it][2025-04-22 20:14:11] (step=0047700) Train Loss: 6.3594, Train Steps/Sec: 0.84 + 17%|█████████████████████████████▋ | 6821/40903 [2:31:36<11:25:50, 1.21s/it][2025-04-22 20:14:41] (step=0047725) Train Loss: 6.3703, Train Steps/Sec: 0.84 + 17%|█████████████████████████████▊ | 6846/40903 [2:32:05<11:13:25, 1.19s/it][2025-04-22 20:15:10] (step=0047750) Train Loss: 6.3487, Train Steps/Sec: 0.84 + 17%|█████████████████████████████▉ | 6871/40903 [2:32:35<11:06:36, 1.18s/it][2025-04-22 20:15:40] (step=0047775) Train Loss: 6.3124, Train Steps/Sec: 0.85 + 17%|██████████████████████████████ | 6896/40903 [2:33:05<11:06:01, 1.18s/it][2025-04-22 20:16:10] (step=0047800) Train Loss: 6.3493, Train Steps/Sec: 0.84 + 17%|██████████████████████████████ | 6921/40903 [2:33:34<11:15:23, 1.19s/it][2025-04-22 20:16:39] (step=0047825) Train Loss: 6.3561, Train Steps/Sec: 0.85 + 17%|██████████████████████████████▏ | 6946/40903 [2:34:04<11:07:56, 1.18s/it][2025-04-22 20:17:09] (step=0047850) Train Loss: 6.3511, Train Steps/Sec: 0.84 + 17%|██████████████████████████████▎ | 6971/40903 [2:34:34<11:06:55, 1.18s/it][2025-04-22 20:17:38] (step=0047875) Train Loss: 6.3614, Train Steps/Sec: 0.84 + 17%|██████████████████████████████▍ | 6996/40903 [2:35:03<11:01:17, 1.17s/it][2025-04-22 20:18:08] (step=0047900) Train Loss: 6.3660, Train Steps/Sec: 0.84 + 17%|██████████████████████████████▌ | 7021/40903 [2:35:33<11:19:29, 1.20s/it][2025-04-22 20:18:38] (step=0047925) Train Loss: 6.3380, Train Steps/Sec: 0.85 + 17%|██████████████████████████████▋ | 7046/40903 [2:36:02<11:03:39, 1.18s/it][2025-04-22 20:19:07] (step=0047950) Train Loss: 6.3669, Train Steps/Sec: 0.84 + 17%|██████████████████████████████▊ | 7071/40903 [2:36:32<11:09:58, 1.19s/it][2025-04-22 20:19:37] (step=0047975) Train Loss: 6.3681, Train Steps/Sec: 0.84 + 17%|██████████████████████████████▉ | 7096/40903 [2:37:02<11:02:14, 1.18s/it][2025-04-22 20:20:07] (step=0048000) Train Loss: 6.3427, Train Steps/Sec: 0.84 +You are using a model of type instructblip to instantiate a model of type blip-2. This is not supported for all configurations of models and can yield errors. +model_name_or_path:/tmp/haozhezhao/model/instructblip-flan-t5-xl +image_place_holder: +Some kwargs in processor config are unused and will not have any effect: num_query_tokens. +tokenizer length after expend 32102 +tokenizer length before expend 32102 +100%|██████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████| 3/3 [02:54<00:00, 58.08s/it] +[2025-04-22 20:24:50] Finish Eval in 48000 steps...███████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████| 3/3 [02:53<00:00, 57.59s/it] +processing left-padding... +processing left-padding... +processing left-padding... +[2025-04-22 20:25:12] Saved checkpoint to checkpoint/BeforeFix_Blip_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_objects/002-GPT-XL/checkpoints/0048000.pt +[2025-04-22 20:25:14] Removed old checkpoint: checkpoint/BeforeFix_Blip_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_objects/002-GPT-XL/checkpoints/0044000.pt + 17%|██████████████████████████████▉ | 7121/40903 [2:42:39<11:39:20, 1.24s/it][2025-04-22 20:25:44] (step=0048025) Train Loss: 6.3675, Train Steps/Sec: 0.07 + 17%|███████████████████████████████ | 7146/40903 [2:43:09<11:12:56, 1.20s/it][2025-04-22 20:26:14] (step=0048050) Train Loss: 6.4020, Train Steps/Sec: 0.84 + 18%|███████████████████████████████▏ | 7171/40903 [2:43:39<11:01:10, 1.18s/it][2025-04-22 20:26:43] (step=0048075) Train Loss: 6.3341, Train Steps/Sec: 0.84 + 18%|███████████████████████████████▎ | 7196/40903 [2:44:08<10:51:11, 1.16s/it][2025-04-22 20:27:13] (step=0048100) Train Loss: 6.3328, Train Steps/Sec: 0.85 + 18%|███████████████████████████████▍ | 7221/40903 [2:44:38<11:20:09, 1.21s/it][2025-04-22 20:27:43] (step=0048125) Train Loss: 6.3608, Train Steps/Sec: 0.85 + 18%|███████████████████████████████▌ | 7246/40903 [2:45:07<11:12:04, 1.20s/it][2025-04-22 20:28:12] (step=0048150) Train Loss: 6.3852, Train Steps/Sec: 0.84 + 18%|███████████████████████████████▋ | 7271/40903 [2:45:37<11:02:19, 1.18s/it][2025-04-22 20:28:42] (step=0048175) Train Loss: 6.4194, Train Steps/Sec: 0.84 + 18%|███████████████████████████████▊ | 7296/40903 [2:46:07<10:56:23, 1.17s/it][2025-04-22 20:29:12] (step=0048200) Train Loss: 6.3544, Train Steps/Sec: 0.85 + 18%|███████████████████████████████▊ | 7321/40903 [2:46:36<11:07:04, 1.19s/it][2025-04-22 20:29:41] (step=0048225) Train Loss: 6.3848, Train Steps/Sec: 0.85 + 18%|███████████████████████████████▉ | 7346/40903 [2:47:12<11:47:27, 1.26s/it][2025-04-22 20:30:17] (step=0048250) Train Loss: 6.3638, Train Steps/Sec: 0.70 + 18%|████████████████████████████████ | 7371/40903 [2:47:42<10:59:51, 1.18s/it][2025-04-22 20:30:47] (step=0048275) Train Loss: 6.3962, Train Steps/Sec: 0.84 + 18%|████████████████████████████████▏ | 7396/40903 [2:48:11<10:56:01, 1.17s/it][2025-04-22 20:31:16] (step=0048300) Train Loss: 6.3620, Train Steps/Sec: 0.84 + 18%|████████████████████████████████▎ | 7421/40903 [2:48:41<11:11:02, 1.20s/it][2025-04-22 20:31:46] (step=0048325) Train Loss: 6.3205, Train Steps/Sec: 0.84 + 18%|████████████████████████████████▍ | 7446/40903 [2:49:11<11:09:48, 1.20s/it][2025-04-22 20:32:16] (step=0048350) Train Loss: 6.3509, Train Steps/Sec: 0.84 + 18%|████████████████████████████████▌ | 7471/40903 [2:49:40<10:53:30, 1.17s/it][2025-04-22 20:32:45] (step=0048375) Train Loss: 6.3693, Train Steps/Sec: 0.85 + 18%|████████████████████████████████▌ | 7496/40903 [2:50:10<10:49:58, 1.17s/it][2025-04-22 20:33:15] (step=0048400) Train Loss: 6.3715, Train Steps/Sec: 0.84 + 18%|████████████████████████████████▋ | 7521/40903 [2:50:40<11:01:11, 1.19s/it][2025-04-22 20:33:44] (step=0048425) Train Loss: 6.3452, Train Steps/Sec: 0.85 + 18%|████████████████████████████████▊ | 7546/40903 [2:51:09<11:04:02, 1.19s/it][2025-04-22 20:34:14] (step=0048450) Train Loss: 6.3810, Train Steps/Sec: 0.84 + 19%|████████████████████████████████▉ | 7571/40903 [2:51:39<10:56:39, 1.18s/it][2025-04-22 20:34:44] (step=0048475) Train Loss: 6.3666, Train Steps/Sec: 0.84 + 19%|█████████████████████████████████ | 7596/40903 [2:52:08<10:50:07, 1.17s/it][2025-04-22 20:35:13] (step=0048500) Train Loss: 6.3337, Train Steps/Sec: 0.84 + 19%|█████████████████████████████████▏ | 7621/40903 [2:52:38<11:03:08, 1.20s/it][2025-04-22 20:35:43] (step=0048525) Train Loss: 6.3847, Train Steps/Sec: 0.85 + 19%|█████████████████████████████████▎ | 7646/40903 [2:53:07<10:52:58, 1.18s/it][2025-04-22 20:36:12] (step=0048550) Train Loss: 6.3342, Train Steps/Sec: 0.85 + 19%|█████████████████████████████████▍ | 7671/40903 [2:53:37<10:46:56, 1.17s/it][2025-04-22 20:36:42] (step=0048575) Train Loss: 6.3634, Train Steps/Sec: 0.85 + 19%|█████████████████████████████████▍ | 7696/40903 [2:54:06<10:56:07, 1.19s/it][2025-04-22 20:37:11] (step=0048600) Train Loss: 6.3193, Train Steps/Sec: 0.84 + 19%|█████████████████████████████████▌ | 7721/40903 [2:54:36<11:04:06, 1.20s/it][2025-04-22 20:37:41] (step=0048625) Train Loss: 6.3557, Train Steps/Sec: 0.84 + 19%|█████████████████████████████████▋ | 7746/40903 [2:55:06<10:49:21, 1.18s/it][2025-04-22 20:38:11] (step=0048650) Train Loss: 6.3800, Train Steps/Sec: 0.84 + 19%|█████████████████████████████████▊ | 7771/40903 [2:55:35<11:00:18, 1.20s/it][2025-04-22 20:38:40] (step=0048675) Train Loss: 6.3499, Train Steps/Sec: 0.85 + 19%|█████████████████████████████████▉ | 7796/40903 [2:56:05<10:48:26, 1.18s/it][2025-04-22 20:39:10] (step=0048700) Train Loss: 6.3624, Train Steps/Sec: 0.84 + 19%|██████████████████████████████████ | 7821/40903 [2:56:35<11:04:36, 1.21s/it][2025-04-22 20:39:40] (step=0048725) Train Loss: 6.3269, Train Steps/Sec: 0.84 + 19%|██████████████████████████████████▏ | 7846/40903 [2:57:04<10:50:00, 1.18s/it][2025-04-22 20:40:09] (step=0048750) Train Loss: 6.3889, Train Steps/Sec: 0.85 + 19%|██████████████████████████████████▎ | 7871/40903 [2:57:34<10:47:21, 1.18s/it][2025-04-22 20:40:39] (step=0048775) Train Loss: 6.3376, Train Steps/Sec: 0.84 + 19%|██████████████████████████████████▎ | 7896/40903 [2:58:04<10:47:07, 1.18s/it][2025-04-22 20:41:09] (step=0048800) Train Loss: 6.3652, Train Steps/Sec: 0.84 + 19%|██████████████████████████████████▍ | 7921/40903 [2:58:33<10:59:04, 1.20s/it][2025-04-22 20:41:38] (step=0048825) Train Loss: 6.3058, Train Steps/Sec: 0.85 + 19%|██████████████████████████████████▌ | 7946/40903 [2:59:03<10:47:02, 1.18s/it][2025-04-22 20:42:08] (step=0048850) Train Loss: 6.3460, Train Steps/Sec: 0.85 + 19%|██████████████████████████████████▋ | 7971/40903 [2:59:32<10:50:57, 1.19s/it][2025-04-22 20:42:37] (step=0048875) Train Loss: 6.3809, Train Steps/Sec: 0.84 + 20%|██████████████████████████████████▊ | 7996/40903 [3:00:02<10:39:54, 1.17s/it][2025-04-22 20:43:07] (step=0048900) Train Loss: 6.3554, Train Steps/Sec: 0.84 + 20%|██████████████████████████████████▉ | 8021/40903 [3:00:32<10:54:19, 1.19s/it][2025-04-22 20:43:36] (step=0048925) Train Loss: 6.3605, Train Steps/Sec: 0.85 + 20%|███████████████████████████████████ | 8046/40903 [3:01:01<10:52:00, 1.19s/it][2025-04-22 20:44:06] (step=0048950) Train Loss: 6.3683, Train Steps/Sec: 0.84 + 20%|███████████████████████████████████ | 8071/40903 [3:01:31<10:50:06, 1.19s/it][2025-04-22 20:44:36] (step=0048975) Train Loss: 6.3110, Train Steps/Sec: 0.84 + 20%|███████████████████████████████████▏ | 8096/40903 [3:02:01<10:38:52, 1.17s/it][2025-04-22 20:45:05] (step=0049000) Train Loss: 6.3722, Train Steps/Sec: 0.84 + 20%|███████████████████████████████████▎ | 8121/40903 [3:02:30<10:57:32, 1.20s/it][2025-04-22 20:45:35] (step=0049025) Train Loss: 6.3578, Train Steps/Sec: 0.85 + 20%|███████████████████████████████████▍ | 8146/40903 [3:03:00<10:46:27, 1.18s/it][2025-04-22 20:46:05] (step=0049050) Train Loss: 6.3975, Train Steps/Sec: 0.84 + 20%|███████████████████████████████████▌ | 8171/40903 [3:03:29<10:42:55, 1.18s/it][2025-04-22 20:46:34] (step=0049075) Train Loss: 6.3417, Train Steps/Sec: 0.85 + 20%|███████████████████████████████████▋ | 8196/40903 [3:03:59<10:35:59, 1.17s/it][2025-04-22 20:47:04] (step=0049100) Train Loss: 6.3858, Train Steps/Sec: 0.84 + 20%|███████████████████████████████████▊ | 8221/40903 [3:04:29<10:53:52, 1.20s/it][2025-04-22 20:47:33] (step=0049125) Train Loss: 6.3428, Train Steps/Sec: 0.85 + 20%|███████████████████████████████████▉ | 8246/40903 [3:04:58<10:39:08, 1.17s/it][2025-04-22 20:48:03] (step=0049150) Train Loss: 6.3872, Train Steps/Sec: 0.85 + 20%|███████████████████████████████████▉ | 8271/40903 [3:05:28<10:36:25, 1.17s/it][2025-04-22 20:48:33] (step=0049175) Train Loss: 6.3822, Train Steps/Sec: 0.84 + 20%|████████████████████████████████████ | 8296/40903 [3:05:57<10:35:39, 1.17s/it][2025-04-22 20:49:02] (step=0049200) Train Loss: 6.3595, Train Steps/Sec: 0.84 + 20%|████████████████████████████████████▏ | 8321/40903 [3:06:27<10:57:41, 1.21s/it][2025-04-22 20:49:32] (step=0049225) Train Loss: 6.3950, Train Steps/Sec: 0.84 + 20%|████████████████████████████████████▎ | 8346/40903 [3:06:57<10:40:40, 1.18s/it][2025-04-22 20:50:02] (step=0049250) Train Loss: 6.3908, Train Steps/Sec: 0.85 + 20%|████████████████████████████████████▍ | 8371/40903 [3:07:26<10:40:33, 1.18s/it][2025-04-22 20:50:31] (step=0049275) Train Loss: 6.3626, Train Steps/Sec: 0.84 + 21%|████████████████████████████████████▌ | 8396/40903 [3:07:56<10:42:29, 1.19s/it][2025-04-22 20:51:01] (step=0049300) Train Loss: 6.3815, Train Steps/Sec: 0.84 + 21%|████████████████████████████████████▋ | 8421/40903 [3:08:26<10:46:58, 1.20s/it][2025-04-22 20:51:30] (step=0049325) Train Loss: 6.3680, Train Steps/Sec: 0.85 + 21%|████████████████████████████████████▊ | 8446/40903 [3:08:55<10:38:32, 1.18s/it][2025-04-22 20:52:00] (step=0049350) Train Loss: 6.3428, Train Steps/Sec: 0.85 + 21%|████████████████████████████████████▊ | 8471/40903 [3:09:25<10:47:40, 1.20s/it][2025-04-22 20:52:30] (step=0049375) Train Loss: 6.3665, Train Steps/Sec: 0.84 + 21%|████████████████████████████████████▉ | 8496/40903 [3:09:55<10:35:21, 1.18s/it][2025-04-22 20:52:59] (step=0049400) Train Loss: 6.3675, Train Steps/Sec: 0.84 + 21%|█████████████████████████████████████ | 8521/40903 [3:10:24<10:46:51, 1.20s/it][2025-04-22 20:53:29] (step=0049425) Train Loss: 6.3356, Train Steps/Sec: 0.84 + 21%|█████████████████████████████████████▏ | 8546/40903 [3:10:54<10:34:19, 1.18s/it][2025-04-22 20:53:59] (step=0049450) Train Loss: 6.3215, Train Steps/Sec: 0.85 + 21%|█████████████████████████████████████▎ | 8571/40903 [3:11:23<10:32:14, 1.17s/it][2025-04-22 20:54:28] (step=0049475) Train Loss: 6.3407, Train Steps/Sec: 0.85 + 21%|█████████████████████████████████████▍ | 8596/40903 [3:11:53<10:37:56, 1.18s/it][2025-04-22 20:54:58] (step=0049500) Train Loss: 6.3615, Train Steps/Sec: 0.84 + 21%|█████████████████████████████████████▌ | 8621/40903 [3:12:29<10:58:54, 1.22s/it][2025-04-22 20:55:34] (step=0049525) Train Loss: 6.3650, Train Steps/Sec: 0.69 + 21%|█████████████████████████████████████▋ | 8646/40903 [3:12:59<10:43:18, 1.20s/it][2025-04-22 20:56:04] (step=0049550) Train Loss: 6.3607, Train Steps/Sec: 0.84 + 21%|█████████████████████████████████████▋ | 8671/40903 [3:13:29<10:40:25, 1.19s/it][2025-04-22 20:56:33] (step=0049575) Train Loss: 6.3545, Train Steps/Sec: 0.84 + 21%|█████████████████████████████████████▊ | 8696/40903 [3:13:58<10:28:51, 1.17s/it][2025-04-22 20:57:03] (step=0049600) Train Loss: 6.3357, Train Steps/Sec: 0.84 + 21%|█████████████████████████████████████▉ | 8721/40903 [3:14:28<10:45:04, 1.20s/it][2025-04-22 20:57:33] (step=0049625) Train Loss: 6.3352, Train Steps/Sec: 0.85 + 21%|██████████████████████████████████████ | 8746/40903 [3:14:57<10:35:32, 1.19s/it][2025-04-22 20:58:02] (step=0049650) Train Loss: 6.3838, Train Steps/Sec: 0.84 + 21%|██████████████████████████████████████▏ | 8771/40903 [3:15:34<10:38:58, 1.19s/it][2025-04-22 20:58:39] (step=0049675) Train Loss: 6.3212, Train Steps/Sec: 0.69 + 22%|██████████████████████████████████████▎ | 8796/40903 [3:16:04<10:23:34, 1.17s/it][2025-04-22 20:59:09] (step=0049700) Train Loss: 6.3437, Train Steps/Sec: 0.84 + 22%|██████████████████████████████████████▍ | 8821/40903 [3:16:33<10:48:27, 1.21s/it][2025-04-22 20:59:38] (step=0049725) Train Loss: 6.4050, Train Steps/Sec: 0.84 + 22%|██████████████████████████████████████▍ | 8846/40903 [3:17:03<10:39:58, 1.20s/it][2025-04-22 21:00:08] (step=0049750) Train Loss: 6.4146, Train Steps/Sec: 0.84 + 22%|██████████████████████████████████████▌ | 8871/40903 [3:17:32<10:29:37, 1.18s/it][2025-04-22 21:00:37] (step=0049775) Train Loss: 6.3308, Train Steps/Sec: 0.85 + 22%|██████████████████████████████████████▋ | 8896/40903 [3:18:02<10:27:01, 1.18s/it][2025-04-22 21:01:07] (step=0049800) Train Loss: 6.3435, Train Steps/Sec: 0.84 + 22%|██████████████████████████████████████▊ | 8921/40903 [3:18:32<10:42:49, 1.21s/it][2025-04-22 21:01:37] (step=0049825) Train Loss: 6.3690, Train Steps/Sec: 0.84 + 22%|██████████████████████████████████████▉ | 8946/40903 [3:19:02<10:34:23, 1.19s/it][2025-04-22 21:02:07] (step=0049850) Train Loss: 6.3600, Train Steps/Sec: 0.84 + 22%|███████████████████████████████████████ | 8971/40903 [3:19:31<10:24:03, 1.17s/it][2025-04-22 21:02:36] (step=0049875) Train Loss: 6.3841, Train Steps/Sec: 0.85 + 22%|███████████████████████████████████████▏ | 8996/40903 [3:20:01<10:15:57, 1.16s/it][2025-04-22 21:03:06] (step=0049900) Train Loss: 6.3517, Train Steps/Sec: 0.84 + 22%|███████████████████████████████████████▎ | 9021/40903 [3:20:30<10:36:45, 1.20s/it][2025-04-22 21:03:35] (step=0049925) Train Loss: 6.3643, Train Steps/Sec: 0.84 + 22%|███████████████████████████████████████▎ | 9046/40903 [3:21:00<10:28:35, 1.18s/it][2025-04-22 21:04:05] (step=0049950) Train Loss: 6.3613, Train Steps/Sec: 0.84 + 22%|███████████████████████████████████████▍ | 9071/40903 [3:21:30<10:27:13, 1.18s/it][2025-04-22 21:04:35] (step=0049975) Train Loss: 6.3952, Train Steps/Sec: 0.85 + 22%|███████████████████████████████████████▌ | 9096/40903 [3:21:59<10:25:30, 1.18s/it][2025-04-22 21:05:04] (step=0050000) Train Loss: 6.3936, Train Steps/Sec: 0.84 +You are using a model of type instructblip to instantiate a model of type blip-2. This is not supported for all configurations of models and can yield errors. +model_name_or_path:/tmp/haozhezhao/model/instructblip-flan-t5-xl +image_place_holder: +Some kwargs in processor config are unused and will not have any effect: num_query_tokens. +tokenizer length after expend 32102 +tokenizer length before expend 32102 +100%|██████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████| 3/3 [02:54<00:00, 58.01s/it] +[2025-04-22 21:09:48] Finish Eval in 50000 steps...███████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████| 3/3 [02:53<00:00, 57.50s/it] +processing left-padding... +processing left-padding... +processing left-padding... +[2025-04-22 21:10:09] Saved checkpoint to checkpoint/BeforeFix_Blip_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_objects/002-GPT-XL/checkpoints/0050000.pt +[2025-04-22 21:10:11] Removed old checkpoint: checkpoint/BeforeFix_Blip_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_objects/002-GPT-XL/checkpoints/0046000.pt + 22%|███████████████████████████████████████▋ | 9121/40903 [3:27:36<10:38:01, 1.20s/it][2025-04-22 21:10:41] (step=0050025) Train Loss: 6.3514, Train Steps/Sec: 0.07 + 22%|███████████████████████████████████████▊ | 9146/40903 [3:28:05<10:25:32, 1.18s/it][2025-04-22 21:11:10] (step=0050050) Train Loss: 6.3385, Train Steps/Sec: 0.85 + 22%|███████████████████████████████████████▉ | 9171/40903 [3:28:35<10:16:30, 1.17s/it][2025-04-22 21:11:40] (step=0050075) Train Loss: 6.3067, Train Steps/Sec: 0.85 + 22%|████████████████████████████████████████ | 9196/40903 [3:29:05<10:19:21, 1.17s/it][2025-04-22 21:12:09] (step=0050100) Train Loss: 6.3481, Train Steps/Sec: 0.84 + 23%|████████████████████████████████████████▏ | 9221/40903 [3:29:34<10:35:39, 1.20s/it][2025-04-22 21:12:39] (step=0050125) Train Loss: 6.4039, Train Steps/Sec: 0.84 + 23%|████████████████████████████████████████▏ | 9246/40903 [3:30:04<10:18:03, 1.17s/it][2025-04-22 21:13:08] (step=0050150) Train Loss: 6.3658, Train Steps/Sec: 0.85 + 23%|████████████████████████████████████████▎ | 9271/40903 [3:30:33<10:18:57, 1.17s/it][2025-04-22 21:13:38] (step=0050175) Train Loss: 6.3445, Train Steps/Sec: 0.85 + 23%|████████████████████████████████████████▍ | 9296/40903 [3:31:09<10:14:28, 1.17s/it][2025-04-22 21:14:14] (step=0050200) Train Loss: 6.3639, Train Steps/Sec: 0.70 + 23%|████████████████████████████████████████▌ | 9321/40903 [3:31:38<10:34:20, 1.21s/it][2025-04-22 21:14:43] (step=0050225) Train Loss: 6.3848, Train Steps/Sec: 0.85 + 23%|████████████████████████████████████████▋ | 9346/40903 [3:32:08<10:16:19, 1.17s/it][2025-04-22 21:15:13] (step=0050250) Train Loss: 6.3718, Train Steps/Sec: 0.85 + 23%|████████████████████████████████████████▊ | 9371/40903 [3:32:44<10:17:18, 1.17s/it][2025-04-22 21:15:49] (step=0050275) Train Loss: 6.3758, Train Steps/Sec: 0.69 + 23%|████████████████████████████████████████▉ | 9396/40903 [3:33:14<10:15:19, 1.17s/it][2025-04-22 21:16:19] (step=0050300) Train Loss: 6.3821, Train Steps/Sec: 0.84 + 23%|████████████████████████████████████████▉ | 9421/40903 [3:33:44<10:34:03, 1.21s/it][2025-04-22 21:16:48] (step=0050325) Train Loss: 6.3327, Train Steps/Sec: 0.84 + 23%|█████████████████████████████████████████ | 9446/40903 [3:34:13<10:12:25, 1.17s/it][2025-04-22 21:17:18] (step=0050350) Train Loss: 6.3675, Train Steps/Sec: 0.85 + 23%|█████████████████████████████████████████▏ | 9471/40903 [3:34:43<10:22:08, 1.19s/it][2025-04-22 21:17:48] (step=0050375) Train Loss: 6.3805, Train Steps/Sec: 0.84 + 23%|█████████████████████████████████████████▎ | 9496/40903 [3:35:12<10:09:59, 1.17s/it][2025-04-22 21:18:17] (step=0050400) Train Loss: 6.3980, Train Steps/Sec: 0.84 + 23%|█████████████████████████████████████████▍ | 9521/40903 [3:35:42<10:32:48, 1.21s/it][2025-04-22 21:18:47] (step=0050425) Train Loss: 6.3152, Train Steps/Sec: 0.85 + 23%|█████████████████████████████████████████▌ | 9546/40903 [3:36:12<10:24:38, 1.20s/it][2025-04-22 21:19:16] (step=0050450) Train Loss: 6.3156, Train Steps/Sec: 0.84 + 23%|█████████████████████████████████████████▋ | 9571/40903 [3:36:41<10:03:54, 1.16s/it][2025-04-22 21:19:46] (step=0050475) Train Loss: 6.3838, Train Steps/Sec: 0.85 + 23%|█████████████████████████████████████████▊ | 9596/40903 [3:37:11<10:17:33, 1.18s/it][2025-04-22 21:20:16] (step=0050500) Train Loss: 6.4255, Train Steps/Sec: 0.84 + 24%|█████████████████████████████████████████▊ | 9621/40903 [3:37:41<10:47:43, 1.24s/it][2025-04-22 21:20:46] (step=0050525) Train Loss: 6.3317, Train Steps/Sec: 0.83 + 24%|█████████████████████████████████████████▉ | 9646/40903 [3:38:10<10:18:32, 1.19s/it][2025-04-22 21:21:15] (step=0050550) Train Loss: 6.3871, Train Steps/Sec: 0.84 + 24%|██████████████████████████████████████████ | 9671/40903 [3:38:40<10:13:06, 1.18s/it][2025-04-22 21:21:45] (step=0050575) Train Loss: 6.3330, Train Steps/Sec: 0.85 + 24%|██████████████████████████████████████████▏ | 9696/40903 [3:39:10<10:14:52, 1.18s/it][2025-04-22 21:22:15] (step=0050600) Train Loss: 6.3627, Train Steps/Sec: 0.84 + 24%|██████████████████████████████████████████▎ | 9721/40903 [3:39:39<10:26:25, 1.21s/it][2025-04-22 21:22:44] (step=0050625) Train Loss: 6.3439, Train Steps/Sec: 0.85 + 24%|██████████████████████████████████████████▍ | 9746/40903 [3:40:09<10:23:50, 1.20s/it][2025-04-22 21:23:14] (step=0050650) Train Loss: 6.3643, Train Steps/Sec: 0.84 + 24%|██████████████████████████████████████████▌ | 9771/40903 [3:40:38<10:13:03, 1.18s/it][2025-04-22 21:23:43] (step=0050675) Train Loss: 6.3381, Train Steps/Sec: 0.85 + 24%|██████████████████████████████████████████▊ | 9796/40903 [3:41:08<9:58:58, 1.16s/it][2025-04-22 21:24:13] (step=0050700) Train Loss: 6.3712, Train Steps/Sec: 0.84 + 24%|██████████████████████████████████████████▋ | 9821/40903 [3:41:38<10:27:38, 1.21s/it][2025-04-22 21:24:42] (step=0050725) Train Loss: 6.3539, Train Steps/Sec: 0.85 + 24%|██████████████████████████████████████████▊ | 9846/40903 [3:42:07<10:14:49, 1.19s/it][2025-04-22 21:25:12] (step=0050750) Train Loss: 6.3167, Train Steps/Sec: 0.85 + 24%|██████████████████████████████████████████▉ | 9871/40903 [3:42:36<10:06:16, 1.17s/it][2025-04-22 21:25:41] (step=0050775) Train Loss: 6.3693, Train Steps/Sec: 0.85 + 24%|███████████████████████████████████████████ | 9896/40903 [3:43:06<10:02:15, 1.17s/it][2025-04-22 21:26:11] (step=0050800) Train Loss: 6.4028, Train Steps/Sec: 0.85 + 24%|███████████████████████████████████████████▏ | 9921/40903 [3:43:36<10:20:17, 1.20s/it][2025-04-22 21:26:40] (step=0050825) Train Loss: 6.3905, Train Steps/Sec: 0.84 + 24%|███████████████████████████████████████████▎ | 9946/40903 [3:44:05<10:13:33, 1.19s/it][2025-04-22 21:27:10] (step=0050850) Train Loss: 6.3747, Train Steps/Sec: 0.84 + 24%|███████████████████████████████████████████▍ | 9971/40903 [3:44:35<10:02:36, 1.17s/it][2025-04-22 21:27:40] (step=0050875) Train Loss: 6.3899, Train Steps/Sec: 0.85 + 24%|███████████████████████████████████████████▌ | 9996/40903 [3:45:05<10:01:53, 1.17s/it][2025-04-22 21:28:10] (step=0050900) Train Loss: 6.3754, Train Steps/Sec: 0.84 + 24%|███████████████████████████████████████████▎ | 10021/40903 [3:45:34<10:17:18, 1.20s/it][2025-04-22 21:28:39] (step=0050925) Train Loss: 6.3260, Train Steps/Sec: 0.85 + 25%|███████████████████████████████████████████▍ | 10046/40903 [3:46:04<10:14:35, 1.20s/it][2025-04-22 21:29:09] (step=0050950) Train Loss: 6.3489, Train Steps/Sec: 0.84 + 25%|███████████████████████████████████████████▌ | 10071/40903 [3:46:33<10:06:34, 1.18s/it][2025-04-22 21:29:38] (step=0050975) Train Loss: 6.3390, Train Steps/Sec: 0.85 + 25%|███████████████████████████████████████████▋ | 10096/40903 [3:47:03<10:09:13, 1.19s/it][2025-04-22 21:30:08] (step=0051000) Train Loss: 6.3635, Train Steps/Sec: 0.84 + 25%|███████████████████████████████████████████▊ | 10121/40903 [3:47:33<10:12:03, 1.19s/it][2025-04-22 21:30:37] (step=0051025) Train Loss: 6.3321, Train Steps/Sec: 0.85 + 25%|███████████████████████████████████████████▉ | 10146/40903 [3:48:02<10:03:22, 1.18s/it][2025-04-22 21:31:07] (step=0051050) Train Loss: 6.3071, Train Steps/Sec: 0.84 + 25%|████████████████████████████████████████████▎ | 10171/40903 [3:48:32<9:56:13, 1.16s/it][2025-04-22 21:31:37] (step=0051075) Train Loss: 6.3728, Train Steps/Sec: 0.85 + 25%|████████████████████████████████████████████▎ | 10196/40903 [3:49:01<9:58:34, 1.17s/it][2025-04-22 21:32:06] (step=0051100) Train Loss: 6.3545, Train Steps/Sec: 0.84 + 25%|████████████████████████████████████████████▏ | 10221/40903 [3:49:31<10:20:17, 1.21s/it][2025-04-22 21:32:36] (step=0051125) Train Loss: 6.3818, Train Steps/Sec: 0.84 + 25%|████████████████████████████████████████████▌ | 10246/40903 [3:50:01<9:58:05, 1.17s/it][2025-04-22 21:33:05] (step=0051150) Train Loss: 6.3304, Train Steps/Sec: 0.85 + 25%|████████████████████████████████████████████▍ | 10271/40903 [3:50:30<10:01:47, 1.18s/it][2025-04-22 21:33:35] (step=0051175) Train Loss: 6.3977, Train Steps/Sec: 0.85 + 25%|████████████████████████████████████████████▌ | 10296/40903 [3:51:00<10:00:34, 1.18s/it][2025-04-22 21:34:05] (step=0051200) Train Loss: 6.3587, Train Steps/Sec: 0.84 + 25%|████████████████████████████████████████████▋ | 10321/40903 [3:51:30<10:19:23, 1.22s/it][2025-04-22 21:34:34] (step=0051225) Train Loss: 6.3230, Train Steps/Sec: 0.85 + 25%|████████████████████████████████████████████▊ | 10346/40903 [3:51:59<10:04:39, 1.19s/it][2025-04-22 21:35:04] (step=0051250) Train Loss: 6.3800, Train Steps/Sec: 0.85 + 25%|█████████████████████████████████████████████▏ | 10371/40903 [3:52:29<9:53:22, 1.17s/it][2025-04-22 21:35:34] (step=0051275) Train Loss: 6.3531, Train Steps/Sec: 0.84 + 25%|█████████████████████████████████████████████▏ | 10396/40903 [3:52:58<9:57:54, 1.18s/it][2025-04-22 21:36:03] (step=0051300) Train Loss: 6.3749, Train Steps/Sec: 0.84 + 25%|█████████████████████████████████████████████ | 10421/40903 [3:53:28<10:02:19, 1.19s/it][2025-04-22 21:36:33] (step=0051325) Train Loss: 6.3458, Train Steps/Sec: 0.85 + 26%|█████████████████████████████████████████████▏ | 10446/40903 [3:53:58<10:03:27, 1.19s/it][2025-04-22 21:37:03] (step=0051350) Train Loss: 6.3734, Train Steps/Sec: 0.84 + 26%|█████████████████████████████████████████████▌ | 10471/40903 [3:54:27<9:57:01, 1.18s/it][2025-04-22 21:37:32] (step=0051375) Train Loss: 6.3654, Train Steps/Sec: 0.84 + 26%|█████████████████████████████████████████████▋ | 10496/40903 [3:54:57<9:50:11, 1.16s/it][2025-04-22 21:38:02] (step=0051400) Train Loss: 6.3498, Train Steps/Sec: 0.85 + 26%|█████████████████████████████████████████████▌ | 10521/40903 [3:55:26<10:10:00, 1.20s/it][2025-04-22 21:38:31] (step=0051425) Train Loss: 6.3604, Train Steps/Sec: 0.85 + 26%|█████████████████████████████████████████████▋ | 10546/40903 [3:55:56<10:00:26, 1.19s/it][2025-04-22 21:39:01] (step=0051450) Train Loss: 6.3374, Train Steps/Sec: 0.85 + 26%|██████████████████████████████████████████████ | 10571/40903 [3:56:25<9:51:48, 1.17s/it][2025-04-22 21:39:30] (step=0051475) Train Loss: 6.3158, Train Steps/Sec: 0.85 + 26%|██████████████████████████████████████████████ | 10596/40903 [3:56:55<9:54:07, 1.18s/it][2025-04-22 21:40:00] (step=0051500) Train Loss: 6.3431, Train Steps/Sec: 0.84 + 26%|█████████████████████████████████████████████▉ | 10621/40903 [3:57:25<10:12:22, 1.21s/it][2025-04-22 21:40:29] (step=0051525) Train Loss: 6.3082, Train Steps/Sec: 0.84 + 26%|██████████████████████████████████████████████ | 10646/40903 [3:57:54<10:06:25, 1.20s/it][2025-04-22 21:40:59] (step=0051550) Train Loss: 6.4122, Train Steps/Sec: 0.84 + 26%|██████████████████████████████████████████████▏ | 10671/40903 [3:58:24<10:02:07, 1.20s/it][2025-04-22 21:41:29] (step=0051575) Train Loss: 6.3284, Train Steps/Sec: 0.84 + 26%|██████████████████████████████████████████████▌ | 10696/40903 [3:58:54<9:46:03, 1.16s/it][2025-04-22 21:41:58] (step=0051600) Train Loss: 6.3458, Train Steps/Sec: 0.85 + 26%|██████████████████████████████████████████████▍ | 10721/40903 [3:59:23<10:07:32, 1.21s/it][2025-04-22 21:42:28] (step=0051625) Train Loss: 6.3633, Train Steps/Sec: 0.85 + 26%|██████████████████████████████████████████████▊ | 10746/40903 [3:59:53<9:51:00, 1.18s/it][2025-04-22 21:42:57] (step=0051650) Train Loss: 6.3330, Train Steps/Sec: 0.85 + 26%|██████████████████████████████████████████████▊ | 10771/40903 [4:00:22<9:48:00, 1.17s/it][2025-04-22 21:43:27] (step=0051675) Train Loss: 6.3701, Train Steps/Sec: 0.85 + 26%|██████████████████████████████████████████████▉ | 10796/40903 [4:00:52<9:51:00, 1.18s/it][2025-04-22 21:43:57] (step=0051700) Train Loss: 6.3758, Train Steps/Sec: 0.84 + 26%|██████████████████████████████████████████████▊ | 10821/40903 [4:01:22<10:07:44, 1.21s/it][2025-04-22 21:44:26] (step=0051725) Train Loss: 6.3409, Train Steps/Sec: 0.85 + 27%|███████████████████████████████████████████████▏ | 10846/40903 [4:01:51<9:51:33, 1.18s/it][2025-04-22 21:44:56] (step=0051750) Train Loss: 6.3523, Train Steps/Sec: 0.84 + 27%|███████████████████████████████████████████████▎ | 10871/40903 [4:02:21<9:45:15, 1.17s/it][2025-04-22 21:45:25] (step=0051775) Train Loss: 6.3365, Train Steps/Sec: 0.85 + 27%|███████████████████████████████████████████████▍ | 10896/40903 [4:02:50<9:49:40, 1.18s/it][2025-04-22 21:45:55] (step=0051800) Train Loss: 6.3439, Train Steps/Sec: 0.84 + 27%|███████████████████████████████████████████████▎ | 10921/40903 [4:03:20<10:00:13, 1.20s/it][2025-04-22 21:46:25] (step=0051825) Train Loss: 6.3647, Train Steps/Sec: 0.85 + 27%|███████████████████████████████████████████████▋ | 10946/40903 [4:03:49<9:52:03, 1.19s/it][2025-04-22 21:46:54] (step=0051850) Train Loss: 6.3590, Train Steps/Sec: 0.85 + 27%|███████████████████████████████████████████████▋ | 10971/40903 [4:04:19<9:39:54, 1.16s/it][2025-04-22 21:47:24] (step=0051875) Train Loss: 6.4129, Train Steps/Sec: 0.84 + 27%|███████████████████████████████████████████████▊ | 10996/40903 [4:04:49<9:39:37, 1.16s/it][2025-04-22 21:47:54] (step=0051900) Train Loss: 6.3705, Train Steps/Sec: 0.85 + 27%|███████████████████████████████████████████████▉ | 11021/40903 [4:05:18<9:56:52, 1.20s/it][2025-04-22 21:48:23] (step=0051925) Train Loss: 6.3339, Train Steps/Sec: 0.84 + 27%|████████████████████████████████████████████████ | 11046/40903 [4:05:48<9:44:26, 1.17s/it][2025-04-22 21:48:53] (step=0051950) Train Loss: 6.3688, Train Steps/Sec: 0.85 + 27%|████████████████████████████████████████████████▏ | 11071/40903 [4:06:17<9:44:38, 1.18s/it][2025-04-22 21:49:22] (step=0051975) Train Loss: 6.3199, Train Steps/Sec: 0.85 + 27%|████████████████████████████████████████████████▎ | 11096/40903 [4:06:47<9:37:08, 1.16s/it][2025-04-22 21:49:52] (step=0052000) Train Loss: 6.2975, Train Steps/Sec: 0.84 +You are using a model of type instructblip to instantiate a model of type blip-2. This is not supported for all configurations of models and can yield errors. +model_name_or_path:/tmp/haozhezhao/model/instructblip-flan-t5-xl +image_place_holder: +Some kwargs in processor config are unused and will not have any effect: num_query_tokens. +tokenizer length after expend 32102 +tokenizer length before expend 32102 +100%|██████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████| 3/3 [02:53<00:00, 57.79s/it] +[2025-04-22 21:54:34] Finish Eval in 52000 steps...███████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████| 3/3 [02:52<00:00, 57.31s/it] +processing left-padding... +processing left-padding... +processing left-padding... +[2025-04-22 21:54:55] Saved checkpoint to checkpoint/BeforeFix_Blip_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_objects/002-GPT-XL/checkpoints/0052000.pt +[2025-04-22 21:54:57] Removed old checkpoint: checkpoint/BeforeFix_Blip_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_objects/002-GPT-XL/checkpoints/0048000.pt + 27%|████████████████████████████████████████████████ | 11121/40903 [4:12:22<10:00:45, 1.21s/it][2025-04-22 21:55:27] (step=0052025) Train Loss: 6.3551, Train Steps/Sec: 0.07 + 27%|████████████████████████████████████████████████▌ | 11146/40903 [4:12:52<9:47:22, 1.18s/it][2025-04-22 21:55:56] (step=0052050) Train Loss: 6.3397, Train Steps/Sec: 0.84 + 27%|████████████████████████████████████████████████▌ | 11171/40903 [4:13:21<9:44:40, 1.18s/it][2025-04-22 21:56:26] (step=0052075) Train Loss: 6.3306, Train Steps/Sec: 0.84 + 27%|████████████████████████████████████████████████▋ | 11196/40903 [4:13:51<9:38:58, 1.17s/it][2025-04-22 21:56:56] (step=0052100) Train Loss: 6.3617, Train Steps/Sec: 0.84 + 27%|████████████████████████████████████████████████▊ | 11221/40903 [4:14:20<9:54:21, 1.20s/it][2025-04-22 21:57:25] (step=0052125) Train Loss: 6.3185, Train Steps/Sec: 0.85 + 27%|████████████████████████████████████████████████▉ | 11246/40903 [4:14:50<9:53:42, 1.20s/it][2025-04-22 21:57:55] (step=0052150) Train Loss: 6.3297, Train Steps/Sec: 0.84 + 28%|█████████████████████████████████████████████████ | 11271/40903 [4:15:20<9:47:37, 1.19s/it][2025-04-22 21:58:24] (step=0052175) Train Loss: 6.3426, Train Steps/Sec: 0.85 + 28%|█████████████████████████████████████████████████▏ | 11296/40903 [4:15:49<9:38:13, 1.17s/it][2025-04-22 21:58:54] (step=0052200) Train Loss: 6.3452, Train Steps/Sec: 0.84 + 28%|█████████████████████████████████████████████████▎ | 11321/40903 [4:16:19<9:47:38, 1.19s/it][2025-04-22 21:59:24] (step=0052225) Train Loss: 6.3689, Train Steps/Sec: 0.85 + 28%|█████████████████████████████████████████████████▍ | 11346/40903 [4:16:48<9:41:00, 1.18s/it][2025-04-22 21:59:53] (step=0052250) Train Loss: 6.3658, Train Steps/Sec: 0.85 + 28%|█████████████████████████████████████████████████▍ | 11371/40903 [4:17:18<9:36:34, 1.17s/it][2025-04-22 22:00:23] (step=0052275) Train Loss: 6.3020, Train Steps/Sec: 0.84 + 28%|█████████████████████████████████████████████████▌ | 11396/40903 [4:17:48<9:39:36, 1.18s/it][2025-04-22 22:00:52] (step=0052300) Train Loss: 6.3408, Train Steps/Sec: 0.84 + 28%|█████████████████████████████████████████████████▋ | 11421/40903 [4:18:17<9:50:07, 1.20s/it][2025-04-22 22:01:22] (step=0052325) Train Loss: 6.3629, Train Steps/Sec: 0.84 + 28%|█████████████████████████████████████████████████▊ | 11446/40903 [4:18:47<9:39:38, 1.18s/it][2025-04-22 22:01:52] (step=0052350) Train Loss: 6.3136, Train Steps/Sec: 0.85 + 28%|█████████████████████████████████████████████████▉ | 11471/40903 [4:19:17<9:44:29, 1.19s/it][2025-04-22 22:02:22] (step=0052375) Train Loss: 6.3242, Train Steps/Sec: 0.84 + 28%|██████████████████████████████████████████████████ | 11496/40903 [4:19:46<9:34:47, 1.17s/it][2025-04-22 22:02:51] (step=0052400) Train Loss: 6.3592, Train Steps/Sec: 0.85 + 28%|██████████████████████████████████████████████████▏ | 11521/40903 [4:20:16<9:45:23, 1.20s/it][2025-04-22 22:03:21] (step=0052425) Train Loss: 6.3891, Train Steps/Sec: 0.84 + 28%|██████████████████████████████████████████████████▏ | 11546/40903 [4:20:45<9:39:29, 1.18s/it][2025-04-22 22:03:50] (step=0052450) Train Loss: 6.3568, Train Steps/Sec: 0.84 + 28%|██████████████████████████████████████████████████▎ | 11571/40903 [4:21:15<9:44:40, 1.20s/it][2025-04-22 22:04:20] (step=0052475) Train Loss: 6.3862, Train Steps/Sec: 0.84 + 28%|██████████████████████████████████████████████████▍ | 11596/40903 [4:21:45<9:34:46, 1.18s/it][2025-04-22 22:04:50] (step=0052500) Train Loss: 6.3509, Train Steps/Sec: 0.85 + 28%|██████████████████████████████████████████████████▌ | 11621/40903 [4:22:14<9:50:20, 1.21s/it][2025-04-22 22:05:19] (step=0052525) Train Loss: 6.3229, Train Steps/Sec: 0.85 + 28%|██████████████████████████████████████████████████▋ | 11646/40903 [4:22:44<9:36:17, 1.18s/it][2025-04-22 22:05:49] (step=0052550) Train Loss: 6.3608, Train Steps/Sec: 0.85 + 29%|██████████████████████████████████████████████████▊ | 11671/40903 [4:23:13<9:29:02, 1.17s/it][2025-04-22 22:06:18] (step=0052575) Train Loss: 6.3128, Train Steps/Sec: 0.85 + 29%|██████████████████████████████████████████████████▉ | 11696/40903 [4:23:43<9:31:27, 1.17s/it][2025-04-22 22:06:48] (step=0052600) Train Loss: 6.3342, Train Steps/Sec: 0.84 + 29%|███████████████████████████████████████████████████ | 11721/40903 [4:24:13<9:36:58, 1.19s/it][2025-04-22 22:07:17] (step=0052625) Train Loss: 6.2980, Train Steps/Sec: 0.85 + 29%|███████████████████████████████████████████████████ | 11746/40903 [4:24:42<9:39:12, 1.19s/it][2025-04-22 22:07:47] (step=0052650) Train Loss: 6.3503, Train Steps/Sec: 0.85 + 29%|███████████████████████████████████████████████████▏ | 11771/40903 [4:25:12<9:32:54, 1.18s/it][2025-04-22 22:08:17] (step=0052675) Train Loss: 6.3839, Train Steps/Sec: 0.84 + 29%|███████████████████████████████████████████████████▎ | 11796/40903 [4:25:41<9:27:32, 1.17s/it][2025-04-22 22:08:46] (step=0052700) Train Loss: 6.3336, Train Steps/Sec: 0.84 + 29%|███████████████████████████████████████████████████▍ | 11821/40903 [4:26:11<9:43:17, 1.20s/it][2025-04-22 22:09:16] (step=0052725) Train Loss: 6.3114, Train Steps/Sec: 0.85 + 29%|███████████████████████████████████████████████████▌ | 11846/40903 [4:26:47<9:48:23, 1.21s/it][2025-04-22 22:09:52] (step=0052750) Train Loss: 6.3173, Train Steps/Sec: 0.70 + 29%|███████████████████████████████████████████████████▋ | 11871/40903 [4:27:16<9:25:11, 1.17s/it][2025-04-22 22:10:21] (step=0052775) Train Loss: 6.3613, Train Steps/Sec: 0.85 + 29%|███████████████████████████████████████████████████▊ | 11896/40903 [4:27:46<9:19:36, 1.16s/it][2025-04-22 22:10:51] (step=0052800) Train Loss: 6.3439, Train Steps/Sec: 0.85 + 29%|███████████████████████████████████████████████████▉ | 11921/40903 [4:28:15<9:31:44, 1.18s/it][2025-04-22 22:11:20] (step=0052825) Train Loss: 6.3260, Train Steps/Sec: 0.85 + 29%|███████████████████████████████████████████████████▉ | 11946/40903 [4:28:45<9:32:14, 1.19s/it][2025-04-22 22:11:50] (step=0052850) Train Loss: 6.3555, Train Steps/Sec: 0.85 + 29%|████████████████████████████████████████████████████ | 11971/40903 [4:29:14<9:22:01, 1.17s/it][2025-04-22 22:12:19] (step=0052875) Train Loss: 6.2772, Train Steps/Sec: 0.85 + 29%|████████████████████████████████████████████████████▏ | 11996/40903 [4:29:44<9:26:19, 1.18s/it][2025-04-22 22:12:49] (step=0052900) Train Loss: 6.3157, Train Steps/Sec: 0.84 + 29%|████████████████████████████████████████████████████▎ | 12021/40903 [4:30:14<9:38:25, 1.20s/it][2025-04-22 22:13:19] (step=0052925) Train Loss: 6.3365, Train Steps/Sec: 0.85 + 29%|████████████████████████████████████████████████████▍ | 12046/40903 [4:30:43<9:35:04, 1.20s/it][2025-04-22 22:13:48] (step=0052950) Train Loss: 6.3617, Train Steps/Sec: 0.85 + 30%|████████████████████████████████████████████████████▌ | 12071/40903 [4:31:13<9:26:15, 1.18s/it][2025-04-22 22:14:18] (step=0052975) Train Loss: 6.3668, Train Steps/Sec: 0.84 + 30%|████████████████████████████████████████████████████▋ | 12096/40903 [4:31:42<9:19:54, 1.17s/it][2025-04-22 22:14:47] (step=0053000) Train Loss: 6.3447, Train Steps/Sec: 0.85 + 30%|████████████████████████████████████████████████████▋ | 12121/40903 [4:32:20<9:40:41, 1.21s/it][2025-04-22 22:15:25] (step=0053025) Train Loss: 6.3301, Train Steps/Sec: 0.67 + 30%|████████████████████████████████████████████████████▊ | 12146/40903 [4:32:49<9:32:48, 1.20s/it][2025-04-22 22:15:54] (step=0053050) Train Loss: 6.3492, Train Steps/Sec: 0.84 + 30%|████████████████████████████████████████████████████▉ | 12171/40903 [4:33:19<9:22:20, 1.17s/it][2025-04-22 22:16:24] (step=0053075) Train Loss: 6.3220, Train Steps/Sec: 0.85 + 30%|█████████████████████████████████████████████████████ | 12196/40903 [4:33:48<9:14:29, 1.16s/it][2025-04-22 22:16:53] (step=0053100) Train Loss: 6.3168, Train Steps/Sec: 0.84 + 30%|█████████████████████████████████████████████████████▏ | 12221/40903 [4:34:18<9:37:26, 1.21s/it][2025-04-22 22:17:23] (step=0053125) Train Loss: 6.3370, Train Steps/Sec: 0.85 + 30%|█████████████████████████████████████████████████████▎ | 12246/40903 [4:34:48<9:20:45, 1.17s/it][2025-04-22 22:17:52] (step=0053150) Train Loss: 6.3471, Train Steps/Sec: 0.85 + 30%|█████████████████████████████████████████████████████▍ | 12271/40903 [4:35:17<9:22:23, 1.18s/it][2025-04-22 22:18:22] (step=0053175) Train Loss: 6.3386, Train Steps/Sec: 0.84 + 30%|█████████████████████████████████████████████████████▌ | 12296/40903 [4:35:47<9:16:48, 1.17s/it][2025-04-22 22:18:52] (step=0053200) Train Loss: 6.3188, Train Steps/Sec: 0.84 + 30%|█████████████████████████████████████████████████████▌ | 12321/40903 [4:36:17<9:33:02, 1.20s/it][2025-04-22 22:19:21] (step=0053225) Train Loss: 6.3524, Train Steps/Sec: 0.85 + 30%|█████████████████████████████████████████████████████▋ | 12346/40903 [4:36:46<9:19:51, 1.18s/it][2025-04-22 22:19:51] (step=0053250) Train Loss: 6.3657, Train Steps/Sec: 0.85 + 30%|█████████████████████████████████████████████████████▊ | 12371/40903 [4:37:15<9:14:07, 1.17s/it][2025-04-22 22:20:20] (step=0053275) Train Loss: 6.3745, Train Steps/Sec: 0.85 + 30%|█████████████████████████████████████████████████████▉ | 12396/40903 [4:37:45<9:23:43, 1.19s/it][2025-04-22 22:20:50] (step=0053300) Train Loss: 6.3740, Train Steps/Sec: 0.84 + 30%|██████████████████████████████████████████████████████ | 12421/40903 [4:38:15<9:30:12, 1.20s/it][2025-04-22 22:21:19] (step=0053325) Train Loss: 6.3402, Train Steps/Sec: 0.85 + 30%|██████████████████████████████████████████████████████▏ | 12446/40903 [4:38:44<9:20:33, 1.18s/it][2025-04-22 22:21:49] (step=0053350) Train Loss: 6.3562, Train Steps/Sec: 0.85 + 30%|██████████████████████████████████████████████████████▎ | 12471/40903 [4:39:13<9:13:27, 1.17s/it][2025-04-22 22:22:18] (step=0053375) Train Loss: 6.3436, Train Steps/Sec: 0.85 + 31%|██████████████████████████████████████████████████████▍ | 12496/40903 [4:39:43<9:19:10, 1.18s/it][2025-04-22 22:22:48] (step=0053400) Train Loss: 6.3244, Train Steps/Sec: 0.84 + 31%|██████████████████████████████████████████████████████▍ | 12521/40903 [4:40:13<9:29:29, 1.20s/it][2025-04-22 22:23:18] (step=0053425) Train Loss: 6.3225, Train Steps/Sec: 0.85 + 31%|██████████████████████████████████████████████████████▌ | 12546/40903 [4:40:42<9:15:25, 1.18s/it][2025-04-22 22:23:47] (step=0053450) Train Loss: 6.3449, Train Steps/Sec: 0.84 + 31%|██████████████████████████████████████████████████████▋ | 12571/40903 [4:41:12<9:18:42, 1.18s/it][2025-04-22 22:24:17] (step=0053475) Train Loss: 6.3376, Train Steps/Sec: 0.84 + 31%|██████████████████████████████████████████████████████▊ | 12596/40903 [4:41:42<9:08:53, 1.16s/it][2025-04-22 22:24:47] (step=0053500) Train Loss: 6.3672, Train Steps/Sec: 0.84 + 31%|██████████████████████████████████████████████████████▉ | 12621/40903 [4:42:11<9:31:05, 1.21s/it][2025-04-22 22:25:16] (step=0053525) Train Loss: 6.3623, Train Steps/Sec: 0.85 + 31%|███████████████████████████████████████████████████████ | 12646/40903 [4:42:41<9:20:18, 1.19s/it][2025-04-22 22:25:46] (step=0053550) Train Loss: 6.3641, Train Steps/Sec: 0.85 + 31%|███████████████████████████████████████████████████████▏ | 12671/40903 [4:43:10<9:13:59, 1.18s/it][2025-04-22 22:26:15] (step=0053575) Train Loss: 6.3872, Train Steps/Sec: 0.85 + 31%|███████████████████████████████████████████████████████▏ | 12696/40903 [4:43:46<9:13:43, 1.18s/it][2025-04-22 22:26:51] (step=0053600) Train Loss: 6.3453, Train Steps/Sec: 0.70 + 31%|███████████████████████████████████████████████████████▎ | 12721/40903 [4:44:16<9:27:15, 1.21s/it][2025-04-22 22:27:21] (step=0053625) Train Loss: 6.3656, Train Steps/Sec: 0.85 + 31%|███████████████████████████████████████████████████████▍ | 12746/40903 [4:44:45<9:14:11, 1.18s/it][2025-04-22 22:27:50] (step=0053650) Train Loss: 6.3811, Train Steps/Sec: 0.85 + 31%|███████████████████████████████████████████████████████▌ | 12771/40903 [4:45:15<9:12:16, 1.18s/it][2025-04-22 22:28:20] (step=0053675) Train Loss: 6.3335, Train Steps/Sec: 0.85 + 31%|███████████████████████████████████████████████████████▋ | 12796/40903 [4:45:44<9:02:35, 1.16s/it][2025-04-22 22:28:49] (step=0053700) Train Loss: 6.3338, Train Steps/Sec: 0.84 + 31%|███████████████████████████████████████████████████████▊ | 12821/40903 [4:46:14<9:21:09, 1.20s/it][2025-04-22 22:29:19] (step=0053725) Train Loss: 6.3603, Train Steps/Sec: 0.85 + 31%|███████████████████████████████████████████████████████▉ | 12846/40903 [4:46:44<9:26:20, 1.21s/it][2025-04-22 22:29:49] (step=0053750) Train Loss: 6.3645, Train Steps/Sec: 0.84 + 31%|████████████████████████████████████████████████████████ | 12871/40903 [4:47:13<9:16:52, 1.19s/it][2025-04-22 22:30:18] (step=0053775) Train Loss: 6.3482, Train Steps/Sec: 0.84 + 32%|████████████████████████████████████████████████████████ | 12896/40903 [4:47:43<8:57:53, 1.15s/it][2025-04-22 22:30:48] (step=0053800) Train Loss: 6.3494, Train Steps/Sec: 0.84 + 32%|████████████████████████████████████████████████████████▏ | 12921/40903 [4:48:13<9:20:51, 1.20s/it][2025-04-22 22:31:17] (step=0053825) Train Loss: 6.3488, Train Steps/Sec: 0.85 + 32%|████████████████████████████████████████████████████████▎ | 12946/40903 [4:48:42<9:14:59, 1.19s/it][2025-04-22 22:31:47] (step=0053850) Train Loss: 6.3424, Train Steps/Sec: 0.85 + 32%|████████████████████████████████████████████████████████▍ | 12971/40903 [4:49:12<9:03:23, 1.17s/it][2025-04-22 22:32:16] (step=0053875) Train Loss: 6.3602, Train Steps/Sec: 0.85 + 32%|████████████████████████████████████████████████████████▌ | 12996/40903 [4:49:41<9:04:37, 1.17s/it][2025-04-22 22:32:46] (step=0053900) Train Loss: 6.3368, Train Steps/Sec: 0.84 + 32%|████████████████████████████████████████████████████████▋ | 13021/40903 [4:50:11<9:17:33, 1.20s/it][2025-04-22 22:33:16] (step=0053925) Train Loss: 6.3475, Train Steps/Sec: 0.85 + 32%|████████████████████████████████████████████████████████▊ | 13046/40903 [4:50:40<9:06:32, 1.18s/it][2025-04-22 22:33:45] (step=0053950) Train Loss: 6.3465, Train Steps/Sec: 0.85 + 32%|████████████████████████████████████████████████████████▉ | 13071/40903 [4:51:10<9:03:41, 1.17s/it][2025-04-22 22:34:15] (step=0053975) Train Loss: 6.4114, Train Steps/Sec: 0.84 + 32%|████████████████████████████████████████████████████████▉ | 13096/40903 [4:51:39<9:00:19, 1.17s/it][2025-04-22 22:34:44] (step=0054000) Train Loss: 6.3212, Train Steps/Sec: 0.85 +You are using a model of type instructblip to instantiate a model of type blip-2. This is not supported for all configurations of models and can yield errors. +model_name_or_path:/tmp/haozhezhao/model/instructblip-flan-t5-xl +image_place_holder: +Some kwargs in processor config are unused and will not have any effect: num_query_tokens. +tokenizer length after expend 32102 +tokenizer length before expend 32102 +100%|██████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████| 3/3 [02:53<00:00, 57.70s/it] +[2025-04-22 22:39:26] Finish Eval in 54000 steps...███████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████| 3/3 [02:52<00:00, 57.29s/it] +processing left-padding... +processing left-padding... +processing left-padding... +[2025-04-22 22:39:47] Saved checkpoint to checkpoint/BeforeFix_Blip_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_objects/002-GPT-XL/checkpoints/0054000.pt +[2025-04-22 22:39:49] Removed old checkpoint: checkpoint/BeforeFix_Blip_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_objects/002-GPT-XL/checkpoints/0050000.pt + 32%|█████████████████████████████████████████████████████████ | 13121/40903 [4:57:14<9:25:25, 1.22s/it][2025-04-22 22:40:19] (step=0054025) Train Loss: 6.3558, Train Steps/Sec: 0.07 + 32%|█████████████████████████████████████████████████████████▏ | 13146/40903 [4:57:44<9:13:54, 1.20s/it][2025-04-22 22:40:49] (step=0054050) Train Loss: 6.3873, Train Steps/Sec: 0.84 + 32%|█████████████████████████████████████████████████████████▎ | 13171/40903 [4:58:13<9:06:33, 1.18s/it][2025-04-22 22:41:18] (step=0054075) Train Loss: 6.3549, Train Steps/Sec: 0.85 + 32%|█████████████████████████████████████████████████████████▍ | 13196/40903 [4:58:43<8:54:10, 1.16s/it][2025-04-22 22:41:48] (step=0054100) Train Loss: 6.2926, Train Steps/Sec: 0.85 + 32%|█████████████████████████████████████████████████████████▌ | 13221/40903 [4:59:12<9:21:57, 1.22s/it][2025-04-22 22:42:17] (step=0054125) Train Loss: 6.3421, Train Steps/Sec: 0.84 + 32%|█████████████████████████████████████████████████████████▋ | 13246/40903 [4:59:42<9:00:46, 1.17s/it][2025-04-22 22:42:47] (step=0054150) Train Loss: 6.3640, Train Steps/Sec: 0.85 + 32%|█████████████████████████████████████████████████████████▊ | 13271/40903 [5:00:11<8:57:21, 1.17s/it][2025-04-22 22:43:16] (step=0054175) Train Loss: 6.3310, Train Steps/Sec: 0.85 + 33%|█████████████████████████████████████████████████████████▊ | 13296/40903 [5:00:41<8:58:27, 1.17s/it][2025-04-22 22:43:46] (step=0054200) Train Loss: 6.3536, Train Steps/Sec: 0.84 + 33%|█████████████████████████████████████████████████████████▉ | 13321/40903 [5:01:11<9:18:54, 1.22s/it][2025-04-22 22:44:16] (step=0054225) Train Loss: 6.3540, Train Steps/Sec: 0.83 + 33%|██████████████████████████████████████████████████████████ | 13346/40903 [5:01:41<9:06:35, 1.19s/it][2025-04-22 22:44:45] (step=0054250) Train Loss: 6.3510, Train Steps/Sec: 0.84 + 33%|██████████████████████████████████████████████████████████▏ | 13371/40903 [5:02:10<8:55:58, 1.17s/it][2025-04-22 22:45:15] (step=0054275) Train Loss: 6.3500, Train Steps/Sec: 0.85 + 33%|██████████████████████████████████████████████████████████▎ | 13396/40903 [5:02:40<8:55:41, 1.17s/it][2025-04-22 22:45:45] (step=0054300) Train Loss: 6.3317, Train Steps/Sec: 0.85 + 33%|██████████████████████████████████████████████████████████▍ | 13421/40903 [5:03:09<9:11:07, 1.20s/it][2025-04-22 22:46:14] (step=0054325) Train Loss: 6.4095, Train Steps/Sec: 0.85 + 33%|██████████████████████████████████████████████████████████▌ | 13446/40903 [5:03:39<9:03:51, 1.19s/it][2025-04-22 22:46:44] (step=0054350) Train Loss: 6.3225, Train Steps/Sec: 0.84 + 33%|██████████████████████████████████████████████████████████▌ | 13471/40903 [5:04:08<9:02:59, 1.19s/it][2025-04-22 22:47:13] (step=0054375) Train Loss: 6.3592, Train Steps/Sec: 0.85 + 33%|██████████████████████████████████████████████████████████▋ | 13496/40903 [5:04:38<8:56:24, 1.17s/it][2025-04-22 22:47:43] (step=0054400) Train Loss: 6.3660, Train Steps/Sec: 0.84 + 33%|██████████████████████████████████████████████████████████▊ | 13521/40903 [5:05:07<9:01:34, 1.19s/it][2025-04-22 22:48:12] (step=0054425) Train Loss: 6.3191, Train Steps/Sec: 0.85 + 33%|██████████████████████████████████████████████████████████▉ | 13546/40903 [5:05:37<9:03:29, 1.19s/it][2025-04-22 22:48:42] (step=0054450) Train Loss: 6.3248, Train Steps/Sec: 0.84 + 33%|███████████████████████████████████████████████████████████ | 13571/40903 [5:06:07<9:00:34, 1.19s/it][2025-04-22 22:49:11] (step=0054475) Train Loss: 6.3690, Train Steps/Sec: 0.84 + 33%|███████████████████████████████████████████████████████████▏ | 13596/40903 [5:06:36<8:52:05, 1.17s/it][2025-04-22 22:49:41] (step=0054500) Train Loss: 6.3310, Train Steps/Sec: 0.84 + 33%|███████████████████████████████████████████████████████████▎ | 13621/40903 [5:07:06<9:06:54, 1.20s/it][2025-04-22 22:50:11] (step=0054525) Train Loss: 6.2947, Train Steps/Sec: 0.84 + 33%|███████████████████████████████████████████████████████████▍ | 13646/40903 [5:07:36<8:58:39, 1.19s/it][2025-04-22 22:50:40] (step=0054550) Train Loss: 6.3424, Train Steps/Sec: 0.85 + 33%|███████████████████████████████████████████████████████████▍ | 13671/40903 [5:08:12<8:53:03, 1.17s/it][2025-04-22 22:51:17] (step=0054575) Train Loss: 6.3365, Train Steps/Sec: 0.69 + 33%|███████████████████████████████████████████████████████████▌ | 13696/40903 [5:08:41<8:53:31, 1.18s/it][2025-04-22 22:51:46] (step=0054600) Train Loss: 6.3928, Train Steps/Sec: 0.84 + 34%|███████████████████████████████████████████████████████████▋ | 13721/40903 [5:09:11<9:01:00, 1.19s/it][2025-04-22 22:52:16] (step=0054625) Train Loss: 6.3631, Train Steps/Sec: 0.85 + 34%|███████████████████████████████████████████████████████████▊ | 13746/40903 [5:09:41<9:01:20, 1.20s/it][2025-04-22 22:52:45] (step=0054650) Train Loss: 6.3537, Train Steps/Sec: 0.84 + 34%|███████████████████████████████████████████████████████████▉ | 13771/40903 [5:10:10<8:56:25, 1.19s/it][2025-04-22 22:53:15] (step=0054675) Train Loss: 6.3417, Train Steps/Sec: 0.84 + 34%|████████████████████████████████████████████████████████████ | 13796/40903 [5:10:40<8:40:25, 1.15s/it][2025-04-22 22:53:45] (step=0054700) Train Loss: 6.3901, Train Steps/Sec: 0.85 + 34%|████████████████████████████████████████████████████████████▏ | 13821/40903 [5:11:09<8:59:57, 1.20s/it][2025-04-22 22:54:14] (step=0054725) Train Loss: 6.3204, Train Steps/Sec: 0.85 + 34%|████████████████████████████████████████████████████████████▎ | 13846/40903 [5:11:39<8:52:52, 1.18s/it][2025-04-22 22:54:44] (step=0054750) Train Loss: 6.3846, Train Steps/Sec: 0.84 + 34%|████████████████████████████████████████████████████████████▎ | 13871/40903 [5:12:15<8:59:47, 1.20s/it][2025-04-22 22:55:20] (step=0054775) Train Loss: 6.3784, Train Steps/Sec: 0.69 + 34%|████████████████████████████████████████████████████████████▍ | 13896/40903 [5:12:45<8:44:50, 1.17s/it][2025-04-22 22:55:50] (step=0054800) Train Loss: 6.3303, Train Steps/Sec: 0.84 + 34%|████████████████████████████████████████████████████████████▌ | 13921/40903 [5:13:15<9:03:19, 1.21s/it][2025-04-22 22:56:19] (step=0054825) Train Loss: 6.3437, Train Steps/Sec: 0.84 + 34%|████████████████████████████████████████████████████████████▋ | 13946/40903 [5:13:44<8:58:25, 1.20s/it][2025-04-22 22:56:49] (step=0054850) Train Loss: 6.3703, Train Steps/Sec: 0.85 + 34%|████████████████████████████████████████████████████████████▊ | 13971/40903 [5:14:14<8:49:14, 1.18s/it][2025-04-22 22:57:19] (step=0054875) Train Loss: 6.3478, Train Steps/Sec: 0.84 + 34%|████████████████████████████████████████████████████████████▉ | 13996/40903 [5:14:43<8:41:53, 1.16s/it][2025-04-22 22:57:48] (step=0054900) Train Loss: 6.3611, Train Steps/Sec: 0.84 + 34%|█████████████████████████████████████████████████████████████ | 14021/40903 [5:15:13<8:53:04, 1.19s/it][2025-04-22 22:58:18] (step=0054925) Train Loss: 6.3167, Train Steps/Sec: 0.85 + 34%|█████████████████████████████████████████████████████████████ | 14046/40903 [5:15:43<8:49:35, 1.18s/it][2025-04-22 22:58:47] (step=0054950) Train Loss: 6.3351, Train Steps/Sec: 0.85 + 34%|█████████████████████████████████████████████████████████████▏ | 14071/40903 [5:16:12<8:42:18, 1.17s/it][2025-04-22 22:59:17] (step=0054975) Train Loss: 6.3810, Train Steps/Sec: 0.85 + 34%|█████████████████████████████████████████████████████████████▎ | 14096/40903 [5:16:41<8:38:24, 1.16s/it][2025-04-22 22:59:46] (step=0055000) Train Loss: 6.3776, Train Steps/Sec: 0.85 + 35%|█████████████████████████████████████████████████████████████▍ | 14121/40903 [5:17:11<9:00:15, 1.21s/it][2025-04-22 23:00:16] (step=0055025) Train Loss: 6.3506, Train Steps/Sec: 0.85 + 35%|█████████████████████████████████████████████████████████████▌ | 14146/40903 [5:17:41<9:00:10, 1.21s/it][2025-04-22 23:00:46] (step=0055050) Train Loss: 6.3278, Train Steps/Sec: 0.84 + 35%|█████████████████████████████████████████████████████████████▋ | 14171/40903 [5:18:10<8:39:47, 1.17s/it][2025-04-22 23:01:15] (step=0055075) Train Loss: 6.3490, Train Steps/Sec: 0.85 + 35%|█████████████████████████████████████████████████████████████▊ | 14196/40903 [5:18:40<8:40:05, 1.17s/it][2025-04-22 23:01:45] (step=0055100) Train Loss: 6.3454, Train Steps/Sec: 0.85 + 35%|█████████████████████████████████████████████████████████████▉ | 14221/40903 [5:19:10<8:59:53, 1.21s/it][2025-04-22 23:02:14] (step=0055125) Train Loss: 6.3790, Train Steps/Sec: 0.84 + 35%|█████████████████████████████████████████████████████████████▉ | 14246/40903 [5:19:39<8:45:24, 1.18s/it][2025-04-22 23:02:44] (step=0055150) Train Loss: 6.4077, Train Steps/Sec: 0.85 + 35%|██████████████████████████████████████████████████████████████ | 14271/40903 [5:20:09<8:39:24, 1.17s/it][2025-04-22 23:03:14] (step=0055175) Train Loss: 6.3145, Train Steps/Sec: 0.84 + 35%|██████████████████████████████████████████████████████████████▏ | 14296/40903 [5:20:38<8:41:47, 1.18s/it][2025-04-22 23:03:43] (step=0055200) Train Loss: 6.3769, Train Steps/Sec: 0.84 + 35%|██████████████████████████████████████████████████████████████▎ | 14321/40903 [5:21:08<8:52:06, 1.20s/it][2025-04-22 23:04:13] (step=0055225) Train Loss: 6.3337, Train Steps/Sec: 0.85 + 35%|██████████████████████████████████████████████████████████████▍ | 14346/40903 [5:21:38<8:42:49, 1.18s/it][2025-04-22 23:04:42] (step=0055250) Train Loss: 6.4037, Train Steps/Sec: 0.84 + 35%|██████████████████████████████████████████████████████████████▌ | 14371/40903 [5:22:13<9:24:17, 1.28s/it][2025-04-22 23:05:18] (step=0055275) Train Loss: 6.3586, Train Steps/Sec: 0.71 + 35%|██████████████████████████████████████████████████████████████▋ | 14396/40903 [5:22:42<8:40:25, 1.18s/it][2025-04-22 23:05:47] (step=0055300) Train Loss: 6.3134, Train Steps/Sec: 0.84 + 35%|██████████████████████████████████████████████████████████████▊ | 14421/40903 [5:23:12<8:45:33, 1.19s/it][2025-04-22 23:06:17] (step=0055325) Train Loss: 6.3562, Train Steps/Sec: 0.85 + 35%|██████████████████████████████████████████████████████████████▊ | 14446/40903 [5:23:42<8:45:18, 1.19s/it][2025-04-22 23:06:47] (step=0055350) Train Loss: 6.3906, Train Steps/Sec: 0.85 + 35%|██████████████████████████████████████████████████████████████▉ | 14471/40903 [5:24:18<8:35:56, 1.17s/it][2025-04-22 23:07:23] (step=0055375) Train Loss: 6.3679, Train Steps/Sec: 0.69 + 35%|███████████████████████████████████████████████████████████████ | 14496/40903 [5:24:47<8:34:23, 1.17s/it][2025-04-22 23:07:52] (step=0055400) Train Loss: 6.3516, Train Steps/Sec: 0.84 + 36%|███████████████████████████████████████████████████████████████▏ | 14521/40903 [5:25:17<8:45:42, 1.20s/it][2025-04-22 23:08:22] (step=0055425) Train Loss: 6.3204, Train Steps/Sec: 0.85 + 36%|███████████████████████████████████████████████████████████████▎ | 14546/40903 [5:25:47<8:40:46, 1.19s/it][2025-04-22 23:08:51] (step=0055450) Train Loss: 6.3659, Train Steps/Sec: 0.84 + 36%|███████████████████████████████████████████████████████████████▍ | 14571/40903 [5:26:16<8:37:47, 1.18s/it][2025-04-22 23:09:21] (step=0055475) Train Loss: 6.3890, Train Steps/Sec: 0.84 + 36%|███████████████████████████████████████████████████████████████▌ | 14596/40903 [5:26:46<8:36:20, 1.18s/it][2025-04-22 23:09:51] (step=0055500) Train Loss: 6.3438, Train Steps/Sec: 0.84 + 36%|███████████████████████████████████████████████████████████████▋ | 14621/40903 [5:27:16<8:43:51, 1.20s/it][2025-04-22 23:10:20] (step=0055525) Train Loss: 6.3565, Train Steps/Sec: 0.85 + 36%|███████████████████████████████████████████████████████████████▋ | 14646/40903 [5:27:45<8:37:22, 1.18s/it][2025-04-22 23:10:50] (step=0055550) Train Loss: 6.3329, Train Steps/Sec: 0.85 + 36%|███████████████████████████████████████████████████████████████▊ | 14671/40903 [5:28:14<8:33:56, 1.18s/it][2025-04-22 23:11:19] (step=0055575) Train Loss: 6.3456, Train Steps/Sec: 0.85 + 36%|███████████████████████████████████████████████████████████████▉ | 14696/40903 [5:28:44<8:35:14, 1.18s/it][2025-04-22 23:11:49] (step=0055600) Train Loss: 6.3777, Train Steps/Sec: 0.84 + 36%|████████████████████████████████████████████████████████████████ | 14721/40903 [5:29:14<8:42:26, 1.20s/it][2025-04-22 23:12:19] (step=0055625) Train Loss: 6.3734, Train Steps/Sec: 0.85 + 36%|████████████████████████████████████████████████████████████████▏ | 14746/40903 [5:29:43<8:41:40, 1.20s/it][2025-04-22 23:12:48] (step=0055650) Train Loss: 6.3530, Train Steps/Sec: 0.85 + 36%|████████████████████████████████████████████████████████████████▎ | 14771/40903 [5:30:13<8:33:18, 1.18s/it][2025-04-22 23:13:18] (step=0055675) Train Loss: 6.3411, Train Steps/Sec: 0.84 + 36%|████████████████████████████████████████████████████████████████▍ | 14796/40903 [5:30:43<8:33:33, 1.18s/it][2025-04-22 23:13:48] (step=0055700) Train Loss: 6.3365, Train Steps/Sec: 0.84 + 36%|████████████████████████████████████████████████████████████████▍ | 14821/40903 [5:31:12<8:44:27, 1.21s/it][2025-04-22 23:14:17] (step=0055725) Train Loss: 6.3414, Train Steps/Sec: 0.85 + 36%|████████████████████████████████████████████████████████████████▌ | 14846/40903 [5:31:42<8:29:30, 1.17s/it][2025-04-22 23:14:46] (step=0055750) Train Loss: 6.3173, Train Steps/Sec: 0.85 + 36%|████████████████████████████████████████████████████████████████▋ | 14871/40903 [5:32:11<8:28:05, 1.17s/it][2025-04-22 23:15:16] (step=0055775) Train Loss: 6.3624, Train Steps/Sec: 0.85 + 36%|████████████████████████████████████████████████████████████████▊ | 14896/40903 [5:32:40<8:20:29, 1.15s/it][2025-04-22 23:15:45] (step=0055800) Train Loss: 6.3532, Train Steps/Sec: 0.85 + 36%|████████████████████████████████████████████████████████████████▉ | 14921/40903 [5:33:10<8:41:43, 1.20s/it][2025-04-22 23:16:15] (step=0055825) Train Loss: 6.3347, Train Steps/Sec: 0.85 + 37%|█████████████████████████████████████████████████████████████████ | 14946/40903 [5:33:40<8:28:05, 1.17s/it][2025-04-22 23:16:45] (step=0055850) Train Loss: 6.3455, Train Steps/Sec: 0.85 + 37%|█████████████████████████████████████████████████████████████████▏ | 14971/40903 [5:34:09<8:28:39, 1.18s/it][2025-04-22 23:17:14] (step=0055875) Train Loss: 6.3213, Train Steps/Sec: 0.85 + 37%|█████████████████████████████████████████████████████████████████▎ | 14996/40903 [5:34:39<8:23:50, 1.17s/it][2025-04-22 23:17:44] (step=0055900) Train Loss: 6.3595, Train Steps/Sec: 0.84 + 37%|█████████████████████████████████████████████████████████████████▎ | 15021/40903 [5:35:08<8:39:54, 1.21s/it][2025-04-22 23:18:13] (step=0055925) Train Loss: 6.4034, Train Steps/Sec: 0.84 + 37%|█████████████████████████████████████████████████████████████████▍ | 15046/40903 [5:35:38<8:35:03, 1.20s/it][2025-04-22 23:18:43] (step=0055950) Train Loss: 6.3050, Train Steps/Sec: 0.85 + 37%|█████████████████████████████████████████████████████████████████▌ | 15071/40903 [5:36:08<8:26:45, 1.18s/it][2025-04-22 23:19:12] (step=0055975) Train Loss: 6.3751, Train Steps/Sec: 0.84 + 37%|█████████████████████████████████████████████████████████████████▋ | 15096/40903 [5:36:37<8:29:15, 1.18s/it][2025-04-22 23:19:42] (step=0056000) Train Loss: 6.4245, Train Steps/Sec: 0.84 +You are using a model of type instructblip to instantiate a model of type blip-2. This is not supported for all configurations of models and can yield errors. +model_name_or_path:/tmp/haozhezhao/model/instructblip-flan-t5-xl +image_place_holder: +Some kwargs in processor config are unused and will not have any effect: num_query_tokens. +tokenizer length after expend 32102 +tokenizer length before expend 32102 +100%|██████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████| 3/3 [02:54<00:00, 58.00s/it] +[2025-04-22 23:24:26] Finish Eval in 56000 steps...███████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████| 3/3 [02:53<00:00, 57.52s/it] +processing left-padding... +processing left-padding... +processing left-padding... +[2025-04-22 23:24:47] Saved checkpoint to checkpoint/BeforeFix_Blip_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_objects/002-GPT-XL/checkpoints/0056000.pt +[2025-04-22 23:24:49] Removed old checkpoint: checkpoint/BeforeFix_Blip_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_objects/002-GPT-XL/checkpoints/0052000.pt + 37%|█████████████████████████████████████████████████████████████████▊ | 15121/40903 [5:42:14<8:43:27, 1.22s/it][2025-04-22 23:25:19] (step=0056025) Train Loss: 6.3377, Train Steps/Sec: 0.07 + 37%|█████████████████████████████████████████████████████████████████▉ | 15146/40903 [5:42:43<8:22:36, 1.17s/it][2025-04-22 23:25:48] (step=0056050) Train Loss: 6.3397, Train Steps/Sec: 0.85 + 37%|██████████████████████████████████████████████████████████████████ | 15171/40903 [5:43:13<8:19:35, 1.16s/it][2025-04-22 23:26:18] (step=0056075) Train Loss: 6.3903, Train Steps/Sec: 0.84 + 37%|██████████████████████████████████████████████████████████████████▏ | 15196/40903 [5:43:42<8:21:53, 1.17s/it][2025-04-22 23:26:47] (step=0056100) Train Loss: 6.3491, Train Steps/Sec: 0.84 + 37%|██████████████████████████████████████████████████████████████████▏ | 15221/40903 [5:44:12<8:30:49, 1.19s/it][2025-04-22 23:27:17] (step=0056125) Train Loss: 6.3250, Train Steps/Sec: 0.85 + 37%|██████████████████████████████████████████████████████████████████▎ | 15246/40903 [5:44:41<8:24:13, 1.18s/it][2025-04-22 23:27:46] (step=0056150) Train Loss: 6.3073, Train Steps/Sec: 0.85 + 37%|██████████████████████████████████████████████████████████████████▍ | 15271/40903 [5:45:11<8:17:08, 1.16s/it][2025-04-22 23:28:16] (step=0056175) Train Loss: 6.3217, Train Steps/Sec: 0.85 + 37%|██████████████████████████████████████████████████████████████████▌ | 15296/40903 [5:45:40<8:24:16, 1.18s/it][2025-04-22 23:28:45] (step=0056200) Train Loss: 6.3457, Train Steps/Sec: 0.85 + 37%|██████████████████████████████████████████████████████████████████▋ | 15321/40903 [5:46:10<8:28:17, 1.19s/it][2025-04-22 23:29:15] (step=0056225) Train Loss: 6.3093, Train Steps/Sec: 0.85 + 38%|██████████████████████████████████████████████████████████████████▊ | 15346/40903 [5:46:39<8:17:30, 1.17s/it][2025-04-22 23:29:44] (step=0056250) Train Loss: 6.3495, Train Steps/Sec: 0.85 + 38%|██████████████████████████████████████████████████████████████████▉ | 15371/40903 [5:47:09<8:26:16, 1.19s/it][2025-04-22 23:30:14] (step=0056275) Train Loss: 6.3593, Train Steps/Sec: 0.84 + 38%|██████████████████████████████████████████████████████████████████▉ | 15396/40903 [5:47:39<8:19:56, 1.18s/it][2025-04-22 23:30:44] (step=0056300) Train Loss: 6.3292, Train Steps/Sec: 0.84 + 38%|███████████████████████████████████████████████████████████████████ | 15421/40903 [5:48:08<8:28:41, 1.20s/it][2025-04-22 23:31:13] (step=0056325) Train Loss: 6.3496, Train Steps/Sec: 0.85 + 38%|███████████████████████████████████████████████████████████████████▏ | 15446/40903 [5:48:38<8:22:54, 1.19s/it][2025-04-22 23:31:43] (step=0056350) Train Loss: 6.3584, Train Steps/Sec: 0.85 + 38%|███████████████████████████████████████████████████████████████████▎ | 15471/40903 [5:49:08<8:19:36, 1.18s/it][2025-04-22 23:32:12] (step=0056375) Train Loss: 6.3291, Train Steps/Sec: 0.84 + 38%|███████████████████████████████████████████████████████████████████▍ | 15496/40903 [5:49:37<8:12:41, 1.16s/it][2025-04-22 23:32:42] (step=0056400) Train Loss: 6.3568, Train Steps/Sec: 0.84 + 38%|███████████████████████████████████████████████████████████████████▌ | 15521/40903 [5:50:07<8:33:53, 1.21s/it][2025-04-22 23:33:12] (step=0056425) Train Loss: 6.3449, Train Steps/Sec: 0.84 + 38%|███████████████████████████████████████████████████████████████████▋ | 15546/40903 [5:50:37<8:22:22, 1.19s/it][2025-04-22 23:33:41] (step=0056450) Train Loss: 6.2899, Train Steps/Sec: 0.84 + 38%|███████████████████████████████████████████████████████████████████▊ | 15571/40903 [5:51:06<8:18:49, 1.18s/it][2025-04-22 23:34:11] (step=0056475) Train Loss: 6.3712, Train Steps/Sec: 0.85 + 38%|███████████████████████████████████████████████████████████████████▊ | 15596/40903 [5:51:36<8:11:48, 1.17s/it][2025-04-22 23:34:41] (step=0056500) Train Loss: 6.3364, Train Steps/Sec: 0.85 + 38%|███████████████████████████████████████████████████████████████████▉ | 15621/40903 [5:52:05<8:23:56, 1.20s/it][2025-04-22 23:35:10] (step=0056525) Train Loss: 6.3889, Train Steps/Sec: 0.84 + 38%|████████████████████████████████████████████████████████████████████ | 15646/40903 [5:52:35<8:11:13, 1.17s/it][2025-04-22 23:35:40] (step=0056550) Train Loss: 6.3981, Train Steps/Sec: 0.85 + 38%|████████████████████████████████████████████████████████████████████▏ | 15671/40903 [5:53:04<8:19:17, 1.19s/it][2025-04-22 23:36:09] (step=0056575) Train Loss: 6.3067, Train Steps/Sec: 0.84 + 38%|████████████████████████████████████████████████████████████████████▎ | 15696/40903 [5:53:34<8:11:30, 1.17s/it][2025-04-22 23:36:39] (step=0056600) Train Loss: 6.3754, Train Steps/Sec: 0.85 + 38%|████████████████████████████████████████████████████████████████████▍ | 15721/40903 [5:54:04<8:20:05, 1.19s/it][2025-04-22 23:37:08] (step=0056625) Train Loss: 6.3566, Train Steps/Sec: 0.85 + 38%|████████████████████████████████████████████████████████████████████▌ | 15746/40903 [5:54:33<8:15:07, 1.18s/it][2025-04-22 23:37:38] (step=0056650) Train Loss: 6.3214, Train Steps/Sec: 0.85 + 39%|████████████████████████████████████████████████████████████████████▋ | 15771/40903 [5:55:02<8:11:38, 1.17s/it][2025-04-22 23:38:07] (step=0056675) Train Loss: 6.3450, Train Steps/Sec: 0.85 + 39%|████████████████████████████████████████████████████████████████████▋ | 15796/40903 [5:55:32<8:10:44, 1.17s/it][2025-04-22 23:38:37] (step=0056700) Train Loss: 6.3693, Train Steps/Sec: 0.84 + 39%|████████████████████████████████████████████████████████████████████▊ | 15821/40903 [5:56:02<8:21:21, 1.20s/it][2025-04-22 23:39:07] (step=0056725) Train Loss: 6.3612, Train Steps/Sec: 0.85 + 39%|████████████████████████████████████████████████████████████████████▉ | 15846/40903 [5:56:31<8:12:15, 1.18s/it][2025-04-22 23:39:36] (step=0056750) Train Loss: 6.3388, Train Steps/Sec: 0.85 + 39%|█████████████████████████████████████████████████████████████████████ | 15871/40903 [5:57:01<8:19:49, 1.20s/it][2025-04-22 23:40:06] (step=0056775) Train Loss: 6.3310, Train Steps/Sec: 0.84 + 39%|█████████████████████████████████████████████████████████████████████▏ | 15896/40903 [5:57:31<8:10:26, 1.18s/it][2025-04-22 23:40:36] (step=0056800) Train Loss: 6.3601, Train Steps/Sec: 0.83 + 39%|█████████████████████████████████████████████████████████████████████▎ | 15921/40903 [5:58:00<8:22:27, 1.21s/it][2025-04-22 23:41:05] (step=0056825) Train Loss: 6.3451, Train Steps/Sec: 0.84 + 39%|█████████████████████████████████████████████████████████████████████▍ | 15946/40903 [5:58:30<8:07:38, 1.17s/it][2025-04-22 23:41:35] (step=0056850) Train Loss: 6.3513, Train Steps/Sec: 0.85 + 39%|█████████████████████████████████████████████████████████████████████▌ | 15971/40903 [5:58:59<8:08:33, 1.18s/it][2025-04-22 23:42:04] (step=0056875) Train Loss: 6.3635, Train Steps/Sec: 0.85 + 39%|█████████████████████████████████████████████████████████████████████▌ | 15996/40903 [5:59:29<8:08:59, 1.18s/it][2025-04-22 23:42:34] (step=0056900) Train Loss: 6.3390, Train Steps/Sec: 0.84 + 39%|█████████████████████████████████████████████████████████████████████▋ | 16021/40903 [5:59:59<8:15:04, 1.19s/it][2025-04-22 23:43:04] (step=0056925) Train Loss: 6.3425, Train Steps/Sec: 0.85 + 39%|█████████████████████████████████████████████████████████████████████▊ | 16046/40903 [6:00:28<8:08:50, 1.18s/it][2025-04-22 23:43:33] (step=0056950) Train Loss: 6.3505, Train Steps/Sec: 0.85 + 39%|█████████████████████████████████████████████████████████████████████▉ | 16071/40903 [6:00:58<8:07:27, 1.18s/it][2025-04-22 23:44:03] (step=0056975) Train Loss: 6.3425, Train Steps/Sec: 0.84 + 39%|██████████████████████████████████████████████████████████████████████ | 16096/40903 [6:01:28<8:07:48, 1.18s/it][2025-04-22 23:44:32] (step=0057000) Train Loss: 6.3702, Train Steps/Sec: 0.84 + 39%|██████████████████████████████████████████████████████████████████████▏ | 16121/40903 [6:01:57<8:11:54, 1.19s/it][2025-04-22 23:45:02] (step=0057025) Train Loss: 6.3826, Train Steps/Sec: 0.85 + 39%|██████████████████████████████████████████████████████████████████████▎ | 16146/40903 [6:02:27<8:05:07, 1.18s/it][2025-04-22 23:45:31] (step=0057050) Train Loss: 6.3433, Train Steps/Sec: 0.85 + 40%|██████████████████████████████████████████████████████████████████████▎ | 16171/40903 [6:02:56<7:59:20, 1.16s/it][2025-04-22 23:46:01] (step=0057075) Train Loss: 6.3489, Train Steps/Sec: 0.85 + 40%|██████████████████████████████████████████████████████████████████████▍ | 16196/40903 [6:03:26<8:03:49, 1.17s/it][2025-04-22 23:46:31] (step=0057100) Train Loss: 6.3365, Train Steps/Sec: 0.84 + 40%|██████████████████████████████████████████████████████████████████████▌ | 16221/40903 [6:03:56<8:11:54, 1.20s/it][2025-04-22 23:47:00] (step=0057125) Train Loss: 6.3767, Train Steps/Sec: 0.85 + 40%|██████████████████████████████████████████████████████████████████████▋ | 16246/40903 [6:04:25<8:07:48, 1.19s/it][2025-04-22 23:47:30] (step=0057150) Train Loss: 6.3572, Train Steps/Sec: 0.85 + 40%|██████████████████████████████████████████████████████████████████████▊ | 16271/40903 [6:04:55<8:02:10, 1.17s/it][2025-04-22 23:47:59] (step=0057175) Train Loss: 6.3373, Train Steps/Sec: 0.85 + 40%|██████████████████████████████████████████████████████████████████████▉ | 16296/40903 [6:05:24<8:01:02, 1.17s/it][2025-04-22 23:48:29] (step=0057200) Train Loss: 6.3375, Train Steps/Sec: 0.85 + 40%|███████████████████████████████████████████████████████████████████████ | 16321/40903 [6:05:54<8:12:44, 1.20s/it][2025-04-22 23:48:59] (step=0057225) Train Loss: 6.3276, Train Steps/Sec: 0.84 + 40%|███████████████████████████████████████████████████████████████████████▏ | 16346/40903 [6:06:23<8:01:36, 1.18s/it][2025-04-22 23:49:28] (step=0057250) Train Loss: 6.3482, Train Steps/Sec: 0.84 + 40%|███████████████████████████████████████████████████████████████████████▏ | 16371/40903 [6:06:53<7:57:09, 1.17s/it][2025-04-22 23:49:58] (step=0057275) Train Loss: 6.3370, Train Steps/Sec: 0.85 + 40%|███████████████████████████████████████████████████████████████████████▎ | 16396/40903 [6:07:22<7:58:13, 1.17s/it][2025-04-22 23:50:27] (step=0057300) Train Loss: 6.3217, Train Steps/Sec: 0.84 + 40%|███████████████████████████████████████████████████████████████████████▍ | 16421/40903 [6:07:52<8:10:56, 1.20s/it][2025-04-22 23:50:57] (step=0057325) Train Loss: 6.3290, Train Steps/Sec: 0.85 + 40%|███████████████████████████████████████████████████████████████████████▌ | 16446/40903 [6:08:22<8:02:26, 1.18s/it][2025-04-22 23:51:26] (step=0057350) Train Loss: 6.3970, Train Steps/Sec: 0.85 + 40%|███████████████████████████████████████████████████████████████████████▋ | 16471/40903 [6:08:51<7:59:15, 1.18s/it][2025-04-22 23:51:56] (step=0057375) Train Loss: 6.3492, Train Steps/Sec: 0.85 + 40%|███████████████████████████████████████████████████████████████████████▊ | 16496/40903 [6:09:21<8:00:20, 1.18s/it][2025-04-22 23:52:26] (step=0057400) Train Loss: 6.3638, Train Steps/Sec: 0.84 + 40%|███████████████████████████████████████████████████████████████████████▉ | 16521/40903 [6:09:50<8:05:56, 1.20s/it][2025-04-22 23:52:55] (step=0057425) Train Loss: 6.3525, Train Steps/Sec: 0.85 + 40%|████████████████████████████████████████████████████████████████████████ | 16546/40903 [6:10:20<8:02:54, 1.19s/it][2025-04-22 23:53:25] (step=0057450) Train Loss: 6.3624, Train Steps/Sec: 0.84 + 41%|████████████████████████████████████████████████████████████████████████ | 16571/40903 [6:10:50<7:54:17, 1.17s/it][2025-04-22 23:53:54] (step=0057475) Train Loss: 6.3671, Train Steps/Sec: 0.85 + 41%|████████████████████████████████████████████████████████████████████████▏ | 16596/40903 [6:11:19<8:01:03, 1.19s/it][2025-04-22 23:54:24] (step=0057500) Train Loss: 6.3372, Train Steps/Sec: 0.84 + 41%|████████████████████████████████████████████████████████████████████████▎ | 16621/40903 [6:11:49<8:04:28, 1.20s/it][2025-04-22 23:54:54] (step=0057525) Train Loss: 6.3775, Train Steps/Sec: 0.84 + 41%|████████████████████████████████████████████████████████████████████████▍ | 16646/40903 [6:12:18<8:03:41, 1.20s/it][2025-04-22 23:55:23] (step=0057550) Train Loss: 6.3511, Train Steps/Sec: 0.84 + 41%|████████████████████████████████████████████████████████████████████████▌ | 16671/40903 [6:12:48<7:49:22, 1.16s/it][2025-04-22 23:55:53] (step=0057575) Train Loss: 6.3765, Train Steps/Sec: 0.85 + 41%|████████████████████████████████████████████████████████████████████████▋ | 16696/40903 [6:13:17<7:49:37, 1.16s/it][2025-04-22 23:56:22] (step=0057600) Train Loss: 6.3573, Train Steps/Sec: 0.85 + 41%|████████████████████████████████████████████████████████████████████████▊ | 16721/40903 [6:13:47<8:02:55, 1.20s/it][2025-04-22 23:56:52] (step=0057625) Train Loss: 6.3730, Train Steps/Sec: 0.85 + 41%|████████████████████████████████████████████████████████████████████████▊ | 16746/40903 [6:14:16<7:56:48, 1.18s/it][2025-04-22 23:57:21] (step=0057650) Train Loss: 6.3445, Train Steps/Sec: 0.85 + 41%|████████████████████████████████████████████████████████████████████████▉ | 16771/40903 [6:14:46<7:51:57, 1.17s/it][2025-04-22 23:57:51] (step=0057675) Train Loss: 6.3342, Train Steps/Sec: 0.84 + 41%|█████████████████████████████████████████████████████████████████████████ | 16796/40903 [6:15:16<7:45:21, 1.16s/it][2025-04-22 23:58:20] (step=0057700) Train Loss: 6.3634, Train Steps/Sec: 0.85 + 41%|█████████████████████████████████████████████████████████████████████████▏ | 16821/40903 [6:15:45<8:01:34, 1.20s/it][2025-04-22 23:58:50] (step=0057725) Train Loss: 6.3528, Train Steps/Sec: 0.84 + 41%|█████████████████████████████████████████████████████████████████████████▎ | 16846/40903 [6:16:15<7:58:35, 1.19s/it][2025-04-22 23:59:20] (step=0057750) Train Loss: 6.3264, Train Steps/Sec: 0.85 + 41%|█████████████████████████████████████████████████████████████████████████▍ | 16871/40903 [6:16:44<7:47:28, 1.17s/it][2025-04-22 23:59:49] (step=0057775) Train Loss: 6.3578, Train Steps/Sec: 0.84 + 41%|█████████████████████████████████████████████████████████████████████████▌ | 16896/40903 [6:17:14<7:51:00, 1.18s/it][2025-04-23 00:00:19] (step=0057800) Train Loss: 6.3556, Train Steps/Sec: 0.84 + 41%|█████████████████████████████████████████████████████████████████████████▋ | 16921/40903 [6:17:44<7:58:15, 1.20s/it][2025-04-23 00:00:49] (step=0057825) Train Loss: 6.3528, Train Steps/Sec: 0.84 + 41%|█████████████████████████████████████████████████████████████████████████▋ | 16946/40903 [6:18:13<7:52:00, 1.18s/it][2025-04-23 00:01:18] (step=0057850) Train Loss: 6.3667, Train Steps/Sec: 0.85 + 41%|█████████████████████████████████████████████████████████████████████████▊ | 16971/40903 [6:18:43<7:51:02, 1.18s/it][2025-04-23 00:01:48] (step=0057875) Train Loss: 6.2504, Train Steps/Sec: 0.85 + 42%|█████████████████████████████████████████████████████████████████████████▉ | 16996/40903 [6:19:12<7:42:51, 1.16s/it][2025-04-23 00:02:17] (step=0057900) Train Loss: 6.3355, Train Steps/Sec: 0.84 + 42%|██████████████████████████████████████████████████████████████████████████ | 17021/40903 [6:19:42<7:54:34, 1.19s/it][2025-04-23 00:02:47] (step=0057925) Train Loss: 6.3522, Train Steps/Sec: 0.84 + 42%|██████████████████████████████████████████████████████████████████████████▏ | 17046/40903 [6:20:12<7:51:29, 1.19s/it][2025-04-23 00:03:16] (step=0057950) Train Loss: 6.3422, Train Steps/Sec: 0.85 + 42%|██████████████████████████████████████████████████████████████████████████▎ | 17071/40903 [6:20:41<7:46:40, 1.17s/it][2025-04-23 00:03:46] (step=0057975) Train Loss: 6.3737, Train Steps/Sec: 0.85 + 42%|██████████████████████████████████████████████████████████████████████████▍ | 17096/40903 [6:21:11<7:48:54, 1.18s/it][2025-04-23 00:04:16] (step=0058000) Train Loss: 6.3704, Train Steps/Sec: 0.84 +You are using a model of type instructblip to instantiate a model of type blip-2. This is not supported for all configurations of models and can yield errors. +model_name_or_path:/tmp/haozhezhao/model/instructblip-flan-t5-xl +image_place_holder: +Some kwargs in processor config are unused and will not have any effect: num_query_tokens. +tokenizer length after expend 32102 +tokenizer length before expend 32102 +100%|██████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████| 3/3 [02:53<00:00, 57.72s/it] +[2025-04-23 00:08:58] Finish Eval in 58000 steps...███████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████| 3/3 [02:52<00:00, 57.30s/it] +processing left-padding... +processing left-padding... +processing left-padding... +[2025-04-23 00:09:19] Saved checkpoint to checkpoint/BeforeFix_Blip_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_objects/002-GPT-XL/checkpoints/0058000.pt +[2025-04-23 00:09:21] Removed old checkpoint: checkpoint/BeforeFix_Blip_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_objects/002-GPT-XL/checkpoints/0054000.pt + 42%|██████████████████████████████████████████████████████████████████████████▌ | 17121/40903 [6:26:46<8:04:59, 1.22s/it][2025-04-23 00:09:51] (step=0058025) Train Loss: 6.3290, Train Steps/Sec: 0.07 + 42%|██████████████████████████████████████████████████████████████████████████▌ | 17146/40903 [6:27:16<7:50:39, 1.19s/it][2025-04-23 00:10:21] (step=0058050) Train Loss: 6.3338, Train Steps/Sec: 0.84 + 42%|██████████████████████████████████████████████████████████████████████████▋ | 17171/40903 [6:27:45<7:43:17, 1.17s/it][2025-04-23 00:10:50] (step=0058075) Train Loss: 6.3431, Train Steps/Sec: 0.85 + 42%|██████████████████████████████████████████████████████████████████████████▊ | 17196/40903 [6:28:15<7:45:14, 1.18s/it][2025-04-23 00:11:20] (step=0058100) Train Loss: 6.3163, Train Steps/Sec: 0.85 + 42%|██████████████████████████████████████████████████████████████████████████▉ | 17221/40903 [6:28:44<7:58:55, 1.21s/it][2025-04-23 00:11:49] (step=0058125) Train Loss: 6.3915, Train Steps/Sec: 0.84 + 42%|███████████████████████████████████████████████████████████████████████████ | 17246/40903 [6:29:14<7:43:35, 1.18s/it][2025-04-23 00:12:19] (step=0058150) Train Loss: 6.3559, Train Steps/Sec: 0.85 + 42%|███████████████████████████████████████████████████████████████████████████▏ | 17271/40903 [6:29:44<7:46:27, 1.18s/it][2025-04-23 00:12:48] (step=0058175) Train Loss: 6.3612, Train Steps/Sec: 0.84 + 42%|███████████████████████████████████████████████████████████████████████████▎ | 17296/40903 [6:30:20<7:38:22, 1.17s/it][2025-04-23 00:13:24] (step=0058200) Train Loss: 6.3006, Train Steps/Sec: 0.69 + 42%|███████████████████████████████████████████████████████████████████████████▍ | 17321/40903 [6:30:49<8:00:24, 1.22s/it][2025-04-23 00:13:54] (step=0058225) Train Loss: 6.3712, Train Steps/Sec: 0.84 + 42%|███████████████████████████████████████████████████████████████████████████▍ | 17346/40903 [6:31:19<7:44:29, 1.18s/it][2025-04-23 00:14:24] (step=0058250) Train Loss: 6.3507, Train Steps/Sec: 0.84 + 42%|███████████████████████████████████████████████████████████████████████████▌ | 17371/40903 [6:31:48<7:42:02, 1.18s/it][2025-04-23 00:14:53] (step=0058275) Train Loss: 6.3322, Train Steps/Sec: 0.85 + 43%|███████████████████████████████████████████████████████████████████████████▋ | 17396/40903 [6:32:18<7:30:18, 1.15s/it][2025-04-23 00:15:23] (step=0058300) Train Loss: 6.3266, Train Steps/Sec: 0.85 + 43%|███████████████████████████████████████████████████████████████████████████▊ | 17421/40903 [6:32:48<7:50:20, 1.20s/it][2025-04-23 00:15:52] (step=0058325) Train Loss: 6.3670, Train Steps/Sec: 0.84 + 43%|███████████████████████████████████████████████████████████████████████████▉ | 17446/40903 [6:33:17<7:42:11, 1.18s/it][2025-04-23 00:16:22] (step=0058350) Train Loss: 6.3547, Train Steps/Sec: 0.85 + 43%|████████████████████████████████████████████████████████████████████████████ | 17471/40903 [6:33:47<7:41:44, 1.18s/it][2025-04-23 00:16:52] (step=0058375) Train Loss: 6.3786, Train Steps/Sec: 0.84 + 43%|████████████████████████████████████████████████████████████████████████████▏ | 17496/40903 [6:34:16<7:34:15, 1.16s/it][2025-04-23 00:17:21] (step=0058400) Train Loss: 6.3510, Train Steps/Sec: 0.85 + 43%|████████████████████████████████████████████████████████████████████████████▏ | 17521/40903 [6:34:46<7:48:25, 1.20s/it][2025-04-23 00:17:51] (step=0058425) Train Loss: 6.3942, Train Steps/Sec: 0.84 + 43%|████████████████████████████████████████████████████████████████████████████▎ | 17546/40903 [6:35:16<7:37:38, 1.18s/it][2025-04-23 00:18:21] (step=0058450) Train Loss: 6.3586, Train Steps/Sec: 0.85 + 43%|████████████████████████████████████████████████████████████████████████████▍ | 17571/40903 [6:35:45<7:36:21, 1.17s/it][2025-04-23 00:18:50] (step=0058475) Train Loss: 6.3456, Train Steps/Sec: 0.85 + 43%|████████████████████████████████████████████████████████████████████████████▌ | 17596/40903 [6:36:23<8:00:55, 1.24s/it][2025-04-23 00:19:27] (step=0058500) Train Loss: 6.3676, Train Steps/Sec: 0.67 + 43%|████████████████████████████████████████████████████████████████████████████▋ | 17621/40903 [6:36:52<7:37:01, 1.18s/it][2025-04-23 00:19:57] (step=0058525) Train Loss: 6.3509, Train Steps/Sec: 0.85 + 43%|████████████████████████████████████████████████████████████████████████████▊ | 17646/40903 [6:37:22<7:43:52, 1.20s/it][2025-04-23 00:20:27] (step=0058550) Train Loss: 6.3403, Train Steps/Sec: 0.83 + 43%|████████████████████████████████████████████████████████████████████████████▉ | 17671/40903 [6:37:52<7:31:34, 1.17s/it][2025-04-23 00:20:56] (step=0058575) Train Loss: 6.3165, Train Steps/Sec: 0.85 + 43%|█████████████████████████████████████████████████████████████████████████████ | 17696/40903 [6:38:21<7:33:31, 1.17s/it][2025-04-23 00:21:26] (step=0058600) Train Loss: 6.3961, Train Steps/Sec: 0.84 + 43%|█████████████████████████████████████████████████████████████████████████████ | 17721/40903 [6:38:51<7:46:32, 1.21s/it][2025-04-23 00:21:56] (step=0058625) Train Loss: 6.3610, Train Steps/Sec: 0.84 + 43%|█████████████████████████████████████████████████████████████████████████████▏ | 17746/40903 [6:39:21<7:35:15, 1.18s/it][2025-04-23 00:22:25] (step=0058650) Train Loss: 6.3470, Train Steps/Sec: 0.85 + 43%|█████████████████████████████████████████████████████████████████████████████▎ | 17771/40903 [6:39:50<7:32:37, 1.17s/it][2025-04-23 00:22:55] (step=0058675) Train Loss: 6.3559, Train Steps/Sec: 0.85 + 44%|█████████████████████████████████████████████████████████████████████████████▍ | 17796/40903 [6:40:20<7:35:10, 1.18s/it][2025-04-23 00:23:25] (step=0058700) Train Loss: 6.3629, Train Steps/Sec: 0.84 + 44%|█████████████████████████████████████████████████████████████████████████████▌ | 17821/40903 [6:40:49<7:45:05, 1.21s/it][2025-04-23 00:23:54] (step=0058725) Train Loss: 6.3521, Train Steps/Sec: 0.85 + 44%|█████████████████████████████████████████████████████████████████████████████▋ | 17846/40903 [6:41:19<7:30:59, 1.17s/it][2025-04-23 00:24:24] (step=0058750) Train Loss: 6.3274, Train Steps/Sec: 0.85 + 44%|█████████████████████████████████████████████████████████████████████████████▊ | 17871/40903 [6:41:48<7:34:04, 1.18s/it][2025-04-23 00:24:53] (step=0058775) Train Loss: 6.3484, Train Steps/Sec: 0.84 + 44%|█████████████████████████████████████████████████████████████████████████████▉ | 17896/40903 [6:42:18<7:24:27, 1.16s/it][2025-04-23 00:25:23] (step=0058800) Train Loss: 6.3098, Train Steps/Sec: 0.85 + 44%|█████████████████████████████████████████████████████████████████████████████▉ | 17921/40903 [6:42:48<7:44:13, 1.21s/it][2025-04-23 00:25:52] (step=0058825) Train Loss: 6.3793, Train Steps/Sec: 0.85 + 44%|██████████████████████████████████████████████████████████████████████████████ | 17946/40903 [6:43:17<7:33:29, 1.19s/it][2025-04-23 00:26:22] (step=0058850) Train Loss: 6.3275, Train Steps/Sec: 0.85 + 44%|██████████████████████████████████████████████████████████████████████████████▏ | 17971/40903 [6:43:47<7:27:15, 1.17s/it][2025-04-23 00:26:51] (step=0058875) Train Loss: 6.3640, Train Steps/Sec: 0.85 + 44%|██████████████████████████████████████████████████████████████████████████████▎ | 17996/40903 [6:44:16<7:26:27, 1.17s/it][2025-04-23 00:27:21] (step=0058900) Train Loss: 6.3613, Train Steps/Sec: 0.84 + 44%|██████████████████████████████████████████████████████████████████████████████▍ | 18021/40903 [6:44:52<7:37:53, 1.20s/it][2025-04-23 00:27:57] (step=0058925) Train Loss: 6.3642, Train Steps/Sec: 0.70 + 44%|██████████████████████████████████████████████████████████████████████████████▌ | 18046/40903 [6:45:22<7:31:49, 1.19s/it][2025-04-23 00:28:27] (step=0058950) Train Loss: 6.4144, Train Steps/Sec: 0.85 + 44%|██████████████████████████████████████████████████████████████████████████████▋ | 18071/40903 [6:45:51<7:32:49, 1.19s/it][2025-04-23 00:28:56] (step=0058975) Train Loss: 6.3442, Train Steps/Sec: 0.84 + 44%|██████████████████████████████████████████████████████████████████████████████▋ | 18096/40903 [6:46:21<7:30:22, 1.18s/it][2025-04-23 00:29:26] (step=0059000) Train Loss: 6.3802, Train Steps/Sec: 0.84 + 44%|██████████████████████████████████████████████████████████████████████████████▊ | 18121/40903 [6:46:51<7:40:15, 1.21s/it][2025-04-23 00:29:56] (step=0059025) Train Loss: 6.3840, Train Steps/Sec: 0.84 + 44%|██████████████████████████████████████████████████████████████████████████████▉ | 18146/40903 [6:47:20<7:30:02, 1.19s/it][2025-04-23 00:30:25] (step=0059050) Train Loss: 6.3740, Train Steps/Sec: 0.85 + 44%|███████████████████████████████████████████████████████████████████████████████ | 18171/40903 [6:47:50<7:25:54, 1.18s/it][2025-04-23 00:30:55] (step=0059075) Train Loss: 6.3302, Train Steps/Sec: 0.84 + 44%|███████████████████████████████████████████████████████████████████████████████▏ | 18196/40903 [6:48:20<7:23:26, 1.17s/it][2025-04-23 00:31:25] (step=0059100) Train Loss: 6.3647, Train Steps/Sec: 0.84 + 45%|███████████████████████████████████████████████████████████████████████████████▎ | 18221/40903 [6:48:49<7:31:06, 1.19s/it][2025-04-23 00:31:54] (step=0059125) Train Loss: 6.3592, Train Steps/Sec: 0.85 + 45%|███████████████████████████████████████████████████████████████████████████████▍ | 18246/40903 [6:49:19<7:30:54, 1.19s/it][2025-04-23 00:32:24] (step=0059150) Train Loss: 6.3390, Train Steps/Sec: 0.84 + 45%|███████████████████████████████████████████████████████████████████████████████▌ | 18271/40903 [6:49:48<7:21:48, 1.17s/it][2025-04-23 00:32:53] (step=0059175) Train Loss: 6.3479, Train Steps/Sec: 0.85 + 45%|███████████████████████████████████████████████████████████████████████████████▌ | 18296/40903 [6:50:18<7:24:23, 1.18s/it][2025-04-23 00:33:23] (step=0059200) Train Loss: 6.3417, Train Steps/Sec: 0.84 + 45%|███████████████████████████████████████████████████████████████████████████████▋ | 18321/40903 [6:50:48<7:31:41, 1.20s/it][2025-04-23 00:33:52] (step=0059225) Train Loss: 6.3618, Train Steps/Sec: 0.85 + 45%|███████████████████████████████████████████████████████████████████████████████▊ | 18346/40903 [6:51:17<7:26:52, 1.19s/it][2025-04-23 00:34:22] (step=0059250) Train Loss: 6.3447, Train Steps/Sec: 0.85 + 45%|███████████████████████████████████████████████████████████████████████████████▉ | 18371/40903 [6:51:47<7:18:07, 1.17s/it][2025-04-23 00:34:52] (step=0059275) Train Loss: 6.3677, Train Steps/Sec: 0.85 + 45%|████████████████████████████████████████████████████████████████████████████████ | 18396/40903 [6:52:16<7:24:21, 1.18s/it][2025-04-23 00:35:21] (step=0059300) Train Loss: 6.3314, Train Steps/Sec: 0.84 + 45%|████████████████████████████████████████████████████████████████████████████████▏ | 18421/40903 [6:52:46<7:33:24, 1.21s/it][2025-04-23 00:35:51] (step=0059325) Train Loss: 6.3300, Train Steps/Sec: 0.84 + 45%|████████████████████████████████████████████████████████████████████████████████▎ | 18446/40903 [6:53:16<7:23:06, 1.18s/it][2025-04-23 00:36:20] (step=0059350) Train Loss: 6.3084, Train Steps/Sec: 0.85 + 45%|████████████████████████████████████████████████████████████████████████████████▍ | 18471/40903 [6:53:45<7:18:19, 1.17s/it][2025-04-23 00:36:50] (step=0059375) Train Loss: 6.3901, Train Steps/Sec: 0.85 + 45%|████████████████████████████████████████████████████████████████████████████████▍ | 18496/40903 [6:54:15<7:22:48, 1.19s/it][2025-04-23 00:37:20] (step=0059400) Train Loss: 6.4052, Train Steps/Sec: 0.84 + 45%|████████████████████████████████████████████████████████████████████████████████▌ | 18521/40903 [6:54:45<7:26:54, 1.20s/it][2025-04-23 00:37:49] (step=0059425) Train Loss: 6.3712, Train Steps/Sec: 0.85 + 45%|████████████████████████████████████████████████████████████████████████████████▋ | 18546/40903 [6:55:14<7:17:45, 1.17s/it][2025-04-23 00:38:19] (step=0059450) Train Loss: 6.3404, Train Steps/Sec: 0.85 + 45%|████████████████████████████████████████████████████████████████████████████████▊ | 18571/40903 [6:55:43<7:18:40, 1.18s/it][2025-04-23 00:38:48] (step=0059475) Train Loss: 6.3754, Train Steps/Sec: 0.85 + 45%|████████████████████████████████████████████████████████████████████████████████▉ | 18596/40903 [6:56:13<7:13:21, 1.17s/it][2025-04-23 00:39:18] (step=0059500) Train Loss: 6.3944, Train Steps/Sec: 0.84 + 46%|█████████████████████████████████████████████████████████████████████████████████ | 18621/40903 [6:56:43<7:26:36, 1.20s/it][2025-04-23 00:39:48] (step=0059525) Train Loss: 6.3459, Train Steps/Sec: 0.85 + 46%|█████████████████████████████████████████████████████████████████████████████████▏ | 18646/40903 [6:57:12<7:19:06, 1.18s/it][2025-04-23 00:40:17] (step=0059550) Train Loss: 6.4229, Train Steps/Sec: 0.84 + 46%|█████████████████████████████████████████████████████████████████████████████████▎ | 18671/40903 [6:57:42<7:16:40, 1.18s/it][2025-04-23 00:40:47] (step=0059575) Train Loss: 6.3403, Train Steps/Sec: 0.85 + 46%|█████████████████████████████████████████████████████████████████████████████████▎ | 18696/40903 [6:58:18<7:14:07, 1.17s/it][2025-04-23 00:41:23] (step=0059600) Train Loss: 6.4009, Train Steps/Sec: 0.69 + 46%|█████████████████████████████████████████████████████████████████████████████████▍ | 18721/40903 [6:58:48<7:23:24, 1.20s/it][2025-04-23 00:41:53] (step=0059625) Train Loss: 6.3561, Train Steps/Sec: 0.84 + 46%|█████████████████████████████████████████████████████████████████████████████████▌ | 18746/40903 [6:59:18<7:22:00, 1.20s/it][2025-04-23 00:42:22] (step=0059650) Train Loss: 6.3420, Train Steps/Sec: 0.84 + 46%|█████████████████████████████████████████████████████████████████████████████████▋ | 18771/40903 [6:59:47<7:16:15, 1.18s/it][2025-04-23 00:42:52] (step=0059675) Train Loss: 6.3317, Train Steps/Sec: 0.85 + 46%|█████████████████████████████████████████████████████████████████████████████████▊ | 18796/40903 [7:00:17<7:12:14, 1.17s/it][2025-04-23 00:43:22] (step=0059700) Train Loss: 6.3246, Train Steps/Sec: 0.84 + 46%|█████████████████████████████████████████████████████████████████████████████████▉ | 18821/40903 [7:00:46<7:15:36, 1.18s/it][2025-04-23 00:43:51] (step=0059725) Train Loss: 6.3087, Train Steps/Sec: 0.85 + 46%|██████████████████████████████████████████████████████████████████████████████████ | 18846/40903 [7:01:16<7:16:26, 1.19s/it][2025-04-23 00:44:21] (step=0059750) Train Loss: 6.3451, Train Steps/Sec: 0.85 + 46%|██████████████████████████████████████████████████████████████████████████████████ | 18871/40903 [7:01:45<7:12:25, 1.18s/it][2025-04-23 00:44:50] (step=0059775) Train Loss: 6.3745, Train Steps/Sec: 0.85 + 46%|██████████████████████████████████████████████████████████████████████████████████▏ | 18896/40903 [7:02:15<7:12:56, 1.18s/it][2025-04-23 00:45:20] (step=0059800) Train Loss: 6.3218, Train Steps/Sec: 0.84 + 46%|██████████████████████████████████████████████████████████████████████████████████▎ | 18921/40903 [7:02:45<7:19:48, 1.20s/it][2025-04-23 00:45:49] (step=0059825) Train Loss: 6.3161, Train Steps/Sec: 0.84 + 46%|██████████████████████████████████████████████████████████████████████████████████▍ | 18946/40903 [7:03:14<7:19:21, 1.20s/it][2025-04-23 00:46:19] (step=0059850) Train Loss: 6.3508, Train Steps/Sec: 0.84 + 46%|██████████████████████████████████████████████████████████████████████████████████▌ | 18971/40903 [7:03:44<7:08:02, 1.17s/it][2025-04-23 00:46:48] (step=0059875) Train Loss: 6.3573, Train Steps/Sec: 0.85 + 46%|██████████████████████████████████████████████████████████████████████████████████▋ | 18996/40903 [7:04:20<7:05:48, 1.17s/it][2025-04-23 00:47:25] (step=0059900) Train Loss: 6.3272, Train Steps/Sec: 0.69 + 47%|██████████████████████████████████████████████████████████████████████████████████▊ | 19021/40903 [7:04:50<7:14:46, 1.19s/it][2025-04-23 00:47:54] (step=0059925) Train Loss: 6.3600, Train Steps/Sec: 0.85 + 47%|██████████████████████████████████████████████████████████████████████████████████▉ | 19046/40903 [7:05:19<7:14:09, 1.19s/it][2025-04-23 00:48:24] (step=0059950) Train Loss: 6.3728, Train Steps/Sec: 0.84 + 47%|██████████████████████████████████████████████████████████████████████████████████▉ | 19071/40903 [7:05:49<7:12:03, 1.19s/it][2025-04-23 00:48:54] (step=0059975) Train Loss: 6.3718, Train Steps/Sec: 0.84 + 47%|███████████████████████████████████████████████████████████████████████████████████ | 19096/40903 [7:06:19<7:02:04, 1.16s/it][2025-04-23 00:49:24] (step=0060000) Train Loss: 6.3603, Train Steps/Sec: 0.84 +You are using a model of type instructblip to instantiate a model of type blip-2. This is not supported for all configurations of models and can yield errors. +model_name_or_path:/tmp/haozhezhao/model/instructblip-flan-t5-xl +image_place_holder: +Some kwargs in processor config are unused and will not have any effect: num_query_tokens. +tokenizer length after expend 32102 +tokenizer length before expend 32102 +100%|██████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████| 3/3 [02:53<00:00, 57.94s/it] +[2025-04-23 00:54:06] Finish Eval in 60000 steps...███████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████| 3/3 [02:53<00:00, 57.40s/it] +processing left-padding... +processing left-padding... +processing left-padding... +[2025-04-23 00:54:28] Saved checkpoint to checkpoint/BeforeFix_Blip_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_objects/002-GPT-XL/checkpoints/0060000.pt +[2025-04-23 00:54:30] Removed old checkpoint: checkpoint/BeforeFix_Blip_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_objects/002-GPT-XL/checkpoints/0056000.pt + 47%|███████████████████████████████████████████████████████████████████████████████████▏ | 19121/40903 [7:11:55<7:22:34, 1.22s/it][2025-04-23 00:55:00] (step=0060025) Train Loss: 6.3512, Train Steps/Sec: 0.07 + 47%|███████████████████████████████████████████████████████████████████████████████████▎ | 19146/40903 [7:12:24<7:15:56, 1.20s/it][2025-04-23 00:55:29] (step=0060050) Train Loss: 6.3060, Train Steps/Sec: 0.84 + 47%|███████████████████████████████████████████████████████████████████████████████████▍ | 19171/40903 [7:12:54<7:14:07, 1.20s/it][2025-04-23 00:55:59] (step=0060075) Train Loss: 6.3510, Train Steps/Sec: 0.85 + 47%|███████████████████████████████████████████████████████████████████████████████████▌ | 19196/40903 [7:13:23<6:59:23, 1.16s/it][2025-04-23 00:56:28] (step=0060100) Train Loss: 6.3394, Train Steps/Sec: 0.85 + 47%|███████████████████████████████████████████████████████████████████████████████████▋ | 19221/40903 [7:13:53<7:18:49, 1.21s/it][2025-04-23 00:56:58] (step=0060125) Train Loss: 6.3834, Train Steps/Sec: 0.84 + 47%|███████████████████████████████████████████████████████████████████████████████████▊ | 19246/40903 [7:14:23<7:04:25, 1.18s/it][2025-04-23 00:57:28] (step=0060150) Train Loss: 6.3614, Train Steps/Sec: 0.85 + 47%|███████████████████████████████████████████████████████████████████████████████████▊ | 19271/40903 [7:14:52<7:04:08, 1.18s/it][2025-04-23 00:57:57] (step=0060175) Train Loss: 6.3555, Train Steps/Sec: 0.84 + 47%|███████████████████████████████████████████████████████████████████████████████████▉ | 19296/40903 [7:15:22<7:01:29, 1.17s/it][2025-04-23 00:58:27] (step=0060200) Train Loss: 6.3691, Train Steps/Sec: 0.85 + 47%|████████████████████████████████████████████████████████████████████████████████████ | 19321/40903 [7:15:51<7:07:31, 1.19s/it][2025-04-23 00:58:56] (step=0060225) Train Loss: 6.3437, Train Steps/Sec: 0.85 + 47%|████████████████████████████████████████████████████████████████████████████████████▏ | 19346/40903 [7:16:21<7:01:56, 1.17s/it][2025-04-23 00:59:26] (step=0060250) Train Loss: 6.4016, Train Steps/Sec: 0.85 + 47%|████████████████████████████████████████████████████████████████████████████████████▎ | 19371/40903 [7:16:50<7:02:32, 1.18s/it][2025-04-23 00:59:55] (step=0060275) Train Loss: 6.3458, Train Steps/Sec: 0.85 + 47%|████████████████████████████████████████████████████████████████████████████████████▍ | 19396/40903 [7:17:20<7:02:03, 1.18s/it][2025-04-23 01:00:25] (step=0060300) Train Loss: 6.3585, Train Steps/Sec: 0.84 + 47%|████████████████████████████████████████████████████████████████████████████████████▌ | 19421/40903 [7:17:50<7:11:06, 1.20s/it][2025-04-23 01:00:55] (step=0060325) Train Loss: 6.3774, Train Steps/Sec: 0.84 + 48%|████████████████████████████████████████████████████████████████████████████████████▌ | 19446/40903 [7:18:19<7:03:57, 1.19s/it][2025-04-23 01:01:24] (step=0060350) Train Loss: 6.3316, Train Steps/Sec: 0.85 + 48%|████████████████████████████████████████████████████████████████████████████████████▋ | 19471/40903 [7:18:49<7:06:30, 1.19s/it][2025-04-23 01:01:54] (step=0060375) Train Loss: 6.3835, Train Steps/Sec: 0.84 + 48%|████████████████████████████████████████████████████████████████████████████████████▊ | 19496/40903 [7:19:25<7:04:45, 1.19s/it][2025-04-23 01:02:30] (step=0060400) Train Loss: 6.3950, Train Steps/Sec: 0.69 + 48%|████████████████████████████████████████████████████████████████████████████████████▉ | 19521/40903 [7:19:55<7:09:49, 1.21s/it][2025-04-23 01:03:00] (step=0060425) Train Loss: 6.3499, Train Steps/Sec: 0.84 + 48%|█████████████████████████████████████████████████████████████████████████████████████ | 19546/40903 [7:20:24<7:03:23, 1.19s/it][2025-04-23 01:03:29] (step=0060450) Train Loss: 6.3143, Train Steps/Sec: 0.85 + 48%|█████████████████████████████████████████████████████████████████████████████████████▏ | 19571/40903 [7:20:54<6:58:17, 1.18s/it][2025-04-23 01:03:59] (step=0060475) Train Loss: 6.3718, Train Steps/Sec: 0.85 + 48%|█████████████████████████████████████████████████████████████████████████████████████▎ | 19596/40903 [7:21:30<6:54:44, 1.17s/it][2025-04-23 01:04:35] (step=0060500) Train Loss: 6.3172, Train Steps/Sec: 0.69 + 48%|█████████████████████████████████████████████████████████████████████████████████████▍ | 19621/40903 [7:22:00<7:04:47, 1.20s/it][2025-04-23 01:05:05] (step=0060525) Train Loss: 6.3282, Train Steps/Sec: 0.85 + 48%|█████████████████████████████████████████████████████████████████████████████████████▍ | 19646/40903 [7:22:29<6:58:24, 1.18s/it][2025-04-23 01:05:34] (step=0060550) Train Loss: 6.3741, Train Steps/Sec: 0.85 + 48%|█████████████████████████████████████████████████████████████████████████████████████▌ | 19671/40903 [7:22:59<6:55:48, 1.18s/it][2025-04-23 01:06:03] (step=0060575) Train Loss: 6.3486, Train Steps/Sec: 0.85 + 48%|█████████████████████████████████████████████████████████████████████████████████████▋ | 19696/40903 [7:23:28<6:52:11, 1.17s/it][2025-04-23 01:06:33] (step=0060600) Train Loss: 6.3623, Train Steps/Sec: 0.85 + 48%|█████████████████████████████████████████████████████████████████████████████████████▊ | 19721/40903 [7:23:58<7:01:58, 1.20s/it][2025-04-23 01:07:03] (step=0060625) Train Loss: 6.2981, Train Steps/Sec: 0.85 + 48%|█████████████████████████████████████████████████████████████████████████████████████▉ | 19746/40903 [7:24:27<6:59:10, 1.19s/it][2025-04-23 01:07:32] (step=0060650) Train Loss: 6.3824, Train Steps/Sec: 0.84 + 48%|██████████████████████████████████████████████████████████████████████████████████████ | 19771/40903 [7:24:57<6:53:10, 1.17s/it][2025-04-23 01:08:02] (step=0060675) Train Loss: 6.3720, Train Steps/Sec: 0.84 + 48%|██████████████████████████████████████████████████████████████████████████████████████▏ | 19796/40903 [7:25:27<6:51:20, 1.17s/it][2025-04-23 01:08:32] (step=0060700) Train Loss: 6.3654, Train Steps/Sec: 0.84 + 48%|██████████████████████████████████████████████████████████████████████████████████████▎ | 19821/40903 [7:25:56<7:03:28, 1.21s/it][2025-04-23 01:09:01] (step=0060725) Train Loss: 6.3973, Train Steps/Sec: 0.85 + 49%|██████████████████████████████████████████████████████████████████████████████████████▎ | 19846/40903 [7:26:26<6:56:36, 1.19s/it][2025-04-23 01:09:31] (step=0060750) Train Loss: 6.3510, Train Steps/Sec: 0.84 + 49%|██████████████████████████████████████████████████████████████████████████████████████▍ | 19871/40903 [7:26:55<6:55:15, 1.18s/it][2025-04-23 01:10:00] (step=0060775) Train Loss: 6.3503, Train Steps/Sec: 0.85 + 49%|██████████████████████████████████████████████████████████████████████████████████████▌ | 19896/40903 [7:27:25<6:54:57, 1.19s/it][2025-04-23 01:10:30] (step=0060800) Train Loss: 6.3566, Train Steps/Sec: 0.84 + 49%|██████████████████████████████████████████████████████████████████████████████████████▋ | 19921/40903 [7:27:55<7:05:05, 1.22s/it][2025-04-23 01:11:00] (step=0060825) Train Loss: 6.3621, Train Steps/Sec: 0.84 + 49%|██████████████████████████████████████████████████████████████████████████████████████▊ | 19946/40903 [7:28:25<6:53:06, 1.18s/it][2025-04-23 01:11:29] (step=0060850) Train Loss: 6.3518, Train Steps/Sec: 0.85 + 49%|██████████████████████████████████████████████████████████████████████████████████████▉ | 19971/40903 [7:28:54<6:49:04, 1.17s/it][2025-04-23 01:11:59] (step=0060875) Train Loss: 6.3714, Train Steps/Sec: 0.85 + 49%|███████████████████████████████████████████████████████████████████████████████████████ | 19996/40903 [7:29:24<6:49:02, 1.17s/it][2025-04-23 01:12:29] (step=0060900) Train Loss: 6.3538, Train Steps/Sec: 0.84 + 49%|███████████████████████████████████████████████████████████████████████████████████████▏ | 20021/40903 [7:29:53<6:53:43, 1.19s/it][2025-04-23 01:12:58] (step=0060925) Train Loss: 6.3483, Train Steps/Sec: 0.85 + 49%|███████████████████████████████████████████████████████████████████████████████████████▏ | 20046/40903 [7:30:23<6:48:56, 1.18s/it][2025-04-23 01:13:28] (step=0060950) Train Loss: 6.3831, Train Steps/Sec: 0.85 + 49%|███████████████████████████████████████████████████████████████████████████████████████▎ | 20071/40903 [7:30:53<6:46:54, 1.17s/it][2025-04-23 01:13:57] (step=0060975) Train Loss: 6.3270, Train Steps/Sec: 0.84 + 49%|███████████████████████████████████████████████████████████████████████████████████████▍ | 20096/40903 [7:31:22<6:49:27, 1.18s/it][2025-04-23 01:14:27] (step=0061000) Train Loss: 6.3770, Train Steps/Sec: 0.84 + 49%|███████████████████████████████████████████████████████████████████████████████████████▌ | 20121/40903 [7:31:52<7:05:17, 1.23s/it][2025-04-23 01:14:57] (step=0061025) Train Loss: 6.3351, Train Steps/Sec: 0.84 + 49%|███████████████████████████████████████████████████████████████████████████████████████▋ | 20146/40903 [7:32:21<6:50:27, 1.19s/it][2025-04-23 01:15:26] (step=0061050) Train Loss: 6.3727, Train Steps/Sec: 0.85 + 49%|███████████████████████████████████████████████████████████████████████████████████████▊ | 20171/40903 [7:32:51<6:41:43, 1.16s/it][2025-04-23 01:15:56] (step=0061075) Train Loss: 6.3652, Train Steps/Sec: 0.84 + 49%|███████████████████████████████████████████████████████████████████████████████████████▉ | 20196/40903 [7:33:21<6:47:42, 1.18s/it][2025-04-23 01:16:26] (step=0061100) Train Loss: 6.3468, Train Steps/Sec: 0.84 + 49%|███████████████████████████████████████████████████████████████████████████████████████▉ | 20221/40903 [7:33:50<6:53:37, 1.20s/it][2025-04-23 01:16:55] (step=0061125) Train Loss: 6.3403, Train Steps/Sec: 0.85 + 49%|████████████████████████████████████████████████████████████████████████████████████████ | 20246/40903 [7:34:20<6:48:12, 1.19s/it][2025-04-23 01:17:25] (step=0061150) Train Loss: 6.3650, Train Steps/Sec: 0.85 + 50%|████████████████████████████████████████████████████████████████████████████████████████▏ | 20271/40903 [7:34:49<6:43:30, 1.17s/it][2025-04-23 01:17:54] (step=0061175) Train Loss: 6.3871, Train Steps/Sec: 0.85 + 50%|████████████████████████████████████████████████████████████████████████████████████████▎ | 20296/40903 [7:35:19<6:46:56, 1.18s/it][2025-04-23 01:18:24] (step=0061200) Train Loss: 6.3918, Train Steps/Sec: 0.84 + 50%|████████████████████████████████████████████████████████████████████████████████████████▍ | 20321/40903 [7:35:49<7:00:17, 1.23s/it][2025-04-23 01:18:53] (step=0061225) Train Loss: 6.3595, Train Steps/Sec: 0.84 + 50%|████████████████████████████████████████████████████████████████████████████████████████▌ | 20346/40903 [7:36:18<6:44:47, 1.18s/it][2025-04-23 01:19:23] (step=0061250) Train Loss: 6.3611, Train Steps/Sec: 0.84 + 50%|████████████████████████████████████████████████████████████████████████████████████████▋ | 20371/40903 [7:36:48<6:40:08, 1.17s/it][2025-04-23 01:19:52] (step=0061275) Train Loss: 6.3463, Train Steps/Sec: 0.85 + 50%|████████████████████████████████████████████████████████████████████████████████████████▊ | 20396/40903 [7:37:17<6:38:25, 1.17s/it][2025-04-23 01:20:22] (step=0061300) Train Loss: 6.3469, Train Steps/Sec: 0.84 + 50%|████████████████████████████████████████████████████████████████████████████████████████▊ | 20421/40903 [7:37:47<6:49:03, 1.20s/it][2025-04-23 01:20:52] (step=0061325) Train Loss: 6.3372, Train Steps/Sec: 0.85 + 50%|████████████████████████████████████████████████████████████████████████████████████████▉ | 20446/40903 [7:38:16<6:47:17, 1.19s/it][2025-04-23 01:21:21] (step=0061350) Train Loss: 6.3679, Train Steps/Sec: 0.85 + 50%|█████████████████████████████████████████████████████████████████████████████████████████ | 20471/40903 [7:38:46<6:44:52, 1.19s/it][2025-04-23 01:21:51] (step=0061375) Train Loss: 6.3137, Train Steps/Sec: 0.84 + 50%|█████████████████████████████████████████████████████████████████████████████████████████▏ | 20496/40903 [7:39:15<6:39:16, 1.17s/it][2025-04-23 01:22:20] (step=0061400) Train Loss: 6.3569, Train Steps/Sec: 0.84 + 50%|█████████████████████████████████████████████████████████████████████████████████████████▎ | 20521/40903 [7:39:45<6:46:18, 1.20s/it][2025-04-23 01:22:50] (step=0061425) Train Loss: 6.3639, Train Steps/Sec: 0.85 + 50%|█████████████████████████████████████████████████████████████████████████████████████████▍ | 20546/40903 [7:40:15<6:49:14, 1.21s/it][2025-04-23 01:23:19] (step=0061450) Train Loss: 6.3690, Train Steps/Sec: 0.85 + 50%|█████████████████████████████████████████████████████████████████████████████████████████▌ | 20571/40903 [7:40:44<6:40:12, 1.18s/it][2025-04-23 01:23:49] (step=0061475) Train Loss: 6.3763, Train Steps/Sec: 0.85 + 50%|█████████████████████████████████████████████████████████████████████████████████████████▋ | 20596/40903 [7:41:14<6:33:44, 1.16s/it][2025-04-23 01:24:19] (step=0061500) Train Loss: 6.3721, Train Steps/Sec: 0.84 + 50%|█████████████████████████████████████████████████████████████████████████████████████████▋ | 20621/40903 [7:41:43<6:42:11, 1.19s/it][2025-04-23 01:24:48] (step=0061525) Train Loss: 6.3231, Train Steps/Sec: 0.85 + 50%|█████████████████████████████████████████████████████████████████████████████████████████▊ | 20646/40903 [7:42:13<6:42:41, 1.19s/it][2025-04-23 01:25:18] (step=0061550) Train Loss: 6.3792, Train Steps/Sec: 0.85 + 51%|█████████████████████████████████████████████████████████████████████████████████████████▉ | 20671/40903 [7:42:42<6:39:51, 1.19s/it][2025-04-23 01:25:47] (step=0061575) Train Loss: 6.4044, Train Steps/Sec: 0.85 + 51%|██████████████████████████████████████████████████████████████████████████████████████████ | 20696/40903 [7:43:12<6:31:49, 1.16s/it][2025-04-23 01:26:17] (step=0061600) Train Loss: 6.3546, Train Steps/Sec: 0.84 + 51%|██████████████████████████████████████████████████████████████████████████████████████████▏ | 20721/40903 [7:43:42<6:41:08, 1.19s/it][2025-04-23 01:26:47] (step=0061625) Train Loss: 6.3706, Train Steps/Sec: 0.84 + 51%|██████████████████████████████████████████████████████████████████████████████████████████▎ | 20746/40903 [7:44:11<6:39:37, 1.19s/it][2025-04-23 01:27:16] (step=0061650) Train Loss: 6.3131, Train Steps/Sec: 0.85 + 51%|██████████████████████████████████████████████████████████████████████████████████████████▍ | 20771/40903 [7:44:41<6:32:10, 1.17s/it][2025-04-23 01:27:46] (step=0061675) Train Loss: 6.3882, Train Steps/Sec: 0.85 + 51%|██████████████████████████████████████████████████████████████████████████████████████████▍ | 20796/40903 [7:45:10<6:38:02, 1.19s/it][2025-04-23 01:28:15] (step=0061700) Train Loss: 6.3468, Train Steps/Sec: 0.84 + 51%|██████████████████████████████████████████████████████████████████████████████████████████▌ | 20821/40903 [7:45:40<6:40:14, 1.20s/it][2025-04-23 01:28:45] (step=0061725) Train Loss: 6.3929, Train Steps/Sec: 0.84 + 51%|██████████████████████████████████████████████████████████████████████████████████████████▋ | 20846/40903 [7:46:10<6:36:42, 1.19s/it][2025-04-23 01:29:15] (step=0061750) Train Loss: 6.3556, Train Steps/Sec: 0.85 + 51%|██████████████████████████████████████████████████████████████████████████████████████████▊ | 20871/40903 [7:46:39<6:31:26, 1.17s/it][2025-04-23 01:29:44] (step=0061775) Train Loss: 6.3595, Train Steps/Sec: 0.85 + 51%|██████████████████████████████████████████████████████████████████████████████████████████▉ | 20896/40903 [7:47:09<6:32:38, 1.18s/it][2025-04-23 01:30:14] (step=0061800) Train Loss: 6.4402, Train Steps/Sec: 0.84 + 51%|███████████████████████████████████████████████████████████████████████████████████████████ | 20921/40903 [7:47:38<6:34:23, 1.18s/it][2025-04-23 01:30:43] (step=0061825) Train Loss: 6.3372, Train Steps/Sec: 0.85 + 51%|███████████████████████████████████████████████████████████████████████████████████████████▏ | 20946/40903 [7:48:08<6:38:12, 1.20s/it][2025-04-23 01:31:13] (step=0061850) Train Loss: 6.3577, Train Steps/Sec: 0.85 + 51%|███████████████████████████████████████████████████████████████████████████████████████████▎ | 20971/40903 [7:48:37<6:29:06, 1.17s/it][2025-04-23 01:31:42] (step=0061875) Train Loss: 6.3141, Train Steps/Sec: 0.85 + 51%|███████████████████████████████████████████████████████████████████████████████████████████▎ | 20996/40903 [7:49:07<6:23:09, 1.15s/it][2025-04-23 01:32:12] (step=0061900) Train Loss: 6.3724, Train Steps/Sec: 0.84 + 51%|███████████████████████████████████████████████████████████████████████████████████████████▍ | 21021/40903 [7:49:36<6:38:33, 1.20s/it][2025-04-23 01:32:41] (step=0061925) Train Loss: 6.3366, Train Steps/Sec: 0.85 + 51%|███████████████████████████████████████████████████████████████████████████████████████████▌ | 21046/40903 [7:50:06<6:31:59, 1.18s/it][2025-04-23 01:33:11] (step=0061950) Train Loss: 6.3379, Train Steps/Sec: 0.85 + 52%|███████████████████████████████████████████████████████████████████████████████████████████▋ | 21071/40903 [7:50:36<6:26:03, 1.17s/it][2025-04-23 01:33:40] (step=0061975) Train Loss: 6.3504, Train Steps/Sec: 0.85 + 52%|███████████████████████████████████████████████████████████████████████████████████████████▊ | 21096/40903 [7:51:05<6:30:46, 1.18s/it][2025-04-23 01:34:10] (step=0062000) Train Loss: 6.3501, Train Steps/Sec: 0.84 +You are using a model of type instructblip to instantiate a model of type blip-2. This is not supported for all configurations of models and can yield errors. +model_name_or_path:/tmp/haozhezhao/model/instructblip-flan-t5-xl +image_place_holder: +Some kwargs in processor config are unused and will not have any effect: num_query_tokens. +tokenizer length after expend 32102 +tokenizer length before expend 32102 +100%|██████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████| 3/3 [02:53<00:00, 57.90s/it] +[2025-04-23 01:38:53] Finish Eval in 62000 steps...███████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████| 3/3 [02:52<00:00, 57.39s/it] +processing left-padding... +processing left-padding... +processing left-padding... +[2025-04-23 01:39:14] Saved checkpoint to checkpoint/BeforeFix_Blip_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_objects/002-GPT-XL/checkpoints/0062000.pt +[2025-04-23 01:39:17] Removed old checkpoint: checkpoint/BeforeFix_Blip_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_objects/002-GPT-XL/checkpoints/0058000.pt + 52%|███████████████████████████████████████████████████████████████████████████████████████████▉ | 21121/40903 [7:56:41<6:43:43, 1.22s/it][2025-04-23 01:39:46] (step=0062025) Train Loss: 6.3557, Train Steps/Sec: 0.07 + 52%|████████████████████████████████████████████████████████████████████████████████████████████ | 21146/40903 [7:57:11<6:28:36, 1.18s/it][2025-04-23 01:40:16] (step=0062050) Train Loss: 6.3626, Train Steps/Sec: 0.85 + 52%|████████████████████████████████████████████████████████████████████████████████████████████▏ | 21171/40903 [7:57:40<6:26:18, 1.17s/it][2025-04-23 01:40:45] (step=0062075) Train Loss: 6.3252, Train Steps/Sec: 0.85 + 52%|████████████████████████████████████████████████████████████████████████████████████████████▏ | 21196/40903 [7:58:10<6:24:18, 1.17s/it][2025-04-23 01:41:15] (step=0062100) Train Loss: 6.3430, Train Steps/Sec: 0.84 + 52%|████████████████████████████████████████████████████████████████████████████████████████████▎ | 21221/40903 [7:58:40<6:36:32, 1.21s/it][2025-04-23 01:41:44] (step=0062125) Train Loss: 6.3151, Train Steps/Sec: 0.84 + 52%|████████████████████████████████████████████████████████████████████████████████████████████▍ | 21246/40903 [7:59:09<6:28:27, 1.19s/it][2025-04-23 01:42:14] (step=0062150) Train Loss: 6.3603, Train Steps/Sec: 0.84 + 52%|████████████████████████████████████████████████████████████████████████████████████████████▌ | 21271/40903 [7:59:39<6:23:05, 1.17s/it][2025-04-23 01:42:44] (step=0062175) Train Loss: 6.3918, Train Steps/Sec: 0.85 + 52%|████████████████████████████████████████████████████████████████████████████████████████████▋ | 21296/40903 [8:00:09<6:26:04, 1.18s/it][2025-04-23 01:43:14] (step=0062200) Train Loss: 6.4090, Train Steps/Sec: 0.84 + 52%|████████████████████████████████████████████████████████████████████████████████████████████▊ | 21321/40903 [8:00:38<6:33:23, 1.21s/it][2025-04-23 01:43:43] (step=0062225) Train Loss: 6.3646, Train Steps/Sec: 0.85 + 52%|████████████████████████████████████████████████████████████████████████████████████████████▉ | 21346/40903 [8:01:08<6:22:20, 1.17s/it][2025-04-23 01:44:13] (step=0062250) Train Loss: 6.3674, Train Steps/Sec: 0.83 + 52%|█████████████████████████████████████████████████████████████████████████████████████████████ | 21371/40903 [8:01:38<6:29:37, 1.20s/it][2025-04-23 01:44:43] (step=0062275) Train Loss: 6.3514, Train Steps/Sec: 0.84 + 52%|█████████████████████████████████████████████████████████████████████████████████████████████ | 21396/40903 [8:02:08<6:17:33, 1.16s/it][2025-04-23 01:45:13] (step=0062300) Train Loss: 6.3793, Train Steps/Sec: 0.84 + 52%|█████████████████████████████████████████████████████████████████████████████████████████████▏ | 21421/40903 [8:02:37<6:31:04, 1.20s/it][2025-04-23 01:45:42] (step=0062325) Train Loss: 6.3635, Train Steps/Sec: 0.85 + 52%|█████████████████████████████████████████████████████████████████████████████████████████████▎ | 21446/40903 [8:03:07<6:27:15, 1.19s/it][2025-04-23 01:46:12] (step=0062350) Train Loss: 6.3360, Train Steps/Sec: 0.85 + 52%|█████████████████████████████████████████████████████████████████████████████████████████████▍ | 21471/40903 [8:03:36<6:21:33, 1.18s/it][2025-04-23 01:46:41] (step=0062375) Train Loss: 6.3926, Train Steps/Sec: 0.85 + 53%|█████████████████████████████████████████████████████████████████████████████████████████████▌ | 21496/40903 [8:04:06<6:18:53, 1.17s/it][2025-04-23 01:47:11] (step=0062400) Train Loss: 6.3639, Train Steps/Sec: 0.84 + 53%|█████████████████████████████████████████████████████████████████████████████████████████████▋ | 21521/40903 [8:04:36<6:31:34, 1.21s/it][2025-04-23 01:47:40] (step=0062425) Train Loss: 6.2681, Train Steps/Sec: 0.85 + 53%|█████████████████████████████████████████████████████████████████████████████████████████████▊ | 21546/40903 [8:05:05<6:19:42, 1.18s/it][2025-04-23 01:48:10] (step=0062450) Train Loss: 6.3176, Train Steps/Sec: 0.84 + 53%|█████████████████████████████████████████████████████████████████████████████████████████████▊ | 21571/40903 [8:05:35<6:15:44, 1.17s/it][2025-04-23 01:48:40] (step=0062475) Train Loss: 6.3695, Train Steps/Sec: 0.85 + 53%|█████████████████████████████████████████████████████████████████████████████████████████████▉ | 21596/40903 [8:06:04<6:16:30, 1.17s/it][2025-04-23 01:49:09] (step=0062500) Train Loss: 6.3309, Train Steps/Sec: 0.84 + 53%|██████████████████████████████████████████████████████████████████████████████████████████████ | 21621/40903 [8:06:34<6:25:38, 1.20s/it][2025-04-23 01:49:39] (step=0062525) Train Loss: 6.3456, Train Steps/Sec: 0.85 + 53%|██████████████████████████████████████████████████████████████████████████████████████████████▏ | 21646/40903 [8:07:04<6:20:18, 1.18s/it][2025-04-23 01:50:09] (step=0062550) Train Loss: 6.3922, Train Steps/Sec: 0.84 + 53%|██████████████████████████████████████████████████████████████████████████████████████████████▎ | 21671/40903 [8:07:33<6:18:48, 1.18s/it][2025-04-23 01:50:38] (step=0062575) Train Loss: 6.3579, Train Steps/Sec: 0.84 + 53%|██████████████████████████████████████████████████████████████████████████████████████████████▍ | 21696/40903 [8:08:03<6:11:22, 1.16s/it][2025-04-23 01:51:08] (step=0062600) Train Loss: 6.3286, Train Steps/Sec: 0.84 + 53%|██████████████████████████████████████████████████████████████████████████████████████████████▌ | 21721/40903 [8:08:33<6:21:17, 1.19s/it][2025-04-23 01:51:37] (step=0062625) Train Loss: 6.3469, Train Steps/Sec: 0.85 + 53%|██████████████████████████████████████████████████████████████████████████████████████████████▋ | 21746/40903 [8:09:02<6:18:09, 1.18s/it][2025-04-23 01:52:07] (step=0062650) Train Loss: 6.3562, Train Steps/Sec: 0.85 + 53%|██████████████████████████████████████████████████████████████████████████████████████████████▋ | 21771/40903 [8:09:32<6:13:14, 1.17s/it][2025-04-23 01:52:37] (step=0062675) Train Loss: 6.3648, Train Steps/Sec: 0.84 + 53%|██████████████████████████████████████████████████████████████████████████████████████████████▊ | 21796/40903 [8:10:01<6:15:07, 1.18s/it][2025-04-23 01:53:06] (step=0062700) Train Loss: 6.3379, Train Steps/Sec: 0.85 + 53%|██████████████████████████████████████████████████████████████████████████████████████████████▉ | 21821/40903 [8:10:31<6:20:39, 1.20s/it][2025-04-23 01:53:36] (step=0062725) Train Loss: 6.3610, Train Steps/Sec: 0.85 + 53%|███████████████████████████████████████████████████████████████████████████████████████████████ | 21846/40903 [8:11:01<6:17:04, 1.19s/it][2025-04-23 01:54:05] (step=0062750) Train Loss: 6.3883, Train Steps/Sec: 0.84 + 53%|███████████████████████████████████████████████████████████████████████████████████████████████▏ | 21871/40903 [8:11:30<6:15:50, 1.18s/it][2025-04-23 01:54:35] (step=0062775) Train Loss: 6.3681, Train Steps/Sec: 0.85 + 54%|███████████████████████████████████████████████████████████████████████████████████████████████▎ | 21896/40903 [8:12:00<6:12:51, 1.18s/it][2025-04-23 01:55:05] (step=0062800) Train Loss: 6.3585, Train Steps/Sec: 0.84 + 54%|███████████████████████████████████████████████████████████████████████████████████████████████▍ | 21921/40903 [8:12:29<6:19:52, 1.20s/it][2025-04-23 01:55:34] (step=0062825) Train Loss: 6.3875, Train Steps/Sec: 0.84 + 54%|███████████████████████████████████████████████████████████████████████████████████████████████▌ | 21946/40903 [8:12:59<6:15:02, 1.19s/it][2025-04-23 01:56:04] (step=0062850) Train Loss: 6.3796, Train Steps/Sec: 0.84 + 54%|███████████████████████████████████████████████████████████████████████████████████████████████▌ | 21971/40903 [8:13:29<6:06:52, 1.16s/it][2025-04-23 01:56:33] (step=0062875) Train Loss: 6.3456, Train Steps/Sec: 0.85 + 54%|███████████████████████████████████████████████████████████████████████████████████████████████▋ | 21996/40903 [8:13:58<6:09:39, 1.17s/it][2025-04-23 01:57:03] (step=0062900) Train Loss: 6.3158, Train Steps/Sec: 0.84 + 54%|███████████████████████████████████████████████████████████████████████████████████████████████▊ | 22021/40903 [8:14:28<6:19:46, 1.21s/it][2025-04-23 01:57:33] (step=0062925) Train Loss: 6.3458, Train Steps/Sec: 0.84 + 54%|███████████████████████████████████████████████████████████████████████████████████████████████▉ | 22046/40903 [8:14:57<6:14:40, 1.19s/it][2025-04-23 01:58:02] (step=0062950) Train Loss: 6.3445, Train Steps/Sec: 0.84 + 54%|████████████████████████████████████████████████████████████████████████████████████████████████ | 22071/40903 [8:15:27<6:09:09, 1.18s/it][2025-04-23 01:58:32] (step=0062975) Train Loss: 6.3467, Train Steps/Sec: 0.85 + 54%|████████████████████████████████████████████████████████████████████████████████████████████████▏ | 22096/40903 [8:15:57<6:10:43, 1.18s/it][2025-04-23 01:59:02] (step=0063000) Train Loss: 6.3569, Train Steps/Sec: 0.84 + 54%|████████████████████████████████████████████████████████████████████████████████████████████████▎ | 22121/40903 [8:16:26<6:13:58, 1.19s/it][2025-04-23 01:59:31] (step=0063025) Train Loss: 6.3482, Train Steps/Sec: 0.84 + 54%|████████████████████████████████████████████████████████████████████████████████████████████████▎ | 22146/40903 [8:16:56<6:15:15, 1.20s/it][2025-04-23 02:00:01] (step=0063050) Train Loss: 6.3249, Train Steps/Sec: 0.85 + 54%|████████████████████████████████████████████████████████████████████████████████████████████████▍ | 22171/40903 [8:17:25<6:04:15, 1.17s/it][2025-04-23 02:00:30] (step=0063075) Train Loss: 6.3804, Train Steps/Sec: 0.85 + 54%|████████████████████████████████████████████████████████████████████████████████████████████████▌ | 22196/40903 [8:17:55<6:05:57, 1.17s/it][2025-04-23 02:01:00] (step=0063100) Train Loss: 6.3363, Train Steps/Sec: 0.84 + 54%|████████████████████████████████████████████████████████████████████████████████████████████████▋ | 22221/40903 [8:18:25<6:11:45, 1.19s/it][2025-04-23 02:01:29] (step=0063125) Train Loss: 6.2975, Train Steps/Sec: 0.85 + 54%|████████████████████████████████████████████████████████████████████████████████████████████████▊ | 22246/40903 [8:18:54<6:06:50, 1.18s/it][2025-04-23 02:01:59] (step=0063150) Train Loss: 6.3370, Train Steps/Sec: 0.85 + 54%|████████████████████████████████████████████████████████████████████████████████████████████████▉ | 22271/40903 [8:19:24<6:13:07, 1.20s/it][2025-04-23 02:02:29] (step=0063175) Train Loss: 6.3386, Train Steps/Sec: 0.84 + 55%|█████████████████████████████████████████████████████████████████████████████████████████████████ | 22296/40903 [8:19:53<5:56:53, 1.15s/it][2025-04-23 02:02:58] (step=0063200) Train Loss: 6.3425, Train Steps/Sec: 0.85 + 55%|█████████████████████████████████████████████████████████████████████████████████████████████████▏ | 22321/40903 [8:20:23<6:09:14, 1.19s/it][2025-04-23 02:03:28] (step=0063225) Train Loss: 6.3789, Train Steps/Sec: 0.85 + 55%|█████████████████████████████████████████████████████████████████████████████████████████████████▏ | 22346/40903 [8:20:52<6:04:56, 1.18s/it][2025-04-23 02:03:57] (step=0063250) Train Loss: 6.3952, Train Steps/Sec: 0.84 + 55%|█████████████████████████████████████████████████████████████████████████████████████████████████▎ | 22371/40903 [8:21:22<6:03:02, 1.18s/it][2025-04-23 02:04:27] (step=0063275) Train Loss: 6.3712, Train Steps/Sec: 0.84 + 55%|█████████████████████████████████████████████████████████████████████████████████████████████████▍ | 22396/40903 [8:21:51<5:58:50, 1.16s/it][2025-04-23 02:04:56] (step=0063300) Train Loss: 6.3246, Train Steps/Sec: 0.84 + 55%|█████████████████████████████████████████████████████████████████████████████████████████████████▌ | 22421/40903 [8:22:21<6:10:43, 1.20s/it][2025-04-23 02:05:26] (step=0063325) Train Loss: 6.3427, Train Steps/Sec: 0.85 + 55%|█████████████████████████████████████████████████████████████████████████████████████████████████▋ | 22446/40903 [8:22:51<6:06:18, 1.19s/it][2025-04-23 02:05:56] (step=0063350) Train Loss: 6.3416, Train Steps/Sec: 0.84 + 55%|█████████████████████████████████████████████████████████████████████████████████████████████████▊ | 22471/40903 [8:23:21<6:00:37, 1.17s/it][2025-04-23 02:06:25] (step=0063375) Train Loss: 6.3771, Train Steps/Sec: 0.84 + 55%|█████████████████████████████████████████████████████████████████████████████████████████████████▉ | 22496/40903 [8:23:50<5:59:00, 1.17s/it][2025-04-23 02:06:55] (step=0063400) Train Loss: 6.3277, Train Steps/Sec: 0.84 + 55%|██████████████████████████████████████████████████████████████████████████████████████████████████ | 22521/40903 [8:24:20<6:06:11, 1.20s/it][2025-04-23 02:07:25] (step=0063425) Train Loss: 6.2981, Train Steps/Sec: 0.85 + 55%|██████████████████████████████████████████████████████████████████████████████████████████████████ | 22546/40903 [8:24:49<6:00:55, 1.18s/it][2025-04-23 02:07:54] (step=0063450) Train Loss: 6.4009, Train Steps/Sec: 0.85 + 55%|██████████████████████████████████████████████████████████████████████████████████████████████████▏ | 22571/40903 [8:25:19<6:00:25, 1.18s/it][2025-04-23 02:08:24] (step=0063475) Train Loss: 6.3537, Train Steps/Sec: 0.85 + 55%|██████████████████████████████████████████████████████████████████████████████████████████████████▎ | 22596/40903 [8:25:48<5:57:02, 1.17s/it][2025-04-23 02:08:53] (step=0063500) Train Loss: 6.3791, Train Steps/Sec: 0.85 + 55%|██████████████████████████████████████████████████████████████████████████████████████████████████▍ | 22621/40903 [8:26:18<6:05:41, 1.20s/it][2025-04-23 02:09:23] (step=0063525) Train Loss: 6.3215, Train Steps/Sec: 0.84 + 55%|██████████████████████████████████████████████████████████████████████████████████████████████████▌ | 22646/40903 [8:26:48<6:01:26, 1.19s/it][2025-04-23 02:09:53] (step=0063550) Train Loss: 6.3749, Train Steps/Sec: 0.84 + 55%|██████████████████████████████████████████████████████████████████████████████████████████████████▋ | 22671/40903 [8:27:17<6:00:50, 1.19s/it][2025-04-23 02:10:22] (step=0063575) Train Loss: 6.3508, Train Steps/Sec: 0.84 + 55%|██████████████████████████████████████████████████████████████████████████████████████████████████▊ | 22696/40903 [8:27:53<6:13:01, 1.23s/it][2025-04-23 02:10:58] (step=0063600) Train Loss: 6.3413, Train Steps/Sec: 0.69 + 56%|██████████████████████████████████████████████████████████████████████████████████████████████████▉ | 22721/40903 [8:28:23<6:06:22, 1.21s/it][2025-04-23 02:11:28] (step=0063625) Train Loss: 6.3008, Train Steps/Sec: 0.85 + 56%|██████████████████████████████████████████████████████████████████████████████████████████████████▉ | 22746/40903 [8:28:53<5:57:02, 1.18s/it][2025-04-23 02:11:57] (step=0063650) Train Loss: 6.3721, Train Steps/Sec: 0.85 + 56%|███████████████████████████████████████████████████████████████████████████████████████████████████ | 22771/40903 [8:29:22<5:53:53, 1.17s/it][2025-04-23 02:12:27] (step=0063675) Train Loss: 6.3174, Train Steps/Sec: 0.84 + 56%|███████████████████████████████████████████████████████████████████████████████████████████████████▏ | 22796/40903 [8:29:52<5:52:05, 1.17s/it][2025-04-23 02:12:57] (step=0063700) Train Loss: 6.3715, Train Steps/Sec: 0.84 + 56%|███████████████████████████████████████████████████████████████████████████████████████████████████▎ | 22821/40903 [8:30:22<5:59:37, 1.19s/it][2025-04-23 02:13:26] (step=0063725) Train Loss: 6.3516, Train Steps/Sec: 0.85 + 56%|███████████████████████████████████████████████████████████████████████████████████████████████████▍ | 22846/40903 [8:30:51<5:57:33, 1.19s/it][2025-04-23 02:13:56] (step=0063750) Train Loss: 6.3485, Train Steps/Sec: 0.84 + 56%|███████████████████████████████████████████████████████████████████████████████████████████████████▌ | 22871/40903 [8:31:21<5:52:11, 1.17s/it][2025-04-23 02:14:26] (step=0063775) Train Loss: 6.3504, Train Steps/Sec: 0.85 + 56%|███████████████████████████████████████████████████████████████████████████████████████████████████▋ | 22896/40903 [8:31:50<5:52:15, 1.17s/it][2025-04-23 02:14:55] (step=0063800) Train Loss: 6.3364, Train Steps/Sec: 0.84 + 56%|███████████████████████████████████████████████████████████████████████████████████████████████████▋ | 22921/40903 [8:32:20<5:59:56, 1.20s/it][2025-04-23 02:15:25] (step=0063825) Train Loss: 6.3256, Train Steps/Sec: 0.85 + 56%|███████████████████████████████████████████████████████████████████████████████████████████████████▊ | 22946/40903 [8:32:49<5:54:22, 1.18s/it][2025-04-23 02:15:54] (step=0063850) Train Loss: 6.4014, Train Steps/Sec: 0.85 + 56%|███████████████████████████████████████████████████████████████████████████████████████████████████▉ | 22971/40903 [8:33:19<5:53:16, 1.18s/it][2025-04-23 02:16:24] (step=0063875) Train Loss: 6.3323, Train Steps/Sec: 0.84 + 56%|████████████████████████████████████████████████████████████████████████████████████████████████████ | 22996/40903 [8:33:49<5:50:48, 1.18s/it][2025-04-23 02:16:53] (step=0063900) Train Loss: 6.3499, Train Steps/Sec: 0.85 + 56%|████████████████████████████████████████████████████████████████████████████████████████████████████▏ | 23021/40903 [8:34:18<5:55:01, 1.19s/it][2025-04-23 02:17:23] (step=0063925) Train Loss: 6.3401, Train Steps/Sec: 0.85 + 56%|████████████████████████████████████████████████████████████████████████████████████████████████████▎ | 23046/40903 [8:34:48<5:49:26, 1.17s/it][2025-04-23 02:17:52] (step=0063950) Train Loss: 6.3843, Train Steps/Sec: 0.85 + 56%|████████████████████████████████████████████████████████████████████████████████████████████████████▍ | 23071/40903 [8:35:17<5:43:12, 1.15s/it][2025-04-23 02:18:22] (step=0063975) Train Loss: 6.3725, Train Steps/Sec: 0.85 + 56%|███████████████████████████████████████████████████████████████████████████████████████████████████▉ | 23096/40903 [8:35:54<13:29:29, 2.73s/it][2025-04-23 02:18:59] (step=0064000) Train Loss: 6.3980, Train Steps/Sec: 0.68 +You are using a model of type instructblip to instantiate a model of type blip-2. This is not supported for all configurations of models and can yield errors. +model_name_or_path:/tmp/haozhezhao/model/instructblip-flan-t5-xl +image_place_holder: +Some kwargs in processor config are unused and will not have any effect: num_query_tokens. +tokenizer length after expend 32102 +tokenizer length before expend 32102 +100%|██████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████| 3/3 [02:53<00:00, 57.95s/it] +[2025-04-23 02:23:42] Finish Eval in 64000 steps...███████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████| 3/3 [02:53<00:00, 57.43s/it] +processing left-padding... +processing left-padding... +processing left-padding... +[2025-04-23 02:24:04] Saved checkpoint to checkpoint/BeforeFix_Blip_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_objects/002-GPT-XL/checkpoints/0064000.pt +[2025-04-23 02:24:06] Removed old checkpoint: checkpoint/BeforeFix_Blip_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_objects/002-GPT-XL/checkpoints/0060000.pt + 57%|████████████████████████████████████████████████████████████████████████████████████████████████████▌ | 23121/40903 [8:41:31<6:05:02, 1.23s/it][2025-04-23 02:24:36] (step=0064025) Train Loss: 6.3392, Train Steps/Sec: 0.07 + 57%|████████████████████████████████████████████████████████████████████████████████████████████████████▋ | 23146/40903 [8:42:00<5:53:07, 1.19s/it][2025-04-23 02:25:05] (step=0064050) Train Loss: 6.3861, Train Steps/Sec: 0.84 + 57%|████████████████████████████████████████████████████████████████████████████████████████████████████▊ | 23171/40903 [8:42:30<5:45:23, 1.17s/it][2025-04-23 02:25:35] (step=0064075) Train Loss: 6.3302, Train Steps/Sec: 0.84 + 57%|████████████████████████████████████████████████████████████████████████████████████████████████████▉ | 23196/40903 [8:43:00<5:45:55, 1.17s/it][2025-04-23 02:26:05] (step=0064100) Train Loss: 6.3661, Train Steps/Sec: 0.83 + 57%|█████████████████████████████████████████████████████████████████████████████████████████████████████ | 23221/40903 [8:43:30<5:56:18, 1.21s/it][2025-04-23 02:26:35] (step=0064125) Train Loss: 6.3796, Train Steps/Sec: 0.84 + 57%|█████████████████████████████████████████████████████████████████████████████████████████████████████▏ | 23246/40903 [8:44:00<5:47:41, 1.18s/it][2025-04-23 02:27:04] (step=0064150) Train Loss: 6.3818, Train Steps/Sec: 0.84 + 57%|█████████████████████████████████████████████████████████████████████████████████████████████████████▎ | 23271/40903 [8:44:29<5:47:16, 1.18s/it][2025-04-23 02:27:34] (step=0064175) Train Loss: 6.3574, Train Steps/Sec: 0.84 + 57%|█████████████████████████████████████████████████████████████████████████████████████████████████████▍ | 23296/40903 [8:44:59<5:41:05, 1.16s/it][2025-04-23 02:28:04] (step=0064200) Train Loss: 6.3455, Train Steps/Sec: 0.85 + 57%|█████████████████████████████████████████████████████████████████████████████████████████████████████▍ | 23321/40903 [8:45:28<5:51:50, 1.20s/it][2025-04-23 02:28:33] (step=0064225) Train Loss: 6.3539, Train Steps/Sec: 0.85 + 57%|█████████████████████████████████████████████████████████████████████████████████████████████████████▌ | 23346/40903 [8:46:04<7:14:53, 1.49s/it][2025-04-23 02:29:08] (step=0064250) Train Loss: 6.3644, Train Steps/Sec: 0.71 + 57%|█████████████████████████████████████████████████████████████████████████████████████████████████████▋ | 23371/40903 [8:46:33<5:46:37, 1.19s/it][2025-04-23 02:29:38] (step=0064275) Train Loss: 6.3313, Train Steps/Sec: 0.85 + 57%|█████████████████████████████████████████████████████████████████████████████████████████████████████▊ | 23396/40903 [8:47:03<5:44:24, 1.18s/it][2025-04-23 02:30:08] (step=0064300) Train Loss: 6.3388, Train Steps/Sec: 0.84 + 57%|█████████████████████████████████████████████████████████████████████████████████████████████████████▉ | 23421/40903 [8:47:32<5:46:38, 1.19s/it][2025-04-23 02:30:37] (step=0064325) Train Loss: 6.3166, Train Steps/Sec: 0.85 + 57%|██████████████████████████████████████████████████████████████████████████████████████████████████████ | 23446/40903 [8:48:02<5:39:25, 1.17s/it][2025-04-23 02:31:06] (step=0064350) Train Loss: 6.3685, Train Steps/Sec: 0.86 + 57%|██████████████████████████████████████████████████████████████████████████████████████████████████████▏ | 23471/40903 [8:48:31<5:38:30, 1.17s/it][2025-04-23 02:31:36] (step=0064375) Train Loss: 6.3484, Train Steps/Sec: 0.85 + 57%|██████████████████████████████████████████████████████████████████████████████████████████████████████▏ | 23496/40903 [8:49:00<5:40:26, 1.17s/it][2025-04-23 02:32:05] (step=0064400) Train Loss: 6.3136, Train Steps/Sec: 0.85 + 58%|██████████████████████████████████████████████████████████████████████████████████████████████████████▎ | 23521/40903 [8:49:30<5:45:47, 1.19s/it][2025-04-23 02:32:35] (step=0064425) Train Loss: 6.3520, Train Steps/Sec: 0.85 + 58%|██████████████████████████████████████████████████████████████████████████████████████████████████████▍ | 23546/40903 [8:50:00<5:42:43, 1.18s/it][2025-04-23 02:33:04] (step=0064450) Train Loss: 6.3871, Train Steps/Sec: 0.85 + 58%|██████████████████████████████████████████████████████████████████████████████████████████████████████▌ | 23571/40903 [8:50:29<5:37:35, 1.17s/it][2025-04-23 02:33:34] (step=0064475) Train Loss: 6.3369, Train Steps/Sec: 0.84 + 58%|██████████████████████████████████████████████████████████████████████████████████████████████████████▋ | 23596/40903 [8:50:59<5:43:11, 1.19s/it][2025-04-23 02:34:04] (step=0064500) Train Loss: 6.3527, Train Steps/Sec: 0.84 + 58%|██████████████████████████████████████████████████████████████████████████████████████████████████████▊ | 23621/40903 [8:51:28<5:44:46, 1.20s/it][2025-04-23 02:34:33] (step=0064525) Train Loss: 6.3190, Train Steps/Sec: 0.85 + 58%|██████████████████████████████████████████████████████████████████████████████████████████████████████▉ | 23646/40903 [8:51:58<5:37:14, 1.17s/it][2025-04-23 02:35:03] (step=0064550) Train Loss: 6.2953, Train Steps/Sec: 0.85 + 58%|███████████████████████████████████████████████████████████████████████████████████████████████████████ | 23671/40903 [8:52:28<5:37:24, 1.17s/it][2025-04-23 02:35:32] (step=0064575) Train Loss: 6.4045, Train Steps/Sec: 0.84 + 58%|███████████████████████████████████████████████████████████████████████████████████████████████████████ | 23696/40903 [8:52:57<5:37:59, 1.18s/it][2025-04-23 02:36:02] (step=0064600) Train Loss: 6.3595, Train Steps/Sec: 0.84 + 58%|███████████████████████████████████████████████████████████████████████████████████████████████████████▏ | 23721/40903 [8:53:33<7:13:04, 1.51s/it][2025-04-23 02:36:38] (step=0064625) Train Loss: 6.3060, Train Steps/Sec: 0.70 + 58%|███████████████████████████████████████████████████████████████████████████████████████████████████████▎ | 23746/40903 [8:54:02<5:34:44, 1.17s/it][2025-04-23 02:37:07] (step=0064650) Train Loss: 6.3351, Train Steps/Sec: 0.85 + 58%|███████████████████████████████████████████████████████████████████████████████████████████████████████▍ | 23771/40903 [8:54:32<5:36:53, 1.18s/it][2025-04-23 02:37:37] (step=0064675) Train Loss: 6.4252, Train Steps/Sec: 0.84 + 58%|███████████████████████████████████████████████████████████████████████████████████████████████████████▌ | 23796/40903 [8:55:01<5:34:07, 1.17s/it][2025-04-23 02:38:06] (step=0064700) Train Loss: 6.3582, Train Steps/Sec: 0.85 + 58%|███████████████████████████████████████████████████████████████████████████████████████████████████████▋ | 23821/40903 [8:55:31<5:41:28, 1.20s/it][2025-04-23 02:38:36] (step=0064725) Train Loss: 6.3378, Train Steps/Sec: 0.85 + 58%|███████████████████████████████████████████████████████████████████████████████████████████████████████▊ | 23846/40903 [8:56:01<5:38:35, 1.19s/it][2025-04-23 02:39:05] (step=0064750) Train Loss: 6.3467, Train Steps/Sec: 0.84 + 58%|███████████████████████████████████████████████████████████████████████████████████████████████████████▉ | 23871/40903 [8:56:30<5:34:18, 1.18s/it][2025-04-23 02:39:35] (step=0064775) Train Loss: 6.4106, Train Steps/Sec: 0.85 + 58%|███████████████████████████████████████████████████████████████████████████████████████████████████████▉ | 23896/40903 [8:57:00<5:30:51, 1.17s/it][2025-04-23 02:40:05] (step=0064800) Train Loss: 6.3491, Train Steps/Sec: 0.84 + 58%|████████████████████████████████████████████████████████████████████████████████████████████████████████ | 23921/40903 [8:57:30<5:43:15, 1.21s/it][2025-04-23 02:40:34] (step=0064825) Train Loss: 6.3646, Train Steps/Sec: 0.84 + 59%|████████████████████████████████████████████████████████████████████████████████████████████████████████▏ | 23946/40903 [8:57:59<5:34:09, 1.18s/it][2025-04-23 02:41:04] (step=0064850) Train Loss: 6.3323, Train Steps/Sec: 0.84 + 59%|████████████████████████████████████████████████████████████████████████████████████████████████████████▎ | 23971/40903 [8:58:29<5:31:27, 1.17s/it][2025-04-23 02:41:34] (step=0064875) Train Loss: 6.3399, Train Steps/Sec: 0.85 + 59%|████████████████████████████████████████████████████████████████████████████████████████████████████████▍ | 23996/40903 [8:58:58<5:29:29, 1.17s/it][2025-04-23 02:42:03] (step=0064900) Train Loss: 6.3531, Train Steps/Sec: 0.84 + 59%|████████████████████████████████████████████████████████████████████████████████████████████████████████▌ | 24021/40903 [8:59:28<5:40:06, 1.21s/it][2025-04-23 02:42:33] (step=0064925) Train Loss: 6.3273, Train Steps/Sec: 0.85 + 59%|████████████████████████████████████████████████████████████████████████████████████████████████████████▋ | 24046/40903 [8:59:57<5:34:30, 1.19s/it][2025-04-23 02:43:02] (step=0064950) Train Loss: 6.3822, Train Steps/Sec: 0.85 + 59%|████████████████████████████████████████████████████████████████████████████████████████████████████████▊ | 24071/40903 [9:00:34<8:48:45, 1.88s/it][2025-04-23 02:43:39] (step=0064975) Train Loss: 6.3431, Train Steps/Sec: 0.69 + 59%|████████████████████████████████████████████████████████████████████████████████████████████████████████▊ | 24096/40903 [9:01:04<5:32:00, 1.19s/it][2025-04-23 02:44:08] (step=0065000) Train Loss: 6.3736, Train Steps/Sec: 0.84 + 59%|████████████████████████████████████████████████████████████████████████████████████████████████████████▉ | 24121/40903 [9:01:33<5:31:41, 1.19s/it][2025-04-23 02:44:38] (step=0065025) Train Loss: 6.3617, Train Steps/Sec: 0.85 + 59%|█████████████████████████████████████████████████████████████████████████████████████████████████████████ | 24146/40903 [9:02:03<5:34:32, 1.20s/it][2025-04-23 02:45:07] (step=0065050) Train Loss: 6.3250, Train Steps/Sec: 0.84 + 59%|█████████████████████████████████████████████████████████████████████████████████████████████████████████▏ | 24171/40903 [9:02:32<5:27:43, 1.18s/it][2025-04-23 02:45:37] (step=0065075) Train Loss: 6.3283, Train Steps/Sec: 0.85 + 59%|█████████████████████████████████████████████████████████████████████████████████████████████████████████▎ | 24196/40903 [9:03:02<5:28:56, 1.18s/it][2025-04-23 02:46:07] (step=0065100) Train Loss: 6.3877, Train Steps/Sec: 0.84 + 59%|█████████████████████████████████████████████████████████████████████████████████████████████████████████▍ | 24221/40903 [9:03:32<5:30:14, 1.19s/it][2025-04-23 02:46:36] (step=0065125) Train Loss: 6.3965, Train Steps/Sec: 0.84 + 59%|█████████████████████████████████████████████████████████████████████████████████████████████████████████▌ | 24246/40903 [9:04:01<5:30:39, 1.19s/it][2025-04-23 02:47:06] (step=0065150) Train Loss: 6.3641, Train Steps/Sec: 0.84 + 59%|█████████████████████████████████████████████████████████████████████████████████████████████████████████▌ | 24271/40903 [9:04:31<5:24:51, 1.17s/it][2025-04-23 02:47:36] (step=0065175) Train Loss: 6.3372, Train Steps/Sec: 0.85 + 59%|█████████████████████████████████████████████████████████████████████████████████████████████████████████▋ | 24296/40903 [9:05:00<5:25:48, 1.18s/it][2025-04-23 02:48:05] (step=0065200) Train Loss: 6.3477, Train Steps/Sec: 0.84 + 59%|█████████████████████████████████████████████████████████████████████████████████████████████████████████▊ | 24321/40903 [9:05:30<5:30:37, 1.20s/it][2025-04-23 02:48:35] (step=0065225) Train Loss: 6.3493, Train Steps/Sec: 0.84 + 60%|█████████████████████████████████████████████████████████████████████████████████████████████████████████▉ | 24346/40903 [9:06:00<5:25:45, 1.18s/it][2025-04-23 02:49:04] (step=0065250) Train Loss: 6.3417, Train Steps/Sec: 0.84 + 60%|██████████████████████████████████████████████████████████████████████████████████████████████████████████ | 24371/40903 [9:06:29<5:25:29, 1.18s/it][2025-04-23 02:49:34] (step=0065275) Train Loss: 6.3010, Train Steps/Sec: 0.85 + 60%|██████████████████████████████████████████████████████████████████████████████████████████████████████████▏ | 24396/40903 [9:06:59<5:25:39, 1.18s/it][2025-04-23 02:50:04] (step=0065300) Train Loss: 6.3587, Train Steps/Sec: 0.84 + 60%|██████████████████████████████████████████████████████████████████████████████████████████████████████████▎ | 24421/40903 [9:07:28<5:26:56, 1.19s/it][2025-04-23 02:50:33] (step=0065325) Train Loss: 6.3424, Train Steps/Sec: 0.85 + 60%|██████████████████████████████████████████████████████████████████████████████████████████████████████████▍ | 24446/40903 [9:07:58<5:28:46, 1.20s/it][2025-04-23 02:51:03] (step=0065350) Train Loss: 6.3805, Train Steps/Sec: 0.84 + 60%|██████████████████████████████████████████████████████████████████████████████████████████████████████████▍ | 24471/40903 [9:08:28<5:24:09, 1.18s/it][2025-04-23 02:51:32] (step=0065375) Train Loss: 6.2927, Train Steps/Sec: 0.84 + 60%|██████████████████████████████████████████████████████████████████████████████████████████████████████████▌ | 24496/40903 [9:08:57<5:18:21, 1.16s/it][2025-04-23 02:52:02] (step=0065400) Train Loss: 6.3656, Train Steps/Sec: 0.85 + 60%|██████████████████████████████████████████████████████████████████████████████████████████████████████████▋ | 24521/40903 [9:09:27<5:25:16, 1.19s/it][2025-04-23 02:52:31] (step=0065425) Train Loss: 6.3667, Train Steps/Sec: 0.85 + 60%|██████████████████████████████████████████████████████████████████████████████████████████████████████████▊ | 24546/40903 [9:09:56<5:19:51, 1.17s/it][2025-04-23 02:53:01] (step=0065450) Train Loss: 6.3276, Train Steps/Sec: 0.85 + 60%|██████████████████████████████████████████████████████████████████████████████████████████████████████████▉ | 24571/40903 [9:10:31<5:46:48, 1.27s/it][2025-04-23 02:53:36] (step=0065475) Train Loss: 6.3375, Train Steps/Sec: 0.70 + 60%|███████████████████████████████████████████████████████████████████████████████████████████████████████████ | 24596/40903 [9:11:01<5:18:20, 1.17s/it][2025-04-23 02:54:06] (step=0065500) Train Loss: 6.3213, Train Steps/Sec: 0.84 + 60%|███████████████████████████████████████████████████████████████████████████████████████████████████████████▏ | 24621/40903 [9:11:31<5:26:58, 1.20s/it][2025-04-23 02:54:36] (step=0065525) Train Loss: 6.3455, Train Steps/Sec: 0.84 + 60%|███████████████████████████████████████████████████████████████████████████████████████████████████████████▎ | 24646/40903 [9:12:00<5:18:00, 1.17s/it][2025-04-23 02:55:05] (step=0065550) Train Loss: 6.3407, Train Steps/Sec: 0.85 + 60%|███████████████████████████████████████████████████████████████████████████████████████████████████████████▎ | 24671/40903 [9:12:30<5:17:30, 1.17s/it][2025-04-23 02:55:35] (step=0065575) Train Loss: 6.3742, Train Steps/Sec: 0.85 + 60%|███████████████████████████████████████████████████████████████████████████████████████████████████████████▍ | 24696/40903 [9:13:06<5:35:48, 1.24s/it][2025-04-23 02:56:11] (step=0065600) Train Loss: 6.3811, Train Steps/Sec: 0.69 + 60%|███████████████████████████████████████████████████████████████████████████████████████████████████████████▌ | 24721/40903 [9:13:36<5:24:16, 1.20s/it][2025-04-23 02:56:41] (step=0065625) Train Loss: 6.3286, Train Steps/Sec: 0.85 + 60%|███████████████████████████████████████████████████████████████████████████████████████████████████████████▋ | 24746/40903 [9:14:05<5:19:26, 1.19s/it][2025-04-23 02:57:10] (step=0065650) Train Loss: 6.3204, Train Steps/Sec: 0.84 + 61%|███████████████████████████████████████████████████████████████████████████████████████████████████████████▊ | 24771/40903 [9:14:35<5:14:59, 1.17s/it][2025-04-23 02:57:40] (step=0065675) Train Loss: 6.3540, Train Steps/Sec: 0.84 + 61%|███████████████████████████████████████████████████████████████████████████████████████████████████████████▉ | 24796/40903 [9:15:05<5:13:19, 1.17s/it][2025-04-23 02:58:10] (step=0065700) Train Loss: 6.3491, Train Steps/Sec: 0.84 + 61%|████████████████████████████████████████████████████████████████████████████████████████████████████████████ | 24821/40903 [9:15:34<5:16:53, 1.18s/it][2025-04-23 02:58:39] (step=0065725) Train Loss: 6.3510, Train Steps/Sec: 0.85 + 61%|████████████████████████████████████████████████████████████████████████████████████████████████████████████ | 24846/40903 [9:16:04<5:17:23, 1.19s/it][2025-04-23 02:59:08] (step=0065750) Train Loss: 6.3026, Train Steps/Sec: 0.85 + 61%|████████████████████████████████████████████████████████████████████████████████████████████████████████████▏ | 24871/40903 [9:16:33<5:15:42, 1.18s/it][2025-04-23 02:59:38] (step=0065775) Train Loss: 6.3619, Train Steps/Sec: 0.85 + 61%|████████████████████████████████████████████████████████████████████████████████████████████████████████████▎ | 24896/40903 [9:17:03<5:13:37, 1.18s/it][2025-04-23 03:00:08] (step=0065800) Train Loss: 6.3727, Train Steps/Sec: 0.84 + 61%|████████████████████████████████████████████████████████████████████████████████████████████████████████████▍ | 24921/40903 [9:17:32<5:22:05, 1.21s/it][2025-04-23 03:00:37] (step=0065825) Train Loss: 6.3767, Train Steps/Sec: 0.85 + 61%|████████████████████████████████████████████████████████████████████████████████████████████████████████████▌ | 24946/40903 [9:18:02<5:16:43, 1.19s/it][2025-04-23 03:01:07] (step=0065850) Train Loss: 6.3487, Train Steps/Sec: 0.85 + 61%|████████████████████████████████████████████████████████████████████████████████████████████████████████████▋ | 24971/40903 [9:18:32<5:12:50, 1.18s/it][2025-04-23 03:01:37] (step=0065875) Train Loss: 6.3888, Train Steps/Sec: 0.84 + 61%|████████████████████████████████████████████████████████████████████████████████████████████████████████████▊ | 24996/40903 [9:19:01<5:07:55, 1.16s/it][2025-04-23 03:02:06] (step=0065900) Train Loss: 6.3431, Train Steps/Sec: 0.85 + 61%|████████████████████████████████████████████████████████████████████████████████████████████████████████████▉ | 25021/40903 [9:19:31<5:21:11, 1.21s/it][2025-04-23 03:02:36] (step=0065925) Train Loss: 6.3807, Train Steps/Sec: 0.84 + 61%|████████████████████████████████████████████████████████████████████████████████████████████████████████████▉ | 25046/40903 [9:20:01<5:11:10, 1.18s/it][2025-04-23 03:03:05] (step=0065950) Train Loss: 6.3260, Train Steps/Sec: 0.85 + 61%|█████████████████████████████████████████████████████████████████████████████████████████████████████████████ | 25071/40903 [9:20:30<5:14:30, 1.19s/it][2025-04-23 03:03:35] (step=0065975) Train Loss: 6.3659, Train Steps/Sec: 0.85 + 61%|█████████████████████████████████████████████████████████████████████████████████████████████████████████████▏ | 25096/40903 [9:20:59<5:06:14, 1.16s/it][2025-04-23 03:04:04] (step=0066000) Train Loss: 6.3234, Train Steps/Sec: 0.85 +You are using a model of type instructblip to instantiate a model of type blip-2. This is not supported for all configurations of models and can yield errors. +model_name_or_path:/tmp/haozhezhao/model/instructblip-flan-t5-xl +image_place_holder: +Some kwargs in processor config are unused and will not have any effect: num_query_tokens. +tokenizer length after expend 32102 +tokenizer length before expend 32102 +100%|██████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████| 3/3 [02:52<00:00, 57.63s/it] +[2025-04-23 03:08:46] Finish Eval in 66000 steps...███████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████| 3/3 [02:52<00:00, 57.13s/it] +processing left-padding... +processing left-padding... +processing left-padding... +[2025-04-23 03:09:07] Saved checkpoint to checkpoint/BeforeFix_Blip_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_objects/002-GPT-XL/checkpoints/0066000.pt +[2025-04-23 03:09:10] Removed old checkpoint: checkpoint/BeforeFix_Blip_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_objects/002-GPT-XL/checkpoints/0062000.pt + 61%|█████████████████████████████████████████████████████████████████████████████████████████████████████████████▎ | 25121/40903 [9:26:35<5:20:51, 1.22s/it][2025-04-23 03:09:39] (step=0066025) Train Loss: 6.3385, Train Steps/Sec: 0.07 + 61%|█████████████████████████████████████████████████████████████████████████████████████████████████████████████▍ | 25146/40903 [9:27:04<5:10:32, 1.18s/it][2025-04-23 03:10:09] (step=0066050) Train Loss: 6.3768, Train Steps/Sec: 0.85 + 62%|█████████████████████████████████████████████████████████████████████████████████████████████████████████████▌ | 25171/40903 [9:27:34<5:04:48, 1.16s/it][2025-04-23 03:10:38] (step=0066075) Train Loss: 6.3173, Train Steps/Sec: 0.84 + 62%|█████████████████████████████████████████████████████████████████████████████████████████████████████████████▋ | 25196/40903 [9:28:03<5:05:02, 1.17s/it][2025-04-23 03:11:08] (step=0066100) Train Loss: 6.3832, Train Steps/Sec: 0.84 + 62%|█████████████████████████████████████████████████████████████████████████████████████████████████████████████▊ | 25221/40903 [9:28:33<5:12:38, 1.20s/it][2025-04-23 03:11:38] (step=0066125) Train Loss: 6.3364, Train Steps/Sec: 0.85 + 62%|█████████████████████████████████████████████████████████████████████████████████████████████████████████████▊ | 25246/40903 [9:29:02<5:06:09, 1.17s/it][2025-04-23 03:12:07] (step=0066150) Train Loss: 6.3700, Train Steps/Sec: 0.85 + 62%|█████████████████████████████████████████████████████████████████████████████████████████████████████████████▉ | 25271/40903 [9:29:32<5:09:45, 1.19s/it][2025-04-23 03:12:36] (step=0066175) Train Loss: 6.3740, Train Steps/Sec: 0.85 + 62%|██████████████████████████████████████████████████████████████████████████████████████████████████████████████ | 25296/40903 [9:30:01<5:04:24, 1.17s/it][2025-04-23 03:13:06] (step=0066200) Train Loss: 6.3260, Train Steps/Sec: 0.85 + 62%|██████████████████████████████████████████████████████████████████████████████████████████████████████████████▏ | 25321/40903 [9:30:31<5:13:45, 1.21s/it][2025-04-23 03:13:36] (step=0066225) Train Loss: 6.3687, Train Steps/Sec: 0.84 + 62%|██████████████████████████████████████████████████████████████████████████████████████████████████████████████▎ | 25346/40903 [9:31:00<5:07:52, 1.19s/it][2025-04-23 03:14:05] (step=0066250) Train Loss: 6.3342, Train Steps/Sec: 0.85 + 62%|██████████████████████████████████████████████████████████████████████████████████████████████████████████████▍ | 25371/40903 [9:31:30<5:02:36, 1.17s/it][2025-04-23 03:14:35] (step=0066275) Train Loss: 6.3542, Train Steps/Sec: 0.85 + 62%|██████████████████████████████████████████████████████████████████████████████████████████████████████████████▌ | 25396/40903 [9:31:59<5:04:51, 1.18s/it][2025-04-23 03:15:04] (step=0066300) Train Loss: 6.3369, Train Steps/Sec: 0.84 + 62%|██████████████████████████████████████████████████████████████████████████████████████████████████████████████▋ | 25421/40903 [9:32:29<5:11:55, 1.21s/it][2025-04-23 03:15:34] (step=0066325) Train Loss: 6.3529, Train Steps/Sec: 0.85 + 62%|██████████████████████████████████████████████████████████████████████████████████████████████████████████████▋ | 25446/40903 [9:32:59<5:04:28, 1.18s/it][2025-04-23 03:16:03] (step=0066350) Train Loss: 6.3651, Train Steps/Sec: 0.84 + 62%|██████████████████████████████████████████████████████████████████████████████████████████████████████████████▊ | 25471/40903 [9:33:28<5:03:17, 1.18s/it][2025-04-23 03:16:33] (step=0066375) Train Loss: 6.3005, Train Steps/Sec: 0.85 + 62%|██████████████████████████████████████████████████████████████████████████████████████████████████████████████▉ | 25496/40903 [9:33:57<4:56:27, 1.15s/it][2025-04-23 03:17:02] (step=0066400) Train Loss: 6.3621, Train Steps/Sec: 0.85 + 62%|███████████████████████████████████████████████████████████████████████████████████████████████████████████████ | 25521/40903 [9:34:27<5:08:46, 1.20s/it][2025-04-23 03:17:32] (step=0066425) Train Loss: 6.3078, Train Steps/Sec: 0.85 + 62%|███████████████████████████████████████████████████████████████████████████████████████████████████████████████▏ | 25546/40903 [9:34:57<5:06:00, 1.20s/it][2025-04-23 03:18:02] (step=0066450) Train Loss: 6.3655, Train Steps/Sec: 0.84 + 63%|███████████████████████████████████████████████████████████████████████████████████████████████████████████████▎ | 25571/40903 [9:35:26<4:58:08, 1.17s/it][2025-04-23 03:18:31] (step=0066475) Train Loss: 6.3260, Train Steps/Sec: 0.84 + 63%|███████████████████████████████████████████████████████████████████████████████████████████████████████████████▍ | 25596/40903 [9:35:56<4:56:22, 1.16s/it][2025-04-23 03:19:01] (step=0066500) Train Loss: 6.3164, Train Steps/Sec: 0.85 + 63%|███████████████████████████████████████████████████████████████████████████████████████████████████████████████▍ | 25621/40903 [9:36:25<5:01:49, 1.18s/it][2025-04-23 03:19:30] (step=0066525) Train Loss: 6.3345, Train Steps/Sec: 0.85 + 63%|███████████████████████████████████████████████████████████████████████████████████████████████████████████████▌ | 25646/40903 [9:36:55<5:03:19, 1.19s/it][2025-04-23 03:19:59] (step=0066550) Train Loss: 6.3707, Train Steps/Sec: 0.85 + 63%|███████████████████████████████████████████████████████████████████████████████████████████████████████████████▋ | 25671/40903 [9:37:25<4:57:01, 1.17s/it][2025-04-23 03:20:29] (step=0066575) Train Loss: 6.3026, Train Steps/Sec: 0.84 + 63%|███████████████████████████████████████████████████████████████████████████████████████████████████████████████▊ | 25696/40903 [9:37:54<4:54:10, 1.16s/it][2025-04-23 03:20:59] (step=0066600) Train Loss: 6.3408, Train Steps/Sec: 0.85 + 63%|███████████████████████████████████████████████████████████████████████████████████████████████████████████████▉ | 25721/40903 [9:38:24<5:04:26, 1.20s/it][2025-04-23 03:21:29] (step=0066625) Train Loss: 6.3819, Train Steps/Sec: 0.84 + 63%|████████████████████████████████████████████████████████████████████████████████████████████████████████████████ | 25746/40903 [9:38:53<5:00:40, 1.19s/it][2025-04-23 03:21:58] (step=0066650) Train Loss: 6.3302, Train Steps/Sec: 0.85 + 63%|████████████████████████████████████████████████████████████████████████████████████████████████████████████████▏ | 25771/40903 [9:39:23<4:58:39, 1.18s/it][2025-04-23 03:22:27] (step=0066675) Train Loss: 6.3462, Train Steps/Sec: 0.85 + 63%|████████████████████████████████████████████████████████████████████████████████████████████████████████████████▎ | 25796/40903 [9:39:52<4:55:35, 1.17s/it][2025-04-23 03:22:57] (step=0066700) Train Loss: 6.3153, Train Steps/Sec: 0.85 + 63%|████████████████████████████████████████████████████████████████████████████████████████████████████████████████▎ | 25821/40903 [9:40:22<5:00:30, 1.20s/it][2025-04-23 03:23:27] (step=0066725) Train Loss: 6.3556, Train Steps/Sec: 0.85 + 63%|████████████████████████████████████████████████████████████████████████████████████████████████████████████████▍ | 25846/40903 [9:40:51<4:54:30, 1.17s/it][2025-04-23 03:23:56] (step=0066750) Train Loss: 6.3270, Train Steps/Sec: 0.84 + 63%|████████████████████████████████████████████████████████████████████████████████████████████████████████████████▌ | 25871/40903 [9:41:21<4:54:47, 1.18s/it][2025-04-23 03:24:26] (step=0066775) Train Loss: 6.3833, Train Steps/Sec: 0.85 + 63%|████████████████████████████████████████████████████████████████████████████████████████████████████████████████▋ | 25896/40903 [9:41:50<4:52:49, 1.17s/it][2025-04-23 03:24:55] (step=0066800) Train Loss: 6.3228, Train Steps/Sec: 0.85 + 63%|████████████████████████████████████████████████████████████████████████████████████████████████████████████████▊ | 25921/40903 [9:42:20<4:57:55, 1.19s/it][2025-04-23 03:25:25] (step=0066825) Train Loss: 6.3594, Train Steps/Sec: 0.84 + 63%|████████████████████████████████████████████████████████████████████████████████████████████████████████████████▉ | 25946/40903 [9:42:50<5:00:03, 1.20s/it][2025-04-23 03:25:54] (step=0066850) Train Loss: 6.3680, Train Steps/Sec: 0.84 + 63%|█████████████████████████████████████████████████████████████████████████████████████████████████████████████████ | 25971/40903 [9:43:19<4:48:33, 1.16s/it][2025-04-23 03:26:24] (step=0066875) Train Loss: 6.3347, Train Steps/Sec: 0.84 + 64%|█████████████████████████████████████████████████████████████████████████████████████████████████████████████████▏ | 25996/40903 [9:43:49<4:53:06, 1.18s/it][2025-04-23 03:26:54] (step=0066900) Train Loss: 6.3713, Train Steps/Sec: 0.84 + 64%|█████████████████████████████████████████████████████████████████████████████████████████████████████████████████▏ | 26021/40903 [9:44:19<4:58:40, 1.20s/it][2025-04-23 03:27:24] (step=0066925) Train Loss: 6.3526, Train Steps/Sec: 0.85 + 64%|█████████████████████████████████████████████████████████████████████████████████████████████████████████████████▎ | 26046/40903 [9:44:48<4:56:50, 1.20s/it][2025-04-23 03:27:53] (step=0066950) Train Loss: 6.3577, Train Steps/Sec: 0.84 + 64%|█████████████████████████████████████████████████████████████████████████████████████████████████████████████████▍ | 26071/40903 [9:45:18<4:49:18, 1.17s/it][2025-04-23 03:28:23] (step=0066975) Train Loss: 6.3842, Train Steps/Sec: 0.84 + 64%|█████████████████████████████████████████████████████████████████████████████████████████████████████████████████▌ | 26096/40903 [9:45:47<4:48:06, 1.17s/it][2025-04-23 03:28:52] (step=0067000) Train Loss: 6.3351, Train Steps/Sec: 0.85 + 64%|█████████████████████████████████████████████████████████████████████████████████████████████████████████████████▋ | 26121/40903 [9:46:17<4:55:02, 1.20s/it][2025-04-23 03:29:22] (step=0067025) Train Loss: 6.3368, Train Steps/Sec: 0.84 + 64%|█████████████████████████████████████████████████████████████████████████████████████████████████████████████████▊ | 26146/40903 [9:46:47<4:52:03, 1.19s/it][2025-04-23 03:29:52] (step=0067050) Train Loss: 6.3699, Train Steps/Sec: 0.84 + 64%|█████████████████████████████████████████████████████████████████████████████████████████████████████████████████▉ | 26171/40903 [9:47:17<4:48:28, 1.17s/it][2025-04-23 03:30:22] (step=0067075) Train Loss: 6.3649, Train Steps/Sec: 0.84 + 64%|█████████████████████████████████████████████████████████████████████████████████████████████████████████████████▉ | 26196/40903 [9:47:46<4:44:45, 1.16s/it][2025-04-23 03:30:51] (step=0067100) Train Loss: 6.3492, Train Steps/Sec: 0.84 + 64%|██████████████████████████████████████████████████████████████████████████████████████████████████████████████████ | 26221/40903 [9:48:16<4:54:31, 1.20s/it][2025-04-23 03:31:21] (step=0067125) Train Loss: 6.3555, Train Steps/Sec: 0.85 + 64%|██████████████████████████████████████████████████████████████████████████████████████████████████████████████████▏ | 26246/40903 [9:48:45<4:49:07, 1.18s/it][2025-04-23 03:31:50] (step=0067150) Train Loss: 6.3542, Train Steps/Sec: 0.85 + 64%|██████████████████████████████████████████████████████████████████████████████████████████████████████████████████▎ | 26271/40903 [9:49:15<4:48:39, 1.18s/it][2025-04-23 03:32:20] (step=0067175) Train Loss: 6.3286, Train Steps/Sec: 0.84 + 64%|██████████████████████████████████████████████████████████████████████████████████████████████████████████████████▍ | 26296/40903 [9:49:45<4:48:09, 1.18s/it][2025-04-23 03:32:50] (step=0067200) Train Loss: 6.2953, Train Steps/Sec: 0.84 + 64%|██████████████████████████████████████████████████████████████████████████████████████████████████████████████████▌ | 26321/40903 [9:50:14<4:52:30, 1.20s/it][2025-04-23 03:33:19] (step=0067225) Train Loss: 6.3171, Train Steps/Sec: 0.84 + 64%|██████████████████████████████████████████████████████████████████████████████████████████████████████████████████▋ | 26346/40903 [9:50:44<4:44:28, 1.17s/it][2025-04-23 03:33:49] (step=0067250) Train Loss: 6.3496, Train Steps/Sec: 0.85 + 64%|██████████████████████████████████████████████████████████████████████████████████████████████████████████████████▊ | 26371/40903 [9:51:13<4:42:29, 1.17s/it][2025-04-23 03:34:18] (step=0067275) Train Loss: 6.3219, Train Steps/Sec: 0.85 + 65%|██████████████████████████████████████████████████████████████████████████████████████████████████████████████████▊ | 26396/40903 [9:51:43<4:42:12, 1.17s/it][2025-04-23 03:34:48] (step=0067300) Train Loss: 6.3881, Train Steps/Sec: 0.85 + 65%|██████████████████████████████████████████████████████████████████████████████████████████████████████████████████▉ | 26421/40903 [9:52:13<4:45:23, 1.18s/it][2025-04-23 03:35:17] (step=0067325) Train Loss: 6.3436, Train Steps/Sec: 0.84 + 65%|███████████████████████████████████████████████████████████████████████████████████████████████████████████████████ | 26446/40903 [9:52:42<4:45:40, 1.19s/it][2025-04-23 03:35:47] (step=0067350) Train Loss: 6.3732, Train Steps/Sec: 0.85 + 65%|███████████████████████████████████████████████████████████████████████████████████████████████████████████████████▏ | 26471/40903 [9:53:12<4:41:49, 1.17s/it][2025-04-23 03:36:17] (step=0067375) Train Loss: 6.3529, Train Steps/Sec: 0.85 + 65%|███████████████████████████████████████████████████████████████████████████████████████████████████████████████████▎ | 26496/40903 [9:53:41<4:44:17, 1.18s/it][2025-04-23 03:36:46] (step=0067400) Train Loss: 6.3418, Train Steps/Sec: 0.84 + 65%|███████████████████████████████████████████████████████████████████████████████████████████████████████████████████▍ | 26521/40903 [9:54:11<4:46:53, 1.20s/it][2025-04-23 03:37:16] (step=0067425) Train Loss: 6.3578, Train Steps/Sec: 0.84 + 65%|███████████████████████████████████████████████████████████████████████████████████████████████████████████████████▌ | 26546/40903 [9:54:41<4:42:59, 1.18s/it][2025-04-23 03:37:45] (step=0067450) Train Loss: 6.3558, Train Steps/Sec: 0.85 + 65%|███████████████████████████████████████████████████████████████████████████████████████████████████████████████████▋ | 26571/40903 [9:55:10<4:39:42, 1.17s/it][2025-04-23 03:38:15] (step=0067475) Train Loss: 6.3481, Train Steps/Sec: 0.85 + 65%|███████████████████████████████████████████████████████████████████████████████████████████████████████████████████▋ | 26596/40903 [9:55:40<4:39:01, 1.17s/it][2025-04-23 03:38:45] (step=0067500) Train Loss: 6.3243, Train Steps/Sec: 0.84 + 65%|███████████████████████████████████████████████████████████████████████████████████████████████████████████████████▊ | 26621/40903 [9:56:09<4:43:43, 1.19s/it][2025-04-23 03:39:14] (step=0067525) Train Loss: 6.3968, Train Steps/Sec: 0.85 + 65%|███████████████████████████████████████████████████████████████████████████████████████████████████████████████████▉ | 26646/40903 [9:56:39<4:40:53, 1.18s/it][2025-04-23 03:39:43] (step=0067550) Train Loss: 6.3015, Train Steps/Sec: 0.85 + 65%|████████████████████████████████████████████████████████████████████████████████████████████████████████████████████ | 26671/40903 [9:57:08<4:39:19, 1.18s/it][2025-04-23 03:40:13] (step=0067575) Train Loss: 6.3854, Train Steps/Sec: 0.85 + 65%|████████████████████████████████████████████████████████████████████████████████████████████████████████████████████▏ | 26696/40903 [9:57:38<4:37:45, 1.17s/it][2025-04-23 03:40:43] (step=0067600) Train Loss: 6.3681, Train Steps/Sec: 0.84 + 65%|████████████████████████████████████████████████████████████████████████████████████████████████████████████████████▎ | 26721/40903 [9:58:07<4:46:59, 1.21s/it][2025-04-23 03:41:12] (step=0067625) Train Loss: 6.3718, Train Steps/Sec: 0.85 + 65%|████████████████████████████████████████████████████████████████████████████████████████████████████████████████████▍ | 26746/40903 [9:58:37<4:39:51, 1.19s/it][2025-04-23 03:41:42] (step=0067650) Train Loss: 6.3531, Train Steps/Sec: 0.84 + 65%|████████████████████████████████████████████████████████████████████████████████████████████████████████████████████▌ | 26771/40903 [9:59:07<4:37:47, 1.18s/it][2025-04-23 03:42:12] (step=0067675) Train Loss: 6.4035, Train Steps/Sec: 0.84 + 66%|████████████████████████████████████████████████████████████████████████████████████████████████████████████████████▌ | 26796/40903 [9:59:36<4:31:20, 1.15s/it][2025-04-23 03:42:41] (step=0067700) Train Loss: 6.3745, Train Steps/Sec: 0.84 + 66%|████████████████████████████████████████████████████████████████████████████████████████████████████████████████████ | 26821/40903 [10:00:06<4:40:54, 1.20s/it][2025-04-23 03:43:11] (step=0067725) Train Loss: 6.3522, Train Steps/Sec: 0.85 + 66%|████████████████████████████████████████████████████████████████████████████████████████████████████████████████████▏ | 26846/40903 [10:00:36<4:38:15, 1.19s/it][2025-04-23 03:43:41] (step=0067750) Train Loss: 6.3945, Train Steps/Sec: 0.84 + 66%|████████████████████████████████████████████████████████████████████████████████████████████████████████████████████▎ | 26871/40903 [10:01:05<4:33:24, 1.17s/it][2025-04-23 03:44:10] (step=0067775) Train Loss: 6.3681, Train Steps/Sec: 0.85 + 66%|████████████████████████████████████████████████████████████████████████████████████████████████████████████████████▍ | 26896/40903 [10:01:35<4:28:48, 1.15s/it][2025-04-23 03:44:40] (step=0067800) Train Loss: 6.3421, Train Steps/Sec: 0.85 + 66%|████████████████████████████████████████████████████████████████████████████████████████████████████████████████████▍ | 26921/40903 [10:02:04<4:44:07, 1.22s/it][2025-04-23 03:45:09] (step=0067825) Train Loss: 6.3226, Train Steps/Sec: 0.85 + 66%|████████████████████████████████████████████████████████████████████████████████████████████████████████████████████▌ | 26946/40903 [10:02:34<4:36:05, 1.19s/it][2025-04-23 03:45:39] (step=0067850) Train Loss: 6.3771, Train Steps/Sec: 0.84 + 66%|████████████████████████████████████████████████████████████████████████████████████████████████████████████████████▋ | 26971/40903 [10:03:03<4:34:26, 1.18s/it][2025-04-23 03:46:08] (step=0067875) Train Loss: 6.3529, Train Steps/Sec: 0.85 + 66%|████████████████████████████████████████████████████████████████████████████████████████████████████████████████████▊ | 26996/40903 [10:03:33<4:31:31, 1.17s/it][2025-04-23 03:46:38] (step=0067900) Train Loss: 6.3673, Train Steps/Sec: 0.85 + 66%|████████████████████████████████████████████████████████████████████████████████████████████████████████████████████▉ | 27021/40903 [10:04:03<4:41:15, 1.22s/it][2025-04-23 03:47:07] (step=0067925) Train Loss: 6.3418, Train Steps/Sec: 0.84 + 66%|█████████████████████████████████████████████████████████████████████████████████████████████████████████████████████ | 27046/40903 [10:04:32<4:33:24, 1.18s/it][2025-04-23 03:47:37] (step=0067950) Train Loss: 6.3299, Train Steps/Sec: 0.85 + 66%|█████████████████████████████████████████████████████████████████████████████████████████████████████████████████████▏ | 27071/40903 [10:05:02<4:29:37, 1.17s/it][2025-04-23 03:48:06] (step=0067975) Train Loss: 6.3490, Train Steps/Sec: 0.85 + 66%|█████████████████████████████████████████████████████████████████████████████████████████████████████████████████████▎ | 27096/40903 [10:05:31<4:29:53, 1.17s/it][2025-04-23 03:48:36] (step=0068000) Train Loss: 6.3817, Train Steps/Sec: 0.84 +You are using a model of type instructblip to instantiate a model of type blip-2. This is not supported for all configurations of models and can yield errors. +model_name_or_path:/tmp/haozhezhao/model/instructblip-flan-t5-xl +image_place_holder: +Some kwargs in processor config are unused and will not have any effect: num_query_tokens. +tokenizer length after expend 32102 +tokenizer length before expend 32102 +100%|██████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████| 3/3 [02:53<00:00, 57.82s/it] +[2025-04-23 03:53:19] Finish Eval in 68000 steps...███████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████| 3/3 [02:52<00:00, 57.31s/it] +processing left-padding... +processing left-padding... +processing left-padding... +[2025-04-23 03:53:40] Saved checkpoint to checkpoint/BeforeFix_Blip_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_objects/002-GPT-XL/checkpoints/0068000.pt +[2025-04-23 03:53:42] Removed old checkpoint: checkpoint/BeforeFix_Blip_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_objects/002-GPT-XL/checkpoints/0064000.pt + 66%|█████████████████████████████████████████████████████████████████████████████████████████████████████████████████████▎ | 27121/40903 [10:11:07<4:40:41, 1.22s/it][2025-04-23 03:54:12] (step=0068025) Train Loss: 6.3556, Train Steps/Sec: 0.07 + 66%|█████████████████████████████████████████████████████████████████████████████████████████████████████████████████████▍ | 27146/40903 [10:11:37<4:29:50, 1.18s/it][2025-04-23 03:54:41] (step=0068050) Train Loss: 6.3482, Train Steps/Sec: 0.85 + 66%|█████████████████████████████████████████████████████████████████████████████████████████████████████████████████████▌ | 27171/40903 [10:12:06<4:28:33, 1.17s/it][2025-04-23 03:55:11] (step=0068075) Train Loss: 6.3781, Train Steps/Sec: 0.84 + 66%|█████████████████████████████████████████████████████████████████████████████████████████████████████████████████████▋ | 27196/40903 [10:12:36<4:29:21, 1.18s/it][2025-04-23 03:55:41] (step=0068100) Train Loss: 6.3700, Train Steps/Sec: 0.84 + 67%|█████████████████████████████████████████████████████████████████████████████████████████████████████████████████████▊ | 27221/40903 [10:13:05<4:31:08, 1.19s/it][2025-04-23 03:56:10] (step=0068125) Train Loss: 6.3353, Train Steps/Sec: 0.85 + 67%|█████████████████████████████████████████████████████████████████████████████████████████████████████████████████████▉ | 27246/40903 [10:13:35<4:31:41, 1.19s/it][2025-04-23 03:56:40] (step=0068150) Train Loss: 6.3502, Train Steps/Sec: 0.84 + 67%|██████████████████████████████████████████████████████████████████████████████████████████████████████████████████████ | 27271/40903 [10:14:05<4:25:31, 1.17s/it][2025-04-23 03:57:10] (step=0068175) Train Loss: 6.3670, Train Steps/Sec: 0.85 + 67%|██████████████████████████████████████████████████████████████████████████████████████████████████████████████████████ | 27296/40903 [10:14:34<4:22:15, 1.16s/it][2025-04-23 03:57:39] (step=0068200) Train Loss: 6.3444, Train Steps/Sec: 0.85 + 67%|██████████████████████████████████████████████████████████████████████████████████████████████████████████████████████▏ | 27321/40903 [10:15:04<4:31:34, 1.20s/it][2025-04-23 03:58:09] (step=0068225) Train Loss: 6.3998, Train Steps/Sec: 0.85 + 67%|██████████████████████████████████████████████████████████████████████████████████████████████████████████████████████▎ | 27346/40903 [10:15:33<4:28:58, 1.19s/it][2025-04-23 03:58:38] (step=0068250) Train Loss: 6.3752, Train Steps/Sec: 0.84 + 67%|██████████████████████████████████████████████████████████████████████████████████████████████████████████████████████▍ | 27371/40903 [10:16:03<4:23:47, 1.17s/it][2025-04-23 03:59:08] (step=0068275) Train Loss: 6.3178, Train Steps/Sec: 0.85 + 67%|██████████████████████████████████████████████████████████████████████████████████████████████████████████████████████▌ | 27396/40903 [10:16:33<4:24:35, 1.18s/it][2025-04-23 03:59:38] (step=0068300) Train Loss: 6.3597, Train Steps/Sec: 0.84 + 67%|██████████████████████████████████████████████████████████████████████████████████████████████████████████████████████▋ | 27421/40903 [10:17:02<4:27:19, 1.19s/it][2025-04-23 04:00:07] (step=0068325) Train Loss: 6.3677, Train Steps/Sec: 0.85 + 67%|██████████████████████████████████████████████████████████████████████████████████████████████████████████████████████▊ | 27446/40903 [10:17:32<4:25:41, 1.18s/it][2025-04-23 04:00:37] (step=0068350) Train Loss: 6.3767, Train Steps/Sec: 0.84 + 67%|██████████████████████████████████████████████████████████████████████████████████████████████████████████████████████▉ | 27471/40903 [10:18:01<4:25:35, 1.19s/it][2025-04-23 04:01:06] (step=0068375) Train Loss: 6.3317, Train Steps/Sec: 0.84 + 67%|██████████████████████████████████████████████████████████████████████████████████████████████████████████████████████▉ | 27496/40903 [10:18:31<4:24:11, 1.18s/it][2025-04-23 04:01:36] (step=0068400) Train Loss: 6.3709, Train Steps/Sec: 0.84 + 67%|███████████████████████████████████████████████████████████████████████████████████████████████████████████████████████ | 27521/40903 [10:19:01<4:27:15, 1.20s/it][2025-04-23 04:02:06] (step=0068425) Train Loss: 6.3203, Train Steps/Sec: 0.83 + 67%|███████████████████████████████████████████████████████████████████████████████████████████████████████████████████████▏ | 27546/40903 [10:19:31<4:23:05, 1.18s/it][2025-04-23 04:02:35] (step=0068450) Train Loss: 6.3157, Train Steps/Sec: 0.85 + 67%|███████████████████████████████████████████████████████████████████████████████████████████████████████████████████████▎ | 27571/40903 [10:20:00<4:21:46, 1.18s/it][2025-04-23 04:03:05] (step=0068475) Train Loss: 6.3468, Train Steps/Sec: 0.84 + 67%|███████████████████████████████████████████████████████████████████████████████████████████████████████████████████████▍ | 27596/40903 [10:20:30<4:20:18, 1.17s/it][2025-04-23 04:03:35] (step=0068500) Train Loss: 6.2917, Train Steps/Sec: 0.84 + 68%|███████████████████████████████████████████████████████████████████████████████████████████████████████████████████████▌ | 27621/40903 [10:21:00<4:26:30, 1.20s/it][2025-04-23 04:04:04] (step=0068525) Train Loss: 6.3486, Train Steps/Sec: 0.84 + 68%|███████████████████████████████████████████████████████████████████████████████████████████████████████████████████████▋ | 27646/40903 [10:21:29<4:23:29, 1.19s/it][2025-04-23 04:04:34] (step=0068550) Train Loss: 6.3864, Train Steps/Sec: 0.84 + 68%|███████████████████████████████████████████████████████████████████████████████████████████████████████████████████████▋ | 27671/40903 [10:21:59<4:20:05, 1.18s/it][2025-04-23 04:05:04] (step=0068575) Train Loss: 6.3577, Train Steps/Sec: 0.85 + 68%|███████████████████████████████████████████████████████████████████████████████████████████████████████████████████████▊ | 27696/40903 [10:22:28<4:15:40, 1.16s/it][2025-04-23 04:05:33] (step=0068600) Train Loss: 6.3270, Train Steps/Sec: 0.85 + 68%|███████████████████████████████████████████████████████████████████████████████████████████████████████████████████████▉ | 27721/40903 [10:22:58<4:23:10, 1.20s/it][2025-04-23 04:06:03] (step=0068625) Train Loss: 6.3440, Train Steps/Sec: 0.85 + 68%|████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████ | 27746/40903 [10:23:27<4:18:17, 1.18s/it][2025-04-23 04:06:32] (step=0068650) Train Loss: 6.3424, Train Steps/Sec: 0.84 + 68%|████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████▏ | 27771/40903 [10:23:57<4:15:29, 1.17s/it][2025-04-23 04:07:02] (step=0068675) Train Loss: 6.3229, Train Steps/Sec: 0.85 + 68%|████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████▎ | 27796/40903 [10:24:26<4:11:38, 1.15s/it][2025-04-23 04:07:31] (step=0068700) Train Loss: 6.3194, Train Steps/Sec: 0.84 + 68%|████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████▍ | 27821/40903 [10:24:56<4:24:54, 1.21s/it][2025-04-23 04:08:01] (step=0068725) Train Loss: 6.3478, Train Steps/Sec: 0.84 + 68%|████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████▍ | 27846/40903 [10:25:26<4:18:57, 1.19s/it][2025-04-23 04:08:31] (step=0068750) Train Loss: 6.3208, Train Steps/Sec: 0.84 + 68%|████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████▌ | 27871/40903 [10:25:55<4:16:22, 1.18s/it][2025-04-23 04:09:00] (step=0068775) Train Loss: 6.3846, Train Steps/Sec: 0.84 + 68%|████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████▋ | 27896/40903 [10:26:25<4:13:10, 1.17s/it][2025-04-23 04:09:30] (step=0068800) Train Loss: 6.3346, Train Steps/Sec: 0.84 + 68%|████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████▊ | 27921/40903 [10:26:55<4:20:15, 1.20s/it][2025-04-23 04:10:00] (step=0068825) Train Loss: 6.3480, Train Steps/Sec: 0.84 + 68%|████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████▉ | 27946/40903 [10:27:24<4:14:23, 1.18s/it][2025-04-23 04:10:29] (step=0068850) Train Loss: 6.3485, Train Steps/Sec: 0.85 + 68%|█████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████ | 27971/40903 [10:27:54<4:14:27, 1.18s/it][2025-04-23 04:10:59] (step=0068875) Train Loss: 6.3587, Train Steps/Sec: 0.85 + 68%|█████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████▏ | 27996/40903 [10:28:23<4:10:56, 1.17s/it][2025-04-23 04:11:28] (step=0068900) Train Loss: 6.3659, Train Steps/Sec: 0.84 + 69%|█████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████▎ | 28021/40903 [10:28:53<4:21:07, 1.22s/it][2025-04-23 04:11:58] (step=0068925) Train Loss: 6.3085, Train Steps/Sec: 0.84 + 69%|█████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████▎ | 28046/40903 [10:29:23<4:13:26, 1.18s/it][2025-04-23 04:12:28] (step=0068950) Train Loss: 6.3548, Train Steps/Sec: 0.85 + 69%|█████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████▍ | 28071/40903 [10:29:52<4:15:56, 1.20s/it][2025-04-23 04:12:57] (step=0068975) Train Loss: 6.3191, Train Steps/Sec: 0.84 + 69%|█████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████▌ | 28096/40903 [10:30:22<4:12:02, 1.18s/it][2025-04-23 04:13:27] (step=0069000) Train Loss: 6.3589, Train Steps/Sec: 0.84 + 69%|█████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████▋ | 28121/40903 [10:30:58<8:59:51, 2.53s/it][2025-04-23 04:14:03] (step=0069025) Train Loss: 6.3698, Train Steps/Sec: 0.70 + 69%|█████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████▊ | 28146/40903 [10:31:27<4:09:59, 1.18s/it][2025-04-23 04:14:32] (step=0069050) Train Loss: 6.3421, Train Steps/Sec: 0.85 + 69%|█████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████▉ | 28171/40903 [10:31:57<4:10:04, 1.18s/it][2025-04-23 04:15:02] (step=0069075) Train Loss: 6.3555, Train Steps/Sec: 0.84 + 69%|██████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████ | 28196/40903 [10:32:27<4:07:05, 1.17s/it][2025-04-23 04:15:32] (step=0069100) Train Loss: 6.3277, Train Steps/Sec: 0.84 + 69%|██████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████ | 28221/40903 [10:32:56<4:12:23, 1.19s/it][2025-04-23 04:16:01] (step=0069125) Train Loss: 6.3516, Train Steps/Sec: 0.85 + 69%|██████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████▏ | 28246/40903 [10:33:26<4:10:04, 1.19s/it][2025-04-23 04:16:31] (step=0069150) Train Loss: 6.3877, Train Steps/Sec: 0.85 + 69%|██████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████▎ | 28271/40903 [10:33:56<4:06:24, 1.17s/it][2025-04-23 04:17:00] (step=0069175) Train Loss: 6.3725, Train Steps/Sec: 0.85 + 69%|██████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████▍ | 28296/40903 [10:34:25<4:04:41, 1.16s/it][2025-04-23 04:17:30] (step=0069200) Train Loss: 6.3530, Train Steps/Sec: 0.84 + 69%|██████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████▌ | 28321/40903 [10:34:55<4:11:09, 1.20s/it][2025-04-23 04:18:00] (step=0069225) Train Loss: 6.2957, Train Steps/Sec: 0.85 + 69%|██████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████▋ | 28346/40903 [10:35:24<4:07:13, 1.18s/it][2025-04-23 04:18:29] (step=0069250) Train Loss: 6.3530, Train Steps/Sec: 0.84 + 69%|██████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████▊ | 28371/40903 [10:35:54<4:05:25, 1.18s/it][2025-04-23 04:18:59] (step=0069275) Train Loss: 6.3147, Train Steps/Sec: 0.85 + 69%|██████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████▉ | 28396/40903 [10:36:24<4:07:45, 1.19s/it][2025-04-23 04:19:29] (step=0069300) Train Loss: 6.3508, Train Steps/Sec: 0.84 + 69%|██████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████▉ | 28421/40903 [10:36:53<4:09:37, 1.20s/it][2025-04-23 04:19:58] (step=0069325) Train Loss: 6.3524, Train Steps/Sec: 0.85 + 70%|███████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████ | 28446/40903 [10:37:23<4:06:57, 1.19s/it][2025-04-23 04:20:28] (step=0069350) Train Loss: 6.3527, Train Steps/Sec: 0.84 + 70%|███████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████▏ | 28471/40903 [10:37:53<4:03:49, 1.18s/it][2025-04-23 04:20:57] (step=0069375) Train Loss: 6.3539, Train Steps/Sec: 0.84 + 70%|███████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████▎ | 28496/40903 [10:38:22<4:00:37, 1.16s/it][2025-04-23 04:21:27] (step=0069400) Train Loss: 6.3681, Train Steps/Sec: 0.85 + 70%|███████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████▍ | 28521/40903 [10:38:52<4:06:52, 1.20s/it][2025-04-23 04:21:56] (step=0069425) Train Loss: 6.3468, Train Steps/Sec: 0.85 + 70%|███████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████▌ | 28546/40903 [10:39:21<4:00:51, 1.17s/it][2025-04-23 04:22:26] (step=0069450) Train Loss: 6.3334, Train Steps/Sec: 0.84 + 70%|███████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████▋ | 28571/40903 [10:39:51<4:03:40, 1.19s/it][2025-04-23 04:22:56] (step=0069475) Train Loss: 6.3455, Train Steps/Sec: 0.84 + 70%|███████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████ | 28596/40903 [10:40:28<11:47:50, 3.45s/it][2025-04-23 04:23:33] (step=0069500) Train Loss: 6.3640, Train Steps/Sec: 0.67 + 70%|███████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████▊ | 28621/40903 [10:40:58<4:06:10, 1.20s/it][2025-04-23 04:24:03] (step=0069525) Train Loss: 6.3660, Train Steps/Sec: 0.84 + 70%|███████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████▉ | 28646/40903 [10:41:27<4:00:36, 1.18s/it][2025-04-23 04:24:32] (step=0069550) Train Loss: 6.3431, Train Steps/Sec: 0.85 + 70%|████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████ | 28671/40903 [10:41:57<4:01:11, 1.18s/it][2025-04-23 04:25:02] (step=0069575) Train Loss: 6.3408, Train Steps/Sec: 0.84 + 70%|████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████▏ | 28696/40903 [10:42:32<3:56:11, 1.16s/it][2025-04-23 04:25:37] (step=0069600) Train Loss: 6.3426, Train Steps/Sec: 0.71 + 70%|████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████▎ | 28721/40903 [10:43:02<4:05:55, 1.21s/it][2025-04-23 04:26:07] (step=0069625) Train Loss: 6.2847, Train Steps/Sec: 0.85 + 70%|████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████▍ | 28746/40903 [10:43:32<3:59:23, 1.18s/it][2025-04-23 04:26:36] (step=0069650) Train Loss: 6.3360, Train Steps/Sec: 0.84 + 70%|████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████▌ | 28771/40903 [10:44:08<4:08:26, 1.23s/it][2025-04-23 04:27:12] (step=0069675) Train Loss: 6.3424, Train Steps/Sec: 0.69 + 70%|████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████▌ | 28796/40903 [10:44:37<3:59:28, 1.19s/it][2025-04-23 04:27:42] (step=0069700) Train Loss: 6.3369, Train Steps/Sec: 0.84 + 70%|████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████▋ | 28821/40903 [10:45:07<3:59:34, 1.19s/it][2025-04-23 04:28:12] (step=0069725) Train Loss: 6.2735, Train Steps/Sec: 0.85 + 71%|████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████▊ | 28846/40903 [10:45:36<3:58:20, 1.19s/it][2025-04-23 04:28:41] (step=0069750) Train Loss: 6.3332, Train Steps/Sec: 0.85 + 71%|████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████▉ | 28871/40903 [10:46:06<3:59:08, 1.19s/it][2025-04-23 04:29:11] (step=0069775) Train Loss: 6.3216, Train Steps/Sec: 0.85 + 71%|█████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████ | 28896/40903 [10:46:35<3:51:22, 1.16s/it][2025-04-23 04:29:40] (step=0069800) Train Loss: 6.3615, Train Steps/Sec: 0.84 + 71%|█████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████▏ | 28921/40903 [10:47:05<3:57:32, 1.19s/it][2025-04-23 04:30:10] (step=0069825) Train Loss: 6.3897, Train Steps/Sec: 0.85 + 71%|█████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████▎ | 28946/40903 [10:47:34<3:54:23, 1.18s/it][2025-04-23 04:30:39] (step=0069850) Train Loss: 6.3381, Train Steps/Sec: 0.85 + 71%|█████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████▎ | 28971/40903 [10:48:04<3:55:59, 1.19s/it][2025-04-23 04:31:09] (step=0069875) Train Loss: 6.3087, Train Steps/Sec: 0.85 + 71%|█████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████▍ | 28996/40903 [10:48:34<3:53:29, 1.18s/it][2025-04-23 04:31:38] (step=0069900) Train Loss: 6.3758, Train Steps/Sec: 0.85 + 71%|█████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████▌ | 29021/40903 [10:49:03<3:57:27, 1.20s/it][2025-04-23 04:32:08] (step=0069925) Train Loss: 6.3304, Train Steps/Sec: 0.84 + 71%|█████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████▋ | 29046/40903 [10:49:33<3:52:47, 1.18s/it][2025-04-23 04:32:38] (step=0069950) Train Loss: 6.3813, Train Steps/Sec: 0.84 + 71%|█████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████▊ | 29071/40903 [10:50:02<3:51:06, 1.17s/it][2025-04-23 04:33:07] (step=0069975) Train Loss: 6.3422, Train Steps/Sec: 0.85 + 71%|█████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████▉ | 29096/40903 [10:50:32<3:49:30, 1.17s/it][2025-04-23 04:33:37] (step=0070000) Train Loss: 6.3426, Train Steps/Sec: 0.84 +You are using a model of type instructblip to instantiate a model of type blip-2. This is not supported for all configurations of models and can yield errors. +model_name_or_path:/tmp/haozhezhao/model/instructblip-flan-t5-xl +image_place_holder: +Some kwargs in processor config are unused and will not have any effect: num_query_tokens. +tokenizer length after expend 32102 +tokenizer length before expend 32102 +100%|███████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████| 3/3 [02:54<00:00, 58.25s/it] +[2025-04-23 04:38:21] Finish Eval in 70000 steps...████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████| 3/3 [02:54<00:00, 57.75s/it] +processing left-padding... +processing left-padding... +processing left-padding... +[2025-04-23 04:38:43] Saved checkpoint to checkpoint/BeforeFix_Blip_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_objects/002-GPT-XL/checkpoints/0070000.pt +[2025-04-23 04:38:45] Removed old checkpoint: checkpoint/BeforeFix_Blip_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_objects/002-GPT-XL/checkpoints/0066000.pt + 71%|██████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████ | 29121/40903 [10:56:10<3:58:14, 1.21s/it][2025-04-23 04:39:14] (step=0070025) Train Loss: 6.3465, Train Steps/Sec: 0.07 + 71%|██████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████ | 29146/40903 [10:56:39<3:53:24, 1.19s/it][2025-04-23 04:39:44] (step=0070050) Train Loss: 6.3475, Train Steps/Sec: 0.85 + 71%|██████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████▏ | 29171/40903 [10:57:16<4:00:29, 1.23s/it][2025-04-23 04:40:20] (step=0070075) Train Loss: 6.3551, Train Steps/Sec: 0.69 + 71%|██████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████▎ | 29196/40903 [10:57:45<3:51:49, 1.19s/it][2025-04-23 04:40:50] (step=0070100) Train Loss: 6.3826, Train Steps/Sec: 0.84 + 71%|██████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████▍ | 29221/40903 [10:58:15<3:55:36, 1.21s/it][2025-04-23 04:41:20] (step=0070125) Train Loss: 6.3390, Train Steps/Sec: 0.84 + 72%|██████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████▌ | 29246/40903 [10:58:45<3:51:02, 1.19s/it][2025-04-23 04:41:50] (step=0070150) Train Loss: 6.3039, Train Steps/Sec: 0.84 + 72%|██████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████▋ | 29271/40903 [10:59:14<3:50:20, 1.19s/it][2025-04-23 04:42:19] (step=0070175) Train Loss: 6.3818, Train Steps/Sec: 0.85 + 72%|██████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████▊ | 29296/40903 [10:59:44<3:46:22, 1.17s/it][2025-04-23 04:42:49] (step=0070200) Train Loss: 6.2835, Train Steps/Sec: 0.84 + 72%|██████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████▉ | 29321/40903 [11:00:14<3:49:32, 1.19s/it][2025-04-23 04:43:19] (step=0070225) Train Loss: 6.3692, Train Steps/Sec: 0.84 + 72%|██████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████▉ | 29346/40903 [11:00:43<3:48:46, 1.19s/it][2025-04-23 04:43:48] (step=0070250) Train Loss: 6.3930, Train Steps/Sec: 0.84 + 72%|███████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████ | 29371/40903 [11:01:13<3:46:32, 1.18s/it][2025-04-23 04:44:18] (step=0070275) Train Loss: 6.3942, Train Steps/Sec: 0.83 + 72%|███████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████▏ | 29396/40903 [11:01:43<3:47:09, 1.18s/it][2025-04-23 04:44:48] (step=0070300) Train Loss: 6.3494, Train Steps/Sec: 0.84 + 72%|███████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████▎ | 29421/40903 [11:02:13<3:51:34, 1.21s/it][2025-04-23 04:45:18] (step=0070325) Train Loss: 6.3470, Train Steps/Sec: 0.84 + 72%|███████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████▍ | 29446/40903 [11:02:42<3:44:04, 1.17s/it][2025-04-23 04:45:47] (step=0070350) Train Loss: 6.3546, Train Steps/Sec: 0.85 + 72%|███████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████▌ | 29471/40903 [11:03:12<3:43:45, 1.17s/it][2025-04-23 04:46:17] (step=0070375) Train Loss: 6.3257, Train Steps/Sec: 0.85 + 72%|███████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████▋ | 29496/40903 [11:03:41<3:42:34, 1.17s/it][2025-04-23 04:46:46] (step=0070400) Train Loss: 6.3703, Train Steps/Sec: 0.84 + 72%|███████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████▋ | 29521/40903 [11:04:11<3:47:07, 1.20s/it][2025-04-23 04:47:16] (step=0070425) Train Loss: 6.3362, Train Steps/Sec: 0.85 + 72%|███████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████▊ | 29546/40903 [11:04:41<3:46:29, 1.20s/it][2025-04-23 04:47:46] (step=0070450) Train Loss: 6.3916, Train Steps/Sec: 0.84 + 72%|███████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████▉ | 29571/40903 [11:05:10<3:43:22, 1.18s/it][2025-04-23 04:48:15] (step=0070475) Train Loss: 6.3729, Train Steps/Sec: 0.84 + 72%|████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████ | 29596/40903 [11:05:40<3:40:52, 1.17s/it][2025-04-23 04:48:45] (step=0070500) Train Loss: 6.3430, Train Steps/Sec: 0.85 + 72%|████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████▏ | 29621/40903 [11:06:10<3:46:00, 1.20s/it][2025-04-23 04:49:14] (step=0070525) Train Loss: 6.3696, Train Steps/Sec: 0.84 + 72%|████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████▎ | 29646/40903 [11:06:39<3:43:16, 1.19s/it][2025-04-23 04:49:44] (step=0070550) Train Loss: 6.3212, Train Steps/Sec: 0.85 + 73%|████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████▍ | 29671/40903 [11:07:09<3:39:47, 1.17s/it][2025-04-23 04:50:13] (step=0070575) Train Loss: 6.3366, Train Steps/Sec: 0.85 + 73%|████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████▌ | 29696/40903 [11:07:44<3:37:53, 1.17s/it][2025-04-23 04:50:49] (step=0070600) Train Loss: 6.3677, Train Steps/Sec: 0.70 + 73%|████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████▌ | 29721/40903 [11:08:14<3:46:03, 1.21s/it][2025-04-23 04:51:19] (step=0070625) Train Loss: 6.3892, Train Steps/Sec: 0.84 + 73%|████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████▋ | 29746/40903 [11:08:43<3:39:07, 1.18s/it][2025-04-23 04:51:48] (step=0070650) Train Loss: 6.3326, Train Steps/Sec: 0.85 + 73%|████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████▊ | 29771/40903 [11:09:13<3:38:55, 1.18s/it][2025-04-23 04:52:18] (step=0070675) Train Loss: 6.3681, Train Steps/Sec: 0.85 + 73%|████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████▉ | 29796/40903 [11:09:49<5:07:55, 1.66s/it][2025-04-23 04:52:54] (step=0070700) Train Loss: 6.3604, Train Steps/Sec: 0.68 + 73%|█████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████ | 29821/40903 [11:10:19<3:40:52, 1.20s/it][2025-04-23 04:53:24] (step=0070725) Train Loss: 6.2997, Train Steps/Sec: 0.84 + 73%|█████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████▏ | 29846/40903 [11:10:49<3:38:06, 1.18s/it][2025-04-23 04:53:53] (step=0070750) Train Loss: 6.3118, Train Steps/Sec: 0.85 + 73%|█████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████▎ | 29871/40903 [11:11:18<3:34:33, 1.17s/it][2025-04-23 04:54:23] (step=0070775) Train Loss: 6.3532, Train Steps/Sec: 0.84 + 73%|█████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████▎ | 29896/40903 [11:11:48<3:32:37, 1.16s/it][2025-04-23 04:54:53] (step=0070800) Train Loss: 6.3230, Train Steps/Sec: 0.85 + 73%|█████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████▍ | 29921/40903 [11:12:17<3:38:11, 1.19s/it][2025-04-23 04:55:22] (step=0070825) Train Loss: 6.3404, Train Steps/Sec: 0.85 + 73%|█████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████▌ | 29946/40903 [11:12:47<3:35:10, 1.18s/it][2025-04-23 04:55:51] (step=0070850) Train Loss: 6.3396, Train Steps/Sec: 0.85 + 73%|█████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████▋ | 29971/40903 [11:13:16<3:36:59, 1.19s/it][2025-04-23 04:56:21] (step=0070875) Train Loss: 6.3589, Train Steps/Sec: 0.85 + 73%|█████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████▊ | 29996/40903 [11:13:46<3:34:23, 1.18s/it][2025-04-23 04:56:51] (step=0070900) Train Loss: 6.3163, Train Steps/Sec: 0.84 + 73%|█████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████▉ | 30021/40903 [11:14:16<3:38:54, 1.21s/it][2025-04-23 04:57:20] (step=0070925) Train Loss: 6.2975, Train Steps/Sec: 0.85 + 73%|██████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████ | 30046/40903 [11:14:45<3:31:44, 1.17s/it][2025-04-23 04:57:50] (step=0070950) Train Loss: 6.3587, Train Steps/Sec: 0.85 + 74%|██████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████▏ | 30071/40903 [11:15:15<3:35:19, 1.19s/it][2025-04-23 04:58:19] (step=0070975) Train Loss: 6.3797, Train Steps/Sec: 0.85 + 74%|██████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████▏ | 30096/40903 [11:15:44<3:31:16, 1.17s/it][2025-04-23 04:58:49] (step=0071000) Train Loss: 6.3171, Train Steps/Sec: 0.84 + 74%|██████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████▎ | 30121/40903 [11:16:14<3:36:45, 1.21s/it][2025-04-23 04:59:19] (step=0071025) Train Loss: 6.3808, Train Steps/Sec: 0.84 + 74%|██████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████▍ | 30146/40903 [11:16:44<3:33:43, 1.19s/it][2025-04-23 04:59:48] (step=0071050) Train Loss: 6.3630, Train Steps/Sec: 0.85 + 74%|██████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████▌ | 30171/40903 [11:17:13<3:30:26, 1.18s/it][2025-04-23 05:00:18] (step=0071075) Train Loss: 6.3412, Train Steps/Sec: 0.84 + 74%|██████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████▋ | 30196/40903 [11:17:43<3:30:07, 1.18s/it][2025-04-23 05:00:48] (step=0071100) Train Loss: 6.3603, Train Steps/Sec: 0.84 + 74%|██████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████▊ | 30221/40903 [11:18:13<3:33:05, 1.20s/it][2025-04-23 05:01:17] (step=0071125) Train Loss: 6.3729, Train Steps/Sec: 0.84 + 74%|██████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████▉ | 30246/40903 [11:18:42<3:31:20, 1.19s/it][2025-04-23 05:01:47] (step=0071150) Train Loss: 6.3781, Train Steps/Sec: 0.84 + 74%|██████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████▉ | 30271/40903 [11:19:12<3:27:48, 1.17s/it][2025-04-23 05:02:17] (step=0071175) Train Loss: 6.3162, Train Steps/Sec: 0.84 + 74%|███████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████ | 30296/40903 [11:19:42<3:25:54, 1.16s/it][2025-04-23 05:02:47] (step=0071200) Train Loss: 6.3495, Train Steps/Sec: 0.84 + 74%|███████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████▏ | 30321/40903 [11:20:11<3:31:28, 1.20s/it][2025-04-23 05:03:16] (step=0071225) Train Loss: 6.3507, Train Steps/Sec: 0.85 + 74%|███████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████▎ | 30346/40903 [11:20:41<3:28:10, 1.18s/it][2025-04-23 05:03:46] (step=0071250) Train Loss: 6.3671, Train Steps/Sec: 0.84 + 74%|███████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████▍ | 30371/40903 [11:21:10<3:24:56, 1.17s/it][2025-04-23 05:04:15] (step=0071275) Train Loss: 6.3732, Train Steps/Sec: 0.85 + 74%|███████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████▌ | 30396/40903 [11:21:40<3:27:22, 1.18s/it][2025-04-23 05:04:45] (step=0071300) Train Loss: 6.4040, Train Steps/Sec: 0.84 + 74%|███████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████▋ | 30421/40903 [11:22:10<3:31:36, 1.21s/it][2025-04-23 05:05:15] (step=0071325) Train Loss: 6.3138, Train Steps/Sec: 0.84 + 74%|███████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████▋ | 30446/40903 [11:22:40<3:27:22, 1.19s/it][2025-04-23 05:05:45] (step=0071350) Train Loss: 6.3619, Train Steps/Sec: 0.84 + 74%|███████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████▊ | 30471/40903 [11:23:10<3:25:39, 1.18s/it][2025-04-23 05:06:15] (step=0071375) Train Loss: 6.3423, Train Steps/Sec: 0.84 + 75%|███████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████▉ | 30496/40903 [11:23:39<3:22:03, 1.16s/it][2025-04-23 05:06:44] (step=0071400) Train Loss: 6.3287, Train Steps/Sec: 0.84 + 75%|████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████ | 30521/40903 [11:24:09<3:27:47, 1.20s/it][2025-04-23 05:07:14] (step=0071425) Train Loss: 6.3940, Train Steps/Sec: 0.85 + 75%|████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████▏ | 30546/40903 [11:24:38<3:23:18, 1.18s/it][2025-04-23 05:07:43] (step=0071450) Train Loss: 6.3545, Train Steps/Sec: 0.85 + 75%|████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████▎ | 30571/40903 [11:25:08<3:20:43, 1.17s/it][2025-04-23 05:08:13] (step=0071475) Train Loss: 6.3348, Train Steps/Sec: 0.85 + 75%|████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████▍ | 30596/40903 [11:25:37<3:20:49, 1.17s/it][2025-04-23 05:08:42] (step=0071500) Train Loss: 6.3816, Train Steps/Sec: 0.84 + 75%|████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████▌ | 30621/40903 [11:26:07<3:24:41, 1.19s/it][2025-04-23 05:09:12] (step=0071525) Train Loss: 6.3782, Train Steps/Sec: 0.85 + 75%|████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████▌ | 30646/40903 [11:26:37<3:25:09, 1.20s/it][2025-04-23 05:09:41] (step=0071550) Train Loss: 6.3726, Train Steps/Sec: 0.85 + 75%|████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████▋ | 30671/40903 [11:27:06<3:21:26, 1.18s/it][2025-04-23 05:10:11] (step=0071575) Train Loss: 6.2966, Train Steps/Sec: 0.85 + 75%|████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████▊ | 30696/40903 [11:27:36<3:20:32, 1.18s/it][2025-04-23 05:10:41] (step=0071600) Train Loss: 6.3758, Train Steps/Sec: 0.84 + 75%|████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████▉ | 30721/40903 [11:28:06<3:24:10, 1.20s/it][2025-04-23 05:11:10] (step=0071625) Train Loss: 6.3487, Train Steps/Sec: 0.84 + 75%|█████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████ | 30746/40903 [11:28:35<3:18:42, 1.17s/it][2025-04-23 05:11:40] (step=0071650) Train Loss: 6.3379, Train Steps/Sec: 0.85 + 75%|█████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████▏ | 30771/40903 [11:29:04<3:15:48, 1.16s/it][2025-04-23 05:12:09] (step=0071675) Train Loss: 6.3350, Train Steps/Sec: 0.85 + 75%|█████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████▎ | 30796/40903 [11:29:34<3:19:07, 1.18s/it][2025-04-23 05:12:39] (step=0071700) Train Loss: 6.3971, Train Steps/Sec: 0.84 + 75%|█████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████▎ | 30821/40903 [11:30:04<3:19:05, 1.18s/it][2025-04-23 05:13:09] (step=0071725) Train Loss: 6.3351, Train Steps/Sec: 0.85 + 75%|█████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████▍ | 30846/40903 [11:30:33<3:17:41, 1.18s/it][2025-04-23 05:13:38] (step=0071750) Train Loss: 6.3553, Train Steps/Sec: 0.85 + 75%|█████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████▌ | 30871/40903 [11:31:03<3:16:22, 1.17s/it][2025-04-23 05:14:08] (step=0071775) Train Loss: 6.3214, Train Steps/Sec: 0.84 + 76%|█████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████▋ | 30896/40903 [11:31:32<3:17:13, 1.18s/it][2025-04-23 05:14:37] (step=0071800) Train Loss: 6.3676, Train Steps/Sec: 0.84 + 76%|█████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████▊ | 30921/40903 [11:32:02<3:19:34, 1.20s/it][2025-04-23 05:15:07] (step=0071825) Train Loss: 6.3562, Train Steps/Sec: 0.85 + 76%|█████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████▉ | 30946/40903 [11:32:32<3:15:43, 1.18s/it][2025-04-23 05:15:36] (step=0071850) Train Loss: 6.3276, Train Steps/Sec: 0.84 + 76%|██████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████ | 30971/40903 [11:33:01<3:14:21, 1.17s/it][2025-04-23 05:16:06] (step=0071875) Train Loss: 6.3389, Train Steps/Sec: 0.85 + 76%|██████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████▏ | 30996/40903 [11:33:31<3:13:43, 1.17s/it][2025-04-23 05:16:35] (step=0071900) Train Loss: 6.3059, Train Steps/Sec: 0.84 + 76%|██████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████▏ | 31021/40903 [11:34:00<3:16:25, 1.19s/it][2025-04-23 05:17:05] (step=0071925) Train Loss: 6.3134, Train Steps/Sec: 0.84 + 76%|██████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████▎ | 31046/40903 [11:34:30<3:16:07, 1.19s/it][2025-04-23 05:17:35] (step=0071950) Train Loss: 6.3801, Train Steps/Sec: 0.85 + 76%|██████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████▍ | 31071/40903 [11:34:59<3:11:58, 1.17s/it][2025-04-23 05:18:04] (step=0071975) Train Loss: 6.3715, Train Steps/Sec: 0.85 + 76%|██████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████▌ | 31096/40903 [11:35:29<3:11:36, 1.17s/it][2025-04-23 05:18:34] (step=0072000) Train Loss: 6.3830, Train Steps/Sec: 0.84 +You are using a model of type instructblip to instantiate a model of type blip-2. This is not supported for all configurations of models and can yield errors. +model_name_or_path:/tmp/haozhezhao/model/instructblip-flan-t5-xl +image_place_holder: +Some kwargs in processor config are unused and will not have any effect: num_query_tokens. +tokenizer length after expend 32102 +tokenizer length before expend 32102 +100%|███████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████| 3/3 [02:53<00:00, 57.89s/it] +[2025-04-23 05:23:17] Finish Eval in 72000 steps...████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████| 3/3 [02:53<00:00, 57.44s/it] +processing left-padding... +processing left-padding... +processing left-padding... +[2025-04-23 05:23:39] Saved checkpoint to checkpoint/BeforeFix_Blip_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_objects/002-GPT-XL/checkpoints/0072000.pt +[2025-04-23 05:23:41] Removed old checkpoint: checkpoint/BeforeFix_Blip_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_objects/002-GPT-XL/checkpoints/0068000.pt + 76%|██████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████▋ | 31121/40903 [11:41:06<3:17:07, 1.21s/it][2025-04-23 05:24:10] (step=0072025) Train Loss: 6.3182, Train Steps/Sec: 0.07 + 76%|██████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████▊ | 31146/40903 [11:41:35<3:13:59, 1.19s/it][2025-04-23 05:24:40] (step=0072050) Train Loss: 6.3494, Train Steps/Sec: 0.85 + 76%|██████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████▉ | 31171/40903 [11:42:05<3:12:43, 1.19s/it][2025-04-23 05:25:10] (step=0072075) Train Loss: 6.3617, Train Steps/Sec: 0.84 + 76%|██████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████▉ | 31196/40903 [11:42:35<3:35:07, 1.33s/it][2025-04-23 05:25:40] (step=0072100) Train Loss: 6.3418, Train Steps/Sec: 0.83 + 76%|███████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████ | 31221/40903 [11:43:04<3:12:59, 1.20s/it][2025-04-23 05:26:09] (step=0072125) Train Loss: 6.3159, Train Steps/Sec: 0.85 + 76%|███████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████▏ | 31246/40903 [11:43:34<3:11:40, 1.19s/it][2025-04-23 05:26:39] (step=0072150) Train Loss: 6.3703, Train Steps/Sec: 0.85 + 76%|███████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████▎ | 31271/40903 [11:44:04<3:08:24, 1.17s/it][2025-04-23 05:27:08] (step=0072175) Train Loss: 6.3450, Train Steps/Sec: 0.85 + 77%|███████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████▍ | 31296/40903 [11:44:33<3:05:38, 1.16s/it][2025-04-23 05:27:38] (step=0072200) Train Loss: 6.3307, Train Steps/Sec: 0.85 + 77%|███████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████▌ | 31321/40903 [11:45:03<3:11:27, 1.20s/it][2025-04-23 05:28:07] (step=0072225) Train Loss: 6.3357, Train Steps/Sec: 0.85 + 77%|███████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████▋ | 31346/40903 [11:45:32<3:06:42, 1.17s/it][2025-04-23 05:28:37] (step=0072250) Train Loss: 6.3656, Train Steps/Sec: 0.85 + 77%|███████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████▊ | 31371/40903 [11:46:01<3:06:25, 1.17s/it][2025-04-23 05:29:06] (step=0072275) Train Loss: 6.3411, Train Steps/Sec: 0.85 + 77%|███████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████▊ | 31396/40903 [11:46:31<3:04:55, 1.17s/it][2025-04-23 05:29:36] (step=0072300) Train Loss: 6.3224, Train Steps/Sec: 0.84 + 77%|███████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████▉ | 31421/40903 [11:47:01<3:09:09, 1.20s/it][2025-04-23 05:30:06] (step=0072325) Train Loss: 6.3501, Train Steps/Sec: 0.85 + 77%|████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████ | 31446/40903 [11:47:30<3:06:42, 1.18s/it][2025-04-23 05:30:35] (step=0072350) Train Loss: 6.3434, Train Steps/Sec: 0.85 + 77%|████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████▏ | 31471/40903 [11:48:00<3:07:14, 1.19s/it][2025-04-23 05:31:05] (step=0072375) Train Loss: 6.3803, Train Steps/Sec: 0.85 + 77%|████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████▎ | 31496/40903 [11:48:30<3:01:48, 1.16s/it][2025-04-23 05:31:34] (step=0072400) Train Loss: 6.3769, Train Steps/Sec: 0.84 + 77%|████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████▍ | 31521/40903 [11:48:59<3:08:34, 1.21s/it][2025-04-23 05:32:04] (step=0072425) Train Loss: 6.3336, Train Steps/Sec: 0.84 + 77%|████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████▌ | 31546/40903 [11:49:29<3:05:06, 1.19s/it][2025-04-23 05:32:34] (step=0072450) Train Loss: 6.3762, Train Steps/Sec: 0.85 + 77%|████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████▌ | 31571/40903 [11:49:58<3:00:59, 1.16s/it][2025-04-23 05:33:03] (step=0072475) Train Loss: 6.3761, Train Steps/Sec: 0.85 + 77%|████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████▋ | 31596/40903 [11:50:28<3:00:11, 1.16s/it][2025-04-23 05:33:33] (step=0072500) Train Loss: 6.3843, Train Steps/Sec: 0.85 + 77%|████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████▊ | 31621/40903 [11:50:57<3:04:21, 1.19s/it][2025-04-23 05:34:02] (step=0072525) Train Loss: 6.2855, Train Steps/Sec: 0.85 + 77%|████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████▉ | 31646/40903 [11:51:27<3:02:08, 1.18s/it][2025-04-23 05:34:32] (step=0072550) Train Loss: 6.3537, Train Steps/Sec: 0.84 + 77%|█████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████ | 31671/40903 [11:51:57<3:00:44, 1.17s/it][2025-04-23 05:35:01] (step=0072575) Train Loss: 6.3624, Train Steps/Sec: 0.85 + 77%|█████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████▏ | 31696/40903 [11:52:26<2:57:03, 1.15s/it][2025-04-23 05:35:31] (step=0072600) Train Loss: 6.3623, Train Steps/Sec: 0.85 + 78%|█████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████▎ | 31721/40903 [11:52:56<3:04:48, 1.21s/it][2025-04-23 05:36:00] (step=0072625) Train Loss: 6.3507, Train Steps/Sec: 0.85 + 78%|█████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████▎ | 31746/40903 [11:53:25<3:00:13, 1.18s/it][2025-04-23 05:36:30] (step=0072650) Train Loss: 6.3444, Train Steps/Sec: 0.84 + 78%|█████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████▍ | 31771/40903 [11:53:55<3:00:06, 1.18s/it][2025-04-23 05:37:00] (step=0072675) Train Loss: 6.3028, Train Steps/Sec: 0.85 + 78%|█████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████▌ | 31796/40903 [11:54:24<2:54:50, 1.15s/it][2025-04-23 05:37:29] (step=0072700) Train Loss: 6.3677, Train Steps/Sec: 0.85 + 78%|█████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████▋ | 31821/40903 [11:54:54<2:59:33, 1.19s/it][2025-04-23 05:37:59] (step=0072725) Train Loss: 6.2953, Train Steps/Sec: 0.85 + 78%|█████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████▊ | 31846/40903 [11:55:24<2:59:21, 1.19s/it][2025-04-23 05:38:28] (step=0072750) Train Loss: 6.3283, Train Steps/Sec: 0.84 + 78%|█████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████▉ | 31871/40903 [11:55:53<2:56:22, 1.17s/it][2025-04-23 05:38:58] (step=0072775) Train Loss: 6.3087, Train Steps/Sec: 0.85 + 78%|██████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████ | 31896/40903 [11:56:23<2:58:19, 1.19s/it][2025-04-23 05:39:28] (step=0072800) Train Loss: 6.3651, Train Steps/Sec: 0.84 + 78%|██████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████▏ | 31921/40903 [11:56:52<2:59:01, 1.20s/it][2025-04-23 05:39:57] (step=0072825) Train Loss: 6.3535, Train Steps/Sec: 0.85 + 78%|██████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████▏ | 31946/40903 [11:57:22<2:57:26, 1.19s/it][2025-04-23 05:40:27] (step=0072850) Train Loss: 6.3297, Train Steps/Sec: 0.85 + 78%|██████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████▎ | 31971/40903 [11:57:52<2:56:39, 1.19s/it][2025-04-23 05:40:57] (step=0072875) Train Loss: 6.3485, Train Steps/Sec: 0.84 + 78%|██████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████▍ | 31996/40903 [11:58:21<2:53:30, 1.17s/it][2025-04-23 05:41:26] (step=0072900) Train Loss: 6.3218, Train Steps/Sec: 0.84 + 78%|██████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████▌ | 32021/40903 [11:58:51<2:57:27, 1.20s/it][2025-04-23 05:41:56] (step=0072925) Train Loss: 6.3673, Train Steps/Sec: 0.85 + 78%|██████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████▋ | 32046/40903 [11:59:21<2:56:01, 1.19s/it][2025-04-23 05:42:25] (step=0072950) Train Loss: 6.3042, Train Steps/Sec: 0.85 + 78%|██████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████▊ | 32071/40903 [11:59:50<2:50:48, 1.16s/it][2025-04-23 05:42:55] (step=0072975) Train Loss: 6.3133, Train Steps/Sec: 0.85 + 78%|██████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████▉ | 32096/40903 [12:00:20<2:53:10, 1.18s/it][2025-04-23 05:43:24] (step=0073000) Train Loss: 6.3545, Train Steps/Sec: 0.84 + 79%|██████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████▉ | 32121/40903 [12:00:49<2:56:36, 1.21s/it][2025-04-23 05:43:54] (step=0073025) Train Loss: 6.3624, Train Steps/Sec: 0.84 + 79%|███████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████ | 32146/40903 [12:01:19<2:53:43, 1.19s/it][2025-04-23 05:44:24] (step=0073050) Train Loss: 6.3356, Train Steps/Sec: 0.85 + 79%|███████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████▏ | 32171/40903 [12:01:48<2:52:25, 1.18s/it][2025-04-23 05:44:53] (step=0073075) Train Loss: 6.3983, Train Steps/Sec: 0.84 + 79%|███████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████▎ | 32196/40903 [12:02:18<2:49:20, 1.17s/it][2025-04-23 05:45:23] (step=0073100) Train Loss: 6.3874, Train Steps/Sec: 0.84 + 79%|███████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████▍ | 32221/40903 [12:02:48<2:53:54, 1.20s/it][2025-04-23 05:45:53] (step=0073125) Train Loss: 6.3402, Train Steps/Sec: 0.85 + 79%|███████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████▌ | 32246/40903 [12:03:17<2:49:13, 1.17s/it][2025-04-23 05:46:22] (step=0073150) Train Loss: 6.2843, Train Steps/Sec: 0.85 + 79%|███████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████▋ | 32271/40903 [12:03:47<2:48:48, 1.17s/it][2025-04-23 05:46:52] (step=0073175) Train Loss: 6.2901, Train Steps/Sec: 0.85 + 79%|███████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████▊ | 32296/40903 [12:04:16<2:47:38, 1.17s/it][2025-04-23 05:47:21] (step=0073200) Train Loss: 6.3595, Train Steps/Sec: 0.84 + 79%|███████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████▊ | 32321/40903 [12:04:46<2:50:05, 1.19s/it][2025-04-23 05:47:51] (step=0073225) Train Loss: 6.3140, Train Steps/Sec: 0.85 + 79%|███████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████▉ | 32346/40903 [12:05:15<2:49:55, 1.19s/it][2025-04-23 05:48:20] (step=0073250) Train Loss: 6.3659, Train Steps/Sec: 0.85 + 79%|████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████ | 32371/40903 [12:05:45<2:48:35, 1.19s/it][2025-04-23 05:48:50] (step=0073275) Train Loss: 6.3957, Train Steps/Sec: 0.85 + 79%|████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████▏ | 32396/40903 [12:06:15<2:43:32, 1.15s/it][2025-04-23 05:49:19] (step=0073300) Train Loss: 6.3874, Train Steps/Sec: 0.84 + 79%|████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████▎ | 32421/40903 [12:06:44<2:51:49, 1.22s/it][2025-04-23 05:49:49] (step=0073325) Train Loss: 6.3419, Train Steps/Sec: 0.84 + 79%|████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████▍ | 32446/40903 [12:07:14<2:46:06, 1.18s/it][2025-04-23 05:50:19] (step=0073350) Train Loss: 6.3496, Train Steps/Sec: 0.84 + 79%|████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████▌ | 32471/40903 [12:07:44<2:47:33, 1.19s/it][2025-04-23 05:50:49] (step=0073375) Train Loss: 6.3650, Train Steps/Sec: 0.84 + 79%|████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████▌ | 32496/40903 [12:08:13<2:44:34, 1.17s/it][2025-04-23 05:51:18] (step=0073400) Train Loss: 6.3479, Train Steps/Sec: 0.84 + 80%|████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████▋ | 32521/40903 [12:08:43<2:48:45, 1.21s/it][2025-04-23 05:51:48] (step=0073425) Train Loss: 6.3033, Train Steps/Sec: 0.85 + 80%|████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████▊ | 32546/40903 [12:09:12<2:44:06, 1.18s/it][2025-04-23 05:52:17] (step=0073450) Train Loss: 6.2895, Train Steps/Sec: 0.85 + 80%|████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████▉ | 32571/40903 [12:09:42<2:43:30, 1.18s/it][2025-04-23 05:52:46] (step=0073475) Train Loss: 6.3464, Train Steps/Sec: 0.85 + 80%|█████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████ | 32596/40903 [12:10:11<2:42:58, 1.18s/it][2025-04-23 05:53:16] (step=0073500) Train Loss: 6.3206, Train Steps/Sec: 0.84 + 80%|█████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████▏ | 32621/40903 [12:10:41<2:47:54, 1.22s/it][2025-04-23 05:53:46] (step=0073525) Train Loss: 6.3357, Train Steps/Sec: 0.84 + 80%|█████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████▎ | 32646/40903 [12:11:11<2:43:32, 1.19s/it][2025-04-23 05:54:16] (step=0073550) Train Loss: 6.3293, Train Steps/Sec: 0.84 + 80%|█████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████▍ | 32671/40903 [12:11:40<2:41:49, 1.18s/it][2025-04-23 05:54:45] (step=0073575) Train Loss: 6.3375, Train Steps/Sec: 0.85 + 80%|█████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████▍ | 32696/40903 [12:12:10<2:40:46, 1.18s/it][2025-04-23 05:55:15] (step=0073600) Train Loss: 6.3500, Train Steps/Sec: 0.84 + 80%|█████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████▌ | 32721/40903 [12:12:40<2:43:09, 1.20s/it][2025-04-23 05:55:45] (step=0073625) Train Loss: 6.3740, Train Steps/Sec: 0.84 + 80%|█████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████▋ | 32746/40903 [12:13:10<2:42:18, 1.19s/it][2025-04-23 05:56:15] (step=0073650) Train Loss: 6.3535, Train Steps/Sec: 0.84 + 80%|█████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████▊ | 32771/40903 [12:13:39<2:39:22, 1.18s/it][2025-04-23 05:56:44] (step=0073675) Train Loss: 6.3706, Train Steps/Sec: 0.85 + 80%|█████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████▉ | 32796/40903 [12:14:09<2:41:30, 1.20s/it][2025-04-23 05:57:14] (step=0073700) Train Loss: 6.3625, Train Steps/Sec: 0.84 + 80%|██████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████ | 32821/40903 [12:14:39<2:42:42, 1.21s/it][2025-04-23 05:57:43] (step=0073725) Train Loss: 6.3679, Train Steps/Sec: 0.84 + 80%|██████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████▏ | 32846/40903 [12:15:08<2:38:23, 1.18s/it][2025-04-23 05:58:13] (step=0073750) Train Loss: 6.3581, Train Steps/Sec: 0.85 + 80%|██████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████▏ | 32871/40903 [12:15:38<2:36:40, 1.17s/it][2025-04-23 05:58:43] (step=0073775) Train Loss: 6.3789, Train Steps/Sec: 0.84 + 80%|██████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████▎ | 32896/40903 [12:16:07<2:36:14, 1.17s/it][2025-04-23 05:59:12] (step=0073800) Train Loss: 6.3911, Train Steps/Sec: 0.84 + 80%|██████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████▍ | 32921/40903 [12:16:37<2:39:27, 1.20s/it][2025-04-23 05:59:42] (step=0073825) Train Loss: 6.3711, Train Steps/Sec: 0.85 + 81%|██████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████▌ | 32946/40903 [12:17:07<2:36:51, 1.18s/it][2025-04-23 06:00:11] (step=0073850) Train Loss: 6.3173, Train Steps/Sec: 0.85 + 81%|██████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████▋ | 32971/40903 [12:17:36<2:35:15, 1.17s/it][2025-04-23 06:00:41] (step=0073875) Train Loss: 6.3534, Train Steps/Sec: 0.85 + 81%|██████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████▊ | 32996/40903 [12:18:06<2:34:00, 1.17s/it][2025-04-23 06:01:11] (step=0073900) Train Loss: 6.3414, Train Steps/Sec: 0.84 + 81%|██████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████▉ | 33021/40903 [12:18:36<2:38:13, 1.20s/it][2025-04-23 06:01:40] (step=0073925) Train Loss: 6.2894, Train Steps/Sec: 0.85 + 81%|███████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████ | 33046/40903 [12:19:05<2:33:58, 1.18s/it][2025-04-23 06:02:10] (step=0073950) Train Loss: 6.3567, Train Steps/Sec: 0.84 + 81%|███████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████ | 33071/40903 [12:19:35<2:33:44, 1.18s/it][2025-04-23 06:02:40] (step=0073975) Train Loss: 6.3291, Train Steps/Sec: 0.84 + 81%|███████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████▏ | 33096/40903 [12:20:04<2:32:53, 1.18s/it][2025-04-23 06:03:09] (step=0074000) Train Loss: 6.3349, Train Steps/Sec: 0.84 +You are using a model of type instructblip to instantiate a model of type blip-2. This is not supported for all configurations of models and can yield errors. +model_name_or_path:/tmp/haozhezhao/model/instructblip-flan-t5-xl +image_place_holder: +Some kwargs in processor config are unused and will not have any effect: num_query_tokens. +tokenizer length after expend 32102 +tokenizer length before expend 32102 +100%|███████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████| 3/3 [02:53<00:00, 57.85s/it] +[2025-04-23 06:07:52] Finish Eval in 74000 steps...████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████| 3/3 [02:53<00:00, 57.40s/it] +processing left-padding... +processing left-padding... +processing left-padding... +[2025-04-23 06:08:13] Saved checkpoint to checkpoint/BeforeFix_Blip_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_objects/002-GPT-XL/checkpoints/0074000.pt +[2025-04-23 06:08:16] Removed old checkpoint: checkpoint/BeforeFix_Blip_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_objects/002-GPT-XL/checkpoints/0070000.pt + 81%|███████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████▎ | 33121/40903 [12:25:41<2:40:27, 1.24s/it][2025-04-23 06:08:46] (step=0074025) Train Loss: 6.3452, Train Steps/Sec: 0.07 + 81%|███████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████▍ | 33146/40903 [12:26:11<2:35:56, 1.21s/it][2025-04-23 06:09:16] (step=0074050) Train Loss: 6.3687, Train Steps/Sec: 0.84 + 81%|███████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████▌ | 33171/40903 [12:26:41<2:32:03, 1.18s/it][2025-04-23 06:09:45] (step=0074075) Train Loss: 6.3676, Train Steps/Sec: 0.84 + 81%|███████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████▋ | 33196/40903 [12:27:10<2:29:13, 1.16s/it][2025-04-23 06:10:15] (step=0074100) Train Loss: 6.3437, Train Steps/Sec: 0.84 + 81%|███████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████▊ | 33221/40903 [12:27:40<2:33:43, 1.20s/it][2025-04-23 06:10:45] (step=0074125) Train Loss: 6.4079, Train Steps/Sec: 0.85 + 81%|███████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████▊ | 33246/40903 [12:28:09<2:29:26, 1.17s/it][2025-04-23 06:11:14] (step=0074150) Train Loss: 6.3216, Train Steps/Sec: 0.85 + 81%|███████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████▉ | 33271/40903 [12:28:39<2:29:05, 1.17s/it][2025-04-23 06:11:44] (step=0074175) Train Loss: 6.3633, Train Steps/Sec: 0.85 + 81%|████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████ | 33296/40903 [12:29:08<2:31:26, 1.19s/it][2025-04-23 06:12:13] (step=0074200) Train Loss: 6.3356, Train Steps/Sec: 0.84 + 81%|████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████▏ | 33321/40903 [12:29:38<2:31:48, 1.20s/it][2025-04-23 06:12:43] (step=0074225) Train Loss: 6.3070, Train Steps/Sec: 0.85 + 82%|████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████▎ | 33346/40903 [12:30:08<2:29:39, 1.19s/it][2025-04-23 06:13:12] (step=0074250) Train Loss: 6.3273, Train Steps/Sec: 0.84 + 82%|████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████▍ | 33371/40903 [12:30:37<2:27:17, 1.17s/it][2025-04-23 06:13:42] (step=0074275) Train Loss: 6.3341, Train Steps/Sec: 0.85 + 82%|████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████▌ | 33396/40903 [12:31:06<2:26:24, 1.17s/it][2025-04-23 06:14:11] (step=0074300) Train Loss: 6.3587, Train Steps/Sec: 0.85 + 82%|████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████▌ | 33421/40903 [12:31:37<2:30:29, 1.21s/it][2025-04-23 06:14:42] (step=0074325) Train Loss: 6.3354, Train Steps/Sec: 0.83 + 82%|████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████▋ | 33446/40903 [12:32:06<2:26:16, 1.18s/it][2025-04-23 06:15:11] (step=0074350) Train Loss: 6.3160, Train Steps/Sec: 0.85 + 82%|████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████▊ | 33471/40903 [12:32:36<2:26:22, 1.18s/it][2025-04-23 06:15:41] (step=0074375) Train Loss: 6.3403, Train Steps/Sec: 0.84 + 82%|████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████▉ | 33496/40903 [12:33:05<2:24:27, 1.17s/it][2025-04-23 06:16:10] (step=0074400) Train Loss: 6.3625, Train Steps/Sec: 0.84 + 82%|█████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████ | 33521/40903 [12:33:35<2:27:41, 1.20s/it][2025-04-23 06:16:40] (step=0074425) Train Loss: 6.3563, Train Steps/Sec: 0.84 + 82%|█████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████▏ | 33546/40903 [12:34:11<6:23:02, 3.12s/it][2025-04-23 06:17:16] (step=0074450) Train Loss: 6.3485, Train Steps/Sec: 0.69 + 82%|█████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████▎ | 33571/40903 [12:34:41<2:23:40, 1.18s/it][2025-04-23 06:17:46] (step=0074475) Train Loss: 6.3460, Train Steps/Sec: 0.84 + 82%|█████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████▍ | 33596/40903 [12:35:11<2:22:43, 1.17s/it][2025-04-23 06:18:16] (step=0074500) Train Loss: 6.3663, Train Steps/Sec: 0.84 + 82%|█████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████▍ | 33621/40903 [12:35:40<2:26:24, 1.21s/it][2025-04-23 06:18:45] (step=0074525) Train Loss: 6.3606, Train Steps/Sec: 0.85 + 82%|█████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████▌ | 33646/40903 [12:36:10<2:25:43, 1.20s/it][2025-04-23 06:19:15] (step=0074550) Train Loss: 6.3881, Train Steps/Sec: 0.84 + 82%|█████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████▋ | 33671/40903 [12:36:39<2:20:45, 1.17s/it][2025-04-23 06:19:44] (step=0074575) Train Loss: 6.3630, Train Steps/Sec: 0.85 + 82%|█████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████▊ | 33696/40903 [12:37:09<2:21:44, 1.18s/it][2025-04-23 06:20:14] (step=0074600) Train Loss: 6.3816, Train Steps/Sec: 0.84 + 82%|█████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████▉ | 33721/40903 [12:37:39<2:23:07, 1.20s/it][2025-04-23 06:20:43] (step=0074625) Train Loss: 6.3447, Train Steps/Sec: 0.85 + 83%|██████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████ | 33746/40903 [12:38:08<2:20:33, 1.18s/it][2025-04-23 06:21:13] (step=0074650) Train Loss: 6.3542, Train Steps/Sec: 0.84 + 83%|██████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████▏ | 33771/40903 [12:38:38<2:21:04, 1.19s/it][2025-04-23 06:21:43] (step=0074675) Train Loss: 6.3629, Train Steps/Sec: 0.84 + 83%|██████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████▏ | 33796/40903 [12:39:07<2:18:34, 1.17s/it][2025-04-23 06:22:19] (step=0074700) Train Loss: 6.3541, Train Steps/Sec: 0.70 + 83%|██████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████▎ | 33821/40903 [12:39:44<2:21:01, 1.19s/it][2025-04-23 06:22:48] (step=0074725) Train Loss: 6.3391, Train Steps/Sec: 0.85 + 83%|██████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████▍ | 33846/40903 [12:40:13<2:21:13, 1.20s/it][2025-04-23 06:23:18] (step=0074750) Train Loss: 6.3226, Train Steps/Sec: 0.84 + 83%|██████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████▌ | 33871/40903 [12:40:43<2:17:01, 1.17s/it][2025-04-23 06:23:48] (step=0074775) Train Loss: 6.3538, Train Steps/Sec: 0.85 + 83%|██████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████▋ | 33896/40903 [12:41:12<2:17:57, 1.18s/it][2025-04-23 06:24:17] (step=0074800) Train Loss: 6.3620, Train Steps/Sec: 0.84 + 83%|██████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████▊ | 33921/40903 [12:41:42<2:19:40, 1.20s/it][2025-04-23 06:24:47] (step=0074825) Train Loss: 6.3569, Train Steps/Sec: 0.85 + 83%|██████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████▉ | 33946/40903 [12:42:11<2:17:51, 1.19s/it][2025-04-23 06:25:16] (step=0074850) Train Loss: 6.3591, Train Steps/Sec: 0.85 + 83%|███████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████ | 33971/40903 [12:42:41<2:15:50, 1.18s/it][2025-04-23 06:25:46] (step=0074875) Train Loss: 6.3874, Train Steps/Sec: 0.85 + 83%|███████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████ | 33996/40903 [12:43:10<2:14:17, 1.17s/it][2025-04-23 06:26:15] (step=0074900) Train Loss: 6.3073, Train Steps/Sec: 0.84 + 83%|███████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████▏ | 34021/40903 [12:43:46<2:21:01, 1.23s/it][2025-04-23 06:26:51] (step=0074925) Train Loss: 6.3462, Train Steps/Sec: 0.71 + 83%|███████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████▎ | 34046/40903 [12:44:15<2:14:17, 1.18s/it][2025-04-23 06:27:20] (step=0074950) Train Loss: 6.3276, Train Steps/Sec: 0.84 + 83%|███████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████▍ | 34071/40903 [12:44:45<2:12:24, 1.16s/it][2025-04-23 06:27:50] (step=0074975) Train Loss: 6.3238, Train Steps/Sec: 0.85 + 83%|███████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████▌ | 34096/40903 [12:45:22<2:34:02, 1.36s/it][2025-04-23 06:28:27] (step=0075000) Train Loss: 6.3976, Train Steps/Sec: 0.67 + 83%|███████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████▋ | 34121/40903 [12:45:52<2:16:00, 1.20s/it][2025-04-23 06:28:57] (step=0075025) Train Loss: 6.3376, Train Steps/Sec: 0.84 + 83%|███████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████▊ | 34146/40903 [12:46:21<2:12:50, 1.18s/it][2025-04-23 06:29:26] (step=0075050) Train Loss: 6.3846, Train Steps/Sec: 0.85 + 84%|███████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████▊ | 34171/40903 [12:46:51<2:11:05, 1.17s/it][2025-04-23 06:29:56] (step=0075075) Train Loss: 6.3756, Train Steps/Sec: 0.85 + 84%|███████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████▉ | 34196/40903 [12:47:20<2:11:35, 1.18s/it][2025-04-23 06:30:25] (step=0075100) Train Loss: 6.3248, Train Steps/Sec: 0.84 + 84%|████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████ | 34221/40903 [12:47:50<2:13:32, 1.20s/it][2025-04-23 06:30:55] (step=0075125) Train Loss: 6.3368, Train Steps/Sec: 0.84 + 84%|████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████▏ | 34246/40903 [12:48:20<2:12:03, 1.19s/it][2025-04-23 06:31:25] (step=0075150) Train Loss: 6.3337, Train Steps/Sec: 0.85 + 84%|████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████▎ | 34271/40903 [12:48:56<2:17:15, 1.24s/it][2025-04-23 06:32:01] (step=0075175) Train Loss: 6.3733, Train Steps/Sec: 0.68 + 84%|████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████▍ | 34296/40903 [12:49:26<2:08:33, 1.17s/it][2025-04-23 06:32:31] (step=0075200) Train Loss: 6.3562, Train Steps/Sec: 0.84 + 84%|████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████▌ | 34321/40903 [12:49:55<2:12:07, 1.20s/it][2025-04-23 06:33:00] (step=0075225) Train Loss: 6.3279, Train Steps/Sec: 0.85 + 84%|████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████▋ | 34346/40903 [12:50:25<2:09:32, 1.19s/it][2025-04-23 06:33:30] (step=0075250) Train Loss: 6.3229, Train Steps/Sec: 0.85 + 84%|████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████▋ | 34371/40903 [12:50:54<2:08:42, 1.18s/it][2025-04-23 06:33:59] (step=0075275) Train Loss: 6.3146, Train Steps/Sec: 0.85 + 84%|████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████▊ | 34396/40903 [12:51:24<2:06:49, 1.17s/it][2025-04-23 06:34:29] (step=0075300) Train Loss: 6.3336, Train Steps/Sec: 0.85 + 84%|████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████▉ | 34421/40903 [12:51:53<2:09:40, 1.20s/it][2025-04-23 06:34:58] (step=0075325) Train Loss: 6.3629, Train Steps/Sec: 0.85 + 84%|█████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████ | 34446/40903 [12:52:23<2:08:00, 1.19s/it][2025-04-23 06:35:28] (step=0075350) Train Loss: 6.3658, Train Steps/Sec: 0.84 + 84%|█████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████▏ | 34471/40903 [12:52:52<2:05:18, 1.17s/it][2025-04-23 06:35:57] (step=0075375) Train Loss: 6.3848, Train Steps/Sec: 0.85 + 84%|█████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████▎ | 34496/40903 [12:53:22<2:05:35, 1.18s/it][2025-04-23 06:36:27] (step=0075400) Train Loss: 6.3446, Train Steps/Sec: 0.84 + 84%|█████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████▍ | 34521/40903 [12:53:52<2:07:13, 1.20s/it][2025-04-23 06:36:56] (step=0075425) Train Loss: 6.3783, Train Steps/Sec: 0.85 + 84%|█████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████▍ | 34546/40903 [12:54:21<2:04:48, 1.18s/it][2025-04-23 06:37:26] (step=0075450) Train Loss: 6.3332, Train Steps/Sec: 0.85 + 85%|█████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████▌ | 34571/40903 [12:54:51<2:05:28, 1.19s/it][2025-04-23 06:37:55] (step=0075475) Train Loss: 6.3575, Train Steps/Sec: 0.85 + 85%|█████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████▋ | 34596/40903 [12:55:20<2:02:50, 1.17s/it][2025-04-23 06:38:25] (step=0075500) Train Loss: 6.3664, Train Steps/Sec: 0.85 + 85%|█████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████▊ | 34621/40903 [12:55:50<2:06:59, 1.21s/it][2025-04-23 06:38:55] (step=0075525) Train Loss: 6.3626, Train Steps/Sec: 0.84 + 85%|█████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████▉ | 34646/40903 [12:56:20<2:03:20, 1.18s/it][2025-04-23 06:39:24] (step=0075550) Train Loss: 6.3466, Train Steps/Sec: 0.84 + 85%|██████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████ | 34671/40903 [12:56:49<2:02:45, 1.18s/it][2025-04-23 06:39:54] (step=0075575) Train Loss: 6.3398, Train Steps/Sec: 0.84 + 85%|██████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████▏ | 34696/40903 [12:57:19<2:00:56, 1.17s/it][2025-04-23 06:40:24] (step=0075600) Train Loss: 6.3229, Train Steps/Sec: 0.84 + 85%|██████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████▏ | 34721/40903 [12:57:48<2:04:59, 1.21s/it][2025-04-23 06:40:53] (step=0075625) Train Loss: 6.3517, Train Steps/Sec: 0.85 + 85%|██████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████▎ | 34746/40903 [12:58:18<2:01:40, 1.19s/it][2025-04-23 06:41:23] (step=0075650) Train Loss: 6.3400, Train Steps/Sec: 0.85 + 85%|██████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████▍ | 34771/40903 [12:58:48<2:00:54, 1.18s/it][2025-04-23 06:41:52] (step=0075675) Train Loss: 6.3695, Train Steps/Sec: 0.85 + 85%|██████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████▌ | 34796/40903 [12:59:23<2:40:55, 1.58s/it][2025-04-23 06:42:28] (step=0075700) Train Loss: 6.3605, Train Steps/Sec: 0.71 + 85%|██████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████▋ | 34821/40903 [12:59:52<2:01:36, 1.20s/it][2025-04-23 06:42:57] (step=0075725) Train Loss: 6.2944, Train Steps/Sec: 0.85 + 85%|██████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████▊ | 34846/40903 [13:00:22<1:59:47, 1.19s/it][2025-04-23 06:43:27] (step=0075750) Train Loss: 6.3800, Train Steps/Sec: 0.84 + 85%|██████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████▉ | 34871/40903 [13:00:51<1:58:11, 1.18s/it][2025-04-23 06:43:56] (step=0075775) Train Loss: 6.3421, Train Steps/Sec: 0.85 + 85%|███████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████ | 34896/40903 [13:01:21<1:57:31, 1.17s/it][2025-04-23 06:44:26] (step=0075800) Train Loss: 6.3526, Train Steps/Sec: 0.84 + 85%|███████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████ | 34921/40903 [13:01:57<2:04:02, 1.24s/it][2025-04-23 06:45:02] (step=0075825) Train Loss: 6.3566, Train Steps/Sec: 0.70 + 85%|███████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████▏ | 34946/40903 [13:02:26<1:58:24, 1.19s/it][2025-04-23 06:45:31] (step=0075850) Train Loss: 6.3599, Train Steps/Sec: 0.84 + 85%|███████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████▎ | 34971/40903 [13:02:56<1:56:32, 1.18s/it][2025-04-23 06:46:01] (step=0075875) Train Loss: 6.3399, Train Steps/Sec: 0.84 + 86%|███████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████▍ | 34996/40903 [13:03:26<1:57:02, 1.19s/it][2025-04-23 06:46:31] (step=0075900) Train Loss: 6.3467, Train Steps/Sec: 0.84 + 86%|███████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████▌ | 35021/40903 [13:03:55<1:57:07, 1.19s/it][2025-04-23 06:47:00] (step=0075925) Train Loss: 6.3376, Train Steps/Sec: 0.85 + 86%|███████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████▋ | 35046/40903 [13:04:25<1:55:43, 1.19s/it][2025-04-23 06:47:30] (step=0075950) Train Loss: 6.3298, Train Steps/Sec: 0.84 + 86%|███████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████▊ | 35071/40903 [13:04:54<1:53:46, 1.17s/it][2025-04-23 06:47:59] (step=0075975) Train Loss: 6.3420, Train Steps/Sec: 0.85 + 86%|███████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████▊ | 35096/40903 [13:05:24<1:53:16, 1.17s/it][2025-04-23 06:48:29] (step=0076000) Train Loss: 6.3395, Train Steps/Sec: 0.84 +You are using a model of type instructblip to instantiate a model of type blip-2. This is not supported for all configurations of models and can yield errors. +model_name_or_path:/tmp/haozhezhao/model/instructblip-flan-t5-xl +image_place_holder: +Some kwargs in processor config are unused and will not have any effect: num_query_tokens. +tokenizer length after expend 32102 +tokenizer length before expend 32102 +100%|███████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████| 3/3 [02:54<00:00, 58.12s/it] +[2025-04-23 06:53:13] Finish Eval in 76000 steps...████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████| 3/3 [02:53<00:00, 57.53s/it] +processing left-padding... +processing left-padding... +processing left-padding... +[2025-04-23 06:53:34] Saved checkpoint to checkpoint/BeforeFix_Blip_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_objects/002-GPT-XL/checkpoints/0076000.pt +[2025-04-23 06:53:36] Removed old checkpoint: checkpoint/BeforeFix_Blip_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_objects/002-GPT-XL/checkpoints/0072000.pt + 86%|███████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████▉ | 35121/40903 [13:11:01<1:57:42, 1.22s/it][2025-04-23 06:54:06] (step=0076025) Train Loss: 6.3710, Train Steps/Sec: 0.07 + 86%|████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████ | 35146/40903 [13:11:31<1:54:40, 1.20s/it][2025-04-23 06:54:35] (step=0076050) Train Loss: 6.3300, Train Steps/Sec: 0.85 + 86%|████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████▏ | 35171/40903 [13:12:00<1:51:58, 1.17s/it][2025-04-23 06:55:05] (step=0076075) Train Loss: 6.3622, Train Steps/Sec: 0.84 + 86%|████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████▎ | 35196/40903 [13:12:30<1:51:26, 1.17s/it][2025-04-23 06:55:35] (step=0076100) Train Loss: 6.3648, Train Steps/Sec: 0.84 + 86%|████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████▍ | 35221/40903 [13:12:59<1:53:43, 1.20s/it][2025-04-23 06:56:04] (step=0076125) Train Loss: 6.3580, Train Steps/Sec: 0.85 + 86%|████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████▌ | 35246/40903 [13:13:29<1:51:59, 1.19s/it][2025-04-23 06:56:34] (step=0076150) Train Loss: 6.3582, Train Steps/Sec: 0.85 + 86%|████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████▋ | 35271/40903 [13:13:59<1:52:31, 1.20s/it][2025-04-23 06:57:04] (step=0076175) Train Loss: 6.3651, Train Steps/Sec: 0.84 + 86%|████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████▋ | 35296/40903 [13:14:28<1:48:57, 1.17s/it][2025-04-23 06:57:33] (step=0076200) Train Loss: 6.3338, Train Steps/Sec: 0.84 + 86%|████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████▊ | 35321/40903 [13:14:58<1:53:13, 1.22s/it][2025-04-23 06:58:03] (step=0076225) Train Loss: 6.3642, Train Steps/Sec: 0.84 + 86%|████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████▉ | 35346/40903 [13:15:28<1:47:57, 1.17s/it][2025-04-23 06:58:32] (step=0076250) Train Loss: 6.3692, Train Steps/Sec: 0.85 + 86%|█████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████ | 35371/40903 [13:15:57<1:48:25, 1.18s/it][2025-04-23 06:59:02] (step=0076275) Train Loss: 6.3740, Train Steps/Sec: 0.84 + 87%|█████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████▏ | 35396/40903 [13:16:27<1:47:12, 1.17s/it][2025-04-23 06:59:32] (step=0076300) Train Loss: 6.3358, Train Steps/Sec: 0.84 + 87%|█████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████▎ | 35421/40903 [13:16:57<1:50:12, 1.21s/it][2025-04-23 07:00:01] (step=0076325) Train Loss: 6.3583, Train Steps/Sec: 0.84 + 87%|█████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████▍ | 35446/40903 [13:17:26<1:46:53, 1.18s/it][2025-04-23 07:00:31] (step=0076350) Train Loss: 6.3391, Train Steps/Sec: 0.85 + 87%|█████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████▍ | 35471/40903 [13:17:56<1:46:52, 1.18s/it][2025-04-23 07:01:00] (step=0076375) Train Loss: 6.3396, Train Steps/Sec: 0.85 + 87%|█████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████▌ | 35496/40903 [13:18:25<1:45:01, 1.17s/it][2025-04-23 07:01:30] (step=0076400) Train Loss: 6.3404, Train Steps/Sec: 0.84 + 87%|█████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████▋ | 35521/40903 [13:18:55<1:50:43, 1.23s/it][2025-04-23 07:02:00] (step=0076425) Train Loss: 6.3365, Train Steps/Sec: 0.83 + 87%|█████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████▊ | 35546/40903 [13:19:25<1:46:26, 1.19s/it][2025-04-23 07:02:30] (step=0076450) Train Loss: 6.3239, Train Steps/Sec: 0.84 + 87%|█████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████▉ | 35571/40903 [13:19:54<1:44:17, 1.17s/it][2025-04-23 07:02:59] (step=0076475) Train Loss: 6.3530, Train Steps/Sec: 0.85 + 87%|██████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████ | 35596/40903 [13:20:24<1:44:08, 1.18s/it][2025-04-23 07:03:29] (step=0076500) Train Loss: 6.3640, Train Steps/Sec: 0.84 + 87%|██████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████▏ | 35621/40903 [13:20:53<1:44:37, 1.19s/it][2025-04-23 07:03:58] (step=0076525) Train Loss: 6.3210, Train Steps/Sec: 0.85 + 87%|██████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████▎ | 35646/40903 [13:21:23<1:43:34, 1.18s/it][2025-04-23 07:04:28] (step=0076550) Train Loss: 6.3785, Train Steps/Sec: 0.84 + 87%|██████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████▎ | 35671/40903 [13:21:53<1:43:53, 1.19s/it][2025-04-23 07:04:58] (step=0076575) Train Loss: 6.3596, Train Steps/Sec: 0.84 + 87%|██████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████▍ | 35696/40903 [13:22:22<1:41:40, 1.17s/it][2025-04-23 07:05:27] (step=0076600) Train Loss: 6.3501, Train Steps/Sec: 0.84 + 87%|██████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████▌ | 35721/40903 [13:22:52<1:44:07, 1.21s/it][2025-04-23 07:05:57] (step=0076625) Train Loss: 6.3432, Train Steps/Sec: 0.85 + 87%|██████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████▋ | 35746/40903 [13:23:21<1:40:58, 1.17s/it][2025-04-23 07:06:26] (step=0076650) Train Loss: 6.3264, Train Steps/Sec: 0.85 + 87%|██████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████▊ | 35771/40903 [13:23:51<1:40:02, 1.17s/it][2025-04-23 07:06:56] (step=0076675) Train Loss: 6.3597, Train Steps/Sec: 0.84 + 88%|██████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████▉ | 35796/40903 [13:24:21<1:38:15, 1.15s/it][2025-04-23 07:07:25] (step=0076700) Train Loss: 6.3682, Train Steps/Sec: 0.84 + 88%|███████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████ | 35821/40903 [13:24:50<1:42:19, 1.21s/it][2025-04-23 07:07:55] (step=0076725) Train Loss: 6.3179, Train Steps/Sec: 0.85 + 88%|███████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████ | 35846/40903 [13:25:20<1:38:59, 1.17s/it][2025-04-23 07:08:25] (step=0076750) Train Loss: 6.3677, Train Steps/Sec: 0.85 + 88%|███████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████▏ | 35871/40903 [13:25:49<1:38:03, 1.17s/it][2025-04-23 07:08:54] (step=0076775) Train Loss: 6.3427, Train Steps/Sec: 0.85 + 88%|███████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████▎ | 35896/40903 [13:26:19<1:39:09, 1.19s/it][2025-04-23 07:09:24] (step=0076800) Train Loss: 6.3266, Train Steps/Sec: 0.84 + 88%|███████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████▍ | 35921/40903 [13:26:49<1:41:13, 1.22s/it][2025-04-23 07:09:53] (step=0076825) Train Loss: 6.3784, Train Steps/Sec: 0.84 + 88%|███████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████▌ | 35946/40903 [13:27:18<1:38:09, 1.19s/it][2025-04-23 07:10:23] (step=0076850) Train Loss: 6.3407, Train Steps/Sec: 0.84 + 88%|███████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████▋ | 35971/40903 [13:27:48<1:38:00, 1.19s/it][2025-04-23 07:10:53] (step=0076875) Train Loss: 6.3178, Train Steps/Sec: 0.84 + 88%|███████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████▊ | 35996/40903 [13:28:18<1:36:32, 1.18s/it][2025-04-23 07:11:23] (step=0076900) Train Loss: 6.3645, Train Steps/Sec: 0.84 + 88%|███████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████▊ | 36021/40903 [13:28:47<1:37:36, 1.20s/it][2025-04-23 07:11:52] (step=0076925) Train Loss: 6.3645, Train Steps/Sec: 0.84 + 88%|███████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████▉ | 36046/40903 [13:29:17<1:36:51, 1.20s/it][2025-04-23 07:12:22] (step=0076950) Train Loss: 6.3682, Train Steps/Sec: 0.84 + 88%|████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████ | 36071/40903 [13:29:47<1:34:37, 1.18s/it][2025-04-23 07:12:52] (step=0076975) Train Loss: 6.3090, Train Steps/Sec: 0.84 + 88%|████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████▏ | 36096/40903 [13:30:16<1:33:05, 1.16s/it][2025-04-23 07:13:21] (step=0077000) Train Loss: 6.3298, Train Steps/Sec: 0.85 + 88%|████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████▎ | 36121/40903 [13:30:46<1:36:10, 1.21s/it][2025-04-23 07:13:51] (step=0077025) Train Loss: 6.3303, Train Steps/Sec: 0.85 + 88%|████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████▍ | 36146/40903 [13:31:15<1:33:44, 1.18s/it][2025-04-23 07:14:20] (step=0077050) Train Loss: 6.3166, Train Steps/Sec: 0.85 + 88%|████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████▌ | 36171/40903 [13:31:45<1:32:15, 1.17s/it][2025-04-23 07:14:50] (step=0077075) Train Loss: 6.3358, Train Steps/Sec: 0.84 + 88%|████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████▋ | 36196/40903 [13:32:15<1:32:43, 1.18s/it][2025-04-23 07:15:20] (step=0077100) Train Loss: 6.3414, Train Steps/Sec: 0.84 + 89%|████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████▋ | 36221/40903 [13:32:44<1:34:25, 1.21s/it][2025-04-23 07:15:49] (step=0077125) Train Loss: 6.3573, Train Steps/Sec: 0.85 + 89%|████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████▊ | 36246/40903 [13:33:14<1:31:16, 1.18s/it][2025-04-23 07:16:19] (step=0077150) Train Loss: 6.3979, Train Steps/Sec: 0.85 + 89%|████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████▉ | 36271/40903 [13:33:43<1:30:29, 1.17s/it][2025-04-23 07:16:48] (step=0077175) Train Loss: 6.3928, Train Steps/Sec: 0.85 + 89%|█████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████ | 36296/40903 [13:34:13<1:30:17, 1.18s/it][2025-04-23 07:17:18] (step=0077200) Train Loss: 6.3438, Train Steps/Sec: 0.85 + 89%|█████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████▏ | 36321/40903 [13:34:42<1:31:32, 1.20s/it][2025-04-23 07:17:47] (step=0077225) Train Loss: 6.3227, Train Steps/Sec: 0.85 + 89%|█████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████▎ | 36346/40903 [13:35:12<1:30:41, 1.19s/it][2025-04-23 07:18:17] (step=0077250) Train Loss: 6.3356, Train Steps/Sec: 0.84 + 89%|█████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████▍ | 36371/40903 [13:35:41<1:28:01, 1.17s/it][2025-04-23 07:18:46] (step=0077275) Train Loss: 6.3373, Train Steps/Sec: 0.85 + 89%|█████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████▍ | 36396/40903 [13:36:11<1:26:59, 1.16s/it][2025-04-23 07:19:16] (step=0077300) Train Loss: 6.3692, Train Steps/Sec: 0.85 + 89%|█████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████▌ | 36421/40903 [13:36:40<1:31:08, 1.22s/it][2025-04-23 07:19:45] (step=0077325) Train Loss: 6.3664, Train Steps/Sec: 0.84 + 89%|█████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████▋ | 36446/40903 [13:37:10<1:27:45, 1.18s/it][2025-04-23 07:20:15] (step=0077350) Train Loss: 6.3117, Train Steps/Sec: 0.84 + 89%|█████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████▊ | 36471/40903 [13:37:40<1:27:06, 1.18s/it][2025-04-23 07:20:45] (step=0077375) Train Loss: 6.3560, Train Steps/Sec: 0.84 + 89%|█████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████▉ | 36496/40903 [13:38:09<1:25:44, 1.17s/it][2025-04-23 07:21:14] (step=0077400) Train Loss: 6.3381, Train Steps/Sec: 0.84 + 89%|██████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████ | 36521/40903 [13:38:39<1:27:35, 1.20s/it][2025-04-23 07:21:44] (step=0077425) Train Loss: 6.3556, Train Steps/Sec: 0.84 + 89%|██████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████▏ | 36546/40903 [13:39:09<1:27:15, 1.20s/it][2025-04-23 07:22:14] (step=0077450) Train Loss: 6.3438, Train Steps/Sec: 0.84 + 89%|██████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████▎ | 36571/40903 [13:39:38<1:24:06, 1.16s/it][2025-04-23 07:22:43] (step=0077475) Train Loss: 6.3435, Train Steps/Sec: 0.85 + 89%|██████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████▎ | 36596/40903 [13:40:08<1:23:57, 1.17s/it][2025-04-23 07:23:13] (step=0077500) Train Loss: 6.3177, Train Steps/Sec: 0.84 + 90%|██████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████▍ | 36621/40903 [13:40:38<1:24:55, 1.19s/it][2025-04-23 07:23:42] (step=0077525) Train Loss: 6.3505, Train Steps/Sec: 0.85 + 90%|██████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████▌ | 36646/40903 [13:41:07<1:24:31, 1.19s/it][2025-04-23 07:24:12] (step=0077550) Train Loss: 6.3179, Train Steps/Sec: 0.84 + 90%|██████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████▋ | 36671/40903 [13:41:37<1:22:14, 1.17s/it][2025-04-23 07:24:42] (step=0077575) Train Loss: 6.3607, Train Steps/Sec: 0.85 + 90%|██████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████▊ | 36696/40903 [13:42:06<1:22:17, 1.17s/it][2025-04-23 07:25:11] (step=0077600) Train Loss: 6.3339, Train Steps/Sec: 0.85 + 90%|██████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████▉ | 36721/40903 [13:42:36<1:23:10, 1.19s/it][2025-04-23 07:25:41] (step=0077625) Train Loss: 6.3192, Train Steps/Sec: 0.84 + 90%|███████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████ | 36746/40903 [13:43:05<1:21:08, 1.17s/it][2025-04-23 07:26:10] (step=0077650) Train Loss: 6.3599, Train Steps/Sec: 0.84 + 90%|███████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████ | 36771/40903 [13:43:35<1:21:10, 1.18s/it][2025-04-23 07:26:40] (step=0077675) Train Loss: 6.3239, Train Steps/Sec: 0.85 + 90%|███████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████▏ | 36796/40903 [13:44:05<1:20:39, 1.18s/it][2025-04-23 07:27:10] (step=0077700) Train Loss: 6.3361, Train Steps/Sec: 0.84 + 90%|███████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████▎ | 36821/40903 [13:44:34<1:21:27, 1.20s/it][2025-04-23 07:27:39] (step=0077725) Train Loss: 6.3436, Train Steps/Sec: 0.85 + 90%|███████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████▍ | 36846/40903 [13:45:04<1:19:48, 1.18s/it][2025-04-23 07:28:09] (step=0077750) Train Loss: 6.3777, Train Steps/Sec: 0.84 + 90%|███████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████▌ | 36871/40903 [13:45:33<1:18:22, 1.17s/it][2025-04-23 07:28:38] (step=0077775) Train Loss: 6.3170, Train Steps/Sec: 0.85 + 90%|███████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████▋ | 36896/40903 [13:46:03<1:17:07, 1.15s/it][2025-04-23 07:29:08] (step=0077800) Train Loss: 6.3717, Train Steps/Sec: 0.85 + 90%|███████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████▊ | 36921/40903 [13:46:33<1:20:48, 1.22s/it][2025-04-23 07:29:37] (step=0077825) Train Loss: 6.3351, Train Steps/Sec: 0.84 + 90%|███████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████▉ | 36946/40903 [13:47:02<1:18:17, 1.19s/it][2025-04-23 07:30:07] (step=0077850) Train Loss: 6.3691, Train Steps/Sec: 0.85 + 90%|███████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████▉ | 36971/40903 [13:47:32<1:16:34, 1.17s/it][2025-04-23 07:30:36] (step=0077875) Train Loss: 6.3989, Train Steps/Sec: 0.85 + 90%|████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████ | 36996/40903 [13:48:01<1:15:48, 1.16s/it][2025-04-23 07:31:06] (step=0077900) Train Loss: 6.3504, Train Steps/Sec: 0.84 + 91%|████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████▏ | 37021/40903 [13:48:31<1:18:13, 1.21s/it][2025-04-23 07:31:36] (step=0077925) Train Loss: 6.3630, Train Steps/Sec: 0.84 + 91%|████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████▎ | 37046/40903 [13:49:01<1:16:26, 1.19s/it][2025-04-23 07:32:05] (step=0077950) Train Loss: 6.2973, Train Steps/Sec: 0.84 + 91%|████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████▍ | 37071/40903 [13:49:30<1:15:30, 1.18s/it][2025-04-23 07:32:35] (step=0077975) Train Loss: 6.3474, Train Steps/Sec: 0.84 + 91%|████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████▌ | 37096/40903 [13:50:00<1:14:24, 1.17s/it][2025-04-23 07:33:05] (step=0078000) Train Loss: 6.2938, Train Steps/Sec: 0.84 +You are using a model of type instructblip to instantiate a model of type blip-2. This is not supported for all configurations of models and can yield errors. +model_name_or_path:/tmp/haozhezhao/model/instructblip-flan-t5-xl +image_place_holder: +Some kwargs in processor config are unused and will not have any effect: num_query_tokens. +tokenizer length after expend 32102 +tokenizer length before expend 32102 +100%|███████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████| 3/3 [02:53<00:00, 57.93s/it] +[2025-04-23 07:37:48] Finish Eval in 78000 steps...████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████| 3/3 [02:52<00:00, 57.24s/it] +processing left-padding... +processing left-padding... +processing left-padding... +[2025-04-23 07:38:11] Saved checkpoint to checkpoint/BeforeFix_Blip_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_objects/002-GPT-XL/checkpoints/0078000.pt +[2025-04-23 07:38:13] Removed old checkpoint: checkpoint/BeforeFix_Blip_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_objects/002-GPT-XL/checkpoints/0074000.pt + 91%|████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████▋ | 37121/40903 [13:55:37<1:16:29, 1.21s/it][2025-04-23 07:38:42] (step=0078025) Train Loss: 6.3359, Train Steps/Sec: 0.07 + 91%|████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████▋ | 37146/40903 [13:56:07<1:13:53, 1.18s/it][2025-04-23 07:39:12] (step=0078050) Train Loss: 6.3523, Train Steps/Sec: 0.85 + 91%|████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████▊ | 37171/40903 [13:56:37<1:13:38, 1.18s/it][2025-04-23 07:39:41] (step=0078075) Train Loss: 6.4101, Train Steps/Sec: 0.84 + 91%|████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████▉ | 37196/40903 [13:57:06<1:12:29, 1.17s/it][2025-04-23 07:40:11] (step=0078100) Train Loss: 6.3501, Train Steps/Sec: 0.84 + 91%|█████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████ | 37221/40903 [13:57:36<1:15:03, 1.22s/it][2025-04-23 07:40:41] (step=0078125) Train Loss: 6.3519, Train Steps/Sec: 0.84 + 91%|█████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████▏ | 37246/40903 [13:58:05<1:12:20, 1.19s/it][2025-04-23 07:41:10] (step=0078150) Train Loss: 6.3507, Train Steps/Sec: 0.85 + 91%|█████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████▎ | 37271/40903 [13:58:35<1:11:13, 1.18s/it][2025-04-23 07:41:40] (step=0078175) Train Loss: 6.3646, Train Steps/Sec: 0.84 + 91%|█████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████▍ | 37296/40903 [13:59:04<1:10:46, 1.18s/it][2025-04-23 07:42:09] (step=0078200) Train Loss: 6.3302, Train Steps/Sec: 0.85 + 91%|█████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████▍ | 37321/40903 [13:59:34<1:11:24, 1.20s/it][2025-04-23 07:42:39] (step=0078225) Train Loss: 6.3469, Train Steps/Sec: 0.85 + 91%|█████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████▌ | 37346/40903 [14:00:04<1:09:58, 1.18s/it][2025-04-23 07:43:08] (step=0078250) Train Loss: 6.3299, Train Steps/Sec: 0.85 + 91%|█████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████▋ | 37371/40903 [14:00:33<1:08:43, 1.17s/it][2025-04-23 07:43:38] (step=0078275) Train Loss: 6.3728, Train Steps/Sec: 0.85 + 91%|█████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████▊ | 37396/40903 [14:01:03<1:08:09, 1.17s/it][2025-04-23 07:44:08] (step=0078300) Train Loss: 6.3820, Train Steps/Sec: 0.84 + 91%|█████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████▉ | 37421/40903 [14:01:32<1:09:38, 1.20s/it][2025-04-23 07:44:37] (step=0078325) Train Loss: 6.3756, Train Steps/Sec: 0.85 + 92%|██████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████ | 37446/40903 [14:02:02<1:08:33, 1.19s/it][2025-04-23 07:45:07] (step=0078350) Train Loss: 6.3242, Train Steps/Sec: 0.84 + 92%|██████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████▏ | 37471/40903 [14:02:32<1:06:58, 1.17s/it][2025-04-23 07:45:36] (step=0078375) Train Loss: 6.3703, Train Steps/Sec: 0.85 + 92%|██████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████▎ | 37496/40903 [14:03:01<1:06:56, 1.18s/it][2025-04-23 07:46:06] (step=0078400) Train Loss: 6.4080, Train Steps/Sec: 0.84 + 92%|██████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████▎ | 37521/40903 [14:03:31<1:07:11, 1.19s/it][2025-04-23 07:46:36] (step=0078425) Train Loss: 6.3824, Train Steps/Sec: 0.85 + 92%|██████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████▍ | 37546/40903 [14:04:01<1:06:18, 1.19s/it][2025-04-23 07:47:05] (step=0078450) Train Loss: 6.3740, Train Steps/Sec: 0.84 + 92%|██████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████▌ | 37571/40903 [14:04:30<1:04:51, 1.17s/it][2025-04-23 07:47:35] (step=0078475) Train Loss: 6.3671, Train Steps/Sec: 0.85 + 92%|██████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████▋ | 37596/40903 [14:05:00<1:04:58, 1.18s/it][2025-04-23 07:48:05] (step=0078500) Train Loss: 6.4051, Train Steps/Sec: 0.84 + 92%|██████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████▊ | 37621/40903 [14:05:29<1:05:31, 1.20s/it][2025-04-23 07:48:34] (step=0078525) Train Loss: 6.3586, Train Steps/Sec: 0.85 + 92%|██████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████▉ | 37646/40903 [14:05:59<1:04:14, 1.18s/it][2025-04-23 07:49:04] (step=0078550) Train Loss: 6.3614, Train Steps/Sec: 0.84 + 92%|███████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████ | 37671/40903 [14:06:28<1:02:45, 1.17s/it][2025-04-23 07:49:33] (step=0078575) Train Loss: 6.3361, Train Steps/Sec: 0.85 + 92%|███████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████ | 37696/40903 [14:06:58<1:02:54, 1.18s/it][2025-04-23 07:50:03] (step=0078600) Train Loss: 6.3664, Train Steps/Sec: 0.84 + 92%|███████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████▏ | 37721/40903 [14:07:27<1:03:10, 1.19s/it][2025-04-23 07:50:32] (step=0078625) Train Loss: 6.3641, Train Steps/Sec: 0.85 + 92%|███████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████▎ | 37746/40903 [14:07:57<1:01:13, 1.16s/it][2025-04-23 07:51:02] (step=0078650) Train Loss: 6.3663, Train Steps/Sec: 0.85 + 92%|███████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████▍ | 37771/40903 [14:08:27<1:01:40, 1.18s/it][2025-04-23 07:51:31] (step=0078675) Train Loss: 6.3432, Train Steps/Sec: 0.84 + 92%|███████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████▌ | 37796/40903 [14:08:56<1:00:22, 1.17s/it][2025-04-23 07:52:01] (step=0078700) Train Loss: 6.3370, Train Steps/Sec: 0.85 + 92%|███████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████▋ | 37821/40903 [14:09:26<1:01:57, 1.21s/it][2025-04-23 07:52:30] (step=0078725) Train Loss: 6.3041, Train Steps/Sec: 0.85 + 93%|███████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████▊ | 37846/40903 [14:09:55<1:00:06, 1.18s/it][2025-04-23 07:53:00] (step=0078750) Train Loss: 6.3254, Train Steps/Sec: 0.85 + 93%|█████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████▋ | 37871/40903 [14:10:24<59:00, 1.17s/it][2025-04-23 07:53:29] (step=0078775) Train Loss: 6.3348, Train Steps/Sec: 0.85 + 93%|█████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████▊ | 37896/40903 [14:10:54<59:10, 1.18s/it][2025-04-23 07:53:59] (step=0078800) Train Loss: 6.3398, Train Steps/Sec: 0.84 + 93%|█████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████▉ | 37921/40903 [14:11:24<59:28, 1.20s/it][2025-04-23 07:54:29] (step=0078825) Train Loss: 6.3608, Train Steps/Sec: 0.85 + 93%|██████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████ | 37946/40903 [14:11:53<58:01, 1.18s/it][2025-04-23 07:54:58] (step=0078850) Train Loss: 6.3802, Train Steps/Sec: 0.85 + 93%|██████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████▏ | 37971/40903 [14:12:23<57:29, 1.18s/it][2025-04-23 07:55:27] (step=0078875) Train Loss: 6.3466, Train Steps/Sec: 0.85 + 93%|██████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████▎ | 37996/40903 [14:12:52<56:54, 1.17s/it][2025-04-23 07:55:57] (step=0078900) Train Loss: 6.3609, Train Steps/Sec: 0.84 + 93%|██████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████▍ | 38021/40903 [14:13:22<57:19, 1.19s/it][2025-04-23 07:56:27] (step=0078925) Train Loss: 6.3449, Train Steps/Sec: 0.85 + 93%|██████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████▍ | 38046/40903 [14:13:51<56:39, 1.19s/it][2025-04-23 07:56:56] (step=0078950) Train Loss: 6.3370, Train Steps/Sec: 0.85 + 93%|██████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████▌ | 38071/40903 [14:14:21<55:00, 1.17s/it][2025-04-23 07:57:26] (step=0078975) Train Loss: 6.3076, Train Steps/Sec: 0.85 + 93%|██████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████▋ | 38096/40903 [14:14:50<54:40, 1.17s/it][2025-04-23 07:57:55] (step=0079000) Train Loss: 6.3324, Train Steps/Sec: 0.84 + 93%|██████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████▊ | 38121/40903 [14:15:20<56:21, 1.22s/it][2025-04-23 07:58:25] (step=0079025) Train Loss: 6.2824, Train Steps/Sec: 0.85 + 93%|██████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████▉ | 38146/40903 [14:15:50<55:00, 1.20s/it][2025-04-23 07:58:55] (step=0079050) Train Loss: 6.3335, Train Steps/Sec: 0.84 + 93%|███████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████ | 38171/40903 [14:16:19<53:16, 1.17s/it][2025-04-23 07:59:24] (step=0079075) Train Loss: 6.3374, Train Steps/Sec: 0.85 + 93%|███████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████▏ | 38196/40903 [14:16:49<52:37, 1.17s/it][2025-04-23 07:59:54] (step=0079100) Train Loss: 6.3786, Train Steps/Sec: 0.85 + 93%|███████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████▎ | 38221/40903 [14:17:18<53:37, 1.20s/it][2025-04-23 08:00:23] (step=0079125) Train Loss: 6.3115, Train Steps/Sec: 0.85 + 94%|███████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████▎ | 38246/40903 [14:17:48<52:02, 1.18s/it][2025-04-23 08:00:53] (step=0079150) Train Loss: 6.3658, Train Steps/Sec: 0.85 + 94%|███████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████▍ | 38271/40903 [14:18:17<52:30, 1.20s/it][2025-04-23 08:01:22] (step=0079175) Train Loss: 6.3507, Train Steps/Sec: 0.85 + 94%|███████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████▌ | 38296/40903 [14:18:47<50:39, 1.17s/it][2025-04-23 08:01:52] (step=0079200) Train Loss: 6.3581, Train Steps/Sec: 0.84 + 94%|███████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████▋ | 38321/40903 [14:19:17<51:19, 1.19s/it][2025-04-23 08:02:21] (step=0079225) Train Loss: 6.3107, Train Steps/Sec: 0.85 + 94%|███████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████▊ | 38346/40903 [14:19:46<50:34, 1.19s/it][2025-04-23 08:02:51] (step=0079250) Train Loss: 6.3426, Train Steps/Sec: 0.85 + 94%|███████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████▉ | 38371/40903 [14:20:16<50:00, 1.18s/it][2025-04-23 08:03:21] (step=0079275) Train Loss: 6.3595, Train Steps/Sec: 0.84 + 94%|████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████ | 38396/40903 [14:20:45<48:40, 1.16s/it][2025-04-23 08:03:50] (step=0079300) Train Loss: 6.3722, Train Steps/Sec: 0.85 + 94%|████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████▏ | 38421/40903 [14:21:15<49:08, 1.19s/it][2025-04-23 08:04:20] (step=0079325) Train Loss: 6.3736, Train Steps/Sec: 0.85 + 94%|████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████▏ | 38446/40903 [14:21:44<48:37, 1.19s/it][2025-04-23 08:04:49] (step=0079350) Train Loss: 6.3806, Train Steps/Sec: 0.85 + 94%|████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████▎ | 38471/40903 [14:22:14<47:37, 1.18s/it][2025-04-23 08:05:19] (step=0079375) Train Loss: 6.3198, Train Steps/Sec: 0.85 + 94%|████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████▍ | 38496/40903 [14:22:44<47:06, 1.17s/it][2025-04-23 08:05:48] (step=0079400) Train Loss: 6.3414, Train Steps/Sec: 0.84 + 94%|████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████▌ | 38521/40903 [14:23:13<47:35, 1.20s/it][2025-04-23 08:06:18] (step=0079425) Train Loss: 6.3912, Train Steps/Sec: 0.84 + 94%|████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████▋ | 38546/40903 [14:23:43<46:10, 1.18s/it][2025-04-23 08:06:48] (step=0079450) Train Loss: 6.4017, Train Steps/Sec: 0.85 + 94%|████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████▊ | 38571/40903 [14:24:12<45:44, 1.18s/it][2025-04-23 08:07:17] (step=0079475) Train Loss: 6.3202, Train Steps/Sec: 0.85 + 94%|████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████▉ | 38596/40903 [14:24:42<44:11, 1.15s/it][2025-04-23 08:07:47] (step=0079500) Train Loss: 6.3548, Train Steps/Sec: 0.85 + 94%|█████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████ | 38621/40903 [14:25:11<46:00, 1.21s/it][2025-04-23 08:08:16] (step=0079525) Train Loss: 6.3089, Train Steps/Sec: 0.85 + 94%|█████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████ | 38646/40903 [14:25:41<43:58, 1.17s/it][2025-04-23 08:08:46] (step=0079550) Train Loss: 6.3707, Train Steps/Sec: 0.85 + 95%|█████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████▏ | 38671/40903 [14:26:10<44:02, 1.18s/it][2025-04-23 08:09:15] (step=0079575) Train Loss: 6.3111, Train Steps/Sec: 0.84 + 95%|█████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████▎ | 38696/40903 [14:26:40<42:42, 1.16s/it][2025-04-23 08:09:45] (step=0079600) Train Loss: 6.3688, Train Steps/Sec: 0.84 + 95%|█████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████▍ | 38721/40903 [14:27:10<43:34, 1.20s/it][2025-04-23 08:10:15] (step=0079625) Train Loss: 6.3791, Train Steps/Sec: 0.84 + 95%|█████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████▌ | 38746/40903 [14:27:39<42:18, 1.18s/it][2025-04-23 08:10:44] (step=0079650) Train Loss: 6.3147, Train Steps/Sec: 0.85 + 95%|█████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████▋ | 38771/40903 [14:28:09<42:20, 1.19s/it][2025-04-23 08:11:14] (step=0079675) Train Loss: 6.3600, Train Steps/Sec: 0.84 + 95%|█████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████▊ | 38796/40903 [14:28:38<40:59, 1.17s/it][2025-04-23 08:11:43] (step=0079700) Train Loss: 6.3574, Train Steps/Sec: 0.84 + 95%|█████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████▉ | 38821/40903 [14:29:08<41:20, 1.19s/it][2025-04-23 08:12:13] (step=0079725) Train Loss: 6.3236, Train Steps/Sec: 0.85 + 95%|█████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████▉ | 38846/40903 [14:29:44<41:22, 1.21s/it][2025-04-23 08:12:49] (step=0079750) Train Loss: 6.3563, Train Steps/Sec: 0.69 + 95%|██████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████ | 38871/40903 [14:30:14<39:32, 1.17s/it][2025-04-23 08:13:18] (step=0079775) Train Loss: 6.3497, Train Steps/Sec: 0.85 + 95%|██████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████▏ | 38896/40903 [14:30:43<38:54, 1.16s/it][2025-04-23 08:13:48] (step=0079800) Train Loss: 6.3228, Train Steps/Sec: 0.84 + 95%|██████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████▎ | 38921/40903 [14:31:13<39:41, 1.20s/it][2025-04-23 08:14:18] (step=0079825) Train Loss: 6.3464, Train Steps/Sec: 0.85 + 95%|██████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████▍ | 38946/40903 [14:31:42<38:18, 1.17s/it][2025-04-23 08:14:47] (step=0079850) Train Loss: 6.3154, Train Steps/Sec: 0.85 + 95%|██████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████▌ | 38971/40903 [14:32:12<37:22, 1.16s/it][2025-04-23 08:15:17] (step=0079875) Train Loss: 6.3322, Train Steps/Sec: 0.85 + 95%|██████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████▋ | 38996/40903 [14:32:41<36:47, 1.16s/it][2025-04-23 08:15:46] (step=0079900) Train Loss: 6.3097, Train Steps/Sec: 0.84 + 95%|██████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████▊ | 39021/40903 [14:33:17<37:15, 1.19s/it][2025-04-23 08:16:22] (step=0079925) Train Loss: 6.3435, Train Steps/Sec: 0.70 + 95%|██████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████▊ | 39046/40903 [14:33:47<36:47, 1.19s/it][2025-04-23 08:16:52] (step=0079950) Train Loss: 6.3956, Train Steps/Sec: 0.85 + 96%|██████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████▉ | 39071/40903 [14:34:16<35:52, 1.18s/it][2025-04-23 08:17:21] (step=0079975) Train Loss: 6.3386, Train Steps/Sec: 0.85 + 96%|███████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████ | 39096/40903 [14:34:46<35:36, 1.18s/it][2025-04-23 08:17:51] (step=0080000) Train Loss: 6.3754, Train Steps/Sec: 0.85 +You are using a model of type instructblip to instantiate a model of type blip-2. This is not supported for all configurations of models and can yield errors. +model_name_or_path:/tmp/haozhezhao/model/instructblip-flan-t5-xl +image_place_holder: +Some kwargs in processor config are unused and will not have any effect: num_query_tokens. +tokenizer length after expend 32102 +tokenizer length before expend 32102 +100%|███████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████| 3/3 [02:52<00:00, 57.52s/it] +[2025-04-23 08:22:32] Finish Eval in 80000 steps...████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████| 3/3 [02:51<00:00, 57.08s/it] +processing left-padding... +processing left-padding... +processing left-padding... +[2025-04-23 08:22:54] Saved checkpoint to checkpoint/BeforeFix_Blip_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_objects/002-GPT-XL/checkpoints/0080000.pt +[2025-04-23 08:22:56] Removed old checkpoint: checkpoint/BeforeFix_Blip_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_objects/002-GPT-XL/checkpoints/0076000.pt + 96%|███████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████▏ | 39121/40903 [14:40:21<36:13, 1.22s/it][2025-04-23 08:23:26] (step=0080025) Train Loss: 6.3246, Train Steps/Sec: 0.07 + 96%|███████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████▎ | 39146/40903 [14:40:51<34:22, 1.17s/it][2025-04-23 08:23:56] (step=0080050) Train Loss: 6.3600, Train Steps/Sec: 0.85 + 96%|███████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████▍ | 39171/40903 [14:41:21<34:02, 1.18s/it][2025-04-23 08:24:26] (step=0080075) Train Loss: 6.3799, Train Steps/Sec: 0.83 + 96%|███████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████▌ | 39196/40903 [14:41:50<32:52, 1.16s/it][2025-04-23 08:24:55] (step=0080100) Train Loss: 6.3017, Train Steps/Sec: 0.85 + 96%|███████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████▋ | 39221/40903 [14:42:20<33:45, 1.20s/it][2025-04-23 08:25:25] (step=0080125) Train Loss: 6.3257, Train Steps/Sec: 0.84 + 96%|███████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████▋ | 39246/40903 [14:42:49<32:39, 1.18s/it][2025-04-23 08:25:54] (step=0080150) Train Loss: 6.3831, Train Steps/Sec: 0.85 + 96%|███████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████▊ | 39271/40903 [14:43:19<31:56, 1.17s/it][2025-04-23 08:26:24] (step=0080175) Train Loss: 6.3443, Train Steps/Sec: 0.85 + 96%|███████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████▉ | 39296/40903 [14:43:48<31:31, 1.18s/it][2025-04-23 08:26:53] (step=0080200) Train Loss: 6.3237, Train Steps/Sec: 0.84 + 96%|████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████ | 39321/40903 [14:44:18<31:49, 1.21s/it][2025-04-23 08:27:23] (step=0080225) Train Loss: 6.4110, Train Steps/Sec: 0.84 + 96%|████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████▏ | 39346/40903 [14:44:48<30:53, 1.19s/it][2025-04-23 08:27:53] (step=0080250) Train Loss: 6.3286, Train Steps/Sec: 0.84 + 96%|████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████▎ | 39371/40903 [14:45:30<30:31, 1.20s/it][2025-04-23 08:28:35] (step=0080275) Train Loss: 6.3791, Train Steps/Sec: 0.59 + 96%|████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████▍ | 39396/40903 [14:46:00<29:17, 1.17s/it][2025-04-23 08:29:05] (step=0080300) Train Loss: 6.3523, Train Steps/Sec: 0.84 + 96%|████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████▌ | 39421/40903 [14:46:29<29:18, 1.19s/it][2025-04-23 08:29:34] (step=0080325) Train Loss: 6.3277, Train Steps/Sec: 0.85 + 96%|████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████▌ | 39446/40903 [14:46:59<28:52, 1.19s/it][2025-04-23 08:30:03] (step=0080350) Train Loss: 6.3378, Train Steps/Sec: 0.85 + 96%|████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████▋ | 39471/40903 [14:47:28<28:14, 1.18s/it][2025-04-23 08:30:33] (step=0080375) Train Loss: 6.3302, Train Steps/Sec: 0.85 + 97%|████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████▊ | 39496/40903 [14:47:58<27:46, 1.18s/it][2025-04-23 08:31:03] (step=0080400) Train Loss: 6.3459, Train Steps/Sec: 0.84 + 97%|████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████▉ | 39521/40903 [14:48:27<27:38, 1.20s/it][2025-04-23 08:31:32] (step=0080425) Train Loss: 6.3570, Train Steps/Sec: 0.85 + 97%|█████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████ | 39546/40903 [14:48:57<26:40, 1.18s/it][2025-04-23 08:32:01] (step=0080450) Train Loss: 6.3320, Train Steps/Sec: 0.85 + 97%|█████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████▏ | 39568/40903 [14:49:23<26:13, 1.18s/it] diff --git a/CKPTS/BeforeFix_Blip_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_objects/wandb/run-20250422_014729-ni4jp6ul/files/requirements.txt b/CKPTS/BeforeFix_Blip_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_objects/wandb/run-20250422_014729-ni4jp6ul/files/requirements.txt new file mode 100644 index 0000000000000000000000000000000000000000..06dc78369ffff807b210006a0e79d705ffe2a7d7 --- /dev/null +++ b/CKPTS/BeforeFix_Blip_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_objects/wandb/run-20250422_014729-ni4jp6ul/files/requirements.txt @@ -0,0 +1,131 @@ +typing_extensions==4.12.2 +pyzmq==26.3.0 +nvidia-cufft-cu12==11.0.2.54 +triton==3.1.0 +nvidia-cublas-cu12==12.1.3.1 +psutil==7.0.0 +nvidia-cuda-cupti-cu12==12.1.105 +smmap==5.0.2 +nvidia-cuda-runtime-cu12==12.1.105 +aiohappyeyeballs==2.6.1 +asttokens==3.0.0 +huggingface-hub==0.29.3 +pyarrow==19.0.1 +fonttools==4.56.0 +python-dateutil==2.9.0.post0 +GitPython==3.1.44 +aiohttp==3.11.14 +wandb==0.19.8 +setproctitle==1.3.5 +PyYAML==6.0.2 +pydantic_core==2.27.2 +safetensors==0.5.3 +nvidia-nvjitlink-cu12==12.1.105 +aiosignal==1.3.2 +dill==0.3.8 +nvidia-cuda-nvrtc-cu12==12.1.105 +multiprocess==0.70.16 +pure_eval==0.2.3 +stack_data==0.6.3 +pydantic==2.10.6 +MarkupSafe==2.1.5 +tornado==6.4.2 +executing==2.1.0 +executing==2.2.0 +opencv-python==4.11.0.86 +nvitop==1.4.2 +multidict==6.2.0 +Jinja2==3.1.4 +torch==2.5.1+cu121 +nvidia-curand-cu12==10.3.2.106 +platformdirs==4.3.6 +six==1.17.0 +mpmath==1.3.0 +zipp==3.21.0 +packaging==24.2 +requests==2.32.3 +certifi==2025.1.31 +docker-pycreds==0.4.0 +torchvision==0.20.1+cu121 +pandas==2.2.3 +networkx==3.3 +exceptiongroup==1.2.2 +pickleshare==0.7.5 +tokenizers==0.21.1 +charset-normalizer==3.4.1 +jupyter_core==5.7.2 +wcwidth==0.2.13 +nvidia-nvtx-cu12==12.1.105 +prompt_toolkit==3.0.50 +fsspec==2024.12.0 +pillow==11.1.0 +propcache==0.3.0 +regex==2024.11.6 +ptyprocess==0.7.0 +contourpy==1.3.1 +importlib_metadata==8.6.1 +idna==3.10 +comm==0.2.2 +protobuf==5.29.3 +yarl==1.18.3 +ipython_pygments_lexers==1.1.1 +pip==25.0 +parso==0.8.4 +joblib==1.4.2 +nvidia-nccl-cu12==2.21.5 +hf_transfer==0.1.9 +Pygments==2.19.1 +decorator==5.2.1 +filelock==3.18.0 +nvidia-cusparse-cu12==12.1.0.106 +debugpy==1.8.13 +urllib3==2.3.0 +traitlets==5.14.3 +tzdata==2025.1 +matplotlib-inline==0.1.7 +matplotlib==3.10.1 +kiwisolver==1.4.8 +nest_asyncio==1.6.0 +frozenlist==1.5.0 +nvidia-ml-py==12.570.86 +transformers==4.49.0 +nltk==3.9.1 +ipykernel==6.29.5 +click==8.1.8 +gitdb==4.0.12 +pyparsing==3.2.1 +attrs==25.3.0 +jedi==0.19.2 +ipython==9.0.2 +nvidia-cudnn-cu12==9.1.0.70 +pexpect==4.9.0 +nvidia-cusolver-cu12==11.4.5.107 +numpy==2.2.4 +tqdm==4.67.1 +pytz==2025.1 +wheel==0.45.1 +sentry-sdk==2.23.1 +torchaudio==2.5.1+cu121 +jupyter_client==8.6.3 +cycler==0.12.1 +annotated-types==0.7.0 +sympy==1.13.1 +xxhash==3.5.0 +datasets==3.4.1 +setuptools==75.8.0 +typing_extensions==4.12.2 +wheel==0.43.0 +importlib_metadata==8.0.0 +backports.tarfile==1.2.0 +autocommand==2.2.2 +packaging==24.2 +tomli==2.0.1 +typeguard==4.3.0 +zipp==3.19.2 +jaraco.context==5.3.0 +jaraco.functools==4.0.1 +more-itertools==10.3.0 +platformdirs==4.2.2 +jaraco.text==3.12.1 +jaraco.collections==5.1.0 +inflect==7.3.1 diff --git a/CKPTS/BeforeFix_Blip_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_objects/wandb/run-20250422_014729-ni4jp6ul/files/wandb-metadata.json b/CKPTS/BeforeFix_Blip_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_objects/wandb/run-20250422_014729-ni4jp6ul/files/wandb-metadata.json new file mode 100644 index 0000000000000000000000000000000000000000..0106078ad963d84e2ea03970e77054912da390de --- /dev/null +++ b/CKPTS/BeforeFix_Blip_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_objects/wandb/run-20250422_014729-ni4jp6ul/files/wandb-metadata.json @@ -0,0 +1,143 @@ +{ + "os": "Linux-5.15.0-1064-azure-x86_64-with-glibc2.31", + "python": "CPython 3.11.11", + "startedAt": "2025-04-22T01:47:29.361967Z", + "args": [ + "--vq-ckpt", + "/tmp/haozhezhao/model/llamagen_t2i/vq_ds16_t2i.pt", + "--data-path", + "/tmp/haozhezhao/MLLMG/jsonl_data/multiobjects_molom_imagenet_flux_qwen_midsource_gen_2_2m_trained.jsonl", + "--dataset", + "ti2i", + "--image-size", + "512", + "--results-dir", + "checkpoint/BeforeFix_Blip_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_objects", + "--cloud-save-path", + "/tmp/haozhezhao/MLLMG/checkpoint", + "--lr", + "5e-5", + "--val_data_path", + "/tmp/haozhezhao/MLLMG/jsonl_data/multiobjects_molom_imagenet_flux_qwen_midsource_gen_val.jsonl", + "--use_vision_tower", + "--model_name_or_path", + "/tmp/haozhezhao/model/instructblip-flan-t5-xl", + "--image_place_holder", + "", + "--do_eval", + "--eval_steps", + "2000", + "--max_eval_samples", + "200", + "--cfg-scale", + "7.5", + "--top-k", + "16384", + "--load_from_checkpoint", + "/tmp/haozhezhao/model/llamagen_t2i/t2i_XL_stage2_512.pt", + "--global-batch-size", + "56", + "--num-workers", + "4", + "--warmup", + "0.05", + "--gradient-accumulation-steps", + "4", + "--train_text_encoder", + "--ckpt-every", + "2000", + "--epochs", + "2", + "--subject_driven", + "--reference_data_path", + "/tmp/haozhezhao/MLLMG/cc12m_reference_tunnel.jsonl", + "--multimodal_encoder", + "blip", + "--do_recovery", + "--find_unused_parameters", + "--cls-token-num", + "512", + "--train_all", + "--fix", + "gpt", + "--gpt-ckpt", + "/tmp/haozhezhao/MLLMG/checkpoint/BLIP2Trainall_just_segment_pretrain_stage3_dreambench_recap_Subject200k_filtered_t2i_flux400k_200kmid_recovery_150k_extract_150k_100_fluxseg_50samseg_trainall_1e4/000-GPT-XL/checkpoints/0010000.pt", + "--load_subject_embedding", + "/tmp/haozhezhao/MLLMG/subject_embedding.bin" + ], + "program": "/tmp/haozhezhao/MLLMG/autoregressive/train/train_t2i.py", + "codePath": "autoregressive/train/train_t2i.py", + "email": "mimazhe55360@gmail.com", + "root": "checkpoint/BeforeFix_Blip_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_objects", + "host": "447cc403a8794092814259713c51c1df00001X", + "executable": "/tmp/haozhezhao/anaconda3/envs/nlp/bin/python", + "codePathLocal": "autoregressive/train/train_t2i.py", + "cpu_count": 96, + "cpu_count_logical": 96, + "gpu": "NVIDIA A100-SXM4-80GB", + "gpu_count": 8, + "disk": { + "/": { + "total": "133003395072", + "used": "65441980416" + } + }, + "memory": { + "total": "1902387884032" + }, + "cpu": { + "count": 96, + "countLogical": 96 + }, + "gpu_nvidia": [ + { + "name": "NVIDIA A100-SXM4-80GB", + "memoryTotal": "85899345920", + "cudaCores": 6912, + "architecture": "Ampere" + }, + { + "name": "NVIDIA A100-SXM4-80GB", + "memoryTotal": "85899345920", + "cudaCores": 6912, + "architecture": "Ampere" + }, + { + "name": "NVIDIA A100-SXM4-80GB", + "memoryTotal": "85899345920", + "cudaCores": 6912, + "architecture": "Ampere" + }, + { + "name": "NVIDIA A100-SXM4-80GB", + "memoryTotal": "85899345920", + "cudaCores": 6912, + "architecture": "Ampere" + }, + { + "name": "NVIDIA A100-SXM4-80GB", + "memoryTotal": "85899345920", + "cudaCores": 6912, + "architecture": "Ampere" + }, + { + "name": "NVIDIA A100-SXM4-80GB", + "memoryTotal": "85899345920", + "cudaCores": 6912, + "architecture": "Ampere" + }, + { + "name": "NVIDIA A100-SXM4-80GB", + "memoryTotal": "85899345920", + "cudaCores": 6912, + "architecture": "Ampere" + }, + { + "name": "NVIDIA A100-SXM4-80GB", + "memoryTotal": "85899345920", + "cudaCores": 6912, + "architecture": "Ampere" + } + ], + "cudaVersion": "12.2" +} \ No newline at end of file diff --git a/CKPTS/BeforeFix_Blip_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_objects/wandb/run-20250422_014729-ni4jp6ul/logs/debug-core.log b/CKPTS/BeforeFix_Blip_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_objects/wandb/run-20250422_014729-ni4jp6ul/logs/debug-core.log new file mode 100644 index 0000000000000000000000000000000000000000..26ac6760ff4c4f7c929747f7e61a3f98486f771c --- /dev/null +++ b/CKPTS/BeforeFix_Blip_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_objects/wandb/run-20250422_014729-ni4jp6ul/logs/debug-core.log @@ -0,0 +1,7 @@ +{"time":"2025-04-22T01:47:28.811477845Z","level":"INFO","msg":"main: starting server","port-filename":"/tmp/tmpi3vn9aea/port-2951284.txt","pid":2951284,"log-level":0,"disable-analytics":false,"shutdown-on-parent-exit":false} +{"time":"2025-04-22T01:47:28.812995645Z","level":"INFO","msg":"Will exit if parent process dies.","ppid":2951284} +{"time":"2025-04-22T01:47:28.812931806Z","level":"INFO","msg":"server is running","addr":{"IP":"127.0.0.1","Port":38103,"Zone":""}} +{"time":"2025-04-22T01:47:28.99757108Z","level":"INFO","msg":"connection: ManageConnectionData: new connection created","id":"127.0.0.1:39098"} +{"time":"2025-04-22T01:47:29.361639272Z","level":"INFO","msg":"handleInformInit: received","streamId":"ni4jp6ul","id":"127.0.0.1:39098"} +{"time":"2025-04-22T01:47:29.467986515Z","level":"INFO","msg":"handleInformInit: stream started","streamId":"ni4jp6ul","id":"127.0.0.1:39098"} +{"time":"2025-04-23T08:32:32.553655939Z","level":"INFO","msg":"Parent process exited, terminating service process."} diff --git a/CKPTS/BeforeFix_Blip_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_objects/wandb/run-20250422_014729-ni4jp6ul/logs/debug-internal.log b/CKPTS/BeforeFix_Blip_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_objects/wandb/run-20250422_014729-ni4jp6ul/logs/debug-internal.log new file mode 100644 index 0000000000000000000000000000000000000000..eb9cc17862e5b3621c9639b15dd2c9d774e83cc0 --- /dev/null +++ b/CKPTS/BeforeFix_Blip_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_objects/wandb/run-20250422_014729-ni4jp6ul/logs/debug-internal.log @@ -0,0 +1,9 @@ +{"time":"2025-04-22T01:47:29.361870194Z","level":"INFO","msg":"stream: starting","core version":"0.19.8","symlink path":"checkpoint/BeforeFix_Blip_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_objects/wandb/run-20250422_014729-ni4jp6ul/logs/debug-core.log"} +{"time":"2025-04-22T01:47:29.467936381Z","level":"INFO","msg":"created new stream","id":"ni4jp6ul"} +{"time":"2025-04-22T01:47:29.467981215Z","level":"INFO","msg":"stream: started","id":"ni4jp6ul"} +{"time":"2025-04-22T01:47:29.46801618Z","level":"INFO","msg":"writer: Do: started","stream_id":"ni4jp6ul"} +{"time":"2025-04-22T01:47:29.468021706Z","level":"INFO","msg":"sender: started","stream_id":"ni4jp6ul"} +{"time":"2025-04-22T01:47:29.468072912Z","level":"INFO","msg":"handler: started","stream_id":"ni4jp6ul"} +{"time":"2025-04-22T01:47:29.833858878Z","level":"INFO","msg":"Starting system monitor"} +{"time":"2025-04-22T18:15:28.577913461Z","level":"INFO","msg":"api: retrying error","error":"Post \"https://api.wandb.ai/graphql\": net/http: request canceled (Client.Timeout exceeded while awaiting headers)"} +{"time":"2025-04-22T21:04:58.809243773Z","level":"INFO","msg":"api: retrying error","error":"Post \"https://api.wandb.ai/graphql\": context deadline exceeded (Client.Timeout exceeded while awaiting headers)"} diff --git a/CKPTS/BeforeFix_Blip_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_objects/wandb/run-20250422_014729-ni4jp6ul/logs/debug.log b/CKPTS/BeforeFix_Blip_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_objects/wandb/run-20250422_014729-ni4jp6ul/logs/debug.log new file mode 100644 index 0000000000000000000000000000000000000000..528f8c3ee609095481f5914449e01f2ac296d6e6 --- /dev/null +++ b/CKPTS/BeforeFix_Blip_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_objects/wandb/run-20250422_014729-ni4jp6ul/logs/debug.log @@ -0,0 +1,22 @@ +2025-04-22 01:47:29,354 INFO MainThread:2951284 [wandb_setup.py:_flush():67] Current SDK version is 0.19.8 +2025-04-22 01:47:29,354 INFO MainThread:2951284 [wandb_setup.py:_flush():67] Configure stats pid to 2951284 +2025-04-22 01:47:29,354 INFO MainThread:2951284 [wandb_setup.py:_flush():67] Loading settings from /tmp/haozhezhao/.config/wandb/settings +2025-04-22 01:47:29,354 INFO MainThread:2951284 [wandb_setup.py:_flush():67] Loading settings from /tmp/haozhezhao/MLLMG/wandb/settings +2025-04-22 01:47:29,355 INFO MainThread:2951284 [wandb_setup.py:_flush():67] Loading settings from environment variables +2025-04-22 01:47:29,355 INFO MainThread:2951284 [wandb_init.py:setup_run_log_directory():647] Logging user logs to checkpoint/BeforeFix_Blip_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_objects/wandb/run-20250422_014729-ni4jp6ul/logs/debug.log +2025-04-22 01:47:29,355 INFO MainThread:2951284 [wandb_init.py:setup_run_log_directory():648] Logging internal logs to checkpoint/BeforeFix_Blip_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_objects/wandb/run-20250422_014729-ni4jp6ul/logs/debug-internal.log +2025-04-22 01:47:29,355 INFO MainThread:2951284 [wandb_init.py:init():761] calling init triggers +2025-04-22 01:47:29,355 INFO MainThread:2951284 [wandb_init.py:init():766] wandb.init called with sweep_config: {} +config: {'data_path': '/tmp/haozhezhao/MLLMG/jsonl_data/multiobjects_molom_imagenet_flux_qwen_midsource_gen_2_2m_trained.jsonl', 'cloud_save_path': '/tmp/haozhezhao/MLLMG/checkpoint', 'no_local_save': False, 'vq_model': 'VQ-16', 'vq_ckpt': '/tmp/haozhezhao/model/llamagen_t2i/vq_ds16_t2i.pt', 'codebook_size': 16384, 'codebook_embed_dim': 8, 'gpt_model': 'GPT-XL', 'gpt_ckpt': '/tmp/haozhezhao/MLLMG/checkpoint/BLIP2Trainall_just_segment_pretrain_stage3_dreambench_recap_Subject200k_filtered_t2i_flux400k_200kmid_recovery_150k_extract_150k_100_fluxseg_50samseg_trainall_1e4/000-GPT-XL/checkpoints/0010000.pt', 'gpt_type': 't2i', 'vocab_size': 16384, 'cls_token_num': 512, 'dropout_p': 0.1, 'token_dropout_p': 0.1, 'drop_path': 0.0, 'no_compile': False, 'results_dir': 'checkpoint/BeforeFix_Blip_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_objects', 'dataset': 'ti2i', 'image_size': 512, 'downsample_size': 16, 'num_classes': 1000, 'epochs': 2, 'lr': 5e-05, 'weight_decay': 0.05, 'beta1': 0.9, 'beta2': 0.95, 'max_grad_norm': 1.0, 'global_batch_size': 56, 'global_seed': 0, 'num_workers': 4, 'log_every': 25, 'ckpt_every': 2000, 'gradient_accumulation_steps': 4, 'mixed_precision': 'bf16', 'val_data_path': '/tmp/haozhezhao/MLLMG/jsonl_data/multiobjects_molom_imagenet_flux_qwen_midsource_gen_val.jsonl', 'use_vision_tower': True, 'model_name_or_path': '/tmp/haozhezhao/model/instructblip-flan-t5-xl', 'image_place_holder': '', 'processor_path': None, 'do_eval': True, 'max_eval_samples': 200, 'train_text_encoder': True, 'no_left_padding': False, 'cfg_scale': 7.5, 'top_k': 16384, 'temperature': 0.9, 'top_p': 1.0, 'eval_steps': 2000, 'project_name': 'llamagen_ti2i', 'load_from_checkpoint': '/tmp/haozhezhao/model/llamagen_t2i/t2i_XL_stage2_512.pt', 'warmup': 0.05, 'lr_decay_style': 'cosine', 'lr_decay_ratio': 0.1, 'train_iters': 500000, 'class_dropout_prob': 0.1, 'with_image_only': False, 'image_only_rate': 0.1, 'stage2': False, 'subject_driven': True, 'load_subject_embedding': '/tmp/haozhezhao/MLLMG/subject_embedding.bin', 'reference_data_path': '/tmp/haozhezhao/MLLMG/cc12m_reference_tunnel.jsonl', 'multimodal_encoder': 'blip', 'do_recovery': True, 'no_replace': False, 'resume': False, 'dreambench_eval': False, 'find_unused_parameters': True, 'load_visual_encoder': False, 'continue_stage1': False, 'replace_subject': False, 'train_all': True, 'save_total_limit': 2, 'load_language_projection': None, 'mm_vision_tower': 'openai/clip-vit-large-patch14', 'load_fixed_llamagen': False, 'unfreeze_output': False, 'fix': 'gpt', 'rank': 0, 'world_size': 8, 'gpu': 0, 'dist_url': 'env://', 'distributed': True, 'dist_backend': 'nccl', '_wandb': {}} +2025-04-22 01:47:29,355 INFO MainThread:2951284 [wandb_init.py:init():784] starting backend +2025-04-22 01:47:29,355 INFO MainThread:2951284 [wandb_init.py:init():788] sending inform_init request +2025-04-22 01:47:29,361 INFO MainThread:2951284 [backend.py:_multiprocessing_setup():101] multiprocessing start_methods=fork,spawn,forkserver, using: spawn +2025-04-22 01:47:29,361 INFO MainThread:2951284 [wandb_init.py:init():798] backend started and connected +2025-04-22 01:47:29,363 INFO MainThread:2951284 [wandb_init.py:init():891] updated telemetry +2025-04-22 01:47:29,364 INFO MainThread:2951284 [wandb_init.py:init():915] communicating run to backend with 90.0 second timeout +2025-04-22 01:47:29,831 INFO MainThread:2951284 [wandb_init.py:init():990] starting run threads in backend +2025-04-22 01:47:29,917 INFO MainThread:2951284 [wandb_run.py:_console_start():2375] atexit reg +2025-04-22 01:47:29,917 INFO MainThread:2951284 [wandb_run.py:_redirect():2227] redirect: wrap_raw +2025-04-22 01:47:29,918 INFO MainThread:2951284 [wandb_run.py:_redirect():2292] Wrapping output streams. +2025-04-22 01:47:29,918 INFO MainThread:2951284 [wandb_run.py:_redirect():2315] Redirects installed. +2025-04-22 01:47:29,919 INFO MainThread:2951284 [wandb_init.py:init():1032] run started, returning control to user process diff --git a/CKPTS/BeforeFix_Blip_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_objects/wandb/run-20250422_014729-ni4jp6ul/run-ni4jp6ul.wandb b/CKPTS/BeforeFix_Blip_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_objects/wandb/run-20250422_014729-ni4jp6ul/run-ni4jp6ul.wandb new file mode 100644 index 0000000000000000000000000000000000000000..b19618d6662bdebfc90c43da60f0398e3942203a --- /dev/null +++ b/CKPTS/BeforeFix_Blip_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_objects/wandb/run-20250422_014729-ni4jp6ul/run-ni4jp6ul.wandb @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:0f1c05d589d418097ae9ad52de71e217eb22315623c5caebd1086fecf24e7540 +size 59801600 diff --git a/CKPTS/EmptyFix_InstructBlip_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_objects/000-GPT-XL/log.txt b/CKPTS/EmptyFix_InstructBlip_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_objects/000-GPT-XL/log.txt new file mode 100644 index 0000000000000000000000000000000000000000..b06a7d47a9f27fc0acfeda73fcd7feb3ebf9db77 --- /dev/null +++ b/CKPTS/EmptyFix_InstructBlip_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_objects/000-GPT-XL/log.txt @@ -0,0 +1,18 @@ +[2025-04-20 16:35:47] Experiment directory created at checkpoint/EmptyFix_InstructBlip_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_objects/000-GPT-XL +[2025-04-20 16:35:47] Namespace(data_path='/tmp/haozhezhao/MLLMG/jsonl_data/multiobjects_molom_imagenet_flux_qwen_midsource_gen_2_2m_trained.jsonl', cloud_save_path='/tmp/haozhezhao/MLLMG/checkpoint', no_local_save=False, vq_model='VQ-16', vq_ckpt='/tmp/haozhezhao/model/llamagen_t2i/vq_ds16_t2i.pt', codebook_size=16384, codebook_embed_dim=8, gpt_model='GPT-XL', gpt_ckpt='/tmp/haozhezhao/MLLMG/checkpoint/EmptyFix_InstructBlip_just_segment_pretrain_stage3_dreambench_recap_Subject400k_filtered_t2i_flux400k_200kmid_recovery_150k_extract_150k_100_fluxseg_50samseg_trainall_1e4_no_replace/007-GPT-XL/checkpoints/0092000.pt', gpt_type='t2i', vocab_size=16384, cls_token_num=512, dropout_p=0.1, token_dropout_p=0.1, drop_path=0.0, no_compile=False, results_dir='checkpoint/EmptyFix_InstructBlip_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_objects', dataset='ti2i', image_size=512, downsample_size=16, num_classes=1000, epochs=10, lr=5e-05, weight_decay=0.05, beta1=0.9, beta2=0.95, max_grad_norm=1.0, global_batch_size=56, global_seed=0, num_workers=4, log_every=25, ckpt_every=2000, gradient_accumulation_steps=4, mixed_precision='bf16', val_data_path='/tmp/haozhezhao/MLLMG/jsonl_data/multiobjects_molom_imagenet_flux_qwen_midsource_gen_val.jsonl', use_vision_tower=True, model_name_or_path='/tmp/haozhezhao/model/instructblip-flan-t5-xl', image_place_holder='', processor_path=None, do_eval=True, max_eval_samples=200, train_text_encoder=True, no_left_padding=False, cfg_scale=7.5, top_k=16384, temperature=0.9, top_p=1.0, eval_steps=2000, project_name='llamagen_ti2i', load_from_checkpoint='/tmp/haozhezhao/MLLMG/checkpoint/FIXed_3M_1epoch_step32000.pt', warmup=0.05, lr_decay_style='cosine', lr_decay_ratio=0.1, train_iters=500000, class_dropout_prob=0.1, with_image_only=False, image_only_rate=0.1, stage2=False, subject_driven=True, load_subject_embedding=None, reference_data_path='/tmp/haozhezhao/MLLMG/cc12m_reference_tunnel.jsonl', multimodal_encoder='instructblip', do_recovery=True, no_replace=False, resume=False, dreambench_eval=False, find_unused_parameters=True, load_visual_encoder=False, continue_stage1=False, replace_subject=False, train_all=True, save_total_limit=2, load_language_projection=None, mm_vision_tower='openai/clip-vit-large-patch14', load_fixed_llamagen=True, unfreeze_output=False, fix='gpt-empty-fix', rank=0, world_size=8, gpu=0, dist_url='env://', distributed=True, dist_backend='nccl') +[2025-04-20 16:35:47] Starting rank=0, seed=0, world_size=8. +[2025-04-20 16:35:47] vision_config is None. initializing the InstructBlipVisionConfig with default values. +[2025-04-20 16:35:47] qformer_config is None. Initializing the InstructBlipQFormerConfig with default values. +[2025-04-20 16:35:47] text_config is None. Initializing the text config with default values (`OPTConfig`). +[2025-04-20 16:37:27] GPT Parameters: 3,174,417,408 +[2025-04-20 16:37:27] num decayed parameter tensors: 637, with 3,173,270,016 parameters +[2025-04-20 16:37:27] num non-decayed parameter tensors: 644, with 1,147,392 parameters +[2025-04-20 16:37:27] using fused AdamW: True +[2025-04-20 16:37:56] Dataset contains 2,290,583 images +[2025-04-20 16:37:56] Train iters 409030 , warmup 20451.5, len of loader 40903 +[2025-04-20 16:38:24] ### LOAD pretraining weights from checkpoint: /tmp/haozhezhao/MLLMG/checkpoint/EmptyFix_InstructBlip_just_segment_pretrain_stage3_dreambench_recap_Subject400k_filtered_t2i_flux400k_200kmid_recovery_150k_extract_150k_100_fluxseg_50samseg_trainall_1e4_no_replace/007-GPT-XL/checkpoints/0092000.pt +[2025-04-20 16:38:24] Initial state: steps=0, epochs=0 +[2025-04-20 16:38:24] compiling the model... (may take several minutes) +[2025-04-20 16:38:25] freeze the vit +[2025-04-20 16:38:25] ***** total param is 3174417408 ***** +[2025-04-20 16:38:25] ***** total trained param is 2188465152 ***** diff --git a/CKPTS/EmptyFix_InstructBlip_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_objects/001-GPT-XL/log.txt b/CKPTS/EmptyFix_InstructBlip_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_objects/001-GPT-XL/log.txt new file mode 100644 index 0000000000000000000000000000000000000000..809d85463c19d051dd46ac61b13d58933c170168 --- /dev/null +++ b/CKPTS/EmptyFix_InstructBlip_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_objects/001-GPT-XL/log.txt @@ -0,0 +1,18 @@ +[2025-04-20 16:38:49] Experiment directory created at checkpoint/EmptyFix_InstructBlip_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_objects/001-GPT-XL +[2025-04-20 16:38:49] Namespace(data_path='/tmp/haozhezhao/MLLMG/jsonl_data/multiobjects_molom_imagenet_flux_qwen_midsource_gen_2_2m_trained.jsonl', cloud_save_path='/tmp/haozhezhao/MLLMG/checkpoint', no_local_save=False, vq_model='VQ-16', vq_ckpt='/tmp/haozhezhao/model/llamagen_t2i/vq_ds16_t2i.pt', codebook_size=16384, codebook_embed_dim=8, gpt_model='GPT-XL', gpt_ckpt='/tmp/haozhezhao/MLLMG/checkpoint/EmptyFix_InstructBlip_just_segment_pretrain_stage3_dreambench_recap_Subject400k_filtered_t2i_flux400k_200kmid_recovery_150k_extract_150k_100_fluxseg_50samseg_trainall_1e4_no_replace/007-GPT-XL/checkpoints/0092000.pt', gpt_type='t2i', vocab_size=16384, cls_token_num=512, dropout_p=0.1, token_dropout_p=0.1, drop_path=0.0, no_compile=False, results_dir='checkpoint/EmptyFix_InstructBlip_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_objects', dataset='ti2i', image_size=512, downsample_size=16, num_classes=1000, epochs=2, lr=5e-05, weight_decay=0.05, beta1=0.9, beta2=0.95, max_grad_norm=1.0, global_batch_size=56, global_seed=0, num_workers=4, log_every=25, ckpt_every=2000, gradient_accumulation_steps=4, mixed_precision='bf16', val_data_path='/tmp/haozhezhao/MLLMG/jsonl_data/multiobjects_molom_imagenet_flux_qwen_midsource_gen_val.jsonl', use_vision_tower=True, model_name_or_path='/tmp/haozhezhao/model/instructblip-flan-t5-xl', image_place_holder='', processor_path=None, do_eval=True, max_eval_samples=200, train_text_encoder=True, no_left_padding=False, cfg_scale=7.5, top_k=16384, temperature=0.9, top_p=1.0, eval_steps=2000, project_name='llamagen_ti2i', load_from_checkpoint='/tmp/haozhezhao/MLLMG/checkpoint/FIXed_3M_1epoch_step32000.pt', warmup=0.05, lr_decay_style='cosine', lr_decay_ratio=0.1, train_iters=500000, class_dropout_prob=0.1, with_image_only=False, image_only_rate=0.1, stage2=False, subject_driven=True, load_subject_embedding=None, reference_data_path='/tmp/haozhezhao/MLLMG/cc12m_reference_tunnel.jsonl', multimodal_encoder='instructblip', do_recovery=True, no_replace=False, resume=False, dreambench_eval=False, find_unused_parameters=True, load_visual_encoder=False, continue_stage1=False, replace_subject=False, train_all=True, save_total_limit=2, load_language_projection=None, mm_vision_tower='openai/clip-vit-large-patch14', load_fixed_llamagen=True, unfreeze_output=False, fix='gpt-empty-fix', rank=0, world_size=8, gpu=0, dist_url='env://', distributed=True, dist_backend='nccl') +[2025-04-20 16:38:49] Starting rank=0, seed=0, world_size=8. +[2025-04-20 16:38:49] vision_config is None. initializing the InstructBlipVisionConfig with default values. +[2025-04-20 16:38:49] qformer_config is None. Initializing the InstructBlipQFormerConfig with default values. +[2025-04-20 16:38:49] text_config is None. Initializing the text config with default values (`OPTConfig`). +[2025-04-20 16:40:29] GPT Parameters: 3,174,417,408 +[2025-04-20 16:40:29] num decayed parameter tensors: 637, with 3,173,270,016 parameters +[2025-04-20 16:40:29] num non-decayed parameter tensors: 644, with 1,147,392 parameters +[2025-04-20 16:40:29] using fused AdamW: True +[2025-04-20 16:40:57] Dataset contains 2,290,583 images +[2025-04-20 16:40:57] Train iters 81806 , warmup 4090.3, len of loader 40903 +[2025-04-20 16:41:17] ### LOAD pretraining weights from checkpoint: /tmp/haozhezhao/MLLMG/checkpoint/EmptyFix_InstructBlip_just_segment_pretrain_stage3_dreambench_recap_Subject400k_filtered_t2i_flux400k_200kmid_recovery_150k_extract_150k_100_fluxseg_50samseg_trainall_1e4_no_replace/007-GPT-XL/checkpoints/0092000.pt +[2025-04-20 16:41:17] Initial state: steps=0, epochs=0 +[2025-04-20 16:41:17] compiling the model... (may take several minutes) +[2025-04-20 16:41:18] freeze the vit +[2025-04-20 16:41:18] ***** total param is 3174417408 ***** +[2025-04-20 16:41:18] ***** total trained param is 2188465152 ***** diff --git a/CKPTS/EmptyFix_InstructBlip_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_objects/002-GPT-XL/checkpoints/0058000.pt b/CKPTS/EmptyFix_InstructBlip_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_objects/002-GPT-XL/checkpoints/0058000.pt new file mode 100644 index 0000000000000000000000000000000000000000..876b822baeee21cdc15eab4985767715a2ddac6a --- /dev/null +++ b/CKPTS/EmptyFix_InstructBlip_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_objects/002-GPT-XL/checkpoints/0058000.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:47fc8c2d030779a259a36288b87ca78c96a0b56237fce3ec0d618796ccf541d0 +size 19774276703 diff --git a/CKPTS/EmptyFix_InstructBlip_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_objects/002-GPT-XL/checkpoints/eval_step_10000/batch_0_cfg_7.5_topk_16384.jpg b/CKPTS/EmptyFix_InstructBlip_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_objects/002-GPT-XL/checkpoints/eval_step_10000/batch_0_cfg_7.5_topk_16384.jpg new file mode 100644 index 0000000000000000000000000000000000000000..4730c816695da500bdf3ebccfd5392fd7c85dc50 --- /dev/null +++ b/CKPTS/EmptyFix_InstructBlip_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_objects/002-GPT-XL/checkpoints/eval_step_10000/batch_0_cfg_7.5_topk_16384.jpg @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:970ea5bde39f22e514eb1a88d7278f947891e0d912213f59be653aa3c4ffc0f7 +size 1133988 diff --git a/CKPTS/EmptyFix_InstructBlip_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_objects/002-GPT-XL/checkpoints/eval_step_10000/batch_1_cfg_7.5_topk_16384.jpg b/CKPTS/EmptyFix_InstructBlip_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_objects/002-GPT-XL/checkpoints/eval_step_10000/batch_1_cfg_7.5_topk_16384.jpg new file mode 100644 index 0000000000000000000000000000000000000000..c26098b24bc387d224123bd370c014322aca4627 --- /dev/null +++ b/CKPTS/EmptyFix_InstructBlip_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_objects/002-GPT-XL/checkpoints/eval_step_10000/batch_1_cfg_7.5_topk_16384.jpg @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:2fe1e3744feadb591fd226209560c34a61f3c31ae3a024a38fc3fe1f5c8f1e4e +size 988871 diff --git a/CKPTS/EmptyFix_InstructBlip_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_objects/002-GPT-XL/checkpoints/eval_step_10000/batch_2_cfg_7.5_topk_16384.jpg b/CKPTS/EmptyFix_InstructBlip_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_objects/002-GPT-XL/checkpoints/eval_step_10000/batch_2_cfg_7.5_topk_16384.jpg new file mode 100644 index 0000000000000000000000000000000000000000..5ccd89ed05ebed47ccdfe46e62a4417163dd69f9 --- /dev/null +++ b/CKPTS/EmptyFix_InstructBlip_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_objects/002-GPT-XL/checkpoints/eval_step_10000/batch_2_cfg_7.5_topk_16384.jpg @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:e9243e73a47417edadf25284fbe7cf2aa7eef4915b76491be019e54ce4e0fcca +size 982069 diff --git a/CKPTS/EmptyFix_InstructBlip_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_objects/002-GPT-XL/checkpoints/eval_step_12000/batch_0_cfg_7.5_topk_16384.jpg b/CKPTS/EmptyFix_InstructBlip_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_objects/002-GPT-XL/checkpoints/eval_step_12000/batch_0_cfg_7.5_topk_16384.jpg new file mode 100644 index 0000000000000000000000000000000000000000..23a712e24ea9b88cb3b9774bc512b456accd9e12 --- /dev/null +++ b/CKPTS/EmptyFix_InstructBlip_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_objects/002-GPT-XL/checkpoints/eval_step_12000/batch_0_cfg_7.5_topk_16384.jpg @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:d9e7ed98bd43b09e32a9e40a71f627ee317448a251c64abb1c24e1c6dd7087f5 +size 1130225 diff --git a/CKPTS/EmptyFix_InstructBlip_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_objects/002-GPT-XL/checkpoints/eval_step_12000/batch_1_cfg_7.5_topk_16384.jpg b/CKPTS/EmptyFix_InstructBlip_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_objects/002-GPT-XL/checkpoints/eval_step_12000/batch_1_cfg_7.5_topk_16384.jpg new file mode 100644 index 0000000000000000000000000000000000000000..ce158ef0f0c3fb2b02b7bd95f17adb35f2a36876 --- /dev/null +++ b/CKPTS/EmptyFix_InstructBlip_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_objects/002-GPT-XL/checkpoints/eval_step_12000/batch_1_cfg_7.5_topk_16384.jpg @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:8ce30a6db280387b7da90b6c361c2b07c6eecccc7e89476689789f8765f91f4d +size 985201 diff --git a/CKPTS/EmptyFix_InstructBlip_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_objects/002-GPT-XL/checkpoints/eval_step_12000/batch_2_cfg_7.5_topk_16384.jpg b/CKPTS/EmptyFix_InstructBlip_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_objects/002-GPT-XL/checkpoints/eval_step_12000/batch_2_cfg_7.5_topk_16384.jpg new file mode 100644 index 0000000000000000000000000000000000000000..82188922eb324376ff1ae56c838cf2bfb34e8b30 --- /dev/null +++ b/CKPTS/EmptyFix_InstructBlip_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_objects/002-GPT-XL/checkpoints/eval_step_12000/batch_2_cfg_7.5_topk_16384.jpg @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:cbb6ca5abc8ea0868e605f41b1e6032beb355cd81988ae53567cb5bc6b2d31e0 +size 976606 diff --git a/CKPTS/EmptyFix_InstructBlip_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_objects/002-GPT-XL/checkpoints/eval_step_14000/batch_0_cfg_7.5_topk_16384.jpg b/CKPTS/EmptyFix_InstructBlip_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_objects/002-GPT-XL/checkpoints/eval_step_14000/batch_0_cfg_7.5_topk_16384.jpg new file mode 100644 index 0000000000000000000000000000000000000000..47f4e4934991d390f125382f3dbe7aeaf27d3fc7 --- /dev/null +++ b/CKPTS/EmptyFix_InstructBlip_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_objects/002-GPT-XL/checkpoints/eval_step_14000/batch_0_cfg_7.5_topk_16384.jpg @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:7adbfdc51c2a128f09eca5bc92daf797bc0ba88fb45b26fb4ff52b2ede373a8e +size 1157241 diff --git a/CKPTS/EmptyFix_InstructBlip_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_objects/002-GPT-XL/checkpoints/eval_step_14000/batch_1_cfg_7.5_topk_16384.jpg b/CKPTS/EmptyFix_InstructBlip_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_objects/002-GPT-XL/checkpoints/eval_step_14000/batch_1_cfg_7.5_topk_16384.jpg new file mode 100644 index 0000000000000000000000000000000000000000..f3ebb6317e128e16ed74000261ae49cd6c79ebac --- /dev/null +++ b/CKPTS/EmptyFix_InstructBlip_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_objects/002-GPT-XL/checkpoints/eval_step_14000/batch_1_cfg_7.5_topk_16384.jpg @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:adceb83a5a0d11d6179a77378e04caa8fab2323b3c0aecf97793694e97aa5697 +size 1003205 diff --git a/CKPTS/EmptyFix_InstructBlip_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_objects/002-GPT-XL/checkpoints/eval_step_14000/batch_2_cfg_7.5_topk_16384.jpg b/CKPTS/EmptyFix_InstructBlip_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_objects/002-GPT-XL/checkpoints/eval_step_14000/batch_2_cfg_7.5_topk_16384.jpg new file mode 100644 index 0000000000000000000000000000000000000000..7e966af93a81302cf6175f2d9a293b88be8dc1c5 --- /dev/null +++ b/CKPTS/EmptyFix_InstructBlip_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_objects/002-GPT-XL/checkpoints/eval_step_14000/batch_2_cfg_7.5_topk_16384.jpg @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:2d38ef437f3f43a0de4caf4e867335df1e7a259d05cda26a1ae38ef8fbff8144 +size 991011 diff --git a/CKPTS/EmptyFix_InstructBlip_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_objects/002-GPT-XL/checkpoints/eval_step_16000/batch_0_cfg_7.5_topk_16384.jpg b/CKPTS/EmptyFix_InstructBlip_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_objects/002-GPT-XL/checkpoints/eval_step_16000/batch_0_cfg_7.5_topk_16384.jpg new file mode 100644 index 0000000000000000000000000000000000000000..fbc20546dda7301552495fc5c4b487a5423e290d --- /dev/null +++ b/CKPTS/EmptyFix_InstructBlip_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_objects/002-GPT-XL/checkpoints/eval_step_16000/batch_0_cfg_7.5_topk_16384.jpg @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:cc4ef2b4cf2a384127a88f656976348d98ab53eed6501d7a46e2e6aa32c0c356 +size 1118086 diff --git a/CKPTS/EmptyFix_InstructBlip_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_objects/002-GPT-XL/checkpoints/eval_step_16000/batch_1_cfg_7.5_topk_16384.jpg b/CKPTS/EmptyFix_InstructBlip_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_objects/002-GPT-XL/checkpoints/eval_step_16000/batch_1_cfg_7.5_topk_16384.jpg new file mode 100644 index 0000000000000000000000000000000000000000..365e7eb635423aeac459563123e7780805c36ff7 --- /dev/null +++ b/CKPTS/EmptyFix_InstructBlip_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_objects/002-GPT-XL/checkpoints/eval_step_16000/batch_1_cfg_7.5_topk_16384.jpg @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:da90fe5bae4c9424c26ac660791c0605835e5a65a1ce6960245161663006cd1e +size 962320 diff --git a/CKPTS/EmptyFix_InstructBlip_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_objects/002-GPT-XL/checkpoints/eval_step_16000/batch_2_cfg_7.5_topk_16384.jpg b/CKPTS/EmptyFix_InstructBlip_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_objects/002-GPT-XL/checkpoints/eval_step_16000/batch_2_cfg_7.5_topk_16384.jpg new file mode 100644 index 0000000000000000000000000000000000000000..a3b457ed2c389f07a291abe299a55c01af154fd9 --- /dev/null +++ b/CKPTS/EmptyFix_InstructBlip_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_objects/002-GPT-XL/checkpoints/eval_step_16000/batch_2_cfg_7.5_topk_16384.jpg @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:7d6c8b76a68fdca1a9e853ed71cc8ae919e326d38aa57730b76b1ed65a7bc380 +size 962632 diff --git a/CKPTS/EmptyFix_InstructBlip_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_objects/002-GPT-XL/checkpoints/eval_step_18000/batch_0_cfg_7.5_topk_16384.jpg b/CKPTS/EmptyFix_InstructBlip_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_objects/002-GPT-XL/checkpoints/eval_step_18000/batch_0_cfg_7.5_topk_16384.jpg new file mode 100644 index 0000000000000000000000000000000000000000..1c62ac730ee7e25a57d38955829c4e7af6780aa4 --- /dev/null +++ b/CKPTS/EmptyFix_InstructBlip_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_objects/002-GPT-XL/checkpoints/eval_step_18000/batch_0_cfg_7.5_topk_16384.jpg @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:b59401ab283385c9d7260b0fe2fd5160bc9ea9ee36ce34f102e2e2c473aa3431 +size 1150884 diff --git a/CKPTS/EmptyFix_InstructBlip_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_objects/002-GPT-XL/checkpoints/eval_step_18000/batch_1_cfg_7.5_topk_16384.jpg b/CKPTS/EmptyFix_InstructBlip_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_objects/002-GPT-XL/checkpoints/eval_step_18000/batch_1_cfg_7.5_topk_16384.jpg new file mode 100644 index 0000000000000000000000000000000000000000..1b27dbe8535b5f214aa56274daaae1393c73813f --- /dev/null +++ b/CKPTS/EmptyFix_InstructBlip_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_objects/002-GPT-XL/checkpoints/eval_step_18000/batch_1_cfg_7.5_topk_16384.jpg @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:d134a6531c9d5f6c5861158d586e31bf8cbbb0b739bb0bd0c180c83b2f25677c +size 970656 diff --git a/CKPTS/EmptyFix_InstructBlip_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_objects/002-GPT-XL/checkpoints/eval_step_18000/batch_2_cfg_7.5_topk_16384.jpg b/CKPTS/EmptyFix_InstructBlip_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_objects/002-GPT-XL/checkpoints/eval_step_18000/batch_2_cfg_7.5_topk_16384.jpg new file mode 100644 index 0000000000000000000000000000000000000000..1e189d5ba5e8b27590f2c94cc9eb0243be356c52 --- /dev/null +++ b/CKPTS/EmptyFix_InstructBlip_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_objects/002-GPT-XL/checkpoints/eval_step_18000/batch_2_cfg_7.5_topk_16384.jpg @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:8d2770ec4860b1b509daa08357fe69e3ef58eb232e4670af1298d181001b4be7 +size 952970 diff --git a/CKPTS/EmptyFix_InstructBlip_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_objects/002-GPT-XL/checkpoints/eval_step_2000/batch_0_cfg_7.5_topk_16384.jpg b/CKPTS/EmptyFix_InstructBlip_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_objects/002-GPT-XL/checkpoints/eval_step_2000/batch_0_cfg_7.5_topk_16384.jpg new file mode 100644 index 0000000000000000000000000000000000000000..0f1f1f54fa81d440abfa39b4910a92c4afb2e1a3 --- /dev/null +++ b/CKPTS/EmptyFix_InstructBlip_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_objects/002-GPT-XL/checkpoints/eval_step_2000/batch_0_cfg_7.5_topk_16384.jpg @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:1ca65e5412874f30af83d4a4b24b5d0933ad2b6a1cf6b086616c58bfaf70122e +size 1134677 diff --git a/CKPTS/EmptyFix_InstructBlip_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_objects/002-GPT-XL/checkpoints/eval_step_2000/batch_1_cfg_7.5_topk_16384.jpg b/CKPTS/EmptyFix_InstructBlip_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_objects/002-GPT-XL/checkpoints/eval_step_2000/batch_1_cfg_7.5_topk_16384.jpg new file mode 100644 index 0000000000000000000000000000000000000000..89c5fac401de33d2f2d5a5603701cbeaa1e7f9d0 --- /dev/null +++ b/CKPTS/EmptyFix_InstructBlip_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_objects/002-GPT-XL/checkpoints/eval_step_2000/batch_1_cfg_7.5_topk_16384.jpg @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:415784afb228f3d35cd55f0f6b350bb4239991b8d4200772b61e11aab303aef8 +size 987740 diff --git a/CKPTS/EmptyFix_InstructBlip_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_objects/002-GPT-XL/checkpoints/eval_step_2000/batch_2_cfg_7.5_topk_16384.jpg b/CKPTS/EmptyFix_InstructBlip_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_objects/002-GPT-XL/checkpoints/eval_step_2000/batch_2_cfg_7.5_topk_16384.jpg new file mode 100644 index 0000000000000000000000000000000000000000..19120dc481100033e274a43daff049030d19e2ff --- /dev/null +++ b/CKPTS/EmptyFix_InstructBlip_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_objects/002-GPT-XL/checkpoints/eval_step_2000/batch_2_cfg_7.5_topk_16384.jpg @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:77c3488450b997324e025521729568027174d73fb3ee8972883991b37b647090 +size 982168 diff --git a/CKPTS/EmptyFix_InstructBlip_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_objects/002-GPT-XL/checkpoints/eval_step_20000/batch_0_cfg_7.5_topk_16384.jpg b/CKPTS/EmptyFix_InstructBlip_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_objects/002-GPT-XL/checkpoints/eval_step_20000/batch_0_cfg_7.5_topk_16384.jpg new file mode 100644 index 0000000000000000000000000000000000000000..974755400c3ac86230139953b6d6ad2ab459d0c5 --- /dev/null +++ b/CKPTS/EmptyFix_InstructBlip_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_objects/002-GPT-XL/checkpoints/eval_step_20000/batch_0_cfg_7.5_topk_16384.jpg @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:e0dbd1c4bf238ec7324eb189772968eef4bc411b171a4c2d5cf3ced527e561c0 +size 1141346 diff --git a/CKPTS/EmptyFix_InstructBlip_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_objects/002-GPT-XL/checkpoints/eval_step_20000/batch_1_cfg_7.5_topk_16384.jpg b/CKPTS/EmptyFix_InstructBlip_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_objects/002-GPT-XL/checkpoints/eval_step_20000/batch_1_cfg_7.5_topk_16384.jpg new file mode 100644 index 0000000000000000000000000000000000000000..2e19769b031731c6e69ae7a844df1b1d5cc63eba --- /dev/null +++ b/CKPTS/EmptyFix_InstructBlip_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_objects/002-GPT-XL/checkpoints/eval_step_20000/batch_1_cfg_7.5_topk_16384.jpg @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:cfa0e98b9c1cde251f2a7843216f24dc9320447e66e4ec79e850f9a8c2f83a85 +size 965765 diff --git a/CKPTS/EmptyFix_InstructBlip_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_objects/002-GPT-XL/checkpoints/eval_step_20000/batch_2_cfg_7.5_topk_16384.jpg b/CKPTS/EmptyFix_InstructBlip_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_objects/002-GPT-XL/checkpoints/eval_step_20000/batch_2_cfg_7.5_topk_16384.jpg new file mode 100644 index 0000000000000000000000000000000000000000..7c3010fc2e840f52c2dab33b291349ad0a4eb54b --- /dev/null +++ b/CKPTS/EmptyFix_InstructBlip_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_objects/002-GPT-XL/checkpoints/eval_step_20000/batch_2_cfg_7.5_topk_16384.jpg @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:493deb90bf0beb0b056fdebff873dfd2250650931131a2aafffcfb9d684cb674 +size 995829 diff --git a/CKPTS/EmptyFix_InstructBlip_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_objects/002-GPT-XL/checkpoints/eval_step_22000/batch_0_cfg_7.5_topk_16384.jpg b/CKPTS/EmptyFix_InstructBlip_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_objects/002-GPT-XL/checkpoints/eval_step_22000/batch_0_cfg_7.5_topk_16384.jpg new file mode 100644 index 0000000000000000000000000000000000000000..db9c36067839d7fb5df4ef67a29ef7a1e5d74a27 --- /dev/null +++ b/CKPTS/EmptyFix_InstructBlip_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_objects/002-GPT-XL/checkpoints/eval_step_22000/batch_0_cfg_7.5_topk_16384.jpg @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:fc016de987d341bf7b6642891b7f523ee4e014ac77b47b1bbbc9f9606ff512e7 +size 1136132 diff --git a/CKPTS/EmptyFix_InstructBlip_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_objects/002-GPT-XL/checkpoints/eval_step_22000/batch_1_cfg_7.5_topk_16384.jpg b/CKPTS/EmptyFix_InstructBlip_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_objects/002-GPT-XL/checkpoints/eval_step_22000/batch_1_cfg_7.5_topk_16384.jpg new file mode 100644 index 0000000000000000000000000000000000000000..a4377cdbc284805d53a81b809ce47996cbd01596 --- /dev/null +++ b/CKPTS/EmptyFix_InstructBlip_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_objects/002-GPT-XL/checkpoints/eval_step_22000/batch_1_cfg_7.5_topk_16384.jpg @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:5be4be1ccbe34bbdd6b54f038db30657503f9068a3cd87bf6a77b587012b0173 +size 998561 diff --git a/CKPTS/EmptyFix_InstructBlip_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_objects/002-GPT-XL/checkpoints/eval_step_22000/batch_2_cfg_7.5_topk_16384.jpg b/CKPTS/EmptyFix_InstructBlip_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_objects/002-GPT-XL/checkpoints/eval_step_22000/batch_2_cfg_7.5_topk_16384.jpg new file mode 100644 index 0000000000000000000000000000000000000000..154cd1c66dea99a0826a5d22c55a23d327ff5d9e --- /dev/null +++ b/CKPTS/EmptyFix_InstructBlip_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_objects/002-GPT-XL/checkpoints/eval_step_22000/batch_2_cfg_7.5_topk_16384.jpg @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:4707375a5d69b61f2ca9e5ec46bbae894ba690dcf3cabac36778fa9a2762e7b2 +size 936880 diff --git a/CKPTS/EmptyFix_InstructBlip_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_objects/002-GPT-XL/checkpoints/eval_step_24000/batch_0_cfg_7.5_topk_16384.jpg b/CKPTS/EmptyFix_InstructBlip_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_objects/002-GPT-XL/checkpoints/eval_step_24000/batch_0_cfg_7.5_topk_16384.jpg new file mode 100644 index 0000000000000000000000000000000000000000..8404ebe0cd12165519e19354974f21354a151ee7 --- /dev/null +++ b/CKPTS/EmptyFix_InstructBlip_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_objects/002-GPT-XL/checkpoints/eval_step_24000/batch_0_cfg_7.5_topk_16384.jpg @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:b7e7e91d6438d70159d9b903bbd643a605c8e3f505a3d1cf07416964179d074a +size 1143685 diff --git a/CKPTS/EmptyFix_InstructBlip_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_objects/002-GPT-XL/checkpoints/eval_step_24000/batch_1_cfg_7.5_topk_16384.jpg b/CKPTS/EmptyFix_InstructBlip_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_objects/002-GPT-XL/checkpoints/eval_step_24000/batch_1_cfg_7.5_topk_16384.jpg new file mode 100644 index 0000000000000000000000000000000000000000..826b1b2ce1e86127c6cbd2e098fc5ad74002e457 --- /dev/null +++ b/CKPTS/EmptyFix_InstructBlip_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_objects/002-GPT-XL/checkpoints/eval_step_24000/batch_1_cfg_7.5_topk_16384.jpg @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:697e086f8950b9535cd2c5037967e9d1e222942fc53ae4a674a8bd818f363f83 +size 977795 diff --git a/CKPTS/EmptyFix_InstructBlip_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_objects/002-GPT-XL/checkpoints/eval_step_24000/batch_2_cfg_7.5_topk_16384.jpg b/CKPTS/EmptyFix_InstructBlip_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_objects/002-GPT-XL/checkpoints/eval_step_24000/batch_2_cfg_7.5_topk_16384.jpg new file mode 100644 index 0000000000000000000000000000000000000000..5b1de1a6ede644e87de056f3170f3341bc4e525e --- /dev/null +++ b/CKPTS/EmptyFix_InstructBlip_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_objects/002-GPT-XL/checkpoints/eval_step_24000/batch_2_cfg_7.5_topk_16384.jpg @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:3aea784680e405cf3f88815c8ac76c86d18627c4ee52b5823ae6bb8e281f15f8 +size 962536 diff --git a/CKPTS/EmptyFix_InstructBlip_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_objects/002-GPT-XL/checkpoints/eval_step_26000/batch_0_cfg_7.5_topk_16384.jpg b/CKPTS/EmptyFix_InstructBlip_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_objects/002-GPT-XL/checkpoints/eval_step_26000/batch_0_cfg_7.5_topk_16384.jpg new file mode 100644 index 0000000000000000000000000000000000000000..92073ac69eb8c442db14ad80d5067c2ffb719453 --- /dev/null +++ b/CKPTS/EmptyFix_InstructBlip_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_objects/002-GPT-XL/checkpoints/eval_step_26000/batch_0_cfg_7.5_topk_16384.jpg @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:c6596b9fc2704e6e0619ca959b16c2f4d1a3e96a07f6838c82f61ce41259c736 +size 1122258 diff --git a/CKPTS/EmptyFix_InstructBlip_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_objects/002-GPT-XL/checkpoints/eval_step_26000/batch_1_cfg_7.5_topk_16384.jpg b/CKPTS/EmptyFix_InstructBlip_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_objects/002-GPT-XL/checkpoints/eval_step_26000/batch_1_cfg_7.5_topk_16384.jpg new file mode 100644 index 0000000000000000000000000000000000000000..2e8ab549b1403235253f665e8a7f7b567a687dda --- /dev/null +++ b/CKPTS/EmptyFix_InstructBlip_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_objects/002-GPT-XL/checkpoints/eval_step_26000/batch_1_cfg_7.5_topk_16384.jpg @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:239875b9642222862982066ebcc0ff0bf93824c72ba6dad4dc7e8226b4f4f17e +size 957455 diff --git a/CKPTS/EmptyFix_InstructBlip_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_objects/002-GPT-XL/checkpoints/eval_step_26000/batch_2_cfg_7.5_topk_16384.jpg b/CKPTS/EmptyFix_InstructBlip_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_objects/002-GPT-XL/checkpoints/eval_step_26000/batch_2_cfg_7.5_topk_16384.jpg new file mode 100644 index 0000000000000000000000000000000000000000..97482e18c3f2f905adec55e81be13a59c44b1590 --- /dev/null +++ b/CKPTS/EmptyFix_InstructBlip_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_objects/002-GPT-XL/checkpoints/eval_step_26000/batch_2_cfg_7.5_topk_16384.jpg @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:2cfec67042a0d4fbb9571c5652c3b3f52dec80a821cbbd2575ef3fec9cb5399e +size 961612 diff --git a/CKPTS/EmptyFix_InstructBlip_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_objects/002-GPT-XL/checkpoints/eval_step_28000/batch_0_cfg_7.5_topk_16384.jpg b/CKPTS/EmptyFix_InstructBlip_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_objects/002-GPT-XL/checkpoints/eval_step_28000/batch_0_cfg_7.5_topk_16384.jpg new file mode 100644 index 0000000000000000000000000000000000000000..d8fd3c35e91dd626119cf3514da549acedef3c7b --- /dev/null +++ b/CKPTS/EmptyFix_InstructBlip_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_objects/002-GPT-XL/checkpoints/eval_step_28000/batch_0_cfg_7.5_topk_16384.jpg @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:fb92dbd041609c9e29333d87f1df65d6298ba2c8023c119e7a0a84ca8c8acfca +size 1138873 diff --git a/CKPTS/EmptyFix_InstructBlip_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_objects/002-GPT-XL/checkpoints/eval_step_28000/batch_1_cfg_7.5_topk_16384.jpg b/CKPTS/EmptyFix_InstructBlip_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_objects/002-GPT-XL/checkpoints/eval_step_28000/batch_1_cfg_7.5_topk_16384.jpg new file mode 100644 index 0000000000000000000000000000000000000000..5f2d2454eeeabaf93a601a4a47b3dc6cb0584507 --- /dev/null +++ b/CKPTS/EmptyFix_InstructBlip_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_objects/002-GPT-XL/checkpoints/eval_step_28000/batch_1_cfg_7.5_topk_16384.jpg @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:91b24a6f09eec3778aa1a9ddb5d3cf920dc8ec5f63725afb8639776e02afdf89 +size 986728 diff --git a/CKPTS/EmptyFix_InstructBlip_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_objects/002-GPT-XL/checkpoints/eval_step_28000/batch_2_cfg_7.5_topk_16384.jpg b/CKPTS/EmptyFix_InstructBlip_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_objects/002-GPT-XL/checkpoints/eval_step_28000/batch_2_cfg_7.5_topk_16384.jpg new file mode 100644 index 0000000000000000000000000000000000000000..d593f4040e9b4d507f1f5a7c6583bec8c3899c1d --- /dev/null +++ b/CKPTS/EmptyFix_InstructBlip_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_objects/002-GPT-XL/checkpoints/eval_step_28000/batch_2_cfg_7.5_topk_16384.jpg @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:f0af67eec6f4c46127e61ed6c8294c902ed8c9860696189a9588fe84baed6ca2 +size 941396 diff --git a/CKPTS/EmptyFix_InstructBlip_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_objects/002-GPT-XL/checkpoints/eval_step_30000/batch_0_cfg_7.5_topk_16384.jpg b/CKPTS/EmptyFix_InstructBlip_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_objects/002-GPT-XL/checkpoints/eval_step_30000/batch_0_cfg_7.5_topk_16384.jpg new file mode 100644 index 0000000000000000000000000000000000000000..bbc55bde3a1b5321dc2d51288a8a7df6ec4a755b --- /dev/null +++ b/CKPTS/EmptyFix_InstructBlip_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_objects/002-GPT-XL/checkpoints/eval_step_30000/batch_0_cfg_7.5_topk_16384.jpg @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:b0e9dd65d169c0d7d6d2d0e543ff81962b4b9be2c45f6704d8dc71177baa77a0 +size 1132553 diff --git a/CKPTS/EmptyFix_InstructBlip_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_objects/002-GPT-XL/checkpoints/eval_step_30000/batch_1_cfg_7.5_topk_16384.jpg b/CKPTS/EmptyFix_InstructBlip_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_objects/002-GPT-XL/checkpoints/eval_step_30000/batch_1_cfg_7.5_topk_16384.jpg new file mode 100644 index 0000000000000000000000000000000000000000..5cd58d44303f1772eb78cf7b836139dbf7a57b97 --- /dev/null +++ b/CKPTS/EmptyFix_InstructBlip_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_objects/002-GPT-XL/checkpoints/eval_step_30000/batch_1_cfg_7.5_topk_16384.jpg @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:48ed0e3e656b66dc6acc5e0675fdea2a75f1c40b11332ec1857eb938ec4b6631 +size 966202 diff --git a/CKPTS/EmptyFix_InstructBlip_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_objects/002-GPT-XL/checkpoints/eval_step_30000/batch_2_cfg_7.5_topk_16384.jpg b/CKPTS/EmptyFix_InstructBlip_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_objects/002-GPT-XL/checkpoints/eval_step_30000/batch_2_cfg_7.5_topk_16384.jpg new file mode 100644 index 0000000000000000000000000000000000000000..c700b054464b4e7f38147843e9c899798795d80f --- /dev/null +++ b/CKPTS/EmptyFix_InstructBlip_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_objects/002-GPT-XL/checkpoints/eval_step_30000/batch_2_cfg_7.5_topk_16384.jpg @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:556cd1d112108286fb80aa016f7bb59ad7e2d33fe06ed19904e488c9950d1db9 +size 958773 diff --git a/CKPTS/EmptyFix_InstructBlip_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_objects/002-GPT-XL/checkpoints/eval_step_32000/batch_0_cfg_7.5_topk_16384.jpg b/CKPTS/EmptyFix_InstructBlip_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_objects/002-GPT-XL/checkpoints/eval_step_32000/batch_0_cfg_7.5_topk_16384.jpg new file mode 100644 index 0000000000000000000000000000000000000000..743821a97032fa2c3c10bc17a9071f71f7a86b6b --- /dev/null +++ b/CKPTS/EmptyFix_InstructBlip_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_objects/002-GPT-XL/checkpoints/eval_step_32000/batch_0_cfg_7.5_topk_16384.jpg @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:489b822f63c6501c38879935ca278ed5755eeb157d65a2c58defc1a3e48dd362 +size 1144674 diff --git a/CKPTS/EmptyFix_InstructBlip_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_objects/002-GPT-XL/checkpoints/eval_step_32000/batch_1_cfg_7.5_topk_16384.jpg b/CKPTS/EmptyFix_InstructBlip_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_objects/002-GPT-XL/checkpoints/eval_step_32000/batch_1_cfg_7.5_topk_16384.jpg new file mode 100644 index 0000000000000000000000000000000000000000..8de66000b6a5527c2f22321df103d18baf827160 --- /dev/null +++ b/CKPTS/EmptyFix_InstructBlip_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_objects/002-GPT-XL/checkpoints/eval_step_32000/batch_1_cfg_7.5_topk_16384.jpg @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:727237b624e8dc583906982ed70f7d5ab288ca2f602c9141bddc96a200162bb9 +size 964860 diff --git a/CKPTS/EmptyFix_InstructBlip_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_objects/002-GPT-XL/checkpoints/eval_step_32000/batch_2_cfg_7.5_topk_16384.jpg b/CKPTS/EmptyFix_InstructBlip_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_objects/002-GPT-XL/checkpoints/eval_step_32000/batch_2_cfg_7.5_topk_16384.jpg new file mode 100644 index 0000000000000000000000000000000000000000..3d00e4a9167c41b3ebf2a93ec11163e545630293 --- /dev/null +++ b/CKPTS/EmptyFix_InstructBlip_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_objects/002-GPT-XL/checkpoints/eval_step_32000/batch_2_cfg_7.5_topk_16384.jpg @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:818bfce8aa126df874dd0745ab7a90203dac7c825f11ebef3b13d5f7ffb3828f +size 979105 diff --git a/CKPTS/EmptyFix_InstructBlip_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_objects/002-GPT-XL/checkpoints/eval_step_34000/batch_0_cfg_7.5_topk_16384.jpg b/CKPTS/EmptyFix_InstructBlip_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_objects/002-GPT-XL/checkpoints/eval_step_34000/batch_0_cfg_7.5_topk_16384.jpg new file mode 100644 index 0000000000000000000000000000000000000000..5d97e7d0b82a9e9b5e0f67337d23741249375a20 --- /dev/null +++ b/CKPTS/EmptyFix_InstructBlip_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_objects/002-GPT-XL/checkpoints/eval_step_34000/batch_0_cfg_7.5_topk_16384.jpg @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:f7efe12bcbf748a2585e246b8e8bfbd04d585daad413e88d1218b76bee0e0590 +size 1119699 diff --git a/CKPTS/EmptyFix_InstructBlip_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_objects/002-GPT-XL/checkpoints/eval_step_34000/batch_1_cfg_7.5_topk_16384.jpg b/CKPTS/EmptyFix_InstructBlip_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_objects/002-GPT-XL/checkpoints/eval_step_34000/batch_1_cfg_7.5_topk_16384.jpg new file mode 100644 index 0000000000000000000000000000000000000000..315511fed3c35448c4ac4ec5abb99f803bae0650 --- /dev/null +++ b/CKPTS/EmptyFix_InstructBlip_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_objects/002-GPT-XL/checkpoints/eval_step_34000/batch_1_cfg_7.5_topk_16384.jpg @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:062d666bc4dbb9387bc2323a5af93f7b16e5e83b6aa092ffe368043beebd694d +size 954308 diff --git a/CKPTS/EmptyFix_InstructBlip_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_objects/002-GPT-XL/checkpoints/eval_step_34000/batch_2_cfg_7.5_topk_16384.jpg b/CKPTS/EmptyFix_InstructBlip_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_objects/002-GPT-XL/checkpoints/eval_step_34000/batch_2_cfg_7.5_topk_16384.jpg new file mode 100644 index 0000000000000000000000000000000000000000..ca9e25edd3cd7a41bc43cf0e33ce304d7b5fa3d4 --- /dev/null +++ b/CKPTS/EmptyFix_InstructBlip_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_objects/002-GPT-XL/checkpoints/eval_step_34000/batch_2_cfg_7.5_topk_16384.jpg @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:de23fa56dedd7671dfa1db9e414cc8d99ad101ba20e752b7d24a46cabcf2da96 +size 1000059 diff --git a/CKPTS/EmptyFix_InstructBlip_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_objects/002-GPT-XL/checkpoints/eval_step_36000/batch_0_cfg_7.5_topk_16384.jpg b/CKPTS/EmptyFix_InstructBlip_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_objects/002-GPT-XL/checkpoints/eval_step_36000/batch_0_cfg_7.5_topk_16384.jpg new file mode 100644 index 0000000000000000000000000000000000000000..0d9a8a7b9f5082b5c93dd74274ead1323492680e --- /dev/null +++ b/CKPTS/EmptyFix_InstructBlip_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_objects/002-GPT-XL/checkpoints/eval_step_36000/batch_0_cfg_7.5_topk_16384.jpg @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:48d36eff8e0b5a3ff4bf6b50b32b84884f2c4e88fc844d9c11cebefc62a6bafe +size 1146274 diff --git a/CKPTS/EmptyFix_InstructBlip_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_objects/002-GPT-XL/checkpoints/eval_step_36000/batch_1_cfg_7.5_topk_16384.jpg b/CKPTS/EmptyFix_InstructBlip_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_objects/002-GPT-XL/checkpoints/eval_step_36000/batch_1_cfg_7.5_topk_16384.jpg new file mode 100644 index 0000000000000000000000000000000000000000..f643ac7e71eee2f9d52ea093e4fd4c30d99c58b2 --- /dev/null +++ b/CKPTS/EmptyFix_InstructBlip_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_objects/002-GPT-XL/checkpoints/eval_step_36000/batch_1_cfg_7.5_topk_16384.jpg @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:22df93ec75423051b1a142124ff7c01e90b595fc48e3eb4250d0f2da2a71310d +size 956366 diff --git a/CKPTS/EmptyFix_InstructBlip_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_objects/002-GPT-XL/checkpoints/eval_step_36000/batch_2_cfg_7.5_topk_16384.jpg b/CKPTS/EmptyFix_InstructBlip_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_objects/002-GPT-XL/checkpoints/eval_step_36000/batch_2_cfg_7.5_topk_16384.jpg new file mode 100644 index 0000000000000000000000000000000000000000..34bc71bf3677027e5bbba96ff4eb120e129a6299 --- /dev/null +++ b/CKPTS/EmptyFix_InstructBlip_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_objects/002-GPT-XL/checkpoints/eval_step_36000/batch_2_cfg_7.5_topk_16384.jpg @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:63b9d63f17f6fdb16d561a6b197de51fd9170cd1052caa6a53ffc2327c121e89 +size 958973 diff --git a/CKPTS/EmptyFix_InstructBlip_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_objects/002-GPT-XL/checkpoints/eval_step_38000/batch_0_cfg_7.5_topk_16384.jpg b/CKPTS/EmptyFix_InstructBlip_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_objects/002-GPT-XL/checkpoints/eval_step_38000/batch_0_cfg_7.5_topk_16384.jpg new file mode 100644 index 0000000000000000000000000000000000000000..3e078e138bc509080753274e75ffcdf71c6bb44f --- /dev/null +++ b/CKPTS/EmptyFix_InstructBlip_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_objects/002-GPT-XL/checkpoints/eval_step_38000/batch_0_cfg_7.5_topk_16384.jpg @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:7751db3580706c1c5d1ffd5fa7cdbb1c4b6dd4bf574798b9864dbdf39810e68f +size 1133041 diff --git a/CKPTS/EmptyFix_InstructBlip_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_objects/002-GPT-XL/checkpoints/eval_step_38000/batch_1_cfg_7.5_topk_16384.jpg b/CKPTS/EmptyFix_InstructBlip_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_objects/002-GPT-XL/checkpoints/eval_step_38000/batch_1_cfg_7.5_topk_16384.jpg new file mode 100644 index 0000000000000000000000000000000000000000..483b0c71ff177e085ec09aff96bb3f5afe430384 --- /dev/null +++ b/CKPTS/EmptyFix_InstructBlip_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_objects/002-GPT-XL/checkpoints/eval_step_38000/batch_1_cfg_7.5_topk_16384.jpg @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:180f7f9466c41a06b814f75605e081e4a7d17a2751df661e70f04de051131ffc +size 981622 diff --git a/CKPTS/EmptyFix_InstructBlip_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_objects/002-GPT-XL/checkpoints/eval_step_38000/batch_2_cfg_7.5_topk_16384.jpg b/CKPTS/EmptyFix_InstructBlip_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_objects/002-GPT-XL/checkpoints/eval_step_38000/batch_2_cfg_7.5_topk_16384.jpg new file mode 100644 index 0000000000000000000000000000000000000000..4d57c2058db59b3ab8e6573811430b08b1072e8e --- /dev/null +++ b/CKPTS/EmptyFix_InstructBlip_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_objects/002-GPT-XL/checkpoints/eval_step_38000/batch_2_cfg_7.5_topk_16384.jpg @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:e35c57713d11b4d20422d15aa0c88ae95e942e985ea05175f2c070a225999777 +size 987053 diff --git a/CKPTS/EmptyFix_InstructBlip_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_objects/002-GPT-XL/checkpoints/eval_step_4000/batch_0_cfg_7.5_topk_16384.jpg b/CKPTS/EmptyFix_InstructBlip_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_objects/002-GPT-XL/checkpoints/eval_step_4000/batch_0_cfg_7.5_topk_16384.jpg new file mode 100644 index 0000000000000000000000000000000000000000..1557222644219f52bbf52b1ef8fc993c3b81f068 --- /dev/null +++ b/CKPTS/EmptyFix_InstructBlip_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_objects/002-GPT-XL/checkpoints/eval_step_4000/batch_0_cfg_7.5_topk_16384.jpg @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:c4226ade14252f356be46a7d1c6cc0e3b2f59b3aaa5e21f2a2b06ec1de579dcc +size 1149793 diff --git a/CKPTS/EmptyFix_InstructBlip_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_objects/002-GPT-XL/checkpoints/eval_step_4000/batch_1_cfg_7.5_topk_16384.jpg b/CKPTS/EmptyFix_InstructBlip_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_objects/002-GPT-XL/checkpoints/eval_step_4000/batch_1_cfg_7.5_topk_16384.jpg new file mode 100644 index 0000000000000000000000000000000000000000..5ed2dbd6b80cfd683d769731913ed2c2e009efa0 --- /dev/null +++ b/CKPTS/EmptyFix_InstructBlip_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_objects/002-GPT-XL/checkpoints/eval_step_4000/batch_1_cfg_7.5_topk_16384.jpg @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:0b52d56d99dba404c25c5a147b33b0546002f1ebd9a0e49a05e582b7763c5e22 +size 1001873 diff --git a/CKPTS/EmptyFix_InstructBlip_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_objects/002-GPT-XL/checkpoints/eval_step_4000/batch_2_cfg_7.5_topk_16384.jpg b/CKPTS/EmptyFix_InstructBlip_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_objects/002-GPT-XL/checkpoints/eval_step_4000/batch_2_cfg_7.5_topk_16384.jpg new file mode 100644 index 0000000000000000000000000000000000000000..5feba600e2e1ef04ea90fcab3b7357711890d23c --- /dev/null +++ b/CKPTS/EmptyFix_InstructBlip_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_objects/002-GPT-XL/checkpoints/eval_step_4000/batch_2_cfg_7.5_topk_16384.jpg @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:644d84544a7a93d8629da4199fd07550f34bff3a846333076175feb196b43885 +size 979776 diff --git a/CKPTS/EmptyFix_InstructBlip_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_objects/002-GPT-XL/checkpoints/eval_step_40000/batch_0_cfg_7.5_topk_16384.jpg b/CKPTS/EmptyFix_InstructBlip_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_objects/002-GPT-XL/checkpoints/eval_step_40000/batch_0_cfg_7.5_topk_16384.jpg new file mode 100644 index 0000000000000000000000000000000000000000..8487e6be925ecb4af121fe347dd7b08203d6a88d --- /dev/null +++ b/CKPTS/EmptyFix_InstructBlip_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_objects/002-GPT-XL/checkpoints/eval_step_40000/batch_0_cfg_7.5_topk_16384.jpg @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:eb52e61cef3a31de2b1a19d8473b6c42d4d4fedc9fcfdd230bea5fd8983abda6 +size 1131025 diff --git a/CKPTS/EmptyFix_InstructBlip_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_objects/002-GPT-XL/checkpoints/eval_step_40000/batch_1_cfg_7.5_topk_16384.jpg b/CKPTS/EmptyFix_InstructBlip_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_objects/002-GPT-XL/checkpoints/eval_step_40000/batch_1_cfg_7.5_topk_16384.jpg new file mode 100644 index 0000000000000000000000000000000000000000..a1e752fc6858964d577b545b169af2c646a5d79b --- /dev/null +++ b/CKPTS/EmptyFix_InstructBlip_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_objects/002-GPT-XL/checkpoints/eval_step_40000/batch_1_cfg_7.5_topk_16384.jpg @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:e557a5a739935906988985bbb3980107d168830f62ae36a1d120a416fbe3e79d +size 984956 diff --git a/CKPTS/EmptyFix_InstructBlip_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_objects/002-GPT-XL/checkpoints/eval_step_40000/batch_2_cfg_7.5_topk_16384.jpg b/CKPTS/EmptyFix_InstructBlip_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_objects/002-GPT-XL/checkpoints/eval_step_40000/batch_2_cfg_7.5_topk_16384.jpg new file mode 100644 index 0000000000000000000000000000000000000000..bd44217ca9e6431b1889a36a69913d389efe78ea --- /dev/null +++ b/CKPTS/EmptyFix_InstructBlip_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_objects/002-GPT-XL/checkpoints/eval_step_40000/batch_2_cfg_7.5_topk_16384.jpg @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:f035a7e466a151709972f4b7de5eda4622ebc20cd64eb764c3453bb3cc416fcc +size 960118 diff --git a/CKPTS/EmptyFix_InstructBlip_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_objects/002-GPT-XL/checkpoints/eval_step_42000/batch_0_cfg_7.5_topk_16384.jpg b/CKPTS/EmptyFix_InstructBlip_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_objects/002-GPT-XL/checkpoints/eval_step_42000/batch_0_cfg_7.5_topk_16384.jpg new file mode 100644 index 0000000000000000000000000000000000000000..d276e235cd5ad03e3408599702fadc94a6964a57 --- /dev/null +++ b/CKPTS/EmptyFix_InstructBlip_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_objects/002-GPT-XL/checkpoints/eval_step_42000/batch_0_cfg_7.5_topk_16384.jpg @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:660065044f0b14b34b9f1a58d8beda5558635527e2bbe71de2823e019d12a811 +size 1124926 diff --git a/CKPTS/EmptyFix_InstructBlip_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_objects/002-GPT-XL/checkpoints/eval_step_42000/batch_1_cfg_7.5_topk_16384.jpg b/CKPTS/EmptyFix_InstructBlip_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_objects/002-GPT-XL/checkpoints/eval_step_42000/batch_1_cfg_7.5_topk_16384.jpg new file mode 100644 index 0000000000000000000000000000000000000000..7a7e78bb58307720905e2e812cdc3ebf09760c45 --- /dev/null +++ b/CKPTS/EmptyFix_InstructBlip_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_objects/002-GPT-XL/checkpoints/eval_step_42000/batch_1_cfg_7.5_topk_16384.jpg @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:f6fd4589b06acf9ff72cce01466a03a171401b86674bc497577529507acdc4c2 +size 960576 diff --git a/CKPTS/EmptyFix_InstructBlip_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_objects/002-GPT-XL/checkpoints/eval_step_42000/batch_2_cfg_7.5_topk_16384.jpg b/CKPTS/EmptyFix_InstructBlip_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_objects/002-GPT-XL/checkpoints/eval_step_42000/batch_2_cfg_7.5_topk_16384.jpg new file mode 100644 index 0000000000000000000000000000000000000000..d502a1b0b144962f8ac45db8b278236e0894cda4 --- /dev/null +++ b/CKPTS/EmptyFix_InstructBlip_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_objects/002-GPT-XL/checkpoints/eval_step_42000/batch_2_cfg_7.5_topk_16384.jpg @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:170e64f0acb3df32b2a683cd3cdc55c1b37756761439c54d92529aaf20469276 +size 940500 diff --git a/CKPTS/EmptyFix_InstructBlip_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_objects/002-GPT-XL/checkpoints/eval_step_44000/batch_0_cfg_7.5_topk_16384.jpg b/CKPTS/EmptyFix_InstructBlip_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_objects/002-GPT-XL/checkpoints/eval_step_44000/batch_0_cfg_7.5_topk_16384.jpg new file mode 100644 index 0000000000000000000000000000000000000000..6a36912274fde29e97a5e791195f94452bc0805f --- /dev/null +++ b/CKPTS/EmptyFix_InstructBlip_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_objects/002-GPT-XL/checkpoints/eval_step_44000/batch_0_cfg_7.5_topk_16384.jpg @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:5d8007310beedfa41edb299545bee7fa879c45af745ee0c6d8244d6e4a52bd7d +size 1137965 diff --git a/CKPTS/EmptyFix_InstructBlip_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_objects/002-GPT-XL/checkpoints/eval_step_44000/batch_1_cfg_7.5_topk_16384.jpg b/CKPTS/EmptyFix_InstructBlip_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_objects/002-GPT-XL/checkpoints/eval_step_44000/batch_1_cfg_7.5_topk_16384.jpg new file mode 100644 index 0000000000000000000000000000000000000000..fd8dfd5b850be200298a24158a2d77a1b39143a8 --- /dev/null +++ b/CKPTS/EmptyFix_InstructBlip_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_objects/002-GPT-XL/checkpoints/eval_step_44000/batch_1_cfg_7.5_topk_16384.jpg @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:95c9e0df58ff33fbf607e6db2d6f3206d9ee19ab9153a359826ac39fdf3b5861 +size 957971 diff --git a/CKPTS/EmptyFix_InstructBlip_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_objects/002-GPT-XL/checkpoints/eval_step_44000/batch_2_cfg_7.5_topk_16384.jpg b/CKPTS/EmptyFix_InstructBlip_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_objects/002-GPT-XL/checkpoints/eval_step_44000/batch_2_cfg_7.5_topk_16384.jpg new file mode 100644 index 0000000000000000000000000000000000000000..75b90e3d9dc837ed9dcde76dd188dfb166dd7b1a --- /dev/null +++ b/CKPTS/EmptyFix_InstructBlip_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_objects/002-GPT-XL/checkpoints/eval_step_44000/batch_2_cfg_7.5_topk_16384.jpg @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:615aeefb8dbd4e38dcc1c045a1270465208bb757296ef1b96a1830a4afdcc872 +size 964044 diff --git a/CKPTS/EmptyFix_InstructBlip_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_objects/002-GPT-XL/checkpoints/eval_step_46000/batch_0_cfg_7.5_topk_16384.jpg b/CKPTS/EmptyFix_InstructBlip_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_objects/002-GPT-XL/checkpoints/eval_step_46000/batch_0_cfg_7.5_topk_16384.jpg new file mode 100644 index 0000000000000000000000000000000000000000..a0629cbaea97663ec9dc4e420cd5a859d484f8e3 --- /dev/null +++ b/CKPTS/EmptyFix_InstructBlip_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_objects/002-GPT-XL/checkpoints/eval_step_46000/batch_0_cfg_7.5_topk_16384.jpg @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:3fb702e1ad2cf3c77d76faf14e92d05555ac088acb509fae281722949036f92f +size 1118342 diff --git a/CKPTS/EmptyFix_InstructBlip_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_objects/002-GPT-XL/checkpoints/eval_step_46000/batch_1_cfg_7.5_topk_16384.jpg b/CKPTS/EmptyFix_InstructBlip_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_objects/002-GPT-XL/checkpoints/eval_step_46000/batch_1_cfg_7.5_topk_16384.jpg new file mode 100644 index 0000000000000000000000000000000000000000..ae8c6ea48cd73a961daadeb2337924c154382712 --- /dev/null +++ b/CKPTS/EmptyFix_InstructBlip_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_objects/002-GPT-XL/checkpoints/eval_step_46000/batch_1_cfg_7.5_topk_16384.jpg @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:e365efda5fbb7e97faf43a9612748cf3d925e581cc4f2125377def3b955d2723 +size 947937 diff --git a/CKPTS/EmptyFix_InstructBlip_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_objects/002-GPT-XL/checkpoints/eval_step_46000/batch_2_cfg_7.5_topk_16384.jpg b/CKPTS/EmptyFix_InstructBlip_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_objects/002-GPT-XL/checkpoints/eval_step_46000/batch_2_cfg_7.5_topk_16384.jpg new file mode 100644 index 0000000000000000000000000000000000000000..7475cfda446ddf4260f4a541187a758528d52463 --- /dev/null +++ b/CKPTS/EmptyFix_InstructBlip_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_objects/002-GPT-XL/checkpoints/eval_step_46000/batch_2_cfg_7.5_topk_16384.jpg @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:f3cda9c33ea5bb62f5762984192e5d8ce33ed3f6a327d2db64c3f6055c6725fc +size 956190 diff --git a/CKPTS/EmptyFix_InstructBlip_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_objects/002-GPT-XL/checkpoints/eval_step_48000/batch_0_cfg_7.5_topk_16384.jpg b/CKPTS/EmptyFix_InstructBlip_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_objects/002-GPT-XL/checkpoints/eval_step_48000/batch_0_cfg_7.5_topk_16384.jpg new file mode 100644 index 0000000000000000000000000000000000000000..a30a231805003a8e91dcc309baecda710f43e0f2 --- /dev/null +++ b/CKPTS/EmptyFix_InstructBlip_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_objects/002-GPT-XL/checkpoints/eval_step_48000/batch_0_cfg_7.5_topk_16384.jpg @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:2d68f1c2eacfd1d8dfa11b00634327f40dd2fefad113dc70d2a649644e93db41 +size 1125542 diff --git a/CKPTS/EmptyFix_InstructBlip_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_objects/002-GPT-XL/checkpoints/eval_step_48000/batch_1_cfg_7.5_topk_16384.jpg b/CKPTS/EmptyFix_InstructBlip_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_objects/002-GPT-XL/checkpoints/eval_step_48000/batch_1_cfg_7.5_topk_16384.jpg new file mode 100644 index 0000000000000000000000000000000000000000..5704ff22386f69f9d60b3a32679b3da05da798da --- /dev/null +++ b/CKPTS/EmptyFix_InstructBlip_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_objects/002-GPT-XL/checkpoints/eval_step_48000/batch_1_cfg_7.5_topk_16384.jpg @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:6584c3504fd41df931cebdc0184323a92084063053a09d209ca20ff73a3a06e1 +size 984017 diff --git a/CKPTS/EmptyFix_InstructBlip_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_objects/002-GPT-XL/checkpoints/eval_step_48000/batch_2_cfg_7.5_topk_16384.jpg b/CKPTS/EmptyFix_InstructBlip_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_objects/002-GPT-XL/checkpoints/eval_step_48000/batch_2_cfg_7.5_topk_16384.jpg new file mode 100644 index 0000000000000000000000000000000000000000..2932cac94ce5a234b651aceb140e217273738813 --- /dev/null +++ b/CKPTS/EmptyFix_InstructBlip_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_objects/002-GPT-XL/checkpoints/eval_step_48000/batch_2_cfg_7.5_topk_16384.jpg @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:a5821d8f66464074fbf3d32791d3615132d052f46c0f09afbdb863fd777a4b5c +size 1004136 diff --git a/CKPTS/EmptyFix_InstructBlip_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_objects/002-GPT-XL/checkpoints/eval_step_50000/batch_0_cfg_7.5_topk_16384.jpg b/CKPTS/EmptyFix_InstructBlip_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_objects/002-GPT-XL/checkpoints/eval_step_50000/batch_0_cfg_7.5_topk_16384.jpg new file mode 100644 index 0000000000000000000000000000000000000000..16ee5e4de50971c6d0f91ffad5f93885c9055916 --- /dev/null +++ b/CKPTS/EmptyFix_InstructBlip_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_objects/002-GPT-XL/checkpoints/eval_step_50000/batch_0_cfg_7.5_topk_16384.jpg @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:e02e887cd08a3521ee47734aa58c3d69b3c01906068f3197dc753f3338491e94 +size 1109112 diff --git a/CKPTS/EmptyFix_InstructBlip_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_objects/002-GPT-XL/checkpoints/eval_step_50000/batch_1_cfg_7.5_topk_16384.jpg b/CKPTS/EmptyFix_InstructBlip_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_objects/002-GPT-XL/checkpoints/eval_step_50000/batch_1_cfg_7.5_topk_16384.jpg new file mode 100644 index 0000000000000000000000000000000000000000..6fc4b6da8911b2f7335da18253cb7095acca4d9f --- /dev/null +++ b/CKPTS/EmptyFix_InstructBlip_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_objects/002-GPT-XL/checkpoints/eval_step_50000/batch_1_cfg_7.5_topk_16384.jpg @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:bef1fa7fcd4b556a1fd7bf10844ee6d82a578285333cb5736aa2577390a66ed5 +size 976435 diff --git a/CKPTS/EmptyFix_InstructBlip_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_objects/002-GPT-XL/checkpoints/eval_step_50000/batch_2_cfg_7.5_topk_16384.jpg b/CKPTS/EmptyFix_InstructBlip_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_objects/002-GPT-XL/checkpoints/eval_step_50000/batch_2_cfg_7.5_topk_16384.jpg new file mode 100644 index 0000000000000000000000000000000000000000..49cd8f5f8a9487708d9dc58fa00a935b307e5ab8 --- /dev/null +++ b/CKPTS/EmptyFix_InstructBlip_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_objects/002-GPT-XL/checkpoints/eval_step_50000/batch_2_cfg_7.5_topk_16384.jpg @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:f6a93a9d1de6505bdb922281354cdb3c62ed8bb024683db6add30515eec77a62 +size 955695 diff --git a/CKPTS/EmptyFix_InstructBlip_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_objects/002-GPT-XL/checkpoints/eval_step_52000/batch_0_cfg_7.5_topk_16384.jpg b/CKPTS/EmptyFix_InstructBlip_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_objects/002-GPT-XL/checkpoints/eval_step_52000/batch_0_cfg_7.5_topk_16384.jpg new file mode 100644 index 0000000000000000000000000000000000000000..1770dcc835f2b9687d4e65ede1db740fb7ac6585 --- /dev/null +++ b/CKPTS/EmptyFix_InstructBlip_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_objects/002-GPT-XL/checkpoints/eval_step_52000/batch_0_cfg_7.5_topk_16384.jpg @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:aa8faf5cb76f004dd890817a41e6c8d551f32996a1dacead5331099bd351d975 +size 1111533 diff --git a/CKPTS/EmptyFix_InstructBlip_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_objects/002-GPT-XL/checkpoints/eval_step_52000/batch_1_cfg_7.5_topk_16384.jpg b/CKPTS/EmptyFix_InstructBlip_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_objects/002-GPT-XL/checkpoints/eval_step_52000/batch_1_cfg_7.5_topk_16384.jpg new file mode 100644 index 0000000000000000000000000000000000000000..09433420149eb3369ca661b2cd027794651c1c5e --- /dev/null +++ b/CKPTS/EmptyFix_InstructBlip_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_objects/002-GPT-XL/checkpoints/eval_step_52000/batch_1_cfg_7.5_topk_16384.jpg @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:b9a1c963554b2798b675c000ffcb9df867da95f14cdf632e9fdb31e26d74c4a9 +size 963505 diff --git a/CKPTS/EmptyFix_InstructBlip_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_objects/002-GPT-XL/checkpoints/eval_step_52000/batch_2_cfg_7.5_topk_16384.jpg b/CKPTS/EmptyFix_InstructBlip_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_objects/002-GPT-XL/checkpoints/eval_step_52000/batch_2_cfg_7.5_topk_16384.jpg new file mode 100644 index 0000000000000000000000000000000000000000..fd3ea1f045640bf4d24cee995e7a27623e3f566a --- /dev/null +++ b/CKPTS/EmptyFix_InstructBlip_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_objects/002-GPT-XL/checkpoints/eval_step_52000/batch_2_cfg_7.5_topk_16384.jpg @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:62f48c29780c0ffc8de499686b3aab46cd176985cd5c3445b6024ed396d8551b +size 983227 diff --git a/CKPTS/EmptyFix_InstructBlip_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_objects/002-GPT-XL/checkpoints/eval_step_54000/batch_0_cfg_7.5_topk_16384.jpg b/CKPTS/EmptyFix_InstructBlip_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_objects/002-GPT-XL/checkpoints/eval_step_54000/batch_0_cfg_7.5_topk_16384.jpg new file mode 100644 index 0000000000000000000000000000000000000000..baa9c51319e7c331d22dc976a594b989af06117d --- /dev/null +++ b/CKPTS/EmptyFix_InstructBlip_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_objects/002-GPT-XL/checkpoints/eval_step_54000/batch_0_cfg_7.5_topk_16384.jpg @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:9e4a526d41a300594edf4e9b97eb7698be3a6b6beed69302f22e8895fb276595 +size 1132531 diff --git a/CKPTS/EmptyFix_InstructBlip_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_objects/002-GPT-XL/checkpoints/eval_step_54000/batch_1_cfg_7.5_topk_16384.jpg b/CKPTS/EmptyFix_InstructBlip_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_objects/002-GPT-XL/checkpoints/eval_step_54000/batch_1_cfg_7.5_topk_16384.jpg new file mode 100644 index 0000000000000000000000000000000000000000..c8e4218c9ad7c4b04f2d25695425d0a60558339e --- /dev/null +++ b/CKPTS/EmptyFix_InstructBlip_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_objects/002-GPT-XL/checkpoints/eval_step_54000/batch_1_cfg_7.5_topk_16384.jpg @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:3a0afdcf19678864b50cb60564dd546e6c48186e768de84ef66f7799f9f35c22 +size 955594 diff --git a/CKPTS/EmptyFix_InstructBlip_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_objects/002-GPT-XL/checkpoints/eval_step_54000/batch_2_cfg_7.5_topk_16384.jpg b/CKPTS/EmptyFix_InstructBlip_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_objects/002-GPT-XL/checkpoints/eval_step_54000/batch_2_cfg_7.5_topk_16384.jpg new file mode 100644 index 0000000000000000000000000000000000000000..e64f90c63da68b4c64e6f7dc802de6e8af491c93 --- /dev/null +++ b/CKPTS/EmptyFix_InstructBlip_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_objects/002-GPT-XL/checkpoints/eval_step_54000/batch_2_cfg_7.5_topk_16384.jpg @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:b87ad686f4a34fe17772e5980b08b33df9364bd37dfd64eaca16891b9ae6ef3a +size 961687 diff --git a/CKPTS/EmptyFix_InstructBlip_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_objects/002-GPT-XL/checkpoints/eval_step_56000/batch_0_cfg_7.5_topk_16384.jpg b/CKPTS/EmptyFix_InstructBlip_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_objects/002-GPT-XL/checkpoints/eval_step_56000/batch_0_cfg_7.5_topk_16384.jpg new file mode 100644 index 0000000000000000000000000000000000000000..841cd0c7363fd65af8a0b2b9b94037903f126b4a --- /dev/null +++ b/CKPTS/EmptyFix_InstructBlip_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_objects/002-GPT-XL/checkpoints/eval_step_56000/batch_0_cfg_7.5_topk_16384.jpg @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:a6c6390285dab4e04156b8203ef1537e5c11b6e7122b05d59133ff4ebc2058a3 +size 1135232 diff --git a/CKPTS/EmptyFix_InstructBlip_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_objects/002-GPT-XL/checkpoints/eval_step_56000/batch_1_cfg_7.5_topk_16384.jpg b/CKPTS/EmptyFix_InstructBlip_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_objects/002-GPT-XL/checkpoints/eval_step_56000/batch_1_cfg_7.5_topk_16384.jpg new file mode 100644 index 0000000000000000000000000000000000000000..5a19ccf045ecc03e72e3fc86bda9780bc39355d3 --- /dev/null +++ b/CKPTS/EmptyFix_InstructBlip_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_objects/002-GPT-XL/checkpoints/eval_step_56000/batch_1_cfg_7.5_topk_16384.jpg @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:1545a31e9a8f80b74902543d2c6b95b5ac8d42f28b6d190e919f90bd1759a83c +size 971316 diff --git a/CKPTS/EmptyFix_InstructBlip_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_objects/002-GPT-XL/checkpoints/eval_step_56000/batch_2_cfg_7.5_topk_16384.jpg b/CKPTS/EmptyFix_InstructBlip_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_objects/002-GPT-XL/checkpoints/eval_step_56000/batch_2_cfg_7.5_topk_16384.jpg new file mode 100644 index 0000000000000000000000000000000000000000..88221dcbd5782675a25d9ae98c64d4b189ce43df --- /dev/null +++ b/CKPTS/EmptyFix_InstructBlip_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_objects/002-GPT-XL/checkpoints/eval_step_56000/batch_2_cfg_7.5_topk_16384.jpg @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:e55a79ecf4f04d95716dfdc52045e304874b8ddb318846c75ff48880cebc1104 +size 1000742 diff --git a/CKPTS/EmptyFix_InstructBlip_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_objects/002-GPT-XL/checkpoints/eval_step_58000/batch_0_cfg_7.5_topk_16384.jpg b/CKPTS/EmptyFix_InstructBlip_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_objects/002-GPT-XL/checkpoints/eval_step_58000/batch_0_cfg_7.5_topk_16384.jpg new file mode 100644 index 0000000000000000000000000000000000000000..106ec41eccbbe8dc9256cebea3808113cf99c2dd --- /dev/null +++ b/CKPTS/EmptyFix_InstructBlip_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_objects/002-GPT-XL/checkpoints/eval_step_58000/batch_0_cfg_7.5_topk_16384.jpg @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:8bf22acf3ee60c0f0a2ce02420d449a8c05335cfc02f0ba545e3e4d6c9ea4630 +size 1110944 diff --git a/CKPTS/EmptyFix_InstructBlip_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_objects/002-GPT-XL/checkpoints/eval_step_58000/batch_1_cfg_7.5_topk_16384.jpg b/CKPTS/EmptyFix_InstructBlip_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_objects/002-GPT-XL/checkpoints/eval_step_58000/batch_1_cfg_7.5_topk_16384.jpg new file mode 100644 index 0000000000000000000000000000000000000000..9690f4169c892a024b441ea6f1625fa7794b5260 --- /dev/null +++ b/CKPTS/EmptyFix_InstructBlip_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_objects/002-GPT-XL/checkpoints/eval_step_58000/batch_1_cfg_7.5_topk_16384.jpg @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:ee559ed7e710bbf2ddfba82283662958354e08883fe1f85e209de082496d6cb2 +size 981214 diff --git a/CKPTS/EmptyFix_InstructBlip_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_objects/002-GPT-XL/checkpoints/eval_step_58000/batch_2_cfg_7.5_topk_16384.jpg b/CKPTS/EmptyFix_InstructBlip_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_objects/002-GPT-XL/checkpoints/eval_step_58000/batch_2_cfg_7.5_topk_16384.jpg new file mode 100644 index 0000000000000000000000000000000000000000..cefe02736ff6793a606823715cf41f35ff9cc423 --- /dev/null +++ b/CKPTS/EmptyFix_InstructBlip_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_objects/002-GPT-XL/checkpoints/eval_step_58000/batch_2_cfg_7.5_topk_16384.jpg @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:0b737454ba30597753df2ae435bfe7d160447080d35e7534f01f0d9c6986449f +size 947066 diff --git a/CKPTS/EmptyFix_InstructBlip_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_objects/002-GPT-XL/checkpoints/eval_step_6000/batch_0_cfg_7.5_topk_16384.jpg b/CKPTS/EmptyFix_InstructBlip_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_objects/002-GPT-XL/checkpoints/eval_step_6000/batch_0_cfg_7.5_topk_16384.jpg new file mode 100644 index 0000000000000000000000000000000000000000..8d75ad7b7ab308fee38ea41ac4f4fb5d318358fe --- /dev/null +++ b/CKPTS/EmptyFix_InstructBlip_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_objects/002-GPT-XL/checkpoints/eval_step_6000/batch_0_cfg_7.5_topk_16384.jpg @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:76718b86faee3018fd2185bca918ded6aa3b39ce352967626feabd68eda5dbf9 +size 1137810 diff --git a/CKPTS/EmptyFix_InstructBlip_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_objects/002-GPT-XL/checkpoints/eval_step_6000/batch_1_cfg_7.5_topk_16384.jpg b/CKPTS/EmptyFix_InstructBlip_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_objects/002-GPT-XL/checkpoints/eval_step_6000/batch_1_cfg_7.5_topk_16384.jpg new file mode 100644 index 0000000000000000000000000000000000000000..c2daf33cabb049d7f1341e42393276427e68c12a --- /dev/null +++ b/CKPTS/EmptyFix_InstructBlip_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_objects/002-GPT-XL/checkpoints/eval_step_6000/batch_1_cfg_7.5_topk_16384.jpg @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:05b02775ccef887178f8b32e76dbdc9713f51382ae62fcb1a6041f2fad11c03a +size 1010624 diff --git a/CKPTS/EmptyFix_InstructBlip_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_objects/002-GPT-XL/checkpoints/eval_step_6000/batch_2_cfg_7.5_topk_16384.jpg b/CKPTS/EmptyFix_InstructBlip_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_objects/002-GPT-XL/checkpoints/eval_step_6000/batch_2_cfg_7.5_topk_16384.jpg new file mode 100644 index 0000000000000000000000000000000000000000..8d95c9508810587155ccf53b63b2ed2ba487a9d3 --- /dev/null +++ b/CKPTS/EmptyFix_InstructBlip_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_objects/002-GPT-XL/checkpoints/eval_step_6000/batch_2_cfg_7.5_topk_16384.jpg @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:51068f9758feccd674ee4ba985927a2733e4a1d4bb3ef80f90aa4964ebcde8bf +size 979207 diff --git a/CKPTS/EmptyFix_InstructBlip_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_objects/002-GPT-XL/checkpoints/eval_step_60000/batch_0_cfg_7.5_topk_16384.jpg b/CKPTS/EmptyFix_InstructBlip_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_objects/002-GPT-XL/checkpoints/eval_step_60000/batch_0_cfg_7.5_topk_16384.jpg new file mode 100644 index 0000000000000000000000000000000000000000..45be60153fc822eb48a8ef822f0dd0f6ef195f52 --- /dev/null +++ b/CKPTS/EmptyFix_InstructBlip_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_objects/002-GPT-XL/checkpoints/eval_step_60000/batch_0_cfg_7.5_topk_16384.jpg @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:6df3658320bfce9808d24ae5b53c99fd947e200c3cf2742c7c7d957a2ad79360 +size 1103883 diff --git a/CKPTS/EmptyFix_InstructBlip_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_objects/002-GPT-XL/checkpoints/eval_step_60000/batch_1_cfg_7.5_topk_16384.jpg b/CKPTS/EmptyFix_InstructBlip_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_objects/002-GPT-XL/checkpoints/eval_step_60000/batch_1_cfg_7.5_topk_16384.jpg new file mode 100644 index 0000000000000000000000000000000000000000..cb3b21422faa0c9e350eb2cf8a914bb8fff97631 --- /dev/null +++ b/CKPTS/EmptyFix_InstructBlip_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_objects/002-GPT-XL/checkpoints/eval_step_60000/batch_1_cfg_7.5_topk_16384.jpg @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:dcd5dc0d8ef3c65b50bbb4a48bfbb2584e6363e190627ca7b373f6391b2f459d +size 957637 diff --git a/CKPTS/EmptyFix_InstructBlip_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_objects/002-GPT-XL/checkpoints/eval_step_60000/batch_2_cfg_7.5_topk_16384.jpg b/CKPTS/EmptyFix_InstructBlip_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_objects/002-GPT-XL/checkpoints/eval_step_60000/batch_2_cfg_7.5_topk_16384.jpg new file mode 100644 index 0000000000000000000000000000000000000000..bada6a2e52535d0d5a193b7290ba86cc8d045518 --- /dev/null +++ b/CKPTS/EmptyFix_InstructBlip_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_objects/002-GPT-XL/checkpoints/eval_step_60000/batch_2_cfg_7.5_topk_16384.jpg @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:70e3d75aedc80acc3afcebdd3fa2cbd24f15f6f41a712091807407457e447f56 +size 992889 diff --git a/CKPTS/EmptyFix_InstructBlip_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_objects/002-GPT-XL/checkpoints/eval_step_8000/batch_0_cfg_7.5_topk_16384.jpg b/CKPTS/EmptyFix_InstructBlip_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_objects/002-GPT-XL/checkpoints/eval_step_8000/batch_0_cfg_7.5_topk_16384.jpg new file mode 100644 index 0000000000000000000000000000000000000000..6af5638ced59f21ce932fe304684de493fdaa391 --- /dev/null +++ b/CKPTS/EmptyFix_InstructBlip_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_objects/002-GPT-XL/checkpoints/eval_step_8000/batch_0_cfg_7.5_topk_16384.jpg @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:0fcb43b27b8331a9a039ff970d7ef0c91739896e7c0df5cc4c446fa0bf745310 +size 1102528 diff --git a/CKPTS/EmptyFix_InstructBlip_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_objects/002-GPT-XL/checkpoints/eval_step_8000/batch_1_cfg_7.5_topk_16384.jpg b/CKPTS/EmptyFix_InstructBlip_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_objects/002-GPT-XL/checkpoints/eval_step_8000/batch_1_cfg_7.5_topk_16384.jpg new file mode 100644 index 0000000000000000000000000000000000000000..4e68cae276a8e0f5f0a23fa8a1df4763bd397603 --- /dev/null +++ b/CKPTS/EmptyFix_InstructBlip_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_objects/002-GPT-XL/checkpoints/eval_step_8000/batch_1_cfg_7.5_topk_16384.jpg @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:e3b9d47f9fdccec42c1bd69f83fa982ebef97ad19a1f244f9086029e281a8dec +size 977231 diff --git a/CKPTS/EmptyFix_InstructBlip_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_objects/002-GPT-XL/checkpoints/eval_step_8000/batch_2_cfg_7.5_topk_16384.jpg b/CKPTS/EmptyFix_InstructBlip_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_objects/002-GPT-XL/checkpoints/eval_step_8000/batch_2_cfg_7.5_topk_16384.jpg new file mode 100644 index 0000000000000000000000000000000000000000..20a1ca605ff0eba053ed814da3c79a6948f99437 --- /dev/null +++ b/CKPTS/EmptyFix_InstructBlip_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_objects/002-GPT-XL/checkpoints/eval_step_8000/batch_2_cfg_7.5_topk_16384.jpg @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:5c6ed9200ce6205f15c12632cd87ab81aea14aecd22e05bc0e4eaa1a4634dd7a +size 968788 diff --git a/CKPTS/EmptyFix_InstructBlip_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_objects/002-GPT-XL/log.txt b/CKPTS/EmptyFix_InstructBlip_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_objects/002-GPT-XL/log.txt new file mode 100644 index 0000000000000000000000000000000000000000..3c1a2ecd6fbb46e2a05bbd33482bb5d616db27a8 --- /dev/null +++ b/CKPTS/EmptyFix_InstructBlip_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_objects/002-GPT-XL/log.txt @@ -0,0 +1,2628 @@ +[2025-04-20 16:53:59] Experiment directory created at checkpoint/EmptyFix_InstructBlip_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_objects/002-GPT-XL +[2025-04-20 16:53:59] Namespace(data_path='/tmp/haozhezhao/MLLMG/jsonl_data/multiobjects_molom_imagenet_flux_qwen_midsource_gen_2_2m_trained.jsonl', cloud_save_path='/tmp/haozhezhao/MLLMG/checkpoint', no_local_save=False, vq_model='VQ-16', vq_ckpt='/tmp/haozhezhao/model/llamagen_t2i/vq_ds16_t2i.pt', codebook_size=16384, codebook_embed_dim=8, gpt_model='GPT-XL', gpt_ckpt='/tmp/haozhezhao/MLLMG/checkpoint/EmptyFix_InstructBlip_just_segment_pretrain_stage3_dreambench_recap_Subject400k_filtered_t2i_flux400k_200kmid_recovery_150k_extract_150k_100_fluxseg_50samseg_trainall_1e4_no_replace/007-GPT-XL/checkpoints/0092000.pt', gpt_type='t2i', vocab_size=16384, cls_token_num=512, dropout_p=0.1, token_dropout_p=0.1, drop_path=0.0, no_compile=False, results_dir='checkpoint/EmptyFix_InstructBlip_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_objects', dataset='ti2i', image_size=512, downsample_size=16, num_classes=1000, epochs=2, lr=5e-05, weight_decay=0.05, beta1=0.9, beta2=0.95, max_grad_norm=1.0, global_batch_size=56, global_seed=0, num_workers=4, log_every=25, ckpt_every=2000, gradient_accumulation_steps=4, mixed_precision='bf16', val_data_path='/tmp/haozhezhao/MLLMG/jsonl_data/multiobjects_molom_imagenet_flux_qwen_midsource_gen_val.jsonl', use_vision_tower=True, model_name_or_path='/tmp/haozhezhao/model/instructblip-flan-t5-xl', image_place_holder='', processor_path=None, do_eval=True, max_eval_samples=200, train_text_encoder=True, no_left_padding=False, cfg_scale=7.5, top_k=16384, temperature=0.9, top_p=1.0, eval_steps=2000, project_name='llamagen_ti2i', load_from_checkpoint='/tmp/haozhezhao/MLLMG/checkpoint/FIXed_3M_1epoch_step32000.pt', warmup=0.05, lr_decay_style='cosine', lr_decay_ratio=0.1, train_iters=500000, class_dropout_prob=0.1, with_image_only=False, image_only_rate=0.1, stage2=False, subject_driven=True, load_subject_embedding=None, reference_data_path='/tmp/haozhezhao/MLLMG/cc12m_reference_tunnel.jsonl', multimodal_encoder='instructblip', do_recovery=True, no_replace=False, resume=False, dreambench_eval=False, find_unused_parameters=True, load_visual_encoder=False, continue_stage1=False, replace_subject=False, train_all=True, save_total_limit=2, load_language_projection=None, mm_vision_tower='openai/clip-vit-large-patch14', load_fixed_llamagen=True, unfreeze_output=False, fix='gpt-empty-fix', rank=0, world_size=8, gpu=0, dist_url='env://', distributed=True, dist_backend='nccl') +[2025-04-20 16:53:59] Starting rank=0, seed=0, world_size=8. +[2025-04-20 16:53:59] vision_config is None. initializing the InstructBlipVisionConfig with default values. +[2025-04-20 16:53:59] qformer_config is None. Initializing the InstructBlipQFormerConfig with default values. +[2025-04-20 16:53:59] text_config is None. Initializing the text config with default values (`OPTConfig`). +[2025-04-20 16:55:39] GPT Parameters: 3,174,417,408 +[2025-04-20 16:55:39] num decayed parameter tensors: 637, with 3,173,270,016 parameters +[2025-04-20 16:55:39] num non-decayed parameter tensors: 644, with 1,147,392 parameters +[2025-04-20 16:55:39] using fused AdamW: True +[2025-04-20 16:56:07] Dataset contains 2,290,583 images +[2025-04-20 16:56:07] Train iters 81806 , warmup 4090.3, len of loader 40903 +[2025-04-20 16:56:27] ### LOAD pretraining weights from checkpoint: /tmp/haozhezhao/MLLMG/checkpoint/EmptyFix_InstructBlip_just_segment_pretrain_stage3_dreambench_recap_Subject400k_filtered_t2i_flux400k_200kmid_recovery_150k_extract_150k_100_fluxseg_50samseg_trainall_1e4_no_replace/007-GPT-XL/checkpoints/0092000.pt +[2025-04-20 16:56:27] Initial state: steps=0, epochs=0 +[2025-04-20 16:56:27] compiling the model... (may take several minutes) +[2025-04-20 16:56:28] freeze the vit +[2025-04-20 16:56:28] ***** total param is 3174417408 ***** +[2025-04-20 16:56:28] ***** total trained param is 2188465152 ***** +[2025-04-20 16:56:34] Training for 2 epochs... +[2025-04-20 16:56:34] Beginning epoch 0... +[2025-04-20 17:28:53] (step=0000025) Train Loss: 6.5057, Train Steps/Sec: 0.01 +[2025-04-20 17:38:07] (step=0000050) Train Loss: 6.4661, Train Steps/Sec: 0.05 +[2025-04-20 17:38:59] (step=0000075) Train Loss: 6.4967, Train Steps/Sec: 0.48 +[2025-04-20 17:41:03] (step=0000100) Train Loss: 6.4912, Train Steps/Sec: 0.20 +[2025-04-20 17:41:32] (step=0000125) Train Loss: 6.4592, Train Steps/Sec: 0.87 +[2025-04-20 17:42:00] (step=0000150) Train Loss: 6.4632, Train Steps/Sec: 0.87 +[2025-04-20 17:42:29] (step=0000175) Train Loss: 6.5350, Train Steps/Sec: 0.86 +[2025-04-20 17:42:58] (step=0000200) Train Loss: 6.4932, Train Steps/Sec: 0.86 +[2025-04-20 17:43:27] (step=0000225) Train Loss: 6.4468, Train Steps/Sec: 0.86 +[2025-04-20 17:43:56] (step=0000250) Train Loss: 6.4591, Train Steps/Sec: 0.86 +[2025-04-20 17:44:25] (step=0000275) Train Loss: 6.4404, Train Steps/Sec: 0.86 +[2025-04-20 17:44:54] (step=0000300) Train Loss: 6.4955, Train Steps/Sec: 0.86 +[2025-04-20 17:45:23] (step=0000325) Train Loss: 6.4684, Train Steps/Sec: 0.87 +[2025-04-20 17:45:52] (step=0000350) Train Loss: 6.4577, Train Steps/Sec: 0.86 +[2025-04-20 17:46:21] (step=0000375) Train Loss: 6.4246, Train Steps/Sec: 0.87 +[2025-04-20 17:46:50] (step=0000400) Train Loss: 6.4903, Train Steps/Sec: 0.86 +[2025-04-20 17:47:19] (step=0000425) Train Loss: 6.4252, Train Steps/Sec: 0.87 +[2025-04-20 17:47:48] (step=0000450) Train Loss: 6.4626, Train Steps/Sec: 0.87 +[2025-04-20 17:48:16] (step=0000475) Train Loss: 6.5006, Train Steps/Sec: 0.87 +[2025-04-20 17:48:46] (step=0000500) Train Loss: 6.4528, Train Steps/Sec: 0.86 +[2025-04-20 17:49:14] (step=0000525) Train Loss: 6.4631, Train Steps/Sec: 0.87 +[2025-04-20 17:49:43] (step=0000550) Train Loss: 6.4537, Train Steps/Sec: 0.86 +[2025-04-20 17:50:12] (step=0000575) Train Loss: 6.4566, Train Steps/Sec: 0.87 +[2025-04-20 17:50:41] (step=0000600) Train Loss: 6.4900, Train Steps/Sec: 0.86 +[2025-04-20 17:51:10] (step=0000625) Train Loss: 6.4453, Train Steps/Sec: 0.87 +[2025-04-20 17:51:39] (step=0000650) Train Loss: 6.4462, Train Steps/Sec: 0.86 +[2025-04-20 17:52:08] (step=0000675) Train Loss: 6.4297, Train Steps/Sec: 0.86 +[2025-04-20 17:52:37] (step=0000700) Train Loss: 6.4266, Train Steps/Sec: 0.86 +[2025-04-20 17:53:06] (step=0000725) Train Loss: 6.4453, Train Steps/Sec: 0.87 +[2025-04-20 17:53:35] (step=0000750) Train Loss: 6.4367, Train Steps/Sec: 0.87 +[2025-04-20 17:54:04] (step=0000775) Train Loss: 6.4228, Train Steps/Sec: 0.87 +[2025-04-20 17:54:33] (step=0000800) Train Loss: 6.4384, Train Steps/Sec: 0.86 +[2025-04-20 17:55:02] (step=0000825) Train Loss: 6.4331, Train Steps/Sec: 0.87 +[2025-04-20 17:55:30] (step=0000850) Train Loss: 6.4629, Train Steps/Sec: 0.87 +[2025-04-20 17:55:59] (step=0000875) Train Loss: 6.4206, Train Steps/Sec: 0.87 +[2025-04-20 17:56:28] (step=0000900) Train Loss: 6.4803, Train Steps/Sec: 0.86 +[2025-04-20 17:56:57] (step=0000925) Train Loss: 6.4405, Train Steps/Sec: 0.87 +[2025-04-20 17:57:26] (step=0000950) Train Loss: 6.4298, Train Steps/Sec: 0.86 +[2025-04-20 17:57:55] (step=0000975) Train Loss: 6.4205, Train Steps/Sec: 0.87 +[2025-04-20 17:58:24] (step=0001000) Train Loss: 6.4658, Train Steps/Sec: 0.86 +[2025-04-20 17:58:53] (step=0001025) Train Loss: 6.4562, Train Steps/Sec: 0.86 +[2025-04-20 17:59:22] (step=0001050) Train Loss: 6.4440, Train Steps/Sec: 0.86 +[2025-04-20 17:59:51] (step=0001075) Train Loss: 6.4449, Train Steps/Sec: 0.86 +[2025-04-20 18:00:20] (step=0001100) Train Loss: 6.4628, Train Steps/Sec: 0.86 +[2025-04-20 18:00:49] (step=0001125) Train Loss: 6.4262, Train Steps/Sec: 0.87 +[2025-04-20 18:01:18] (step=0001150) Train Loss: 6.4035, Train Steps/Sec: 0.86 +[2025-04-20 18:01:47] (step=0001175) Train Loss: 6.4520, Train Steps/Sec: 0.86 +[2025-04-20 18:02:16] (step=0001200) Train Loss: 6.4313, Train Steps/Sec: 0.86 +[2025-04-20 18:02:45] (step=0001225) Train Loss: 6.4554, Train Steps/Sec: 0.86 +[2025-04-20 18:03:13] (step=0001250) Train Loss: 6.4194, Train Steps/Sec: 0.86 +[2025-04-20 18:03:42] (step=0001275) Train Loss: 6.3933, Train Steps/Sec: 0.87 +[2025-04-20 18:04:11] (step=0001300) Train Loss: 6.4842, Train Steps/Sec: 0.87 +[2025-04-20 18:04:40] (step=0001325) Train Loss: 6.4172, Train Steps/Sec: 0.87 +[2025-04-20 18:05:09] (step=0001350) Train Loss: 6.4004, Train Steps/Sec: 0.86 +[2025-04-20 18:05:38] (step=0001375) Train Loss: 6.4349, Train Steps/Sec: 0.86 +[2025-04-20 18:06:07] (step=0001400) Train Loss: 6.4317, Train Steps/Sec: 0.86 +[2025-04-20 18:06:36] (step=0001425) Train Loss: 6.4534, Train Steps/Sec: 0.86 +[2025-04-20 18:07:05] (step=0001450) Train Loss: 6.4484, Train Steps/Sec: 0.87 +[2025-04-20 18:07:34] (step=0001475) Train Loss: 6.4463, Train Steps/Sec: 0.86 +[2025-04-20 18:08:03] (step=0001500) Train Loss: 6.4355, Train Steps/Sec: 0.86 +[2025-04-20 18:08:32] (step=0001525) Train Loss: 6.4485, Train Steps/Sec: 0.87 +[2025-04-20 18:09:01] (step=0001550) Train Loss: 6.4276, Train Steps/Sec: 0.87 +[2025-04-20 18:09:30] (step=0001575) Train Loss: 6.4632, Train Steps/Sec: 0.86 +[2025-04-20 18:09:59] (step=0001600) Train Loss: 6.3607, Train Steps/Sec: 0.86 +[2025-04-20 18:10:27] (step=0001625) Train Loss: 6.4375, Train Steps/Sec: 0.87 +[2025-04-20 18:10:56] (step=0001650) Train Loss: 6.4503, Train Steps/Sec: 0.87 +[2025-04-20 18:11:25] (step=0001675) Train Loss: 6.4590, Train Steps/Sec: 0.87 +[2025-04-20 18:11:54] (step=0001700) Train Loss: 6.3970, Train Steps/Sec: 0.87 +[2025-04-20 18:12:23] (step=0001725) Train Loss: 6.4129, Train Steps/Sec: 0.87 +[2025-04-20 18:12:52] (step=0001750) Train Loss: 6.4354, Train Steps/Sec: 0.86 +[2025-04-20 18:13:20] (step=0001775) Train Loss: 6.3983, Train Steps/Sec: 0.86 +[2025-04-20 18:13:49] (step=0001800) Train Loss: 6.4709, Train Steps/Sec: 0.87 +[2025-04-20 18:14:18] (step=0001825) Train Loss: 6.4512, Train Steps/Sec: 0.87 +[2025-04-20 18:14:47] (step=0001850) Train Loss: 6.4830, Train Steps/Sec: 0.86 +[2025-04-20 18:15:16] (step=0001875) Train Loss: 6.4413, Train Steps/Sec: 0.87 +[2025-04-20 18:15:45] (step=0001900) Train Loss: 6.4012, Train Steps/Sec: 0.86 +[2025-04-20 18:16:14] (step=0001925) Train Loss: 6.3938, Train Steps/Sec: 0.86 +[2025-04-20 18:16:43] (step=0001950) Train Loss: 6.4060, Train Steps/Sec: 0.86 +[2025-04-20 18:17:12] (step=0001975) Train Loss: 6.4073, Train Steps/Sec: 0.87 +[2025-04-20 18:17:41] (step=0002000) Train Loss: 6.3882, Train Steps/Sec: 0.86 +[2025-04-20 18:17:41] vision_config is None. initializing the InstructBlipVisionConfig with default values. +[2025-04-20 18:17:41] qformer_config is None. Initializing the InstructBlipQFormerConfig with default values. +[2025-04-20 18:17:41] text_config is None. Initializing the text config with default values (`OPTConfig`). +[2025-04-20 18:22:38] Finish Eval in 2000 steps... +[2025-04-20 18:22:58] Saved checkpoint to checkpoint/EmptyFix_InstructBlip_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_objects/002-GPT-XL/checkpoints/0002000.pt +[2025-04-20 18:23:28] (step=0002025) Train Loss: 6.4317, Train Steps/Sec: 0.07 +[2025-04-20 18:23:57] (step=0002050) Train Loss: 6.4282, Train Steps/Sec: 0.86 +[2025-04-20 18:24:26] (step=0002075) Train Loss: 6.4583, Train Steps/Sec: 0.86 +[2025-04-20 18:24:55] (step=0002100) Train Loss: 6.4021, Train Steps/Sec: 0.86 +[2025-04-20 18:25:24] (step=0002125) Train Loss: 6.4306, Train Steps/Sec: 0.86 +[2025-04-20 18:25:53] (step=0002150) Train Loss: 6.4259, Train Steps/Sec: 0.86 +[2025-04-20 18:26:22] (step=0002175) Train Loss: 6.4573, Train Steps/Sec: 0.86 +[2025-04-20 18:26:51] (step=0002200) Train Loss: 6.4362, Train Steps/Sec: 0.86 +[2025-04-20 18:27:20] (step=0002225) Train Loss: 6.3764, Train Steps/Sec: 0.86 +[2025-04-20 18:27:49] (step=0002250) Train Loss: 6.4350, Train Steps/Sec: 0.87 +[2025-04-20 18:28:18] (step=0002275) Train Loss: 6.4154, Train Steps/Sec: 0.85 +[2025-04-20 18:28:47] (step=0002300) Train Loss: 6.4466, Train Steps/Sec: 0.86 +[2025-04-20 18:29:16] (step=0002325) Train Loss: 6.4372, Train Steps/Sec: 0.87 +[2025-04-20 18:29:45] (step=0002350) Train Loss: 6.3879, Train Steps/Sec: 0.87 +[2025-04-20 18:30:14] (step=0002375) Train Loss: 6.3905, Train Steps/Sec: 0.87 +[2025-04-20 18:30:43] (step=0002400) Train Loss: 6.4216, Train Steps/Sec: 0.86 +[2025-04-20 18:31:12] (step=0002425) Train Loss: 6.3905, Train Steps/Sec: 0.86 +[2025-04-20 18:31:41] (step=0002450) Train Loss: 6.4440, Train Steps/Sec: 0.86 +[2025-04-20 18:32:10] (step=0002475) Train Loss: 6.3693, Train Steps/Sec: 0.86 +[2025-04-20 18:32:39] (step=0002500) Train Loss: 6.3873, Train Steps/Sec: 0.86 +[2025-04-20 18:33:08] (step=0002525) Train Loss: 6.4046, Train Steps/Sec: 0.86 +[2025-04-20 18:33:37] (step=0002550) Train Loss: 6.3648, Train Steps/Sec: 0.87 +[2025-04-20 18:34:05] (step=0002575) Train Loss: 6.4183, Train Steps/Sec: 0.87 +[2025-04-20 18:34:34] (step=0002600) Train Loss: 6.4004, Train Steps/Sec: 0.87 +[2025-04-20 18:35:03] (step=0002625) Train Loss: 6.4420, Train Steps/Sec: 0.87 +[2025-04-20 18:35:32] (step=0002650) Train Loss: 6.4250, Train Steps/Sec: 0.86 +[2025-04-20 18:36:01] (step=0002675) Train Loss: 6.4597, Train Steps/Sec: 0.86 +[2025-04-20 18:36:30] (step=0002700) Train Loss: 6.4171, Train Steps/Sec: 0.86 +[2025-04-20 18:36:59] (step=0002725) Train Loss: 6.4164, Train Steps/Sec: 0.86 +[2025-04-20 18:37:28] (step=0002750) Train Loss: 6.4378, Train Steps/Sec: 0.87 +[2025-04-20 18:37:57] (step=0002775) Train Loss: 6.4483, Train Steps/Sec: 0.87 +[2025-04-20 18:38:26] (step=0002800) Train Loss: 6.4112, Train Steps/Sec: 0.86 +[2025-04-20 18:38:55] (step=0002825) Train Loss: 6.3953, Train Steps/Sec: 0.87 +[2025-04-20 18:39:24] (step=0002850) Train Loss: 6.4340, Train Steps/Sec: 0.87 +[2025-04-20 18:39:53] (step=0002875) Train Loss: 6.4026, Train Steps/Sec: 0.87 +[2025-04-20 18:40:22] (step=0002900) Train Loss: 6.4041, Train Steps/Sec: 0.86 +[2025-04-20 18:40:50] (step=0002925) Train Loss: 6.3717, Train Steps/Sec: 0.87 +[2025-04-20 18:41:19] (step=0002950) Train Loss: 6.3824, Train Steps/Sec: 0.87 +[2025-04-20 18:41:48] (step=0002975) Train Loss: 6.4330, Train Steps/Sec: 0.86 +[2025-04-20 18:42:17] (step=0003000) Train Loss: 6.4103, Train Steps/Sec: 0.86 +[2025-04-20 18:42:46] (step=0003025) Train Loss: 6.3851, Train Steps/Sec: 0.87 +[2025-04-20 18:43:15] (step=0003050) Train Loss: 6.4296, Train Steps/Sec: 0.87 +[2025-04-20 18:43:44] (step=0003075) Train Loss: 6.4462, Train Steps/Sec: 0.86 +[2025-04-20 18:44:21] (step=0003100) Train Loss: 6.4000, Train Steps/Sec: 0.67 +[2025-04-20 18:44:50] (step=0003125) Train Loss: 6.4226, Train Steps/Sec: 0.87 +[2025-04-20 18:45:19] (step=0003150) Train Loss: 6.3912, Train Steps/Sec: 0.87 +[2025-04-20 18:45:48] (step=0003175) Train Loss: 6.4448, Train Steps/Sec: 0.87 +[2025-04-20 18:46:17] (step=0003200) Train Loss: 6.3975, Train Steps/Sec: 0.86 +[2025-04-20 18:46:46] (step=0003225) Train Loss: 6.4188, Train Steps/Sec: 0.86 +[2025-04-20 18:47:15] (step=0003250) Train Loss: 6.4158, Train Steps/Sec: 0.86 +[2025-04-20 18:47:44] (step=0003275) Train Loss: 6.4110, Train Steps/Sec: 0.86 +[2025-04-20 18:48:22] (step=0003300) Train Loss: 6.4131, Train Steps/Sec: 0.66 +[2025-04-20 18:48:50] (step=0003325) Train Loss: 6.4438, Train Steps/Sec: 0.87 +[2025-04-20 18:49:20] (step=0003350) Train Loss: 6.4193, Train Steps/Sec: 0.86 +[2025-04-20 18:49:48] (step=0003375) Train Loss: 6.4168, Train Steps/Sec: 0.86 +[2025-04-20 18:50:18] (step=0003400) Train Loss: 6.4184, Train Steps/Sec: 0.86 +[2025-04-20 18:50:46] (step=0003425) Train Loss: 6.4455, Train Steps/Sec: 0.87 +[2025-04-20 18:51:16] (step=0003450) Train Loss: 6.4266, Train Steps/Sec: 0.86 +[2025-04-20 18:51:44] (step=0003475) Train Loss: 6.4238, Train Steps/Sec: 0.87 +[2025-04-20 18:52:21] (step=0003500) Train Loss: 6.3790, Train Steps/Sec: 0.69 +[2025-04-20 18:52:57] (step=0003525) Train Loss: 6.3894, Train Steps/Sec: 0.69 +[2025-04-20 18:53:26] (step=0003550) Train Loss: 6.3901, Train Steps/Sec: 0.86 +[2025-04-20 18:53:55] (step=0003575) Train Loss: 6.3879, Train Steps/Sec: 0.86 +[2025-04-20 18:54:32] (step=0003600) Train Loss: 6.4110, Train Steps/Sec: 0.68 +[2025-04-20 18:55:11] (step=0003625) Train Loss: 6.3892, Train Steps/Sec: 0.63 +[2025-04-20 18:55:40] (step=0003650) Train Loss: 6.3665, Train Steps/Sec: 0.87 +[2025-04-20 18:56:09] (step=0003675) Train Loss: 6.4501, Train Steps/Sec: 0.87 +[2025-04-20 18:56:38] (step=0003700) Train Loss: 6.4340, Train Steps/Sec: 0.85 +[2025-04-20 18:57:07] (step=0003725) Train Loss: 6.3840, Train Steps/Sec: 0.87 +[2025-04-20 18:57:36] (step=0003750) Train Loss: 6.3956, Train Steps/Sec: 0.86 +[2025-04-20 18:58:05] (step=0003775) Train Loss: 6.4174, Train Steps/Sec: 0.87 +[2025-04-20 18:58:34] (step=0003800) Train Loss: 6.4015, Train Steps/Sec: 0.86 +[2025-04-20 18:59:03] (step=0003825) Train Loss: 6.3467, Train Steps/Sec: 0.87 +[2025-04-20 18:59:32] (step=0003850) Train Loss: 6.4053, Train Steps/Sec: 0.86 +[2025-04-20 19:00:01] (step=0003875) Train Loss: 6.3870, Train Steps/Sec: 0.86 +[2025-04-20 19:00:30] (step=0003900) Train Loss: 6.4049, Train Steps/Sec: 0.86 +[2025-04-20 19:00:59] (step=0003925) Train Loss: 6.4392, Train Steps/Sec: 0.86 +[2025-04-20 19:01:28] (step=0003950) Train Loss: 6.4453, Train Steps/Sec: 0.86 +[2025-04-20 19:01:57] (step=0003975) Train Loss: 6.4215, Train Steps/Sec: 0.87 +[2025-04-20 19:02:26] (step=0004000) Train Loss: 6.4013, Train Steps/Sec: 0.86 +[2025-04-20 19:02:26] vision_config is None. initializing the InstructBlipVisionConfig with default values. +[2025-04-20 19:02:26] qformer_config is None. Initializing the InstructBlipQFormerConfig with default values. +[2025-04-20 19:02:26] text_config is None. Initializing the text config with default values (`OPTConfig`). +[2025-04-20 19:07:23] Finish Eval in 4000 steps... +[2025-04-20 19:07:44] Saved checkpoint to checkpoint/EmptyFix_InstructBlip_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_objects/002-GPT-XL/checkpoints/0004000.pt +[2025-04-20 19:08:13] (step=0004025) Train Loss: 6.3798, Train Steps/Sec: 0.07 +[2025-04-20 19:08:42] (step=0004050) Train Loss: 6.4162, Train Steps/Sec: 0.86 +[2025-04-20 19:09:11] (step=0004075) Train Loss: 6.4138, Train Steps/Sec: 0.86 +[2025-04-20 19:09:40] (step=0004100) Train Loss: 6.4085, Train Steps/Sec: 0.86 +[2025-04-20 19:10:09] (step=0004125) Train Loss: 6.4676, Train Steps/Sec: 0.86 +[2025-04-20 19:10:38] (step=0004150) Train Loss: 6.3727, Train Steps/Sec: 0.86 +[2025-04-20 19:11:07] (step=0004175) Train Loss: 6.4526, Train Steps/Sec: 0.87 +[2025-04-20 19:11:36] (step=0004200) Train Loss: 6.4306, Train Steps/Sec: 0.86 +[2025-04-20 19:12:05] (step=0004225) Train Loss: 6.4266, Train Steps/Sec: 0.86 +[2025-04-20 19:12:34] (step=0004250) Train Loss: 6.3537, Train Steps/Sec: 0.85 +[2025-04-20 19:13:03] (step=0004275) Train Loss: 6.4255, Train Steps/Sec: 0.87 +[2025-04-20 19:13:32] (step=0004300) Train Loss: 6.4346, Train Steps/Sec: 0.86 +[2025-04-20 19:14:01] (step=0004325) Train Loss: 6.4124, Train Steps/Sec: 0.86 +[2025-04-20 19:14:30] (step=0004350) Train Loss: 6.4066, Train Steps/Sec: 0.87 +[2025-04-20 19:14:59] (step=0004375) Train Loss: 6.4033, Train Steps/Sec: 0.87 +[2025-04-20 19:15:28] (step=0004400) Train Loss: 6.3907, Train Steps/Sec: 0.86 +[2025-04-20 19:15:57] (step=0004425) Train Loss: 6.4029, Train Steps/Sec: 0.86 +[2025-04-20 19:16:26] (step=0004450) Train Loss: 6.3747, Train Steps/Sec: 0.87 +[2025-04-20 19:16:55] (step=0004475) Train Loss: 6.4041, Train Steps/Sec: 0.87 +[2025-04-20 19:17:24] (step=0004500) Train Loss: 6.3704, Train Steps/Sec: 0.86 +[2025-04-20 19:17:53] (step=0004525) Train Loss: 6.4040, Train Steps/Sec: 0.86 +[2025-04-20 19:18:22] (step=0004550) Train Loss: 6.4572, Train Steps/Sec: 0.86 +[2025-04-20 19:18:50] (step=0004575) Train Loss: 6.3539, Train Steps/Sec: 0.87 +[2025-04-20 19:19:20] (step=0004600) Train Loss: 6.4472, Train Steps/Sec: 0.86 +[2025-04-20 19:19:48] (step=0004625) Train Loss: 6.3996, Train Steps/Sec: 0.87 +[2025-04-20 19:20:17] (step=0004650) Train Loss: 6.4231, Train Steps/Sec: 0.86 +[2025-04-20 19:20:46] (step=0004675) Train Loss: 6.4317, Train Steps/Sec: 0.86 +[2025-04-20 19:21:16] (step=0004700) Train Loss: 6.4144, Train Steps/Sec: 0.86 +[2025-04-20 19:21:44] (step=0004725) Train Loss: 6.3825, Train Steps/Sec: 0.87 +[2025-04-20 19:22:13] (step=0004750) Train Loss: 6.4110, Train Steps/Sec: 0.87 +[2025-04-20 19:22:42] (step=0004775) Train Loss: 6.3894, Train Steps/Sec: 0.86 +[2025-04-20 19:23:11] (step=0004800) Train Loss: 6.3890, Train Steps/Sec: 0.86 +[2025-04-20 19:23:40] (step=0004825) Train Loss: 6.3873, Train Steps/Sec: 0.87 +[2025-04-20 19:24:09] (step=0004850) Train Loss: 6.4305, Train Steps/Sec: 0.86 +[2025-04-20 19:24:38] (step=0004875) Train Loss: 6.4459, Train Steps/Sec: 0.86 +[2025-04-20 19:25:07] (step=0004900) Train Loss: 6.3918, Train Steps/Sec: 0.86 +[2025-04-20 19:25:36] (step=0004925) Train Loss: 6.4141, Train Steps/Sec: 0.86 +[2025-04-20 19:26:05] (step=0004950) Train Loss: 6.4233, Train Steps/Sec: 0.87 +[2025-04-20 19:26:34] (step=0004975) Train Loss: 6.3474, Train Steps/Sec: 0.86 +[2025-04-20 19:27:03] (step=0005000) Train Loss: 6.4056, Train Steps/Sec: 0.86 +[2025-04-20 19:27:32] (step=0005025) Train Loss: 6.4960, Train Steps/Sec: 0.86 +[2025-04-20 19:28:01] (step=0005050) Train Loss: 6.4010, Train Steps/Sec: 0.86 +[2025-04-20 19:28:30] (step=0005075) Train Loss: 6.4534, Train Steps/Sec: 0.87 +[2025-04-20 19:28:59] (step=0005100) Train Loss: 6.3942, Train Steps/Sec: 0.86 +[2025-04-20 19:29:28] (step=0005125) Train Loss: 6.4175, Train Steps/Sec: 0.86 +[2025-04-20 19:29:57] (step=0005150) Train Loss: 6.3711, Train Steps/Sec: 0.86 +[2025-04-20 19:30:26] (step=0005175) Train Loss: 6.4427, Train Steps/Sec: 0.86 +[2025-04-20 19:30:55] (step=0005200) Train Loss: 6.4294, Train Steps/Sec: 0.86 +[2025-04-20 19:31:31] (step=0005225) Train Loss: 6.4102, Train Steps/Sec: 0.68 +[2025-04-20 19:32:00] (step=0005250) Train Loss: 6.3955, Train Steps/Sec: 0.86 +[2025-04-20 19:32:29] (step=0005275) Train Loss: 6.4068, Train Steps/Sec: 0.86 +[2025-04-20 19:32:58] (step=0005300) Train Loss: 6.3637, Train Steps/Sec: 0.87 +[2025-04-20 19:33:27] (step=0005325) Train Loss: 6.4061, Train Steps/Sec: 0.86 +[2025-04-20 19:33:56] (step=0005350) Train Loss: 6.4148, Train Steps/Sec: 0.86 +[2025-04-20 19:34:25] (step=0005375) Train Loss: 6.4038, Train Steps/Sec: 0.87 +[2025-04-20 19:34:54] (step=0005400) Train Loss: 6.4185, Train Steps/Sec: 0.85 +[2025-04-20 19:35:23] (step=0005425) Train Loss: 6.3802, Train Steps/Sec: 0.87 +[2025-04-20 19:35:52] (step=0005450) Train Loss: 6.3825, Train Steps/Sec: 0.87 +[2025-04-20 19:36:21] (step=0005475) Train Loss: 6.3806, Train Steps/Sec: 0.86 +[2025-04-20 19:36:50] (step=0005500) Train Loss: 6.3928, Train Steps/Sec: 0.86 +[2025-04-20 19:37:19] (step=0005525) Train Loss: 6.3680, Train Steps/Sec: 0.86 +[2025-04-20 19:37:48] (step=0005550) Train Loss: 6.4031, Train Steps/Sec: 0.86 +[2025-04-20 19:38:17] (step=0005575) Train Loss: 6.4179, Train Steps/Sec: 0.86 +[2025-04-20 19:38:47] (step=0005600) Train Loss: 6.3998, Train Steps/Sec: 0.86 +[2025-04-20 19:39:16] (step=0005625) Train Loss: 6.3929, Train Steps/Sec: 0.86 +[2025-04-20 19:39:44] (step=0005650) Train Loss: 6.4087, Train Steps/Sec: 0.87 +[2025-04-20 19:40:14] (step=0005675) Train Loss: 6.3870, Train Steps/Sec: 0.86 +[2025-04-20 19:40:43] (step=0005700) Train Loss: 6.3952, Train Steps/Sec: 0.86 +[2025-04-20 19:41:12] (step=0005725) Train Loss: 6.4217, Train Steps/Sec: 0.86 +[2025-04-20 19:41:41] (step=0005750) Train Loss: 6.3573, Train Steps/Sec: 0.86 +[2025-04-20 19:42:09] (step=0005775) Train Loss: 6.4025, Train Steps/Sec: 0.87 +[2025-04-20 19:42:38] (step=0005800) Train Loss: 6.3411, Train Steps/Sec: 0.86 +[2025-04-20 19:43:07] (step=0005825) Train Loss: 6.4026, Train Steps/Sec: 0.86 +[2025-04-20 19:43:36] (step=0005850) Train Loss: 6.4254, Train Steps/Sec: 0.86 +[2025-04-20 19:44:05] (step=0005875) Train Loss: 6.3602, Train Steps/Sec: 0.87 +[2025-04-20 19:44:34] (step=0005900) Train Loss: 6.4234, Train Steps/Sec: 0.86 +[2025-04-20 19:45:03] (step=0005925) Train Loss: 6.4573, Train Steps/Sec: 0.86 +[2025-04-20 19:45:32] (step=0005950) Train Loss: 6.3904, Train Steps/Sec: 0.87 +[2025-04-20 19:46:01] (step=0005975) Train Loss: 6.3817, Train Steps/Sec: 0.86 +[2025-04-20 19:46:30] (step=0006000) Train Loss: 6.3946, Train Steps/Sec: 0.86 +[2025-04-20 19:46:30] vision_config is None. initializing the InstructBlipVisionConfig with default values. +[2025-04-20 19:46:30] qformer_config is None. Initializing the InstructBlipQFormerConfig with default values. +[2025-04-20 19:46:30] text_config is None. Initializing the text config with default values (`OPTConfig`). +[2025-04-20 19:51:29] Finish Eval in 6000 steps... +[2025-04-20 19:51:51] Saved checkpoint to checkpoint/EmptyFix_InstructBlip_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_objects/002-GPT-XL/checkpoints/0006000.pt +[2025-04-20 19:51:53] Removed old checkpoint: checkpoint/EmptyFix_InstructBlip_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_objects/002-GPT-XL/checkpoints/0002000.pt +[2025-04-20 19:52:22] (step=0006025) Train Loss: 6.3639, Train Steps/Sec: 0.07 +[2025-04-20 19:52:51] (step=0006050) Train Loss: 6.3682, Train Steps/Sec: 0.87 +[2025-04-20 19:53:20] (step=0006075) Train Loss: 6.4023, Train Steps/Sec: 0.87 +[2025-04-20 19:53:49] (step=0006100) Train Loss: 6.4317, Train Steps/Sec: 0.85 +[2025-04-20 19:54:18] (step=0006125) Train Loss: 6.4124, Train Steps/Sec: 0.87 +[2025-04-20 19:54:47] (step=0006150) Train Loss: 6.4234, Train Steps/Sec: 0.87 +[2025-04-20 19:55:16] (step=0006175) Train Loss: 6.3582, Train Steps/Sec: 0.86 +[2025-04-20 19:55:45] (step=0006200) Train Loss: 6.4322, Train Steps/Sec: 0.86 +[2025-04-20 19:56:14] (step=0006225) Train Loss: 6.4129, Train Steps/Sec: 0.86 +[2025-04-20 19:56:43] (step=0006250) Train Loss: 6.4372, Train Steps/Sec: 0.86 +[2025-04-20 19:57:12] (step=0006275) Train Loss: 6.3692, Train Steps/Sec: 0.86 +[2025-04-20 19:57:41] (step=0006300) Train Loss: 6.4202, Train Steps/Sec: 0.86 +[2025-04-20 19:58:10] (step=0006325) Train Loss: 6.3626, Train Steps/Sec: 0.87 +[2025-04-20 19:58:39] (step=0006350) Train Loss: 6.3946, Train Steps/Sec: 0.86 +[2025-04-20 19:59:08] (step=0006375) Train Loss: 6.3853, Train Steps/Sec: 0.87 +[2025-04-20 19:59:37] (step=0006400) Train Loss: 6.3988, Train Steps/Sec: 0.86 +[2025-04-20 20:00:06] (step=0006425) Train Loss: 6.4258, Train Steps/Sec: 0.86 +[2025-04-20 20:00:35] (step=0006450) Train Loss: 6.3749, Train Steps/Sec: 0.87 +[2025-04-20 20:01:04] (step=0006475) Train Loss: 6.3733, Train Steps/Sec: 0.86 +[2025-04-20 20:01:33] (step=0006500) Train Loss: 6.4008, Train Steps/Sec: 0.86 +[2025-04-20 20:02:02] (step=0006525) Train Loss: 6.4189, Train Steps/Sec: 0.86 +[2025-04-20 20:02:31] (step=0006550) Train Loss: 6.3724, Train Steps/Sec: 0.86 +[2025-04-20 20:03:00] (step=0006575) Train Loss: 6.3892, Train Steps/Sec: 0.86 +[2025-04-20 20:03:29] (step=0006600) Train Loss: 6.4250, Train Steps/Sec: 0.86 +[2025-04-20 20:03:58] (step=0006625) Train Loss: 6.4626, Train Steps/Sec: 0.86 +[2025-04-20 20:04:27] (step=0006650) Train Loss: 6.3847, Train Steps/Sec: 0.86 +[2025-04-20 20:04:56] (step=0006675) Train Loss: 6.3932, Train Steps/Sec: 0.86 +[2025-04-20 20:05:25] (step=0006700) Train Loss: 6.3909, Train Steps/Sec: 0.86 +[2025-04-20 20:05:54] (step=0006725) Train Loss: 6.3934, Train Steps/Sec: 0.87 +[2025-04-20 20:06:23] (step=0006750) Train Loss: 6.3592, Train Steps/Sec: 0.87 +[2025-04-20 20:06:52] (step=0006775) Train Loss: 6.4392, Train Steps/Sec: 0.86 +[2025-04-20 20:07:21] (step=0006800) Train Loss: 6.3956, Train Steps/Sec: 0.86 +[2025-04-20 20:07:50] (step=0006825) Train Loss: 6.3609, Train Steps/Sec: 0.86 +[2025-04-20 20:08:19] (step=0006850) Train Loss: 6.4154, Train Steps/Sec: 0.86 +[2025-04-20 20:08:47] (step=0006875) Train Loss: 6.3889, Train Steps/Sec: 0.87 +[2025-04-20 20:09:16] (step=0006900) Train Loss: 6.3961, Train Steps/Sec: 0.86 +[2025-04-20 20:09:45] (step=0006925) Train Loss: 6.3351, Train Steps/Sec: 0.86 +[2025-04-20 20:10:14] (step=0006950) Train Loss: 6.3856, Train Steps/Sec: 0.86 +[2025-04-20 20:10:43] (step=0006975) Train Loss: 6.3620, Train Steps/Sec: 0.86 +[2025-04-20 20:11:12] (step=0007000) Train Loss: 6.3816, Train Steps/Sec: 0.86 +[2025-04-20 20:11:41] (step=0007025) Train Loss: 6.3824, Train Steps/Sec: 0.87 +[2025-04-20 20:12:10] (step=0007050) Train Loss: 6.4235, Train Steps/Sec: 0.86 +[2025-04-20 20:12:39] (step=0007075) Train Loss: 6.3622, Train Steps/Sec: 0.87 +[2025-04-20 20:13:08] (step=0007100) Train Loss: 6.3758, Train Steps/Sec: 0.86 +[2025-04-20 20:13:37] (step=0007125) Train Loss: 6.4410, Train Steps/Sec: 0.86 +[2025-04-20 20:14:06] (step=0007150) Train Loss: 6.4216, Train Steps/Sec: 0.86 +[2025-04-20 20:14:35] (step=0007175) Train Loss: 6.3759, Train Steps/Sec: 0.87 +[2025-04-20 20:15:04] (step=0007200) Train Loss: 6.4032, Train Steps/Sec: 0.87 +[2025-04-20 20:15:32] (step=0007225) Train Loss: 6.3882, Train Steps/Sec: 0.87 +[2025-04-20 20:16:01] (step=0007250) Train Loss: 6.4284, Train Steps/Sec: 0.86 +[2025-04-20 20:16:30] (step=0007275) Train Loss: 6.3438, Train Steps/Sec: 0.86 +[2025-04-20 20:16:59] (step=0007300) Train Loss: 6.3605, Train Steps/Sec: 0.86 +[2025-04-20 20:17:28] (step=0007325) Train Loss: 6.3738, Train Steps/Sec: 0.87 +[2025-04-20 20:17:57] (step=0007350) Train Loss: 6.3574, Train Steps/Sec: 0.87 +[2025-04-20 20:18:26] (step=0007375) Train Loss: 6.3941, Train Steps/Sec: 0.86 +[2025-04-20 20:18:55] (step=0007400) Train Loss: 6.3649, Train Steps/Sec: 0.86 +[2025-04-20 20:19:24] (step=0007425) Train Loss: 6.3685, Train Steps/Sec: 0.87 +[2025-04-20 20:19:53] (step=0007450) Train Loss: 6.4298, Train Steps/Sec: 0.87 +[2025-04-20 20:20:22] (step=0007475) Train Loss: 6.4004, Train Steps/Sec: 0.86 +[2025-04-20 20:20:51] (step=0007500) Train Loss: 6.3570, Train Steps/Sec: 0.86 +[2025-04-20 20:21:20] (step=0007525) Train Loss: 6.3585, Train Steps/Sec: 0.87 +[2025-04-20 20:21:49] (step=0007550) Train Loss: 6.3927, Train Steps/Sec: 0.87 +[2025-04-20 20:22:18] (step=0007575) Train Loss: 6.3808, Train Steps/Sec: 0.87 +[2025-04-20 20:22:47] (step=0007600) Train Loss: 6.3883, Train Steps/Sec: 0.86 +[2025-04-20 20:23:16] (step=0007625) Train Loss: 6.4050, Train Steps/Sec: 0.87 +[2025-04-20 20:23:45] (step=0007650) Train Loss: 6.4035, Train Steps/Sec: 0.86 +[2025-04-20 20:24:14] (step=0007675) Train Loss: 6.4132, Train Steps/Sec: 0.86 +[2025-04-20 20:24:43] (step=0007700) Train Loss: 6.4007, Train Steps/Sec: 0.86 +[2025-04-20 20:25:12] (step=0007725) Train Loss: 6.3731, Train Steps/Sec: 0.87 +[2025-04-20 20:25:41] (step=0007750) Train Loss: 6.4080, Train Steps/Sec: 0.86 +[2025-04-20 20:26:10] (step=0007775) Train Loss: 6.3848, Train Steps/Sec: 0.87 +[2025-04-20 20:26:39] (step=0007800) Train Loss: 6.3688, Train Steps/Sec: 0.86 +[2025-04-20 20:27:08] (step=0007825) Train Loss: 6.4151, Train Steps/Sec: 0.86 +[2025-04-20 20:27:37] (step=0007850) Train Loss: 6.3808, Train Steps/Sec: 0.87 +[2025-04-20 20:28:05] (step=0007875) Train Loss: 6.3819, Train Steps/Sec: 0.87 +[2025-04-20 20:28:35] (step=0007900) Train Loss: 6.3806, Train Steps/Sec: 0.86 +[2025-04-20 20:29:04] (step=0007925) Train Loss: 6.4112, Train Steps/Sec: 0.86 +[2025-04-20 20:29:33] (step=0007950) Train Loss: 6.4041, Train Steps/Sec: 0.86 +[2025-04-20 20:30:01] (step=0007975) Train Loss: 6.3668, Train Steps/Sec: 0.86 +[2025-04-20 20:30:31] (step=0008000) Train Loss: 6.3944, Train Steps/Sec: 0.85 +[2025-04-20 20:30:31] vision_config is None. initializing the InstructBlipVisionConfig with default values. +[2025-04-20 20:30:31] qformer_config is None. Initializing the InstructBlipQFormerConfig with default values. +[2025-04-20 20:30:31] text_config is None. Initializing the text config with default values (`OPTConfig`). +[2025-04-20 20:35:30] Finish Eval in 8000 steps... +[2025-04-20 20:35:50] Saved checkpoint to checkpoint/EmptyFix_InstructBlip_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_objects/002-GPT-XL/checkpoints/0008000.pt +[2025-04-20 20:35:53] Removed old checkpoint: checkpoint/EmptyFix_InstructBlip_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_objects/002-GPT-XL/checkpoints/0004000.pt +[2025-04-20 20:36:22] (step=0008025) Train Loss: 6.3706, Train Steps/Sec: 0.07 +[2025-04-20 20:36:51] (step=0008050) Train Loss: 6.3633, Train Steps/Sec: 0.86 +[2025-04-20 20:37:21] (step=0008075) Train Loss: 6.4134, Train Steps/Sec: 0.86 +[2025-04-20 20:37:50] (step=0008100) Train Loss: 6.4166, Train Steps/Sec: 0.86 +[2025-04-20 20:38:19] (step=0008125) Train Loss: 6.3949, Train Steps/Sec: 0.86 +[2025-04-20 20:38:48] (step=0008150) Train Loss: 6.4234, Train Steps/Sec: 0.86 +[2025-04-20 20:39:17] (step=0008175) Train Loss: 6.3986, Train Steps/Sec: 0.86 +[2025-04-20 20:39:46] (step=0008200) Train Loss: 6.4047, Train Steps/Sec: 0.86 +[2025-04-20 20:40:15] (step=0008225) Train Loss: 6.4041, Train Steps/Sec: 0.87 +[2025-04-20 20:40:44] (step=0008250) Train Loss: 6.4457, Train Steps/Sec: 0.86 +[2025-04-20 20:41:13] (step=0008275) Train Loss: 6.3644, Train Steps/Sec: 0.86 +[2025-04-20 20:41:42] (step=0008300) Train Loss: 6.4133, Train Steps/Sec: 0.86 +[2025-04-20 20:42:11] (step=0008325) Train Loss: 6.3571, Train Steps/Sec: 0.87 +[2025-04-20 20:42:39] (step=0008350) Train Loss: 6.4061, Train Steps/Sec: 0.87 +[2025-04-20 20:43:08] (step=0008375) Train Loss: 6.4055, Train Steps/Sec: 0.86 +[2025-04-20 20:43:37] (step=0008400) Train Loss: 6.4204, Train Steps/Sec: 0.86 +[2025-04-20 20:44:06] (step=0008425) Train Loss: 6.3955, Train Steps/Sec: 0.87 +[2025-04-20 20:44:35] (step=0008450) Train Loss: 6.3687, Train Steps/Sec: 0.86 +[2025-04-20 20:45:04] (step=0008475) Train Loss: 6.3981, Train Steps/Sec: 0.87 +[2025-04-20 20:45:33] (step=0008500) Train Loss: 6.3823, Train Steps/Sec: 0.86 +[2025-04-20 20:46:02] (step=0008525) Train Loss: 6.3724, Train Steps/Sec: 0.86 +[2025-04-20 20:46:31] (step=0008550) Train Loss: 6.3739, Train Steps/Sec: 0.86 +[2025-04-20 20:47:01] (step=0008575) Train Loss: 6.3688, Train Steps/Sec: 0.85 +[2025-04-20 20:47:30] (step=0008600) Train Loss: 6.3924, Train Steps/Sec: 0.85 +[2025-04-20 20:47:59] (step=0008625) Train Loss: 6.4045, Train Steps/Sec: 0.86 +[2025-04-20 20:48:35] (step=0008650) Train Loss: 6.3730, Train Steps/Sec: 0.70 +[2025-04-20 20:49:04] (step=0008675) Train Loss: 6.3526, Train Steps/Sec: 0.86 +[2025-04-20 20:49:33] (step=0008700) Train Loss: 6.4079, Train Steps/Sec: 0.86 +[2025-04-20 20:50:08] (step=0008725) Train Loss: 6.4042, Train Steps/Sec: 0.71 +[2025-04-20 20:50:49] (step=0008750) Train Loss: 6.3655, Train Steps/Sec: 0.61 +[2025-04-20 20:51:18] (step=0008775) Train Loss: 6.3824, Train Steps/Sec: 0.86 +[2025-04-20 20:51:47] (step=0008800) Train Loss: 6.3272, Train Steps/Sec: 0.86 +[2025-04-20 20:52:16] (step=0008825) Train Loss: 6.3610, Train Steps/Sec: 0.86 +[2025-04-20 20:52:53] (step=0008850) Train Loss: 6.3850, Train Steps/Sec: 0.69 +[2025-04-20 20:53:21] (step=0008875) Train Loss: 6.3465, Train Steps/Sec: 0.87 +[2025-04-20 20:53:57] (step=0008900) Train Loss: 6.3950, Train Steps/Sec: 0.70 +[2025-04-20 20:54:26] (step=0008925) Train Loss: 6.3569, Train Steps/Sec: 0.86 +[2025-04-20 20:54:55] (step=0008950) Train Loss: 6.3923, Train Steps/Sec: 0.87 +[2025-04-20 20:55:24] (step=0008975) Train Loss: 6.3860, Train Steps/Sec: 0.86 +[2025-04-20 20:55:53] (step=0009000) Train Loss: 6.3863, Train Steps/Sec: 0.86 +[2025-04-20 20:56:22] (step=0009025) Train Loss: 6.4070, Train Steps/Sec: 0.86 +[2025-04-20 20:56:52] (step=0009050) Train Loss: 6.3906, Train Steps/Sec: 0.86 +[2025-04-20 20:57:21] (step=0009075) Train Loss: 6.3658, Train Steps/Sec: 0.86 +[2025-04-20 20:57:50] (step=0009100) Train Loss: 6.3843, Train Steps/Sec: 0.85 +[2025-04-20 20:58:19] (step=0009125) Train Loss: 6.3939, Train Steps/Sec: 0.86 +[2025-04-20 20:58:48] (step=0009150) Train Loss: 6.3952, Train Steps/Sec: 0.86 +[2025-04-20 20:59:17] (step=0009175) Train Loss: 6.3775, Train Steps/Sec: 0.86 +[2025-04-20 20:59:46] (step=0009200) Train Loss: 6.4193, Train Steps/Sec: 0.86 +[2025-04-20 21:00:15] (step=0009225) Train Loss: 6.3914, Train Steps/Sec: 0.87 +[2025-04-20 21:00:44] (step=0009250) Train Loss: 6.3672, Train Steps/Sec: 0.86 +[2025-04-20 21:01:13] (step=0009275) Train Loss: 6.3718, Train Steps/Sec: 0.86 +[2025-04-20 21:01:42] (step=0009300) Train Loss: 6.3824, Train Steps/Sec: 0.86 +[2025-04-20 21:02:11] (step=0009325) Train Loss: 6.3804, Train Steps/Sec: 0.86 +[2025-04-20 21:02:40] (step=0009350) Train Loss: 6.4055, Train Steps/Sec: 0.86 +[2025-04-20 21:03:09] (step=0009375) Train Loss: 6.3529, Train Steps/Sec: 0.86 +[2025-04-20 21:03:38] (step=0009400) Train Loss: 6.3810, Train Steps/Sec: 0.86 +[2025-04-20 21:04:07] (step=0009425) Train Loss: 6.3819, Train Steps/Sec: 0.86 +[2025-04-20 21:04:36] (step=0009450) Train Loss: 6.3966, Train Steps/Sec: 0.86 +[2025-04-20 21:05:05] (step=0009475) Train Loss: 6.3731, Train Steps/Sec: 0.87 +[2025-04-20 21:05:34] (step=0009500) Train Loss: 6.3268, Train Steps/Sec: 0.86 +[2025-04-20 21:06:03] (step=0009525) Train Loss: 6.4038, Train Steps/Sec: 0.86 +[2025-04-20 21:06:32] (step=0009550) Train Loss: 6.4151, Train Steps/Sec: 0.86 +[2025-04-20 21:07:01] (step=0009575) Train Loss: 6.3854, Train Steps/Sec: 0.85 +[2025-04-20 21:07:30] (step=0009600) Train Loss: 6.3520, Train Steps/Sec: 0.86 +[2025-04-20 21:07:59] (step=0009625) Train Loss: 6.3630, Train Steps/Sec: 0.86 +[2025-04-20 21:08:28] (step=0009650) Train Loss: 6.3795, Train Steps/Sec: 0.86 +[2025-04-20 21:08:57] (step=0009675) Train Loss: 6.4091, Train Steps/Sec: 0.86 +[2025-04-20 21:09:27] (step=0009700) Train Loss: 6.3828, Train Steps/Sec: 0.86 +[2025-04-20 21:09:55] (step=0009725) Train Loss: 6.3852, Train Steps/Sec: 0.86 +[2025-04-20 21:10:24] (step=0009750) Train Loss: 6.3458, Train Steps/Sec: 0.86 +[2025-04-20 21:10:53] (step=0009775) Train Loss: 6.3804, Train Steps/Sec: 0.86 +[2025-04-20 21:11:23] (step=0009800) Train Loss: 6.3956, Train Steps/Sec: 0.86 +[2025-04-20 21:11:52] (step=0009825) Train Loss: 6.3620, Train Steps/Sec: 0.86 +[2025-04-20 21:12:20] (step=0009850) Train Loss: 6.3812, Train Steps/Sec: 0.87 +[2025-04-20 21:12:50] (step=0009875) Train Loss: 6.3835, Train Steps/Sec: 0.86 +[2025-04-20 21:13:19] (step=0009900) Train Loss: 6.3651, Train Steps/Sec: 0.86 +[2025-04-20 21:13:48] (step=0009925) Train Loss: 6.3761, Train Steps/Sec: 0.87 +[2025-04-20 21:14:17] (step=0009950) Train Loss: 6.3658, Train Steps/Sec: 0.87 +[2025-04-20 21:14:46] (step=0009975) Train Loss: 6.3681, Train Steps/Sec: 0.86 +[2025-04-20 21:15:14] (step=0010000) Train Loss: 6.3482, Train Steps/Sec: 0.86 +[2025-04-20 21:15:15] vision_config is None. initializing the InstructBlipVisionConfig with default values. +[2025-04-20 21:15:15] qformer_config is None. Initializing the InstructBlipQFormerConfig with default values. +[2025-04-20 21:15:15] text_config is None. Initializing the text config with default values (`OPTConfig`). +[2025-04-20 21:20:12] Finish Eval in 10000 steps... +[2025-04-20 21:20:33] Saved checkpoint to checkpoint/EmptyFix_InstructBlip_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_objects/002-GPT-XL/checkpoints/0010000.pt +[2025-04-20 21:20:36] Removed old checkpoint: checkpoint/EmptyFix_InstructBlip_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_objects/002-GPT-XL/checkpoints/0006000.pt +[2025-04-20 21:21:06] (step=0010025) Train Loss: 6.3416, Train Steps/Sec: 0.07 +[2025-04-20 21:21:35] (step=0010050) Train Loss: 6.4356, Train Steps/Sec: 0.86 +[2025-04-20 21:22:04] (step=0010075) Train Loss: 6.3961, Train Steps/Sec: 0.86 +[2025-04-20 21:22:33] (step=0010100) Train Loss: 6.3849, Train Steps/Sec: 0.86 +[2025-04-20 21:23:02] (step=0010125) Train Loss: 6.3769, Train Steps/Sec: 0.86 +[2025-04-20 21:23:31] (step=0010150) Train Loss: 6.3623, Train Steps/Sec: 0.86 +[2025-04-20 21:24:00] (step=0010175) Train Loss: 6.3687, Train Steps/Sec: 0.87 +[2025-04-20 21:24:29] (step=0010200) Train Loss: 6.4005, Train Steps/Sec: 0.86 +[2025-04-20 21:24:58] (step=0010225) Train Loss: 6.3519, Train Steps/Sec: 0.86 +[2025-04-20 21:25:27] (step=0010250) Train Loss: 6.3230, Train Steps/Sec: 0.86 +[2025-04-20 21:25:56] (step=0010275) Train Loss: 6.3681, Train Steps/Sec: 0.86 +[2025-04-20 21:26:25] (step=0010300) Train Loss: 6.3738, Train Steps/Sec: 0.86 +[2025-04-20 21:26:54] (step=0010325) Train Loss: 6.3513, Train Steps/Sec: 0.86 +[2025-04-20 21:27:23] (step=0010350) Train Loss: 6.3873, Train Steps/Sec: 0.87 +[2025-04-20 21:27:52] (step=0010375) Train Loss: 6.3754, Train Steps/Sec: 0.86 +[2025-04-20 21:28:21] (step=0010400) Train Loss: 6.4202, Train Steps/Sec: 0.86 +[2025-04-20 21:28:50] (step=0010425) Train Loss: 6.3954, Train Steps/Sec: 0.86 +[2025-04-20 21:29:20] (step=0010450) Train Loss: 6.4095, Train Steps/Sec: 0.85 +[2025-04-20 21:29:49] (step=0010475) Train Loss: 6.4071, Train Steps/Sec: 0.86 +[2025-04-20 21:30:18] (step=0010500) Train Loss: 6.3886, Train Steps/Sec: 0.85 +[2025-04-20 21:30:47] (step=0010525) Train Loss: 6.3823, Train Steps/Sec: 0.86 +[2025-04-20 21:31:16] (step=0010550) Train Loss: 6.3945, Train Steps/Sec: 0.86 +[2025-04-20 21:31:45] (step=0010575) Train Loss: 6.4118, Train Steps/Sec: 0.87 +[2025-04-20 21:32:14] (step=0010600) Train Loss: 6.3712, Train Steps/Sec: 0.85 +[2025-04-20 21:32:43] (step=0010625) Train Loss: 6.4013, Train Steps/Sec: 0.86 +[2025-04-20 21:33:12] (step=0010650) Train Loss: 6.3921, Train Steps/Sec: 0.86 +[2025-04-20 21:33:48] (step=0010675) Train Loss: 6.3912, Train Steps/Sec: 0.70 +[2025-04-20 21:34:17] (step=0010700) Train Loss: 6.3835, Train Steps/Sec: 0.85 +[2025-04-20 21:34:46] (step=0010725) Train Loss: 6.3389, Train Steps/Sec: 0.87 +[2025-04-20 21:35:15] (step=0010750) Train Loss: 6.3830, Train Steps/Sec: 0.86 +[2025-04-20 21:35:44] (step=0010775) Train Loss: 6.3867, Train Steps/Sec: 0.86 +[2025-04-20 21:36:13] (step=0010800) Train Loss: 6.3722, Train Steps/Sec: 0.86 +[2025-04-20 21:36:42] (step=0010825) Train Loss: 6.4261, Train Steps/Sec: 0.86 +[2025-04-20 21:37:11] (step=0010850) Train Loss: 6.3747, Train Steps/Sec: 0.87 +[2025-04-20 21:37:40] (step=0010875) Train Loss: 6.3466, Train Steps/Sec: 0.86 +[2025-04-20 21:38:09] (step=0010900) Train Loss: 6.4164, Train Steps/Sec: 0.86 +[2025-04-20 21:38:38] (step=0010925) Train Loss: 6.4021, Train Steps/Sec: 0.86 +[2025-04-20 21:39:08] (step=0010950) Train Loss: 6.3983, Train Steps/Sec: 0.86 +[2025-04-20 21:39:37] (step=0010975) Train Loss: 6.3489, Train Steps/Sec: 0.86 +[2025-04-20 21:40:06] (step=0011000) Train Loss: 6.4121, Train Steps/Sec: 0.86 +[2025-04-20 21:40:35] (step=0011025) Train Loss: 6.3742, Train Steps/Sec: 0.86 +[2025-04-20 21:41:04] (step=0011050) Train Loss: 6.3350, Train Steps/Sec: 0.86 +[2025-04-20 21:41:33] (step=0011075) Train Loss: 6.3824, Train Steps/Sec: 0.86 +[2025-04-20 21:42:02] (step=0011100) Train Loss: 6.3826, Train Steps/Sec: 0.86 +[2025-04-20 21:42:31] (step=0011125) Train Loss: 6.3944, Train Steps/Sec: 0.86 +[2025-04-20 21:43:00] (step=0011150) Train Loss: 6.3676, Train Steps/Sec: 0.86 +[2025-04-20 21:43:29] (step=0011175) Train Loss: 6.4154, Train Steps/Sec: 0.86 +[2025-04-20 21:43:58] (step=0011200) Train Loss: 6.3902, Train Steps/Sec: 0.86 +[2025-04-20 21:44:27] (step=0011225) Train Loss: 6.3435, Train Steps/Sec: 0.87 +[2025-04-20 21:44:56] (step=0011250) Train Loss: 6.3809, Train Steps/Sec: 0.86 +[2025-04-20 21:45:25] (step=0011275) Train Loss: 6.3705, Train Steps/Sec: 0.86 +[2025-04-20 21:45:54] (step=0011300) Train Loss: 6.3579, Train Steps/Sec: 0.86 +[2025-04-20 21:46:23] (step=0011325) Train Loss: 6.3900, Train Steps/Sec: 0.86 +[2025-04-20 21:46:52] (step=0011350) Train Loss: 6.3850, Train Steps/Sec: 0.86 +[2025-04-20 21:47:21] (step=0011375) Train Loss: 6.3749, Train Steps/Sec: 0.86 +[2025-04-20 21:47:50] (step=0011400) Train Loss: 6.3975, Train Steps/Sec: 0.86 +[2025-04-20 21:48:19] (step=0011425) Train Loss: 6.3925, Train Steps/Sec: 0.86 +[2025-04-20 21:48:48] (step=0011450) Train Loss: 6.3605, Train Steps/Sec: 0.86 +[2025-04-20 21:49:17] (step=0011475) Train Loss: 6.3957, Train Steps/Sec: 0.87 +[2025-04-20 21:49:46] (step=0011500) Train Loss: 6.3514, Train Steps/Sec: 0.86 +[2025-04-20 21:50:15] (step=0011525) Train Loss: 6.3738, Train Steps/Sec: 0.87 +[2025-04-20 21:50:44] (step=0011550) Train Loss: 6.3985, Train Steps/Sec: 0.86 +[2025-04-20 21:51:13] (step=0011575) Train Loss: 6.4023, Train Steps/Sec: 0.86 +[2025-04-20 21:51:42] (step=0011600) Train Loss: 6.3330, Train Steps/Sec: 0.86 +[2025-04-20 21:52:11] (step=0011625) Train Loss: 6.3534, Train Steps/Sec: 0.87 +[2025-04-20 21:52:40] (step=0011650) Train Loss: 6.3525, Train Steps/Sec: 0.87 +[2025-04-20 21:53:09] (step=0011675) Train Loss: 6.3878, Train Steps/Sec: 0.87 +[2025-04-20 21:53:38] (step=0011700) Train Loss: 6.3886, Train Steps/Sec: 0.86 +[2025-04-20 21:54:07] (step=0011725) Train Loss: 6.3759, Train Steps/Sec: 0.86 +[2025-04-20 21:54:36] (step=0011750) Train Loss: 6.3945, Train Steps/Sec: 0.86 +[2025-04-20 21:55:05] (step=0011775) Train Loss: 6.3888, Train Steps/Sec: 0.87 +[2025-04-20 21:55:34] (step=0011800) Train Loss: 6.3875, Train Steps/Sec: 0.86 +[2025-04-20 21:56:03] (step=0011825) Train Loss: 6.3748, Train Steps/Sec: 0.86 +[2025-04-20 21:56:32] (step=0011850) Train Loss: 6.4215, Train Steps/Sec: 0.87 +[2025-04-20 21:57:01] (step=0011875) Train Loss: 6.3591, Train Steps/Sec: 0.87 +[2025-04-20 21:57:30] (step=0011900) Train Loss: 6.3571, Train Steps/Sec: 0.85 +[2025-04-20 21:57:59] (step=0011925) Train Loss: 6.3885, Train Steps/Sec: 0.86 +[2025-04-20 21:58:28] (step=0011950) Train Loss: 6.4215, Train Steps/Sec: 0.86 +[2025-04-20 21:58:57] (step=0011975) Train Loss: 6.3789, Train Steps/Sec: 0.87 +[2025-04-20 21:59:26] (step=0012000) Train Loss: 6.3799, Train Steps/Sec: 0.85 +[2025-04-20 21:59:26] vision_config is None. initializing the InstructBlipVisionConfig with default values. +[2025-04-20 21:59:26] qformer_config is None. Initializing the InstructBlipQFormerConfig with default values. +[2025-04-20 21:59:26] text_config is None. Initializing the text config with default values (`OPTConfig`). +[2025-04-20 22:04:23] Finish Eval in 12000 steps... +[2025-04-20 22:04:45] Saved checkpoint to checkpoint/EmptyFix_InstructBlip_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_objects/002-GPT-XL/checkpoints/0012000.pt +[2025-04-20 22:04:47] Removed old checkpoint: checkpoint/EmptyFix_InstructBlip_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_objects/002-GPT-XL/checkpoints/0008000.pt +[2025-04-20 22:05:17] (step=0012025) Train Loss: 6.3647, Train Steps/Sec: 0.07 +[2025-04-20 22:05:46] (step=0012050) Train Loss: 6.3851, Train Steps/Sec: 0.86 +[2025-04-20 22:06:15] (step=0012075) Train Loss: 6.3810, Train Steps/Sec: 0.86 +[2025-04-20 22:06:44] (step=0012100) Train Loss: 6.3371, Train Steps/Sec: 0.86 +[2025-04-20 22:07:12] (step=0012125) Train Loss: 6.3599, Train Steps/Sec: 0.86 +[2025-04-20 22:07:41] (step=0012150) Train Loss: 6.3541, Train Steps/Sec: 0.87 +[2025-04-20 22:08:10] (step=0012175) Train Loss: 6.3631, Train Steps/Sec: 0.86 +[2025-04-20 22:08:40] (step=0012200) Train Loss: 6.3736, Train Steps/Sec: 0.86 +[2025-04-20 22:09:09] (step=0012225) Train Loss: 6.4124, Train Steps/Sec: 0.86 +[2025-04-20 22:09:38] (step=0012250) Train Loss: 6.3955, Train Steps/Sec: 0.86 +[2025-04-20 22:10:07] (step=0012275) Train Loss: 6.3887, Train Steps/Sec: 0.86 +[2025-04-20 22:10:36] (step=0012300) Train Loss: 6.4028, Train Steps/Sec: 0.85 +[2025-04-20 22:11:05] (step=0012325) Train Loss: 6.3651, Train Steps/Sec: 0.86 +[2025-04-20 22:11:34] (step=0012350) Train Loss: 6.3376, Train Steps/Sec: 0.86 +[2025-04-20 22:12:03] (step=0012375) Train Loss: 6.3763, Train Steps/Sec: 0.86 +[2025-04-20 22:12:32] (step=0012400) Train Loss: 6.3825, Train Steps/Sec: 0.86 +[2025-04-20 22:13:01] (step=0012425) Train Loss: 6.4091, Train Steps/Sec: 0.86 +[2025-04-20 22:13:30] (step=0012450) Train Loss: 6.3682, Train Steps/Sec: 0.87 +[2025-04-20 22:13:59] (step=0012475) Train Loss: 6.3610, Train Steps/Sec: 0.86 +[2025-04-20 22:14:28] (step=0012500) Train Loss: 6.3800, Train Steps/Sec: 0.85 +[2025-04-20 22:14:57] (step=0012525) Train Loss: 6.4002, Train Steps/Sec: 0.87 +[2025-04-20 22:15:27] (step=0012550) Train Loss: 6.4250, Train Steps/Sec: 0.85 +[2025-04-20 22:15:56] (step=0012575) Train Loss: 6.3958, Train Steps/Sec: 0.86 +[2025-04-20 22:16:25] (step=0012600) Train Loss: 6.4005, Train Steps/Sec: 0.86 +[2025-04-20 22:16:54] (step=0012625) Train Loss: 6.4057, Train Steps/Sec: 0.86 +[2025-04-20 22:17:23] (step=0012650) Train Loss: 6.3585, Train Steps/Sec: 0.86 +[2025-04-20 22:17:52] (step=0012675) Train Loss: 6.3624, Train Steps/Sec: 0.86 +[2025-04-20 22:18:21] (step=0012700) Train Loss: 6.3976, Train Steps/Sec: 0.86 +[2025-04-20 22:18:50] (step=0012725) Train Loss: 6.3468, Train Steps/Sec: 0.86 +[2025-04-20 22:19:19] (step=0012750) Train Loss: 6.3925, Train Steps/Sec: 0.87 +[2025-04-20 22:19:48] (step=0012775) Train Loss: 6.3538, Train Steps/Sec: 0.86 +[2025-04-20 22:20:17] (step=0012800) Train Loss: 6.4290, Train Steps/Sec: 0.86 +[2025-04-20 22:20:46] (step=0012825) Train Loss: 6.4017, Train Steps/Sec: 0.86 +[2025-04-20 22:21:15] (step=0012850) Train Loss: 6.4120, Train Steps/Sec: 0.86 +[2025-04-20 22:21:44] (step=0012875) Train Loss: 6.4340, Train Steps/Sec: 0.86 +[2025-04-20 22:22:13] (step=0012900) Train Loss: 6.4128, Train Steps/Sec: 0.85 +[2025-04-20 22:22:42] (step=0012925) Train Loss: 6.3787, Train Steps/Sec: 0.86 +[2025-04-20 22:23:11] (step=0012950) Train Loss: 6.4141, Train Steps/Sec: 0.87 +[2025-04-20 22:23:40] (step=0012975) Train Loss: 6.4113, Train Steps/Sec: 0.86 +[2025-04-20 22:24:09] (step=0013000) Train Loss: 6.3888, Train Steps/Sec: 0.86 +[2025-04-20 22:24:38] (step=0013025) Train Loss: 6.3751, Train Steps/Sec: 0.87 +[2025-04-20 22:25:07] (step=0013050) Train Loss: 6.3941, Train Steps/Sec: 0.86 +[2025-04-20 22:25:36] (step=0013075) Train Loss: 6.3027, Train Steps/Sec: 0.86 +[2025-04-20 22:26:05] (step=0013100) Train Loss: 6.4030, Train Steps/Sec: 0.86 +[2025-04-20 22:26:34] (step=0013125) Train Loss: 6.3696, Train Steps/Sec: 0.86 +[2025-04-20 22:27:03] (step=0013150) Train Loss: 6.3846, Train Steps/Sec: 0.86 +[2025-04-20 22:27:32] (step=0013175) Train Loss: 6.3513, Train Steps/Sec: 0.86 +[2025-04-20 22:28:01] (step=0013200) Train Loss: 6.3768, Train Steps/Sec: 0.86 +[2025-04-20 22:28:30] (step=0013225) Train Loss: 6.3535, Train Steps/Sec: 0.86 +[2025-04-20 22:28:59] (step=0013250) Train Loss: 6.3706, Train Steps/Sec: 0.86 +[2025-04-20 22:29:28] (step=0013275) Train Loss: 6.3682, Train Steps/Sec: 0.86 +[2025-04-20 22:29:57] (step=0013300) Train Loss: 6.3791, Train Steps/Sec: 0.86 +[2025-04-20 22:30:26] (step=0013325) Train Loss: 6.3947, Train Steps/Sec: 0.87 +[2025-04-20 22:30:55] (step=0013350) Train Loss: 6.3811, Train Steps/Sec: 0.86 +[2025-04-20 22:31:24] (step=0013375) Train Loss: 6.4003, Train Steps/Sec: 0.87 +[2025-04-20 22:31:53] (step=0013400) Train Loss: 6.3587, Train Steps/Sec: 0.86 +[2025-04-20 22:32:22] (step=0013425) Train Loss: 6.3949, Train Steps/Sec: 0.86 +[2025-04-20 22:32:51] (step=0013450) Train Loss: 6.3825, Train Steps/Sec: 0.86 +[2025-04-20 22:33:20] (step=0013475) Train Loss: 6.4066, Train Steps/Sec: 0.86 +[2025-04-20 22:33:49] (step=0013500) Train Loss: 6.3577, Train Steps/Sec: 0.86 +[2025-04-20 22:34:18] (step=0013525) Train Loss: 6.3027, Train Steps/Sec: 0.87 +[2025-04-20 22:34:47] (step=0013550) Train Loss: 6.3652, Train Steps/Sec: 0.86 +[2025-04-20 22:35:16] (step=0013575) Train Loss: 6.3487, Train Steps/Sec: 0.87 +[2025-04-20 22:35:46] (step=0013600) Train Loss: 6.4049, Train Steps/Sec: 0.85 +[2025-04-20 22:36:15] (step=0013625) Train Loss: 6.3775, Train Steps/Sec: 0.86 +[2025-04-20 22:36:44] (step=0013650) Train Loss: 6.3642, Train Steps/Sec: 0.86 +[2025-04-20 22:37:13] (step=0013675) Train Loss: 6.4123, Train Steps/Sec: 0.86 +[2025-04-20 22:37:42] (step=0013700) Train Loss: 6.3916, Train Steps/Sec: 0.86 +[2025-04-20 22:38:11] (step=0013725) Train Loss: 6.3725, Train Steps/Sec: 0.87 +[2025-04-20 22:38:40] (step=0013750) Train Loss: 6.3867, Train Steps/Sec: 0.86 +[2025-04-20 22:39:09] (step=0013775) Train Loss: 6.3989, Train Steps/Sec: 0.85 +[2025-04-20 22:39:38] (step=0013800) Train Loss: 6.3713, Train Steps/Sec: 0.86 +[2025-04-20 22:40:07] (step=0013825) Train Loss: 6.3642, Train Steps/Sec: 0.86 +[2025-04-20 22:40:42] (step=0013850) Train Loss: 6.3685, Train Steps/Sec: 0.72 +[2025-04-20 22:41:18] (step=0013875) Train Loss: 6.3532, Train Steps/Sec: 0.71 +[2025-04-20 22:41:53] (step=0013900) Train Loss: 6.3685, Train Steps/Sec: 0.71 +[2025-04-20 22:42:22] (step=0013925) Train Loss: 6.3808, Train Steps/Sec: 0.86 +[2025-04-20 22:42:51] (step=0013950) Train Loss: 6.3450, Train Steps/Sec: 0.87 +[2025-04-20 22:43:20] (step=0013975) Train Loss: 6.4151, Train Steps/Sec: 0.86 +[2025-04-20 22:43:49] (step=0014000) Train Loss: 6.3505, Train Steps/Sec: 0.86 +[2025-04-20 22:43:49] vision_config is None. initializing the InstructBlipVisionConfig with default values. +[2025-04-20 22:43:49] qformer_config is None. Initializing the InstructBlipQFormerConfig with default values. +[2025-04-20 22:43:49] text_config is None. Initializing the text config with default values (`OPTConfig`). +[2025-04-20 22:48:48] Finish Eval in 14000 steps... +[2025-04-20 22:49:08] Saved checkpoint to checkpoint/EmptyFix_InstructBlip_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_objects/002-GPT-XL/checkpoints/0014000.pt +[2025-04-20 22:49:10] Removed old checkpoint: checkpoint/EmptyFix_InstructBlip_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_objects/002-GPT-XL/checkpoints/0010000.pt +[2025-04-20 22:49:47] (step=0014025) Train Loss: 6.4268, Train Steps/Sec: 0.07 +[2025-04-20 22:50:16] (step=0014050) Train Loss: 6.3833, Train Steps/Sec: 0.86 +[2025-04-20 22:50:45] (step=0014075) Train Loss: 6.3749, Train Steps/Sec: 0.86 +[2025-04-20 22:51:15] (step=0014100) Train Loss: 6.3774, Train Steps/Sec: 0.86 +[2025-04-20 22:51:51] (step=0014125) Train Loss: 6.3558, Train Steps/Sec: 0.70 +[2025-04-20 22:52:20] (step=0014150) Train Loss: 6.3758, Train Steps/Sec: 0.85 +[2025-04-20 22:52:49] (step=0014175) Train Loss: 6.4110, Train Steps/Sec: 0.86 +[2025-04-20 22:53:18] (step=0014200) Train Loss: 6.4086, Train Steps/Sec: 0.85 +[2025-04-20 22:53:47] (step=0014225) Train Loss: 6.3940, Train Steps/Sec: 0.87 +[2025-04-20 22:54:16] (step=0014250) Train Loss: 6.3854, Train Steps/Sec: 0.86 +[2025-04-20 22:54:45] (step=0014275) Train Loss: 6.3896, Train Steps/Sec: 0.86 +[2025-04-20 22:55:14] (step=0014300) Train Loss: 6.3480, Train Steps/Sec: 0.86 +[2025-04-20 22:55:43] (step=0014325) Train Loss: 6.3828, Train Steps/Sec: 0.86 +[2025-04-20 22:56:13] (step=0014350) Train Loss: 6.3776, Train Steps/Sec: 0.86 +[2025-04-20 22:56:42] (step=0014375) Train Loss: 6.3960, Train Steps/Sec: 0.86 +[2025-04-20 22:57:11] (step=0014400) Train Loss: 6.3866, Train Steps/Sec: 0.86 +[2025-04-20 22:57:40] (step=0014425) Train Loss: 6.3974, Train Steps/Sec: 0.86 +[2025-04-20 22:58:09] (step=0014450) Train Loss: 6.3840, Train Steps/Sec: 0.86 +[2025-04-20 22:58:45] (step=0014475) Train Loss: 6.3865, Train Steps/Sec: 0.70 +[2025-04-20 22:59:14] (step=0014500) Train Loss: 6.3690, Train Steps/Sec: 0.86 +[2025-04-20 22:59:43] (step=0014525) Train Loss: 6.4192, Train Steps/Sec: 0.86 +[2025-04-20 23:00:12] (step=0014550) Train Loss: 6.3542, Train Steps/Sec: 0.87 +[2025-04-20 23:00:41] (step=0014575) Train Loss: 6.3777, Train Steps/Sec: 0.87 +[2025-04-20 23:01:10] (step=0014600) Train Loss: 6.3402, Train Steps/Sec: 0.85 +[2025-04-20 23:01:39] (step=0014625) Train Loss: 6.3705, Train Steps/Sec: 0.86 +[2025-04-20 23:02:08] (step=0014650) Train Loss: 6.3783, Train Steps/Sec: 0.86 +[2025-04-20 23:02:37] (step=0014675) Train Loss: 6.3703, Train Steps/Sec: 0.86 +[2025-04-20 23:03:06] (step=0014700) Train Loss: 6.3537, Train Steps/Sec: 0.86 +[2025-04-20 23:03:35] (step=0014725) Train Loss: 6.3969, Train Steps/Sec: 0.86 +[2025-04-20 23:04:04] (step=0014750) Train Loss: 6.4090, Train Steps/Sec: 0.86 +[2025-04-20 23:04:33] (step=0014775) Train Loss: 6.3850, Train Steps/Sec: 0.87 +[2025-04-20 23:05:02] (step=0014800) Train Loss: 6.3831, Train Steps/Sec: 0.86 +[2025-04-20 23:05:31] (step=0014825) Train Loss: 6.4021, Train Steps/Sec: 0.86 +[2025-04-20 23:06:00] (step=0014850) Train Loss: 6.3623, Train Steps/Sec: 0.87 +[2025-04-20 23:06:29] (step=0014875) Train Loss: 6.4184, Train Steps/Sec: 0.86 +[2025-04-20 23:06:58] (step=0014900) Train Loss: 6.3629, Train Steps/Sec: 0.85 +[2025-04-20 23:07:27] (step=0014925) Train Loss: 6.4217, Train Steps/Sec: 0.87 +[2025-04-20 23:07:56] (step=0014950) Train Loss: 6.3455, Train Steps/Sec: 0.86 +[2025-04-20 23:08:25] (step=0014975) Train Loss: 6.3853, Train Steps/Sec: 0.86 +[2025-04-20 23:08:54] (step=0015000) Train Loss: 6.3686, Train Steps/Sec: 0.86 +[2025-04-20 23:09:23] (step=0015025) Train Loss: 6.3725, Train Steps/Sec: 0.87 +[2025-04-20 23:09:52] (step=0015050) Train Loss: 6.4011, Train Steps/Sec: 0.87 +[2025-04-20 23:10:21] (step=0015075) Train Loss: 6.3604, Train Steps/Sec: 0.86 +[2025-04-20 23:10:50] (step=0015100) Train Loss: 6.3671, Train Steps/Sec: 0.86 +[2025-04-20 23:11:19] (step=0015125) Train Loss: 6.3969, Train Steps/Sec: 0.86 +[2025-04-20 23:11:48] (step=0015150) Train Loss: 6.3618, Train Steps/Sec: 0.86 +[2025-04-20 23:12:17] (step=0015175) Train Loss: 6.3758, Train Steps/Sec: 0.86 +[2025-04-20 23:12:46] (step=0015200) Train Loss: 6.4084, Train Steps/Sec: 0.86 +[2025-04-20 23:13:14] (step=0015225) Train Loss: 6.3245, Train Steps/Sec: 0.87 +[2025-04-20 23:13:43] (step=0015250) Train Loss: 6.3929, Train Steps/Sec: 0.86 +[2025-04-20 23:14:12] (step=0015275) Train Loss: 6.3435, Train Steps/Sec: 0.87 +[2025-04-20 23:14:42] (step=0015300) Train Loss: 6.4148, Train Steps/Sec: 0.86 +[2025-04-20 23:15:10] (step=0015325) Train Loss: 6.3818, Train Steps/Sec: 0.87 +[2025-04-20 23:15:39] (step=0015350) Train Loss: 6.3365, Train Steps/Sec: 0.87 +[2025-04-20 23:16:08] (step=0015375) Train Loss: 6.3969, Train Steps/Sec: 0.87 +[2025-04-20 23:16:37] (step=0015400) Train Loss: 6.3595, Train Steps/Sec: 0.86 +[2025-04-20 23:17:06] (step=0015425) Train Loss: 6.3598, Train Steps/Sec: 0.86 +[2025-04-20 23:17:35] (step=0015450) Train Loss: 6.3636, Train Steps/Sec: 0.87 +[2025-04-20 23:18:04] (step=0015475) Train Loss: 6.3962, Train Steps/Sec: 0.86 +[2025-04-20 23:18:33] (step=0015500) Train Loss: 6.3863, Train Steps/Sec: 0.86 +[2025-04-20 23:19:02] (step=0015525) Train Loss: 6.3677, Train Steps/Sec: 0.87 +[2025-04-20 23:19:31] (step=0015550) Train Loss: 6.3568, Train Steps/Sec: 0.86 +[2025-04-20 23:20:00] (step=0015575) Train Loss: 6.3498, Train Steps/Sec: 0.87 +[2025-04-20 23:20:29] (step=0015600) Train Loss: 6.3701, Train Steps/Sec: 0.86 +[2025-04-20 23:20:58] (step=0015625) Train Loss: 6.3735, Train Steps/Sec: 0.87 +[2025-04-20 23:21:27] (step=0015650) Train Loss: 6.3737, Train Steps/Sec: 0.86 +[2025-04-20 23:21:56] (step=0015675) Train Loss: 6.3929, Train Steps/Sec: 0.87 +[2025-04-20 23:22:25] (step=0015700) Train Loss: 6.4092, Train Steps/Sec: 0.86 +[2025-04-20 23:22:54] (step=0015725) Train Loss: 6.3625, Train Steps/Sec: 0.86 +[2025-04-20 23:23:23] (step=0015750) Train Loss: 6.3774, Train Steps/Sec: 0.86 +[2025-04-20 23:23:52] (step=0015775) Train Loss: 6.3587, Train Steps/Sec: 0.86 +[2025-04-20 23:24:21] (step=0015800) Train Loss: 6.3575, Train Steps/Sec: 0.86 +[2025-04-20 23:24:50] (step=0015825) Train Loss: 6.3569, Train Steps/Sec: 0.86 +[2025-04-20 23:25:20] (step=0015850) Train Loss: 6.3757, Train Steps/Sec: 0.85 +[2025-04-20 23:25:49] (step=0015875) Train Loss: 6.4235, Train Steps/Sec: 0.86 +[2025-04-20 23:26:18] (step=0015900) Train Loss: 6.3705, Train Steps/Sec: 0.86 +[2025-04-20 23:26:47] (step=0015925) Train Loss: 6.3307, Train Steps/Sec: 0.87 +[2025-04-20 23:27:16] (step=0015950) Train Loss: 6.3993, Train Steps/Sec: 0.86 +[2025-04-20 23:27:45] (step=0015975) Train Loss: 6.3985, Train Steps/Sec: 0.87 +[2025-04-20 23:28:14] (step=0016000) Train Loss: 6.3998, Train Steps/Sec: 0.86 +[2025-04-20 23:28:14] vision_config is None. initializing the InstructBlipVisionConfig with default values. +[2025-04-20 23:28:14] qformer_config is None. Initializing the InstructBlipQFormerConfig with default values. +[2025-04-20 23:28:14] text_config is None. Initializing the text config with default values (`OPTConfig`). +[2025-04-20 23:33:11] Finish Eval in 16000 steps... +[2025-04-20 23:33:32] Saved checkpoint to checkpoint/EmptyFix_InstructBlip_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_objects/002-GPT-XL/checkpoints/0016000.pt +[2025-04-20 23:33:34] Removed old checkpoint: checkpoint/EmptyFix_InstructBlip_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_objects/002-GPT-XL/checkpoints/0012000.pt +[2025-04-20 23:34:03] (step=0016025) Train Loss: 6.3523, Train Steps/Sec: 0.07 +[2025-04-20 23:34:38] (step=0016050) Train Loss: 6.3671, Train Steps/Sec: 0.71 +[2025-04-20 23:35:07] (step=0016075) Train Loss: 6.3546, Train Steps/Sec: 0.87 +[2025-04-20 23:35:37] (step=0016100) Train Loss: 6.3700, Train Steps/Sec: 0.86 +[2025-04-20 23:36:05] (step=0016125) Train Loss: 6.3669, Train Steps/Sec: 0.87 +[2025-04-20 23:36:34] (step=0016150) Train Loss: 6.3727, Train Steps/Sec: 0.86 +[2025-04-20 23:37:03] (step=0016175) Train Loss: 6.3665, Train Steps/Sec: 0.86 +[2025-04-20 23:37:32] (step=0016200) Train Loss: 6.3796, Train Steps/Sec: 0.87 +[2025-04-20 23:38:01] (step=0016225) Train Loss: 6.4009, Train Steps/Sec: 0.87 +[2025-04-20 23:38:30] (step=0016250) Train Loss: 6.4004, Train Steps/Sec: 0.86 +[2025-04-20 23:38:59] (step=0016275) Train Loss: 6.3455, Train Steps/Sec: 0.86 +[2025-04-20 23:39:29] (step=0016300) Train Loss: 6.3837, Train Steps/Sec: 0.85 +[2025-04-20 23:39:57] (step=0016325) Train Loss: 6.3633, Train Steps/Sec: 0.87 +[2025-04-20 23:40:26] (step=0016350) Train Loss: 6.3845, Train Steps/Sec: 0.86 +[2025-04-20 23:40:55] (step=0016375) Train Loss: 6.3522, Train Steps/Sec: 0.87 +[2025-04-20 23:41:24] (step=0016400) Train Loss: 6.3824, Train Steps/Sec: 0.86 +[2025-04-20 23:41:53] (step=0016425) Train Loss: 6.3643, Train Steps/Sec: 0.87 +[2025-04-20 23:42:22] (step=0016450) Train Loss: 6.3979, Train Steps/Sec: 0.86 +[2025-04-20 23:42:51] (step=0016475) Train Loss: 6.3899, Train Steps/Sec: 0.87 +[2025-04-20 23:43:20] (step=0016500) Train Loss: 6.3922, Train Steps/Sec: 0.86 +[2025-04-20 23:43:49] (step=0016525) Train Loss: 6.3777, Train Steps/Sec: 0.87 +[2025-04-20 23:44:18] (step=0016550) Train Loss: 6.3541, Train Steps/Sec: 0.86 +[2025-04-20 23:44:47] (step=0016575) Train Loss: 6.3560, Train Steps/Sec: 0.86 +[2025-04-20 23:45:16] (step=0016600) Train Loss: 6.4027, Train Steps/Sec: 0.87 +[2025-04-20 23:45:45] (step=0016625) Train Loss: 6.3683, Train Steps/Sec: 0.86 +[2025-04-20 23:46:14] (step=0016650) Train Loss: 6.3835, Train Steps/Sec: 0.86 +[2025-04-20 23:46:43] (step=0016675) Train Loss: 6.3814, Train Steps/Sec: 0.86 +[2025-04-20 23:47:12] (step=0016700) Train Loss: 6.3790, Train Steps/Sec: 0.86 +[2025-04-20 23:47:41] (step=0016725) Train Loss: 6.3799, Train Steps/Sec: 0.86 +[2025-04-20 23:48:10] (step=0016750) Train Loss: 6.4049, Train Steps/Sec: 0.87 +[2025-04-20 23:48:39] (step=0016775) Train Loss: 6.3800, Train Steps/Sec: 0.87 +[2025-04-20 23:49:08] (step=0016800) Train Loss: 6.3872, Train Steps/Sec: 0.86 +[2025-04-20 23:49:37] (step=0016825) Train Loss: 6.3822, Train Steps/Sec: 0.86 +[2025-04-20 23:50:06] (step=0016850) Train Loss: 6.3462, Train Steps/Sec: 0.86 +[2025-04-20 23:50:35] (step=0016875) Train Loss: 6.3791, Train Steps/Sec: 0.86 +[2025-04-20 23:51:04] (step=0016900) Train Loss: 6.3639, Train Steps/Sec: 0.85 +[2025-04-20 23:51:33] (step=0016925) Train Loss: 6.3443, Train Steps/Sec: 0.87 +[2025-04-20 23:52:02] (step=0016950) Train Loss: 6.3584, Train Steps/Sec: 0.86 +[2025-04-20 23:52:31] (step=0016975) Train Loss: 6.3316, Train Steps/Sec: 0.86 +[2025-04-20 23:53:00] (step=0017000) Train Loss: 6.3659, Train Steps/Sec: 0.86 +[2025-04-20 23:53:29] (step=0017025) Train Loss: 6.4050, Train Steps/Sec: 0.86 +[2025-04-20 23:53:58] (step=0017050) Train Loss: 6.3851, Train Steps/Sec: 0.87 +[2025-04-20 23:54:27] (step=0017075) Train Loss: 6.4069, Train Steps/Sec: 0.87 +[2025-04-20 23:54:56] (step=0017100) Train Loss: 6.4126, Train Steps/Sec: 0.86 +[2025-04-20 23:55:25] (step=0017125) Train Loss: 6.3844, Train Steps/Sec: 0.86 +[2025-04-20 23:55:54] (step=0017150) Train Loss: 6.3670, Train Steps/Sec: 0.86 +[2025-04-20 23:56:23] (step=0017175) Train Loss: 6.3529, Train Steps/Sec: 0.86 +[2025-04-20 23:56:52] (step=0017200) Train Loss: 6.4183, Train Steps/Sec: 0.86 +[2025-04-20 23:57:21] (step=0017225) Train Loss: 6.3256, Train Steps/Sec: 0.86 +[2025-04-20 23:57:50] (step=0017250) Train Loss: 6.3411, Train Steps/Sec: 0.87 +[2025-04-20 23:58:19] (step=0017275) Train Loss: 6.4084, Train Steps/Sec: 0.86 +[2025-04-20 23:58:48] (step=0017300) Train Loss: 6.4136, Train Steps/Sec: 0.86 +[2025-04-20 23:59:17] (step=0017325) Train Loss: 6.3704, Train Steps/Sec: 0.86 +[2025-04-20 23:59:46] (step=0017350) Train Loss: 6.3687, Train Steps/Sec: 0.87 +[2025-04-21 00:00:15] (step=0017375) Train Loss: 6.3995, Train Steps/Sec: 0.86 +[2025-04-21 00:00:44] (step=0017400) Train Loss: 6.3207, Train Steps/Sec: 0.86 +[2025-04-21 00:01:13] (step=0017425) Train Loss: 6.3558, Train Steps/Sec: 0.87 +[2025-04-21 00:01:42] (step=0017450) Train Loss: 6.3778, Train Steps/Sec: 0.86 +[2025-04-21 00:02:11] (step=0017475) Train Loss: 6.3614, Train Steps/Sec: 0.87 +[2025-04-21 00:02:40] (step=0017500) Train Loss: 6.3744, Train Steps/Sec: 0.85 +[2025-04-21 00:03:09] (step=0017525) Train Loss: 6.4081, Train Steps/Sec: 0.86 +[2025-04-21 00:03:38] (step=0017550) Train Loss: 6.4076, Train Steps/Sec: 0.86 +[2025-04-21 00:04:07] (step=0017575) Train Loss: 6.3773, Train Steps/Sec: 0.86 +[2025-04-21 00:04:36] (step=0017600) Train Loss: 6.3976, Train Steps/Sec: 0.86 +[2025-04-21 00:05:05] (step=0017625) Train Loss: 6.3619, Train Steps/Sec: 0.87 +[2025-04-21 00:05:34] (step=0017650) Train Loss: 6.3763, Train Steps/Sec: 0.86 +[2025-04-21 00:06:03] (step=0017675) Train Loss: 6.3551, Train Steps/Sec: 0.87 +[2025-04-21 00:06:32] (step=0017700) Train Loss: 6.3766, Train Steps/Sec: 0.86 +[2025-04-21 00:07:01] (step=0017725) Train Loss: 6.3670, Train Steps/Sec: 0.87 +[2025-04-21 00:07:30] (step=0017750) Train Loss: 6.3686, Train Steps/Sec: 0.86 +[2025-04-21 00:07:59] (step=0017775) Train Loss: 6.3492, Train Steps/Sec: 0.87 +[2025-04-21 00:08:28] (step=0017800) Train Loss: 6.4030, Train Steps/Sec: 0.86 +[2025-04-21 00:08:56] (step=0017825) Train Loss: 6.3719, Train Steps/Sec: 0.87 +[2025-04-21 00:09:25] (step=0017850) Train Loss: 6.3572, Train Steps/Sec: 0.87 +[2025-04-21 00:09:54] (step=0017875) Train Loss: 6.3440, Train Steps/Sec: 0.86 +[2025-04-21 00:10:23] (step=0017900) Train Loss: 6.3783, Train Steps/Sec: 0.86 +[2025-04-21 00:10:52] (step=0017925) Train Loss: 6.3824, Train Steps/Sec: 0.86 +[2025-04-21 00:11:21] (step=0017950) Train Loss: 6.4424, Train Steps/Sec: 0.86 +[2025-04-21 00:11:50] (step=0017975) Train Loss: 6.3693, Train Steps/Sec: 0.87 +[2025-04-21 00:12:19] (step=0018000) Train Loss: 6.3021, Train Steps/Sec: 0.86 +[2025-04-21 00:12:19] vision_config is None. initializing the InstructBlipVisionConfig with default values. +[2025-04-21 00:12:19] qformer_config is None. Initializing the InstructBlipQFormerConfig with default values. +[2025-04-21 00:12:19] text_config is None. Initializing the text config with default values (`OPTConfig`). +[2025-04-21 00:17:15] Finish Eval in 18000 steps... +[2025-04-21 00:17:36] Saved checkpoint to checkpoint/EmptyFix_InstructBlip_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_objects/002-GPT-XL/checkpoints/0018000.pt +[2025-04-21 00:17:39] Removed old checkpoint: checkpoint/EmptyFix_InstructBlip_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_objects/002-GPT-XL/checkpoints/0014000.pt +[2025-04-21 00:18:08] (step=0018025) Train Loss: 6.3746, Train Steps/Sec: 0.07 +[2025-04-21 00:18:37] (step=0018050) Train Loss: 6.3455, Train Steps/Sec: 0.86 +[2025-04-21 00:19:06] (step=0018075) Train Loss: 6.3788, Train Steps/Sec: 0.86 +[2025-04-21 00:19:34] (step=0018100) Train Loss: 6.4125, Train Steps/Sec: 0.87 +[2025-04-21 00:20:04] (step=0018125) Train Loss: 6.3575, Train Steps/Sec: 0.86 +[2025-04-21 00:20:32] (step=0018150) Train Loss: 6.3689, Train Steps/Sec: 0.87 +[2025-04-21 00:21:01] (step=0018175) Train Loss: 6.3596, Train Steps/Sec: 0.87 +[2025-04-21 00:21:30] (step=0018200) Train Loss: 6.3527, Train Steps/Sec: 0.86 +[2025-04-21 00:21:59] (step=0018225) Train Loss: 6.3432, Train Steps/Sec: 0.87 +[2025-04-21 00:22:28] (step=0018250) Train Loss: 6.3721, Train Steps/Sec: 0.86 +[2025-04-21 00:22:57] (step=0018275) Train Loss: 6.3824, Train Steps/Sec: 0.87 +[2025-04-21 00:23:26] (step=0018300) Train Loss: 6.3473, Train Steps/Sec: 0.86 +[2025-04-21 00:23:55] (step=0018325) Train Loss: 6.3399, Train Steps/Sec: 0.86 +[2025-04-21 00:24:24] (step=0018350) Train Loss: 6.3774, Train Steps/Sec: 0.86 +[2025-04-21 00:24:53] (step=0018375) Train Loss: 6.3665, Train Steps/Sec: 0.87 +[2025-04-21 00:25:22] (step=0018400) Train Loss: 6.3844, Train Steps/Sec: 0.86 +[2025-04-21 00:25:50] (step=0018425) Train Loss: 6.3861, Train Steps/Sec: 0.87 +[2025-04-21 00:26:19] (step=0018450) Train Loss: 6.3966, Train Steps/Sec: 0.87 +[2025-04-21 00:26:49] (step=0018475) Train Loss: 6.3617, Train Steps/Sec: 0.86 +[2025-04-21 00:27:18] (step=0018500) Train Loss: 6.3432, Train Steps/Sec: 0.86 +[2025-04-21 00:27:46] (step=0018525) Train Loss: 6.3543, Train Steps/Sec: 0.87 +[2025-04-21 00:28:16] (step=0018550) Train Loss: 6.3576, Train Steps/Sec: 0.86 +[2025-04-21 00:28:45] (step=0018575) Train Loss: 6.4320, Train Steps/Sec: 0.86 +[2025-04-21 00:29:13] (step=0018600) Train Loss: 6.3880, Train Steps/Sec: 0.87 +[2025-04-21 00:29:42] (step=0018625) Train Loss: 6.3513, Train Steps/Sec: 0.86 +[2025-04-21 00:30:11] (step=0018650) Train Loss: 6.3604, Train Steps/Sec: 0.86 +[2025-04-21 00:30:40] (step=0018675) Train Loss: 6.3923, Train Steps/Sec: 0.86 +[2025-04-21 00:31:09] (step=0018700) Train Loss: 6.3354, Train Steps/Sec: 0.86 +[2025-04-21 00:31:38] (step=0018725) Train Loss: 6.3351, Train Steps/Sec: 0.87 +[2025-04-21 00:32:07] (step=0018750) Train Loss: 6.3858, Train Steps/Sec: 0.85 +[2025-04-21 00:32:36] (step=0018775) Train Loss: 6.3527, Train Steps/Sec: 0.86 +[2025-04-21 00:33:05] (step=0018800) Train Loss: 6.3586, Train Steps/Sec: 0.86 +[2025-04-21 00:33:34] (step=0018825) Train Loss: 6.4239, Train Steps/Sec: 0.86 +[2025-04-21 00:34:03] (step=0018850) Train Loss: 6.4103, Train Steps/Sec: 0.86 +[2025-04-21 00:34:33] (step=0018875) Train Loss: 6.3972, Train Steps/Sec: 0.86 +[2025-04-21 00:35:08] (step=0018900) Train Loss: 6.3541, Train Steps/Sec: 0.71 +[2025-04-21 00:35:36] (step=0018925) Train Loss: 6.3782, Train Steps/Sec: 0.87 +[2025-04-21 00:36:06] (step=0018950) Train Loss: 6.3705, Train Steps/Sec: 0.86 +[2025-04-21 00:36:35] (step=0018975) Train Loss: 6.3574, Train Steps/Sec: 0.86 +[2025-04-21 00:37:10] (step=0019000) Train Loss: 6.3524, Train Steps/Sec: 0.71 +[2025-04-21 00:37:45] (step=0019025) Train Loss: 6.3442, Train Steps/Sec: 0.71 +[2025-04-21 00:38:14] (step=0019050) Train Loss: 6.3618, Train Steps/Sec: 0.86 +[2025-04-21 00:38:43] (step=0019075) Train Loss: 6.3709, Train Steps/Sec: 0.86 +[2025-04-21 00:39:12] (step=0019100) Train Loss: 6.3712, Train Steps/Sec: 0.86 +[2025-04-21 00:39:41] (step=0019125) Train Loss: 6.3677, Train Steps/Sec: 0.86 +[2025-04-21 00:40:10] (step=0019150) Train Loss: 6.3478, Train Steps/Sec: 0.87 +[2025-04-21 00:40:45] (step=0019175) Train Loss: 6.3293, Train Steps/Sec: 0.70 +[2025-04-21 00:41:15] (step=0019200) Train Loss: 6.3818, Train Steps/Sec: 0.85 +[2025-04-21 00:41:44] (step=0019225) Train Loss: 6.3957, Train Steps/Sec: 0.86 +[2025-04-21 00:42:13] (step=0019250) Train Loss: 6.3767, Train Steps/Sec: 0.87 +[2025-04-21 00:42:42] (step=0019275) Train Loss: 6.4022, Train Steps/Sec: 0.86 +[2025-04-21 00:43:11] (step=0019300) Train Loss: 6.3559, Train Steps/Sec: 0.86 +[2025-04-21 00:43:40] (step=0019325) Train Loss: 6.3869, Train Steps/Sec: 0.86 +[2025-04-21 00:44:09] (step=0019350) Train Loss: 6.3561, Train Steps/Sec: 0.86 +[2025-04-21 00:44:38] (step=0019375) Train Loss: 6.3513, Train Steps/Sec: 0.86 +[2025-04-21 00:45:07] (step=0019400) Train Loss: 6.3877, Train Steps/Sec: 0.86 +[2025-04-21 00:45:36] (step=0019425) Train Loss: 6.3591, Train Steps/Sec: 0.86 +[2025-04-21 00:46:05] (step=0019450) Train Loss: 6.3699, Train Steps/Sec: 0.87 +[2025-04-21 00:46:34] (step=0019475) Train Loss: 6.3576, Train Steps/Sec: 0.86 +[2025-04-21 00:47:03] (step=0019500) Train Loss: 6.3756, Train Steps/Sec: 0.86 +[2025-04-21 00:47:32] (step=0019525) Train Loss: 6.3557, Train Steps/Sec: 0.86 +[2025-04-21 00:48:01] (step=0019550) Train Loss: 6.3574, Train Steps/Sec: 0.86 +[2025-04-21 00:48:30] (step=0019575) Train Loss: 6.4242, Train Steps/Sec: 0.86 +[2025-04-21 00:48:59] (step=0019600) Train Loss: 6.3368, Train Steps/Sec: 0.86 +[2025-04-21 00:49:35] (step=0019625) Train Loss: 6.4080, Train Steps/Sec: 0.70 +[2025-04-21 00:50:04] (step=0019650) Train Loss: 6.3751, Train Steps/Sec: 0.86 +[2025-04-21 00:50:33] (step=0019675) Train Loss: 6.3850, Train Steps/Sec: 0.86 +[2025-04-21 00:51:02] (step=0019700) Train Loss: 6.3372, Train Steps/Sec: 0.87 +[2025-04-21 00:51:31] (step=0019725) Train Loss: 6.3665, Train Steps/Sec: 0.87 +[2025-04-21 00:52:00] (step=0019750) Train Loss: 6.4102, Train Steps/Sec: 0.87 +[2025-04-21 00:52:28] (step=0019775) Train Loss: 6.3881, Train Steps/Sec: 0.87 +[2025-04-21 00:52:57] (step=0019800) Train Loss: 6.3235, Train Steps/Sec: 0.86 +[2025-04-21 00:53:26] (step=0019825) Train Loss: 6.3425, Train Steps/Sec: 0.87 +[2025-04-21 00:53:55] (step=0019850) Train Loss: 6.3572, Train Steps/Sec: 0.87 +[2025-04-21 00:54:24] (step=0019875) Train Loss: 6.3215, Train Steps/Sec: 0.86 +[2025-04-21 00:54:53] (step=0019900) Train Loss: 6.3700, Train Steps/Sec: 0.86 +[2025-04-21 00:55:22] (step=0019925) Train Loss: 6.3583, Train Steps/Sec: 0.87 +[2025-04-21 00:55:51] (step=0019950) Train Loss: 6.3661, Train Steps/Sec: 0.86 +[2025-04-21 00:56:20] (step=0019975) Train Loss: 6.3822, Train Steps/Sec: 0.86 +[2025-04-21 00:56:49] (step=0020000) Train Loss: 6.3599, Train Steps/Sec: 0.86 +[2025-04-21 00:56:49] vision_config is None. initializing the InstructBlipVisionConfig with default values. +[2025-04-21 00:56:49] qformer_config is None. Initializing the InstructBlipQFormerConfig with default values. +[2025-04-21 00:56:49] text_config is None. Initializing the text config with default values (`OPTConfig`). +[2025-04-21 01:01:45] Finish Eval in 20000 steps... +[2025-04-21 01:02:06] Saved checkpoint to checkpoint/EmptyFix_InstructBlip_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_objects/002-GPT-XL/checkpoints/0020000.pt +[2025-04-21 01:02:09] Removed old checkpoint: checkpoint/EmptyFix_InstructBlip_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_objects/002-GPT-XL/checkpoints/0016000.pt +[2025-04-21 01:02:45] (step=0020025) Train Loss: 6.4007, Train Steps/Sec: 0.07 +[2025-04-21 01:03:14] (step=0020050) Train Loss: 6.3626, Train Steps/Sec: 0.86 +[2025-04-21 01:03:43] (step=0020075) Train Loss: 6.3841, Train Steps/Sec: 0.86 +[2025-04-21 01:04:12] (step=0020100) Train Loss: 6.3703, Train Steps/Sec: 0.86 +[2025-04-21 01:04:41] (step=0020125) Train Loss: 6.3846, Train Steps/Sec: 0.86 +[2025-04-21 01:05:10] (step=0020150) Train Loss: 6.4086, Train Steps/Sec: 0.86 +[2025-04-21 01:05:39] (step=0020175) Train Loss: 6.3703, Train Steps/Sec: 0.86 +[2025-04-21 01:06:08] (step=0020200) Train Loss: 6.4060, Train Steps/Sec: 0.85 +[2025-04-21 01:06:37] (step=0020225) Train Loss: 6.3570, Train Steps/Sec: 0.86 +[2025-04-21 01:07:06] (step=0020250) Train Loss: 6.3316, Train Steps/Sec: 0.87 +[2025-04-21 01:07:35] (step=0020275) Train Loss: 6.4294, Train Steps/Sec: 0.86 +[2025-04-21 01:08:04] (step=0020300) Train Loss: 6.3632, Train Steps/Sec: 0.86 +[2025-04-21 01:08:34] (step=0020325) Train Loss: 6.3973, Train Steps/Sec: 0.85 +[2025-04-21 01:09:03] (step=0020350) Train Loss: 6.3975, Train Steps/Sec: 0.86 +[2025-04-21 01:09:32] (step=0020375) Train Loss: 6.3380, Train Steps/Sec: 0.86 +[2025-04-21 01:10:01] (step=0020400) Train Loss: 6.3360, Train Steps/Sec: 0.86 +[2025-04-21 01:10:30] (step=0020425) Train Loss: 6.3581, Train Steps/Sec: 0.87 +[2025-04-21 01:10:59] (step=0020450) Train Loss: 6.3592, Train Steps/Sec: 0.86 +[2025-04-21 01:11:28] (step=0020475) Train Loss: 6.3668, Train Steps/Sec: 0.87 +[2025-04-21 01:11:57] (step=0020500) Train Loss: 6.3508, Train Steps/Sec: 0.86 +[2025-04-21 01:12:26] (step=0020525) Train Loss: 6.4139, Train Steps/Sec: 0.86 +[2025-04-21 01:12:55] (step=0020550) Train Loss: 6.3422, Train Steps/Sec: 0.86 +[2025-04-21 01:13:24] (step=0020575) Train Loss: 6.3582, Train Steps/Sec: 0.87 +[2025-04-21 01:13:53] (step=0020600) Train Loss: 6.3568, Train Steps/Sec: 0.86 +[2025-04-21 01:14:21] (step=0020625) Train Loss: 6.4143, Train Steps/Sec: 0.86 +[2025-04-21 01:14:50] (step=0020650) Train Loss: 6.3262, Train Steps/Sec: 0.86 +[2025-04-21 01:15:19] (step=0020675) Train Loss: 6.3309, Train Steps/Sec: 0.87 +[2025-04-21 01:15:48] (step=0020700) Train Loss: 6.3539, Train Steps/Sec: 0.86 +[2025-04-21 01:16:17] (step=0020725) Train Loss: 6.3750, Train Steps/Sec: 0.87 +[2025-04-21 01:16:46] (step=0020750) Train Loss: 6.3620, Train Steps/Sec: 0.86 +[2025-04-21 01:17:15] (step=0020775) Train Loss: 6.3677, Train Steps/Sec: 0.86 +[2025-04-21 01:17:44] (step=0020800) Train Loss: 6.3786, Train Steps/Sec: 0.86 +[2025-04-21 01:18:13] (step=0020825) Train Loss: 6.3137, Train Steps/Sec: 0.87 +[2025-04-21 01:18:42] (step=0020850) Train Loss: 6.3338, Train Steps/Sec: 0.86 +[2025-04-21 01:19:11] (step=0020875) Train Loss: 6.3547, Train Steps/Sec: 0.86 +[2025-04-21 01:19:40] (step=0020900) Train Loss: 6.3664, Train Steps/Sec: 0.86 +[2025-04-21 01:20:09] (step=0020925) Train Loss: 6.3483, Train Steps/Sec: 0.86 +[2025-04-21 01:20:38] (step=0020950) Train Loss: 6.3652, Train Steps/Sec: 0.86 +[2025-04-21 01:21:07] (step=0020975) Train Loss: 6.4175, Train Steps/Sec: 0.86 +[2025-04-21 01:21:36] (step=0021000) Train Loss: 6.3432, Train Steps/Sec: 0.86 +[2025-04-21 01:22:05] (step=0021025) Train Loss: 6.4031, Train Steps/Sec: 0.86 +[2025-04-21 01:22:34] (step=0021050) Train Loss: 6.4267, Train Steps/Sec: 0.86 +[2025-04-21 01:23:03] (step=0021075) Train Loss: 6.3494, Train Steps/Sec: 0.87 +[2025-04-21 01:23:32] (step=0021100) Train Loss: 6.3606, Train Steps/Sec: 0.86 +[2025-04-21 01:24:01] (step=0021125) Train Loss: 6.3786, Train Steps/Sec: 0.86 +[2025-04-21 01:24:30] (step=0021150) Train Loss: 6.3281, Train Steps/Sec: 0.86 +[2025-04-21 01:24:59] (step=0021175) Train Loss: 6.3491, Train Steps/Sec: 0.87 +[2025-04-21 01:25:28] (step=0021200) Train Loss: 6.3257, Train Steps/Sec: 0.86 +[2025-04-21 01:25:57] (step=0021225) Train Loss: 6.3485, Train Steps/Sec: 0.87 +[2025-04-21 01:26:26] (step=0021250) Train Loss: 6.3734, Train Steps/Sec: 0.86 +[2025-04-21 01:26:55] (step=0021275) Train Loss: 6.3573, Train Steps/Sec: 0.86 +[2025-04-21 01:27:24] (step=0021300) Train Loss: 6.3836, Train Steps/Sec: 0.86 +[2025-04-21 01:27:53] (step=0021325) Train Loss: 6.3561, Train Steps/Sec: 0.87 +[2025-04-21 01:28:22] (step=0021350) Train Loss: 6.3639, Train Steps/Sec: 0.87 +[2025-04-21 01:28:51] (step=0021375) Train Loss: 6.3065, Train Steps/Sec: 0.87 +[2025-04-21 01:29:26] (step=0021400) Train Loss: 6.3759, Train Steps/Sec: 0.70 +[2025-04-21 01:29:55] (step=0021425) Train Loss: 6.3554, Train Steps/Sec: 0.87 +[2025-04-21 01:30:24] (step=0021450) Train Loss: 6.4091, Train Steps/Sec: 0.86 +[2025-04-21 01:30:53] (step=0021475) Train Loss: 6.3991, Train Steps/Sec: 0.86 +[2025-04-21 01:31:22] (step=0021500) Train Loss: 6.3791, Train Steps/Sec: 0.86 +[2025-04-21 01:31:51] (step=0021525) Train Loss: 6.3652, Train Steps/Sec: 0.87 +[2025-04-21 01:32:20] (step=0021550) Train Loss: 6.3358, Train Steps/Sec: 0.87 +[2025-04-21 01:32:49] (step=0021575) Train Loss: 6.3758, Train Steps/Sec: 0.86 +[2025-04-21 01:33:18] (step=0021600) Train Loss: 6.3597, Train Steps/Sec: 0.86 +[2025-04-21 01:33:47] (step=0021625) Train Loss: 6.3683, Train Steps/Sec: 0.86 +[2025-04-21 01:34:16] (step=0021650) Train Loss: 6.3814, Train Steps/Sec: 0.86 +[2025-04-21 01:34:45] (step=0021675) Train Loss: 6.3356, Train Steps/Sec: 0.86 +[2025-04-21 01:35:14] (step=0021700) Train Loss: 6.3326, Train Steps/Sec: 0.86 +[2025-04-21 01:35:43] (step=0021725) Train Loss: 6.3467, Train Steps/Sec: 0.86 +[2025-04-21 01:36:12] (step=0021750) Train Loss: 6.3465, Train Steps/Sec: 0.86 +[2025-04-21 01:36:41] (step=0021775) Train Loss: 6.3814, Train Steps/Sec: 0.87 +[2025-04-21 01:37:10] (step=0021800) Train Loss: 6.3341, Train Steps/Sec: 0.86 +[2025-04-21 01:37:39] (step=0021825) Train Loss: 6.3846, Train Steps/Sec: 0.86 +[2025-04-21 01:38:08] (step=0021850) Train Loss: 6.3585, Train Steps/Sec: 0.86 +[2025-04-21 01:38:37] (step=0021875) Train Loss: 6.3686, Train Steps/Sec: 0.86 +[2025-04-21 01:39:06] (step=0021900) Train Loss: 6.3652, Train Steps/Sec: 0.86 +[2025-04-21 01:39:35] (step=0021925) Train Loss: 6.4040, Train Steps/Sec: 0.86 +[2025-04-21 01:40:04] (step=0021950) Train Loss: 6.3196, Train Steps/Sec: 0.86 +[2025-04-21 01:40:33] (step=0021975) Train Loss: 6.3810, Train Steps/Sec: 0.87 +[2025-04-21 01:41:02] (step=0022000) Train Loss: 6.3708, Train Steps/Sec: 0.86 +[2025-04-21 01:41:02] vision_config is None. initializing the InstructBlipVisionConfig with default values. +[2025-04-21 01:41:02] qformer_config is None. Initializing the InstructBlipQFormerConfig with default values. +[2025-04-21 01:41:02] text_config is None. Initializing the text config with default values (`OPTConfig`). +[2025-04-21 01:45:59] Finish Eval in 22000 steps... +[2025-04-21 01:46:20] Saved checkpoint to checkpoint/EmptyFix_InstructBlip_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_objects/002-GPT-XL/checkpoints/0022000.pt +[2025-04-21 01:46:22] Removed old checkpoint: checkpoint/EmptyFix_InstructBlip_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_objects/002-GPT-XL/checkpoints/0018000.pt +[2025-04-21 01:46:52] (step=0022025) Train Loss: 6.3658, Train Steps/Sec: 0.07 +[2025-04-21 01:47:20] (step=0022050) Train Loss: 6.3666, Train Steps/Sec: 0.87 +[2025-04-21 01:47:49] (step=0022075) Train Loss: 6.3475, Train Steps/Sec: 0.86 +[2025-04-21 01:48:20] (step=0022100) Train Loss: 6.3346, Train Steps/Sec: 0.81 +[2025-04-21 01:48:49] (step=0022125) Train Loss: 6.2973, Train Steps/Sec: 0.86 +[2025-04-21 01:49:18] (step=0022150) Train Loss: 6.3615, Train Steps/Sec: 0.86 +[2025-04-21 01:49:47] (step=0022175) Train Loss: 6.4504, Train Steps/Sec: 0.85 +[2025-04-21 01:50:17] (step=0022200) Train Loss: 6.3179, Train Steps/Sec: 0.86 +[2025-04-21 01:50:45] (step=0022225) Train Loss: 6.3681, Train Steps/Sec: 0.86 +[2025-04-21 01:51:14] (step=0022250) Train Loss: 6.3423, Train Steps/Sec: 0.87 +[2025-04-21 01:51:43] (step=0022275) Train Loss: 6.3634, Train Steps/Sec: 0.87 +[2025-04-21 01:52:12] (step=0022300) Train Loss: 6.3722, Train Steps/Sec: 0.86 +[2025-04-21 01:52:41] (step=0022325) Train Loss: 6.3243, Train Steps/Sec: 0.86 +[2025-04-21 01:53:10] (step=0022350) Train Loss: 6.3421, Train Steps/Sec: 0.86 +[2025-04-21 01:53:39] (step=0022375) Train Loss: 6.3052, Train Steps/Sec: 0.86 +[2025-04-21 01:54:08] (step=0022400) Train Loss: 6.3891, Train Steps/Sec: 0.86 +[2025-04-21 01:54:37] (step=0022425) Train Loss: 6.3591, Train Steps/Sec: 0.86 +[2025-04-21 01:55:06] (step=0022450) Train Loss: 6.3219, Train Steps/Sec: 0.86 +[2025-04-21 01:55:35] (step=0022475) Train Loss: 6.4234, Train Steps/Sec: 0.86 +[2025-04-21 01:56:05] (step=0022500) Train Loss: 6.3545, Train Steps/Sec: 0.86 +[2025-04-21 01:56:34] (step=0022525) Train Loss: 6.3259, Train Steps/Sec: 0.86 +[2025-04-21 01:57:02] (step=0022550) Train Loss: 6.3740, Train Steps/Sec: 0.87 +[2025-04-21 01:57:31] (step=0022575) Train Loss: 6.3532, Train Steps/Sec: 0.86 +[2025-04-21 01:58:01] (step=0022600) Train Loss: 6.3768, Train Steps/Sec: 0.86 +[2025-04-21 01:58:30] (step=0022625) Train Loss: 6.3661, Train Steps/Sec: 0.86 +[2025-04-21 01:58:59] (step=0022650) Train Loss: 6.3682, Train Steps/Sec: 0.86 +[2025-04-21 01:59:28] (step=0022675) Train Loss: 6.3293, Train Steps/Sec: 0.87 +[2025-04-21 01:59:57] (step=0022700) Train Loss: 6.3710, Train Steps/Sec: 0.86 +[2025-04-21 02:00:26] (step=0022725) Train Loss: 6.4127, Train Steps/Sec: 0.87 +[2025-04-21 02:00:55] (step=0022750) Train Loss: 6.3910, Train Steps/Sec: 0.86 +[2025-04-21 02:01:23] (step=0022775) Train Loss: 6.3761, Train Steps/Sec: 0.87 +[2025-04-21 02:01:52] (step=0022800) Train Loss: 6.3517, Train Steps/Sec: 0.86 +[2025-04-21 02:02:21] (step=0022825) Train Loss: 6.4041, Train Steps/Sec: 0.86 +[2025-04-21 02:02:50] (step=0022850) Train Loss: 6.3618, Train Steps/Sec: 0.87 +[2025-04-21 02:03:19] (step=0022875) Train Loss: 6.3327, Train Steps/Sec: 0.86 +[2025-04-21 02:03:48] (step=0022900) Train Loss: 6.3585, Train Steps/Sec: 0.86 +[2025-04-21 02:04:17] (step=0022925) Train Loss: 6.3672, Train Steps/Sec: 0.87 +[2025-04-21 02:04:46] (step=0022950) Train Loss: 6.3508, Train Steps/Sec: 0.86 +[2025-04-21 02:05:15] (step=0022975) Train Loss: 6.3416, Train Steps/Sec: 0.86 +[2025-04-21 02:05:44] (step=0023000) Train Loss: 6.3727, Train Steps/Sec: 0.86 +[2025-04-21 02:06:13] (step=0023025) Train Loss: 6.3308, Train Steps/Sec: 0.87 +[2025-04-21 02:06:42] (step=0023050) Train Loss: 6.3897, Train Steps/Sec: 0.86 +[2025-04-21 02:07:11] (step=0023075) Train Loss: 6.3515, Train Steps/Sec: 0.87 +[2025-04-21 02:07:40] (step=0023100) Train Loss: 6.3310, Train Steps/Sec: 0.87 +[2025-04-21 02:08:09] (step=0023125) Train Loss: 6.3806, Train Steps/Sec: 0.86 +[2025-04-21 02:08:38] (step=0023150) Train Loss: 6.3478, Train Steps/Sec: 0.87 +[2025-04-21 02:09:07] (step=0023175) Train Loss: 6.3624, Train Steps/Sec: 0.87 +[2025-04-21 02:09:36] (step=0023200) Train Loss: 6.4122, Train Steps/Sec: 0.86 +[2025-04-21 02:10:05] (step=0023225) Train Loss: 6.3619, Train Steps/Sec: 0.86 +[2025-04-21 02:10:34] (step=0023250) Train Loss: 6.3752, Train Steps/Sec: 0.86 +[2025-04-21 02:11:03] (step=0023275) Train Loss: 6.3531, Train Steps/Sec: 0.86 +[2025-04-21 02:11:32] (step=0023300) Train Loss: 6.3249, Train Steps/Sec: 0.86 +[2025-04-21 02:12:01] (step=0023325) Train Loss: 6.3501, Train Steps/Sec: 0.86 +[2025-04-21 02:12:30] (step=0023350) Train Loss: 6.3412, Train Steps/Sec: 0.87 +[2025-04-21 02:12:59] (step=0023375) Train Loss: 6.3464, Train Steps/Sec: 0.87 +[2025-04-21 02:13:28] (step=0023400) Train Loss: 6.3408, Train Steps/Sec: 0.85 +[2025-04-21 02:13:57] (step=0023425) Train Loss: 6.3395, Train Steps/Sec: 0.86 +[2025-04-21 02:14:26] (step=0023450) Train Loss: 6.3297, Train Steps/Sec: 0.87 +[2025-04-21 02:14:55] (step=0023475) Train Loss: 6.3751, Train Steps/Sec: 0.86 +[2025-04-21 02:15:24] (step=0023500) Train Loss: 6.3128, Train Steps/Sec: 0.86 +[2025-04-21 02:15:53] (step=0023525) Train Loss: 6.3439, Train Steps/Sec: 0.86 +[2025-04-21 02:16:22] (step=0023550) Train Loss: 6.3733, Train Steps/Sec: 0.86 +[2025-04-21 02:16:51] (step=0023575) Train Loss: 6.3629, Train Steps/Sec: 0.86 +[2025-04-21 02:17:20] (step=0023600) Train Loss: 6.3712, Train Steps/Sec: 0.86 +[2025-04-21 02:17:49] (step=0023625) Train Loss: 6.3663, Train Steps/Sec: 0.86 +[2025-04-21 02:18:18] (step=0023650) Train Loss: 6.3499, Train Steps/Sec: 0.87 +[2025-04-21 02:18:47] (step=0023675) Train Loss: 6.4199, Train Steps/Sec: 0.86 +[2025-04-21 02:19:16] (step=0023700) Train Loss: 6.3506, Train Steps/Sec: 0.87 +[2025-04-21 02:19:45] (step=0023725) Train Loss: 6.3917, Train Steps/Sec: 0.86 +[2025-04-21 02:20:14] (step=0023750) Train Loss: 6.3995, Train Steps/Sec: 0.86 +[2025-04-21 02:20:43] (step=0023775) Train Loss: 6.3579, Train Steps/Sec: 0.87 +[2025-04-21 02:21:12] (step=0023800) Train Loss: 6.3829, Train Steps/Sec: 0.86 +[2025-04-21 02:21:41] (step=0023825) Train Loss: 6.3539, Train Steps/Sec: 0.86 +[2025-04-21 02:22:10] (step=0023850) Train Loss: 6.3477, Train Steps/Sec: 0.86 +[2025-04-21 02:22:39] (step=0023875) Train Loss: 6.3316, Train Steps/Sec: 0.86 +[2025-04-21 02:23:08] (step=0023900) Train Loss: 6.3634, Train Steps/Sec: 0.86 +[2025-04-21 02:23:37] (step=0023925) Train Loss: 6.3681, Train Steps/Sec: 0.86 +[2025-04-21 02:24:06] (step=0023950) Train Loss: 6.3486, Train Steps/Sec: 0.86 +[2025-04-21 02:24:35] (step=0023975) Train Loss: 6.3461, Train Steps/Sec: 0.87 +[2025-04-21 02:25:10] (step=0024000) Train Loss: 6.4093, Train Steps/Sec: 0.72 +[2025-04-21 02:25:10] vision_config is None. initializing the InstructBlipVisionConfig with default values. +[2025-04-21 02:25:10] qformer_config is None. Initializing the InstructBlipQFormerConfig with default values. +[2025-04-21 02:25:10] text_config is None. Initializing the text config with default values (`OPTConfig`). +[2025-04-21 02:30:12] Finish Eval in 24000 steps... +[2025-04-21 02:30:34] Saved checkpoint to checkpoint/EmptyFix_InstructBlip_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_objects/002-GPT-XL/checkpoints/0024000.pt +[2025-04-21 02:30:36] Removed old checkpoint: checkpoint/EmptyFix_InstructBlip_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_objects/002-GPT-XL/checkpoints/0020000.pt +[2025-04-21 02:31:06] (step=0024025) Train Loss: 6.3558, Train Steps/Sec: 0.07 +[2025-04-21 02:31:35] (step=0024050) Train Loss: 6.3518, Train Steps/Sec: 0.86 +[2025-04-21 02:32:04] (step=0024075) Train Loss: 6.3659, Train Steps/Sec: 0.86 +[2025-04-21 02:32:33] (step=0024100) Train Loss: 6.4010, Train Steps/Sec: 0.86 +[2025-04-21 02:33:02] (step=0024125) Train Loss: 6.3769, Train Steps/Sec: 0.86 +[2025-04-21 02:33:37] (step=0024150) Train Loss: 6.3587, Train Steps/Sec: 0.71 +[2025-04-21 02:34:11] (step=0024175) Train Loss: 6.3630, Train Steps/Sec: 0.73 +[2025-04-21 02:34:41] (step=0024200) Train Loss: 6.3830, Train Steps/Sec: 0.85 +[2025-04-21 02:35:10] (step=0024225) Train Loss: 6.3690, Train Steps/Sec: 0.86 +[2025-04-21 02:35:39] (step=0024250) Train Loss: 6.3824, Train Steps/Sec: 0.86 +[2025-04-21 02:36:08] (step=0024275) Train Loss: 6.3836, Train Steps/Sec: 0.86 +[2025-04-21 02:36:37] (step=0024300) Train Loss: 6.3332, Train Steps/Sec: 0.86 +[2025-04-21 02:37:13] (step=0024325) Train Loss: 6.3419, Train Steps/Sec: 0.69 +[2025-04-21 02:37:42] (step=0024350) Train Loss: 6.3480, Train Steps/Sec: 0.86 +[2025-04-21 02:38:11] (step=0024375) Train Loss: 6.3769, Train Steps/Sec: 0.87 +[2025-04-21 02:38:40] (step=0024400) Train Loss: 6.3625, Train Steps/Sec: 0.86 +[2025-04-21 02:39:09] (step=0024425) Train Loss: 6.3737, Train Steps/Sec: 0.87 +[2025-04-21 02:39:38] (step=0024450) Train Loss: 6.3468, Train Steps/Sec: 0.86 +[2025-04-21 02:40:07] (step=0024475) Train Loss: 6.3555, Train Steps/Sec: 0.86 +[2025-04-21 02:40:36] (step=0024500) Train Loss: 6.3516, Train Steps/Sec: 0.86 +[2025-04-21 02:41:05] (step=0024525) Train Loss: 6.3670, Train Steps/Sec: 0.86 +[2025-04-21 02:41:34] (step=0024550) Train Loss: 6.3716, Train Steps/Sec: 0.86 +[2025-04-21 02:42:03] (step=0024575) Train Loss: 6.3686, Train Steps/Sec: 0.86 +[2025-04-21 02:42:32] (step=0024600) Train Loss: 6.3718, Train Steps/Sec: 0.86 +[2025-04-21 02:43:01] (step=0024625) Train Loss: 6.3501, Train Steps/Sec: 0.87 +[2025-04-21 02:43:30] (step=0024650) Train Loss: 6.3442, Train Steps/Sec: 0.85 +[2025-04-21 02:43:59] (step=0024675) Train Loss: 6.3780, Train Steps/Sec: 0.86 +[2025-04-21 02:44:28] (step=0024700) Train Loss: 6.3514, Train Steps/Sec: 0.86 +[2025-04-21 02:44:57] (step=0024725) Train Loss: 6.3843, Train Steps/Sec: 0.87 +[2025-04-21 02:45:26] (step=0024750) Train Loss: 6.3634, Train Steps/Sec: 0.87 +[2025-04-21 02:45:55] (step=0024775) Train Loss: 6.3045, Train Steps/Sec: 0.86 +[2025-04-21 02:46:24] (step=0024800) Train Loss: 6.4016, Train Steps/Sec: 0.86 +[2025-04-21 02:46:53] (step=0024825) Train Loss: 6.3643, Train Steps/Sec: 0.86 +[2025-04-21 02:47:22] (step=0024850) Train Loss: 6.3527, Train Steps/Sec: 0.86 +[2025-04-21 02:47:51] (step=0024875) Train Loss: 6.3577, Train Steps/Sec: 0.86 +[2025-04-21 02:48:20] (step=0024900) Train Loss: 6.3228, Train Steps/Sec: 0.86 +[2025-04-21 02:48:50] (step=0024925) Train Loss: 6.3498, Train Steps/Sec: 0.85 +[2025-04-21 02:49:19] (step=0024950) Train Loss: 6.3428, Train Steps/Sec: 0.86 +[2025-04-21 02:49:48] (step=0024975) Train Loss: 6.3687, Train Steps/Sec: 0.86 +[2025-04-21 02:50:17] (step=0025000) Train Loss: 6.3424, Train Steps/Sec: 0.86 +[2025-04-21 02:50:46] (step=0025025) Train Loss: 6.3565, Train Steps/Sec: 0.86 +[2025-04-21 02:51:15] (step=0025050) Train Loss: 6.3661, Train Steps/Sec: 0.87 +[2025-04-21 02:51:44] (step=0025075) Train Loss: 6.3491, Train Steps/Sec: 0.86 +[2025-04-21 02:52:20] (step=0025100) Train Loss: 6.3811, Train Steps/Sec: 0.70 +[2025-04-21 02:52:49] (step=0025125) Train Loss: 6.3843, Train Steps/Sec: 0.86 +[2025-04-21 02:53:18] (step=0025150) Train Loss: 6.3637, Train Steps/Sec: 0.86 +[2025-04-21 02:53:47] (step=0025175) Train Loss: 6.3255, Train Steps/Sec: 0.86 +[2025-04-21 02:54:16] (step=0025200) Train Loss: 6.3621, Train Steps/Sec: 0.86 +[2025-04-21 02:54:45] (step=0025225) Train Loss: 6.3791, Train Steps/Sec: 0.86 +[2025-04-21 02:55:14] (step=0025250) Train Loss: 6.4018, Train Steps/Sec: 0.86 +[2025-04-21 02:55:43] (step=0025275) Train Loss: 6.3284, Train Steps/Sec: 0.86 +[2025-04-21 02:56:12] (step=0025300) Train Loss: 6.3475, Train Steps/Sec: 0.86 +[2025-04-21 02:56:42] (step=0025325) Train Loss: 6.3283, Train Steps/Sec: 0.86 +[2025-04-21 02:57:11] (step=0025350) Train Loss: 6.3511, Train Steps/Sec: 0.86 +[2025-04-21 02:57:40] (step=0025375) Train Loss: 6.3670, Train Steps/Sec: 0.86 +[2025-04-21 02:58:09] (step=0025400) Train Loss: 6.3386, Train Steps/Sec: 0.86 +[2025-04-21 02:58:38] (step=0025425) Train Loss: 6.3750, Train Steps/Sec: 0.86 +[2025-04-21 02:59:07] (step=0025450) Train Loss: 6.3268, Train Steps/Sec: 0.87 +[2025-04-21 02:59:36] (step=0025475) Train Loss: 6.3573, Train Steps/Sec: 0.87 +[2025-04-21 03:00:05] (step=0025500) Train Loss: 6.3382, Train Steps/Sec: 0.86 +[2025-04-21 03:00:34] (step=0025525) Train Loss: 6.3784, Train Steps/Sec: 0.86 +[2025-04-21 03:01:02] (step=0025550) Train Loss: 6.3222, Train Steps/Sec: 0.87 +[2025-04-21 03:01:38] (step=0025575) Train Loss: 6.3341, Train Steps/Sec: 0.70 +[2025-04-21 03:02:07] (step=0025600) Train Loss: 6.3751, Train Steps/Sec: 0.86 +[2025-04-21 03:02:36] (step=0025625) Train Loss: 6.3363, Train Steps/Sec: 0.86 +[2025-04-21 03:03:05] (step=0025650) Train Loss: 6.3295, Train Steps/Sec: 0.87 +[2025-04-21 03:03:34] (step=0025675) Train Loss: 6.3694, Train Steps/Sec: 0.86 +[2025-04-21 03:04:03] (step=0025700) Train Loss: 6.3719, Train Steps/Sec: 0.86 +[2025-04-21 03:04:32] (step=0025725) Train Loss: 6.3420, Train Steps/Sec: 0.85 +[2025-04-21 03:05:01] (step=0025750) Train Loss: 6.3204, Train Steps/Sec: 0.87 +[2025-04-21 03:05:30] (step=0025775) Train Loss: 6.3874, Train Steps/Sec: 0.86 +[2025-04-21 03:05:59] (step=0025800) Train Loss: 6.3793, Train Steps/Sec: 0.86 +[2025-04-21 03:06:28] (step=0025825) Train Loss: 6.3670, Train Steps/Sec: 0.87 +[2025-04-21 03:06:57] (step=0025850) Train Loss: 6.4039, Train Steps/Sec: 0.87 +[2025-04-21 03:07:26] (step=0025875) Train Loss: 6.3791, Train Steps/Sec: 0.86 +[2025-04-21 03:07:55] (step=0025900) Train Loss: 6.3482, Train Steps/Sec: 0.86 +[2025-04-21 03:08:24] (step=0025925) Train Loss: 6.3696, Train Steps/Sec: 0.86 +[2025-04-21 03:08:53] (step=0025950) Train Loss: 6.3965, Train Steps/Sec: 0.86 +[2025-04-21 03:09:22] (step=0025975) Train Loss: 6.3744, Train Steps/Sec: 0.87 +[2025-04-21 03:09:51] (step=0026000) Train Loss: 6.3677, Train Steps/Sec: 0.86 +[2025-04-21 03:09:51] vision_config is None. initializing the InstructBlipVisionConfig with default values. +[2025-04-21 03:09:51] qformer_config is None. Initializing the InstructBlipQFormerConfig with default values. +[2025-04-21 03:09:51] text_config is None. Initializing the text config with default values (`OPTConfig`). +[2025-04-21 03:14:49] Finish Eval in 26000 steps... +[2025-04-21 03:15:10] Saved checkpoint to checkpoint/EmptyFix_InstructBlip_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_objects/002-GPT-XL/checkpoints/0026000.pt +[2025-04-21 03:15:13] Removed old checkpoint: checkpoint/EmptyFix_InstructBlip_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_objects/002-GPT-XL/checkpoints/0022000.pt +[2025-04-21 03:15:42] (step=0026025) Train Loss: 6.3470, Train Steps/Sec: 0.07 +[2025-04-21 03:16:11] (step=0026050) Train Loss: 6.3629, Train Steps/Sec: 0.86 +[2025-04-21 03:16:40] (step=0026075) Train Loss: 6.3461, Train Steps/Sec: 0.87 +[2025-04-21 03:17:09] (step=0026100) Train Loss: 6.3876, Train Steps/Sec: 0.86 +[2025-04-21 03:17:38] (step=0026125) Train Loss: 6.3786, Train Steps/Sec: 0.86 +[2025-04-21 03:18:06] (step=0026150) Train Loss: 6.3170, Train Steps/Sec: 0.87 +[2025-04-21 03:18:35] (step=0026175) Train Loss: 6.3566, Train Steps/Sec: 0.86 +[2025-04-21 03:19:05] (step=0026200) Train Loss: 6.3644, Train Steps/Sec: 0.86 +[2025-04-21 03:19:34] (step=0026225) Train Loss: 6.3755, Train Steps/Sec: 0.86 +[2025-04-21 03:20:02] (step=0026250) Train Loss: 6.3697, Train Steps/Sec: 0.87 +[2025-04-21 03:20:31] (step=0026275) Train Loss: 6.3471, Train Steps/Sec: 0.87 +[2025-04-21 03:21:00] (step=0026300) Train Loss: 6.3177, Train Steps/Sec: 0.86 +[2025-04-21 03:21:29] (step=0026325) Train Loss: 6.3616, Train Steps/Sec: 0.87 +[2025-04-21 03:21:58] (step=0026350) Train Loss: 6.3272, Train Steps/Sec: 0.86 +[2025-04-21 03:22:27] (step=0026375) Train Loss: 6.3418, Train Steps/Sec: 0.85 +[2025-04-21 03:22:57] (step=0026400) Train Loss: 6.3755, Train Steps/Sec: 0.86 +[2025-04-21 03:23:25] (step=0026425) Train Loss: 6.3510, Train Steps/Sec: 0.87 +[2025-04-21 03:23:55] (step=0026450) Train Loss: 6.3413, Train Steps/Sec: 0.86 +[2025-04-21 03:24:23] (step=0026475) Train Loss: 6.3838, Train Steps/Sec: 0.86 +[2025-04-21 03:24:52] (step=0026500) Train Loss: 6.3173, Train Steps/Sec: 0.86 +[2025-04-21 03:25:22] (step=0026525) Train Loss: 6.3553, Train Steps/Sec: 0.85 +[2025-04-21 03:25:51] (step=0026550) Train Loss: 6.3551, Train Steps/Sec: 0.86 +[2025-04-21 03:26:19] (step=0026575) Train Loss: 6.3361, Train Steps/Sec: 0.87 +[2025-04-21 03:26:49] (step=0026600) Train Loss: 6.3232, Train Steps/Sec: 0.86 +[2025-04-21 03:27:17] (step=0026625) Train Loss: 6.3677, Train Steps/Sec: 0.87 +[2025-04-21 03:27:46] (step=0026650) Train Loss: 6.3657, Train Steps/Sec: 0.87 +[2025-04-21 03:28:15] (step=0026675) Train Loss: 6.3276, Train Steps/Sec: 0.86 +[2025-04-21 03:28:44] (step=0026700) Train Loss: 6.3333, Train Steps/Sec: 0.86 +[2025-04-21 03:29:13] (step=0026725) Train Loss: 6.3867, Train Steps/Sec: 0.86 +[2025-04-21 03:29:42] (step=0026750) Train Loss: 6.3364, Train Steps/Sec: 0.86 +[2025-04-21 03:30:18] (step=0026775) Train Loss: 6.3518, Train Steps/Sec: 0.71 +[2025-04-21 03:30:47] (step=0026800) Train Loss: 6.4067, Train Steps/Sec: 0.86 +[2025-04-21 03:31:16] (step=0026825) Train Loss: 6.3702, Train Steps/Sec: 0.87 +[2025-04-21 03:31:44] (step=0026850) Train Loss: 6.3551, Train Steps/Sec: 0.87 +[2025-04-21 03:32:13] (step=0026875) Train Loss: 6.3477, Train Steps/Sec: 0.87 +[2025-04-21 03:32:42] (step=0026900) Train Loss: 6.3795, Train Steps/Sec: 0.86 +[2025-04-21 03:33:11] (step=0026925) Train Loss: 6.3482, Train Steps/Sec: 0.86 +[2025-04-21 03:33:40] (step=0026950) Train Loss: 6.3259, Train Steps/Sec: 0.87 +[2025-04-21 03:34:09] (step=0026975) Train Loss: 6.3949, Train Steps/Sec: 0.87 +[2025-04-21 03:34:38] (step=0027000) Train Loss: 6.3009, Train Steps/Sec: 0.86 +[2025-04-21 03:35:07] (step=0027025) Train Loss: 6.3842, Train Steps/Sec: 0.87 +[2025-04-21 03:35:36] (step=0027050) Train Loss: 6.3529, Train Steps/Sec: 0.87 +[2025-04-21 03:36:05] (step=0027075) Train Loss: 6.3504, Train Steps/Sec: 0.86 +[2025-04-21 03:36:34] (step=0027100) Train Loss: 6.3667, Train Steps/Sec: 0.86 +[2025-04-21 03:37:02] (step=0027125) Train Loss: 6.3233, Train Steps/Sec: 0.87 +[2025-04-21 03:37:31] (step=0027150) Train Loss: 6.3892, Train Steps/Sec: 0.86 +[2025-04-21 03:38:00] (step=0027175) Train Loss: 6.3368, Train Steps/Sec: 0.86 +[2025-04-21 03:38:29] (step=0027200) Train Loss: 6.3567, Train Steps/Sec: 0.86 +[2025-04-21 03:38:58] (step=0027225) Train Loss: 6.3344, Train Steps/Sec: 0.86 +[2025-04-21 03:39:28] (step=0027250) Train Loss: 6.3033, Train Steps/Sec: 0.86 +[2025-04-21 03:39:57] (step=0027275) Train Loss: 6.3923, Train Steps/Sec: 0.86 +[2025-04-21 03:40:26] (step=0027300) Train Loss: 6.3692, Train Steps/Sec: 0.86 +[2025-04-21 03:40:55] (step=0027325) Train Loss: 6.3704, Train Steps/Sec: 0.86 +[2025-04-21 03:41:23] (step=0027350) Train Loss: 6.3613, Train Steps/Sec: 0.86 +[2025-04-21 03:41:52] (step=0027375) Train Loss: 6.3764, Train Steps/Sec: 0.86 +[2025-04-21 03:42:22] (step=0027400) Train Loss: 6.4086, Train Steps/Sec: 0.85 +[2025-04-21 03:42:51] (step=0027425) Train Loss: 6.3465, Train Steps/Sec: 0.86 +[2025-04-21 03:43:20] (step=0027450) Train Loss: 6.3396, Train Steps/Sec: 0.87 +[2025-04-21 03:43:49] (step=0027475) Train Loss: 6.3792, Train Steps/Sec: 0.86 +[2025-04-21 03:44:18] (step=0027500) Train Loss: 6.4072, Train Steps/Sec: 0.86 +[2025-04-21 03:44:47] (step=0027525) Train Loss: 6.3606, Train Steps/Sec: 0.86 +[2025-04-21 03:45:16] (step=0027550) Train Loss: 6.3618, Train Steps/Sec: 0.86 +[2025-04-21 03:45:45] (step=0027575) Train Loss: 6.3251, Train Steps/Sec: 0.86 +[2025-04-21 03:46:14] (step=0027600) Train Loss: 6.3383, Train Steps/Sec: 0.86 +[2025-04-21 03:46:43] (step=0027625) Train Loss: 6.3349, Train Steps/Sec: 0.86 +[2025-04-21 03:47:12] (step=0027650) Train Loss: 6.3574, Train Steps/Sec: 0.86 +[2025-04-21 03:47:41] (step=0027675) Train Loss: 6.4096, Train Steps/Sec: 0.85 +[2025-04-21 03:48:10] (step=0027700) Train Loss: 6.3193, Train Steps/Sec: 0.86 +[2025-04-21 03:48:40] (step=0027725) Train Loss: 6.3214, Train Steps/Sec: 0.86 +[2025-04-21 03:49:09] (step=0027750) Train Loss: 6.2980, Train Steps/Sec: 0.86 +[2025-04-21 03:49:38] (step=0027775) Train Loss: 6.3813, Train Steps/Sec: 0.86 +[2025-04-21 03:50:07] (step=0027800) Train Loss: 6.3219, Train Steps/Sec: 0.86 +[2025-04-21 03:50:36] (step=0027825) Train Loss: 6.3490, Train Steps/Sec: 0.86 +[2025-04-21 03:51:05] (step=0027850) Train Loss: 6.3964, Train Steps/Sec: 0.86 +[2025-04-21 03:51:34] (step=0027875) Train Loss: 6.3413, Train Steps/Sec: 0.86 +[2025-04-21 03:52:03] (step=0027900) Train Loss: 6.3581, Train Steps/Sec: 0.86 +[2025-04-21 03:52:32] (step=0027925) Train Loss: 6.3296, Train Steps/Sec: 0.86 +[2025-04-21 03:53:02] (step=0027950) Train Loss: 6.3410, Train Steps/Sec: 0.85 +[2025-04-21 03:53:31] (step=0027975) Train Loss: 6.3782, Train Steps/Sec: 0.86 +[2025-04-21 03:54:00] (step=0028000) Train Loss: 6.3464, Train Steps/Sec: 0.86 +[2025-04-21 03:54:00] vision_config is None. initializing the InstructBlipVisionConfig with default values. +[2025-04-21 03:54:00] qformer_config is None. Initializing the InstructBlipQFormerConfig with default values. +[2025-04-21 03:54:00] text_config is None. Initializing the text config with default values (`OPTConfig`). +[2025-04-21 03:59:02] Finish Eval in 28000 steps... +[2025-04-21 03:59:23] Saved checkpoint to checkpoint/EmptyFix_InstructBlip_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_objects/002-GPT-XL/checkpoints/0028000.pt +[2025-04-21 03:59:26] Removed old checkpoint: checkpoint/EmptyFix_InstructBlip_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_objects/002-GPT-XL/checkpoints/0024000.pt +[2025-04-21 03:59:56] (step=0028025) Train Loss: 6.3965, Train Steps/Sec: 0.07 +[2025-04-21 04:00:25] (step=0028050) Train Loss: 6.3218, Train Steps/Sec: 0.87 +[2025-04-21 04:00:53] (step=0028075) Train Loss: 6.3401, Train Steps/Sec: 0.87 +[2025-04-21 04:01:23] (step=0028100) Train Loss: 6.3767, Train Steps/Sec: 0.86 +[2025-04-21 04:01:51] (step=0028125) Train Loss: 6.3418, Train Steps/Sec: 0.87 +[2025-04-21 04:02:20] (step=0028150) Train Loss: 6.3603, Train Steps/Sec: 0.86 +[2025-04-21 04:02:49] (step=0028175) Train Loss: 6.3529, Train Steps/Sec: 0.87 +[2025-04-21 04:03:18] (step=0028200) Train Loss: 6.3611, Train Steps/Sec: 0.86 +[2025-04-21 04:03:47] (step=0028225) Train Loss: 6.3530, Train Steps/Sec: 0.86 +[2025-04-21 04:04:16] (step=0028250) Train Loss: 6.3598, Train Steps/Sec: 0.87 +[2025-04-21 04:04:45] (step=0028275) Train Loss: 6.3987, Train Steps/Sec: 0.87 +[2025-04-21 04:05:14] (step=0028300) Train Loss: 6.3398, Train Steps/Sec: 0.86 +[2025-04-21 04:05:43] (step=0028325) Train Loss: 6.3578, Train Steps/Sec: 0.87 +[2025-04-21 04:06:11] (step=0028350) Train Loss: 6.3460, Train Steps/Sec: 0.87 +[2025-04-21 04:06:41] (step=0028375) Train Loss: 6.3581, Train Steps/Sec: 0.85 +[2025-04-21 04:07:10] (step=0028400) Train Loss: 6.3677, Train Steps/Sec: 0.86 +[2025-04-21 04:07:39] (step=0028425) Train Loss: 6.3485, Train Steps/Sec: 0.86 +[2025-04-21 04:08:08] (step=0028450) Train Loss: 6.3729, Train Steps/Sec: 0.86 +[2025-04-21 04:08:37] (step=0028475) Train Loss: 6.3527, Train Steps/Sec: 0.87 +[2025-04-21 04:09:06] (step=0028500) Train Loss: 6.3969, Train Steps/Sec: 0.86 +[2025-04-21 04:09:35] (step=0028525) Train Loss: 6.3828, Train Steps/Sec: 0.86 +[2025-04-21 04:10:04] (step=0028550) Train Loss: 6.3429, Train Steps/Sec: 0.86 +[2025-04-21 04:10:33] (step=0028575) Train Loss: 6.3425, Train Steps/Sec: 0.86 +[2025-04-21 04:11:02] (step=0028600) Train Loss: 6.3549, Train Steps/Sec: 0.85 +[2025-04-21 04:11:31] (step=0028625) Train Loss: 6.3332, Train Steps/Sec: 0.86 +[2025-04-21 04:12:00] (step=0028650) Train Loss: 6.3944, Train Steps/Sec: 0.86 +[2025-04-21 04:12:29] (step=0028675) Train Loss: 6.3774, Train Steps/Sec: 0.86 +[2025-04-21 04:12:58] (step=0028700) Train Loss: 6.3918, Train Steps/Sec: 0.86 +[2025-04-21 04:13:27] (step=0028725) Train Loss: 6.3310, Train Steps/Sec: 0.86 +[2025-04-21 04:13:56] (step=0028750) Train Loss: 6.3728, Train Steps/Sec: 0.87 +[2025-04-21 04:14:25] (step=0028775) Train Loss: 6.3210, Train Steps/Sec: 0.87 +[2025-04-21 04:14:54] (step=0028800) Train Loss: 6.3504, Train Steps/Sec: 0.86 +[2025-04-21 04:15:23] (step=0028825) Train Loss: 6.3486, Train Steps/Sec: 0.86 +[2025-04-21 04:15:52] (step=0028850) Train Loss: 6.3896, Train Steps/Sec: 0.86 +[2025-04-21 04:16:21] (step=0028875) Train Loss: 6.3681, Train Steps/Sec: 0.87 +[2025-04-21 04:16:50] (step=0028900) Train Loss: 6.3774, Train Steps/Sec: 0.86 +[2025-04-21 04:17:19] (step=0028925) Train Loss: 6.3660, Train Steps/Sec: 0.86 +[2025-04-21 04:17:48] (step=0028950) Train Loss: 6.3879, Train Steps/Sec: 0.87 +[2025-04-21 04:18:17] (step=0028975) Train Loss: 6.3435, Train Steps/Sec: 0.86 +[2025-04-21 04:18:46] (step=0029000) Train Loss: 6.3313, Train Steps/Sec: 0.86 +[2025-04-21 04:19:15] (step=0029025) Train Loss: 6.3023, Train Steps/Sec: 0.86 +[2025-04-21 04:19:43] (step=0029050) Train Loss: 6.3433, Train Steps/Sec: 0.87 +[2025-04-21 04:20:18] (step=0029075) Train Loss: 6.3457, Train Steps/Sec: 0.72 +[2025-04-21 04:20:47] (step=0029100) Train Loss: 6.3513, Train Steps/Sec: 0.86 +[2025-04-21 04:21:16] (step=0029125) Train Loss: 6.3515, Train Steps/Sec: 0.86 +[2025-04-21 04:21:45] (step=0029150) Train Loss: 6.3628, Train Steps/Sec: 0.87 +[2025-04-21 04:22:14] (step=0029175) Train Loss: 6.3556, Train Steps/Sec: 0.87 +[2025-04-21 04:22:43] (step=0029200) Train Loss: 6.3554, Train Steps/Sec: 0.86 +[2025-04-21 04:23:12] (step=0029225) Train Loss: 6.3380, Train Steps/Sec: 0.87 +[2025-04-21 04:23:41] (step=0029250) Train Loss: 6.3629, Train Steps/Sec: 0.87 +[2025-04-21 04:24:10] (step=0029275) Train Loss: 6.4024, Train Steps/Sec: 0.86 +[2025-04-21 04:24:51] (step=0029300) Train Loss: 6.3460, Train Steps/Sec: 0.62 +[2025-04-21 04:25:20] (step=0029325) Train Loss: 6.3666, Train Steps/Sec: 0.86 +[2025-04-21 04:25:49] (step=0029350) Train Loss: 6.3581, Train Steps/Sec: 0.86 +[2025-04-21 04:26:17] (step=0029375) Train Loss: 6.3585, Train Steps/Sec: 0.87 +[2025-04-21 04:26:46] (step=0029400) Train Loss: 6.3851, Train Steps/Sec: 0.86 +[2025-04-21 04:27:15] (step=0029425) Train Loss: 6.3995, Train Steps/Sec: 0.86 +[2025-04-21 04:27:44] (step=0029450) Train Loss: 6.3841, Train Steps/Sec: 0.87 +[2025-04-21 04:28:20] (step=0029475) Train Loss: 6.3521, Train Steps/Sec: 0.70 +[2025-04-21 04:28:49] (step=0029500) Train Loss: 6.3437, Train Steps/Sec: 0.86 +[2025-04-21 04:29:18] (step=0029525) Train Loss: 6.3278, Train Steps/Sec: 0.86 +[2025-04-21 04:29:47] (step=0029550) Train Loss: 6.3310, Train Steps/Sec: 0.87 +[2025-04-21 04:30:16] (step=0029575) Train Loss: 6.3715, Train Steps/Sec: 0.87 +[2025-04-21 04:30:45] (step=0029600) Train Loss: 6.4138, Train Steps/Sec: 0.86 +[2025-04-21 04:31:14] (step=0029625) Train Loss: 6.3451, Train Steps/Sec: 0.86 +[2025-04-21 04:31:43] (step=0029650) Train Loss: 6.3330, Train Steps/Sec: 0.86 +[2025-04-21 04:32:12] (step=0029675) Train Loss: 6.3198, Train Steps/Sec: 0.87 +[2025-04-21 04:32:41] (step=0029700) Train Loss: 6.3112, Train Steps/Sec: 0.86 +[2025-04-21 04:33:10] (step=0029725) Train Loss: 6.3301, Train Steps/Sec: 0.86 +[2025-04-21 04:33:39] (step=0029750) Train Loss: 6.3633, Train Steps/Sec: 0.87 +[2025-04-21 04:34:07] (step=0029775) Train Loss: 6.3884, Train Steps/Sec: 0.87 +[2025-04-21 04:34:36] (step=0029800) Train Loss: 6.3829, Train Steps/Sec: 0.86 +[2025-04-21 04:35:05] (step=0029825) Train Loss: 6.3908, Train Steps/Sec: 0.87 +[2025-04-21 04:35:34] (step=0029850) Train Loss: 6.3840, Train Steps/Sec: 0.86 +[2025-04-21 04:36:03] (step=0029875) Train Loss: 6.3226, Train Steps/Sec: 0.87 +[2025-04-21 04:36:32] (step=0029900) Train Loss: 6.3468, Train Steps/Sec: 0.86 +[2025-04-21 04:37:01] (step=0029925) Train Loss: 6.3095, Train Steps/Sec: 0.86 +[2025-04-21 04:37:30] (step=0029950) Train Loss: 6.3676, Train Steps/Sec: 0.86 +[2025-04-21 04:37:59] (step=0029975) Train Loss: 6.3829, Train Steps/Sec: 0.87 +[2025-04-21 04:38:28] (step=0030000) Train Loss: 6.3785, Train Steps/Sec: 0.86 +[2025-04-21 04:38:28] vision_config is None. initializing the InstructBlipVisionConfig with default values. +[2025-04-21 04:38:28] qformer_config is None. Initializing the InstructBlipQFormerConfig with default values. +[2025-04-21 04:38:28] text_config is None. Initializing the text config with default values (`OPTConfig`). +[2025-04-21 04:43:29] Finish Eval in 30000 steps... +[2025-04-21 04:43:51] Saved checkpoint to checkpoint/EmptyFix_InstructBlip_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_objects/002-GPT-XL/checkpoints/0030000.pt +[2025-04-21 04:43:54] Removed old checkpoint: checkpoint/EmptyFix_InstructBlip_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_objects/002-GPT-XL/checkpoints/0026000.pt +[2025-04-21 04:44:23] (step=0030025) Train Loss: 6.3251, Train Steps/Sec: 0.07 +[2025-04-21 04:44:52] (step=0030050) Train Loss: 6.3458, Train Steps/Sec: 0.86 +[2025-04-21 04:45:21] (step=0030075) Train Loss: 6.3593, Train Steps/Sec: 0.87 +[2025-04-21 04:45:50] (step=0030100) Train Loss: 6.3500, Train Steps/Sec: 0.85 +[2025-04-21 04:46:19] (step=0030125) Train Loss: 6.3606, Train Steps/Sec: 0.86 +[2025-04-21 04:46:48] (step=0030150) Train Loss: 6.3750, Train Steps/Sec: 0.86 +[2025-04-21 04:47:17] (step=0030175) Train Loss: 6.3188, Train Steps/Sec: 0.86 +[2025-04-21 04:47:46] (step=0030200) Train Loss: 6.3371, Train Steps/Sec: 0.86 +[2025-04-21 04:48:16] (step=0030225) Train Loss: 6.3688, Train Steps/Sec: 0.85 +[2025-04-21 04:48:44] (step=0030250) Train Loss: 6.3324, Train Steps/Sec: 0.87 +[2025-04-21 04:49:13] (step=0030275) Train Loss: 6.3763, Train Steps/Sec: 0.86 +[2025-04-21 04:49:43] (step=0030300) Train Loss: 6.3825, Train Steps/Sec: 0.86 +[2025-04-21 04:50:12] (step=0030325) Train Loss: 6.3558, Train Steps/Sec: 0.86 +[2025-04-21 04:50:41] (step=0030350) Train Loss: 6.3528, Train Steps/Sec: 0.86 +[2025-04-21 04:51:10] (step=0030375) Train Loss: 6.3142, Train Steps/Sec: 0.86 +[2025-04-21 04:51:38] (step=0030400) Train Loss: 6.3463, Train Steps/Sec: 0.86 +[2025-04-21 04:52:07] (step=0030425) Train Loss: 6.3527, Train Steps/Sec: 0.87 +[2025-04-21 04:52:36] (step=0030450) Train Loss: 6.3439, Train Steps/Sec: 0.86 +[2025-04-21 04:53:05] (step=0030475) Train Loss: 6.3689, Train Steps/Sec: 0.86 +[2025-04-21 04:53:35] (step=0030500) Train Loss: 6.3332, Train Steps/Sec: 0.85 +[2025-04-21 04:54:04] (step=0030525) Train Loss: 6.3732, Train Steps/Sec: 0.86 +[2025-04-21 04:54:33] (step=0030550) Train Loss: 6.3370, Train Steps/Sec: 0.86 +[2025-04-21 04:55:09] (step=0030575) Train Loss: 6.3300, Train Steps/Sec: 0.69 +[2025-04-21 04:55:38] (step=0030600) Train Loss: 6.3515, Train Steps/Sec: 0.86 +[2025-04-21 04:56:07] (step=0030625) Train Loss: 6.3603, Train Steps/Sec: 0.86 +[2025-04-21 04:56:36] (step=0030650) Train Loss: 6.3487, Train Steps/Sec: 0.86 +[2025-04-21 04:57:05] (step=0030675) Train Loss: 6.3759, Train Steps/Sec: 0.86 +[2025-04-21 04:57:34] (step=0030700) Train Loss: 6.3838, Train Steps/Sec: 0.85 +[2025-04-21 04:58:03] (step=0030725) Train Loss: 6.3653, Train Steps/Sec: 0.86 +[2025-04-21 04:58:33] (step=0030750) Train Loss: 6.3391, Train Steps/Sec: 0.86 +[2025-04-21 04:59:01] (step=0030775) Train Loss: 6.3525, Train Steps/Sec: 0.87 +[2025-04-21 04:59:30] (step=0030800) Train Loss: 6.3178, Train Steps/Sec: 0.86 +[2025-04-21 04:59:59] (step=0030825) Train Loss: 6.3290, Train Steps/Sec: 0.86 +[2025-04-21 05:00:29] (step=0030850) Train Loss: 6.3983, Train Steps/Sec: 0.86 +[2025-04-21 05:00:57] (step=0030875) Train Loss: 6.3580, Train Steps/Sec: 0.87 +[2025-04-21 05:01:27] (step=0030900) Train Loss: 6.3417, Train Steps/Sec: 0.86 +[2025-04-21 05:01:56] (step=0030925) Train Loss: 6.3549, Train Steps/Sec: 0.86 +[2025-04-21 05:02:25] (step=0030950) Train Loss: 6.3405, Train Steps/Sec: 0.86 +[2025-04-21 05:02:54] (step=0030975) Train Loss: 6.3642, Train Steps/Sec: 0.86 +[2025-04-21 05:03:23] (step=0031000) Train Loss: 6.3546, Train Steps/Sec: 0.86 +[2025-04-21 05:03:52] (step=0031025) Train Loss: 6.3407, Train Steps/Sec: 0.86 +[2025-04-21 05:04:21] (step=0031050) Train Loss: 6.3662, Train Steps/Sec: 0.86 +[2025-04-21 05:04:50] (step=0031075) Train Loss: 6.3439, Train Steps/Sec: 0.86 +[2025-04-21 05:05:19] (step=0031100) Train Loss: 6.3715, Train Steps/Sec: 0.85 +[2025-04-21 05:05:55] (step=0031125) Train Loss: 6.3476, Train Steps/Sec: 0.70 +[2025-04-21 05:06:24] (step=0031150) Train Loss: 6.3596, Train Steps/Sec: 0.87 +[2025-04-21 05:06:53] (step=0031175) Train Loss: 6.3599, Train Steps/Sec: 0.86 +[2025-04-21 05:07:21] (step=0031200) Train Loss: 6.3425, Train Steps/Sec: 0.87 +[2025-04-21 05:07:50] (step=0031225) Train Loss: 6.3417, Train Steps/Sec: 0.87 +[2025-04-21 05:08:19] (step=0031250) Train Loss: 6.3524, Train Steps/Sec: 0.87 +[2025-04-21 05:08:48] (step=0031275) Train Loss: 6.3695, Train Steps/Sec: 0.86 +[2025-04-21 05:09:18] (step=0031300) Train Loss: 6.3959, Train Steps/Sec: 0.85 +[2025-04-21 05:09:46] (step=0031325) Train Loss: 6.3677, Train Steps/Sec: 0.87 +[2025-04-21 05:10:16] (step=0031350) Train Loss: 6.3689, Train Steps/Sec: 0.86 +[2025-04-21 05:10:44] (step=0031375) Train Loss: 6.3324, Train Steps/Sec: 0.86 +[2025-04-21 05:11:14] (step=0031400) Train Loss: 6.3580, Train Steps/Sec: 0.86 +[2025-04-21 05:11:42] (step=0031425) Train Loss: 6.3699, Train Steps/Sec: 0.87 +[2025-04-21 05:12:12] (step=0031450) Train Loss: 6.3302, Train Steps/Sec: 0.86 +[2025-04-21 05:12:41] (step=0031475) Train Loss: 6.3314, Train Steps/Sec: 0.86 +[2025-04-21 05:13:10] (step=0031500) Train Loss: 6.3686, Train Steps/Sec: 0.86 +[2025-04-21 05:13:39] (step=0031525) Train Loss: 6.3277, Train Steps/Sec: 0.86 +[2025-04-21 05:14:08] (step=0031550) Train Loss: 6.3329, Train Steps/Sec: 0.86 +[2025-04-21 05:14:37] (step=0031575) Train Loss: 6.3432, Train Steps/Sec: 0.87 +[2025-04-21 05:15:06] (step=0031600) Train Loss: 6.3270, Train Steps/Sec: 0.86 +[2025-04-21 05:15:35] (step=0031625) Train Loss: 6.3325, Train Steps/Sec: 0.86 +[2025-04-21 05:16:04] (step=0031650) Train Loss: 6.4057, Train Steps/Sec: 0.87 +[2025-04-21 05:16:33] (step=0031675) Train Loss: 6.3654, Train Steps/Sec: 0.86 +[2025-04-21 05:17:02] (step=0031700) Train Loss: 6.3739, Train Steps/Sec: 0.86 +[2025-04-21 05:17:31] (step=0031725) Train Loss: 6.3742, Train Steps/Sec: 0.86 +[2025-04-21 05:18:00] (step=0031750) Train Loss: 6.3282, Train Steps/Sec: 0.86 +[2025-04-21 05:18:29] (step=0031775) Train Loss: 6.3641, Train Steps/Sec: 0.86 +[2025-04-21 05:18:58] (step=0031800) Train Loss: 6.3450, Train Steps/Sec: 0.86 +[2025-04-21 05:19:27] (step=0031825) Train Loss: 6.3420, Train Steps/Sec: 0.87 +[2025-04-21 05:19:56] (step=0031850) Train Loss: 6.3177, Train Steps/Sec: 0.85 +[2025-04-21 05:20:25] (step=0031875) Train Loss: 6.3552, Train Steps/Sec: 0.87 +[2025-04-21 05:20:54] (step=0031900) Train Loss: 6.3628, Train Steps/Sec: 0.86 +[2025-04-21 05:21:23] (step=0031925) Train Loss: 6.3566, Train Steps/Sec: 0.87 +[2025-04-21 05:21:52] (step=0031950) Train Loss: 6.3674, Train Steps/Sec: 0.86 +[2025-04-21 05:22:21] (step=0031975) Train Loss: 6.3699, Train Steps/Sec: 0.87 +[2025-04-21 05:22:50] (step=0032000) Train Loss: 6.3100, Train Steps/Sec: 0.86 +[2025-04-21 05:22:50] vision_config is None. initializing the InstructBlipVisionConfig with default values. +[2025-04-21 05:22:50] qformer_config is None. Initializing the InstructBlipQFormerConfig with default values. +[2025-04-21 05:22:50] text_config is None. Initializing the text config with default values (`OPTConfig`). +[2025-04-21 05:27:52] Finish Eval in 32000 steps... +[2025-04-21 05:28:14] Saved checkpoint to checkpoint/EmptyFix_InstructBlip_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_objects/002-GPT-XL/checkpoints/0032000.pt +[2025-04-21 05:28:17] Removed old checkpoint: checkpoint/EmptyFix_InstructBlip_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_objects/002-GPT-XL/checkpoints/0028000.pt +[2025-04-21 05:28:46] (step=0032025) Train Loss: 6.3682, Train Steps/Sec: 0.07 +[2025-04-21 05:29:15] (step=0032050) Train Loss: 6.3654, Train Steps/Sec: 0.86 +[2025-04-21 05:29:44] (step=0032075) Train Loss: 6.3242, Train Steps/Sec: 0.86 +[2025-04-21 05:30:13] (step=0032100) Train Loss: 6.3301, Train Steps/Sec: 0.86 +[2025-04-21 05:30:49] (step=0032125) Train Loss: 6.3159, Train Steps/Sec: 0.71 +[2025-04-21 05:31:18] (step=0032150) Train Loss: 6.3528, Train Steps/Sec: 0.86 +[2025-04-21 05:31:47] (step=0032175) Train Loss: 6.3530, Train Steps/Sec: 0.86 +[2025-04-21 05:32:16] (step=0032200) Train Loss: 6.3250, Train Steps/Sec: 0.85 +[2025-04-21 05:32:45] (step=0032225) Train Loss: 6.3218, Train Steps/Sec: 0.87 +[2025-04-21 05:33:14] (step=0032250) Train Loss: 6.3472, Train Steps/Sec: 0.86 +[2025-04-21 05:33:43] (step=0032275) Train Loss: 6.3511, Train Steps/Sec: 0.87 +[2025-04-21 05:34:12] (step=0032300) Train Loss: 6.3678, Train Steps/Sec: 0.86 +[2025-04-21 05:34:41] (step=0032325) Train Loss: 6.3574, Train Steps/Sec: 0.87 +[2025-04-21 05:35:10] (step=0032350) Train Loss: 6.3731, Train Steps/Sec: 0.86 +[2025-04-21 05:35:39] (step=0032375) Train Loss: 6.3507, Train Steps/Sec: 0.86 +[2025-04-21 05:36:08] (step=0032400) Train Loss: 6.3262, Train Steps/Sec: 0.86 +[2025-04-21 05:36:37] (step=0032425) Train Loss: 6.3810, Train Steps/Sec: 0.86 +[2025-04-21 05:37:06] (step=0032450) Train Loss: 6.3593, Train Steps/Sec: 0.86 +[2025-04-21 05:37:35] (step=0032475) Train Loss: 6.3953, Train Steps/Sec: 0.86 +[2025-04-21 05:38:04] (step=0032500) Train Loss: 6.4023, Train Steps/Sec: 0.86 +[2025-04-21 05:38:33] (step=0032525) Train Loss: 6.2976, Train Steps/Sec: 0.86 +[2025-04-21 05:39:02] (step=0032550) Train Loss: 6.3994, Train Steps/Sec: 0.86 +[2025-04-21 05:39:31] (step=0032575) Train Loss: 6.3647, Train Steps/Sec: 0.87 +[2025-04-21 05:40:00] (step=0032600) Train Loss: 6.3146, Train Steps/Sec: 0.86 +[2025-04-21 05:40:29] (step=0032625) Train Loss: 6.3330, Train Steps/Sec: 0.86 +[2025-04-21 05:40:58] (step=0032650) Train Loss: 6.3411, Train Steps/Sec: 0.86 +[2025-04-21 05:41:27] (step=0032675) Train Loss: 6.3686, Train Steps/Sec: 0.86 +[2025-04-21 05:41:56] (step=0032700) Train Loss: 6.3673, Train Steps/Sec: 0.86 +[2025-04-21 05:42:25] (step=0032725) Train Loss: 6.3501, Train Steps/Sec: 0.86 +[2025-04-21 05:42:54] (step=0032750) Train Loss: 6.3925, Train Steps/Sec: 0.86 +[2025-04-21 05:43:23] (step=0032775) Train Loss: 6.3518, Train Steps/Sec: 0.86 +[2025-04-21 05:43:53] (step=0032800) Train Loss: 6.4033, Train Steps/Sec: 0.86 +[2025-04-21 05:44:21] (step=0032825) Train Loss: 6.3515, Train Steps/Sec: 0.87 +[2025-04-21 05:44:51] (step=0032850) Train Loss: 6.3209, Train Steps/Sec: 0.86 +[2025-04-21 05:45:19] (step=0032875) Train Loss: 6.3796, Train Steps/Sec: 0.86 +[2025-04-21 05:45:49] (step=0032900) Train Loss: 6.3573, Train Steps/Sec: 0.86 +[2025-04-21 05:46:18] (step=0032925) Train Loss: 6.3432, Train Steps/Sec: 0.86 +[2025-04-21 05:46:47] (step=0032950) Train Loss: 6.3513, Train Steps/Sec: 0.86 +[2025-04-21 05:47:15] (step=0032975) Train Loss: 6.3672, Train Steps/Sec: 0.86 +[2025-04-21 05:47:45] (step=0033000) Train Loss: 6.3484, Train Steps/Sec: 0.86 +[2025-04-21 05:48:14] (step=0033025) Train Loss: 6.3555, Train Steps/Sec: 0.85 +[2025-04-21 05:48:43] (step=0033050) Train Loss: 6.3730, Train Steps/Sec: 0.87 +[2025-04-21 05:49:12] (step=0033075) Train Loss: 6.3439, Train Steps/Sec: 0.86 +[2025-04-21 05:49:41] (step=0033100) Train Loss: 6.3187, Train Steps/Sec: 0.86 +[2025-04-21 05:50:10] (step=0033125) Train Loss: 6.3008, Train Steps/Sec: 0.87 +[2025-04-21 05:50:39] (step=0033150) Train Loss: 6.3250, Train Steps/Sec: 0.86 +[2025-04-21 05:51:08] (step=0033175) Train Loss: 6.3351, Train Steps/Sec: 0.86 +[2025-04-21 05:51:37] (step=0033200) Train Loss: 6.3551, Train Steps/Sec: 0.86 +[2025-04-21 05:52:06] (step=0033225) Train Loss: 6.3575, Train Steps/Sec: 0.87 +[2025-04-21 05:52:35] (step=0033250) Train Loss: 6.3817, Train Steps/Sec: 0.87 +[2025-04-21 05:53:04] (step=0033275) Train Loss: 6.3643, Train Steps/Sec: 0.86 +[2025-04-21 05:53:33] (step=0033300) Train Loss: 6.3442, Train Steps/Sec: 0.86 +[2025-04-21 05:54:02] (step=0033325) Train Loss: 6.3446, Train Steps/Sec: 0.87 +[2025-04-21 05:54:30] (step=0033350) Train Loss: 6.3794, Train Steps/Sec: 0.87 +[2025-04-21 05:54:59] (step=0033375) Train Loss: 6.4010, Train Steps/Sec: 0.86 +[2025-04-21 05:55:29] (step=0033400) Train Loss: 6.3447, Train Steps/Sec: 0.86 +[2025-04-21 05:55:58] (step=0033425) Train Loss: 6.3829, Train Steps/Sec: 0.87 +[2025-04-21 05:56:27] (step=0033450) Train Loss: 6.3589, Train Steps/Sec: 0.86 +[2025-04-21 05:56:55] (step=0033475) Train Loss: 6.3233, Train Steps/Sec: 0.87 +[2025-04-21 05:57:24] (step=0033500) Train Loss: 6.3657, Train Steps/Sec: 0.86 +[2025-04-21 05:57:53] (step=0033525) Train Loss: 6.3759, Train Steps/Sec: 0.86 +[2025-04-21 05:58:22] (step=0033550) Train Loss: 6.3201, Train Steps/Sec: 0.87 +[2025-04-21 05:58:51] (step=0033575) Train Loss: 6.3209, Train Steps/Sec: 0.86 +[2025-04-21 05:59:20] (step=0033600) Train Loss: 6.3367, Train Steps/Sec: 0.87 +[2025-04-21 05:59:49] (step=0033625) Train Loss: 6.3821, Train Steps/Sec: 0.86 +[2025-04-21 06:00:18] (step=0033650) Train Loss: 6.3537, Train Steps/Sec: 0.87 +[2025-04-21 06:00:47] (step=0033675) Train Loss: 6.3577, Train Steps/Sec: 0.87 +[2025-04-21 06:01:16] (step=0033700) Train Loss: 6.3625, Train Steps/Sec: 0.86 +[2025-04-21 06:01:45] (step=0033725) Train Loss: 6.3389, Train Steps/Sec: 0.86 +[2025-04-21 06:02:14] (step=0033750) Train Loss: 6.3792, Train Steps/Sec: 0.86 +[2025-04-21 06:02:43] (step=0033775) Train Loss: 6.3802, Train Steps/Sec: 0.86 +[2025-04-21 06:03:13] (step=0033800) Train Loss: 6.3645, Train Steps/Sec: 0.85 +[2025-04-21 06:03:41] (step=0033825) Train Loss: 6.4032, Train Steps/Sec: 0.87 +[2025-04-21 06:04:11] (step=0033850) Train Loss: 6.3200, Train Steps/Sec: 0.86 +[2025-04-21 06:04:40] (step=0033875) Train Loss: 6.3618, Train Steps/Sec: 0.86 +[2025-04-21 06:05:09] (step=0033900) Train Loss: 6.3471, Train Steps/Sec: 0.86 +[2025-04-21 06:05:38] (step=0033925) Train Loss: 6.3791, Train Steps/Sec: 0.86 +[2025-04-21 06:06:07] (step=0033950) Train Loss: 6.3608, Train Steps/Sec: 0.86 +[2025-04-21 06:06:36] (step=0033975) Train Loss: 6.3317, Train Steps/Sec: 0.86 +[2025-04-21 06:07:05] (step=0034000) Train Loss: 6.3671, Train Steps/Sec: 0.86 +[2025-04-21 06:07:05] vision_config is None. initializing the InstructBlipVisionConfig with default values. +[2025-04-21 06:07:05] qformer_config is None. Initializing the InstructBlipQFormerConfig with default values. +[2025-04-21 06:07:05] text_config is None. Initializing the text config with default values (`OPTConfig`). +[2025-04-21 06:12:08] Finish Eval in 34000 steps... +[2025-04-21 06:12:29] Saved checkpoint to checkpoint/EmptyFix_InstructBlip_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_objects/002-GPT-XL/checkpoints/0034000.pt +[2025-04-21 06:12:32] Removed old checkpoint: checkpoint/EmptyFix_InstructBlip_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_objects/002-GPT-XL/checkpoints/0030000.pt +[2025-04-21 06:13:01] (step=0034025) Train Loss: 6.3782, Train Steps/Sec: 0.07 +[2025-04-21 06:13:30] (step=0034050) Train Loss: 6.3714, Train Steps/Sec: 0.86 +[2025-04-21 06:13:59] (step=0034075) Train Loss: 6.3418, Train Steps/Sec: 0.87 +[2025-04-21 06:14:28] (step=0034100) Train Loss: 6.3584, Train Steps/Sec: 0.86 +[2025-04-21 06:14:57] (step=0034125) Train Loss: 6.3528, Train Steps/Sec: 0.86 +[2025-04-21 06:15:31] (step=0034150) Train Loss: 6.3564, Train Steps/Sec: 0.73 +[2025-04-21 06:16:00] (step=0034175) Train Loss: 6.3125, Train Steps/Sec: 0.86 +[2025-04-21 06:16:29] (step=0034200) Train Loss: 6.2817, Train Steps/Sec: 0.87 +[2025-04-21 06:16:58] (step=0034225) Train Loss: 6.3578, Train Steps/Sec: 0.87 +[2025-04-21 06:17:27] (step=0034250) Train Loss: 6.3890, Train Steps/Sec: 0.86 +[2025-04-21 06:17:56] (step=0034275) Train Loss: 6.2661, Train Steps/Sec: 0.87 +[2025-04-21 06:18:25] (step=0034300) Train Loss: 6.3455, Train Steps/Sec: 0.86 +[2025-04-21 06:18:54] (step=0034325) Train Loss: 6.3299, Train Steps/Sec: 0.86 +[2025-04-21 06:19:23] (step=0034350) Train Loss: 6.3544, Train Steps/Sec: 0.86 +[2025-04-21 06:19:52] (step=0034375) Train Loss: 6.3600, Train Steps/Sec: 0.86 +[2025-04-21 06:20:21] (step=0034400) Train Loss: 6.3690, Train Steps/Sec: 0.86 +[2025-04-21 06:20:55] (step=0034425) Train Loss: 6.3434, Train Steps/Sec: 0.73 +[2025-04-21 06:21:31] (step=0034450) Train Loss: 6.4024, Train Steps/Sec: 0.71 +[2025-04-21 06:22:00] (step=0034475) Train Loss: 6.3563, Train Steps/Sec: 0.86 +[2025-04-21 06:22:29] (step=0034500) Train Loss: 6.3718, Train Steps/Sec: 0.86 +[2025-04-21 06:22:58] (step=0034525) Train Loss: 6.3515, Train Steps/Sec: 0.86 +[2025-04-21 06:23:27] (step=0034550) Train Loss: 6.3355, Train Steps/Sec: 0.86 +[2025-04-21 06:23:56] (step=0034575) Train Loss: 6.3577, Train Steps/Sec: 0.86 +[2025-04-21 06:24:25] (step=0034600) Train Loss: 6.3654, Train Steps/Sec: 0.86 +[2025-04-21 06:24:54] (step=0034625) Train Loss: 6.3273, Train Steps/Sec: 0.87 +[2025-04-21 06:25:29] (step=0034650) Train Loss: 6.3047, Train Steps/Sec: 0.70 +[2025-04-21 06:25:59] (step=0034675) Train Loss: 6.3644, Train Steps/Sec: 0.86 +[2025-04-21 06:26:28] (step=0034700) Train Loss: 6.3885, Train Steps/Sec: 0.86 +[2025-04-21 06:26:57] (step=0034725) Train Loss: 6.3484, Train Steps/Sec: 0.86 +[2025-04-21 06:27:25] (step=0034750) Train Loss: 6.3720, Train Steps/Sec: 0.86 +[2025-04-21 06:27:54] (step=0034775) Train Loss: 6.3282, Train Steps/Sec: 0.87 +[2025-04-21 06:28:23] (step=0034800) Train Loss: 6.3536, Train Steps/Sec: 0.86 +[2025-04-21 06:28:53] (step=0034825) Train Loss: 6.3315, Train Steps/Sec: 0.86 +[2025-04-21 06:29:22] (step=0034850) Train Loss: 6.3914, Train Steps/Sec: 0.86 +[2025-04-21 06:29:50] (step=0034875) Train Loss: 6.3303, Train Steps/Sec: 0.87 +[2025-04-21 06:30:19] (step=0034900) Train Loss: 6.3694, Train Steps/Sec: 0.86 +[2025-04-21 06:30:48] (step=0034925) Train Loss: 6.3111, Train Steps/Sec: 0.87 +[2025-04-21 06:31:17] (step=0034950) Train Loss: 6.3494, Train Steps/Sec: 0.86 +[2025-04-21 06:31:46] (step=0034975) Train Loss: 6.3639, Train Steps/Sec: 0.86 +[2025-04-21 06:32:15] (step=0035000) Train Loss: 6.3413, Train Steps/Sec: 0.86 +[2025-04-21 06:32:44] (step=0035025) Train Loss: 6.3636, Train Steps/Sec: 0.87 +[2025-04-21 06:33:13] (step=0035050) Train Loss: 6.3986, Train Steps/Sec: 0.86 +[2025-04-21 06:33:42] (step=0035075) Train Loss: 6.3165, Train Steps/Sec: 0.87 +[2025-04-21 06:34:11] (step=0035100) Train Loss: 6.3545, Train Steps/Sec: 0.85 +[2025-04-21 06:34:40] (step=0035125) Train Loss: 6.3564, Train Steps/Sec: 0.87 +[2025-04-21 06:35:09] (step=0035150) Train Loss: 6.2909, Train Steps/Sec: 0.86 +[2025-04-21 06:35:38] (step=0035175) Train Loss: 6.3569, Train Steps/Sec: 0.86 +[2025-04-21 06:36:07] (step=0035200) Train Loss: 6.3050, Train Steps/Sec: 0.86 +[2025-04-21 06:36:36] (step=0035225) Train Loss: 6.3530, Train Steps/Sec: 0.87 +[2025-04-21 06:37:05] (step=0035250) Train Loss: 6.3751, Train Steps/Sec: 0.86 +[2025-04-21 06:37:34] (step=0035275) Train Loss: 6.3640, Train Steps/Sec: 0.86 +[2025-04-21 06:38:03] (step=0035300) Train Loss: 6.3587, Train Steps/Sec: 0.86 +[2025-04-21 06:38:32] (step=0035325) Train Loss: 6.3018, Train Steps/Sec: 0.87 +[2025-04-21 06:39:01] (step=0035350) Train Loss: 6.3812, Train Steps/Sec: 0.86 +[2025-04-21 06:39:30] (step=0035375) Train Loss: 6.3254, Train Steps/Sec: 0.86 +[2025-04-21 06:39:59] (step=0035400) Train Loss: 6.3218, Train Steps/Sec: 0.86 +[2025-04-21 06:40:28] (step=0035425) Train Loss: 6.3515, Train Steps/Sec: 0.87 +[2025-04-21 06:40:57] (step=0035450) Train Loss: 6.3738, Train Steps/Sec: 0.86 +[2025-04-21 06:41:26] (step=0035475) Train Loss: 6.3773, Train Steps/Sec: 0.86 +[2025-04-21 06:41:55] (step=0035500) Train Loss: 6.3686, Train Steps/Sec: 0.86 +[2025-04-21 06:42:24] (step=0035525) Train Loss: 6.3396, Train Steps/Sec: 0.87 +[2025-04-21 06:42:53] (step=0035550) Train Loss: 6.3475, Train Steps/Sec: 0.87 +[2025-04-21 06:43:22] (step=0035575) Train Loss: 6.3448, Train Steps/Sec: 0.86 +[2025-04-21 06:43:51] (step=0035600) Train Loss: 6.3247, Train Steps/Sec: 0.86 +[2025-04-21 06:44:20] (step=0035625) Train Loss: 6.3567, Train Steps/Sec: 0.86 +[2025-04-21 06:44:49] (step=0035650) Train Loss: 6.3511, Train Steps/Sec: 0.86 +[2025-04-21 06:45:18] (step=0035675) Train Loss: 6.3892, Train Steps/Sec: 0.86 +[2025-04-21 06:45:47] (step=0035700) Train Loss: 6.3473, Train Steps/Sec: 0.86 +[2025-04-21 06:46:16] (step=0035725) Train Loss: 6.3324, Train Steps/Sec: 0.86 +[2025-04-21 06:46:45] (step=0035750) Train Loss: 6.3351, Train Steps/Sec: 0.86 +[2025-04-21 06:47:14] (step=0035775) Train Loss: 6.3669, Train Steps/Sec: 0.86 +[2025-04-21 06:47:43] (step=0035800) Train Loss: 6.3392, Train Steps/Sec: 0.86 +[2025-04-21 06:48:12] (step=0035825) Train Loss: 6.3484, Train Steps/Sec: 0.87 +[2025-04-21 06:48:41] (step=0035850) Train Loss: 6.3254, Train Steps/Sec: 0.86 +[2025-04-21 06:49:10] (step=0035875) Train Loss: 6.3521, Train Steps/Sec: 0.86 +[2025-04-21 06:49:39] (step=0035900) Train Loss: 6.3560, Train Steps/Sec: 0.86 +[2025-04-21 06:50:08] (step=0035925) Train Loss: 6.3514, Train Steps/Sec: 0.86 +[2025-04-21 06:50:37] (step=0035950) Train Loss: 6.3311, Train Steps/Sec: 0.87 +[2025-04-21 06:51:06] (step=0035975) Train Loss: 6.3172, Train Steps/Sec: 0.86 +[2025-04-21 06:51:35] (step=0036000) Train Loss: 6.3432, Train Steps/Sec: 0.86 +[2025-04-21 06:51:35] vision_config is None. initializing the InstructBlipVisionConfig with default values. +[2025-04-21 06:51:35] qformer_config is None. Initializing the InstructBlipQFormerConfig with default values. +[2025-04-21 06:51:35] text_config is None. Initializing the text config with default values (`OPTConfig`). +[2025-04-21 06:56:35] Finish Eval in 36000 steps... +[2025-04-21 06:56:57] Saved checkpoint to checkpoint/EmptyFix_InstructBlip_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_objects/002-GPT-XL/checkpoints/0036000.pt +[2025-04-21 06:56:59] Removed old checkpoint: checkpoint/EmptyFix_InstructBlip_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_objects/002-GPT-XL/checkpoints/0032000.pt +[2025-04-21 06:57:28] (step=0036025) Train Loss: 6.3512, Train Steps/Sec: 0.07 +[2025-04-21 06:58:04] (step=0036050) Train Loss: 6.3443, Train Steps/Sec: 0.71 +[2025-04-21 06:58:33] (step=0036075) Train Loss: 6.3503, Train Steps/Sec: 0.86 +[2025-04-21 06:59:02] (step=0036100) Train Loss: 6.3222, Train Steps/Sec: 0.86 +[2025-04-21 06:59:31] (step=0036125) Train Loss: 6.3539, Train Steps/Sec: 0.87 +[2025-04-21 07:00:00] (step=0036150) Train Loss: 6.3735, Train Steps/Sec: 0.87 +[2025-04-21 07:00:28] (step=0036175) Train Loss: 6.3481, Train Steps/Sec: 0.87 +[2025-04-21 07:00:58] (step=0036200) Train Loss: 6.3727, Train Steps/Sec: 0.85 +[2025-04-21 07:01:27] (step=0036225) Train Loss: 6.3583, Train Steps/Sec: 0.86 +[2025-04-21 07:01:56] (step=0036250) Train Loss: 6.3597, Train Steps/Sec: 0.86 +[2025-04-21 07:02:24] (step=0036275) Train Loss: 6.3344, Train Steps/Sec: 0.87 +[2025-04-21 07:02:54] (step=0036300) Train Loss: 6.3535, Train Steps/Sec: 0.86 +[2025-04-21 07:03:22] (step=0036325) Train Loss: 6.3246, Train Steps/Sec: 0.86 +[2025-04-21 07:03:51] (step=0036350) Train Loss: 6.3377, Train Steps/Sec: 0.87 +[2025-04-21 07:04:20] (step=0036375) Train Loss: 6.3380, Train Steps/Sec: 0.86 +[2025-04-21 07:04:50] (step=0036400) Train Loss: 6.3153, Train Steps/Sec: 0.85 +[2025-04-21 07:05:19] (step=0036425) Train Loss: 6.3124, Train Steps/Sec: 0.86 +[2025-04-21 07:05:47] (step=0036450) Train Loss: 6.3288, Train Steps/Sec: 0.87 +[2025-04-21 07:06:17] (step=0036475) Train Loss: 6.3996, Train Steps/Sec: 0.86 +[2025-04-21 07:06:46] (step=0036500) Train Loss: 6.3455, Train Steps/Sec: 0.86 +[2025-04-21 07:07:15] (step=0036525) Train Loss: 6.3838, Train Steps/Sec: 0.86 +[2025-04-21 07:07:44] (step=0036550) Train Loss: 6.3471, Train Steps/Sec: 0.86 +[2025-04-21 07:08:13] (step=0036575) Train Loss: 6.3403, Train Steps/Sec: 0.86 +[2025-04-21 07:08:42] (step=0036600) Train Loss: 6.3594, Train Steps/Sec: 0.86 +[2025-04-21 07:09:11] (step=0036625) Train Loss: 6.3033, Train Steps/Sec: 0.87 +[2025-04-21 07:09:47] (step=0036650) Train Loss: 6.3267, Train Steps/Sec: 0.69 +[2025-04-21 07:10:16] (step=0036675) Train Loss: 6.3850, Train Steps/Sec: 0.87 +[2025-04-21 07:10:45] (step=0036700) Train Loss: 6.3499, Train Steps/Sec: 0.86 +[2025-04-21 07:11:14] (step=0036725) Train Loss: 6.3429, Train Steps/Sec: 0.87 +[2025-04-21 07:11:43] (step=0036750) Train Loss: 6.3251, Train Steps/Sec: 0.86 +[2025-04-21 07:12:12] (step=0036775) Train Loss: 6.3110, Train Steps/Sec: 0.87 +[2025-04-21 07:12:41] (step=0036800) Train Loss: 6.3927, Train Steps/Sec: 0.86 +[2025-04-21 07:13:10] (step=0036825) Train Loss: 6.3142, Train Steps/Sec: 0.86 +[2025-04-21 07:13:38] (step=0036850) Train Loss: 6.3475, Train Steps/Sec: 0.87 +[2025-04-21 07:14:07] (step=0036875) Train Loss: 6.2929, Train Steps/Sec: 0.86 +[2025-04-21 07:14:37] (step=0036900) Train Loss: 6.3314, Train Steps/Sec: 0.86 +[2025-04-21 07:15:05] (step=0036925) Train Loss: 6.3500, Train Steps/Sec: 0.87 +[2025-04-21 07:15:34] (step=0036950) Train Loss: 6.4056, Train Steps/Sec: 0.86 +[2025-04-21 07:16:03] (step=0036975) Train Loss: 6.3125, Train Steps/Sec: 0.87 +[2025-04-21 07:16:33] (step=0037000) Train Loss: 6.3668, Train Steps/Sec: 0.86 +[2025-04-21 07:17:02] (step=0037025) Train Loss: 6.3610, Train Steps/Sec: 0.86 +[2025-04-21 07:17:30] (step=0037050) Train Loss: 6.3637, Train Steps/Sec: 0.86 +[2025-04-21 07:17:59] (step=0037075) Train Loss: 6.3473, Train Steps/Sec: 0.87 +[2025-04-21 07:18:29] (step=0037100) Train Loss: 6.3363, Train Steps/Sec: 0.86 +[2025-04-21 07:18:57] (step=0037125) Train Loss: 6.3611, Train Steps/Sec: 0.87 +[2025-04-21 07:19:26] (step=0037150) Train Loss: 6.3327, Train Steps/Sec: 0.86 +[2025-04-21 07:19:55] (step=0037175) Train Loss: 6.3202, Train Steps/Sec: 0.87 +[2025-04-21 07:20:24] (step=0037200) Train Loss: 6.3310, Train Steps/Sec: 0.86 +[2025-04-21 07:20:53] (step=0037225) Train Loss: 6.3426, Train Steps/Sec: 0.87 +[2025-04-21 07:21:22] (step=0037250) Train Loss: 6.3455, Train Steps/Sec: 0.87 +[2025-04-21 07:21:51] (step=0037275) Train Loss: 6.3697, Train Steps/Sec: 0.87 +[2025-04-21 07:22:20] (step=0037300) Train Loss: 6.3396, Train Steps/Sec: 0.86 +[2025-04-21 07:22:49] (step=0037325) Train Loss: 6.3309, Train Steps/Sec: 0.86 +[2025-04-21 07:23:18] (step=0037350) Train Loss: 6.3560, Train Steps/Sec: 0.87 +[2025-04-21 07:23:47] (step=0037375) Train Loss: 6.3322, Train Steps/Sec: 0.86 +[2025-04-21 07:24:16] (step=0037400) Train Loss: 6.3518, Train Steps/Sec: 0.86 +[2025-04-21 07:24:45] (step=0037425) Train Loss: 6.3518, Train Steps/Sec: 0.87 +[2025-04-21 07:25:14] (step=0037450) Train Loss: 6.3780, Train Steps/Sec: 0.86 +[2025-04-21 07:25:49] (step=0037475) Train Loss: 6.3342, Train Steps/Sec: 0.70 +[2025-04-21 07:26:18] (step=0037500) Train Loss: 6.3120, Train Steps/Sec: 0.87 +[2025-04-21 07:26:47] (step=0037525) Train Loss: 6.3522, Train Steps/Sec: 0.86 +[2025-04-21 07:27:16] (step=0037550) Train Loss: 6.3282, Train Steps/Sec: 0.86 +[2025-04-21 07:27:45] (step=0037575) Train Loss: 6.3630, Train Steps/Sec: 0.86 +[2025-04-21 07:28:15] (step=0037600) Train Loss: 6.2970, Train Steps/Sec: 0.86 +[2025-04-21 07:28:43] (step=0037625) Train Loss: 6.3258, Train Steps/Sec: 0.87 +[2025-04-21 07:29:13] (step=0037650) Train Loss: 6.3495, Train Steps/Sec: 0.86 +[2025-04-21 07:29:42] (step=0037675) Train Loss: 6.3584, Train Steps/Sec: 0.86 +[2025-04-21 07:30:11] (step=0037700) Train Loss: 6.3177, Train Steps/Sec: 0.86 +[2025-04-21 07:30:40] (step=0037725) Train Loss: 6.3937, Train Steps/Sec: 0.86 +[2025-04-21 07:31:09] (step=0037750) Train Loss: 6.2960, Train Steps/Sec: 0.86 +[2025-04-21 07:31:38] (step=0037775) Train Loss: 6.3160, Train Steps/Sec: 0.87 +[2025-04-21 07:32:06] (step=0037800) Train Loss: 6.3358, Train Steps/Sec: 0.87 +[2025-04-21 07:32:36] (step=0037825) Train Loss: 6.3448, Train Steps/Sec: 0.86 +[2025-04-21 07:33:05] (step=0037850) Train Loss: 6.3323, Train Steps/Sec: 0.86 +[2025-04-21 07:33:34] (step=0037875) Train Loss: 6.3514, Train Steps/Sec: 0.86 +[2025-04-21 07:34:03] (step=0037900) Train Loss: 6.3267, Train Steps/Sec: 0.85 +[2025-04-21 07:34:32] (step=0037925) Train Loss: 6.3423, Train Steps/Sec: 0.87 +[2025-04-21 07:35:00] (step=0037950) Train Loss: 6.3190, Train Steps/Sec: 0.87 +[2025-04-21 07:35:29] (step=0037975) Train Loss: 6.3968, Train Steps/Sec: 0.86 +[2025-04-21 07:35:58] (step=0038000) Train Loss: 6.3746, Train Steps/Sec: 0.86 +[2025-04-21 07:35:58] vision_config is None. initializing the InstructBlipVisionConfig with default values. +[2025-04-21 07:35:58] qformer_config is None. Initializing the InstructBlipQFormerConfig with default values. +[2025-04-21 07:35:58] text_config is None. Initializing the text config with default values (`OPTConfig`). +[2025-04-21 07:41:01] Finish Eval in 38000 steps... +[2025-04-21 07:41:22] Saved checkpoint to checkpoint/EmptyFix_InstructBlip_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_objects/002-GPT-XL/checkpoints/0038000.pt +[2025-04-21 07:41:24] Removed old checkpoint: checkpoint/EmptyFix_InstructBlip_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_objects/002-GPT-XL/checkpoints/0034000.pt +[2025-04-21 07:41:54] (step=0038025) Train Loss: 6.3458, Train Steps/Sec: 0.07 +[2025-04-21 07:42:22] (step=0038050) Train Loss: 6.3498, Train Steps/Sec: 0.87 +[2025-04-21 07:42:52] (step=0038075) Train Loss: 6.3843, Train Steps/Sec: 0.86 +[2025-04-21 07:43:20] (step=0038100) Train Loss: 6.3375, Train Steps/Sec: 0.86 +[2025-04-21 07:43:50] (step=0038125) Train Loss: 6.3763, Train Steps/Sec: 0.86 +[2025-04-21 07:44:18] (step=0038150) Train Loss: 6.3122, Train Steps/Sec: 0.87 +[2025-04-21 07:44:47] (step=0038175) Train Loss: 6.3205, Train Steps/Sec: 0.87 +[2025-04-21 07:45:16] (step=0038200) Train Loss: 6.3253, Train Steps/Sec: 0.86 +[2025-04-21 07:45:45] (step=0038225) Train Loss: 6.3043, Train Steps/Sec: 0.86 +[2025-04-21 07:46:15] (step=0038250) Train Loss: 6.3291, Train Steps/Sec: 0.86 +[2025-04-21 07:46:44] (step=0038275) Train Loss: 6.3330, Train Steps/Sec: 0.86 +[2025-04-21 07:47:13] (step=0038300) Train Loss: 6.3704, Train Steps/Sec: 0.85 +[2025-04-21 07:47:42] (step=0038325) Train Loss: 6.3672, Train Steps/Sec: 0.86 +[2025-04-21 07:48:11] (step=0038350) Train Loss: 6.3459, Train Steps/Sec: 0.86 +[2025-04-21 07:48:40] (step=0038375) Train Loss: 6.3592, Train Steps/Sec: 0.87 +[2025-04-21 07:49:09] (step=0038400) Train Loss: 6.3584, Train Steps/Sec: 0.86 +[2025-04-21 07:49:38] (step=0038425) Train Loss: 6.4017, Train Steps/Sec: 0.86 +[2025-04-21 07:50:07] (step=0038450) Train Loss: 6.3158, Train Steps/Sec: 0.86 +[2025-04-21 07:50:36] (step=0038475) Train Loss: 6.3427, Train Steps/Sec: 0.87 +[2025-04-21 07:51:05] (step=0038500) Train Loss: 6.3746, Train Steps/Sec: 0.85 +[2025-04-21 07:51:34] (step=0038525) Train Loss: 6.3166, Train Steps/Sec: 0.87 +[2025-04-21 07:52:03] (step=0038550) Train Loss: 6.3380, Train Steps/Sec: 0.86 +[2025-04-21 07:52:32] (step=0038575) Train Loss: 6.3544, Train Steps/Sec: 0.87 +[2025-04-21 07:53:01] (step=0038600) Train Loss: 6.3234, Train Steps/Sec: 0.86 +[2025-04-21 07:53:30] (step=0038625) Train Loss: 6.3427, Train Steps/Sec: 0.87 +[2025-04-21 07:53:59] (step=0038650) Train Loss: 6.3417, Train Steps/Sec: 0.86 +[2025-04-21 07:54:28] (step=0038675) Train Loss: 6.3687, Train Steps/Sec: 0.86 +[2025-04-21 07:55:04] (step=0038700) Train Loss: 6.3568, Train Steps/Sec: 0.86 +[2025-04-21 07:55:33] (step=0038725) Train Loss: 6.3656, Train Steps/Sec: 0.86 +[2025-04-21 07:56:02] (step=0038750) Train Loss: 6.3732, Train Steps/Sec: 0.86 +[2025-04-21 07:56:31] (step=0038775) Train Loss: 6.3440, Train Steps/Sec: 0.86 +[2025-04-21 07:57:00] (step=0038800) Train Loss: 6.3895, Train Steps/Sec: 0.86 +[2025-04-21 07:57:29] (step=0038825) Train Loss: 6.3425, Train Steps/Sec: 0.86 +[2025-04-21 07:57:58] (step=0038850) Train Loss: 6.3325, Train Steps/Sec: 0.86 +[2025-04-21 07:58:27] (step=0038875) Train Loss: 6.3478, Train Steps/Sec: 0.87 +[2025-04-21 07:58:56] (step=0038900) Train Loss: 6.3157, Train Steps/Sec: 0.85 +[2025-04-21 07:59:25] (step=0038925) Train Loss: 6.3480, Train Steps/Sec: 0.87 +[2025-04-21 07:59:54] (step=0038950) Train Loss: 6.3238, Train Steps/Sec: 0.86 +[2025-04-21 08:00:23] (step=0038975) Train Loss: 6.3272, Train Steps/Sec: 0.86 +[2025-04-21 08:00:52] (step=0039000) Train Loss: 6.3403, Train Steps/Sec: 0.86 +[2025-04-21 08:01:21] (step=0039025) Train Loss: 6.3470, Train Steps/Sec: 0.86 +[2025-04-21 08:01:50] (step=0039050) Train Loss: 6.3534, Train Steps/Sec: 0.86 +[2025-04-21 08:02:18] (step=0039075) Train Loss: 6.2928, Train Steps/Sec: 0.87 +[2025-04-21 08:02:48] (step=0039100) Train Loss: 6.3658, Train Steps/Sec: 0.85 +[2025-04-21 08:03:17] (step=0039125) Train Loss: 6.3640, Train Steps/Sec: 0.86 +[2025-04-21 08:03:46] (step=0039150) Train Loss: 6.3221, Train Steps/Sec: 0.86 +[2025-04-21 08:04:15] (step=0039175) Train Loss: 6.3509, Train Steps/Sec: 0.87 +[2025-04-21 08:04:50] (step=0039200) Train Loss: 6.3545, Train Steps/Sec: 0.70 +[2025-04-21 08:05:19] (step=0039225) Train Loss: 6.3093, Train Steps/Sec: 0.86 +[2025-04-21 08:05:48] (step=0039250) Train Loss: 6.3438, Train Steps/Sec: 0.86 +[2025-04-21 08:06:17] (step=0039275) Train Loss: 6.3087, Train Steps/Sec: 0.86 +[2025-04-21 08:06:46] (step=0039300) Train Loss: 6.3728, Train Steps/Sec: 0.86 +[2025-04-21 08:07:15] (step=0039325) Train Loss: 6.3312, Train Steps/Sec: 0.87 +[2025-04-21 08:07:44] (step=0039350) Train Loss: 6.3773, Train Steps/Sec: 0.86 +[2025-04-21 08:08:13] (step=0039375) Train Loss: 6.3174, Train Steps/Sec: 0.86 +[2025-04-21 08:08:42] (step=0039400) Train Loss: 6.3439, Train Steps/Sec: 0.86 +[2025-04-21 08:09:11] (step=0039425) Train Loss: 6.3225, Train Steps/Sec: 0.86 +[2025-04-21 08:09:40] (step=0039450) Train Loss: 6.3096, Train Steps/Sec: 0.86 +[2025-04-21 08:10:09] (step=0039475) Train Loss: 6.3392, Train Steps/Sec: 0.87 +[2025-04-21 08:10:38] (step=0039500) Train Loss: 6.3228, Train Steps/Sec: 0.87 +[2025-04-21 08:11:07] (step=0039525) Train Loss: 6.3177, Train Steps/Sec: 0.87 +[2025-04-21 08:11:36] (step=0039550) Train Loss: 6.3412, Train Steps/Sec: 0.86 +[2025-04-21 08:12:05] (step=0039575) Train Loss: 6.4107, Train Steps/Sec: 0.86 +[2025-04-21 08:12:45] (step=0039600) Train Loss: 6.2908, Train Steps/Sec: 0.62 +[2025-04-21 08:13:14] (step=0039625) Train Loss: 6.3588, Train Steps/Sec: 0.86 +[2025-04-21 08:13:43] (step=0039650) Train Loss: 6.3531, Train Steps/Sec: 0.86 +[2025-04-21 08:14:12] (step=0039675) Train Loss: 6.3719, Train Steps/Sec: 0.86 +[2025-04-21 08:14:41] (step=0039700) Train Loss: 6.3606, Train Steps/Sec: 0.86 +[2025-04-21 08:15:10] (step=0039725) Train Loss: 6.3331, Train Steps/Sec: 0.86 +[2025-04-21 08:15:39] (step=0039750) Train Loss: 6.3333, Train Steps/Sec: 0.86 +[2025-04-21 08:16:08] (step=0039775) Train Loss: 6.3528, Train Steps/Sec: 0.86 +[2025-04-21 08:16:44] (step=0039800) Train Loss: 6.3159, Train Steps/Sec: 0.70 +[2025-04-21 08:17:13] (step=0039825) Train Loss: 6.3681, Train Steps/Sec: 0.86 +[2025-04-21 08:17:42] (step=0039850) Train Loss: 6.3231, Train Steps/Sec: 0.86 +[2025-04-21 08:18:11] (step=0039875) Train Loss: 6.3493, Train Steps/Sec: 0.87 +[2025-04-21 08:18:40] (step=0039900) Train Loss: 6.3536, Train Steps/Sec: 0.86 +[2025-04-21 08:19:09] (step=0039925) Train Loss: 6.3566, Train Steps/Sec: 0.87 +[2025-04-21 08:19:38] (step=0039950) Train Loss: 6.3813, Train Steps/Sec: 0.86 +[2025-04-21 08:20:07] (step=0039975) Train Loss: 6.3212, Train Steps/Sec: 0.86 +[2025-04-21 08:20:36] (step=0040000) Train Loss: 6.3735, Train Steps/Sec: 0.86 +[2025-04-21 08:20:36] vision_config is None. initializing the InstructBlipVisionConfig with default values. +[2025-04-21 08:20:36] qformer_config is None. Initializing the InstructBlipQFormerConfig with default values. +[2025-04-21 08:20:36] text_config is None. Initializing the text config with default values (`OPTConfig`). +[2025-04-21 08:25:44] Finish Eval in 40000 steps... +[2025-04-21 08:26:06] Saved checkpoint to checkpoint/EmptyFix_InstructBlip_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_objects/002-GPT-XL/checkpoints/0040000.pt +[2025-04-21 08:26:09] Removed old checkpoint: checkpoint/EmptyFix_InstructBlip_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_objects/002-GPT-XL/checkpoints/0036000.pt +[2025-04-21 08:26:38] (step=0040025) Train Loss: 6.3967, Train Steps/Sec: 0.07 +[2025-04-21 08:27:07] (step=0040050) Train Loss: 6.3284, Train Steps/Sec: 0.86 +[2025-04-21 08:27:36] (step=0040075) Train Loss: 6.3149, Train Steps/Sec: 0.86 +[2025-04-21 08:28:06] (step=0040100) Train Loss: 6.3586, Train Steps/Sec: 0.85 +[2025-04-21 08:28:35] (step=0040125) Train Loss: 6.3523, Train Steps/Sec: 0.86 +[2025-04-21 08:29:04] (step=0040150) Train Loss: 6.3411, Train Steps/Sec: 0.86 +[2025-04-21 08:29:33] (step=0040175) Train Loss: 6.3492, Train Steps/Sec: 0.87 +[2025-04-21 08:30:02] (step=0040200) Train Loss: 6.3589, Train Steps/Sec: 0.86 +[2025-04-21 08:30:31] (step=0040225) Train Loss: 6.3615, Train Steps/Sec: 0.86 +[2025-04-21 08:31:00] (step=0040250) Train Loss: 6.3064, Train Steps/Sec: 0.86 +[2025-04-21 08:31:28] (step=0040275) Train Loss: 6.3484, Train Steps/Sec: 0.87 +[2025-04-21 08:31:58] (step=0040300) Train Loss: 6.3621, Train Steps/Sec: 0.86 +[2025-04-21 08:32:27] (step=0040325) Train Loss: 6.3579, Train Steps/Sec: 0.86 +[2025-04-21 08:32:56] (step=0040350) Train Loss: 6.3565, Train Steps/Sec: 0.86 +[2025-04-21 08:33:24] (step=0040375) Train Loss: 6.3377, Train Steps/Sec: 0.87 +[2025-04-21 08:33:53] (step=0040400) Train Loss: 6.3935, Train Steps/Sec: 0.86 +[2025-04-21 08:34:22] (step=0040425) Train Loss: 6.3442, Train Steps/Sec: 0.87 +[2025-04-21 08:34:51] (step=0040450) Train Loss: 6.3519, Train Steps/Sec: 0.87 +[2025-04-21 08:35:20] (step=0040475) Train Loss: 6.3160, Train Steps/Sec: 0.86 +[2025-04-21 08:35:49] (step=0040500) Train Loss: 6.3269, Train Steps/Sec: 0.86 +[2025-04-21 08:36:18] (step=0040525) Train Loss: 6.3457, Train Steps/Sec: 0.86 +[2025-04-21 08:36:47] (step=0040550) Train Loss: 6.3718, Train Steps/Sec: 0.87 +[2025-04-21 08:37:16] (step=0040575) Train Loss: 6.3177, Train Steps/Sec: 0.86 +[2025-04-21 08:37:45] (step=0040600) Train Loss: 6.3849, Train Steps/Sec: 0.86 +[2025-04-21 08:38:13] (step=0040625) Train Loss: 6.3518, Train Steps/Sec: 0.87 +[2025-04-21 08:38:42] (step=0040650) Train Loss: 6.3771, Train Steps/Sec: 0.86 +[2025-04-21 08:39:11] (step=0040675) Train Loss: 6.3330, Train Steps/Sec: 0.87 +[2025-04-21 08:39:40] (step=0040700) Train Loss: 6.3644, Train Steps/Sec: 0.87 +[2025-04-21 08:40:09] (step=0040725) Train Loss: 6.3491, Train Steps/Sec: 0.87 +[2025-04-21 08:40:38] (step=0040750) Train Loss: 6.3437, Train Steps/Sec: 0.87 +[2025-04-21 08:41:07] (step=0040775) Train Loss: 6.3393, Train Steps/Sec: 0.86 +[2025-04-21 08:41:36] (step=0040800) Train Loss: 6.3309, Train Steps/Sec: 0.85 +[2025-04-21 08:42:05] (step=0040825) Train Loss: 6.3604, Train Steps/Sec: 0.87 +[2025-04-21 08:42:34] (step=0040850) Train Loss: 6.3307, Train Steps/Sec: 0.86 +[2025-04-21 08:43:03] (step=0040875) Train Loss: 6.3429, Train Steps/Sec: 0.87 +[2025-04-21 08:43:32] (step=0040900) Train Loss: 6.3512, Train Steps/Sec: 0.86 +[2025-04-21 08:43:37] Beginning epoch 1... +[2025-04-21 08:44:05] (step=0040925) Train Loss: 6.3596, Train Steps/Sec: 0.75 +[2025-04-21 08:44:34] (step=0040950) Train Loss: 6.3512, Train Steps/Sec: 0.86 +[2025-04-21 08:45:03] (step=0040975) Train Loss: 6.3151, Train Steps/Sec: 0.86 +[2025-04-21 08:45:32] (step=0041000) Train Loss: 6.2929, Train Steps/Sec: 0.86 +[2025-04-21 08:46:01] (step=0041025) Train Loss: 6.2896, Train Steps/Sec: 0.86 +[2025-04-21 08:46:30] (step=0041050) Train Loss: 6.3067, Train Steps/Sec: 0.86 +[2025-04-21 08:46:59] (step=0041075) Train Loss: 6.2805, Train Steps/Sec: 0.86 +[2025-04-21 08:47:28] (step=0041100) Train Loss: 6.3232, Train Steps/Sec: 0.86 +[2025-04-21 08:47:57] (step=0041125) Train Loss: 6.3318, Train Steps/Sec: 0.87 +[2025-04-21 08:48:26] (step=0041150) Train Loss: 6.3513, Train Steps/Sec: 0.86 +[2025-04-21 08:48:56] (step=0041175) Train Loss: 6.3623, Train Steps/Sec: 0.86 +[2025-04-21 08:49:25] (step=0041200) Train Loss: 6.3247, Train Steps/Sec: 0.86 +[2025-04-21 08:49:54] (step=0041225) Train Loss: 6.3178, Train Steps/Sec: 0.86 +[2025-04-21 08:50:23] (step=0041250) Train Loss: 6.2936, Train Steps/Sec: 0.86 +[2025-04-21 08:50:52] (step=0041275) Train Loss: 6.3588, Train Steps/Sec: 0.86 +[2025-04-21 08:51:21] (step=0041300) Train Loss: 6.3148, Train Steps/Sec: 0.86 +[2025-04-21 08:51:50] (step=0041325) Train Loss: 6.2817, Train Steps/Sec: 0.86 +[2025-04-21 08:52:19] (step=0041350) Train Loss: 6.3594, Train Steps/Sec: 0.86 +[2025-04-21 08:52:48] (step=0041375) Train Loss: 6.3353, Train Steps/Sec: 0.87 +[2025-04-21 08:53:17] (step=0041400) Train Loss: 6.3569, Train Steps/Sec: 0.86 +[2025-04-21 08:53:46] (step=0041425) Train Loss: 6.3383, Train Steps/Sec: 0.86 +[2025-04-21 08:54:15] (step=0041450) Train Loss: 6.3326, Train Steps/Sec: 0.86 +[2025-04-21 08:54:44] (step=0041475) Train Loss: 6.3481, Train Steps/Sec: 0.86 +[2025-04-21 08:55:13] (step=0041500) Train Loss: 6.3145, Train Steps/Sec: 0.86 +[2025-04-21 08:55:42] (step=0041525) Train Loss: 6.3153, Train Steps/Sec: 0.86 +[2025-04-21 08:56:20] (step=0041550) Train Loss: 6.3619, Train Steps/Sec: 0.65 +[2025-04-21 08:56:49] (step=0041575) Train Loss: 6.3373, Train Steps/Sec: 0.86 +[2025-04-21 08:57:19] (step=0041600) Train Loss: 6.3153, Train Steps/Sec: 0.85 +[2025-04-21 08:57:48] (step=0041625) Train Loss: 6.2986, Train Steps/Sec: 0.86 +[2025-04-21 08:58:17] (step=0041650) Train Loss: 6.3021, Train Steps/Sec: 0.87 +[2025-04-21 08:58:45] (step=0041675) Train Loss: 6.3171, Train Steps/Sec: 0.87 +[2025-04-21 08:59:15] (step=0041700) Train Loss: 6.3390, Train Steps/Sec: 0.86 +[2025-04-21 08:59:44] (step=0041725) Train Loss: 6.3166, Train Steps/Sec: 0.86 +[2025-04-21 09:00:13] (step=0041750) Train Loss: 6.2793, Train Steps/Sec: 0.86 +[2025-04-21 09:00:42] (step=0041775) Train Loss: 6.3067, Train Steps/Sec: 0.86 +[2025-04-21 09:01:11] (step=0041800) Train Loss: 6.2992, Train Steps/Sec: 0.86 +[2025-04-21 09:01:40] (step=0041825) Train Loss: 6.3134, Train Steps/Sec: 0.86 +[2025-04-21 09:02:09] (step=0041850) Train Loss: 6.3616, Train Steps/Sec: 0.86 +[2025-04-21 09:02:38] (step=0041875) Train Loss: 6.3514, Train Steps/Sec: 0.86 +[2025-04-21 09:03:07] (step=0041900) Train Loss: 6.3211, Train Steps/Sec: 0.86 +[2025-04-21 09:03:36] (step=0041925) Train Loss: 6.3218, Train Steps/Sec: 0.86 +[2025-04-21 09:04:05] (step=0041950) Train Loss: 6.3305, Train Steps/Sec: 0.86 +[2025-04-21 09:04:34] (step=0041975) Train Loss: 6.3133, Train Steps/Sec: 0.86 +[2025-04-21 09:05:04] (step=0042000) Train Loss: 6.3283, Train Steps/Sec: 0.85 +[2025-04-21 09:05:04] vision_config is None. initializing the InstructBlipVisionConfig with default values. +[2025-04-21 09:05:04] qformer_config is None. Initializing the InstructBlipQFormerConfig with default values. +[2025-04-21 09:05:04] text_config is None. Initializing the text config with default values (`OPTConfig`). +[2025-04-21 09:10:07] Finish Eval in 42000 steps... +[2025-04-21 09:10:29] Saved checkpoint to checkpoint/EmptyFix_InstructBlip_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_objects/002-GPT-XL/checkpoints/0042000.pt +[2025-04-21 09:10:32] Removed old checkpoint: checkpoint/EmptyFix_InstructBlip_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_objects/002-GPT-XL/checkpoints/0038000.pt +[2025-04-21 09:11:01] (step=0042025) Train Loss: 6.2914, Train Steps/Sec: 0.07 +[2025-04-21 09:11:30] (step=0042050) Train Loss: 6.3593, Train Steps/Sec: 0.86 +[2025-04-21 09:11:58] (step=0042075) Train Loss: 6.3028, Train Steps/Sec: 0.87 +[2025-04-21 09:12:28] (step=0042100) Train Loss: 6.3152, Train Steps/Sec: 0.86 +[2025-04-21 09:12:56] (step=0042125) Train Loss: 6.3577, Train Steps/Sec: 0.87 +[2025-04-21 09:13:26] (step=0042150) Train Loss: 6.3234, Train Steps/Sec: 0.86 +[2025-04-21 09:13:55] (step=0042175) Train Loss: 6.3580, Train Steps/Sec: 0.86 +[2025-04-21 09:14:33] (step=0042200) Train Loss: 6.2841, Train Steps/Sec: 0.65 +[2025-04-21 09:15:02] (step=0042225) Train Loss: 6.3235, Train Steps/Sec: 0.87 +[2025-04-21 09:15:31] (step=0042250) Train Loss: 6.2882, Train Steps/Sec: 0.86 +[2025-04-21 09:16:00] (step=0042275) Train Loss: 6.3025, Train Steps/Sec: 0.86 +[2025-04-21 09:16:29] (step=0042300) Train Loss: 6.3415, Train Steps/Sec: 0.86 +[2025-04-21 09:16:58] (step=0042325) Train Loss: 6.3108, Train Steps/Sec: 0.87 +[2025-04-21 09:17:27] (step=0042350) Train Loss: 6.2797, Train Steps/Sec: 0.86 +[2025-04-21 09:17:56] (step=0042375) Train Loss: 6.2853, Train Steps/Sec: 0.86 +[2025-04-21 09:18:26] (step=0042400) Train Loss: 6.3110, Train Steps/Sec: 0.86 +[2025-04-21 09:18:55] (step=0042425) Train Loss: 6.3531, Train Steps/Sec: 0.86 +[2025-04-21 09:19:24] (step=0042450) Train Loss: 6.3467, Train Steps/Sec: 0.86 +[2025-04-21 09:19:53] (step=0042475) Train Loss: 6.3481, Train Steps/Sec: 0.86 +[2025-04-21 09:20:22] (step=0042500) Train Loss: 6.3150, Train Steps/Sec: 0.86 +[2025-04-21 09:20:51] (step=0042525) Train Loss: 6.2922, Train Steps/Sec: 0.86 +[2025-04-21 09:21:20] (step=0042550) Train Loss: 6.3322, Train Steps/Sec: 0.86 +[2025-04-21 09:21:49] (step=0042575) Train Loss: 6.3406, Train Steps/Sec: 0.86 +[2025-04-21 09:22:19] (step=0042600) Train Loss: 6.3244, Train Steps/Sec: 0.85 +[2025-04-21 09:22:48] (step=0042625) Train Loss: 6.2994, Train Steps/Sec: 0.86 +[2025-04-21 09:23:17] (step=0042650) Train Loss: 6.2766, Train Steps/Sec: 0.87 +[2025-04-21 09:23:46] (step=0042675) Train Loss: 6.3568, Train Steps/Sec: 0.86 +[2025-04-21 09:24:15] (step=0042700) Train Loss: 6.3285, Train Steps/Sec: 0.86 +[2025-04-21 09:24:44] (step=0042725) Train Loss: 6.3018, Train Steps/Sec: 0.86 +[2025-04-21 09:25:13] (step=0042750) Train Loss: 6.3434, Train Steps/Sec: 0.86 +[2025-04-21 09:25:42] (step=0042775) Train Loss: 6.3013, Train Steps/Sec: 0.86 +[2025-04-21 09:26:11] (step=0042800) Train Loss: 6.2784, Train Steps/Sec: 0.86 +[2025-04-21 09:26:40] (step=0042825) Train Loss: 6.3155, Train Steps/Sec: 0.86 +[2025-04-21 09:27:18] (step=0042850) Train Loss: 6.2855, Train Steps/Sec: 0.67 +[2025-04-21 09:27:47] (step=0042875) Train Loss: 6.3155, Train Steps/Sec: 0.86 +[2025-04-21 09:28:16] (step=0042900) Train Loss: 6.3452, Train Steps/Sec: 0.86 +[2025-04-21 09:28:45] (step=0042925) Train Loss: 6.3092, Train Steps/Sec: 0.86 +[2025-04-21 09:29:14] (step=0042950) Train Loss: 6.2984, Train Steps/Sec: 0.86 +[2025-04-21 09:29:43] (step=0042975) Train Loss: 6.2929, Train Steps/Sec: 0.86 +[2025-04-21 09:30:12] (step=0043000) Train Loss: 6.3106, Train Steps/Sec: 0.87 +[2025-04-21 09:30:41] (step=0043025) Train Loss: 6.2829, Train Steps/Sec: 0.86 +[2025-04-21 09:31:10] (step=0043050) Train Loss: 6.3297, Train Steps/Sec: 0.86 +[2025-04-21 09:31:39] (step=0043075) Train Loss: 6.3457, Train Steps/Sec: 0.85 +[2025-04-21 09:32:08] (step=0043100) Train Loss: 6.3224, Train Steps/Sec: 0.86 +[2025-04-21 09:32:37] (step=0043125) Train Loss: 6.3026, Train Steps/Sec: 0.86 +[2025-04-21 09:33:06] (step=0043150) Train Loss: 6.3288, Train Steps/Sec: 0.86 +[2025-04-21 09:33:35] (step=0043175) Train Loss: 6.3085, Train Steps/Sec: 0.86 +[2025-04-21 09:34:04] (step=0043200) Train Loss: 6.3326, Train Steps/Sec: 0.86 +[2025-04-21 09:34:34] (step=0043225) Train Loss: 6.3197, Train Steps/Sec: 0.86 +[2025-04-21 09:35:03] (step=0043250) Train Loss: 6.3063, Train Steps/Sec: 0.86 +[2025-04-21 09:35:32] (step=0043275) Train Loss: 6.3041, Train Steps/Sec: 0.87 +[2025-04-21 09:36:01] (step=0043300) Train Loss: 6.3308, Train Steps/Sec: 0.86 +[2025-04-21 09:36:30] (step=0043325) Train Loss: 6.3103, Train Steps/Sec: 0.87 +[2025-04-21 09:36:59] (step=0043350) Train Loss: 6.3368, Train Steps/Sec: 0.86 +[2025-04-21 09:37:28] (step=0043375) Train Loss: 6.3489, Train Steps/Sec: 0.86 +[2025-04-21 09:37:57] (step=0043400) Train Loss: 6.2886, Train Steps/Sec: 0.86 +[2025-04-21 09:38:26] (step=0043425) Train Loss: 6.3115, Train Steps/Sec: 0.86 +[2025-04-21 09:38:55] (step=0043450) Train Loss: 6.3122, Train Steps/Sec: 0.86 +[2025-04-21 09:39:24] (step=0043475) Train Loss: 6.3447, Train Steps/Sec: 0.86 +[2025-04-21 09:39:53] (step=0043500) Train Loss: 6.3194, Train Steps/Sec: 0.85 +[2025-04-21 09:40:22] (step=0043525) Train Loss: 6.3513, Train Steps/Sec: 0.86 +[2025-04-21 09:40:51] (step=0043550) Train Loss: 6.3005, Train Steps/Sec: 0.86 +[2025-04-21 09:41:20] (step=0043575) Train Loss: 6.3220, Train Steps/Sec: 0.87 +[2025-04-21 09:41:49] (step=0043600) Train Loss: 6.3659, Train Steps/Sec: 0.86 +[2025-04-21 09:42:18] (step=0043625) Train Loss: 6.3240, Train Steps/Sec: 0.86 +[2025-04-21 09:42:47] (step=0043650) Train Loss: 6.2759, Train Steps/Sec: 0.86 +[2025-04-21 09:43:16] (step=0043675) Train Loss: 6.3215, Train Steps/Sec: 0.86 +[2025-04-21 09:43:45] (step=0043700) Train Loss: 6.3394, Train Steps/Sec: 0.86 +[2025-04-21 09:44:14] (step=0043725) Train Loss: 6.3077, Train Steps/Sec: 0.86 +[2025-04-21 09:44:44] (step=0043750) Train Loss: 6.3277, Train Steps/Sec: 0.86 +[2025-04-21 09:45:12] (step=0043775) Train Loss: 6.3043, Train Steps/Sec: 0.86 +[2025-04-21 09:45:41] (step=0043800) Train Loss: 6.2903, Train Steps/Sec: 0.86 +[2025-04-21 09:46:11] (step=0043825) Train Loss: 6.2843, Train Steps/Sec: 0.86 +[2025-04-21 09:46:40] (step=0043850) Train Loss: 6.3233, Train Steps/Sec: 0.86 +[2025-04-21 09:47:09] (step=0043875) Train Loss: 6.3341, Train Steps/Sec: 0.86 +[2025-04-21 09:47:38] (step=0043900) Train Loss: 6.3591, Train Steps/Sec: 0.86 +[2025-04-21 09:48:07] (step=0043925) Train Loss: 6.3205, Train Steps/Sec: 0.86 +[2025-04-21 09:48:36] (step=0043950) Train Loss: 6.3255, Train Steps/Sec: 0.86 +[2025-04-21 09:49:05] (step=0043975) Train Loss: 6.2868, Train Steps/Sec: 0.86 +[2025-04-21 09:49:34] (step=0044000) Train Loss: 6.3435, Train Steps/Sec: 0.86 +[2025-04-21 09:49:34] vision_config is None. initializing the InstructBlipVisionConfig with default values. +[2025-04-21 09:49:34] qformer_config is None. Initializing the InstructBlipQFormerConfig with default values. +[2025-04-21 09:49:34] text_config is None. Initializing the text config with default values (`OPTConfig`). +[2025-04-21 09:54:33] Finish Eval in 44000 steps... +[2025-04-21 09:54:54] Saved checkpoint to checkpoint/EmptyFix_InstructBlip_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_objects/002-GPT-XL/checkpoints/0044000.pt +[2025-04-21 09:54:57] Removed old checkpoint: checkpoint/EmptyFix_InstructBlip_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_objects/002-GPT-XL/checkpoints/0040000.pt +[2025-04-21 09:55:26] (step=0044025) Train Loss: 6.3354, Train Steps/Sec: 0.07 +[2025-04-21 09:55:55] (step=0044050) Train Loss: 6.3011, Train Steps/Sec: 0.86 +[2025-04-21 09:56:24] (step=0044075) Train Loss: 6.3236, Train Steps/Sec: 0.86 +[2025-04-21 09:56:53] (step=0044100) Train Loss: 6.3273, Train Steps/Sec: 0.86 +[2025-04-21 09:57:22] (step=0044125) Train Loss: 6.3029, Train Steps/Sec: 0.86 +[2025-04-21 09:57:51] (step=0044150) Train Loss: 6.3283, Train Steps/Sec: 0.86 +[2025-04-21 09:58:20] (step=0044175) Train Loss: 6.3083, Train Steps/Sec: 0.87 +[2025-04-21 09:58:49] (step=0044200) Train Loss: 6.3270, Train Steps/Sec: 0.86 +[2025-04-21 09:59:18] (step=0044225) Train Loss: 6.2861, Train Steps/Sec: 0.87 +[2025-04-21 09:59:55] (step=0044250) Train Loss: 6.3370, Train Steps/Sec: 0.67 +[2025-04-21 10:00:25] (step=0044275) Train Loss: 6.3234, Train Steps/Sec: 0.86 +[2025-04-21 10:00:54] (step=0044300) Train Loss: 6.3125, Train Steps/Sec: 0.86 +[2025-04-21 10:01:23] (step=0044325) Train Loss: 6.3417, Train Steps/Sec: 0.86 +[2025-04-21 10:01:52] (step=0044350) Train Loss: 6.2993, Train Steps/Sec: 0.87 +[2025-04-21 10:02:21] (step=0044375) Train Loss: 6.3324, Train Steps/Sec: 0.87 +[2025-04-21 10:02:50] (step=0044400) Train Loss: 6.3464, Train Steps/Sec: 0.86 +[2025-04-21 10:03:19] (step=0044425) Train Loss: 6.3215, Train Steps/Sec: 0.86 +[2025-04-21 10:03:48] (step=0044450) Train Loss: 6.3056, Train Steps/Sec: 0.86 +[2025-04-21 10:04:16] (step=0044475) Train Loss: 6.3270, Train Steps/Sec: 0.87 +[2025-04-21 10:04:46] (step=0044500) Train Loss: 6.3626, Train Steps/Sec: 0.86 +[2025-04-21 10:05:15] (step=0044525) Train Loss: 6.3275, Train Steps/Sec: 0.86 +[2025-04-21 10:05:44] (step=0044550) Train Loss: 6.3680, Train Steps/Sec: 0.86 +[2025-04-21 10:06:13] (step=0044575) Train Loss: 6.3433, Train Steps/Sec: 0.86 +[2025-04-21 10:06:42] (step=0044600) Train Loss: 6.3451, Train Steps/Sec: 0.86 +[2025-04-21 10:07:11] (step=0044625) Train Loss: 6.3268, Train Steps/Sec: 0.86 +[2025-04-21 10:07:40] (step=0044650) Train Loss: 6.2971, Train Steps/Sec: 0.86 +[2025-04-21 10:08:09] (step=0044675) Train Loss: 6.3329, Train Steps/Sec: 0.86 +[2025-04-21 10:08:38] (step=0044700) Train Loss: 6.3139, Train Steps/Sec: 0.86 +[2025-04-21 10:09:15] (step=0044725) Train Loss: 6.3074, Train Steps/Sec: 0.67 +[2025-04-21 10:09:52] (step=0044750) Train Loss: 6.3248, Train Steps/Sec: 0.68 +[2025-04-21 10:10:21] (step=0044775) Train Loss: 6.3198, Train Steps/Sec: 0.86 +[2025-04-21 10:10:50] (step=0044800) Train Loss: 6.3100, Train Steps/Sec: 0.86 +[2025-04-21 10:11:19] (step=0044825) Train Loss: 6.3170, Train Steps/Sec: 0.86 +[2025-04-21 10:11:48] (step=0044850) Train Loss: 6.3096, Train Steps/Sec: 0.86 +[2025-04-21 10:12:17] (step=0044875) Train Loss: 6.2746, Train Steps/Sec: 0.86 +[2025-04-21 10:12:46] (step=0044900) Train Loss: 6.3082, Train Steps/Sec: 0.86 +[2025-04-21 10:13:15] (step=0044925) Train Loss: 6.3104, Train Steps/Sec: 0.87 +[2025-04-21 10:13:53] (step=0044950) Train Loss: 6.3234, Train Steps/Sec: 0.66 +[2025-04-21 10:14:22] (step=0044975) Train Loss: 6.3291, Train Steps/Sec: 0.86 +[2025-04-21 10:14:51] (step=0045000) Train Loss: 6.3519, Train Steps/Sec: 0.86 +[2025-04-21 10:15:20] (step=0045025) Train Loss: 6.3733, Train Steps/Sec: 0.86 +[2025-04-21 10:15:49] (step=0045050) Train Loss: 6.2938, Train Steps/Sec: 0.86 +[2025-04-21 10:16:18] (step=0045075) Train Loss: 6.3451, Train Steps/Sec: 0.86 +[2025-04-21 10:16:47] (step=0045100) Train Loss: 6.3121, Train Steps/Sec: 0.86 +[2025-04-21 10:17:16] (step=0045125) Train Loss: 6.3278, Train Steps/Sec: 0.86 +[2025-04-21 10:17:45] (step=0045150) Train Loss: 6.3351, Train Steps/Sec: 0.86 +[2025-04-21 10:18:14] (step=0045175) Train Loss: 6.3167, Train Steps/Sec: 0.86 +[2025-04-21 10:18:44] (step=0045200) Train Loss: 6.3075, Train Steps/Sec: 0.85 +[2025-04-21 10:19:13] (step=0045225) Train Loss: 6.3395, Train Steps/Sec: 0.87 +[2025-04-21 10:19:42] (step=0045250) Train Loss: 6.3293, Train Steps/Sec: 0.86 +[2025-04-21 10:20:11] (step=0045275) Train Loss: 6.2912, Train Steps/Sec: 0.86 +[2025-04-21 10:20:40] (step=0045300) Train Loss: 6.3216, Train Steps/Sec: 0.86 +[2025-04-21 10:21:09] (step=0045325) Train Loss: 6.3026, Train Steps/Sec: 0.86 +[2025-04-21 10:21:38] (step=0045350) Train Loss: 6.3473, Train Steps/Sec: 0.86 +[2025-04-21 10:22:07] (step=0045375) Train Loss: 6.3236, Train Steps/Sec: 0.86 +[2025-04-21 10:22:36] (step=0045400) Train Loss: 6.2711, Train Steps/Sec: 0.86 +[2025-04-21 10:23:05] (step=0045425) Train Loss: 6.3082, Train Steps/Sec: 0.87 +[2025-04-21 10:23:34] (step=0045450) Train Loss: 6.3221, Train Steps/Sec: 0.86 +[2025-04-21 10:24:03] (step=0045475) Train Loss: 6.3410, Train Steps/Sec: 0.86 +[2025-04-21 10:24:32] (step=0045500) Train Loss: 6.3400, Train Steps/Sec: 0.86 +[2025-04-21 10:25:01] (step=0045525) Train Loss: 6.3109, Train Steps/Sec: 0.86 +[2025-04-21 10:25:30] (step=0045550) Train Loss: 6.3051, Train Steps/Sec: 0.86 +[2025-04-21 10:25:59] (step=0045575) Train Loss: 6.3305, Train Steps/Sec: 0.86 +[2025-04-21 10:26:29] (step=0045600) Train Loss: 6.3241, Train Steps/Sec: 0.85 +[2025-04-21 10:26:57] (step=0045625) Train Loss: 6.3009, Train Steps/Sec: 0.87 +[2025-04-21 10:27:26] (step=0045650) Train Loss: 6.2843, Train Steps/Sec: 0.86 +[2025-04-21 10:27:56] (step=0045675) Train Loss: 6.3439, Train Steps/Sec: 0.86 +[2025-04-21 10:28:25] (step=0045700) Train Loss: 6.3526, Train Steps/Sec: 0.86 +[2025-04-21 10:28:53] (step=0045725) Train Loss: 6.3496, Train Steps/Sec: 0.87 +[2025-04-21 10:29:22] (step=0045750) Train Loss: 6.3461, Train Steps/Sec: 0.87 +[2025-04-21 10:29:51] (step=0045775) Train Loss: 6.3333, Train Steps/Sec: 0.86 +[2025-04-21 10:30:20] (step=0045800) Train Loss: 6.3397, Train Steps/Sec: 0.86 +[2025-04-21 10:30:49] (step=0045825) Train Loss: 6.3179, Train Steps/Sec: 0.86 +[2025-04-21 10:31:19] (step=0045850) Train Loss: 6.3172, Train Steps/Sec: 0.86 +[2025-04-21 10:31:48] (step=0045875) Train Loss: 6.3247, Train Steps/Sec: 0.86 +[2025-04-21 10:32:17] (step=0045900) Train Loss: 6.2785, Train Steps/Sec: 0.86 +[2025-04-21 10:32:46] (step=0045925) Train Loss: 6.2978, Train Steps/Sec: 0.86 +[2025-04-21 10:33:15] (step=0045950) Train Loss: 6.3534, Train Steps/Sec: 0.86 +[2025-04-21 10:33:44] (step=0045975) Train Loss: 6.2923, Train Steps/Sec: 0.86 +[2025-04-21 10:34:13] (step=0046000) Train Loss: 6.3102, Train Steps/Sec: 0.87 +[2025-04-21 10:34:13] vision_config is None. initializing the InstructBlipVisionConfig with default values. +[2025-04-21 10:34:13] qformer_config is None. Initializing the InstructBlipQFormerConfig with default values. +[2025-04-21 10:34:13] text_config is None. Initializing the text config with default values (`OPTConfig`). +[2025-04-21 10:39:09] Finish Eval in 46000 steps... +[2025-04-21 10:39:29] Saved checkpoint to checkpoint/EmptyFix_InstructBlip_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_objects/002-GPT-XL/checkpoints/0046000.pt +[2025-04-21 10:39:32] Removed old checkpoint: checkpoint/EmptyFix_InstructBlip_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_objects/002-GPT-XL/checkpoints/0042000.pt +[2025-04-21 10:40:01] (step=0046025) Train Loss: 6.3476, Train Steps/Sec: 0.07 +[2025-04-21 10:40:30] (step=0046050) Train Loss: 6.3122, Train Steps/Sec: 0.86 +[2025-04-21 10:40:59] (step=0046075) Train Loss: 6.2914, Train Steps/Sec: 0.86 +[2025-04-21 10:41:28] (step=0046100) Train Loss: 6.3655, Train Steps/Sec: 0.86 +[2025-04-21 10:41:57] (step=0046125) Train Loss: 6.3304, Train Steps/Sec: 0.86 +[2025-04-21 10:42:26] (step=0046150) Train Loss: 6.2863, Train Steps/Sec: 0.86 +[2025-04-21 10:42:55] (step=0046175) Train Loss: 6.3163, Train Steps/Sec: 0.86 +[2025-04-21 10:43:24] (step=0046200) Train Loss: 6.3544, Train Steps/Sec: 0.86 +[2025-04-21 10:43:53] (step=0046225) Train Loss: 6.2471, Train Steps/Sec: 0.86 +[2025-04-21 10:44:22] (step=0046250) Train Loss: 6.2934, Train Steps/Sec: 0.86 +[2025-04-21 10:44:51] (step=0046275) Train Loss: 6.3249, Train Steps/Sec: 0.85 +[2025-04-21 10:45:20] (step=0046300) Train Loss: 6.3292, Train Steps/Sec: 0.86 +[2025-04-21 10:45:49] (step=0046325) Train Loss: 6.2799, Train Steps/Sec: 0.86 +[2025-04-21 10:46:18] (step=0046350) Train Loss: 6.3178, Train Steps/Sec: 0.86 +[2025-04-21 10:46:47] (step=0046375) Train Loss: 6.3761, Train Steps/Sec: 0.86 +[2025-04-21 10:47:17] (step=0046400) Train Loss: 6.3497, Train Steps/Sec: 0.86 +[2025-04-21 10:47:46] (step=0046425) Train Loss: 6.2901, Train Steps/Sec: 0.86 +[2025-04-21 10:48:15] (step=0046450) Train Loss: 6.3485, Train Steps/Sec: 0.86 +[2025-04-21 10:48:44] (step=0046475) Train Loss: 6.3303, Train Steps/Sec: 0.85 +[2025-04-21 10:49:13] (step=0046500) Train Loss: 6.3570, Train Steps/Sec: 0.85 +[2025-04-21 10:49:42] (step=0046525) Train Loss: 6.3524, Train Steps/Sec: 0.86 +[2025-04-21 10:50:11] (step=0046550) Train Loss: 6.2666, Train Steps/Sec: 0.85 +[2025-04-21 10:50:41] (step=0046575) Train Loss: 6.3613, Train Steps/Sec: 0.86 +[2025-04-21 10:51:10] (step=0046600) Train Loss: 6.3481, Train Steps/Sec: 0.86 +[2025-04-21 10:51:39] (step=0046625) Train Loss: 6.3352, Train Steps/Sec: 0.86 +[2025-04-21 10:52:08] (step=0046650) Train Loss: 6.3644, Train Steps/Sec: 0.86 +[2025-04-21 10:52:37] (step=0046675) Train Loss: 6.3267, Train Steps/Sec: 0.86 +[2025-04-21 10:53:06] (step=0046700) Train Loss: 6.3071, Train Steps/Sec: 0.85 +[2025-04-21 10:53:35] (step=0046725) Train Loss: 6.2829, Train Steps/Sec: 0.87 +[2025-04-21 10:54:04] (step=0046750) Train Loss: 6.3315, Train Steps/Sec: 0.86 +[2025-04-21 10:54:33] (step=0046775) Train Loss: 6.2801, Train Steps/Sec: 0.87 +[2025-04-21 10:55:02] (step=0046800) Train Loss: 6.3091, Train Steps/Sec: 0.86 +[2025-04-21 10:55:31] (step=0046825) Train Loss: 6.3194, Train Steps/Sec: 0.86 +[2025-04-21 10:56:00] (step=0046850) Train Loss: 6.3146, Train Steps/Sec: 0.87 +[2025-04-21 10:56:29] (step=0046875) Train Loss: 6.3319, Train Steps/Sec: 0.86 +[2025-04-21 10:56:58] (step=0046900) Train Loss: 6.3189, Train Steps/Sec: 0.86 +[2025-04-21 10:57:27] (step=0046925) Train Loss: 6.2935, Train Steps/Sec: 0.87 +[2025-04-21 10:57:56] (step=0046950) Train Loss: 6.2771, Train Steps/Sec: 0.86 +[2025-04-21 10:58:25] (step=0046975) Train Loss: 6.3162, Train Steps/Sec: 0.86 +[2025-04-21 10:58:54] (step=0047000) Train Loss: 6.3255, Train Steps/Sec: 0.85 +[2025-04-21 10:59:30] (step=0047025) Train Loss: 6.3668, Train Steps/Sec: 0.69 +[2025-04-21 10:59:59] (step=0047050) Train Loss: 6.3226, Train Steps/Sec: 0.86 +[2025-04-21 11:00:28] (step=0047075) Train Loss: 6.3328, Train Steps/Sec: 0.86 +[2025-04-21 11:00:57] (step=0047100) Train Loss: 6.2894, Train Steps/Sec: 0.86 +[2025-04-21 11:01:26] (step=0047125) Train Loss: 6.3496, Train Steps/Sec: 0.86 +[2025-04-21 11:01:55] (step=0047150) Train Loss: 6.3189, Train Steps/Sec: 0.87 +[2025-04-21 11:02:24] (step=0047175) Train Loss: 6.3534, Train Steps/Sec: 0.86 +[2025-04-21 11:02:53] (step=0047200) Train Loss: 6.3049, Train Steps/Sec: 0.86 +[2025-04-21 11:03:22] (step=0047225) Train Loss: 6.3044, Train Steps/Sec: 0.86 +[2025-04-21 11:03:51] (step=0047250) Train Loss: 6.3306, Train Steps/Sec: 0.86 +[2025-04-21 11:04:20] (step=0047275) Train Loss: 6.3129, Train Steps/Sec: 0.87 +[2025-04-21 11:04:49] (step=0047300) Train Loss: 6.3335, Train Steps/Sec: 0.86 +[2025-04-21 11:05:18] (step=0047325) Train Loss: 6.3338, Train Steps/Sec: 0.85 +[2025-04-21 11:05:48] (step=0047350) Train Loss: 6.2869, Train Steps/Sec: 0.85 +[2025-04-21 11:06:17] (step=0047375) Train Loss: 6.3344, Train Steps/Sec: 0.86 +[2025-04-21 11:06:46] (step=0047400) Train Loss: 6.3273, Train Steps/Sec: 0.86 +[2025-04-21 11:07:15] (step=0047425) Train Loss: 6.3607, Train Steps/Sec: 0.86 +[2025-04-21 11:07:44] (step=0047450) Train Loss: 6.3263, Train Steps/Sec: 0.86 +[2025-04-21 11:08:13] (step=0047475) Train Loss: 6.3096, Train Steps/Sec: 0.86 +[2025-04-21 11:08:42] (step=0047500) Train Loss: 6.3534, Train Steps/Sec: 0.85 +[2025-04-21 11:09:11] (step=0047525) Train Loss: 6.3307, Train Steps/Sec: 0.86 +[2025-04-21 11:09:40] (step=0047550) Train Loss: 6.3380, Train Steps/Sec: 0.86 +[2025-04-21 11:10:09] (step=0047575) Train Loss: 6.3429, Train Steps/Sec: 0.87 +[2025-04-21 11:10:39] (step=0047600) Train Loss: 6.3511, Train Steps/Sec: 0.85 +[2025-04-21 11:11:08] (step=0047625) Train Loss: 6.3041, Train Steps/Sec: 0.87 +[2025-04-21 11:11:37] (step=0047650) Train Loss: 6.3403, Train Steps/Sec: 0.86 +[2025-04-21 11:12:06] (step=0047675) Train Loss: 6.3353, Train Steps/Sec: 0.86 +[2025-04-21 11:12:35] (step=0047700) Train Loss: 6.3233, Train Steps/Sec: 0.86 +[2025-04-21 11:13:11] (step=0047725) Train Loss: 6.3378, Train Steps/Sec: 0.69 +[2025-04-21 11:13:40] (step=0047750) Train Loss: 6.3107, Train Steps/Sec: 0.86 +[2025-04-21 11:14:09] (step=0047775) Train Loss: 6.2776, Train Steps/Sec: 0.87 +[2025-04-21 11:14:38] (step=0047800) Train Loss: 6.3173, Train Steps/Sec: 0.86 +[2025-04-21 11:15:07] (step=0047825) Train Loss: 6.3170, Train Steps/Sec: 0.86 +[2025-04-21 11:15:36] (step=0047850) Train Loss: 6.3146, Train Steps/Sec: 0.86 +[2025-04-21 11:16:05] (step=0047875) Train Loss: 6.3275, Train Steps/Sec: 0.86 +[2025-04-21 11:16:34] (step=0047900) Train Loss: 6.3333, Train Steps/Sec: 0.86 +[2025-04-21 11:17:03] (step=0047925) Train Loss: 6.3005, Train Steps/Sec: 0.86 +[2025-04-21 11:17:32] (step=0047950) Train Loss: 6.3272, Train Steps/Sec: 0.86 +[2025-04-21 11:18:01] (step=0047975) Train Loss: 6.3293, Train Steps/Sec: 0.86 +[2025-04-21 11:18:30] (step=0048000) Train Loss: 6.3100, Train Steps/Sec: 0.86 +[2025-04-21 11:18:30] vision_config is None. initializing the InstructBlipVisionConfig with default values. +[2025-04-21 11:18:30] qformer_config is None. Initializing the InstructBlipQFormerConfig with default values. +[2025-04-21 11:18:30] text_config is None. Initializing the text config with default values (`OPTConfig`). +[2025-04-21 11:23:30] Finish Eval in 48000 steps... +[2025-04-21 11:23:52] Saved checkpoint to checkpoint/EmptyFix_InstructBlip_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_objects/002-GPT-XL/checkpoints/0048000.pt +[2025-04-21 11:23:54] Removed old checkpoint: checkpoint/EmptyFix_InstructBlip_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_objects/002-GPT-XL/checkpoints/0044000.pt +[2025-04-21 11:24:23] (step=0048025) Train Loss: 6.3307, Train Steps/Sec: 0.07 +[2025-04-21 11:24:52] (step=0048050) Train Loss: 6.3693, Train Steps/Sec: 0.86 +[2025-04-21 11:25:21] (step=0048075) Train Loss: 6.3038, Train Steps/Sec: 0.86 +[2025-04-21 11:25:50] (step=0048100) Train Loss: 6.2995, Train Steps/Sec: 0.86 +[2025-04-21 11:26:19] (step=0048125) Train Loss: 6.3273, Train Steps/Sec: 0.86 +[2025-04-21 11:26:48] (step=0048150) Train Loss: 6.3498, Train Steps/Sec: 0.86 +[2025-04-21 11:27:17] (step=0048175) Train Loss: 6.3889, Train Steps/Sec: 0.86 +[2025-04-21 11:27:46] (step=0048200) Train Loss: 6.3187, Train Steps/Sec: 0.87 +[2025-04-21 11:28:21] (step=0048225) Train Loss: 6.3433, Train Steps/Sec: 0.71 +[2025-04-21 11:28:50] (step=0048250) Train Loss: 6.3242, Train Steps/Sec: 0.86 +[2025-04-21 11:29:19] (step=0048275) Train Loss: 6.3623, Train Steps/Sec: 0.86 +[2025-04-21 11:29:48] (step=0048300) Train Loss: 6.3247, Train Steps/Sec: 0.86 +[2025-04-21 11:30:18] (step=0048325) Train Loss: 6.2803, Train Steps/Sec: 0.86 +[2025-04-21 11:30:47] (step=0048350) Train Loss: 6.3139, Train Steps/Sec: 0.86 +[2025-04-21 11:31:16] (step=0048375) Train Loss: 6.3342, Train Steps/Sec: 0.86 +[2025-04-21 11:31:46] (step=0048400) Train Loss: 6.3422, Train Steps/Sec: 0.84 +[2025-04-21 11:32:14] (step=0048425) Train Loss: 6.3109, Train Steps/Sec: 0.87 +[2025-04-21 11:32:43] (step=0048450) Train Loss: 6.3452, Train Steps/Sec: 0.86 +[2025-04-21 11:33:12] (step=0048475) Train Loss: 6.3323, Train Steps/Sec: 0.86 +[2025-04-21 11:33:42] (step=0048500) Train Loss: 6.2980, Train Steps/Sec: 0.86 +[2025-04-21 11:34:10] (step=0048525) Train Loss: 6.3517, Train Steps/Sec: 0.86 +[2025-04-21 11:34:39] (step=0048550) Train Loss: 6.3010, Train Steps/Sec: 0.87 +[2025-04-21 11:35:08] (step=0048575) Train Loss: 6.3295, Train Steps/Sec: 0.87 +[2025-04-21 11:35:37] (step=0048600) Train Loss: 6.2818, Train Steps/Sec: 0.86 +[2025-04-21 11:36:06] (step=0048625) Train Loss: 6.3201, Train Steps/Sec: 0.86 +[2025-04-21 11:36:35] (step=0048650) Train Loss: 6.3439, Train Steps/Sec: 0.86 +[2025-04-21 11:37:04] (step=0048675) Train Loss: 6.3155, Train Steps/Sec: 0.86 +[2025-04-21 11:37:33] (step=0048700) Train Loss: 6.3259, Train Steps/Sec: 0.85 +[2025-04-21 11:38:02] (step=0048725) Train Loss: 6.2939, Train Steps/Sec: 0.86 +[2025-04-21 11:38:31] (step=0048750) Train Loss: 6.3494, Train Steps/Sec: 0.86 +[2025-04-21 11:39:00] (step=0048775) Train Loss: 6.3000, Train Steps/Sec: 0.86 +[2025-04-21 11:39:29] (step=0048800) Train Loss: 6.3319, Train Steps/Sec: 0.86 +[2025-04-21 11:39:58] (step=0048825) Train Loss: 6.2680, Train Steps/Sec: 0.87 +[2025-04-21 11:40:27] (step=0048850) Train Loss: 6.3062, Train Steps/Sec: 0.87 +[2025-04-21 11:40:56] (step=0048875) Train Loss: 6.3486, Train Steps/Sec: 0.86 +[2025-04-21 11:41:25] (step=0048900) Train Loss: 6.3180, Train Steps/Sec: 0.86 +[2025-04-21 11:41:54] (step=0048925) Train Loss: 6.3265, Train Steps/Sec: 0.87 +[2025-04-21 11:42:23] (step=0048950) Train Loss: 6.3284, Train Steps/Sec: 0.86 +[2025-04-21 11:42:52] (step=0048975) Train Loss: 6.2738, Train Steps/Sec: 0.86 +[2025-04-21 11:43:21] (step=0049000) Train Loss: 6.3306, Train Steps/Sec: 0.86 +[2025-04-21 11:43:50] (step=0049025) Train Loss: 6.3226, Train Steps/Sec: 0.86 +[2025-04-21 11:44:19] (step=0049050) Train Loss: 6.3647, Train Steps/Sec: 0.86 +[2025-04-21 11:44:48] (step=0049075) Train Loss: 6.3065, Train Steps/Sec: 0.87 +[2025-04-21 11:45:17] (step=0049100) Train Loss: 6.3465, Train Steps/Sec: 0.86 +[2025-04-21 11:45:46] (step=0049125) Train Loss: 6.3090, Train Steps/Sec: 0.86 +[2025-04-21 11:46:15] (step=0049150) Train Loss: 6.3530, Train Steps/Sec: 0.86 +[2025-04-21 11:46:44] (step=0049175) Train Loss: 6.3465, Train Steps/Sec: 0.86 +[2025-04-21 11:47:13] (step=0049200) Train Loss: 6.3266, Train Steps/Sec: 0.86 +[2025-04-21 11:47:43] (step=0049225) Train Loss: 6.3596, Train Steps/Sec: 0.86 +[2025-04-21 11:48:11] (step=0049250) Train Loss: 6.3600, Train Steps/Sec: 0.87 +[2025-04-21 11:48:40] (step=0049275) Train Loss: 6.3299, Train Steps/Sec: 0.86 +[2025-04-21 11:49:16] (step=0049300) Train Loss: 6.3447, Train Steps/Sec: 0.71 +[2025-04-21 11:49:45] (step=0049325) Train Loss: 6.3304, Train Steps/Sec: 0.86 +[2025-04-21 11:50:14] (step=0049350) Train Loss: 6.3083, Train Steps/Sec: 0.86 +[2025-04-21 11:50:43] (step=0049375) Train Loss: 6.3340, Train Steps/Sec: 0.86 +[2025-04-21 11:51:12] (step=0049400) Train Loss: 6.3347, Train Steps/Sec: 0.86 +[2025-04-21 11:51:41] (step=0049425) Train Loss: 6.2996, Train Steps/Sec: 0.86 +[2025-04-21 11:52:10] (step=0049450) Train Loss: 6.2878, Train Steps/Sec: 0.87 +[2025-04-21 11:52:39] (step=0049475) Train Loss: 6.3077, Train Steps/Sec: 0.86 +[2025-04-21 11:53:08] (step=0049500) Train Loss: 6.3274, Train Steps/Sec: 0.86 +[2025-04-21 11:53:37] (step=0049525) Train Loss: 6.3279, Train Steps/Sec: 0.86 +[2025-04-21 11:54:06] (step=0049550) Train Loss: 6.3245, Train Steps/Sec: 0.86 +[2025-04-21 11:54:35] (step=0049575) Train Loss: 6.3192, Train Steps/Sec: 0.86 +[2025-04-21 11:55:04] (step=0049600) Train Loss: 6.3014, Train Steps/Sec: 0.86 +[2025-04-21 11:55:33] (step=0049625) Train Loss: 6.2970, Train Steps/Sec: 0.86 +[2025-04-21 11:56:02] (step=0049650) Train Loss: 6.3456, Train Steps/Sec: 0.86 +[2025-04-21 11:56:31] (step=0049675) Train Loss: 6.2865, Train Steps/Sec: 0.86 +[2025-04-21 11:57:00] (step=0049700) Train Loss: 6.3068, Train Steps/Sec: 0.85 +[2025-04-21 11:57:29] (step=0049725) Train Loss: 6.3723, Train Steps/Sec: 0.86 +[2025-04-21 11:57:59] (step=0049750) Train Loss: 6.3741, Train Steps/Sec: 0.86 +[2025-04-21 11:58:28] (step=0049775) Train Loss: 6.2909, Train Steps/Sec: 0.86 +[2025-04-21 11:58:57] (step=0049800) Train Loss: 6.3112, Train Steps/Sec: 0.85 +[2025-04-21 11:59:26] (step=0049825) Train Loss: 6.3344, Train Steps/Sec: 0.86 +[2025-04-21 12:00:02] (step=0049850) Train Loss: 6.3260, Train Steps/Sec: 0.70 +[2025-04-21 12:00:36] (step=0049875) Train Loss: 6.3525, Train Steps/Sec: 0.72 +[2025-04-21 12:01:05] (step=0049900) Train Loss: 6.3135, Train Steps/Sec: 0.86 +[2025-04-21 12:01:35] (step=0049925) Train Loss: 6.3306, Train Steps/Sec: 0.86 +[2025-04-21 12:02:04] (step=0049950) Train Loss: 6.3246, Train Steps/Sec: 0.86 +[2025-04-21 12:02:33] (step=0049975) Train Loss: 6.3607, Train Steps/Sec: 0.86 +[2025-04-21 12:03:02] (step=0050000) Train Loss: 6.3621, Train Steps/Sec: 0.86 +[2025-04-21 12:03:02] vision_config is None. initializing the InstructBlipVisionConfig with default values. +[2025-04-21 12:03:02] qformer_config is None. Initializing the InstructBlipQFormerConfig with default values. +[2025-04-21 12:03:02] text_config is None. Initializing the text config with default values (`OPTConfig`). +[2025-04-21 12:08:01] Finish Eval in 50000 steps... +[2025-04-21 12:08:23] Saved checkpoint to checkpoint/EmptyFix_InstructBlip_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_objects/002-GPT-XL/checkpoints/0050000.pt +[2025-04-21 12:08:25] Removed old checkpoint: checkpoint/EmptyFix_InstructBlip_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_objects/002-GPT-XL/checkpoints/0046000.pt +[2025-04-21 12:08:54] (step=0050025) Train Loss: 6.3110, Train Steps/Sec: 0.07 +[2025-04-21 12:09:23] (step=0050050) Train Loss: 6.3067, Train Steps/Sec: 0.87 +[2025-04-21 12:09:52] (step=0050075) Train Loss: 6.2759, Train Steps/Sec: 0.86 +[2025-04-21 12:10:28] (step=0050100) Train Loss: 6.3168, Train Steps/Sec: 0.69 +[2025-04-21 12:10:57] (step=0050125) Train Loss: 6.3686, Train Steps/Sec: 0.86 +[2025-04-21 12:11:25] (step=0050150) Train Loss: 6.3304, Train Steps/Sec: 0.87 +[2025-04-21 12:11:54] (step=0050175) Train Loss: 6.3052, Train Steps/Sec: 0.86 +[2025-04-21 12:12:24] (step=0050200) Train Loss: 6.3333, Train Steps/Sec: 0.86 +[2025-04-21 12:12:53] (step=0050225) Train Loss: 6.3543, Train Steps/Sec: 0.86 +[2025-04-21 12:13:22] (step=0050250) Train Loss: 6.3323, Train Steps/Sec: 0.85 +[2025-04-21 12:13:51] (step=0050275) Train Loss: 6.3409, Train Steps/Sec: 0.86 +[2025-04-21 12:14:20] (step=0050300) Train Loss: 6.3490, Train Steps/Sec: 0.86 +[2025-04-21 12:14:49] (step=0050325) Train Loss: 6.3003, Train Steps/Sec: 0.86 +[2025-04-21 12:15:18] (step=0050350) Train Loss: 6.3305, Train Steps/Sec: 0.87 +[2025-04-21 12:15:47] (step=0050375) Train Loss: 6.3412, Train Steps/Sec: 0.86 +[2025-04-21 12:16:17] (step=0050400) Train Loss: 6.3583, Train Steps/Sec: 0.86 +[2025-04-21 12:16:45] (step=0050425) Train Loss: 6.2790, Train Steps/Sec: 0.87 +[2025-04-21 12:17:14] (step=0050450) Train Loss: 6.2790, Train Steps/Sec: 0.86 +[2025-04-21 12:17:43] (step=0050475) Train Loss: 6.3491, Train Steps/Sec: 0.87 +[2025-04-21 12:18:12] (step=0050500) Train Loss: 6.3907, Train Steps/Sec: 0.86 +[2025-04-21 12:18:41] (step=0050525) Train Loss: 6.2939, Train Steps/Sec: 0.86 +[2025-04-21 12:19:10] (step=0050550) Train Loss: 6.3549, Train Steps/Sec: 0.86 +[2025-04-21 12:19:39] (step=0050575) Train Loss: 6.2975, Train Steps/Sec: 0.86 +[2025-04-21 12:20:09] (step=0050600) Train Loss: 6.3272, Train Steps/Sec: 0.85 +[2025-04-21 12:20:37] (step=0050625) Train Loss: 6.3089, Train Steps/Sec: 0.87 +[2025-04-21 12:21:07] (step=0050650) Train Loss: 6.3296, Train Steps/Sec: 0.86 +[2025-04-21 12:21:35] (step=0050675) Train Loss: 6.3030, Train Steps/Sec: 0.87 +[2025-04-21 12:22:05] (step=0050700) Train Loss: 6.3354, Train Steps/Sec: 0.86 +[2025-04-21 12:22:34] (step=0050725) Train Loss: 6.3165, Train Steps/Sec: 0.86 +[2025-04-21 12:23:02] (step=0050750) Train Loss: 6.2831, Train Steps/Sec: 0.87 +[2025-04-21 12:23:31] (step=0050775) Train Loss: 6.3335, Train Steps/Sec: 0.87 +[2025-04-21 12:24:00] (step=0050800) Train Loss: 6.3676, Train Steps/Sec: 0.86 +[2025-04-21 12:24:29] (step=0050825) Train Loss: 6.3512, Train Steps/Sec: 0.86 +[2025-04-21 12:24:58] (step=0050850) Train Loss: 6.3359, Train Steps/Sec: 0.86 +[2025-04-21 12:25:27] (step=0050875) Train Loss: 6.3557, Train Steps/Sec: 0.86 +[2025-04-21 12:25:57] (step=0050900) Train Loss: 6.3436, Train Steps/Sec: 0.86 +[2025-04-21 12:26:26] (step=0050925) Train Loss: 6.2907, Train Steps/Sec: 0.86 +[2025-04-21 12:26:55] (step=0050950) Train Loss: 6.3172, Train Steps/Sec: 0.86 +[2025-04-21 12:27:24] (step=0050975) Train Loss: 6.3054, Train Steps/Sec: 0.86 +[2025-04-21 12:27:53] (step=0051000) Train Loss: 6.3295, Train Steps/Sec: 0.85 +[2025-04-21 12:28:22] (step=0051025) Train Loss: 6.2941, Train Steps/Sec: 0.86 +[2025-04-21 12:28:51] (step=0051050) Train Loss: 6.2734, Train Steps/Sec: 0.86 +[2025-04-21 12:29:20] (step=0051075) Train Loss: 6.3378, Train Steps/Sec: 0.86 +[2025-04-21 12:29:49] (step=0051100) Train Loss: 6.3198, Train Steps/Sec: 0.86 +[2025-04-21 12:30:18] (step=0051125) Train Loss: 6.3433, Train Steps/Sec: 0.86 +[2025-04-21 12:30:47] (step=0051150) Train Loss: 6.2950, Train Steps/Sec: 0.87 +[2025-04-21 12:31:16] (step=0051175) Train Loss: 6.3624, Train Steps/Sec: 0.86 +[2025-04-21 12:31:45] (step=0051200) Train Loss: 6.3254, Train Steps/Sec: 0.86 +[2025-04-21 12:32:14] (step=0051225) Train Loss: 6.2876, Train Steps/Sec: 0.86 +[2025-04-21 12:32:43] (step=0051250) Train Loss: 6.3454, Train Steps/Sec: 0.86 +[2025-04-21 12:33:12] (step=0051275) Train Loss: 6.3194, Train Steps/Sec: 0.86 +[2025-04-21 12:33:42] (step=0051300) Train Loss: 6.3387, Train Steps/Sec: 0.85 +[2025-04-21 12:34:11] (step=0051325) Train Loss: 6.3111, Train Steps/Sec: 0.86 +[2025-04-21 12:34:40] (step=0051350) Train Loss: 6.3351, Train Steps/Sec: 0.86 +[2025-04-21 12:35:09] (step=0051375) Train Loss: 6.3350, Train Steps/Sec: 0.86 +[2025-04-21 12:35:38] (step=0051400) Train Loss: 6.3204, Train Steps/Sec: 0.86 +[2025-04-21 12:36:06] (step=0051425) Train Loss: 6.3268, Train Steps/Sec: 0.87 +[2025-04-21 12:36:35] (step=0051450) Train Loss: 6.3045, Train Steps/Sec: 0.86 +[2025-04-21 12:37:04] (step=0051475) Train Loss: 6.2805, Train Steps/Sec: 0.87 +[2025-04-21 12:37:33] (step=0051500) Train Loss: 6.3105, Train Steps/Sec: 0.86 +[2025-04-21 12:38:02] (step=0051525) Train Loss: 6.2738, Train Steps/Sec: 0.86 +[2025-04-21 12:38:32] (step=0051550) Train Loss: 6.3812, Train Steps/Sec: 0.85 +[2025-04-21 12:39:01] (step=0051575) Train Loss: 6.2925, Train Steps/Sec: 0.86 +[2025-04-21 12:39:30] (step=0051600) Train Loss: 6.3124, Train Steps/Sec: 0.87 +[2025-04-21 12:39:59] (step=0051625) Train Loss: 6.3267, Train Steps/Sec: 0.86 +[2025-04-21 12:40:28] (step=0051650) Train Loss: 6.2963, Train Steps/Sec: 0.86 +[2025-04-21 12:40:57] (step=0051675) Train Loss: 6.3355, Train Steps/Sec: 0.86 +[2025-04-21 12:41:26] (step=0051700) Train Loss: 6.3396, Train Steps/Sec: 0.85 +[2025-04-21 12:41:55] (step=0051725) Train Loss: 6.3061, Train Steps/Sec: 0.86 +[2025-04-21 12:42:24] (step=0051750) Train Loss: 6.3164, Train Steps/Sec: 0.86 +[2025-04-21 12:42:53] (step=0051775) Train Loss: 6.2977, Train Steps/Sec: 0.86 +[2025-04-21 12:43:22] (step=0051800) Train Loss: 6.3072, Train Steps/Sec: 0.86 +[2025-04-21 12:43:51] (step=0051825) Train Loss: 6.3308, Train Steps/Sec: 0.86 +[2025-04-21 12:44:20] (step=0051850) Train Loss: 6.3273, Train Steps/Sec: 0.86 +[2025-04-21 12:44:49] (step=0051875) Train Loss: 6.3805, Train Steps/Sec: 0.86 +[2025-04-21 12:45:18] (step=0051900) Train Loss: 6.3349, Train Steps/Sec: 0.86 +[2025-04-21 12:45:47] (step=0051925) Train Loss: 6.2981, Train Steps/Sec: 0.86 +[2025-04-21 12:46:16] (step=0051950) Train Loss: 6.3306, Train Steps/Sec: 0.87 +[2025-04-21 12:46:45] (step=0051975) Train Loss: 6.2865, Train Steps/Sec: 0.86 +[2025-04-21 12:47:14] (step=0052000) Train Loss: 6.2633, Train Steps/Sec: 0.86 +[2025-04-21 12:47:14] vision_config is None. initializing the InstructBlipVisionConfig with default values. +[2025-04-21 12:47:14] qformer_config is None. Initializing the InstructBlipQFormerConfig with default values. +[2025-04-21 12:47:14] text_config is None. Initializing the text config with default values (`OPTConfig`). +[2025-04-21 12:52:16] Finish Eval in 52000 steps... +[2025-04-21 12:52:36] Saved checkpoint to checkpoint/EmptyFix_InstructBlip_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_objects/002-GPT-XL/checkpoints/0052000.pt +[2025-04-21 12:52:39] Removed old checkpoint: checkpoint/EmptyFix_InstructBlip_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_objects/002-GPT-XL/checkpoints/0048000.pt +[2025-04-21 12:53:08] (step=0052025) Train Loss: 6.3210, Train Steps/Sec: 0.07 +[2025-04-21 12:53:37] (step=0052050) Train Loss: 6.3018, Train Steps/Sec: 0.86 +[2025-04-21 12:54:06] (step=0052075) Train Loss: 6.2939, Train Steps/Sec: 0.86 +[2025-04-21 12:54:35] (step=0052100) Train Loss: 6.3251, Train Steps/Sec: 0.84 +[2025-04-21 12:55:04] (step=0052125) Train Loss: 6.2857, Train Steps/Sec: 0.86 +[2025-04-21 12:55:33] (step=0052150) Train Loss: 6.2933, Train Steps/Sec: 0.86 +[2025-04-21 12:56:02] (step=0052175) Train Loss: 6.3043, Train Steps/Sec: 0.87 +[2025-04-21 12:56:31] (step=0052200) Train Loss: 6.3133, Train Steps/Sec: 0.85 +[2025-04-21 12:57:00] (step=0052225) Train Loss: 6.3338, Train Steps/Sec: 0.87 +[2025-04-21 12:57:29] (step=0052250) Train Loss: 6.3298, Train Steps/Sec: 0.87 +[2025-04-21 12:57:58] (step=0052275) Train Loss: 6.2633, Train Steps/Sec: 0.86 +[2025-04-21 12:58:28] (step=0052300) Train Loss: 6.3057, Train Steps/Sec: 0.85 +[2025-04-21 12:58:57] (step=0052325) Train Loss: 6.3253, Train Steps/Sec: 0.86 +[2025-04-21 12:59:26] (step=0052350) Train Loss: 6.2797, Train Steps/Sec: 0.87 +[2025-04-21 12:59:55] (step=0052375) Train Loss: 6.2842, Train Steps/Sec: 0.86 +[2025-04-21 13:00:24] (step=0052400) Train Loss: 6.3232, Train Steps/Sec: 0.86 +[2025-04-21 13:00:53] (step=0052425) Train Loss: 6.3505, Train Steps/Sec: 0.86 +[2025-04-21 13:01:22] (step=0052450) Train Loss: 6.3254, Train Steps/Sec: 0.86 +[2025-04-21 13:01:51] (step=0052475) Train Loss: 6.3495, Train Steps/Sec: 0.86 +[2025-04-21 13:02:27] (step=0052500) Train Loss: 6.3111, Train Steps/Sec: 0.69 +[2025-04-21 13:02:56] (step=0052525) Train Loss: 6.2883, Train Steps/Sec: 0.86 +[2025-04-21 13:03:25] (step=0052550) Train Loss: 6.3268, Train Steps/Sec: 0.86 +[2025-04-21 13:03:54] (step=0052575) Train Loss: 6.2731, Train Steps/Sec: 0.86 +[2025-04-21 13:04:23] (step=0052600) Train Loss: 6.2945, Train Steps/Sec: 0.86 +[2025-04-21 13:04:52] (step=0052625) Train Loss: 6.2613, Train Steps/Sec: 0.86 +[2025-04-21 13:05:21] (step=0052650) Train Loss: 6.3179, Train Steps/Sec: 0.86 +[2025-04-21 13:05:50] (step=0052675) Train Loss: 6.3497, Train Steps/Sec: 0.86 +[2025-04-21 13:06:19] (step=0052700) Train Loss: 6.3000, Train Steps/Sec: 0.86 +[2025-04-21 13:06:48] (step=0052725) Train Loss: 6.2762, Train Steps/Sec: 0.86 +[2025-04-21 13:07:17] (step=0052750) Train Loss: 6.2859, Train Steps/Sec: 0.86 +[2025-04-21 13:07:46] (step=0052775) Train Loss: 6.3259, Train Steps/Sec: 0.87 +[2025-04-21 13:08:15] (step=0052800) Train Loss: 6.3093, Train Steps/Sec: 0.86 +[2025-04-21 13:08:44] (step=0052825) Train Loss: 6.2971, Train Steps/Sec: 0.87 +[2025-04-21 13:09:13] (step=0052850) Train Loss: 6.3200, Train Steps/Sec: 0.87 +[2025-04-21 13:09:42] (step=0052875) Train Loss: 6.2389, Train Steps/Sec: 0.86 +[2025-04-21 13:10:11] (step=0052900) Train Loss: 6.2755, Train Steps/Sec: 0.86 +[2025-04-21 13:10:40] (step=0052925) Train Loss: 6.2961, Train Steps/Sec: 0.86 +[2025-04-21 13:11:09] (step=0052950) Train Loss: 6.3259, Train Steps/Sec: 0.86 +[2025-04-21 13:11:38] (step=0052975) Train Loss: 6.3337, Train Steps/Sec: 0.86 +[2025-04-21 13:12:07] (step=0053000) Train Loss: 6.3088, Train Steps/Sec: 0.86 +[2025-04-21 13:12:36] (step=0053025) Train Loss: 6.2921, Train Steps/Sec: 0.86 +[2025-04-21 13:13:05] (step=0053050) Train Loss: 6.3145, Train Steps/Sec: 0.86 +[2025-04-21 13:13:34] (step=0053075) Train Loss: 6.2853, Train Steps/Sec: 0.86 +[2025-04-21 13:14:03] (step=0053100) Train Loss: 6.2808, Train Steps/Sec: 0.86 +[2025-04-21 13:14:32] (step=0053125) Train Loss: 6.3016, Train Steps/Sec: 0.86 +[2025-04-21 13:15:01] (step=0053150) Train Loss: 6.3075, Train Steps/Sec: 0.86 +[2025-04-21 13:15:30] (step=0053175) Train Loss: 6.3057, Train Steps/Sec: 0.86 +[2025-04-21 13:15:59] (step=0053200) Train Loss: 6.2811, Train Steps/Sec: 0.86 +[2025-04-21 13:16:28] (step=0053225) Train Loss: 6.3121, Train Steps/Sec: 0.86 +[2025-04-21 13:16:57] (step=0053250) Train Loss: 6.3311, Train Steps/Sec: 0.87 +[2025-04-21 13:17:33] (step=0053275) Train Loss: 6.3409, Train Steps/Sec: 0.70 +[2025-04-21 13:18:02] (step=0053300) Train Loss: 6.3421, Train Steps/Sec: 0.86 +[2025-04-21 13:18:31] (step=0053325) Train Loss: 6.3071, Train Steps/Sec: 0.87 +[2025-04-21 13:19:00] (step=0053350) Train Loss: 6.3249, Train Steps/Sec: 0.87 +[2025-04-21 13:19:29] (step=0053375) Train Loss: 6.2999, Train Steps/Sec: 0.87 +[2025-04-21 13:19:58] (step=0053400) Train Loss: 6.2867, Train Steps/Sec: 0.86 +[2025-04-21 13:20:27] (step=0053425) Train Loss: 6.2825, Train Steps/Sec: 0.86 +[2025-04-21 13:20:56] (step=0053450) Train Loss: 6.3099, Train Steps/Sec: 0.86 +[2025-04-21 13:21:25] (step=0053475) Train Loss: 6.3040, Train Steps/Sec: 0.86 +[2025-04-21 13:21:54] (step=0053500) Train Loss: 6.3303, Train Steps/Sec: 0.86 +[2025-04-21 13:22:23] (step=0053525) Train Loss: 6.3250, Train Steps/Sec: 0.86 +[2025-04-21 13:22:52] (step=0053550) Train Loss: 6.3291, Train Steps/Sec: 0.86 +[2025-04-21 13:23:21] (step=0053575) Train Loss: 6.3560, Train Steps/Sec: 0.86 +[2025-04-21 13:24:00] (step=0053600) Train Loss: 6.3092, Train Steps/Sec: 0.65 +[2025-04-21 13:24:29] (step=0053625) Train Loss: 6.3301, Train Steps/Sec: 0.86 +[2025-04-21 13:24:58] (step=0053650) Train Loss: 6.3483, Train Steps/Sec: 0.86 +[2025-04-21 13:25:27] (step=0053675) Train Loss: 6.2990, Train Steps/Sec: 0.86 +[2025-04-21 13:25:56] (step=0053700) Train Loss: 6.2975, Train Steps/Sec: 0.86 +[2025-04-21 13:26:25] (step=0053725) Train Loss: 6.3266, Train Steps/Sec: 0.87 +[2025-04-21 13:26:54] (step=0053750) Train Loss: 6.3287, Train Steps/Sec: 0.86 +[2025-04-21 13:27:23] (step=0053775) Train Loss: 6.3103, Train Steps/Sec: 0.86 +[2025-04-21 13:27:52] (step=0053800) Train Loss: 6.3171, Train Steps/Sec: 0.86 +[2025-04-21 13:28:21] (step=0053825) Train Loss: 6.3113, Train Steps/Sec: 0.86 +[2025-04-21 13:28:50] (step=0053850) Train Loss: 6.3044, Train Steps/Sec: 0.86 +[2025-04-21 13:29:19] (step=0053875) Train Loss: 6.3302, Train Steps/Sec: 0.87 +[2025-04-21 13:29:48] (step=0053900) Train Loss: 6.3015, Train Steps/Sec: 0.85 +[2025-04-21 13:30:17] (step=0053925) Train Loss: 6.3102, Train Steps/Sec: 0.87 +[2025-04-21 13:30:46] (step=0053950) Train Loss: 6.3114, Train Steps/Sec: 0.87 +[2025-04-21 13:31:15] (step=0053975) Train Loss: 6.3732, Train Steps/Sec: 0.86 +[2025-04-21 13:31:44] (step=0054000) Train Loss: 6.2860, Train Steps/Sec: 0.86 +[2025-04-21 13:31:44] vision_config is None. initializing the InstructBlipVisionConfig with default values. +[2025-04-21 13:31:44] qformer_config is None. Initializing the InstructBlipQFormerConfig with default values. +[2025-04-21 13:31:44] text_config is None. Initializing the text config with default values (`OPTConfig`). +[2025-04-21 13:36:44] Finish Eval in 54000 steps... +[2025-04-21 13:37:06] Saved checkpoint to checkpoint/EmptyFix_InstructBlip_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_objects/002-GPT-XL/checkpoints/0054000.pt +[2025-04-21 13:37:08] Removed old checkpoint: checkpoint/EmptyFix_InstructBlip_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_objects/002-GPT-XL/checkpoints/0050000.pt +[2025-04-21 13:37:37] (step=0054025) Train Loss: 6.3181, Train Steps/Sec: 0.07 +[2025-04-21 13:38:06] (step=0054050) Train Loss: 6.3544, Train Steps/Sec: 0.86 +[2025-04-21 13:38:35] (step=0054075) Train Loss: 6.3208, Train Steps/Sec: 0.86 +[2025-04-21 13:39:04] (step=0054100) Train Loss: 6.2614, Train Steps/Sec: 0.86 +[2025-04-21 13:39:33] (step=0054125) Train Loss: 6.3071, Train Steps/Sec: 0.86 +[2025-04-21 13:40:02] (step=0054150) Train Loss: 6.3313, Train Steps/Sec: 0.87 +[2025-04-21 13:40:31] (step=0054175) Train Loss: 6.2956, Train Steps/Sec: 0.87 +[2025-04-21 13:41:00] (step=0054200) Train Loss: 6.3168, Train Steps/Sec: 0.86 +[2025-04-21 13:41:29] (step=0054225) Train Loss: 6.3233, Train Steps/Sec: 0.86 +[2025-04-21 13:41:58] (step=0054250) Train Loss: 6.3155, Train Steps/Sec: 0.86 +[2025-04-21 13:42:27] (step=0054275) Train Loss: 6.3173, Train Steps/Sec: 0.86 +[2025-04-21 13:42:56] (step=0054300) Train Loss: 6.2947, Train Steps/Sec: 0.86 +[2025-04-21 13:43:25] (step=0054325) Train Loss: 6.3775, Train Steps/Sec: 0.87 +[2025-04-21 13:43:54] (step=0054350) Train Loss: 6.2870, Train Steps/Sec: 0.86 +[2025-04-21 13:44:29] (step=0054375) Train Loss: 6.3230, Train Steps/Sec: 0.71 +[2025-04-21 13:44:58] (step=0054400) Train Loss: 6.3316, Train Steps/Sec: 0.86 +[2025-04-21 13:45:27] (step=0054425) Train Loss: 6.2772, Train Steps/Sec: 0.87 +[2025-04-21 13:45:56] (step=0054450) Train Loss: 6.2887, Train Steps/Sec: 0.86 +[2025-04-21 13:46:25] (step=0054475) Train Loss: 6.3333, Train Steps/Sec: 0.86 +[2025-04-21 13:46:54] (step=0054500) Train Loss: 6.2980, Train Steps/Sec: 0.86 +[2025-04-21 13:47:23] (step=0054525) Train Loss: 6.2585, Train Steps/Sec: 0.86 +[2025-04-21 13:47:52] (step=0054550) Train Loss: 6.3052, Train Steps/Sec: 0.86 +[2025-04-21 13:48:21] (step=0054575) Train Loss: 6.3014, Train Steps/Sec: 0.86 +[2025-04-21 13:48:51] (step=0054600) Train Loss: 6.3585, Train Steps/Sec: 0.84 +[2025-04-21 13:49:20] (step=0054625) Train Loss: 6.3282, Train Steps/Sec: 0.87 +[2025-04-21 13:49:49] (step=0054650) Train Loss: 6.3160, Train Steps/Sec: 0.86 +[2025-04-21 13:50:18] (step=0054675) Train Loss: 6.3076, Train Steps/Sec: 0.86 +[2025-04-21 13:50:47] (step=0054700) Train Loss: 6.3545, Train Steps/Sec: 0.86 +[2025-04-21 13:51:16] (step=0054725) Train Loss: 6.2812, Train Steps/Sec: 0.87 +[2025-04-21 13:51:45] (step=0054750) Train Loss: 6.3502, Train Steps/Sec: 0.86 +[2025-04-21 13:52:14] (step=0054775) Train Loss: 6.3415, Train Steps/Sec: 0.87 +[2025-04-21 13:52:43] (step=0054800) Train Loss: 6.2951, Train Steps/Sec: 0.86 +[2025-04-21 13:53:12] (step=0054825) Train Loss: 6.3100, Train Steps/Sec: 0.86 +[2025-04-21 13:53:41] (step=0054850) Train Loss: 6.3337, Train Steps/Sec: 0.87 +[2025-04-21 13:54:10] (step=0054875) Train Loss: 6.3060, Train Steps/Sec: 0.86 +[2025-04-21 13:54:39] (step=0054900) Train Loss: 6.3239, Train Steps/Sec: 0.86 +[2025-04-21 13:55:08] (step=0054925) Train Loss: 6.2841, Train Steps/Sec: 0.86 +[2025-04-21 13:55:37] (step=0054950) Train Loss: 6.2974, Train Steps/Sec: 0.86 +[2025-04-21 13:56:13] (step=0054975) Train Loss: 6.3491, Train Steps/Sec: 0.70 +[2025-04-21 13:56:42] (step=0055000) Train Loss: 6.3432, Train Steps/Sec: 0.86 +[2025-04-21 13:57:16] (step=0055025) Train Loss: 6.3160, Train Steps/Sec: 0.72 +[2025-04-21 13:57:46] (step=0055050) Train Loss: 6.2929, Train Steps/Sec: 0.85 +[2025-04-21 13:58:14] (step=0055075) Train Loss: 6.3156, Train Steps/Sec: 0.87 +[2025-04-21 13:58:43] (step=0055100) Train Loss: 6.3081, Train Steps/Sec: 0.86 +[2025-04-21 13:59:13] (step=0055125) Train Loss: 6.3462, Train Steps/Sec: 0.86 +[2025-04-21 13:59:42] (step=0055150) Train Loss: 6.3735, Train Steps/Sec: 0.86 +[2025-04-21 14:00:11] (step=0055175) Train Loss: 6.2770, Train Steps/Sec: 0.86 +[2025-04-21 14:00:40] (step=0055200) Train Loss: 6.3434, Train Steps/Sec: 0.86 +[2025-04-21 14:01:09] (step=0055225) Train Loss: 6.2983, Train Steps/Sec: 0.87 +[2025-04-21 14:01:38] (step=0055250) Train Loss: 6.3702, Train Steps/Sec: 0.86 +[2025-04-21 14:02:14] (step=0055275) Train Loss: 6.3186, Train Steps/Sec: 0.70 +[2025-04-21 14:02:43] (step=0055300) Train Loss: 6.2767, Train Steps/Sec: 0.86 +[2025-04-21 14:03:12] (step=0055325) Train Loss: 6.3232, Train Steps/Sec: 0.86 +[2025-04-21 14:03:41] (step=0055350) Train Loss: 6.3526, Train Steps/Sec: 0.86 +[2025-04-21 14:04:10] (step=0055375) Train Loss: 6.3322, Train Steps/Sec: 0.86 +[2025-04-21 14:04:39] (step=0055400) Train Loss: 6.3153, Train Steps/Sec: 0.86 +[2025-04-21 14:05:08] (step=0055425) Train Loss: 6.2843, Train Steps/Sec: 0.86 +[2025-04-21 14:05:37] (step=0055450) Train Loss: 6.3261, Train Steps/Sec: 0.86 +[2025-04-21 14:06:06] (step=0055475) Train Loss: 6.3460, Train Steps/Sec: 0.86 +[2025-04-21 14:06:35] (step=0055500) Train Loss: 6.3093, Train Steps/Sec: 0.86 +[2025-04-21 14:07:04] (step=0055525) Train Loss: 6.3176, Train Steps/Sec: 0.86 +[2025-04-21 14:07:33] (step=0055550) Train Loss: 6.2970, Train Steps/Sec: 0.86 +[2025-04-21 14:08:02] (step=0055575) Train Loss: 6.3079, Train Steps/Sec: 0.87 +[2025-04-21 14:08:31] (step=0055600) Train Loss: 6.3381, Train Steps/Sec: 0.86 +[2025-04-21 14:09:00] (step=0055625) Train Loss: 6.3387, Train Steps/Sec: 0.87 +[2025-04-21 14:09:29] (step=0055650) Train Loss: 6.3162, Train Steps/Sec: 0.86 +[2025-04-21 14:09:58] (step=0055675) Train Loss: 6.3058, Train Steps/Sec: 0.86 +[2025-04-21 14:10:27] (step=0055700) Train Loss: 6.3010, Train Steps/Sec: 0.86 +[2025-04-21 14:10:56] (step=0055725) Train Loss: 6.3054, Train Steps/Sec: 0.87 +[2025-04-21 14:11:25] (step=0055750) Train Loss: 6.2818, Train Steps/Sec: 0.87 +[2025-04-21 14:11:54] (step=0055775) Train Loss: 6.3272, Train Steps/Sec: 0.86 +[2025-04-21 14:12:23] (step=0055800) Train Loss: 6.3193, Train Steps/Sec: 0.86 +[2025-04-21 14:12:52] (step=0055825) Train Loss: 6.2963, Train Steps/Sec: 0.86 +[2025-04-21 14:13:21] (step=0055850) Train Loss: 6.3086, Train Steps/Sec: 0.86 +[2025-04-21 14:13:50] (step=0055875) Train Loss: 6.2784, Train Steps/Sec: 0.86 +[2025-04-21 14:14:19] (step=0055900) Train Loss: 6.3207, Train Steps/Sec: 0.86 +[2025-04-21 14:14:48] (step=0055925) Train Loss: 6.3656, Train Steps/Sec: 0.86 +[2025-04-21 14:15:17] (step=0055950) Train Loss: 6.2659, Train Steps/Sec: 0.86 +[2025-04-21 14:15:46] (step=0055975) Train Loss: 6.3355, Train Steps/Sec: 0.86 +[2025-04-21 14:16:15] (step=0056000) Train Loss: 6.3826, Train Steps/Sec: 0.85 +[2025-04-21 14:16:15] vision_config is None. initializing the InstructBlipVisionConfig with default values. +[2025-04-21 14:16:15] qformer_config is None. Initializing the InstructBlipQFormerConfig with default values. +[2025-04-21 14:16:15] text_config is None. Initializing the text config with default values (`OPTConfig`). +[2025-04-21 14:21:14] Finish Eval in 56000 steps... +[2025-04-21 14:21:36] Saved checkpoint to checkpoint/EmptyFix_InstructBlip_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_objects/002-GPT-XL/checkpoints/0056000.pt +[2025-04-21 14:21:38] Removed old checkpoint: checkpoint/EmptyFix_InstructBlip_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_objects/002-GPT-XL/checkpoints/0052000.pt +[2025-04-21 14:22:07] (step=0056025) Train Loss: 6.3026, Train Steps/Sec: 0.07 +[2025-04-21 14:22:36] (step=0056050) Train Loss: 6.3045, Train Steps/Sec: 0.87 +[2025-04-21 14:23:05] (step=0056075) Train Loss: 6.3541, Train Steps/Sec: 0.87 +[2025-04-21 14:23:34] (step=0056100) Train Loss: 6.3105, Train Steps/Sec: 0.86 +[2025-04-21 14:24:03] (step=0056125) Train Loss: 6.2820, Train Steps/Sec: 0.86 +[2025-04-21 14:24:32] (step=0056150) Train Loss: 6.2695, Train Steps/Sec: 0.87 +[2025-04-21 14:25:00] (step=0056175) Train Loss: 6.2865, Train Steps/Sec: 0.87 +[2025-04-21 14:25:29] (step=0056200) Train Loss: 6.3090, Train Steps/Sec: 0.86 +[2025-04-21 14:25:58] (step=0056225) Train Loss: 6.2766, Train Steps/Sec: 0.86 +[2025-04-21 14:26:27] (step=0056250) Train Loss: 6.3111, Train Steps/Sec: 0.87 +[2025-04-21 14:26:56] (step=0056275) Train Loss: 6.3247, Train Steps/Sec: 0.86 +[2025-04-21 14:27:25] (step=0056300) Train Loss: 6.2936, Train Steps/Sec: 0.86 +[2025-04-21 14:27:54] (step=0056325) Train Loss: 6.3156, Train Steps/Sec: 0.86 +[2025-04-21 14:28:23] (step=0056350) Train Loss: 6.3244, Train Steps/Sec: 0.87 +[2025-04-21 14:28:52] (step=0056375) Train Loss: 6.2930, Train Steps/Sec: 0.86 +[2025-04-21 14:29:21] (step=0056400) Train Loss: 6.3243, Train Steps/Sec: 0.86 +[2025-04-21 14:29:51] (step=0056425) Train Loss: 6.3066, Train Steps/Sec: 0.85 +[2025-04-21 14:30:20] (step=0056450) Train Loss: 6.2511, Train Steps/Sec: 0.84 +[2025-04-21 14:30:49] (step=0056475) Train Loss: 6.3366, Train Steps/Sec: 0.86 +[2025-04-21 14:31:18] (step=0056500) Train Loss: 6.2942, Train Steps/Sec: 0.86 +[2025-04-21 14:31:47] (step=0056525) Train Loss: 6.3526, Train Steps/Sec: 0.86 +[2025-04-21 14:32:16] (step=0056550) Train Loss: 6.3618, Train Steps/Sec: 0.87 +[2025-04-21 14:32:45] (step=0056575) Train Loss: 6.2675, Train Steps/Sec: 0.86 +[2025-04-21 14:33:14] (step=0056600) Train Loss: 6.3376, Train Steps/Sec: 0.86 +[2025-04-21 14:33:43] (step=0056625) Train Loss: 6.3207, Train Steps/Sec: 0.86 +[2025-04-21 14:34:12] (step=0056650) Train Loss: 6.2857, Train Steps/Sec: 0.86 +[2025-04-21 14:34:41] (step=0056675) Train Loss: 6.3077, Train Steps/Sec: 0.87 +[2025-04-21 14:35:10] (step=0056700) Train Loss: 6.3314, Train Steps/Sec: 0.86 +[2025-04-21 14:35:39] (step=0056725) Train Loss: 6.3239, Train Steps/Sec: 0.86 +[2025-04-21 14:36:08] (step=0056750) Train Loss: 6.3030, Train Steps/Sec: 0.87 +[2025-04-21 14:36:37] (step=0056775) Train Loss: 6.2913, Train Steps/Sec: 0.86 +[2025-04-21 14:37:06] (step=0056800) Train Loss: 6.3209, Train Steps/Sec: 0.85 +[2025-04-21 14:37:35] (step=0056825) Train Loss: 6.3150, Train Steps/Sec: 0.86 +[2025-04-21 14:38:04] (step=0056850) Train Loss: 6.3179, Train Steps/Sec: 0.86 +[2025-04-21 14:38:33] (step=0056875) Train Loss: 6.3286, Train Steps/Sec: 0.87 +[2025-04-21 14:39:03] (step=0056900) Train Loss: 6.3038, Train Steps/Sec: 0.85 +[2025-04-21 14:39:31] (step=0056925) Train Loss: 6.3023, Train Steps/Sec: 0.87 +[2025-04-21 14:40:00] (step=0056950) Train Loss: 6.3067, Train Steps/Sec: 0.86 +[2025-04-21 14:40:29] (step=0056975) Train Loss: 6.3090, Train Steps/Sec: 0.86 +[2025-04-21 14:40:59] (step=0057000) Train Loss: 6.3325, Train Steps/Sec: 0.86 +[2025-04-21 14:41:27] (step=0057025) Train Loss: 6.3487, Train Steps/Sec: 0.87 +[2025-04-21 14:41:56] (step=0057050) Train Loss: 6.3047, Train Steps/Sec: 0.86 +[2025-04-21 14:42:25] (step=0057075) Train Loss: 6.3124, Train Steps/Sec: 0.86 +[2025-04-21 14:42:55] (step=0057100) Train Loss: 6.2957, Train Steps/Sec: 0.86 +[2025-04-21 14:43:23] (step=0057125) Train Loss: 6.3383, Train Steps/Sec: 0.87 +[2025-04-21 14:43:52] (step=0057150) Train Loss: 6.3176, Train Steps/Sec: 0.86 +[2025-04-21 14:44:21] (step=0057175) Train Loss: 6.2984, Train Steps/Sec: 0.87 +[2025-04-21 14:44:50] (step=0057200) Train Loss: 6.2984, Train Steps/Sec: 0.86 +[2025-04-21 14:45:19] (step=0057225) Train Loss: 6.2883, Train Steps/Sec: 0.86 +[2025-04-21 14:45:48] (step=0057250) Train Loss: 6.3115, Train Steps/Sec: 0.86 +[2025-04-21 14:46:17] (step=0057275) Train Loss: 6.3005, Train Steps/Sec: 0.87 +[2025-04-21 14:46:46] (step=0057300) Train Loss: 6.2880, Train Steps/Sec: 0.86 +[2025-04-21 14:47:15] (step=0057325) Train Loss: 6.2940, Train Steps/Sec: 0.87 +[2025-04-21 14:47:44] (step=0057350) Train Loss: 6.3557, Train Steps/Sec: 0.86 +[2025-04-21 14:48:13] (step=0057375) Train Loss: 6.3110, Train Steps/Sec: 0.87 +[2025-04-21 14:48:42] (step=0057400) Train Loss: 6.3221, Train Steps/Sec: 0.86 +[2025-04-21 14:49:11] (step=0057425) Train Loss: 6.3181, Train Steps/Sec: 0.86 +[2025-04-21 14:49:40] (step=0057450) Train Loss: 6.3229, Train Steps/Sec: 0.86 +[2025-04-21 14:50:09] (step=0057475) Train Loss: 6.3325, Train Steps/Sec: 0.87 +[2025-04-21 14:50:38] (step=0057500) Train Loss: 6.2991, Train Steps/Sec: 0.86 +[2025-04-21 14:51:07] (step=0057525) Train Loss: 6.3373, Train Steps/Sec: 0.86 +[2025-04-21 14:51:36] (step=0057550) Train Loss: 6.3133, Train Steps/Sec: 0.86 +[2025-04-21 14:52:05] (step=0057575) Train Loss: 6.3435, Train Steps/Sec: 0.87 +[2025-04-21 14:52:34] (step=0057600) Train Loss: 6.3203, Train Steps/Sec: 0.86 +[2025-04-21 14:53:03] (step=0057625) Train Loss: 6.3365, Train Steps/Sec: 0.87 +[2025-04-21 14:53:32] (step=0057650) Train Loss: 6.3089, Train Steps/Sec: 0.87 +[2025-04-21 14:54:01] (step=0057675) Train Loss: 6.2960, Train Steps/Sec: 0.86 +[2025-04-21 14:54:30] (step=0057700) Train Loss: 6.3310, Train Steps/Sec: 0.86 +[2025-04-21 14:54:59] (step=0057725) Train Loss: 6.3167, Train Steps/Sec: 0.86 +[2025-04-21 14:55:28] (step=0057750) Train Loss: 6.2913, Train Steps/Sec: 0.86 +[2025-04-21 14:55:57] (step=0057775) Train Loss: 6.3194, Train Steps/Sec: 0.86 +[2025-04-21 14:56:26] (step=0057800) Train Loss: 6.3194, Train Steps/Sec: 0.86 +[2025-04-21 14:56:55] (step=0057825) Train Loss: 6.3164, Train Steps/Sec: 0.86 +[2025-04-21 14:57:24] (step=0057850) Train Loss: 6.3267, Train Steps/Sec: 0.87 +[2025-04-21 14:57:53] (step=0057875) Train Loss: 6.2173, Train Steps/Sec: 0.87 +[2025-04-21 14:58:22] (step=0057900) Train Loss: 6.3016, Train Steps/Sec: 0.86 +[2025-04-21 14:58:51] (step=0057925) Train Loss: 6.3150, Train Steps/Sec: 0.86 +[2025-04-21 14:59:20] (step=0057950) Train Loss: 6.3060, Train Steps/Sec: 0.87 +[2025-04-21 14:59:56] (step=0057975) Train Loss: 6.3365, Train Steps/Sec: 0.69 +[2025-04-21 15:00:25] (step=0058000) Train Loss: 6.3304, Train Steps/Sec: 0.86 +[2025-04-21 15:00:25] vision_config is None. initializing the InstructBlipVisionConfig with default values. +[2025-04-21 15:00:25] qformer_config is None. Initializing the InstructBlipQFormerConfig with default values. +[2025-04-21 15:00:25] text_config is None. Initializing the text config with default values (`OPTConfig`). +[2025-04-21 15:05:25] Finish Eval in 58000 steps... +[2025-04-21 15:05:46] Saved checkpoint to checkpoint/EmptyFix_InstructBlip_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_objects/002-GPT-XL/checkpoints/0058000.pt +[2025-04-21 15:05:48] Removed old checkpoint: checkpoint/EmptyFix_InstructBlip_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_objects/002-GPT-XL/checkpoints/0054000.pt +[2025-04-21 15:06:17] (step=0058025) Train Loss: 6.2943, Train Steps/Sec: 0.07 +[2025-04-21 15:06:46] (step=0058050) Train Loss: 6.2973, Train Steps/Sec: 0.86 +[2025-04-21 15:07:15] (step=0058075) Train Loss: 6.3070, Train Steps/Sec: 0.87 +[2025-04-21 15:07:44] (step=0058100) Train Loss: 6.2786, Train Steps/Sec: 0.86 +[2025-04-21 15:08:13] (step=0058125) Train Loss: 6.3566, Train Steps/Sec: 0.86 +[2025-04-21 15:08:42] (step=0058150) Train Loss: 6.3208, Train Steps/Sec: 0.87 +[2025-04-21 15:09:11] (step=0058175) Train Loss: 6.3247, Train Steps/Sec: 0.86 +[2025-04-21 15:09:40] (step=0058200) Train Loss: 6.2622, Train Steps/Sec: 0.86 +[2025-04-21 15:10:09] (step=0058225) Train Loss: 6.3316, Train Steps/Sec: 0.86 +[2025-04-21 15:10:38] (step=0058250) Train Loss: 6.3125, Train Steps/Sec: 0.86 +[2025-04-21 15:11:07] (step=0058275) Train Loss: 6.2978, Train Steps/Sec: 0.87 +[2025-04-21 15:11:37] (step=0058300) Train Loss: 6.2895, Train Steps/Sec: 0.85 +[2025-04-21 15:12:06] (step=0058325) Train Loss: 6.3290, Train Steps/Sec: 0.86 +[2025-04-21 15:12:35] (step=0058350) Train Loss: 6.3170, Train Steps/Sec: 0.86 +[2025-04-21 15:13:04] (step=0058375) Train Loss: 6.3442, Train Steps/Sec: 0.86 +[2025-04-21 15:13:33] (step=0058400) Train Loss: 6.3154, Train Steps/Sec: 0.86 +[2025-04-21 15:14:02] (step=0058425) Train Loss: 6.3598, Train Steps/Sec: 0.86 +[2025-04-21 15:14:31] (step=0058450) Train Loss: 6.3203, Train Steps/Sec: 0.86 +[2025-04-21 15:15:00] (step=0058475) Train Loss: 6.3095, Train Steps/Sec: 0.86 +[2025-04-21 15:15:29] (step=0058500) Train Loss: 6.3282, Train Steps/Sec: 0.86 +[2025-04-21 15:15:58] (step=0058525) Train Loss: 6.3115, Train Steps/Sec: 0.86 +[2025-04-21 15:16:27] (step=0058550) Train Loss: 6.3015, Train Steps/Sec: 0.86 +[2025-04-21 15:16:56] (step=0058575) Train Loss: 6.2804, Train Steps/Sec: 0.87 +[2025-04-21 15:17:25] (step=0058600) Train Loss: 6.3549, Train Steps/Sec: 0.86 +[2025-04-21 15:17:54] (step=0058625) Train Loss: 6.3253, Train Steps/Sec: 0.86 +[2025-04-21 15:18:23] (step=0058650) Train Loss: 6.3115, Train Steps/Sec: 0.86 +[2025-04-21 15:18:52] (step=0058675) Train Loss: 6.3172, Train Steps/Sec: 0.87 +[2025-04-21 15:19:21] (step=0058700) Train Loss: 6.3260, Train Steps/Sec: 0.86 +[2025-04-21 15:19:50] (step=0058725) Train Loss: 6.3166, Train Steps/Sec: 0.86 +[2025-04-21 15:20:19] (step=0058750) Train Loss: 6.2932, Train Steps/Sec: 0.87 +[2025-04-21 15:20:48] (step=0058775) Train Loss: 6.3144, Train Steps/Sec: 0.86 +[2025-04-21 15:21:17] (step=0058800) Train Loss: 6.2687, Train Steps/Sec: 0.86 +[2025-04-21 15:21:53] (step=0058825) Train Loss: 6.3394, Train Steps/Sec: 0.69 +[2025-04-21 15:22:22] (step=0058850) Train Loss: 6.2880, Train Steps/Sec: 0.87 +[2025-04-21 15:22:50] (step=0058875) Train Loss: 6.3287, Train Steps/Sec: 0.86 +[2025-04-21 15:23:20] (step=0058900) Train Loss: 6.3241, Train Steps/Sec: 0.86 +[2025-04-21 15:23:49] (step=0058925) Train Loss: 6.3269, Train Steps/Sec: 0.86 +[2025-04-21 15:24:18] (step=0058950) Train Loss: 6.3794, Train Steps/Sec: 0.86 +[2025-04-21 15:24:54] (step=0058975) Train Loss: 6.3080, Train Steps/Sec: 0.70 +[2025-04-21 15:25:23] (step=0059000) Train Loss: 6.3446, Train Steps/Sec: 0.86 +[2025-04-21 15:25:52] (step=0059025) Train Loss: 6.3452, Train Steps/Sec: 0.86 +[2025-04-21 15:26:21] (step=0059050) Train Loss: 6.3407, Train Steps/Sec: 0.86 +[2025-04-21 15:26:50] (step=0059075) Train Loss: 6.2928, Train Steps/Sec: 0.86 +[2025-04-21 15:27:19] (step=0059100) Train Loss: 6.3278, Train Steps/Sec: 0.86 +[2025-04-21 15:27:48] (step=0059125) Train Loss: 6.3246, Train Steps/Sec: 0.87 +[2025-04-21 15:28:17] (step=0059150) Train Loss: 6.3073, Train Steps/Sec: 0.86 +[2025-04-21 15:28:46] (step=0059175) Train Loss: 6.3117, Train Steps/Sec: 0.86 +[2025-04-21 15:29:15] (step=0059200) Train Loss: 6.3037, Train Steps/Sec: 0.86 +[2025-04-21 15:29:44] (step=0059225) Train Loss: 6.3290, Train Steps/Sec: 0.86 +[2025-04-21 15:30:13] (step=0059250) Train Loss: 6.3048, Train Steps/Sec: 0.86 +[2025-04-21 15:30:42] (step=0059275) Train Loss: 6.3318, Train Steps/Sec: 0.86 +[2025-04-21 15:31:11] (step=0059300) Train Loss: 6.2981, Train Steps/Sec: 0.85 +[2025-04-21 15:31:40] (step=0059325) Train Loss: 6.2948, Train Steps/Sec: 0.86 +[2025-04-21 15:32:09] (step=0059350) Train Loss: 6.2735, Train Steps/Sec: 0.87 +[2025-04-21 15:32:38] (step=0059375) Train Loss: 6.3541, Train Steps/Sec: 0.86 +[2025-04-21 15:33:07] (step=0059400) Train Loss: 6.3715, Train Steps/Sec: 0.86 +[2025-04-21 15:33:36] (step=0059425) Train Loss: 6.3332, Train Steps/Sec: 0.86 +[2025-04-21 15:34:11] (step=0059450) Train Loss: 6.3060, Train Steps/Sec: 0.71 +[2025-04-21 15:34:40] (step=0059475) Train Loss: 6.3299, Train Steps/Sec: 0.86 +[2025-04-21 15:35:09] (step=0059500) Train Loss: 6.3614, Train Steps/Sec: 0.85 +[2025-04-21 15:35:38] (step=0059525) Train Loss: 6.3147, Train Steps/Sec: 0.87 +[2025-04-21 15:36:07] (step=0059550) Train Loss: 6.3884, Train Steps/Sec: 0.86 +[2025-04-21 15:36:36] (step=0059575) Train Loss: 6.3023, Train Steps/Sec: 0.86 +[2025-04-21 15:37:06] (step=0059600) Train Loss: 6.3623, Train Steps/Sec: 0.85 +[2025-04-21 15:37:35] (step=0059625) Train Loss: 6.3213, Train Steps/Sec: 0.86 +[2025-04-21 15:38:04] (step=0059650) Train Loss: 6.3028, Train Steps/Sec: 0.86 +[2025-04-21 15:38:33] (step=0059675) Train Loss: 6.2971, Train Steps/Sec: 0.86 +[2025-04-21 15:39:02] (step=0059700) Train Loss: 6.2909, Train Steps/Sec: 0.86 +[2025-04-21 15:39:31] (step=0059725) Train Loss: 6.2699, Train Steps/Sec: 0.87 +[2025-04-21 15:40:00] (step=0059750) Train Loss: 6.3094, Train Steps/Sec: 0.87 +[2025-04-21 15:40:29] (step=0059775) Train Loss: 6.3392, Train Steps/Sec: 0.86 +[2025-04-21 15:40:58] (step=0059800) Train Loss: 6.2904, Train Steps/Sec: 0.86 +[2025-04-21 15:41:27] (step=0059825) Train Loss: 6.2817, Train Steps/Sec: 0.86 +[2025-04-21 15:41:56] (step=0059850) Train Loss: 6.3173, Train Steps/Sec: 0.86 +[2025-04-21 15:42:25] (step=0059875) Train Loss: 6.3231, Train Steps/Sec: 0.87 +[2025-04-21 15:42:54] (step=0059900) Train Loss: 6.2935, Train Steps/Sec: 0.86 +[2025-04-21 15:43:23] (step=0059925) Train Loss: 6.3229, Train Steps/Sec: 0.86 +[2025-04-21 15:43:52] (step=0059950) Train Loss: 6.3369, Train Steps/Sec: 0.86 +[2025-04-21 15:44:21] (step=0059975) Train Loss: 6.3329, Train Steps/Sec: 0.86 +[2025-04-21 15:44:50] (step=0060000) Train Loss: 6.3256, Train Steps/Sec: 0.86 +[2025-04-21 15:44:50] vision_config is None. initializing the InstructBlipVisionConfig with default values. +[2025-04-21 15:44:50] qformer_config is None. Initializing the InstructBlipQFormerConfig with default values. +[2025-04-21 15:44:50] text_config is None. Initializing the text config with default values (`OPTConfig`). +[2025-04-21 15:49:51] Finish Eval in 60000 steps... +[2025-04-21 15:50:13] Saved checkpoint to checkpoint/EmptyFix_InstructBlip_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_objects/002-GPT-XL/checkpoints/0060000.pt +[2025-04-21 15:50:15] Removed old checkpoint: checkpoint/EmptyFix_InstructBlip_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_objects/002-GPT-XL/checkpoints/0056000.pt +[2025-04-21 15:50:43] (step=0060025) Train Loss: 6.3152, Train Steps/Sec: 0.07 +[2025-04-21 15:51:12] (step=0060050) Train Loss: 6.2679, Train Steps/Sec: 0.86 +[2025-04-21 15:51:41] (step=0060075) Train Loss: 6.3155, Train Steps/Sec: 0.86 +[2025-04-21 15:52:10] (step=0060100) Train Loss: 6.3024, Train Steps/Sec: 0.86 +[2025-04-21 15:52:46] (step=0060125) Train Loss: 6.3491, Train Steps/Sec: 0.70 +[2025-04-21 15:53:21] (step=0060150) Train Loss: 6.3295, Train Steps/Sec: 0.72 +[2025-04-21 15:53:50] (step=0060175) Train Loss: 6.3198, Train Steps/Sec: 0.86 +[2025-04-21 15:54:19] (step=0060200) Train Loss: 6.3365, Train Steps/Sec: 0.86 +[2025-04-21 15:54:48] (step=0060225) Train Loss: 6.3078, Train Steps/Sec: 0.86 +[2025-04-21 15:55:17] (step=0060250) Train Loss: 6.3634, Train Steps/Sec: 0.87 +[2025-04-21 15:55:46] (step=0060275) Train Loss: 6.3078, Train Steps/Sec: 0.87 +[2025-04-21 15:56:15] (step=0060300) Train Loss: 6.3207, Train Steps/Sec: 0.86 +[2025-04-21 15:56:44] (step=0060325) Train Loss: 6.3432, Train Steps/Sec: 0.86 +[2025-04-21 15:57:13] (step=0060350) Train Loss: 6.2936, Train Steps/Sec: 0.87 +[2025-04-21 15:57:42] (step=0060375) Train Loss: 6.3447, Train Steps/Sec: 0.86 +[2025-04-21 15:58:11] (step=0060400) Train Loss: 6.3580, Train Steps/Sec: 0.85 +[2025-04-21 15:58:47] (step=0060425) Train Loss: 6.3166, Train Steps/Sec: 0.70 +[2025-04-21 15:59:16] (step=0060450) Train Loss: 6.2779, Train Steps/Sec: 0.86 +[2025-04-21 15:59:45] (step=0060475) Train Loss: 6.3291, Train Steps/Sec: 0.86 +[2025-04-21 16:00:14] (step=0060500) Train Loss: 6.2802, Train Steps/Sec: 0.86 +[2025-04-21 16:00:43] (step=0060525) Train Loss: 6.2887, Train Steps/Sec: 0.86 +[2025-04-21 16:01:12] (step=0060550) Train Loss: 6.3375, Train Steps/Sec: 0.87 +[2025-04-21 16:01:41] (step=0060575) Train Loss: 6.3152, Train Steps/Sec: 0.87 +[2025-04-21 16:02:10] (step=0060600) Train Loss: 6.3254, Train Steps/Sec: 0.86 +[2025-04-21 16:02:39] (step=0060625) Train Loss: 6.2551, Train Steps/Sec: 0.87 +[2025-04-21 16:03:08] (step=0060650) Train Loss: 6.3456, Train Steps/Sec: 0.86 +[2025-04-21 16:03:37] (step=0060675) Train Loss: 6.3353, Train Steps/Sec: 0.86 +[2025-04-21 16:04:06] (step=0060700) Train Loss: 6.3289, Train Steps/Sec: 0.86 +[2025-04-21 16:04:35] (step=0060725) Train Loss: 6.3616, Train Steps/Sec: 0.86 diff --git a/CKPTS/EmptyFix_InstructBlip_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_objects/wandb/debug-internal.log b/CKPTS/EmptyFix_InstructBlip_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_objects/wandb/debug-internal.log new file mode 100644 index 0000000000000000000000000000000000000000..74dc30b07844c3579e969ff38961c446e5e4d1ac --- /dev/null +++ b/CKPTS/EmptyFix_InstructBlip_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_objects/wandb/debug-internal.log @@ -0,0 +1,7 @@ +{"time":"2025-04-20T16:56:33.404420738Z","level":"INFO","msg":"stream: starting","core version":"0.19.8","symlink path":"checkpoint/EmptyFix_InstructBlip_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_objects/wandb/run-20250420_165633-wzvwagyn/logs/debug-core.log"} +{"time":"2025-04-20T16:56:33.612529261Z","level":"INFO","msg":"created new stream","id":"wzvwagyn"} +{"time":"2025-04-20T16:56:33.612568585Z","level":"INFO","msg":"stream: started","id":"wzvwagyn"} +{"time":"2025-04-20T16:56:33.612590145Z","level":"INFO","msg":"writer: Do: started","stream_id":"wzvwagyn"} +{"time":"2025-04-20T16:56:33.612631518Z","level":"INFO","msg":"sender: started","stream_id":"wzvwagyn"} +{"time":"2025-04-20T16:56:33.612642178Z","level":"INFO","msg":"handler: started","stream_id":"wzvwagyn"} +{"time":"2025-04-20T16:56:34.210323838Z","level":"INFO","msg":"Starting system monitor"} diff --git a/CKPTS/EmptyFix_InstructBlip_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_objects/wandb/debug.log b/CKPTS/EmptyFix_InstructBlip_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_objects/wandb/debug.log new file mode 100644 index 0000000000000000000000000000000000000000..415d8bc1826ba5ebf0c507c7de3e27bbd188809c --- /dev/null +++ b/CKPTS/EmptyFix_InstructBlip_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_objects/wandb/debug.log @@ -0,0 +1,22 @@ +2025-04-20 16:56:33,394 INFO MainThread:2696278 [wandb_setup.py:_flush():67] Current SDK version is 0.19.8 +2025-04-20 16:56:33,394 INFO MainThread:2696278 [wandb_setup.py:_flush():67] Configure stats pid to 2696278 +2025-04-20 16:56:33,395 INFO MainThread:2696278 [wandb_setup.py:_flush():67] Loading settings from /tmp/haozhezhao/.config/wandb/settings +2025-04-20 16:56:33,395 INFO MainThread:2696278 [wandb_setup.py:_flush():67] Loading settings from /tmp/haozhezhao/MLLMG/wandb/settings +2025-04-20 16:56:33,395 INFO MainThread:2696278 [wandb_setup.py:_flush():67] Loading settings from environment variables +2025-04-20 16:56:33,395 INFO MainThread:2696278 [wandb_init.py:setup_run_log_directory():647] Logging user logs to checkpoint/EmptyFix_InstructBlip_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_objects/wandb/run-20250420_165633-wzvwagyn/logs/debug.log +2025-04-20 16:56:33,395 INFO MainThread:2696278 [wandb_init.py:setup_run_log_directory():648] Logging internal logs to checkpoint/EmptyFix_InstructBlip_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_objects/wandb/run-20250420_165633-wzvwagyn/logs/debug-internal.log +2025-04-20 16:56:33,395 INFO MainThread:2696278 [wandb_init.py:init():761] calling init triggers +2025-04-20 16:56:33,395 INFO MainThread:2696278 [wandb_init.py:init():766] wandb.init called with sweep_config: {} +config: {'data_path': '/tmp/haozhezhao/MLLMG/jsonl_data/multiobjects_molom_imagenet_flux_qwen_midsource_gen_2_2m_trained.jsonl', 'cloud_save_path': '/tmp/haozhezhao/MLLMG/checkpoint', 'no_local_save': False, 'vq_model': 'VQ-16', 'vq_ckpt': '/tmp/haozhezhao/model/llamagen_t2i/vq_ds16_t2i.pt', 'codebook_size': 16384, 'codebook_embed_dim': 8, 'gpt_model': 'GPT-XL', 'gpt_ckpt': '/tmp/haozhezhao/MLLMG/checkpoint/EmptyFix_InstructBlip_just_segment_pretrain_stage3_dreambench_recap_Subject400k_filtered_t2i_flux400k_200kmid_recovery_150k_extract_150k_100_fluxseg_50samseg_trainall_1e4_no_replace/007-GPT-XL/checkpoints/0092000.pt', 'gpt_type': 't2i', 'vocab_size': 16384, 'cls_token_num': 512, 'dropout_p': 0.1, 'token_dropout_p': 0.1, 'drop_path': 0.0, 'no_compile': False, 'results_dir': 'checkpoint/EmptyFix_InstructBlip_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_objects', 'dataset': 'ti2i', 'image_size': 512, 'downsample_size': 16, 'num_classes': 1000, 'epochs': 2, 'lr': 5e-05, 'weight_decay': 0.05, 'beta1': 0.9, 'beta2': 0.95, 'max_grad_norm': 1.0, 'global_batch_size': 56, 'global_seed': 0, 'num_workers': 4, 'log_every': 25, 'ckpt_every': 2000, 'gradient_accumulation_steps': 4, 'mixed_precision': 'bf16', 'val_data_path': '/tmp/haozhezhao/MLLMG/jsonl_data/multiobjects_molom_imagenet_flux_qwen_midsource_gen_val.jsonl', 'use_vision_tower': True, 'model_name_or_path': '/tmp/haozhezhao/model/instructblip-flan-t5-xl', 'image_place_holder': '', 'processor_path': None, 'do_eval': True, 'max_eval_samples': 200, 'train_text_encoder': True, 'no_left_padding': False, 'cfg_scale': 7.5, 'top_k': 16384, 'temperature': 0.9, 'top_p': 1.0, 'eval_steps': 2000, 'project_name': 'llamagen_ti2i', 'load_from_checkpoint': '/tmp/haozhezhao/MLLMG/checkpoint/FIXed_3M_1epoch_step32000.pt', 'warmup': 0.05, 'lr_decay_style': 'cosine', 'lr_decay_ratio': 0.1, 'train_iters': 500000, 'class_dropout_prob': 0.1, 'with_image_only': False, 'image_only_rate': 0.1, 'stage2': False, 'subject_driven': True, 'load_subject_embedding': None, 'reference_data_path': '/tmp/haozhezhao/MLLMG/cc12m_reference_tunnel.jsonl', 'multimodal_encoder': 'instructblip', 'do_recovery': True, 'no_replace': False, 'resume': False, 'dreambench_eval': False, 'find_unused_parameters': True, 'load_visual_encoder': False, 'continue_stage1': False, 'replace_subject': False, 'train_all': True, 'save_total_limit': 2, 'load_language_projection': None, 'mm_vision_tower': 'openai/clip-vit-large-patch14', 'load_fixed_llamagen': True, 'unfreeze_output': False, 'fix': 'gpt-empty-fix', 'rank': 0, 'world_size': 8, 'gpu': 0, 'dist_url': 'env://', 'distributed': True, 'dist_backend': 'nccl', '_wandb': {}} +2025-04-20 16:56:33,395 INFO MainThread:2696278 [wandb_init.py:init():784] starting backend +2025-04-20 16:56:33,395 INFO MainThread:2696278 [wandb_init.py:init():788] sending inform_init request +2025-04-20 16:56:33,401 INFO MainThread:2696278 [backend.py:_multiprocessing_setup():101] multiprocessing start_methods=fork,spawn,forkserver, using: spawn +2025-04-20 16:56:33,401 INFO MainThread:2696278 [wandb_init.py:init():798] backend started and connected +2025-04-20 16:56:33,409 INFO MainThread:2696278 [wandb_init.py:init():891] updated telemetry +2025-04-20 16:56:33,410 INFO MainThread:2696278 [wandb_init.py:init():915] communicating run to backend with 90.0 second timeout +2025-04-20 16:56:34,207 INFO MainThread:2696278 [wandb_init.py:init():990] starting run threads in backend +2025-04-20 16:56:34,307 INFO MainThread:2696278 [wandb_run.py:_console_start():2375] atexit reg +2025-04-20 16:56:34,307 INFO MainThread:2696278 [wandb_run.py:_redirect():2227] redirect: wrap_raw +2025-04-20 16:56:34,308 INFO MainThread:2696278 [wandb_run.py:_redirect():2292] Wrapping output streams. +2025-04-20 16:56:34,308 INFO MainThread:2696278 [wandb_run.py:_redirect():2315] Redirects installed. +2025-04-20 16:56:34,310 INFO MainThread:2696278 [wandb_init.py:init():1032] run started, returning control to user process diff --git a/CKPTS/EmptyFix_InstructBlip_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_objects/wandb/run-20250420_165633-wzvwagyn/files/output.log b/CKPTS/EmptyFix_InstructBlip_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_objects/wandb/run-20250420_165633-wzvwagyn/files/output.log new file mode 100644 index 0000000000000000000000000000000000000000..7bb433312d77ad5231313fa85a7705eae2a8be96 --- /dev/null +++ b/CKPTS/EmptyFix_InstructBlip_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_objects/wandb/run-20250420_165633-wzvwagyn/files/output.log @@ -0,0 +1,2892 @@ +[2025-04-20 16:56:34] Training for 2 epochs... +[2025-04-20 16:56:34] Beginning epoch 0... + 0%| | 0/40903 [00:00 +tokenizer length after expend 32102 +tokenizer length before expend 32102 + /tmp/haozhezhao/anaconda3/envs/nlp/lib/python3.11/contextlib.py:105: FutureWarning: `torch.backends.cuda.sdp_kernel()` is deprecated. In the future, this context manager will be removed. Please see `torch.nn.attention.sdpa_kernel()` for the new context manager, with updated signature. + self.gen = func(*args, **kwds) | 0/3 [00:00 +tokenizer length after expend 32102 +tokenizer length before expend 32102 +100%|████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████| 3/3 [03:06<00:00, 62.10s/it] +[2025-04-20 19:07:23] Finish Eval in 4000 steps...██████████████████████████████████████████████████████████████████████| 3/3 [03:05<00:00, 61.67s/it] +processing left-padding... +processing left-padding... +processing left-padding... +[2025-04-20 19:07:44] Saved checkpoint to checkpoint/EmptyFix_InstructBlip_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_objects/002-GPT-XL/checkpoints/0004000.pt + 10%|███████████ | 4024/40903 [2:11:38<12:15:06, 1.20s/it][2025-04-20 19:08:13] (step=0004025) Train Loss: 6.3798, Train Steps/Sec: 0.07 + 10%|███████████ | 4049/40903 [2:12:06<11:45:58, 1.15s/it][2025-04-20 19:08:42] (step=0004050) Train Loss: 6.4162, Train Steps/Sec: 0.86 + 10%|███████████▏ | 4074/40903 [2:12:35<11:40:26, 1.14s/it][2025-04-20 19:09:11] (step=0004075) Train Loss: 6.4138, Train Steps/Sec: 0.86 + 10%|███████████▏ | 4099/40903 [2:13:04<11:39:27, 1.14s/it][2025-04-20 19:09:40] (step=0004100) Train Loss: 6.4085, Train Steps/Sec: 0.86 + 10%|███████████▎ | 4124/40903 [2:13:34<11:55:08, 1.17s/it][2025-04-20 19:10:09] (step=0004125) Train Loss: 6.4676, Train Steps/Sec: 0.86 + 10%|███████████▎ | 4149/40903 [2:14:03<11:50:14, 1.16s/it][2025-04-20 19:10:38] (step=0004150) Train Loss: 6.3727, Train Steps/Sec: 0.86 + 10%|███████████▍ | 4174/40903 [2:14:31<11:41:08, 1.15s/it][2025-04-20 19:11:07] (step=0004175) Train Loss: 6.4526, Train Steps/Sec: 0.87 + 10%|███████████▍ | 4199/40903 [2:15:00<11:44:00, 1.15s/it][2025-04-20 19:11:36] (step=0004200) Train Loss: 6.4306, Train Steps/Sec: 0.86 + 10%|███████████▌ | 4224/40903 [2:15:29<12:12:38, 1.20s/it][2025-04-20 19:12:05] (step=0004225) Train Loss: 6.4266, Train Steps/Sec: 0.86 + 10%|███████████▋ | 4249/40903 [2:15:59<11:52:12, 1.17s/it][2025-04-20 19:12:34] (step=0004250) Train Loss: 6.3537, Train Steps/Sec: 0.85 + 10%|███████████▋ | 4274/40903 [2:16:28<11:44:09, 1.15s/it][2025-04-20 19:13:03] (step=0004275) Train Loss: 6.4255, Train Steps/Sec: 0.87 + 11%|███████████▊ | 4299/40903 [2:16:57<11:38:48, 1.15s/it][2025-04-20 19:13:32] (step=0004300) Train Loss: 6.4346, Train Steps/Sec: 0.86 + 11%|███████████▊ | 4324/40903 [2:17:26<11:55:15, 1.17s/it][2025-04-20 19:14:01] (step=0004325) Train Loss: 6.4124, Train Steps/Sec: 0.86 + 11%|███████████▉ | 4349/40903 [2:17:54<11:37:24, 1.14s/it][2025-04-20 19:14:30] (step=0004350) Train Loss: 6.4066, Train Steps/Sec: 0.87 + 11%|███████████▉ | 4374/40903 [2:18:23<11:42:51, 1.15s/it][2025-04-20 19:14:59] (step=0004375) Train Loss: 6.4033, Train Steps/Sec: 0.87 + 11%|████████████ | 4399/40903 [2:18:52<11:35:59, 1.14s/it][2025-04-20 19:15:28] (step=0004400) Train Loss: 6.3907, Train Steps/Sec: 0.86 + 11%|████████████ | 4424/40903 [2:19:21<11:48:27, 1.17s/it][2025-04-20 19:15:57] (step=0004425) Train Loss: 6.4029, Train Steps/Sec: 0.86 + 11%|████████████▏ | 4449/40903 [2:19:50<11:39:28, 1.15s/it][2025-04-20 19:16:26] (step=0004450) Train Loss: 6.3747, Train Steps/Sec: 0.87 + 11%|████████████▎ | 4474/40903 [2:20:19<11:29:37, 1.14s/it][2025-04-20 19:16:55] (step=0004475) Train Loss: 6.4041, Train Steps/Sec: 0.87 + 11%|████████████▎ | 4499/40903 [2:20:48<11:37:13, 1.15s/it][2025-04-20 19:17:24] (step=0004500) Train Loss: 6.3704, Train Steps/Sec: 0.86 + 11%|████████████▍ | 4524/40903 [2:21:17<11:56:51, 1.18s/it][2025-04-20 19:17:53] (step=0004525) Train Loss: 6.4040, Train Steps/Sec: 0.86 + 11%|████████████▍ | 4549/40903 [2:21:46<11:40:15, 1.16s/it][2025-04-20 19:18:22] (step=0004550) Train Loss: 6.4572, Train Steps/Sec: 0.86 + 11%|████████████▌ | 4574/40903 [2:22:15<11:36:47, 1.15s/it][2025-04-20 19:18:50] (step=0004575) Train Loss: 6.3539, Train Steps/Sec: 0.87 + 11%|████████████▌ | 4599/40903 [2:22:44<11:41:31, 1.16s/it][2025-04-20 19:19:20] (step=0004600) Train Loss: 6.4472, Train Steps/Sec: 0.86 + 11%|████████████▋ | 4624/40903 [2:23:13<11:46:16, 1.17s/it][2025-04-20 19:19:48] (step=0004625) Train Loss: 6.3996, Train Steps/Sec: 0.87 + 11%|████████████▋ | 4649/40903 [2:23:42<11:35:21, 1.15s/it][2025-04-20 19:20:17] (step=0004650) Train Loss: 6.4231, Train Steps/Sec: 0.86 + 11%|████████████▊ | 4674/40903 [2:24:11<11:34:03, 1.15s/it][2025-04-20 19:20:46] (step=0004675) Train Loss: 6.4317, Train Steps/Sec: 0.86 + 11%|████████████▊ | 4699/40903 [2:24:40<11:37:22, 1.16s/it][2025-04-20 19:21:16] (step=0004700) Train Loss: 6.4144, Train Steps/Sec: 0.86 + 12%|████████████▉ | 4724/40903 [2:25:09<11:45:42, 1.17s/it][2025-04-20 19:21:44] (step=0004725) Train Loss: 6.3825, Train Steps/Sec: 0.87 + 12%|█████████████ | 4749/40903 [2:25:38<11:32:04, 1.15s/it][2025-04-20 19:22:13] (step=0004750) Train Loss: 6.4110, Train Steps/Sec: 0.87 + 12%|█████████████ | 4774/40903 [2:26:06<11:38:30, 1.16s/it][2025-04-20 19:22:42] (step=0004775) Train Loss: 6.3894, Train Steps/Sec: 0.86 + 12%|█████████████▏ | 4799/40903 [2:26:35<11:32:22, 1.15s/it][2025-04-20 19:23:11] (step=0004800) Train Loss: 6.3890, Train Steps/Sec: 0.86 + 12%|█████████████▏ | 4824/40903 [2:27:04<11:45:18, 1.17s/it][2025-04-20 19:23:40] (step=0004825) Train Loss: 6.3873, Train Steps/Sec: 0.87 + 12%|█████████████▎ | 4849/40903 [2:27:33<11:22:02, 1.14s/it][2025-04-20 19:24:09] (step=0004850) Train Loss: 6.4305, Train Steps/Sec: 0.86 + 12%|█████████████▎ | 4874/40903 [2:28:02<11:29:28, 1.15s/it][2025-04-20 19:24:38] (step=0004875) Train Loss: 6.4459, Train Steps/Sec: 0.86 + 12%|█████████████▍ | 4899/40903 [2:28:31<11:29:11, 1.15s/it][2025-04-20 19:25:07] (step=0004900) Train Loss: 6.3918, Train Steps/Sec: 0.86 + 12%|█████████████▍ | 4924/40903 [2:29:00<11:45:56, 1.18s/it][2025-04-20 19:25:36] (step=0004925) Train Loss: 6.4141, Train Steps/Sec: 0.86 + 12%|█████████████▌ | 4949/40903 [2:29:29<11:26:07, 1.14s/it][2025-04-20 19:26:05] (step=0004950) Train Loss: 6.4233, Train Steps/Sec: 0.87 + 12%|█████████████▌ | 4974/40903 [2:29:58<11:32:49, 1.16s/it][2025-04-20 19:26:34] (step=0004975) Train Loss: 6.3474, Train Steps/Sec: 0.86 + 12%|█████████████▋ | 4999/40903 [2:30:27<11:27:28, 1.15s/it][2025-04-20 19:27:03] (step=0005000) Train Loss: 6.4056, Train Steps/Sec: 0.86 + 12%|█████████████▊ | 5024/40903 [2:30:56<11:44:45, 1.18s/it][2025-04-20 19:27:32] (step=0005025) Train Loss: 6.4960, Train Steps/Sec: 0.86 + 12%|█████████████▊ | 5049/40903 [2:31:25<11:29:29, 1.15s/it][2025-04-20 19:28:01] (step=0005050) Train Loss: 6.4010, Train Steps/Sec: 0.86 + 12%|█████████████▉ | 5074/40903 [2:31:54<11:31:10, 1.16s/it][2025-04-20 19:28:30] (step=0005075) Train Loss: 6.4534, Train Steps/Sec: 0.87 + 12%|█████████████▉ | 5099/40903 [2:32:23<11:13:21, 1.13s/it][2025-04-20 19:28:59] (step=0005100) Train Loss: 6.3942, Train Steps/Sec: 0.86 + 13%|██████████████ | 5124/40903 [2:32:52<11:33:53, 1.16s/it][2025-04-20 19:29:28] (step=0005125) Train Loss: 6.4175, Train Steps/Sec: 0.86 + 13%|██████████████ | 5149/40903 [2:33:21<11:27:35, 1.15s/it][2025-04-20 19:29:57] (step=0005150) Train Loss: 6.3711, Train Steps/Sec: 0.86 + 13%|██████████████▏ | 5174/40903 [2:33:50<11:25:48, 1.15s/it][2025-04-20 19:30:26] (step=0005175) Train Loss: 6.4427, Train Steps/Sec: 0.86 + 13%|██████████████▏ | 5199/40903 [2:34:19<11:26:32, 1.15s/it][2025-04-20 19:30:55] (step=0005200) Train Loss: 6.4294, Train Steps/Sec: 0.86 + 13%|██████████████▎ | 5224/40903 [2:34:56<12:02:30, 1.22s/it][2025-04-20 19:31:31] (step=0005225) Train Loss: 6.4102, Train Steps/Sec: 0.68 + 13%|██████████████▎ | 5249/40903 [2:35:25<11:31:23, 1.16s/it][2025-04-20 19:32:00] (step=0005250) Train Loss: 6.3955, Train Steps/Sec: 0.86 + 13%|██████████████▍ | 5274/40903 [2:35:54<11:30:11, 1.16s/it][2025-04-20 19:32:29] (step=0005275) Train Loss: 6.4068, Train Steps/Sec: 0.86 + 13%|██████████████▌ | 5299/40903 [2:36:22<11:12:05, 1.13s/it][2025-04-20 19:32:58] (step=0005300) Train Loss: 6.3637, Train Steps/Sec: 0.87 + 13%|██████████████▌ | 5324/40903 [2:36:51<11:39:28, 1.18s/it][2025-04-20 19:33:27] (step=0005325) Train Loss: 6.4061, Train Steps/Sec: 0.86 + 13%|██████████████▋ | 5349/40903 [2:37:20<11:30:17, 1.16s/it][2025-04-20 19:33:56] (step=0005350) Train Loss: 6.4148, Train Steps/Sec: 0.86 + 13%|██████████████▋ | 5374/40903 [2:37:49<11:14:01, 1.14s/it][2025-04-20 19:34:25] (step=0005375) Train Loss: 6.4038, Train Steps/Sec: 0.87 + 13%|██████████████▊ | 5399/40903 [2:38:19<11:16:11, 1.14s/it][2025-04-20 19:34:54] (step=0005400) Train Loss: 6.4185, Train Steps/Sec: 0.85 + 13%|██████████████▊ | 5424/40903 [2:38:48<11:35:48, 1.18s/it][2025-04-20 19:35:23] (step=0005425) Train Loss: 6.3802, Train Steps/Sec: 0.87 + 13%|██████████████▉ | 5449/40903 [2:39:16<11:22:34, 1.16s/it][2025-04-20 19:35:52] (step=0005450) Train Loss: 6.3825, Train Steps/Sec: 0.87 + 13%|██████████████▉ | 5474/40903 [2:39:45<11:25:01, 1.16s/it][2025-04-20 19:36:21] (step=0005475) Train Loss: 6.3806, Train Steps/Sec: 0.86 + 13%|███████████████ | 5499/40903 [2:40:14<11:17:02, 1.15s/it][2025-04-20 19:36:50] (step=0005500) Train Loss: 6.3928, Train Steps/Sec: 0.86 + 14%|███████████████▏ | 5524/40903 [2:40:44<11:31:20, 1.17s/it][2025-04-20 19:37:19] (step=0005525) Train Loss: 6.3680, Train Steps/Sec: 0.86 + 14%|███████████████▏ | 5549/40903 [2:41:13<11:33:55, 1.18s/it][2025-04-20 19:37:48] (step=0005550) Train Loss: 6.4031, Train Steps/Sec: 0.86 + 14%|███████████████▎ | 5574/40903 [2:41:42<11:21:42, 1.16s/it][2025-04-20 19:38:17] (step=0005575) Train Loss: 6.4179, Train Steps/Sec: 0.86 + 14%|███████████████▎ | 5599/40903 [2:42:11<11:20:06, 1.16s/it][2025-04-20 19:38:47] (step=0005600) Train Loss: 6.3998, Train Steps/Sec: 0.86 + 14%|███████████████▍ | 5624/40903 [2:42:40<11:44:27, 1.20s/it][2025-04-20 19:39:16] (step=0005625) Train Loss: 6.3929, Train Steps/Sec: 0.86 + 14%|███████████████▍ | 5649/40903 [2:43:09<11:25:52, 1.17s/it][2025-04-20 19:39:44] (step=0005650) Train Loss: 6.4087, Train Steps/Sec: 0.87 + 14%|███████████████▌ | 5674/40903 [2:43:38<11:13:09, 1.15s/it][2025-04-20 19:40:14] (step=0005675) Train Loss: 6.3870, Train Steps/Sec: 0.86 + 14%|███████████████▌ | 5699/40903 [2:44:07<11:12:25, 1.15s/it][2025-04-20 19:40:43] (step=0005700) Train Loss: 6.3952, Train Steps/Sec: 0.86 + 14%|███████████████▋ | 5724/40903 [2:44:36<11:39:18, 1.19s/it][2025-04-20 19:41:12] (step=0005725) Train Loss: 6.4217, Train Steps/Sec: 0.86 + 14%|███████████████▋ | 5749/40903 [2:45:05<11:24:10, 1.17s/it][2025-04-20 19:41:41] (step=0005750) Train Loss: 6.3573, Train Steps/Sec: 0.86 + 14%|███████████████▊ | 5774/40903 [2:45:34<11:12:40, 1.15s/it][2025-04-20 19:42:09] (step=0005775) Train Loss: 6.4025, Train Steps/Sec: 0.87 + 14%|███████████████▉ | 5799/40903 [2:46:03<11:07:18, 1.14s/it][2025-04-20 19:42:38] (step=0005800) Train Loss: 6.3411, Train Steps/Sec: 0.86 + 14%|███████████████▉ | 5824/40903 [2:46:32<11:29:30, 1.18s/it][2025-04-20 19:43:07] (step=0005825) Train Loss: 6.4026, Train Steps/Sec: 0.86 + 14%|████████████████ | 5849/40903 [2:47:01<11:19:10, 1.16s/it][2025-04-20 19:43:36] (step=0005850) Train Loss: 6.4254, Train Steps/Sec: 0.86 + 14%|████████████████ | 5874/40903 [2:47:30<11:15:03, 1.16s/it][2025-04-20 19:44:05] (step=0005875) Train Loss: 6.3602, Train Steps/Sec: 0.87 + 14%|████████████████▏ | 5899/40903 [2:47:59<11:04:31, 1.14s/it][2025-04-20 19:44:34] (step=0005900) Train Loss: 6.4234, Train Steps/Sec: 0.86 + 14%|████████████████▏ | 5924/40903 [2:48:28<11:27:20, 1.18s/it][2025-04-20 19:45:03] (step=0005925) Train Loss: 6.4573, Train Steps/Sec: 0.86 + 15%|████████████████▎ | 5949/40903 [2:48:56<11:04:50, 1.14s/it][2025-04-20 19:45:32] (step=0005950) Train Loss: 6.3904, Train Steps/Sec: 0.87 + 15%|████████████████▎ | 5974/40903 [2:49:26<11:10:53, 1.15s/it][2025-04-20 19:46:01] (step=0005975) Train Loss: 6.3817, Train Steps/Sec: 0.86 + 15%|████████████████▍ | 5999/40903 [2:49:55<11:12:36, 1.16s/it][2025-04-20 19:46:30] (step=0006000) Train Loss: 6.3946, Train Steps/Sec: 0.86 +[2025-04-20 19:46:30] vision_config is None. initializing the InstructBlipVisionConfig with default values. +[2025-04-20 19:46:30] qformer_config is None. Initializing the InstructBlipQFormerConfig with default values. +[2025-04-20 19:46:30] text_config is None. Initializing the text config with default values (`OPTConfig`). +model_name_or_path:/tmp/haozhezhao/model/instructblip-flan-t5-xl +image_place_holder: +tokenizer length after expend 32102 +tokenizer length before expend 32102 +100%|████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████| 3/3 [03:06<00:00, 62.32s/it] +[2025-04-20 19:51:29] Finish Eval in 6000 steps...██████████████████████████████████████████████████████████████████████| 3/3 [03:06<00:00, 61.94s/it] +processing left-padding... +processing left-padding... +processing left-padding... +[2025-04-20 19:51:51] Saved checkpoint to checkpoint/EmptyFix_InstructBlip_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_objects/002-GPT-XL/checkpoints/0006000.pt +[2025-04-20 19:51:53] Removed old checkpoint: checkpoint/EmptyFix_InstructBlip_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_objects/002-GPT-XL/checkpoints/0002000.pt + 15%|████████████████▍ | 6024/40903 [2:55:47<11:39:23, 1.20s/it][2025-04-20 19:52:22] (step=0006025) Train Loss: 6.3639, Train Steps/Sec: 0.07 + 15%|████████████████▌ | 6049/40903 [2:56:15<11:13:42, 1.16s/it][2025-04-20 19:52:51] (step=0006050) Train Loss: 6.3682, Train Steps/Sec: 0.87 + 15%|████████████████▋ | 6074/40903 [2:56:44<11:02:23, 1.14s/it][2025-04-20 19:53:20] (step=0006075) Train Loss: 6.4023, Train Steps/Sec: 0.87 + 15%|████████████████▋ | 6099/40903 [2:57:13<11:09:31, 1.15s/it][2025-04-20 19:53:49] (step=0006100) Train Loss: 6.4317, Train Steps/Sec: 0.85 + 15%|████████████████▊ | 6124/40903 [2:57:42<11:24:19, 1.18s/it][2025-04-20 19:54:18] (step=0006125) Train Loss: 6.4124, Train Steps/Sec: 0.87 + 15%|████████████████▊ | 6149/40903 [2:58:11<11:10:41, 1.16s/it][2025-04-20 19:54:47] (step=0006150) Train Loss: 6.4234, Train Steps/Sec: 0.87 + 15%|████████████████▉ | 6174/40903 [2:58:40<11:05:50, 1.15s/it][2025-04-20 19:55:16] (step=0006175) Train Loss: 6.3582, Train Steps/Sec: 0.86 + 15%|████████████████▉ | 6199/40903 [2:59:09<11:01:17, 1.14s/it][2025-04-20 19:55:45] (step=0006200) Train Loss: 6.4322, Train Steps/Sec: 0.86 + 15%|█████████████████ | 6224/40903 [2:59:38<11:19:39, 1.18s/it][2025-04-20 19:56:14] (step=0006225) Train Loss: 6.4129, Train Steps/Sec: 0.86 + 15%|█████████████████ | 6249/40903 [3:00:07<11:15:36, 1.17s/it][2025-04-20 19:56:43] (step=0006250) Train Loss: 6.4372, Train Steps/Sec: 0.86 + 15%|█████████████████▏ | 6274/40903 [3:00:36<11:03:26, 1.15s/it][2025-04-20 19:57:12] (step=0006275) Train Loss: 6.3692, Train Steps/Sec: 0.86 + 15%|█████████████████▏ | 6299/40903 [3:01:05<11:08:08, 1.16s/it][2025-04-20 19:57:41] (step=0006300) Train Loss: 6.4202, Train Steps/Sec: 0.86 + 15%|█████████████████▎ | 6324/40903 [3:01:34<11:14:10, 1.17s/it][2025-04-20 19:58:10] (step=0006325) Train Loss: 6.3626, Train Steps/Sec: 0.87 + 16%|█████████████████▍ | 6349/40903 [3:02:03<11:11:03, 1.17s/it][2025-04-20 19:58:39] (step=0006350) Train Loss: 6.3946, Train Steps/Sec: 0.86 + 16%|█████████████████▍ | 6374/40903 [3:02:32<11:05:13, 1.16s/it][2025-04-20 19:59:08] (step=0006375) Train Loss: 6.3853, Train Steps/Sec: 0.87 + 16%|█████████████████▌ | 6399/40903 [3:03:01<11:00:59, 1.15s/it][2025-04-20 19:59:37] (step=0006400) Train Loss: 6.3988, Train Steps/Sec: 0.86 + 16%|█████████████████▌ | 6424/40903 [3:03:30<11:19:41, 1.18s/it][2025-04-20 20:00:06] (step=0006425) Train Loss: 6.4258, Train Steps/Sec: 0.86 + 16%|█████████████████▋ | 6449/40903 [3:03:59<11:02:54, 1.15s/it][2025-04-20 20:00:35] (step=0006450) Train Loss: 6.3749, Train Steps/Sec: 0.87 + 16%|█████████████████▋ | 6474/40903 [3:04:28<10:56:24, 1.14s/it][2025-04-20 20:01:04] (step=0006475) Train Loss: 6.3733, Train Steps/Sec: 0.86 + 16%|█████████████████▊ | 6499/40903 [3:04:57<10:55:40, 1.14s/it][2025-04-20 20:01:33] (step=0006500) Train Loss: 6.4008, Train Steps/Sec: 0.86 + 16%|█████████████████▊ | 6524/40903 [3:05:26<11:12:12, 1.17s/it][2025-04-20 20:02:02] (step=0006525) Train Loss: 6.4189, Train Steps/Sec: 0.86 + 16%|█████████████████▉ | 6549/40903 [3:05:55<11:09:55, 1.17s/it][2025-04-20 20:02:31] (step=0006550) Train Loss: 6.3724, Train Steps/Sec: 0.86 + 16%|██████████████████ | 6574/40903 [3:06:24<11:03:41, 1.16s/it][2025-04-20 20:03:00] (step=0006575) Train Loss: 6.3892, Train Steps/Sec: 0.86 + 16%|██████████████████ | 6599/40903 [3:06:53<10:56:00, 1.15s/it][2025-04-20 20:03:29] (step=0006600) Train Loss: 6.4250, Train Steps/Sec: 0.86 + 16%|██████████████████▏ | 6624/40903 [3:07:22<11:17:52, 1.19s/it][2025-04-20 20:03:58] (step=0006625) Train Loss: 6.4626, Train Steps/Sec: 0.86 + 16%|██████████████████▏ | 6649/40903 [3:07:51<11:07:48, 1.17s/it][2025-04-20 20:04:27] (step=0006650) Train Loss: 6.3847, Train Steps/Sec: 0.86 + 16%|██████████████████▎ | 6674/40903 [3:08:20<10:57:13, 1.15s/it][2025-04-20 20:04:56] (step=0006675) Train Loss: 6.3932, Train Steps/Sec: 0.86 + 16%|██████████████████▎ | 6699/40903 [3:08:49<11:05:42, 1.17s/it][2025-04-20 20:05:25] (step=0006700) Train Loss: 6.3909, Train Steps/Sec: 0.86 + 16%|██████████████████▍ | 6724/40903 [3:09:18<11:10:57, 1.18s/it][2025-04-20 20:05:54] (step=0006725) Train Loss: 6.3934, Train Steps/Sec: 0.87 + 17%|██████████████████▍ | 6749/40903 [3:09:47<10:54:13, 1.15s/it][2025-04-20 20:06:23] (step=0006750) Train Loss: 6.3592, Train Steps/Sec: 0.87 + 17%|██████████████████▌ | 6774/40903 [3:10:16<10:57:22, 1.16s/it][2025-04-20 20:06:52] (step=0006775) Train Loss: 6.4392, Train Steps/Sec: 0.86 + 17%|██████████████████▌ | 6799/40903 [3:10:45<10:42:05, 1.13s/it][2025-04-20 20:07:21] (step=0006800) Train Loss: 6.3956, Train Steps/Sec: 0.86 + 17%|██████████████████▋ | 6824/40903 [3:11:14<11:04:45, 1.17s/it][2025-04-20 20:07:50] (step=0006825) Train Loss: 6.3609, Train Steps/Sec: 0.86 + 17%|██████████████████▊ | 6849/40903 [3:11:43<11:05:07, 1.17s/it][2025-04-20 20:08:19] (step=0006850) Train Loss: 6.4154, Train Steps/Sec: 0.86 + 17%|██████████████████▊ | 6874/40903 [3:12:12<10:53:43, 1.15s/it][2025-04-20 20:08:47] (step=0006875) Train Loss: 6.3889, Train Steps/Sec: 0.87 + 17%|██████████████████▉ | 6899/40903 [3:12:41<10:52:11, 1.15s/it][2025-04-20 20:09:16] (step=0006900) Train Loss: 6.3961, Train Steps/Sec: 0.86 + 17%|██████████████████▉ | 6924/40903 [3:13:10<11:02:27, 1.17s/it][2025-04-20 20:09:45] (step=0006925) Train Loss: 6.3351, Train Steps/Sec: 0.86 + 17%|███████████████████ | 6949/40903 [3:13:39<11:00:21, 1.17s/it][2025-04-20 20:10:14] (step=0006950) Train Loss: 6.3856, Train Steps/Sec: 0.86 + 17%|███████████████████ | 6974/40903 [3:14:08<10:57:03, 1.16s/it][2025-04-20 20:10:43] (step=0006975) Train Loss: 6.3620, Train Steps/Sec: 0.86 + 17%|███████████████████▏ | 6999/40903 [3:14:37<10:46:46, 1.14s/it][2025-04-20 20:11:12] (step=0007000) Train Loss: 6.3816, Train Steps/Sec: 0.86 + 17%|███████████████████▏ | 7024/40903 [3:15:06<10:59:58, 1.17s/it][2025-04-20 20:11:41] (step=0007025) Train Loss: 6.3824, Train Steps/Sec: 0.87 + 17%|███████████████████▎ | 7049/40903 [3:15:34<10:55:58, 1.16s/it][2025-04-20 20:12:10] (step=0007050) Train Loss: 6.4235, Train Steps/Sec: 0.86 + 17%|███████████████████▎ | 7074/40903 [3:16:03<10:47:39, 1.15s/it][2025-04-20 20:12:39] (step=0007075) Train Loss: 6.3622, Train Steps/Sec: 0.87 + 17%|███████████████████▍ | 7099/40903 [3:16:32<10:40:02, 1.14s/it][2025-04-20 20:13:08] (step=0007100) Train Loss: 6.3758, Train Steps/Sec: 0.86 + 17%|███████████████████▌ | 7124/40903 [3:17:01<11:10:38, 1.19s/it][2025-04-20 20:13:37] (step=0007125) Train Loss: 6.4410, Train Steps/Sec: 0.86 + 17%|███████████████████▌ | 7149/40903 [3:17:30<10:57:31, 1.17s/it][2025-04-20 20:14:06] (step=0007150) Train Loss: 6.4216, Train Steps/Sec: 0.86 + 18%|███████████████████▋ | 7174/40903 [3:17:59<10:44:12, 1.15s/it][2025-04-20 20:14:35] (step=0007175) Train Loss: 6.3759, Train Steps/Sec: 0.87 + 18%|███████████████████▋ | 7199/40903 [3:18:28<10:45:47, 1.15s/it][2025-04-20 20:15:04] (step=0007200) Train Loss: 6.4032, Train Steps/Sec: 0.87 + 18%|███████████████████▊ | 7224/40903 [3:18:57<11:03:50, 1.18s/it][2025-04-20 20:15:32] (step=0007225) Train Loss: 6.3882, Train Steps/Sec: 0.87 + 18%|███████████████████▊ | 7249/40903 [3:19:26<10:48:59, 1.16s/it][2025-04-20 20:16:01] (step=0007250) Train Loss: 6.4284, Train Steps/Sec: 0.86 + 18%|███████████████████▉ | 7274/40903 [3:19:55<10:47:17, 1.15s/it][2025-04-20 20:16:30] (step=0007275) Train Loss: 6.3438, Train Steps/Sec: 0.86 + 18%|███████████████████▉ | 7299/40903 [3:20:24<10:46:40, 1.15s/it][2025-04-20 20:16:59] (step=0007300) Train Loss: 6.3605, Train Steps/Sec: 0.86 + 18%|████████████████████ | 7324/40903 [3:20:53<10:47:58, 1.16s/it][2025-04-20 20:17:28] (step=0007325) Train Loss: 6.3738, Train Steps/Sec: 0.87 + 18%|████████████████████ | 7349/40903 [3:21:22<10:43:02, 1.15s/it][2025-04-20 20:17:57] (step=0007350) Train Loss: 6.3574, Train Steps/Sec: 0.87 + 18%|████████████████████▏ | 7374/40903 [3:21:51<10:36:57, 1.14s/it][2025-04-20 20:18:26] (step=0007375) Train Loss: 6.3941, Train Steps/Sec: 0.86 + 18%|████████████████████▎ | 7399/40903 [3:22:20<10:41:23, 1.15s/it][2025-04-20 20:18:55] (step=0007400) Train Loss: 6.3649, Train Steps/Sec: 0.86 + 18%|████████████████████▎ | 7424/40903 [3:22:49<10:48:54, 1.16s/it][2025-04-20 20:19:24] (step=0007425) Train Loss: 6.3685, Train Steps/Sec: 0.87 + 18%|████████████████████▍ | 7449/40903 [3:23:17<10:50:17, 1.17s/it][2025-04-20 20:19:53] (step=0007450) Train Loss: 6.4298, Train Steps/Sec: 0.87 + 18%|████████████████████▍ | 7474/40903 [3:23:47<10:46:21, 1.16s/it][2025-04-20 20:20:22] (step=0007475) Train Loss: 6.4004, Train Steps/Sec: 0.86 + 18%|████████████████████▌ | 7499/40903 [3:24:16<10:39:42, 1.15s/it][2025-04-20 20:20:51] (step=0007500) Train Loss: 6.3570, Train Steps/Sec: 0.86 + 18%|████████████████████▌ | 7524/40903 [3:24:45<10:45:16, 1.16s/it][2025-04-20 20:21:20] (step=0007525) Train Loss: 6.3585, Train Steps/Sec: 0.87 + 18%|████████████████████▋ | 7549/40903 [3:25:13<10:42:06, 1.16s/it][2025-04-20 20:21:49] (step=0007550) Train Loss: 6.3927, Train Steps/Sec: 0.87 + 19%|████████████████████▋ | 7574/40903 [3:25:42<10:31:20, 1.14s/it][2025-04-20 20:22:18] (step=0007575) Train Loss: 6.3808, Train Steps/Sec: 0.87 + 19%|████████████████████▊ | 7599/40903 [3:26:11<10:48:35, 1.17s/it][2025-04-20 20:22:47] (step=0007600) Train Loss: 6.3883, Train Steps/Sec: 0.86 + 19%|████████████████████▉ | 7624/40903 [3:26:40<10:48:20, 1.17s/it][2025-04-20 20:23:16] (step=0007625) Train Loss: 6.4050, Train Steps/Sec: 0.87 + 19%|████████████████████▉ | 7649/40903 [3:27:09<10:50:33, 1.17s/it][2025-04-20 20:23:45] (step=0007650) Train Loss: 6.4035, Train Steps/Sec: 0.86 + 19%|█████████████████████ | 7674/40903 [3:27:38<10:35:29, 1.15s/it][2025-04-20 20:24:14] (step=0007675) Train Loss: 6.4132, Train Steps/Sec: 0.86 + 19%|█████████████████████ | 7699/40903 [3:28:07<10:36:05, 1.15s/it][2025-04-20 20:24:43] (step=0007700) Train Loss: 6.4007, Train Steps/Sec: 0.86 + 19%|█████████████████████▏ | 7724/40903 [3:28:36<10:47:00, 1.17s/it][2025-04-20 20:25:12] (step=0007725) Train Loss: 6.3731, Train Steps/Sec: 0.87 + 19%|█████████████████████▏ | 7749/40903 [3:29:05<10:40:25, 1.16s/it][2025-04-20 20:25:41] (step=0007750) Train Loss: 6.4080, Train Steps/Sec: 0.86 + 19%|█████████████████████▎ | 7774/40903 [3:29:34<10:36:12, 1.15s/it][2025-04-20 20:26:10] (step=0007775) Train Loss: 6.3848, Train Steps/Sec: 0.87 + 19%|█████████████████████▎ | 7799/40903 [3:30:03<10:30:05, 1.14s/it][2025-04-20 20:26:39] (step=0007800) Train Loss: 6.3688, Train Steps/Sec: 0.86 + 19%|█████████████████████▍ | 7824/40903 [3:30:32<11:03:06, 1.20s/it][2025-04-20 20:27:08] (step=0007825) Train Loss: 6.4151, Train Steps/Sec: 0.86 + 19%|█████████████████████▍ | 7849/40903 [3:31:01<10:38:12, 1.16s/it][2025-04-20 20:27:37] (step=0007850) Train Loss: 6.3808, Train Steps/Sec: 0.87 + 19%|█████████████████████▌ | 7874/40903 [3:31:30<10:27:09, 1.14s/it][2025-04-20 20:28:05] (step=0007875) Train Loss: 6.3819, Train Steps/Sec: 0.87 + 19%|█████████████████████▋ | 7899/40903 [3:31:59<10:33:38, 1.15s/it][2025-04-20 20:28:35] (step=0007900) Train Loss: 6.3806, Train Steps/Sec: 0.86 + 19%|█████████████████████▋ | 7924/40903 [3:32:28<10:41:39, 1.17s/it][2025-04-20 20:29:04] (step=0007925) Train Loss: 6.4112, Train Steps/Sec: 0.86 + 19%|█████████████████████▊ | 7949/40903 [3:32:57<10:32:38, 1.15s/it][2025-04-20 20:29:33] (step=0007950) Train Loss: 6.4041, Train Steps/Sec: 0.86 + 19%|█████████████████████▊ | 7974/40903 [3:33:26<10:34:41, 1.16s/it][2025-04-20 20:30:01] (step=0007975) Train Loss: 6.3668, Train Steps/Sec: 0.86 + 20%|█████████████████████▉ | 7999/40903 [3:33:55<10:43:01, 1.17s/it][2025-04-20 20:30:31] (step=0008000) Train Loss: 6.3944, Train Steps/Sec: 0.85 +[2025-04-20 20:30:31] vision_config is None. initializing the InstructBlipVisionConfig with default values. +[2025-04-20 20:30:31] qformer_config is None. Initializing the InstructBlipQFormerConfig with default values. +[2025-04-20 20:30:31] text_config is None. Initializing the text config with default values (`OPTConfig`). +model_name_or_path:/tmp/haozhezhao/model/instructblip-flan-t5-xl +image_place_holder: +tokenizer length after expend 32102 +tokenizer length before expend 32102 +100%|████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████| 3/3 [03:06<00:00, 62.33s/it] +[2025-04-20 20:35:30] Finish Eval in 8000 steps...██████████████████████████████████████████████████████████████████████| 3/3 [03:06<00:00, 61.86s/it] +processing left-padding... +processing left-padding... +processing left-padding... +[2025-04-20 20:35:50] Saved checkpoint to checkpoint/EmptyFix_InstructBlip_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_objects/002-GPT-XL/checkpoints/0008000.pt +[2025-04-20 20:35:53] Removed old checkpoint: checkpoint/EmptyFix_InstructBlip_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_objects/002-GPT-XL/checkpoints/0004000.pt + 20%|█████████████████████▉ | 8024/40903 [3:39:47<10:57:27, 1.20s/it][2025-04-20 20:36:22] (step=0008025) Train Loss: 6.3706, Train Steps/Sec: 0.07 + 20%|██████████████████████ | 8049/40903 [3:40:16<10:31:43, 1.15s/it][2025-04-20 20:36:51] (step=0008050) Train Loss: 6.3633, Train Steps/Sec: 0.86 + 20%|██████████████████████ | 8074/40903 [3:40:45<10:38:37, 1.17s/it][2025-04-20 20:37:21] (step=0008075) Train Loss: 6.4134, Train Steps/Sec: 0.86 + 20%|██████████████████████▏ | 8099/40903 [3:41:14<10:25:30, 1.14s/it][2025-04-20 20:37:50] (step=0008100) Train Loss: 6.4166, Train Steps/Sec: 0.86 + 20%|██████████████████████▏ | 8124/40903 [3:41:43<10:37:46, 1.17s/it][2025-04-20 20:38:19] (step=0008125) Train Loss: 6.3949, Train Steps/Sec: 0.86 + 20%|██████████████████████▎ | 8149/40903 [3:42:12<10:38:52, 1.17s/it][2025-04-20 20:38:48] (step=0008150) Train Loss: 6.4234, Train Steps/Sec: 0.86 + 20%|██████████████████████▍ | 8174/40903 [3:42:41<10:35:15, 1.16s/it][2025-04-20 20:39:17] (step=0008175) Train Loss: 6.3986, Train Steps/Sec: 0.86 + 20%|██████████████████████▍ | 8199/40903 [3:43:10<10:24:08, 1.15s/it][2025-04-20 20:39:46] (step=0008200) Train Loss: 6.4047, Train Steps/Sec: 0.86 + 20%|██████████████████████▌ | 8224/40903 [3:43:39<10:34:53, 1.17s/it][2025-04-20 20:40:15] (step=0008225) Train Loss: 6.4041, Train Steps/Sec: 0.87 + 20%|██████████████████████▌ | 8249/40903 [3:44:08<10:34:22, 1.17s/it][2025-04-20 20:40:44] (step=0008250) Train Loss: 6.4457, Train Steps/Sec: 0.86 + 20%|██████████████████████▋ | 8274/40903 [3:44:37<10:30:00, 1.16s/it][2025-04-20 20:41:13] (step=0008275) Train Loss: 6.3644, Train Steps/Sec: 0.86 + 20%|██████████████████████▋ | 8299/40903 [3:45:06<10:20:20, 1.14s/it][2025-04-20 20:41:42] (step=0008300) Train Loss: 6.4133, Train Steps/Sec: 0.86 + 20%|██████████████████████▊ | 8324/40903 [3:45:35<10:38:46, 1.18s/it][2025-04-20 20:42:11] (step=0008325) Train Loss: 6.3571, Train Steps/Sec: 0.87 + 20%|██████████████████████▊ | 8349/40903 [3:46:04<10:29:10, 1.16s/it][2025-04-20 20:42:39] (step=0008350) Train Loss: 6.4061, Train Steps/Sec: 0.87 + 20%|██████████████████████▉ | 8374/40903 [3:46:33<10:29:27, 1.16s/it][2025-04-20 20:43:08] (step=0008375) Train Loss: 6.4055, Train Steps/Sec: 0.86 + 21%|██████████████████████▉ | 8399/40903 [3:47:02<10:19:35, 1.14s/it][2025-04-20 20:43:37] (step=0008400) Train Loss: 6.4204, Train Steps/Sec: 0.86 + 21%|███████████████████████ | 8424/40903 [3:47:31<10:32:16, 1.17s/it][2025-04-20 20:44:06] (step=0008425) Train Loss: 6.3955, Train Steps/Sec: 0.87 + 21%|███████████████████████▏ | 8449/40903 [3:48:00<10:27:11, 1.16s/it][2025-04-20 20:44:35] (step=0008450) Train Loss: 6.3687, Train Steps/Sec: 0.86 + 21%|███████████████████████▏ | 8474/40903 [3:48:29<10:18:53, 1.15s/it][2025-04-20 20:45:04] (step=0008475) Train Loss: 6.3981, Train Steps/Sec: 0.87 + 21%|███████████████████████▎ | 8499/40903 [3:48:58<10:09:22, 1.13s/it][2025-04-20 20:45:33] (step=0008500) Train Loss: 6.3823, Train Steps/Sec: 0.86 + 21%|███████████████████████▎ | 8524/40903 [3:49:27<10:33:47, 1.17s/it][2025-04-20 20:46:02] (step=0008525) Train Loss: 6.3724, Train Steps/Sec: 0.86 + 21%|███████████████████████▍ | 8549/40903 [3:49:56<10:23:29, 1.16s/it][2025-04-20 20:46:31] (step=0008550) Train Loss: 6.3739, Train Steps/Sec: 0.86 + 21%|███████████████████████▍ | 8574/40903 [3:50:25<10:31:36, 1.17s/it][2025-04-20 20:47:01] (step=0008575) Train Loss: 6.3688, Train Steps/Sec: 0.85 + 21%|███████████████████████▌ | 8599/40903 [3:50:54<10:22:12, 1.16s/it][2025-04-20 20:47:30] (step=0008600) Train Loss: 6.3924, Train Steps/Sec: 0.85 + 21%|███████████████████████▌ | 8624/40903 [3:51:23<10:36:28, 1.18s/it][2025-04-20 20:47:59] (step=0008625) Train Loss: 6.4045, Train Steps/Sec: 0.86 + 21%|███████████████████████▋ | 8649/40903 [3:51:59<10:21:27, 1.16s/it][2025-04-20 20:48:35] (step=0008650) Train Loss: 6.3730, Train Steps/Sec: 0.70 + 21%|███████████████████████▊ | 8674/40903 [3:52:28<10:20:10, 1.15s/it][2025-04-20 20:49:04] (step=0008675) Train Loss: 6.3526, Train Steps/Sec: 0.86 + 21%|███████████████████████▊ | 8699/40903 [3:52:57<10:14:22, 1.14s/it][2025-04-20 20:49:33] (step=0008700) Train Loss: 6.4079, Train Steps/Sec: 0.86 + 21%|███████████████████████▉ | 8724/40903 [3:53:26<10:28:55, 1.17s/it][2025-04-20 20:50:08] (step=0008725) Train Loss: 6.4042, Train Steps/Sec: 0.71 + 21%|███████████████████████▉ | 8749/40903 [3:54:14<14:16:31, 1.60s/it][2025-04-20 20:50:49] (step=0008750) Train Loss: 6.3655, Train Steps/Sec: 0.61 + 21%|████████████████████████ | 8774/40903 [3:54:43<10:19:36, 1.16s/it][2025-04-20 20:51:18] (step=0008775) Train Loss: 6.3824, Train Steps/Sec: 0.86 + 22%|████████████████████████ | 8799/40903 [3:55:12<10:09:12, 1.14s/it][2025-04-20 20:51:47] (step=0008800) Train Loss: 6.3272, Train Steps/Sec: 0.86 + 22%|████████████████████████▏ | 8824/40903 [3:55:41<10:32:40, 1.18s/it][2025-04-20 20:52:16] (step=0008825) Train Loss: 6.3610, Train Steps/Sec: 0.86 + 22%|████████████████████████▏ | 8849/40903 [3:56:17<11:26:54, 1.29s/it][2025-04-20 20:52:53] (step=0008850) Train Loss: 6.3850, Train Steps/Sec: 0.69 + 22%|████████████████████████▎ | 8874/40903 [3:56:46<10:13:37, 1.15s/it][2025-04-20 20:53:21] (step=0008875) Train Loss: 6.3465, Train Steps/Sec: 0.87 + 22%|████████████████████████▎ | 8899/40903 [3:57:22<23:04:56, 2.60s/it][2025-04-20 20:53:57] (step=0008900) Train Loss: 6.3950, Train Steps/Sec: 0.70 + 22%|████████████████████████▍ | 8924/40903 [3:57:51<10:29:11, 1.18s/it][2025-04-20 20:54:26] (step=0008925) Train Loss: 6.3569, Train Steps/Sec: 0.86 + 22%|████████████████████████▌ | 8949/40903 [3:58:20<10:13:39, 1.15s/it][2025-04-20 20:54:55] (step=0008950) Train Loss: 6.3923, Train Steps/Sec: 0.87 + 22%|████████████████████████▌ | 8974/40903 [3:58:49<10:11:45, 1.15s/it][2025-04-20 20:55:24] (step=0008975) Train Loss: 6.3860, Train Steps/Sec: 0.86 + 22%|████████████████████████▋ | 8999/40903 [3:59:18<10:17:43, 1.16s/it][2025-04-20 20:55:53] (step=0009000) Train Loss: 6.3863, Train Steps/Sec: 0.86 + 22%|████████████████████████▋ | 9024/40903 [3:59:47<10:23:44, 1.17s/it][2025-04-20 20:56:22] (step=0009025) Train Loss: 6.4070, Train Steps/Sec: 0.86 + 22%|████████████████████████▊ | 9049/40903 [4:00:16<10:26:31, 1.18s/it][2025-04-20 20:56:52] (step=0009050) Train Loss: 6.3906, Train Steps/Sec: 0.86 + 22%|████████████████████████▊ | 9074/40903 [4:00:45<10:11:31, 1.15s/it][2025-04-20 20:57:21] (step=0009075) Train Loss: 6.3658, Train Steps/Sec: 0.86 + 22%|████████████████████████▉ | 9099/40903 [4:01:14<10:10:19, 1.15s/it][2025-04-20 20:57:50] (step=0009100) Train Loss: 6.3843, Train Steps/Sec: 0.85 + 22%|████████████████████████▉ | 9124/40903 [4:01:43<10:27:15, 1.18s/it][2025-04-20 20:58:19] (step=0009125) Train Loss: 6.3939, Train Steps/Sec: 0.86 + 22%|█████████████████████████ | 9149/40903 [4:02:13<10:19:04, 1.17s/it][2025-04-20 20:58:48] (step=0009150) Train Loss: 6.3952, Train Steps/Sec: 0.86 + 22%|█████████████████████████ | 9174/40903 [4:02:42<10:12:25, 1.16s/it][2025-04-20 20:59:17] (step=0009175) Train Loss: 6.3775, Train Steps/Sec: 0.86 + 22%|█████████████████████████▏ | 9199/40903 [4:03:10<10:01:40, 1.14s/it][2025-04-20 20:59:46] (step=0009200) Train Loss: 6.4193, Train Steps/Sec: 0.86 + 23%|█████████████████████████▎ | 9224/40903 [4:03:39<10:18:33, 1.17s/it][2025-04-20 21:00:15] (step=0009225) Train Loss: 6.3914, Train Steps/Sec: 0.87 + 23%|█████████████████████████▎ | 9249/40903 [4:04:08<10:07:22, 1.15s/it][2025-04-20 21:00:44] (step=0009250) Train Loss: 6.3672, Train Steps/Sec: 0.86 + 23%|█████████████████████████▍ | 9274/40903 [4:04:37<10:05:54, 1.15s/it][2025-04-20 21:01:13] (step=0009275) Train Loss: 6.3718, Train Steps/Sec: 0.86 + 23%|█████████████████████████▋ | 9299/40903 [4:05:06<9:57:13, 1.13s/it][2025-04-20 21:01:42] (step=0009300) Train Loss: 6.3824, Train Steps/Sec: 0.86 + 23%|█████████████████████████▌ | 9324/40903 [4:05:35<10:24:05, 1.19s/it][2025-04-20 21:02:11] (step=0009325) Train Loss: 6.3804, Train Steps/Sec: 0.86 + 23%|█████████████████████████▌ | 9349/40903 [4:06:04<10:17:56, 1.18s/it][2025-04-20 21:02:40] (step=0009350) Train Loss: 6.4055, Train Steps/Sec: 0.86 + 23%|█████████████████████████▋ | 9374/40903 [4:06:33<10:16:18, 1.17s/it][2025-04-20 21:03:09] (step=0009375) Train Loss: 6.3529, Train Steps/Sec: 0.86 + 23%|█████████████████████████▋ | 9399/40903 [4:07:02<10:01:21, 1.15s/it][2025-04-20 21:03:38] (step=0009400) Train Loss: 6.3810, Train Steps/Sec: 0.86 + 23%|█████████████████████████▊ | 9424/40903 [4:07:32<10:20:05, 1.18s/it][2025-04-20 21:04:07] (step=0009425) Train Loss: 6.3819, Train Steps/Sec: 0.86 + 23%|█████████████████████████▊ | 9449/40903 [4:08:00<10:06:17, 1.16s/it][2025-04-20 21:04:36] (step=0009450) Train Loss: 6.3966, Train Steps/Sec: 0.86 + 23%|█████████████████████████▉ | 9474/40903 [4:08:29<10:01:30, 1.15s/it][2025-04-20 21:05:05] (step=0009475) Train Loss: 6.3731, Train Steps/Sec: 0.87 + 23%|██████████████████████████ | 9499/40903 [4:08:58<10:06:37, 1.16s/it][2025-04-20 21:05:34] (step=0009500) Train Loss: 6.3268, Train Steps/Sec: 0.86 + 23%|██████████████████████████ | 9524/40903 [4:09:27<10:11:36, 1.17s/it][2025-04-20 21:06:03] (step=0009525) Train Loss: 6.4038, Train Steps/Sec: 0.86 + 23%|██████████████████████████▏ | 9549/40903 [4:09:56<10:07:18, 1.16s/it][2025-04-20 21:06:32] (step=0009550) Train Loss: 6.4151, Train Steps/Sec: 0.86 + 23%|██████████████████████████▏ | 9574/40903 [4:10:26<10:12:57, 1.17s/it][2025-04-20 21:07:01] (step=0009575) Train Loss: 6.3854, Train Steps/Sec: 0.85 + 23%|██████████████████████████▌ | 9599/40903 [4:10:55<9:57:43, 1.15s/it][2025-04-20 21:07:30] (step=0009600) Train Loss: 6.3520, Train Steps/Sec: 0.86 + 24%|██████████████████████████▎ | 9624/40903 [4:11:24<10:08:19, 1.17s/it][2025-04-20 21:07:59] (step=0009625) Train Loss: 6.3630, Train Steps/Sec: 0.86 + 24%|██████████████████████████▍ | 9649/40903 [4:11:53<10:07:07, 1.17s/it][2025-04-20 21:08:28] (step=0009650) Train Loss: 6.3795, Train Steps/Sec: 0.86 + 24%|██████████████████████████▍ | 9674/40903 [4:12:22<10:03:32, 1.16s/it][2025-04-20 21:08:57] (step=0009675) Train Loss: 6.4091, Train Steps/Sec: 0.86 + 24%|██████████████████████████▊ | 9699/40903 [4:12:51<9:50:32, 1.14s/it][2025-04-20 21:09:27] (step=0009700) Train Loss: 6.3828, Train Steps/Sec: 0.86 + 24%|██████████████████████████▋ | 9724/40903 [4:13:20<10:10:14, 1.17s/it][2025-04-20 21:09:55] (step=0009725) Train Loss: 6.3852, Train Steps/Sec: 0.86 + 24%|██████████████████████████▋ | 9749/40903 [4:13:49<10:00:30, 1.16s/it][2025-04-20 21:10:24] (step=0009750) Train Loss: 6.3458, Train Steps/Sec: 0.86 + 24%|███████████████████████████ | 9774/40903 [4:14:18<9:58:48, 1.15s/it][2025-04-20 21:10:53] (step=0009775) Train Loss: 6.3804, Train Steps/Sec: 0.86 + 24%|███████████████████████████ | 9799/40903 [4:14:47<9:52:47, 1.14s/it][2025-04-20 21:11:23] (step=0009800) Train Loss: 6.3956, Train Steps/Sec: 0.86 + 24%|██████████████████████████▉ | 9824/40903 [4:15:16<10:05:45, 1.17s/it][2025-04-20 21:11:52] (step=0009825) Train Loss: 6.3620, Train Steps/Sec: 0.86 + 24%|███████████████████████████▏ | 9849/40903 [4:15:45<9:58:49, 1.16s/it][2025-04-20 21:12:20] (step=0009850) Train Loss: 6.3812, Train Steps/Sec: 0.87 + 24%|███████████████████████████▎ | 9874/40903 [4:16:14<9:56:44, 1.15s/it][2025-04-20 21:12:50] (step=0009875) Train Loss: 6.3835, Train Steps/Sec: 0.86 + 24%|███████████████████████████▎ | 9899/40903 [4:16:43<9:51:58, 1.15s/it][2025-04-20 21:13:19] (step=0009900) Train Loss: 6.3651, Train Steps/Sec: 0.86 + 24%|███████████████████████████▏ | 9924/40903 [4:17:12<10:03:29, 1.17s/it][2025-04-20 21:13:48] (step=0009925) Train Loss: 6.3761, Train Steps/Sec: 0.87 + 24%|███████████████████████████▏ | 9949/40903 [4:17:41<10:00:53, 1.16s/it][2025-04-20 21:14:17] (step=0009950) Train Loss: 6.3658, Train Steps/Sec: 0.87 + 24%|███████████████████████████▌ | 9974/40903 [4:18:10<9:51:49, 1.15s/it][2025-04-20 21:14:46] (step=0009975) Train Loss: 6.3681, Train Steps/Sec: 0.86 + 24%|███████████████████████████▌ | 9999/40903 [4:18:39<9:45:54, 1.14s/it][2025-04-20 21:15:14] (step=0010000) Train Loss: 6.3482, Train Steps/Sec: 0.86 +[2025-04-20 21:15:15] vision_config is None. initializing the InstructBlipVisionConfig with default values. +[2025-04-20 21:15:15] qformer_config is None. Initializing the InstructBlipQFormerConfig with default values. +[2025-04-20 21:15:15] text_config is None. Initializing the text config with default values (`OPTConfig`). +model_name_or_path:/tmp/haozhezhao/model/instructblip-flan-t5-xl +image_place_holder: +tokenizer length after expend 32102 +tokenizer length before expend 32102 +100%|████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████| 3/3 [03:06<00:00, 62.12s/it] +[2025-04-20 21:20:12] Finish Eval in 10000 steps...█████████████████████████████████████████████████████████████████████| 3/3 [03:05<00:00, 61.69s/it] +processing left-padding... +processing left-padding... +processing left-padding... +[2025-04-20 21:20:33] Saved checkpoint to checkpoint/EmptyFix_InstructBlip_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_objects/002-GPT-XL/checkpoints/0010000.pt +[2025-04-20 21:20:36] Removed old checkpoint: checkpoint/EmptyFix_InstructBlip_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_objects/002-GPT-XL/checkpoints/0006000.pt + 25%|███████████████████████████▏ | 10024/40903 [4:24:30<10:25:00, 1.21s/it][2025-04-20 21:21:06] (step=0010025) Train Loss: 6.3416, Train Steps/Sec: 0.07 + 25%|███████████████████████████▎ | 10049/40903 [4:24:59<10:01:45, 1.17s/it][2025-04-20 21:21:35] (step=0010050) Train Loss: 6.4356, Train Steps/Sec: 0.86 + 25%|███████████████████████████▌ | 10074/40903 [4:25:28<9:50:13, 1.15s/it][2025-04-20 21:22:04] (step=0010075) Train Loss: 6.3961, Train Steps/Sec: 0.86 + 25%|███████████████████████████▋ | 10099/40903 [4:25:57<9:51:00, 1.15s/it][2025-04-20 21:22:33] (step=0010100) Train Loss: 6.3849, Train Steps/Sec: 0.86 + 25%|███████████████████████████▍ | 10124/40903 [4:26:26<10:00:15, 1.17s/it][2025-04-20 21:23:02] (step=0010125) Train Loss: 6.3769, Train Steps/Sec: 0.86 + 25%|███████████████████████████▊ | 10149/40903 [4:26:55<9:51:02, 1.15s/it][2025-04-20 21:23:31] (step=0010150) Train Loss: 6.3623, Train Steps/Sec: 0.86 + 25%|███████████████████████████▊ | 10174/40903 [4:27:24<9:55:09, 1.16s/it][2025-04-20 21:24:00] (step=0010175) Train Loss: 6.3687, Train Steps/Sec: 0.87 + 25%|███████████████████████████▉ | 10199/40903 [4:27:53<9:51:18, 1.16s/it][2025-04-20 21:24:29] (step=0010200) Train Loss: 6.4005, Train Steps/Sec: 0.86 + 25%|███████████████████████████▋ | 10224/40903 [4:28:23<10:05:57, 1.19s/it][2025-04-20 21:24:58] (step=0010225) Train Loss: 6.3519, Train Steps/Sec: 0.86 + 25%|████████████████████████████ | 10249/40903 [4:28:51<9:57:34, 1.17s/it][2025-04-20 21:25:27] (step=0010250) Train Loss: 6.3230, Train Steps/Sec: 0.86 + 25%|████████████████████████████▏ | 10274/40903 [4:29:21<9:47:24, 1.15s/it][2025-04-20 21:25:56] (step=0010275) Train Loss: 6.3681, Train Steps/Sec: 0.86 + 25%|████████████████████████████▏ | 10299/40903 [4:29:50<9:46:09, 1.15s/it][2025-04-20 21:26:25] (step=0010300) Train Loss: 6.3738, Train Steps/Sec: 0.86 + 25%|████████████████████████████ | 10324/40903 [4:30:19<10:05:09, 1.19s/it][2025-04-20 21:26:54] (step=0010325) Train Loss: 6.3513, Train Steps/Sec: 0.86 + 25%|████████████████████████████▎ | 10349/40903 [4:30:48<9:46:05, 1.15s/it][2025-04-20 21:27:23] (step=0010350) Train Loss: 6.3873, Train Steps/Sec: 0.87 + 25%|████████████████████████████▍ | 10374/40903 [4:31:17<9:54:20, 1.17s/it][2025-04-20 21:27:52] (step=0010375) Train Loss: 6.3754, Train Steps/Sec: 0.86 + 25%|████████████████████████████▍ | 10399/40903 [4:31:46<9:44:12, 1.15s/it][2025-04-20 21:28:21] (step=0010400) Train Loss: 6.4202, Train Steps/Sec: 0.86 + 25%|████████████████████████████▌ | 10424/40903 [4:32:15<9:52:05, 1.17s/it][2025-04-20 21:28:50] (step=0010425) Train Loss: 6.3954, Train Steps/Sec: 0.86 + 26%|████████████████████████████▌ | 10449/40903 [4:32:44<9:54:39, 1.17s/it][2025-04-20 21:29:20] (step=0010450) Train Loss: 6.4095, Train Steps/Sec: 0.85 + 26%|████████████████████████████▋ | 10474/40903 [4:33:13<9:45:00, 1.15s/it][2025-04-20 21:29:49] (step=0010475) Train Loss: 6.4071, Train Steps/Sec: 0.86 + 26%|████████████████████████████▋ | 10499/40903 [4:33:42<9:38:35, 1.14s/it][2025-04-20 21:30:18] (step=0010500) Train Loss: 6.3886, Train Steps/Sec: 0.85 + 26%|████████████████████████████▊ | 10524/40903 [4:34:11<9:50:16, 1.17s/it][2025-04-20 21:30:47] (step=0010525) Train Loss: 6.3823, Train Steps/Sec: 0.86 + 26%|████████████████████████████▉ | 10549/40903 [4:34:40<9:53:55, 1.17s/it][2025-04-20 21:31:16] (step=0010550) Train Loss: 6.3945, Train Steps/Sec: 0.86 + 26%|████████████████████████████▉ | 10574/40903 [4:35:09<9:40:54, 1.15s/it][2025-04-20 21:31:45] (step=0010575) Train Loss: 6.4118, Train Steps/Sec: 0.87 + 26%|█████████████████████████████ | 10599/40903 [4:35:39<9:45:29, 1.16s/it][2025-04-20 21:32:14] (step=0010600) Train Loss: 6.3712, Train Steps/Sec: 0.85 + 26%|█████████████████████████████ | 10624/40903 [4:36:08<9:59:26, 1.19s/it][2025-04-20 21:32:43] (step=0010625) Train Loss: 6.4013, Train Steps/Sec: 0.86 + 26%|█████████████████████████████▏ | 10649/40903 [4:36:37<9:51:11, 1.17s/it][2025-04-20 21:33:12] (step=0010650) Train Loss: 6.3921, Train Steps/Sec: 0.86 + 26%|████████████████████████████▉ | 10674/40903 [4:37:12<10:35:20, 1.26s/it][2025-04-20 21:33:48] (step=0010675) Train Loss: 6.3912, Train Steps/Sec: 0.70 + 26%|█████████████████████████████▎ | 10699/40903 [4:37:41<9:39:17, 1.15s/it][2025-04-20 21:34:17] (step=0010700) Train Loss: 6.3835, Train Steps/Sec: 0.85 + 26%|█████████████████████████████▎ | 10724/40903 [4:38:10<9:52:45, 1.18s/it][2025-04-20 21:34:46] (step=0010725) Train Loss: 6.3389, Train Steps/Sec: 0.87 + 26%|█████████████████████████████▍ | 10749/40903 [4:38:40<9:47:59, 1.17s/it][2025-04-20 21:35:15] (step=0010750) Train Loss: 6.3830, Train Steps/Sec: 0.86 + 26%|█████████████████████████████▌ | 10774/40903 [4:39:09<9:34:21, 1.14s/it][2025-04-20 21:35:44] (step=0010775) Train Loss: 6.3867, Train Steps/Sec: 0.86 + 26%|█████████████████████████████▌ | 10799/40903 [4:39:38<9:37:05, 1.15s/it][2025-04-20 21:36:13] (step=0010800) Train Loss: 6.3722, Train Steps/Sec: 0.86 + 26%|█████████████████████████████▋ | 10824/40903 [4:40:07<9:56:49, 1.19s/it][2025-04-20 21:36:42] (step=0010825) Train Loss: 6.4261, Train Steps/Sec: 0.86 + 27%|█████████████████████████████▋ | 10849/40903 [4:40:35<9:40:01, 1.16s/it][2025-04-20 21:37:11] (step=0010850) Train Loss: 6.3747, Train Steps/Sec: 0.87 + 27%|█████████████████████████████▊ | 10874/40903 [4:41:05<9:45:50, 1.17s/it][2025-04-20 21:37:40] (step=0010875) Train Loss: 6.3466, Train Steps/Sec: 0.86 + 27%|█████████████████████████████▊ | 10899/40903 [4:41:34<9:30:28, 1.14s/it][2025-04-20 21:38:09] (step=0010900) Train Loss: 6.4164, Train Steps/Sec: 0.86 + 27%|█████████████████████████████▉ | 10924/40903 [4:42:03<9:47:57, 1.18s/it][2025-04-20 21:38:38] (step=0010925) Train Loss: 6.4021, Train Steps/Sec: 0.86 + 27%|█████████████████████████████▉ | 10949/40903 [4:42:32<9:43:34, 1.17s/it][2025-04-20 21:39:08] (step=0010950) Train Loss: 6.3983, Train Steps/Sec: 0.86 + 27%|██████████████████████████████ | 10974/40903 [4:43:01<9:33:52, 1.15s/it][2025-04-20 21:39:37] (step=0010975) Train Loss: 6.3489, Train Steps/Sec: 0.86 + 27%|██████████████████████████████ | 10999/40903 [4:43:30<9:39:45, 1.16s/it][2025-04-20 21:40:06] (step=0011000) Train Loss: 6.4121, Train Steps/Sec: 0.86 + 27%|██████████████████████████████▏ | 11024/40903 [4:43:59<9:56:28, 1.20s/it][2025-04-20 21:40:35] (step=0011025) Train Loss: 6.3742, Train Steps/Sec: 0.86 + 27%|██████████████████████████████▎ | 11049/40903 [4:44:28<9:42:51, 1.17s/it][2025-04-20 21:41:04] (step=0011050) Train Loss: 6.3350, Train Steps/Sec: 0.86 + 27%|██████████████████████████████▎ | 11074/40903 [4:44:57<9:41:54, 1.17s/it][2025-04-20 21:41:33] (step=0011075) Train Loss: 6.3824, Train Steps/Sec: 0.86 + 27%|██████████████████████████████▍ | 11099/40903 [4:45:26<9:30:44, 1.15s/it][2025-04-20 21:42:02] (step=0011100) Train Loss: 6.3826, Train Steps/Sec: 0.86 + 27%|██████████████████████████████▍ | 11124/40903 [4:45:56<9:52:19, 1.19s/it][2025-04-20 21:42:31] (step=0011125) Train Loss: 6.3944, Train Steps/Sec: 0.86 + 27%|██████████████████████████████▌ | 11149/40903 [4:46:25<9:33:18, 1.16s/it][2025-04-20 21:43:00] (step=0011150) Train Loss: 6.3676, Train Steps/Sec: 0.86 + 27%|██████████████████████████████▌ | 11174/40903 [4:46:54<9:39:54, 1.17s/it][2025-04-20 21:43:29] (step=0011175) Train Loss: 6.4154, Train Steps/Sec: 0.86 + 27%|██████████████████████████████▋ | 11199/40903 [4:47:22<9:23:33, 1.14s/it][2025-04-20 21:43:58] (step=0011200) Train Loss: 6.3902, Train Steps/Sec: 0.86 + 27%|██████████████████████████████▋ | 11224/40903 [4:47:51<9:37:20, 1.17s/it][2025-04-20 21:44:27] (step=0011225) Train Loss: 6.3435, Train Steps/Sec: 0.87 + 28%|██████████████████████████████▊ | 11249/40903 [4:48:20<9:43:50, 1.18s/it][2025-04-20 21:44:56] (step=0011250) Train Loss: 6.3809, Train Steps/Sec: 0.86 + 28%|██████████████████████████████▊ | 11274/40903 [4:48:49<9:20:21, 1.13s/it][2025-04-20 21:45:25] (step=0011275) Train Loss: 6.3705, Train Steps/Sec: 0.86 + 28%|██████████████████████████████▉ | 11299/40903 [4:49:18<9:24:44, 1.14s/it][2025-04-20 21:45:54] (step=0011300) Train Loss: 6.3579, Train Steps/Sec: 0.86 + 28%|███████████████████████████████ | 11324/40903 [4:49:48<9:43:51, 1.18s/it][2025-04-20 21:46:23] (step=0011325) Train Loss: 6.3900, Train Steps/Sec: 0.86 + 28%|███████████████████████████████ | 11349/40903 [4:50:17<9:26:30, 1.15s/it][2025-04-20 21:46:52] (step=0011350) Train Loss: 6.3850, Train Steps/Sec: 0.86 + 28%|███████████████████████████████▏ | 11374/40903 [4:50:45<9:21:57, 1.14s/it][2025-04-20 21:47:21] (step=0011375) Train Loss: 6.3749, Train Steps/Sec: 0.86 + 28%|███████████████████████████████▏ | 11399/40903 [4:51:14<9:22:00, 1.14s/it][2025-04-20 21:47:50] (step=0011400) Train Loss: 6.3975, Train Steps/Sec: 0.86 + 28%|███████████████████████████████▎ | 11424/40903 [4:51:44<9:44:57, 1.19s/it][2025-04-20 21:48:19] (step=0011425) Train Loss: 6.3925, Train Steps/Sec: 0.86 + 28%|███████████████████████████████▎ | 11449/40903 [4:52:13<9:32:08, 1.17s/it][2025-04-20 21:48:48] (step=0011450) Train Loss: 6.3605, Train Steps/Sec: 0.86 + 28%|███████████████████████████████▍ | 11474/40903 [4:52:42<9:16:37, 1.13s/it][2025-04-20 21:49:17] (step=0011475) Train Loss: 6.3957, Train Steps/Sec: 0.87 + 28%|███████████████████████████████▍ | 11499/40903 [4:53:10<9:18:57, 1.14s/it][2025-04-20 21:49:46] (step=0011500) Train Loss: 6.3514, Train Steps/Sec: 0.86 + 28%|███████████████████████████████▌ | 11524/40903 [4:53:39<9:33:47, 1.17s/it][2025-04-20 21:50:15] (step=0011525) Train Loss: 6.3738, Train Steps/Sec: 0.87 + 28%|███████████████████████████████▌ | 11549/40903 [4:54:08<9:28:18, 1.16s/it][2025-04-20 21:50:44] (step=0011550) Train Loss: 6.3985, Train Steps/Sec: 0.86 + 28%|███████████████████████████████▋ | 11574/40903 [4:54:37<9:25:09, 1.16s/it][2025-04-20 21:51:13] (step=0011575) Train Loss: 6.4023, Train Steps/Sec: 0.86 + 28%|███████████████████████████████▊ | 11599/40903 [4:55:06<9:16:43, 1.14s/it][2025-04-20 21:51:42] (step=0011600) Train Loss: 6.3330, Train Steps/Sec: 0.86 + 28%|███████████████████████████████▊ | 11624/40903 [4:55:35<9:34:07, 1.18s/it][2025-04-20 21:52:11] (step=0011625) Train Loss: 6.3534, Train Steps/Sec: 0.87 + 28%|███████████████████████████████▉ | 11649/40903 [4:56:04<9:28:15, 1.17s/it][2025-04-20 21:52:40] (step=0011650) Train Loss: 6.3525, Train Steps/Sec: 0.87 + 29%|███████████████████████████████▉ | 11674/40903 [4:56:33<9:14:29, 1.14s/it][2025-04-20 21:53:09] (step=0011675) Train Loss: 6.3878, Train Steps/Sec: 0.87 + 29%|████████████████████████████████ | 11699/40903 [4:57:02<9:12:13, 1.13s/it][2025-04-20 21:53:38] (step=0011700) Train Loss: 6.3886, Train Steps/Sec: 0.86 + 29%|████████████████████████████████ | 11724/40903 [4:57:31<9:32:57, 1.18s/it][2025-04-20 21:54:07] (step=0011725) Train Loss: 6.3759, Train Steps/Sec: 0.86 + 29%|████████████████████████████████▏ | 11749/40903 [4:58:00<9:30:07, 1.17s/it][2025-04-20 21:54:36] (step=0011750) Train Loss: 6.3945, Train Steps/Sec: 0.86 + 29%|████████████████████████████████▏ | 11774/40903 [4:58:29<9:25:38, 1.17s/it][2025-04-20 21:55:05] (step=0011775) Train Loss: 6.3888, Train Steps/Sec: 0.87 + 29%|████████████████████████████████▎ | 11799/40903 [4:58:58<9:10:07, 1.13s/it][2025-04-20 21:55:34] (step=0011800) Train Loss: 6.3875, Train Steps/Sec: 0.86 + 29%|████████████████████████████████▍ | 11824/40903 [4:59:27<9:26:43, 1.17s/it][2025-04-20 21:56:03] (step=0011825) Train Loss: 6.3748, Train Steps/Sec: 0.86 + 29%|████████████████████████████████▍ | 11849/40903 [4:59:56<9:28:11, 1.17s/it][2025-04-20 21:56:32] (step=0011850) Train Loss: 6.4215, Train Steps/Sec: 0.87 + 29%|████████████████████████████████▌ | 11874/40903 [5:00:25<9:12:23, 1.14s/it][2025-04-20 21:57:01] (step=0011875) Train Loss: 6.3591, Train Steps/Sec: 0.87 + 29%|████████████████████████████████▌ | 11899/40903 [5:00:55<9:21:31, 1.16s/it][2025-04-20 21:57:30] (step=0011900) Train Loss: 6.3571, Train Steps/Sec: 0.85 + 29%|████████████████████████████████▋ | 11924/40903 [5:01:24<9:33:09, 1.19s/it][2025-04-20 21:57:59] (step=0011925) Train Loss: 6.3885, Train Steps/Sec: 0.86 + 29%|████████████████████████████████▋ | 11949/40903 [5:01:53<9:11:36, 1.14s/it][2025-04-20 21:58:28] (step=0011950) Train Loss: 6.4215, Train Steps/Sec: 0.86 + 29%|████████████████████████████████▊ | 11974/40903 [5:02:22<9:16:54, 1.16s/it][2025-04-20 21:58:57] (step=0011975) Train Loss: 6.3789, Train Steps/Sec: 0.87 + 29%|████████████████████████████████▊ | 11999/40903 [5:02:51<9:24:20, 1.17s/it][2025-04-20 21:59:26] (step=0012000) Train Loss: 6.3799, Train Steps/Sec: 0.85 +[2025-04-20 21:59:26] vision_config is None. initializing the InstructBlipVisionConfig with default values. +[2025-04-20 21:59:26] qformer_config is None. Initializing the InstructBlipQFormerConfig with default values. +[2025-04-20 21:59:26] text_config is None. Initializing the text config with default values (`OPTConfig`). +model_name_or_path:/tmp/haozhezhao/model/instructblip-flan-t5-xl +image_place_holder: +tokenizer length after expend 32102 +tokenizer length before expend 32102 +100%|████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████| 3/3 [03:05<00:00, 61.96s/it] +[2025-04-20 22:04:23] Finish Eval in 12000 steps...█████████████████████████████████████████████████████████████████████| 3/3 [03:05<00:00, 61.51s/it] +processing left-padding... +processing left-padding... +processing left-padding... +[2025-04-20 22:04:45] Saved checkpoint to checkpoint/EmptyFix_InstructBlip_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_objects/002-GPT-XL/checkpoints/0012000.pt +[2025-04-20 22:04:47] Removed old checkpoint: checkpoint/EmptyFix_InstructBlip_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_objects/002-GPT-XL/checkpoints/0008000.pt + 29%|████████████████████████████████▉ | 12024/40903 [5:08:41<9:44:13, 1.21s/it][2025-04-20 22:05:17] (step=0012025) Train Loss: 6.3647, Train Steps/Sec: 0.07 + 29%|████████████████████████████████▉ | 12049/40903 [5:09:10<9:13:02, 1.15s/it][2025-04-20 22:05:46] (step=0012050) Train Loss: 6.3851, Train Steps/Sec: 0.86 + 30%|█████████████████████████████████ | 12074/40903 [5:09:39<9:07:39, 1.14s/it][2025-04-20 22:06:15] (step=0012075) Train Loss: 6.3810, Train Steps/Sec: 0.86 + 30%|█████████████████████████████████▏ | 12099/40903 [5:10:08<9:05:40, 1.14s/it][2025-04-20 22:06:44] (step=0012100) Train Loss: 6.3371, Train Steps/Sec: 0.86 + 30%|█████████████████████████████████▏ | 12124/40903 [5:10:37<9:24:24, 1.18s/it][2025-04-20 22:07:12] (step=0012125) Train Loss: 6.3599, Train Steps/Sec: 0.86 + 30%|█████████████████████████████████▎ | 12149/40903 [5:11:06<9:20:13, 1.17s/it][2025-04-20 22:07:41] (step=0012150) Train Loss: 6.3541, Train Steps/Sec: 0.87 + 30%|█████████████████████████████████▎ | 12174/40903 [5:11:35<9:14:37, 1.16s/it][2025-04-20 22:08:10] (step=0012175) Train Loss: 6.3631, Train Steps/Sec: 0.86 + 30%|█████████████████████████████████▍ | 12199/40903 [5:12:04<9:09:39, 1.15s/it][2025-04-20 22:08:40] (step=0012200) Train Loss: 6.3736, Train Steps/Sec: 0.86 + 30%|█████████████████████████████████▍ | 12224/40903 [5:12:33<9:19:43, 1.17s/it][2025-04-20 22:09:09] (step=0012225) Train Loss: 6.4124, Train Steps/Sec: 0.86 + 30%|█████████████████████████████████▌ | 12249/40903 [5:13:02<9:21:16, 1.18s/it][2025-04-20 22:09:38] (step=0012250) Train Loss: 6.3955, Train Steps/Sec: 0.86 + 30%|█████████████████████████████████▌ | 12274/40903 [5:13:31<9:04:21, 1.14s/it][2025-04-20 22:10:07] (step=0012275) Train Loss: 6.3887, Train Steps/Sec: 0.86 + 30%|█████████████████████████████████▋ | 12299/40903 [5:14:00<9:06:48, 1.15s/it][2025-04-20 22:10:36] (step=0012300) Train Loss: 6.4028, Train Steps/Sec: 0.85 + 30%|█████████████████████████████████▋ | 12324/40903 [5:14:29<9:16:02, 1.17s/it][2025-04-20 22:11:05] (step=0012325) Train Loss: 6.3651, Train Steps/Sec: 0.86 + 30%|█████████████████████████████████▊ | 12349/40903 [5:14:58<9:11:59, 1.16s/it][2025-04-20 22:11:34] (step=0012350) Train Loss: 6.3376, Train Steps/Sec: 0.86 + 30%|█████████████████████████████████▉ | 12374/40903 [5:15:27<9:05:37, 1.15s/it][2025-04-20 22:12:03] (step=0012375) Train Loss: 6.3763, Train Steps/Sec: 0.86 + 30%|█████████████████████████████████▉ | 12399/40903 [5:15:56<9:12:10, 1.16s/it][2025-04-20 22:12:32] (step=0012400) Train Loss: 6.3825, Train Steps/Sec: 0.86 + 30%|██████████████████████████████████ | 12424/40903 [5:16:25<9:24:05, 1.19s/it][2025-04-20 22:13:01] (step=0012425) Train Loss: 6.4091, Train Steps/Sec: 0.86 + 30%|██████████████████████████████████ | 12449/40903 [5:16:54<9:06:25, 1.15s/it][2025-04-20 22:13:30] (step=0012450) Train Loss: 6.3682, Train Steps/Sec: 0.87 + 30%|██████████████████████████████████▏ | 12474/40903 [5:17:23<9:03:50, 1.15s/it][2025-04-20 22:13:59] (step=0012475) Train Loss: 6.3610, Train Steps/Sec: 0.86 + 31%|██████████████████████████████████▏ | 12499/40903 [5:17:53<9:08:37, 1.16s/it][2025-04-20 22:14:28] (step=0012500) Train Loss: 6.3800, Train Steps/Sec: 0.85 + 31%|██████████████████████████████████▎ | 12524/40903 [5:18:22<9:14:12, 1.17s/it][2025-04-20 22:14:57] (step=0012525) Train Loss: 6.4002, Train Steps/Sec: 0.87 + 31%|██████████████████████████████████▎ | 12549/40903 [5:18:51<9:17:12, 1.18s/it][2025-04-20 22:15:27] (step=0012550) Train Loss: 6.4250, Train Steps/Sec: 0.85 + 31%|██████████████████████████████████▍ | 12574/40903 [5:19:20<9:12:19, 1.17s/it][2025-04-20 22:15:56] (step=0012575) Train Loss: 6.3958, Train Steps/Sec: 0.86 + 31%|██████████████████████████████████▍ | 12599/40903 [5:19:49<9:04:23, 1.15s/it][2025-04-20 22:16:25] (step=0012600) Train Loss: 6.4005, Train Steps/Sec: 0.86 + 31%|██████████████████████████████████▌ | 12624/40903 [5:20:18<9:14:24, 1.18s/it][2025-04-20 22:16:54] (step=0012625) Train Loss: 6.4057, Train Steps/Sec: 0.86 + 31%|██████████████████████████████████▋ | 12649/40903 [5:20:47<9:06:05, 1.16s/it][2025-04-20 22:17:23] (step=0012650) Train Loss: 6.3585, Train Steps/Sec: 0.86 + 31%|██████████████████████████████████▋ | 12674/40903 [5:21:16<9:03:25, 1.16s/it][2025-04-20 22:17:52] (step=0012675) Train Loss: 6.3624, Train Steps/Sec: 0.86 + 31%|██████████████████████████████████▊ | 12699/40903 [5:21:45<8:57:07, 1.14s/it][2025-04-20 22:18:21] (step=0012700) Train Loss: 6.3976, Train Steps/Sec: 0.86 + 31%|██████████████████████████████████▊ | 12724/40903 [5:22:14<9:08:40, 1.17s/it][2025-04-20 22:18:50] (step=0012725) Train Loss: 6.3468, Train Steps/Sec: 0.86 + 31%|██████████████████████████████████▉ | 12749/40903 [5:22:43<9:02:59, 1.16s/it][2025-04-20 22:19:19] (step=0012750) Train Loss: 6.3925, Train Steps/Sec: 0.87 + 31%|██████████████████████████████████▉ | 12774/40903 [5:23:12<9:07:20, 1.17s/it][2025-04-20 22:19:48] (step=0012775) Train Loss: 6.3538, Train Steps/Sec: 0.86 + 31%|███████████████████████████████████ | 12799/40903 [5:23:41<9:02:14, 1.16s/it][2025-04-20 22:20:17] (step=0012800) Train Loss: 6.4290, Train Steps/Sec: 0.86 + 31%|███████████████████████████████████ | 12824/40903 [5:24:10<9:08:02, 1.17s/it][2025-04-20 22:20:46] (step=0012825) Train Loss: 6.4017, Train Steps/Sec: 0.86 + 31%|███████████████████████████████████▏ | 12849/40903 [5:24:39<9:06:50, 1.17s/it][2025-04-20 22:21:15] (step=0012850) Train Loss: 6.4120, Train Steps/Sec: 0.86 + 31%|███████████████████████████████████▎ | 12874/40903 [5:25:08<8:55:31, 1.15s/it][2025-04-20 22:21:44] (step=0012875) Train Loss: 6.4340, Train Steps/Sec: 0.86 + 32%|███████████████████████████████████▎ | 12899/40903 [5:25:38<8:56:17, 1.15s/it][2025-04-20 22:22:13] (step=0012900) Train Loss: 6.4128, Train Steps/Sec: 0.85 + 32%|███████████████████████████████████▍ | 12924/40903 [5:26:07<9:11:40, 1.18s/it][2025-04-20 22:22:42] (step=0012925) Train Loss: 6.3787, Train Steps/Sec: 0.86 + 32%|███████████████████████████████████▍ | 12949/40903 [5:26:36<8:58:20, 1.16s/it][2025-04-20 22:23:11] (step=0012950) Train Loss: 6.4141, Train Steps/Sec: 0.87 + 32%|███████████████████████████████████▌ | 12974/40903 [5:27:05<9:04:32, 1.17s/it][2025-04-20 22:23:40] (step=0012975) Train Loss: 6.4113, Train Steps/Sec: 0.86 + 32%|███████████████████████████████████▌ | 12999/40903 [5:27:33<8:55:10, 1.15s/it][2025-04-20 22:24:09] (step=0013000) Train Loss: 6.3888, Train Steps/Sec: 0.86 + 32%|███████████████████████████████████▋ | 13024/40903 [5:28:03<9:06:52, 1.18s/it][2025-04-20 22:24:38] (step=0013025) Train Loss: 6.3751, Train Steps/Sec: 0.87 + 32%|███████████████████████████████████▋ | 13049/40903 [5:28:32<8:59:14, 1.16s/it][2025-04-20 22:25:07] (step=0013050) Train Loss: 6.3941, Train Steps/Sec: 0.86 + 32%|███████████████████████████████████▊ | 13074/40903 [5:29:01<8:53:03, 1.15s/it][2025-04-20 22:25:36] (step=0013075) Train Loss: 6.3027, Train Steps/Sec: 0.86 + 32%|███████████████████████████████████▊ | 13099/40903 [5:29:30<8:51:52, 1.15s/it][2025-04-20 22:26:05] (step=0013100) Train Loss: 6.4030, Train Steps/Sec: 0.86 + 32%|███████████████████████████████████▉ | 13124/40903 [5:29:59<9:01:46, 1.17s/it][2025-04-20 22:26:34] (step=0013125) Train Loss: 6.3696, Train Steps/Sec: 0.86 + 32%|████████████████████████████████████ | 13149/40903 [5:30:27<8:55:12, 1.16s/it][2025-04-20 22:27:03] (step=0013150) Train Loss: 6.3846, Train Steps/Sec: 0.86 + 32%|████████████████████████████████████ | 13174/40903 [5:30:57<8:58:41, 1.17s/it][2025-04-20 22:27:32] (step=0013175) Train Loss: 6.3513, Train Steps/Sec: 0.86 + 32%|████████████████████████████████████▏ | 13199/40903 [5:31:26<8:52:17, 1.15s/it][2025-04-20 22:28:01] (step=0013200) Train Loss: 6.3768, Train Steps/Sec: 0.86 + 32%|████████████████████████████████████▏ | 13224/40903 [5:31:55<8:58:37, 1.17s/it][2025-04-20 22:28:30] (step=0013225) Train Loss: 6.3535, Train Steps/Sec: 0.86 + 32%|████████████████████████████████████▎ | 13249/40903 [5:32:24<9:02:55, 1.18s/it][2025-04-20 22:28:59] (step=0013250) Train Loss: 6.3706, Train Steps/Sec: 0.86 + 32%|████████████████████████████████████▎ | 13274/40903 [5:32:53<8:43:11, 1.14s/it][2025-04-20 22:29:28] (step=0013275) Train Loss: 6.3682, Train Steps/Sec: 0.86 + 33%|████████████████████████████████████▍ | 13299/40903 [5:33:22<8:48:40, 1.15s/it][2025-04-20 22:29:57] (step=0013300) Train Loss: 6.3791, Train Steps/Sec: 0.86 + 33%|████████████████████████████████████▍ | 13324/40903 [5:33:51<8:57:00, 1.17s/it][2025-04-20 22:30:26] (step=0013325) Train Loss: 6.3947, Train Steps/Sec: 0.87 + 33%|████████████████████████████████████▌ | 13349/40903 [5:34:20<8:57:26, 1.17s/it][2025-04-20 22:30:55] (step=0013350) Train Loss: 6.3811, Train Steps/Sec: 0.86 + 33%|████████████████████████████████████▌ | 13374/40903 [5:34:49<8:47:31, 1.15s/it][2025-04-20 22:31:24] (step=0013375) Train Loss: 6.4003, Train Steps/Sec: 0.87 + 33%|████████████████████████████████████▋ | 13399/40903 [5:35:18<8:48:28, 1.15s/it][2025-04-20 22:31:53] (step=0013400) Train Loss: 6.3587, Train Steps/Sec: 0.86 + 33%|████████████████████████████████████▊ | 13424/40903 [5:35:47<8:59:52, 1.18s/it][2025-04-20 22:32:22] (step=0013425) Train Loss: 6.3949, Train Steps/Sec: 0.86 + 33%|████████████████████████████████████▊ | 13449/40903 [5:36:16<8:54:55, 1.17s/it][2025-04-20 22:32:51] (step=0013450) Train Loss: 6.3825, Train Steps/Sec: 0.86 + 33%|████████████████████████████████████▉ | 13474/40903 [5:36:45<8:50:30, 1.16s/it][2025-04-20 22:33:20] (step=0013475) Train Loss: 6.4066, Train Steps/Sec: 0.86 + 33%|████████████████████████████████████▉ | 13499/40903 [5:37:14<8:47:09, 1.15s/it][2025-04-20 22:33:49] (step=0013500) Train Loss: 6.3577, Train Steps/Sec: 0.86 + 33%|█████████████████████████████████████ | 13524/40903 [5:37:43<8:57:03, 1.18s/it][2025-04-20 22:34:18] (step=0013525) Train Loss: 6.3027, Train Steps/Sec: 0.87 + 33%|█████████████████████████████████████ | 13549/40903 [5:38:12<8:55:01, 1.17s/it][2025-04-20 22:34:47] (step=0013550) Train Loss: 6.3652, Train Steps/Sec: 0.86 + 33%|█████████████████████████████████████▏ | 13574/40903 [5:38:41<8:43:11, 1.15s/it][2025-04-20 22:35:16] (step=0013575) Train Loss: 6.3487, Train Steps/Sec: 0.87 + 33%|█████████████████████████████████████▏ | 13599/40903 [5:39:10<8:46:58, 1.16s/it][2025-04-20 22:35:46] (step=0013600) Train Loss: 6.4049, Train Steps/Sec: 0.85 + 33%|█████████████████████████████████████▎ | 13624/40903 [5:39:39<8:54:13, 1.18s/it][2025-04-20 22:36:15] (step=0013625) Train Loss: 6.3775, Train Steps/Sec: 0.86 + 33%|█████████████████████████████████████▎ | 13649/40903 [5:40:08<8:51:41, 1.17s/it][2025-04-20 22:36:44] (step=0013650) Train Loss: 6.3642, Train Steps/Sec: 0.86 + 33%|█████████████████████████████████████▍ | 13674/40903 [5:40:37<8:51:21, 1.17s/it][2025-04-20 22:37:13] (step=0013675) Train Loss: 6.4123, Train Steps/Sec: 0.86 + 33%|█████████████████████████████████████▌ | 13699/40903 [5:41:06<8:42:07, 1.15s/it][2025-04-20 22:37:42] (step=0013700) Train Loss: 6.3916, Train Steps/Sec: 0.86 + 34%|█████████████████████████████████████▌ | 13724/40903 [5:41:35<8:49:39, 1.17s/it][2025-04-20 22:38:11] (step=0013725) Train Loss: 6.3725, Train Steps/Sec: 0.87 + 34%|█████████████████████████████████████▋ | 13749/40903 [5:42:04<8:42:42, 1.16s/it][2025-04-20 22:38:40] (step=0013750) Train Loss: 6.3867, Train Steps/Sec: 0.86 + 34%|█████████████████████████████████████▋ | 13774/40903 [5:42:34<8:50:54, 1.17s/it][2025-04-20 22:39:09] (step=0013775) Train Loss: 6.3989, Train Steps/Sec: 0.85 + 34%|█████████████████████████████████████▊ | 13799/40903 [5:43:03<8:39:57, 1.15s/it][2025-04-20 22:39:38] (step=0013800) Train Loss: 6.3713, Train Steps/Sec: 0.86 + 34%|█████████████████████████████████████▊ | 13824/40903 [5:43:32<8:47:38, 1.17s/it][2025-04-20 22:40:07] (step=0013825) Train Loss: 6.3642, Train Steps/Sec: 0.86 + 34%|█████████████████████████████████████▉ | 13849/40903 [5:44:07<8:51:23, 1.18s/it][2025-04-20 22:40:42] (step=0013850) Train Loss: 6.3685, Train Steps/Sec: 0.72 + 34%|█████████████████████████████████████▋ | 13874/40903 [5:44:42<11:09:04, 1.49s/it][2025-04-20 22:41:18] (step=0013875) Train Loss: 6.3532, Train Steps/Sec: 0.71 + 34%|██████████████████████████████████████ | 13899/40903 [5:45:17<8:34:28, 1.14s/it][2025-04-20 22:41:53] (step=0013900) Train Loss: 6.3685, Train Steps/Sec: 0.71 + 34%|██████████████████████████████████████▏ | 13924/40903 [5:45:46<8:41:55, 1.16s/it][2025-04-20 22:42:22] (step=0013925) Train Loss: 6.3808, Train Steps/Sec: 0.86 + 34%|██████████████████████████████████████▏ | 13949/40903 [5:46:15<8:36:34, 1.15s/it][2025-04-20 22:42:51] (step=0013950) Train Loss: 6.3450, Train Steps/Sec: 0.87 + 34%|██████████████████████████████████████▎ | 13974/40903 [5:46:44<8:39:54, 1.16s/it][2025-04-20 22:43:20] (step=0013975) Train Loss: 6.4151, Train Steps/Sec: 0.86 + 34%|██████████████████████████████████████▎ | 13999/40903 [5:47:13<8:37:15, 1.15s/it][2025-04-20 22:43:49] (step=0014000) Train Loss: 6.3505, Train Steps/Sec: 0.86 +[2025-04-20 22:43:49] vision_config is None. initializing the InstructBlipVisionConfig with default values. +[2025-04-20 22:43:49] qformer_config is None. Initializing the InstructBlipQFormerConfig with default values. +[2025-04-20 22:43:49] text_config is None. Initializing the text config with default values (`OPTConfig`). +model_name_or_path:/tmp/haozhezhao/model/instructblip-flan-t5-xl +image_place_holder: +tokenizer length after expend 32102 +tokenizer length before expend 32102 +100%|████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████| 3/3 [03:07<00:00, 62.63s/it] +[2025-04-20 22:48:48] Finish Eval in 14000 steps...█████████████████████████████████████████████████████████████████████| 3/3 [03:07<00:00, 62.04s/it] +processing left-padding... +processing left-padding... +processing left-padding... +[2025-04-20 22:49:08] Saved checkpoint to checkpoint/EmptyFix_InstructBlip_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_objects/002-GPT-XL/checkpoints/0014000.pt +[2025-04-20 22:49:10] Removed old checkpoint: checkpoint/EmptyFix_InstructBlip_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_objects/002-GPT-XL/checkpoints/0010000.pt + 34%|██████████████████████████████████████ | 14024/40903 [5:53:12<10:18:32, 1.38s/it][2025-04-20 22:49:47] (step=0014025) Train Loss: 6.4268, Train Steps/Sec: 0.07 + 34%|██████████████████████████████████████▍ | 14049/40903 [5:53:41<8:43:44, 1.17s/it][2025-04-20 22:50:16] (step=0014050) Train Loss: 6.3833, Train Steps/Sec: 0.86 + 34%|██████████████████████████████████████▌ | 14074/40903 [5:54:10<8:37:23, 1.16s/it][2025-04-20 22:50:45] (step=0014075) Train Loss: 6.3749, Train Steps/Sec: 0.86 + 34%|██████████████████████████████████████▌ | 14099/40903 [5:54:39<8:30:53, 1.14s/it][2025-04-20 22:51:15] (step=0014100) Train Loss: 6.3774, Train Steps/Sec: 0.86 + 35%|██████████████████████████████████████▎ | 14124/40903 [5:55:15<24:07:16, 3.24s/it][2025-04-20 22:51:51] (step=0014125) Train Loss: 6.3558, Train Steps/Sec: 0.70 + 35%|██████████████████████████████████████▋ | 14149/40903 [5:55:44<8:34:17, 1.15s/it][2025-04-20 22:52:20] (step=0014150) Train Loss: 6.3758, Train Steps/Sec: 0.85 + 35%|██████████████████████████████████████▊ | 14174/40903 [5:56:13<8:38:47, 1.16s/it][2025-04-20 22:52:49] (step=0014175) Train Loss: 6.4110, Train Steps/Sec: 0.86 + 35%|██████████████████████████████████████▉ | 14199/40903 [5:56:42<8:36:41, 1.16s/it][2025-04-20 22:53:18] (step=0014200) Train Loss: 6.4086, Train Steps/Sec: 0.85 + 35%|██████████████████████████████████████▉ | 14224/40903 [5:57:12<8:49:54, 1.19s/it][2025-04-20 22:53:47] (step=0014225) Train Loss: 6.3940, Train Steps/Sec: 0.87 + 35%|███████████████████████████████████████ | 14249/40903 [5:57:41<8:36:51, 1.16s/it][2025-04-20 22:54:16] (step=0014250) Train Loss: 6.3854, Train Steps/Sec: 0.86 + 35%|███████████████████████████████████████ | 14274/40903 [5:58:09<8:34:13, 1.16s/it][2025-04-20 22:54:45] (step=0014275) Train Loss: 6.3896, Train Steps/Sec: 0.86 + 35%|███████████████████████████████████████▏ | 14299/40903 [5:58:38<8:24:43, 1.14s/it][2025-04-20 22:55:14] (step=0014300) Train Loss: 6.3480, Train Steps/Sec: 0.86 + 35%|███████████████████████████████████████▏ | 14324/40903 [5:59:08<8:40:48, 1.18s/it][2025-04-20 22:55:43] (step=0014325) Train Loss: 6.3828, Train Steps/Sec: 0.86 + 35%|███████████████████████████████████████▎ | 14349/40903 [5:59:37<8:38:29, 1.17s/it][2025-04-20 22:56:13] (step=0014350) Train Loss: 6.3776, Train Steps/Sec: 0.86 + 35%|███████████████████████████████████████▎ | 14374/40903 [6:00:06<8:25:36, 1.14s/it][2025-04-20 22:56:42] (step=0014375) Train Loss: 6.3960, Train Steps/Sec: 0.86 + 35%|███████████████████████████████████████▍ | 14399/40903 [6:00:35<8:26:06, 1.15s/it][2025-04-20 22:57:11] (step=0014400) Train Loss: 6.3866, Train Steps/Sec: 0.86 + 35%|███████████████████████████████████████▍ | 14424/40903 [6:01:04<8:52:13, 1.21s/it][2025-04-20 22:57:40] (step=0014425) Train Loss: 6.3974, Train Steps/Sec: 0.86 + 35%|███████████████████████████████████████▌ | 14449/40903 [6:01:33<8:40:16, 1.18s/it][2025-04-20 22:58:09] (step=0014450) Train Loss: 6.3840, Train Steps/Sec: 0.86 + 35%|███████████████████████████████████████▋ | 14474/40903 [6:02:09<8:44:05, 1.19s/it][2025-04-20 22:58:45] (step=0014475) Train Loss: 6.3865, Train Steps/Sec: 0.70 + 35%|███████████████████████████████████████▋ | 14499/40903 [6:02:38<8:25:51, 1.15s/it][2025-04-20 22:59:14] (step=0014500) Train Loss: 6.3690, Train Steps/Sec: 0.86 + 36%|███████████████████████████████████████▊ | 14524/40903 [6:03:08<8:41:03, 1.19s/it][2025-04-20 22:59:43] (step=0014525) Train Loss: 6.4192, Train Steps/Sec: 0.86 + 36%|███████████████████████████████████████▊ | 14549/40903 [6:03:36<8:25:16, 1.15s/it][2025-04-20 23:00:12] (step=0014550) Train Loss: 6.3542, Train Steps/Sec: 0.87 + 36%|███████████████████████████████████████▉ | 14574/40903 [6:04:05<8:20:58, 1.14s/it][2025-04-20 23:00:41] (step=0014575) Train Loss: 6.3777, Train Steps/Sec: 0.87 + 36%|███████████████████████████████████████▉ | 14599/40903 [6:04:34<8:28:23, 1.16s/it][2025-04-20 23:01:10] (step=0014600) Train Loss: 6.3402, Train Steps/Sec: 0.85 + 36%|████████████████████████████████████████ | 14624/40903 [6:05:04<8:36:23, 1.18s/it][2025-04-20 23:01:39] (step=0014625) Train Loss: 6.3705, Train Steps/Sec: 0.86 + 36%|████████████████████████████████████████ | 14649/40903 [6:05:33<8:31:28, 1.17s/it][2025-04-20 23:02:08] (step=0014650) Train Loss: 6.3783, Train Steps/Sec: 0.86 + 36%|████████████████████████████████████████▏ | 14674/40903 [6:06:02<8:22:21, 1.15s/it][2025-04-20 23:02:37] (step=0014675) Train Loss: 6.3703, Train Steps/Sec: 0.86 + 36%|████████████████████████████████████████▏ | 14699/40903 [6:06:30<8:22:10, 1.15s/it][2025-04-20 23:03:06] (step=0014700) Train Loss: 6.3537, Train Steps/Sec: 0.86 + 36%|████████████████████████████████████████▎ | 14724/40903 [6:07:00<8:33:31, 1.18s/it][2025-04-20 23:03:35] (step=0014725) Train Loss: 6.3969, Train Steps/Sec: 0.86 + 36%|████████████████████████████████████████▍ | 14749/40903 [6:07:29<8:31:09, 1.17s/it][2025-04-20 23:04:04] (step=0014750) Train Loss: 6.4090, Train Steps/Sec: 0.86 + 36%|████████████████████████████████████████▍ | 14774/40903 [6:07:58<8:25:16, 1.16s/it][2025-04-20 23:04:33] (step=0014775) Train Loss: 6.3850, Train Steps/Sec: 0.87 + 36%|████████████████████████████████████████▌ | 14799/40903 [6:08:26<8:16:24, 1.14s/it][2025-04-20 23:05:02] (step=0014800) Train Loss: 6.3831, Train Steps/Sec: 0.86 + 36%|████████████████████████████████████████▌ | 14824/40903 [6:08:55<8:28:00, 1.17s/it][2025-04-20 23:05:31] (step=0014825) Train Loss: 6.4021, Train Steps/Sec: 0.86 + 36%|████████████████████████████████████████▋ | 14849/40903 [6:09:24<8:30:12, 1.17s/it][2025-04-20 23:06:00] (step=0014850) Train Loss: 6.3623, Train Steps/Sec: 0.87 + 36%|████████████████████████████████████████▋ | 14874/40903 [6:09:53<8:24:03, 1.16s/it][2025-04-20 23:06:29] (step=0014875) Train Loss: 6.4184, Train Steps/Sec: 0.86 + 36%|████████████████████████████████████████▊ | 14899/40903 [6:10:22<8:27:33, 1.17s/it][2025-04-20 23:06:58] (step=0014900) Train Loss: 6.3629, Train Steps/Sec: 0.85 + 36%|████████████████████████████████████████▊ | 14924/40903 [6:10:51<8:26:34, 1.17s/it][2025-04-20 23:07:27] (step=0014925) Train Loss: 6.4217, Train Steps/Sec: 0.87 + 37%|████████████████████████████████████████▉ | 14949/40903 [6:11:20<8:17:55, 1.15s/it][2025-04-20 23:07:56] (step=0014950) Train Loss: 6.3455, Train Steps/Sec: 0.86 + 37%|█████████████████████████████████████████ | 14974/40903 [6:11:49<8:15:06, 1.15s/it][2025-04-20 23:08:25] (step=0014975) Train Loss: 6.3853, Train Steps/Sec: 0.86 + 37%|█████████████████████████████████████████ | 14999/40903 [6:12:18<8:21:11, 1.16s/it][2025-04-20 23:08:54] (step=0015000) Train Loss: 6.3686, Train Steps/Sec: 0.86 + 37%|█████████████████████████████████████████▏ | 15024/40903 [6:12:47<8:33:27, 1.19s/it][2025-04-20 23:09:23] (step=0015025) Train Loss: 6.3725, Train Steps/Sec: 0.87 + 37%|█████████████████████████████████████████▏ | 15049/40903 [6:13:16<8:18:40, 1.16s/it][2025-04-20 23:09:52] (step=0015050) Train Loss: 6.4011, Train Steps/Sec: 0.87 + 37%|█████████████████████████████████████████▎ | 15074/40903 [6:13:45<8:21:50, 1.17s/it][2025-04-20 23:10:21] (step=0015075) Train Loss: 6.3604, Train Steps/Sec: 0.86 + 37%|█████████████████████████████████████████▎ | 15099/40903 [6:14:14<8:17:09, 1.16s/it][2025-04-20 23:10:50] (step=0015100) Train Loss: 6.3671, Train Steps/Sec: 0.86 + 37%|█████████████████████████████████████████▍ | 15124/40903 [6:14:43<8:28:33, 1.18s/it][2025-04-20 23:11:19] (step=0015125) Train Loss: 6.3969, Train Steps/Sec: 0.86 + 37%|█████████████████████████████████████████▍ | 15149/40903 [6:15:12<8:17:17, 1.16s/it][2025-04-20 23:11:48] (step=0015150) Train Loss: 6.3618, Train Steps/Sec: 0.86 + 37%|█████████████████████████████████████████▌ | 15174/40903 [6:15:41<8:20:40, 1.17s/it][2025-04-20 23:12:17] (step=0015175) Train Loss: 6.3758, Train Steps/Sec: 0.86 + 37%|█████████████████████████████████████████▌ | 15199/40903 [6:16:10<8:10:06, 1.14s/it][2025-04-20 23:12:46] (step=0015200) Train Loss: 6.4084, Train Steps/Sec: 0.86 + 37%|█████████████████████████████████████████▋ | 15224/40903 [6:16:39<8:19:35, 1.17s/it][2025-04-20 23:13:14] (step=0015225) Train Loss: 6.3245, Train Steps/Sec: 0.87 + 37%|█████████████████████████████████████████▊ | 15249/40903 [6:17:08<8:22:31, 1.18s/it][2025-04-20 23:13:43] (step=0015250) Train Loss: 6.3929, Train Steps/Sec: 0.86 + 37%|█████████████████████████████████████████▊ | 15274/40903 [6:17:37<8:12:39, 1.15s/it][2025-04-20 23:14:12] (step=0015275) Train Loss: 6.3435, Train Steps/Sec: 0.87 + 37%|█████████████████████████████████████████▉ | 15299/40903 [6:18:06<8:09:44, 1.15s/it][2025-04-20 23:14:42] (step=0015300) Train Loss: 6.4148, Train Steps/Sec: 0.86 + 37%|█████████████████████████████████████████▉ | 15324/40903 [6:18:35<8:20:57, 1.18s/it][2025-04-20 23:15:10] (step=0015325) Train Loss: 6.3818, Train Steps/Sec: 0.87 + 38%|██████████████████████████████████████████ | 15349/40903 [6:19:04<8:15:56, 1.16s/it][2025-04-20 23:15:39] (step=0015350) Train Loss: 6.3365, Train Steps/Sec: 0.87 + 38%|██████████████████████████████████████████ | 15374/40903 [6:19:33<8:01:22, 1.13s/it][2025-04-20 23:16:08] (step=0015375) Train Loss: 6.3969, Train Steps/Sec: 0.87 + 38%|██████████████████████████████████████████▏ | 15399/40903 [6:20:02<8:07:14, 1.15s/it][2025-04-20 23:16:37] (step=0015400) Train Loss: 6.3595, Train Steps/Sec: 0.86 + 38%|██████████████████████████████████████████▏ | 15424/40903 [6:20:31<8:22:18, 1.18s/it][2025-04-20 23:17:06] (step=0015425) Train Loss: 6.3598, Train Steps/Sec: 0.86 + 38%|██████████████████████████████████████████▎ | 15449/40903 [6:21:00<8:08:56, 1.15s/it][2025-04-20 23:17:35] (step=0015450) Train Loss: 6.3636, Train Steps/Sec: 0.87 + 38%|██████████████████████████████████████████▎ | 15474/40903 [6:21:29<8:03:19, 1.14s/it][2025-04-20 23:18:04] (step=0015475) Train Loss: 6.3962, Train Steps/Sec: 0.86 + 38%|██████████████████████████████████████████▍ | 15499/40903 [6:21:58<8:09:24, 1.16s/it][2025-04-20 23:18:33] (step=0015500) Train Loss: 6.3863, Train Steps/Sec: 0.86 + 38%|██████████████████████████████████████████▌ | 15524/40903 [6:22:27<8:17:53, 1.18s/it][2025-04-20 23:19:02] (step=0015525) Train Loss: 6.3677, Train Steps/Sec: 0.87 + 38%|██████████████████████████████████████████▌ | 15549/40903 [6:22:56<8:05:15, 1.15s/it][2025-04-20 23:19:31] (step=0015550) Train Loss: 6.3568, Train Steps/Sec: 0.86 + 38%|██████████████████████████████████████████▋ | 15574/40903 [6:23:24<8:08:56, 1.16s/it][2025-04-20 23:20:00] (step=0015575) Train Loss: 6.3498, Train Steps/Sec: 0.87 + 38%|██████████████████████████████████████████▋ | 15599/40903 [6:23:53<8:00:58, 1.14s/it][2025-04-20 23:20:29] (step=0015600) Train Loss: 6.3701, Train Steps/Sec: 0.86 + 38%|██████████████████████████████████████████▊ | 15624/40903 [6:24:22<8:13:45, 1.17s/it][2025-04-20 23:20:58] (step=0015625) Train Loss: 6.3735, Train Steps/Sec: 0.87 + 38%|██████████████████████████████████████████▊ | 15649/40903 [6:24:52<8:14:41, 1.18s/it][2025-04-20 23:21:27] (step=0015650) Train Loss: 6.3737, Train Steps/Sec: 0.86 + 38%|██████████████████████████████████████████▉ | 15674/40903 [6:25:21<7:59:11, 1.14s/it][2025-04-20 23:21:56] (step=0015675) Train Loss: 6.3929, Train Steps/Sec: 0.87 + 38%|██████████████████████████████████████████▉ | 15699/40903 [6:25:50<8:01:15, 1.15s/it][2025-04-20 23:22:25] (step=0015700) Train Loss: 6.4092, Train Steps/Sec: 0.86 + 38%|███████████████████████████████████████████ | 15724/40903 [6:26:19<8:13:02, 1.17s/it][2025-04-20 23:22:54] (step=0015725) Train Loss: 6.3625, Train Steps/Sec: 0.86 + 39%|███████████████████████████████████████████ | 15749/40903 [6:26:48<8:07:34, 1.16s/it][2025-04-20 23:23:23] (step=0015750) Train Loss: 6.3774, Train Steps/Sec: 0.86 + 39%|███████████████████████████████████████████▏ | 15774/40903 [6:27:17<8:07:21, 1.16s/it][2025-04-20 23:23:52] (step=0015775) Train Loss: 6.3587, Train Steps/Sec: 0.86 + 39%|███████████████████████████████████████████▎ | 15799/40903 [6:27:46<8:03:39, 1.16s/it][2025-04-20 23:24:21] (step=0015800) Train Loss: 6.3575, Train Steps/Sec: 0.86 + 39%|███████████████████████████████████████████▎ | 15824/40903 [6:28:15<8:13:16, 1.18s/it][2025-04-20 23:24:50] (step=0015825) Train Loss: 6.3569, Train Steps/Sec: 0.86 + 39%|███████████████████████████████████████████▍ | 15849/40903 [6:28:44<8:11:19, 1.18s/it][2025-04-20 23:25:20] (step=0015850) Train Loss: 6.3757, Train Steps/Sec: 0.85 + 39%|███████████████████████████████████████████▍ | 15874/40903 [6:29:13<8:04:44, 1.16s/it][2025-04-20 23:25:49] (step=0015875) Train Loss: 6.4235, Train Steps/Sec: 0.86 + 39%|███████████████████████████████████████████▌ | 15899/40903 [6:29:42<7:50:50, 1.13s/it][2025-04-20 23:26:18] (step=0015900) Train Loss: 6.3705, Train Steps/Sec: 0.86 + 39%|███████████████████████████████████████████▌ | 15924/40903 [6:30:11<8:04:11, 1.16s/it][2025-04-20 23:26:47] (step=0015925) Train Loss: 6.3307, Train Steps/Sec: 0.87 + 39%|███████████████████████████████████████████▋ | 15949/40903 [6:30:40<8:08:58, 1.18s/it][2025-04-20 23:27:16] (step=0015950) Train Loss: 6.3993, Train Steps/Sec: 0.86 + 39%|███████████████████████████████████████████▋ | 15974/40903 [6:31:09<7:58:08, 1.15s/it][2025-04-20 23:27:45] (step=0015975) Train Loss: 6.3985, Train Steps/Sec: 0.87 + 39%|███████████████████████████████████████████▊ | 15999/40903 [6:31:38<7:54:09, 1.14s/it][2025-04-20 23:28:14] (step=0016000) Train Loss: 6.3998, Train Steps/Sec: 0.86 +[2025-04-20 23:28:14] vision_config is None. initializing the InstructBlipVisionConfig with default values. +[2025-04-20 23:28:14] qformer_config is None. Initializing the InstructBlipQFormerConfig with default values. +[2025-04-20 23:28:14] text_config is None. Initializing the text config with default values (`OPTConfig`). +model_name_or_path:/tmp/haozhezhao/model/instructblip-flan-t5-xl +image_place_holder: +tokenizer length after expend 32102 +tokenizer length before expend 32102 +100%|████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████| 3/3 [03:06<00:00, 62.17s/it] +[2025-04-20 23:33:11] Finish Eval in 16000 steps...█████████████████████████████████████████████████████████████████████| 3/3 [03:05<00:00, 61.64s/it] +processing left-padding... +processing left-padding... +processing left-padding... +[2025-04-20 23:33:32] Saved checkpoint to checkpoint/EmptyFix_InstructBlip_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_objects/002-GPT-XL/checkpoints/0016000.pt +[2025-04-20 23:33:34] Removed old checkpoint: checkpoint/EmptyFix_InstructBlip_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_objects/002-GPT-XL/checkpoints/0012000.pt + 39%|███████████████████████████████████████████▉ | 16024/40903 [6:37:27<8:17:44, 1.20s/it][2025-04-20 23:34:03] (step=0016025) Train Loss: 6.3523, Train Steps/Sec: 0.07 + 39%|███████████████████████████████████████████▉ | 16049/40903 [6:38:03<9:30:52, 1.38s/it][2025-04-20 23:34:38] (step=0016050) Train Loss: 6.3671, Train Steps/Sec: 0.71 + 39%|████████████████████████████████████████████ | 16074/40903 [6:38:32<7:51:34, 1.14s/it][2025-04-20 23:35:07] (step=0016075) Train Loss: 6.3546, Train Steps/Sec: 0.87 + 39%|████████████████████████████████████████████ | 16099/40903 [6:39:01<7:55:51, 1.15s/it][2025-04-20 23:35:37] (step=0016100) Train Loss: 6.3700, Train Steps/Sec: 0.86 + 39%|████████████████████████████████████████████▏ | 16124/40903 [6:39:30<8:02:33, 1.17s/it][2025-04-20 23:36:05] (step=0016125) Train Loss: 6.3669, Train Steps/Sec: 0.87 + 39%|████████████████████████████████████████████▏ | 16149/40903 [6:39:59<8:02:50, 1.17s/it][2025-04-20 23:36:34] (step=0016150) Train Loss: 6.3727, Train Steps/Sec: 0.86 + 40%|████████████████████████████████████████████▎ | 16174/40903 [6:40:28<7:49:42, 1.14s/it][2025-04-20 23:37:03] (step=0016175) Train Loss: 6.3665, Train Steps/Sec: 0.86 + 40%|████████████████████████████████████████████▎ | 16199/40903 [6:40:57<7:51:21, 1.14s/it][2025-04-20 23:37:32] (step=0016200) Train Loss: 6.3796, Train Steps/Sec: 0.87 + 40%|████████████████████████████████████████████▍ | 16224/40903 [6:41:26<8:02:45, 1.17s/it][2025-04-20 23:38:01] (step=0016225) Train Loss: 6.4009, Train Steps/Sec: 0.87 + 40%|████████████████████████████████████████████▍ | 16249/40903 [6:41:55<8:00:57, 1.17s/it][2025-04-20 23:38:30] (step=0016250) Train Loss: 6.4004, Train Steps/Sec: 0.86 + 40%|████████████████████████████████████████████▌ | 16274/40903 [6:42:24<7:52:14, 1.15s/it][2025-04-20 23:38:59] (step=0016275) Train Loss: 6.3455, Train Steps/Sec: 0.86 + 40%|████████████████████████████████████████████▋ | 16299/40903 [6:42:53<7:56:29, 1.16s/it][2025-04-20 23:39:29] (step=0016300) Train Loss: 6.3837, Train Steps/Sec: 0.85 + 40%|████████████████████████████████████████████▋ | 16324/40903 [6:43:22<7:56:41, 1.16s/it][2025-04-20 23:39:57] (step=0016325) Train Loss: 6.3633, Train Steps/Sec: 0.87 + 40%|████████████████████████████████████████████▊ | 16349/40903 [6:43:51<7:50:57, 1.15s/it][2025-04-20 23:40:26] (step=0016350) Train Loss: 6.3845, Train Steps/Sec: 0.86 + 40%|████████████████████████████████████████████▊ | 16374/40903 [6:44:20<7:48:01, 1.14s/it][2025-04-20 23:40:55] (step=0016375) Train Loss: 6.3522, Train Steps/Sec: 0.87 + 40%|████████████████████████████████████████████▉ | 16399/40903 [6:44:49<7:45:16, 1.14s/it][2025-04-20 23:41:24] (step=0016400) Train Loss: 6.3824, Train Steps/Sec: 0.86 + 40%|████████████████████████████████████████████▉ | 16424/40903 [6:45:17<8:01:58, 1.18s/it][2025-04-20 23:41:53] (step=0016425) Train Loss: 6.3643, Train Steps/Sec: 0.87 + 40%|█████████████████████████████████████████████ | 16449/40903 [6:45:47<8:03:47, 1.19s/it][2025-04-20 23:42:22] (step=0016450) Train Loss: 6.3979, Train Steps/Sec: 0.86 + 40%|█████████████████████████████████████████████ | 16474/40903 [6:46:15<7:43:46, 1.14s/it][2025-04-20 23:42:51] (step=0016475) Train Loss: 6.3899, Train Steps/Sec: 0.87 + 40%|█████████████████████████████████████████████▏ | 16499/40903 [6:46:44<7:44:19, 1.14s/it][2025-04-20 23:43:20] (step=0016500) Train Loss: 6.3922, Train Steps/Sec: 0.86 + 40%|█████████████████████████████████████████████▏ | 16524/40903 [6:47:13<7:53:16, 1.16s/it][2025-04-20 23:43:49] (step=0016525) Train Loss: 6.3777, Train Steps/Sec: 0.87 + 40%|█████████████████████████████████████████████▎ | 16549/40903 [6:47:43<8:02:57, 1.19s/it][2025-04-20 23:44:18] (step=0016550) Train Loss: 6.3541, Train Steps/Sec: 0.86 + 41%|█████████████████████████████████████████████▍ | 16574/40903 [6:48:12<7:53:51, 1.17s/it][2025-04-20 23:44:47] (step=0016575) Train Loss: 6.3560, Train Steps/Sec: 0.86 + 41%|█████████████████████████████████████████████▍ | 16599/40903 [6:48:40<7:45:21, 1.15s/it][2025-04-20 23:45:16] (step=0016600) Train Loss: 6.4027, Train Steps/Sec: 0.87 + 41%|█████████████████████████████████████████████▌ | 16624/40903 [6:49:09<7:55:52, 1.18s/it][2025-04-20 23:45:45] (step=0016625) Train Loss: 6.3683, Train Steps/Sec: 0.86 + 41%|█████████████████████████████████████████████▌ | 16649/40903 [6:49:39<7:49:35, 1.16s/it][2025-04-20 23:46:14] (step=0016650) Train Loss: 6.3835, Train Steps/Sec: 0.86 + 41%|█████████████████████████████████████████████▋ | 16674/40903 [6:50:07<7:47:14, 1.16s/it][2025-04-20 23:46:43] (step=0016675) Train Loss: 6.3814, Train Steps/Sec: 0.86 + 41%|█████████████████████████████████████████████▋ | 16699/40903 [6:50:36<7:45:12, 1.15s/it][2025-04-20 23:47:12] (step=0016700) Train Loss: 6.3790, Train Steps/Sec: 0.86 + 41%|█████████████████████████████████████████████▊ | 16724/40903 [6:51:06<7:56:02, 1.18s/it][2025-04-20 23:47:41] (step=0016725) Train Loss: 6.3799, Train Steps/Sec: 0.86 + 41%|█████████████████████████████████████████████▊ | 16749/40903 [6:51:34<7:47:19, 1.16s/it][2025-04-20 23:48:10] (step=0016750) Train Loss: 6.4049, Train Steps/Sec: 0.87 + 41%|█████████████████████████████████████████████▉ | 16774/40903 [6:52:03<7:39:19, 1.14s/it][2025-04-20 23:48:39] (step=0016775) Train Loss: 6.3800, Train Steps/Sec: 0.87 + 41%|█████████████████████████████████████████████▉ | 16799/40903 [6:52:32<7:40:21, 1.15s/it][2025-04-20 23:49:08] (step=0016800) Train Loss: 6.3872, Train Steps/Sec: 0.86 + 41%|██████████████████████████████████████████████ | 16824/40903 [6:53:01<7:54:00, 1.18s/it][2025-04-20 23:49:37] (step=0016825) Train Loss: 6.3822, Train Steps/Sec: 0.86 + 41%|██████████████████████████████████████████████▏ | 16849/40903 [6:53:30<7:43:50, 1.16s/it][2025-04-20 23:50:06] (step=0016850) Train Loss: 6.3462, Train Steps/Sec: 0.86 + 41%|██████████████████████████████████████████████▏ | 16874/40903 [6:53:59<7:40:08, 1.15s/it][2025-04-20 23:50:35] (step=0016875) Train Loss: 6.3791, Train Steps/Sec: 0.86 + 41%|██████████████████████████████████████████████▎ | 16899/40903 [6:54:28<7:42:02, 1.15s/it][2025-04-20 23:51:04] (step=0016900) Train Loss: 6.3639, Train Steps/Sec: 0.85 + 41%|██████████████████████████████████████████████▎ | 16924/40903 [6:54:57<7:45:40, 1.17s/it][2025-04-20 23:51:33] (step=0016925) Train Loss: 6.3443, Train Steps/Sec: 0.87 + 41%|██████████████████████████████████████████████▍ | 16949/40903 [6:55:26<7:45:31, 1.17s/it][2025-04-20 23:52:02] (step=0016950) Train Loss: 6.3584, Train Steps/Sec: 0.86 + 41%|██████████████████████████████████████████████▍ | 16974/40903 [6:55:55<7:40:54, 1.16s/it][2025-04-20 23:52:31] (step=0016975) Train Loss: 6.3316, Train Steps/Sec: 0.86 + 42%|██████████████████████████████████████████████▌ | 16999/40903 [6:56:24<7:39:47, 1.15s/it][2025-04-20 23:53:00] (step=0017000) Train Loss: 6.3659, Train Steps/Sec: 0.86 + 42%|██████████████████████████████████████████████▌ | 17024/40903 [6:56:53<7:46:37, 1.17s/it][2025-04-20 23:53:29] (step=0017025) Train Loss: 6.4050, Train Steps/Sec: 0.86 + 42%|██████████████████████████████████████████████▋ | 17049/40903 [6:57:22<7:44:18, 1.17s/it][2025-04-20 23:53:58] (step=0017050) Train Loss: 6.3851, Train Steps/Sec: 0.87 + 42%|██████████████████████████████████████████████▊ | 17074/40903 [6:57:51<7:33:26, 1.14s/it][2025-04-20 23:54:27] (step=0017075) Train Loss: 6.4069, Train Steps/Sec: 0.87 + 42%|██████████████████████████████████████████████▊ | 17099/40903 [6:58:20<7:30:11, 1.13s/it][2025-04-20 23:54:56] (step=0017100) Train Loss: 6.4126, Train Steps/Sec: 0.86 + 42%|██████████████████████████████████████████████▉ | 17124/40903 [6:58:49<7:48:22, 1.18s/it][2025-04-20 23:55:25] (step=0017125) Train Loss: 6.3844, Train Steps/Sec: 0.86 + 42%|██████████████████████████████████████████████▉ | 17149/40903 [6:59:18<7:39:44, 1.16s/it][2025-04-20 23:55:54] (step=0017150) Train Loss: 6.3670, Train Steps/Sec: 0.86 + 42%|███████████████████████████████████████████████ | 17174/40903 [6:59:47<7:32:07, 1.14s/it][2025-04-20 23:56:23] (step=0017175) Train Loss: 6.3529, Train Steps/Sec: 0.86 + 42%|███████████████████████████████████████████████ | 17199/40903 [7:00:16<7:34:34, 1.15s/it][2025-04-20 23:56:52] (step=0017200) Train Loss: 6.4183, Train Steps/Sec: 0.86 + 42%|███████████████████████████████████████████████▏ | 17224/40903 [7:00:45<7:41:18, 1.17s/it][2025-04-20 23:57:21] (step=0017225) Train Loss: 6.3256, Train Steps/Sec: 0.86 + 42%|███████████████████████████████████████████████▏ | 17249/40903 [7:01:14<7:38:36, 1.16s/it][2025-04-20 23:57:50] (step=0017250) Train Loss: 6.3411, Train Steps/Sec: 0.87 + 42%|███████████████████████████████████████████████▎ | 17274/40903 [7:01:43<7:34:28, 1.15s/it][2025-04-20 23:58:19] (step=0017275) Train Loss: 6.4084, Train Steps/Sec: 0.86 + 42%|███████████████████████████████████████████████▎ | 17299/40903 [7:02:12<7:30:00, 1.14s/it][2025-04-20 23:58:48] (step=0017300) Train Loss: 6.4136, Train Steps/Sec: 0.86 + 42%|███████████████████████████████████████████████▍ | 17324/40903 [7:02:41<7:39:37, 1.17s/it][2025-04-20 23:59:17] (step=0017325) Train Loss: 6.3704, Train Steps/Sec: 0.86 + 42%|███████████████████████████████████████████████▌ | 17349/40903 [7:03:10<7:31:32, 1.15s/it][2025-04-20 23:59:46] (step=0017350) Train Loss: 6.3687, Train Steps/Sec: 0.87 + 42%|███████████████████████████████████████████████▌ | 17374/40903 [7:03:39<7:30:53, 1.15s/it][2025-04-21 00:00:15] (step=0017375) Train Loss: 6.3995, Train Steps/Sec: 0.86 + 43%|███████████████████████████████████████████████▋ | 17399/40903 [7:04:08<7:30:24, 1.15s/it][2025-04-21 00:00:44] (step=0017400) Train Loss: 6.3207, Train Steps/Sec: 0.86 + 43%|███████████████████████████████████████████████▋ | 17424/40903 [7:04:37<7:41:25, 1.18s/it][2025-04-21 00:01:13] (step=0017425) Train Loss: 6.3558, Train Steps/Sec: 0.87 + 43%|███████████████████████████████████████████████▊ | 17449/40903 [7:05:06<7:35:41, 1.17s/it][2025-04-21 00:01:42] (step=0017450) Train Loss: 6.3778, Train Steps/Sec: 0.86 + 43%|███████████████████████████████████████████████▊ | 17474/40903 [7:05:35<7:26:43, 1.14s/it][2025-04-21 00:02:11] (step=0017475) Train Loss: 6.3614, Train Steps/Sec: 0.87 + 43%|███████████████████████████████████████████████▉ | 17499/40903 [7:06:04<7:28:16, 1.15s/it][2025-04-21 00:02:40] (step=0017500) Train Loss: 6.3744, Train Steps/Sec: 0.85 + 43%|███████████████████████████████████████████████▉ | 17524/40903 [7:06:33<7:40:07, 1.18s/it][2025-04-21 00:03:09] (step=0017525) Train Loss: 6.4081, Train Steps/Sec: 0.86 + 43%|████████████████████████████████████████████████ | 17549/40903 [7:07:02<7:37:30, 1.18s/it][2025-04-21 00:03:38] (step=0017550) Train Loss: 6.4076, Train Steps/Sec: 0.86 + 43%|████████████████████████████████████████████████ | 17574/40903 [7:07:31<7:25:02, 1.14s/it][2025-04-21 00:04:07] (step=0017575) Train Loss: 6.3773, Train Steps/Sec: 0.86 + 43%|████████████████████████████████████████████████▏ | 17599/40903 [7:08:00<7:26:19, 1.15s/it][2025-04-21 00:04:36] (step=0017600) Train Loss: 6.3976, Train Steps/Sec: 0.86 + 43%|████████████████████████████████████████████████▎ | 17624/40903 [7:08:29<7:36:46, 1.18s/it][2025-04-21 00:05:05] (step=0017625) Train Loss: 6.3619, Train Steps/Sec: 0.87 + 43%|████████████████████████████████████████████████▎ | 17649/40903 [7:08:58<7:28:26, 1.16s/it][2025-04-21 00:05:34] (step=0017650) Train Loss: 6.3763, Train Steps/Sec: 0.86 + 43%|████████████████████████████████████████████████▍ | 17674/40903 [7:09:27<7:22:35, 1.14s/it][2025-04-21 00:06:03] (step=0017675) Train Loss: 6.3551, Train Steps/Sec: 0.87 + 43%|████████████████████████████████████████████████▍ | 17699/40903 [7:09:56<7:28:22, 1.16s/it][2025-04-21 00:06:32] (step=0017700) Train Loss: 6.3766, Train Steps/Sec: 0.86 + 43%|████████████████████████████████████████████████▌ | 17724/40903 [7:10:25<7:34:07, 1.18s/it][2025-04-21 00:07:01] (step=0017725) Train Loss: 6.3670, Train Steps/Sec: 0.87 + 43%|████████████████████████████████████████████████▌ | 17749/40903 [7:10:54<7:33:43, 1.18s/it][2025-04-21 00:07:30] (step=0017750) Train Loss: 6.3686, Train Steps/Sec: 0.86 + 43%|████████████████████████████████████████████████▋ | 17774/40903 [7:11:23<7:20:34, 1.14s/it][2025-04-21 00:07:59] (step=0017775) Train Loss: 6.3492, Train Steps/Sec: 0.87 + 44%|████████████████████████████████████████████████▋ | 17799/40903 [7:11:52<7:23:31, 1.15s/it][2025-04-21 00:08:28] (step=0017800) Train Loss: 6.4030, Train Steps/Sec: 0.86 + 44%|████████████████████████████████████████████████▊ | 17824/40903 [7:12:21<7:29:32, 1.17s/it][2025-04-21 00:08:56] (step=0017825) Train Loss: 6.3719, Train Steps/Sec: 0.87 + 44%|████████████████████████████████████████████████▊ | 17849/40903 [7:12:50<7:23:22, 1.15s/it][2025-04-21 00:09:25] (step=0017850) Train Loss: 6.3572, Train Steps/Sec: 0.87 + 44%|████████████████████████████████████████████████▉ | 17874/40903 [7:13:19<7:21:09, 1.15s/it][2025-04-21 00:09:54] (step=0017875) Train Loss: 6.3440, Train Steps/Sec: 0.86 + 44%|█████████████████████████████████████████████████ | 17899/40903 [7:13:48<7:23:09, 1.16s/it][2025-04-21 00:10:23] (step=0017900) Train Loss: 6.3783, Train Steps/Sec: 0.86 + 44%|█████████████████████████████████████████████████ | 17924/40903 [7:14:17<7:30:58, 1.18s/it][2025-04-21 00:10:52] (step=0017925) Train Loss: 6.3824, Train Steps/Sec: 0.86 + 44%|█████████████████████████████████████████████████▏ | 17949/40903 [7:14:46<7:30:03, 1.18s/it][2025-04-21 00:11:21] (step=0017950) Train Loss: 6.4424, Train Steps/Sec: 0.86 + 44%|█████████████████████████████████████████████████▏ | 17974/40903 [7:15:15<7:21:43, 1.16s/it][2025-04-21 00:11:50] (step=0017975) Train Loss: 6.3693, Train Steps/Sec: 0.87 + 44%|█████████████████████████████████████████████████▎ | 17999/40903 [7:15:44<7:12:32, 1.13s/it][2025-04-21 00:12:19] (step=0018000) Train Loss: 6.3021, Train Steps/Sec: 0.86 +[2025-04-21 00:12:19] vision_config is None. initializing the InstructBlipVisionConfig with default values. +[2025-04-21 00:12:19] qformer_config is None. Initializing the InstructBlipQFormerConfig with default values. +[2025-04-21 00:12:19] text_config is None. Initializing the text config with default values (`OPTConfig`). +model_name_or_path:/tmp/haozhezhao/model/instructblip-flan-t5-xl +image_place_holder: +tokenizer length after expend 32102 +tokenizer length before expend 32102 +100%|████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████| 3/3 [03:05<00:00, 61.79s/it] +[2025-04-21 00:17:15] Finish Eval in 18000 steps...█████████████████████████████████████████████████████████████████████| 3/3 [03:04<00:00, 61.42s/it] +processing left-padding... +processing left-padding... +processing left-padding... +[2025-04-21 00:17:36] Saved checkpoint to checkpoint/EmptyFix_InstructBlip_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_objects/002-GPT-XL/checkpoints/0018000.pt +[2025-04-21 00:17:39] Removed old checkpoint: checkpoint/EmptyFix_InstructBlip_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_objects/002-GPT-XL/checkpoints/0014000.pt + 44%|█████████████████████████████████████████████████▎ | 18024/40903 [7:21:32<7:38:10, 1.20s/it][2025-04-21 00:18:08] (step=0018025) Train Loss: 6.3746, Train Steps/Sec: 0.07 + 44%|█████████████████████████████████████████████████▍ | 18049/40903 [7:22:01<7:24:27, 1.17s/it][2025-04-21 00:18:37] (step=0018050) Train Loss: 6.3455, Train Steps/Sec: 0.86 + 44%|█████████████████████████████████████████████████▍ | 18074/40903 [7:22:30<7:20:10, 1.16s/it][2025-04-21 00:19:06] (step=0018075) Train Loss: 6.3788, Train Steps/Sec: 0.86 + 44%|█████████████████████████████████████████████████▌ | 18099/40903 [7:22:59<7:14:52, 1.14s/it][2025-04-21 00:19:34] (step=0018100) Train Loss: 6.4125, Train Steps/Sec: 0.87 + 44%|█████████████████████████████████████████████████▋ | 18124/40903 [7:23:28<7:28:25, 1.18s/it][2025-04-21 00:20:04] (step=0018125) Train Loss: 6.3575, Train Steps/Sec: 0.86 + 44%|█████████████████████████████████████████████████▋ | 18149/40903 [7:23:57<7:24:03, 1.17s/it][2025-04-21 00:20:32] (step=0018150) Train Loss: 6.3689, Train Steps/Sec: 0.87 + 44%|█████████████████████████████████████████████████▊ | 18174/40903 [7:24:26<7:21:17, 1.16s/it][2025-04-21 00:21:01] (step=0018175) Train Loss: 6.3596, Train Steps/Sec: 0.87 + 44%|█████████████████████████████████████████████████▊ | 18199/40903 [7:24:55<7:15:26, 1.15s/it][2025-04-21 00:21:30] (step=0018200) Train Loss: 6.3527, Train Steps/Sec: 0.86 + 45%|█████████████████████████████████████████████████▉ | 18224/40903 [7:25:24<7:24:29, 1.18s/it][2025-04-21 00:21:59] (step=0018225) Train Loss: 6.3432, Train Steps/Sec: 0.87 + 45%|█████████████████████████████████████████████████▉ | 18249/40903 [7:25:53<7:16:03, 1.15s/it][2025-04-21 00:22:28] (step=0018250) Train Loss: 6.3721, Train Steps/Sec: 0.86 + 45%|██████████████████████████████████████████████████ | 18274/40903 [7:26:21<7:19:33, 1.17s/it][2025-04-21 00:22:57] (step=0018275) Train Loss: 6.3824, Train Steps/Sec: 0.87 + 45%|██████████████████████████████████████████████████ | 18299/40903 [7:26:50<7:08:06, 1.14s/it][2025-04-21 00:23:26] (step=0018300) Train Loss: 6.3473, Train Steps/Sec: 0.86 + 45%|██████████████████████████████████████████████████▏ | 18324/40903 [7:27:19<7:26:26, 1.19s/it][2025-04-21 00:23:55] (step=0018325) Train Loss: 6.3399, Train Steps/Sec: 0.86 + 45%|██████████████████████████████████████████████████▏ | 18349/40903 [7:27:48<7:11:38, 1.15s/it][2025-04-21 00:24:24] (step=0018350) Train Loss: 6.3774, Train Steps/Sec: 0.86 + 45%|██████████████████████████████████████████████████▎ | 18374/40903 [7:28:17<7:12:37, 1.15s/it][2025-04-21 00:24:53] (step=0018375) Train Loss: 6.3665, Train Steps/Sec: 0.87 + 45%|██████████████████████████████████████████████████▍ | 18399/40903 [7:28:46<7:10:24, 1.15s/it][2025-04-21 00:25:22] (step=0018400) Train Loss: 6.3844, Train Steps/Sec: 0.86 + 45%|██████████████████████████████████████████████████▍ | 18424/40903 [7:29:15<7:13:31, 1.16s/it][2025-04-21 00:25:50] (step=0018425) Train Loss: 6.3861, Train Steps/Sec: 0.87 + 45%|██████████████████████████████████████████████████▌ | 18449/40903 [7:29:44<7:10:41, 1.15s/it][2025-04-21 00:26:19] (step=0018450) Train Loss: 6.3966, Train Steps/Sec: 0.87 + 45%|██████████████████████████████████████████████████▌ | 18474/40903 [7:30:13<7:22:17, 1.18s/it][2025-04-21 00:26:49] (step=0018475) Train Loss: 6.3617, Train Steps/Sec: 0.86 + 45%|██████████████████████████████████████████████████▋ | 18499/40903 [7:30:42<7:07:32, 1.15s/it][2025-04-21 00:27:18] (step=0018500) Train Loss: 6.3432, Train Steps/Sec: 0.86 + 45%|██████████████████████████████████████████████████▋ | 18524/40903 [7:31:11<7:16:20, 1.17s/it][2025-04-21 00:27:46] (step=0018525) Train Loss: 6.3543, Train Steps/Sec: 0.87 + 45%|██████████████████████████████████████████████████▊ | 18549/40903 [7:31:40<7:16:38, 1.17s/it][2025-04-21 00:28:16] (step=0018550) Train Loss: 6.3576, Train Steps/Sec: 0.86 + 45%|██████████████████████████████████████████████████▊ | 18574/40903 [7:32:09<7:10:57, 1.16s/it][2025-04-21 00:28:45] (step=0018575) Train Loss: 6.4320, Train Steps/Sec: 0.86 + 45%|██████████████████████████████████████████████████▉ | 18599/40903 [7:32:38<7:03:29, 1.14s/it][2025-04-21 00:29:13] (step=0018600) Train Loss: 6.3880, Train Steps/Sec: 0.87 + 46%|██████████████████████████████████████████████████▉ | 18624/40903 [7:33:07<7:19:41, 1.18s/it][2025-04-21 00:29:42] (step=0018625) Train Loss: 6.3513, Train Steps/Sec: 0.86 + 46%|███████████████████████████████████████████████████ | 18649/40903 [7:33:36<7:09:31, 1.16s/it][2025-04-21 00:30:11] (step=0018650) Train Loss: 6.3604, Train Steps/Sec: 0.86 + 46%|███████████████████████████████████████████████████▏ | 18674/40903 [7:34:05<7:13:43, 1.17s/it][2025-04-21 00:30:40] (step=0018675) Train Loss: 6.3923, Train Steps/Sec: 0.86 + 46%|███████████████████████████████████████████████████▏ | 18699/40903 [7:34:34<7:01:16, 1.14s/it][2025-04-21 00:31:09] (step=0018700) Train Loss: 6.3354, Train Steps/Sec: 0.86 + 46%|███████████████████████████████████████████████████▎ | 18724/40903 [7:35:03<7:14:02, 1.17s/it][2025-04-21 00:31:38] (step=0018725) Train Loss: 6.3351, Train Steps/Sec: 0.87 + 46%|███████████████████████████████████████████████████▎ | 18749/40903 [7:35:32<7:12:13, 1.17s/it][2025-04-21 00:32:07] (step=0018750) Train Loss: 6.3858, Train Steps/Sec: 0.85 + 46%|███████████████████████████████████████████████████▍ | 18774/40903 [7:36:01<7:03:33, 1.15s/it][2025-04-21 00:32:36] (step=0018775) Train Loss: 6.3527, Train Steps/Sec: 0.86 + 46%|███████████████████████████████████████████████████▍ | 18799/40903 [7:36:30<6:52:52, 1.12s/it][2025-04-21 00:33:05] (step=0018800) Train Loss: 6.3586, Train Steps/Sec: 0.86 + 46%|███████████████████████████████████████████████████▌ | 18824/40903 [7:36:59<7:15:06, 1.18s/it][2025-04-21 00:33:34] (step=0018825) Train Loss: 6.4239, Train Steps/Sec: 0.86 + 46%|███████████████████████████████████████████████████▌ | 18849/40903 [7:37:28<7:04:22, 1.15s/it][2025-04-21 00:34:03] (step=0018850) Train Loss: 6.4103, Train Steps/Sec: 0.86 + 46%|███████████████████████████████████████████████████▋ | 18874/40903 [7:37:57<7:04:27, 1.16s/it][2025-04-21 00:34:33] (step=0018875) Train Loss: 6.3972, Train Steps/Sec: 0.86 + 46%|███████████████████████████████████████████████████▋ | 18899/40903 [7:38:32<8:14:35, 1.35s/it][2025-04-21 00:35:08] (step=0018900) Train Loss: 6.3541, Train Steps/Sec: 0.71 + 46%|███████████████████████████████████████████████████▊ | 18924/40903 [7:39:01<7:08:16, 1.17s/it][2025-04-21 00:35:36] (step=0018925) Train Loss: 6.3782, Train Steps/Sec: 0.87 + 46%|███████████████████████████████████████████████████▉ | 18949/40903 [7:39:30<7:08:19, 1.17s/it][2025-04-21 00:36:06] (step=0018950) Train Loss: 6.3705, Train Steps/Sec: 0.86 + 46%|███████████████████████████████████████████████████▉ | 18974/40903 [7:39:59<7:00:04, 1.15s/it][2025-04-21 00:36:35] (step=0018975) Train Loss: 6.3574, Train Steps/Sec: 0.86 + 46%|████████████████████████████████████████████████████ | 18999/40903 [7:40:34<7:06:09, 1.17s/it][2025-04-21 00:37:10] (step=0019000) Train Loss: 6.3524, Train Steps/Sec: 0.71 + 47%|████████████████████████████████████████████████████ | 19024/40903 [7:41:09<8:02:17, 1.32s/it][2025-04-21 00:37:45] (step=0019025) Train Loss: 6.3442, Train Steps/Sec: 0.71 + 47%|████████████████████████████████████████████████████▏ | 19049/40903 [7:41:38<7:04:44, 1.17s/it][2025-04-21 00:38:14] (step=0019050) Train Loss: 6.3618, Train Steps/Sec: 0.86 + 47%|████████████████████████████████████████████████████▏ | 19074/40903 [7:42:07<6:59:28, 1.15s/it][2025-04-21 00:38:43] (step=0019075) Train Loss: 6.3709, Train Steps/Sec: 0.86 + 47%|████████████████████████████████████████████████████▎ | 19099/40903 [7:42:36<6:55:10, 1.14s/it][2025-04-21 00:39:12] (step=0019100) Train Loss: 6.3712, Train Steps/Sec: 0.86 + 47%|████████████████████████████████████████████████████▎ | 19124/40903 [7:43:05<7:06:05, 1.17s/it][2025-04-21 00:39:41] (step=0019125) Train Loss: 6.3677, Train Steps/Sec: 0.86 + 47%|████████████████████████████████████████████████████▍ | 19149/40903 [7:43:34<7:00:53, 1.16s/it][2025-04-21 00:40:10] (step=0019150) Train Loss: 6.3478, Train Steps/Sec: 0.87 + 47%|████████████████████████████████████████████████████▌ | 19174/40903 [7:44:10<6:56:07, 1.15s/it][2025-04-21 00:40:45] (step=0019175) Train Loss: 6.3293, Train Steps/Sec: 0.70 + 47%|████████████████████████████████████████████████████▌ | 19199/40903 [7:44:39<6:57:10, 1.15s/it][2025-04-21 00:41:15] (step=0019200) Train Loss: 6.3818, Train Steps/Sec: 0.85 + 47%|████████████████████████████████████████████████████▋ | 19224/40903 [7:45:08<7:04:43, 1.18s/it][2025-04-21 00:41:44] (step=0019225) Train Loss: 6.3957, Train Steps/Sec: 0.86 + 47%|████████████████████████████████████████████████████▋ | 19249/40903 [7:45:37<6:54:02, 1.15s/it][2025-04-21 00:42:13] (step=0019250) Train Loss: 6.3767, Train Steps/Sec: 0.87 + 47%|████████████████████████████████████████████████████▊ | 19274/40903 [7:46:06<6:55:08, 1.15s/it][2025-04-21 00:42:42] (step=0019275) Train Loss: 6.4022, Train Steps/Sec: 0.86 + 47%|████████████████████████████████████████████████████▊ | 19299/40903 [7:46:35<6:48:34, 1.13s/it][2025-04-21 00:43:11] (step=0019300) Train Loss: 6.3559, Train Steps/Sec: 0.86 + 47%|████████████████████████████████████████████████████▉ | 19324/40903 [7:47:04<7:09:58, 1.20s/it][2025-04-21 00:43:40] (step=0019325) Train Loss: 6.3869, Train Steps/Sec: 0.86 + 47%|████████████████████████████████████████████████████▉ | 19349/40903 [7:47:33<6:59:53, 1.17s/it][2025-04-21 00:44:09] (step=0019350) Train Loss: 6.3561, Train Steps/Sec: 0.86 + 47%|█████████████████████████████████████████████████████ | 19374/40903 [7:48:02<6:54:07, 1.15s/it][2025-04-21 00:44:38] (step=0019375) Train Loss: 6.3513, Train Steps/Sec: 0.86 + 47%|█████████████████████████████████████████████████████ | 19399/40903 [7:48:31<6:57:15, 1.16s/it][2025-04-21 00:45:07] (step=0019400) Train Loss: 6.3877, Train Steps/Sec: 0.86 + 47%|█████████████████████████████████████████████████████▏ | 19424/40903 [7:49:00<7:05:58, 1.19s/it][2025-04-21 00:45:36] (step=0019425) Train Loss: 6.3591, Train Steps/Sec: 0.86 + 48%|█████████████████████████████████████████████████████▎ | 19449/40903 [7:49:29<6:51:06, 1.15s/it][2025-04-21 00:46:05] (step=0019450) Train Loss: 6.3699, Train Steps/Sec: 0.87 + 48%|█████████████████████████████████████████████████████▎ | 19474/40903 [7:49:58<6:53:42, 1.16s/it][2025-04-21 00:46:34] (step=0019475) Train Loss: 6.3576, Train Steps/Sec: 0.86 + 48%|█████████████████████████████████████████████████████▍ | 19499/40903 [7:50:27<6:49:22, 1.15s/it][2025-04-21 00:47:03] (step=0019500) Train Loss: 6.3756, Train Steps/Sec: 0.86 + 48%|█████████████████████████████████████████████████████▍ | 19524/40903 [7:50:57<6:54:19, 1.16s/it][2025-04-21 00:47:32] (step=0019525) Train Loss: 6.3557, Train Steps/Sec: 0.86 + 48%|█████████████████████████████████████████████████████▌ | 19549/40903 [7:51:25<6:51:13, 1.16s/it][2025-04-21 00:48:01] (step=0019550) Train Loss: 6.3574, Train Steps/Sec: 0.86 + 48%|█████████████████████████████████████████████████████▌ | 19574/40903 [7:51:55<6:55:31, 1.17s/it][2025-04-21 00:48:30] (step=0019575) Train Loss: 6.4242, Train Steps/Sec: 0.86 + 48%|█████████████████████████████████████████████████████▋ | 19599/40903 [7:52:23<6:44:40, 1.14s/it][2025-04-21 00:48:59] (step=0019600) Train Loss: 6.3368, Train Steps/Sec: 0.86 + 48%|█████████████████████████████████████████████████████▋ | 19624/40903 [7:52:59<6:58:28, 1.18s/it][2025-04-21 00:49:35] (step=0019625) Train Loss: 6.4080, Train Steps/Sec: 0.70 + 48%|█████████████████████████████████████████████████████▊ | 19649/40903 [7:53:28<6:45:11, 1.14s/it][2025-04-21 00:50:04] (step=0019650) Train Loss: 6.3751, Train Steps/Sec: 0.86 + 48%|█████████████████████████████████████████████████████▊ | 19674/40903 [7:53:57<6:46:48, 1.15s/it][2025-04-21 00:50:33] (step=0019675) Train Loss: 6.3850, Train Steps/Sec: 0.86 + 48%|█████████████████████████████████████████████████████▉ | 19699/40903 [7:54:26<6:42:09, 1.14s/it][2025-04-21 00:51:02] (step=0019700) Train Loss: 6.3372, Train Steps/Sec: 0.87 + 48%|██████████████████████████████████████████████████████ | 19724/40903 [7:54:55<6:57:55, 1.18s/it][2025-04-21 00:51:31] (step=0019725) Train Loss: 6.3665, Train Steps/Sec: 0.87 + 48%|██████████████████████████████████████████████████████ | 19749/40903 [7:55:24<6:46:49, 1.15s/it][2025-04-21 00:52:00] (step=0019750) Train Loss: 6.4102, Train Steps/Sec: 0.87 + 48%|██████████████████████████████████████████████████████▏ | 19774/40903 [7:55:53<6:42:22, 1.14s/it][2025-04-21 00:52:28] (step=0019775) Train Loss: 6.3881, Train Steps/Sec: 0.87 + 48%|██████████████████████████████████████████████████████▏ | 19799/40903 [7:56:22<6:43:15, 1.15s/it][2025-04-21 00:52:57] (step=0019800) Train Loss: 6.3235, Train Steps/Sec: 0.86 + 48%|██████████████████████████████████████████████████████▎ | 19824/40903 [7:56:51<6:50:27, 1.17s/it][2025-04-21 00:53:26] (step=0019825) Train Loss: 6.3425, Train Steps/Sec: 0.87 + 49%|██████████████████████████████████████████████████████▎ | 19849/40903 [7:57:20<6:48:02, 1.16s/it][2025-04-21 00:53:55] (step=0019850) Train Loss: 6.3572, Train Steps/Sec: 0.87 + 49%|██████████████████████████████████████████████████████▍ | 19874/40903 [7:57:49<6:48:10, 1.16s/it][2025-04-21 00:54:24] (step=0019875) Train Loss: 6.3215, Train Steps/Sec: 0.86 + 49%|██████████████████████████████████████████████████████▍ | 19899/40903 [7:58:17<6:39:46, 1.14s/it][2025-04-21 00:54:53] (step=0019900) Train Loss: 6.3700, Train Steps/Sec: 0.86 + 49%|██████████████████████████████████████████████████████▌ | 19924/40903 [7:58:46<6:47:25, 1.17s/it][2025-04-21 00:55:22] (step=0019925) Train Loss: 6.3583, Train Steps/Sec: 0.87 + 49%|██████████████████████████████████████████████████████▌ | 19949/40903 [7:59:15<6:43:28, 1.16s/it][2025-04-21 00:55:51] (step=0019950) Train Loss: 6.3661, Train Steps/Sec: 0.86 + 49%|██████████████████████████████████████████████████████▋ | 19974/40903 [7:59:44<6:38:54, 1.14s/it][2025-04-21 00:56:20] (step=0019975) Train Loss: 6.3822, Train Steps/Sec: 0.86 + 49%|██████████████████████████████████████████████████████▊ | 19999/40903 [8:00:13<6:40:48, 1.15s/it][2025-04-21 00:56:49] (step=0020000) Train Loss: 6.3599, Train Steps/Sec: 0.86 +[2025-04-21 00:56:49] vision_config is None. initializing the InstructBlipVisionConfig with default values. +[2025-04-21 00:56:49] qformer_config is None. Initializing the InstructBlipQFormerConfig with default values. +[2025-04-21 00:56:49] text_config is None. Initializing the text config with default values (`OPTConfig`). +model_name_or_path:/tmp/haozhezhao/model/instructblip-flan-t5-xl +image_place_holder: +tokenizer length after expend 32102 +tokenizer length before expend 32102 +100%|████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████| 3/3 [03:04<00:00, 61.50s/it] +[2025-04-21 01:01:45] Finish Eval in 20000 steps...█████████████████████████████████████████████████████████████████████| 3/3 [03:04<00:00, 61.15s/it] +processing left-padding... +processing left-padding... +processing left-padding... +[2025-04-21 01:02:06] Saved checkpoint to checkpoint/EmptyFix_InstructBlip_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_objects/002-GPT-XL/checkpoints/0020000.pt +[2025-04-21 01:02:09] Removed old checkpoint: checkpoint/EmptyFix_InstructBlip_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_objects/002-GPT-XL/checkpoints/0016000.pt + 49%|██████████████████████████████████████████████████████▊ | 20024/40903 [8:06:09<7:04:24, 1.22s/it][2025-04-21 01:02:45] (step=0020025) Train Loss: 6.4007, Train Steps/Sec: 0.07 + 49%|██████████████████████████████████████████████████████▉ | 20049/40903 [8:06:38<6:44:52, 1.16s/it][2025-04-21 01:03:14] (step=0020050) Train Loss: 6.3626, Train Steps/Sec: 0.86 + 49%|██████████████████████████████████████████████████████▉ | 20074/40903 [8:07:07<6:41:17, 1.16s/it][2025-04-21 01:03:43] (step=0020075) Train Loss: 6.3841, Train Steps/Sec: 0.86 + 49%|███████████████████████████████████████████████████████ | 20099/40903 [8:07:36<6:34:00, 1.14s/it][2025-04-21 01:04:12] (step=0020100) Train Loss: 6.3703, Train Steps/Sec: 0.86 + 49%|███████████████████████████████████████████████████████ | 20124/40903 [8:08:05<6:43:49, 1.17s/it][2025-04-21 01:04:41] (step=0020125) Train Loss: 6.3846, Train Steps/Sec: 0.86 + 49%|███████████████████████████████████████████████████████▏ | 20149/40903 [8:08:34<6:38:05, 1.15s/it][2025-04-21 01:05:10] (step=0020150) Train Loss: 6.4086, Train Steps/Sec: 0.86 + 49%|███████████████████████████████████████████████████████▏ | 20174/40903 [8:09:03<6:41:36, 1.16s/it][2025-04-21 01:05:39] (step=0020175) Train Loss: 6.3703, Train Steps/Sec: 0.86 + 49%|███████████████████████████████████████████████████████▎ | 20199/40903 [8:09:33<6:41:41, 1.16s/it][2025-04-21 01:06:08] (step=0020200) Train Loss: 6.4060, Train Steps/Sec: 0.85 + 49%|███████████████████████████████████████████████████████▍ | 20224/40903 [8:10:02<6:41:16, 1.16s/it][2025-04-21 01:06:37] (step=0020225) Train Loss: 6.3570, Train Steps/Sec: 0.86 + 50%|███████████████████████████████████████████████████████▍ | 20249/40903 [8:10:31<6:33:28, 1.14s/it][2025-04-21 01:07:06] (step=0020250) Train Loss: 6.3316, Train Steps/Sec: 0.87 + 50%|███████████████████████████████████████████████████████▌ | 20274/40903 [8:11:00<6:38:46, 1.16s/it][2025-04-21 01:07:35] (step=0020275) Train Loss: 6.4294, Train Steps/Sec: 0.86 + 50%|███████████████████████████████████████████████████████▌ | 20299/40903 [8:11:29<6:33:35, 1.15s/it][2025-04-21 01:08:04] (step=0020300) Train Loss: 6.3632, Train Steps/Sec: 0.86 + 50%|███████████████████████████████████████████████████████▋ | 20324/40903 [8:11:58<6:55:10, 1.21s/it][2025-04-21 01:08:34] (step=0020325) Train Loss: 6.3973, Train Steps/Sec: 0.85 + 50%|███████████████████████████████████████████████████████▋ | 20349/40903 [8:12:27<6:36:29, 1.16s/it][2025-04-21 01:09:03] (step=0020350) Train Loss: 6.3975, Train Steps/Sec: 0.86 + 50%|███████████████████████████████████████████████████████▊ | 20374/40903 [8:12:56<6:42:27, 1.18s/it][2025-04-21 01:09:32] (step=0020375) Train Loss: 6.3380, Train Steps/Sec: 0.86 + 50%|███████████████████████████████████████████████████████▊ | 20399/40903 [8:13:25<6:34:17, 1.15s/it][2025-04-21 01:10:01] (step=0020400) Train Loss: 6.3360, Train Steps/Sec: 0.86 + 50%|███████████████████████████████████████████████████████▉ | 20424/40903 [8:13:54<6:40:53, 1.17s/it][2025-04-21 01:10:30] (step=0020425) Train Loss: 6.3581, Train Steps/Sec: 0.87 + 50%|███████████████████████████████████████████████████████▉ | 20449/40903 [8:14:23<6:33:01, 1.15s/it][2025-04-21 01:10:59] (step=0020450) Train Loss: 6.3592, Train Steps/Sec: 0.86 + 50%|████████████████████████████████████████████████████████ | 20474/40903 [8:14:52<6:29:54, 1.15s/it][2025-04-21 01:11:28] (step=0020475) Train Loss: 6.3668, Train Steps/Sec: 0.87 + 50%|████████████████████████████████████████████████████████▏ | 20499/40903 [8:15:21<6:35:41, 1.16s/it][2025-04-21 01:11:57] (step=0020500) Train Loss: 6.3508, Train Steps/Sec: 0.86 + 50%|████████████████████████████████████████████████████████▏ | 20524/40903 [8:15:50<6:46:05, 1.20s/it][2025-04-21 01:12:26] (step=0020525) Train Loss: 6.4139, Train Steps/Sec: 0.86 + 50%|████████████████████████████████████████████████████████▎ | 20549/40903 [8:16:19<6:35:48, 1.17s/it][2025-04-21 01:12:55] (step=0020550) Train Loss: 6.3422, Train Steps/Sec: 0.86 + 50%|████████████████████████████████████████████████████████▎ | 20574/40903 [8:16:48<6:25:24, 1.14s/it][2025-04-21 01:13:24] (step=0020575) Train Loss: 6.3582, Train Steps/Sec: 0.87 + 50%|████████████████████████████████████████████████████████▍ | 20599/40903 [8:17:17<6:24:33, 1.14s/it][2025-04-21 01:13:53] (step=0020600) Train Loss: 6.3568, Train Steps/Sec: 0.86 + 50%|████████████████████████████████████████████████████████▍ | 20624/40903 [8:17:46<6:35:44, 1.17s/it][2025-04-21 01:14:21] (step=0020625) Train Loss: 6.4143, Train Steps/Sec: 0.86 + 50%|████████████████████████████████████████████████████████▌ | 20649/40903 [8:18:15<6:29:45, 1.15s/it][2025-04-21 01:14:50] (step=0020650) Train Loss: 6.3262, Train Steps/Sec: 0.86 + 51%|████████████████████████████████████████████████████████▌ | 20674/40903 [8:18:44<6:25:12, 1.14s/it][2025-04-21 01:15:19] (step=0020675) Train Loss: 6.3309, Train Steps/Sec: 0.87 + 51%|████████████████████████████████████████████████████████▋ | 20699/40903 [8:19:13<6:25:14, 1.14s/it][2025-04-21 01:15:48] (step=0020700) Train Loss: 6.3539, Train Steps/Sec: 0.86 + 51%|████████████████████████████████████████████████████████▋ | 20724/40903 [8:19:42<6:37:37, 1.18s/it][2025-04-21 01:16:17] (step=0020725) Train Loss: 6.3750, Train Steps/Sec: 0.87 + 51%|████████████████████████████████████████████████████████▊ | 20749/40903 [8:20:11<6:32:35, 1.17s/it][2025-04-21 01:16:46] (step=0020750) Train Loss: 6.3620, Train Steps/Sec: 0.86 + 51%|████████████████████████████████████████████████████████▉ | 20774/40903 [8:20:40<6:27:33, 1.16s/it][2025-04-21 01:17:15] (step=0020775) Train Loss: 6.3677, Train Steps/Sec: 0.86 + 51%|████████████████████████████████████████████████████████▉ | 20799/40903 [8:21:09<6:21:35, 1.14s/it][2025-04-21 01:17:44] (step=0020800) Train Loss: 6.3786, Train Steps/Sec: 0.86 + 51%|█████████████████████████████████████████████████████████ | 20824/40903 [8:21:38<6:34:40, 1.18s/it][2025-04-21 01:18:13] (step=0020825) Train Loss: 6.3137, Train Steps/Sec: 0.87 + 51%|█████████████████████████████████████████████████████████ | 20849/40903 [8:22:07<6:26:49, 1.16s/it][2025-04-21 01:18:42] (step=0020850) Train Loss: 6.3338, Train Steps/Sec: 0.86 + 51%|█████████████████████████████████████████████████████████▏ | 20874/40903 [8:22:36<6:22:57, 1.15s/it][2025-04-21 01:19:11] (step=0020875) Train Loss: 6.3547, Train Steps/Sec: 0.86 + 51%|█████████████████████████████████████████████████████████▏ | 20899/40903 [8:23:05<6:22:20, 1.15s/it][2025-04-21 01:19:40] (step=0020900) Train Loss: 6.3664, Train Steps/Sec: 0.86 + 51%|█████████████████████████████████████████████████████████▎ | 20924/40903 [8:23:34<6:31:43, 1.18s/it][2025-04-21 01:20:09] (step=0020925) Train Loss: 6.3483, Train Steps/Sec: 0.86 + 51%|█████████████████████████████████████████████████████████▎ | 20949/40903 [8:24:03<6:29:02, 1.17s/it][2025-04-21 01:20:38] (step=0020950) Train Loss: 6.3652, Train Steps/Sec: 0.86 + 51%|█████████████████████████████████████████████████████████▍ | 20974/40903 [8:24:32<6:22:45, 1.15s/it][2025-04-21 01:21:07] (step=0020975) Train Loss: 6.4175, Train Steps/Sec: 0.86 + 51%|█████████████████████████████████████████████████████████▍ | 20999/40903 [8:25:01<6:22:12, 1.15s/it][2025-04-21 01:21:36] (step=0021000) Train Loss: 6.3432, Train Steps/Sec: 0.86 + 51%|█████████████████████████████████████████████████████████▌ | 21024/40903 [8:25:30<6:31:51, 1.18s/it][2025-04-21 01:22:05] (step=0021025) Train Loss: 6.4031, Train Steps/Sec: 0.86 + 51%|█████████████████████████████████████████████████████████▋ | 21049/40903 [8:25:59<6:25:20, 1.16s/it][2025-04-21 01:22:34] (step=0021050) Train Loss: 6.4267, Train Steps/Sec: 0.86 + 52%|█████████████████████████████████████████████████████████▋ | 21074/40903 [8:26:28<6:18:29, 1.15s/it][2025-04-21 01:23:03] (step=0021075) Train Loss: 6.3494, Train Steps/Sec: 0.87 + 52%|█████████████████████████████████████████████████████████▊ | 21099/40903 [8:26:56<6:21:16, 1.16s/it][2025-04-21 01:23:32] (step=0021100) Train Loss: 6.3606, Train Steps/Sec: 0.86 + 52%|█████████████████████████████████████████████████████████▊ | 21124/40903 [8:27:26<6:30:21, 1.18s/it][2025-04-21 01:24:01] (step=0021125) Train Loss: 6.3786, Train Steps/Sec: 0.86 + 52%|█████████████████████████████████████████████████████████▉ | 21149/40903 [8:27:55<6:24:56, 1.17s/it][2025-04-21 01:24:30] (step=0021150) Train Loss: 6.3281, Train Steps/Sec: 0.86 + 52%|█████████████████████████████████████████████████████████▉ | 21174/40903 [8:28:24<6:15:36, 1.14s/it][2025-04-21 01:24:59] (step=0021175) Train Loss: 6.3491, Train Steps/Sec: 0.87 + 52%|██████████████████████████████████████████████████████████ | 21199/40903 [8:28:52<6:12:58, 1.14s/it][2025-04-21 01:25:28] (step=0021200) Train Loss: 6.3257, Train Steps/Sec: 0.86 + 52%|██████████████████████████████████████████████████████████ | 21224/40903 [8:29:21<6:21:52, 1.16s/it][2025-04-21 01:25:57] (step=0021225) Train Loss: 6.3485, Train Steps/Sec: 0.87 + 52%|██████████████████████████████████████████████████████████▏ | 21249/40903 [8:29:51<6:22:08, 1.17s/it][2025-04-21 01:26:26] (step=0021250) Train Loss: 6.3734, Train Steps/Sec: 0.86 + 52%|██████████████████████████████████████████████████████████▎ | 21274/40903 [8:30:19<6:15:59, 1.15s/it][2025-04-21 01:26:55] (step=0021275) Train Loss: 6.3573, Train Steps/Sec: 0.86 + 52%|██████████████████████████████████████████████████████████▎ | 21299/40903 [8:30:48<6:14:06, 1.15s/it][2025-04-21 01:27:24] (step=0021300) Train Loss: 6.3836, Train Steps/Sec: 0.86 + 52%|██████████████████████████████████████████████████████████▍ | 21324/40903 [8:31:17<6:18:42, 1.16s/it][2025-04-21 01:27:53] (step=0021325) Train Loss: 6.3561, Train Steps/Sec: 0.87 + 52%|██████████████████████████████████████████████████████████▍ | 21349/40903 [8:31:46<6:13:31, 1.15s/it][2025-04-21 01:28:22] (step=0021350) Train Loss: 6.3639, Train Steps/Sec: 0.87 + 52%|██████████████████████████████████████████████████████████▌ | 21374/40903 [8:32:15<6:14:01, 1.15s/it][2025-04-21 01:28:51] (step=0021375) Train Loss: 6.3065, Train Steps/Sec: 0.87 + 52%|██████████████████████████████████████████████████████████ | 21399/40903 [8:32:50<11:22:35, 2.10s/it][2025-04-21 01:29:26] (step=0021400) Train Loss: 6.3759, Train Steps/Sec: 0.70 + 52%|██████████████████████████████████████████████████████████▋ | 21424/40903 [8:33:19<6:16:36, 1.16s/it][2025-04-21 01:29:55] (step=0021425) Train Loss: 6.3554, Train Steps/Sec: 0.87 + 52%|██████████████████████████████████████████████████████████▋ | 21449/40903 [8:33:48<6:15:24, 1.16s/it][2025-04-21 01:30:24] (step=0021450) Train Loss: 6.4091, Train Steps/Sec: 0.86 + 52%|██████████████████████████████████████████████████████████▊ | 21474/40903 [8:34:17<6:09:02, 1.14s/it][2025-04-21 01:30:53] (step=0021475) Train Loss: 6.3991, Train Steps/Sec: 0.86 + 53%|██████████████████████████████████████████████████████████▊ | 21499/40903 [8:34:46<6:12:56, 1.15s/it][2025-04-21 01:31:22] (step=0021500) Train Loss: 6.3791, Train Steps/Sec: 0.86 + 53%|██████████████████████████████████████████████████████████▉ | 21524/40903 [8:35:15<6:14:11, 1.16s/it][2025-04-21 01:31:51] (step=0021525) Train Loss: 6.3652, Train Steps/Sec: 0.87 + 53%|███████████████████████████████████████████████████████████ | 21549/40903 [8:35:44<6:12:38, 1.16s/it][2025-04-21 01:32:20] (step=0021550) Train Loss: 6.3358, Train Steps/Sec: 0.87 + 53%|███████████████████████████████████████████████████████████ | 21574/40903 [8:36:13<6:08:40, 1.14s/it][2025-04-21 01:32:49] (step=0021575) Train Loss: 6.3758, Train Steps/Sec: 0.86 + 53%|███████████████████████████████████████████████████████████▏ | 21599/40903 [8:36:42<6:10:35, 1.15s/it][2025-04-21 01:33:18] (step=0021600) Train Loss: 6.3597, Train Steps/Sec: 0.86 + 53%|███████████████████████████████████████████████████████████▏ | 21624/40903 [8:37:11<6:20:31, 1.18s/it][2025-04-21 01:33:47] (step=0021625) Train Loss: 6.3683, Train Steps/Sec: 0.86 + 53%|███████████████████████████████████████████████████████████▎ | 21649/40903 [8:37:40<6:12:12, 1.16s/it][2025-04-21 01:34:16] (step=0021650) Train Loss: 6.3814, Train Steps/Sec: 0.86 + 53%|███████████████████████████████████████████████████████████▎ | 21674/40903 [8:38:09<6:11:47, 1.16s/it][2025-04-21 01:34:45] (step=0021675) Train Loss: 6.3356, Train Steps/Sec: 0.86 + 53%|███████████████████████████████████████████████████████████▍ | 21699/40903 [8:38:38<6:04:42, 1.14s/it][2025-04-21 01:35:14] (step=0021700) Train Loss: 6.3326, Train Steps/Sec: 0.86 + 53%|███████████████████████████████████████████████████████████▍ | 21724/40903 [8:39:07<6:14:17, 1.17s/it][2025-04-21 01:35:43] (step=0021725) Train Loss: 6.3467, Train Steps/Sec: 0.86 + 53%|███████████████████████████████████████████████████████████▌ | 21749/40903 [8:39:36<6:10:02, 1.16s/it][2025-04-21 01:36:12] (step=0021750) Train Loss: 6.3465, Train Steps/Sec: 0.86 + 53%|███████████████████████████████████████████████████████████▌ | 21774/40903 [8:40:05<6:06:19, 1.15s/it][2025-04-21 01:36:41] (step=0021775) Train Loss: 6.3814, Train Steps/Sec: 0.87 + 53%|███████████████████████████████████████████████████████████▋ | 21799/40903 [8:40:34<6:05:50, 1.15s/it][2025-04-21 01:37:10] (step=0021800) Train Loss: 6.3341, Train Steps/Sec: 0.86 + 53%|███████████████████████████████████████████████████████████▊ | 21824/40903 [8:41:03<6:15:04, 1.18s/it][2025-04-21 01:37:39] (step=0021825) Train Loss: 6.3846, Train Steps/Sec: 0.86 + 53%|███████████████████████████████████████████████████████████▊ | 21849/40903 [8:41:32<6:11:19, 1.17s/it][2025-04-21 01:38:08] (step=0021850) Train Loss: 6.3585, Train Steps/Sec: 0.86 + 53%|███████████████████████████████████████████████████████████▉ | 21874/40903 [8:42:01<6:01:15, 1.14s/it][2025-04-21 01:38:37] (step=0021875) Train Loss: 6.3686, Train Steps/Sec: 0.86 + 54%|███████████████████████████████████████████████████████████▉ | 21899/40903 [8:42:30<6:07:02, 1.16s/it][2025-04-21 01:39:06] (step=0021900) Train Loss: 6.3652, Train Steps/Sec: 0.86 + 54%|████████████████████████████████████████████████████████████ | 21924/40903 [8:42:59<6:13:01, 1.18s/it][2025-04-21 01:39:35] (step=0021925) Train Loss: 6.4040, Train Steps/Sec: 0.86 + 54%|████████████████████████████████████████████████████████████ | 21949/40903 [8:43:28<6:07:53, 1.16s/it][2025-04-21 01:40:04] (step=0021950) Train Loss: 6.3196, Train Steps/Sec: 0.86 + 54%|████████████████████████████████████████████████████████████▏ | 21974/40903 [8:43:57<6:00:47, 1.14s/it][2025-04-21 01:40:33] (step=0021975) Train Loss: 6.3810, Train Steps/Sec: 0.87 + 54%|████████████████████████████████████████████████████████████▏ | 21999/40903 [8:44:26<6:01:03, 1.15s/it][2025-04-21 01:41:02] (step=0022000) Train Loss: 6.3708, Train Steps/Sec: 0.86 +[2025-04-21 01:41:02] vision_config is None. initializing the InstructBlipVisionConfig with default values. +[2025-04-21 01:41:02] qformer_config is None. Initializing the InstructBlipQFormerConfig with default values. +[2025-04-21 01:41:02] text_config is None. Initializing the text config with default values (`OPTConfig`). +model_name_or_path:/tmp/haozhezhao/model/instructblip-flan-t5-xl +image_place_holder: +tokenizer length after expend 32102 +tokenizer length before expend 32102 +100%|████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████| 3/3 [03:06<00:00, 62.10s/it] +[2025-04-21 01:45:59] Finish Eval in 22000 steps...█████████████████████████████████████████████████████████████████████| 3/3 [03:05<00:00, 61.65s/it] +processing left-padding... +processing left-padding... +processing left-padding... +[2025-04-21 01:46:20] Saved checkpoint to checkpoint/EmptyFix_InstructBlip_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_objects/002-GPT-XL/checkpoints/0022000.pt +[2025-04-21 01:46:22] Removed old checkpoint: checkpoint/EmptyFix_InstructBlip_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_objects/002-GPT-XL/checkpoints/0018000.pt + 54%|████████████████████████████████████████████████████████████▎ | 22024/40903 [8:50:16<6:14:18, 1.19s/it][2025-04-21 01:46:52] (step=0022025) Train Loss: 6.3658, Train Steps/Sec: 0.07 + 54%|████████████████████████████████████████████████████████████▎ | 22049/40903 [8:50:45<6:04:48, 1.16s/it][2025-04-21 01:47:20] (step=0022050) Train Loss: 6.3666, Train Steps/Sec: 0.87 + 54%|████████████████████████████████████████████████████████████▍ | 22074/40903 [8:51:14<6:01:00, 1.15s/it][2025-04-21 01:47:49] (step=0022075) Train Loss: 6.3475, Train Steps/Sec: 0.86 + 54%|████████████████████████████████████████████████████████████▌ | 22099/40903 [8:51:44<7:02:24, 1.35s/it][2025-04-21 01:48:20] (step=0022100) Train Loss: 6.3346, Train Steps/Sec: 0.81 + 54%|████████████████████████████████████████████████████████████▌ | 22124/40903 [8:52:14<6:11:16, 1.19s/it][2025-04-21 01:48:49] (step=0022125) Train Loss: 6.2973, Train Steps/Sec: 0.86 + 54%|████████████████████████████████████████████████████████████▋ | 22149/40903 [8:52:43<6:06:11, 1.17s/it][2025-04-21 01:49:18] (step=0022150) Train Loss: 6.3615, Train Steps/Sec: 0.86 + 54%|████████████████████████████████████████████████████████████▋ | 22174/40903 [8:53:12<6:00:54, 1.16s/it][2025-04-21 01:49:47] (step=0022175) Train Loss: 6.4504, Train Steps/Sec: 0.85 + 54%|████████████████████████████████████████████████████████████▊ | 22199/40903 [8:53:41<5:54:51, 1.14s/it][2025-04-21 01:50:17] (step=0022200) Train Loss: 6.3179, Train Steps/Sec: 0.86 + 54%|████████████████████████████████████████████████████████████▊ | 22224/40903 [8:54:10<6:03:52, 1.17s/it][2025-04-21 01:50:45] (step=0022225) Train Loss: 6.3681, Train Steps/Sec: 0.86 + 54%|████████████████████████████████████████████████████████████▉ | 22249/40903 [8:54:39<6:02:18, 1.17s/it][2025-04-21 01:51:14] (step=0022250) Train Loss: 6.3423, Train Steps/Sec: 0.87 + 54%|████████████████████████████████████████████████████████████▉ | 22274/40903 [8:55:08<5:54:58, 1.14s/it][2025-04-21 01:51:43] (step=0022275) Train Loss: 6.3634, Train Steps/Sec: 0.87 + 55%|█████████████████████████████████████████████████████████████ | 22299/40903 [8:55:36<5:57:42, 1.15s/it][2025-04-21 01:52:12] (step=0022300) Train Loss: 6.3722, Train Steps/Sec: 0.86 + 55%|█████████████████████████████████████████████████████████████▏ | 22324/40903 [8:56:06<6:04:27, 1.18s/it][2025-04-21 01:52:41] (step=0022325) Train Loss: 6.3243, Train Steps/Sec: 0.86 + 55%|█████████████████████████████████████████████████████████████▏ | 22349/40903 [8:56:34<6:01:32, 1.17s/it][2025-04-21 01:53:10] (step=0022350) Train Loss: 6.3421, Train Steps/Sec: 0.86 + 55%|█████████████████████████████████████████████████████████████▎ | 22374/40903 [8:57:04<5:53:27, 1.14s/it][2025-04-21 01:53:39] (step=0022375) Train Loss: 6.3052, Train Steps/Sec: 0.86 + 55%|█████████████████████████████████████████████████████████████▎ | 22399/40903 [8:57:33<5:51:35, 1.14s/it][2025-04-21 01:54:08] (step=0022400) Train Loss: 6.3891, Train Steps/Sec: 0.86 + 55%|█████████████████████████████████████████████████████████████▍ | 22424/40903 [8:58:02<6:01:44, 1.17s/it][2025-04-21 01:54:37] (step=0022425) Train Loss: 6.3591, Train Steps/Sec: 0.86 + 55%|█████████████████████████████████████████████████████████████▍ | 22449/40903 [8:58:31<5:59:47, 1.17s/it][2025-04-21 01:55:06] (step=0022450) Train Loss: 6.3219, Train Steps/Sec: 0.86 + 55%|█████████████████████████████████████████████████████████████▌ | 22474/40903 [8:59:00<5:58:07, 1.17s/it][2025-04-21 01:55:35] (step=0022475) Train Loss: 6.4234, Train Steps/Sec: 0.86 + 55%|█████████████████████████████████████████████████████████████▌ | 22499/40903 [8:59:29<5:56:22, 1.16s/it][2025-04-21 01:56:05] (step=0022500) Train Loss: 6.3545, Train Steps/Sec: 0.86 + 55%|█████████████████████████████████████████████████████████████▋ | 22524/40903 [8:59:58<5:56:03, 1.16s/it][2025-04-21 01:56:34] (step=0022525) Train Loss: 6.3259, Train Steps/Sec: 0.86 + 55%|█████████████████████████████████████████████████████████████▋ | 22549/40903 [9:00:27<5:53:34, 1.16s/it][2025-04-21 01:57:02] (step=0022550) Train Loss: 6.3740, Train Steps/Sec: 0.87 + 55%|█████████████████████████████████████████████████████████████▊ | 22574/40903 [9:00:56<5:52:59, 1.16s/it][2025-04-21 01:57:31] (step=0022575) Train Loss: 6.3532, Train Steps/Sec: 0.86 + 55%|█████████████████████████████████████████████████████████████▉ | 22599/40903 [9:01:25<5:53:47, 1.16s/it][2025-04-21 01:58:01] (step=0022600) Train Loss: 6.3768, Train Steps/Sec: 0.86 + 55%|█████████████████████████████████████████████████████████████▉ | 22624/40903 [9:01:54<6:02:18, 1.19s/it][2025-04-21 01:58:30] (step=0022625) Train Loss: 6.3661, Train Steps/Sec: 0.86 + 55%|██████████████████████████████████████████████████████████████ | 22649/40903 [9:02:23<5:53:30, 1.16s/it][2025-04-21 01:58:59] (step=0022650) Train Loss: 6.3682, Train Steps/Sec: 0.86 + 55%|██████████████████████████████████████████████████████████████ | 22674/40903 [9:02:52<5:55:07, 1.17s/it][2025-04-21 01:59:28] (step=0022675) Train Loss: 6.3293, Train Steps/Sec: 0.87 + 55%|██████████████████████████████████████████████████████████████▏ | 22699/40903 [9:03:21<5:46:14, 1.14s/it][2025-04-21 01:59:57] (step=0022700) Train Loss: 6.3710, Train Steps/Sec: 0.86 + 56%|██████████████████████████████████████████████████████████████▏ | 22724/40903 [9:03:50<5:57:26, 1.18s/it][2025-04-21 02:00:26] (step=0022725) Train Loss: 6.4127, Train Steps/Sec: 0.87 + 56%|██████████████████████████████████████████████████████████████▎ | 22749/40903 [9:04:19<5:49:05, 1.15s/it][2025-04-21 02:00:55] (step=0022750) Train Loss: 6.3910, Train Steps/Sec: 0.86 + 56%|██████████████████████████████████████████████████████████████▎ | 22774/40903 [9:04:48<5:47:31, 1.15s/it][2025-04-21 02:01:23] (step=0022775) Train Loss: 6.3761, Train Steps/Sec: 0.87 + 56%|██████████████████████████████████████████████████████████████▍ | 22799/40903 [9:05:17<5:45:09, 1.14s/it][2025-04-21 02:01:52] (step=0022800) Train Loss: 6.3517, Train Steps/Sec: 0.86 + 56%|██████████████████████████████████████████████████████████████▍ | 22824/40903 [9:05:46<6:00:21, 1.20s/it][2025-04-21 02:02:21] (step=0022825) Train Loss: 6.4041, Train Steps/Sec: 0.86 + 56%|██████████████████████████████████████████████████████████████▌ | 22849/40903 [9:06:15<5:50:05, 1.16s/it][2025-04-21 02:02:50] (step=0022850) Train Loss: 6.3618, Train Steps/Sec: 0.87 + 56%|██████████████████████████████████████████████████████████████▋ | 22874/40903 [9:06:44<5:42:27, 1.14s/it][2025-04-21 02:03:19] (step=0022875) Train Loss: 6.3327, Train Steps/Sec: 0.86 + 56%|██████████████████████████████████████████████████████████████▋ | 22899/40903 [9:07:12<5:43:41, 1.15s/it][2025-04-21 02:03:48] (step=0022900) Train Loss: 6.3585, Train Steps/Sec: 0.86 + 56%|██████████████████████████████████████████████████████████████▊ | 22924/40903 [9:07:41<5:51:36, 1.17s/it][2025-04-21 02:04:17] (step=0022925) Train Loss: 6.3672, Train Steps/Sec: 0.87 + 56%|██████████████████████████████████████████████████████████████▊ | 22949/40903 [9:08:10<5:46:03, 1.16s/it][2025-04-21 02:04:46] (step=0022950) Train Loss: 6.3508, Train Steps/Sec: 0.86 + 56%|██████████████████████████████████████████████████████████████▉ | 22974/40903 [9:08:39<5:44:44, 1.15s/it][2025-04-21 02:05:15] (step=0022975) Train Loss: 6.3416, Train Steps/Sec: 0.86 + 56%|██████████████████████████████████████████████████████████████▉ | 22999/40903 [9:09:09<5:42:58, 1.15s/it][2025-04-21 02:05:44] (step=0023000) Train Loss: 6.3727, Train Steps/Sec: 0.86 + 56%|███████████████████████████████████████████████████████████████ | 23024/40903 [9:09:37<5:45:08, 1.16s/it][2025-04-21 02:06:13] (step=0023025) Train Loss: 6.3308, Train Steps/Sec: 0.87 + 56%|███████████████████████████████████████████████████████████████ | 23049/40903 [9:10:07<5:45:39, 1.16s/it][2025-04-21 02:06:42] (step=0023050) Train Loss: 6.3897, Train Steps/Sec: 0.86 + 56%|███████████████████████████████████████████████████████████████▏ | 23074/40903 [9:10:35<5:38:40, 1.14s/it][2025-04-21 02:07:11] (step=0023075) Train Loss: 6.3515, Train Steps/Sec: 0.87 + 56%|███████████████████████████████████████████████████████████████▏ | 23099/40903 [9:11:04<5:39:22, 1.14s/it][2025-04-21 02:07:40] (step=0023100) Train Loss: 6.3310, Train Steps/Sec: 0.87 + 57%|███████████████████████████████████████████████████████████████▎ | 23124/40903 [9:11:33<5:47:56, 1.17s/it][2025-04-21 02:08:09] (step=0023125) Train Loss: 6.3806, Train Steps/Sec: 0.86 + 57%|███████████████████████████████████████████████████████████████▍ | 23149/40903 [9:12:02<5:41:58, 1.16s/it][2025-04-21 02:08:38] (step=0023150) Train Loss: 6.3478, Train Steps/Sec: 0.87 + 57%|███████████████████████████████████████████████████████████████▍ | 23174/40903 [9:12:31<5:39:43, 1.15s/it][2025-04-21 02:09:07] (step=0023175) Train Loss: 6.3624, Train Steps/Sec: 0.87 + 57%|███████████████████████████████████████████████████████████████▌ | 23199/40903 [9:13:00<5:40:45, 1.15s/it][2025-04-21 02:09:36] (step=0023200) Train Loss: 6.4122, Train Steps/Sec: 0.86 + 57%|███████████████████████████████████████████████████████████████▌ | 23224/40903 [9:13:29<5:47:11, 1.18s/it][2025-04-21 02:10:05] (step=0023225) Train Loss: 6.3619, Train Steps/Sec: 0.86 + 57%|███████████████████████████████████████████████████████████████▋ | 23249/40903 [9:13:58<5:41:49, 1.16s/it][2025-04-21 02:10:34] (step=0023250) Train Loss: 6.3752, Train Steps/Sec: 0.86 + 57%|███████████████████████████████████████████████████████████████▋ | 23274/40903 [9:14:27<5:38:31, 1.15s/it][2025-04-21 02:11:03] (step=0023275) Train Loss: 6.3531, Train Steps/Sec: 0.86 + 57%|███████████████████████████████████████████████████████████████▊ | 23299/40903 [9:14:56<5:36:49, 1.15s/it][2025-04-21 02:11:32] (step=0023300) Train Loss: 6.3249, Train Steps/Sec: 0.86 + 57%|███████████████████████████████████████████████████████████████▊ | 23324/40903 [9:15:25<5:47:06, 1.18s/it][2025-04-21 02:12:01] (step=0023325) Train Loss: 6.3501, Train Steps/Sec: 0.86 + 57%|███████████████████████████████████████████████████████████████▉ | 23349/40903 [9:15:54<5:37:33, 1.15s/it][2025-04-21 02:12:30] (step=0023350) Train Loss: 6.3412, Train Steps/Sec: 0.87 + 57%|████████████████████████████████████████████████████████████████ | 23374/40903 [9:16:23<5:33:47, 1.14s/it][2025-04-21 02:12:59] (step=0023375) Train Loss: 6.3464, Train Steps/Sec: 0.87 + 57%|████████████████████████████████████████████████████████████████ | 23399/40903 [9:16:52<5:38:04, 1.16s/it][2025-04-21 02:13:28] (step=0023400) Train Loss: 6.3408, Train Steps/Sec: 0.85 + 57%|████████████████████████████████████████████████████████████████▏ | 23424/40903 [9:17:21<5:41:57, 1.17s/it][2025-04-21 02:13:57] (step=0023425) Train Loss: 6.3395, Train Steps/Sec: 0.86 + 57%|████████████████████████████████████████████████████████████████▏ | 23449/40903 [9:17:50<5:35:59, 1.16s/it][2025-04-21 02:14:26] (step=0023450) Train Loss: 6.3297, Train Steps/Sec: 0.87 + 57%|████████████████████████████████████████████████████████████████▎ | 23474/40903 [9:18:19<5:37:26, 1.16s/it][2025-04-21 02:14:55] (step=0023475) Train Loss: 6.3751, Train Steps/Sec: 0.86 + 57%|████████████████████████████████████████████████████████████████▎ | 23499/40903 [9:18:48<5:36:11, 1.16s/it][2025-04-21 02:15:24] (step=0023500) Train Loss: 6.3128, Train Steps/Sec: 0.86 + 58%|████████████████████████████████████████████████████████████████▍ | 23524/40903 [9:19:17<5:43:01, 1.18s/it][2025-04-21 02:15:53] (step=0023525) Train Loss: 6.3439, Train Steps/Sec: 0.86 + 58%|████████████████████████████████████████████████████████████████▍ | 23549/40903 [9:19:46<5:35:57, 1.16s/it][2025-04-21 02:16:22] (step=0023550) Train Loss: 6.3733, Train Steps/Sec: 0.86 + 58%|████████████████████████████████████████████████████████████████▌ | 23574/40903 [9:20:15<5:33:29, 1.15s/it][2025-04-21 02:16:51] (step=0023575) Train Loss: 6.3629, Train Steps/Sec: 0.86 + 58%|████████████████████████████████████████████████████████████████▌ | 23599/40903 [9:20:44<5:32:05, 1.15s/it][2025-04-21 02:17:20] (step=0023600) Train Loss: 6.3712, Train Steps/Sec: 0.86 + 58%|████████████████████████████████████████████████████████████████▋ | 23624/40903 [9:21:14<5:38:24, 1.18s/it][2025-04-21 02:17:49] (step=0023625) Train Loss: 6.3663, Train Steps/Sec: 0.86 + 58%|████████████████████████████████████████████████████████████████▊ | 23649/40903 [9:21:42<5:33:19, 1.16s/it][2025-04-21 02:18:18] (step=0023650) Train Loss: 6.3499, Train Steps/Sec: 0.87 + 58%|████████████████████████████████████████████████████████████████▊ | 23674/40903 [9:22:11<5:32:19, 1.16s/it][2025-04-21 02:18:47] (step=0023675) Train Loss: 6.4199, Train Steps/Sec: 0.86 + 58%|████████████████████████████████████████████████████████████████▉ | 23699/40903 [9:22:40<5:25:46, 1.14s/it][2025-04-21 02:19:16] (step=0023700) Train Loss: 6.3506, Train Steps/Sec: 0.87 + 58%|████████████████████████████████████████████████████████████████▉ | 23724/40903 [9:23:09<5:40:58, 1.19s/it][2025-04-21 02:19:45] (step=0023725) Train Loss: 6.3917, Train Steps/Sec: 0.86 + 58%|█████████████████████████████████████████████████████████████████ | 23749/40903 [9:23:38<5:35:07, 1.17s/it][2025-04-21 02:20:14] (step=0023750) Train Loss: 6.3995, Train Steps/Sec: 0.86 + 58%|█████████████████████████████████████████████████████████████████ | 23774/40903 [9:24:07<5:26:15, 1.14s/it][2025-04-21 02:20:43] (step=0023775) Train Loss: 6.3579, Train Steps/Sec: 0.87 + 58%|█████████████████████████████████████████████████████████████████▏ | 23799/40903 [9:24:36<5:26:15, 1.14s/it][2025-04-21 02:21:12] (step=0023800) Train Loss: 6.3829, Train Steps/Sec: 0.86 + 58%|█████████████████████████████████████████████████████████████████▏ | 23824/40903 [9:25:05<5:36:31, 1.18s/it][2025-04-21 02:21:41] (step=0023825) Train Loss: 6.3539, Train Steps/Sec: 0.86 + 58%|█████████████████████████████████████████████████████████████████▎ | 23849/40903 [9:25:34<5:29:18, 1.16s/it][2025-04-21 02:22:10] (step=0023850) Train Loss: 6.3477, Train Steps/Sec: 0.86 + 58%|█████████████████████████████████████████████████████████████████▎ | 23874/40903 [9:26:03<5:26:28, 1.15s/it][2025-04-21 02:22:39] (step=0023875) Train Loss: 6.3316, Train Steps/Sec: 0.86 + 58%|█████████████████████████████████████████████████████████████████▍ | 23899/40903 [9:26:32<5:26:29, 1.15s/it][2025-04-21 02:23:08] (step=0023900) Train Loss: 6.3634, Train Steps/Sec: 0.86 + 58%|█████████████████████████████████████████████████████████████████▌ | 23924/40903 [9:27:01<5:40:14, 1.20s/it][2025-04-21 02:23:37] (step=0023925) Train Loss: 6.3681, Train Steps/Sec: 0.86 + 59%|█████████████████████████████████████████████████████████████████▌ | 23949/40903 [9:27:30<5:30:07, 1.17s/it][2025-04-21 02:24:06] (step=0023950) Train Loss: 6.3486, Train Steps/Sec: 0.86 + 59%|█████████████████████████████████████████████████████████████████▋ | 23974/40903 [9:27:59<5:28:37, 1.16s/it][2025-04-21 02:24:35] (step=0023975) Train Loss: 6.3461, Train Steps/Sec: 0.87 + 59%|█████████████████████████████████████████████████████████████████▋ | 23999/40903 [9:28:34<9:27:09, 2.01s/it][2025-04-21 02:25:10] (step=0024000) Train Loss: 6.4093, Train Steps/Sec: 0.72 +[2025-04-21 02:25:10] vision_config is None. initializing the InstructBlipVisionConfig with default values. +[2025-04-21 02:25:10] qformer_config is None. Initializing the InstructBlipQFormerConfig with default values. +[2025-04-21 02:25:10] text_config is None. Initializing the text config with default values (`OPTConfig`). +model_name_or_path:/tmp/haozhezhao/model/instructblip-flan-t5-xl +image_place_holder: +tokenizer length after expend 32102 +tokenizer length before expend 32102 +100%|████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████| 3/3 [03:10<00:00, 63.44s/it] +[2025-04-21 02:30:12] Finish Eval in 24000 steps...█████████████████████████████████████████████████████████████████████| 3/3 [03:09<00:00, 63.05s/it] +processing left-padding... +processing left-padding... +processing left-padding... +[2025-04-21 02:30:34] Saved checkpoint to checkpoint/EmptyFix_InstructBlip_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_objects/002-GPT-XL/checkpoints/0024000.pt +[2025-04-21 02:30:36] Removed old checkpoint: checkpoint/EmptyFix_InstructBlip_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_objects/002-GPT-XL/checkpoints/0020000.pt + 59%|█████████████████████████████████████████████████████████████████▊ | 24024/40903 [9:34:30<5:40:40, 1.21s/it][2025-04-21 02:31:06] (step=0024025) Train Loss: 6.3558, Train Steps/Sec: 0.07 + 59%|█████████████████████████████████████████████████████████████████▊ | 24049/40903 [9:34:59<5:28:07, 1.17s/it][2025-04-21 02:31:35] (step=0024050) Train Loss: 6.3518, Train Steps/Sec: 0.86 + 59%|█████████████████████████████████████████████████████████████████▉ | 24074/40903 [9:35:28<5:22:35, 1.15s/it][2025-04-21 02:32:04] (step=0024075) Train Loss: 6.3659, Train Steps/Sec: 0.86 + 59%|█████████████████████████████████████████████████████████████████▉ | 24099/40903 [9:35:57<5:24:56, 1.16s/it][2025-04-21 02:32:33] (step=0024100) Train Loss: 6.4010, Train Steps/Sec: 0.86 + 59%|██████████████████████████████████████████████████████████████████ | 24124/40903 [9:36:26<5:23:32, 1.16s/it][2025-04-21 02:33:02] (step=0024125) Train Loss: 6.3769, Train Steps/Sec: 0.86 + 59%|██████████████████████████████████████████████████████████████████ | 24149/40903 [9:37:01<5:24:50, 1.16s/it][2025-04-21 02:33:37] (step=0024150) Train Loss: 6.3587, Train Steps/Sec: 0.71 + 59%|██████████████████████████████████████████████████████████████████▏ | 24174/40903 [9:37:36<5:24:38, 1.16s/it][2025-04-21 02:34:11] (step=0024175) Train Loss: 6.3630, Train Steps/Sec: 0.73 + 59%|██████████████████████████████████████████████████████████████████▎ | 24199/40903 [9:38:05<5:24:13, 1.16s/it][2025-04-21 02:34:41] (step=0024200) Train Loss: 6.3830, Train Steps/Sec: 0.85 + 59%|██████████████████████████████████████████████████████████████████▎ | 24224/40903 [9:38:34<5:25:23, 1.17s/it][2025-04-21 02:35:10] (step=0024225) Train Loss: 6.3690, Train Steps/Sec: 0.86 + 59%|██████████████████████████████████████████████████████████████████▍ | 24249/40903 [9:39:03<5:30:36, 1.19s/it][2025-04-21 02:35:39] (step=0024250) Train Loss: 6.3824, Train Steps/Sec: 0.86 + 59%|██████████████████████████████████████████████████████████████████▍ | 24274/40903 [9:39:32<5:18:58, 1.15s/it][2025-04-21 02:36:08] (step=0024275) Train Loss: 6.3836, Train Steps/Sec: 0.86 + 59%|██████████████████████████████████████████████████████████████████▌ | 24299/40903 [9:40:01<5:19:42, 1.16s/it][2025-04-21 02:36:37] (step=0024300) Train Loss: 6.3332, Train Steps/Sec: 0.86 + 59%|██████████████████████████████████████████████████████████████████▌ | 24324/40903 [9:40:38<5:29:01, 1.19s/it][2025-04-21 02:37:13] (step=0024325) Train Loss: 6.3419, Train Steps/Sec: 0.69 + 60%|██████████████████████████████████████████████████████████████████▋ | 24349/40903 [9:41:07<5:27:29, 1.19s/it][2025-04-21 02:37:42] (step=0024350) Train Loss: 6.3480, Train Steps/Sec: 0.86 + 60%|██████████████████████████████████████████████████████████████████▋ | 24374/40903 [9:41:35<5:15:37, 1.15s/it][2025-04-21 02:38:11] (step=0024375) Train Loss: 6.3769, Train Steps/Sec: 0.87 + 60%|██████████████████████████████████████████████████████████████████▊ | 24399/40903 [9:42:04<5:14:15, 1.14s/it][2025-04-21 02:38:40] (step=0024400) Train Loss: 6.3625, Train Steps/Sec: 0.86 + 60%|██████████████████████████████████████████████████████████████████▉ | 24424/40903 [9:42:33<5:20:05, 1.17s/it][2025-04-21 02:39:09] (step=0024425) Train Loss: 6.3737, Train Steps/Sec: 0.87 + 60%|██████████████████████████████████████████████████████████████████▉ | 24449/40903 [9:43:02<5:19:31, 1.17s/it][2025-04-21 02:39:38] (step=0024450) Train Loss: 6.3468, Train Steps/Sec: 0.86 + 60%|███████████████████████████████████████████████████████████████████ | 24474/40903 [9:43:31<5:12:21, 1.14s/it][2025-04-21 02:40:07] (step=0024475) Train Loss: 6.3555, Train Steps/Sec: 0.86 + 60%|███████████████████████████████████████████████████████████████████ | 24499/40903 [9:44:00<5:10:01, 1.13s/it][2025-04-21 02:40:36] (step=0024500) Train Loss: 6.3516, Train Steps/Sec: 0.86 + 60%|███████████████████████████████████████████████████████████████████▏ | 24524/40903 [9:44:29<5:22:05, 1.18s/it][2025-04-21 02:41:05] (step=0024525) Train Loss: 6.3670, Train Steps/Sec: 0.86 + 60%|███████████████████████████████████████████████████████████████████▏ | 24549/40903 [9:44:59<5:16:33, 1.16s/it][2025-04-21 02:41:34] (step=0024550) Train Loss: 6.3716, Train Steps/Sec: 0.86 + 60%|███████████████████████████████████████████████████████████████████▎ | 24574/40903 [9:45:28<5:14:48, 1.16s/it][2025-04-21 02:42:03] (step=0024575) Train Loss: 6.3686, Train Steps/Sec: 0.86 + 60%|███████████████████████████████████████████████████████████████████▎ | 24599/40903 [9:45:56<5:11:04, 1.14s/it][2025-04-21 02:42:32] (step=0024600) Train Loss: 6.3718, Train Steps/Sec: 0.86 + 60%|███████████████████████████████████████████████████████████████████▍ | 24624/40903 [9:46:25<5:18:43, 1.17s/it][2025-04-21 02:43:01] (step=0024625) Train Loss: 6.3501, Train Steps/Sec: 0.87 + 60%|███████████████████████████████████████████████████████████████████▍ | 24649/40903 [9:46:55<5:19:08, 1.18s/it][2025-04-21 02:43:30] (step=0024650) Train Loss: 6.3442, Train Steps/Sec: 0.85 + 60%|███████████████████████████████████████████████████████████████████▌ | 24674/40903 [9:47:24<5:08:25, 1.14s/it][2025-04-21 02:43:59] (step=0024675) Train Loss: 6.3780, Train Steps/Sec: 0.86 + 60%|███████████████████████████████████████████████████████████████████▋ | 24699/40903 [9:47:52<5:08:39, 1.14s/it][2025-04-21 02:44:28] (step=0024700) Train Loss: 6.3514, Train Steps/Sec: 0.86 + 60%|███████████████████████████████████████████████████████████████████▋ | 24724/40903 [9:48:21<5:19:46, 1.19s/it][2025-04-21 02:44:57] (step=0024725) Train Loss: 6.3843, Train Steps/Sec: 0.87 + 61%|███████████████████████████████████████████████████████████████████▊ | 24749/40903 [9:48:50<5:16:12, 1.17s/it][2025-04-21 02:45:26] (step=0024750) Train Loss: 6.3634, Train Steps/Sec: 0.87 + 61%|███████████████████████████████████████████████████████████████████▊ | 24774/40903 [9:49:19<5:14:17, 1.17s/it][2025-04-21 02:45:55] (step=0024775) Train Loss: 6.3045, Train Steps/Sec: 0.86 + 61%|███████████████████████████████████████████████████████████████████▉ | 24799/40903 [9:49:48<5:08:26, 1.15s/it][2025-04-21 02:46:24] (step=0024800) Train Loss: 6.4016, Train Steps/Sec: 0.86 + 61%|███████████████████████████████████████████████████████████████████▉ | 24824/40903 [9:50:18<5:15:53, 1.18s/it][2025-04-21 02:46:53] (step=0024825) Train Loss: 6.3643, Train Steps/Sec: 0.86 + 61%|████████████████████████████████████████████████████████████████████ | 24849/40903 [9:50:47<5:13:45, 1.17s/it][2025-04-21 02:47:22] (step=0024850) Train Loss: 6.3527, Train Steps/Sec: 0.86 + 61%|████████████████████████████████████████████████████████████████████ | 24874/40903 [9:51:16<5:07:35, 1.15s/it][2025-04-21 02:47:51] (step=0024875) Train Loss: 6.3577, Train Steps/Sec: 0.86 + 61%|████████████████████████████████████████████████████████████████████▏ | 24899/40903 [9:51:45<5:07:19, 1.15s/it][2025-04-21 02:48:20] (step=0024900) Train Loss: 6.3228, Train Steps/Sec: 0.86 + 61%|████████████████████████████████████████████████████████████████████▏ | 24924/40903 [9:52:14<5:22:45, 1.21s/it][2025-04-21 02:48:50] (step=0024925) Train Loss: 6.3498, Train Steps/Sec: 0.85 + 61%|████████████████████████████████████████████████████████████████████▎ | 24949/40903 [9:52:43<5:10:03, 1.17s/it][2025-04-21 02:49:19] (step=0024950) Train Loss: 6.3428, Train Steps/Sec: 0.86 + 61%|████████████████████████████████████████████████████████████████████▍ | 24974/40903 [9:53:12<5:04:06, 1.15s/it][2025-04-21 02:49:48] (step=0024975) Train Loss: 6.3687, Train Steps/Sec: 0.86 + 61%|████████████████████████████████████████████████████████████████████▍ | 24999/40903 [9:53:41<5:07:07, 1.16s/it][2025-04-21 02:50:17] (step=0025000) Train Loss: 6.3424, Train Steps/Sec: 0.86 + 61%|████████████████████████████████████████████████████████████████████▌ | 25024/40903 [9:54:11<5:12:43, 1.18s/it][2025-04-21 02:50:46] (step=0025025) Train Loss: 6.3565, Train Steps/Sec: 0.86 + 61%|████████████████████████████████████████████████████████████████████▌ | 25049/40903 [9:54:39<5:01:38, 1.14s/it][2025-04-21 02:51:15] (step=0025050) Train Loss: 6.3661, Train Steps/Sec: 0.87 + 61%|████████████████████████████████████████████████████████████████████▋ | 25074/40903 [9:55:08<5:08:00, 1.17s/it][2025-04-21 02:51:44] (step=0025075) Train Loss: 6.3491, Train Steps/Sec: 0.86 + 61%|████████████████████████████████████████████████████████████████████▋ | 25099/40903 [9:55:44<6:07:24, 1.39s/it][2025-04-21 02:52:20] (step=0025100) Train Loss: 6.3811, Train Steps/Sec: 0.70 + 61%|████████████████████████████████████████████████████████████████████▊ | 25124/40903 [9:56:13<5:10:03, 1.18s/it][2025-04-21 02:52:49] (step=0025125) Train Loss: 6.3843, Train Steps/Sec: 0.86 + 61%|████████████████████████████████████████████████████████████████████▊ | 25149/40903 [9:56:42<5:04:19, 1.16s/it][2025-04-21 02:53:18] (step=0025150) Train Loss: 6.3637, Train Steps/Sec: 0.86 + 62%|████████████████████████████████████████████████████████████████████▉ | 25174/40903 [9:57:11<5:03:25, 1.16s/it][2025-04-21 02:53:47] (step=0025175) Train Loss: 6.3255, Train Steps/Sec: 0.86 + 62%|████████████████████████████████████████████████████████████████████▉ | 25199/40903 [9:57:40<4:58:43, 1.14s/it][2025-04-21 02:54:16] (step=0025200) Train Loss: 6.3621, Train Steps/Sec: 0.86 + 62%|█████████████████████████████████████████████████████████████████████ | 25224/40903 [9:58:10<5:12:38, 1.20s/it][2025-04-21 02:54:45] (step=0025225) Train Loss: 6.3791, Train Steps/Sec: 0.86 + 62%|█████████████████████████████████████████████████████████████████████▏ | 25249/40903 [9:58:39<5:02:36, 1.16s/it][2025-04-21 02:55:14] (step=0025250) Train Loss: 6.4018, Train Steps/Sec: 0.86 + 62%|█████████████████████████████████████████████████████████████████████▏ | 25274/40903 [9:59:08<4:58:58, 1.15s/it][2025-04-21 02:55:43] (step=0025275) Train Loss: 6.3284, Train Steps/Sec: 0.86 + 62%|█████████████████████████████████████████████████████████████████████▎ | 25299/40903 [9:59:37<4:55:42, 1.14s/it][2025-04-21 02:56:12] (step=0025300) Train Loss: 6.3475, Train Steps/Sec: 0.86 + 62%|████████████████████████████████████████████████████████████████████▋ | 25324/40903 [10:00:06<5:06:29, 1.18s/it][2025-04-21 02:56:42] (step=0025325) Train Loss: 6.3283, Train Steps/Sec: 0.86 + 62%|████████████████████████████████████████████████████████████████████▊ | 25349/40903 [10:00:35<4:57:56, 1.15s/it][2025-04-21 02:57:11] (step=0025350) Train Loss: 6.3511, Train Steps/Sec: 0.86 + 62%|████████████████████████████████████████████████████████████████████▊ | 25374/40903 [10:01:04<5:02:05, 1.17s/it][2025-04-21 02:57:40] (step=0025375) Train Loss: 6.3670, Train Steps/Sec: 0.86 + 62%|████████████████████████████████████████████████████████████████████▉ | 25399/40903 [10:01:33<5:00:05, 1.16s/it][2025-04-21 02:58:09] (step=0025400) Train Loss: 6.3386, Train Steps/Sec: 0.86 + 62%|████████████████████████████████████████████████████████████████████▉ | 25424/40903 [10:02:02<5:02:38, 1.17s/it][2025-04-21 02:58:38] (step=0025425) Train Loss: 6.3750, Train Steps/Sec: 0.86 + 62%|█████████████████████████████████████████████████████████████████████ | 25449/40903 [10:02:31<4:57:10, 1.15s/it][2025-04-21 02:59:07] (step=0025450) Train Loss: 6.3268, Train Steps/Sec: 0.87 + 62%|█████████████████████████████████████████████████████████████████████▏ | 25474/40903 [10:03:00<4:54:08, 1.14s/it][2025-04-21 02:59:36] (step=0025475) Train Loss: 6.3573, Train Steps/Sec: 0.87 + 62%|█████████████████████████████████████████████████████████████████████▏ | 25499/40903 [10:03:29<4:54:53, 1.15s/it][2025-04-21 03:00:05] (step=0025500) Train Loss: 6.3382, Train Steps/Sec: 0.86 + 62%|█████████████████████████████████████████████████████████████████████▎ | 25524/40903 [10:03:58<4:58:50, 1.17s/it][2025-04-21 03:00:34] (step=0025525) Train Loss: 6.3784, Train Steps/Sec: 0.86 + 62%|█████████████████████████████████████████████████████████████████████▎ | 25549/40903 [10:04:27<4:54:43, 1.15s/it][2025-04-21 03:01:02] (step=0025550) Train Loss: 6.3222, Train Steps/Sec: 0.87 + 63%|█████████████████████████████████████████████████████████████████████▍ | 25574/40903 [10:05:02<5:19:06, 1.25s/it][2025-04-21 03:01:38] (step=0025575) Train Loss: 6.3341, Train Steps/Sec: 0.70 + 63%|█████████████████████████████████████████████████████████████████████▍ | 25599/40903 [10:05:31<4:51:00, 1.14s/it][2025-04-21 03:02:07] (step=0025600) Train Loss: 6.3751, Train Steps/Sec: 0.86 + 63%|█████████████████████████████████████████████████████████████████████▌ | 25624/40903 [10:06:01<4:59:43, 1.18s/it][2025-04-21 03:02:36] (step=0025625) Train Loss: 6.3363, Train Steps/Sec: 0.86 + 63%|█████████████████████████████████████████████████████████████████████▌ | 25649/40903 [10:06:29<4:55:01, 1.16s/it][2025-04-21 03:03:05] (step=0025650) Train Loss: 6.3295, Train Steps/Sec: 0.87 + 63%|█████████████████████████████████████████████████████████████████████▋ | 25674/40903 [10:06:58<4:50:57, 1.15s/it][2025-04-21 03:03:34] (step=0025675) Train Loss: 6.3694, Train Steps/Sec: 0.86 + 63%|█████████████████████████████████████████████████████████████████████▋ | 25699/40903 [10:07:27<4:53:56, 1.16s/it][2025-04-21 03:04:03] (step=0025700) Train Loss: 6.3719, Train Steps/Sec: 0.86 + 63%|█████████████████████████████████████████████████████████████████████▊ | 25724/40903 [10:07:57<5:03:04, 1.20s/it][2025-04-21 03:04:32] (step=0025725) Train Loss: 6.3420, Train Steps/Sec: 0.85 + 63%|█████████████████████████████████████████████████████████████████████▉ | 25749/40903 [10:08:26<4:54:56, 1.17s/it][2025-04-21 03:05:01] (step=0025750) Train Loss: 6.3204, Train Steps/Sec: 0.87 + 63%|█████████████████████████████████████████████████████████████████████▉ | 25774/40903 [10:08:54<4:51:19, 1.16s/it][2025-04-21 03:05:30] (step=0025775) Train Loss: 6.3874, Train Steps/Sec: 0.86 + 63%|██████████████████████████████████████████████████████████████████████ | 25799/40903 [10:09:23<4:43:59, 1.13s/it][2025-04-21 03:05:59] (step=0025800) Train Loss: 6.3793, Train Steps/Sec: 0.86 + 63%|██████████████████████████████████████████████████████████████████████ | 25824/40903 [10:09:52<4:57:12, 1.18s/it][2025-04-21 03:06:28] (step=0025825) Train Loss: 6.3670, Train Steps/Sec: 0.87 + 63%|██████████████████████████████████████████████████████████████████████▏ | 25849/40903 [10:10:21<4:49:31, 1.15s/it][2025-04-21 03:06:57] (step=0025850) Train Loss: 6.4039, Train Steps/Sec: 0.87 + 63%|██████████████████████████████████████████████████████████████████████▏ | 25874/40903 [10:10:50<4:50:35, 1.16s/it][2025-04-21 03:07:26] (step=0025875) Train Loss: 6.3791, Train Steps/Sec: 0.86 + 63%|██████████████████████████████████████████████████████████████████████▎ | 25899/40903 [10:11:19<4:46:02, 1.14s/it][2025-04-21 03:07:55] (step=0025900) Train Loss: 6.3482, Train Steps/Sec: 0.86 + 63%|██████████████████████████████████████████████████████████████████████▎ | 25924/40903 [10:11:48<4:56:56, 1.19s/it][2025-04-21 03:08:24] (step=0025925) Train Loss: 6.3696, Train Steps/Sec: 0.86 + 63%|██████████████████████████████████████████████████████████████████████▍ | 25949/40903 [10:12:18<4:47:49, 1.15s/it][2025-04-21 03:08:53] (step=0025950) Train Loss: 6.3965, Train Steps/Sec: 0.86 + 64%|██████████████████████████████████████████████████████████████████████▍ | 25974/40903 [10:12:46<4:46:28, 1.15s/it][2025-04-21 03:09:22] (step=0025975) Train Loss: 6.3744, Train Steps/Sec: 0.87 + 64%|██████████████████████████████████████████████████████████████████████▌ | 25999/40903 [10:13:15<4:46:28, 1.15s/it][2025-04-21 03:09:51] (step=0026000) Train Loss: 6.3677, Train Steps/Sec: 0.86 +[2025-04-21 03:09:51] vision_config is None. initializing the InstructBlipVisionConfig with default values. +[2025-04-21 03:09:51] qformer_config is None. Initializing the InstructBlipQFormerConfig with default values. +[2025-04-21 03:09:51] text_config is None. Initializing the text config with default values (`OPTConfig`). +model_name_or_path:/tmp/haozhezhao/model/instructblip-flan-t5-xl +image_place_holder: +tokenizer length after expend 32102 +tokenizer length before expend 32102 +100%|████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████| 3/3 [03:06<00:00, 62.05s/it] +[2025-04-21 03:14:49] Finish Eval in 26000 steps...█████████████████████████████████████████████████████████████████████| 3/3 [03:05<00:00, 61.61s/it] +processing left-padding... +processing left-padding... +processing left-padding... +[2025-04-21 03:15:10] Saved checkpoint to checkpoint/EmptyFix_InstructBlip_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_objects/002-GPT-XL/checkpoints/0026000.pt +[2025-04-21 03:15:13] Removed old checkpoint: checkpoint/EmptyFix_InstructBlip_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_objects/002-GPT-XL/checkpoints/0022000.pt + 64%|██████████████████████████████████████████████████████████████████████▌ | 26024/40903 [10:19:06<4:54:48, 1.19s/it][2025-04-21 03:15:42] (step=0026025) Train Loss: 6.3470, Train Steps/Sec: 0.07 + 64%|██████████████████████████████████████████████████████████████████████▋ | 26049/40903 [10:19:35<4:48:30, 1.17s/it][2025-04-21 03:16:11] (step=0026050) Train Loss: 6.3629, Train Steps/Sec: 0.86 + 64%|██████████████████████████████████████████████████████████████████████▊ | 26074/40903 [10:20:04<4:46:33, 1.16s/it][2025-04-21 03:16:40] (step=0026075) Train Loss: 6.3461, Train Steps/Sec: 0.87 + 64%|██████████████████████████████████████████████████████████████████████▊ | 26099/40903 [10:20:33<4:43:44, 1.15s/it][2025-04-21 03:17:09] (step=0026100) Train Loss: 6.3876, Train Steps/Sec: 0.86 + 64%|██████████████████████████████████████████████████████████████████████▉ | 26124/40903 [10:21:02<4:50:58, 1.18s/it][2025-04-21 03:17:38] (step=0026125) Train Loss: 6.3786, Train Steps/Sec: 0.86 + 64%|██████████████████████████████████████████████████████████████████████▉ | 26149/40903 [10:21:31<4:43:11, 1.15s/it][2025-04-21 03:18:06] (step=0026150) Train Loss: 6.3170, Train Steps/Sec: 0.87 + 64%|███████████████████████████████████████████████████████████████████████ | 26174/40903 [10:22:00<4:44:15, 1.16s/it][2025-04-21 03:18:35] (step=0026175) Train Loss: 6.3566, Train Steps/Sec: 0.86 + 64%|███████████████████████████████████████████████████████████████████████ | 26199/40903 [10:22:29<4:43:06, 1.16s/it][2025-04-21 03:19:05] (step=0026200) Train Loss: 6.3644, Train Steps/Sec: 0.86 + 64%|███████████████████████████████████████████████████████████████████████▏ | 26224/40903 [10:22:58<4:48:16, 1.18s/it][2025-04-21 03:19:34] (step=0026225) Train Loss: 6.3755, Train Steps/Sec: 0.86 + 64%|███████████████████████████████████████████████████████████████████████▏ | 26249/40903 [10:23:27<4:42:42, 1.16s/it][2025-04-21 03:20:02] (step=0026250) Train Loss: 6.3697, Train Steps/Sec: 0.87 + 64%|███████████████████████████████████████████████████████████████████████▎ | 26274/40903 [10:23:56<4:41:52, 1.16s/it][2025-04-21 03:20:31] (step=0026275) Train Loss: 6.3471, Train Steps/Sec: 0.87 + 64%|███████████████████████████████████████████████████████████████████████▎ | 26299/40903 [10:24:24<4:36:19, 1.14s/it][2025-04-21 03:21:00] (step=0026300) Train Loss: 6.3177, Train Steps/Sec: 0.86 + 64%|███████████████████████████████████████████████████████████████████████▍ | 26324/40903 [10:24:54<4:43:20, 1.17s/it][2025-04-21 03:21:29] (step=0026325) Train Loss: 6.3616, Train Steps/Sec: 0.87 + 64%|███████████████████████████████████████████████████████████████████████▌ | 26349/40903 [10:25:23<4:39:23, 1.15s/it][2025-04-21 03:21:58] (step=0026350) Train Loss: 6.3272, Train Steps/Sec: 0.86 + 64%|███████████████████████████████████████████████████████████████████████▌ | 26374/40903 [10:25:52<4:37:27, 1.15s/it][2025-04-21 03:22:27] (step=0026375) Train Loss: 6.3418, Train Steps/Sec: 0.85 + 65%|███████████████████████████████████████████████████████████████████████▋ | 26399/40903 [10:26:21<4:36:35, 1.14s/it][2025-04-21 03:22:57] (step=0026400) Train Loss: 6.3755, Train Steps/Sec: 0.86 + 65%|███████████████████████████████████████████████████████████████████████▋ | 26424/40903 [10:26:50<4:42:09, 1.17s/it][2025-04-21 03:23:25] (step=0026425) Train Loss: 6.3510, Train Steps/Sec: 0.87 + 65%|███████████████████████████████████████████████████████████████████████▊ | 26449/40903 [10:27:19<4:41:00, 1.17s/it][2025-04-21 03:23:55] (step=0026450) Train Loss: 6.3413, Train Steps/Sec: 0.86 + 65%|███████████████████████████████████████████████████████████████████████▊ | 26474/40903 [10:27:48<4:33:17, 1.14s/it][2025-04-21 03:24:23] (step=0026475) Train Loss: 6.3838, Train Steps/Sec: 0.86 + 65%|███████████████████████████████████████████████████████████████████████▉ | 26499/40903 [10:28:17<4:37:24, 1.16s/it][2025-04-21 03:24:52] (step=0026500) Train Loss: 6.3173, Train Steps/Sec: 0.86 + 65%|███████████████████████████████████████████████████████████████████████▉ | 26524/40903 [10:28:46<4:41:46, 1.18s/it][2025-04-21 03:25:22] (step=0026525) Train Loss: 6.3553, Train Steps/Sec: 0.85 + 65%|████████████████████████████████████████████████████████████████████████ | 26549/40903 [10:29:15<4:37:45, 1.16s/it][2025-04-21 03:25:51] (step=0026550) Train Loss: 6.3551, Train Steps/Sec: 0.86 + 65%|████████████████████████████████████████████████████████████████████████ | 26574/40903 [10:29:44<4:33:38, 1.15s/it][2025-04-21 03:26:19] (step=0026575) Train Loss: 6.3361, Train Steps/Sec: 0.87 + 65%|████████████████████████████████████████████████████████████████████████▏ | 26599/40903 [10:30:13<4:34:14, 1.15s/it][2025-04-21 03:26:49] (step=0026600) Train Loss: 6.3232, Train Steps/Sec: 0.86 + 65%|████████████████████████████████████████████████████████████████████████▎ | 26624/40903 [10:30:42<4:42:57, 1.19s/it][2025-04-21 03:27:17] (step=0026625) Train Loss: 6.3677, Train Steps/Sec: 0.87 + 65%|████████████████████████████████████████████████████████████████████████▎ | 26649/40903 [10:31:11<4:36:58, 1.17s/it][2025-04-21 03:27:46] (step=0026650) Train Loss: 6.3657, Train Steps/Sec: 0.87 + 65%|████████████████████████████████████████████████████████████████████████▍ | 26674/40903 [10:31:40<4:33:10, 1.15s/it][2025-04-21 03:28:15] (step=0026675) Train Loss: 6.3276, Train Steps/Sec: 0.86 + 65%|████████████████████████████████████████████████████████████████████████▍ | 26699/40903 [10:32:08<4:32:32, 1.15s/it][2025-04-21 03:28:44] (step=0026700) Train Loss: 6.3333, Train Steps/Sec: 0.86 + 65%|████████████████████████████████████████████████████████████████████████▌ | 26724/40903 [10:32:38<4:40:14, 1.19s/it][2025-04-21 03:29:13] (step=0026725) Train Loss: 6.3867, Train Steps/Sec: 0.86 + 65%|████████████████████████████████████████████████████████████████████████▌ | 26749/40903 [10:33:07<4:32:50, 1.16s/it][2025-04-21 03:29:42] (step=0026750) Train Loss: 6.3364, Train Steps/Sec: 0.86 + 65%|████████████████████████████████████████████████████████████████████████▋ | 26774/40903 [10:33:42<4:35:21, 1.17s/it][2025-04-21 03:30:18] (step=0026775) Train Loss: 6.3518, Train Steps/Sec: 0.71 + 66%|████████████████████████████████████████████████████████████████████████▋ | 26799/40903 [10:34:11<4:31:06, 1.15s/it][2025-04-21 03:30:47] (step=0026800) Train Loss: 6.4067, Train Steps/Sec: 0.86 + 66%|████████████████████████████████████████████████████████████████████████▊ | 26824/40903 [10:34:40<4:36:23, 1.18s/it][2025-04-21 03:31:16] (step=0026825) Train Loss: 6.3702, Train Steps/Sec: 0.87 + 66%|████████████████████████████████████████████████████████████████████████▊ | 26849/40903 [10:35:09<4:33:20, 1.17s/it][2025-04-21 03:31:44] (step=0026850) Train Loss: 6.3551, Train Steps/Sec: 0.87 + 66%|████████████████████████████████████████████████████████████████████████▉ | 26874/40903 [10:35:38<4:26:28, 1.14s/it][2025-04-21 03:32:13] (step=0026875) Train Loss: 6.3477, Train Steps/Sec: 0.87 + 66%|████████████████████████████████████████████████████████████████████████▉ | 26899/40903 [10:36:07<4:29:05, 1.15s/it][2025-04-21 03:32:42] (step=0026900) Train Loss: 6.3795, Train Steps/Sec: 0.86 + 66%|█████████████████████████████████████████████████████████████████████████ | 26924/40903 [10:36:36<4:34:41, 1.18s/it][2025-04-21 03:33:11] (step=0026925) Train Loss: 6.3482, Train Steps/Sec: 0.86 + 66%|█████████████████████████████████████████████████████████████████████████▏ | 26949/40903 [10:37:05<4:25:38, 1.14s/it][2025-04-21 03:33:40] (step=0026950) Train Loss: 6.3259, Train Steps/Sec: 0.87 + 66%|█████████████████████████████████████████████████████████████████████████▏ | 26974/40903 [10:37:33<4:25:25, 1.14s/it][2025-04-21 03:34:09] (step=0026975) Train Loss: 6.3949, Train Steps/Sec: 0.87 + 66%|█████████████████████████████████████████████████████████████████████████▎ | 26999/40903 [10:38:02<4:27:51, 1.16s/it][2025-04-21 03:34:38] (step=0027000) Train Loss: 6.3009, Train Steps/Sec: 0.86 + 66%|█████████████████████████████████████████████████████████████████████████▎ | 27024/40903 [10:38:31<4:32:59, 1.18s/it][2025-04-21 03:35:07] (step=0027025) Train Loss: 6.3842, Train Steps/Sec: 0.87 + 66%|█████████████████████████████████████████████████████████████████████████▍ | 27049/40903 [10:39:00<4:28:54, 1.16s/it][2025-04-21 03:35:36] (step=0027050) Train Loss: 6.3529, Train Steps/Sec: 0.87 + 66%|█████████████████████████████████████████████████████████████████████████▍ | 27074/40903 [10:39:29<4:27:30, 1.16s/it][2025-04-21 03:36:05] (step=0027075) Train Loss: 6.3504, Train Steps/Sec: 0.86 + 66%|█████████████████████████████████████████████████████████████████████████▌ | 27099/40903 [10:39:58<4:23:50, 1.15s/it][2025-04-21 03:36:34] (step=0027100) Train Loss: 6.3667, Train Steps/Sec: 0.86 + 66%|█████████████████████████████████████████████████████████████████████████▌ | 27124/40903 [10:40:27<4:27:35, 1.17s/it][2025-04-21 03:37:02] (step=0027125) Train Loss: 6.3233, Train Steps/Sec: 0.87 + 66%|█████████████████████████████████████████████████████████████████████████▋ | 27149/40903 [10:40:56<4:25:56, 1.16s/it][2025-04-21 03:37:31] (step=0027150) Train Loss: 6.3892, Train Steps/Sec: 0.86 + 66%|█████████████████████████████████████████████████████████████████████████▋ | 27174/40903 [10:41:25<4:25:49, 1.16s/it][2025-04-21 03:38:00] (step=0027175) Train Loss: 6.3368, Train Steps/Sec: 0.86 + 66%|█████████████████████████████████████████████████████████████████████████▊ | 27199/40903 [10:41:54<4:22:55, 1.15s/it][2025-04-21 03:38:29] (step=0027200) Train Loss: 6.3567, Train Steps/Sec: 0.86 + 67%|█████████████████████████████████████████████████████████████████████████▉ | 27224/40903 [10:42:23<4:31:43, 1.19s/it][2025-04-21 03:38:58] (step=0027225) Train Loss: 6.3344, Train Steps/Sec: 0.86 + 67%|█████████████████████████████████████████████████████████████████████████▉ | 27249/40903 [10:42:52<4:23:43, 1.16s/it][2025-04-21 03:39:28] (step=0027250) Train Loss: 6.3033, Train Steps/Sec: 0.86 + 67%|██████████████████████████████████████████████████████████████████████████ | 27274/40903 [10:43:21<4:20:38, 1.15s/it][2025-04-21 03:39:57] (step=0027275) Train Loss: 6.3923, Train Steps/Sec: 0.86 + 67%|██████████████████████████████████████████████████████████████████████████ | 27299/40903 [10:43:50<4:22:37, 1.16s/it][2025-04-21 03:40:26] (step=0027300) Train Loss: 6.3692, Train Steps/Sec: 0.86 + 67%|██████████████████████████████████████████████████████████████████████████▏ | 27324/40903 [10:44:19<4:26:27, 1.18s/it][2025-04-21 03:40:55] (step=0027325) Train Loss: 6.3704, Train Steps/Sec: 0.86 + 67%|██████████████████████████████████████████████████████████████████████████▏ | 27349/40903 [10:44:48<4:22:59, 1.16s/it][2025-04-21 03:41:23] (step=0027350) Train Loss: 6.3613, Train Steps/Sec: 0.86 + 67%|██████████████████████████████████████████████████████████████████████████▎ | 27374/40903 [10:45:17<4:19:37, 1.15s/it][2025-04-21 03:41:52] (step=0027375) Train Loss: 6.3764, Train Steps/Sec: 0.86 + 67%|██████████████████████████████████████████████████████████████████████████▎ | 27399/40903 [10:45:46<4:17:54, 1.15s/it][2025-04-21 03:42:22] (step=0027400) Train Loss: 6.4086, Train Steps/Sec: 0.85 + 67%|██████████████████████████████████████████████████████████████████████████▍ | 27424/40903 [10:46:15<4:24:48, 1.18s/it][2025-04-21 03:42:51] (step=0027425) Train Loss: 6.3465, Train Steps/Sec: 0.86 + 67%|██████████████████████████████████████████████████████████████████████████▍ | 27449/40903 [10:46:44<4:20:26, 1.16s/it][2025-04-21 03:43:20] (step=0027450) Train Loss: 6.3396, Train Steps/Sec: 0.87 + 67%|██████████████████████████████████████████████████████████████████████████▌ | 27474/40903 [10:47:13<4:13:42, 1.13s/it][2025-04-21 03:43:49] (step=0027475) Train Loss: 6.3792, Train Steps/Sec: 0.86 + 67%|██████████████████████████████████████████████████████████████████████████▋ | 27499/40903 [10:47:42<4:16:01, 1.15s/it][2025-04-21 03:44:18] (step=0027500) Train Loss: 6.4072, Train Steps/Sec: 0.86 + 67%|██████████████████████████████████████████████████████████████████████████▋ | 27524/40903 [10:48:11<4:23:46, 1.18s/it][2025-04-21 03:44:47] (step=0027525) Train Loss: 6.3606, Train Steps/Sec: 0.86 + 67%|██████████████████████████████████████████████████████████████████████████▊ | 27549/40903 [10:48:40<4:14:16, 1.14s/it][2025-04-21 03:45:16] (step=0027550) Train Loss: 6.3618, Train Steps/Sec: 0.86 + 67%|██████████████████████████████████████████████████████████████████████████▊ | 27574/40903 [10:49:09<4:14:43, 1.15s/it][2025-04-21 03:45:45] (step=0027575) Train Loss: 6.3251, Train Steps/Sec: 0.86 + 67%|██████████████████████████████████████████████████████████████████████████▉ | 27599/40903 [10:49:38<4:15:29, 1.15s/it][2025-04-21 03:46:14] (step=0027600) Train Loss: 6.3383, Train Steps/Sec: 0.86 + 68%|██████████████████████████████████████████████████████████████████████████▉ | 27624/40903 [10:50:07<4:19:48, 1.17s/it][2025-04-21 03:46:43] (step=0027625) Train Loss: 6.3349, Train Steps/Sec: 0.86 + 68%|███████████████████████████████████████████████████████████████████████████ | 27649/40903 [10:50:36<4:17:58, 1.17s/it][2025-04-21 03:47:12] (step=0027650) Train Loss: 6.3574, Train Steps/Sec: 0.86 + 68%|███████████████████████████████████████████████████████████████████████████ | 27674/40903 [10:51:06<4:20:36, 1.18s/it][2025-04-21 03:47:41] (step=0027675) Train Loss: 6.4096, Train Steps/Sec: 0.85 + 68%|███████████████████████████████████████████████████████████████████████████▏ | 27699/40903 [10:51:35<4:13:55, 1.15s/it][2025-04-21 03:48:10] (step=0027700) Train Loss: 6.3193, Train Steps/Sec: 0.86 + 68%|███████████████████████████████████████████████████████████████████████████▏ | 27724/40903 [10:52:04<4:18:19, 1.18s/it][2025-04-21 03:48:40] (step=0027725) Train Loss: 6.3214, Train Steps/Sec: 0.86 + 68%|███████████████████████████████████████████████████████████████████████████▎ | 27749/40903 [10:52:33<4:15:40, 1.17s/it][2025-04-21 03:49:09] (step=0027750) Train Loss: 6.2980, Train Steps/Sec: 0.86 + 68%|███████████████████████████████████████████████████████████████████████████▎ | 27774/40903 [10:53:02<4:10:30, 1.14s/it][2025-04-21 03:49:38] (step=0027775) Train Loss: 6.3813, Train Steps/Sec: 0.86 + 68%|███████████████████████████████████████████████████████████████████████████▍ | 27799/40903 [10:53:31<4:14:15, 1.16s/it][2025-04-21 03:50:07] (step=0027800) Train Loss: 6.3219, Train Steps/Sec: 0.86 + 68%|███████████████████████████████████████████████████████████████████████████▌ | 27824/40903 [10:54:01<4:15:40, 1.17s/it][2025-04-21 03:50:36] (step=0027825) Train Loss: 6.3490, Train Steps/Sec: 0.86 + 68%|███████████████████████████████████████████████████████████████████████████▌ | 27849/40903 [10:54:29<4:13:39, 1.17s/it][2025-04-21 03:51:05] (step=0027850) Train Loss: 6.3964, Train Steps/Sec: 0.86 + 68%|███████████████████████████████████████████████████████████████████████████▋ | 27874/40903 [10:54:58<4:09:27, 1.15s/it][2025-04-21 03:51:34] (step=0027875) Train Loss: 6.3413, Train Steps/Sec: 0.86 + 68%|███████████████████████████████████████████████████████████████████████████▋ | 27899/40903 [10:55:27<4:09:23, 1.15s/it][2025-04-21 03:52:03] (step=0027900) Train Loss: 6.3581, Train Steps/Sec: 0.86 + 68%|███████████████████████████████████████████████████████████████████████████▊ | 27924/40903 [10:55:57<4:15:09, 1.18s/it][2025-04-21 03:52:32] (step=0027925) Train Loss: 6.3296, Train Steps/Sec: 0.86 + 68%|███████████████████████████████████████████████████████████████████████████▊ | 27949/40903 [10:56:26<4:11:12, 1.16s/it][2025-04-21 03:53:02] (step=0027950) Train Loss: 6.3410, Train Steps/Sec: 0.85 + 68%|███████████████████████████████████████████████████████████████████████████▉ | 27974/40903 [10:56:55<4:11:08, 1.17s/it][2025-04-21 03:53:31] (step=0027975) Train Loss: 6.3782, Train Steps/Sec: 0.86 + 68%|███████████████████████████████████████████████████████████████████████████▉ | 27999/40903 [10:57:24<4:08:31, 1.16s/it][2025-04-21 03:54:00] (step=0028000) Train Loss: 6.3464, Train Steps/Sec: 0.86 +[2025-04-21 03:54:00] vision_config is None. initializing the InstructBlipVisionConfig with default values. +[2025-04-21 03:54:00] qformer_config is None. Initializing the InstructBlipQFormerConfig with default values. +[2025-04-21 03:54:00] text_config is None. Initializing the text config with default values (`OPTConfig`). +model_name_or_path:/tmp/haozhezhao/model/instructblip-flan-t5-xl +image_place_holder: +tokenizer length after expend 32102 +tokenizer length before expend 32102 +100%|████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████| 3/3 [03:09<00:00, 63.04s/it] +[2025-04-21 03:59:02] Finish Eval in 28000 steps...█████████████████████████████████████████████████████████████████████| 3/3 [03:08<00:00, 62.52s/it] +processing left-padding... +processing left-padding... +processing left-padding... +[2025-04-21 03:59:23] Saved checkpoint to checkpoint/EmptyFix_InstructBlip_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_objects/002-GPT-XL/checkpoints/0028000.pt +[2025-04-21 03:59:26] Removed old checkpoint: checkpoint/EmptyFix_InstructBlip_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_objects/002-GPT-XL/checkpoints/0024000.pt + 69%|████████████████████████████████████████████████████████████████████████████ | 28024/40903 [11:03:20<4:18:53, 1.21s/it][2025-04-21 03:59:56] (step=0028025) Train Loss: 6.3965, Train Steps/Sec: 0.07 + 69%|████████████████████████████████████████████████████████████████████████████ | 28049/40903 [11:03:49<4:07:41, 1.16s/it][2025-04-21 04:00:25] (step=0028050) Train Loss: 6.3218, Train Steps/Sec: 0.87 + 69%|████████████████████████████████████████████████████████████████████████████▏ | 28074/40903 [11:04:18<4:05:45, 1.15s/it][2025-04-21 04:00:53] (step=0028075) Train Loss: 6.3401, Train Steps/Sec: 0.87 + 69%|████████████████████████████████████████████████████████████████████████████▎ | 28099/40903 [11:04:47<4:03:28, 1.14s/it][2025-04-21 04:01:23] (step=0028100) Train Loss: 6.3767, Train Steps/Sec: 0.86 + 69%|████████████████████████████████████████████████████████████████████████████▎ | 28124/40903 [11:05:16<4:12:10, 1.18s/it][2025-04-21 04:01:51] (step=0028125) Train Loss: 6.3418, Train Steps/Sec: 0.87 + 69%|████████████████████████████████████████████████████████████████████████████▍ | 28149/40903 [11:05:45<4:07:44, 1.17s/it][2025-04-21 04:02:20] (step=0028150) Train Loss: 6.3603, Train Steps/Sec: 0.86 + 69%|████████████████████████████████████████████████████████████████████████████▍ | 28174/40903 [11:06:14<4:02:44, 1.14s/it][2025-04-21 04:02:49] (step=0028175) Train Loss: 6.3529, Train Steps/Sec: 0.87 + 69%|████████████████████████████████████████████████████████████████████████████▌ | 28199/40903 [11:06:42<4:02:42, 1.15s/it][2025-04-21 04:03:18] (step=0028200) Train Loss: 6.3611, Train Steps/Sec: 0.86 + 69%|████████████████████████████████████████████████████████████████████████████▌ | 28224/40903 [11:07:12<4:11:18, 1.19s/it][2025-04-21 04:03:47] (step=0028225) Train Loss: 6.3530, Train Steps/Sec: 0.86 + 69%|████████████████████████████████████████████████████████████████████████████▋ | 28249/40903 [11:07:40<4:01:05, 1.14s/it][2025-04-21 04:04:16] (step=0028250) Train Loss: 6.3598, Train Steps/Sec: 0.87 + 69%|████████████████████████████████████████████████████████████████████████████▋ | 28274/40903 [11:08:09<4:01:36, 1.15s/it][2025-04-21 04:04:45] (step=0028275) Train Loss: 6.3987, Train Steps/Sec: 0.87 + 69%|████████████████████████████████████████████████████████████████████████████▊ | 28299/40903 [11:08:38<4:01:34, 1.15s/it][2025-04-21 04:05:14] (step=0028300) Train Loss: 6.3398, Train Steps/Sec: 0.86 + 69%|████████████████████████████████████████████████████████████████████████████▊ | 28324/40903 [11:09:07<4:04:42, 1.17s/it][2025-04-21 04:05:43] (step=0028325) Train Loss: 6.3578, Train Steps/Sec: 0.87 + 69%|████████████████████████████████████████████████████████████████████████████▉ | 28349/40903 [11:09:36<4:01:31, 1.15s/it][2025-04-21 04:06:11] (step=0028350) Train Loss: 6.3460, Train Steps/Sec: 0.87 + 69%|████████████████████████████████████████████████████████████████████████████▉ | 28374/40903 [11:10:05<4:02:44, 1.16s/it][2025-04-21 04:06:41] (step=0028375) Train Loss: 6.3581, Train Steps/Sec: 0.85 + 69%|█████████████████████████████████████████████████████████████████████████████ | 28399/40903 [11:10:34<3:57:36, 1.14s/it][2025-04-21 04:07:10] (step=0028400) Train Loss: 6.3677, Train Steps/Sec: 0.86 + 69%|█████████████████████████████████████████████████████████████████████████████▏ | 28424/40903 [11:11:03<4:05:21, 1.18s/it][2025-04-21 04:07:39] (step=0028425) Train Loss: 6.3485, Train Steps/Sec: 0.86 + 70%|█████████████████████████████████████████████████████████████████████████████▏ | 28449/40903 [11:11:32<3:59:36, 1.15s/it][2025-04-21 04:08:08] (step=0028450) Train Loss: 6.3729, Train Steps/Sec: 0.86 + 70%|█████████████████████████████████████████████████████████████████████████████▎ | 28474/40903 [11:12:01<3:57:55, 1.15s/it][2025-04-21 04:08:37] (step=0028475) Train Loss: 6.3527, Train Steps/Sec: 0.87 + 70%|█████████████████████████████████████████████████████████████████████████████▎ | 28499/40903 [11:12:30<3:56:53, 1.15s/it][2025-04-21 04:09:06] (step=0028500) Train Loss: 6.3969, Train Steps/Sec: 0.86 + 70%|█████████████████████████████████████████████████████████████████████████████▍ | 28524/40903 [11:12:59<4:02:10, 1.17s/it][2025-04-21 04:09:35] (step=0028525) Train Loss: 6.3828, Train Steps/Sec: 0.86 + 70%|█████████████████████████████████████████████████████████████████████████████▍ | 28549/40903 [11:13:28<3:56:44, 1.15s/it][2025-04-21 04:10:04] (step=0028550) Train Loss: 6.3429, Train Steps/Sec: 0.86 + 70%|█████████████████████████████████████████████████████████████████████████████▌ | 28574/40903 [11:13:57<3:57:18, 1.15s/it][2025-04-21 04:10:33] (step=0028575) Train Loss: 6.3425, Train Steps/Sec: 0.86 + 70%|█████████████████████████████████████████████████████████████████████████████▌ | 28599/40903 [11:14:26<3:58:49, 1.16s/it][2025-04-21 04:11:02] (step=0028600) Train Loss: 6.3549, Train Steps/Sec: 0.85 + 70%|█████████████████████████████████████████████████████████████████████████████▋ | 28624/40903 [11:14:55<4:02:56, 1.19s/it][2025-04-21 04:11:31] (step=0028625) Train Loss: 6.3332, Train Steps/Sec: 0.86 + 70%|█████████████████████████████████████████████████████████████████████████████▋ | 28649/40903 [11:15:24<3:57:49, 1.16s/it][2025-04-21 04:12:00] (step=0028650) Train Loss: 6.3944, Train Steps/Sec: 0.86 + 70%|█████████████████████████████████████████████████████████████████████████████▊ | 28674/40903 [11:15:53<3:54:37, 1.15s/it][2025-04-21 04:12:29] (step=0028675) Train Loss: 6.3774, Train Steps/Sec: 0.86 + 70%|█████████████████████████████████████████████████████████████████████████████▉ | 28699/40903 [11:16:22<3:52:32, 1.14s/it][2025-04-21 04:12:58] (step=0028700) Train Loss: 6.3918, Train Steps/Sec: 0.86 + 70%|█████████████████████████████████████████████████████████████████████████████▉ | 28724/40903 [11:16:51<4:00:28, 1.18s/it][2025-04-21 04:13:27] (step=0028725) Train Loss: 6.3310, Train Steps/Sec: 0.86 + 70%|██████████████████████████████████████████████████████████████████████████████ | 28749/40903 [11:17:20<3:54:36, 1.16s/it][2025-04-21 04:13:56] (step=0028750) Train Loss: 6.3728, Train Steps/Sec: 0.87 + 70%|██████████████████████████████████████████████████████████████████████████████ | 28774/40903 [11:17:49<3:50:45, 1.14s/it][2025-04-21 04:14:25] (step=0028775) Train Loss: 6.3210, Train Steps/Sec: 0.87 + 70%|██████████████████████████████████████████████████████████████████████████████▏ | 28799/40903 [11:18:18<3:50:34, 1.14s/it][2025-04-21 04:14:54] (step=0028800) Train Loss: 6.3504, Train Steps/Sec: 0.86 + 70%|██████████████████████████████████████████████████████████████████████████████▏ | 28824/40903 [11:18:47<3:59:24, 1.19s/it][2025-04-21 04:15:23] (step=0028825) Train Loss: 6.3486, Train Steps/Sec: 0.86 + 71%|██████████████████████████████████████████████████████████████████████████████▎ | 28849/40903 [11:19:16<3:52:52, 1.16s/it][2025-04-21 04:15:52] (step=0028850) Train Loss: 6.3896, Train Steps/Sec: 0.86 + 71%|██████████████████████████████████████████████████████████████████████████████▎ | 28874/40903 [11:19:45<3:50:42, 1.15s/it][2025-04-21 04:16:21] (step=0028875) Train Loss: 6.3681, Train Steps/Sec: 0.87 + 71%|██████████████████████████████████████████████████████████████████████████████▍ | 28899/40903 [11:20:14<3:48:21, 1.14s/it][2025-04-21 04:16:50] (step=0028900) Train Loss: 6.3774, Train Steps/Sec: 0.86 + 71%|██████████████████████████████████████████████████████████████████████████████▍ | 28924/40903 [11:20:43<3:56:39, 1.19s/it][2025-04-21 04:17:19] (step=0028925) Train Loss: 6.3660, Train Steps/Sec: 0.86 + 71%|██████████████████████████████████████████████████████████████████████████████▌ | 28949/40903 [11:21:12<3:51:21, 1.16s/it][2025-04-21 04:17:48] (step=0028950) Train Loss: 6.3879, Train Steps/Sec: 0.87 + 71%|██████████████████████████████████████████████████████████████████████████████▋ | 28974/40903 [11:21:41<3:49:19, 1.15s/it][2025-04-21 04:18:17] (step=0028975) Train Loss: 6.3435, Train Steps/Sec: 0.86 + 71%|██████████████████████████████████████████████████████████████████████████████▋ | 28999/40903 [11:22:10<3:49:18, 1.16s/it][2025-04-21 04:18:46] (step=0029000) Train Loss: 6.3313, Train Steps/Sec: 0.86 + 71%|██████████████████████████████████████████████████████████████████████████████▊ | 29024/40903 [11:22:39<3:53:25, 1.18s/it][2025-04-21 04:19:15] (step=0029025) Train Loss: 6.3023, Train Steps/Sec: 0.86 + 71%|██████████████████████████████████████████████████████████████████████████████▊ | 29049/40903 [11:23:08<3:48:29, 1.16s/it][2025-04-21 04:19:43] (step=0029050) Train Loss: 6.3433, Train Steps/Sec: 0.87 + 71%|██████████████████████████████████████████████████████████████████████████████▉ | 29074/40903 [11:23:43<4:27:47, 1.36s/it][2025-04-21 04:20:18] (step=0029075) Train Loss: 6.3457, Train Steps/Sec: 0.72 + 71%|██████████████████████████████████████████████████████████████████████████████▉ | 29099/40903 [11:24:12<3:47:57, 1.16s/it][2025-04-21 04:20:47] (step=0029100) Train Loss: 6.3513, Train Steps/Sec: 0.86 + 71%|███████████████████████████████████████████████████████████████████████████████ | 29124/40903 [11:24:41<3:52:55, 1.19s/it][2025-04-21 04:21:16] (step=0029125) Train Loss: 6.3515, Train Steps/Sec: 0.86 + 71%|███████████████████████████████████████████████████████████████████████████████ | 29149/40903 [11:25:10<3:46:20, 1.16s/it][2025-04-21 04:21:45] (step=0029150) Train Loss: 6.3628, Train Steps/Sec: 0.87 + 71%|███████████████████████████████████████████████████████████████████████████████▏ | 29174/40903 [11:25:39<3:43:09, 1.14s/it][2025-04-21 04:22:14] (step=0029175) Train Loss: 6.3556, Train Steps/Sec: 0.87 + 71%|███████████████████████████████████████████████████████████████████████████████▏ | 29199/40903 [11:26:08<3:43:04, 1.14s/it][2025-04-21 04:22:43] (step=0029200) Train Loss: 6.3554, Train Steps/Sec: 0.86 + 71%|███████████████████████████████████████████████████████████████████████████████▎ | 29224/40903 [11:26:36<3:45:45, 1.16s/it][2025-04-21 04:23:12] (step=0029225) Train Loss: 6.3380, Train Steps/Sec: 0.87 + 72%|███████████████████████████████████████████████████████████████████████████████▎ | 29249/40903 [11:27:05<3:46:54, 1.17s/it][2025-04-21 04:23:41] (step=0029250) Train Loss: 6.3629, Train Steps/Sec: 0.87 + 72%|███████████████████████████████████████████████████████████████████████████████▍ | 29274/40903 [11:27:34<3:43:21, 1.15s/it][2025-04-21 04:24:10] (step=0029275) Train Loss: 6.4024, Train Steps/Sec: 0.86 + 72%|███████████████████████████████████████████████████████████████████████████████▌ | 29299/40903 [11:28:15<3:44:47, 1.16s/it][2025-04-21 04:24:51] (step=0029300) Train Loss: 6.3460, Train Steps/Sec: 0.62 + 72%|███████████████████████████████████████████████████████████████████████████████▌ | 29324/40903 [11:28:44<3:46:49, 1.18s/it][2025-04-21 04:25:20] (step=0029325) Train Loss: 6.3666, Train Steps/Sec: 0.86 + 72%|███████████████████████████████████████████████████████████████████████████████▋ | 29349/40903 [11:29:13<3:42:00, 1.15s/it][2025-04-21 04:25:49] (step=0029350) Train Loss: 6.3581, Train Steps/Sec: 0.86 + 72%|███████████████████████████████████████████████████████████████████████████████▋ | 29374/40903 [11:29:42<3:38:33, 1.14s/it][2025-04-21 04:26:17] (step=0029375) Train Loss: 6.3585, Train Steps/Sec: 0.87 + 72%|███████████████████████████████████████████████████████████████████████████████▊ | 29399/40903 [11:30:11<3:39:50, 1.15s/it][2025-04-21 04:26:46] (step=0029400) Train Loss: 6.3851, Train Steps/Sec: 0.86 + 72%|███████████████████████████████████████████████████████████████████████████████▊ | 29424/40903 [11:30:40<3:44:23, 1.17s/it][2025-04-21 04:27:15] (step=0029425) Train Loss: 6.3995, Train Steps/Sec: 0.86 + 72%|███████████████████████████████████████████████████████████████████████████████▉ | 29449/40903 [11:31:09<3:40:43, 1.16s/it][2025-04-21 04:27:44] (step=0029450) Train Loss: 6.3841, Train Steps/Sec: 0.87 + 72%|███████████████████████████████████████████████████████████████████████████████▉ | 29474/40903 [11:31:44<3:47:03, 1.19s/it][2025-04-21 04:28:20] (step=0029475) Train Loss: 6.3521, Train Steps/Sec: 0.70 + 72%|████████████████████████████████████████████████████████████████████████████████ | 29499/40903 [11:32:13<3:37:36, 1.14s/it][2025-04-21 04:28:49] (step=0029500) Train Loss: 6.3437, Train Steps/Sec: 0.86 + 72%|████████████████████████████████████████████████████████████████████████████████ | 29524/40903 [11:32:42<3:43:50, 1.18s/it][2025-04-21 04:29:18] (step=0029525) Train Loss: 6.3278, Train Steps/Sec: 0.86 + 72%|████████████████████████████████████████████████████████████████████████████████▏ | 29549/40903 [11:33:11<3:37:23, 1.15s/it][2025-04-21 04:29:47] (step=0029550) Train Loss: 6.3310, Train Steps/Sec: 0.87 + 72%|████████████████████████████████████████████████████████████████████████████████▎ | 29574/40903 [11:33:40<3:36:36, 1.15s/it][2025-04-21 04:30:16] (step=0029575) Train Loss: 6.3715, Train Steps/Sec: 0.87 + 72%|████████████████████████████████████████████████████████████████████████████████▎ | 29599/40903 [11:34:09<3:35:48, 1.15s/it][2025-04-21 04:30:45] (step=0029600) Train Loss: 6.4138, Train Steps/Sec: 0.86 + 72%|████████████████████████████████████████████████████████████████████████████████▍ | 29624/40903 [11:34:38<3:42:12, 1.18s/it][2025-04-21 04:31:14] (step=0029625) Train Loss: 6.3451, Train Steps/Sec: 0.86 + 72%|████████████████████████████████████████████████████████████████████████████████▍ | 29649/40903 [11:35:07<3:38:26, 1.16s/it][2025-04-21 04:31:43] (step=0029650) Train Loss: 6.3330, Train Steps/Sec: 0.86 + 73%|████████████████████████████████████████████████████████████████████████████████▌ | 29674/40903 [11:35:36<3:32:38, 1.14s/it][2025-04-21 04:32:12] (step=0029675) Train Loss: 6.3198, Train Steps/Sec: 0.87 + 73%|████████████████████████████████████████████████████████████████████████████████▌ | 29699/40903 [11:36:05<3:33:33, 1.14s/it][2025-04-21 04:32:41] (step=0029700) Train Loss: 6.3112, Train Steps/Sec: 0.86 + 73%|████████████████████████████████████████████████████████████████████████████████▋ | 29724/40903 [11:36:34<3:37:33, 1.17s/it][2025-04-21 04:33:10] (step=0029725) Train Loss: 6.3301, Train Steps/Sec: 0.86 + 73%|████████████████████████████████████████████████████████████████████████████████▋ | 29749/40903 [11:37:03<3:33:05, 1.15s/it][2025-04-21 04:33:39] (step=0029750) Train Loss: 6.3633, Train Steps/Sec: 0.87 + 73%|████████████████████████████████████████████████████████████████████████████████▊ | 29774/40903 [11:37:32<3:33:33, 1.15s/it][2025-04-21 04:34:07] (step=0029775) Train Loss: 6.3884, Train Steps/Sec: 0.87 + 73%|████████████████████████████████████████████████████████████████████████████████▊ | 29799/40903 [11:38:01<3:31:39, 1.14s/it][2025-04-21 04:34:36] (step=0029800) Train Loss: 6.3829, Train Steps/Sec: 0.86 + 73%|████████████████████████████████████████████████████████████████████████████████▉ | 29824/40903 [11:38:30<3:36:24, 1.17s/it][2025-04-21 04:35:05] (step=0029825) Train Loss: 6.3908, Train Steps/Sec: 0.87 + 73%|█████████████████████████████████████████████████████████████████████████████████ | 29849/40903 [11:38:59<3:33:36, 1.16s/it][2025-04-21 04:35:34] (step=0029850) Train Loss: 6.3840, Train Steps/Sec: 0.86 + 73%|█████████████████████████████████████████████████████████████████████████████████ | 29874/40903 [11:39:28<3:30:40, 1.15s/it][2025-04-21 04:36:03] (step=0029875) Train Loss: 6.3226, Train Steps/Sec: 0.87 + 73%|█████████████████████████████████████████████████████████████████████████████████▏ | 29899/40903 [11:39:56<3:29:46, 1.14s/it][2025-04-21 04:36:32] (step=0029900) Train Loss: 6.3468, Train Steps/Sec: 0.86 + 73%|█████████████████████████████████████████████████████████████████████████████████▏ | 29924/40903 [11:40:25<3:35:27, 1.18s/it][2025-04-21 04:37:01] (step=0029925) Train Loss: 6.3095, Train Steps/Sec: 0.86 + 73%|█████████████████████████████████████████████████████████████████████████████████▎ | 29949/40903 [11:40:55<3:33:59, 1.17s/it][2025-04-21 04:37:30] (step=0029950) Train Loss: 6.3676, Train Steps/Sec: 0.86 + 73%|█████████████████████████████████████████████████████████████████████████████████▎ | 29974/40903 [11:41:23<3:27:37, 1.14s/it][2025-04-21 04:37:59] (step=0029975) Train Loss: 6.3829, Train Steps/Sec: 0.87 + 73%|█████████████████████████████████████████████████████████████████████████████████▍ | 29999/40903 [11:41:52<3:27:49, 1.14s/it][2025-04-21 04:38:28] (step=0030000) Train Loss: 6.3785, Train Steps/Sec: 0.86 +[2025-04-21 04:38:28] vision_config is None. initializing the InstructBlipVisionConfig with default values. +[2025-04-21 04:38:28] qformer_config is None. Initializing the InstructBlipQFormerConfig with default values. +[2025-04-21 04:38:28] text_config is None. Initializing the text config with default values (`OPTConfig`). +model_name_or_path:/tmp/haozhezhao/model/instructblip-flan-t5-xl +image_place_holder: +tokenizer length after expend 32102 +tokenizer length before expend 32102 +100%|████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████| 3/3 [03:09<00:00, 63.01s/it] +[2025-04-21 04:43:29] Finish Eval in 30000 steps...█████████████████████████████████████████████████████████████████████| 3/3 [03:08<00:00, 62.61s/it] +processing left-padding... +processing left-padding... +processing left-padding... +[2025-04-21 04:43:51] Saved checkpoint to checkpoint/EmptyFix_InstructBlip_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_objects/002-GPT-XL/checkpoints/0030000.pt +[2025-04-21 04:43:54] Removed old checkpoint: checkpoint/EmptyFix_InstructBlip_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_objects/002-GPT-XL/checkpoints/0026000.pt + 73%|█████████████████████████████████████████████████████████████████████████████████▍ | 30024/40903 [11:47:47<3:36:58, 1.20s/it][2025-04-21 04:44:23] (step=0030025) Train Loss: 6.3251, Train Steps/Sec: 0.07 + 73%|█████████████████████████████████████████████████████████████████████████████████▌ | 30049/40903 [11:48:16<3:31:35, 1.17s/it][2025-04-21 04:44:52] (step=0030050) Train Loss: 6.3458, Train Steps/Sec: 0.86 + 74%|█████████████████████████████████████████████████████████████████████████████████▌ | 30074/40903 [11:48:45<3:27:32, 1.15s/it][2025-04-21 04:45:21] (step=0030075) Train Loss: 6.3593, Train Steps/Sec: 0.87 + 74%|█████████████████████████████████████████████████████████████████████████████████▋ | 30099/40903 [11:49:14<3:27:57, 1.15s/it][2025-04-21 04:45:50] (step=0030100) Train Loss: 6.3500, Train Steps/Sec: 0.85 + 74%|█████████████████████████████████████████████████████████████████████████████████▋ | 30124/40903 [11:49:44<3:30:53, 1.17s/it][2025-04-21 04:46:19] (step=0030125) Train Loss: 6.3606, Train Steps/Sec: 0.86 + 74%|█████████████████████████████████████████████████████████████████████████████████▊ | 30149/40903 [11:50:13<3:28:54, 1.17s/it][2025-04-21 04:46:48] (step=0030150) Train Loss: 6.3750, Train Steps/Sec: 0.86 + 74%|█████████████████████████████████████████████████████████████████████████████████▉ | 30174/40903 [11:50:42<3:29:38, 1.17s/it][2025-04-21 04:47:17] (step=0030175) Train Loss: 6.3188, Train Steps/Sec: 0.86 + 74%|█████████████████████████████████████████████████████████████████████████████████▉ | 30199/40903 [11:51:11<3:23:10, 1.14s/it][2025-04-21 04:47:46] (step=0030200) Train Loss: 6.3371, Train Steps/Sec: 0.86 + 74%|██████████████████████████████████████████████████████████████████████████████████ | 30224/40903 [11:51:40<3:30:44, 1.18s/it][2025-04-21 04:48:16] (step=0030225) Train Loss: 6.3688, Train Steps/Sec: 0.85 + 74%|██████████████████████████████████████████████████████████████████████████████████ | 30249/40903 [11:52:09<3:25:12, 1.16s/it][2025-04-21 04:48:44] (step=0030250) Train Loss: 6.3324, Train Steps/Sec: 0.87 + 74%|██████████████████████████████████████████████████████████████████████████████████▏ | 30274/40903 [11:52:38<3:25:20, 1.16s/it][2025-04-21 04:49:13] (step=0030275) Train Loss: 6.3763, Train Steps/Sec: 0.86 + 74%|██████████████████████████████████████████████████████████████████████████████████▏ | 30299/40903 [11:53:07<3:23:15, 1.15s/it][2025-04-21 04:49:43] (step=0030300) Train Loss: 6.3825, Train Steps/Sec: 0.86 + 74%|██████████████████████████████████████████████████████████████████████████████████▎ | 30324/40903 [11:53:36<3:26:23, 1.17s/it][2025-04-21 04:50:12] (step=0030325) Train Loss: 6.3558, Train Steps/Sec: 0.86 + 74%|██████████████████████████████████████████████████████████████████████████████████▎ | 30349/40903 [11:54:05<3:24:45, 1.16s/it][2025-04-21 04:50:41] (step=0030350) Train Loss: 6.3528, Train Steps/Sec: 0.86 + 74%|██████████████████████████████████████████████████████████████████████████████████▍ | 30374/40903 [11:54:34<3:19:13, 1.14s/it][2025-04-21 04:51:10] (step=0030375) Train Loss: 6.3142, Train Steps/Sec: 0.86 + 74%|██████████████████████████████████████████████████████████████████████████████████▍ | 30399/40903 [11:55:03<3:18:41, 1.13s/it][2025-04-21 04:51:38] (step=0030400) Train Loss: 6.3463, Train Steps/Sec: 0.86 + 74%|██████████████████████████████████████████████████████████████████████████████████▌ | 30424/40903 [11:55:32<3:24:36, 1.17s/it][2025-04-21 04:52:07] (step=0030425) Train Loss: 6.3527, Train Steps/Sec: 0.87 + 74%|██████████████████████████████████████████████████████████████████████████████████▋ | 30449/40903 [11:56:01<3:20:14, 1.15s/it][2025-04-21 04:52:36] (step=0030450) Train Loss: 6.3439, Train Steps/Sec: 0.86 + 75%|██████████████████████████████████████████████████████████████████████████████████▋ | 30474/40903 [11:56:30<3:20:07, 1.15s/it][2025-04-21 04:53:05] (step=0030475) Train Loss: 6.3689, Train Steps/Sec: 0.86 + 75%|██████████████████████████████████████████████████████████████████████████████████▊ | 30499/40903 [11:56:59<3:20:59, 1.16s/it][2025-04-21 04:53:35] (step=0030500) Train Loss: 6.3332, Train Steps/Sec: 0.85 + 75%|██████████████████████████████████████████████████████████████████████████████████▊ | 30524/40903 [11:57:28<3:27:22, 1.20s/it][2025-04-21 04:54:04] (step=0030525) Train Loss: 6.3732, Train Steps/Sec: 0.86 + 75%|██████████████████████████████████████████████████████████████████████████████████▉ | 30549/40903 [11:57:57<3:17:40, 1.15s/it][2025-04-21 04:54:33] (step=0030550) Train Loss: 6.3370, Train Steps/Sec: 0.86 + 75%|██████████████████████████████████████████████████████████████████████████████████▉ | 30574/40903 [11:58:33<3:23:17, 1.18s/it][2025-04-21 04:55:09] (step=0030575) Train Loss: 6.3300, Train Steps/Sec: 0.69 + 75%|███████████████████████████████████████████████████████████████████████████████████ | 30599/40903 [11:59:02<3:18:46, 1.16s/it][2025-04-21 04:55:38] (step=0030600) Train Loss: 6.3515, Train Steps/Sec: 0.86 + 75%|███████████████████████████████████████████████████████████████████████████████████ | 30624/40903 [11:59:32<3:21:44, 1.18s/it][2025-04-21 04:56:07] (step=0030625) Train Loss: 6.3603, Train Steps/Sec: 0.86 + 75%|███████████████████████████████████████████████████████████████████████████████████▏ | 30649/40903 [12:00:01<3:19:12, 1.17s/it][2025-04-21 04:56:36] (step=0030650) Train Loss: 6.3487, Train Steps/Sec: 0.86 + 75%|███████████████████████████████████████████████████████████████████████████████████▏ | 30674/40903 [12:00:30<3:15:29, 1.15s/it][2025-04-21 04:57:05] (step=0030675) Train Loss: 6.3759, Train Steps/Sec: 0.86 + 75%|███████████████████████████████████████████████████████████████████████████████████▎ | 30699/40903 [12:00:59<3:19:53, 1.18s/it][2025-04-21 04:57:34] (step=0030700) Train Loss: 6.3838, Train Steps/Sec: 0.85 + 75%|███████████████████████████████████████████████████████████████████████████████████▍ | 30724/40903 [12:01:28<3:21:09, 1.19s/it][2025-04-21 04:58:03] (step=0030725) Train Loss: 6.3653, Train Steps/Sec: 0.86 + 75%|███████████████████████████████████████████████████████████████████████████████████▍ | 30749/40903 [12:01:57<3:14:42, 1.15s/it][2025-04-21 04:58:33] (step=0030750) Train Loss: 6.3391, Train Steps/Sec: 0.86 + 75%|███████████████████████████████████████████████████████████████████████████████████▌ | 30774/40903 [12:02:26<3:13:42, 1.15s/it][2025-04-21 04:59:01] (step=0030775) Train Loss: 6.3525, Train Steps/Sec: 0.87 + 75%|███████████████████████████████████████████████████████████████████████████████████▌ | 30799/40903 [12:02:55<3:13:39, 1.15s/it][2025-04-21 04:59:30] (step=0030800) Train Loss: 6.3178, Train Steps/Sec: 0.86 + 75%|███████████████████████████████████████████████████████████████████████████████████▋ | 30824/40903 [12:03:24<3:21:47, 1.20s/it][2025-04-21 04:59:59] (step=0030825) Train Loss: 6.3290, Train Steps/Sec: 0.86 + 75%|███████████████████████████████████████████████████████████████████████████████████▋ | 30849/40903 [12:03:53<3:14:53, 1.16s/it][2025-04-21 05:00:29] (step=0030850) Train Loss: 6.3983, Train Steps/Sec: 0.86 + 75%|███████████████████████████████████████████████████████████████████████████████████▊ | 30874/40903 [12:04:22<3:12:45, 1.15s/it][2025-04-21 05:00:57] (step=0030875) Train Loss: 6.3580, Train Steps/Sec: 0.87 + 76%|███████████████████████████████████████████████████████████████████████████████████▊ | 30899/40903 [12:04:51<3:10:43, 1.14s/it][2025-04-21 05:01:27] (step=0030900) Train Loss: 6.3417, Train Steps/Sec: 0.86 + 76%|███████████████████████████████████████████████████████████████████████████████████▉ | 30924/40903 [12:05:20<3:16:54, 1.18s/it][2025-04-21 05:01:56] (step=0030925) Train Loss: 6.3549, Train Steps/Sec: 0.86 + 76%|███████████████████████████████████████████████████████████████████████████████████▉ | 30949/40903 [12:05:49<3:11:53, 1.16s/it][2025-04-21 05:02:25] (step=0030950) Train Loss: 6.3405, Train Steps/Sec: 0.86 + 76%|████████████████████████████████████████████████████████████████████████████████████ | 30974/40903 [12:06:18<3:12:54, 1.17s/it][2025-04-21 05:02:54] (step=0030975) Train Loss: 6.3642, Train Steps/Sec: 0.86 + 76%|████████████████████████████████████████████████████████████████████████████████████ | 30999/40903 [12:06:47<3:08:07, 1.14s/it][2025-04-21 05:03:23] (step=0031000) Train Loss: 6.3546, Train Steps/Sec: 0.86 + 76%|████████████████████████████████████████████████████████████████████████████████████▏ | 31024/40903 [12:07:16<3:14:02, 1.18s/it][2025-04-21 05:03:52] (step=0031025) Train Loss: 6.3407, Train Steps/Sec: 0.86 + 76%|████████████████████████████████████████████████████████████████████████████████████▎ | 31049/40903 [12:07:45<3:09:43, 1.16s/it][2025-04-21 05:04:21] (step=0031050) Train Loss: 6.3662, Train Steps/Sec: 0.86 + 76%|████████████████████████████████████████████████████████████████████████████████████▎ | 31074/40903 [12:08:14<3:10:14, 1.16s/it][2025-04-21 05:04:50] (step=0031075) Train Loss: 6.3439, Train Steps/Sec: 0.86 + 76%|████████████████████████████████████████████████████████████████████████████████████▍ | 31099/40903 [12:08:43<3:09:56, 1.16s/it][2025-04-21 05:05:19] (step=0031100) Train Loss: 6.3715, Train Steps/Sec: 0.85 + 76%|████████████████████████████████████████████████████████████████████████████████████▍ | 31124/40903 [12:09:19<3:09:01, 1.16s/it][2025-04-21 05:05:55] (step=0031125) Train Loss: 6.3476, Train Steps/Sec: 0.70 + 76%|████████████████████████████████████████████████████████████████████████████████████▌ | 31149/40903 [12:09:48<3:08:20, 1.16s/it][2025-04-21 05:06:24] (step=0031150) Train Loss: 6.3596, Train Steps/Sec: 0.87 + 76%|████████████████████████████████████████████████████████████████████████████████████▌ | 31174/40903 [12:10:17<3:08:20, 1.16s/it][2025-04-21 05:06:53] (step=0031175) Train Loss: 6.3599, Train Steps/Sec: 0.86 + 76%|████████████████████████████████████████████████████████████████████████████████████▋ | 31199/40903 [12:10:46<3:02:14, 1.13s/it][2025-04-21 05:07:21] (step=0031200) Train Loss: 6.3425, Train Steps/Sec: 0.87 + 76%|████████████████████████████████████████████████████████████████████████████████████▋ | 31224/40903 [12:11:15<3:07:09, 1.16s/it][2025-04-21 05:07:50] (step=0031225) Train Loss: 6.3417, Train Steps/Sec: 0.87 + 76%|████████████████████████████████████████████████████████████████████████████████████▊ | 31249/40903 [12:11:43<3:04:10, 1.14s/it][2025-04-21 05:08:19] (step=0031250) Train Loss: 6.3524, Train Steps/Sec: 0.87 + 76%|████████████████████████████████████████████████████████████████████████████████████▊ | 31274/40903 [12:12:13<3:04:35, 1.15s/it][2025-04-21 05:08:48] (step=0031275) Train Loss: 6.3695, Train Steps/Sec: 0.86 + 77%|████████████████████████████████████████████████████████████████████████████████████▉ | 31299/40903 [12:12:42<3:05:53, 1.16s/it][2025-04-21 05:09:18] (step=0031300) Train Loss: 6.3959, Train Steps/Sec: 0.85 + 77%|█████████████████████████████████████████████████████████████████████████████████████ | 31324/40903 [12:13:11<3:05:13, 1.16s/it][2025-04-21 05:09:46] (step=0031325) Train Loss: 6.3677, Train Steps/Sec: 0.87 + 77%|█████████████████████████████████████████████████████████████████████████████████████ | 31349/40903 [12:13:40<3:02:38, 1.15s/it][2025-04-21 05:10:16] (step=0031350) Train Loss: 6.3689, Train Steps/Sec: 0.86 + 77%|█████████████████████████████████████████████████████████████████████████████████████▏ | 31374/40903 [12:14:09<3:06:25, 1.17s/it][2025-04-21 05:10:44] (step=0031375) Train Loss: 6.3324, Train Steps/Sec: 0.86 + 77%|█████████████████████████████████████████████████████████████████████████████████████▏ | 31399/40903 [12:14:38<3:03:27, 1.16s/it][2025-04-21 05:11:14] (step=0031400) Train Loss: 6.3580, Train Steps/Sec: 0.86 + 77%|█████████████████████████████████████████████████████████████████████████████████████▎ | 31424/40903 [12:15:07<3:04:23, 1.17s/it][2025-04-21 05:11:42] (step=0031425) Train Loss: 6.3699, Train Steps/Sec: 0.87 + 77%|█████████████████████████████████████████████████████████████████████████████████████▎ | 31449/40903 [12:15:36<3:01:27, 1.15s/it][2025-04-21 05:12:12] (step=0031450) Train Loss: 6.3302, Train Steps/Sec: 0.86 + 77%|█████████████████████████████████████████████████████████████████████████████████████▍ | 31474/40903 [12:16:05<3:03:28, 1.17s/it][2025-04-21 05:12:41] (step=0031475) Train Loss: 6.3314, Train Steps/Sec: 0.86 + 77%|█████████████████████████████████████████████████████████████████████████████████████▍ | 31499/40903 [12:16:34<2:56:44, 1.13s/it][2025-04-21 05:13:10] (step=0031500) Train Loss: 6.3686, Train Steps/Sec: 0.86 + 77%|█████████████████████████████████████████████████████████████████████████████████████▌ | 31524/40903 [12:17:03<3:06:39, 1.19s/it][2025-04-21 05:13:39] (step=0031525) Train Loss: 6.3277, Train Steps/Sec: 0.86 + 77%|█████████████████████████████████████████████████████████████████████████████████████▌ | 31549/40903 [12:17:32<3:01:44, 1.17s/it][2025-04-21 05:14:08] (step=0031550) Train Loss: 6.3329, Train Steps/Sec: 0.86 + 77%|█████████████████████████████████████████████████████████████████████████████████████▋ | 31574/40903 [12:18:01<2:59:18, 1.15s/it][2025-04-21 05:14:37] (step=0031575) Train Loss: 6.3432, Train Steps/Sec: 0.87 + 77%|█████████████████████████████████████████████████████████████████████████████████████▊ | 31599/40903 [12:18:30<2:56:43, 1.14s/it][2025-04-21 05:15:06] (step=0031600) Train Loss: 6.3270, Train Steps/Sec: 0.86 + 77%|█████████████████████████████████████████████████████████████████████████████████████▊ | 31624/40903 [12:18:59<3:01:29, 1.17s/it][2025-04-21 05:15:35] (step=0031625) Train Loss: 6.3325, Train Steps/Sec: 0.86 + 77%|█████████████████████████████████████████████████████████████████████████████████████▉ | 31649/40903 [12:19:28<2:59:56, 1.17s/it][2025-04-21 05:16:04] (step=0031650) Train Loss: 6.4057, Train Steps/Sec: 0.87 + 77%|█████████████████████████████████████████████████████████████████████████████████████▉ | 31674/40903 [12:19:57<2:58:52, 1.16s/it][2025-04-21 05:16:33] (step=0031675) Train Loss: 6.3654, Train Steps/Sec: 0.86 + 77%|██████████████████████████████████████████████████████████████████████████████████████ | 31699/40903 [12:20:26<2:54:13, 1.14s/it][2025-04-21 05:17:02] (step=0031700) Train Loss: 6.3739, Train Steps/Sec: 0.86 + 78%|██████████████████████████████████████████████████████████████████████████████████████ | 31724/40903 [12:20:55<3:00:54, 1.18s/it][2025-04-21 05:17:31] (step=0031725) Train Loss: 6.3742, Train Steps/Sec: 0.86 + 78%|██████████████████████████████████████████████████████████████████████████████████████▏ | 31749/40903 [12:21:24<2:58:23, 1.17s/it][2025-04-21 05:18:00] (step=0031750) Train Loss: 6.3282, Train Steps/Sec: 0.86 + 78%|██████████████████████████████████████████████████████████████████████████████████████▏ | 31774/40903 [12:21:53<2:53:58, 1.14s/it][2025-04-21 05:18:29] (step=0031775) Train Loss: 6.3641, Train Steps/Sec: 0.86 + 78%|██████████████████████████████████████████████████████████████████████████████████████▎ | 31799/40903 [12:22:22<2:54:23, 1.15s/it][2025-04-21 05:18:58] (step=0031800) Train Loss: 6.3450, Train Steps/Sec: 0.86 + 78%|██████████████████████████████████████████████████████████████████████████████████████▎ | 31824/40903 [12:22:51<2:57:24, 1.17s/it][2025-04-21 05:19:27] (step=0031825) Train Loss: 6.3420, Train Steps/Sec: 0.87 + 78%|██████████████████████████████████████████████████████████████████████████████████████▍ | 31849/40903 [12:23:21<2:59:32, 1.19s/it][2025-04-21 05:19:56] (step=0031850) Train Loss: 6.3177, Train Steps/Sec: 0.85 + 78%|██████████████████████████████████████████████████████████████████████████████████████▍ | 31874/40903 [12:23:50<2:53:25, 1.15s/it][2025-04-21 05:20:25] (step=0031875) Train Loss: 6.3552, Train Steps/Sec: 0.87 + 78%|██████████████████████████████████████████████████████████████████████████████████████▌ | 31899/40903 [12:24:19<2:52:10, 1.15s/it][2025-04-21 05:20:54] (step=0031900) Train Loss: 6.3628, Train Steps/Sec: 0.86 + 78%|██████████████████████████████████████████████████████████████████████████████████████▋ | 31924/40903 [12:24:48<2:55:29, 1.17s/it][2025-04-21 05:21:23] (step=0031925) Train Loss: 6.3566, Train Steps/Sec: 0.87 + 78%|██████████████████████████████████████████████████████████████████████████████████████▋ | 31949/40903 [12:25:17<2:54:05, 1.17s/it][2025-04-21 05:21:52] (step=0031950) Train Loss: 6.3674, Train Steps/Sec: 0.86 + 78%|██████████████████████████████████████████████████████████████████████████████████████▊ | 31974/40903 [12:25:46<2:49:05, 1.14s/it][2025-04-21 05:22:21] (step=0031975) Train Loss: 6.3699, Train Steps/Sec: 0.87 + 78%|██████████████████████████████████████████████████████████████████████████████████████▊ | 31999/40903 [12:26:15<2:53:46, 1.17s/it][2025-04-21 05:22:50] (step=0032000) Train Loss: 6.3100, Train Steps/Sec: 0.86 +[2025-04-21 05:22:50] vision_config is None. initializing the InstructBlipVisionConfig with default values. +[2025-04-21 05:22:50] qformer_config is None. Initializing the InstructBlipQFormerConfig with default values. +[2025-04-21 05:22:50] text_config is None. Initializing the text config with default values (`OPTConfig`). +model_name_or_path:/tmp/haozhezhao/model/instructblip-flan-t5-xl +image_place_holder: +tokenizer length after expend 32102 +tokenizer length before expend 32102 +100%|████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████| 3/3 [03:10<00:00, 63.49s/it] +[2025-04-21 05:27:52] Finish Eval in 32000 steps...█████████████████████████████████████████████████████████████████████| 3/3 [03:09<00:00, 62.94s/it] +processing left-padding... +processing left-padding... +processing left-padding... +[2025-04-21 05:28:14] Saved checkpoint to checkpoint/EmptyFix_InstructBlip_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_objects/002-GPT-XL/checkpoints/0032000.pt +[2025-04-21 05:28:17] Removed old checkpoint: checkpoint/EmptyFix_InstructBlip_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_objects/002-GPT-XL/checkpoints/0028000.pt + 78%|██████████████████████████████████████████████████████████████████████████████████████▉ | 32024/40903 [12:32:10<2:57:14, 1.20s/it][2025-04-21 05:28:46] (step=0032025) Train Loss: 6.3682, Train Steps/Sec: 0.07 + 78%|██████████████████████████████████████████████████████████████████████████████████████▉ | 32049/40903 [12:32:39<2:50:34, 1.16s/it][2025-04-21 05:29:15] (step=0032050) Train Loss: 6.3654, Train Steps/Sec: 0.86 + 78%|███████████████████████████████████████████████████████████████████████████████████████ | 32074/40903 [12:33:09<2:48:19, 1.14s/it][2025-04-21 05:29:44] (step=0032075) Train Loss: 6.3242, Train Steps/Sec: 0.86 + 78%|███████████████████████████████████████████████████████████████████████████████████████ | 32099/40903 [12:33:38<2:48:06, 1.15s/it][2025-04-21 05:30:13] (step=0032100) Train Loss: 6.3301, Train Steps/Sec: 0.86 + 79%|███████████████████████████████████████████████████████████████████████████████████████▏ | 32124/40903 [12:34:13<3:13:19, 1.32s/it][2025-04-21 05:30:49] (step=0032125) Train Loss: 6.3159, Train Steps/Sec: 0.71 + 79%|███████████████████████████████████████████████████████████████████████████████████████▏ | 32149/40903 [12:34:42<2:48:31, 1.16s/it][2025-04-21 05:31:18] (step=0032150) Train Loss: 6.3528, Train Steps/Sec: 0.86 + 79%|███████████████████████████████████████████████████████████████████████████████████████▎ | 32174/40903 [12:35:11<2:48:30, 1.16s/it][2025-04-21 05:31:47] (step=0032175) Train Loss: 6.3530, Train Steps/Sec: 0.86 + 79%|███████████████████████████████████████████████████████████████████████████████████████▍ | 32199/40903 [12:35:40<2:48:06, 1.16s/it][2025-04-21 05:32:16] (step=0032200) Train Loss: 6.3250, Train Steps/Sec: 0.85 + 79%|███████████████████████████████████████████████████████████████████████████████████████▍ | 32224/40903 [12:36:09<2:50:20, 1.18s/it][2025-04-21 05:32:45] (step=0032225) Train Loss: 6.3218, Train Steps/Sec: 0.87 + 79%|███████████████████████████████████████████████████████████████████████████████████████▌ | 32249/40903 [12:36:38<2:46:25, 1.15s/it][2025-04-21 05:33:14] (step=0032250) Train Loss: 6.3472, Train Steps/Sec: 0.86 + 79%|███████████████████████████████████████████████████████████████████████████████████████▌ | 32274/40903 [12:37:07<2:46:36, 1.16s/it][2025-04-21 05:33:43] (step=0032275) Train Loss: 6.3511, Train Steps/Sec: 0.87 + 79%|███████████████████████████████████████████████████████████████████████████████████████▋ | 32299/40903 [12:37:36<2:46:46, 1.16s/it][2025-04-21 05:34:12] (step=0032300) Train Loss: 6.3678, Train Steps/Sec: 0.86 + 79%|███████████████████████████████████████████████████████████████████████████████████████▋ | 32324/40903 [12:38:05<2:46:28, 1.16s/it][2025-04-21 05:34:41] (step=0032325) Train Loss: 6.3574, Train Steps/Sec: 0.87 + 79%|███████████████████████████████████████████████████████████████████████████████████████▊ | 32349/40903 [12:38:34<2:44:27, 1.15s/it][2025-04-21 05:35:10] (step=0032350) Train Loss: 6.3731, Train Steps/Sec: 0.86 + 79%|███████████████████████████████████████████████████████████████████████████████████████▊ | 32374/40903 [12:39:03<2:45:28, 1.16s/it][2025-04-21 05:35:39] (step=0032375) Train Loss: 6.3507, Train Steps/Sec: 0.86 + 79%|███████████████████████████████████████████████████████████████████████████████████████▉ | 32399/40903 [12:39:32<2:43:19, 1.15s/it][2025-04-21 05:36:08] (step=0032400) Train Loss: 6.3262, Train Steps/Sec: 0.86 + 79%|███████████████████████████████████████████████████████████████████████████████████████▉ | 32424/40903 [12:40:02<2:47:05, 1.18s/it][2025-04-21 05:36:37] (step=0032425) Train Loss: 6.3810, Train Steps/Sec: 0.86 + 79%|████████████████████████████████████████████████████████████████████████████████████████ | 32449/40903 [12:40:30<2:42:59, 1.16s/it][2025-04-21 05:37:06] (step=0032450) Train Loss: 6.3593, Train Steps/Sec: 0.86 + 79%|████████████████████████████████████████████████████████████████████████████████████████▏ | 32474/40903 [12:41:00<2:41:52, 1.15s/it][2025-04-21 05:37:35] (step=0032475) Train Loss: 6.3953, Train Steps/Sec: 0.86 + 79%|████████████████████████████████████████████████████████████████████████████████████████▏ | 32499/40903 [12:41:29<2:43:00, 1.16s/it][2025-04-21 05:38:04] (step=0032500) Train Loss: 6.4023, Train Steps/Sec: 0.86 + 80%|████████████████████████████████████████████████████████████████████████████████████████▎ | 32524/40903 [12:41:58<2:46:05, 1.19s/it][2025-04-21 05:38:33] (step=0032525) Train Loss: 6.2976, Train Steps/Sec: 0.86 + 80%|████████████████████████████████████████████████████████████████████████████████████████▎ | 32549/40903 [12:42:27<2:41:11, 1.16s/it][2025-04-21 05:39:02] (step=0032550) Train Loss: 6.3994, Train Steps/Sec: 0.86 + 80%|████████████████████████████████████████████████████████████████████████████████████████▍ | 32574/40903 [12:42:55<2:39:16, 1.15s/it][2025-04-21 05:39:31] (step=0032575) Train Loss: 6.3647, Train Steps/Sec: 0.87 + 80%|████████████████████████████████████████████████████████████████████████████████████████▍ | 32599/40903 [12:43:24<2:39:55, 1.16s/it][2025-04-21 05:40:00] (step=0032600) Train Loss: 6.3146, Train Steps/Sec: 0.86 + 80%|████████████████████████████████████████████████████████████████████████████████████████▌ | 32624/40903 [12:43:53<2:41:57, 1.17s/it][2025-04-21 05:40:29] (step=0032625) Train Loss: 6.3330, Train Steps/Sec: 0.86 + 80%|████████████████████████████████████████████████████████████████████████████████████████▌ | 32649/40903 [12:44:22<2:41:59, 1.18s/it][2025-04-21 05:40:58] (step=0032650) Train Loss: 6.3411, Train Steps/Sec: 0.86 + 80%|████████████████████████████████████████████████████████████████████████████████████████▋ | 32674/40903 [12:44:52<2:34:42, 1.13s/it][2025-04-21 05:41:27] (step=0032675) Train Loss: 6.3686, Train Steps/Sec: 0.86 + 80%|████████████████████████████████████████████████████████████████████████████████████████▋ | 32699/40903 [12:45:21<2:36:44, 1.15s/it][2025-04-21 05:41:56] (step=0032700) Train Loss: 6.3673, Train Steps/Sec: 0.86 + 80%|████████████████████████████████████████████████████████████████████████████████████████▊ | 32724/40903 [12:45:50<2:40:13, 1.18s/it][2025-04-21 05:42:25] (step=0032725) Train Loss: 6.3501, Train Steps/Sec: 0.86 + 80%|████████████████████████████████████████████████████████████████████████████████████████▊ | 32749/40903 [12:46:19<2:37:44, 1.16s/it][2025-04-21 05:42:54] (step=0032750) Train Loss: 6.3925, Train Steps/Sec: 0.86 + 80%|████████████████████████████████████████████████████████████████████████████████████████▉ | 32774/40903 [12:46:48<2:34:23, 1.14s/it][2025-04-21 05:43:23] (step=0032775) Train Loss: 6.3518, Train Steps/Sec: 0.86 + 80%|█████████████████████████████████████████████████████████████████████████████████████████ | 32799/40903 [12:47:17<2:33:37, 1.14s/it][2025-04-21 05:43:53] (step=0032800) Train Loss: 6.4033, Train Steps/Sec: 0.86 + 80%|█████████████████████████████████████████████████████████████████████████████████████████ | 32824/40903 [12:47:46<2:39:31, 1.18s/it][2025-04-21 05:44:21] (step=0032825) Train Loss: 6.3515, Train Steps/Sec: 0.87 + 80%|█████████████████████████████████████████████████████████████████████████████████████████▏ | 32849/40903 [12:48:15<2:37:42, 1.17s/it][2025-04-21 05:44:51] (step=0032850) Train Loss: 6.3209, Train Steps/Sec: 0.86 + 80%|█████████████████████████████████████████████████████████████████████████████████████████▏ | 32874/40903 [12:48:44<2:34:28, 1.15s/it][2025-04-21 05:45:19] (step=0032875) Train Loss: 6.3796, Train Steps/Sec: 0.86 + 80%|█████████████████████████████████████████████████████████████████████████████████████████▎ | 32899/40903 [12:49:13<2:31:49, 1.14s/it][2025-04-21 05:45:49] (step=0032900) Train Loss: 6.3573, Train Steps/Sec: 0.86 + 80%|█████████████████████████████████████████████████████████████████████████████████████████▎ | 32924/40903 [12:49:42<2:36:27, 1.18s/it][2025-04-21 05:46:18] (step=0032925) Train Loss: 6.3432, Train Steps/Sec: 0.86 + 81%|█████████████████████████████████████████████████████████████████████████████████████████▍ | 32949/40903 [12:50:11<2:35:33, 1.17s/it][2025-04-21 05:46:47] (step=0032950) Train Loss: 6.3513, Train Steps/Sec: 0.86 + 81%|█████████████████████████████████████████████████████████████████████████████████████████▍ | 32974/40903 [12:50:40<2:31:48, 1.15s/it][2025-04-21 05:47:15] (step=0032975) Train Loss: 6.3672, Train Steps/Sec: 0.86 + 81%|█████████████████████████████████████████████████████████████████████████████████████████▌ | 32999/40903 [12:51:09<2:31:51, 1.15s/it][2025-04-21 05:47:45] (step=0033000) Train Loss: 6.3484, Train Steps/Sec: 0.86 + 81%|█████████████████████████████████████████████████████████████████████████████████████████▌ | 33024/40903 [12:51:38<2:34:16, 1.17s/it][2025-04-21 05:48:14] (step=0033025) Train Loss: 6.3555, Train Steps/Sec: 0.85 + 81%|█████████████████████████████████████████████████████████████████████████████████████████▋ | 33049/40903 [12:52:07<2:31:22, 1.16s/it][2025-04-21 05:48:43] (step=0033050) Train Loss: 6.3730, Train Steps/Sec: 0.87 + 81%|█████████████████████████████████████████████████████████████████████████████████████████▊ | 33074/40903 [12:52:36<2:27:44, 1.13s/it][2025-04-21 05:49:12] (step=0033075) Train Loss: 6.3439, Train Steps/Sec: 0.86 + 81%|█████████████████████████████████████████████████████████████████████████████████████████▊ | 33099/40903 [12:53:05<2:29:59, 1.15s/it][2025-04-21 05:49:41] (step=0033100) Train Loss: 6.3187, Train Steps/Sec: 0.86 + 81%|█████████████████████████████████████████████████████████████████████████████████████████▉ | 33124/40903 [12:53:34<2:33:00, 1.18s/it][2025-04-21 05:50:10] (step=0033125) Train Loss: 6.3008, Train Steps/Sec: 0.87 + 81%|█████████████████████████████████████████████████████████████████████████████████████████▉ | 33149/40903 [12:54:03<2:29:50, 1.16s/it][2025-04-21 05:50:39] (step=0033150) Train Loss: 6.3250, Train Steps/Sec: 0.86 + 81%|██████████████████████████████████████████████████████████████████████████████████████████ | 33174/40903 [12:54:32<2:29:19, 1.16s/it][2025-04-21 05:51:08] (step=0033175) Train Loss: 6.3351, Train Steps/Sec: 0.86 + 81%|██████████████████████████████████████████████████████████████████████████████████████████ | 33199/40903 [12:55:01<2:27:29, 1.15s/it][2025-04-21 05:51:37] (step=0033200) Train Loss: 6.3551, Train Steps/Sec: 0.86 + 81%|██████████████████████████████████████████████████████████████████████████████████████████▏ | 33224/40903 [12:55:30<2:30:39, 1.18s/it][2025-04-21 05:52:06] (step=0033225) Train Loss: 6.3575, Train Steps/Sec: 0.87 + 81%|██████████████████████████████████████████████████████████████████████████████████████████▏ | 33249/40903 [12:55:59<2:28:48, 1.17s/it][2025-04-21 05:52:35] (step=0033250) Train Loss: 6.3817, Train Steps/Sec: 0.87 + 81%|██████████████████████████████████████████████████████████████████████████████████████████▎ | 33274/40903 [12:56:28<2:26:39, 1.15s/it][2025-04-21 05:53:04] (step=0033275) Train Loss: 6.3643, Train Steps/Sec: 0.86 + 81%|██████████████████████████████████████████████████████████████████████████████████████████▎ | 33299/40903 [12:56:57<2:25:17, 1.15s/it][2025-04-21 05:53:33] (step=0033300) Train Loss: 6.3442, Train Steps/Sec: 0.86 + 81%|██████████████████████████████████████████████████████████████████████████████████████████▍ | 33324/40903 [12:57:26<2:29:09, 1.18s/it][2025-04-21 05:54:02] (step=0033325) Train Loss: 6.3446, Train Steps/Sec: 0.87 + 82%|██████████████████████████████████████████████████████████████████████████████████████████▌ | 33349/40903 [12:57:55<2:26:54, 1.17s/it][2025-04-21 05:54:30] (step=0033350) Train Loss: 6.3794, Train Steps/Sec: 0.87 + 82%|██████████████████████████████████████████████████████████████████████████████████████████▌ | 33374/40903 [12:58:24<2:24:40, 1.15s/it][2025-04-21 05:54:59] (step=0033375) Train Loss: 6.4010, Train Steps/Sec: 0.86 + 82%|██████████████████████████████████████████████████████████████████████████████████████████▋ | 33399/40903 [12:58:53<2:25:06, 1.16s/it][2025-04-21 05:55:29] (step=0033400) Train Loss: 6.3447, Train Steps/Sec: 0.86 + 82%|██████████████████████████████████████████████████████████████████████████████████████████▋ | 33424/40903 [12:59:22<2:26:16, 1.17s/it][2025-04-21 05:55:58] (step=0033425) Train Loss: 6.3829, Train Steps/Sec: 0.87 + 82%|██████████████████████████████████████████████████████████████████████████████████████████▊ | 33449/40903 [12:59:51<2:23:37, 1.16s/it][2025-04-21 05:56:27] (step=0033450) Train Loss: 6.3589, Train Steps/Sec: 0.86 + 82%|██████████████████████████████████████████████████████████████████████████████████████████▊ | 33474/40903 [13:00:20<2:19:53, 1.13s/it][2025-04-21 05:56:55] (step=0033475) Train Loss: 6.3233, Train Steps/Sec: 0.87 + 82%|██████████████████████████████████████████████████████████████████████████████████████████▉ | 33499/40903 [13:00:49<2:20:45, 1.14s/it][2025-04-21 05:57:24] (step=0033500) Train Loss: 6.3657, Train Steps/Sec: 0.86 + 82%|██████████████████████████████████████████████████████████████████████████████████████████▉ | 33524/40903 [13:01:18<2:24:41, 1.18s/it][2025-04-21 05:57:53] (step=0033525) Train Loss: 6.3759, Train Steps/Sec: 0.86 + 82%|███████████████████████████████████████████████████████████████████████████████████████████ | 33549/40903 [13:01:47<2:21:21, 1.15s/it][2025-04-21 05:58:22] (step=0033550) Train Loss: 6.3201, Train Steps/Sec: 0.87 + 82%|███████████████████████████████████████████████████████████████████████████████████████████ | 33574/40903 [13:02:16<2:22:24, 1.17s/it][2025-04-21 05:58:51] (step=0033575) Train Loss: 6.3209, Train Steps/Sec: 0.86 + 82%|███████████████████████████████████████████████████████████████████████████████████████████▏ | 33599/40903 [13:02:45<2:19:06, 1.14s/it][2025-04-21 05:59:20] (step=0033600) Train Loss: 6.3367, Train Steps/Sec: 0.87 + 82%|███████████████████████████████████████████████████████████████████████████████████████████▏ | 33624/40903 [13:03:14<2:23:28, 1.18s/it][2025-04-21 05:59:49] (step=0033625) Train Loss: 6.3821, Train Steps/Sec: 0.86 + 82%|███████████████████████████████████████████████████████████████████████████████████████████▎ | 33649/40903 [13:03:42<2:21:28, 1.17s/it][2025-04-21 06:00:18] (step=0033650) Train Loss: 6.3537, Train Steps/Sec: 0.87 + 82%|███████████████████████████████████████████████████████████████████████████████████████████▍ | 33674/40903 [13:04:11<2:17:38, 1.14s/it][2025-04-21 06:00:47] (step=0033675) Train Loss: 6.3577, Train Steps/Sec: 0.87 + 82%|███████████████████████████████████████████████████████████████████████████████████████████▍ | 33699/40903 [13:04:40<2:17:37, 1.15s/it][2025-04-21 06:01:16] (step=0033700) Train Loss: 6.3625, Train Steps/Sec: 0.86 + 82%|███████████████████████████████████████████████████████████████████████████████████████████▌ | 33724/40903 [13:05:09<2:21:25, 1.18s/it][2025-04-21 06:01:45] (step=0033725) Train Loss: 6.3389, Train Steps/Sec: 0.86 + 83%|███████████████████████████████████████████████████████████████████████████████████████████▌ | 33749/40903 [13:05:39<2:22:00, 1.19s/it][2025-04-21 06:02:14] (step=0033750) Train Loss: 6.3792, Train Steps/Sec: 0.86 + 83%|███████████████████████████████████████████████████████████████████████████████████████████▋ | 33774/40903 [13:06:08<2:15:26, 1.14s/it][2025-04-21 06:02:43] (step=0033775) Train Loss: 6.3802, Train Steps/Sec: 0.86 + 83%|███████████████████████████████████████████████████████████████████████████████████████████▋ | 33799/40903 [13:06:37<2:17:13, 1.16s/it][2025-04-21 06:03:13] (step=0033800) Train Loss: 6.3645, Train Steps/Sec: 0.85 + 83%|███████████████████████████████████████████████████████████████████████████████████████████▊ | 33824/40903 [13:07:06<2:17:39, 1.17s/it][2025-04-21 06:03:41] (step=0033825) Train Loss: 6.4032, Train Steps/Sec: 0.87 + 83%|███████████████████████████████████████████████████████████████████████████████████████████▊ | 33849/40903 [13:07:35<2:16:24, 1.16s/it][2025-04-21 06:04:11] (step=0033850) Train Loss: 6.3200, Train Steps/Sec: 0.86 + 83%|███████████████████████████████████████████████████████████████████████████████████████████▉ | 33874/40903 [13:08:04<2:15:29, 1.16s/it][2025-04-21 06:04:40] (step=0033875) Train Loss: 6.3618, Train Steps/Sec: 0.86 + 83%|███████████████████████████████████████████████████████████████████████████████████████████▉ | 33899/40903 [13:08:33<2:13:38, 1.14s/it][2025-04-21 06:05:09] (step=0033900) Train Loss: 6.3471, Train Steps/Sec: 0.86 + 83%|████████████████████████████████████████████████████████████████████████████████████████████ | 33924/40903 [13:09:02<2:17:13, 1.18s/it][2025-04-21 06:05:38] (step=0033925) Train Loss: 6.3791, Train Steps/Sec: 0.86 + 83%|████████████████████████████████████████████████████████████████████████████████████████████▏ | 33949/40903 [13:09:31<2:14:38, 1.16s/it][2025-04-21 06:06:07] (step=0033950) Train Loss: 6.3608, Train Steps/Sec: 0.86 + 83%|████████████████████████████████████████████████████████████████████████████████████████████▏ | 33974/40903 [13:10:00<2:14:27, 1.16s/it][2025-04-21 06:06:36] (step=0033975) Train Loss: 6.3317, Train Steps/Sec: 0.86 + 83%|████████████████████████████████████████████████████████████████████████████████████████████▎ | 33999/40903 [13:10:30<2:13:05, 1.16s/it][2025-04-21 06:07:05] (step=0034000) Train Loss: 6.3671, Train Steps/Sec: 0.86 +[2025-04-21 06:07:05] vision_config is None. initializing the InstructBlipVisionConfig with default values. +[2025-04-21 06:07:05] qformer_config is None. Initializing the InstructBlipQFormerConfig with default values. +[2025-04-21 06:07:05] text_config is None. Initializing the text config with default values (`OPTConfig`). +model_name_or_path:/tmp/haozhezhao/model/instructblip-flan-t5-xl +image_place_holder: +tokenizer length after expend 32102 +tokenizer length before expend 32102 +100%|████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████| 3/3 [03:10<00:00, 63.65s/it] +[2025-04-21 06:12:08] Finish Eval in 34000 steps...█████████████████████████████████████████████████████████████████████| 3/3 [03:10<00:00, 63.17s/it] +processing left-padding... +processing left-padding... +processing left-padding... +[2025-04-21 06:12:29] Saved checkpoint to checkpoint/EmptyFix_InstructBlip_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_objects/002-GPT-XL/checkpoints/0034000.pt +[2025-04-21 06:12:32] Removed old checkpoint: checkpoint/EmptyFix_InstructBlip_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_objects/002-GPT-XL/checkpoints/0030000.pt + 83%|████████████████████████████████████████████████████████████████████████████████████████████▎ | 34024/40903 [13:16:25<2:17:59, 1.20s/it][2025-04-21 06:13:01] (step=0034025) Train Loss: 6.3782, Train Steps/Sec: 0.07 + 83%|████████████████████████████████████████████████████████████████████████████████████████████▍ | 34049/40903 [13:16:54<2:11:13, 1.15s/it][2025-04-21 06:13:30] (step=0034050) Train Loss: 6.3714, Train Steps/Sec: 0.86 + 83%|████████████████████████████████████████████████████████████████████████████████████████████▍ | 34074/40903 [13:17:23<2:12:14, 1.16s/it][2025-04-21 06:13:59] (step=0034075) Train Loss: 6.3418, Train Steps/Sec: 0.87 + 83%|████████████████████████████████████████████████████████████████████████████████████████████▌ | 34099/40903 [13:17:52<2:09:34, 1.14s/it][2025-04-21 06:14:28] (step=0034100) Train Loss: 6.3584, Train Steps/Sec: 0.86 + 83%|████████████████████████████████████████████████████████████████████████████████████████████▌ | 34124/40903 [13:18:21<2:13:46, 1.18s/it][2025-04-21 06:14:57] (step=0034125) Train Loss: 6.3528, Train Steps/Sec: 0.86 + 83%|████████████████████████████████████████████████████████████████████████████████████████████▋ | 34149/40903 [13:18:56<2:13:24, 1.19s/it][2025-04-21 06:15:31] (step=0034150) Train Loss: 6.3564, Train Steps/Sec: 0.73 + 84%|████████████████████████████████████████████████████████████████████████████████████████████▋ | 34174/40903 [13:19:25<2:09:33, 1.16s/it][2025-04-21 06:16:00] (step=0034175) Train Loss: 6.3125, Train Steps/Sec: 0.86 + 84%|████████████████████████████████████████████████████████████████████████████████████████████▊ | 34199/40903 [13:19:53<2:06:23, 1.13s/it][2025-04-21 06:16:29] (step=0034200) Train Loss: 6.2817, Train Steps/Sec: 0.87 + 84%|████████████████████████████████████████████████████████████████████████████████████████████▊ | 34224/40903 [13:20:22<2:10:01, 1.17s/it][2025-04-21 06:16:58] (step=0034225) Train Loss: 6.3578, Train Steps/Sec: 0.87 + 84%|████████████████████████████████████████████████████████████████████████████████████████████▉ | 34249/40903 [13:20:51<2:08:33, 1.16s/it][2025-04-21 06:17:27] (step=0034250) Train Loss: 6.3890, Train Steps/Sec: 0.86 + 84%|█████████████████████████████████████████████████████████████████████████████████████████████ | 34274/40903 [13:21:20<2:06:02, 1.14s/it][2025-04-21 06:17:56] (step=0034275) Train Loss: 6.2661, Train Steps/Sec: 0.87 + 84%|█████████████████████████████████████████████████████████████████████████████████████████████ | 34299/40903 [13:21:49<2:06:57, 1.15s/it][2025-04-21 06:18:25] (step=0034300) Train Loss: 6.3455, Train Steps/Sec: 0.86 + 84%|█████████████████████████████████████████████████████████████████████████████████████████████▏ | 34324/40903 [13:22:18<2:07:49, 1.17s/it][2025-04-21 06:18:54] (step=0034325) Train Loss: 6.3299, Train Steps/Sec: 0.86 + 84%|█████████████████████████████████████████████████████████████████████████████████████████████▏ | 34349/40903 [13:22:47<2:07:30, 1.17s/it][2025-04-21 06:19:23] (step=0034350) Train Loss: 6.3544, Train Steps/Sec: 0.86 + 84%|█████████████████████████████████████████████████████████████████████████████████████████████▎ | 34374/40903 [13:23:16<2:05:07, 1.15s/it][2025-04-21 06:19:52] (step=0034375) Train Loss: 6.3600, Train Steps/Sec: 0.86 + 84%|█████████████████████████████████████████████████████████████████████████████████████████████▎ | 34399/40903 [13:23:45<2:04:45, 1.15s/it][2025-04-21 06:20:21] (step=0034400) Train Loss: 6.3690, Train Steps/Sec: 0.86 + 84%|█████████████████████████████████████████████████████████████████████████████████████████████▍ | 34424/40903 [13:24:14<2:07:55, 1.18s/it][2025-04-21 06:20:55] (step=0034425) Train Loss: 6.3434, Train Steps/Sec: 0.73 + 84%|█████████████████████████████████████████████████████████████████████████████████████████████▍ | 34449/40903 [13:24:55<2:12:38, 1.23s/it][2025-04-21 06:21:31] (step=0034450) Train Loss: 6.4024, Train Steps/Sec: 0.71 + 84%|█████████████████████████████████████████████████████████████████████████████████████████████▌ | 34474/40903 [13:25:24<2:04:28, 1.16s/it][2025-04-21 06:22:00] (step=0034475) Train Loss: 6.3563, Train Steps/Sec: 0.86 + 84%|█████████████████████████████████████████████████████████████████████████████████████████████▌ | 34499/40903 [13:25:53<2:02:52, 1.15s/it][2025-04-21 06:22:29] (step=0034500) Train Loss: 6.3718, Train Steps/Sec: 0.86 + 84%|█████████████████████████████████████████████████████████████████████████████████████████████▋ | 34524/40903 [13:26:22<2:05:53, 1.18s/it][2025-04-21 06:22:58] (step=0034525) Train Loss: 6.3515, Train Steps/Sec: 0.86 + 84%|█████████████████████████████████████████████████████████████████████████████████████████████▊ | 34549/40903 [13:26:51<2:01:53, 1.15s/it][2025-04-21 06:23:27] (step=0034550) Train Loss: 6.3355, Train Steps/Sec: 0.86 + 85%|█████████████████████████████████████████████████████████████████████████████████████████████▊ | 34574/40903 [13:27:20<2:02:56, 1.17s/it][2025-04-21 06:23:56] (step=0034575) Train Loss: 6.3577, Train Steps/Sec: 0.86 + 85%|█████████████████████████████████████████████████████████████████████████████████████████████▉ | 34599/40903 [13:27:49<2:01:45, 1.16s/it][2025-04-21 06:24:25] (step=0034600) Train Loss: 6.3654, Train Steps/Sec: 0.86 + 85%|█████████████████████████████████████████████████████████████████████████████████████████████▉ | 34624/40903 [13:28:18<2:02:14, 1.17s/it][2025-04-21 06:24:54] (step=0034625) Train Loss: 6.3273, Train Steps/Sec: 0.87 + 85%|██████████████████████████████████████████████████████████████████████████████████████████████ | 34649/40903 [13:28:54<2:00:17, 1.15s/it][2025-04-21 06:25:29] (step=0034650) Train Loss: 6.3047, Train Steps/Sec: 0.70 + 85%|██████████████████████████████████████████████████████████████████████████████████████████████ | 34674/40903 [13:29:23<2:01:19, 1.17s/it][2025-04-21 06:25:59] (step=0034675) Train Loss: 6.3644, Train Steps/Sec: 0.86 + 85%|██████████████████████████████████████████████████████████████████████████████████████████████▏ | 34699/40903 [13:29:52<1:58:36, 1.15s/it][2025-04-21 06:26:28] (step=0034700) Train Loss: 6.3885, Train Steps/Sec: 0.86 + 85%|██████████████████████████████████████████████████████████████████████████████████████████████▏ | 34724/40903 [13:30:21<2:00:14, 1.17s/it][2025-04-21 06:26:57] (step=0034725) Train Loss: 6.3484, Train Steps/Sec: 0.86 + 85%|██████████████████████████████████████████████████████████████████████████████████████████████▎ | 34749/40903 [13:30:50<1:59:17, 1.16s/it][2025-04-21 06:27:25] (step=0034750) Train Loss: 6.3720, Train Steps/Sec: 0.86 + 85%|██████████████████████████████████████████████████████████████████████████████████████████████▎ | 34774/40903 [13:31:19<1:58:03, 1.16s/it][2025-04-21 06:27:54] (step=0034775) Train Loss: 6.3282, Train Steps/Sec: 0.87 + 85%|██████████████████████████████████████████████████████████████████████████████████████████████▍ | 34799/40903 [13:31:48<1:56:25, 1.14s/it][2025-04-21 06:28:23] (step=0034800) Train Loss: 6.3536, Train Steps/Sec: 0.86 + 85%|██████████████████████████████████████████████████████████████████████████████████████████████▌ | 34824/40903 [13:32:17<2:00:02, 1.18s/it][2025-04-21 06:28:53] (step=0034825) Train Loss: 6.3315, Train Steps/Sec: 0.86 + 85%|██████████████████████████████████████████████████████████████████████████████████████████████▌ | 34849/40903 [13:32:46<1:57:31, 1.16s/it][2025-04-21 06:29:22] (step=0034850) Train Loss: 6.3914, Train Steps/Sec: 0.86 + 85%|██████████████████████████████████████████████████████████████████████████████████████████████▋ | 34874/40903 [13:33:15<1:56:47, 1.16s/it][2025-04-21 06:29:50] (step=0034875) Train Loss: 6.3303, Train Steps/Sec: 0.87 + 85%|██████████████████████████████████████████████████████████████████████████████████████████████▋ | 34899/40903 [13:33:44<1:54:27, 1.14s/it][2025-04-21 06:30:19] (step=0034900) Train Loss: 6.3694, Train Steps/Sec: 0.86 + 85%|██████████████████████████████████████████████████████████████████████████████████████████████▊ | 34924/40903 [13:34:12<1:56:53, 1.17s/it][2025-04-21 06:30:48] (step=0034925) Train Loss: 6.3111, Train Steps/Sec: 0.87 + 85%|██████████████████████████████████████████████████████████████████████████████████████████████▊ | 34949/40903 [13:34:41<1:56:05, 1.17s/it][2025-04-21 06:31:17] (step=0034950) Train Loss: 6.3494, Train Steps/Sec: 0.86 + 86%|██████████████████████████████████████████████████████████████████████████████████████████████▉ | 34974/40903 [13:35:10<1:54:14, 1.16s/it][2025-04-21 06:31:46] (step=0034975) Train Loss: 6.3639, Train Steps/Sec: 0.86 + 86%|██████████████████████████████████████████████████████████████████████████████████████████████▉ | 34999/40903 [13:35:39<1:51:34, 1.13s/it][2025-04-21 06:32:15] (step=0035000) Train Loss: 6.3413, Train Steps/Sec: 0.86 + 86%|███████████████████████████████████████████████████████████████████████████████████████████████ | 35024/40903 [13:36:08<1:53:32, 1.16s/it][2025-04-21 06:32:44] (step=0035025) Train Loss: 6.3636, Train Steps/Sec: 0.87 + 86%|███████████████████████████████████████████████████████████████████████████████████████████████ | 35049/40903 [13:36:37<1:54:30, 1.17s/it][2025-04-21 06:33:13] (step=0035050) Train Loss: 6.3986, Train Steps/Sec: 0.86 + 86%|███████████████████████████████████████████████████████████████████████████████████████████████▏ | 35074/40903 [13:37:06<1:52:22, 1.16s/it][2025-04-21 06:33:42] (step=0035075) Train Loss: 6.3165, Train Steps/Sec: 0.87 + 86%|███████████████████████████████████████████████████████████████████████████████████████████████▏ | 35099/40903 [13:37:36<1:54:18, 1.18s/it][2025-04-21 06:34:11] (step=0035100) Train Loss: 6.3545, Train Steps/Sec: 0.85 + 86%|███████████████████████████████████████████████████████████████████████████████████████████████▎ | 35124/40903 [13:38:04<1:53:11, 1.18s/it][2025-04-21 06:34:40] (step=0035125) Train Loss: 6.3564, Train Steps/Sec: 0.87 + 86%|███████████████████████████████████████████████████████████████████████████████████████████████▍ | 35149/40903 [13:38:33<1:50:27, 1.15s/it][2025-04-21 06:35:09] (step=0035150) Train Loss: 6.2909, Train Steps/Sec: 0.86 + 86%|███████████████████████████████████████████████████████████████████████████████████████████████▍ | 35174/40903 [13:39:02<1:50:36, 1.16s/it][2025-04-21 06:35:38] (step=0035175) Train Loss: 6.3569, Train Steps/Sec: 0.86 + 86%|███████████████████████████████████████████████████████████████████████████████████████████████▌ | 35199/40903 [13:39:31<1:49:28, 1.15s/it][2025-04-21 06:36:07] (step=0035200) Train Loss: 6.3050, Train Steps/Sec: 0.86 + 86%|███████████████████████████████████████████████████████████████████████████████████████████████▌ | 35224/40903 [13:40:00<1:50:31, 1.17s/it][2025-04-21 06:36:36] (step=0035225) Train Loss: 6.3530, Train Steps/Sec: 0.87 + 86%|███████████████████████████████████████████████████████████████████████████████████████████████▋ | 35249/40903 [13:40:29<1:49:49, 1.17s/it][2025-04-21 06:37:05] (step=0035250) Train Loss: 6.3751, Train Steps/Sec: 0.86 + 86%|███████████████████████████████████████████████████████████████████████████████████████████████▋ | 35274/40903 [13:40:58<1:49:40, 1.17s/it][2025-04-21 06:37:34] (step=0035275) Train Loss: 6.3640, Train Steps/Sec: 0.86 + 86%|███████████████████████████████████████████████████████████████████████████████████████████████▊ | 35299/40903 [13:41:27<1:49:47, 1.18s/it][2025-04-21 06:38:03] (step=0035300) Train Loss: 6.3587, Train Steps/Sec: 0.86 + 86%|███████████████████████████████████████████████████████████████████████████████████████████████▊ | 35324/40903 [13:41:56<1:48:24, 1.17s/it][2025-04-21 06:38:32] (step=0035325) Train Loss: 6.3018, Train Steps/Sec: 0.87 + 86%|███████████████████████████████████████████████████████████████████████████████████████████████▉ | 35349/40903 [13:42:25<1:47:08, 1.16s/it][2025-04-21 06:39:01] (step=0035350) Train Loss: 6.3812, Train Steps/Sec: 0.86 + 86%|███████████████████████████████████████████████████████████████████████████████████████████████▉ | 35374/40903 [13:42:54<1:46:52, 1.16s/it][2025-04-21 06:39:30] (step=0035375) Train Loss: 6.3254, Train Steps/Sec: 0.86 + 87%|████████████████████████████████████████████████████████████████████████████████████████████████ | 35399/40903 [13:43:23<1:44:54, 1.14s/it][2025-04-21 06:39:59] (step=0035400) Train Loss: 6.3218, Train Steps/Sec: 0.86 + 87%|████████████████████████████████████████████████████████████████████████████████████████████████▏ | 35424/40903 [13:43:52<1:46:41, 1.17s/it][2025-04-21 06:40:28] (step=0035425) Train Loss: 6.3515, Train Steps/Sec: 0.87 + 87%|████████████████████████████████████████████████████████████████████████████████████████████████▏ | 35449/40903 [13:44:21<1:44:59, 1.15s/it][2025-04-21 06:40:57] (step=0035450) Train Loss: 6.3738, Train Steps/Sec: 0.86 + 87%|████████████████████████████████████████████████████████████████████████████████████████████████▎ | 35474/40903 [13:44:50<1:44:40, 1.16s/it][2025-04-21 06:41:26] (step=0035475) Train Loss: 6.3773, Train Steps/Sec: 0.86 + 87%|████████████████████████████████████████████████████████████████████████████████████████████████▎ | 35499/40903 [13:45:19<1:44:24, 1.16s/it][2025-04-21 06:41:55] (step=0035500) Train Loss: 6.3686, Train Steps/Sec: 0.86 + 87%|████████████████████████████████████████████████████████████████████████████████████████████████▍ | 35524/40903 [13:45:48<1:45:23, 1.18s/it][2025-04-21 06:42:24] (step=0035525) Train Loss: 6.3396, Train Steps/Sec: 0.87 + 87%|████████████████████████████████████████████████████████████████████████████████████████████████▍ | 35549/40903 [13:46:17<1:41:59, 1.14s/it][2025-04-21 06:42:53] (step=0035550) Train Loss: 6.3475, Train Steps/Sec: 0.87 + 87%|████████████████████████████████████████████████████████████████████████████████████████████████▌ | 35574/40903 [13:46:46<1:42:18, 1.15s/it][2025-04-21 06:43:22] (step=0035575) Train Loss: 6.3448, Train Steps/Sec: 0.86 + 87%|████████████████████████████████████████████████████████████████████████████████████████████████▌ | 35599/40903 [13:47:15<1:42:03, 1.15s/it][2025-04-21 06:43:51] (step=0035600) Train Loss: 6.3247, Train Steps/Sec: 0.86 + 87%|████████████████████████████████████████████████████████████████████████████████████████████████▋ | 35624/40903 [13:47:44<1:42:58, 1.17s/it][2025-04-21 06:44:20] (step=0035625) Train Loss: 6.3567, Train Steps/Sec: 0.86 + 87%|████████████████████████████████████████████████████████████████████████████████████████████████▋ | 35649/40903 [13:48:13<1:42:04, 1.17s/it][2025-04-21 06:44:49] (step=0035650) Train Loss: 6.3511, Train Steps/Sec: 0.86 + 87%|████████████████████████████████████████████████████████████████████████████████████████████████▊ | 35674/40903 [13:48:42<1:40:38, 1.15s/it][2025-04-21 06:45:18] (step=0035675) Train Loss: 6.3892, Train Steps/Sec: 0.86 + 87%|████████████████████████████████████████████████████████████████████████████████████████████████▉ | 35699/40903 [13:49:11<1:40:29, 1.16s/it][2025-04-21 06:45:47] (step=0035700) Train Loss: 6.3473, Train Steps/Sec: 0.86 + 87%|████████████████████████████████████████████████████████████████████████████████████████████████▉ | 35724/40903 [13:49:40<1:40:57, 1.17s/it][2025-04-21 06:46:16] (step=0035725) Train Loss: 6.3324, Train Steps/Sec: 0.86 + 87%|█████████████████████████████████████████████████████████████████████████████████████████████████ | 35749/40903 [13:50:09<1:39:46, 1.16s/it][2025-04-21 06:46:45] (step=0035750) Train Loss: 6.3351, Train Steps/Sec: 0.86 + 87%|█████████████████████████████████████████████████████████████████████████████████████████████████ | 35774/40903 [13:50:38<1:40:10, 1.17s/it][2025-04-21 06:47:14] (step=0035775) Train Loss: 6.3669, Train Steps/Sec: 0.86 + 88%|█████████████████████████████████████████████████████████████████████████████████████████████████▏ | 35799/40903 [13:51:07<1:36:54, 1.14s/it][2025-04-21 06:47:43] (step=0035800) Train Loss: 6.3392, Train Steps/Sec: 0.86 + 88%|█████████████████████████████████████████████████████████████████████████████████████████████████▏ | 35824/40903 [13:51:36<1:39:26, 1.17s/it][2025-04-21 06:48:12] (step=0035825) Train Loss: 6.3484, Train Steps/Sec: 0.87 + 88%|█████████████████████████████████████████████████████████████████████████████████████████████████▎ | 35849/40903 [13:52:05<1:38:13, 1.17s/it][2025-04-21 06:48:41] (step=0035850) Train Loss: 6.3254, Train Steps/Sec: 0.86 + 88%|█████████████████████████████████████████████████████████████████████████████████████████████████▎ | 35874/40903 [13:52:34<1:36:42, 1.15s/it][2025-04-21 06:49:10] (step=0035875) Train Loss: 6.3521, Train Steps/Sec: 0.86 + 88%|█████████████████████████████████████████████████████████████████████████████████████████████████▍ | 35899/40903 [13:53:03<1:37:01, 1.16s/it][2025-04-21 06:49:39] (step=0035900) Train Loss: 6.3560, Train Steps/Sec: 0.86 + 88%|█████████████████████████████████████████████████████████████████████████████████████████████████▍ | 35924/40903 [13:53:32<1:37:56, 1.18s/it][2025-04-21 06:50:08] (step=0035925) Train Loss: 6.3514, Train Steps/Sec: 0.86 + 88%|█████████████████████████████████████████████████████████████████████████████████████████████████▌ | 35949/40903 [13:54:01<1:34:49, 1.15s/it][2025-04-21 06:50:37] (step=0035950) Train Loss: 6.3311, Train Steps/Sec: 0.87 + 88%|█████████████████████████████████████████████████████████████████████████████████████████████████▌ | 35974/40903 [13:54:30<1:33:49, 1.14s/it][2025-04-21 06:51:06] (step=0035975) Train Loss: 6.3172, Train Steps/Sec: 0.86 + 88%|█████████████████████████████████████████████████████████████████████████████████████████████████▋ | 35999/40903 [13:54:59<1:33:50, 1.15s/it][2025-04-21 06:51:35] (step=0036000) Train Loss: 6.3432, Train Steps/Sec: 0.86 +[2025-04-21 06:51:35] vision_config is None. initializing the InstructBlipVisionConfig with default values. +[2025-04-21 06:51:35] qformer_config is None. Initializing the InstructBlipQFormerConfig with default values. +[2025-04-21 06:51:35] text_config is None. Initializing the text config with default values (`OPTConfig`). +model_name_or_path:/tmp/haozhezhao/model/instructblip-flan-t5-xl +image_place_holder: +tokenizer length after expend 32102 +tokenizer length before expend 32102 +100%|████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████| 3/3 [03:08<00:00, 62.84s/it] +[2025-04-21 06:56:35] Finish Eval in 36000 steps...█████████████████████████████████████████████████████████████████████| 3/3 [03:08<00:00, 62.52s/it] +processing left-padding... +processing left-padding... +processing left-padding... +[2025-04-21 06:56:57] Saved checkpoint to checkpoint/EmptyFix_InstructBlip_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_objects/002-GPT-XL/checkpoints/0036000.pt +[2025-04-21 06:56:59] Removed old checkpoint: checkpoint/EmptyFix_InstructBlip_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_objects/002-GPT-XL/checkpoints/0032000.pt + 88%|█████████████████████████████████████████████████████████████████████████████████████████████████▊ | 36024/40903 [14:00:53<1:37:43, 1.20s/it][2025-04-21 06:57:28] (step=0036025) Train Loss: 6.3512, Train Steps/Sec: 0.07 + 88%|█████████████████████████████████████████████████████████████████████████████████████████████████▊ | 36049/40903 [14:01:28<1:41:41, 1.26s/it][2025-04-21 06:58:04] (step=0036050) Train Loss: 6.3443, Train Steps/Sec: 0.71 + 88%|█████████████████████████████████████████████████████████████████████████████████████████████████▉ | 36074/40903 [14:01:57<1:32:47, 1.15s/it][2025-04-21 06:58:33] (step=0036075) Train Loss: 6.3503, Train Steps/Sec: 0.86 + 88%|█████████████████████████████████████████████████████████████████████████████████████████████████▉ | 36099/40903 [14:02:26<1:31:41, 1.15s/it][2025-04-21 06:59:02] (step=0036100) Train Loss: 6.3222, Train Steps/Sec: 0.86 + 88%|██████████████████████████████████████████████████████████████████████████████████████████████████ | 36124/40903 [14:02:55<1:34:41, 1.19s/it][2025-04-21 06:59:31] (step=0036125) Train Loss: 6.3539, Train Steps/Sec: 0.87 + 88%|██████████████████████████████████████████████████████████████████████████████████████████████████ | 36149/40903 [14:03:24<1:31:49, 1.16s/it][2025-04-21 07:00:00] (step=0036150) Train Loss: 6.3735, Train Steps/Sec: 0.87 + 88%|██████████████████████████████████████████████████████████████████████████████████████████████████▏ | 36174/40903 [14:03:53<1:30:36, 1.15s/it][2025-04-21 07:00:28] (step=0036175) Train Loss: 6.3481, Train Steps/Sec: 0.87 + 88%|██████████████████████████████████████████████████████████████████████████████████████████████████▏ | 36199/40903 [14:04:22<1:32:03, 1.17s/it][2025-04-21 07:00:58] (step=0036200) Train Loss: 6.3727, Train Steps/Sec: 0.85 + 89%|██████████████████████████████████████████████████████████████████████████████████████████████████▎ | 36224/40903 [14:04:51<1:30:15, 1.16s/it][2025-04-21 07:01:27] (step=0036225) Train Loss: 6.3583, Train Steps/Sec: 0.86 + 89%|██████████████████████████████████████████████████████████████████████████████████████████████████▎ | 36249/40903 [14:05:20<1:29:46, 1.16s/it][2025-04-21 07:01:56] (step=0036250) Train Loss: 6.3597, Train Steps/Sec: 0.86 + 89%|██████████████████████████████████████████████████████████████████████████████████████████████████▍ | 36274/40903 [14:05:49<1:28:42, 1.15s/it][2025-04-21 07:02:24] (step=0036275) Train Loss: 6.3344, Train Steps/Sec: 0.87 + 89%|██████████████████████████████████████████████████████████████████████████████████████████████████▌ | 36299/40903 [14:06:18<1:29:08, 1.16s/it][2025-04-21 07:02:54] (step=0036300) Train Loss: 6.3535, Train Steps/Sec: 0.86 + 89%|██████████████████████████████████████████████████████████████████████████████████████████████████▌ | 36324/40903 [14:06:47<1:29:00, 1.17s/it][2025-04-21 07:03:22] (step=0036325) Train Loss: 6.3246, Train Steps/Sec: 0.86 + 89%|██████████████████████████████████████████████████████████████████████████████████████████████████▋ | 36349/40903 [14:07:16<1:27:36, 1.15s/it][2025-04-21 07:03:51] (step=0036350) Train Loss: 6.3377, Train Steps/Sec: 0.87 + 89%|██████████████████████████████████████████████████████████████████████████████████████████████████▋ | 36374/40903 [14:07:45<1:25:54, 1.14s/it][2025-04-21 07:04:20] (step=0036375) Train Loss: 6.3380, Train Steps/Sec: 0.86 + 89%|██████████████████████████████████████████████████████████████████████████████████████████████████▊ | 36399/40903 [14:08:14<1:25:40, 1.14s/it][2025-04-21 07:04:50] (step=0036400) Train Loss: 6.3153, Train Steps/Sec: 0.85 + 89%|██████████████████████████████████████████████████████████████████████████████████████████████████▊ | 36424/40903 [14:08:43<1:28:11, 1.18s/it][2025-04-21 07:05:19] (step=0036425) Train Loss: 6.3124, Train Steps/Sec: 0.86 + 89%|██████████████████████████████████████████████████████████████████████████████████████████████████▉ | 36449/40903 [14:09:12<1:25:32, 1.15s/it][2025-04-21 07:05:47] (step=0036450) Train Loss: 6.3288, Train Steps/Sec: 0.87 + 89%|██████████████████████████████████████████████████████████████████████████████████████████████████▉ | 36474/40903 [14:09:41<1:26:24, 1.17s/it][2025-04-21 07:06:17] (step=0036475) Train Loss: 6.3996, Train Steps/Sec: 0.86 + 89%|███████████████████████████████████████████████████████████████████████████████████████████████████ | 36499/40903 [14:10:10<1:23:43, 1.14s/it][2025-04-21 07:06:46] (step=0036500) Train Loss: 6.3455, Train Steps/Sec: 0.86 + 89%|███████████████████████████████████████████████████████████████████████████████████████████████████ | 36524/40903 [14:10:39<1:27:10, 1.19s/it][2025-04-21 07:07:15] (step=0036525) Train Loss: 6.3838, Train Steps/Sec: 0.86 + 89%|███████████████████████████████████████████████████████████████████████████████████████████████████▏ | 36549/40903 [14:11:08<1:23:56, 1.16s/it][2025-04-21 07:07:44] (step=0036550) Train Loss: 6.3471, Train Steps/Sec: 0.86 + 89%|███████████████████████████████████████████████████████████████████████████████████████████████████▎ | 36574/40903 [14:11:37<1:24:37, 1.17s/it][2025-04-21 07:08:13] (step=0036575) Train Loss: 6.3403, Train Steps/Sec: 0.86 + 89%|███████████████████████████████████████████████████████████████████████████████████████████████████▎ | 36599/40903 [14:12:06<1:22:56, 1.16s/it][2025-04-21 07:08:42] (step=0036600) Train Loss: 6.3594, Train Steps/Sec: 0.86 + 90%|███████████████████████████████████████████████████████████████████████████████████████████████████▍ | 36624/40903 [14:12:35<1:23:20, 1.17s/it][2025-04-21 07:09:11] (step=0036625) Train Loss: 6.3033, Train Steps/Sec: 0.87 + 90%|███████████████████████████████████████████████████████████████████████████████████████████████████▍ | 36649/40903 [14:13:12<1:28:43, 1.25s/it][2025-04-21 07:09:47] (step=0036650) Train Loss: 6.3267, Train Steps/Sec: 0.69 + 90%|███████████████████████████████████████████████████████████████████████████████████████████████████▌ | 36674/40903 [14:13:41<1:20:32, 1.14s/it][2025-04-21 07:10:16] (step=0036675) Train Loss: 6.3850, Train Steps/Sec: 0.87 + 90%|███████████████████████████████████████████████████████████████████████████████████████████████████▌ | 36699/40903 [14:14:09<1:19:49, 1.14s/it][2025-04-21 07:10:45] (step=0036700) Train Loss: 6.3499, Train Steps/Sec: 0.86 + 90%|███████████████████████████████████████████████████████████████████████████████████████████████████▋ | 36724/40903 [14:14:38<1:21:46, 1.17s/it][2025-04-21 07:11:14] (step=0036725) Train Loss: 6.3429, Train Steps/Sec: 0.87 + 90%|███████████████████████████████████████████████████████████████████████████████████████████████████▋ | 36749/40903 [14:15:07<1:19:43, 1.15s/it][2025-04-21 07:11:43] (step=0036750) Train Loss: 6.3251, Train Steps/Sec: 0.86 + 90%|███████████████████████████████████████████████████████████████████████████████████████████████████▊ | 36774/40903 [14:15:36<1:18:36, 1.14s/it][2025-04-21 07:12:12] (step=0036775) Train Loss: 6.3110, Train Steps/Sec: 0.87 + 90%|███████████████████████████████████████████████████████████████████████████████████████████████████▊ | 36799/40903 [14:16:05<1:17:57, 1.14s/it][2025-04-21 07:12:41] (step=0036800) Train Loss: 6.3927, Train Steps/Sec: 0.86 + 90%|███████████████████████████████████████████████████████████████████████████████████████████████████▉ | 36824/40903 [14:16:34<1:19:15, 1.17s/it][2025-04-21 07:13:10] (step=0036825) Train Loss: 6.3142, Train Steps/Sec: 0.86 + 90%|███████████████████████████████████████████████████████████████████████████████████████████████████▉ | 36849/40903 [14:17:03<1:18:47, 1.17s/it][2025-04-21 07:13:38] (step=0036850) Train Loss: 6.3475, Train Steps/Sec: 0.87 + 90%|████████████████████████████████████████████████████████████████████████████████████████████████████ | 36874/40903 [14:17:32<1:17:35, 1.16s/it][2025-04-21 07:14:07] (step=0036875) Train Loss: 6.2929, Train Steps/Sec: 0.86 + 90%|████████████████████████████████████████████████████████████████████████████████████████████████████▏ | 36899/40903 [14:18:01<1:16:44, 1.15s/it][2025-04-21 07:14:37] (step=0036900) Train Loss: 6.3314, Train Steps/Sec: 0.86 + 90%|████████████████████████████████████████████████████████████████████████████████████████████████████▏ | 36924/40903 [14:18:30<1:17:16, 1.17s/it][2025-04-21 07:15:05] (step=0036925) Train Loss: 6.3500, Train Steps/Sec: 0.87 + 90%|████████████████████████████████████████████████████████████████████████████████████████████████████▎ | 36949/40903 [14:18:59<1:16:21, 1.16s/it][2025-04-21 07:15:34] (step=0036950) Train Loss: 6.4056, Train Steps/Sec: 0.86 + 90%|████████████████████████████████████████████████████████████████████████████████████████████████████▎ | 36974/40903 [14:19:28<1:15:11, 1.15s/it][2025-04-21 07:16:03] (step=0036975) Train Loss: 6.3125, Train Steps/Sec: 0.87 + 90%|████████████████████████████████████████████████████████████████████████████████████████████████████▍ | 36999/40903 [14:19:57<1:14:55, 1.15s/it][2025-04-21 07:16:33] (step=0037000) Train Loss: 6.3668, Train Steps/Sec: 0.86 + 91%|████████████████████████████████████████████████████████████████████████████████████████████████████▍ | 37024/40903 [14:20:26<1:16:04, 1.18s/it][2025-04-21 07:17:02] (step=0037025) Train Loss: 6.3610, Train Steps/Sec: 0.86 + 91%|████████████████████████████████████████████████████████████████████████████████████████████████████▌ | 37049/40903 [14:20:55<1:14:12, 1.16s/it][2025-04-21 07:17:30] (step=0037050) Train Loss: 6.3637, Train Steps/Sec: 0.86 + 91%|████████████████████████████████████████████████████████████████████████████████████████████████████▌ | 37074/40903 [14:21:24<1:14:08, 1.16s/it][2025-04-21 07:17:59] (step=0037075) Train Loss: 6.3473, Train Steps/Sec: 0.87 + 91%|████████████████████████████████████████████████████████████████████████████████████████████████████▋ | 37099/40903 [14:21:53<1:13:45, 1.16s/it][2025-04-21 07:18:29] (step=0037100) Train Loss: 6.3363, Train Steps/Sec: 0.86 + 91%|████████████████████████████████████████████████████████████████████████████████████████████████████▋ | 37124/40903 [14:22:22<1:13:30, 1.17s/it][2025-04-21 07:18:57] (step=0037125) Train Loss: 6.3611, Train Steps/Sec: 0.87 + 91%|████████████████████████████████████████████████████████████████████████████████████████████████████▊ | 37149/40903 [14:22:51<1:12:57, 1.17s/it][2025-04-21 07:19:26] (step=0037150) Train Loss: 6.3327, Train Steps/Sec: 0.86 + 91%|████████████████████████████████████████████████████████████████████████████████████████████████████▉ | 37174/40903 [14:23:19<1:10:56, 1.14s/it][2025-04-21 07:19:55] (step=0037175) Train Loss: 6.3202, Train Steps/Sec: 0.87 + 91%|████████████████████████████████████████████████████████████████████████████████████████████████████▉ | 37199/40903 [14:23:48<1:10:33, 1.14s/it][2025-04-21 07:20:24] (step=0037200) Train Loss: 6.3310, Train Steps/Sec: 0.86 + 91%|█████████████████████████████████████████████████████████████████████████████████████████████████████ | 37224/40903 [14:24:18<1:12:08, 1.18s/it][2025-04-21 07:20:53] (step=0037225) Train Loss: 6.3426, Train Steps/Sec: 0.87 + 91%|█████████████████████████████████████████████████████████████████████████████████████████████████████ | 37249/40903 [14:24:46<1:10:22, 1.16s/it][2025-04-21 07:21:22] (step=0037250) Train Loss: 6.3455, Train Steps/Sec: 0.87 + 91%|█████████████████████████████████████████████████████████████████████████████████████████████████████▏ | 37274/40903 [14:25:15<1:09:40, 1.15s/it][2025-04-21 07:21:51] (step=0037275) Train Loss: 6.3697, Train Steps/Sec: 0.87 + 91%|█████████████████████████████████████████████████████████████████████████████████████████████████████▏ | 37299/40903 [14:25:44<1:08:49, 1.15s/it][2025-04-21 07:22:20] (step=0037300) Train Loss: 6.3396, Train Steps/Sec: 0.86 + 91%|█████████████████████████████████████████████████████████████████████████████████████████████████████▎ | 37324/40903 [14:26:13<1:10:12, 1.18s/it][2025-04-21 07:22:49] (step=0037325) Train Loss: 6.3309, Train Steps/Sec: 0.86 + 91%|█████████████████████████████████████████████████████████████████████████████████████████████████████▎ | 37349/40903 [14:26:42<1:08:31, 1.16s/it][2025-04-21 07:23:18] (step=0037350) Train Loss: 6.3560, Train Steps/Sec: 0.87 + 91%|█████████████████████████████████████████████████████████████████████████████████████████████████████▍ | 37374/40903 [14:27:11<1:08:54, 1.17s/it][2025-04-21 07:23:47] (step=0037375) Train Loss: 6.3322, Train Steps/Sec: 0.86 + 91%|█████████████████████████████████████████████████████████████████████████████████████████████████████▍ | 37399/40903 [14:27:40<1:06:50, 1.14s/it][2025-04-21 07:24:16] (step=0037400) Train Loss: 6.3518, Train Steps/Sec: 0.86 + 91%|█████████████████████████████████████████████████████████████████████████████████████████████████████▌ | 37424/40903 [14:28:09<1:08:10, 1.18s/it][2025-04-21 07:24:45] (step=0037425) Train Loss: 6.3518, Train Steps/Sec: 0.87 + 92%|█████████████████████████████████████████████████████████████████████████████████████████████████████▋ | 37449/40903 [14:28:38<1:07:52, 1.18s/it][2025-04-21 07:25:14] (step=0037450) Train Loss: 6.3780, Train Steps/Sec: 0.86 + 92%|█████████████████████████████████████████████████████████████████████████████████████████████████████▋ | 37474/40903 [14:29:14<1:10:45, 1.24s/it][2025-04-21 07:25:49] (step=0037475) Train Loss: 6.3342, Train Steps/Sec: 0.70 + 92%|█████████████████████████████████████████████████████████████████████████████████████████████████████▊ | 37499/40903 [14:29:42<1:05:02, 1.15s/it][2025-04-21 07:26:18] (step=0037500) Train Loss: 6.3120, Train Steps/Sec: 0.87 + 92%|█████████████████████████████████████████████████████████████████████████████████████████████████████▊ | 37524/40903 [14:30:12<1:06:47, 1.19s/it][2025-04-21 07:26:47] (step=0037525) Train Loss: 6.3522, Train Steps/Sec: 0.86 + 92%|█████████████████████████████████████████████████████████████████████████████████████████████████████▉ | 37549/40903 [14:30:41<1:05:29, 1.17s/it][2025-04-21 07:27:16] (step=0037550) Train Loss: 6.3282, Train Steps/Sec: 0.86 + 92%|█████████████████████████████████████████████████████████████████████████████████████████████████████▉ | 37574/40903 [14:31:10<1:03:23, 1.14s/it][2025-04-21 07:27:45] (step=0037575) Train Loss: 6.3630, Train Steps/Sec: 0.86 + 92%|██████████████████████████████████████████████████████████████████████████████████████████████████████ | 37599/40903 [14:31:39<1:03:55, 1.16s/it][2025-04-21 07:28:15] (step=0037600) Train Loss: 6.2970, Train Steps/Sec: 0.86 + 92%|██████████████████████████████████████████████████████████████████████████████████████████████████████ | 37624/40903 [14:32:08<1:04:36, 1.18s/it][2025-04-21 07:28:43] (step=0037625) Train Loss: 6.3258, Train Steps/Sec: 0.87 + 92%|██████████████████████████████████████████████████████████████████████████████████████████████████████▏ | 37649/40903 [14:32:37<1:03:08, 1.16s/it][2025-04-21 07:29:13] (step=0037650) Train Loss: 6.3495, Train Steps/Sec: 0.86 + 92%|██████████████████████████████████████████████████████████████████████████████████████████████████████▏ | 37674/40903 [14:33:06<1:02:14, 1.16s/it][2025-04-21 07:29:42] (step=0037675) Train Loss: 6.3584, Train Steps/Sec: 0.86 + 92%|██████████████████████████████████████████████████████████████████████████████████████████████████████▎ | 37699/40903 [14:33:35<1:01:45, 1.16s/it][2025-04-21 07:30:11] (step=0037700) Train Loss: 6.3177, Train Steps/Sec: 0.86 + 92%|██████████████████████████████████████████████████████████████████████████████████████████████████████▎ | 37724/40903 [14:34:04<1:04:02, 1.21s/it][2025-04-21 07:30:40] (step=0037725) Train Loss: 6.3937, Train Steps/Sec: 0.86 + 92%|██████████████████████████████████████████████████████████████████████████████████████████████████████▍ | 37749/40903 [14:34:33<1:01:55, 1.18s/it][2025-04-21 07:31:09] (step=0037750) Train Loss: 6.2960, Train Steps/Sec: 0.86 + 92%|████████████████████████████████████████████████████████████████████████████████████████████████████████▎ | 37774/40903 [14:35:02<59:04, 1.13s/it][2025-04-21 07:31:38] (step=0037775) Train Loss: 6.3160, Train Steps/Sec: 0.87 + 92%|████████████████████████████████████████████████████████████████████████████████████████████████████████▍ | 37799/40903 [14:35:31<58:36, 1.13s/it][2025-04-21 07:32:06] (step=0037800) Train Loss: 6.3358, Train Steps/Sec: 0.87 + 92%|██████████████████████████████████████████████████████████████████████████████████████████████████████▋ | 37824/40903 [14:36:00<1:01:03, 1.19s/it][2025-04-21 07:32:36] (step=0037825) Train Loss: 6.3448, Train Steps/Sec: 0.86 + 93%|████████████████████████████████████████████████████████████████████████████████████████████████████████▌ | 37849/40903 [14:36:29<59:00, 1.16s/it][2025-04-21 07:33:05] (step=0037850) Train Loss: 6.3323, Train Steps/Sec: 0.86 + 93%|████████████████████████████████████████████████████████████████████████████████████████████████████████▋ | 37874/40903 [14:36:58<57:39, 1.14s/it][2025-04-21 07:33:34] (step=0037875) Train Loss: 6.3514, Train Steps/Sec: 0.86 + 93%|████████████████████████████████████████████████████████████████████████████████████████████████████████▋ | 37899/40903 [14:37:27<57:43, 1.15s/it][2025-04-21 07:34:03] (step=0037900) Train Loss: 6.3267, Train Steps/Sec: 0.85 + 93%|████████████████████████████████████████████████████████████████████████████████████████████████████████▊ | 37924/40903 [14:37:56<58:40, 1.18s/it][2025-04-21 07:34:32] (step=0037925) Train Loss: 6.3423, Train Steps/Sec: 0.87 + 93%|████████████████████████████████████████████████████████████████████████████████████████████████████████▊ | 37949/40903 [14:38:25<56:58, 1.16s/it][2025-04-21 07:35:00] (step=0037950) Train Loss: 6.3190, Train Steps/Sec: 0.87 + 93%|████████████████████████████████████████████████████████████████████████████████████████████████████████▉ | 37974/40903 [14:38:54<55:42, 1.14s/it][2025-04-21 07:35:29] (step=0037975) Train Loss: 6.3968, Train Steps/Sec: 0.86 + 93%|████████████████████████████████████████████████████████████████████████████████████████████████████████▉ | 37999/40903 [14:39:23<55:40, 1.15s/it][2025-04-21 07:35:58] (step=0038000) Train Loss: 6.3746, Train Steps/Sec: 0.86 +[2025-04-21 07:35:58] vision_config is None. initializing the InstructBlipVisionConfig with default values. +[2025-04-21 07:35:58] qformer_config is None. Initializing the InstructBlipQFormerConfig with default values. +[2025-04-21 07:35:58] text_config is None. Initializing the text config with default values (`OPTConfig`). +model_name_or_path:/tmp/haozhezhao/model/instructblip-flan-t5-xl +image_place_holder: +tokenizer length after expend 32102 +tokenizer length before expend 32102 +100%|████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████| 3/3 [03:10<00:00, 63.50s/it] +[2025-04-21 07:41:01] Finish Eval in 38000 steps...█████████████████████████████████████████████████████████████████████| 3/3 [03:10<00:00, 63.17s/it] +processing left-padding... +processing left-padding... +processing left-padding... +[2025-04-21 07:41:22] Saved checkpoint to checkpoint/EmptyFix_InstructBlip_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_objects/002-GPT-XL/checkpoints/0038000.pt +[2025-04-21 07:41:24] Removed old checkpoint: checkpoint/EmptyFix_InstructBlip_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_objects/002-GPT-XL/checkpoints/0034000.pt + 93%|█████████████████████████████████████████████████████████████████████████████████████████████████████████ | 38024/40903 [14:45:18<58:06, 1.21s/it][2025-04-21 07:41:54] (step=0038025) Train Loss: 6.3458, Train Steps/Sec: 0.07 + 93%|█████████████████████████████████████████████████████████████████████████████████████████████████████████ | 38049/40903 [14:45:47<55:20, 1.16s/it][2025-04-21 07:42:22] (step=0038050) Train Loss: 6.3498, Train Steps/Sec: 0.87 + 93%|█████████████████████████████████████████████████████████████████████████████████████████████████████████▏ | 38074/40903 [14:46:16<54:22, 1.15s/it][2025-04-21 07:42:52] (step=0038075) Train Loss: 6.3843, Train Steps/Sec: 0.86 + 93%|█████████████████████████████████████████████████████████████████████████████████████████████████████████▎ | 38099/40903 [14:46:45<53:04, 1.14s/it][2025-04-21 07:43:20] (step=0038100) Train Loss: 6.3375, Train Steps/Sec: 0.86 + 93%|█████████████████████████████████████████████████████████████████████████████████████████████████████████▎ | 38124/40903 [14:47:14<54:07, 1.17s/it][2025-04-21 07:43:50] (step=0038125) Train Loss: 6.3763, Train Steps/Sec: 0.86 + 93%|█████████████████████████████████████████████████████████████████████████████████████████████████████████▍ | 38149/40903 [14:47:43<53:08, 1.16s/it][2025-04-21 07:44:18] (step=0038150) Train Loss: 6.3122, Train Steps/Sec: 0.87 + 93%|█████████████████████████████████████████████████████████████████████████████████████████████████████████▍ | 38174/40903 [14:48:12<53:15, 1.17s/it][2025-04-21 07:44:47] (step=0038175) Train Loss: 6.3205, Train Steps/Sec: 0.87 + 93%|█████████████████████████████████████████████████████████████████████████████████████████████████████████▌ | 38199/40903 [14:48:40<51:04, 1.13s/it][2025-04-21 07:45:16] (step=0038200) Train Loss: 6.3253, Train Steps/Sec: 0.86 + 93%|█████████████████████████████████████████████████████████████████████████████████████████████████████████▌ | 38224/40903 [14:49:10<52:58, 1.19s/it][2025-04-21 07:45:45] (step=0038225) Train Loss: 6.3043, Train Steps/Sec: 0.86 + 94%|█████████████████████████████████████████████████████████████████████████████████████████████████████████▋ | 38249/40903 [14:49:39<50:35, 1.14s/it][2025-04-21 07:46:15] (step=0038250) Train Loss: 6.3291, Train Steps/Sec: 0.86 + 94%|█████████████████████████████████████████████████████████████████████████████████████████████████████████▋ | 38274/40903 [14:50:08<50:28, 1.15s/it][2025-04-21 07:46:44] (step=0038275) Train Loss: 6.3330, Train Steps/Sec: 0.86 + 94%|█████████████████████████████████████████████████████████████████████████████████████████████████████████▊ | 38299/40903 [14:50:37<50:05, 1.15s/it][2025-04-21 07:47:13] (step=0038300) Train Loss: 6.3704, Train Steps/Sec: 0.85 + 94%|█████████████████████████████████████████████████████████████████████████████████████████████████████████▉ | 38324/40903 [14:51:06<50:28, 1.17s/it][2025-04-21 07:47:42] (step=0038325) Train Loss: 6.3672, Train Steps/Sec: 0.86 + 94%|█████████████████████████████████████████████████████████████████████████████████████████████████████████▉ | 38349/40903 [14:51:35<50:05, 1.18s/it][2025-04-21 07:48:11] (step=0038350) Train Loss: 6.3459, Train Steps/Sec: 0.86 + 94%|██████████████████████████████████████████████████████████████████████████████████████████████████████████ | 38374/40903 [14:52:04<48:14, 1.14s/it][2025-04-21 07:48:40] (step=0038375) Train Loss: 6.3592, Train Steps/Sec: 0.87 + 94%|██████████████████████████████████████████████████████████████████████████████████████████████████████████ | 38399/40903 [14:52:33<48:05, 1.15s/it][2025-04-21 07:49:09] (step=0038400) Train Loss: 6.3584, Train Steps/Sec: 0.86 + 94%|██████████████████████████████████████████████████████████████████████████████████████████████████████████▏ | 38424/40903 [14:53:02<47:53, 1.16s/it][2025-04-21 07:49:38] (step=0038425) Train Loss: 6.4017, Train Steps/Sec: 0.86 + 94%|██████████████████████████████████████████████████████████████████████████████████████████████████████████▏ | 38449/40903 [14:53:31<47:31, 1.16s/it][2025-04-21 07:50:07] (step=0038450) Train Loss: 6.3158, Train Steps/Sec: 0.86 + 94%|██████████████████████████████████████████████████████████████████████████████████████████████████████████▎ | 38474/40903 [14:54:00<46:39, 1.15s/it][2025-04-21 07:50:36] (step=0038475) Train Loss: 6.3427, Train Steps/Sec: 0.87 + 94%|██████████████████████████████████████████████████████████████████████████████████████████████████████████▎ | 38499/40903 [14:54:29<45:53, 1.15s/it][2025-04-21 07:51:05] (step=0038500) Train Loss: 6.3746, Train Steps/Sec: 0.85 + 94%|██████████████████████████████████████████████████████████████████████████████████████████████████████████▍ | 38524/40903 [14:54:58<46:14, 1.17s/it][2025-04-21 07:51:34] (step=0038525) Train Loss: 6.3166, Train Steps/Sec: 0.87 + 94%|██████████████████████████████████████████████████████████████████████████████████████████████████████████▍ | 38549/40903 [14:55:27<45:36, 1.16s/it][2025-04-21 07:52:03] (step=0038550) Train Loss: 6.3380, Train Steps/Sec: 0.86 + 94%|██████████████████████████████████████████████████████████████████████████████████████████████████████████▌ | 38574/40903 [14:55:56<44:29, 1.15s/it][2025-04-21 07:52:32] (step=0038575) Train Loss: 6.3544, Train Steps/Sec: 0.87 + 94%|██████████████████████████████████████████████████████████████████████████████████████████████████████████▋ | 38599/40903 [14:56:25<43:44, 1.14s/it][2025-04-21 07:53:01] (step=0038600) Train Loss: 6.3234, Train Steps/Sec: 0.86 + 94%|██████████████████████████████████████████████████████████████████████████████████████████████████████████▋ | 38624/40903 [14:56:54<44:29, 1.17s/it][2025-04-21 07:53:30] (step=0038625) Train Loss: 6.3427, Train Steps/Sec: 0.87 + 94%|██████████████████████████████████████████████████████████████████████████████████████████████████████████▊ | 38649/40903 [14:57:23<44:25, 1.18s/it][2025-04-21 07:53:59] (step=0038650) Train Loss: 6.3417, Train Steps/Sec: 0.86 + 95%|██████████████████████████████████████████████████████████████████████████████████████████████████████████▊ | 38674/40903 [14:57:52<42:53, 1.15s/it][2025-04-21 07:54:28] (step=0038675) Train Loss: 6.3687, Train Steps/Sec: 0.86 + 95%|██████████████████████████████████████████████████████████████████████████████████████████████████████████▉ | 38699/40903 [14:58:28<42:08, 1.15s/it][2025-04-21 07:55:04] (step=0038700) Train Loss: 6.3568, Train Steps/Sec: 0.86 + 95%|██████████████████████████████████████████████████████████████████████████████████████████████████████████▉ | 38724/40903 [14:58:57<42:53, 1.18s/it][2025-04-21 07:55:33] (step=0038725) Train Loss: 6.3656, Train Steps/Sec: 0.86 + 95%|███████████████████████████████████████████████████████████████████████████████████████████████████████████ | 38749/40903 [14:59:26<41:21, 1.15s/it][2025-04-21 07:56:02] (step=0038750) Train Loss: 6.3732, Train Steps/Sec: 0.86 + 95%|███████████████████████████████████████████████████████████████████████████████████████████████████████████ | 38774/40903 [14:59:55<42:16, 1.19s/it][2025-04-21 07:56:31] (step=0038775) Train Loss: 6.3440, Train Steps/Sec: 0.86 + 95%|███████████████████████████████████████████████████████████████████████████████████████████████████████████▏ | 38799/40903 [15:00:24<40:17, 1.15s/it][2025-04-21 07:57:00] (step=0038800) Train Loss: 6.3895, Train Steps/Sec: 0.86 + 95%|███████████████████████████████████████████████████████████████████████████████████████████████████████████▎ | 38824/40903 [15:00:53<41:04, 1.19s/it][2025-04-21 07:57:29] (step=0038825) Train Loss: 6.3425, Train Steps/Sec: 0.86 + 95%|███████████████████████████████████████████████████████████████████████████████████████████████████████████▎ | 38849/40903 [15:01:22<39:34, 1.16s/it][2025-04-21 07:57:58] (step=0038850) Train Loss: 6.3325, Train Steps/Sec: 0.86 + 95%|███████████████████████████████████████████████████████████████████████████████████████████████████████████▍ | 38874/40903 [15:01:51<38:49, 1.15s/it][2025-04-21 07:58:27] (step=0038875) Train Loss: 6.3478, Train Steps/Sec: 0.87 + 95%|███████████████████████████████████████████████████████████████████████████████████████████████████████████▍ | 38899/40903 [15:02:20<38:44, 1.16s/it][2025-04-21 07:58:56] (step=0038900) Train Loss: 6.3157, Train Steps/Sec: 0.85 + 95%|███████████████████████████████████████████████████████████████████████████████████████████████████████████▌ | 38924/40903 [15:02:49<39:02, 1.18s/it][2025-04-21 07:59:25] (step=0038925) Train Loss: 6.3480, Train Steps/Sec: 0.87 + 95%|███████████████████████████████████████████████████████████████████████████████████████████████████████████▌ | 38949/40903 [15:03:18<37:54, 1.16s/it][2025-04-21 07:59:54] (step=0038950) Train Loss: 6.3238, Train Steps/Sec: 0.86 + 95%|███████████████████████████████████████████████████████████████████████████████████████████████████████████▋ | 38974/40903 [15:03:47<37:07, 1.15s/it][2025-04-21 08:00:23] (step=0038975) Train Loss: 6.3272, Train Steps/Sec: 0.86 + 95%|███████████████████████████████████████████████████████████████████████████████████████████████████████████▋ | 38999/40903 [15:04:16<36:11, 1.14s/it][2025-04-21 08:00:52] (step=0039000) Train Loss: 6.3403, Train Steps/Sec: 0.86 + 95%|███████████████████████████████████████████████████████████████████████████████████████████████████████████▊ | 39024/40903 [15:04:45<36:56, 1.18s/it][2025-04-21 08:01:21] (step=0039025) Train Loss: 6.3470, Train Steps/Sec: 0.86 + 95%|███████████████████████████████████████████████████████████████████████████████████████████████████████████▉ | 39049/40903 [15:05:14<35:38, 1.15s/it][2025-04-21 08:01:50] (step=0039050) Train Loss: 6.3534, Train Steps/Sec: 0.86 + 96%|███████████████████████████████████████████████████████████████████████████████████████████████████████████▉ | 39074/40903 [15:05:43<34:49, 1.14s/it][2025-04-21 08:02:18] (step=0039075) Train Loss: 6.2928, Train Steps/Sec: 0.87 + 96%|████████████████████████████████████████████████████████████████████████████████████████████████████████████ | 39099/40903 [15:06:12<34:20, 1.14s/it][2025-04-21 08:02:48] (step=0039100) Train Loss: 6.3658, Train Steps/Sec: 0.85 + 96%|████████████████████████████████████████████████████████████████████████████████████████████████████████████ | 39124/40903 [15:06:41<34:48, 1.17s/it][2025-04-21 08:03:17] (step=0039125) Train Loss: 6.3640, Train Steps/Sec: 0.86 + 96%|████████████████████████████████████████████████████████████████████████████████████████████████████████████▏ | 39149/40903 [15:07:10<34:01, 1.16s/it][2025-04-21 08:03:46] (step=0039150) Train Loss: 6.3221, Train Steps/Sec: 0.86 + 96%|████████████████████████████████████████████████████████████████████████████████████████████████████████████▏ | 39174/40903 [15:07:39<32:59, 1.14s/it][2025-04-21 08:04:15] (step=0039175) Train Loss: 6.3509, Train Steps/Sec: 0.87 + 96%|████████████████████████████████████████████████████████████████████████████████████████████████████████████▎ | 39199/40903 [15:08:14<41:40, 1.47s/it][2025-04-21 08:04:50] (step=0039200) Train Loss: 6.3545, Train Steps/Sec: 0.70 + 96%|████████████████████████████████████████████████████████████████████████████████████████████████████████████▎ | 39224/40903 [15:08:44<32:57, 1.18s/it][2025-04-21 08:05:19] (step=0039225) Train Loss: 6.3093, Train Steps/Sec: 0.86 + 96%|████████████████████████████████████████████████████████████████████████████████████████████████████████████▍ | 39249/40903 [15:09:13<32:02, 1.16s/it][2025-04-21 08:05:48] (step=0039250) Train Loss: 6.3438, Train Steps/Sec: 0.86 + 96%|████████████████████████████████████████████████████████████████████████████████████████████████████████████▍ | 39274/40903 [15:09:42<30:57, 1.14s/it][2025-04-21 08:06:17] (step=0039275) Train Loss: 6.3087, Train Steps/Sec: 0.86 + 96%|████████████████████████████████████████████████████████████████████████████████████████████████████████████▌ | 39299/40903 [15:10:10<30:49, 1.15s/it][2025-04-21 08:06:46] (step=0039300) Train Loss: 6.3728, Train Steps/Sec: 0.86 + 96%|████████████████████████████████████████████████████████████████████████████████████████████████████████████▋ | 39324/40903 [15:10:39<30:44, 1.17s/it][2025-04-21 08:07:15] (step=0039325) Train Loss: 6.3312, Train Steps/Sec: 0.87 + 96%|████████████████████████████████████████████████████████████████████████████████████████████████████████████▋ | 39349/40903 [15:11:09<29:42, 1.15s/it][2025-04-21 08:07:44] (step=0039350) Train Loss: 6.3773, Train Steps/Sec: 0.86 + 96%|████████████████████████████████████████████████████████████████████████████████████████████████████████████▊ | 39374/40903 [15:11:38<29:08, 1.14s/it][2025-04-21 08:08:13] (step=0039375) Train Loss: 6.3174, Train Steps/Sec: 0.86 + 96%|████████████████████████████████████████████████████████████████████████████████████████████████████████████▊ | 39399/40903 [15:12:06<28:46, 1.15s/it][2025-04-21 08:08:42] (step=0039400) Train Loss: 6.3439, Train Steps/Sec: 0.86 + 96%|████████████████████████████████████████████████████████████████████████████████████████████████████████████▉ | 39424/40903 [15:12:36<29:32, 1.20s/it][2025-04-21 08:09:11] (step=0039425) Train Loss: 6.3225, Train Steps/Sec: 0.86 + 96%|████████████████████████████████████████████████████████████████████████████████████████████████████████████▉ | 39449/40903 [15:13:05<28:21, 1.17s/it][2025-04-21 08:09:40] (step=0039450) Train Loss: 6.3096, Train Steps/Sec: 0.86 + 97%|█████████████████████████████████████████████████████████████████████████████████████████████████████████████ | 39474/40903 [15:13:34<27:30, 1.15s/it][2025-04-21 08:10:09] (step=0039475) Train Loss: 6.3392, Train Steps/Sec: 0.87 + 97%|█████████████████████████████████████████████████████████████████████████████████████████████████████████████ | 39499/40903 [15:14:02<26:19, 1.12s/it][2025-04-21 08:10:38] (step=0039500) Train Loss: 6.3228, Train Steps/Sec: 0.87 + 97%|█████████████████████████████████████████████████████████████████████████████████████████████████████████████▏ | 39524/40903 [15:14:31<26:51, 1.17s/it][2025-04-21 08:11:07] (step=0039525) Train Loss: 6.3177, Train Steps/Sec: 0.87 + 97%|█████████████████████████████████████████████████████████████████████████████████████████████████████████████▎ | 39549/40903 [15:15:00<26:14, 1.16s/it][2025-04-21 08:11:36] (step=0039550) Train Loss: 6.3412, Train Steps/Sec: 0.86 + 97%|█████████████████████████████████████████████████████████████████████████████████████████████████████████████▎ | 39574/40903 [15:15:29<25:39, 1.16s/it][2025-04-21 08:12:05] (step=0039575) Train Loss: 6.4107, Train Steps/Sec: 0.86 + 97%|█████████████████████████████████████████████████████████████████████████████████████████████████████████████▍ | 39599/40903 [15:16:10<27:26, 1.26s/it][2025-04-21 08:12:45] (step=0039600) Train Loss: 6.2908, Train Steps/Sec: 0.62 + 97%|█████████████████████████████████████████████████████████████████████████████████████████████████████████████▍ | 39624/40903 [15:16:39<25:08, 1.18s/it][2025-04-21 08:13:14] (step=0039625) Train Loss: 6.3588, Train Steps/Sec: 0.86 + 97%|█████████████████████████████████████████████████████████████████████████████████████████████████████████████▌ | 39649/40903 [15:17:08<24:20, 1.16s/it][2025-04-21 08:13:43] (step=0039650) Train Loss: 6.3531, Train Steps/Sec: 0.86 + 97%|█████████████████████████████████████████████████████████████████████████████████████████████████████████████▌ | 39674/40903 [15:17:37<23:38, 1.15s/it][2025-04-21 08:14:12] (step=0039675) Train Loss: 6.3719, Train Steps/Sec: 0.86 + 97%|█████████████████████████████████████████████████████████████████████████████████████████████████████████████▋ | 39699/40903 [15:18:06<22:59, 1.15s/it][2025-04-21 08:14:41] (step=0039700) Train Loss: 6.3606, Train Steps/Sec: 0.86 + 97%|█████████████████████████████████████████████████████████████████████████████████████████████████████████████▋ | 39724/40903 [15:18:35<23:09, 1.18s/it][2025-04-21 08:15:10] (step=0039725) Train Loss: 6.3331, Train Steps/Sec: 0.86 + 97%|█████████████████████████████████████████████████████████████████████████████████████████████████████████████▊ | 39749/40903 [15:19:04<22:35, 1.17s/it][2025-04-21 08:15:39] (step=0039750) Train Loss: 6.3333, Train Steps/Sec: 0.86 + 97%|█████████████████████████████████████████████████████████████████████████████████████████████████████████████▉ | 39774/40903 [15:19:33<21:46, 1.16s/it][2025-04-21 08:16:08] (step=0039775) Train Loss: 6.3528, Train Steps/Sec: 0.86 + 97%|█████████████████████████████████████████████████████████████████████████████████████████████████████████████▉ | 39799/40903 [15:20:08<21:09, 1.15s/it][2025-04-21 08:16:44] (step=0039800) Train Loss: 6.3159, Train Steps/Sec: 0.70 + 97%|██████████████████████████████████████████████████████████████████████████████████████████████████████████████ | 39824/40903 [15:20:38<21:17, 1.18s/it][2025-04-21 08:17:13] (step=0039825) Train Loss: 6.3681, Train Steps/Sec: 0.86 + 97%|██████████████████████████████████████████████████████████████████████████████████████████████████████████████ | 39849/40903 [15:21:07<20:34, 1.17s/it][2025-04-21 08:17:42] (step=0039850) Train Loss: 6.3231, Train Steps/Sec: 0.86 + 97%|██████████████████████████████████████████████████████████████████████████████████████████████████████████████▏ | 39874/40903 [15:21:35<19:59, 1.17s/it][2025-04-21 08:18:11] (step=0039875) Train Loss: 6.3493, Train Steps/Sec: 0.87 + 98%|██████████████████████████████████████████████████████████████████████████████████████████████████████████████▏ | 39899/40903 [15:22:04<19:19, 1.15s/it][2025-04-21 08:18:40] (step=0039900) Train Loss: 6.3536, Train Steps/Sec: 0.86 + 98%|██████████████████████████████████████████████████████████████████████████████████████████████████████████████▎ | 39924/40903 [15:22:33<19:11, 1.18s/it][2025-04-21 08:19:09] (step=0039925) Train Loss: 6.3566, Train Steps/Sec: 0.87 + 98%|██████████████████████████████████████████████████████████████████████████████████████████████████████████████▎ | 39949/40903 [15:23:02<18:25, 1.16s/it][2025-04-21 08:19:38] (step=0039950) Train Loss: 6.3813, Train Steps/Sec: 0.86 + 98%|██████████████████████████████████████████████████████████████████████████████████████████████████████████████▍ | 39974/40903 [15:23:31<17:43, 1.14s/it][2025-04-21 08:20:07] (step=0039975) Train Loss: 6.3212, Train Steps/Sec: 0.86 + 98%|██████████████████████████████████████████████████████████████████████████████████████████████████████████████▌ | 39999/40903 [15:24:00<17:23, 1.15s/it][2025-04-21 08:20:36] (step=0040000) Train Loss: 6.3735, Train Steps/Sec: 0.86 +[2025-04-21 08:20:36] vision_config is None. initializing the InstructBlipVisionConfig with default values. +[2025-04-21 08:20:36] qformer_config is None. Initializing the InstructBlipQFormerConfig with default values. +[2025-04-21 08:20:36] text_config is None. Initializing the text config with default values (`OPTConfig`). +model_name_or_path:/tmp/haozhezhao/model/instructblip-flan-t5-xl +image_place_holder: +tokenizer length after expend 32102 +tokenizer length before expend 32102 +100%|████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████| 3/3 [03:13<00:00, 64.35s/it] +[2025-04-21 08:25:44] Finish Eval in 40000 steps...█████████████████████████████████████████████████████████████████████| 3/3 [03:12<00:00, 63.62s/it] +processing left-padding... +processing left-padding... +processing left-padding... +[2025-04-21 08:26:06] Saved checkpoint to checkpoint/EmptyFix_InstructBlip_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_objects/002-GPT-XL/checkpoints/0040000.pt +[2025-04-21 08:26:09] Removed old checkpoint: checkpoint/EmptyFix_InstructBlip_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_objects/002-GPT-XL/checkpoints/0036000.pt + 98%|██████████████████████████████████████████████████████████████████████████████████████████████████████████████▌ | 40024/40903 [15:30:02<17:45, 1.21s/it][2025-04-21 08:26:38] (step=0040025) Train Loss: 6.3967, Train Steps/Sec: 0.07 + 98%|██████████████████████████████████████████████████████████████████████████████████████████████████████████████▋ | 40049/40903 [15:30:32<16:30, 1.16s/it][2025-04-21 08:27:07] (step=0040050) Train Loss: 6.3284, Train Steps/Sec: 0.86 + 98%|██████████████████████████████████████████████████████████████████████████████████████████████████████████████▋ | 40074/40903 [15:31:01<15:58, 1.16s/it][2025-04-21 08:27:36] (step=0040075) Train Loss: 6.3149, Train Steps/Sec: 0.86 + 98%|██████████████████████████████████████████████████████████████████████████████████████████████████████████████▊ | 40099/40903 [15:31:30<15:29, 1.16s/it][2025-04-21 08:28:06] (step=0040100) Train Loss: 6.3586, Train Steps/Sec: 0.85 + 98%|██████████████████████████████████████████████████████████████████████████████████████████████████████████████▊ | 40124/40903 [15:31:59<15:21, 1.18s/it][2025-04-21 08:28:35] (step=0040125) Train Loss: 6.3523, Train Steps/Sec: 0.86 + 98%|██████████████████████████████████████████████████████████████████████████████████████████████████████████████▉ | 40149/40903 [15:32:28<14:36, 1.16s/it][2025-04-21 08:29:04] (step=0040150) Train Loss: 6.3411, Train Steps/Sec: 0.86 + 98%|██████████████████████████████████████████████████████████████████████████████████████████████████████████████▉ | 40174/40903 [15:32:57<14:11, 1.17s/it][2025-04-21 08:29:33] (step=0040175) Train Loss: 6.3492, Train Steps/Sec: 0.87 + 98%|███████████████████████████████████████████████████████████████████████████████████████████████████████████████ | 40199/40903 [15:33:26<13:23, 1.14s/it][2025-04-21 08:30:02] (step=0040200) Train Loss: 6.3589, Train Steps/Sec: 0.86 + 98%|███████████████████████████████████████████████████████████████████████████████████████████████████████████████ | 40224/40903 [15:33:55<13:15, 1.17s/it][2025-04-21 08:30:31] (step=0040225) Train Loss: 6.3615, Train Steps/Sec: 0.86 + 98%|███████████████████████████████████████████████████████████████████████████████████████████████████████████████▏ | 40249/40903 [15:34:24<12:45, 1.17s/it][2025-04-21 08:31:00] (step=0040250) Train Loss: 6.3064, Train Steps/Sec: 0.86 + 98%|███████████████████████████████████████████████████████████████████████████████████████████████████████████████▎ | 40274/40903 [15:34:53<11:56, 1.14s/it][2025-04-21 08:31:28] (step=0040275) Train Loss: 6.3484, Train Steps/Sec: 0.87 + 99%|███████████████████████████████████████████████████████████████████████████████████████████████████████████████▎ | 40299/40903 [15:35:22<11:29, 1.14s/it][2025-04-21 08:31:58] (step=0040300) Train Loss: 6.3621, Train Steps/Sec: 0.86 + 99%|███████████████████████████████████████████████████████████████████████████████████████████████████████████████▍ | 40324/40903 [15:35:51<11:16, 1.17s/it][2025-04-21 08:32:27] (step=0040325) Train Loss: 6.3579, Train Steps/Sec: 0.86 + 99%|███████████████████████████████████████████████████████████████████████████████████████████████████████████████▍ | 40349/40903 [15:36:20<10:44, 1.16s/it][2025-04-21 08:32:56] (step=0040350) Train Loss: 6.3565, Train Steps/Sec: 0.86 + 99%|███████████████████████████████████████████████████████████████████████████████████████████████████████████████▌ | 40374/40903 [15:36:49<10:02, 1.14s/it][2025-04-21 08:33:24] (step=0040375) Train Loss: 6.3377, Train Steps/Sec: 0.87 + 99%|███████████████████████████████████████████████████████████████████████████████████████████████████████████████▌ | 40399/40903 [15:37:18<09:38, 1.15s/it][2025-04-21 08:33:53] (step=0040400) Train Loss: 6.3935, Train Steps/Sec: 0.86 + 99%|███████████████████████████████████████████████████████████████████████████████████████████████████████████████▋ | 40424/40903 [15:37:46<09:18, 1.16s/it][2025-04-21 08:34:22] (step=0040425) Train Loss: 6.3442, Train Steps/Sec: 0.87 + 99%|███████████████████████████████████████████████████████████████████████████████████████████████████████████████▋ | 40449/40903 [15:38:15<08:42, 1.15s/it][2025-04-21 08:34:51] (step=0040450) Train Loss: 6.3519, Train Steps/Sec: 0.87 + 99%|███████████████████████████████████████████████████████████████████████████████████████████████████████████████▊ | 40474/40903 [15:38:44<08:13, 1.15s/it][2025-04-21 08:35:20] (step=0040475) Train Loss: 6.3160, Train Steps/Sec: 0.86 + 99%|███████████████████████████████████████████████████████████████████████████████████████████████████████████████▉ | 40499/40903 [15:39:13<07:44, 1.15s/it][2025-04-21 08:35:49] (step=0040500) Train Loss: 6.3269, Train Steps/Sec: 0.86 + 99%|███████████████████████████████████████████████████████████████████████████████████████████████████████████████▉ | 40524/40903 [15:39:42<07:28, 1.18s/it][2025-04-21 08:36:18] (step=0040525) Train Loss: 6.3457, Train Steps/Sec: 0.86 + 99%|████████████████████████████████████████████████████████████████████████████████████████████████████████████████ | 40549/40903 [15:40:11<06:47, 1.15s/it][2025-04-21 08:36:47] (step=0040550) Train Loss: 6.3718, Train Steps/Sec: 0.87 + 99%|████████████████████████████████████████████████████████████████████████████████████████████████████████████████ | 40574/40903 [15:40:40<06:15, 1.14s/it][2025-04-21 08:37:16] (step=0040575) Train Loss: 6.3177, Train Steps/Sec: 0.86 + 99%|████████████████████████████████████████████████████████████████████████████████████████████████████████████████▏| 40599/40903 [15:41:09<05:49, 1.15s/it][2025-04-21 08:37:45] (step=0040600) Train Loss: 6.3849, Train Steps/Sec: 0.86 + 99%|████████████████████████████████████████████████████████████████████████████████████████████████████████████████▏| 40624/40903 [15:41:38<05:28, 1.18s/it][2025-04-21 08:38:13] (step=0040625) Train Loss: 6.3518, Train Steps/Sec: 0.87 + 99%|████████████████████████████████████████████████████████████████████████████████████████████████████████████████▎| 40649/40903 [15:42:07<04:54, 1.16s/it][2025-04-21 08:38:42] (step=0040650) Train Loss: 6.3771, Train Steps/Sec: 0.86 + 99%|████████████████████████████████████████████████████████████████████████████████████████████████████████████████▎| 40674/40903 [15:42:36<04:27, 1.17s/it][2025-04-21 08:39:11] (step=0040675) Train Loss: 6.3330, Train Steps/Sec: 0.87 +100%|████████████████████████████████████████████████████████████████████████████████████████████████████████████████▍| 40699/40903 [15:43:04<03:48, 1.12s/it][2025-04-21 08:39:40] (step=0040700) Train Loss: 6.3644, Train Steps/Sec: 0.87 +100%|████████████████████████████████████████████████████████████████████████████████████████████████████████████████▌| 40724/40903 [15:43:33<03:30, 1.17s/it][2025-04-21 08:40:09] (step=0040725) Train Loss: 6.3491, Train Steps/Sec: 0.87 +100%|████████████████████████████████████████████████████████████████████████████████████████████████████████████████▌| 40749/40903 [15:44:02<02:56, 1.14s/it][2025-04-21 08:40:38] (step=0040750) Train Loss: 6.3437, Train Steps/Sec: 0.87 +100%|████████████████████████████████████████████████████████████████████████████████████████████████████████████████▋| 40774/40903 [15:44:31<02:28, 1.15s/it][2025-04-21 08:41:07] (step=0040775) Train Loss: 6.3393, Train Steps/Sec: 0.86 +100%|████████████████████████████████████████████████████████████████████████████████████████████████████████████████▋| 40799/40903 [15:45:00<01:59, 1.15s/it][2025-04-21 08:41:36] (step=0040800) Train Loss: 6.3309, Train Steps/Sec: 0.85 +100%|████████████████████████████████████████████████████████████████████████████████████████████████████████████████▊| 40824/40903 [15:45:29<01:32, 1.17s/it][2025-04-21 08:42:05] (step=0040825) Train Loss: 6.3604, Train Steps/Sec: 0.87 +100%|████████████████████████████████████████████████████████████████████████████████████████████████████████████████▊| 40849/40903 [15:45:58<01:03, 1.17s/it][2025-04-21 08:42:34] (step=0040850) Train Loss: 6.3307, Train Steps/Sec: 0.86 +100%|████████████████████████████████████████████████████████████████████████████████████████████████████████████████▉| 40874/40903 [15:46:27<00:33, 1.16s/it][2025-04-21 08:43:03] (step=0040875) Train Loss: 6.3429, Train Steps/Sec: 0.87 +100%|████████████████████████████████████████████████████████████████████████████████████████████████████████████████▉| 40899/40903 [15:46:56<00:04, 1.16s/it][2025-04-21 08:43:32] (step=0040900) Train Loss: 6.3512, Train Steps/Sec: 0.86 +100%|█████████████████████████████████████████████████████████████████████████████████████████████████████████████████| 40903/40903 [15:47:02<00:00, 1.39s/it] +[2025-04-21 08:43:37] Beginning epoch 1... + 0%| | 21/40903 [00:27<13:32:47, 1.19s/it][2025-04-21 08:44:05] (step=0040925) Train Loss: 6.3596, Train Steps/Sec: 0.75 + 0%|▏ | 46/40903 [00:56<13:14:34, 1.17s/it][2025-04-21 08:44:34] (step=0040950) Train Loss: 6.3512, Train Steps/Sec: 0.86 + 0%|▏ | 71/40903 [01:25<13:09:46, 1.16s/it][2025-04-21 08:45:03] (step=0040975) Train Loss: 6.3151, Train Steps/Sec: 0.86 + 0%|▎ | 96/40903 [01:54<12:54:25, 1.14s/it][2025-04-21 08:45:32] (step=0041000) Train Loss: 6.2929, Train Steps/Sec: 0.86 + 0%|▎ | 121/40903 [02:23<13:24:32, 1.18s/it][2025-04-21 08:46:01] (step=0041025) Train Loss: 6.2896, Train Steps/Sec: 0.86 + 0%|▍ | 146/40903 [02:52<13:07:48, 1.16s/it][2025-04-21 08:46:30] (step=0041050) Train Loss: 6.3067, Train Steps/Sec: 0.86 + 0%|▍ | 171/40903 [03:21<13:05:35, 1.16s/it][2025-04-21 08:46:59] (step=0041075) Train Loss: 6.2805, Train Steps/Sec: 0.86 + 0%|▌ | 196/40903 [03:50<13:01:59, 1.15s/it][2025-04-21 08:47:28] (step=0041100) Train Loss: 6.3232, Train Steps/Sec: 0.86 + 1%|▌ | 221/40903 [04:19<13:11:27, 1.17s/it][2025-04-21 08:47:57] (step=0041125) Train Loss: 6.3318, Train Steps/Sec: 0.87 + 1%|▋ | 246/40903 [04:48<13:06:26, 1.16s/it][2025-04-21 08:48:26] (step=0041150) Train Loss: 6.3513, Train Steps/Sec: 0.86 + 1%|▊ | 271/40903 [05:17<12:58:12, 1.15s/it][2025-04-21 08:48:56] (step=0041175) Train Loss: 6.3623, Train Steps/Sec: 0.86 + 1%|▊ | 296/40903 [05:46<13:08:09, 1.16s/it][2025-04-21 08:49:25] (step=0041200) Train Loss: 6.3247, Train Steps/Sec: 0.86 + 1%|▉ | 321/40903 [06:15<13:12:45, 1.17s/it][2025-04-21 08:49:54] (step=0041225) Train Loss: 6.3178, Train Steps/Sec: 0.86 + 1%|▉ | 346/40903 [06:44<13:04:05, 1.16s/it][2025-04-21 08:50:23] (step=0041250) Train Loss: 6.2936, Train Steps/Sec: 0.86 + 1%|█ | 371/40903 [07:13<12:56:42, 1.15s/it][2025-04-21 08:50:52] (step=0041275) Train Loss: 6.3588, Train Steps/Sec: 0.86 + 1%|█ | 396/40903 [07:42<12:56:33, 1.15s/it][2025-04-21 08:51:21] (step=0041300) Train Loss: 6.3148, Train Steps/Sec: 0.86 + 1%|█▏ | 421/40903 [08:11<13:12:00, 1.17s/it][2025-04-21 08:51:50] (step=0041325) Train Loss: 6.2817, Train Steps/Sec: 0.86 + 1%|█▎ | 446/40903 [08:40<13:06:42, 1.17s/it][2025-04-21 08:52:19] (step=0041350) Train Loss: 6.3594, Train Steps/Sec: 0.86 + 1%|█▎ | 471/40903 [09:09<12:54:10, 1.15s/it][2025-04-21 08:52:48] (step=0041375) Train Loss: 6.3353, Train Steps/Sec: 0.87 + 1%|█▍ | 496/40903 [09:38<12:54:37, 1.15s/it][2025-04-21 08:53:17] (step=0041400) Train Loss: 6.3569, Train Steps/Sec: 0.86 + 1%|█▍ | 521/40903 [10:08<13:19:42, 1.19s/it][2025-04-21 08:53:46] (step=0041425) Train Loss: 6.3383, Train Steps/Sec: 0.86 + 1%|█▌ | 546/40903 [10:37<13:00:38, 1.16s/it][2025-04-21 08:54:15] (step=0041450) Train Loss: 6.3326, Train Steps/Sec: 0.86 + 1%|█▌ | 571/40903 [11:06<12:57:43, 1.16s/it][2025-04-21 08:54:44] (step=0041475) Train Loss: 6.3481, Train Steps/Sec: 0.86 + 1%|█▋ | 596/40903 [11:35<12:54:36, 1.15s/it][2025-04-21 08:55:13] (step=0041500) Train Loss: 6.3145, Train Steps/Sec: 0.86 + 2%|█▋ | 621/40903 [12:04<13:20:53, 1.19s/it][2025-04-21 08:55:42] (step=0041525) Train Loss: 6.3153, Train Steps/Sec: 0.86 + 2%|█▊ | 646/40903 [12:42<13:03:09, 1.17s/it][2025-04-21 08:56:20] (step=0041550) Train Loss: 6.3619, Train Steps/Sec: 0.65 + 2%|█▉ | 671/40903 [13:11<13:03:44, 1.17s/it][2025-04-21 08:56:49] (step=0041575) Train Loss: 6.3373, Train Steps/Sec: 0.86 + 2%|█▉ | 696/40903 [13:40<12:45:05, 1.14s/it][2025-04-21 08:57:19] (step=0041600) Train Loss: 6.3153, Train Steps/Sec: 0.85 + 2%|██ | 721/40903 [14:09<13:08:14, 1.18s/it][2025-04-21 08:57:48] (step=0041625) Train Loss: 6.2986, Train Steps/Sec: 0.86 + 2%|██ | 746/40903 [14:38<12:46:14, 1.14s/it][2025-04-21 08:58:17] (step=0041650) Train Loss: 6.3021, Train Steps/Sec: 0.87 + 2%|██▏ | 771/40903 [15:07<12:44:56, 1.14s/it][2025-04-21 08:58:45] (step=0041675) Train Loss: 6.3171, Train Steps/Sec: 0.87 + 2%|██▏ | 796/40903 [15:36<12:47:43, 1.15s/it][2025-04-21 08:59:15] (step=0041700) Train Loss: 6.3390, Train Steps/Sec: 0.86 + 2%|██▎ | 821/40903 [16:05<13:03:03, 1.17s/it][2025-04-21 08:59:44] (step=0041725) Train Loss: 6.3166, Train Steps/Sec: 0.86 + 2%|██▍ | 846/40903 [16:34<13:08:52, 1.18s/it][2025-04-21 09:00:13] (step=0041750) Train Loss: 6.2793, Train Steps/Sec: 0.86 + 2%|██▍ | 871/40903 [17:03<12:53:18, 1.16s/it][2025-04-21 09:00:42] (step=0041775) Train Loss: 6.3067, Train Steps/Sec: 0.86 + 2%|██▌ | 896/40903 [17:32<12:40:40, 1.14s/it][2025-04-21 09:01:11] (step=0041800) Train Loss: 6.2992, Train Steps/Sec: 0.86 + 2%|██▌ | 921/40903 [18:01<13:00:15, 1.17s/it][2025-04-21 09:01:40] (step=0041825) Train Loss: 6.3134, Train Steps/Sec: 0.86 + 2%|██▋ | 946/40903 [18:30<12:56:21, 1.17s/it][2025-04-21 09:02:09] (step=0041850) Train Loss: 6.3616, Train Steps/Sec: 0.86 + 2%|██▋ | 971/40903 [18:59<12:52:06, 1.16s/it][2025-04-21 09:02:38] (step=0041875) Train Loss: 6.3514, Train Steps/Sec: 0.86 + 2%|██▊ | 996/40903 [19:28<12:33:57, 1.13s/it][2025-04-21 09:03:07] (step=0041900) Train Loss: 6.3211, Train Steps/Sec: 0.86 + 2%|██▊ | 1021/40903 [19:57<12:54:22, 1.16s/it][2025-04-21 09:03:36] (step=0041925) Train Loss: 6.3218, Train Steps/Sec: 0.86 + 3%|██▉ | 1046/40903 [20:26<12:48:32, 1.16s/it][2025-04-21 09:04:05] (step=0041950) Train Loss: 6.3305, Train Steps/Sec: 0.86 + 3%|██▉ | 1071/40903 [20:56<12:50:55, 1.16s/it][2025-04-21 09:04:34] (step=0041975) Train Loss: 6.3133, Train Steps/Sec: 0.86 + 3%|███ | 1096/40903 [21:25<13:04:22, 1.18s/it][2025-04-21 09:05:04] (step=0042000) Train Loss: 6.3283, Train Steps/Sec: 0.85 +[2025-04-21 09:05:04] vision_config is None. initializing the InstructBlipVisionConfig with default values. +[2025-04-21 09:05:04] qformer_config is None. Initializing the InstructBlipQFormerConfig with default values. +[2025-04-21 09:05:04] text_config is None. Initializing the text config with default values (`OPTConfig`). +model_name_or_path:/tmp/haozhezhao/model/instructblip-flan-t5-xl +image_place_holder: +tokenizer length after expend 32102 +tokenizer length before expend 32102 +100%|████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████| 3/3 [03:11<00:00, 63.77s/it] +[2025-04-21 09:10:07] Finish Eval in 42000 steps...█████████████████████████████████████████████████████████████████████| 3/3 [03:10<00:00, 63.07s/it] +processing left-padding... +processing left-padding... +processing left-padding... +[2025-04-21 09:10:29] Saved checkpoint to checkpoint/EmptyFix_InstructBlip_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_objects/002-GPT-XL/checkpoints/0042000.pt +[2025-04-21 09:10:32] Removed old checkpoint: checkpoint/EmptyFix_InstructBlip_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_objects/002-GPT-XL/checkpoints/0038000.pt + 3%|███ | 1121/40903 [27:22<13:12:10, 1.19s/it][2025-04-21 09:11:01] (step=0042025) Train Loss: 6.2914, Train Steps/Sec: 0.07 + 3%|███▏ | 1146/40903 [27:51<12:56:33, 1.17s/it][2025-04-21 09:11:30] (step=0042050) Train Loss: 6.3593, Train Steps/Sec: 0.86 + 3%|███▎ | 1171/40903 [28:20<12:38:23, 1.15s/it][2025-04-21 09:11:58] (step=0042075) Train Loss: 6.3028, Train Steps/Sec: 0.87 + 3%|███▎ | 1196/40903 [28:49<12:52:14, 1.17s/it][2025-04-21 09:12:28] (step=0042100) Train Loss: 6.3152, Train Steps/Sec: 0.86 + 3%|███▍ | 1221/40903 [29:18<12:53:36, 1.17s/it][2025-04-21 09:12:56] (step=0042125) Train Loss: 6.3577, Train Steps/Sec: 0.87 + 3%|███▍ | 1246/40903 [29:47<12:53:37, 1.17s/it][2025-04-21 09:13:26] (step=0042150) Train Loss: 6.3234, Train Steps/Sec: 0.86 + 3%|███▌ | 1271/40903 [30:16<12:47:14, 1.16s/it][2025-04-21 09:13:55] (step=0042175) Train Loss: 6.3580, Train Steps/Sec: 0.86 + 3%|███▌ | 1296/40903 [30:55<17:45:31, 1.61s/it][2025-04-21 09:14:33] (step=0042200) Train Loss: 6.2841, Train Steps/Sec: 0.65 + 3%|███▋ | 1321/40903 [31:24<12:51:31, 1.17s/it][2025-04-21 09:15:02] (step=0042225) Train Loss: 6.3235, Train Steps/Sec: 0.87 + 3%|███▊ | 1346/40903 [31:53<13:03:03, 1.19s/it][2025-04-21 09:15:31] (step=0042250) Train Loss: 6.2882, Train Steps/Sec: 0.86 + 3%|███▊ | 1371/40903 [32:22<12:37:26, 1.15s/it][2025-04-21 09:16:00] (step=0042275) Train Loss: 6.3025, Train Steps/Sec: 0.86 + 3%|███▉ | 1396/40903 [32:51<12:42:59, 1.16s/it][2025-04-21 09:16:29] (step=0042300) Train Loss: 6.3415, Train Steps/Sec: 0.86 + 3%|███▉ | 1421/40903 [33:20<12:43:02, 1.16s/it][2025-04-21 09:16:58] (step=0042325) Train Loss: 6.3108, Train Steps/Sec: 0.87 + 4%|████ | 1446/40903 [33:49<12:51:02, 1.17s/it][2025-04-21 09:17:27] (step=0042350) Train Loss: 6.2797, Train Steps/Sec: 0.86 + 4%|████ | 1471/40903 [34:18<12:52:44, 1.18s/it][2025-04-21 09:17:56] (step=0042375) Train Loss: 6.2853, Train Steps/Sec: 0.86 + 4%|████▏ | 1496/40903 [34:47<12:38:36, 1.16s/it][2025-04-21 09:18:26] (step=0042400) Train Loss: 6.3110, Train Steps/Sec: 0.86 + 4%|████▏ | 1521/40903 [35:16<12:49:42, 1.17s/it][2025-04-21 09:18:55] (step=0042425) Train Loss: 6.3531, Train Steps/Sec: 0.86 + 4%|████▎ | 1546/40903 [35:45<12:34:41, 1.15s/it][2025-04-21 09:19:24] (step=0042450) Train Loss: 6.3467, Train Steps/Sec: 0.86 + 4%|████▍ | 1571/40903 [36:14<12:44:21, 1.17s/it][2025-04-21 09:19:53] (step=0042475) Train Loss: 6.3481, Train Steps/Sec: 0.86 + 4%|████▍ | 1596/40903 [36:43<12:31:32, 1.15s/it][2025-04-21 09:20:22] (step=0042500) Train Loss: 6.3150, Train Steps/Sec: 0.86 + 4%|████▌ | 1621/40903 [37:13<12:57:53, 1.19s/it][2025-04-21 09:20:51] (step=0042525) Train Loss: 6.2922, Train Steps/Sec: 0.86 + 4%|████▌ | 1646/40903 [37:42<12:52:22, 1.18s/it][2025-04-21 09:21:20] (step=0042550) Train Loss: 6.3322, Train Steps/Sec: 0.86 + 4%|████▋ | 1671/40903 [38:11<12:36:34, 1.16s/it][2025-04-21 09:21:49] (step=0042575) Train Loss: 6.3406, Train Steps/Sec: 0.86 + 4%|████▋ | 1696/40903 [38:40<12:27:03, 1.14s/it][2025-04-21 09:22:19] (step=0042600) Train Loss: 6.3244, Train Steps/Sec: 0.85 + 4%|████▊ | 1721/40903 [39:09<12:52:13, 1.18s/it][2025-04-21 09:22:48] (step=0042625) Train Loss: 6.2994, Train Steps/Sec: 0.86 + 4%|████▊ | 1746/40903 [39:38<12:43:28, 1.17s/it][2025-04-21 09:23:17] (step=0042650) Train Loss: 6.2766, Train Steps/Sec: 0.87 + 4%|████▉ | 1771/40903 [40:07<12:34:10, 1.16s/it][2025-04-21 09:23:46] (step=0042675) Train Loss: 6.3568, Train Steps/Sec: 0.86 + 4%|█████ | 1796/40903 [40:36<12:35:23, 1.16s/it][2025-04-21 09:24:15] (step=0042700) Train Loss: 6.3285, Train Steps/Sec: 0.86 + 4%|█████ | 1821/40903 [41:05<12:46:22, 1.18s/it][2025-04-21 09:24:44] (step=0042725) Train Loss: 6.3018, Train Steps/Sec: 0.86 + 5%|█████▏ | 1846/40903 [41:34<12:40:42, 1.17s/it][2025-04-21 09:25:13] (step=0042750) Train Loss: 6.3434, Train Steps/Sec: 0.86 + 5%|█████▏ | 1871/40903 [42:03<12:31:25, 1.16s/it][2025-04-21 09:25:42] (step=0042775) Train Loss: 6.3013, Train Steps/Sec: 0.86 + 5%|█████▎ | 1896/40903 [42:32<12:21:38, 1.14s/it][2025-04-21 09:26:11] (step=0042800) Train Loss: 6.2784, Train Steps/Sec: 0.86 + 5%|█████▎ | 1921/40903 [43:02<12:47:44, 1.18s/it][2025-04-21 09:26:40] (step=0042825) Train Loss: 6.3155, Train Steps/Sec: 0.86 + 5%|█████▍ | 1946/40903 [43:39<13:43:52, 1.27s/it][2025-04-21 09:27:18] (step=0042850) Train Loss: 6.2855, Train Steps/Sec: 0.67 + 5%|█████▍ | 1971/40903 [44:08<12:23:15, 1.15s/it][2025-04-21 09:27:47] (step=0042875) Train Loss: 6.3155, Train Steps/Sec: 0.86 + 5%|█████▌ | 1996/40903 [44:37<12:22:37, 1.15s/it][2025-04-21 09:28:16] (step=0042900) Train Loss: 6.3452, Train Steps/Sec: 0.86 + 5%|█████▋ | 2021/40903 [45:06<12:43:37, 1.18s/it][2025-04-21 09:28:45] (step=0042925) Train Loss: 6.3092, Train Steps/Sec: 0.86 + 5%|█████▋ | 2046/40903 [45:35<12:28:31, 1.16s/it][2025-04-21 09:29:14] (step=0042950) Train Loss: 6.2984, Train Steps/Sec: 0.86 + 5%|█████▊ | 2071/40903 [46:04<12:26:32, 1.15s/it][2025-04-21 09:29:43] (step=0042975) Train Loss: 6.2929, Train Steps/Sec: 0.86 + 5%|█████▊ | 2096/40903 [46:33<12:11:50, 1.13s/it][2025-04-21 09:30:12] (step=0043000) Train Loss: 6.3106, Train Steps/Sec: 0.87 + 5%|█████▉ | 2121/40903 [47:02<12:47:32, 1.19s/it][2025-04-21 09:30:41] (step=0043025) Train Loss: 6.2829, Train Steps/Sec: 0.86 + 5%|█████▉ | 2146/40903 [47:31<12:36:23, 1.17s/it][2025-04-21 09:31:10] (step=0043050) Train Loss: 6.3297, Train Steps/Sec: 0.86 + 5%|██████ | 2171/40903 [48:01<12:34:06, 1.17s/it][2025-04-21 09:31:39] (step=0043075) Train Loss: 6.3457, Train Steps/Sec: 0.85 + 5%|██████ | 2196/40903 [48:30<12:16:14, 1.14s/it][2025-04-21 09:32:08] (step=0043100) Train Loss: 6.3224, Train Steps/Sec: 0.86 + 5%|██████▏ | 2221/40903 [48:59<12:31:25, 1.17s/it][2025-04-21 09:32:37] (step=0043125) Train Loss: 6.3026, Train Steps/Sec: 0.86 + 5%|██████▎ | 2246/40903 [49:28<12:29:40, 1.16s/it][2025-04-21 09:33:06] (step=0043150) Train Loss: 6.3288, Train Steps/Sec: 0.86 + 6%|██████▎ | 2271/40903 [49:57<12:29:21, 1.16s/it][2025-04-21 09:33:35] (step=0043175) Train Loss: 6.3085, Train Steps/Sec: 0.86 + 6%|██████▍ | 2296/40903 [50:26<12:06:34, 1.13s/it][2025-04-21 09:34:04] (step=0043200) Train Loss: 6.3326, Train Steps/Sec: 0.86 + 6%|██████▍ | 2321/40903 [50:55<12:31:04, 1.17s/it][2025-04-21 09:34:34] (step=0043225) Train Loss: 6.3197, Train Steps/Sec: 0.86 + 6%|██████▌ | 2346/40903 [51:24<12:30:12, 1.17s/it][2025-04-21 09:35:03] (step=0043250) Train Loss: 6.3063, Train Steps/Sec: 0.86 + 6%|██████▌ | 2371/40903 [51:53<12:18:37, 1.15s/it][2025-04-21 09:35:32] (step=0043275) Train Loss: 6.3041, Train Steps/Sec: 0.87 + 6%|██████▋ | 2396/40903 [52:22<12:16:37, 1.15s/it][2025-04-21 09:36:01] (step=0043300) Train Loss: 6.3308, Train Steps/Sec: 0.86 + 6%|██████▋ | 2421/40903 [52:51<12:31:44, 1.17s/it][2025-04-21 09:36:30] (step=0043325) Train Loss: 6.3103, Train Steps/Sec: 0.87 + 6%|██████▊ | 2446/40903 [53:20<12:19:35, 1.15s/it][2025-04-21 09:36:59] (step=0043350) Train Loss: 6.3368, Train Steps/Sec: 0.86 + 6%|██████▉ | 2471/40903 [53:49<12:12:50, 1.14s/it][2025-04-21 09:37:28] (step=0043375) Train Loss: 6.3489, Train Steps/Sec: 0.86 + 6%|██████▉ | 2496/40903 [54:18<12:11:03, 1.14s/it][2025-04-21 09:37:57] (step=0043400) Train Loss: 6.2886, Train Steps/Sec: 0.86 + 6%|███████ | 2521/40903 [54:47<12:31:11, 1.17s/it][2025-04-21 09:38:26] (step=0043425) Train Loss: 6.3115, Train Steps/Sec: 0.86 + 6%|███████ | 2546/40903 [55:16<12:27:02, 1.17s/it][2025-04-21 09:38:55] (step=0043450) Train Loss: 6.3122, Train Steps/Sec: 0.86 + 6%|███████▏ | 2571/40903 [55:45<12:14:08, 1.15s/it][2025-04-21 09:39:24] (step=0043475) Train Loss: 6.3447, Train Steps/Sec: 0.86 + 6%|███████▏ | 2596/40903 [56:15<12:11:38, 1.15s/it][2025-04-21 09:39:53] (step=0043500) Train Loss: 6.3194, Train Steps/Sec: 0.85 + 6%|███████▎ | 2621/40903 [56:44<12:33:22, 1.18s/it][2025-04-21 09:40:22] (step=0043525) Train Loss: 6.3513, Train Steps/Sec: 0.86 + 6%|███████▎ | 2646/40903 [57:13<12:25:54, 1.17s/it][2025-04-21 09:40:51] (step=0043550) Train Loss: 6.3005, Train Steps/Sec: 0.86 + 7%|███████▍ | 2671/40903 [57:42<12:06:43, 1.14s/it][2025-04-21 09:41:20] (step=0043575) Train Loss: 6.3220, Train Steps/Sec: 0.87 + 7%|███████▌ | 2696/40903 [58:11<12:13:44, 1.15s/it][2025-04-21 09:41:49] (step=0043600) Train Loss: 6.3659, Train Steps/Sec: 0.86 + 7%|███████▌ | 2721/40903 [58:40<12:23:42, 1.17s/it][2025-04-21 09:42:18] (step=0043625) Train Loss: 6.3240, Train Steps/Sec: 0.86 + 7%|███████▋ | 2746/40903 [59:09<12:11:34, 1.15s/it][2025-04-21 09:42:47] (step=0043650) Train Loss: 6.2759, Train Steps/Sec: 0.86 + 7%|███████▋ | 2771/40903 [59:38<12:17:38, 1.16s/it][2025-04-21 09:43:16] (step=0043675) Train Loss: 6.3215, Train Steps/Sec: 0.86 + 7%|███████▋ | 2796/40903 [1:00:07<12:04:15, 1.14s/it][2025-04-21 09:43:45] (step=0043700) Train Loss: 6.3394, Train Steps/Sec: 0.86 + 7%|███████▋ | 2821/40903 [1:00:36<12:26:22, 1.18s/it][2025-04-21 09:44:14] (step=0043725) Train Loss: 6.3077, Train Steps/Sec: 0.86 + 7%|███████▊ | 2846/40903 [1:01:05<12:19:45, 1.17s/it][2025-04-21 09:44:44] (step=0043750) Train Loss: 6.3277, Train Steps/Sec: 0.86 + 7%|███████▊ | 2871/40903 [1:01:34<12:09:56, 1.15s/it][2025-04-21 09:45:12] (step=0043775) Train Loss: 6.3043, Train Steps/Sec: 0.86 + 7%|███████▉ | 2896/40903 [1:02:03<12:05:49, 1.15s/it][2025-04-21 09:45:41] (step=0043800) Train Loss: 6.2903, Train Steps/Sec: 0.86 + 7%|███████▉ | 2921/40903 [1:02:32<12:23:21, 1.17s/it][2025-04-21 09:46:11] (step=0043825) Train Loss: 6.2843, Train Steps/Sec: 0.86 + 7%|████████ | 2946/40903 [1:03:01<12:19:57, 1.17s/it][2025-04-21 09:46:40] (step=0043850) Train Loss: 6.3233, Train Steps/Sec: 0.86 + 7%|████████▏ | 2971/40903 [1:03:30<12:21:22, 1.17s/it][2025-04-21 09:47:09] (step=0043875) Train Loss: 6.3341, Train Steps/Sec: 0.86 + 7%|████████▏ | 2996/40903 [1:03:59<12:07:52, 1.15s/it][2025-04-21 09:47:38] (step=0043900) Train Loss: 6.3591, Train Steps/Sec: 0.86 + 7%|████████▎ | 3021/40903 [1:04:29<12:36:19, 1.20s/it][2025-04-21 09:48:07] (step=0043925) Train Loss: 6.3205, Train Steps/Sec: 0.86 + 7%|████████▎ | 3046/40903 [1:04:58<12:13:05, 1.16s/it][2025-04-21 09:48:36] (step=0043950) Train Loss: 6.3255, Train Steps/Sec: 0.86 + 8%|████████▍ | 3071/40903 [1:05:27<12:02:15, 1.15s/it][2025-04-21 09:49:05] (step=0043975) Train Loss: 6.2868, Train Steps/Sec: 0.86 + 8%|████████▍ | 3096/40903 [1:05:55<11:58:47, 1.14s/it][2025-04-21 09:49:34] (step=0044000) Train Loss: 6.3435, Train Steps/Sec: 0.86 +[2025-04-21 09:49:34] vision_config is None. initializing the InstructBlipVisionConfig with default values. +[2025-04-21 09:49:34] qformer_config is None. Initializing the InstructBlipQFormerConfig with default values. +[2025-04-21 09:49:34] text_config is None. Initializing the text config with default values (`OPTConfig`). +model_name_or_path:/tmp/haozhezhao/model/instructblip-flan-t5-xl +image_place_holder: +tokenizer length after expend 32102 +tokenizer length before expend 32102 +100%|████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████| 3/3 [03:09<00:00, 63.30s/it] +[2025-04-21 09:54:33] Finish Eval in 44000 steps...█████████████████████████████████████████████████████████████████████| 3/3 [03:08<00:00, 62.71s/it] +processing left-padding... +processing left-padding... +processing left-padding... +[2025-04-21 09:54:54] Saved checkpoint to checkpoint/EmptyFix_InstructBlip_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_objects/002-GPT-XL/checkpoints/0044000.pt +[2025-04-21 09:54:57] Removed old checkpoint: checkpoint/EmptyFix_InstructBlip_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_objects/002-GPT-XL/checkpoints/0040000.pt + 8%|████████▌ | 3121/40903 [1:11:48<12:30:18, 1.19s/it][2025-04-21 09:55:26] (step=0044025) Train Loss: 6.3354, Train Steps/Sec: 0.07 + 8%|████████▌ | 3146/40903 [1:12:17<12:14:41, 1.17s/it][2025-04-21 09:55:55] (step=0044050) Train Loss: 6.3011, Train Steps/Sec: 0.86 + 8%|████████▋ | 3171/40903 [1:12:46<12:11:00, 1.16s/it][2025-04-21 09:56:24] (step=0044075) Train Loss: 6.3236, Train Steps/Sec: 0.86 + 8%|████████▊ | 3196/40903 [1:13:15<12:04:45, 1.15s/it][2025-04-21 09:56:53] (step=0044100) Train Loss: 6.3273, Train Steps/Sec: 0.86 + 8%|████████▊ | 3221/40903 [1:13:44<12:15:58, 1.17s/it][2025-04-21 09:57:22] (step=0044125) Train Loss: 6.3029, Train Steps/Sec: 0.86 + 8%|████████▉ | 3246/40903 [1:14:13<12:06:36, 1.16s/it][2025-04-21 09:57:51] (step=0044150) Train Loss: 6.3283, Train Steps/Sec: 0.86 + 8%|████████▉ | 3271/40903 [1:14:42<11:51:34, 1.13s/it][2025-04-21 09:58:20] (step=0044175) Train Loss: 6.3083, Train Steps/Sec: 0.87 + 8%|█████████ | 3296/40903 [1:15:11<12:05:36, 1.16s/it][2025-04-21 09:58:49] (step=0044200) Train Loss: 6.3270, Train Steps/Sec: 0.86 + 8%|█████████ | 3321/40903 [1:15:40<12:05:46, 1.16s/it][2025-04-21 09:59:18] (step=0044225) Train Loss: 6.2861, Train Steps/Sec: 0.87 + 8%|█████████▏ | 3346/40903 [1:16:17<15:04:25, 1.44s/it][2025-04-21 09:59:55] (step=0044250) Train Loss: 6.3370, Train Steps/Sec: 0.67 + 8%|█████████▏ | 3371/40903 [1:16:46<12:17:53, 1.18s/it][2025-04-21 10:00:25] (step=0044275) Train Loss: 6.3234, Train Steps/Sec: 0.86 + 8%|█████████▎ | 3396/40903 [1:17:15<11:55:56, 1.15s/it][2025-04-21 10:00:54] (step=0044300) Train Loss: 6.3125, Train Steps/Sec: 0.86 + 8%|█████████▎ | 3421/40903 [1:17:44<12:11:23, 1.17s/it][2025-04-21 10:01:23] (step=0044325) Train Loss: 6.3417, Train Steps/Sec: 0.86 + 8%|█████████▍ | 3446/40903 [1:18:13<12:00:26, 1.15s/it][2025-04-21 10:01:52] (step=0044350) Train Loss: 6.2993, Train Steps/Sec: 0.87 + 8%|█████████▌ | 3471/40903 [1:18:42<11:58:20, 1.15s/it][2025-04-21 10:02:21] (step=0044375) Train Loss: 6.3324, Train Steps/Sec: 0.87 + 9%|█████████▌ | 3496/40903 [1:19:11<12:01:33, 1.16s/it][2025-04-21 10:02:50] (step=0044400) Train Loss: 6.3464, Train Steps/Sec: 0.86 + 9%|█████████▋ | 3521/40903 [1:19:40<12:25:53, 1.20s/it][2025-04-21 10:03:19] (step=0044425) Train Loss: 6.3215, Train Steps/Sec: 0.86 + 9%|█████████▋ | 3546/40903 [1:20:09<11:59:02, 1.15s/it][2025-04-21 10:03:48] (step=0044450) Train Loss: 6.3056, Train Steps/Sec: 0.86 + 9%|█████████▊ | 3571/40903 [1:20:38<11:42:30, 1.13s/it][2025-04-21 10:04:16] (step=0044475) Train Loss: 6.3270, Train Steps/Sec: 0.87 + 9%|█████████▊ | 3596/40903 [1:21:07<11:48:43, 1.14s/it][2025-04-21 10:04:46] (step=0044500) Train Loss: 6.3626, Train Steps/Sec: 0.86 + 9%|█████████▉ | 3621/40903 [1:21:36<12:15:31, 1.18s/it][2025-04-21 10:05:15] (step=0044525) Train Loss: 6.3275, Train Steps/Sec: 0.86 + 9%|█████████▉ | 3646/40903 [1:22:05<11:53:39, 1.15s/it][2025-04-21 10:05:44] (step=0044550) Train Loss: 6.3680, Train Steps/Sec: 0.86 + 9%|██████████ | 3671/40903 [1:22:34<12:02:50, 1.16s/it][2025-04-21 10:06:13] (step=0044575) Train Loss: 6.3433, Train Steps/Sec: 0.86 + 9%|██████████ | 3696/40903 [1:23:03<11:51:27, 1.15s/it][2025-04-21 10:06:42] (step=0044600) Train Loss: 6.3451, Train Steps/Sec: 0.86 + 9%|██████████▏ | 3721/40903 [1:23:32<12:06:47, 1.17s/it][2025-04-21 10:07:11] (step=0044625) Train Loss: 6.3268, Train Steps/Sec: 0.86 + 9%|██████████▎ | 3746/40903 [1:24:01<11:54:41, 1.15s/it][2025-04-21 10:07:40] (step=0044650) Train Loss: 6.2971, Train Steps/Sec: 0.86 + 9%|██████████▎ | 3771/40903 [1:24:30<11:55:52, 1.16s/it][2025-04-21 10:08:09] (step=0044675) Train Loss: 6.3329, Train Steps/Sec: 0.86 + 9%|██████████▍ | 3796/40903 [1:24:59<11:50:11, 1.15s/it][2025-04-21 10:08:38] (step=0044700) Train Loss: 6.3139, Train Steps/Sec: 0.86 + 9%|██████████▍ | 3821/40903 [1:25:36<13:44:36, 1.33s/it][2025-04-21 10:09:15] (step=0044725) Train Loss: 6.3074, Train Steps/Sec: 0.67 + 9%|██████████▌ | 3846/40903 [1:26:13<12:21:55, 1.20s/it][2025-04-21 10:09:52] (step=0044750) Train Loss: 6.3248, Train Steps/Sec: 0.68 + 9%|██████████▌ | 3871/40903 [1:26:42<11:44:32, 1.14s/it][2025-04-21 10:10:21] (step=0044775) Train Loss: 6.3198, Train Steps/Sec: 0.86 + 10%|██████████▋ | 3896/40903 [1:27:12<11:54:06, 1.16s/it][2025-04-21 10:10:50] (step=0044800) Train Loss: 6.3100, Train Steps/Sec: 0.86 + 10%|██████████▋ | 3921/40903 [1:27:41<12:04:56, 1.18s/it][2025-04-21 10:11:19] (step=0044825) Train Loss: 6.3170, Train Steps/Sec: 0.86 + 10%|██████████▊ | 3946/40903 [1:28:10<11:53:00, 1.16s/it][2025-04-21 10:11:48] (step=0044850) Train Loss: 6.3096, Train Steps/Sec: 0.86 + 10%|██████████▊ | 3971/40903 [1:28:39<11:49:10, 1.15s/it][2025-04-21 10:12:17] (step=0044875) Train Loss: 6.2746, Train Steps/Sec: 0.86 + 10%|██████████▉ | 3996/40903 [1:29:08<11:57:17, 1.17s/it][2025-04-21 10:12:46] (step=0044900) Train Loss: 6.3082, Train Steps/Sec: 0.86 + 10%|███████████ | 4021/40903 [1:29:36<11:50:52, 1.16s/it][2025-04-21 10:13:15] (step=0044925) Train Loss: 6.3104, Train Steps/Sec: 0.87 + 10%|███████████ | 4046/40903 [1:30:15<12:04:18, 1.18s/it][2025-04-21 10:13:53] (step=0044950) Train Loss: 6.3234, Train Steps/Sec: 0.66 + 10%|███████████▏ | 4071/40903 [1:30:44<11:38:12, 1.14s/it][2025-04-21 10:14:22] (step=0044975) Train Loss: 6.3291, Train Steps/Sec: 0.86 + 10%|███████████▏ | 4096/40903 [1:31:13<11:35:28, 1.13s/it][2025-04-21 10:14:51] (step=0045000) Train Loss: 6.3519, Train Steps/Sec: 0.86 + 10%|███████████▎ | 4121/40903 [1:31:42<12:04:09, 1.18s/it][2025-04-21 10:15:20] (step=0045025) Train Loss: 6.3733, Train Steps/Sec: 0.86 + 10%|███████████▎ | 4146/40903 [1:32:11<11:52:16, 1.16s/it][2025-04-21 10:15:49] (step=0045050) Train Loss: 6.2938, Train Steps/Sec: 0.86 + 10%|███████████▍ | 4171/40903 [1:32:40<11:58:45, 1.17s/it][2025-04-21 10:16:18] (step=0045075) Train Loss: 6.3451, Train Steps/Sec: 0.86 + 10%|███████████▍ | 4196/40903 [1:33:09<11:42:02, 1.15s/it][2025-04-21 10:16:47] (step=0045100) Train Loss: 6.3121, Train Steps/Sec: 0.86 + 10%|███████████▌ | 4221/40903 [1:33:38<12:01:20, 1.18s/it][2025-04-21 10:17:16] (step=0045125) Train Loss: 6.3278, Train Steps/Sec: 0.86 + 10%|███████████▋ | 4246/40903 [1:34:07<12:00:38, 1.18s/it][2025-04-21 10:17:45] (step=0045150) Train Loss: 6.3351, Train Steps/Sec: 0.86 + 10%|███████████▋ | 4271/40903 [1:34:36<11:44:37, 1.15s/it][2025-04-21 10:18:14] (step=0045175) Train Loss: 6.3167, Train Steps/Sec: 0.86 + 11%|███████████▊ | 4296/40903 [1:35:05<11:52:18, 1.17s/it][2025-04-21 10:18:44] (step=0045200) Train Loss: 6.3075, Train Steps/Sec: 0.85 + 11%|███████████▊ | 4321/40903 [1:35:34<11:58:29, 1.18s/it][2025-04-21 10:19:13] (step=0045225) Train Loss: 6.3395, Train Steps/Sec: 0.87 + 11%|███████████▉ | 4346/40903 [1:36:03<11:41:48, 1.15s/it][2025-04-21 10:19:42] (step=0045250) Train Loss: 6.3293, Train Steps/Sec: 0.86 + 11%|███████████▉ | 4371/40903 [1:36:32<11:42:39, 1.15s/it][2025-04-21 10:20:11] (step=0045275) Train Loss: 6.2912, Train Steps/Sec: 0.86 + 11%|████████████ | 4396/40903 [1:37:01<11:37:58, 1.15s/it][2025-04-21 10:20:40] (step=0045300) Train Loss: 6.3216, Train Steps/Sec: 0.86 + 11%|████████████ | 4421/40903 [1:37:31<12:05:33, 1.19s/it][2025-04-21 10:21:09] (step=0045325) Train Loss: 6.3026, Train Steps/Sec: 0.86 + 11%|████████████▏ | 4446/40903 [1:38:00<11:56:39, 1.18s/it][2025-04-21 10:21:38] (step=0045350) Train Loss: 6.3473, Train Steps/Sec: 0.86 + 11%|████████████▏ | 4471/40903 [1:38:29<11:43:48, 1.16s/it][2025-04-21 10:22:07] (step=0045375) Train Loss: 6.3236, Train Steps/Sec: 0.86 + 11%|████████████▎ | 4496/40903 [1:38:58<11:39:37, 1.15s/it][2025-04-21 10:22:36] (step=0045400) Train Loss: 6.2711, Train Steps/Sec: 0.86 + 11%|████████████▍ | 4521/40903 [1:39:26<11:44:42, 1.16s/it][2025-04-21 10:23:05] (step=0045425) Train Loss: 6.3082, Train Steps/Sec: 0.87 + 11%|████████████▍ | 4546/40903 [1:39:55<11:42:35, 1.16s/it][2025-04-21 10:23:34] (step=0045450) Train Loss: 6.3221, Train Steps/Sec: 0.86 + 11%|████████████▌ | 4571/40903 [1:40:25<11:46:19, 1.17s/it][2025-04-21 10:24:03] (step=0045475) Train Loss: 6.3410, Train Steps/Sec: 0.86 + 11%|████████████▌ | 4596/40903 [1:40:54<11:32:10, 1.14s/it][2025-04-21 10:24:32] (step=0045500) Train Loss: 6.3400, Train Steps/Sec: 0.86 + 11%|████████████▋ | 4621/40903 [1:41:23<11:53:10, 1.18s/it][2025-04-21 10:25:01] (step=0045525) Train Loss: 6.3109, Train Steps/Sec: 0.86 + 11%|████████████▋ | 4646/40903 [1:41:52<11:46:58, 1.17s/it][2025-04-21 10:25:30] (step=0045550) Train Loss: 6.3051, Train Steps/Sec: 0.86 + 11%|████████████▊ | 4671/40903 [1:42:21<11:42:06, 1.16s/it][2025-04-21 10:25:59] (step=0045575) Train Loss: 6.3305, Train Steps/Sec: 0.86 + 11%|████████████▊ | 4696/40903 [1:42:50<11:32:06, 1.15s/it][2025-04-21 10:26:29] (step=0045600) Train Loss: 6.3241, Train Steps/Sec: 0.85 + 12%|████████████▉ | 4721/40903 [1:43:19<11:54:07, 1.18s/it][2025-04-21 10:26:57] (step=0045625) Train Loss: 6.3009, Train Steps/Sec: 0.87 + 12%|████████████▉ | 4746/40903 [1:43:48<11:45:17, 1.17s/it][2025-04-21 10:27:26] (step=0045650) Train Loss: 6.2843, Train Steps/Sec: 0.86 + 12%|█████████████ | 4771/40903 [1:44:17<11:28:52, 1.14s/it][2025-04-21 10:27:56] (step=0045675) Train Loss: 6.3439, Train Steps/Sec: 0.86 + 12%|█████████████▏ | 4796/40903 [1:44:46<11:25:48, 1.14s/it][2025-04-21 10:28:25] (step=0045700) Train Loss: 6.3526, Train Steps/Sec: 0.86 + 12%|█████████████▏ | 4821/40903 [1:45:15<11:47:36, 1.18s/it][2025-04-21 10:28:53] (step=0045725) Train Loss: 6.3496, Train Steps/Sec: 0.87 + 12%|█████████████▎ | 4846/40903 [1:45:44<11:37:17, 1.16s/it][2025-04-21 10:29:22] (step=0045750) Train Loss: 6.3461, Train Steps/Sec: 0.87 + 12%|█████████████▎ | 4871/40903 [1:46:13<11:37:18, 1.16s/it][2025-04-21 10:29:51] (step=0045775) Train Loss: 6.3333, Train Steps/Sec: 0.86 + 12%|█████████████▍ | 4896/40903 [1:46:42<11:31:10, 1.15s/it][2025-04-21 10:30:20] (step=0045800) Train Loss: 6.3397, Train Steps/Sec: 0.86 + 12%|█████████████▍ | 4921/40903 [1:47:11<11:42:04, 1.17s/it][2025-04-21 10:30:49] (step=0045825) Train Loss: 6.3179, Train Steps/Sec: 0.86 + 12%|█████████████▌ | 4946/40903 [1:47:40<11:25:56, 1.14s/it][2025-04-21 10:31:19] (step=0045850) Train Loss: 6.3172, Train Steps/Sec: 0.86 + 12%|█████████████▌ | 4971/40903 [1:48:09<11:36:27, 1.16s/it][2025-04-21 10:31:48] (step=0045875) Train Loss: 6.3247, Train Steps/Sec: 0.86 + 12%|█████████████▋ | 4996/40903 [1:48:38<11:34:42, 1.16s/it][2025-04-21 10:32:17] (step=0045900) Train Loss: 6.2785, Train Steps/Sec: 0.86 + 12%|█████████████▋ | 5021/40903 [1:49:07<11:44:20, 1.18s/it][2025-04-21 10:32:46] (step=0045925) Train Loss: 6.2978, Train Steps/Sec: 0.86 + 12%|█████████████▊ | 5046/40903 [1:49:37<11:40:15, 1.17s/it][2025-04-21 10:33:15] (step=0045950) Train Loss: 6.3534, Train Steps/Sec: 0.86 + 12%|█████████████▉ | 5071/40903 [1:50:05<11:34:30, 1.16s/it][2025-04-21 10:33:44] (step=0045975) Train Loss: 6.2923, Train Steps/Sec: 0.86 + 12%|█████████████▉ | 5096/40903 [1:50:34<11:25:12, 1.15s/it][2025-04-21 10:34:13] (step=0046000) Train Loss: 6.3102, Train Steps/Sec: 0.87 +[2025-04-21 10:34:13] vision_config is None. initializing the InstructBlipVisionConfig with default values. +[2025-04-21 10:34:13] qformer_config is None. Initializing the InstructBlipQFormerConfig with default values. +[2025-04-21 10:34:13] text_config is None. Initializing the text config with default values (`OPTConfig`). +model_name_or_path:/tmp/haozhezhao/model/instructblip-flan-t5-xl +image_place_holder: +tokenizer length after expend 32102 +tokenizer length before expend 32102 +100%|████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████| 3/3 [03:06<00:00, 62.19s/it] +[2025-04-21 10:39:09] Finish Eval in 46000 steps...█████████████████████████████████████████████████████████████████████| 3/3 [03:06<00:00, 61.82s/it] +processing left-padding... +processing left-padding... +processing left-padding... +[2025-04-21 10:39:29] Saved checkpoint to checkpoint/EmptyFix_InstructBlip_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_objects/002-GPT-XL/checkpoints/0046000.pt +[2025-04-21 10:39:32] Removed old checkpoint: checkpoint/EmptyFix_InstructBlip_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_objects/002-GPT-XL/checkpoints/0042000.pt + 13%|██████████████ | 5121/40903 [1:56:22<12:00:00, 1.21s/it][2025-04-21 10:40:01] (step=0046025) Train Loss: 6.3476, Train Steps/Sec: 0.07 + 13%|██████████████ | 5146/40903 [1:56:51<11:32:25, 1.16s/it][2025-04-21 10:40:30] (step=0046050) Train Loss: 6.3122, Train Steps/Sec: 0.86 + 13%|██████████████▏ | 5171/40903 [1:57:20<11:24:58, 1.15s/it][2025-04-21 10:40:59] (step=0046075) Train Loss: 6.2914, Train Steps/Sec: 0.86 + 13%|██████████████▏ | 5196/40903 [1:57:49<11:26:59, 1.15s/it][2025-04-21 10:41:28] (step=0046100) Train Loss: 6.3655, Train Steps/Sec: 0.86 + 13%|██████████████▎ | 5221/40903 [1:58:19<11:40:59, 1.18s/it][2025-04-21 10:41:57] (step=0046125) Train Loss: 6.3304, Train Steps/Sec: 0.86 + 13%|██████████████▎ | 5246/40903 [1:58:48<11:31:01, 1.16s/it][2025-04-21 10:42:26] (step=0046150) Train Loss: 6.2863, Train Steps/Sec: 0.86 + 13%|██████████████▍ | 5271/40903 [1:59:16<11:24:31, 1.15s/it][2025-04-21 10:42:55] (step=0046175) Train Loss: 6.3163, Train Steps/Sec: 0.86 + 13%|██████████████▌ | 5296/40903 [1:59:46<11:18:16, 1.14s/it][2025-04-21 10:43:24] (step=0046200) Train Loss: 6.3544, Train Steps/Sec: 0.86 + 13%|██████████████▌ | 5321/40903 [2:00:15<11:31:52, 1.17s/it][2025-04-21 10:43:53] (step=0046225) Train Loss: 6.2471, Train Steps/Sec: 0.86 + 13%|██████████████▋ | 5346/40903 [2:00:44<11:24:02, 1.15s/it][2025-04-21 10:44:22] (step=0046250) Train Loss: 6.2934, Train Steps/Sec: 0.86 + 13%|██████████████▋ | 5371/40903 [2:01:13<11:26:51, 1.16s/it][2025-04-21 10:44:51] (step=0046275) Train Loss: 6.3249, Train Steps/Sec: 0.85 + 13%|██████████████▊ | 5396/40903 [2:01:42<11:17:39, 1.15s/it][2025-04-21 10:45:20] (step=0046300) Train Loss: 6.3292, Train Steps/Sec: 0.86 + 13%|██████████████▊ | 5421/40903 [2:02:11<11:37:41, 1.18s/it][2025-04-21 10:45:49] (step=0046325) Train Loss: 6.2799, Train Steps/Sec: 0.86 + 13%|██████████████▉ | 5446/40903 [2:02:40<11:23:18, 1.16s/it][2025-04-21 10:46:18] (step=0046350) Train Loss: 6.3178, Train Steps/Sec: 0.86 + 13%|██████████████▉ | 5471/40903 [2:03:09<11:20:49, 1.15s/it][2025-04-21 10:46:47] (step=0046375) Train Loss: 6.3761, Train Steps/Sec: 0.86 + 13%|███████████████ | 5496/40903 [2:03:38<11:19:46, 1.15s/it][2025-04-21 10:47:17] (step=0046400) Train Loss: 6.3497, Train Steps/Sec: 0.86 + 13%|███████████████ | 5521/40903 [2:04:07<11:41:30, 1.19s/it][2025-04-21 10:47:46] (step=0046425) Train Loss: 6.2901, Train Steps/Sec: 0.86 + 14%|███████████████▏ | 5546/40903 [2:04:36<11:28:41, 1.17s/it][2025-04-21 10:48:15] (step=0046450) Train Loss: 6.3485, Train Steps/Sec: 0.86 + 14%|███████████████▎ | 5571/40903 [2:05:05<11:23:34, 1.16s/it][2025-04-21 10:48:44] (step=0046475) Train Loss: 6.3303, Train Steps/Sec: 0.85 + 14%|███████████████▎ | 5596/40903 [2:05:35<11:20:12, 1.16s/it][2025-04-21 10:49:13] (step=0046500) Train Loss: 6.3570, Train Steps/Sec: 0.85 + 14%|███████████████▍ | 5621/40903 [2:06:04<11:41:28, 1.19s/it][2025-04-21 10:49:42] (step=0046525) Train Loss: 6.3524, Train Steps/Sec: 0.86 + 14%|███████████████▍ | 5646/40903 [2:06:33<12:03:25, 1.23s/it][2025-04-21 10:50:11] (step=0046550) Train Loss: 6.2666, Train Steps/Sec: 0.85 + 14%|███████████████▌ | 5671/40903 [2:07:02<11:20:18, 1.16s/it][2025-04-21 10:50:41] (step=0046575) Train Loss: 6.3613, Train Steps/Sec: 0.86 + 14%|███████████████▌ | 5696/40903 [2:07:31<11:19:44, 1.16s/it][2025-04-21 10:51:10] (step=0046600) Train Loss: 6.3481, Train Steps/Sec: 0.86 + 14%|███████████████▋ | 5721/40903 [2:08:00<11:37:13, 1.19s/it][2025-04-21 10:51:39] (step=0046625) Train Loss: 6.3352, Train Steps/Sec: 0.86 + 14%|███████████████▋ | 5746/40903 [2:08:29<11:27:56, 1.17s/it][2025-04-21 10:52:08] (step=0046650) Train Loss: 6.3644, Train Steps/Sec: 0.86 + 14%|███████████████▊ | 5771/40903 [2:08:58<11:18:25, 1.16s/it][2025-04-21 10:52:37] (step=0046675) Train Loss: 6.3267, Train Steps/Sec: 0.86 + 14%|███████████████▊ | 5796/40903 [2:09:28<11:17:36, 1.16s/it][2025-04-21 10:53:06] (step=0046700) Train Loss: 6.3071, Train Steps/Sec: 0.85 + 14%|███████████████▉ | 5821/40903 [2:09:57<11:25:47, 1.17s/it][2025-04-21 10:53:35] (step=0046725) Train Loss: 6.2829, Train Steps/Sec: 0.87 + 14%|████████████████ | 5846/40903 [2:10:26<11:29:39, 1.18s/it][2025-04-21 10:54:04] (step=0046750) Train Loss: 6.3315, Train Steps/Sec: 0.86 + 14%|████████████████ | 5871/40903 [2:10:55<11:14:39, 1.16s/it][2025-04-21 10:54:33] (step=0046775) Train Loss: 6.2801, Train Steps/Sec: 0.87 + 14%|████████████████▏ | 5896/40903 [2:11:23<11:05:16, 1.14s/it][2025-04-21 10:55:02] (step=0046800) Train Loss: 6.3091, Train Steps/Sec: 0.86 + 14%|████████████████▏ | 5921/40903 [2:11:53<11:33:16, 1.19s/it][2025-04-21 10:55:31] (step=0046825) Train Loss: 6.3194, Train Steps/Sec: 0.86 + 15%|████████████████▎ | 5946/40903 [2:12:21<11:15:03, 1.16s/it][2025-04-21 10:56:00] (step=0046850) Train Loss: 6.3146, Train Steps/Sec: 0.87 + 15%|████████████████▎ | 5971/40903 [2:12:50<11:16:46, 1.16s/it][2025-04-21 10:56:29] (step=0046875) Train Loss: 6.3319, Train Steps/Sec: 0.86 + 15%|████████████████▍ | 5996/40903 [2:13:19<11:05:47, 1.14s/it][2025-04-21 10:56:58] (step=0046900) Train Loss: 6.3189, Train Steps/Sec: 0.86 + 15%|████████████████▍ | 6021/40903 [2:13:48<11:10:14, 1.15s/it][2025-04-21 10:57:27] (step=0046925) Train Loss: 6.2935, Train Steps/Sec: 0.87 + 15%|████████████████▌ | 6046/40903 [2:14:17<11:18:55, 1.17s/it][2025-04-21 10:57:56] (step=0046950) Train Loss: 6.2771, Train Steps/Sec: 0.86 + 15%|████████████████▌ | 6071/40903 [2:14:46<11:04:41, 1.14s/it][2025-04-21 10:58:25] (step=0046975) Train Loss: 6.3162, Train Steps/Sec: 0.86 + 15%|████████████████▋ | 6096/40903 [2:15:15<11:06:18, 1.15s/it][2025-04-21 10:58:54] (step=0047000) Train Loss: 6.3255, Train Steps/Sec: 0.85 + 15%|████████████████▊ | 6121/40903 [2:15:52<11:28:23, 1.19s/it][2025-04-21 10:59:30] (step=0047025) Train Loss: 6.3668, Train Steps/Sec: 0.69 + 15%|████████████████▊ | 6146/40903 [2:16:21<11:18:13, 1.17s/it][2025-04-21 10:59:59] (step=0047050) Train Loss: 6.3226, Train Steps/Sec: 0.86 + 15%|████████████████▉ | 6171/40903 [2:16:50<11:08:52, 1.16s/it][2025-04-21 11:00:28] (step=0047075) Train Loss: 6.3328, Train Steps/Sec: 0.86 + 15%|████████████████▉ | 6196/40903 [2:17:19<11:07:41, 1.15s/it][2025-04-21 11:00:57] (step=0047100) Train Loss: 6.2894, Train Steps/Sec: 0.86 + 15%|█████████████████ | 6221/40903 [2:17:48<11:19:58, 1.18s/it][2025-04-21 11:01:26] (step=0047125) Train Loss: 6.3496, Train Steps/Sec: 0.86 + 15%|█████████████████ | 6246/40903 [2:18:17<11:06:49, 1.15s/it][2025-04-21 11:01:55] (step=0047150) Train Loss: 6.3189, Train Steps/Sec: 0.87 + 15%|█████████████████▏ | 6271/40903 [2:18:46<11:12:18, 1.16s/it][2025-04-21 11:02:24] (step=0047175) Train Loss: 6.3534, Train Steps/Sec: 0.86 + 15%|█████████████████▏ | 6296/40903 [2:19:15<11:06:32, 1.16s/it][2025-04-21 11:02:53] (step=0047200) Train Loss: 6.3049, Train Steps/Sec: 0.86 + 15%|█████████████████▎ | 6321/40903 [2:19:44<11:14:58, 1.17s/it][2025-04-21 11:03:22] (step=0047225) Train Loss: 6.3044, Train Steps/Sec: 0.86 + 16%|█████████████████▍ | 6346/40903 [2:20:13<11:13:34, 1.17s/it][2025-04-21 11:03:51] (step=0047250) Train Loss: 6.3306, Train Steps/Sec: 0.86 + 16%|█████████████████▍ | 6371/40903 [2:20:42<10:55:15, 1.14s/it][2025-04-21 11:04:20] (step=0047275) Train Loss: 6.3129, Train Steps/Sec: 0.87 + 16%|█████████████████▌ | 6396/40903 [2:21:11<10:58:33, 1.15s/it][2025-04-21 11:04:49] (step=0047300) Train Loss: 6.3335, Train Steps/Sec: 0.86 + 16%|█████████████████▌ | 6421/40903 [2:21:40<11:20:44, 1.18s/it][2025-04-21 11:05:18] (step=0047325) Train Loss: 6.3338, Train Steps/Sec: 0.85 + 16%|█████████████████▋ | 6446/40903 [2:22:09<11:07:59, 1.16s/it][2025-04-21 11:05:48] (step=0047350) Train Loss: 6.2869, Train Steps/Sec: 0.85 + 16%|█████████████████▋ | 6471/40903 [2:22:38<11:08:17, 1.16s/it][2025-04-21 11:06:17] (step=0047375) Train Loss: 6.3344, Train Steps/Sec: 0.86 + 16%|█████████████████▊ | 6496/40903 [2:23:07<11:00:11, 1.15s/it][2025-04-21 11:06:46] (step=0047400) Train Loss: 6.3273, Train Steps/Sec: 0.86 + 16%|█████████████████▊ | 6521/40903 [2:23:37<11:09:57, 1.17s/it][2025-04-21 11:07:15] (step=0047425) Train Loss: 6.3607, Train Steps/Sec: 0.86 + 16%|█████████████████▉ | 6546/40903 [2:24:05<11:00:59, 1.15s/it][2025-04-21 11:07:44] (step=0047450) Train Loss: 6.3263, Train Steps/Sec: 0.86 + 16%|█████████████████▉ | 6571/40903 [2:24:34<11:02:14, 1.16s/it][2025-04-21 11:08:13] (step=0047475) Train Loss: 6.3096, Train Steps/Sec: 0.86 + 16%|██████████████████ | 6596/40903 [2:25:04<10:59:57, 1.15s/it][2025-04-21 11:08:42] (step=0047500) Train Loss: 6.3534, Train Steps/Sec: 0.85 + 16%|██████████████████▏ | 6621/40903 [2:25:33<11:13:53, 1.18s/it][2025-04-21 11:09:11] (step=0047525) Train Loss: 6.3307, Train Steps/Sec: 0.86 + 16%|██████████████████▏ | 6646/40903 [2:26:02<11:13:32, 1.18s/it][2025-04-21 11:09:40] (step=0047550) Train Loss: 6.3380, Train Steps/Sec: 0.86 + 16%|██████████████████▎ | 6671/40903 [2:26:31<10:58:12, 1.15s/it][2025-04-21 11:10:09] (step=0047575) Train Loss: 6.3429, Train Steps/Sec: 0.87 + 16%|██████████████████▎ | 6696/40903 [2:27:00<11:01:05, 1.16s/it][2025-04-21 11:10:39] (step=0047600) Train Loss: 6.3511, Train Steps/Sec: 0.85 + 16%|██████████████████▍ | 6721/40903 [2:27:29<11:06:48, 1.17s/it][2025-04-21 11:11:08] (step=0047625) Train Loss: 6.3041, Train Steps/Sec: 0.87 + 16%|██████████████████▍ | 6746/40903 [2:27:58<10:59:08, 1.16s/it][2025-04-21 11:11:37] (step=0047650) Train Loss: 6.3403, Train Steps/Sec: 0.86 + 17%|██████████████████▌ | 6771/40903 [2:28:27<10:59:47, 1.16s/it][2025-04-21 11:12:06] (step=0047675) Train Loss: 6.3353, Train Steps/Sec: 0.86 + 17%|██████████████████▌ | 6796/40903 [2:28:56<10:47:51, 1.14s/it][2025-04-21 11:12:35] (step=0047700) Train Loss: 6.3233, Train Steps/Sec: 0.86 + 17%|██████████████████▋ | 6821/40903 [2:29:25<11:13:13, 1.19s/it][2025-04-21 11:13:11] (step=0047725) Train Loss: 6.3378, Train Steps/Sec: 0.69 + 17%|██████████████████▋ | 6846/40903 [2:30:02<11:01:30, 1.17s/it][2025-04-21 11:13:40] (step=0047750) Train Loss: 6.3107, Train Steps/Sec: 0.86 + 17%|██████████████████▊ | 6871/40903 [2:30:31<10:53:03, 1.15s/it][2025-04-21 11:14:09] (step=0047775) Train Loss: 6.2776, Train Steps/Sec: 0.87 + 17%|██████████████████▉ | 6896/40903 [2:30:59<10:53:16, 1.15s/it][2025-04-21 11:14:38] (step=0047800) Train Loss: 6.3173, Train Steps/Sec: 0.86 + 17%|██████████████████▉ | 6921/40903 [2:31:29<11:05:52, 1.18s/it][2025-04-21 11:15:07] (step=0047825) Train Loss: 6.3170, Train Steps/Sec: 0.86 + 17%|███████████████████ | 6946/40903 [2:31:58<10:54:22, 1.16s/it][2025-04-21 11:15:36] (step=0047850) Train Loss: 6.3146, Train Steps/Sec: 0.86 + 17%|███████████████████ | 6971/40903 [2:32:27<10:52:43, 1.15s/it][2025-04-21 11:16:05] (step=0047875) Train Loss: 6.3275, Train Steps/Sec: 0.86 + 17%|███████████████████▏ | 6996/40903 [2:32:56<10:46:01, 1.14s/it][2025-04-21 11:16:34] (step=0047900) Train Loss: 6.3333, Train Steps/Sec: 0.86 + 17%|███████████████████▏ | 7021/40903 [2:33:25<11:08:37, 1.18s/it][2025-04-21 11:17:03] (step=0047925) Train Loss: 6.3005, Train Steps/Sec: 0.86 + 17%|███████████████████▎ | 7046/40903 [2:33:54<10:47:10, 1.15s/it][2025-04-21 11:17:32] (step=0047950) Train Loss: 6.3272, Train Steps/Sec: 0.86 + 17%|███████████████████▎ | 7071/40903 [2:34:23<10:58:17, 1.17s/it][2025-04-21 11:18:01] (step=0047975) Train Loss: 6.3293, Train Steps/Sec: 0.86 + 17%|███████████████████▍ | 7096/40903 [2:34:52<10:46:20, 1.15s/it][2025-04-21 11:18:30] (step=0048000) Train Loss: 6.3100, Train Steps/Sec: 0.86 +[2025-04-21 11:18:30] vision_config is None. initializing the InstructBlipVisionConfig with default values. +[2025-04-21 11:18:30] qformer_config is None. Initializing the InstructBlipQFormerConfig with default values. +[2025-04-21 11:18:30] text_config is None. Initializing the text config with default values (`OPTConfig`). +model_name_or_path:/tmp/haozhezhao/model/instructblip-flan-t5-xl +image_place_holder: +tokenizer length after expend 32102 +tokenizer length before expend 32102 +100%|████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████| 3/3 [03:09<00:00, 63.00s/it] +[2025-04-21 11:23:30] Finish Eval in 48000 steps...█████████████████████████████████████████████████████████████████████| 3/3 [03:07<00:00, 62.35s/it] +processing left-padding... +processing left-padding... +processing left-padding... +[2025-04-21 11:23:52] Saved checkpoint to checkpoint/EmptyFix_InstructBlip_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_objects/002-GPT-XL/checkpoints/0048000.pt +[2025-04-21 11:23:54] Removed old checkpoint: checkpoint/EmptyFix_InstructBlip_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_objects/002-GPT-XL/checkpoints/0044000.pt + 17%|███████████████████▍ | 7121/40903 [2:40:45<11:23:37, 1.21s/it][2025-04-21 11:24:23] (step=0048025) Train Loss: 6.3307, Train Steps/Sec: 0.07 + 17%|███████████████████▌ | 7146/40903 [2:41:14<10:56:30, 1.17s/it][2025-04-21 11:24:52] (step=0048050) Train Loss: 6.3693, Train Steps/Sec: 0.86 + 18%|███████████████████▋ | 7171/40903 [2:41:43<10:44:09, 1.15s/it][2025-04-21 11:25:21] (step=0048075) Train Loss: 6.3038, Train Steps/Sec: 0.86 + 18%|███████████████████▋ | 7196/40903 [2:42:12<10:39:23, 1.14s/it][2025-04-21 11:25:50] (step=0048100) Train Loss: 6.2995, Train Steps/Sec: 0.86 + 18%|███████████████████▊ | 7221/40903 [2:42:41<11:09:12, 1.19s/it][2025-04-21 11:26:19] (step=0048125) Train Loss: 6.3273, Train Steps/Sec: 0.86 + 18%|███████████████████▊ | 7246/40903 [2:43:10<10:57:41, 1.17s/it][2025-04-21 11:26:48] (step=0048150) Train Loss: 6.3498, Train Steps/Sec: 0.86 + 18%|███████████████████▉ | 7271/40903 [2:43:39<10:49:29, 1.16s/it][2025-04-21 11:27:17] (step=0048175) Train Loss: 6.3889, Train Steps/Sec: 0.86 + 18%|███████████████████▉ | 7296/40903 [2:44:08<10:42:49, 1.15s/it][2025-04-21 11:27:46] (step=0048200) Train Loss: 6.3187, Train Steps/Sec: 0.87 + 18%|████████████████████ | 7321/40903 [2:44:43<15:10:04, 1.63s/it][2025-04-21 11:28:21] (step=0048225) Train Loss: 6.3433, Train Steps/Sec: 0.71 + 18%|████████████████████ | 7346/40903 [2:45:12<10:50:44, 1.16s/it][2025-04-21 11:28:50] (step=0048250) Train Loss: 6.3242, Train Steps/Sec: 0.86 + 18%|████████████████████▏ | 7371/40903 [2:45:41<10:48:27, 1.16s/it][2025-04-21 11:29:19] (step=0048275) Train Loss: 6.3623, Train Steps/Sec: 0.86 + 18%|████████████████████▎ | 7396/40903 [2:46:10<10:44:03, 1.15s/it][2025-04-21 11:29:48] (step=0048300) Train Loss: 6.3247, Train Steps/Sec: 0.86 + 18%|████████████████████▎ | 7421/40903 [2:46:39<10:58:33, 1.18s/it][2025-04-21 11:30:18] (step=0048325) Train Loss: 6.2803, Train Steps/Sec: 0.86 + 18%|████████████████████▍ | 7446/40903 [2:47:08<10:55:30, 1.18s/it][2025-04-21 11:30:47] (step=0048350) Train Loss: 6.3139, Train Steps/Sec: 0.86 + 18%|████████████████████▍ | 7471/40903 [2:47:37<10:41:21, 1.15s/it][2025-04-21 11:31:16] (step=0048375) Train Loss: 6.3342, Train Steps/Sec: 0.86 + 18%|████████████████████▌ | 7496/40903 [2:48:07<11:01:10, 1.19s/it][2025-04-21 11:31:46] (step=0048400) Train Loss: 6.3422, Train Steps/Sec: 0.84 + 18%|████████████████████▌ | 7521/40903 [2:48:36<10:47:56, 1.16s/it][2025-04-21 11:32:14] (step=0048425) Train Loss: 6.3109, Train Steps/Sec: 0.87 + 18%|████████████████████▋ | 7546/40903 [2:49:05<10:52:38, 1.17s/it][2025-04-21 11:32:43] (step=0048450) Train Loss: 6.3452, Train Steps/Sec: 0.86 + 19%|████████████████████▋ | 7571/40903 [2:49:34<10:45:01, 1.16s/it][2025-04-21 11:33:12] (step=0048475) Train Loss: 6.3323, Train Steps/Sec: 0.86 + 19%|████████████████████▊ | 7596/40903 [2:50:03<10:36:54, 1.15s/it][2025-04-21 11:33:42] (step=0048500) Train Loss: 6.2980, Train Steps/Sec: 0.86 + 19%|████████████████████▊ | 7621/40903 [2:50:32<10:49:35, 1.17s/it][2025-04-21 11:34:10] (step=0048525) Train Loss: 6.3517, Train Steps/Sec: 0.86 + 19%|████████████████████▉ | 7646/40903 [2:51:01<10:38:29, 1.15s/it][2025-04-21 11:34:39] (step=0048550) Train Loss: 6.3010, Train Steps/Sec: 0.87 + 19%|█████████████████████ | 7671/40903 [2:51:30<10:32:48, 1.14s/it][2025-04-21 11:35:08] (step=0048575) Train Loss: 6.3295, Train Steps/Sec: 0.87 + 19%|█████████████████████ | 7696/40903 [2:51:59<10:40:25, 1.16s/it][2025-04-21 11:35:37] (step=0048600) Train Loss: 6.2818, Train Steps/Sec: 0.86 + 19%|█████████████████████▏ | 7721/40903 [2:52:28<10:51:00, 1.18s/it][2025-04-21 11:36:06] (step=0048625) Train Loss: 6.3201, Train Steps/Sec: 0.86 + 19%|█████████████████████▏ | 7746/40903 [2:52:57<10:36:43, 1.15s/it][2025-04-21 11:36:35] (step=0048650) Train Loss: 6.3439, Train Steps/Sec: 0.86 + 19%|█████████████████████▎ | 7771/40903 [2:53:26<10:42:43, 1.16s/it][2025-04-21 11:37:04] (step=0048675) Train Loss: 6.3155, Train Steps/Sec: 0.86 + 19%|█████████████████████▎ | 7796/40903 [2:53:55<10:37:04, 1.15s/it][2025-04-21 11:37:33] (step=0048700) Train Loss: 6.3259, Train Steps/Sec: 0.85 + 19%|█████████████████████▍ | 7821/40903 [2:54:24<10:52:02, 1.18s/it][2025-04-21 11:38:02] (step=0048725) Train Loss: 6.2939, Train Steps/Sec: 0.86 + 19%|█████████████████████▍ | 7846/40903 [2:54:53<10:36:07, 1.15s/it][2025-04-21 11:38:31] (step=0048750) Train Loss: 6.3494, Train Steps/Sec: 0.86 + 19%|█████████████████████▌ | 7871/40903 [2:55:22<10:34:52, 1.15s/it][2025-04-21 11:39:00] (step=0048775) Train Loss: 6.3000, Train Steps/Sec: 0.86 + 19%|█████████████████████▌ | 7896/40903 [2:55:51<10:31:28, 1.15s/it][2025-04-21 11:39:29] (step=0048800) Train Loss: 6.3319, Train Steps/Sec: 0.86 + 19%|█████████████████████▋ | 7921/40903 [2:56:20<10:45:02, 1.17s/it][2025-04-21 11:39:58] (step=0048825) Train Loss: 6.2680, Train Steps/Sec: 0.87 + 19%|█████████████████████▊ | 7946/40903 [2:56:49<10:31:18, 1.15s/it][2025-04-21 11:40:27] (step=0048850) Train Loss: 6.3062, Train Steps/Sec: 0.87 + 19%|█████████████████████▊ | 7971/40903 [2:57:18<10:38:53, 1.16s/it][2025-04-21 11:40:56] (step=0048875) Train Loss: 6.3486, Train Steps/Sec: 0.86 + 20%|█████████████████████▉ | 7996/40903 [2:57:47<10:28:05, 1.15s/it][2025-04-21 11:41:25] (step=0048900) Train Loss: 6.3180, Train Steps/Sec: 0.86 + 20%|█████████████████████▉ | 8021/40903 [2:58:16<10:40:56, 1.17s/it][2025-04-21 11:41:54] (step=0048925) Train Loss: 6.3265, Train Steps/Sec: 0.87 + 20%|██████████████████████ | 8046/40903 [2:58:45<10:39:05, 1.17s/it][2025-04-21 11:42:23] (step=0048950) Train Loss: 6.3284, Train Steps/Sec: 0.86 + 20%|██████████████████████ | 8071/40903 [2:59:14<10:38:50, 1.17s/it][2025-04-21 11:42:52] (step=0048975) Train Loss: 6.2738, Train Steps/Sec: 0.86 + 20%|██████████████████████▏ | 8096/40903 [2:59:43<10:24:28, 1.14s/it][2025-04-21 11:43:21] (step=0049000) Train Loss: 6.3306, Train Steps/Sec: 0.86 + 20%|██████████████████████▏ | 8121/40903 [3:00:12<10:42:37, 1.18s/it][2025-04-21 11:43:50] (step=0049025) Train Loss: 6.3226, Train Steps/Sec: 0.86 + 20%|██████████████████████▎ | 8146/40903 [3:00:41<10:34:42, 1.16s/it][2025-04-21 11:44:19] (step=0049050) Train Loss: 6.3647, Train Steps/Sec: 0.86 + 20%|██████████████████████▎ | 8171/40903 [3:01:10<10:29:37, 1.15s/it][2025-04-21 11:44:48] (step=0049075) Train Loss: 6.3065, Train Steps/Sec: 0.87 + 20%|██████████████████████▍ | 8196/40903 [3:01:39<10:24:07, 1.14s/it][2025-04-21 11:45:17] (step=0049100) Train Loss: 6.3465, Train Steps/Sec: 0.86 + 20%|██████████████████████▌ | 8221/40903 [3:02:08<10:41:32, 1.18s/it][2025-04-21 11:45:46] (step=0049125) Train Loss: 6.3090, Train Steps/Sec: 0.86 + 20%|██████████████████████▌ | 8246/40903 [3:02:37<10:28:02, 1.15s/it][2025-04-21 11:46:15] (step=0049150) Train Loss: 6.3530, Train Steps/Sec: 0.86 + 20%|██████████████████████▋ | 8271/40903 [3:03:06<10:24:57, 1.15s/it][2025-04-21 11:46:44] (step=0049175) Train Loss: 6.3465, Train Steps/Sec: 0.86 + 20%|██████████████████████▋ | 8296/40903 [3:03:35<10:20:46, 1.14s/it][2025-04-21 11:47:13] (step=0049200) Train Loss: 6.3266, Train Steps/Sec: 0.86 + 20%|██████████████████████▊ | 8321/40903 [3:04:04<10:44:34, 1.19s/it][2025-04-21 11:47:43] (step=0049225) Train Loss: 6.3596, Train Steps/Sec: 0.86 + 20%|██████████████████████▊ | 8346/40903 [3:04:33<10:27:47, 1.16s/it][2025-04-21 11:48:11] (step=0049250) Train Loss: 6.3600, Train Steps/Sec: 0.87 + 20%|██████████████████████▉ | 8371/40903 [3:05:02<10:31:37, 1.16s/it][2025-04-21 11:48:40] (step=0049275) Train Loss: 6.3299, Train Steps/Sec: 0.86 + 21%|██████████████████████▉ | 8396/40903 [3:05:37<22:13:09, 2.46s/it][2025-04-21 11:49:16] (step=0049300) Train Loss: 6.3447, Train Steps/Sec: 0.71 + 21%|███████████████████████ | 8421/40903 [3:06:06<10:32:38, 1.17s/it][2025-04-21 11:49:45] (step=0049325) Train Loss: 6.3304, Train Steps/Sec: 0.86 + 21%|███████████████████████▏ | 8446/40903 [3:06:35<10:24:51, 1.16s/it][2025-04-21 11:50:14] (step=0049350) Train Loss: 6.3083, Train Steps/Sec: 0.86 + 21%|███████████████████████▏ | 8471/40903 [3:07:04<10:34:57, 1.17s/it][2025-04-21 11:50:43] (step=0049375) Train Loss: 6.3340, Train Steps/Sec: 0.86 + 21%|███████████████████████▎ | 8496/40903 [3:07:33<10:20:33, 1.15s/it][2025-04-21 11:51:12] (step=0049400) Train Loss: 6.3347, Train Steps/Sec: 0.86 + 21%|███████████████████████▎ | 8521/40903 [3:08:02<10:31:59, 1.17s/it][2025-04-21 11:51:41] (step=0049425) Train Loss: 6.2996, Train Steps/Sec: 0.86 + 21%|███████████████████████▍ | 8546/40903 [3:08:31<10:21:51, 1.15s/it][2025-04-21 11:52:10] (step=0049450) Train Loss: 6.2878, Train Steps/Sec: 0.87 + 21%|███████████████████████▍ | 8571/40903 [3:09:00<10:20:28, 1.15s/it][2025-04-21 11:52:39] (step=0049475) Train Loss: 6.3077, Train Steps/Sec: 0.86 + 21%|███████████████████████▌ | 8596/40903 [3:09:29<10:26:31, 1.16s/it][2025-04-21 11:53:08] (step=0049500) Train Loss: 6.3274, Train Steps/Sec: 0.86 + 21%|███████████████████████▌ | 8621/40903 [3:09:58<10:36:22, 1.18s/it][2025-04-21 11:53:37] (step=0049525) Train Loss: 6.3279, Train Steps/Sec: 0.86 + 21%|███████████████████████▋ | 8646/40903 [3:10:27<10:29:53, 1.17s/it][2025-04-21 11:54:06] (step=0049550) Train Loss: 6.3245, Train Steps/Sec: 0.86 + 21%|███████████████████████▋ | 8671/40903 [3:10:57<10:24:59, 1.16s/it][2025-04-21 11:54:35] (step=0049575) Train Loss: 6.3192, Train Steps/Sec: 0.86 + 21%|███████████████████████▊ | 8696/40903 [3:11:26<10:15:52, 1.15s/it][2025-04-21 11:55:04] (step=0049600) Train Loss: 6.3014, Train Steps/Sec: 0.86 + 21%|███████████████████████▉ | 8721/40903 [3:11:55<10:31:15, 1.18s/it][2025-04-21 11:55:33] (step=0049625) Train Loss: 6.2970, Train Steps/Sec: 0.86 + 21%|███████████████████████▉ | 8746/40903 [3:12:24<10:23:59, 1.16s/it][2025-04-21 11:56:02] (step=0049650) Train Loss: 6.3456, Train Steps/Sec: 0.86 + 21%|████████████████████████ | 8771/40903 [3:12:53<10:22:19, 1.16s/it][2025-04-21 11:56:31] (step=0049675) Train Loss: 6.2865, Train Steps/Sec: 0.86 + 22%|████████████████████████ | 8796/40903 [3:13:22<10:10:43, 1.14s/it][2025-04-21 11:57:00] (step=0049700) Train Loss: 6.3068, Train Steps/Sec: 0.85 + 22%|████████████████████████▏ | 8821/40903 [3:13:51<10:39:52, 1.20s/it][2025-04-21 11:57:29] (step=0049725) Train Loss: 6.3723, Train Steps/Sec: 0.86 + 22%|████████████████████████▏ | 8846/40903 [3:14:20<10:32:48, 1.18s/it][2025-04-21 11:57:59] (step=0049750) Train Loss: 6.3741, Train Steps/Sec: 0.86 + 22%|████████████████████████▎ | 8871/40903 [3:14:49<10:17:03, 1.16s/it][2025-04-21 11:58:28] (step=0049775) Train Loss: 6.2909, Train Steps/Sec: 0.86 + 22%|████████████████████████▎ | 8896/40903 [3:15:18<10:18:35, 1.16s/it][2025-04-21 11:58:57] (step=0049800) Train Loss: 6.3112, Train Steps/Sec: 0.85 + 22%|████████████████████████▍ | 8921/40903 [3:15:48<10:32:35, 1.19s/it][2025-04-21 11:59:26] (step=0049825) Train Loss: 6.3344, Train Steps/Sec: 0.86 + 22%|████████████████████████▍ | 8946/40903 [3:16:23<12:23:51, 1.40s/it][2025-04-21 12:00:02] (step=0049850) Train Loss: 6.3260, Train Steps/Sec: 0.70 + 22%|████████████████████████▌ | 8971/40903 [3:16:58<20:56:26, 2.36s/it][2025-04-21 12:00:36] (step=0049875) Train Loss: 6.3525, Train Steps/Sec: 0.72 + 22%|████████████████████████▋ | 8996/40903 [3:17:27<10:01:39, 1.13s/it][2025-04-21 12:01:05] (step=0049900) Train Loss: 6.3135, Train Steps/Sec: 0.86 + 22%|████████████████████████▋ | 9021/40903 [3:17:56<10:26:06, 1.18s/it][2025-04-21 12:01:35] (step=0049925) Train Loss: 6.3306, Train Steps/Sec: 0.86 + 22%|████████████████████████▊ | 9046/40903 [3:18:25<10:11:34, 1.15s/it][2025-04-21 12:02:04] (step=0049950) Train Loss: 6.3246, Train Steps/Sec: 0.86 + 22%|████████████████████████▊ | 9071/40903 [3:18:54<10:15:33, 1.16s/it][2025-04-21 12:02:33] (step=0049975) Train Loss: 6.3607, Train Steps/Sec: 0.86 + 22%|████████████████████████▉ | 9096/40903 [3:19:23<10:11:49, 1.15s/it][2025-04-21 12:03:02] (step=0050000) Train Loss: 6.3621, Train Steps/Sec: 0.86 +[2025-04-21 12:03:02] vision_config is None. initializing the InstructBlipVisionConfig with default values. +[2025-04-21 12:03:02] qformer_config is None. Initializing the InstructBlipQFormerConfig with default values. +[2025-04-21 12:03:02] text_config is None. Initializing the text config with default values (`OPTConfig`). +model_name_or_path:/tmp/haozhezhao/model/instructblip-flan-t5-xl +image_place_holder: +tokenizer length after expend 32102 +tokenizer length before expend 32102 +100%|████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████| 3/3 [03:08<00:00, 62.91s/it] +[2025-04-21 12:08:01] Finish Eval in 50000 steps...█████████████████████████████████████████████████████████████████████| 3/3 [03:07<00:00, 62.33s/it] +processing left-padding... +processing left-padding... +processing left-padding... +[2025-04-21 12:08:23] Saved checkpoint to checkpoint/EmptyFix_InstructBlip_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_objects/002-GPT-XL/checkpoints/0050000.pt +[2025-04-21 12:08:25] Removed old checkpoint: checkpoint/EmptyFix_InstructBlip_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_objects/002-GPT-XL/checkpoints/0046000.pt + 22%|████████████████████████▉ | 9121/40903 [3:25:15<10:25:09, 1.18s/it][2025-04-21 12:08:54] (step=0050025) Train Loss: 6.3110, Train Steps/Sec: 0.07 + 22%|█████████████████████████ | 9146/40903 [3:25:44<10:14:21, 1.16s/it][2025-04-21 12:09:23] (step=0050050) Train Loss: 6.3067, Train Steps/Sec: 0.87 + 22%|█████████████████████████ | 9171/40903 [3:26:13<10:03:39, 1.14s/it][2025-04-21 12:09:52] (step=0050075) Train Loss: 6.2759, Train Steps/Sec: 0.86 + 22%|█████████████████████████▏ | 9196/40903 [3:26:49<10:13:16, 1.16s/it][2025-04-21 12:10:28] (step=0050100) Train Loss: 6.3168, Train Steps/Sec: 0.69 + 23%|█████████████████████████▏ | 9221/40903 [3:27:18<10:24:53, 1.18s/it][2025-04-21 12:10:57] (step=0050125) Train Loss: 6.3686, Train Steps/Sec: 0.86 + 23%|█████████████████████████▎ | 9246/40903 [3:27:47<10:04:43, 1.15s/it][2025-04-21 12:11:25] (step=0050150) Train Loss: 6.3304, Train Steps/Sec: 0.87 + 23%|█████████████████████████▍ | 9271/40903 [3:28:16<10:09:36, 1.16s/it][2025-04-21 12:11:54] (step=0050175) Train Loss: 6.3052, Train Steps/Sec: 0.86 + 23%|█████████████████████████▍ | 9296/40903 [3:28:45<10:01:25, 1.14s/it][2025-04-21 12:12:24] (step=0050200) Train Loss: 6.3333, Train Steps/Sec: 0.86 + 23%|█████████████████████████▌ | 9321/40903 [3:29:14<10:22:21, 1.18s/it][2025-04-21 12:12:53] (step=0050225) Train Loss: 6.3543, Train Steps/Sec: 0.86 + 23%|█████████████████████████▌ | 9346/40903 [3:29:43<10:08:59, 1.16s/it][2025-04-21 12:13:22] (step=0050250) Train Loss: 6.3323, Train Steps/Sec: 0.85 + 23%|█████████████████████████▋ | 9371/40903 [3:30:12<10:03:46, 1.15s/it][2025-04-21 12:13:51] (step=0050275) Train Loss: 6.3409, Train Steps/Sec: 0.86 + 23%|█████████████████████████▋ | 9396/40903 [3:30:42<10:02:19, 1.15s/it][2025-04-21 12:14:20] (step=0050300) Train Loss: 6.3490, Train Steps/Sec: 0.86 + 23%|█████████████████████████▊ | 9421/40903 [3:31:11<10:22:50, 1.19s/it][2025-04-21 12:14:49] (step=0050325) Train Loss: 6.3003, Train Steps/Sec: 0.86 + 23%|█████████████████████████▊ | 9446/40903 [3:31:40<10:00:36, 1.15s/it][2025-04-21 12:15:18] (step=0050350) Train Loss: 6.3305, Train Steps/Sec: 0.87 + 23%|█████████████████████████▉ | 9471/40903 [3:32:09<10:06:59, 1.16s/it][2025-04-21 12:15:47] (step=0050375) Train Loss: 6.3412, Train Steps/Sec: 0.86 + 23%|██████████████████████████▏ | 9496/40903 [3:32:38<9:58:14, 1.14s/it][2025-04-21 12:16:17] (step=0050400) Train Loss: 6.3583, Train Steps/Sec: 0.86 + 23%|██████████████████████████ | 9521/40903 [3:33:07<10:20:38, 1.19s/it][2025-04-21 12:16:45] (step=0050425) Train Loss: 6.2790, Train Steps/Sec: 0.87 + 23%|██████████████████████████▏ | 9546/40903 [3:33:36<10:11:49, 1.17s/it][2025-04-21 12:17:14] (step=0050450) Train Loss: 6.2790, Train Steps/Sec: 0.86 + 23%|██████████████████████████▍ | 9571/40903 [3:34:05<9:50:44, 1.13s/it][2025-04-21 12:17:43] (step=0050475) Train Loss: 6.3491, Train Steps/Sec: 0.87 + 23%|██████████████████████████▎ | 9596/40903 [3:34:34<10:06:31, 1.16s/it][2025-04-21 12:18:12] (step=0050500) Train Loss: 6.3907, Train Steps/Sec: 0.86 + 24%|██████████████████████████▎ | 9621/40903 [3:35:03<10:12:32, 1.17s/it][2025-04-21 12:18:41] (step=0050525) Train Loss: 6.2939, Train Steps/Sec: 0.86 + 24%|██████████████████████████▍ | 9646/40903 [3:35:32<10:07:34, 1.17s/it][2025-04-21 12:19:10] (step=0050550) Train Loss: 6.3549, Train Steps/Sec: 0.86 + 24%|██████████████████████████▍ | 9671/40903 [3:36:01<10:01:35, 1.16s/it][2025-04-21 12:19:39] (step=0050575) Train Loss: 6.2975, Train Steps/Sec: 0.86 + 24%|██████████████████████████▌ | 9696/40903 [3:36:30<10:04:07, 1.16s/it][2025-04-21 12:20:09] (step=0050600) Train Loss: 6.3272, Train Steps/Sec: 0.85 + 24%|██████████████████████████▌ | 9721/40903 [3:36:59<10:15:29, 1.18s/it][2025-04-21 12:20:37] (step=0050625) Train Loss: 6.3089, Train Steps/Sec: 0.87 + 24%|██████████████████████████▋ | 9746/40903 [3:37:28<10:12:40, 1.18s/it][2025-04-21 12:21:07] (step=0050650) Train Loss: 6.3296, Train Steps/Sec: 0.86 + 24%|██████████████████████████▊ | 9771/40903 [3:37:57<10:00:42, 1.16s/it][2025-04-21 12:21:35] (step=0050675) Train Loss: 6.3030, Train Steps/Sec: 0.87 + 24%|███████████████████████████ | 9796/40903 [3:38:26<9:46:43, 1.13s/it][2025-04-21 12:22:05] (step=0050700) Train Loss: 6.3354, Train Steps/Sec: 0.86 + 24%|██████████████████████████▉ | 9821/40903 [3:38:55<10:14:47, 1.19s/it][2025-04-21 12:22:34] (step=0050725) Train Loss: 6.3165, Train Steps/Sec: 0.86 + 24%|██████████████████████████▉ | 9846/40903 [3:39:24<10:03:14, 1.17s/it][2025-04-21 12:23:02] (step=0050750) Train Loss: 6.2831, Train Steps/Sec: 0.87 + 24%|███████████████████████████▎ | 9871/40903 [3:39:53<9:54:30, 1.15s/it][2025-04-21 12:23:31] (step=0050775) Train Loss: 6.3335, Train Steps/Sec: 0.87 + 24%|███████████████████████████▎ | 9896/40903 [3:40:22<9:51:41, 1.14s/it][2025-04-21 12:24:00] (step=0050800) Train Loss: 6.3676, Train Steps/Sec: 0.86 + 24%|███████████████████████████▏ | 9921/40903 [3:40:51<10:08:36, 1.18s/it][2025-04-21 12:24:29] (step=0050825) Train Loss: 6.3512, Train Steps/Sec: 0.86 + 24%|███████████████████████████▏ | 9946/40903 [3:41:20<10:02:11, 1.17s/it][2025-04-21 12:24:58] (step=0050850) Train Loss: 6.3359, Train Steps/Sec: 0.86 + 24%|███████████████████████████▌ | 9971/40903 [3:41:49<9:51:37, 1.15s/it][2025-04-21 12:25:27] (step=0050875) Train Loss: 6.3557, Train Steps/Sec: 0.86 + 24%|███████████████████████████▌ | 9996/40903 [3:42:18<9:50:04, 1.15s/it][2025-04-21 12:25:57] (step=0050900) Train Loss: 6.3436, Train Steps/Sec: 0.86 + 24%|███████████████████████████▏ | 10021/40903 [3:42:47<10:08:16, 1.18s/it][2025-04-21 12:26:26] (step=0050925) Train Loss: 6.2907, Train Steps/Sec: 0.86 + 25%|███████████████████████████▎ | 10046/40903 [3:43:16<10:05:11, 1.18s/it][2025-04-21 12:26:55] (step=0050950) Train Loss: 6.3172, Train Steps/Sec: 0.86 + 25%|███████████████████████████▌ | 10071/40903 [3:43:45<9:54:04, 1.16s/it][2025-04-21 12:27:24] (step=0050975) Train Loss: 6.3054, Train Steps/Sec: 0.86 + 25%|███████████████████████████▋ | 10096/40903 [3:44:14<9:55:24, 1.16s/it][2025-04-21 12:27:53] (step=0051000) Train Loss: 6.3295, Train Steps/Sec: 0.85 + 25%|███████████████████████████▋ | 10121/40903 [3:44:43<9:59:38, 1.17s/it][2025-04-21 12:28:22] (step=0051025) Train Loss: 6.2941, Train Steps/Sec: 0.86 + 25%|███████████████████████████▊ | 10146/40903 [3:45:13<9:54:05, 1.16s/it][2025-04-21 12:28:51] (step=0051050) Train Loss: 6.2734, Train Steps/Sec: 0.86 + 25%|███████████████████████████▊ | 10171/40903 [3:45:41<9:44:45, 1.14s/it][2025-04-21 12:29:20] (step=0051075) Train Loss: 6.3378, Train Steps/Sec: 0.86 + 25%|███████████████████████████▉ | 10196/40903 [3:46:11<9:46:47, 1.15s/it][2025-04-21 12:29:49] (step=0051100) Train Loss: 6.3198, Train Steps/Sec: 0.86 + 25%|███████████████████████████▋ | 10221/40903 [3:46:40<10:09:36, 1.19s/it][2025-04-21 12:30:18] (step=0051125) Train Loss: 6.3433, Train Steps/Sec: 0.86 + 25%|████████████████████████████ | 10246/40903 [3:47:09<9:44:58, 1.14s/it][2025-04-21 12:30:47] (step=0051150) Train Loss: 6.2950, Train Steps/Sec: 0.87 + 25%|████████████████████████████ | 10271/40903 [3:47:38<9:48:25, 1.15s/it][2025-04-21 12:31:16] (step=0051175) Train Loss: 6.3624, Train Steps/Sec: 0.86 + 25%|████████████████████████████▏ | 10296/40903 [3:48:07<9:50:46, 1.16s/it][2025-04-21 12:31:45] (step=0051200) Train Loss: 6.3254, Train Steps/Sec: 0.86 + 25%|████████████████████████████ | 10321/40903 [3:48:36<10:09:33, 1.20s/it][2025-04-21 12:32:14] (step=0051225) Train Loss: 6.2876, Train Steps/Sec: 0.86 + 25%|████████████████████████████▎ | 10346/40903 [3:49:05<9:51:23, 1.16s/it][2025-04-21 12:32:43] (step=0051250) Train Loss: 6.3454, Train Steps/Sec: 0.86 + 25%|████████████████████████████▍ | 10371/40903 [3:49:34<9:43:54, 1.15s/it][2025-04-21 12:33:12] (step=0051275) Train Loss: 6.3194, Train Steps/Sec: 0.86 + 25%|████████████████████████████▍ | 10396/40903 [3:50:03<9:44:15, 1.15s/it][2025-04-21 12:33:42] (step=0051300) Train Loss: 6.3387, Train Steps/Sec: 0.85 + 25%|████████████████████████████▌ | 10421/40903 [3:50:32<9:51:07, 1.16s/it][2025-04-21 12:34:11] (step=0051325) Train Loss: 6.3111, Train Steps/Sec: 0.86 + 26%|████████████████████████████▌ | 10446/40903 [3:51:01<9:53:13, 1.17s/it][2025-04-21 12:34:40] (step=0051350) Train Loss: 6.3351, Train Steps/Sec: 0.86 + 26%|████████████████████████████▋ | 10471/40903 [3:51:30<9:45:44, 1.15s/it][2025-04-21 12:35:09] (step=0051375) Train Loss: 6.3350, Train Steps/Sec: 0.86 + 26%|████████████████████████████▋ | 10496/40903 [3:51:59<9:37:53, 1.14s/it][2025-04-21 12:35:38] (step=0051400) Train Loss: 6.3204, Train Steps/Sec: 0.86 + 26%|████████████████████████████▊ | 10521/40903 [3:52:28<9:57:19, 1.18s/it][2025-04-21 12:36:06] (step=0051425) Train Loss: 6.3268, Train Steps/Sec: 0.87 + 26%|████████████████████████████▉ | 10546/40903 [3:52:57<9:48:17, 1.16s/it][2025-04-21 12:36:35] (step=0051450) Train Loss: 6.3045, Train Steps/Sec: 0.86 + 26%|████████████████████████████▉ | 10571/40903 [3:53:26<9:40:11, 1.15s/it][2025-04-21 12:37:04] (step=0051475) Train Loss: 6.2805, Train Steps/Sec: 0.87 + 26%|█████████████████████████████ | 10596/40903 [3:53:55<9:43:41, 1.16s/it][2025-04-21 12:37:33] (step=0051500) Train Loss: 6.3105, Train Steps/Sec: 0.86 + 26%|████████████████████████████▊ | 10621/40903 [3:54:24<10:03:37, 1.20s/it][2025-04-21 12:38:02] (step=0051525) Train Loss: 6.2738, Train Steps/Sec: 0.86 + 26%|█████████████████████████████▏ | 10646/40903 [3:54:53<9:52:34, 1.18s/it][2025-04-21 12:38:32] (step=0051550) Train Loss: 6.3812, Train Steps/Sec: 0.85 + 26%|█████████████████████████████▏ | 10671/40903 [3:55:22<9:49:13, 1.17s/it][2025-04-21 12:39:01] (step=0051575) Train Loss: 6.2925, Train Steps/Sec: 0.86 + 26%|█████████████████████████████▎ | 10696/40903 [3:55:51<9:34:42, 1.14s/it][2025-04-21 12:39:30] (step=0051600) Train Loss: 6.3124, Train Steps/Sec: 0.87 + 26%|█████████████████████████████▎ | 10721/40903 [3:56:20<9:55:21, 1.18s/it][2025-04-21 12:39:59] (step=0051625) Train Loss: 6.3267, Train Steps/Sec: 0.86 + 26%|█████████████████████████████▍ | 10746/40903 [3:56:49<9:41:16, 1.16s/it][2025-04-21 12:40:28] (step=0051650) Train Loss: 6.2963, Train Steps/Sec: 0.86 + 26%|█████████████████████████████▍ | 10771/40903 [3:57:18<9:36:50, 1.15s/it][2025-04-21 12:40:57] (step=0051675) Train Loss: 6.3355, Train Steps/Sec: 0.86 + 26%|█████████████████████████████▌ | 10796/40903 [3:57:47<9:40:00, 1.16s/it][2025-04-21 12:41:26] (step=0051700) Train Loss: 6.3396, Train Steps/Sec: 0.85 + 26%|█████████████████████████████▋ | 10821/40903 [3:58:17<9:56:52, 1.19s/it][2025-04-21 12:41:55] (step=0051725) Train Loss: 6.3061, Train Steps/Sec: 0.86 + 27%|█████████████████████████████▋ | 10846/40903 [3:58:46<9:40:37, 1.16s/it][2025-04-21 12:42:24] (step=0051750) Train Loss: 6.3164, Train Steps/Sec: 0.86 + 27%|█████████████████████████████▊ | 10871/40903 [3:59:15<9:33:42, 1.15s/it][2025-04-21 12:42:53] (step=0051775) Train Loss: 6.2977, Train Steps/Sec: 0.86 + 27%|█████████████████████████████▊ | 10896/40903 [3:59:43<9:37:32, 1.15s/it][2025-04-21 12:43:22] (step=0051800) Train Loss: 6.3072, Train Steps/Sec: 0.86 + 27%|█████████████████████████████▉ | 10921/40903 [4:00:13<9:47:52, 1.18s/it][2025-04-21 12:43:51] (step=0051825) Train Loss: 6.3308, Train Steps/Sec: 0.86 + 27%|█████████████████████████████▉ | 10946/40903 [4:00:42<9:42:50, 1.17s/it][2025-04-21 12:44:20] (step=0051850) Train Loss: 6.3273, Train Steps/Sec: 0.86 + 27%|██████████████████████████████ | 10971/40903 [4:01:11<9:28:06, 1.14s/it][2025-04-21 12:44:49] (step=0051875) Train Loss: 6.3805, Train Steps/Sec: 0.86 + 27%|██████████████████████████████ | 10996/40903 [4:01:40<9:27:02, 1.14s/it][2025-04-21 12:45:18] (step=0051900) Train Loss: 6.3349, Train Steps/Sec: 0.86 + 27%|██████████████████████████████▏ | 11021/40903 [4:02:09<9:45:42, 1.18s/it][2025-04-21 12:45:47] (step=0051925) Train Loss: 6.2981, Train Steps/Sec: 0.86 + 27%|██████████████████████████████▏ | 11046/40903 [4:02:38<9:30:59, 1.15s/it][2025-04-21 12:46:16] (step=0051950) Train Loss: 6.3306, Train Steps/Sec: 0.87 + 27%|██████████████████████████████▎ | 11071/40903 [4:03:07<9:32:10, 1.15s/it][2025-04-21 12:46:45] (step=0051975) Train Loss: 6.2865, Train Steps/Sec: 0.86 + 27%|██████████████████████████████▍ | 11096/40903 [4:03:35<9:27:10, 1.14s/it][2025-04-21 12:47:14] (step=0052000) Train Loss: 6.2633, Train Steps/Sec: 0.86 +[2025-04-21 12:47:14] vision_config is None. initializing the InstructBlipVisionConfig with default values. +[2025-04-21 12:47:14] qformer_config is None. Initializing the InstructBlipQFormerConfig with default values. +[2025-04-21 12:47:14] text_config is None. Initializing the text config with default values (`OPTConfig`). +model_name_or_path:/tmp/haozhezhao/model/instructblip-flan-t5-xl +image_place_holder: +tokenizer length after expend 32102 +tokenizer length before expend 32102 +100%|████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████| 3/3 [03:10<00:00, 63.43s/it] +[2025-04-21 12:52:16] Finish Eval in 52000 steps...█████████████████████████████████████████████████████████████████████| 3/3 [03:09<00:00, 62.70s/it] +processing left-padding... +processing left-padding... +processing left-padding... +[2025-04-21 12:52:36] Saved checkpoint to checkpoint/EmptyFix_InstructBlip_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_objects/002-GPT-XL/checkpoints/0052000.pt +[2025-04-21 12:52:39] Removed old checkpoint: checkpoint/EmptyFix_InstructBlip_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_objects/002-GPT-XL/checkpoints/0048000.pt + 27%|██████████████████████████████▍ | 11121/40903 [4:09:29<9:49:54, 1.19s/it][2025-04-21 12:53:08] (step=0052025) Train Loss: 6.3210, Train Steps/Sec: 0.07 + 27%|██████████████████████████████▌ | 11146/40903 [4:09:58<9:35:49, 1.16s/it][2025-04-21 12:53:37] (step=0052050) Train Loss: 6.3018, Train Steps/Sec: 0.86 + 27%|██████████████████████████████▌ | 11171/40903 [4:10:27<9:32:17, 1.15s/it][2025-04-21 12:54:06] (step=0052075) Train Loss: 6.2939, Train Steps/Sec: 0.86 + 27%|██████████████████████████████▋ | 11196/40903 [4:10:57<9:37:28, 1.17s/it][2025-04-21 12:54:35] (step=0052100) Train Loss: 6.3251, Train Steps/Sec: 0.84 + 27%|██████████████████████████████▋ | 11221/40903 [4:11:26<9:43:23, 1.18s/it][2025-04-21 12:55:04] (step=0052125) Train Loss: 6.2857, Train Steps/Sec: 0.86 + 27%|██████████████████████████████▊ | 11246/40903 [4:11:55<9:42:56, 1.18s/it][2025-04-21 12:55:33] (step=0052150) Train Loss: 6.2933, Train Steps/Sec: 0.86 + 28%|██████████████████████████████▊ | 11271/40903 [4:12:24<9:37:53, 1.17s/it][2025-04-21 12:56:02] (step=0052175) Train Loss: 6.3043, Train Steps/Sec: 0.87 + 28%|██████████████████████████████▉ | 11296/40903 [4:12:53<9:28:50, 1.15s/it][2025-04-21 12:56:31] (step=0052200) Train Loss: 6.3133, Train Steps/Sec: 0.85 + 28%|██████████████████████████████▉ | 11321/40903 [4:13:22<9:34:45, 1.17s/it][2025-04-21 12:57:00] (step=0052225) Train Loss: 6.3338, Train Steps/Sec: 0.87 + 28%|███████████████████████████████ | 11346/40903 [4:13:51<9:28:47, 1.15s/it][2025-04-21 12:57:29] (step=0052250) Train Loss: 6.3298, Train Steps/Sec: 0.87 + 28%|███████████████████████████████▏ | 11371/40903 [4:14:20<9:25:29, 1.15s/it][2025-04-21 12:57:58] (step=0052275) Train Loss: 6.2633, Train Steps/Sec: 0.86 + 28%|███████████████████████████████▏ | 11396/40903 [4:14:49<9:29:15, 1.16s/it][2025-04-21 12:58:28] (step=0052300) Train Loss: 6.3057, Train Steps/Sec: 0.85 + 28%|███████████████████████████████▎ | 11421/40903 [4:15:18<9:39:04, 1.18s/it][2025-04-21 12:58:57] (step=0052325) Train Loss: 6.3253, Train Steps/Sec: 0.86 + 28%|███████████████████████████████▎ | 11446/40903 [4:15:47<9:29:04, 1.16s/it][2025-04-21 12:59:26] (step=0052350) Train Loss: 6.2797, Train Steps/Sec: 0.87 + 28%|███████████████████████████████▍ | 11471/40903 [4:16:16<9:21:46, 1.15s/it][2025-04-21 12:59:55] (step=0052375) Train Loss: 6.2842, Train Steps/Sec: 0.86 + 28%|███████████████████████████████▍ | 11496/40903 [4:16:45<9:24:57, 1.15s/it][2025-04-21 13:00:24] (step=0052400) Train Loss: 6.3232, Train Steps/Sec: 0.86 + 28%|███████████████████████████████▌ | 11521/40903 [4:17:14<9:32:06, 1.17s/it][2025-04-21 13:00:53] (step=0052425) Train Loss: 6.3505, Train Steps/Sec: 0.86 + 28%|███████████████████████████████▌ | 11546/40903 [4:17:43<9:27:13, 1.16s/it][2025-04-21 13:01:22] (step=0052450) Train Loss: 6.3254, Train Steps/Sec: 0.86 + 28%|███████████████████████████████▋ | 11571/40903 [4:18:12<9:33:19, 1.17s/it][2025-04-21 13:01:51] (step=0052475) Train Loss: 6.3495, Train Steps/Sec: 0.86 + 28%|███████████████████████████████▊ | 11596/40903 [4:18:48<9:25:34, 1.16s/it][2025-04-21 13:02:27] (step=0052500) Train Loss: 6.3111, Train Steps/Sec: 0.69 + 28%|███████████████████████████████▊ | 11621/40903 [4:19:18<9:40:11, 1.19s/it][2025-04-21 13:02:56] (step=0052525) Train Loss: 6.2883, Train Steps/Sec: 0.86 + 28%|███████████████████████████████▉ | 11646/40903 [4:19:46<9:24:23, 1.16s/it][2025-04-21 13:03:25] (step=0052550) Train Loss: 6.3268, Train Steps/Sec: 0.86 + 29%|███████████████████████████████▉ | 11671/40903 [4:20:15<9:20:27, 1.15s/it][2025-04-21 13:03:54] (step=0052575) Train Loss: 6.2731, Train Steps/Sec: 0.86 + 29%|████████████████████████████████ | 11696/40903 [4:20:45<9:20:42, 1.15s/it][2025-04-21 13:04:23] (step=0052600) Train Loss: 6.2945, Train Steps/Sec: 0.86 + 29%|████████████████████████████████ | 11721/40903 [4:21:14<9:24:37, 1.16s/it][2025-04-21 13:04:52] (step=0052625) Train Loss: 6.2613, Train Steps/Sec: 0.86 + 29%|████████████████████████████████▏ | 11746/40903 [4:21:43<9:26:26, 1.17s/it][2025-04-21 13:05:21] (step=0052650) Train Loss: 6.3179, Train Steps/Sec: 0.86 + 29%|████████████████████████████████▏ | 11771/40903 [4:22:12<9:22:36, 1.16s/it][2025-04-21 13:05:50] (step=0052675) Train Loss: 6.3497, Train Steps/Sec: 0.86 + 29%|████████████████████████████████▎ | 11796/40903 [4:22:41<9:17:59, 1.15s/it][2025-04-21 13:06:19] (step=0052700) Train Loss: 6.3000, Train Steps/Sec: 0.86 + 29%|████████████████████████████████▎ | 11821/40903 [4:23:10<9:32:17, 1.18s/it][2025-04-21 13:06:48] (step=0052725) Train Loss: 6.2762, Train Steps/Sec: 0.86 + 29%|████████████████████████████████▍ | 11846/40903 [4:23:39<9:21:07, 1.16s/it][2025-04-21 13:07:17] (step=0052750) Train Loss: 6.2859, Train Steps/Sec: 0.86 + 29%|████████████████████████████████▌ | 11871/40903 [4:24:08<9:12:32, 1.14s/it][2025-04-21 13:07:46] (step=0052775) Train Loss: 6.3259, Train Steps/Sec: 0.87 + 29%|████████████████████████████████▌ | 11896/40903 [4:24:36<9:07:26, 1.13s/it][2025-04-21 13:08:15] (step=0052800) Train Loss: 6.3093, Train Steps/Sec: 0.86 + 29%|████████████████████████████████▋ | 11921/40903 [4:25:05<9:19:35, 1.16s/it][2025-04-21 13:08:44] (step=0052825) Train Loss: 6.2971, Train Steps/Sec: 0.87 + 29%|████████████████████████████████▋ | 11946/40903 [4:25:34<9:20:25, 1.16s/it][2025-04-21 13:09:13] (step=0052850) Train Loss: 6.3200, Train Steps/Sec: 0.87 + 29%|████████████████████████████████▊ | 11971/40903 [4:26:03<9:11:00, 1.14s/it][2025-04-21 13:09:42] (step=0052875) Train Loss: 6.2389, Train Steps/Sec: 0.86 + 29%|████████████████████████████████▊ | 11996/40903 [4:26:32<9:17:36, 1.16s/it][2025-04-21 13:10:11] (step=0052900) Train Loss: 6.2755, Train Steps/Sec: 0.86 + 29%|████████████████████████████████▉ | 12021/40903 [4:27:02<9:31:50, 1.19s/it][2025-04-21 13:10:40] (step=0052925) Train Loss: 6.2961, Train Steps/Sec: 0.86 + 29%|████████████████████████████████▉ | 12046/40903 [4:27:31<9:23:49, 1.17s/it][2025-04-21 13:11:09] (step=0052950) Train Loss: 6.3259, Train Steps/Sec: 0.86 + 30%|█████████████████████████████████ | 12071/40903 [4:28:00<9:14:53, 1.15s/it][2025-04-21 13:11:38] (step=0052975) Train Loss: 6.3337, Train Steps/Sec: 0.86 + 30%|█████████████████████████████████ | 12096/40903 [4:28:28<9:09:26, 1.14s/it][2025-04-21 13:12:07] (step=0053000) Train Loss: 6.3088, Train Steps/Sec: 0.86 + 30%|█████████████████████████████████▏ | 12121/40903 [4:28:58<9:28:32, 1.19s/it][2025-04-21 13:12:36] (step=0053025) Train Loss: 6.2921, Train Steps/Sec: 0.86 + 30%|█████████████████████████████████▎ | 12146/40903 [4:29:27<9:23:04, 1.17s/it][2025-04-21 13:13:05] (step=0053050) Train Loss: 6.3145, Train Steps/Sec: 0.86 + 30%|█████████████████████████████████▎ | 12171/40903 [4:29:56<9:11:35, 1.15s/it][2025-04-21 13:13:34] (step=0053075) Train Loss: 6.2853, Train Steps/Sec: 0.86 + 30%|█████████████████████████████████▍ | 12196/40903 [4:30:25<9:03:36, 1.14s/it][2025-04-21 13:14:03] (step=0053100) Train Loss: 6.2808, Train Steps/Sec: 0.86 + 30%|█████████████████████████████████▍ | 12221/40903 [4:30:54<9:28:32, 1.19s/it][2025-04-21 13:14:32] (step=0053125) Train Loss: 6.3016, Train Steps/Sec: 0.86 + 30%|█████████████████████████████████▌ | 12246/40903 [4:31:23<9:09:29, 1.15s/it][2025-04-21 13:15:01] (step=0053150) Train Loss: 6.3075, Train Steps/Sec: 0.86 + 30%|█████████████████████████████████▌ | 12271/40903 [4:31:52<9:11:32, 1.16s/it][2025-04-21 13:15:30] (step=0053175) Train Loss: 6.3057, Train Steps/Sec: 0.86 + 30%|█████████████████████████████████▋ | 12296/40903 [4:32:21<9:06:16, 1.15s/it][2025-04-21 13:15:59] (step=0053200) Train Loss: 6.2811, Train Steps/Sec: 0.86 + 30%|█████████████████████████████████▋ | 12321/40903 [4:32:50<9:21:43, 1.18s/it][2025-04-21 13:16:28] (step=0053225) Train Loss: 6.3121, Train Steps/Sec: 0.86 + 30%|█████████████████████████████████▊ | 12346/40903 [4:33:19<9:07:32, 1.15s/it][2025-04-21 13:16:57] (step=0053250) Train Loss: 6.3311, Train Steps/Sec: 0.87 + 30%|█████████████████████████████████▌ | 12371/40903 [4:33:55<14:49:23, 1.87s/it][2025-04-21 13:17:33] (step=0053275) Train Loss: 6.3409, Train Steps/Sec: 0.70 + 30%|█████████████████████████████████▉ | 12396/40903 [4:34:24<9:14:12, 1.17s/it][2025-04-21 13:18:02] (step=0053300) Train Loss: 6.3421, Train Steps/Sec: 0.86 + 30%|██████████████████████████████████ | 12421/40903 [4:34:53<9:18:15, 1.18s/it][2025-04-21 13:18:31] (step=0053325) Train Loss: 6.3071, Train Steps/Sec: 0.87 + 30%|██████████████████████████████████ | 12446/40903 [4:35:22<9:06:51, 1.15s/it][2025-04-21 13:19:00] (step=0053350) Train Loss: 6.3249, Train Steps/Sec: 0.87 + 30%|██████████████████████████████████▏ | 12471/40903 [4:35:51<9:02:30, 1.14s/it][2025-04-21 13:19:29] (step=0053375) Train Loss: 6.2999, Train Steps/Sec: 0.87 + 31%|██████████████████████████████████▏ | 12496/40903 [4:36:20<9:09:55, 1.16s/it][2025-04-21 13:19:58] (step=0053400) Train Loss: 6.2867, Train Steps/Sec: 0.86 + 31%|██████████████████████████████████▎ | 12521/40903 [4:36:49<9:20:52, 1.19s/it][2025-04-21 13:20:27] (step=0053425) Train Loss: 6.2825, Train Steps/Sec: 0.86 + 31%|██████████████████████████████████▎ | 12546/40903 [4:37:18<9:05:02, 1.15s/it][2025-04-21 13:20:56] (step=0053450) Train Loss: 6.3099, Train Steps/Sec: 0.86 + 31%|██████████████████████████████████▍ | 12571/40903 [4:37:47<9:06:16, 1.16s/it][2025-04-21 13:21:25] (step=0053475) Train Loss: 6.3040, Train Steps/Sec: 0.86 + 31%|██████████████████████████████████▍ | 12596/40903 [4:38:16<8:57:30, 1.14s/it][2025-04-21 13:21:54] (step=0053500) Train Loss: 6.3303, Train Steps/Sec: 0.86 + 31%|██████████████████████████████████▌ | 12621/40903 [4:38:45<9:20:29, 1.19s/it][2025-04-21 13:22:23] (step=0053525) Train Loss: 6.3250, Train Steps/Sec: 0.86 + 31%|██████████████████████████████████▋ | 12646/40903 [4:39:14<9:09:59, 1.17s/it][2025-04-21 13:22:52] (step=0053550) Train Loss: 6.3291, Train Steps/Sec: 0.86 + 31%|██████████████████████████████████▋ | 12671/40903 [4:39:43<9:03:16, 1.15s/it][2025-04-21 13:23:21] (step=0053575) Train Loss: 6.3560, Train Steps/Sec: 0.86 + 31%|██████████████████████████████████▍ | 12696/40903 [4:40:21<11:32:02, 1.47s/it][2025-04-21 13:24:00] (step=0053600) Train Loss: 6.3092, Train Steps/Sec: 0.65 + 31%|██████████████████████████████████▊ | 12721/40903 [4:40:50<9:16:36, 1.19s/it][2025-04-21 13:24:29] (step=0053625) Train Loss: 6.3301, Train Steps/Sec: 0.86 + 31%|██████████████████████████████████▉ | 12746/40903 [4:41:19<9:02:36, 1.16s/it][2025-04-21 13:24:58] (step=0053650) Train Loss: 6.3483, Train Steps/Sec: 0.86 + 31%|██████████████████████████████████▉ | 12771/40903 [4:41:48<9:00:04, 1.15s/it][2025-04-21 13:25:27] (step=0053675) Train Loss: 6.2990, Train Steps/Sec: 0.86 + 31%|███████████████████████████████████ | 12796/40903 [4:42:17<8:51:23, 1.13s/it][2025-04-21 13:25:56] (step=0053700) Train Loss: 6.2975, Train Steps/Sec: 0.86 + 31%|███████████████████████████████████ | 12821/40903 [4:42:46<9:11:07, 1.18s/it][2025-04-21 13:26:25] (step=0053725) Train Loss: 6.3266, Train Steps/Sec: 0.87 + 31%|███████████████████████████████████▏ | 12846/40903 [4:43:15<9:15:18, 1.19s/it][2025-04-21 13:26:54] (step=0053750) Train Loss: 6.3287, Train Steps/Sec: 0.86 + 31%|███████████████████████████████████▏ | 12871/40903 [4:43:45<9:07:36, 1.17s/it][2025-04-21 13:27:23] (step=0053775) Train Loss: 6.3103, Train Steps/Sec: 0.86 + 32%|███████████████████████████████████▎ | 12896/40903 [4:44:13<8:46:57, 1.13s/it][2025-04-21 13:27:52] (step=0053800) Train Loss: 6.3171, Train Steps/Sec: 0.86 + 32%|███████████████████████████████████▍ | 12921/40903 [4:44:43<9:10:04, 1.18s/it][2025-04-21 13:28:21] (step=0053825) Train Loss: 6.3113, Train Steps/Sec: 0.86 + 32%|███████████████████████████████████▍ | 12946/40903 [4:45:12<9:04:50, 1.17s/it][2025-04-21 13:28:50] (step=0053850) Train Loss: 6.3044, Train Steps/Sec: 0.86 + 32%|███████████████████████████████████▌ | 12971/40903 [4:45:40<8:53:21, 1.15s/it][2025-04-21 13:29:19] (step=0053875) Train Loss: 6.3302, Train Steps/Sec: 0.87 + 32%|███████████████████████████████████▌ | 12996/40903 [4:46:10<8:54:12, 1.15s/it][2025-04-21 13:29:48] (step=0053900) Train Loss: 6.3015, Train Steps/Sec: 0.85 + 32%|███████████████████████████████████▋ | 13021/40903 [4:46:39<9:04:46, 1.17s/it][2025-04-21 13:30:17] (step=0053925) Train Loss: 6.3102, Train Steps/Sec: 0.87 + 32%|███████████████████████████████████▋ | 13046/40903 [4:47:07<8:56:15, 1.16s/it][2025-04-21 13:30:46] (step=0053950) Train Loss: 6.3114, Train Steps/Sec: 0.87 + 32%|███████████████████████████████████▊ | 13071/40903 [4:47:36<8:53:18, 1.15s/it][2025-04-21 13:31:15] (step=0053975) Train Loss: 6.3732, Train Steps/Sec: 0.86 + 32%|███████████████████████████████████▊ | 13096/40903 [4:48:05<8:49:57, 1.14s/it][2025-04-21 13:31:44] (step=0054000) Train Loss: 6.2860, Train Steps/Sec: 0.86 +[2025-04-21 13:31:44] vision_config is None. initializing the InstructBlipVisionConfig with default values. +[2025-04-21 13:31:44] qformer_config is None. Initializing the InstructBlipQFormerConfig with default values. +[2025-04-21 13:31:44] text_config is None. Initializing the text config with default values (`OPTConfig`). +model_name_or_path:/tmp/haozhezhao/model/instructblip-flan-t5-xl +image_place_holder: +tokenizer length after expend 32102 +tokenizer length before expend 32102 +100%|████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████| 3/3 [03:09<00:00, 63.31s/it] +[2025-04-21 13:36:44] Finish Eval in 54000 steps...█████████████████████████████████████████████████████████████████████| 3/3 [03:08<00:00, 62.69s/it] +processing left-padding... +processing left-padding... +processing left-padding... +[2025-04-21 13:37:06] Saved checkpoint to checkpoint/EmptyFix_InstructBlip_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_objects/002-GPT-XL/checkpoints/0054000.pt +[2025-04-21 13:37:08] Removed old checkpoint: checkpoint/EmptyFix_InstructBlip_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_objects/002-GPT-XL/checkpoints/0050000.pt + 32%|███████████████████████████████████▉ | 13121/40903 [4:53:59<9:15:57, 1.20s/it][2025-04-21 13:37:37] (step=0054025) Train Loss: 6.3181, Train Steps/Sec: 0.07 + 32%|███████████████████████████████████▉ | 13146/40903 [4:54:28<9:02:59, 1.17s/it][2025-04-21 13:38:06] (step=0054050) Train Loss: 6.3544, Train Steps/Sec: 0.86 + 32%|████████████████████████████████████ | 13171/40903 [4:54:57<8:54:51, 1.16s/it][2025-04-21 13:38:35] (step=0054075) Train Loss: 6.3208, Train Steps/Sec: 0.86 + 32%|████████████████████████████████████▏ | 13196/40903 [4:55:26<8:45:50, 1.14s/it][2025-04-21 13:39:04] (step=0054100) Train Loss: 6.2614, Train Steps/Sec: 0.86 + 32%|████████████████████████████████████▏ | 13221/40903 [4:55:55<9:08:13, 1.19s/it][2025-04-21 13:39:33] (step=0054125) Train Loss: 6.3071, Train Steps/Sec: 0.86 + 32%|████████████████████████████████████▎ | 13246/40903 [4:56:24<8:50:18, 1.15s/it][2025-04-21 13:40:02] (step=0054150) Train Loss: 6.3313, Train Steps/Sec: 0.87 + 32%|████████████████████████████████████▎ | 13271/40903 [4:56:52<8:45:44, 1.14s/it][2025-04-21 13:40:31] (step=0054175) Train Loss: 6.2956, Train Steps/Sec: 0.87 + 33%|████████████████████████████████████▍ | 13296/40903 [4:57:21<8:47:31, 1.15s/it][2025-04-21 13:41:00] (step=0054200) Train Loss: 6.3168, Train Steps/Sec: 0.86 + 33%|████████████████████████████████████▍ | 13321/40903 [4:57:50<9:05:50, 1.19s/it][2025-04-21 13:41:29] (step=0054225) Train Loss: 6.3233, Train Steps/Sec: 0.86 + 33%|████████████████████████████████████▌ | 13346/40903 [4:58:19<8:55:33, 1.17s/it][2025-04-21 13:41:58] (step=0054250) Train Loss: 6.3155, Train Steps/Sec: 0.86 + 33%|████████████████████████████████████▌ | 13371/40903 [4:58:48<8:44:40, 1.14s/it][2025-04-21 13:42:27] (step=0054275) Train Loss: 6.3173, Train Steps/Sec: 0.86 + 33%|████████████████████████████████████▋ | 13396/40903 [4:59:17<8:44:19, 1.14s/it][2025-04-21 13:42:56] (step=0054300) Train Loss: 6.2947, Train Steps/Sec: 0.86 + 33%|████████████████████████████████████▋ | 13421/40903 [4:59:46<9:01:24, 1.18s/it][2025-04-21 13:43:25] (step=0054325) Train Loss: 6.3775, Train Steps/Sec: 0.87 + 33%|████████████████████████████████████▊ | 13446/40903 [5:00:15<8:52:54, 1.16s/it][2025-04-21 13:43:54] (step=0054350) Train Loss: 6.2870, Train Steps/Sec: 0.86 + 33%|████████████████████████████████████▌ | 13471/40903 [5:00:51<12:17:50, 1.61s/it][2025-04-21 13:44:29] (step=0054375) Train Loss: 6.3230, Train Steps/Sec: 0.71 + 33%|████████████████████████████████████▉ | 13496/40903 [5:01:20<8:46:11, 1.15s/it][2025-04-21 13:44:58] (step=0054400) Train Loss: 6.3316, Train Steps/Sec: 0.86 + 33%|█████████████████████████████████████ | 13521/40903 [5:01:48<8:51:13, 1.16s/it][2025-04-21 13:45:27] (step=0054425) Train Loss: 6.2772, Train Steps/Sec: 0.87 + 33%|█████████████████████████████████████ | 13546/40903 [5:02:18<8:51:56, 1.17s/it][2025-04-21 13:45:56] (step=0054450) Train Loss: 6.2887, Train Steps/Sec: 0.86 + 33%|█████████████████████████████████████▏ | 13571/40903 [5:02:47<8:49:46, 1.16s/it][2025-04-21 13:46:25] (step=0054475) Train Loss: 6.3333, Train Steps/Sec: 0.86 + 33%|█████████████████████████████████████▏ | 13596/40903 [5:03:16<8:42:22, 1.15s/it][2025-04-21 13:46:54] (step=0054500) Train Loss: 6.2980, Train Steps/Sec: 0.86 + 33%|█████████████████████████████████████▎ | 13621/40903 [5:03:45<8:57:30, 1.18s/it][2025-04-21 13:47:23] (step=0054525) Train Loss: 6.2585, Train Steps/Sec: 0.86 + 33%|█████████████████████████████████████▎ | 13646/40903 [5:04:14<8:48:43, 1.16s/it][2025-04-21 13:47:52] (step=0054550) Train Loss: 6.3052, Train Steps/Sec: 0.86 + 33%|█████████████████████████████████████▍ | 13671/40903 [5:04:43<8:41:45, 1.15s/it][2025-04-21 13:48:21] (step=0054575) Train Loss: 6.3014, Train Steps/Sec: 0.86 + 33%|█████████████████████████████████████▌ | 13696/40903 [5:05:13<8:43:09, 1.15s/it][2025-04-21 13:48:51] (step=0054600) Train Loss: 6.3585, Train Steps/Sec: 0.84 + 34%|█████████████████████████████████████▌ | 13721/40903 [5:05:42<8:49:33, 1.17s/it][2025-04-21 13:49:20] (step=0054625) Train Loss: 6.3282, Train Steps/Sec: 0.87 + 34%|█████████████████████████████████████▋ | 13746/40903 [5:06:11<8:54:17, 1.18s/it][2025-04-21 13:49:49] (step=0054650) Train Loss: 6.3160, Train Steps/Sec: 0.86 + 34%|█████████████████████████████████████▋ | 13771/40903 [5:06:40<8:46:19, 1.16s/it][2025-04-21 13:50:18] (step=0054675) Train Loss: 6.3076, Train Steps/Sec: 0.86 + 34%|█████████████████████████████████████▊ | 13796/40903 [5:07:09<8:30:46, 1.13s/it][2025-04-21 13:50:47] (step=0054700) Train Loss: 6.3545, Train Steps/Sec: 0.86 + 34%|█████████████████████████████████████▊ | 13821/40903 [5:07:38<8:48:32, 1.17s/it][2025-04-21 13:51:16] (step=0054725) Train Loss: 6.2812, Train Steps/Sec: 0.87 + 34%|█████████████████████████████████████▉ | 13846/40903 [5:08:07<8:43:05, 1.16s/it][2025-04-21 13:51:45] (step=0054750) Train Loss: 6.3502, Train Steps/Sec: 0.86 + 34%|█████████████████████████████████████▉ | 13871/40903 [5:08:36<8:34:54, 1.14s/it][2025-04-21 13:52:14] (step=0054775) Train Loss: 6.3415, Train Steps/Sec: 0.87 + 34%|██████████████████████████████████████ | 13896/40903 [5:09:05<8:35:27, 1.15s/it][2025-04-21 13:52:43] (step=0054800) Train Loss: 6.2951, Train Steps/Sec: 0.86 + 34%|██████████████████████████████████████ | 13921/40903 [5:09:34<8:54:28, 1.19s/it][2025-04-21 13:53:12] (step=0054825) Train Loss: 6.3100, Train Steps/Sec: 0.86 + 34%|██████████████████████████████████████▏ | 13946/40903 [5:10:03<8:47:03, 1.17s/it][2025-04-21 13:53:41] (step=0054850) Train Loss: 6.3337, Train Steps/Sec: 0.87 + 34%|██████████████████████████████████████▎ | 13971/40903 [5:10:32<8:38:24, 1.15s/it][2025-04-21 13:54:10] (step=0054875) Train Loss: 6.3060, Train Steps/Sec: 0.86 + 34%|██████████████████████████████████████▎ | 13996/40903 [5:11:01<8:30:27, 1.14s/it][2025-04-21 13:54:39] (step=0054900) Train Loss: 6.3239, Train Steps/Sec: 0.86 + 34%|██████████████████████████████████████▍ | 14021/40903 [5:11:30<8:42:40, 1.17s/it][2025-04-21 13:55:08] (step=0054925) Train Loss: 6.2841, Train Steps/Sec: 0.86 + 34%|██████████████████████████████████████▍ | 14046/40903 [5:11:59<8:39:22, 1.16s/it][2025-04-21 13:55:37] (step=0054950) Train Loss: 6.2974, Train Steps/Sec: 0.86 + 34%|██████████████████████████████████████▌ | 14071/40903 [5:12:28<8:32:37, 1.15s/it][2025-04-21 13:56:13] (step=0054975) Train Loss: 6.3491, Train Steps/Sec: 0.70 + 34%|██████████████████████████████████████▌ | 14096/40903 [5:13:03<8:27:20, 1.14s/it][2025-04-21 13:56:42] (step=0055000) Train Loss: 6.3432, Train Steps/Sec: 0.86 + 35%|██████████████████████████████████████▎ | 14121/40903 [5:13:38<14:57:17, 2.01s/it][2025-04-21 13:57:16] (step=0055025) Train Loss: 6.3160, Train Steps/Sec: 0.72 + 35%|██████████████████████████████████████▋ | 14146/40903 [5:14:07<8:50:05, 1.19s/it][2025-04-21 13:57:46] (step=0055050) Train Loss: 6.2929, Train Steps/Sec: 0.85 + 35%|██████████████████████████████████████▊ | 14171/40903 [5:14:36<8:29:24, 1.14s/it][2025-04-21 13:58:14] (step=0055075) Train Loss: 6.3156, Train Steps/Sec: 0.87 + 35%|██████████████████████████████████████▊ | 14196/40903 [5:15:05<8:30:45, 1.15s/it][2025-04-21 13:58:43] (step=0055100) Train Loss: 6.3081, Train Steps/Sec: 0.86 + 35%|██████████████████████████████████████▉ | 14221/40903 [5:15:34<8:49:53, 1.19s/it][2025-04-21 13:59:13] (step=0055125) Train Loss: 6.3462, Train Steps/Sec: 0.86 + 35%|███████████████████████████████████████ | 14246/40903 [5:16:03<8:35:07, 1.16s/it][2025-04-21 13:59:42] (step=0055150) Train Loss: 6.3735, Train Steps/Sec: 0.86 + 35%|███████████████████████████████████████ | 14271/40903 [5:16:32<8:30:03, 1.15s/it][2025-04-21 14:00:11] (step=0055175) Train Loss: 6.2770, Train Steps/Sec: 0.86 + 35%|███████████████████████████████████████▏ | 14296/40903 [5:17:01<8:32:08, 1.15s/it][2025-04-21 14:00:40] (step=0055200) Train Loss: 6.3434, Train Steps/Sec: 0.86 + 35%|███████████████████████████████████████▏ | 14321/40903 [5:17:30<8:41:21, 1.18s/it][2025-04-21 14:01:09] (step=0055225) Train Loss: 6.2983, Train Steps/Sec: 0.87 + 35%|███████████████████████████████████████▎ | 14346/40903 [5:17:59<8:32:22, 1.16s/it][2025-04-21 14:01:38] (step=0055250) Train Loss: 6.3702, Train Steps/Sec: 0.86 + 35%|███████████████████████████████████████▎ | 14371/40903 [5:18:35<9:06:01, 1.23s/it][2025-04-21 14:02:14] (step=0055275) Train Loss: 6.3186, Train Steps/Sec: 0.70 + 35%|███████████████████████████████████████▍ | 14396/40903 [5:19:04<8:27:50, 1.15s/it][2025-04-21 14:02:43] (step=0055300) Train Loss: 6.2767, Train Steps/Sec: 0.86 + 35%|███████████████████████████████████████▍ | 14421/40903 [5:19:33<8:36:04, 1.17s/it][2025-04-21 14:03:12] (step=0055325) Train Loss: 6.3232, Train Steps/Sec: 0.86 + 35%|███████████████████████████████████████▌ | 14446/40903 [5:20:02<8:35:12, 1.17s/it][2025-04-21 14:03:41] (step=0055350) Train Loss: 6.3526, Train Steps/Sec: 0.86 + 35%|███████████████████████████████████████▌ | 14471/40903 [5:20:31<8:24:47, 1.15s/it][2025-04-21 14:04:10] (step=0055375) Train Loss: 6.3322, Train Steps/Sec: 0.86 + 35%|███████████████████████████████████████▋ | 14496/40903 [5:21:00<8:23:52, 1.14s/it][2025-04-21 14:04:39] (step=0055400) Train Loss: 6.3153, Train Steps/Sec: 0.86 + 36%|███████████████████████████████████████▊ | 14521/40903 [5:21:29<8:37:06, 1.18s/it][2025-04-21 14:05:08] (step=0055425) Train Loss: 6.2843, Train Steps/Sec: 0.86 + 36%|███████████████████████████████████████▊ | 14546/40903 [5:21:58<8:29:43, 1.16s/it][2025-04-21 14:05:37] (step=0055450) Train Loss: 6.3261, Train Steps/Sec: 0.86 + 36%|███████████████████████████████████████▉ | 14571/40903 [5:22:28<8:29:13, 1.16s/it][2025-04-21 14:06:06] (step=0055475) Train Loss: 6.3460, Train Steps/Sec: 0.86 + 36%|███████████████████████████████████████▉ | 14596/40903 [5:22:57<8:25:55, 1.15s/it][2025-04-21 14:06:35] (step=0055500) Train Loss: 6.3093, Train Steps/Sec: 0.86 + 36%|████████████████████████████████████████ | 14621/40903 [5:23:26<8:34:32, 1.17s/it][2025-04-21 14:07:04] (step=0055525) Train Loss: 6.3176, Train Steps/Sec: 0.86 + 36%|████████████████████████████████████████ | 14646/40903 [5:23:55<8:27:41, 1.16s/it][2025-04-21 14:07:33] (step=0055550) Train Loss: 6.2970, Train Steps/Sec: 0.86 + 36%|████████████████████████████████████████▏ | 14671/40903 [5:24:23<8:23:36, 1.15s/it][2025-04-21 14:08:02] (step=0055575) Train Loss: 6.3079, Train Steps/Sec: 0.87 + 36%|████████████████████████████████████████▏ | 14696/40903 [5:24:53<8:25:34, 1.16s/it][2025-04-21 14:08:31] (step=0055600) Train Loss: 6.3381, Train Steps/Sec: 0.86 + 36%|████████████████████████████████████████▎ | 14721/40903 [5:25:22<8:32:07, 1.17s/it][2025-04-21 14:09:00] (step=0055625) Train Loss: 6.3387, Train Steps/Sec: 0.87 + 36%|████████████████████████████████████████▍ | 14746/40903 [5:25:50<8:32:23, 1.18s/it][2025-04-21 14:09:29] (step=0055650) Train Loss: 6.3162, Train Steps/Sec: 0.86 + 36%|████████████████████████████████████████▍ | 14771/40903 [5:26:20<8:22:31, 1.15s/it][2025-04-21 14:09:58] (step=0055675) Train Loss: 6.3058, Train Steps/Sec: 0.86 + 36%|████████████████████████████████████████▌ | 14796/40903 [5:26:49<8:24:40, 1.16s/it][2025-04-21 14:10:27] (step=0055700) Train Loss: 6.3010, Train Steps/Sec: 0.86 + 36%|████████████████████████████████████████▌ | 14821/40903 [5:27:18<8:34:37, 1.18s/it][2025-04-21 14:10:56] (step=0055725) Train Loss: 6.3054, Train Steps/Sec: 0.87 + 36%|████████████████████████████████████████▋ | 14846/40903 [5:27:46<8:19:09, 1.15s/it][2025-04-21 14:11:25] (step=0055750) Train Loss: 6.2818, Train Steps/Sec: 0.87 + 36%|████████████████████████████████████████▋ | 14871/40903 [5:28:15<8:18:17, 1.15s/it][2025-04-21 14:11:54] (step=0055775) Train Loss: 6.3272, Train Steps/Sec: 0.86 + 36%|████████████████████████████████████████▊ | 14896/40903 [5:28:44<8:09:29, 1.13s/it][2025-04-21 14:12:23] (step=0055800) Train Loss: 6.3193, Train Steps/Sec: 0.86 + 36%|████████████████████████████████████████▊ | 14921/40903 [5:29:13<8:32:15, 1.18s/it][2025-04-21 14:12:52] (step=0055825) Train Loss: 6.2963, Train Steps/Sec: 0.86 + 37%|████████████████████████████████████████▉ | 14946/40903 [5:29:42<8:16:01, 1.15s/it][2025-04-21 14:13:21] (step=0055850) Train Loss: 6.3086, Train Steps/Sec: 0.86 + 37%|████████████████████████████████████████▉ | 14971/40903 [5:30:11<8:17:22, 1.15s/it][2025-04-21 14:13:50] (step=0055875) Train Loss: 6.2784, Train Steps/Sec: 0.86 + 37%|█████████████████████████████████████████ | 14996/40903 [5:30:40<8:13:42, 1.14s/it][2025-04-21 14:14:19] (step=0055900) Train Loss: 6.3207, Train Steps/Sec: 0.86 + 37%|█████████████████████████████████████████▏ | 15021/40903 [5:31:09<8:30:02, 1.18s/it][2025-04-21 14:14:48] (step=0055925) Train Loss: 6.3656, Train Steps/Sec: 0.86 + 37%|█████████████████████████████████████████▏ | 15046/40903 [5:31:38<8:25:48, 1.17s/it][2025-04-21 14:15:17] (step=0055950) Train Loss: 6.2659, Train Steps/Sec: 0.86 + 37%|█████████████████████████████████████████▎ | 15071/40903 [5:32:07<8:16:47, 1.15s/it][2025-04-21 14:15:46] (step=0055975) Train Loss: 6.3355, Train Steps/Sec: 0.86 + 37%|█████████████████████████████████████████▎ | 15096/40903 [5:32:36<8:20:04, 1.16s/it][2025-04-21 14:16:15] (step=0056000) Train Loss: 6.3826, Train Steps/Sec: 0.85 +[2025-04-21 14:16:15] vision_config is None. initializing the InstructBlipVisionConfig with default values. +[2025-04-21 14:16:15] qformer_config is None. Initializing the InstructBlipQFormerConfig with default values. +[2025-04-21 14:16:15] text_config is None. Initializing the text config with default values (`OPTConfig`). +model_name_or_path:/tmp/haozhezhao/model/instructblip-flan-t5-xl +image_place_holder: +tokenizer length after expend 32102 +tokenizer length before expend 32102 +100%|████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████| 3/3 [03:08<00:00, 62.86s/it] +[2025-04-21 14:21:14] Finish Eval in 56000 steps...█████████████████████████████████████████████████████████████████████| 3/3 [03:07<00:00, 62.28s/it] +processing left-padding... +processing left-padding... +processing left-padding... +[2025-04-21 14:21:36] Saved checkpoint to checkpoint/EmptyFix_InstructBlip_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_objects/002-GPT-XL/checkpoints/0056000.pt +[2025-04-21 14:21:38] Removed old checkpoint: checkpoint/EmptyFix_InstructBlip_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_objects/002-GPT-XL/checkpoints/0052000.pt + 37%|█████████████████████████████████████████▍ | 15121/40903 [5:38:29<8:34:27, 1.20s/it][2025-04-21 14:22:07] (step=0056025) Train Loss: 6.3026, Train Steps/Sec: 0.07 + 37%|█████████████████████████████████████████▍ | 15146/40903 [5:38:57<8:11:39, 1.15s/it][2025-04-21 14:22:36] (step=0056050) Train Loss: 6.3045, Train Steps/Sec: 0.87 + 37%|█████████████████████████████████████████▌ | 15171/40903 [5:39:26<8:09:12, 1.14s/it][2025-04-21 14:23:05] (step=0056075) Train Loss: 6.3541, Train Steps/Sec: 0.87 + 37%|█████████████████████████████████████████▌ | 15196/40903 [5:39:55<8:11:51, 1.15s/it][2025-04-21 14:23:34] (step=0056100) Train Loss: 6.3105, Train Steps/Sec: 0.86 + 37%|█████████████████████████████████████████▋ | 15221/40903 [5:40:24<8:19:46, 1.17s/it][2025-04-21 14:24:03] (step=0056125) Train Loss: 6.2820, Train Steps/Sec: 0.86 + 37%|█████████████████████████████████████████▋ | 15246/40903 [5:40:53<8:14:02, 1.16s/it][2025-04-21 14:24:32] (step=0056150) Train Loss: 6.2695, Train Steps/Sec: 0.87 + 37%|█████████████████████████████████████████▊ | 15271/40903 [5:41:22<8:06:44, 1.14s/it][2025-04-21 14:25:00] (step=0056175) Train Loss: 6.2865, Train Steps/Sec: 0.87 + 37%|█████████████████████████████████████████▉ | 15296/40903 [5:41:51<8:13:47, 1.16s/it][2025-04-21 14:25:29] (step=0056200) Train Loss: 6.3090, Train Steps/Sec: 0.86 + 37%|█████████████████████████████████████████▉ | 15321/40903 [5:42:20<8:17:39, 1.17s/it][2025-04-21 14:25:58] (step=0056225) Train Loss: 6.2766, Train Steps/Sec: 0.86 + 38%|██████████████████████████████████████████ | 15346/40903 [5:42:49<8:07:32, 1.14s/it][2025-04-21 14:26:27] (step=0056250) Train Loss: 6.3111, Train Steps/Sec: 0.87 + 38%|██████████████████████████████████████████ | 15371/40903 [5:43:18<8:16:27, 1.17s/it][2025-04-21 14:26:56] (step=0056275) Train Loss: 6.3247, Train Steps/Sec: 0.86 + 38%|██████████████████████████████████████████▏ | 15396/40903 [5:43:47<8:09:42, 1.15s/it][2025-04-21 14:27:25] (step=0056300) Train Loss: 6.2936, Train Steps/Sec: 0.86 + 38%|██████████████████████████████████████████▏ | 15421/40903 [5:44:16<8:15:11, 1.17s/it][2025-04-21 14:27:54] (step=0056325) Train Loss: 6.3156, Train Steps/Sec: 0.86 + 38%|██████████████████████████████████████████▎ | 15446/40903 [5:44:45<8:12:50, 1.16s/it][2025-04-21 14:28:23] (step=0056350) Train Loss: 6.3244, Train Steps/Sec: 0.87 + 38%|██████████████████████████████████████████▎ | 15471/40903 [5:45:14<8:08:41, 1.15s/it][2025-04-21 14:28:52] (step=0056375) Train Loss: 6.2930, Train Steps/Sec: 0.86 + 38%|██████████████████████████████████████████▍ | 15496/40903 [5:45:43<8:02:10, 1.14s/it][2025-04-21 14:29:21] (step=0056400) Train Loss: 6.3243, Train Steps/Sec: 0.86 + 38%|██████████████████████████████████████████▍ | 15521/40903 [5:46:12<8:23:54, 1.19s/it][2025-04-21 14:29:51] (step=0056425) Train Loss: 6.3066, Train Steps/Sec: 0.85 + 38%|██████████████████████████████████████████▌ | 15546/40903 [5:46:42<8:12:47, 1.17s/it][2025-04-21 14:30:20] (step=0056450) Train Loss: 6.2511, Train Steps/Sec: 0.84 + 38%|██████████████████████████████████████████▋ | 15571/40903 [5:47:11<8:07:46, 1.16s/it][2025-04-21 14:30:49] (step=0056475) Train Loss: 6.3366, Train Steps/Sec: 0.86 + 38%|██████████████████████████████████████████▋ | 15596/40903 [5:47:40<8:01:56, 1.14s/it][2025-04-21 14:31:18] (step=0056500) Train Loss: 6.2942, Train Steps/Sec: 0.86 + 38%|██████████████████████████████████████████▊ | 15621/40903 [5:48:09<8:14:51, 1.17s/it][2025-04-21 14:31:47] (step=0056525) Train Loss: 6.3526, Train Steps/Sec: 0.86 + 38%|██████████████████████████████████████████▊ | 15646/40903 [5:48:38<8:00:12, 1.14s/it][2025-04-21 14:32:16] (step=0056550) Train Loss: 6.3618, Train Steps/Sec: 0.87 + 38%|██████████████████████████████████████████▉ | 15671/40903 [5:49:07<8:09:18, 1.16s/it][2025-04-21 14:32:45] (step=0056575) Train Loss: 6.2675, Train Steps/Sec: 0.86 + 38%|██████████████████████████████████████████▉ | 15696/40903 [5:49:36<8:02:26, 1.15s/it][2025-04-21 14:33:14] (step=0056600) Train Loss: 6.3376, Train Steps/Sec: 0.86 + 38%|███████████████████████████████████████████ | 15721/40903 [5:50:05<8:10:25, 1.17s/it][2025-04-21 14:33:43] (step=0056625) Train Loss: 6.3207, Train Steps/Sec: 0.86 + 38%|███████████████████████████████████████████ | 15746/40903 [5:50:34<8:03:29, 1.15s/it][2025-04-21 14:34:12] (step=0056650) Train Loss: 6.2857, Train Steps/Sec: 0.86 + 39%|███████████████████████████████████████████▏ | 15771/40903 [5:51:03<8:02:05, 1.15s/it][2025-04-21 14:34:41] (step=0056675) Train Loss: 6.3077, Train Steps/Sec: 0.87 + 39%|███████████████████████████████████████████▎ | 15796/40903 [5:51:32<8:00:47, 1.15s/it][2025-04-21 14:35:10] (step=0056700) Train Loss: 6.3314, Train Steps/Sec: 0.86 + 39%|███████████████████████████████████████████▎ | 15821/40903 [5:52:01<8:11:22, 1.18s/it][2025-04-21 14:35:39] (step=0056725) Train Loss: 6.3239, Train Steps/Sec: 0.86 + 39%|███████████████████████████████████████████▍ | 15846/40903 [5:52:29<8:02:15, 1.15s/it][2025-04-21 14:36:08] (step=0056750) Train Loss: 6.3030, Train Steps/Sec: 0.87 + 39%|███████████████████████████████████████████▍ | 15871/40903 [5:52:59<8:08:59, 1.17s/it][2025-04-21 14:36:37] (step=0056775) Train Loss: 6.2913, Train Steps/Sec: 0.86 + 39%|███████████████████████████████████████████▌ | 15896/40903 [5:53:28<8:00:41, 1.15s/it][2025-04-21 14:37:06] (step=0056800) Train Loss: 6.3209, Train Steps/Sec: 0.85 + 39%|███████████████████████████████████████████▌ | 15921/40903 [5:53:57<8:11:22, 1.18s/it][2025-04-21 14:37:35] (step=0056825) Train Loss: 6.3150, Train Steps/Sec: 0.86 + 39%|███████████████████████████████████████████▋ | 15946/40903 [5:54:26<7:58:36, 1.15s/it][2025-04-21 14:38:04] (step=0056850) Train Loss: 6.3179, Train Steps/Sec: 0.86 + 39%|███████████████████████████████████████████▋ | 15971/40903 [5:54:55<7:57:35, 1.15s/it][2025-04-21 14:38:33] (step=0056875) Train Loss: 6.3286, Train Steps/Sec: 0.87 + 39%|███████████████████████████████████████████▊ | 15996/40903 [5:55:24<7:59:01, 1.15s/it][2025-04-21 14:39:03] (step=0056900) Train Loss: 6.3038, Train Steps/Sec: 0.85 + 39%|███████████████████████████████████████████▊ | 16021/40903 [5:55:53<8:06:17, 1.17s/it][2025-04-21 14:39:31] (step=0056925) Train Loss: 6.3023, Train Steps/Sec: 0.87 + 39%|███████████████████████████████████████████▉ | 16046/40903 [5:56:22<7:59:54, 1.16s/it][2025-04-21 14:40:00] (step=0056950) Train Loss: 6.3067, Train Steps/Sec: 0.86 + 39%|████████████████████████████████████████████ | 16071/40903 [5:56:51<7:57:23, 1.15s/it][2025-04-21 14:40:29] (step=0056975) Train Loss: 6.3090, Train Steps/Sec: 0.86 + 39%|████████████████████████████████████████████ | 16096/40903 [5:57:20<7:57:47, 1.16s/it][2025-04-21 14:40:59] (step=0057000) Train Loss: 6.3325, Train Steps/Sec: 0.86 + 39%|████████████████████████████████████████████▏ | 16121/40903 [5:57:49<8:01:29, 1.17s/it][2025-04-21 14:41:27] (step=0057025) Train Loss: 6.3487, Train Steps/Sec: 0.87 + 39%|████████████████████████████████████████████▏ | 16146/40903 [5:58:18<7:55:27, 1.15s/it][2025-04-21 14:41:56] (step=0057050) Train Loss: 6.3047, Train Steps/Sec: 0.86 + 40%|████████████████████████████████████████████▎ | 16171/40903 [5:58:47<7:48:47, 1.14s/it][2025-04-21 14:42:25] (step=0057075) Train Loss: 6.3124, Train Steps/Sec: 0.86 + 40%|████████████████████████████████████████████▎ | 16196/40903 [5:59:16<7:54:29, 1.15s/it][2025-04-21 14:42:55] (step=0057100) Train Loss: 6.2957, Train Steps/Sec: 0.86 + 40%|████████████████████████████████████████████▍ | 16221/40903 [5:59:45<7:58:08, 1.16s/it][2025-04-21 14:43:23] (step=0057125) Train Loss: 6.3383, Train Steps/Sec: 0.87 + 40%|████████████████████████████████████████████▍ | 16246/40903 [6:00:14<8:01:37, 1.17s/it][2025-04-21 14:43:52] (step=0057150) Train Loss: 6.3176, Train Steps/Sec: 0.86 + 40%|████████████████████████████████████████████▌ | 16271/40903 [6:00:43<7:51:59, 1.15s/it][2025-04-21 14:44:21] (step=0057175) Train Loss: 6.2984, Train Steps/Sec: 0.87 + 40%|████████████████████████████████████████████▌ | 16296/40903 [6:01:12<7:53:18, 1.15s/it][2025-04-21 14:44:50] (step=0057200) Train Loss: 6.2984, Train Steps/Sec: 0.86 + 40%|████████████████████████████████████████████▋ | 16321/40903 [6:01:41<8:03:28, 1.18s/it][2025-04-21 14:45:19] (step=0057225) Train Loss: 6.2883, Train Steps/Sec: 0.86 + 40%|████████████████████████████████████████████▊ | 16346/40903 [6:02:10<7:51:29, 1.15s/it][2025-04-21 14:45:48] (step=0057250) Train Loss: 6.3115, Train Steps/Sec: 0.86 + 40%|████████████████████████████████████████████▊ | 16371/40903 [6:02:39<7:46:46, 1.14s/it][2025-04-21 14:46:17] (step=0057275) Train Loss: 6.3005, Train Steps/Sec: 0.87 + 40%|████████████████████████████████████████████▉ | 16396/40903 [6:03:08<7:48:11, 1.15s/it][2025-04-21 14:46:46] (step=0057300) Train Loss: 6.2880, Train Steps/Sec: 0.86 + 40%|████████████████████████████████████████████▉ | 16421/40903 [6:03:37<8:00:51, 1.18s/it][2025-04-21 14:47:15] (step=0057325) Train Loss: 6.2940, Train Steps/Sec: 0.87 + 40%|█████████████████████████████████████████████ | 16446/40903 [6:04:06<7:54:38, 1.16s/it][2025-04-21 14:47:44] (step=0057350) Train Loss: 6.3557, Train Steps/Sec: 0.86 + 40%|█████████████████████████████████████████████ | 16471/40903 [6:04:35<7:49:45, 1.15s/it][2025-04-21 14:48:13] (step=0057375) Train Loss: 6.3110, Train Steps/Sec: 0.87 + 40%|█████████████████████████████████████████████▏ | 16496/40903 [6:05:04<7:51:04, 1.16s/it][2025-04-21 14:48:42] (step=0057400) Train Loss: 6.3221, Train Steps/Sec: 0.86 + 40%|█████████████████████████████████████████████▏ | 16521/40903 [6:05:33<7:56:30, 1.17s/it][2025-04-21 14:49:11] (step=0057425) Train Loss: 6.3181, Train Steps/Sec: 0.86 + 40%|█████████████████████████████████████████████▎ | 16546/40903 [6:06:02<7:54:26, 1.17s/it][2025-04-21 14:49:40] (step=0057450) Train Loss: 6.3229, Train Steps/Sec: 0.86 + 41%|█████████████████████████████████████████████▎ | 16571/40903 [6:06:31<7:44:05, 1.14s/it][2025-04-21 14:50:09] (step=0057475) Train Loss: 6.3325, Train Steps/Sec: 0.87 + 41%|█████████████████████████████████████████████▍ | 16596/40903 [6:07:00<7:50:35, 1.16s/it][2025-04-21 14:50:38] (step=0057500) Train Loss: 6.2991, Train Steps/Sec: 0.86 + 41%|█████████████████████████████████████████████▌ | 16621/40903 [6:07:29<7:53:13, 1.17s/it][2025-04-21 14:51:07] (step=0057525) Train Loss: 6.3373, Train Steps/Sec: 0.86 + 41%|█████████████████████████████████████████████▌ | 16646/40903 [6:07:58<7:54:33, 1.17s/it][2025-04-21 14:51:36] (step=0057550) Train Loss: 6.3133, Train Steps/Sec: 0.86 + 41%|█████████████████████████████████████████████▋ | 16671/40903 [6:08:27<7:40:32, 1.14s/it][2025-04-21 14:52:05] (step=0057575) Train Loss: 6.3435, Train Steps/Sec: 0.87 + 41%|█████████████████████████████████████████████▋ | 16696/40903 [6:08:56<7:38:40, 1.14s/it][2025-04-21 14:52:34] (step=0057600) Train Loss: 6.3203, Train Steps/Sec: 0.86 + 41%|█████████████████████████████████████████████▊ | 16721/40903 [6:09:25<7:54:18, 1.18s/it][2025-04-21 14:53:03] (step=0057625) Train Loss: 6.3365, Train Steps/Sec: 0.87 + 41%|█████████████████████████████████████████████▊ | 16746/40903 [6:09:53<7:45:55, 1.16s/it][2025-04-21 14:53:32] (step=0057650) Train Loss: 6.3089, Train Steps/Sec: 0.87 + 41%|█████████████████████████████████████████████▉ | 16771/40903 [6:10:23<7:43:18, 1.15s/it][2025-04-21 14:54:01] (step=0057675) Train Loss: 6.2960, Train Steps/Sec: 0.86 + 41%|█████████████████████████████████████████████▉ | 16796/40903 [6:10:51<7:35:26, 1.13s/it][2025-04-21 14:54:30] (step=0057700) Train Loss: 6.3310, Train Steps/Sec: 0.86 + 41%|██████████████████████████████████████████████ | 16821/40903 [6:11:21<7:50:00, 1.17s/it][2025-04-21 14:54:59] (step=0057725) Train Loss: 6.3167, Train Steps/Sec: 0.86 + 41%|██████████████████████████████████████████████▏ | 16846/40903 [6:11:49<7:49:17, 1.17s/it][2025-04-21 14:55:28] (step=0057750) Train Loss: 6.2913, Train Steps/Sec: 0.86 + 41%|██████████████████████████████████████████████▏ | 16871/40903 [6:12:18<7:38:44, 1.15s/it][2025-04-21 14:55:57] (step=0057775) Train Loss: 6.3194, Train Steps/Sec: 0.86 + 41%|██████████████████████████████████████████████▎ | 16896/40903 [6:12:48<7:41:23, 1.15s/it][2025-04-21 14:56:26] (step=0057800) Train Loss: 6.3194, Train Steps/Sec: 0.86 + 41%|██████████████████████████████████████████████▎ | 16921/40903 [6:13:17<7:47:16, 1.17s/it][2025-04-21 14:56:55] (step=0057825) Train Loss: 6.3164, Train Steps/Sec: 0.86 + 41%|██████████████████████████████████████████████▍ | 16946/40903 [6:13:46<7:42:04, 1.16s/it][2025-04-21 14:57:24] (step=0057850) Train Loss: 6.3267, Train Steps/Sec: 0.87 + 41%|██████████████████████████████████████████████▍ | 16971/40903 [6:14:15<7:39:57, 1.15s/it][2025-04-21 14:57:53] (step=0057875) Train Loss: 6.2173, Train Steps/Sec: 0.87 + 42%|██████████████████████████████████████████████▌ | 16996/40903 [6:14:44<7:33:24, 1.14s/it][2025-04-21 14:58:22] (step=0057900) Train Loss: 6.3016, Train Steps/Sec: 0.86 + 42%|██████████████████████████████████████████████▌ | 17021/40903 [6:15:13<7:43:59, 1.17s/it][2025-04-21 14:58:51] (step=0057925) Train Loss: 6.3150, Train Steps/Sec: 0.86 + 42%|██████████████████████████████████████████████▋ | 17046/40903 [6:15:42<7:42:50, 1.16s/it][2025-04-21 14:59:20] (step=0057950) Train Loss: 6.3060, Train Steps/Sec: 0.87 + 42%|██████████████████████████████████████████████▎ | 17071/40903 [6:16:18<17:41:14, 2.67s/it][2025-04-21 14:59:56] (step=0057975) Train Loss: 6.3365, Train Steps/Sec: 0.69 + 42%|██████████████████████████████████████████████▊ | 17096/40903 [6:16:47<7:41:08, 1.16s/it][2025-04-21 15:00:25] (step=0058000) Train Loss: 6.3304, Train Steps/Sec: 0.86 +[2025-04-21 15:00:25] vision_config is None. initializing the InstructBlipVisionConfig with default values. +[2025-04-21 15:00:25] qformer_config is None. Initializing the InstructBlipQFormerConfig with default values. +[2025-04-21 15:00:25] text_config is None. Initializing the text config with default values (`OPTConfig`). +model_name_or_path:/tmp/haozhezhao/model/instructblip-flan-t5-xl +image_place_holder: +tokenizer length after expend 32102 +tokenizer length before expend 32102 +100%|████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████| 3/3 [03:08<00:00, 62.98s/it] +[2025-04-21 15:05:25] Finish Eval in 58000 steps...█████████████████████████████████████████████████████████████████████| 3/3 [03:07<00:00, 62.36s/it] +processing left-padding... +processing left-padding... +processing left-padding... +[2025-04-21 15:05:46] Saved checkpoint to checkpoint/EmptyFix_InstructBlip_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_objects/002-GPT-XL/checkpoints/0058000.pt +[2025-04-21 15:05:48] Removed old checkpoint: checkpoint/EmptyFix_InstructBlip_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_objects/002-GPT-XL/checkpoints/0054000.pt + 42%|██████████████████████████████████████████████▉ | 17121/40903 [6:22:39<7:56:56, 1.20s/it][2025-04-21 15:06:17] (step=0058025) Train Loss: 6.2943, Train Steps/Sec: 0.07 + 42%|██████████████████████████████████████████████▉ | 17146/40903 [6:23:08<7:42:39, 1.17s/it][2025-04-21 15:06:46] (step=0058050) Train Loss: 6.2973, Train Steps/Sec: 0.86 + 42%|███████████████████████████████████████████████ | 17171/40903 [6:23:37<7:33:57, 1.15s/it][2025-04-21 15:07:15] (step=0058075) Train Loss: 6.3070, Train Steps/Sec: 0.87 + 42%|███████████████████████████████████████████████ | 17196/40903 [6:24:05<7:38:30, 1.16s/it][2025-04-21 15:07:44] (step=0058100) Train Loss: 6.2786, Train Steps/Sec: 0.86 + 42%|███████████████████████████████████████████████▏ | 17221/40903 [6:24:35<7:49:13, 1.19s/it][2025-04-21 15:08:13] (step=0058125) Train Loss: 6.3566, Train Steps/Sec: 0.86 + 42%|███████████████████████████████████████████████▏ | 17246/40903 [6:25:04<7:32:38, 1.15s/it][2025-04-21 15:08:42] (step=0058150) Train Loss: 6.3208, Train Steps/Sec: 0.87 + 42%|███████████████████████████████████████████████▎ | 17271/40903 [6:25:33<7:36:38, 1.16s/it][2025-04-21 15:09:11] (step=0058175) Train Loss: 6.3247, Train Steps/Sec: 0.86 + 42%|███████████████████████████████████████████████▎ | 17296/40903 [6:26:02<7:27:03, 1.14s/it][2025-04-21 15:09:40] (step=0058200) Train Loss: 6.2622, Train Steps/Sec: 0.86 + 42%|███████████████████████████████████████████████▍ | 17321/40903 [6:26:31<7:49:09, 1.19s/it][2025-04-21 15:10:09] (step=0058225) Train Loss: 6.3316, Train Steps/Sec: 0.86 + 42%|███████████████████████████████████████████████▍ | 17346/40903 [6:27:00<7:35:41, 1.16s/it][2025-04-21 15:10:38] (step=0058250) Train Loss: 6.3125, Train Steps/Sec: 0.86 + 42%|███████████████████████████████████████████████▌ | 17371/40903 [6:27:29<7:31:42, 1.15s/it][2025-04-21 15:11:07] (step=0058275) Train Loss: 6.2978, Train Steps/Sec: 0.87 + 43%|███████████████████████████████████████████████▋ | 17396/40903 [6:27:58<7:20:08, 1.12s/it][2025-04-21 15:11:37] (step=0058300) Train Loss: 6.2895, Train Steps/Sec: 0.85 + 43%|███████████████████████████████████████████████▋ | 17421/40903 [6:28:27<7:39:27, 1.17s/it][2025-04-21 15:12:06] (step=0058325) Train Loss: 6.3290, Train Steps/Sec: 0.86 + 43%|███████████████████████████████████████████████▊ | 17446/40903 [6:28:56<7:34:33, 1.16s/it][2025-04-21 15:12:35] (step=0058350) Train Loss: 6.3170, Train Steps/Sec: 0.86 + 43%|███████████████████████████████████████████████▊ | 17471/40903 [6:29:25<7:32:09, 1.16s/it][2025-04-21 15:13:04] (step=0058375) Train Loss: 6.3442, Train Steps/Sec: 0.86 + 43%|███████████████████████████████████████████████▉ | 17496/40903 [6:29:54<7:24:43, 1.14s/it][2025-04-21 15:13:33] (step=0058400) Train Loss: 6.3154, Train Steps/Sec: 0.86 + 43%|███████████████████████████████████████████████▉ | 17521/40903 [6:30:23<7:38:39, 1.18s/it][2025-04-21 15:14:02] (step=0058425) Train Loss: 6.3598, Train Steps/Sec: 0.86 + 43%|████████████████████████████████████████████████ | 17546/40903 [6:30:52<7:30:01, 1.16s/it][2025-04-21 15:14:31] (step=0058450) Train Loss: 6.3203, Train Steps/Sec: 0.86 + 43%|████████████████████████████████████████████████ | 17571/40903 [6:31:21<7:27:52, 1.15s/it][2025-04-21 15:15:00] (step=0058475) Train Loss: 6.3095, Train Steps/Sec: 0.86 + 43%|████████████████████████████████████████████████▏ | 17596/40903 [6:31:50<7:26:44, 1.15s/it][2025-04-21 15:15:29] (step=0058500) Train Loss: 6.3282, Train Steps/Sec: 0.86 + 43%|████████████████████████████████████████████████▏ | 17621/40903 [6:32:19<7:27:24, 1.15s/it][2025-04-21 15:15:58] (step=0058525) Train Loss: 6.3115, Train Steps/Sec: 0.86 + 43%|████████████████████████████████████████████████▎ | 17646/40903 [6:32:48<7:30:02, 1.16s/it][2025-04-21 15:16:27] (step=0058550) Train Loss: 6.3015, Train Steps/Sec: 0.86 + 43%|████████████████████████████████████████████████▍ | 17671/40903 [6:33:17<7:22:50, 1.14s/it][2025-04-21 15:16:56] (step=0058575) Train Loss: 6.2804, Train Steps/Sec: 0.87 + 43%|████████████████████████████████████████████████▍ | 17696/40903 [6:33:46<7:23:32, 1.15s/it][2025-04-21 15:17:25] (step=0058600) Train Loss: 6.3549, Train Steps/Sec: 0.86 + 43%|████████████████████████████████████████████████▌ | 17721/40903 [6:34:15<7:38:19, 1.19s/it][2025-04-21 15:17:54] (step=0058625) Train Loss: 6.3253, Train Steps/Sec: 0.86 + 43%|████████████████████████████████████████████████▌ | 17746/40903 [6:34:44<7:25:09, 1.15s/it][2025-04-21 15:18:23] (step=0058650) Train Loss: 6.3115, Train Steps/Sec: 0.86 + 43%|████████████████████████████████████████████████▋ | 17771/40903 [6:35:13<7:22:28, 1.15s/it][2025-04-21 15:18:52] (step=0058675) Train Loss: 6.3172, Train Steps/Sec: 0.87 + 44%|████████████████████████████████████████████████▋ | 17796/40903 [6:35:42<7:26:23, 1.16s/it][2025-04-21 15:19:21] (step=0058700) Train Loss: 6.3260, Train Steps/Sec: 0.86 + 44%|████████████████████████████████████████████████▊ | 17821/40903 [6:36:11<7:35:41, 1.18s/it][2025-04-21 15:19:50] (step=0058725) Train Loss: 6.3166, Train Steps/Sec: 0.86 + 44%|████████████████████████████████████████████████▊ | 17846/40903 [6:36:40<7:22:45, 1.15s/it][2025-04-21 15:20:19] (step=0058750) Train Loss: 6.2932, Train Steps/Sec: 0.87 + 44%|████████████████████████████████████████████████▉ | 17871/40903 [6:37:09<7:25:42, 1.16s/it][2025-04-21 15:20:48] (step=0058775) Train Loss: 6.3144, Train Steps/Sec: 0.86 + 44%|█████████████████████████████████████████████████ | 17896/40903 [6:37:38<7:14:23, 1.13s/it][2025-04-21 15:21:17] (step=0058800) Train Loss: 6.2687, Train Steps/Sec: 0.86 + 44%|████████████████████████████████████████████████▋ | 17921/40903 [6:38:14<21:15:30, 3.33s/it][2025-04-21 15:21:53] (step=0058825) Train Loss: 6.3394, Train Steps/Sec: 0.69 + 44%|█████████████████████████████████████████████████▏ | 17946/40903 [6:38:43<7:24:08, 1.16s/it][2025-04-21 15:22:22] (step=0058850) Train Loss: 6.2880, Train Steps/Sec: 0.87 + 44%|█████████████████████████████████████████████████▏ | 17971/40903 [6:39:12<7:18:05, 1.15s/it][2025-04-21 15:22:50] (step=0058875) Train Loss: 6.3287, Train Steps/Sec: 0.86 + 44%|█████████████████████████████████████████████████▎ | 17996/40903 [6:39:41<7:17:49, 1.15s/it][2025-04-21 15:23:20] (step=0058900) Train Loss: 6.3241, Train Steps/Sec: 0.86 + 44%|█████████████████████████████████████████████████▎ | 18021/40903 [6:40:10<7:23:32, 1.16s/it][2025-04-21 15:23:49] (step=0058925) Train Loss: 6.3269, Train Steps/Sec: 0.86 + 44%|█████████████████████████████████████████████████▍ | 18046/40903 [6:40:39<7:23:50, 1.17s/it][2025-04-21 15:24:18] (step=0058950) Train Loss: 6.3794, Train Steps/Sec: 0.86 + 44%|█████████████████████████████████████████████████ | 18071/40903 [6:41:15<16:33:00, 2.61s/it][2025-04-21 15:24:54] (step=0058975) Train Loss: 6.3080, Train Steps/Sec: 0.70 + 44%|█████████████████████████████████████████████████▌ | 18096/40903 [6:41:44<7:22:38, 1.16s/it][2025-04-21 15:25:23] (step=0059000) Train Loss: 6.3446, Train Steps/Sec: 0.86 + 44%|█████████████████████████████████████████████████▌ | 18121/40903 [6:42:13<7:30:55, 1.19s/it][2025-04-21 15:25:52] (step=0059025) Train Loss: 6.3452, Train Steps/Sec: 0.86 + 44%|█████████████████████████████████████████████████▋ | 18146/40903 [6:42:42<7:21:23, 1.16s/it][2025-04-21 15:26:21] (step=0059050) Train Loss: 6.3407, Train Steps/Sec: 0.86 + 44%|█████████████████████████████████████████████████▊ | 18171/40903 [6:43:11<7:15:50, 1.15s/it][2025-04-21 15:26:50] (step=0059075) Train Loss: 6.2928, Train Steps/Sec: 0.86 + 44%|█████████████████████████████████████████████████▊ | 18196/40903 [6:43:40<7:15:31, 1.15s/it][2025-04-21 15:27:19] (step=0059100) Train Loss: 6.3278, Train Steps/Sec: 0.86 + 45%|█████████████████████████████████████████████████▉ | 18221/40903 [6:44:09<7:22:12, 1.17s/it][2025-04-21 15:27:48] (step=0059125) Train Loss: 6.3246, Train Steps/Sec: 0.87 + 45%|█████████████████████████████████████████████████▉ | 18246/40903 [6:44:38<7:23:18, 1.17s/it][2025-04-21 15:28:17] (step=0059150) Train Loss: 6.3073, Train Steps/Sec: 0.86 + 45%|██████████████████████████████████████████████████ | 18271/40903 [6:45:07<7:12:52, 1.15s/it][2025-04-21 15:28:46] (step=0059175) Train Loss: 6.3117, Train Steps/Sec: 0.86 + 45%|██████████████████████████████████████████████████ | 18296/40903 [6:45:36<7:15:40, 1.16s/it][2025-04-21 15:29:15] (step=0059200) Train Loss: 6.3037, Train Steps/Sec: 0.86 + 45%|██████████████████████████████████████████████████▏ | 18321/40903 [6:46:05<7:22:18, 1.18s/it][2025-04-21 15:29:44] (step=0059225) Train Loss: 6.3290, Train Steps/Sec: 0.86 + 45%|██████████████████████████████████████████████████▏ | 18346/40903 [6:46:34<7:19:27, 1.17s/it][2025-04-21 15:30:13] (step=0059250) Train Loss: 6.3048, Train Steps/Sec: 0.86 + 45%|██████████████████████████████████████████████████▎ | 18371/40903 [6:47:03<7:09:33, 1.14s/it][2025-04-21 15:30:42] (step=0059275) Train Loss: 6.3318, Train Steps/Sec: 0.86 + 45%|██████████████████████████████████████████████████▎ | 18396/40903 [6:47:33<7:16:36, 1.16s/it][2025-04-21 15:31:11] (step=0059300) Train Loss: 6.2981, Train Steps/Sec: 0.85 + 45%|██████████████████████████████████████████████████▍ | 18421/40903 [6:48:02<7:23:41, 1.18s/it][2025-04-21 15:31:40] (step=0059325) Train Loss: 6.2948, Train Steps/Sec: 0.86 + 45%|██████████████████████████████████████████████████▌ | 18446/40903 [6:48:31<7:14:11, 1.16s/it][2025-04-21 15:32:09] (step=0059350) Train Loss: 6.2735, Train Steps/Sec: 0.87 + 45%|██████████████████████████████████████████████████▌ | 18471/40903 [6:49:00<7:09:42, 1.15s/it][2025-04-21 15:32:38] (step=0059375) Train Loss: 6.3541, Train Steps/Sec: 0.86 + 45%|██████████████████████████████████████████████████▋ | 18496/40903 [6:49:29<7:15:07, 1.17s/it][2025-04-21 15:33:07] (step=0059400) Train Loss: 6.3715, Train Steps/Sec: 0.86 + 45%|██████████████████████████████████████████████████▋ | 18521/40903 [6:49:58<7:18:26, 1.18s/it][2025-04-21 15:33:36] (step=0059425) Train Loss: 6.3332, Train Steps/Sec: 0.86 + 45%|██████████████████████████████████████████████████▊ | 18546/40903 [6:50:33<7:11:15, 1.16s/it][2025-04-21 15:34:11] (step=0059450) Train Loss: 6.3060, Train Steps/Sec: 0.71 + 45%|██████████████████████████████████████████████████▊ | 18571/40903 [6:51:02<7:10:25, 1.16s/it][2025-04-21 15:34:40] (step=0059475) Train Loss: 6.3299, Train Steps/Sec: 0.86 + 45%|██████████████████████████████████████████████████▉ | 18596/40903 [6:51:31<7:04:33, 1.14s/it][2025-04-21 15:35:09] (step=0059500) Train Loss: 6.3614, Train Steps/Sec: 0.85 + 46%|██████████████████████████████████████████████████▉ | 18621/40903 [6:52:00<7:18:26, 1.18s/it][2025-04-21 15:35:38] (step=0059525) Train Loss: 6.3147, Train Steps/Sec: 0.87 + 46%|███████████████████████████████████████████████████ | 18646/40903 [6:52:29<7:10:49, 1.16s/it][2025-04-21 15:36:07] (step=0059550) Train Loss: 6.3884, Train Steps/Sec: 0.86 + 46%|███████████████████████████████████████████████████ | 18671/40903 [6:52:58<7:09:07, 1.16s/it][2025-04-21 15:36:36] (step=0059575) Train Loss: 6.3023, Train Steps/Sec: 0.86 + 46%|███████████████████████████████████████████████████▏ | 18696/40903 [6:53:27<7:07:10, 1.15s/it][2025-04-21 15:37:06] (step=0059600) Train Loss: 6.3623, Train Steps/Sec: 0.85 + 46%|███████████████████████████████████████████████████▎ | 18721/40903 [6:53:56<7:15:21, 1.18s/it][2025-04-21 15:37:35] (step=0059625) Train Loss: 6.3213, Train Steps/Sec: 0.86 + 46%|███████████████████████████████████████████████████▎ | 18746/40903 [6:54:25<7:13:46, 1.17s/it][2025-04-21 15:38:04] (step=0059650) Train Loss: 6.3028, Train Steps/Sec: 0.86 + 46%|███████████████████████████████████████████████████▍ | 18771/40903 [6:54:54<7:07:44, 1.16s/it][2025-04-21 15:38:33] (step=0059675) Train Loss: 6.2971, Train Steps/Sec: 0.86 + 46%|███████████████████████████████████████████████████▍ | 18796/40903 [6:55:23<7:03:19, 1.15s/it][2025-04-21 15:39:02] (step=0059700) Train Loss: 6.2909, Train Steps/Sec: 0.86 + 46%|███████████████████████████████████████████████████▌ | 18821/40903 [6:55:52<7:07:51, 1.16s/it][2025-04-21 15:39:31] (step=0059725) Train Loss: 6.2699, Train Steps/Sec: 0.87 + 46%|███████████████████████████████████████████████████▌ | 18846/40903 [6:56:21<7:08:42, 1.17s/it][2025-04-21 15:40:00] (step=0059750) Train Loss: 6.3094, Train Steps/Sec: 0.87 + 46%|███████████████████████████████████████████████████▋ | 18871/40903 [6:56:50<7:04:46, 1.16s/it][2025-04-21 15:40:29] (step=0059775) Train Loss: 6.3392, Train Steps/Sec: 0.86 + 46%|███████████████████████████████████████████████████▋ | 18896/40903 [6:57:19<7:04:37, 1.16s/it][2025-04-21 15:40:58] (step=0059800) Train Loss: 6.2904, Train Steps/Sec: 0.86 + 46%|███████████████████████████████████████████████████▊ | 18921/40903 [6:57:48<7:12:03, 1.18s/it][2025-04-21 15:41:27] (step=0059825) Train Loss: 6.2817, Train Steps/Sec: 0.86 + 46%|███████████████████████████████████████████████████▉ | 18946/40903 [6:58:17<7:09:53, 1.17s/it][2025-04-21 15:41:56] (step=0059850) Train Loss: 6.3173, Train Steps/Sec: 0.86 + 46%|███████████████████████████████████████████████████▉ | 18971/40903 [6:58:46<6:59:22, 1.15s/it][2025-04-21 15:42:25] (step=0059875) Train Loss: 6.3231, Train Steps/Sec: 0.87 + 46%|████████████████████████████████████████████████████ | 18996/40903 [6:59:15<6:57:07, 1.14s/it][2025-04-21 15:42:54] (step=0059900) Train Loss: 6.2935, Train Steps/Sec: 0.86 + 47%|████████████████████████████████████████████████████ | 19021/40903 [6:59:44<7:04:15, 1.16s/it][2025-04-21 15:43:23] (step=0059925) Train Loss: 6.3229, Train Steps/Sec: 0.86 + 47%|████████████████████████████████████████████████████▏ | 19046/40903 [7:00:13<7:06:12, 1.17s/it][2025-04-21 15:43:52] (step=0059950) Train Loss: 6.3369, Train Steps/Sec: 0.86 + 47%|████████████████████████████████████████████████████▏ | 19071/40903 [7:00:43<7:04:01, 1.17s/it][2025-04-21 15:44:21] (step=0059975) Train Loss: 6.3329, Train Steps/Sec: 0.86 + 47%|████████████████████████████████████████████████████▎ | 19096/40903 [7:01:12<6:53:20, 1.14s/it][2025-04-21 15:44:50] (step=0060000) Train Loss: 6.3256, Train Steps/Sec: 0.86 +[2025-04-21 15:44:50] vision_config is None. initializing the InstructBlipVisionConfig with default values. +[2025-04-21 15:44:50] qformer_config is None. Initializing the InstructBlipQFormerConfig with default values. +[2025-04-21 15:44:50] text_config is None. Initializing the text config with default values (`OPTConfig`). +model_name_or_path:/tmp/haozhezhao/model/instructblip-flan-t5-xl +image_place_holder: +tokenizer length after expend 32102 +tokenizer length before expend 32102 +100%|████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████| 3/3 [03:10<00:00, 63.37s/it] +[2025-04-21 15:49:51] Finish Eval in 60000 steps...█████████████████████████████████████████████████████████████████████| 3/3 [03:09<00:00, 62.74s/it] +processing left-padding... +processing left-padding... +processing left-padding... +[2025-04-21 15:50:13] Saved checkpoint to checkpoint/EmptyFix_InstructBlip_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_objects/002-GPT-XL/checkpoints/0060000.pt +[2025-04-21 15:50:15] Removed old checkpoint: checkpoint/EmptyFix_InstructBlip_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_objects/002-GPT-XL/checkpoints/0056000.pt + 47%|████████████████████████████████████████████████████▎ | 19121/40903 [7:07:05<7:15:32, 1.20s/it][2025-04-21 15:50:43] (step=0060025) Train Loss: 6.3152, Train Steps/Sec: 0.07 + 47%|████████████████████████████████████████████████████▍ | 19146/40903 [7:07:34<7:06:36, 1.18s/it][2025-04-21 15:51:12] (step=0060050) Train Loss: 6.2679, Train Steps/Sec: 0.86 + 47%|████████████████████████████████████████████████████▍ | 19171/40903 [7:08:03<7:05:11, 1.17s/it][2025-04-21 15:51:41] (step=0060075) Train Loss: 6.3155, Train Steps/Sec: 0.86 + 47%|████████████████████████████████████████████████████▌ | 19196/40903 [7:08:32<6:52:19, 1.14s/it][2025-04-21 15:52:10] (step=0060100) Train Loss: 6.3024, Train Steps/Sec: 0.86 + 47%|████████████████████████████████████████████████████▋ | 19221/40903 [7:09:08<7:11:51, 1.20s/it][2025-04-21 15:52:46] (step=0060125) Train Loss: 6.3491, Train Steps/Sec: 0.70 + 47%|████████████████████████████████████████████████████▋ | 19246/40903 [7:09:37<6:56:28, 1.15s/it][2025-04-21 15:53:21] (step=0060150) Train Loss: 6.3295, Train Steps/Sec: 0.72 + 47%|████████████████████████████████████████████████████▊ | 19271/40903 [7:10:12<6:55:22, 1.15s/it][2025-04-21 15:53:50] (step=0060175) Train Loss: 6.3198, Train Steps/Sec: 0.86 + 47%|████████████████████████████████████████████████████▊ | 19296/40903 [7:10:41<6:52:50, 1.15s/it][2025-04-21 15:54:19] (step=0060200) Train Loss: 6.3365, Train Steps/Sec: 0.86 + 47%|████████████████████████████████████████████████████▉ | 19321/40903 [7:11:10<6:57:55, 1.16s/it][2025-04-21 15:54:48] (step=0060225) Train Loss: 6.3078, Train Steps/Sec: 0.86 + 47%|████████████████████████████████████████████████████▉ | 19346/40903 [7:11:39<6:54:21, 1.15s/it][2025-04-21 15:55:17] (step=0060250) Train Loss: 6.3634, Train Steps/Sec: 0.87 + 47%|█████████████████████████████████████████████████████ | 19371/40903 [7:12:07<6:53:59, 1.15s/it][2025-04-21 15:55:46] (step=0060275) Train Loss: 6.3078, Train Steps/Sec: 0.87 + 47%|█████████████████████████████████████████████████████ | 19396/40903 [7:12:36<6:52:50, 1.15s/it][2025-04-21 15:56:15] (step=0060300) Train Loss: 6.3207, Train Steps/Sec: 0.86 + 47%|█████████████████████████████████████████████████████▏ | 19421/40903 [7:13:06<7:03:51, 1.18s/it][2025-04-21 15:56:44] (step=0060325) Train Loss: 6.3432, Train Steps/Sec: 0.86 + 48%|█████████████████████████████████████████████████████▏ | 19446/40903 [7:13:34<6:55:05, 1.16s/it][2025-04-21 15:57:13] (step=0060350) Train Loss: 6.2936, Train Steps/Sec: 0.87 + 48%|█████████████████████████████████████████████████████▎ | 19471/40903 [7:14:04<6:58:32, 1.17s/it][2025-04-21 15:57:42] (step=0060375) Train Loss: 6.3447, Train Steps/Sec: 0.86 + 48%|█████████████████████████████████████████████████████▍ | 19496/40903 [7:14:33<6:55:55, 1.17s/it][2025-04-21 15:58:11] (step=0060400) Train Loss: 6.3580, Train Steps/Sec: 0.85 + 48%|█████████████████████████████████████████████████████▍ | 19521/40903 [7:15:09<7:01:36, 1.18s/it][2025-04-21 15:58:47] (step=0060425) Train Loss: 6.3166, Train Steps/Sec: 0.70 + 48%|█████████████████████████████████████████████████████▌ | 19546/40903 [7:15:38<6:55:30, 1.17s/it][2025-04-21 15:59:16] (step=0060450) Train Loss: 6.2779, Train Steps/Sec: 0.86 + 48%|█████████████████████████████████████████████████████▌ | 19571/40903 [7:16:07<6:49:40, 1.15s/it][2025-04-21 15:59:45] (step=0060475) Train Loss: 6.3291, Train Steps/Sec: 0.86 + 48%|█████████████████████████████████████████████████████▋ | 19596/40903 [7:16:36<6:45:49, 1.14s/it][2025-04-21 16:00:14] (step=0060500) Train Loss: 6.2802, Train Steps/Sec: 0.86 + 48%|█████████████████████████████████████████████████████▋ | 19621/40903 [7:17:05<6:57:20, 1.18s/it][2025-04-21 16:00:43] (step=0060525) Train Loss: 6.2887, Train Steps/Sec: 0.86 + 48%|█████████████████████████████████████████████████████▊ | 19646/40903 [7:17:34<6:49:07, 1.15s/it][2025-04-21 16:01:12] (step=0060550) Train Loss: 6.3375, Train Steps/Sec: 0.87 + 48%|█████████████████████████████████████████████████████▊ | 19671/40903 [7:18:02<6:46:51, 1.15s/it][2025-04-21 16:01:41] (step=0060575) Train Loss: 6.3152, Train Steps/Sec: 0.87 + 48%|█████████████████████████████████████████████████████▉ | 19696/40903 [7:18:31<6:43:30, 1.14s/it][2025-04-21 16:02:10] (step=0060600) Train Loss: 6.3254, Train Steps/Sec: 0.86 + 48%|█████████████████████████████████████████████████████▉ | 19721/40903 [7:19:00<6:53:48, 1.17s/it][2025-04-21 16:02:39] (step=0060625) Train Loss: 6.2551, Train Steps/Sec: 0.87 + 48%|██████████████████████████████████████████████████████ | 19746/40903 [7:19:29<6:52:09, 1.17s/it][2025-04-21 16:03:08] (step=0060650) Train Loss: 6.3456, Train Steps/Sec: 0.86 + 48%|██████████████████████████████████████████████████████▏ | 19771/40903 [7:19:58<6:44:47, 1.15s/it][2025-04-21 16:03:37] (step=0060675) Train Loss: 6.3353, Train Steps/Sec: 0.86 + 48%|██████████████████████████████████████████████████████▏ | 19796/40903 [7:20:28<6:43:35, 1.15s/it][2025-04-21 16:04:06] (step=0060700) Train Loss: 6.3289, Train Steps/Sec: 0.86 + 48%|██████████████████████████████████████████████████████▎ | 19821/40903 [7:20:57<6:55:33, 1.18s/it][2025-04-21 16:04:35] (step=0060725) Train Loss: 6.3616, Train Steps/Sec: 0.86 + 48%|██████████████████████████████████████████████████████▎ | 19834/40903 [7:21:12<6:51:39, 1.17s/it] diff --git a/CKPTS/EmptyFix_InstructBlip_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_objects/wandb/run-20250420_165633-wzvwagyn/files/requirements.txt b/CKPTS/EmptyFix_InstructBlip_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_objects/wandb/run-20250420_165633-wzvwagyn/files/requirements.txt new file mode 100644 index 0000000000000000000000000000000000000000..06dc78369ffff807b210006a0e79d705ffe2a7d7 --- /dev/null +++ b/CKPTS/EmptyFix_InstructBlip_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_objects/wandb/run-20250420_165633-wzvwagyn/files/requirements.txt @@ -0,0 +1,131 @@ +typing_extensions==4.12.2 +pyzmq==26.3.0 +nvidia-cufft-cu12==11.0.2.54 +triton==3.1.0 +nvidia-cublas-cu12==12.1.3.1 +psutil==7.0.0 +nvidia-cuda-cupti-cu12==12.1.105 +smmap==5.0.2 +nvidia-cuda-runtime-cu12==12.1.105 +aiohappyeyeballs==2.6.1 +asttokens==3.0.0 +huggingface-hub==0.29.3 +pyarrow==19.0.1 +fonttools==4.56.0 +python-dateutil==2.9.0.post0 +GitPython==3.1.44 +aiohttp==3.11.14 +wandb==0.19.8 +setproctitle==1.3.5 +PyYAML==6.0.2 +pydantic_core==2.27.2 +safetensors==0.5.3 +nvidia-nvjitlink-cu12==12.1.105 +aiosignal==1.3.2 +dill==0.3.8 +nvidia-cuda-nvrtc-cu12==12.1.105 +multiprocess==0.70.16 +pure_eval==0.2.3 +stack_data==0.6.3 +pydantic==2.10.6 +MarkupSafe==2.1.5 +tornado==6.4.2 +executing==2.1.0 +executing==2.2.0 +opencv-python==4.11.0.86 +nvitop==1.4.2 +multidict==6.2.0 +Jinja2==3.1.4 +torch==2.5.1+cu121 +nvidia-curand-cu12==10.3.2.106 +platformdirs==4.3.6 +six==1.17.0 +mpmath==1.3.0 +zipp==3.21.0 +packaging==24.2 +requests==2.32.3 +certifi==2025.1.31 +docker-pycreds==0.4.0 +torchvision==0.20.1+cu121 +pandas==2.2.3 +networkx==3.3 +exceptiongroup==1.2.2 +pickleshare==0.7.5 +tokenizers==0.21.1 +charset-normalizer==3.4.1 +jupyter_core==5.7.2 +wcwidth==0.2.13 +nvidia-nvtx-cu12==12.1.105 +prompt_toolkit==3.0.50 +fsspec==2024.12.0 +pillow==11.1.0 +propcache==0.3.0 +regex==2024.11.6 +ptyprocess==0.7.0 +contourpy==1.3.1 +importlib_metadata==8.6.1 +idna==3.10 +comm==0.2.2 +protobuf==5.29.3 +yarl==1.18.3 +ipython_pygments_lexers==1.1.1 +pip==25.0 +parso==0.8.4 +joblib==1.4.2 +nvidia-nccl-cu12==2.21.5 +hf_transfer==0.1.9 +Pygments==2.19.1 +decorator==5.2.1 +filelock==3.18.0 +nvidia-cusparse-cu12==12.1.0.106 +debugpy==1.8.13 +urllib3==2.3.0 +traitlets==5.14.3 +tzdata==2025.1 +matplotlib-inline==0.1.7 +matplotlib==3.10.1 +kiwisolver==1.4.8 +nest_asyncio==1.6.0 +frozenlist==1.5.0 +nvidia-ml-py==12.570.86 +transformers==4.49.0 +nltk==3.9.1 +ipykernel==6.29.5 +click==8.1.8 +gitdb==4.0.12 +pyparsing==3.2.1 +attrs==25.3.0 +jedi==0.19.2 +ipython==9.0.2 +nvidia-cudnn-cu12==9.1.0.70 +pexpect==4.9.0 +nvidia-cusolver-cu12==11.4.5.107 +numpy==2.2.4 +tqdm==4.67.1 +pytz==2025.1 +wheel==0.45.1 +sentry-sdk==2.23.1 +torchaudio==2.5.1+cu121 +jupyter_client==8.6.3 +cycler==0.12.1 +annotated-types==0.7.0 +sympy==1.13.1 +xxhash==3.5.0 +datasets==3.4.1 +setuptools==75.8.0 +typing_extensions==4.12.2 +wheel==0.43.0 +importlib_metadata==8.0.0 +backports.tarfile==1.2.0 +autocommand==2.2.2 +packaging==24.2 +tomli==2.0.1 +typeguard==4.3.0 +zipp==3.19.2 +jaraco.context==5.3.0 +jaraco.functools==4.0.1 +more-itertools==10.3.0 +platformdirs==4.2.2 +jaraco.text==3.12.1 +jaraco.collections==5.1.0 +inflect==7.3.1 diff --git a/CKPTS/EmptyFix_InstructBlip_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_objects/wandb/run-20250420_165633-wzvwagyn/files/wandb-metadata.json b/CKPTS/EmptyFix_InstructBlip_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_objects/wandb/run-20250420_165633-wzvwagyn/files/wandb-metadata.json new file mode 100644 index 0000000000000000000000000000000000000000..0d0d39675746853cbab1b0e8c179f8d2b0fe3a46 --- /dev/null +++ b/CKPTS/EmptyFix_InstructBlip_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_objects/wandb/run-20250420_165633-wzvwagyn/files/wandb-metadata.json @@ -0,0 +1,142 @@ +{ + "os": "Linux-5.15.0-1064-azure-x86_64-with-glibc2.31", + "python": "CPython 3.11.11", + "startedAt": "2025-04-20T16:56:33.401938Z", + "args": [ + "--vq-ckpt", + "/tmp/haozhezhao/model/llamagen_t2i/vq_ds16_t2i.pt", + "--data-path", + "/tmp/haozhezhao/MLLMG/jsonl_data/multiobjects_molom_imagenet_flux_qwen_midsource_gen_2_2m_trained.jsonl", + "--dataset", + "ti2i", + "--image-size", + "512", + "--results-dir", + "checkpoint/EmptyFix_InstructBlip_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_objects", + "--cloud-save-path", + "/tmp/haozhezhao/MLLMG/checkpoint", + "--lr", + "5e-5", + "--val_data_path", + "/tmp/haozhezhao/MLLMG/jsonl_data/multiobjects_molom_imagenet_flux_qwen_midsource_gen_val.jsonl", + "--use_vision_tower", + "--model_name_or_path", + "/tmp/haozhezhao/model/instructblip-flan-t5-xl", + "--image_place_holder", + "", + "--do_eval", + "--eval_steps", + "2000", + "--max_eval_samples", + "200", + "--cfg-scale", + "7.5", + "--top-k", + "16384", + "--load_from_checkpoint", + "/tmp/haozhezhao/MLLMG/checkpoint/FIXed_3M_1epoch_step32000.pt", + "--global-batch-size", + "56", + "--num-workers", + "4", + "--warmup", + "0.05", + "--gradient-accumulation-steps", + "4", + "--train_text_encoder", + "--ckpt-every", + "2000", + "--epochs", + "2", + "--subject_driven", + "--reference_data_path", + "/tmp/haozhezhao/MLLMG/cc12m_reference_tunnel.jsonl", + "--multimodal_encoder", + "instructblip", + "--do_recovery", + "--find_unused_parameters", + "--cls-token-num", + "512", + "--train_all", + "--load_fixed_llamagen", + "--fix", + "gpt-empty-fix", + "--gpt-ckpt", + "/tmp/haozhezhao/MLLMG/checkpoint/EmptyFix_InstructBlip_just_segment_pretrain_stage3_dreambench_recap_Subject400k_filtered_t2i_flux400k_200kmid_recovery_150k_extract_150k_100_fluxseg_50samseg_trainall_1e4_no_replace/007-GPT-XL/checkpoints/0092000.pt" + ], + "program": "/tmp/haozhezhao/MLLMG/autoregressive/train/train_t2i.py", + "codePath": "autoregressive/train/train_t2i.py", + "email": "mimazhe55360@gmail.com", + "root": "checkpoint/EmptyFix_InstructBlip_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_objects", + "host": "447cc403a8794092814259713c51c1df00001X", + "executable": "/tmp/haozhezhao/anaconda3/envs/nlp/bin/python", + "codePathLocal": "autoregressive/train/train_t2i.py", + "cpu_count": 96, + "cpu_count_logical": 96, + "gpu": "NVIDIA A100-SXM4-80GB", + "gpu_count": 8, + "disk": { + "/": { + "total": "133003395072", + "used": "54909009920" + } + }, + "memory": { + "total": "1902387884032" + }, + "cpu": { + "count": 96, + "countLogical": 96 + }, + "gpu_nvidia": [ + { + "name": "NVIDIA A100-SXM4-80GB", + "memoryTotal": "85899345920", + "cudaCores": 6912, + "architecture": "Ampere" + }, + { + "name": "NVIDIA A100-SXM4-80GB", + "memoryTotal": "85899345920", + "cudaCores": 6912, + "architecture": "Ampere" + }, + { + "name": "NVIDIA A100-SXM4-80GB", + "memoryTotal": "85899345920", + "cudaCores": 6912, + "architecture": "Ampere" + }, + { + "name": "NVIDIA A100-SXM4-80GB", + "memoryTotal": "85899345920", + "cudaCores": 6912, + "architecture": "Ampere" + }, + { + "name": "NVIDIA A100-SXM4-80GB", + "memoryTotal": "85899345920", + "cudaCores": 6912, + "architecture": "Ampere" + }, + { + "name": "NVIDIA A100-SXM4-80GB", + "memoryTotal": "85899345920", + "cudaCores": 6912, + "architecture": "Ampere" + }, + { + "name": "NVIDIA A100-SXM4-80GB", + "memoryTotal": "85899345920", + "cudaCores": 6912, + "architecture": "Ampere" + }, + { + "name": "NVIDIA A100-SXM4-80GB", + "memoryTotal": "85899345920", + "cudaCores": 6912, + "architecture": "Ampere" + } + ], + "cudaVersion": "12.2" +} \ No newline at end of file diff --git a/CKPTS/EmptyFix_InstructBlip_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_objects/wandb/run-20250420_165633-wzvwagyn/logs/debug-core.log b/CKPTS/EmptyFix_InstructBlip_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_objects/wandb/run-20250420_165633-wzvwagyn/logs/debug-core.log new file mode 100644 index 0000000000000000000000000000000000000000..b5ec9ae56f7f6436bb39158eec54207c3a701294 --- /dev/null +++ b/CKPTS/EmptyFix_InstructBlip_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_objects/wandb/run-20250420_165633-wzvwagyn/logs/debug-core.log @@ -0,0 +1,7 @@ +{"time":"2025-04-20T16:56:32.675835693Z","level":"INFO","msg":"main: starting server","port-filename":"/tmp/tmp5qehytl9/port-2696278.txt","pid":2696278,"log-level":0,"disable-analytics":false,"shutdown-on-parent-exit":false} +{"time":"2025-04-20T16:56:32.676919588Z","level":"INFO","msg":"Will exit if parent process dies.","ppid":2696278} +{"time":"2025-04-20T16:56:32.676842969Z","level":"INFO","msg":"server is running","addr":{"IP":"127.0.0.1","Port":38411,"Zone":""}} +{"time":"2025-04-20T16:56:32.861731843Z","level":"INFO","msg":"connection: ManageConnectionData: new connection created","id":"127.0.0.1:58238"} +{"time":"2025-04-20T16:56:33.403993658Z","level":"INFO","msg":"handleInformInit: received","streamId":"wzvwagyn","id":"127.0.0.1:58238"} +{"time":"2025-04-20T16:56:33.612574526Z","level":"INFO","msg":"handleInformInit: stream started","streamId":"wzvwagyn","id":"127.0.0.1:58238"} +{"time":"2025-04-21T16:04:54.143654649Z","level":"INFO","msg":"Parent process exited, terminating service process."} diff --git a/CKPTS/EmptyFix_InstructBlip_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_objects/wandb/run-20250420_165633-wzvwagyn/logs/debug-internal.log b/CKPTS/EmptyFix_InstructBlip_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_objects/wandb/run-20250420_165633-wzvwagyn/logs/debug-internal.log new file mode 100644 index 0000000000000000000000000000000000000000..74dc30b07844c3579e969ff38961c446e5e4d1ac --- /dev/null +++ b/CKPTS/EmptyFix_InstructBlip_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_objects/wandb/run-20250420_165633-wzvwagyn/logs/debug-internal.log @@ -0,0 +1,7 @@ +{"time":"2025-04-20T16:56:33.404420738Z","level":"INFO","msg":"stream: starting","core version":"0.19.8","symlink path":"checkpoint/EmptyFix_InstructBlip_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_objects/wandb/run-20250420_165633-wzvwagyn/logs/debug-core.log"} +{"time":"2025-04-20T16:56:33.612529261Z","level":"INFO","msg":"created new stream","id":"wzvwagyn"} +{"time":"2025-04-20T16:56:33.612568585Z","level":"INFO","msg":"stream: started","id":"wzvwagyn"} +{"time":"2025-04-20T16:56:33.612590145Z","level":"INFO","msg":"writer: Do: started","stream_id":"wzvwagyn"} +{"time":"2025-04-20T16:56:33.612631518Z","level":"INFO","msg":"sender: started","stream_id":"wzvwagyn"} +{"time":"2025-04-20T16:56:33.612642178Z","level":"INFO","msg":"handler: started","stream_id":"wzvwagyn"} +{"time":"2025-04-20T16:56:34.210323838Z","level":"INFO","msg":"Starting system monitor"} diff --git a/CKPTS/EmptyFix_InstructBlip_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_objects/wandb/run-20250420_165633-wzvwagyn/logs/debug.log b/CKPTS/EmptyFix_InstructBlip_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_objects/wandb/run-20250420_165633-wzvwagyn/logs/debug.log new file mode 100644 index 0000000000000000000000000000000000000000..415d8bc1826ba5ebf0c507c7de3e27bbd188809c --- /dev/null +++ b/CKPTS/EmptyFix_InstructBlip_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_objects/wandb/run-20250420_165633-wzvwagyn/logs/debug.log @@ -0,0 +1,22 @@ +2025-04-20 16:56:33,394 INFO MainThread:2696278 [wandb_setup.py:_flush():67] Current SDK version is 0.19.8 +2025-04-20 16:56:33,394 INFO MainThread:2696278 [wandb_setup.py:_flush():67] Configure stats pid to 2696278 +2025-04-20 16:56:33,395 INFO MainThread:2696278 [wandb_setup.py:_flush():67] Loading settings from /tmp/haozhezhao/.config/wandb/settings +2025-04-20 16:56:33,395 INFO MainThread:2696278 [wandb_setup.py:_flush():67] Loading settings from /tmp/haozhezhao/MLLMG/wandb/settings +2025-04-20 16:56:33,395 INFO MainThread:2696278 [wandb_setup.py:_flush():67] Loading settings from environment variables +2025-04-20 16:56:33,395 INFO MainThread:2696278 [wandb_init.py:setup_run_log_directory():647] Logging user logs to checkpoint/EmptyFix_InstructBlip_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_objects/wandb/run-20250420_165633-wzvwagyn/logs/debug.log +2025-04-20 16:56:33,395 INFO MainThread:2696278 [wandb_init.py:setup_run_log_directory():648] Logging internal logs to checkpoint/EmptyFix_InstructBlip_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_objects/wandb/run-20250420_165633-wzvwagyn/logs/debug-internal.log +2025-04-20 16:56:33,395 INFO MainThread:2696278 [wandb_init.py:init():761] calling init triggers +2025-04-20 16:56:33,395 INFO MainThread:2696278 [wandb_init.py:init():766] wandb.init called with sweep_config: {} +config: {'data_path': '/tmp/haozhezhao/MLLMG/jsonl_data/multiobjects_molom_imagenet_flux_qwen_midsource_gen_2_2m_trained.jsonl', 'cloud_save_path': '/tmp/haozhezhao/MLLMG/checkpoint', 'no_local_save': False, 'vq_model': 'VQ-16', 'vq_ckpt': '/tmp/haozhezhao/model/llamagen_t2i/vq_ds16_t2i.pt', 'codebook_size': 16384, 'codebook_embed_dim': 8, 'gpt_model': 'GPT-XL', 'gpt_ckpt': '/tmp/haozhezhao/MLLMG/checkpoint/EmptyFix_InstructBlip_just_segment_pretrain_stage3_dreambench_recap_Subject400k_filtered_t2i_flux400k_200kmid_recovery_150k_extract_150k_100_fluxseg_50samseg_trainall_1e4_no_replace/007-GPT-XL/checkpoints/0092000.pt', 'gpt_type': 't2i', 'vocab_size': 16384, 'cls_token_num': 512, 'dropout_p': 0.1, 'token_dropout_p': 0.1, 'drop_path': 0.0, 'no_compile': False, 'results_dir': 'checkpoint/EmptyFix_InstructBlip_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_objects', 'dataset': 'ti2i', 'image_size': 512, 'downsample_size': 16, 'num_classes': 1000, 'epochs': 2, 'lr': 5e-05, 'weight_decay': 0.05, 'beta1': 0.9, 'beta2': 0.95, 'max_grad_norm': 1.0, 'global_batch_size': 56, 'global_seed': 0, 'num_workers': 4, 'log_every': 25, 'ckpt_every': 2000, 'gradient_accumulation_steps': 4, 'mixed_precision': 'bf16', 'val_data_path': '/tmp/haozhezhao/MLLMG/jsonl_data/multiobjects_molom_imagenet_flux_qwen_midsource_gen_val.jsonl', 'use_vision_tower': True, 'model_name_or_path': '/tmp/haozhezhao/model/instructblip-flan-t5-xl', 'image_place_holder': '', 'processor_path': None, 'do_eval': True, 'max_eval_samples': 200, 'train_text_encoder': True, 'no_left_padding': False, 'cfg_scale': 7.5, 'top_k': 16384, 'temperature': 0.9, 'top_p': 1.0, 'eval_steps': 2000, 'project_name': 'llamagen_ti2i', 'load_from_checkpoint': '/tmp/haozhezhao/MLLMG/checkpoint/FIXed_3M_1epoch_step32000.pt', 'warmup': 0.05, 'lr_decay_style': 'cosine', 'lr_decay_ratio': 0.1, 'train_iters': 500000, 'class_dropout_prob': 0.1, 'with_image_only': False, 'image_only_rate': 0.1, 'stage2': False, 'subject_driven': True, 'load_subject_embedding': None, 'reference_data_path': '/tmp/haozhezhao/MLLMG/cc12m_reference_tunnel.jsonl', 'multimodal_encoder': 'instructblip', 'do_recovery': True, 'no_replace': False, 'resume': False, 'dreambench_eval': False, 'find_unused_parameters': True, 'load_visual_encoder': False, 'continue_stage1': False, 'replace_subject': False, 'train_all': True, 'save_total_limit': 2, 'load_language_projection': None, 'mm_vision_tower': 'openai/clip-vit-large-patch14', 'load_fixed_llamagen': True, 'unfreeze_output': False, 'fix': 'gpt-empty-fix', 'rank': 0, 'world_size': 8, 'gpu': 0, 'dist_url': 'env://', 'distributed': True, 'dist_backend': 'nccl', '_wandb': {}} +2025-04-20 16:56:33,395 INFO MainThread:2696278 [wandb_init.py:init():784] starting backend +2025-04-20 16:56:33,395 INFO MainThread:2696278 [wandb_init.py:init():788] sending inform_init request +2025-04-20 16:56:33,401 INFO MainThread:2696278 [backend.py:_multiprocessing_setup():101] multiprocessing start_methods=fork,spawn,forkserver, using: spawn +2025-04-20 16:56:33,401 INFO MainThread:2696278 [wandb_init.py:init():798] backend started and connected +2025-04-20 16:56:33,409 INFO MainThread:2696278 [wandb_init.py:init():891] updated telemetry +2025-04-20 16:56:33,410 INFO MainThread:2696278 [wandb_init.py:init():915] communicating run to backend with 90.0 second timeout +2025-04-20 16:56:34,207 INFO MainThread:2696278 [wandb_init.py:init():990] starting run threads in backend +2025-04-20 16:56:34,307 INFO MainThread:2696278 [wandb_run.py:_console_start():2375] atexit reg +2025-04-20 16:56:34,307 INFO MainThread:2696278 [wandb_run.py:_redirect():2227] redirect: wrap_raw +2025-04-20 16:56:34,308 INFO MainThread:2696278 [wandb_run.py:_redirect():2292] Wrapping output streams. +2025-04-20 16:56:34,308 INFO MainThread:2696278 [wandb_run.py:_redirect():2315] Redirects installed. +2025-04-20 16:56:34,310 INFO MainThread:2696278 [wandb_init.py:init():1032] run started, returning control to user process diff --git a/CKPTS/EmptyFix_InstructBlip_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_objects/wandb/run-20250420_165633-wzvwagyn/run-wzvwagyn.wandb b/CKPTS/EmptyFix_InstructBlip_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_objects/wandb/run-20250420_165633-wzvwagyn/run-wzvwagyn.wandb new file mode 100644 index 0000000000000000000000000000000000000000..4d9a430fe0a18ce240e54b5ba6b904473114b847 --- /dev/null +++ b/CKPTS/EmptyFix_InstructBlip_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_objects/wandb/run-20250420_165633-wzvwagyn/run-wzvwagyn.wandb @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:eed2700ec8e50e7b8d5cdbf9f7019011ca2769e123561e4acd381849ffaa8eee +size 40140800 diff --git a/CKPTS/X2I_700k_3mask_200k_LLava_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_filtered_objects_3mask_888_context/000-GPT-XL/log.txt b/CKPTS/X2I_700k_3mask_200k_LLava_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_filtered_objects_3mask_888_context/000-GPT-XL/log.txt new file mode 100644 index 0000000000000000000000000000000000000000..4a4fa323025cbeb470881fd8f2c630d0c02d83ea --- /dev/null +++ b/CKPTS/X2I_700k_3mask_200k_LLava_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_filtered_objects_3mask_888_context/000-GPT-XL/log.txt @@ -0,0 +1,18 @@ +[2025-04-29 02:44:26] Experiment directory created at checkpoint/X2I_700k_3mask_200k_LLava_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_filtered_objects_3mask_888_context/000-GPT-XL +[2025-04-29 02:44:26] Namespace(data_path='/tmp/haozhezhao/MLLMG/jsonl_data/X2I_Filtered_700k_segmentmask_3mask_200k_data_train.jsonl', cloud_save_path='/tmp/haozhezhao/MLLMG/checkpoint', no_local_save=False, vq_model='VQ-16', vq_ckpt='/tmp/haozhezhao/model/llamagen_t2i/vq_ds16_t2i.pt', codebook_size=16384, codebook_embed_dim=8, gpt_model='GPT-XL', gpt_ckpt='/tmp/haozhezhao/MLLMG/checkpoint/Afteri2i_100kckpt_LLAVAT5_empty_Fixed_just_segment_pretrain_stage3_dreambench__recap_400k_t2i_400k_flux_200k_midjourney_150k_recovery_150k_grounding_100fluxseg_50samseg/005-GPT-XL/checkpoints/0078000.pt', gpt_type='t2i', vocab_size=16384, cls_token_num=888, dropout_p=0.1, token_dropout_p=0.1, drop_path=0.0, no_compile=False, results_dir='checkpoint/X2I_700k_3mask_200k_LLava_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_filtered_objects_3mask_888_context', dataset='ti2i', image_size=512, downsample_size=16, num_classes=1000, epochs=2, lr=0.0003, weight_decay=0.05, beta1=0.9, beta2=0.95, max_grad_norm=1.0, global_batch_size=48, global_seed=0, num_workers=4, log_every=25, ckpt_every=2000, gradient_accumulation_steps=8, mixed_precision='bf16', val_data_path='/tmp/haozhezhao/MLLMG/jsonl_data/multiimage_val_for_llava_X2I_threemask_dreambenplus.jsonl', use_vision_tower=True, model_name_or_path='/tmp/haozhezhao/model/blip2-flan-t5-xl', image_place_holder='', processor_path=None, do_eval=True, max_eval_samples=200, train_text_encoder=True, no_left_padding=False, cfg_scale=7.5, top_k=16384, temperature=0.9, top_p=1.0, eval_steps=2000, project_name='llamagen_ti2i', load_from_checkpoint='/tmp/haozhezhao/MLLMG/checkpoint/FIXed_3M_1epoch_step32000.pt', warmup=0.05, lr_decay_style='cosine', lr_decay_ratio=0.1, train_iters=500000, class_dropout_prob=0.1, with_image_only=False, image_only_rate=0.1, stage2=False, subject_driven=True, load_subject_embedding=None, reference_data_path='/tmp/haozhezhao/MLLMG/cc12m_reference_tunnel.jsonl', multimodal_encoder='llava', do_recovery=True, no_replace=False, resume=False, dreambench_eval=True, find_unused_parameters=True, load_visual_encoder=False, continue_stage1=False, replace_subject=False, train_all=True, save_total_limit=1, load_language_projection='/tmp/haozhezhao/MLLMG/llava-v1.5-flant5_fixed-pretrain/mm_projector.bin', mm_vision_tower='openai/clip-vit-large-patch14', load_fixed_llamagen=True, unfreeze_output=False, fix='gpt-empty-fix', rank=0, world_size=8, gpu=0, dist_url='env://', distributed=True, dist_backend='nccl') +[2025-04-29 02:44:26] Starting rank=0, seed=0, world_size=8. +[2025-04-29 02:44:26] text_config is None. Initializing the text config with default values (`OPTConfig`). +[2025-04-29 02:45:31] GPT Parameters: 2,310,680,832 +[2025-04-29 02:45:31] num decayed parameter tensors: 356, with 2,007,303,168 parameters +[2025-04-29 02:45:31] num non-decayed parameter tensors: 124, with 197,888 parameters +[2025-04-29 02:45:31] using fused AdamW: True +[2025-04-29 02:45:45] Dataset contains 908,625 images +[2025-04-29 02:45:45] Train iters 37858 , warmup 1892.9, len of loader 18929 +[2025-04-29 02:46:07] ### LOAD pretraining weights from checkpoint: /tmp/haozhezhao/MLLMG/checkpoint/Afteri2i_100kckpt_LLAVAT5_empty_Fixed_just_segment_pretrain_stage3_dreambench__recap_400k_t2i_400k_flux_200k_midjourney_150k_recovery_150k_grounding_100fluxseg_50samseg/005-GPT-XL/checkpoints/0078000.pt +[2025-04-29 02:46:07] Initial state: steps=0, epochs=0 +[2025-04-29 02:46:07] compiling the model... (may take several minutes) +[2025-04-29 02:46:07] freeze the vit +[2025-04-29 02:46:07] ***** total param is 2310680832 ***** +[2025-04-29 02:46:07] ***** total trained param is 2007501056 ***** +[2025-04-29 02:46:15] Training for 2 epochs... +[2025-04-29 02:46:15] Beginning epoch 0... diff --git a/CKPTS/X2I_700k_3mask_200k_LLava_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_filtered_objects_3mask_888_context/002-GPT-XL/log.txt b/CKPTS/X2I_700k_3mask_200k_LLava_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_filtered_objects_3mask_888_context/002-GPT-XL/log.txt new file mode 100644 index 0000000000000000000000000000000000000000..34810ac1e1f2c5f7b2fb5c2f8a5dc10ede657d89 --- /dev/null +++ b/CKPTS/X2I_700k_3mask_200k_LLava_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_filtered_objects_3mask_888_context/002-GPT-XL/log.txt @@ -0,0 +1,36 @@ +[2025-04-29 03:06:28] Experiment directory created at checkpoint/X2I_700k_3mask_200k_LLava_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_filtered_objects_3mask_888_context/002-GPT-XL +[2025-04-29 03:06:28] Namespace(data_path='/tmp/haozhezhao/MLLMG/jsonl_data/X2I_Filtered_700k_segmentmask_3mask_200k_data_train.jsonl', cloud_save_path='/tmp/haozhezhao/MLLMG/checkpoint', no_local_save=False, vq_model='VQ-16', vq_ckpt='/tmp/haozhezhao/model/llamagen_t2i/vq_ds16_t2i.pt', codebook_size=16384, codebook_embed_dim=8, gpt_model='GPT-XL', gpt_ckpt='/tmp/haozhezhao/MLLMG/checkpoint/Afteri2i_100kckpt_LLAVAT5_empty_Fixed_just_segment_pretrain_stage3_dreambench__recap_400k_t2i_400k_flux_200k_midjourney_150k_recovery_150k_grounding_100fluxseg_50samseg/005-GPT-XL/checkpoints/0078000.pt', gpt_type='t2i', vocab_size=16384, cls_token_num=888, dropout_p=0.1, token_dropout_p=0.1, drop_path=0.0, no_compile=False, results_dir='checkpoint/X2I_700k_3mask_200k_LLava_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_filtered_objects_3mask_888_context', dataset='ti2i', image_size=512, downsample_size=16, num_classes=1000, epochs=2, lr=0.0003, weight_decay=0.05, beta1=0.9, beta2=0.95, max_grad_norm=1.0, global_batch_size=40, global_seed=0, num_workers=4, log_every=25, ckpt_every=2000, gradient_accumulation_steps=8, mixed_precision='bf16', val_data_path='/tmp/haozhezhao/MLLMG/jsonl_data/multiimage_val_for_llava_X2I_threemask_dreambenplus.jsonl', use_vision_tower=True, model_name_or_path='/tmp/haozhezhao/model/blip2-flan-t5-xl', image_place_holder='', processor_path=None, do_eval=True, max_eval_samples=200, train_text_encoder=True, no_left_padding=False, cfg_scale=7.5, top_k=16384, temperature=0.9, top_p=1.0, eval_steps=2000, project_name='llamagen_ti2i', load_from_checkpoint='/tmp/haozhezhao/MLLMG/checkpoint/FIXed_3M_1epoch_step32000.pt', warmup=0.05, lr_decay_style='cosine', lr_decay_ratio=0.1, train_iters=500000, class_dropout_prob=0.1, with_image_only=False, image_only_rate=0.1, stage2=False, subject_driven=True, load_subject_embedding=None, reference_data_path='/tmp/haozhezhao/MLLMG/cc12m_reference_tunnel.jsonl', multimodal_encoder='llava', do_recovery=True, no_replace=False, resume=False, dreambench_eval=True, find_unused_parameters=True, load_visual_encoder=False, continue_stage1=False, replace_subject=False, train_all=True, save_total_limit=1, load_language_projection='/tmp/haozhezhao/MLLMG/llava-v1.5-flant5_fixed-pretrain/mm_projector.bin', mm_vision_tower='openai/clip-vit-large-patch14', load_fixed_llamagen=True, unfreeze_output=False, fix='gpt-empty-fix', rank=0, world_size=8, gpu=0, dist_url='env://', distributed=True, dist_backend='nccl') +[2025-04-29 03:06:28] Starting rank=0, seed=0, world_size=8. +[2025-04-29 03:06:28] text_config is None. Initializing the text config with default values (`OPTConfig`). +[2025-04-29 03:07:33] GPT Parameters: 2,310,680,832 +[2025-04-29 03:07:33] num decayed parameter tensors: 356, with 2,007,303,168 parameters +[2025-04-29 03:07:33] num non-decayed parameter tensors: 124, with 197,888 parameters +[2025-04-29 03:07:33] using fused AdamW: True +[2025-04-29 03:07:47] Dataset contains 908,625 images +[2025-04-29 03:07:47] Train iters 45430 , warmup 2271.5, len of loader 22715 +[2025-04-29 03:08:09] ### LOAD pretraining weights from checkpoint: /tmp/haozhezhao/MLLMG/checkpoint/Afteri2i_100kckpt_LLAVAT5_empty_Fixed_just_segment_pretrain_stage3_dreambench__recap_400k_t2i_400k_flux_200k_midjourney_150k_recovery_150k_grounding_100fluxseg_50samseg/005-GPT-XL/checkpoints/0078000.pt +[2025-04-29 03:08:09] Initial state: steps=0, epochs=0 +[2025-04-29 03:08:09] compiling the model... (may take several minutes) +[2025-04-29 03:08:09] freeze the vit +[2025-04-29 03:08:09] ***** total param is 2310680832 ***** +[2025-04-29 03:08:09] ***** total trained param is 2007501056 ***** +[2025-04-29 03:08:14] Training for 2 epochs... +[2025-04-29 03:08:14] Beginning epoch 0... +[2025-04-29 03:13:00] (step=0000025) Train Loss: 1.6057, Train Steps/Sec: 0.09 +[2025-04-29 03:13:23] (step=0000050) Train Loss: 1.5536, Train Steps/Sec: 1.06 +[2025-04-29 03:13:47] (step=0000075) Train Loss: 1.6046, Train Steps/Sec: 1.06 +[2025-04-29 03:14:10] (step=0000100) Train Loss: 1.5534, Train Steps/Sec: 1.07 +[2025-04-29 03:14:34] (step=0000125) Train Loss: 1.6340, Train Steps/Sec: 1.07 +[2025-04-29 03:14:57] (step=0000150) Train Loss: 1.5523, Train Steps/Sec: 1.06 +[2025-04-29 03:15:21] (step=0000175) Train Loss: 1.4454, Train Steps/Sec: 1.05 +[2025-04-29 03:15:45] (step=0000200) Train Loss: 1.4766, Train Steps/Sec: 1.06 +[2025-04-29 03:16:08] (step=0000225) Train Loss: 1.6255, Train Steps/Sec: 1.06 +[2025-04-29 03:16:32] (step=0000250) Train Loss: 1.5149, Train Steps/Sec: 1.06 +[2025-04-29 03:16:55] (step=0000275) Train Loss: 1.5378, Train Steps/Sec: 1.07 +[2025-04-29 03:17:19] (step=0000300) Train Loss: 1.4487, Train Steps/Sec: 1.07 +[2025-04-29 03:17:42] (step=0000325) Train Loss: 1.3856, Train Steps/Sec: 1.07 +[2025-04-29 03:18:05] (step=0000350) Train Loss: 1.5109, Train Steps/Sec: 1.07 +[2025-04-29 03:18:28] (step=0000375) Train Loss: 1.3958, Train Steps/Sec: 1.07 +[2025-04-29 03:18:52] (step=0000400) Train Loss: 1.4455, Train Steps/Sec: 1.07 +[2025-04-29 03:19:15] (step=0000425) Train Loss: 1.5334, Train Steps/Sec: 1.07 +[2025-04-29 03:19:38] (step=0000450) Train Loss: 1.3969, Train Steps/Sec: 1.07 diff --git a/CKPTS/X2I_700k_3mask_200k_LLava_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_filtered_objects_3mask_888_context/003-GPT-XL/checkpoints/eval_step_10000/batch_0_cfg_7.5_topk_16384.jpg b/CKPTS/X2I_700k_3mask_200k_LLava_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_filtered_objects_3mask_888_context/003-GPT-XL/checkpoints/eval_step_10000/batch_0_cfg_7.5_topk_16384.jpg new file mode 100644 index 0000000000000000000000000000000000000000..7ab1c3298dc45557c8cc46f5386e273f5dc2ab64 --- /dev/null +++ b/CKPTS/X2I_700k_3mask_200k_LLava_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_filtered_objects_3mask_888_context/003-GPT-XL/checkpoints/eval_step_10000/batch_0_cfg_7.5_topk_16384.jpg @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:ba897d3c22be57d20c5e281b791615b70953e00b42b6302d547870ea08cd4a98 +size 528707 diff --git a/CKPTS/X2I_700k_3mask_200k_LLava_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_filtered_objects_3mask_888_context/003-GPT-XL/checkpoints/eval_step_10000/batch_1_cfg_7.5_topk_16384.jpg b/CKPTS/X2I_700k_3mask_200k_LLava_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_filtered_objects_3mask_888_context/003-GPT-XL/checkpoints/eval_step_10000/batch_1_cfg_7.5_topk_16384.jpg new file mode 100644 index 0000000000000000000000000000000000000000..bb72a455dcbb5b5e48a8ed0988f56ef394396738 --- /dev/null +++ b/CKPTS/X2I_700k_3mask_200k_LLava_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_filtered_objects_3mask_888_context/003-GPT-XL/checkpoints/eval_step_10000/batch_1_cfg_7.5_topk_16384.jpg @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:cee59812e80ad589d1c953d97a3af7fd3866cd9ca18b9966d9e8bb64d8c3764a +size 577701 diff --git a/CKPTS/X2I_700k_3mask_200k_LLava_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_filtered_objects_3mask_888_context/003-GPT-XL/checkpoints/eval_step_10000/batch_2_cfg_7.5_topk_16384.jpg b/CKPTS/X2I_700k_3mask_200k_LLava_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_filtered_objects_3mask_888_context/003-GPT-XL/checkpoints/eval_step_10000/batch_2_cfg_7.5_topk_16384.jpg new file mode 100644 index 0000000000000000000000000000000000000000..b3c1c56674d6078b6d6bb52aeb8ba44ff3002013 --- /dev/null +++ b/CKPTS/X2I_700k_3mask_200k_LLava_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_filtered_objects_3mask_888_context/003-GPT-XL/checkpoints/eval_step_10000/batch_2_cfg_7.5_topk_16384.jpg @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:3bb34c93d21c9f31723721ba5c14198c56156f9133d89e47f168aa78c29ee548 +size 672067 diff --git a/CKPTS/X2I_700k_3mask_200k_LLava_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_filtered_objects_3mask_888_context/003-GPT-XL/checkpoints/eval_step_10000/batch_3_cfg_7.5_topk_16384.jpg b/CKPTS/X2I_700k_3mask_200k_LLava_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_filtered_objects_3mask_888_context/003-GPT-XL/checkpoints/eval_step_10000/batch_3_cfg_7.5_topk_16384.jpg new file mode 100644 index 0000000000000000000000000000000000000000..1d28dab6d1bb44b3a8d32ba6018a882bb03c2bba --- /dev/null +++ b/CKPTS/X2I_700k_3mask_200k_LLava_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_filtered_objects_3mask_888_context/003-GPT-XL/checkpoints/eval_step_10000/batch_3_cfg_7.5_topk_16384.jpg @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:ed2ad6a7d012b51c431b161b4fbd88b39f8087d742617e348a5d09766ce7294f +size 522807 diff --git a/CKPTS/X2I_700k_3mask_200k_LLava_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_filtered_objects_3mask_888_context/003-GPT-XL/checkpoints/eval_step_10000/batch_4_cfg_7.5_topk_16384.jpg b/CKPTS/X2I_700k_3mask_200k_LLava_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_filtered_objects_3mask_888_context/003-GPT-XL/checkpoints/eval_step_10000/batch_4_cfg_7.5_topk_16384.jpg new file mode 100644 index 0000000000000000000000000000000000000000..a897c4a70500b1c578ac771ff4c28172b7ce79a4 --- /dev/null +++ b/CKPTS/X2I_700k_3mask_200k_LLava_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_filtered_objects_3mask_888_context/003-GPT-XL/checkpoints/eval_step_10000/batch_4_cfg_7.5_topk_16384.jpg @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:3e61c6714ab8e4dfd50966fa787052a5766b01bedd0538b84d686eb8e99faf01 +size 548588 diff --git a/CKPTS/X2I_700k_3mask_200k_LLava_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_filtered_objects_3mask_888_context/003-GPT-XL/checkpoints/eval_step_12000/batch_0_cfg_7.5_topk_16384.jpg b/CKPTS/X2I_700k_3mask_200k_LLava_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_filtered_objects_3mask_888_context/003-GPT-XL/checkpoints/eval_step_12000/batch_0_cfg_7.5_topk_16384.jpg new file mode 100644 index 0000000000000000000000000000000000000000..9ae0251ad0e200cdc4b50a230a37269d77320ff0 --- /dev/null +++ b/CKPTS/X2I_700k_3mask_200k_LLava_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_filtered_objects_3mask_888_context/003-GPT-XL/checkpoints/eval_step_12000/batch_0_cfg_7.5_topk_16384.jpg @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:733c13275f823da4c960d05f1540a0b891d2274b1c61a20185036a884458b197 +size 543058 diff --git a/CKPTS/X2I_700k_3mask_200k_LLava_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_filtered_objects_3mask_888_context/003-GPT-XL/checkpoints/eval_step_12000/batch_1_cfg_7.5_topk_16384.jpg b/CKPTS/X2I_700k_3mask_200k_LLava_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_filtered_objects_3mask_888_context/003-GPT-XL/checkpoints/eval_step_12000/batch_1_cfg_7.5_topk_16384.jpg new file mode 100644 index 0000000000000000000000000000000000000000..2a4e744803f901ddabb5497108f00e99306fbf91 --- /dev/null +++ b/CKPTS/X2I_700k_3mask_200k_LLava_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_filtered_objects_3mask_888_context/003-GPT-XL/checkpoints/eval_step_12000/batch_1_cfg_7.5_topk_16384.jpg @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:cbd379cc65b3f14819587b18e95f376d79675c3abd13260734a4d89c003f5563 +size 565757 diff --git a/CKPTS/X2I_700k_3mask_200k_LLava_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_filtered_objects_3mask_888_context/003-GPT-XL/checkpoints/eval_step_12000/batch_2_cfg_7.5_topk_16384.jpg b/CKPTS/X2I_700k_3mask_200k_LLava_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_filtered_objects_3mask_888_context/003-GPT-XL/checkpoints/eval_step_12000/batch_2_cfg_7.5_topk_16384.jpg new file mode 100644 index 0000000000000000000000000000000000000000..ce4453a70645a0fa18fab74139e6bc6d54f791d2 --- /dev/null +++ b/CKPTS/X2I_700k_3mask_200k_LLava_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_filtered_objects_3mask_888_context/003-GPT-XL/checkpoints/eval_step_12000/batch_2_cfg_7.5_topk_16384.jpg @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:3c4debc204e790b9439e799bd2b8a0b3f3c8022efa07514c1c3f27dfb7fd8d64 +size 665120 diff --git a/CKPTS/X2I_700k_3mask_200k_LLava_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_filtered_objects_3mask_888_context/003-GPT-XL/checkpoints/eval_step_12000/batch_3_cfg_7.5_topk_16384.jpg b/CKPTS/X2I_700k_3mask_200k_LLava_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_filtered_objects_3mask_888_context/003-GPT-XL/checkpoints/eval_step_12000/batch_3_cfg_7.5_topk_16384.jpg new file mode 100644 index 0000000000000000000000000000000000000000..b25c1a9dd4f962b3d96e1b82ad79ded56f52a517 --- /dev/null +++ b/CKPTS/X2I_700k_3mask_200k_LLava_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_filtered_objects_3mask_888_context/003-GPT-XL/checkpoints/eval_step_12000/batch_3_cfg_7.5_topk_16384.jpg @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:6e90aae2542f2234c6bab627f9e4157d3bcd42fd572e6fc78e5fd1c8f5ec03f0 +size 533105 diff --git a/CKPTS/X2I_700k_3mask_200k_LLava_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_filtered_objects_3mask_888_context/003-GPT-XL/checkpoints/eval_step_12000/batch_4_cfg_7.5_topk_16384.jpg b/CKPTS/X2I_700k_3mask_200k_LLava_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_filtered_objects_3mask_888_context/003-GPT-XL/checkpoints/eval_step_12000/batch_4_cfg_7.5_topk_16384.jpg new file mode 100644 index 0000000000000000000000000000000000000000..d3cb6e0a3966b8698de125916f2d7a598c97805d --- /dev/null +++ b/CKPTS/X2I_700k_3mask_200k_LLava_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_filtered_objects_3mask_888_context/003-GPT-XL/checkpoints/eval_step_12000/batch_4_cfg_7.5_topk_16384.jpg @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:91fb9ddadc5b5293117d2cb240e7c09bc7c40cc09b78117e39bd121360339f96 +size 534294 diff --git a/CKPTS/X2I_700k_3mask_200k_LLava_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_filtered_objects_3mask_888_context/003-GPT-XL/checkpoints/eval_step_14000/batch_0_cfg_7.5_topk_16384.jpg b/CKPTS/X2I_700k_3mask_200k_LLava_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_filtered_objects_3mask_888_context/003-GPT-XL/checkpoints/eval_step_14000/batch_0_cfg_7.5_topk_16384.jpg new file mode 100644 index 0000000000000000000000000000000000000000..4d9704f4a9b278305d1b76890b2577115e1e35fa --- /dev/null +++ b/CKPTS/X2I_700k_3mask_200k_LLava_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_filtered_objects_3mask_888_context/003-GPT-XL/checkpoints/eval_step_14000/batch_0_cfg_7.5_topk_16384.jpg @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:cfb3e116c4d23cda81e5d4c1669b78772e45516ff9b5cf3bed5748c42bcf7973 +size 529094 diff --git a/CKPTS/X2I_700k_3mask_200k_LLava_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_filtered_objects_3mask_888_context/003-GPT-XL/checkpoints/eval_step_14000/batch_1_cfg_7.5_topk_16384.jpg b/CKPTS/X2I_700k_3mask_200k_LLava_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_filtered_objects_3mask_888_context/003-GPT-XL/checkpoints/eval_step_14000/batch_1_cfg_7.5_topk_16384.jpg new file mode 100644 index 0000000000000000000000000000000000000000..8a64ce5f6463fa46f0f016bbe5536219f1b89e0a --- /dev/null +++ b/CKPTS/X2I_700k_3mask_200k_LLava_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_filtered_objects_3mask_888_context/003-GPT-XL/checkpoints/eval_step_14000/batch_1_cfg_7.5_topk_16384.jpg @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:544e9d73e2ea0bced2280c0144fe8f4e34730bb1fdb20d68c23661418c56b267 +size 570485 diff --git a/CKPTS/X2I_700k_3mask_200k_LLava_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_filtered_objects_3mask_888_context/003-GPT-XL/checkpoints/eval_step_14000/batch_2_cfg_7.5_topk_16384.jpg b/CKPTS/X2I_700k_3mask_200k_LLava_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_filtered_objects_3mask_888_context/003-GPT-XL/checkpoints/eval_step_14000/batch_2_cfg_7.5_topk_16384.jpg new file mode 100644 index 0000000000000000000000000000000000000000..9edab4b170f9f53ee5b8ad8361c75b6e20cd46fa --- /dev/null +++ b/CKPTS/X2I_700k_3mask_200k_LLava_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_filtered_objects_3mask_888_context/003-GPT-XL/checkpoints/eval_step_14000/batch_2_cfg_7.5_topk_16384.jpg @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:3c07fd930d42e3c606ba5f5fb7570a76f8008e61d8d6d7477611069bca6812bd +size 661582 diff --git a/CKPTS/X2I_700k_3mask_200k_LLava_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_filtered_objects_3mask_888_context/003-GPT-XL/checkpoints/eval_step_14000/batch_3_cfg_7.5_topk_16384.jpg b/CKPTS/X2I_700k_3mask_200k_LLava_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_filtered_objects_3mask_888_context/003-GPT-XL/checkpoints/eval_step_14000/batch_3_cfg_7.5_topk_16384.jpg new file mode 100644 index 0000000000000000000000000000000000000000..c5b88fd3920b45d6173207e2dbe78c4adf693413 --- /dev/null +++ b/CKPTS/X2I_700k_3mask_200k_LLava_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_filtered_objects_3mask_888_context/003-GPT-XL/checkpoints/eval_step_14000/batch_3_cfg_7.5_topk_16384.jpg @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:b1dfc8a43275276631cd1906f5d89a0d90a95b4c0f29856b4370f4ccfa8960dd +size 522649 diff --git a/CKPTS/X2I_700k_3mask_200k_LLava_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_filtered_objects_3mask_888_context/003-GPT-XL/checkpoints/eval_step_14000/batch_4_cfg_7.5_topk_16384.jpg b/CKPTS/X2I_700k_3mask_200k_LLava_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_filtered_objects_3mask_888_context/003-GPT-XL/checkpoints/eval_step_14000/batch_4_cfg_7.5_topk_16384.jpg new file mode 100644 index 0000000000000000000000000000000000000000..c63ef66d4d69a420d291c936f54fa03602f9c397 --- /dev/null +++ b/CKPTS/X2I_700k_3mask_200k_LLava_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_filtered_objects_3mask_888_context/003-GPT-XL/checkpoints/eval_step_14000/batch_4_cfg_7.5_topk_16384.jpg @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:24a0b45710b4338bc86a22e2acdab42eb23e716cc147fe3423cb27581f1ef238 +size 545766 diff --git a/CKPTS/X2I_700k_3mask_200k_LLava_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_filtered_objects_3mask_888_context/003-GPT-XL/checkpoints/eval_step_16000/batch_0_cfg_7.5_topk_16384.jpg b/CKPTS/X2I_700k_3mask_200k_LLava_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_filtered_objects_3mask_888_context/003-GPT-XL/checkpoints/eval_step_16000/batch_0_cfg_7.5_topk_16384.jpg new file mode 100644 index 0000000000000000000000000000000000000000..de925c0124a11882d46788e9d1a9bc6cf8b275e8 --- /dev/null +++ b/CKPTS/X2I_700k_3mask_200k_LLava_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_filtered_objects_3mask_888_context/003-GPT-XL/checkpoints/eval_step_16000/batch_0_cfg_7.5_topk_16384.jpg @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:bf9596f5bcc6156b16bcc15d9526d015ab29743646f9cf162defaeeccf7d93a5 +size 542377 diff --git a/CKPTS/X2I_700k_3mask_200k_LLava_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_filtered_objects_3mask_888_context/003-GPT-XL/checkpoints/eval_step_16000/batch_1_cfg_7.5_topk_16384.jpg b/CKPTS/X2I_700k_3mask_200k_LLava_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_filtered_objects_3mask_888_context/003-GPT-XL/checkpoints/eval_step_16000/batch_1_cfg_7.5_topk_16384.jpg new file mode 100644 index 0000000000000000000000000000000000000000..5e22d2487fce070b3b9d4b4a47a18c6bce9a80f8 --- /dev/null +++ b/CKPTS/X2I_700k_3mask_200k_LLava_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_filtered_objects_3mask_888_context/003-GPT-XL/checkpoints/eval_step_16000/batch_1_cfg_7.5_topk_16384.jpg @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:56a74616db79407167c2fd6ea3c5a7b1c1aa0db83395169295fda83aa536e8fa +size 556315 diff --git a/CKPTS/X2I_700k_3mask_200k_LLava_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_filtered_objects_3mask_888_context/003-GPT-XL/checkpoints/eval_step_16000/batch_2_cfg_7.5_topk_16384.jpg b/CKPTS/X2I_700k_3mask_200k_LLava_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_filtered_objects_3mask_888_context/003-GPT-XL/checkpoints/eval_step_16000/batch_2_cfg_7.5_topk_16384.jpg new file mode 100644 index 0000000000000000000000000000000000000000..0c02c89f6c3ff5d1e87ff99f7580e8e551ae1d8d --- /dev/null +++ b/CKPTS/X2I_700k_3mask_200k_LLava_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_filtered_objects_3mask_888_context/003-GPT-XL/checkpoints/eval_step_16000/batch_2_cfg_7.5_topk_16384.jpg @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:47be1f7e6ac45595c9d2c451893ba26cab956fee296d271d68a055e9668c3b53 +size 678429 diff --git a/CKPTS/X2I_700k_3mask_200k_LLava_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_filtered_objects_3mask_888_context/003-GPT-XL/checkpoints/eval_step_16000/batch_3_cfg_7.5_topk_16384.jpg b/CKPTS/X2I_700k_3mask_200k_LLava_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_filtered_objects_3mask_888_context/003-GPT-XL/checkpoints/eval_step_16000/batch_3_cfg_7.5_topk_16384.jpg new file mode 100644 index 0000000000000000000000000000000000000000..57c700543c3ec202039a3f3416452c2943ffa4f8 --- /dev/null +++ b/CKPTS/X2I_700k_3mask_200k_LLava_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_filtered_objects_3mask_888_context/003-GPT-XL/checkpoints/eval_step_16000/batch_3_cfg_7.5_topk_16384.jpg @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:ebeb6f2de2b5e304df50c769ebd7dc742fe613baab7974d218b96030375f2911 +size 532929 diff --git a/CKPTS/X2I_700k_3mask_200k_LLava_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_filtered_objects_3mask_888_context/003-GPT-XL/checkpoints/eval_step_16000/batch_4_cfg_7.5_topk_16384.jpg b/CKPTS/X2I_700k_3mask_200k_LLava_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_filtered_objects_3mask_888_context/003-GPT-XL/checkpoints/eval_step_16000/batch_4_cfg_7.5_topk_16384.jpg new file mode 100644 index 0000000000000000000000000000000000000000..20b1cf6eebce8e031e02e3b6f057b80b28d70231 --- /dev/null +++ b/CKPTS/X2I_700k_3mask_200k_LLava_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_filtered_objects_3mask_888_context/003-GPT-XL/checkpoints/eval_step_16000/batch_4_cfg_7.5_topk_16384.jpg @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:e32f9c29083c537ccce278b340862f2169a030e7ce2ae26e4e41c526a9722301 +size 544271 diff --git a/CKPTS/X2I_700k_3mask_200k_LLava_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_filtered_objects_3mask_888_context/003-GPT-XL/checkpoints/eval_step_18000/batch_0_cfg_7.5_topk_16384.jpg b/CKPTS/X2I_700k_3mask_200k_LLava_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_filtered_objects_3mask_888_context/003-GPT-XL/checkpoints/eval_step_18000/batch_0_cfg_7.5_topk_16384.jpg new file mode 100644 index 0000000000000000000000000000000000000000..c10927834fb2caf1d154625d0fab3b939e7e7ca4 --- /dev/null +++ b/CKPTS/X2I_700k_3mask_200k_LLava_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_filtered_objects_3mask_888_context/003-GPT-XL/checkpoints/eval_step_18000/batch_0_cfg_7.5_topk_16384.jpg @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:441b6868b1369c31228c22cb4fb93bca5d39eec9ac077f53e462eb23987c3791 +size 526628 diff --git a/CKPTS/X2I_700k_3mask_200k_LLava_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_filtered_objects_3mask_888_context/003-GPT-XL/checkpoints/eval_step_18000/batch_1_cfg_7.5_topk_16384.jpg b/CKPTS/X2I_700k_3mask_200k_LLava_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_filtered_objects_3mask_888_context/003-GPT-XL/checkpoints/eval_step_18000/batch_1_cfg_7.5_topk_16384.jpg new file mode 100644 index 0000000000000000000000000000000000000000..5e50b66eed0b04056482d8f54c63b9118e2b08fb --- /dev/null +++ b/CKPTS/X2I_700k_3mask_200k_LLava_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_filtered_objects_3mask_888_context/003-GPT-XL/checkpoints/eval_step_18000/batch_1_cfg_7.5_topk_16384.jpg @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:717c49dc6d8da889dd35713821b576b608a8d731962cf5a99c82b98e76692276 +size 561748 diff --git a/CKPTS/X2I_700k_3mask_200k_LLava_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_filtered_objects_3mask_888_context/003-GPT-XL/checkpoints/eval_step_18000/batch_2_cfg_7.5_topk_16384.jpg b/CKPTS/X2I_700k_3mask_200k_LLava_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_filtered_objects_3mask_888_context/003-GPT-XL/checkpoints/eval_step_18000/batch_2_cfg_7.5_topk_16384.jpg new file mode 100644 index 0000000000000000000000000000000000000000..3888388e64f34c58d828c869add76e263ed792e4 --- /dev/null +++ b/CKPTS/X2I_700k_3mask_200k_LLava_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_filtered_objects_3mask_888_context/003-GPT-XL/checkpoints/eval_step_18000/batch_2_cfg_7.5_topk_16384.jpg @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:d11600bb034cd6faf4dd61c9526a49aa7f0f799d61a9c3de896e9a44a2c5bc96 +size 671069 diff --git a/CKPTS/X2I_700k_3mask_200k_LLava_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_filtered_objects_3mask_888_context/003-GPT-XL/checkpoints/eval_step_18000/batch_3_cfg_7.5_topk_16384.jpg b/CKPTS/X2I_700k_3mask_200k_LLava_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_filtered_objects_3mask_888_context/003-GPT-XL/checkpoints/eval_step_18000/batch_3_cfg_7.5_topk_16384.jpg new file mode 100644 index 0000000000000000000000000000000000000000..547258d27e2fc89282df401b71aae793c0d7d6db --- /dev/null +++ b/CKPTS/X2I_700k_3mask_200k_LLava_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_filtered_objects_3mask_888_context/003-GPT-XL/checkpoints/eval_step_18000/batch_3_cfg_7.5_topk_16384.jpg @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:b2bbff55d72dd35d7a607fc3028d1c54d452c1401c9f98390b7023fc9d8f820b +size 523833 diff --git a/CKPTS/X2I_700k_3mask_200k_LLava_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_filtered_objects_3mask_888_context/003-GPT-XL/checkpoints/eval_step_18000/batch_4_cfg_7.5_topk_16384.jpg b/CKPTS/X2I_700k_3mask_200k_LLava_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_filtered_objects_3mask_888_context/003-GPT-XL/checkpoints/eval_step_18000/batch_4_cfg_7.5_topk_16384.jpg new file mode 100644 index 0000000000000000000000000000000000000000..b1a2eaec4b4f06052dfce93abf0a40d40f33bd46 --- /dev/null +++ b/CKPTS/X2I_700k_3mask_200k_LLava_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_filtered_objects_3mask_888_context/003-GPT-XL/checkpoints/eval_step_18000/batch_4_cfg_7.5_topk_16384.jpg @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:b382b269e7633e436c46621b9cafa5b39cd0106bbfbb2dbc91545504abbed18f +size 543397 diff --git a/CKPTS/X2I_700k_3mask_200k_LLava_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_filtered_objects_3mask_888_context/003-GPT-XL/checkpoints/eval_step_2000/batch_0_cfg_7.5_topk_16384.jpg b/CKPTS/X2I_700k_3mask_200k_LLava_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_filtered_objects_3mask_888_context/003-GPT-XL/checkpoints/eval_step_2000/batch_0_cfg_7.5_topk_16384.jpg new file mode 100644 index 0000000000000000000000000000000000000000..f03e7ee403ecd90b821a07d1a6624c74f47d0867 --- /dev/null +++ b/CKPTS/X2I_700k_3mask_200k_LLava_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_filtered_objects_3mask_888_context/003-GPT-XL/checkpoints/eval_step_2000/batch_0_cfg_7.5_topk_16384.jpg @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:8ec0cbc631ee2decc12617e1acbe9a5fc6e0d37fb0210741022b3c98d5eae0ce +size 561862 diff --git a/CKPTS/X2I_700k_3mask_200k_LLava_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_filtered_objects_3mask_888_context/003-GPT-XL/checkpoints/eval_step_2000/batch_1_cfg_7.5_topk_16384.jpg b/CKPTS/X2I_700k_3mask_200k_LLava_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_filtered_objects_3mask_888_context/003-GPT-XL/checkpoints/eval_step_2000/batch_1_cfg_7.5_topk_16384.jpg new file mode 100644 index 0000000000000000000000000000000000000000..3e319ebc552e48cb251280ed3524d44c0fca91a8 --- /dev/null +++ b/CKPTS/X2I_700k_3mask_200k_LLava_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_filtered_objects_3mask_888_context/003-GPT-XL/checkpoints/eval_step_2000/batch_1_cfg_7.5_topk_16384.jpg @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:ed90fdf3a382312860619770b4073a60e35b4d3c0a786f60478698118ffce4c4 +size 574062 diff --git a/CKPTS/X2I_700k_3mask_200k_LLava_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_filtered_objects_3mask_888_context/003-GPT-XL/checkpoints/eval_step_2000/batch_2_cfg_7.5_topk_16384.jpg b/CKPTS/X2I_700k_3mask_200k_LLava_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_filtered_objects_3mask_888_context/003-GPT-XL/checkpoints/eval_step_2000/batch_2_cfg_7.5_topk_16384.jpg new file mode 100644 index 0000000000000000000000000000000000000000..f5d35214d85fd560b25537df5c012e3a1bc261ec --- /dev/null +++ b/CKPTS/X2I_700k_3mask_200k_LLava_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_filtered_objects_3mask_888_context/003-GPT-XL/checkpoints/eval_step_2000/batch_2_cfg_7.5_topk_16384.jpg @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:13d85294174841fcb34894bfa40d5f97c8bbb96607013d3ad501797bb44885ab +size 663470 diff --git a/CKPTS/X2I_700k_3mask_200k_LLava_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_filtered_objects_3mask_888_context/003-GPT-XL/checkpoints/eval_step_2000/batch_3_cfg_7.5_topk_16384.jpg b/CKPTS/X2I_700k_3mask_200k_LLava_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_filtered_objects_3mask_888_context/003-GPT-XL/checkpoints/eval_step_2000/batch_3_cfg_7.5_topk_16384.jpg new file mode 100644 index 0000000000000000000000000000000000000000..50b13ac12949cb23ed1acae56d35846267e618df --- /dev/null +++ b/CKPTS/X2I_700k_3mask_200k_LLava_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_filtered_objects_3mask_888_context/003-GPT-XL/checkpoints/eval_step_2000/batch_3_cfg_7.5_topk_16384.jpg @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:03c82a9f04d72661df0ff607db0a97f788c9dbec79d3a3bf23e746e794ef176d +size 578538 diff --git a/CKPTS/X2I_700k_3mask_200k_LLava_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_filtered_objects_3mask_888_context/003-GPT-XL/checkpoints/eval_step_2000/batch_4_cfg_7.5_topk_16384.jpg b/CKPTS/X2I_700k_3mask_200k_LLava_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_filtered_objects_3mask_888_context/003-GPT-XL/checkpoints/eval_step_2000/batch_4_cfg_7.5_topk_16384.jpg new file mode 100644 index 0000000000000000000000000000000000000000..a511588d3e294b4ec099ac3fe21eb79fbafa420c --- /dev/null +++ b/CKPTS/X2I_700k_3mask_200k_LLava_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_filtered_objects_3mask_888_context/003-GPT-XL/checkpoints/eval_step_2000/batch_4_cfg_7.5_topk_16384.jpg @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:eface1e8ecaf0691e165b6df92a670303e6cc63b58092f1c232c22692bdb5abf +size 537520 diff --git a/CKPTS/X2I_700k_3mask_200k_LLava_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_filtered_objects_3mask_888_context/003-GPT-XL/checkpoints/eval_step_20000/batch_0_cfg_7.5_topk_16384.jpg b/CKPTS/X2I_700k_3mask_200k_LLava_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_filtered_objects_3mask_888_context/003-GPT-XL/checkpoints/eval_step_20000/batch_0_cfg_7.5_topk_16384.jpg new file mode 100644 index 0000000000000000000000000000000000000000..090c54cb8163bea75661b1739d5e3c7ef7d338b0 --- /dev/null +++ b/CKPTS/X2I_700k_3mask_200k_LLava_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_filtered_objects_3mask_888_context/003-GPT-XL/checkpoints/eval_step_20000/batch_0_cfg_7.5_topk_16384.jpg @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:911fb4fd0eedea0b2b2363fb4085b071d1082b8d7639d9d29b1bb521d5b40594 +size 528997 diff --git a/CKPTS/X2I_700k_3mask_200k_LLava_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_filtered_objects_3mask_888_context/003-GPT-XL/checkpoints/eval_step_20000/batch_1_cfg_7.5_topk_16384.jpg b/CKPTS/X2I_700k_3mask_200k_LLava_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_filtered_objects_3mask_888_context/003-GPT-XL/checkpoints/eval_step_20000/batch_1_cfg_7.5_topk_16384.jpg new file mode 100644 index 0000000000000000000000000000000000000000..2f314ab54b142c209676348bba9fdda4a5621c1e --- /dev/null +++ b/CKPTS/X2I_700k_3mask_200k_LLava_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_filtered_objects_3mask_888_context/003-GPT-XL/checkpoints/eval_step_20000/batch_1_cfg_7.5_topk_16384.jpg @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:61a2b385d293c74b5bc56c1086259c3ec61e595d4461e2ac29805d61bd1a5fd1 +size 563431 diff --git a/CKPTS/X2I_700k_3mask_200k_LLava_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_filtered_objects_3mask_888_context/003-GPT-XL/checkpoints/eval_step_20000/batch_2_cfg_7.5_topk_16384.jpg b/CKPTS/X2I_700k_3mask_200k_LLava_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_filtered_objects_3mask_888_context/003-GPT-XL/checkpoints/eval_step_20000/batch_2_cfg_7.5_topk_16384.jpg new file mode 100644 index 0000000000000000000000000000000000000000..352bef6258fc71b02f391c5d6d9bfb4429db01a5 --- /dev/null +++ b/CKPTS/X2I_700k_3mask_200k_LLava_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_filtered_objects_3mask_888_context/003-GPT-XL/checkpoints/eval_step_20000/batch_2_cfg_7.5_topk_16384.jpg @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:9bcff0ec6d20a382638cc8afa6cebf57670530147fbae3d51ff3b7c9cb90f79d +size 683824 diff --git a/CKPTS/X2I_700k_3mask_200k_LLava_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_filtered_objects_3mask_888_context/003-GPT-XL/checkpoints/eval_step_20000/batch_3_cfg_7.5_topk_16384.jpg b/CKPTS/X2I_700k_3mask_200k_LLava_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_filtered_objects_3mask_888_context/003-GPT-XL/checkpoints/eval_step_20000/batch_3_cfg_7.5_topk_16384.jpg new file mode 100644 index 0000000000000000000000000000000000000000..1410adf7eb20a4b7ae3dc08852a7498660bee8fb --- /dev/null +++ b/CKPTS/X2I_700k_3mask_200k_LLava_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_filtered_objects_3mask_888_context/003-GPT-XL/checkpoints/eval_step_20000/batch_3_cfg_7.5_topk_16384.jpg @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:2663841b811cd6f616e7676362cca90945142cedcd8ad19b4c92659a15bfc617 +size 516362 diff --git a/CKPTS/X2I_700k_3mask_200k_LLava_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_filtered_objects_3mask_888_context/003-GPT-XL/checkpoints/eval_step_20000/batch_4_cfg_7.5_topk_16384.jpg b/CKPTS/X2I_700k_3mask_200k_LLava_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_filtered_objects_3mask_888_context/003-GPT-XL/checkpoints/eval_step_20000/batch_4_cfg_7.5_topk_16384.jpg new file mode 100644 index 0000000000000000000000000000000000000000..17521631b01a672f96a96851e3c58973c473ae15 --- /dev/null +++ b/CKPTS/X2I_700k_3mask_200k_LLava_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_filtered_objects_3mask_888_context/003-GPT-XL/checkpoints/eval_step_20000/batch_4_cfg_7.5_topk_16384.jpg @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:d270fbc129fb5e0e9ae120dd8c83f63f6a9a50e1cf8b80ab847112e88847b5d4 +size 542855 diff --git a/CKPTS/X2I_700k_3mask_200k_LLava_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_filtered_objects_3mask_888_context/003-GPT-XL/checkpoints/eval_step_22000/batch_0_cfg_7.5_topk_16384.jpg b/CKPTS/X2I_700k_3mask_200k_LLava_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_filtered_objects_3mask_888_context/003-GPT-XL/checkpoints/eval_step_22000/batch_0_cfg_7.5_topk_16384.jpg new file mode 100644 index 0000000000000000000000000000000000000000..b0fa83a940e6f1b1e4f3e890c899f750c91e1797 --- /dev/null +++ b/CKPTS/X2I_700k_3mask_200k_LLava_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_filtered_objects_3mask_888_context/003-GPT-XL/checkpoints/eval_step_22000/batch_0_cfg_7.5_topk_16384.jpg @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:de7c3f2db2c04d4991fbb1a67e406e2efa6d0a3dd549ca18e1f49522d91cdc23 +size 527708 diff --git a/CKPTS/X2I_700k_3mask_200k_LLava_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_filtered_objects_3mask_888_context/003-GPT-XL/checkpoints/eval_step_22000/batch_1_cfg_7.5_topk_16384.jpg b/CKPTS/X2I_700k_3mask_200k_LLava_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_filtered_objects_3mask_888_context/003-GPT-XL/checkpoints/eval_step_22000/batch_1_cfg_7.5_topk_16384.jpg new file mode 100644 index 0000000000000000000000000000000000000000..24a0510a0941689c2912a3528894e361a0380e0c --- /dev/null +++ b/CKPTS/X2I_700k_3mask_200k_LLava_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_filtered_objects_3mask_888_context/003-GPT-XL/checkpoints/eval_step_22000/batch_1_cfg_7.5_topk_16384.jpg @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:b5148bbb1723486868b801bfddd4b1feed0ca47ad07fe47ff281ffc6e298cffe +size 552858 diff --git a/CKPTS/X2I_700k_3mask_200k_LLava_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_filtered_objects_3mask_888_context/003-GPT-XL/checkpoints/eval_step_22000/batch_2_cfg_7.5_topk_16384.jpg b/CKPTS/X2I_700k_3mask_200k_LLava_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_filtered_objects_3mask_888_context/003-GPT-XL/checkpoints/eval_step_22000/batch_2_cfg_7.5_topk_16384.jpg new file mode 100644 index 0000000000000000000000000000000000000000..a1cb71fdf67d37a9dcdd2d4c2640738fc46794dd --- /dev/null +++ b/CKPTS/X2I_700k_3mask_200k_LLava_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_filtered_objects_3mask_888_context/003-GPT-XL/checkpoints/eval_step_22000/batch_2_cfg_7.5_topk_16384.jpg @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:3eecce08f1e02ccdedc2496ffd0a0a57371b97211c6d7dc3e679b46379eeb21b +size 670634 diff --git a/CKPTS/X2I_700k_3mask_200k_LLava_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_filtered_objects_3mask_888_context/003-GPT-XL/checkpoints/eval_step_22000/batch_3_cfg_7.5_topk_16384.jpg b/CKPTS/X2I_700k_3mask_200k_LLava_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_filtered_objects_3mask_888_context/003-GPT-XL/checkpoints/eval_step_22000/batch_3_cfg_7.5_topk_16384.jpg new file mode 100644 index 0000000000000000000000000000000000000000..472eca86783ba37a9392bd10230c679e03cb2d1c --- /dev/null +++ b/CKPTS/X2I_700k_3mask_200k_LLava_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_filtered_objects_3mask_888_context/003-GPT-XL/checkpoints/eval_step_22000/batch_3_cfg_7.5_topk_16384.jpg @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:bdcfe7cb1f1d3b6324d4afd4b07063875e3b63558dee611757ea8e28b94128e7 +size 527274 diff --git a/CKPTS/X2I_700k_3mask_200k_LLava_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_filtered_objects_3mask_888_context/003-GPT-XL/checkpoints/eval_step_22000/batch_4_cfg_7.5_topk_16384.jpg b/CKPTS/X2I_700k_3mask_200k_LLava_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_filtered_objects_3mask_888_context/003-GPT-XL/checkpoints/eval_step_22000/batch_4_cfg_7.5_topk_16384.jpg new file mode 100644 index 0000000000000000000000000000000000000000..99db3b015fe06e412c40e684f38123e2f7b1fcd6 --- /dev/null +++ b/CKPTS/X2I_700k_3mask_200k_LLava_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_filtered_objects_3mask_888_context/003-GPT-XL/checkpoints/eval_step_22000/batch_4_cfg_7.5_topk_16384.jpg @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:4497236631359f74576b3e53c84cd715b8b719f522c2be28cb3966b0ef148d95 +size 549314 diff --git a/CKPTS/X2I_700k_3mask_200k_LLava_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_filtered_objects_3mask_888_context/003-GPT-XL/checkpoints/eval_step_24000/batch_0_cfg_7.5_topk_16384.jpg b/CKPTS/X2I_700k_3mask_200k_LLava_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_filtered_objects_3mask_888_context/003-GPT-XL/checkpoints/eval_step_24000/batch_0_cfg_7.5_topk_16384.jpg new file mode 100644 index 0000000000000000000000000000000000000000..ab68124aaa15944b2967db2d62138f05e74b3787 --- /dev/null +++ b/CKPTS/X2I_700k_3mask_200k_LLava_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_filtered_objects_3mask_888_context/003-GPT-XL/checkpoints/eval_step_24000/batch_0_cfg_7.5_topk_16384.jpg @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:e3bbef9599677b3a57a942b94ad6c3bf796874bf29e348a58227515c4d69b5ae +size 527883 diff --git a/CKPTS/X2I_700k_3mask_200k_LLava_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_filtered_objects_3mask_888_context/003-GPT-XL/checkpoints/eval_step_24000/batch_1_cfg_7.5_topk_16384.jpg b/CKPTS/X2I_700k_3mask_200k_LLava_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_filtered_objects_3mask_888_context/003-GPT-XL/checkpoints/eval_step_24000/batch_1_cfg_7.5_topk_16384.jpg new file mode 100644 index 0000000000000000000000000000000000000000..e651b5a802bb2cd155c81d7665119943231e15a9 --- /dev/null +++ b/CKPTS/X2I_700k_3mask_200k_LLava_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_filtered_objects_3mask_888_context/003-GPT-XL/checkpoints/eval_step_24000/batch_1_cfg_7.5_topk_16384.jpg @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:ba8305bf7d01f51a7decac3bfcd36b884545a56f2b90caffae957178e1535c55 +size 565724 diff --git a/CKPTS/X2I_700k_3mask_200k_LLava_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_filtered_objects_3mask_888_context/003-GPT-XL/checkpoints/eval_step_24000/batch_2_cfg_7.5_topk_16384.jpg b/CKPTS/X2I_700k_3mask_200k_LLava_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_filtered_objects_3mask_888_context/003-GPT-XL/checkpoints/eval_step_24000/batch_2_cfg_7.5_topk_16384.jpg new file mode 100644 index 0000000000000000000000000000000000000000..66b33f03cfcb6d11688c359267f87c026dfa70c0 --- /dev/null +++ b/CKPTS/X2I_700k_3mask_200k_LLava_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_filtered_objects_3mask_888_context/003-GPT-XL/checkpoints/eval_step_24000/batch_2_cfg_7.5_topk_16384.jpg @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:f5899cb8dfced11bfb63ceaf8eee36956b8424b4f3bda6957138db486625a756 +size 664259 diff --git a/CKPTS/X2I_700k_3mask_200k_LLava_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_filtered_objects_3mask_888_context/003-GPT-XL/checkpoints/eval_step_24000/batch_3_cfg_7.5_topk_16384.jpg b/CKPTS/X2I_700k_3mask_200k_LLava_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_filtered_objects_3mask_888_context/003-GPT-XL/checkpoints/eval_step_24000/batch_3_cfg_7.5_topk_16384.jpg new file mode 100644 index 0000000000000000000000000000000000000000..def4b00758b566972b72a9295edb8f247a5a4120 --- /dev/null +++ b/CKPTS/X2I_700k_3mask_200k_LLava_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_filtered_objects_3mask_888_context/003-GPT-XL/checkpoints/eval_step_24000/batch_3_cfg_7.5_topk_16384.jpg @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:242bfae3eab8c02306ee52fdc7e1e0665b70806447ccd636fb8d0c4df6f9ed3e +size 516561 diff --git a/CKPTS/X2I_700k_3mask_200k_LLava_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_filtered_objects_3mask_888_context/003-GPT-XL/checkpoints/eval_step_24000/batch_4_cfg_7.5_topk_16384.jpg b/CKPTS/X2I_700k_3mask_200k_LLava_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_filtered_objects_3mask_888_context/003-GPT-XL/checkpoints/eval_step_24000/batch_4_cfg_7.5_topk_16384.jpg new file mode 100644 index 0000000000000000000000000000000000000000..8f714c706155afdedb5d67b29ba5ee411c196138 --- /dev/null +++ b/CKPTS/X2I_700k_3mask_200k_LLava_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_filtered_objects_3mask_888_context/003-GPT-XL/checkpoints/eval_step_24000/batch_4_cfg_7.5_topk_16384.jpg @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:c48cb3ac24429f24a07df0f9316b98fb0b97240ffa1656f6684ea4f91a5a2bab +size 538606 diff --git a/CKPTS/X2I_700k_3mask_200k_LLava_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_filtered_objects_3mask_888_context/003-GPT-XL/checkpoints/eval_step_26000/batch_0_cfg_7.5_topk_16384.jpg b/CKPTS/X2I_700k_3mask_200k_LLava_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_filtered_objects_3mask_888_context/003-GPT-XL/checkpoints/eval_step_26000/batch_0_cfg_7.5_topk_16384.jpg new file mode 100644 index 0000000000000000000000000000000000000000..becb414865138d46858d91f5de010ba77c0c6f48 --- /dev/null +++ b/CKPTS/X2I_700k_3mask_200k_LLava_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_filtered_objects_3mask_888_context/003-GPT-XL/checkpoints/eval_step_26000/batch_0_cfg_7.5_topk_16384.jpg @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:0b88163241826f7cbf89a7bccfe975774803dfd96b3488ca42e329404fbf0b2b +size 537670 diff --git a/CKPTS/X2I_700k_3mask_200k_LLava_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_filtered_objects_3mask_888_context/003-GPT-XL/checkpoints/eval_step_26000/batch_1_cfg_7.5_topk_16384.jpg b/CKPTS/X2I_700k_3mask_200k_LLava_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_filtered_objects_3mask_888_context/003-GPT-XL/checkpoints/eval_step_26000/batch_1_cfg_7.5_topk_16384.jpg new file mode 100644 index 0000000000000000000000000000000000000000..4148f7b172c65dc4d2fdf38f09e8f9849d7a93b5 --- /dev/null +++ b/CKPTS/X2I_700k_3mask_200k_LLava_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_filtered_objects_3mask_888_context/003-GPT-XL/checkpoints/eval_step_26000/batch_1_cfg_7.5_topk_16384.jpg @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:5954c9675584562c55d7590db01750699fc061bbae31b052145d90611ac21be4 +size 561974 diff --git a/CKPTS/X2I_700k_3mask_200k_LLava_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_filtered_objects_3mask_888_context/003-GPT-XL/checkpoints/eval_step_26000/batch_2_cfg_7.5_topk_16384.jpg b/CKPTS/X2I_700k_3mask_200k_LLava_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_filtered_objects_3mask_888_context/003-GPT-XL/checkpoints/eval_step_26000/batch_2_cfg_7.5_topk_16384.jpg new file mode 100644 index 0000000000000000000000000000000000000000..11ff0f188a598cdb82dd4de21ca3e04803dadd1d --- /dev/null +++ b/CKPTS/X2I_700k_3mask_200k_LLava_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_filtered_objects_3mask_888_context/003-GPT-XL/checkpoints/eval_step_26000/batch_2_cfg_7.5_topk_16384.jpg @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:0338d93933a3ca89a59e83512f75f03080befafcafd0d783ccebeeb0656ec4b0 +size 671228 diff --git a/CKPTS/X2I_700k_3mask_200k_LLava_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_filtered_objects_3mask_888_context/003-GPT-XL/checkpoints/eval_step_26000/batch_3_cfg_7.5_topk_16384.jpg b/CKPTS/X2I_700k_3mask_200k_LLava_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_filtered_objects_3mask_888_context/003-GPT-XL/checkpoints/eval_step_26000/batch_3_cfg_7.5_topk_16384.jpg new file mode 100644 index 0000000000000000000000000000000000000000..d0ced96948c0ded3ea789ffbe21e6172bea0a202 --- /dev/null +++ b/CKPTS/X2I_700k_3mask_200k_LLava_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_filtered_objects_3mask_888_context/003-GPT-XL/checkpoints/eval_step_26000/batch_3_cfg_7.5_topk_16384.jpg @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:ad248ca35b209302a41381ccf2e7721d70e469f48f048743b849a5910e782591 +size 509044 diff --git a/CKPTS/X2I_700k_3mask_200k_LLava_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_filtered_objects_3mask_888_context/003-GPT-XL/checkpoints/eval_step_26000/batch_4_cfg_7.5_topk_16384.jpg b/CKPTS/X2I_700k_3mask_200k_LLava_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_filtered_objects_3mask_888_context/003-GPT-XL/checkpoints/eval_step_26000/batch_4_cfg_7.5_topk_16384.jpg new file mode 100644 index 0000000000000000000000000000000000000000..813768a4fa75a5747ff9f3536f748689e0c70666 --- /dev/null +++ b/CKPTS/X2I_700k_3mask_200k_LLava_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_filtered_objects_3mask_888_context/003-GPT-XL/checkpoints/eval_step_26000/batch_4_cfg_7.5_topk_16384.jpg @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:fae710a62c0cc4755b19d221c80d31274527afed1f055b462b3688ecbaf6948a +size 540755 diff --git a/CKPTS/X2I_700k_3mask_200k_LLava_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_filtered_objects_3mask_888_context/003-GPT-XL/checkpoints/eval_step_28000/batch_0_cfg_7.5_topk_16384.jpg b/CKPTS/X2I_700k_3mask_200k_LLava_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_filtered_objects_3mask_888_context/003-GPT-XL/checkpoints/eval_step_28000/batch_0_cfg_7.5_topk_16384.jpg new file mode 100644 index 0000000000000000000000000000000000000000..dfee275803db45871e7b850ce1edc2173db3d7a1 --- /dev/null +++ b/CKPTS/X2I_700k_3mask_200k_LLava_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_filtered_objects_3mask_888_context/003-GPT-XL/checkpoints/eval_step_28000/batch_0_cfg_7.5_topk_16384.jpg @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:103b36e9baf4eaed0f87e7fa1b888177c2cfb3e12204b48ab344ab90b4379912 +size 528364 diff --git a/CKPTS/X2I_700k_3mask_200k_LLava_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_filtered_objects_3mask_888_context/003-GPT-XL/checkpoints/eval_step_28000/batch_1_cfg_7.5_topk_16384.jpg b/CKPTS/X2I_700k_3mask_200k_LLava_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_filtered_objects_3mask_888_context/003-GPT-XL/checkpoints/eval_step_28000/batch_1_cfg_7.5_topk_16384.jpg new file mode 100644 index 0000000000000000000000000000000000000000..bec95f5cc2b425b1360f87b8185c229aee40d588 --- /dev/null +++ b/CKPTS/X2I_700k_3mask_200k_LLava_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_filtered_objects_3mask_888_context/003-GPT-XL/checkpoints/eval_step_28000/batch_1_cfg_7.5_topk_16384.jpg @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:3fad9e463c0819ed3b039a7012d8bc07ef8e8a472b5b92c5922ade93a41958d6 +size 568839 diff --git a/CKPTS/X2I_700k_3mask_200k_LLava_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_filtered_objects_3mask_888_context/003-GPT-XL/checkpoints/eval_step_28000/batch_2_cfg_7.5_topk_16384.jpg b/CKPTS/X2I_700k_3mask_200k_LLava_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_filtered_objects_3mask_888_context/003-GPT-XL/checkpoints/eval_step_28000/batch_2_cfg_7.5_topk_16384.jpg new file mode 100644 index 0000000000000000000000000000000000000000..5091041c7b2bbac61325b63a6b0fa5826d1f8406 --- /dev/null +++ b/CKPTS/X2I_700k_3mask_200k_LLava_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_filtered_objects_3mask_888_context/003-GPT-XL/checkpoints/eval_step_28000/batch_2_cfg_7.5_topk_16384.jpg @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:e861c6aecf42869cb33059f96d06748cfd33fa2e9e99c97633c4816e03dc4262 +size 662309 diff --git a/CKPTS/X2I_700k_3mask_200k_LLava_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_filtered_objects_3mask_888_context/003-GPT-XL/checkpoints/eval_step_28000/batch_3_cfg_7.5_topk_16384.jpg b/CKPTS/X2I_700k_3mask_200k_LLava_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_filtered_objects_3mask_888_context/003-GPT-XL/checkpoints/eval_step_28000/batch_3_cfg_7.5_topk_16384.jpg new file mode 100644 index 0000000000000000000000000000000000000000..915ebf8c9d999879fa678dabadb7586830faf1e5 --- /dev/null +++ b/CKPTS/X2I_700k_3mask_200k_LLava_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_filtered_objects_3mask_888_context/003-GPT-XL/checkpoints/eval_step_28000/batch_3_cfg_7.5_topk_16384.jpg @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:4283c34f9ce2660c6743e2c846cb51e436eb9a01c2b7aa097013e948c8d13f7c +size 509625 diff --git a/CKPTS/X2I_700k_3mask_200k_LLava_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_filtered_objects_3mask_888_context/003-GPT-XL/checkpoints/eval_step_28000/batch_4_cfg_7.5_topk_16384.jpg b/CKPTS/X2I_700k_3mask_200k_LLava_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_filtered_objects_3mask_888_context/003-GPT-XL/checkpoints/eval_step_28000/batch_4_cfg_7.5_topk_16384.jpg new file mode 100644 index 0000000000000000000000000000000000000000..09e886cef28cafd38360947efb45feeb22675f12 --- /dev/null +++ b/CKPTS/X2I_700k_3mask_200k_LLava_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_filtered_objects_3mask_888_context/003-GPT-XL/checkpoints/eval_step_28000/batch_4_cfg_7.5_topk_16384.jpg @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:722d1e1f09e4d1b11d22ada0c25006675d14c26fe53fb9c025bd666cf24030a8 +size 540252 diff --git a/CKPTS/X2I_700k_3mask_200k_LLava_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_filtered_objects_3mask_888_context/003-GPT-XL/checkpoints/eval_step_30000/batch_0_cfg_7.5_topk_16384.jpg b/CKPTS/X2I_700k_3mask_200k_LLava_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_filtered_objects_3mask_888_context/003-GPT-XL/checkpoints/eval_step_30000/batch_0_cfg_7.5_topk_16384.jpg new file mode 100644 index 0000000000000000000000000000000000000000..051122380a3273cf8e8838d537a4ec59ce01dbd2 --- /dev/null +++ b/CKPTS/X2I_700k_3mask_200k_LLava_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_filtered_objects_3mask_888_context/003-GPT-XL/checkpoints/eval_step_30000/batch_0_cfg_7.5_topk_16384.jpg @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:a99b4547444654971fc8e54a7b9234cf2019197f54d8aba56ec9e62282febe2b +size 521706 diff --git a/CKPTS/X2I_700k_3mask_200k_LLava_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_filtered_objects_3mask_888_context/003-GPT-XL/checkpoints/eval_step_30000/batch_1_cfg_7.5_topk_16384.jpg b/CKPTS/X2I_700k_3mask_200k_LLava_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_filtered_objects_3mask_888_context/003-GPT-XL/checkpoints/eval_step_30000/batch_1_cfg_7.5_topk_16384.jpg new file mode 100644 index 0000000000000000000000000000000000000000..1de4d0737c1f9687f6968e01759c56545b6dd315 --- /dev/null +++ b/CKPTS/X2I_700k_3mask_200k_LLava_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_filtered_objects_3mask_888_context/003-GPT-XL/checkpoints/eval_step_30000/batch_1_cfg_7.5_topk_16384.jpg @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:ac6170f820ca820bd44aa7b685474c0d66e2b849614de21be75efa8f71d6ad55 +size 564889 diff --git a/CKPTS/X2I_700k_3mask_200k_LLava_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_filtered_objects_3mask_888_context/003-GPT-XL/checkpoints/eval_step_30000/batch_2_cfg_7.5_topk_16384.jpg b/CKPTS/X2I_700k_3mask_200k_LLava_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_filtered_objects_3mask_888_context/003-GPT-XL/checkpoints/eval_step_30000/batch_2_cfg_7.5_topk_16384.jpg new file mode 100644 index 0000000000000000000000000000000000000000..47723c8f145861f67708806843ea538dd72fa677 --- /dev/null +++ b/CKPTS/X2I_700k_3mask_200k_LLava_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_filtered_objects_3mask_888_context/003-GPT-XL/checkpoints/eval_step_30000/batch_2_cfg_7.5_topk_16384.jpg @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:2e208376628e1bf52752a49bea565e0d01dfbcad0d150974c7cdd0c249d17b11 +size 675928 diff --git a/CKPTS/X2I_700k_3mask_200k_LLava_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_filtered_objects_3mask_888_context/003-GPT-XL/checkpoints/eval_step_30000/batch_3_cfg_7.5_topk_16384.jpg b/CKPTS/X2I_700k_3mask_200k_LLava_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_filtered_objects_3mask_888_context/003-GPT-XL/checkpoints/eval_step_30000/batch_3_cfg_7.5_topk_16384.jpg new file mode 100644 index 0000000000000000000000000000000000000000..8411299907c50d4d8a0f86f558ae44db21b53b35 --- /dev/null +++ b/CKPTS/X2I_700k_3mask_200k_LLava_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_filtered_objects_3mask_888_context/003-GPT-XL/checkpoints/eval_step_30000/batch_3_cfg_7.5_topk_16384.jpg @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:cf3d1d4e1c1d986f95f64895c5a1b3eaf772d7534ef7e5100b2a8c849c94fb1a +size 518096 diff --git a/CKPTS/X2I_700k_3mask_200k_LLava_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_filtered_objects_3mask_888_context/003-GPT-XL/checkpoints/eval_step_30000/batch_4_cfg_7.5_topk_16384.jpg b/CKPTS/X2I_700k_3mask_200k_LLava_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_filtered_objects_3mask_888_context/003-GPT-XL/checkpoints/eval_step_30000/batch_4_cfg_7.5_topk_16384.jpg new file mode 100644 index 0000000000000000000000000000000000000000..083817ce2d149aacdaa3bc16d504f1101bb4d776 --- /dev/null +++ b/CKPTS/X2I_700k_3mask_200k_LLava_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_filtered_objects_3mask_888_context/003-GPT-XL/checkpoints/eval_step_30000/batch_4_cfg_7.5_topk_16384.jpg @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:6e9f2c113cf658738479b6b4a0b6d1d0aed9ae6241840c4e8dc29880431a81f5 +size 544797 diff --git a/CKPTS/X2I_700k_3mask_200k_LLava_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_filtered_objects_3mask_888_context/003-GPT-XL/checkpoints/eval_step_32000/batch_0_cfg_7.5_topk_16384.jpg b/CKPTS/X2I_700k_3mask_200k_LLava_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_filtered_objects_3mask_888_context/003-GPT-XL/checkpoints/eval_step_32000/batch_0_cfg_7.5_topk_16384.jpg new file mode 100644 index 0000000000000000000000000000000000000000..f24d81f60db7e17a4f13c2f8519eb2986d5104b6 --- /dev/null +++ b/CKPTS/X2I_700k_3mask_200k_LLava_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_filtered_objects_3mask_888_context/003-GPT-XL/checkpoints/eval_step_32000/batch_0_cfg_7.5_topk_16384.jpg @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:0af9f7bcc0d82f380c12c15b783e9e1b029d6e8f33f142e273fb95b3362dc75c +size 542230 diff --git a/CKPTS/X2I_700k_3mask_200k_LLava_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_filtered_objects_3mask_888_context/003-GPT-XL/checkpoints/eval_step_32000/batch_1_cfg_7.5_topk_16384.jpg b/CKPTS/X2I_700k_3mask_200k_LLava_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_filtered_objects_3mask_888_context/003-GPT-XL/checkpoints/eval_step_32000/batch_1_cfg_7.5_topk_16384.jpg new file mode 100644 index 0000000000000000000000000000000000000000..2432bb54b0300ffc0b932510dc97410afe049ea2 --- /dev/null +++ b/CKPTS/X2I_700k_3mask_200k_LLava_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_filtered_objects_3mask_888_context/003-GPT-XL/checkpoints/eval_step_32000/batch_1_cfg_7.5_topk_16384.jpg @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:d774fefc6fe7058dfcea4ec9995b304b13e2d9f118e3cf0edfac95c8bd386320 +size 556021 diff --git a/CKPTS/X2I_700k_3mask_200k_LLava_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_filtered_objects_3mask_888_context/003-GPT-XL/checkpoints/eval_step_32000/batch_2_cfg_7.5_topk_16384.jpg b/CKPTS/X2I_700k_3mask_200k_LLava_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_filtered_objects_3mask_888_context/003-GPT-XL/checkpoints/eval_step_32000/batch_2_cfg_7.5_topk_16384.jpg new file mode 100644 index 0000000000000000000000000000000000000000..6254b6ac996cfef05d1543e5f67323aa871b9cb9 --- /dev/null +++ b/CKPTS/X2I_700k_3mask_200k_LLava_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_filtered_objects_3mask_888_context/003-GPT-XL/checkpoints/eval_step_32000/batch_2_cfg_7.5_topk_16384.jpg @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:9139882d6ddcccf3896137edd1f9eb75afd7ed2a2f1f458c60968df5df7ee874 +size 681497 diff --git a/CKPTS/X2I_700k_3mask_200k_LLava_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_filtered_objects_3mask_888_context/003-GPT-XL/checkpoints/eval_step_32000/batch_3_cfg_7.5_topk_16384.jpg b/CKPTS/X2I_700k_3mask_200k_LLava_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_filtered_objects_3mask_888_context/003-GPT-XL/checkpoints/eval_step_32000/batch_3_cfg_7.5_topk_16384.jpg new file mode 100644 index 0000000000000000000000000000000000000000..8bfcbe0741a5d94d34c308babbba422f38bc86e0 --- /dev/null +++ b/CKPTS/X2I_700k_3mask_200k_LLava_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_filtered_objects_3mask_888_context/003-GPT-XL/checkpoints/eval_step_32000/batch_3_cfg_7.5_topk_16384.jpg @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:5915f6c76f10ecbd414f80bf1250efb217845df2f5f69acbffdb0f4cd4a8b9b4 +size 510336 diff --git a/CKPTS/X2I_700k_3mask_200k_LLava_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_filtered_objects_3mask_888_context/003-GPT-XL/checkpoints/eval_step_32000/batch_4_cfg_7.5_topk_16384.jpg b/CKPTS/X2I_700k_3mask_200k_LLava_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_filtered_objects_3mask_888_context/003-GPT-XL/checkpoints/eval_step_32000/batch_4_cfg_7.5_topk_16384.jpg new file mode 100644 index 0000000000000000000000000000000000000000..422e85eed4fe9042542cb26d98adb99b53e55f42 --- /dev/null +++ b/CKPTS/X2I_700k_3mask_200k_LLava_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_filtered_objects_3mask_888_context/003-GPT-XL/checkpoints/eval_step_32000/batch_4_cfg_7.5_topk_16384.jpg @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:ccd1c7d45992b43629b672089bd2c51063497dff746afa27d496ec1dacc2ade8 +size 562431 diff --git a/CKPTS/X2I_700k_3mask_200k_LLava_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_filtered_objects_3mask_888_context/003-GPT-XL/checkpoints/eval_step_34000/batch_0_cfg_7.5_topk_16384.jpg b/CKPTS/X2I_700k_3mask_200k_LLava_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_filtered_objects_3mask_888_context/003-GPT-XL/checkpoints/eval_step_34000/batch_0_cfg_7.5_topk_16384.jpg new file mode 100644 index 0000000000000000000000000000000000000000..2ddaaf9d9d4b25af8c18401750cb8b989b29d71d --- /dev/null +++ b/CKPTS/X2I_700k_3mask_200k_LLava_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_filtered_objects_3mask_888_context/003-GPT-XL/checkpoints/eval_step_34000/batch_0_cfg_7.5_topk_16384.jpg @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:a98ba70c59e4318aff2fd4cf4db9fcf93e2558a49f7b9bba5352842bf9492a77 +size 519774 diff --git a/CKPTS/X2I_700k_3mask_200k_LLava_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_filtered_objects_3mask_888_context/003-GPT-XL/checkpoints/eval_step_34000/batch_1_cfg_7.5_topk_16384.jpg b/CKPTS/X2I_700k_3mask_200k_LLava_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_filtered_objects_3mask_888_context/003-GPT-XL/checkpoints/eval_step_34000/batch_1_cfg_7.5_topk_16384.jpg new file mode 100644 index 0000000000000000000000000000000000000000..bfb581722617d3846eb4d8b1e1243818fb31f452 --- /dev/null +++ b/CKPTS/X2I_700k_3mask_200k_LLava_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_filtered_objects_3mask_888_context/003-GPT-XL/checkpoints/eval_step_34000/batch_1_cfg_7.5_topk_16384.jpg @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:8eb2da53523c10a9d65b8c87848a88a41dcc8b0b5ab33d26b6301461879f7797 +size 562564 diff --git a/CKPTS/X2I_700k_3mask_200k_LLava_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_filtered_objects_3mask_888_context/003-GPT-XL/checkpoints/eval_step_34000/batch_2_cfg_7.5_topk_16384.jpg b/CKPTS/X2I_700k_3mask_200k_LLava_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_filtered_objects_3mask_888_context/003-GPT-XL/checkpoints/eval_step_34000/batch_2_cfg_7.5_topk_16384.jpg new file mode 100644 index 0000000000000000000000000000000000000000..f5d97289129f11c5a11bc2180699b46ee3d68713 --- /dev/null +++ b/CKPTS/X2I_700k_3mask_200k_LLava_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_filtered_objects_3mask_888_context/003-GPT-XL/checkpoints/eval_step_34000/batch_2_cfg_7.5_topk_16384.jpg @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:311d201859affccba58bd799815a1495c35094238a60869c047a2e641c0d6c68 +size 672512 diff --git a/CKPTS/X2I_700k_3mask_200k_LLava_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_filtered_objects_3mask_888_context/003-GPT-XL/checkpoints/eval_step_4000/batch_0_cfg_7.5_topk_16384.jpg b/CKPTS/X2I_700k_3mask_200k_LLava_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_filtered_objects_3mask_888_context/003-GPT-XL/checkpoints/eval_step_4000/batch_0_cfg_7.5_topk_16384.jpg new file mode 100644 index 0000000000000000000000000000000000000000..fdc06cdc00e041694c1552e3e65a00a8fbd38e5d --- /dev/null +++ b/CKPTS/X2I_700k_3mask_200k_LLava_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_filtered_objects_3mask_888_context/003-GPT-XL/checkpoints/eval_step_4000/batch_0_cfg_7.5_topk_16384.jpg @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:721355b6d0ae6485ae0279dd9155742d10d742f5ba9e1938b7c101dc0cd75209 +size 542235 diff --git a/CKPTS/X2I_700k_3mask_200k_LLava_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_filtered_objects_3mask_888_context/003-GPT-XL/checkpoints/eval_step_4000/batch_1_cfg_7.5_topk_16384.jpg b/CKPTS/X2I_700k_3mask_200k_LLava_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_filtered_objects_3mask_888_context/003-GPT-XL/checkpoints/eval_step_4000/batch_1_cfg_7.5_topk_16384.jpg new file mode 100644 index 0000000000000000000000000000000000000000..fd21ffef5f89484206977ce23cfff0b8af828b41 --- /dev/null +++ b/CKPTS/X2I_700k_3mask_200k_LLava_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_filtered_objects_3mask_888_context/003-GPT-XL/checkpoints/eval_step_4000/batch_1_cfg_7.5_topk_16384.jpg @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:1be389bed13e6999847a5f0031b9b4c6236882edc814dbc9c4bea9dbafaec216 +size 591671 diff --git a/CKPTS/X2I_700k_3mask_200k_LLava_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_filtered_objects_3mask_888_context/003-GPT-XL/checkpoints/eval_step_4000/batch_2_cfg_7.5_topk_16384.jpg b/CKPTS/X2I_700k_3mask_200k_LLava_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_filtered_objects_3mask_888_context/003-GPT-XL/checkpoints/eval_step_4000/batch_2_cfg_7.5_topk_16384.jpg new file mode 100644 index 0000000000000000000000000000000000000000..6a645eb6ef1ad25e53f2bcf1d34969885ff2ba8e --- /dev/null +++ b/CKPTS/X2I_700k_3mask_200k_LLava_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_filtered_objects_3mask_888_context/003-GPT-XL/checkpoints/eval_step_4000/batch_2_cfg_7.5_topk_16384.jpg @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:8630293e49ed0958563cb6ed3464632ab46331ccc97217b718b7f98be6a75775 +size 667619 diff --git a/CKPTS/X2I_700k_3mask_200k_LLava_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_filtered_objects_3mask_888_context/003-GPT-XL/checkpoints/eval_step_4000/batch_3_cfg_7.5_topk_16384.jpg b/CKPTS/X2I_700k_3mask_200k_LLava_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_filtered_objects_3mask_888_context/003-GPT-XL/checkpoints/eval_step_4000/batch_3_cfg_7.5_topk_16384.jpg new file mode 100644 index 0000000000000000000000000000000000000000..bf6c2429876e702f70c6e7e085eeefea175022f6 --- /dev/null +++ b/CKPTS/X2I_700k_3mask_200k_LLava_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_filtered_objects_3mask_888_context/003-GPT-XL/checkpoints/eval_step_4000/batch_3_cfg_7.5_topk_16384.jpg @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:3715206cfd52ef0d5f2c8082aa94f3c852782952eb33ecc783e69cd6e8d41aec +size 551029 diff --git a/CKPTS/X2I_700k_3mask_200k_LLava_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_filtered_objects_3mask_888_context/003-GPT-XL/checkpoints/eval_step_4000/batch_4_cfg_7.5_topk_16384.jpg b/CKPTS/X2I_700k_3mask_200k_LLava_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_filtered_objects_3mask_888_context/003-GPT-XL/checkpoints/eval_step_4000/batch_4_cfg_7.5_topk_16384.jpg new file mode 100644 index 0000000000000000000000000000000000000000..f5f316ed8167b190cb6bcf4b09aae74bae930ed3 --- /dev/null +++ b/CKPTS/X2I_700k_3mask_200k_LLava_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_filtered_objects_3mask_888_context/003-GPT-XL/checkpoints/eval_step_4000/batch_4_cfg_7.5_topk_16384.jpg @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:88590d0c5fde87d65904cca26e53c9c3312339b636ebc917df3212ce7744ed1e +size 548641 diff --git a/CKPTS/X2I_700k_3mask_200k_LLava_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_filtered_objects_3mask_888_context/003-GPT-XL/checkpoints/eval_step_6000/batch_0_cfg_7.5_topk_16384.jpg b/CKPTS/X2I_700k_3mask_200k_LLava_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_filtered_objects_3mask_888_context/003-GPT-XL/checkpoints/eval_step_6000/batch_0_cfg_7.5_topk_16384.jpg new file mode 100644 index 0000000000000000000000000000000000000000..cd4ec4be8826adc7f85be35b8fca85aeead93d12 --- /dev/null +++ b/CKPTS/X2I_700k_3mask_200k_LLava_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_filtered_objects_3mask_888_context/003-GPT-XL/checkpoints/eval_step_6000/batch_0_cfg_7.5_topk_16384.jpg @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:e7fb777e0cd6faf8b52da15cf095e926e3747d5ca9476bb1134de72c7e33f6bc +size 542815 diff --git a/CKPTS/X2I_700k_3mask_200k_LLava_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_filtered_objects_3mask_888_context/003-GPT-XL/checkpoints/eval_step_6000/batch_1_cfg_7.5_topk_16384.jpg b/CKPTS/X2I_700k_3mask_200k_LLava_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_filtered_objects_3mask_888_context/003-GPT-XL/checkpoints/eval_step_6000/batch_1_cfg_7.5_topk_16384.jpg new file mode 100644 index 0000000000000000000000000000000000000000..085ee2e42426bd44486a5b5bbe8c9309ef4e6763 --- /dev/null +++ b/CKPTS/X2I_700k_3mask_200k_LLava_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_filtered_objects_3mask_888_context/003-GPT-XL/checkpoints/eval_step_6000/batch_1_cfg_7.5_topk_16384.jpg @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:fb4fd027fd32785631f19aeb48822122759c336025030fb6053441875af71aab +size 595903 diff --git a/CKPTS/X2I_700k_3mask_200k_LLava_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_filtered_objects_3mask_888_context/003-GPT-XL/checkpoints/eval_step_6000/batch_2_cfg_7.5_topk_16384.jpg b/CKPTS/X2I_700k_3mask_200k_LLava_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_filtered_objects_3mask_888_context/003-GPT-XL/checkpoints/eval_step_6000/batch_2_cfg_7.5_topk_16384.jpg new file mode 100644 index 0000000000000000000000000000000000000000..57d27a79089a9592bdf03ffa55ed623ecd9563fe --- /dev/null +++ b/CKPTS/X2I_700k_3mask_200k_LLava_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_filtered_objects_3mask_888_context/003-GPT-XL/checkpoints/eval_step_6000/batch_2_cfg_7.5_topk_16384.jpg @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:b27fa2862580123741ddfb8ea12a807e75a61a5aa2181aff3c6e6af4ee75d698 +size 687200 diff --git a/CKPTS/X2I_700k_3mask_200k_LLava_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_filtered_objects_3mask_888_context/003-GPT-XL/checkpoints/eval_step_6000/batch_3_cfg_7.5_topk_16384.jpg b/CKPTS/X2I_700k_3mask_200k_LLava_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_filtered_objects_3mask_888_context/003-GPT-XL/checkpoints/eval_step_6000/batch_3_cfg_7.5_topk_16384.jpg new file mode 100644 index 0000000000000000000000000000000000000000..91c9127fc59070e9249971a116943ba6e0e705b8 --- /dev/null +++ b/CKPTS/X2I_700k_3mask_200k_LLava_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_filtered_objects_3mask_888_context/003-GPT-XL/checkpoints/eval_step_6000/batch_3_cfg_7.5_topk_16384.jpg @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:2b76502e6a901cd0153da6d2cdbcb814b443006794ec91ac05b40f0b5e927c6d +size 561323 diff --git a/CKPTS/X2I_700k_3mask_200k_LLava_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_filtered_objects_3mask_888_context/003-GPT-XL/checkpoints/eval_step_6000/batch_4_cfg_7.5_topk_16384.jpg b/CKPTS/X2I_700k_3mask_200k_LLava_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_filtered_objects_3mask_888_context/003-GPT-XL/checkpoints/eval_step_6000/batch_4_cfg_7.5_topk_16384.jpg new file mode 100644 index 0000000000000000000000000000000000000000..0a075bba1ee6863bb7caba511ef12c891234177b --- /dev/null +++ b/CKPTS/X2I_700k_3mask_200k_LLava_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_filtered_objects_3mask_888_context/003-GPT-XL/checkpoints/eval_step_6000/batch_4_cfg_7.5_topk_16384.jpg @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:dbf15960a057fe4f56b8836db2ba4173f85e7673dacbe095087d5ee8edb667ed +size 538104 diff --git a/CKPTS/X2I_700k_3mask_200k_LLava_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_filtered_objects_3mask_888_context/003-GPT-XL/checkpoints/eval_step_8000/batch_0_cfg_7.5_topk_16384.jpg b/CKPTS/X2I_700k_3mask_200k_LLava_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_filtered_objects_3mask_888_context/003-GPT-XL/checkpoints/eval_step_8000/batch_0_cfg_7.5_topk_16384.jpg new file mode 100644 index 0000000000000000000000000000000000000000..61c11d9e4215828e06572990d6d43e0fd36637d5 --- /dev/null +++ b/CKPTS/X2I_700k_3mask_200k_LLava_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_filtered_objects_3mask_888_context/003-GPT-XL/checkpoints/eval_step_8000/batch_0_cfg_7.5_topk_16384.jpg @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:af46dea4fba5161a1377b56f74afef69ba851e2119181039ade4603dcf2fc9ad +size 540342 diff --git a/CKPTS/X2I_700k_3mask_200k_LLava_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_filtered_objects_3mask_888_context/003-GPT-XL/checkpoints/eval_step_8000/batch_1_cfg_7.5_topk_16384.jpg b/CKPTS/X2I_700k_3mask_200k_LLava_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_filtered_objects_3mask_888_context/003-GPT-XL/checkpoints/eval_step_8000/batch_1_cfg_7.5_topk_16384.jpg new file mode 100644 index 0000000000000000000000000000000000000000..7673fdf291ca4fd549748f5a8d37dd11ac5624ce --- /dev/null +++ b/CKPTS/X2I_700k_3mask_200k_LLava_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_filtered_objects_3mask_888_context/003-GPT-XL/checkpoints/eval_step_8000/batch_1_cfg_7.5_topk_16384.jpg @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:ba2d9877f0779c42bce4cbee38fe5b01c656038866bf08bd1f5ffb18e313c96d +size 574535 diff --git a/CKPTS/X2I_700k_3mask_200k_LLava_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_filtered_objects_3mask_888_context/003-GPT-XL/checkpoints/eval_step_8000/batch_2_cfg_7.5_topk_16384.jpg b/CKPTS/X2I_700k_3mask_200k_LLava_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_filtered_objects_3mask_888_context/003-GPT-XL/checkpoints/eval_step_8000/batch_2_cfg_7.5_topk_16384.jpg new file mode 100644 index 0000000000000000000000000000000000000000..44a74133ef55e023447cbb6551e17407ac20f1ed --- /dev/null +++ b/CKPTS/X2I_700k_3mask_200k_LLava_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_filtered_objects_3mask_888_context/003-GPT-XL/checkpoints/eval_step_8000/batch_2_cfg_7.5_topk_16384.jpg @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:264ee93c5aa9e1c45d6fbbf7d0e6de06fd2fc6db3543b94a5ca3c4a231e90dfb +size 663122 diff --git a/CKPTS/X2I_700k_3mask_200k_LLava_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_filtered_objects_3mask_888_context/003-GPT-XL/checkpoints/eval_step_8000/batch_3_cfg_7.5_topk_16384.jpg b/CKPTS/X2I_700k_3mask_200k_LLava_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_filtered_objects_3mask_888_context/003-GPT-XL/checkpoints/eval_step_8000/batch_3_cfg_7.5_topk_16384.jpg new file mode 100644 index 0000000000000000000000000000000000000000..b1b2c4418368cc1a186003a19c68fa93795b1b64 --- /dev/null +++ b/CKPTS/X2I_700k_3mask_200k_LLava_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_filtered_objects_3mask_888_context/003-GPT-XL/checkpoints/eval_step_8000/batch_3_cfg_7.5_topk_16384.jpg @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:205c5a54e23f3854ce3dab9a9473d23c907abadde47e6e1f50c37643b8d2d69e +size 542699 diff --git a/CKPTS/X2I_700k_3mask_200k_LLava_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_filtered_objects_3mask_888_context/003-GPT-XL/checkpoints/eval_step_8000/batch_4_cfg_7.5_topk_16384.jpg b/CKPTS/X2I_700k_3mask_200k_LLava_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_filtered_objects_3mask_888_context/003-GPT-XL/checkpoints/eval_step_8000/batch_4_cfg_7.5_topk_16384.jpg new file mode 100644 index 0000000000000000000000000000000000000000..edeeedfcde89f1ddb7cb04831d241f51efa43abf --- /dev/null +++ b/CKPTS/X2I_700k_3mask_200k_LLava_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_filtered_objects_3mask_888_context/003-GPT-XL/checkpoints/eval_step_8000/batch_4_cfg_7.5_topk_16384.jpg @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:a71a53495d9225b8ef5658df6e45cfdd970f0895859349a25d703cb713523f31 +size 561086 diff --git a/CKPTS/X2I_700k_3mask_200k_LLava_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_filtered_objects_3mask_888_context/003-GPT-XL/log.txt b/CKPTS/X2I_700k_3mask_200k_LLava_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_filtered_objects_3mask_888_context/003-GPT-XL/log.txt new file mode 100644 index 0000000000000000000000000000000000000000..fe151477cda0d94acff01ec77792b54af4a6b89a --- /dev/null +++ b/CKPTS/X2I_700k_3mask_200k_LLava_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_filtered_objects_3mask_888_context/003-GPT-XL/log.txt @@ -0,0 +1,1443 @@ +[2025-04-29 03:26:20] Experiment directory created at checkpoint/X2I_700k_3mask_200k_LLava_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_filtered_objects_3mask_888_context/003-GPT-XL +[2025-04-29 03:26:20] Namespace(data_path='/tmp/haozhezhao/MLLMG/jsonl_data/X2I_Filtered_700k_segmentmask_3mask_200k_data_train.jsonl', cloud_save_path='/tmp/haozhezhao/MLLMG/checkpoint', no_local_save=False, vq_model='VQ-16', vq_ckpt='/tmp/haozhezhao/model/llamagen_t2i/vq_ds16_t2i.pt', codebook_size=16384, codebook_embed_dim=8, gpt_model='GPT-XL', gpt_ckpt='/tmp/haozhezhao/MLLMG/checkpoint/Afteri2i_100kckpt_LLAVAT5_empty_Fixed_just_segment_pretrain_stage3_dreambench__recap_400k_t2i_400k_flux_200k_midjourney_150k_recovery_150k_grounding_100fluxseg_50samseg/005-GPT-XL/checkpoints/0078000.pt', gpt_type='t2i', vocab_size=16384, cls_token_num=888, dropout_p=0.1, token_dropout_p=0.1, drop_path=0.0, no_compile=False, results_dir='checkpoint/X2I_700k_3mask_200k_LLava_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_filtered_objects_3mask_888_context', dataset='ti2i', image_size=512, downsample_size=16, num_classes=1000, epochs=2, lr=0.0003, weight_decay=0.05, beta1=0.9, beta2=0.95, max_grad_norm=1.0, global_batch_size=40, global_seed=0, num_workers=4, log_every=25, ckpt_every=2000, gradient_accumulation_steps=8, mixed_precision='bf16', val_data_path='/tmp/haozhezhao/MLLMG/jsonl_data/multiimage_val_for_llava_X2I_threemask_dreambenplus.jsonl', use_vision_tower=True, model_name_or_path='/tmp/haozhezhao/model/blip2-flan-t5-xl', image_place_holder='', processor_path=None, do_eval=True, max_eval_samples=200, train_text_encoder=True, no_left_padding=False, cfg_scale=7.5, top_k=16384, temperature=0.9, top_p=1.0, eval_steps=2000, project_name='llamagen_ti2i', load_from_checkpoint='/tmp/haozhezhao/MLLMG/checkpoint/FIXed_3M_1epoch_step32000.pt', warmup=0.05, lr_decay_style='cosine', lr_decay_ratio=0.1, train_iters=500000, class_dropout_prob=0.1, with_image_only=False, image_only_rate=0.1, stage2=False, subject_driven=True, load_subject_embedding=None, reference_data_path='/tmp/haozhezhao/MLLMG/cc12m_reference_tunnel.jsonl', multimodal_encoder='llava', do_recovery=True, no_replace=False, resume=False, dreambench_eval=True, find_unused_parameters=True, load_visual_encoder=False, continue_stage1=False, replace_subject=False, train_all=True, save_total_limit=1, load_language_projection='/tmp/haozhezhao/MLLMG/llava-v1.5-flant5_fixed-pretrain/mm_projector.bin', mm_vision_tower='openai/clip-vit-large-patch14', load_fixed_llamagen=True, unfreeze_output=False, fix='gpt-empty-fix', rank=0, world_size=8, gpu=0, dist_url='env://', distributed=True, dist_backend='nccl') +[2025-04-29 03:26:20] Starting rank=0, seed=0, world_size=8. +[2025-04-29 03:26:20] text_config is None. Initializing the text config with default values (`OPTConfig`). +[2025-04-29 03:27:25] GPT Parameters: 2,310,680,832 +[2025-04-29 03:27:25] num decayed parameter tensors: 356, with 2,007,303,168 parameters +[2025-04-29 03:27:25] num non-decayed parameter tensors: 124, with 197,888 parameters +[2025-04-29 03:27:25] using fused AdamW: True +[2025-04-29 03:27:40] Dataset contains 908,625 images +[2025-04-29 03:27:40] Train iters 45430 , warmup 2271.5, len of loader 22715 +[2025-04-29 03:28:01] ### LOAD pretraining weights from checkpoint: /tmp/haozhezhao/MLLMG/checkpoint/Afteri2i_100kckpt_LLAVAT5_empty_Fixed_just_segment_pretrain_stage3_dreambench__recap_400k_t2i_400k_flux_200k_midjourney_150k_recovery_150k_grounding_100fluxseg_50samseg/005-GPT-XL/checkpoints/0078000.pt +[2025-04-29 03:28:01] Initial state: steps=0, epochs=0 +[2025-04-29 03:28:01] compiling the model... (may take several minutes) +[2025-04-29 03:28:02] freeze the vit +[2025-04-29 03:28:02] ***** total param is 2310680832 ***** +[2025-04-29 03:28:02] ***** total trained param is 2007501056 ***** +[2025-04-29 03:28:04] Training for 2 epochs... +[2025-04-29 03:28:04] Beginning epoch 0... +[2025-04-29 03:32:28] (step=0000025) Train Loss: 6.3148, Train Steps/Sec: 0.09 +[2025-04-29 03:32:52] (step=0000050) Train Loss: 6.3111, Train Steps/Sec: 1.06 +[2025-04-29 03:33:15] (step=0000075) Train Loss: 6.3244, Train Steps/Sec: 1.07 +[2025-04-29 03:33:39] (step=0000100) Train Loss: 6.3339, Train Steps/Sec: 1.07 +[2025-04-29 03:34:02] (step=0000125) Train Loss: 6.3132, Train Steps/Sec: 1.07 +[2025-04-29 03:34:26] (step=0000150) Train Loss: 6.2904, Train Steps/Sec: 1.06 +[2025-04-29 03:34:49] (step=0000175) Train Loss: 6.2718, Train Steps/Sec: 1.06 +[2025-04-29 03:35:13] (step=0000200) Train Loss: 6.2099, Train Steps/Sec: 1.06 +[2025-04-29 03:35:36] (step=0000225) Train Loss: 6.2370, Train Steps/Sec: 1.06 +[2025-04-29 03:36:00] (step=0000250) Train Loss: 6.2412, Train Steps/Sec: 1.07 +[2025-04-29 03:36:23] (step=0000275) Train Loss: 6.2081, Train Steps/Sec: 1.07 +[2025-04-29 03:36:46] (step=0000300) Train Loss: 6.2125, Train Steps/Sec: 1.07 +[2025-04-29 03:37:10] (step=0000325) Train Loss: 6.2095, Train Steps/Sec: 1.07 +[2025-04-29 03:37:33] (step=0000350) Train Loss: 6.1541, Train Steps/Sec: 1.07 +[2025-04-29 03:37:56] (step=0000375) Train Loss: 6.1582, Train Steps/Sec: 1.07 +[2025-04-29 03:38:20] (step=0000400) Train Loss: 6.1623, Train Steps/Sec: 1.07 +[2025-04-29 03:38:43] (step=0000425) Train Loss: 6.1617, Train Steps/Sec: 1.08 +[2025-04-29 03:39:06] (step=0000450) Train Loss: 6.1396, Train Steps/Sec: 1.07 +[2025-04-29 03:39:30] (step=0000475) Train Loss: 6.1146, Train Steps/Sec: 1.07 +[2025-04-29 03:39:53] (step=0000500) Train Loss: 6.1260, Train Steps/Sec: 1.07 +[2025-04-29 03:40:16] (step=0000525) Train Loss: 6.1332, Train Steps/Sec: 1.07 +[2025-04-29 03:40:40] (step=0000550) Train Loss: 6.1139, Train Steps/Sec: 1.07 +[2025-04-29 03:41:03] (step=0000575) Train Loss: 6.0858, Train Steps/Sec: 1.07 +[2025-04-29 03:41:26] (step=0000600) Train Loss: 6.1361, Train Steps/Sec: 1.07 +[2025-04-29 03:41:50] (step=0000625) Train Loss: 6.1084, Train Steps/Sec: 1.07 +[2025-04-29 03:42:13] (step=0000650) Train Loss: 6.0751, Train Steps/Sec: 1.07 +[2025-04-29 03:42:36] (step=0000675) Train Loss: 6.0876, Train Steps/Sec: 1.07 +[2025-04-29 03:43:00] (step=0000700) Train Loss: 6.1121, Train Steps/Sec: 1.07 +[2025-04-29 03:43:23] (step=0000725) Train Loss: 6.1070, Train Steps/Sec: 1.07 +[2025-04-29 03:43:46] (step=0000750) Train Loss: 6.0869, Train Steps/Sec: 1.07 +[2025-04-29 03:44:10] (step=0000775) Train Loss: 6.0407, Train Steps/Sec: 1.07 +[2025-04-29 03:44:33] (step=0000800) Train Loss: 6.0711, Train Steps/Sec: 1.07 +[2025-04-29 03:44:57] (step=0000825) Train Loss: 6.0568, Train Steps/Sec: 1.07 +[2025-04-29 03:45:20] (step=0000850) Train Loss: 6.0616, Train Steps/Sec: 1.07 +[2025-04-29 03:45:43] (step=0000875) Train Loss: 6.0502, Train Steps/Sec: 1.07 +[2025-04-29 03:46:07] (step=0000900) Train Loss: 6.0438, Train Steps/Sec: 1.07 +[2025-04-29 03:46:30] (step=0000925) Train Loss: 6.0659, Train Steps/Sec: 1.07 +[2025-04-29 03:46:53] (step=0000950) Train Loss: 6.0466, Train Steps/Sec: 1.07 +[2025-04-29 03:47:16] (step=0000975) Train Loss: 6.0293, Train Steps/Sec: 1.07 +[2025-04-29 03:47:40] (step=0001000) Train Loss: 6.0187, Train Steps/Sec: 1.07 +[2025-04-29 03:48:03] (step=0001025) Train Loss: 6.0104, Train Steps/Sec: 1.07 +[2025-04-29 03:48:26] (step=0001050) Train Loss: 6.0311, Train Steps/Sec: 1.07 +[2025-04-29 03:48:50] (step=0001075) Train Loss: 5.9987, Train Steps/Sec: 1.07 +[2025-04-29 03:49:13] (step=0001100) Train Loss: 6.0397, Train Steps/Sec: 1.07 +[2025-04-29 03:49:36] (step=0001125) Train Loss: 6.0385, Train Steps/Sec: 1.07 +[2025-04-29 03:50:00] (step=0001150) Train Loss: 6.0469, Train Steps/Sec: 1.07 +[2025-04-29 03:50:23] (step=0001175) Train Loss: 6.0547, Train Steps/Sec: 1.07 +[2025-04-29 03:50:46] (step=0001200) Train Loss: 6.0135, Train Steps/Sec: 1.07 +[2025-04-29 03:51:10] (step=0001225) Train Loss: 6.0262, Train Steps/Sec: 1.07 +[2025-04-29 03:51:33] (step=0001250) Train Loss: 6.0007, Train Steps/Sec: 1.07 +[2025-04-29 03:51:56] (step=0001275) Train Loss: 5.9986, Train Steps/Sec: 1.07 +[2025-04-29 03:52:20] (step=0001300) Train Loss: 6.0471, Train Steps/Sec: 1.07 +[2025-04-29 03:52:43] (step=0001325) Train Loss: 6.0259, Train Steps/Sec: 1.07 +[2025-04-29 03:53:10] (step=0001350) Train Loss: 6.0321, Train Steps/Sec: 0.92 +[2025-04-29 03:53:33] (step=0001375) Train Loss: 5.9999, Train Steps/Sec: 1.07 +[2025-04-29 03:54:09] (step=0001400) Train Loss: 5.9755, Train Steps/Sec: 0.71 +[2025-04-29 03:54:47] (step=0001425) Train Loss: 6.0278, Train Steps/Sec: 0.64 +[2025-04-29 03:55:15] (step=0001450) Train Loss: 6.0065, Train Steps/Sec: 0.92 +[2025-04-29 03:55:38] (step=0001475) Train Loss: 5.9920, Train Steps/Sec: 1.07 +[2025-04-29 03:56:01] (step=0001500) Train Loss: 6.0109, Train Steps/Sec: 1.07 +[2025-04-29 03:56:25] (step=0001525) Train Loss: 5.9801, Train Steps/Sec: 1.07 +[2025-04-29 03:56:48] (step=0001550) Train Loss: 5.9769, Train Steps/Sec: 1.07 +[2025-04-29 03:57:11] (step=0001575) Train Loss: 5.9956, Train Steps/Sec: 1.07 +[2025-04-29 03:57:35] (step=0001600) Train Loss: 6.0028, Train Steps/Sec: 1.07 +[2025-04-29 03:57:58] (step=0001625) Train Loss: 5.9921, Train Steps/Sec: 1.07 +[2025-04-29 03:58:21] (step=0001650) Train Loss: 5.9967, Train Steps/Sec: 1.07 +[2025-04-29 03:58:45] (step=0001675) Train Loss: 5.9623, Train Steps/Sec: 1.07 +[2025-04-29 03:59:08] (step=0001700) Train Loss: 5.9835, Train Steps/Sec: 1.07 +[2025-04-29 03:59:31] (step=0001725) Train Loss: 5.9825, Train Steps/Sec: 1.07 +[2025-04-29 03:59:55] (step=0001750) Train Loss: 5.9814, Train Steps/Sec: 1.07 +[2025-04-29 04:00:18] (step=0001775) Train Loss: 5.9900, Train Steps/Sec: 1.07 +[2025-04-29 04:00:41] (step=0001800) Train Loss: 5.9686, Train Steps/Sec: 1.07 +[2025-04-29 04:01:05] (step=0001825) Train Loss: 5.9759, Train Steps/Sec: 1.07 +[2025-04-29 04:01:28] (step=0001850) Train Loss: 5.9971, Train Steps/Sec: 1.07 +[2025-04-29 04:01:51] (step=0001875) Train Loss: 6.0042, Train Steps/Sec: 1.07 +[2025-04-29 04:02:15] (step=0001900) Train Loss: 5.9778, Train Steps/Sec: 1.07 +[2025-04-29 04:02:38] (step=0001925) Train Loss: 5.9749, Train Steps/Sec: 1.07 +[2025-04-29 04:03:01] (step=0001950) Train Loss: 5.9526, Train Steps/Sec: 1.07 +[2025-04-29 04:03:25] (step=0001975) Train Loss: 5.9250, Train Steps/Sec: 1.07 +[2025-04-29 04:03:48] (step=0002000) Train Loss: 5.9830, Train Steps/Sec: 1.07 +[2025-04-29 04:03:48] text_config is None. Initializing the text config with default values (`OPTConfig`). +[2025-04-29 04:09:56] Finish Eval in 2000 steps... +[2025-04-29 04:10:16] Saved checkpoint to checkpoint/X2I_700k_3mask_200k_LLava_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_filtered_objects_3mask_888_context/003-GPT-XL/checkpoints/0002000.pt +[2025-04-29 04:10:40] (step=0002025) Train Loss: 5.9736, Train Steps/Sec: 0.06 +[2025-04-29 04:11:03] (step=0002050) Train Loss: 5.9343, Train Steps/Sec: 1.07 +[2025-04-29 04:11:26] (step=0002075) Train Loss: 5.9470, Train Steps/Sec: 1.07 +[2025-04-29 04:11:50] (step=0002100) Train Loss: 5.9414, Train Steps/Sec: 1.07 +[2025-04-29 04:12:13] (step=0002125) Train Loss: 5.9614, Train Steps/Sec: 1.07 +[2025-04-29 04:12:36] (step=0002150) Train Loss: 5.9472, Train Steps/Sec: 1.07 +[2025-04-29 04:13:00] (step=0002175) Train Loss: 5.9521, Train Steps/Sec: 1.07 +[2025-04-29 04:13:23] (step=0002200) Train Loss: 5.9375, Train Steps/Sec: 1.07 +[2025-04-29 04:13:46] (step=0002225) Train Loss: 5.8890, Train Steps/Sec: 1.07 +[2025-04-29 04:14:10] (step=0002250) Train Loss: 5.9150, Train Steps/Sec: 1.07 +[2025-04-29 04:14:33] (step=0002275) Train Loss: 5.8895, Train Steps/Sec: 1.07 +[2025-04-29 04:14:56] (step=0002300) Train Loss: 5.9525, Train Steps/Sec: 1.07 +[2025-04-29 04:15:20] (step=0002325) Train Loss: 5.9601, Train Steps/Sec: 1.07 +[2025-04-29 04:15:43] (step=0002350) Train Loss: 5.9263, Train Steps/Sec: 1.07 +[2025-04-29 04:16:06] (step=0002375) Train Loss: 5.9501, Train Steps/Sec: 1.07 +[2025-04-29 04:16:30] (step=0002400) Train Loss: 5.9288, Train Steps/Sec: 1.07 +[2025-04-29 04:16:53] (step=0002425) Train Loss: 5.9398, Train Steps/Sec: 1.07 +[2025-04-29 04:17:16] (step=0002450) Train Loss: 5.9082, Train Steps/Sec: 1.07 +[2025-04-29 04:17:40] (step=0002475) Train Loss: 5.9303, Train Steps/Sec: 1.07 +[2025-04-29 04:18:03] (step=0002500) Train Loss: 5.9484, Train Steps/Sec: 1.07 +[2025-04-29 04:18:26] (step=0002525) Train Loss: 5.9123, Train Steps/Sec: 1.07 +[2025-04-29 04:18:50] (step=0002550) Train Loss: 5.9276, Train Steps/Sec: 1.07 +[2025-04-29 04:19:13] (step=0002575) Train Loss: 5.9187, Train Steps/Sec: 1.07 +[2025-04-29 04:19:37] (step=0002600) Train Loss: 5.9264, Train Steps/Sec: 1.07 +[2025-04-29 04:20:00] (step=0002625) Train Loss: 5.9608, Train Steps/Sec: 1.07 +[2025-04-29 04:20:23] (step=0002650) Train Loss: 5.9263, Train Steps/Sec: 1.07 +[2025-04-29 04:20:47] (step=0002675) Train Loss: 5.8694, Train Steps/Sec: 1.07 +[2025-04-29 04:21:10] (step=0002700) Train Loss: 5.9068, Train Steps/Sec: 1.07 +[2025-04-29 04:21:33] (step=0002725) Train Loss: 5.9183, Train Steps/Sec: 1.07 +[2025-04-29 04:21:57] (step=0002750) Train Loss: 5.9510, Train Steps/Sec: 1.07 +[2025-04-29 04:22:20] (step=0002775) Train Loss: 5.9227, Train Steps/Sec: 1.07 +[2025-04-29 04:22:43] (step=0002800) Train Loss: 5.9466, Train Steps/Sec: 1.07 +[2025-04-29 04:23:07] (step=0002825) Train Loss: 5.9318, Train Steps/Sec: 1.07 +[2025-04-29 04:23:30] (step=0002850) Train Loss: 5.9197, Train Steps/Sec: 1.07 +[2025-04-29 04:23:53] (step=0002875) Train Loss: 5.8796, Train Steps/Sec: 1.07 +[2025-04-29 04:24:17] (step=0002900) Train Loss: 5.9328, Train Steps/Sec: 1.07 +[2025-04-29 04:24:40] (step=0002925) Train Loss: 5.8925, Train Steps/Sec: 1.07 +[2025-04-29 04:25:03] (step=0002950) Train Loss: 5.8933, Train Steps/Sec: 1.07 +[2025-04-29 04:25:27] (step=0002975) Train Loss: 5.8922, Train Steps/Sec: 1.07 +[2025-04-29 04:25:50] (step=0003000) Train Loss: 5.8985, Train Steps/Sec: 1.06 +[2025-04-29 04:26:14] (step=0003025) Train Loss: 5.8918, Train Steps/Sec: 1.07 +[2025-04-29 04:26:37] (step=0003050) Train Loss: 5.8963, Train Steps/Sec: 1.07 +[2025-04-29 04:27:01] (step=0003075) Train Loss: 5.9446, Train Steps/Sec: 1.07 +[2025-04-29 04:27:24] (step=0003100) Train Loss: 5.9254, Train Steps/Sec: 1.07 +[2025-04-29 04:27:47] (step=0003125) Train Loss: 5.8754, Train Steps/Sec: 1.07 +[2025-04-29 04:28:11] (step=0003150) Train Loss: 5.8835, Train Steps/Sec: 1.07 +[2025-04-29 04:28:34] (step=0003175) Train Loss: 5.9081, Train Steps/Sec: 1.07 +[2025-04-29 04:29:01] (step=0003200) Train Loss: 5.9129, Train Steps/Sec: 0.92 +[2025-04-29 04:29:36] (step=0003225) Train Loss: 5.8796, Train Steps/Sec: 0.72 +[2025-04-29 04:30:08] (step=0003250) Train Loss: 5.9465, Train Steps/Sec: 0.78 +[2025-04-29 04:30:35] (step=0003275) Train Loss: 5.8695, Train Steps/Sec: 0.92 +[2025-04-29 04:30:58] (step=0003300) Train Loss: 5.8977, Train Steps/Sec: 1.07 +[2025-04-29 04:31:21] (step=0003325) Train Loss: 5.8810, Train Steps/Sec: 1.07 +[2025-04-29 04:31:45] (step=0003350) Train Loss: 5.8849, Train Steps/Sec: 1.07 +[2025-04-29 04:32:08] (step=0003375) Train Loss: 5.9070, Train Steps/Sec: 1.07 +[2025-04-29 04:32:31] (step=0003400) Train Loss: 5.9019, Train Steps/Sec: 1.07 +[2025-04-29 04:32:55] (step=0003425) Train Loss: 5.8988, Train Steps/Sec: 1.07 +[2025-04-29 04:33:18] (step=0003450) Train Loss: 5.8612, Train Steps/Sec: 1.07 +[2025-04-29 04:33:42] (step=0003475) Train Loss: 5.8806, Train Steps/Sec: 1.07 +[2025-04-29 04:34:05] (step=0003500) Train Loss: 5.8402, Train Steps/Sec: 1.07 +[2025-04-29 04:34:28] (step=0003525) Train Loss: 5.9245, Train Steps/Sec: 1.07 +[2025-04-29 04:34:52] (step=0003550) Train Loss: 5.8566, Train Steps/Sec: 1.07 +[2025-04-29 04:35:15] (step=0003575) Train Loss: 5.8998, Train Steps/Sec: 1.07 +[2025-04-29 04:35:38] (step=0003600) Train Loss: 5.8761, Train Steps/Sec: 1.07 +[2025-04-29 04:36:02] (step=0003625) Train Loss: 5.9042, Train Steps/Sec: 1.07 +[2025-04-29 04:36:25] (step=0003650) Train Loss: 5.9033, Train Steps/Sec: 1.07 +[2025-04-29 04:36:49] (step=0003675) Train Loss: 5.8840, Train Steps/Sec: 1.07 +[2025-04-29 04:37:12] (step=0003700) Train Loss: 5.8597, Train Steps/Sec: 1.07 +[2025-04-29 04:37:35] (step=0003725) Train Loss: 5.8832, Train Steps/Sec: 1.07 +[2025-04-29 04:37:59] (step=0003750) Train Loss: 5.8638, Train Steps/Sec: 1.07 +[2025-04-29 04:38:22] (step=0003775) Train Loss: 5.8704, Train Steps/Sec: 1.07 +[2025-04-29 04:38:45] (step=0003800) Train Loss: 5.8557, Train Steps/Sec: 1.07 +[2025-04-29 04:39:14] (step=0003825) Train Loss: 5.8703, Train Steps/Sec: 0.88 +[2025-04-29 04:39:37] (step=0003850) Train Loss: 5.8552, Train Steps/Sec: 1.07 +[2025-04-29 04:40:00] (step=0003875) Train Loss: 5.8254, Train Steps/Sec: 1.07 +[2025-04-29 04:40:24] (step=0003900) Train Loss: 5.8560, Train Steps/Sec: 1.07 +[2025-04-29 04:40:47] (step=0003925) Train Loss: 5.8705, Train Steps/Sec: 1.07 +[2025-04-29 04:41:10] (step=0003950) Train Loss: 5.8411, Train Steps/Sec: 1.07 +[2025-04-29 04:41:34] (step=0003975) Train Loss: 5.8803, Train Steps/Sec: 1.07 +[2025-04-29 04:41:57] (step=0004000) Train Loss: 5.8482, Train Steps/Sec: 1.07 +[2025-04-29 04:41:57] text_config is None. Initializing the text config with default values (`OPTConfig`). +[2025-04-29 04:48:07] Finish Eval in 4000 steps... +[2025-04-29 04:48:26] Saved checkpoint to checkpoint/X2I_700k_3mask_200k_LLava_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_filtered_objects_3mask_888_context/003-GPT-XL/checkpoints/0004000.pt +[2025-04-29 04:48:28] Removed old checkpoint: checkpoint/X2I_700k_3mask_200k_LLava_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_filtered_objects_3mask_888_context/003-GPT-XL/checkpoints/0002000.pt +[2025-04-29 04:48:52] (step=0004025) Train Loss: 5.8419, Train Steps/Sec: 0.06 +[2025-04-29 04:49:15] (step=0004050) Train Loss: 5.8479, Train Steps/Sec: 1.07 +[2025-04-29 04:49:39] (step=0004075) Train Loss: 5.8302, Train Steps/Sec: 1.07 +[2025-04-29 04:50:02] (step=0004100) Train Loss: 5.8721, Train Steps/Sec: 1.07 +[2025-04-29 04:50:25] (step=0004125) Train Loss: 5.8621, Train Steps/Sec: 1.07 +[2025-04-29 04:50:49] (step=0004150) Train Loss: 5.8252, Train Steps/Sec: 1.07 +[2025-04-29 04:51:12] (step=0004175) Train Loss: 5.8497, Train Steps/Sec: 1.07 +[2025-04-29 04:51:35] (step=0004200) Train Loss: 5.8545, Train Steps/Sec: 1.07 +[2025-04-29 04:51:59] (step=0004225) Train Loss: 5.8269, Train Steps/Sec: 1.07 +[2025-04-29 04:52:22] (step=0004250) Train Loss: 5.8774, Train Steps/Sec: 1.07 +[2025-04-29 04:52:45] (step=0004275) Train Loss: 5.8499, Train Steps/Sec: 1.07 +[2025-04-29 04:53:09] (step=0004300) Train Loss: 5.8770, Train Steps/Sec: 1.07 +[2025-04-29 04:53:32] (step=0004325) Train Loss: 5.8492, Train Steps/Sec: 1.07 +[2025-04-29 04:53:55] (step=0004350) Train Loss: 5.8460, Train Steps/Sec: 1.07 +[2025-04-29 04:54:19] (step=0004375) Train Loss: 5.8522, Train Steps/Sec: 1.07 +[2025-04-29 04:54:42] (step=0004400) Train Loss: 5.7965, Train Steps/Sec: 1.07 +[2025-04-29 04:55:06] (step=0004425) Train Loss: 5.8620, Train Steps/Sec: 1.07 +[2025-04-29 04:55:29] (step=0004450) Train Loss: 5.8760, Train Steps/Sec: 1.07 +[2025-04-29 04:55:52] (step=0004475) Train Loss: 5.8182, Train Steps/Sec: 1.07 +[2025-04-29 04:56:16] (step=0004500) Train Loss: 5.8286, Train Steps/Sec: 1.07 +[2025-04-29 04:56:39] (step=0004525) Train Loss: 5.8227, Train Steps/Sec: 1.07 +[2025-04-29 04:57:02] (step=0004550) Train Loss: 5.8005, Train Steps/Sec: 1.07 +[2025-04-29 04:57:26] (step=0004575) Train Loss: 5.8455, Train Steps/Sec: 1.07 +[2025-04-29 04:57:49] (step=0004600) Train Loss: 5.8291, Train Steps/Sec: 1.07 +[2025-04-29 04:58:12] (step=0004625) Train Loss: 5.8473, Train Steps/Sec: 1.07 +[2025-04-29 04:58:36] (step=0004650) Train Loss: 5.8307, Train Steps/Sec: 1.07 +[2025-04-29 04:58:59] (step=0004675) Train Loss: 5.8195, Train Steps/Sec: 1.07 +[2025-04-29 04:59:22] (step=0004700) Train Loss: 5.8784, Train Steps/Sec: 1.07 +[2025-04-29 04:59:46] (step=0004725) Train Loss: 5.8428, Train Steps/Sec: 1.07 +[2025-04-29 05:00:09] (step=0004750) Train Loss: 5.8260, Train Steps/Sec: 1.07 +[2025-04-29 05:00:32] (step=0004775) Train Loss: 5.8336, Train Steps/Sec: 1.07 +[2025-04-29 05:00:56] (step=0004800) Train Loss: 5.8216, Train Steps/Sec: 1.07 +[2025-04-29 05:01:19] (step=0004825) Train Loss: 5.8269, Train Steps/Sec: 1.07 +[2025-04-29 05:01:43] (step=0004850) Train Loss: 5.7651, Train Steps/Sec: 1.07 +[2025-04-29 05:02:06] (step=0004875) Train Loss: 5.8340, Train Steps/Sec: 1.07 +[2025-04-29 05:02:29] (step=0004900) Train Loss: 5.8161, Train Steps/Sec: 1.07 +[2025-04-29 05:02:53] (step=0004925) Train Loss: 5.7862, Train Steps/Sec: 1.07 +[2025-04-29 05:03:16] (step=0004950) Train Loss: 5.8722, Train Steps/Sec: 1.07 +[2025-04-29 05:03:39] (step=0004975) Train Loss: 5.8339, Train Steps/Sec: 1.07 +[2025-04-29 05:04:03] (step=0005000) Train Loss: 5.8255, Train Steps/Sec: 1.07 +[2025-04-29 05:04:30] (step=0005025) Train Loss: 5.8025, Train Steps/Sec: 0.93 +[2025-04-29 05:04:57] (step=0005050) Train Loss: 5.8219, Train Steps/Sec: 0.92 +[2025-04-29 05:05:33] (step=0005075) Train Loss: 5.8094, Train Steps/Sec: 0.70 +[2025-04-29 05:06:00] (step=0005100) Train Loss: 5.8356, Train Steps/Sec: 0.92 +[2025-04-29 05:06:23] (step=0005125) Train Loss: 5.8141, Train Steps/Sec: 1.07 +[2025-04-29 05:06:46] (step=0005150) Train Loss: 5.8313, Train Steps/Sec: 1.07 +[2025-04-29 05:07:10] (step=0005175) Train Loss: 5.7879, Train Steps/Sec: 1.07 +[2025-04-29 05:07:33] (step=0005200) Train Loss: 5.8382, Train Steps/Sec: 1.07 +[2025-04-29 05:07:57] (step=0005225) Train Loss: 5.8006, Train Steps/Sec: 1.07 +[2025-04-29 05:08:20] (step=0005250) Train Loss: 5.8410, Train Steps/Sec: 1.07 +[2025-04-29 05:08:43] (step=0005275) Train Loss: 5.8144, Train Steps/Sec: 1.07 +[2025-04-29 05:09:07] (step=0005300) Train Loss: 5.8186, Train Steps/Sec: 1.07 +[2025-04-29 05:09:30] (step=0005325) Train Loss: 5.7685, Train Steps/Sec: 1.07 +[2025-04-29 05:09:53] (step=0005350) Train Loss: 5.8306, Train Steps/Sec: 1.07 +[2025-04-29 05:10:17] (step=0005375) Train Loss: 5.8085, Train Steps/Sec: 1.07 +[2025-04-29 05:10:40] (step=0005400) Train Loss: 5.8187, Train Steps/Sec: 1.07 +[2025-04-29 05:11:03] (step=0005425) Train Loss: 5.7996, Train Steps/Sec: 1.07 +[2025-04-29 05:11:27] (step=0005450) Train Loss: 5.8233, Train Steps/Sec: 1.07 +[2025-04-29 05:11:50] (step=0005475) Train Loss: 5.8172, Train Steps/Sec: 1.07 +[2025-04-29 05:12:13] (step=0005500) Train Loss: 5.8079, Train Steps/Sec: 1.07 +[2025-04-29 05:12:37] (step=0005525) Train Loss: 5.7883, Train Steps/Sec: 1.07 +[2025-04-29 05:13:00] (step=0005550) Train Loss: 5.7907, Train Steps/Sec: 1.07 +[2025-04-29 05:13:24] (step=0005575) Train Loss: 5.8063, Train Steps/Sec: 1.07 +[2025-04-29 05:13:47] (step=0005600) Train Loss: 5.8248, Train Steps/Sec: 1.07 +[2025-04-29 05:14:10] (step=0005625) Train Loss: 5.7927, Train Steps/Sec: 1.07 +[2025-04-29 05:14:34] (step=0005650) Train Loss: 5.7976, Train Steps/Sec: 1.07 +[2025-04-29 05:14:57] (step=0005675) Train Loss: 5.7999, Train Steps/Sec: 1.07 +[2025-04-29 05:15:20] (step=0005700) Train Loss: 5.7964, Train Steps/Sec: 1.07 +[2025-04-29 05:15:44] (step=0005725) Train Loss: 5.7962, Train Steps/Sec: 1.07 +[2025-04-29 05:16:07] (step=0005750) Train Loss: 5.8085, Train Steps/Sec: 1.07 +[2025-04-29 05:16:30] (step=0005775) Train Loss: 5.7798, Train Steps/Sec: 1.07 +[2025-04-29 05:16:54] (step=0005800) Train Loss: 5.8068, Train Steps/Sec: 1.07 +[2025-04-29 05:17:22] (step=0005825) Train Loss: 5.7916, Train Steps/Sec: 0.88 +[2025-04-29 05:17:46] (step=0005850) Train Loss: 5.7875, Train Steps/Sec: 1.07 +[2025-04-29 05:18:09] (step=0005875) Train Loss: 5.7840, Train Steps/Sec: 1.07 +[2025-04-29 05:18:32] (step=0005900) Train Loss: 5.7750, Train Steps/Sec: 1.07 +[2025-04-29 05:18:56] (step=0005925) Train Loss: 5.8125, Train Steps/Sec: 1.07 +[2025-04-29 05:19:19] (step=0005950) Train Loss: 5.7677, Train Steps/Sec: 1.07 +[2025-04-29 05:19:42] (step=0005975) Train Loss: 5.7858, Train Steps/Sec: 1.07 +[2025-04-29 05:20:06] (step=0006000) Train Loss: 5.8140, Train Steps/Sec: 1.07 +[2025-04-29 05:20:06] text_config is None. Initializing the text config with default values (`OPTConfig`). +[2025-04-29 05:26:21] Finish Eval in 6000 steps... +[2025-04-29 05:26:42] Saved checkpoint to checkpoint/X2I_700k_3mask_200k_LLava_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_filtered_objects_3mask_888_context/003-GPT-XL/checkpoints/0006000.pt +[2025-04-29 05:26:44] Removed old checkpoint: checkpoint/X2I_700k_3mask_200k_LLava_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_filtered_objects_3mask_888_context/003-GPT-XL/checkpoints/0004000.pt +[2025-04-29 05:27:07] (step=0006025) Train Loss: 5.7683, Train Steps/Sec: 0.06 +[2025-04-29 05:27:31] (step=0006050) Train Loss: 5.7708, Train Steps/Sec: 1.07 +[2025-04-29 05:27:54] (step=0006075) Train Loss: 5.7630, Train Steps/Sec: 1.07 +[2025-04-29 05:28:17] (step=0006100) Train Loss: 5.7745, Train Steps/Sec: 1.07 +[2025-04-29 05:28:41] (step=0006125) Train Loss: 5.7903, Train Steps/Sec: 1.07 +[2025-04-29 05:29:04] (step=0006150) Train Loss: 5.7961, Train Steps/Sec: 1.07 +[2025-04-29 05:29:27] (step=0006175) Train Loss: 5.7876, Train Steps/Sec: 1.07 +[2025-04-29 05:29:51] (step=0006200) Train Loss: 5.8221, Train Steps/Sec: 1.06 +[2025-04-29 05:30:14] (step=0006225) Train Loss: 5.7478, Train Steps/Sec: 1.07 +[2025-04-29 05:30:38] (step=0006250) Train Loss: 5.7541, Train Steps/Sec: 1.07 +[2025-04-29 05:31:01] (step=0006275) Train Loss: 5.7691, Train Steps/Sec: 1.07 +[2025-04-29 05:31:24] (step=0006300) Train Loss: 5.7651, Train Steps/Sec: 1.07 +[2025-04-29 05:31:48] (step=0006325) Train Loss: 5.7623, Train Steps/Sec: 1.07 +[2025-04-29 05:32:11] (step=0006350) Train Loss: 5.7490, Train Steps/Sec: 1.07 +[2025-04-29 05:32:34] (step=0006375) Train Loss: 5.7679, Train Steps/Sec: 1.07 +[2025-04-29 05:32:58] (step=0006400) Train Loss: 5.7582, Train Steps/Sec: 1.06 +[2025-04-29 05:33:21] (step=0006425) Train Loss: 5.7595, Train Steps/Sec: 1.07 +[2025-04-29 05:33:45] (step=0006450) Train Loss: 5.7550, Train Steps/Sec: 1.07 +[2025-04-29 05:34:08] (step=0006475) Train Loss: 5.7961, Train Steps/Sec: 1.07 +[2025-04-29 05:34:31] (step=0006500) Train Loss: 5.7498, Train Steps/Sec: 1.07 +[2025-04-29 05:34:55] (step=0006525) Train Loss: 5.7668, Train Steps/Sec: 1.07 +[2025-04-29 05:35:18] (step=0006550) Train Loss: 5.7491, Train Steps/Sec: 1.07 +[2025-04-29 05:35:42] (step=0006575) Train Loss: 5.7654, Train Steps/Sec: 1.07 +[2025-04-29 05:36:05] (step=0006600) Train Loss: 5.7437, Train Steps/Sec: 1.06 +[2025-04-29 05:36:29] (step=0006625) Train Loss: 5.7555, Train Steps/Sec: 1.07 +[2025-04-29 05:36:52] (step=0006650) Train Loss: 5.7789, Train Steps/Sec: 1.07 +[2025-04-29 05:37:15] (step=0006675) Train Loss: 5.7788, Train Steps/Sec: 1.07 +[2025-04-29 05:37:39] (step=0006700) Train Loss: 5.7270, Train Steps/Sec: 1.07 +[2025-04-29 05:38:02] (step=0006725) Train Loss: 5.8227, Train Steps/Sec: 1.07 +[2025-04-29 05:38:26] (step=0006750) Train Loss: 5.7623, Train Steps/Sec: 1.07 +[2025-04-29 05:38:49] (step=0006775) Train Loss: 5.7860, Train Steps/Sec: 1.07 +[2025-04-29 05:39:12] (step=0006800) Train Loss: 5.7637, Train Steps/Sec: 1.06 +[2025-04-29 05:39:36] (step=0006825) Train Loss: 5.7720, Train Steps/Sec: 1.07 +[2025-04-29 05:40:03] (step=0006850) Train Loss: 5.7599, Train Steps/Sec: 0.93 +[2025-04-29 05:40:26] (step=0006875) Train Loss: 5.7749, Train Steps/Sec: 1.07 +[2025-04-29 05:41:04] (step=0006900) Train Loss: 5.7879, Train Steps/Sec: 0.65 +[2025-04-29 05:41:36] (step=0006925) Train Loss: 5.7678, Train Steps/Sec: 0.78 +[2025-04-29 05:42:00] (step=0006950) Train Loss: 5.7947, Train Steps/Sec: 1.07 +[2025-04-29 05:42:23] (step=0006975) Train Loss: 5.7184, Train Steps/Sec: 1.07 +[2025-04-29 05:42:46] (step=0007000) Train Loss: 5.7606, Train Steps/Sec: 1.07 +[2025-04-29 05:43:10] (step=0007025) Train Loss: 5.7921, Train Steps/Sec: 1.07 +[2025-04-29 05:43:33] (step=0007050) Train Loss: 5.7639, Train Steps/Sec: 1.07 +[2025-04-29 05:43:56] (step=0007075) Train Loss: 5.8013, Train Steps/Sec: 1.07 +[2025-04-29 05:44:20] (step=0007100) Train Loss: 5.7403, Train Steps/Sec: 1.07 +[2025-04-29 05:44:43] (step=0007125) Train Loss: 5.7578, Train Steps/Sec: 1.07 +[2025-04-29 05:45:07] (step=0007150) Train Loss: 5.7762, Train Steps/Sec: 1.07 +[2025-04-29 05:45:30] (step=0007175) Train Loss: 5.8014, Train Steps/Sec: 1.07 +[2025-04-29 05:45:53] (step=0007200) Train Loss: 5.7654, Train Steps/Sec: 1.07 +[2025-04-29 05:46:17] (step=0007225) Train Loss: 5.7449, Train Steps/Sec: 1.07 +[2025-04-29 05:46:40] (step=0007250) Train Loss: 5.7375, Train Steps/Sec: 1.07 +[2025-04-29 05:47:03] (step=0007275) Train Loss: 5.7704, Train Steps/Sec: 1.07 +[2025-04-29 05:47:27] (step=0007300) Train Loss: 5.7566, Train Steps/Sec: 1.07 +[2025-04-29 05:47:50] (step=0007325) Train Loss: 5.7424, Train Steps/Sec: 1.07 +[2025-04-29 05:48:14] (step=0007350) Train Loss: 5.7205, Train Steps/Sec: 1.07 +[2025-04-29 05:48:37] (step=0007375) Train Loss: 5.7476, Train Steps/Sec: 1.07 +[2025-04-29 05:49:00] (step=0007400) Train Loss: 5.7343, Train Steps/Sec: 1.06 +[2025-04-29 05:49:24] (step=0007425) Train Loss: 5.7910, Train Steps/Sec: 1.07 +[2025-04-29 05:49:47] (step=0007450) Train Loss: 5.7537, Train Steps/Sec: 1.07 +[2025-04-29 05:50:10] (step=0007475) Train Loss: 5.7322, Train Steps/Sec: 1.07 +[2025-04-29 05:50:34] (step=0007500) Train Loss: 5.7615, Train Steps/Sec: 1.07 +[2025-04-29 05:50:57] (step=0007525) Train Loss: 5.7558, Train Steps/Sec: 1.07 +[2025-04-29 05:51:21] (step=0007550) Train Loss: 5.7631, Train Steps/Sec: 1.07 +[2025-04-29 05:51:44] (step=0007575) Train Loss: 5.7593, Train Steps/Sec: 1.07 +[2025-04-29 05:52:07] (step=0007600) Train Loss: 5.7422, Train Steps/Sec: 1.07 +[2025-04-29 05:52:31] (step=0007625) Train Loss: 5.7696, Train Steps/Sec: 1.07 +[2025-04-29 05:52:54] (step=0007650) Train Loss: 5.7334, Train Steps/Sec: 1.07 +[2025-04-29 05:53:17] (step=0007675) Train Loss: 5.7379, Train Steps/Sec: 1.07 +[2025-04-29 05:53:41] (step=0007700) Train Loss: 5.7010, Train Steps/Sec: 1.07 +[2025-04-29 05:54:04] (step=0007725) Train Loss: 5.7493, Train Steps/Sec: 1.07 +[2025-04-29 05:54:28] (step=0007750) Train Loss: 5.7252, Train Steps/Sec: 1.07 +[2025-04-29 05:54:51] (step=0007775) Train Loss: 5.7779, Train Steps/Sec: 1.07 +[2025-04-29 05:55:14] (step=0007800) Train Loss: 5.7351, Train Steps/Sec: 1.07 +[2025-04-29 05:55:43] (step=0007825) Train Loss: 5.7014, Train Steps/Sec: 0.86 +[2025-04-29 05:56:07] (step=0007850) Train Loss: 5.7300, Train Steps/Sec: 1.07 +[2025-04-29 05:56:30] (step=0007875) Train Loss: 5.7403, Train Steps/Sec: 1.07 +[2025-04-29 05:56:54] (step=0007900) Train Loss: 5.7498, Train Steps/Sec: 1.07 +[2025-04-29 05:57:17] (step=0007925) Train Loss: 5.7240, Train Steps/Sec: 1.07 +[2025-04-29 05:57:40] (step=0007950) Train Loss: 5.7253, Train Steps/Sec: 1.07 +[2025-04-29 05:58:04] (step=0007975) Train Loss: 5.7284, Train Steps/Sec: 1.07 +[2025-04-29 05:58:27] (step=0008000) Train Loss: 5.7196, Train Steps/Sec: 1.07 +[2025-04-29 05:58:27] text_config is None. Initializing the text config with default values (`OPTConfig`). +[2025-04-29 06:04:42] Finish Eval in 8000 steps... +[2025-04-29 06:05:02] Saved checkpoint to checkpoint/X2I_700k_3mask_200k_LLava_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_filtered_objects_3mask_888_context/003-GPT-XL/checkpoints/0008000.pt +[2025-04-29 06:05:04] Removed old checkpoint: checkpoint/X2I_700k_3mask_200k_LLava_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_filtered_objects_3mask_888_context/003-GPT-XL/checkpoints/0006000.pt +[2025-04-29 06:05:28] (step=0008025) Train Loss: 5.7476, Train Steps/Sec: 0.06 +[2025-04-29 06:05:51] (step=0008050) Train Loss: 5.7010, Train Steps/Sec: 1.08 +[2025-04-29 06:06:14] (step=0008075) Train Loss: 5.6957, Train Steps/Sec: 1.07 +[2025-04-29 06:06:38] (step=0008100) Train Loss: 5.7206, Train Steps/Sec: 1.07 +[2025-04-29 06:07:01] (step=0008125) Train Loss: 5.6835, Train Steps/Sec: 1.07 +[2025-04-29 06:07:24] (step=0008150) Train Loss: 5.6986, Train Steps/Sec: 1.07 +[2025-04-29 06:07:48] (step=0008175) Train Loss: 5.6932, Train Steps/Sec: 1.07 +[2025-04-29 06:08:11] (step=0008200) Train Loss: 5.7386, Train Steps/Sec: 1.07 +[2025-04-29 06:08:34] (step=0008225) Train Loss: 5.7430, Train Steps/Sec: 1.07 +[2025-04-29 06:08:58] (step=0008250) Train Loss: 5.7117, Train Steps/Sec: 1.07 +[2025-04-29 06:09:21] (step=0008275) Train Loss: 5.7085, Train Steps/Sec: 1.07 +[2025-04-29 06:09:44] (step=0008300) Train Loss: 5.6827, Train Steps/Sec: 1.07 +[2025-04-29 06:10:08] (step=0008325) Train Loss: 5.7108, Train Steps/Sec: 1.07 +[2025-04-29 06:10:31] (step=0008350) Train Loss: 5.7244, Train Steps/Sec: 1.07 +[2025-04-29 06:10:55] (step=0008375) Train Loss: 5.7078, Train Steps/Sec: 1.07 +[2025-04-29 06:11:18] (step=0008400) Train Loss: 5.7299, Train Steps/Sec: 1.07 +[2025-04-29 06:11:41] (step=0008425) Train Loss: 5.7218, Train Steps/Sec: 1.07 +[2025-04-29 06:12:05] (step=0008450) Train Loss: 5.7360, Train Steps/Sec: 1.07 +[2025-04-29 06:12:28] (step=0008475) Train Loss: 5.7127, Train Steps/Sec: 1.07 +[2025-04-29 06:12:51] (step=0008500) Train Loss: 5.7754, Train Steps/Sec: 1.07 +[2025-04-29 06:13:15] (step=0008525) Train Loss: 5.7064, Train Steps/Sec: 1.07 +[2025-04-29 06:13:38] (step=0008550) Train Loss: 5.6630, Train Steps/Sec: 1.07 +[2025-04-29 06:14:01] (step=0008575) Train Loss: 5.7362, Train Steps/Sec: 1.07 +[2025-04-29 06:14:25] (step=0008600) Train Loss: 5.7099, Train Steps/Sec: 1.07 +[2025-04-29 06:14:48] (step=0008625) Train Loss: 5.6972, Train Steps/Sec: 1.07 +[2025-04-29 06:15:12] (step=0008650) Train Loss: 5.7154, Train Steps/Sec: 1.07 +[2025-04-29 06:15:39] (step=0008675) Train Loss: 5.6796, Train Steps/Sec: 0.92 +[2025-04-29 06:16:02] (step=0008700) Train Loss: 5.6929, Train Steps/Sec: 1.07 +[2025-04-29 06:16:40] (step=0008725) Train Loss: 5.7029, Train Steps/Sec: 0.65 +[2025-04-29 06:17:12] (step=0008750) Train Loss: 5.6816, Train Steps/Sec: 0.78 +[2025-04-29 06:17:36] (step=0008775) Train Loss: 5.6979, Train Steps/Sec: 1.07 +[2025-04-29 06:17:59] (step=0008800) Train Loss: 5.7061, Train Steps/Sec: 1.06 +[2025-04-29 06:18:23] (step=0008825) Train Loss: 5.6726, Train Steps/Sec: 1.07 +[2025-04-29 06:18:46] (step=0008850) Train Loss: 5.6719, Train Steps/Sec: 1.07 +[2025-04-29 06:19:09] (step=0008875) Train Loss: 5.6925, Train Steps/Sec: 1.07 +[2025-04-29 06:19:33] (step=0008900) Train Loss: 5.6645, Train Steps/Sec: 1.07 +[2025-04-29 06:19:56] (step=0008925) Train Loss: 5.6890, Train Steps/Sec: 1.07 +[2025-04-29 06:20:19] (step=0008950) Train Loss: 5.7225, Train Steps/Sec: 1.07 +[2025-04-29 06:20:43] (step=0008975) Train Loss: 5.7335, Train Steps/Sec: 1.07 +[2025-04-29 06:21:06] (step=0009000) Train Loss: 5.7100, Train Steps/Sec: 1.07 +[2025-04-29 06:21:30] (step=0009025) Train Loss: 5.6942, Train Steps/Sec: 1.07 +[2025-04-29 06:21:53] (step=0009050) Train Loss: 5.6565, Train Steps/Sec: 1.07 +[2025-04-29 06:22:16] (step=0009075) Train Loss: 5.6747, Train Steps/Sec: 1.07 +[2025-04-29 06:22:40] (step=0009100) Train Loss: 5.6518, Train Steps/Sec: 1.07 +[2025-04-29 06:23:03] (step=0009125) Train Loss: 5.7103, Train Steps/Sec: 1.07 +[2025-04-29 06:23:26] (step=0009150) Train Loss: 5.6759, Train Steps/Sec: 1.07 +[2025-04-29 06:23:50] (step=0009175) Train Loss: 5.7104, Train Steps/Sec: 1.07 +[2025-04-29 06:24:13] (step=0009200) Train Loss: 5.6650, Train Steps/Sec: 1.06 +[2025-04-29 06:24:37] (step=0009225) Train Loss: 5.6941, Train Steps/Sec: 1.07 +[2025-04-29 06:25:00] (step=0009250) Train Loss: 5.6783, Train Steps/Sec: 1.07 +[2025-04-29 06:25:23] (step=0009275) Train Loss: 5.7203, Train Steps/Sec: 1.07 +[2025-04-29 06:25:47] (step=0009300) Train Loss: 5.7089, Train Steps/Sec: 1.07 +[2025-04-29 06:26:10] (step=0009325) Train Loss: 5.6509, Train Steps/Sec: 1.07 +[2025-04-29 06:26:33] (step=0009350) Train Loss: 5.6673, Train Steps/Sec: 1.07 +[2025-04-29 06:26:57] (step=0009375) Train Loss: 5.6835, Train Steps/Sec: 1.07 +[2025-04-29 06:27:20] (step=0009400) Train Loss: 5.7033, Train Steps/Sec: 1.06 +[2025-04-29 06:27:44] (step=0009425) Train Loss: 5.6874, Train Steps/Sec: 1.07 +[2025-04-29 06:28:07] (step=0009450) Train Loss: 5.6879, Train Steps/Sec: 1.07 +[2025-04-29 06:28:30] (step=0009475) Train Loss: 5.6940, Train Steps/Sec: 1.07 +[2025-04-29 06:28:54] (step=0009500) Train Loss: 5.6876, Train Steps/Sec: 1.07 +[2025-04-29 06:29:17] (step=0009525) Train Loss: 5.6962, Train Steps/Sec: 1.07 +[2025-04-29 06:29:40] (step=0009550) Train Loss: 5.6532, Train Steps/Sec: 1.07 +[2025-04-29 06:30:04] (step=0009575) Train Loss: 5.6856, Train Steps/Sec: 1.07 +[2025-04-29 06:30:27] (step=0009600) Train Loss: 5.6514, Train Steps/Sec: 1.06 +[2025-04-29 06:30:51] (step=0009625) Train Loss: 5.6477, Train Steps/Sec: 1.07 +[2025-04-29 06:31:14] (step=0009650) Train Loss: 5.6648, Train Steps/Sec: 1.07 +[2025-04-29 06:31:37] (step=0009675) Train Loss: 5.6626, Train Steps/Sec: 1.07 +[2025-04-29 06:32:01] (step=0009700) Train Loss: 5.6780, Train Steps/Sec: 1.07 +[2025-04-29 06:32:24] (step=0009725) Train Loss: 5.6756, Train Steps/Sec: 1.07 +[2025-04-29 06:32:47] (step=0009750) Train Loss: 5.6596, Train Steps/Sec: 1.07 +[2025-04-29 06:33:11] (step=0009775) Train Loss: 5.6764, Train Steps/Sec: 1.07 +[2025-04-29 06:33:34] (step=0009800) Train Loss: 5.7234, Train Steps/Sec: 1.07 +[2025-04-29 06:34:03] (step=0009825) Train Loss: 5.6852, Train Steps/Sec: 0.88 +[2025-04-29 06:34:26] (step=0009850) Train Loss: 5.6784, Train Steps/Sec: 1.07 +[2025-04-29 06:34:49] (step=0009875) Train Loss: 5.6849, Train Steps/Sec: 1.07 +[2025-04-29 06:35:13] (step=0009900) Train Loss: 5.6862, Train Steps/Sec: 1.07 +[2025-04-29 06:35:36] (step=0009925) Train Loss: 5.6747, Train Steps/Sec: 1.07 +[2025-04-29 06:36:00] (step=0009950) Train Loss: 5.6979, Train Steps/Sec: 1.07 +[2025-04-29 06:36:23] (step=0009975) Train Loss: 5.6643, Train Steps/Sec: 1.07 +[2025-04-29 06:36:46] (step=0010000) Train Loss: 5.6813, Train Steps/Sec: 1.07 +[2025-04-29 06:36:46] text_config is None. Initializing the text config with default values (`OPTConfig`). +[2025-04-29 06:43:00] Finish Eval in 10000 steps... +[2025-04-29 06:43:20] Saved checkpoint to checkpoint/X2I_700k_3mask_200k_LLava_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_filtered_objects_3mask_888_context/003-GPT-XL/checkpoints/0010000.pt +[2025-04-29 06:43:23] Removed old checkpoint: checkpoint/X2I_700k_3mask_200k_LLava_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_filtered_objects_3mask_888_context/003-GPT-XL/checkpoints/0008000.pt +[2025-04-29 06:43:46] (step=0010025) Train Loss: 5.6626, Train Steps/Sec: 0.06 +[2025-04-29 06:44:09] (step=0010050) Train Loss: 5.6262, Train Steps/Sec: 1.07 +[2025-04-29 06:44:33] (step=0010075) Train Loss: 5.6851, Train Steps/Sec: 1.07 +[2025-04-29 06:44:56] (step=0010100) Train Loss: 5.6767, Train Steps/Sec: 1.07 +[2025-04-29 06:45:19] (step=0010125) Train Loss: 5.6597, Train Steps/Sec: 1.07 +[2025-04-29 06:45:42] (step=0010150) Train Loss: 5.6600, Train Steps/Sec: 1.07 +[2025-04-29 06:46:06] (step=0010175) Train Loss: 5.6687, Train Steps/Sec: 1.07 +[2025-04-29 06:46:29] (step=0010200) Train Loss: 5.6509, Train Steps/Sec: 1.07 +[2025-04-29 06:46:52] (step=0010225) Train Loss: 5.6283, Train Steps/Sec: 1.07 +[2025-04-29 06:47:16] (step=0010250) Train Loss: 5.6811, Train Steps/Sec: 1.07 +[2025-04-29 06:47:39] (step=0010275) Train Loss: 5.6529, Train Steps/Sec: 1.07 +[2025-04-29 06:48:02] (step=0010300) Train Loss: 5.6939, Train Steps/Sec: 1.07 +[2025-04-29 06:48:26] (step=0010325) Train Loss: 5.6217, Train Steps/Sec: 1.07 +[2025-04-29 06:48:49] (step=0010350) Train Loss: 5.6551, Train Steps/Sec: 1.07 +[2025-04-29 06:49:12] (step=0010375) Train Loss: 5.6804, Train Steps/Sec: 1.07 +[2025-04-29 06:49:36] (step=0010400) Train Loss: 5.6409, Train Steps/Sec: 1.07 +[2025-04-29 06:49:59] (step=0010425) Train Loss: 5.6080, Train Steps/Sec: 1.07 +[2025-04-29 06:50:22] (step=0010450) Train Loss: 5.6682, Train Steps/Sec: 1.07 +[2025-04-29 06:50:46] (step=0010475) Train Loss: 5.6738, Train Steps/Sec: 1.07 +[2025-04-29 06:51:13] (step=0010500) Train Loss: 5.6090, Train Steps/Sec: 0.93 +[2025-04-29 06:51:36] (step=0010525) Train Loss: 5.6532, Train Steps/Sec: 1.07 +[2025-04-29 06:52:14] (step=0010550) Train Loss: 5.6482, Train Steps/Sec: 0.66 +[2025-04-29 06:52:46] (step=0010575) Train Loss: 5.6383, Train Steps/Sec: 0.78 +[2025-04-29 06:53:09] (step=0010600) Train Loss: 5.6556, Train Steps/Sec: 1.07 +[2025-04-29 06:53:33] (step=0010625) Train Loss: 5.6372, Train Steps/Sec: 1.07 +[2025-04-29 06:53:56] (step=0010650) Train Loss: 5.6698, Train Steps/Sec: 1.07 +[2025-04-29 06:54:19] (step=0010675) Train Loss: 5.6394, Train Steps/Sec: 1.07 +[2025-04-29 06:54:43] (step=0010700) Train Loss: 5.6825, Train Steps/Sec: 1.07 +[2025-04-29 06:55:06] (step=0010725) Train Loss: 5.6149, Train Steps/Sec: 1.07 +[2025-04-29 06:55:29] (step=0010750) Train Loss: 5.6448, Train Steps/Sec: 1.07 +[2025-04-29 06:55:53] (step=0010775) Train Loss: 5.6573, Train Steps/Sec: 1.07 +[2025-04-29 06:56:16] (step=0010800) Train Loss: 5.6343, Train Steps/Sec: 1.07 +[2025-04-29 06:56:39] (step=0010825) Train Loss: 5.6453, Train Steps/Sec: 1.07 +[2025-04-29 06:57:03] (step=0010850) Train Loss: 5.6445, Train Steps/Sec: 1.07 +[2025-04-29 06:57:26] (step=0010875) Train Loss: 5.6419, Train Steps/Sec: 1.07 +[2025-04-29 06:57:49] (step=0010900) Train Loss: 5.6506, Train Steps/Sec: 1.07 +[2025-04-29 06:58:13] (step=0010925) Train Loss: 5.6356, Train Steps/Sec: 1.07 +[2025-04-29 06:58:36] (step=0010950) Train Loss: 5.6830, Train Steps/Sec: 1.07 +[2025-04-29 06:58:59] (step=0010975) Train Loss: 5.6555, Train Steps/Sec: 1.07 +[2025-04-29 06:59:23] (step=0011000) Train Loss: 5.6609, Train Steps/Sec: 1.07 +[2025-04-29 06:59:46] (step=0011025) Train Loss: 5.6156, Train Steps/Sec: 1.07 +[2025-04-29 07:00:09] (step=0011050) Train Loss: 5.6043, Train Steps/Sec: 1.07 +[2025-04-29 07:00:33] (step=0011075) Train Loss: 5.6439, Train Steps/Sec: 1.07 +[2025-04-29 07:00:56] (step=0011100) Train Loss: 5.6384, Train Steps/Sec: 1.07 +[2025-04-29 07:01:19] (step=0011125) Train Loss: 5.6232, Train Steps/Sec: 1.07 +[2025-04-29 07:01:42] (step=0011150) Train Loss: 5.6529, Train Steps/Sec: 1.07 +[2025-04-29 07:02:06] (step=0011175) Train Loss: 5.6555, Train Steps/Sec: 1.07 +[2025-04-29 07:02:29] (step=0011200) Train Loss: 5.6780, Train Steps/Sec: 1.06 +[2025-04-29 07:02:53] (step=0011225) Train Loss: 5.6328, Train Steps/Sec: 1.07 +[2025-04-29 07:03:16] (step=0011250) Train Loss: 5.6603, Train Steps/Sec: 1.07 +[2025-04-29 07:03:39] (step=0011275) Train Loss: 5.6633, Train Steps/Sec: 1.07 +[2025-04-29 07:04:03] (step=0011300) Train Loss: 5.6483, Train Steps/Sec: 1.07 +[2025-04-29 07:04:26] (step=0011325) Train Loss: 5.6910, Train Steps/Sec: 1.07 +[2025-04-29 07:04:49] (step=0011350) Train Loss: 5.5944, Train Steps/Sec: 1.07 +[2025-04-29 07:05:13] (step=0011375) Train Loss: 5.6404, Train Steps/Sec: 1.07 +[2025-04-29 07:05:36] (step=0011400) Train Loss: 5.5784, Train Steps/Sec: 1.07 +[2025-04-29 07:05:59] (step=0011425) Train Loss: 5.6240, Train Steps/Sec: 1.07 +[2025-04-29 07:06:23] (step=0011450) Train Loss: 5.5962, Train Steps/Sec: 1.07 +[2025-04-29 07:06:46] (step=0011475) Train Loss: 5.6259, Train Steps/Sec: 1.07 +[2025-04-29 07:07:09] (step=0011500) Train Loss: 5.6149, Train Steps/Sec: 1.07 +[2025-04-29 07:07:33] (step=0011525) Train Loss: 5.6181, Train Steps/Sec: 1.07 +[2025-04-29 07:07:56] (step=0011550) Train Loss: 5.6109, Train Steps/Sec: 1.07 +[2025-04-29 07:08:19] (step=0011575) Train Loss: 5.6214, Train Steps/Sec: 1.07 +[2025-04-29 07:08:43] (step=0011600) Train Loss: 5.6483, Train Steps/Sec: 1.07 +[2025-04-29 07:09:06] (step=0011625) Train Loss: 5.6076, Train Steps/Sec: 1.07 +[2025-04-29 07:09:29] (step=0011650) Train Loss: 5.6130, Train Steps/Sec: 1.07 +[2025-04-29 07:09:53] (step=0011675) Train Loss: 5.6370, Train Steps/Sec: 1.07 +[2025-04-29 07:10:16] (step=0011700) Train Loss: 5.6049, Train Steps/Sec: 1.07 +[2025-04-29 07:10:39] (step=0011725) Train Loss: 5.6355, Train Steps/Sec: 1.07 +[2025-04-29 07:11:03] (step=0011750) Train Loss: 5.6030, Train Steps/Sec: 1.07 +[2025-04-29 07:11:26] (step=0011775) Train Loss: 5.6066, Train Steps/Sec: 1.07 +[2025-04-29 07:11:49] (step=0011800) Train Loss: 5.6152, Train Steps/Sec: 1.07 +[2025-04-29 07:12:18] (step=0011825) Train Loss: 5.6392, Train Steps/Sec: 0.88 +[2025-04-29 07:12:41] (step=0011850) Train Loss: 5.5828, Train Steps/Sec: 1.07 +[2025-04-29 07:13:04] (step=0011875) Train Loss: 5.5971, Train Steps/Sec: 1.07 +[2025-04-29 07:13:28] (step=0011900) Train Loss: 5.5842, Train Steps/Sec: 1.07 +[2025-04-29 07:13:51] (step=0011925) Train Loss: 5.5815, Train Steps/Sec: 1.07 +[2025-04-29 07:14:14] (step=0011950) Train Loss: 5.6122, Train Steps/Sec: 1.07 +[2025-04-29 07:14:38] (step=0011975) Train Loss: 5.6280, Train Steps/Sec: 1.07 +[2025-04-29 07:15:01] (step=0012000) Train Loss: 5.6161, Train Steps/Sec: 1.07 +[2025-04-29 07:15:01] text_config is None. Initializing the text config with default values (`OPTConfig`). +[2025-04-29 07:21:06] Finish Eval in 12000 steps... +[2025-04-29 07:21:26] Saved checkpoint to checkpoint/X2I_700k_3mask_200k_LLava_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_filtered_objects_3mask_888_context/003-GPT-XL/checkpoints/0012000.pt +[2025-04-29 07:21:28] Removed old checkpoint: checkpoint/X2I_700k_3mask_200k_LLava_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_filtered_objects_3mask_888_context/003-GPT-XL/checkpoints/0010000.pt +[2025-04-29 07:21:52] (step=0012025) Train Loss: 5.6102, Train Steps/Sec: 0.06 +[2025-04-29 07:22:15] (step=0012050) Train Loss: 5.6099, Train Steps/Sec: 1.07 +[2025-04-29 07:22:38] (step=0012075) Train Loss: 5.6332, Train Steps/Sec: 1.07 +[2025-04-29 07:23:02] (step=0012100) Train Loss: 5.5788, Train Steps/Sec: 1.07 +[2025-04-29 07:23:25] (step=0012125) Train Loss: 5.6247, Train Steps/Sec: 1.07 +[2025-04-29 07:23:48] (step=0012150) Train Loss: 5.6152, Train Steps/Sec: 1.07 +[2025-04-29 07:24:12] (step=0012175) Train Loss: 5.6174, Train Steps/Sec: 1.07 +[2025-04-29 07:24:35] (step=0012200) Train Loss: 5.6034, Train Steps/Sec: 1.07 +[2025-04-29 07:24:58] (step=0012225) Train Loss: 5.6604, Train Steps/Sec: 1.07 +[2025-04-29 07:25:22] (step=0012250) Train Loss: 5.6412, Train Steps/Sec: 1.07 +[2025-04-29 07:25:45] (step=0012275) Train Loss: 5.6142, Train Steps/Sec: 1.07 +[2025-04-29 07:26:08] (step=0012300) Train Loss: 5.6229, Train Steps/Sec: 1.07 +[2025-04-29 07:26:35] (step=0012325) Train Loss: 5.6354, Train Steps/Sec: 0.93 +[2025-04-29 07:26:59] (step=0012350) Train Loss: 5.5657, Train Steps/Sec: 1.07 +[2025-04-29 07:27:37] (step=0012375) Train Loss: 5.6202, Train Steps/Sec: 0.66 +[2025-04-29 07:28:09] (step=0012400) Train Loss: 5.6375, Train Steps/Sec: 0.77 +[2025-04-29 07:28:33] (step=0012425) Train Loss: 5.6353, Train Steps/Sec: 1.07 +[2025-04-29 07:28:56] (step=0012450) Train Loss: 5.6245, Train Steps/Sec: 1.07 +[2025-04-29 07:29:19] (step=0012475) Train Loss: 5.6059, Train Steps/Sec: 1.07 +[2025-04-29 07:29:42] (step=0012500) Train Loss: 5.5688, Train Steps/Sec: 1.07 +[2025-04-29 07:30:06] (step=0012525) Train Loss: 5.6064, Train Steps/Sec: 1.07 +[2025-04-29 07:30:29] (step=0012550) Train Loss: 5.6176, Train Steps/Sec: 1.07 +[2025-04-29 07:30:52] (step=0012575) Train Loss: 5.6182, Train Steps/Sec: 1.07 +[2025-04-29 07:31:16] (step=0012600) Train Loss: 5.6079, Train Steps/Sec: 1.07 +[2025-04-29 07:31:39] (step=0012625) Train Loss: 5.5929, Train Steps/Sec: 1.07 +[2025-04-29 07:32:03] (step=0012650) Train Loss: 5.5959, Train Steps/Sec: 1.07 +[2025-04-29 07:32:26] (step=0012675) Train Loss: 5.6068, Train Steps/Sec: 1.07 +[2025-04-29 07:32:49] (step=0012700) Train Loss: 5.5997, Train Steps/Sec: 1.07 +[2025-04-29 07:33:12] (step=0012725) Train Loss: 5.5990, Train Steps/Sec: 1.07 +[2025-04-29 07:33:36] (step=0012750) Train Loss: 5.5619, Train Steps/Sec: 1.07 +[2025-04-29 07:33:59] (step=0012775) Train Loss: 5.6062, Train Steps/Sec: 1.07 +[2025-04-29 07:34:23] (step=0012800) Train Loss: 5.6018, Train Steps/Sec: 1.07 +[2025-04-29 07:34:46] (step=0012825) Train Loss: 5.6039, Train Steps/Sec: 1.07 +[2025-04-29 07:35:09] (step=0012850) Train Loss: 5.5776, Train Steps/Sec: 1.07 +[2025-04-29 07:35:32] (step=0012875) Train Loss: 5.5906, Train Steps/Sec: 1.07 +[2025-04-29 07:35:56] (step=0012900) Train Loss: 5.5995, Train Steps/Sec: 1.07 +[2025-04-29 07:36:19] (step=0012925) Train Loss: 5.6090, Train Steps/Sec: 1.07 +[2025-04-29 07:36:42] (step=0012950) Train Loss: 5.5639, Train Steps/Sec: 1.07 +[2025-04-29 07:37:06] (step=0012975) Train Loss: 5.5988, Train Steps/Sec: 1.07 +[2025-04-29 07:37:29] (step=0013000) Train Loss: 5.5443, Train Steps/Sec: 1.07 +[2025-04-29 07:37:52] (step=0013025) Train Loss: 5.5398, Train Steps/Sec: 1.07 +[2025-04-29 07:38:16] (step=0013050) Train Loss: 5.6135, Train Steps/Sec: 1.07 +[2025-04-29 07:38:39] (step=0013075) Train Loss: 5.5620, Train Steps/Sec: 1.07 +[2025-04-29 07:39:02] (step=0013100) Train Loss: 5.5969, Train Steps/Sec: 1.07 +[2025-04-29 07:39:26] (step=0013125) Train Loss: 5.6075, Train Steps/Sec: 1.07 +[2025-04-29 07:39:49] (step=0013150) Train Loss: 5.5943, Train Steps/Sec: 1.07 +[2025-04-29 07:40:12] (step=0013175) Train Loss: 5.5932, Train Steps/Sec: 1.07 +[2025-04-29 07:40:36] (step=0013200) Train Loss: 5.5700, Train Steps/Sec: 1.07 +[2025-04-29 07:40:59] (step=0013225) Train Loss: 5.5841, Train Steps/Sec: 1.07 +[2025-04-29 07:41:22] (step=0013250) Train Loss: 5.6120, Train Steps/Sec: 1.07 +[2025-04-29 07:41:46] (step=0013275) Train Loss: 5.5631, Train Steps/Sec: 1.07 +[2025-04-29 07:42:09] (step=0013300) Train Loss: 5.6001, Train Steps/Sec: 1.07 +[2025-04-29 07:42:32] (step=0013325) Train Loss: 5.5546, Train Steps/Sec: 1.07 +[2025-04-29 07:42:56] (step=0013350) Train Loss: 5.5810, Train Steps/Sec: 1.07 +[2025-04-29 07:43:19] (step=0013375) Train Loss: 5.6021, Train Steps/Sec: 1.07 +[2025-04-29 07:43:42] (step=0013400) Train Loss: 5.5799, Train Steps/Sec: 1.07 +[2025-04-29 07:44:06] (step=0013425) Train Loss: 5.5587, Train Steps/Sec: 1.07 +[2025-04-29 07:44:29] (step=0013450) Train Loss: 5.5708, Train Steps/Sec: 1.07 +[2025-04-29 07:44:52] (step=0013475) Train Loss: 5.6145, Train Steps/Sec: 1.07 +[2025-04-29 07:45:16] (step=0013500) Train Loss: 5.5925, Train Steps/Sec: 1.07 +[2025-04-29 07:45:39] (step=0013525) Train Loss: 5.5827, Train Steps/Sec: 1.07 +[2025-04-29 07:46:02] (step=0013550) Train Loss: 5.5588, Train Steps/Sec: 1.07 +[2025-04-29 07:46:26] (step=0013575) Train Loss: 5.6055, Train Steps/Sec: 1.07 +[2025-04-29 07:46:49] (step=0013600) Train Loss: 5.5701, Train Steps/Sec: 1.07 +[2025-04-29 07:47:12] (step=0013625) Train Loss: 5.5817, Train Steps/Sec: 1.07 +[2025-04-29 07:47:36] (step=0013650) Train Loss: 5.5826, Train Steps/Sec: 1.07 +[2025-04-29 07:47:59] (step=0013675) Train Loss: 5.5967, Train Steps/Sec: 1.07 +[2025-04-29 07:48:22] (step=0013700) Train Loss: 5.5624, Train Steps/Sec: 1.07 +[2025-04-29 07:48:46] (step=0013725) Train Loss: 5.5557, Train Steps/Sec: 1.07 +[2025-04-29 07:49:09] (step=0013750) Train Loss: 5.5652, Train Steps/Sec: 1.07 +[2025-04-29 07:49:32] (step=0013775) Train Loss: 5.5830, Train Steps/Sec: 1.07 +[2025-04-29 07:49:56] (step=0013800) Train Loss: 5.5423, Train Steps/Sec: 1.07 +[2025-04-29 07:50:24] (step=0013825) Train Loss: 5.5709, Train Steps/Sec: 0.88 +[2025-04-29 07:50:48] (step=0013850) Train Loss: 5.5457, Train Steps/Sec: 1.07 +[2025-04-29 07:51:11] (step=0013875) Train Loss: 5.5809, Train Steps/Sec: 1.07 +[2025-04-29 07:51:34] (step=0013900) Train Loss: 5.5725, Train Steps/Sec: 1.07 +[2025-04-29 07:51:58] (step=0013925) Train Loss: 5.5868, Train Steps/Sec: 1.07 +[2025-04-29 07:52:21] (step=0013950) Train Loss: 5.5737, Train Steps/Sec: 1.07 +[2025-04-29 07:52:44] (step=0013975) Train Loss: 5.5902, Train Steps/Sec: 1.07 +[2025-04-29 07:53:08] (step=0014000) Train Loss: 5.5533, Train Steps/Sec: 1.07 +[2025-04-29 07:53:08] text_config is None. Initializing the text config with default values (`OPTConfig`). +[2025-04-29 07:59:14] Finish Eval in 14000 steps... +[2025-04-29 07:59:33] Saved checkpoint to checkpoint/X2I_700k_3mask_200k_LLava_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_filtered_objects_3mask_888_context/003-GPT-XL/checkpoints/0014000.pt +[2025-04-29 07:59:36] Removed old checkpoint: checkpoint/X2I_700k_3mask_200k_LLava_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_filtered_objects_3mask_888_context/003-GPT-XL/checkpoints/0012000.pt +[2025-04-29 07:59:59] (step=0014025) Train Loss: 5.5546, Train Steps/Sec: 0.06 +[2025-04-29 08:00:22] (step=0014050) Train Loss: 5.6075, Train Steps/Sec: 1.07 +[2025-04-29 08:00:46] (step=0014075) Train Loss: 5.5798, Train Steps/Sec: 1.07 +[2025-04-29 08:01:09] (step=0014100) Train Loss: 5.5966, Train Steps/Sec: 1.07 +[2025-04-29 08:01:32] (step=0014125) Train Loss: 5.5397, Train Steps/Sec: 1.07 +[2025-04-29 08:01:59] (step=0014150) Train Loss: 5.6072, Train Steps/Sec: 0.93 +[2025-04-29 08:02:22] (step=0014175) Train Loss: 5.5601, Train Steps/Sec: 1.07 +[2025-04-29 08:02:57] (step=0014200) Train Loss: 5.5875, Train Steps/Sec: 0.73 +[2025-04-29 08:03:29] (step=0014225) Train Loss: 5.5446, Train Steps/Sec: 0.78 +[2025-04-29 08:03:56] (step=0014250) Train Loss: 5.5813, Train Steps/Sec: 0.92 +[2025-04-29 08:04:19] (step=0014275) Train Loss: 5.5751, Train Steps/Sec: 1.07 +[2025-04-29 08:04:43] (step=0014300) Train Loss: 5.5394, Train Steps/Sec: 1.06 +[2025-04-29 08:05:06] (step=0014325) Train Loss: 5.5298, Train Steps/Sec: 1.07 +[2025-04-29 08:05:30] (step=0014350) Train Loss: 5.5870, Train Steps/Sec: 1.07 +[2025-04-29 08:05:53] (step=0014375) Train Loss: 5.5338, Train Steps/Sec: 1.07 +[2025-04-29 08:06:16] (step=0014400) Train Loss: 5.5658, Train Steps/Sec: 1.07 +[2025-04-29 08:06:40] (step=0014425) Train Loss: 5.6030, Train Steps/Sec: 1.07 +[2025-04-29 08:07:03] (step=0014450) Train Loss: 5.5906, Train Steps/Sec: 1.07 +[2025-04-29 08:07:26] (step=0014475) Train Loss: 5.5478, Train Steps/Sec: 1.07 +[2025-04-29 08:07:50] (step=0014500) Train Loss: 5.5179, Train Steps/Sec: 1.07 +[2025-04-29 08:08:13] (step=0014525) Train Loss: 5.5665, Train Steps/Sec: 1.07 +[2025-04-29 08:08:36] (step=0014550) Train Loss: 5.5528, Train Steps/Sec: 1.07 +[2025-04-29 08:09:00] (step=0014575) Train Loss: 5.5221, Train Steps/Sec: 1.07 +[2025-04-29 08:09:23] (step=0014600) Train Loss: 5.5813, Train Steps/Sec: 1.07 +[2025-04-29 08:09:46] (step=0014625) Train Loss: 5.5635, Train Steps/Sec: 1.07 +[2025-04-29 08:10:10] (step=0014650) Train Loss: 5.5568, Train Steps/Sec: 1.07 +[2025-04-29 08:10:33] (step=0014675) Train Loss: 5.5543, Train Steps/Sec: 1.07 +[2025-04-29 08:10:56] (step=0014700) Train Loss: 5.5417, Train Steps/Sec: 1.07 +[2025-04-29 08:11:20] (step=0014725) Train Loss: 5.5511, Train Steps/Sec: 1.07 +[2025-04-29 08:11:43] (step=0014750) Train Loss: 5.5200, Train Steps/Sec: 1.07 +[2025-04-29 08:12:06] (step=0014775) Train Loss: 5.5722, Train Steps/Sec: 1.07 +[2025-04-29 08:12:30] (step=0014800) Train Loss: 5.5240, Train Steps/Sec: 1.07 +[2025-04-29 08:12:53] (step=0014825) Train Loss: 5.5490, Train Steps/Sec: 1.07 +[2025-04-29 08:13:16] (step=0014850) Train Loss: 5.5081, Train Steps/Sec: 1.07 +[2025-04-29 08:13:40] (step=0014875) Train Loss: 5.5414, Train Steps/Sec: 1.07 +[2025-04-29 08:14:03] (step=0014900) Train Loss: 5.5642, Train Steps/Sec: 1.07 +[2025-04-29 08:14:26] (step=0014925) Train Loss: 5.5723, Train Steps/Sec: 1.07 +[2025-04-29 08:14:50] (step=0014950) Train Loss: 5.5581, Train Steps/Sec: 1.07 +[2025-04-29 08:15:13] (step=0014975) Train Loss: 5.5734, Train Steps/Sec: 1.07 +[2025-04-29 08:15:36] (step=0015000) Train Loss: 5.5174, Train Steps/Sec: 1.07 +[2025-04-29 08:16:00] (step=0015025) Train Loss: 5.5478, Train Steps/Sec: 1.07 +[2025-04-29 08:16:23] (step=0015050) Train Loss: 5.5366, Train Steps/Sec: 1.07 +[2025-04-29 08:16:46] (step=0015075) Train Loss: 5.5593, Train Steps/Sec: 1.07 +[2025-04-29 08:17:10] (step=0015100) Train Loss: 5.5522, Train Steps/Sec: 1.07 +[2025-04-29 08:17:33] (step=0015125) Train Loss: 5.5422, Train Steps/Sec: 1.07 +[2025-04-29 08:17:56] (step=0015150) Train Loss: 5.5012, Train Steps/Sec: 1.07 +[2025-04-29 08:18:19] (step=0015175) Train Loss: 5.5454, Train Steps/Sec: 1.07 +[2025-04-29 08:18:43] (step=0015200) Train Loss: 5.5244, Train Steps/Sec: 1.07 +[2025-04-29 08:19:06] (step=0015225) Train Loss: 5.5776, Train Steps/Sec: 1.07 +[2025-04-29 08:19:29] (step=0015250) Train Loss: 5.5335, Train Steps/Sec: 1.07 +[2025-04-29 08:19:53] (step=0015275) Train Loss: 5.5761, Train Steps/Sec: 1.07 +[2025-04-29 08:20:16] (step=0015300) Train Loss: 5.5541, Train Steps/Sec: 1.07 +[2025-04-29 08:20:39] (step=0015325) Train Loss: 5.5309, Train Steps/Sec: 1.07 +[2025-04-29 08:21:03] (step=0015350) Train Loss: 5.4955, Train Steps/Sec: 1.07 +[2025-04-29 08:21:26] (step=0015375) Train Loss: 5.5297, Train Steps/Sec: 1.07 +[2025-04-29 08:21:50] (step=0015400) Train Loss: 5.5132, Train Steps/Sec: 1.07 +[2025-04-29 08:22:13] (step=0015425) Train Loss: 5.5043, Train Steps/Sec: 1.07 +[2025-04-29 08:22:36] (step=0015450) Train Loss: 5.5396, Train Steps/Sec: 1.07 +[2025-04-29 08:22:59] (step=0015475) Train Loss: 5.5255, Train Steps/Sec: 1.07 +[2025-04-29 08:23:23] (step=0015500) Train Loss: 5.5703, Train Steps/Sec: 1.07 +[2025-04-29 08:23:46] (step=0015525) Train Loss: 5.5352, Train Steps/Sec: 1.07 +[2025-04-29 08:24:09] (step=0015550) Train Loss: 5.5093, Train Steps/Sec: 1.07 +[2025-04-29 08:24:33] (step=0015575) Train Loss: 5.5172, Train Steps/Sec: 1.07 +[2025-04-29 08:24:56] (step=0015600) Train Loss: 5.5130, Train Steps/Sec: 1.07 +[2025-04-29 08:25:20] (step=0015625) Train Loss: 5.5154, Train Steps/Sec: 1.07 +[2025-04-29 08:25:43] (step=0015650) Train Loss: 5.5052, Train Steps/Sec: 1.07 +[2025-04-29 08:26:06] (step=0015675) Train Loss: 5.4906, Train Steps/Sec: 1.07 +[2025-04-29 08:26:30] (step=0015700) Train Loss: 5.5407, Train Steps/Sec: 1.07 +[2025-04-29 08:26:53] (step=0015725) Train Loss: 5.5064, Train Steps/Sec: 1.07 +[2025-04-29 08:27:16] (step=0015750) Train Loss: 5.5544, Train Steps/Sec: 1.07 +[2025-04-29 08:27:39] (step=0015775) Train Loss: 5.5401, Train Steps/Sec: 1.07 +[2025-04-29 08:28:03] (step=0015800) Train Loss: 5.5560, Train Steps/Sec: 1.07 +[2025-04-29 08:28:31] (step=0015825) Train Loss: 5.5762, Train Steps/Sec: 0.88 +[2025-04-29 08:28:55] (step=0015850) Train Loss: 5.5345, Train Steps/Sec: 1.07 +[2025-04-29 08:29:18] (step=0015875) Train Loss: 5.5260, Train Steps/Sec: 1.07 +[2025-04-29 08:29:41] (step=0015900) Train Loss: 5.5095, Train Steps/Sec: 1.07 +[2025-04-29 08:30:05] (step=0015925) Train Loss: 5.4569, Train Steps/Sec: 1.07 +[2025-04-29 08:30:28] (step=0015950) Train Loss: 5.5358, Train Steps/Sec: 1.07 +[2025-04-29 08:30:55] (step=0015975) Train Loss: 5.5510, Train Steps/Sec: 0.93 +[2025-04-29 08:31:18] (step=0016000) Train Loss: 5.5132, Train Steps/Sec: 1.07 +[2025-04-29 08:31:18] text_config is None. Initializing the text config with default values (`OPTConfig`). +[2025-04-29 08:37:27] Finish Eval in 16000 steps... +[2025-04-29 08:37:46] Saved checkpoint to checkpoint/X2I_700k_3mask_200k_LLava_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_filtered_objects_3mask_888_context/003-GPT-XL/checkpoints/0016000.pt +[2025-04-29 08:37:49] Removed old checkpoint: checkpoint/X2I_700k_3mask_200k_LLava_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_filtered_objects_3mask_888_context/003-GPT-XL/checkpoints/0014000.pt +[2025-04-29 08:38:20] (step=0016025) Train Loss: 5.4989, Train Steps/Sec: 0.06 +[2025-04-29 08:38:52] (step=0016050) Train Loss: 5.5173, Train Steps/Sec: 0.78 +[2025-04-29 08:39:19] (step=0016075) Train Loss: 5.5201, Train Steps/Sec: 0.92 +[2025-04-29 08:39:42] (step=0016100) Train Loss: 5.5091, Train Steps/Sec: 1.08 +[2025-04-29 08:40:05] (step=0016125) Train Loss: 5.5275, Train Steps/Sec: 1.07 +[2025-04-29 08:40:29] (step=0016150) Train Loss: 5.5176, Train Steps/Sec: 1.07 +[2025-04-29 08:40:52] (step=0016175) Train Loss: 5.4848, Train Steps/Sec: 1.07 +[2025-04-29 08:41:16] (step=0016200) Train Loss: 5.4903, Train Steps/Sec: 1.07 +[2025-04-29 08:41:39] (step=0016225) Train Loss: 5.5039, Train Steps/Sec: 1.07 +[2025-04-29 08:42:02] (step=0016250) Train Loss: 5.4891, Train Steps/Sec: 1.07 +[2025-04-29 08:42:25] (step=0016275) Train Loss: 5.5368, Train Steps/Sec: 1.07 +[2025-04-29 08:42:49] (step=0016300) Train Loss: 5.4952, Train Steps/Sec: 1.07 +[2025-04-29 08:43:12] (step=0016325) Train Loss: 5.5017, Train Steps/Sec: 1.07 +[2025-04-29 08:43:35] (step=0016350) Train Loss: 5.5291, Train Steps/Sec: 1.07 +[2025-04-29 08:43:59] (step=0016375) Train Loss: 5.5355, Train Steps/Sec: 1.07 +[2025-04-29 08:44:22] (step=0016400) Train Loss: 5.5142, Train Steps/Sec: 1.07 +[2025-04-29 08:44:46] (step=0016425) Train Loss: 5.5458, Train Steps/Sec: 1.07 +[2025-04-29 08:45:09] (step=0016450) Train Loss: 5.4864, Train Steps/Sec: 1.07 +[2025-04-29 08:45:32] (step=0016475) Train Loss: 5.5509, Train Steps/Sec: 1.07 +[2025-04-29 08:45:56] (step=0016500) Train Loss: 5.4937, Train Steps/Sec: 1.07 +[2025-04-29 08:46:19] (step=0016525) Train Loss: 5.5346, Train Steps/Sec: 1.07 +[2025-04-29 08:46:42] (step=0016550) Train Loss: 5.4860, Train Steps/Sec: 1.07 +[2025-04-29 08:47:05] (step=0016575) Train Loss: 5.4857, Train Steps/Sec: 1.07 +[2025-04-29 08:47:29] (step=0016600) Train Loss: 5.4978, Train Steps/Sec: 1.07 +[2025-04-29 08:47:52] (step=0016625) Train Loss: 5.4841, Train Steps/Sec: 1.07 +[2025-04-29 08:48:16] (step=0016650) Train Loss: 5.5010, Train Steps/Sec: 1.07 +[2025-04-29 08:48:39] (step=0016675) Train Loss: 5.4658, Train Steps/Sec: 1.07 +[2025-04-29 08:49:02] (step=0016700) Train Loss: 5.4964, Train Steps/Sec: 1.07 +[2025-04-29 08:49:26] (step=0016725) Train Loss: 5.5127, Train Steps/Sec: 1.07 +[2025-04-29 08:49:49] (step=0016750) Train Loss: 5.4751, Train Steps/Sec: 1.07 +[2025-04-29 08:50:12] (step=0016775) Train Loss: 5.4721, Train Steps/Sec: 1.07 +[2025-04-29 08:50:36] (step=0016800) Train Loss: 5.5270, Train Steps/Sec: 1.07 +[2025-04-29 08:50:59] (step=0016825) Train Loss: 5.4932, Train Steps/Sec: 1.07 +[2025-04-29 08:51:22] (step=0016850) Train Loss: 5.5325, Train Steps/Sec: 1.07 +[2025-04-29 08:51:46] (step=0016875) Train Loss: 5.5165, Train Steps/Sec: 1.07 +[2025-04-29 08:52:09] (step=0016900) Train Loss: 5.4917, Train Steps/Sec: 1.07 +[2025-04-29 08:52:32] (step=0016925) Train Loss: 5.4518, Train Steps/Sec: 1.07 +[2025-04-29 08:52:56] (step=0016950) Train Loss: 5.4897, Train Steps/Sec: 1.07 +[2025-04-29 08:53:19] (step=0016975) Train Loss: 5.4738, Train Steps/Sec: 1.07 +[2025-04-29 08:53:42] (step=0017000) Train Loss: 5.5400, Train Steps/Sec: 1.07 +[2025-04-29 08:54:06] (step=0017025) Train Loss: 5.5009, Train Steps/Sec: 1.07 +[2025-04-29 08:54:29] (step=0017050) Train Loss: 5.4947, Train Steps/Sec: 1.07 +[2025-04-29 08:54:52] (step=0017075) Train Loss: 5.5011, Train Steps/Sec: 1.07 +[2025-04-29 08:55:16] (step=0017100) Train Loss: 5.5044, Train Steps/Sec: 1.07 +[2025-04-29 08:55:39] (step=0017125) Train Loss: 5.4795, Train Steps/Sec: 1.07 +[2025-04-29 08:56:02] (step=0017150) Train Loss: 5.5091, Train Steps/Sec: 1.07 +[2025-04-29 08:56:26] (step=0017175) Train Loss: 5.4738, Train Steps/Sec: 1.07 +[2025-04-29 08:56:49] (step=0017200) Train Loss: 5.4797, Train Steps/Sec: 1.07 +[2025-04-29 08:57:12] (step=0017225) Train Loss: 5.5094, Train Steps/Sec: 1.07 +[2025-04-29 08:57:36] (step=0017250) Train Loss: 5.4896, Train Steps/Sec: 1.07 +[2025-04-29 08:57:59] (step=0017275) Train Loss: 5.4999, Train Steps/Sec: 1.07 +[2025-04-29 08:58:22] (step=0017300) Train Loss: 5.5186, Train Steps/Sec: 1.07 +[2025-04-29 08:58:46] (step=0017325) Train Loss: 5.5137, Train Steps/Sec: 1.07 +[2025-04-29 08:59:09] (step=0017350) Train Loss: 5.4977, Train Steps/Sec: 1.07 +[2025-04-29 08:59:32] (step=0017375) Train Loss: 5.4916, Train Steps/Sec: 1.07 +[2025-04-29 08:59:56] (step=0017400) Train Loss: 5.5100, Train Steps/Sec: 1.07 +[2025-04-29 09:00:19] (step=0017425) Train Loss: 5.4911, Train Steps/Sec: 1.07 +[2025-04-29 09:00:42] (step=0017450) Train Loss: 5.5129, Train Steps/Sec: 1.07 +[2025-04-29 09:01:06] (step=0017475) Train Loss: 5.5060, Train Steps/Sec: 1.07 +[2025-04-29 09:01:29] (step=0017500) Train Loss: 5.4596, Train Steps/Sec: 1.07 +[2025-04-29 09:01:52] (step=0017525) Train Loss: 5.5039, Train Steps/Sec: 1.07 +[2025-04-29 09:02:16] (step=0017550) Train Loss: 5.4754, Train Steps/Sec: 1.07 +[2025-04-29 09:02:39] (step=0017575) Train Loss: 5.4865, Train Steps/Sec: 1.07 +[2025-04-29 09:03:02] (step=0017600) Train Loss: 5.4845, Train Steps/Sec: 1.07 +[2025-04-29 09:03:26] (step=0017625) Train Loss: 5.4899, Train Steps/Sec: 1.07 +[2025-04-29 09:03:49] (step=0017650) Train Loss: 5.5368, Train Steps/Sec: 1.07 +[2025-04-29 09:04:12] (step=0017675) Train Loss: 5.4715, Train Steps/Sec: 1.07 +[2025-04-29 09:04:36] (step=0017700) Train Loss: 5.4872, Train Steps/Sec: 1.07 +[2025-04-29 09:04:59] (step=0017725) Train Loss: 5.4449, Train Steps/Sec: 1.07 +[2025-04-29 09:05:22] (step=0017750) Train Loss: 5.4830, Train Steps/Sec: 1.07 +[2025-04-29 09:05:46] (step=0017775) Train Loss: 5.5085, Train Steps/Sec: 1.07 +[2025-04-29 09:06:13] (step=0017800) Train Loss: 5.5013, Train Steps/Sec: 0.92 +[2025-04-29 09:06:41] (step=0017825) Train Loss: 5.4543, Train Steps/Sec: 0.88 +[2025-04-29 09:07:15] (step=0017850) Train Loss: 5.4765, Train Steps/Sec: 0.73 +[2025-04-29 09:07:42] (step=0017875) Train Loss: 5.5138, Train Steps/Sec: 0.92 +[2025-04-29 09:08:14] (step=0017900) Train Loss: 5.5145, Train Steps/Sec: 0.78 +[2025-04-29 09:08:38] (step=0017925) Train Loss: 5.4551, Train Steps/Sec: 1.07 +[2025-04-29 09:09:01] (step=0017950) Train Loss: 5.4655, Train Steps/Sec: 1.07 +[2025-04-29 09:09:24] (step=0017975) Train Loss: 5.4681, Train Steps/Sec: 1.07 +[2025-04-29 09:09:48] (step=0018000) Train Loss: 5.4585, Train Steps/Sec: 1.07 +[2025-04-29 09:09:48] text_config is None. Initializing the text config with default values (`OPTConfig`). +[2025-04-29 09:15:54] Finish Eval in 18000 steps... +[2025-04-29 09:16:13] Saved checkpoint to checkpoint/X2I_700k_3mask_200k_LLava_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_filtered_objects_3mask_888_context/003-GPT-XL/checkpoints/0018000.pt +[2025-04-29 09:16:16] Removed old checkpoint: checkpoint/X2I_700k_3mask_200k_LLava_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_filtered_objects_3mask_888_context/003-GPT-XL/checkpoints/0016000.pt +[2025-04-29 09:16:39] (step=0018025) Train Loss: 5.4648, Train Steps/Sec: 0.06 +[2025-04-29 09:17:02] (step=0018050) Train Loss: 5.4403, Train Steps/Sec: 1.07 +[2025-04-29 09:17:26] (step=0018075) Train Loss: 5.5071, Train Steps/Sec: 1.06 +[2025-04-29 09:17:49] (step=0018100) Train Loss: 5.4691, Train Steps/Sec: 1.07 +[2025-04-29 09:18:12] (step=0018125) Train Loss: 5.4943, Train Steps/Sec: 1.07 +[2025-04-29 09:18:36] (step=0018150) Train Loss: 5.4989, Train Steps/Sec: 1.07 +[2025-04-29 09:18:59] (step=0018175) Train Loss: 5.5170, Train Steps/Sec: 1.07 +[2025-04-29 09:19:22] (step=0018200) Train Loss: 5.5043, Train Steps/Sec: 1.07 +[2025-04-29 09:19:46] (step=0018225) Train Loss: 5.4651, Train Steps/Sec: 1.07 +[2025-04-29 09:20:09] (step=0018250) Train Loss: 5.4732, Train Steps/Sec: 1.07 +[2025-04-29 09:20:32] (step=0018275) Train Loss: 5.4628, Train Steps/Sec: 1.07 +[2025-04-29 09:20:56] (step=0018300) Train Loss: 5.4716, Train Steps/Sec: 1.07 +[2025-04-29 09:21:19] (step=0018325) Train Loss: 5.4583, Train Steps/Sec: 1.07 +[2025-04-29 09:21:42] (step=0018350) Train Loss: 5.4479, Train Steps/Sec: 1.07 +[2025-04-29 09:22:05] (step=0018375) Train Loss: 5.4735, Train Steps/Sec: 1.07 +[2025-04-29 09:22:29] (step=0018400) Train Loss: 5.4485, Train Steps/Sec: 1.07 +[2025-04-29 09:22:52] (step=0018425) Train Loss: 5.4465, Train Steps/Sec: 1.07 +[2025-04-29 09:23:16] (step=0018450) Train Loss: 5.4573, Train Steps/Sec: 1.07 +[2025-04-29 09:23:39] (step=0018475) Train Loss: 5.4743, Train Steps/Sec: 1.07 +[2025-04-29 09:24:02] (step=0018500) Train Loss: 5.4736, Train Steps/Sec: 1.07 +[2025-04-29 09:24:26] (step=0018525) Train Loss: 5.4825, Train Steps/Sec: 1.07 +[2025-04-29 09:24:49] (step=0018550) Train Loss: 5.4365, Train Steps/Sec: 1.07 +[2025-04-29 09:25:12] (step=0018575) Train Loss: 5.4780, Train Steps/Sec: 1.07 +[2025-04-29 09:25:36] (step=0018600) Train Loss: 5.4439, Train Steps/Sec: 1.07 +[2025-04-29 09:25:59] (step=0018625) Train Loss: 5.4172, Train Steps/Sec: 1.07 +[2025-04-29 09:26:22] (step=0018650) Train Loss: 5.4675, Train Steps/Sec: 1.07 +[2025-04-29 09:26:46] (step=0018675) Train Loss: 5.4421, Train Steps/Sec: 1.07 +[2025-04-29 09:27:09] (step=0018700) Train Loss: 5.4884, Train Steps/Sec: 1.07 +[2025-04-29 09:27:32] (step=0018725) Train Loss: 5.4299, Train Steps/Sec: 1.07 +[2025-04-29 09:27:56] (step=0018750) Train Loss: 5.4688, Train Steps/Sec: 1.07 +[2025-04-29 09:28:19] (step=0018775) Train Loss: 5.4478, Train Steps/Sec: 1.07 +[2025-04-29 09:28:43] (step=0018800) Train Loss: 5.4857, Train Steps/Sec: 1.06 +[2025-04-29 09:29:06] (step=0018825) Train Loss: 5.4486, Train Steps/Sec: 1.07 +[2025-04-29 09:29:29] (step=0018850) Train Loss: 5.4134, Train Steps/Sec: 1.07 +[2025-04-29 09:29:53] (step=0018875) Train Loss: 5.4713, Train Steps/Sec: 1.07 +[2025-04-29 09:30:16] (step=0018900) Train Loss: 5.4576, Train Steps/Sec: 1.07 +[2025-04-29 09:30:39] (step=0018925) Train Loss: 5.4791, Train Steps/Sec: 1.07 +[2025-04-29 09:31:03] (step=0018950) Train Loss: 5.4137, Train Steps/Sec: 1.07 +[2025-04-29 09:31:26] (step=0018975) Train Loss: 5.4501, Train Steps/Sec: 1.07 +[2025-04-29 09:31:49] (step=0019000) Train Loss: 5.4553, Train Steps/Sec: 1.07 +[2025-04-29 09:32:13] (step=0019025) Train Loss: 5.4304, Train Steps/Sec: 1.07 +[2025-04-29 09:32:36] (step=0019050) Train Loss: 5.3881, Train Steps/Sec: 1.07 +[2025-04-29 09:32:59] (step=0019075) Train Loss: 5.4838, Train Steps/Sec: 1.07 +[2025-04-29 09:33:23] (step=0019100) Train Loss: 5.4403, Train Steps/Sec: 1.07 +[2025-04-29 09:33:46] (step=0019125) Train Loss: 5.4572, Train Steps/Sec: 1.07 +[2025-04-29 09:34:09] (step=0019150) Train Loss: 5.4832, Train Steps/Sec: 1.07 +[2025-04-29 09:34:33] (step=0019175) Train Loss: 5.4566, Train Steps/Sec: 1.07 +[2025-04-29 09:34:56] (step=0019200) Train Loss: 5.4452, Train Steps/Sec: 1.07 +[2025-04-29 09:35:20] (step=0019225) Train Loss: 5.4772, Train Steps/Sec: 1.07 +[2025-04-29 09:35:43] (step=0019250) Train Loss: 5.4370, Train Steps/Sec: 1.07 +[2025-04-29 09:36:06] (step=0019275) Train Loss: 5.4299, Train Steps/Sec: 1.07 +[2025-04-29 09:36:30] (step=0019300) Train Loss: 5.4360, Train Steps/Sec: 1.07 +[2025-04-29 09:36:53] (step=0019325) Train Loss: 5.4435, Train Steps/Sec: 1.07 +[2025-04-29 09:37:16] (step=0019350) Train Loss: 5.4418, Train Steps/Sec: 1.07 +[2025-04-29 09:37:39] (step=0019375) Train Loss: 5.4819, Train Steps/Sec: 1.07 +[2025-04-29 09:38:03] (step=0019400) Train Loss: 5.4311, Train Steps/Sec: 1.07 +[2025-04-29 09:38:26] (step=0019425) Train Loss: 5.4420, Train Steps/Sec: 1.07 +[2025-04-29 09:38:50] (step=0019450) Train Loss: 5.3994, Train Steps/Sec: 1.07 +[2025-04-29 09:39:13] (step=0019475) Train Loss: 5.4396, Train Steps/Sec: 1.07 +[2025-04-29 09:39:36] (step=0019500) Train Loss: 5.3774, Train Steps/Sec: 1.07 +[2025-04-29 09:40:00] (step=0019525) Train Loss: 5.4365, Train Steps/Sec: 1.07 +[2025-04-29 09:40:23] (step=0019550) Train Loss: 5.4819, Train Steps/Sec: 1.07 +[2025-04-29 09:40:46] (step=0019575) Train Loss: 5.4261, Train Steps/Sec: 1.07 +[2025-04-29 09:41:10] (step=0019600) Train Loss: 5.4151, Train Steps/Sec: 1.07 +[2025-04-29 09:41:33] (step=0019625) Train Loss: 5.4233, Train Steps/Sec: 1.07 +[2025-04-29 09:42:00] (step=0019650) Train Loss: 5.4292, Train Steps/Sec: 0.93 +[2025-04-29 09:42:33] (step=0019675) Train Loss: 5.4674, Train Steps/Sec: 0.75 +[2025-04-29 09:43:00] (step=0019700) Train Loss: 5.4311, Train Steps/Sec: 0.92 +[2025-04-29 09:43:32] (step=0019725) Train Loss: 5.4307, Train Steps/Sec: 0.78 +[2025-04-29 09:43:55] (step=0019750) Train Loss: 5.4188, Train Steps/Sec: 1.07 +[2025-04-29 09:44:19] (step=0019775) Train Loss: 5.4614, Train Steps/Sec: 1.07 +[2025-04-29 09:44:42] (step=0019800) Train Loss: 5.4207, Train Steps/Sec: 1.07 +[2025-04-29 09:45:11] (step=0019825) Train Loss: 5.4205, Train Steps/Sec: 0.88 +[2025-04-29 09:45:34] (step=0019850) Train Loss: 5.4217, Train Steps/Sec: 1.07 +[2025-04-29 09:45:57] (step=0019875) Train Loss: 5.4642, Train Steps/Sec: 1.07 +[2025-04-29 09:46:20] (step=0019900) Train Loss: 5.4332, Train Steps/Sec: 1.07 +[2025-04-29 09:46:44] (step=0019925) Train Loss: 5.4270, Train Steps/Sec: 1.07 +[2025-04-29 09:47:07] (step=0019950) Train Loss: 5.4052, Train Steps/Sec: 1.07 +[2025-04-29 09:47:30] (step=0019975) Train Loss: 5.4391, Train Steps/Sec: 1.07 +[2025-04-29 09:47:54] (step=0020000) Train Loss: 5.4363, Train Steps/Sec: 1.07 +[2025-04-29 09:47:54] text_config is None. Initializing the text config with default values (`OPTConfig`). +[2025-04-29 09:53:59] Finish Eval in 20000 steps... +[2025-04-29 09:54:19] Saved checkpoint to checkpoint/X2I_700k_3mask_200k_LLava_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_filtered_objects_3mask_888_context/003-GPT-XL/checkpoints/0020000.pt +[2025-04-29 09:54:21] Removed old checkpoint: checkpoint/X2I_700k_3mask_200k_LLava_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_filtered_objects_3mask_888_context/003-GPT-XL/checkpoints/0018000.pt +[2025-04-29 09:54:45] (step=0020025) Train Loss: 5.4268, Train Steps/Sec: 0.06 +[2025-04-29 09:55:08] (step=0020050) Train Loss: 5.4171, Train Steps/Sec: 1.07 +[2025-04-29 09:55:31] (step=0020075) Train Loss: 5.3948, Train Steps/Sec: 1.07 +[2025-04-29 09:55:54] (step=0020100) Train Loss: 5.3965, Train Steps/Sec: 1.07 +[2025-04-29 09:56:18] (step=0020125) Train Loss: 5.4149, Train Steps/Sec: 1.07 +[2025-04-29 09:56:41] (step=0020150) Train Loss: 5.4199, Train Steps/Sec: 1.07 +[2025-04-29 09:57:04] (step=0020175) Train Loss: 5.4380, Train Steps/Sec: 1.07 +[2025-04-29 09:57:28] (step=0020200) Train Loss: 5.3877, Train Steps/Sec: 1.07 +[2025-04-29 09:57:51] (step=0020225) Train Loss: 5.4076, Train Steps/Sec: 1.07 +[2025-04-29 09:58:14] (step=0020250) Train Loss: 5.4203, Train Steps/Sec: 1.07 +[2025-04-29 09:58:38] (step=0020275) Train Loss: 5.4541, Train Steps/Sec: 1.07 +[2025-04-29 09:59:01] (step=0020300) Train Loss: 5.4349, Train Steps/Sec: 1.07 +[2025-04-29 09:59:24] (step=0020325) Train Loss: 5.3799, Train Steps/Sec: 1.07 +[2025-04-29 09:59:48] (step=0020350) Train Loss: 5.4510, Train Steps/Sec: 1.06 +[2025-04-29 10:00:11] (step=0020375) Train Loss: 5.4777, Train Steps/Sec: 1.07 +[2025-04-29 10:00:35] (step=0020400) Train Loss: 5.4012, Train Steps/Sec: 1.07 +[2025-04-29 10:00:58] (step=0020425) Train Loss: 5.3798, Train Steps/Sec: 1.07 +[2025-04-29 10:01:22] (step=0020450) Train Loss: 5.4311, Train Steps/Sec: 1.07 +[2025-04-29 10:01:45] (step=0020475) Train Loss: 5.3861, Train Steps/Sec: 1.07 +[2025-04-29 10:02:08] (step=0020500) Train Loss: 5.3770, Train Steps/Sec: 1.07 +[2025-04-29 10:02:32] (step=0020525) Train Loss: 5.4088, Train Steps/Sec: 1.07 +[2025-04-29 10:02:55] (step=0020550) Train Loss: 5.4109, Train Steps/Sec: 1.07 +[2025-04-29 10:03:18] (step=0020575) Train Loss: 5.3746, Train Steps/Sec: 1.07 +[2025-04-29 10:03:42] (step=0020600) Train Loss: 5.3871, Train Steps/Sec: 1.07 +[2025-04-29 10:04:05] (step=0020625) Train Loss: 5.3330, Train Steps/Sec: 1.07 +[2025-04-29 10:04:28] (step=0020650) Train Loss: 5.4121, Train Steps/Sec: 1.07 +[2025-04-29 10:04:52] (step=0020675) Train Loss: 5.3987, Train Steps/Sec: 1.07 +[2025-04-29 10:05:15] (step=0020700) Train Loss: 5.4512, Train Steps/Sec: 1.07 +[2025-04-29 10:05:38] (step=0020725) Train Loss: 5.4253, Train Steps/Sec: 1.07 +[2025-04-29 10:06:01] (step=0020750) Train Loss: 5.3962, Train Steps/Sec: 1.07 +[2025-04-29 10:06:25] (step=0020775) Train Loss: 5.3879, Train Steps/Sec: 1.07 +[2025-04-29 10:06:48] (step=0020800) Train Loss: 5.4017, Train Steps/Sec: 1.07 +[2025-04-29 10:07:11] (step=0020825) Train Loss: 5.4864, Train Steps/Sec: 1.07 +[2025-04-29 10:07:35] (step=0020850) Train Loss: 5.4045, Train Steps/Sec: 1.07 +[2025-04-29 10:07:58] (step=0020875) Train Loss: 5.3460, Train Steps/Sec: 1.07 +[2025-04-29 10:08:21] (step=0020900) Train Loss: 5.4009, Train Steps/Sec: 1.07 +[2025-04-29 10:08:45] (step=0020925) Train Loss: 5.3862, Train Steps/Sec: 1.07 +[2025-04-29 10:09:08] (step=0020950) Train Loss: 5.4224, Train Steps/Sec: 1.07 +[2025-04-29 10:09:31] (step=0020975) Train Loss: 5.3661, Train Steps/Sec: 1.07 +[2025-04-29 10:09:55] (step=0021000) Train Loss: 5.3816, Train Steps/Sec: 1.07 +[2025-04-29 10:10:18] (step=0021025) Train Loss: 5.3846, Train Steps/Sec: 1.07 +[2025-04-29 10:10:41] (step=0021050) Train Loss: 5.4074, Train Steps/Sec: 1.07 +[2025-04-29 10:11:05] (step=0021075) Train Loss: 5.3808, Train Steps/Sec: 1.07 +[2025-04-29 10:11:28] (step=0021100) Train Loss: 5.3822, Train Steps/Sec: 1.07 +[2025-04-29 10:11:51] (step=0021125) Train Loss: 5.3822, Train Steps/Sec: 1.07 +[2025-04-29 10:12:15] (step=0021150) Train Loss: 5.4112, Train Steps/Sec: 1.07 +[2025-04-29 10:12:38] (step=0021175) Train Loss: 5.3690, Train Steps/Sec: 1.07 +[2025-04-29 10:13:01] (step=0021200) Train Loss: 5.3775, Train Steps/Sec: 1.07 +[2025-04-29 10:13:25] (step=0021225) Train Loss: 5.3547, Train Steps/Sec: 1.07 +[2025-04-29 10:13:48] (step=0021250) Train Loss: 5.4142, Train Steps/Sec: 1.07 +[2025-04-29 10:14:11] (step=0021275) Train Loss: 5.4065, Train Steps/Sec: 1.07 +[2025-04-29 10:14:35] (step=0021300) Train Loss: 5.3926, Train Steps/Sec: 1.07 +[2025-04-29 10:14:58] (step=0021325) Train Loss: 5.3736, Train Steps/Sec: 1.07 +[2025-04-29 10:15:21] (step=0021350) Train Loss: 5.3554, Train Steps/Sec: 1.07 +[2025-04-29 10:15:45] (step=0021375) Train Loss: 5.3536, Train Steps/Sec: 1.07 +[2025-04-29 10:16:08] (step=0021400) Train Loss: 5.4314, Train Steps/Sec: 1.07 +[2025-04-29 10:16:31] (step=0021425) Train Loss: 5.4026, Train Steps/Sec: 1.07 +[2025-04-29 10:16:55] (step=0021450) Train Loss: 5.3922, Train Steps/Sec: 1.07 +[2025-04-29 10:17:22] (step=0021475) Train Loss: 5.3846, Train Steps/Sec: 0.93 +[2025-04-29 10:17:51] (step=0021500) Train Loss: 5.4061, Train Steps/Sec: 0.85 +[2025-04-29 10:18:18] (step=0021525) Train Loss: 5.4147, Train Steps/Sec: 0.93 +[2025-04-29 10:18:50] (step=0021550) Train Loss: 5.4143, Train Steps/Sec: 0.78 +[2025-04-29 10:19:13] (step=0021575) Train Loss: 5.3790, Train Steps/Sec: 1.07 +[2025-04-29 10:19:37] (step=0021600) Train Loss: 5.3788, Train Steps/Sec: 1.07 +[2025-04-29 10:20:00] (step=0021625) Train Loss: 5.4109, Train Steps/Sec: 1.07 +[2025-04-29 10:20:23] (step=0021650) Train Loss: 5.3888, Train Steps/Sec: 1.07 +[2025-04-29 10:20:47] (step=0021675) Train Loss: 5.3486, Train Steps/Sec: 1.07 +[2025-04-29 10:21:10] (step=0021700) Train Loss: 5.3622, Train Steps/Sec: 1.07 +[2025-04-29 10:21:33] (step=0021725) Train Loss: 5.3279, Train Steps/Sec: 1.07 +[2025-04-29 10:21:57] (step=0021750) Train Loss: 5.3784, Train Steps/Sec: 1.07 +[2025-04-29 10:22:20] (step=0021775) Train Loss: 5.3495, Train Steps/Sec: 1.07 +[2025-04-29 10:22:43] (step=0021800) Train Loss: 5.3971, Train Steps/Sec: 1.07 +[2025-04-29 10:23:12] (step=0021825) Train Loss: 5.3605, Train Steps/Sec: 0.88 +[2025-04-29 10:23:35] (step=0021850) Train Loss: 5.3797, Train Steps/Sec: 1.07 +[2025-04-29 10:23:58] (step=0021875) Train Loss: 5.3925, Train Steps/Sec: 1.07 +[2025-04-29 10:24:22] (step=0021900) Train Loss: 5.3783, Train Steps/Sec: 1.07 +[2025-04-29 10:24:45] (step=0021925) Train Loss: 5.3551, Train Steps/Sec: 1.07 +[2025-04-29 10:25:08] (step=0021950) Train Loss: 5.3834, Train Steps/Sec: 1.07 +[2025-04-29 10:25:32] (step=0021975) Train Loss: 5.3904, Train Steps/Sec: 1.07 +[2025-04-29 10:25:55] (step=0022000) Train Loss: 5.3987, Train Steps/Sec: 1.07 +[2025-04-29 10:25:55] text_config is None. Initializing the text config with default values (`OPTConfig`). +[2025-04-29 10:32:00] Finish Eval in 22000 steps... +[2025-04-29 10:32:20] Saved checkpoint to checkpoint/X2I_700k_3mask_200k_LLava_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_filtered_objects_3mask_888_context/003-GPT-XL/checkpoints/0022000.pt +[2025-04-29 10:32:22] Removed old checkpoint: checkpoint/X2I_700k_3mask_200k_LLava_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_filtered_objects_3mask_888_context/003-GPT-XL/checkpoints/0020000.pt +[2025-04-29 10:32:46] (step=0022025) Train Loss: 5.3955, Train Steps/Sec: 0.06 +[2025-04-29 10:33:09] (step=0022050) Train Loss: 5.4084, Train Steps/Sec: 1.07 +[2025-04-29 10:33:32] (step=0022075) Train Loss: 5.3696, Train Steps/Sec: 1.07 +[2025-04-29 10:33:56] (step=0022100) Train Loss: 5.3669, Train Steps/Sec: 1.07 +[2025-04-29 10:34:19] (step=0022125) Train Loss: 5.3708, Train Steps/Sec: 1.07 +[2025-04-29 10:34:42] (step=0022150) Train Loss: 5.3647, Train Steps/Sec: 1.07 +[2025-04-29 10:35:05] (step=0022175) Train Loss: 5.3857, Train Steps/Sec: 1.07 +[2025-04-29 10:35:29] (step=0022200) Train Loss: 5.3871, Train Steps/Sec: 1.07 +[2025-04-29 10:35:52] (step=0022225) Train Loss: 5.3834, Train Steps/Sec: 1.07 +[2025-04-29 10:36:15] (step=0022250) Train Loss: 5.3720, Train Steps/Sec: 1.07 +[2025-04-29 10:36:39] (step=0022275) Train Loss: 5.3898, Train Steps/Sec: 1.08 +[2025-04-29 10:37:02] (step=0022300) Train Loss: 5.3591, Train Steps/Sec: 1.07 +[2025-04-29 10:37:25] (step=0022325) Train Loss: 5.3790, Train Steps/Sec: 1.07 +[2025-04-29 10:37:49] (step=0022350) Train Loss: 5.3982, Train Steps/Sec: 1.07 +[2025-04-29 10:38:12] (step=0022375) Train Loss: 5.3641, Train Steps/Sec: 1.07 +[2025-04-29 10:38:35] (step=0022400) Train Loss: 5.3342, Train Steps/Sec: 1.07 +[2025-04-29 10:38:59] (step=0022425) Train Loss: 5.3685, Train Steps/Sec: 1.07 +[2025-04-29 10:39:22] (step=0022450) Train Loss: 5.3928, Train Steps/Sec: 1.07 +[2025-04-29 10:39:45] (step=0022475) Train Loss: 5.3594, Train Steps/Sec: 1.07 +[2025-04-29 10:40:09] (step=0022500) Train Loss: 5.3335, Train Steps/Sec: 1.07 +[2025-04-29 10:40:32] (step=0022525) Train Loss: 5.3560, Train Steps/Sec: 1.07 +[2025-04-29 10:40:55] (step=0022550) Train Loss: 5.3433, Train Steps/Sec: 1.07 +[2025-04-29 10:41:19] (step=0022575) Train Loss: 5.3243, Train Steps/Sec: 1.08 +[2025-04-29 10:41:42] (step=0022600) Train Loss: 5.3299, Train Steps/Sec: 1.07 +[2025-04-29 10:42:05] (step=0022625) Train Loss: 5.3828, Train Steps/Sec: 1.07 +[2025-04-29 10:42:29] (step=0022650) Train Loss: 5.3508, Train Steps/Sec: 1.07 +[2025-04-29 10:42:52] (step=0022675) Train Loss: 5.3410, Train Steps/Sec: 1.07 +[2025-04-29 10:43:15] (step=0022700) Train Loss: 5.3587, Train Steps/Sec: 1.07 +[2025-04-29 10:43:30] Beginning epoch 1... +[2025-04-29 10:43:41] (step=0022725) Train Loss: 5.3310, Train Steps/Sec: 0.97 +[2025-04-29 10:44:05] (step=0022750) Train Loss: 5.3631, Train Steps/Sec: 1.07 +[2025-04-29 10:44:28] (step=0022775) Train Loss: 5.3275, Train Steps/Sec: 1.07 +[2025-04-29 10:44:52] (step=0022800) Train Loss: 5.3543, Train Steps/Sec: 1.07 +[2025-04-29 10:45:15] (step=0022825) Train Loss: 5.3428, Train Steps/Sec: 1.07 +[2025-04-29 10:45:38] (step=0022850) Train Loss: 5.3593, Train Steps/Sec: 1.07 +[2025-04-29 10:46:02] (step=0022875) Train Loss: 5.3529, Train Steps/Sec: 1.07 +[2025-04-29 10:46:25] (step=0022900) Train Loss: 5.3342, Train Steps/Sec: 1.07 +[2025-04-29 10:46:48] (step=0022925) Train Loss: 5.3419, Train Steps/Sec: 1.07 +[2025-04-29 10:47:12] (step=0022950) Train Loss: 5.3800, Train Steps/Sec: 1.07 +[2025-04-29 10:47:35] (step=0022975) Train Loss: 5.3806, Train Steps/Sec: 1.07 +[2025-04-29 10:47:59] (step=0023000) Train Loss: 5.3415, Train Steps/Sec: 1.07 +[2025-04-29 10:48:22] (step=0023025) Train Loss: 5.3212, Train Steps/Sec: 1.07 +[2025-04-29 10:48:45] (step=0023050) Train Loss: 5.3440, Train Steps/Sec: 1.07 +[2025-04-29 10:49:09] (step=0023075) Train Loss: 5.3573, Train Steps/Sec: 1.07 +[2025-04-29 10:49:32] (step=0023100) Train Loss: 5.3567, Train Steps/Sec: 1.07 +[2025-04-29 10:49:55] (step=0023125) Train Loss: 5.3897, Train Steps/Sec: 1.07 +[2025-04-29 10:50:19] (step=0023150) Train Loss: 5.2901, Train Steps/Sec: 1.07 +[2025-04-29 10:50:42] (step=0023175) Train Loss: 5.3289, Train Steps/Sec: 1.07 +[2025-04-29 10:51:06] (step=0023200) Train Loss: 5.3431, Train Steps/Sec: 1.06 +[2025-04-29 10:51:29] (step=0023225) Train Loss: 5.3050, Train Steps/Sec: 1.07 +[2025-04-29 10:51:52] (step=0023250) Train Loss: 5.3344, Train Steps/Sec: 1.07 +[2025-04-29 10:52:16] (step=0023275) Train Loss: 5.3365, Train Steps/Sec: 1.07 +[2025-04-29 10:52:44] (step=0023300) Train Loss: 5.3361, Train Steps/Sec: 0.89 +[2025-04-29 10:53:17] (step=0023325) Train Loss: 5.3467, Train Steps/Sec: 0.77 +[2025-04-29 10:53:54] (step=0023350) Train Loss: 5.3220, Train Steps/Sec: 0.67 +[2025-04-29 10:54:23] (step=0023375) Train Loss: 5.3772, Train Steps/Sec: 0.86 +[2025-04-29 10:54:46] (step=0023400) Train Loss: 5.3291, Train Steps/Sec: 1.07 +[2025-04-29 10:55:10] (step=0023425) Train Loss: 5.3216, Train Steps/Sec: 1.07 +[2025-04-29 10:55:33] (step=0023450) Train Loss: 5.3298, Train Steps/Sec: 1.07 +[2025-04-29 10:55:57] (step=0023475) Train Loss: 5.3357, Train Steps/Sec: 1.07 +[2025-04-29 10:56:20] (step=0023500) Train Loss: 5.3131, Train Steps/Sec: 1.07 +[2025-04-29 10:56:43] (step=0023525) Train Loss: 5.3172, Train Steps/Sec: 1.07 +[2025-04-29 10:57:07] (step=0023550) Train Loss: 5.3142, Train Steps/Sec: 1.07 +[2025-04-29 10:57:30] (step=0023575) Train Loss: 5.3333, Train Steps/Sec: 1.07 +[2025-04-29 10:57:53] (step=0023600) Train Loss: 5.3597, Train Steps/Sec: 1.06 +[2025-04-29 10:58:17] (step=0023625) Train Loss: 5.3118, Train Steps/Sec: 1.07 +[2025-04-29 10:58:40] (step=0023650) Train Loss: 5.3399, Train Steps/Sec: 1.07 +[2025-04-29 10:59:04] (step=0023675) Train Loss: 5.3148, Train Steps/Sec: 1.07 +[2025-04-29 10:59:27] (step=0023700) Train Loss: 5.2945, Train Steps/Sec: 1.07 +[2025-04-29 10:59:50] (step=0023725) Train Loss: 5.3206, Train Steps/Sec: 1.07 +[2025-04-29 11:00:14] (step=0023750) Train Loss: 5.3269, Train Steps/Sec: 1.07 +[2025-04-29 11:00:37] (step=0023775) Train Loss: 5.2823, Train Steps/Sec: 1.07 +[2025-04-29 11:01:01] (step=0023800) Train Loss: 5.3338, Train Steps/Sec: 1.06 +[2025-04-29 11:01:30] (step=0023825) Train Loss: 5.3446, Train Steps/Sec: 0.85 +[2025-04-29 11:01:53] (step=0023850) Train Loss: 5.3359, Train Steps/Sec: 1.07 +[2025-04-29 11:02:17] (step=0023875) Train Loss: 5.3156, Train Steps/Sec: 1.06 +[2025-04-29 11:02:41] (step=0023900) Train Loss: 5.3365, Train Steps/Sec: 1.06 +[2025-04-29 11:03:04] (step=0023925) Train Loss: 5.3224, Train Steps/Sec: 1.06 +[2025-04-29 11:03:28] (step=0023950) Train Loss: 5.3429, Train Steps/Sec: 1.06 +[2025-04-29 11:03:52] (step=0023975) Train Loss: 5.3303, Train Steps/Sec: 1.06 +[2025-04-29 11:04:15] (step=0024000) Train Loss: 5.3276, Train Steps/Sec: 1.06 +[2025-04-29 11:04:15] text_config is None. Initializing the text config with default values (`OPTConfig`). +[2025-04-29 11:10:24] Finish Eval in 24000 steps... +[2025-04-29 11:10:44] Saved checkpoint to checkpoint/X2I_700k_3mask_200k_LLava_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_filtered_objects_3mask_888_context/003-GPT-XL/checkpoints/0024000.pt +[2025-04-29 11:10:46] Removed old checkpoint: checkpoint/X2I_700k_3mask_200k_LLava_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_filtered_objects_3mask_888_context/003-GPT-XL/checkpoints/0022000.pt +[2025-04-29 11:11:10] (step=0024025) Train Loss: 5.3626, Train Steps/Sec: 0.06 +[2025-04-29 11:11:33] (step=0024050) Train Loss: 5.3224, Train Steps/Sec: 1.07 +[2025-04-29 11:11:57] (step=0024075) Train Loss: 5.3315, Train Steps/Sec: 1.07 +[2025-04-29 11:12:20] (step=0024100) Train Loss: 5.3309, Train Steps/Sec: 1.06 +[2025-04-29 11:12:44] (step=0024125) Train Loss: 5.3038, Train Steps/Sec: 1.06 +[2025-04-29 11:13:07] (step=0024150) Train Loss: 5.3084, Train Steps/Sec: 1.07 +[2025-04-29 11:13:30] (step=0024175) Train Loss: 5.3069, Train Steps/Sec: 1.07 +[2025-04-29 11:13:54] (step=0024200) Train Loss: 5.2831, Train Steps/Sec: 1.07 +[2025-04-29 11:14:17] (step=0024225) Train Loss: 5.2939, Train Steps/Sec: 1.07 +[2025-04-29 11:14:41] (step=0024250) Train Loss: 5.3085, Train Steps/Sec: 1.07 +[2025-04-29 11:15:04] (step=0024275) Train Loss: 5.3424, Train Steps/Sec: 1.07 +[2025-04-29 11:15:27] (step=0024300) Train Loss: 5.3353, Train Steps/Sec: 1.07 +[2025-04-29 11:15:51] (step=0024325) Train Loss: 5.3295, Train Steps/Sec: 1.07 +[2025-04-29 11:16:14] (step=0024350) Train Loss: 5.2970, Train Steps/Sec: 1.07 +[2025-04-29 11:16:38] (step=0024375) Train Loss: 5.3185, Train Steps/Sec: 1.07 +[2025-04-29 11:17:01] (step=0024400) Train Loss: 5.3046, Train Steps/Sec: 1.06 +[2025-04-29 11:17:25] (step=0024425) Train Loss: 5.2742, Train Steps/Sec: 1.07 +[2025-04-29 11:17:48] (step=0024450) Train Loss: 5.3287, Train Steps/Sec: 1.06 +[2025-04-29 11:18:12] (step=0024475) Train Loss: 5.3067, Train Steps/Sec: 1.06 +[2025-04-29 11:18:35] (step=0024500) Train Loss: 5.3079, Train Steps/Sec: 1.07 +[2025-04-29 11:18:59] (step=0024525) Train Loss: 5.2972, Train Steps/Sec: 1.05 +[2025-04-29 11:19:22] (step=0024550) Train Loss: 5.3579, Train Steps/Sec: 1.06 +[2025-04-29 11:19:46] (step=0024575) Train Loss: 5.3506, Train Steps/Sec: 1.06 +[2025-04-29 11:20:10] (step=0024600) Train Loss: 5.3599, Train Steps/Sec: 1.06 +[2025-04-29 11:20:33] (step=0024625) Train Loss: 5.3310, Train Steps/Sec: 1.05 +[2025-04-29 11:20:57] (step=0024650) Train Loss: 5.2626, Train Steps/Sec: 1.06 +[2025-04-29 11:21:20] (step=0024675) Train Loss: 5.3115, Train Steps/Sec: 1.06 +[2025-04-29 11:21:44] (step=0024700) Train Loss: 5.3001, Train Steps/Sec: 1.07 +[2025-04-29 11:22:07] (step=0024725) Train Loss: 5.3046, Train Steps/Sec: 1.07 +[2025-04-29 11:22:31] (step=0024750) Train Loss: 5.3241, Train Steps/Sec: 1.06 +[2025-04-29 11:22:54] (step=0024775) Train Loss: 5.3387, Train Steps/Sec: 1.06 +[2025-04-29 11:23:18] (step=0024800) Train Loss: 5.3060, Train Steps/Sec: 1.06 +[2025-04-29 11:23:42] (step=0024825) Train Loss: 5.3179, Train Steps/Sec: 1.07 +[2025-04-29 11:24:05] (step=0024850) Train Loss: 5.2771, Train Steps/Sec: 1.07 +[2025-04-29 11:24:29] (step=0024875) Train Loss: 5.2433, Train Steps/Sec: 1.06 +[2025-04-29 11:24:52] (step=0024900) Train Loss: 5.3349, Train Steps/Sec: 1.06 +[2025-04-29 11:25:15] (step=0024925) Train Loss: 5.3182, Train Steps/Sec: 1.07 +[2025-04-29 11:25:39] (step=0024950) Train Loss: 5.3138, Train Steps/Sec: 1.07 +[2025-04-29 11:26:02] (step=0024975) Train Loss: 5.2866, Train Steps/Sec: 1.07 +[2025-04-29 11:26:26] (step=0025000) Train Loss: 5.2903, Train Steps/Sec: 1.06 +[2025-04-29 11:26:49] (step=0025025) Train Loss: 5.2670, Train Steps/Sec: 1.07 +[2025-04-29 11:27:13] (step=0025050) Train Loss: 5.2971, Train Steps/Sec: 1.07 +[2025-04-29 11:27:36] (step=0025075) Train Loss: 5.2747, Train Steps/Sec: 1.07 +[2025-04-29 11:27:59] (step=0025100) Train Loss: 5.2715, Train Steps/Sec: 1.07 +[2025-04-29 11:28:27] (step=0025125) Train Loss: 5.3430, Train Steps/Sec: 0.92 +[2025-04-29 11:28:57] (step=0025150) Train Loss: 5.2703, Train Steps/Sec: 0.81 +[2025-04-29 11:29:32] (step=0025175) Train Loss: 5.2976, Train Steps/Sec: 0.72 +[2025-04-29 11:30:01] (step=0025200) Train Loss: 5.2657, Train Steps/Sec: 1.07 +[2025-04-29 11:30:24] (step=0025225) Train Loss: 5.3184, Train Steps/Sec: 1.07 +[2025-04-29 11:30:48] (step=0025250) Train Loss: 5.3179, Train Steps/Sec: 1.07 +[2025-04-29 11:31:11] (step=0025275) Train Loss: 5.2808, Train Steps/Sec: 1.07 +[2025-04-29 11:31:34] (step=0025300) Train Loss: 5.3077, Train Steps/Sec: 1.07 +[2025-04-29 11:31:58] (step=0025325) Train Loss: 5.3034, Train Steps/Sec: 1.06 +[2025-04-29 11:32:22] (step=0025350) Train Loss: 5.3178, Train Steps/Sec: 1.06 +[2025-04-29 11:32:45] (step=0025375) Train Loss: 5.2518, Train Steps/Sec: 1.07 +[2025-04-29 11:33:09] (step=0025400) Train Loss: 5.2760, Train Steps/Sec: 1.06 +[2025-04-29 11:33:32] (step=0025425) Train Loss: 5.2901, Train Steps/Sec: 1.07 +[2025-04-29 11:33:56] (step=0025450) Train Loss: 5.2834, Train Steps/Sec: 1.06 +[2025-04-29 11:34:19] (step=0025475) Train Loss: 5.2922, Train Steps/Sec: 1.07 +[2025-04-29 11:34:42] (step=0025500) Train Loss: 5.2564, Train Steps/Sec: 1.07 +[2025-04-29 11:35:06] (step=0025525) Train Loss: 5.2840, Train Steps/Sec: 1.07 +[2025-04-29 11:35:29] (step=0025550) Train Loss: 5.3201, Train Steps/Sec: 1.07 +[2025-04-29 11:35:53] (step=0025575) Train Loss: 5.3377, Train Steps/Sec: 1.07 +[2025-04-29 11:36:16] (step=0025600) Train Loss: 5.2707, Train Steps/Sec: 1.07 +[2025-04-29 11:36:39] (step=0025625) Train Loss: 5.2530, Train Steps/Sec: 1.07 +[2025-04-29 11:37:03] (step=0025650) Train Loss: 5.2929, Train Steps/Sec: 1.07 +[2025-04-29 11:37:26] (step=0025675) Train Loss: 5.2941, Train Steps/Sec: 1.06 +[2025-04-29 11:37:50] (step=0025700) Train Loss: 5.3233, Train Steps/Sec: 1.07 +[2025-04-29 11:38:13] (step=0025725) Train Loss: 5.2903, Train Steps/Sec: 1.06 +[2025-04-29 11:38:37] (step=0025750) Train Loss: 5.3044, Train Steps/Sec: 1.07 +[2025-04-29 11:39:00] (step=0025775) Train Loss: 5.2795, Train Steps/Sec: 1.07 +[2025-04-29 11:39:24] (step=0025800) Train Loss: 5.3064, Train Steps/Sec: 1.06 +[2025-04-29 11:39:53] (step=0025825) Train Loss: 5.2581, Train Steps/Sec: 0.85 +[2025-04-29 11:40:17] (step=0025850) Train Loss: 5.2914, Train Steps/Sec: 1.07 +[2025-04-29 11:40:40] (step=0025875) Train Loss: 5.3002, Train Steps/Sec: 1.07 +[2025-04-29 11:41:03] (step=0025900) Train Loss: 5.2849, Train Steps/Sec: 1.07 +[2025-04-29 11:41:27] (step=0025925) Train Loss: 5.2949, Train Steps/Sec: 1.07 +[2025-04-29 11:41:50] (step=0025950) Train Loss: 5.2418, Train Steps/Sec: 1.07 +[2025-04-29 11:42:14] (step=0025975) Train Loss: 5.3101, Train Steps/Sec: 1.07 +[2025-04-29 11:42:37] (step=0026000) Train Loss: 5.2828, Train Steps/Sec: 1.06 +[2025-04-29 11:42:37] text_config is None. Initializing the text config with default values (`OPTConfig`). +[2025-04-29 11:48:44] Finish Eval in 26000 steps... +[2025-04-29 11:49:04] Saved checkpoint to checkpoint/X2I_700k_3mask_200k_LLava_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_filtered_objects_3mask_888_context/003-GPT-XL/checkpoints/0026000.pt +[2025-04-29 11:49:07] Removed old checkpoint: checkpoint/X2I_700k_3mask_200k_LLava_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_filtered_objects_3mask_888_context/003-GPT-XL/checkpoints/0024000.pt +[2025-04-29 11:49:30] (step=0026025) Train Loss: 5.3020, Train Steps/Sec: 0.06 +[2025-04-29 11:49:53] (step=0026050) Train Loss: 5.2898, Train Steps/Sec: 1.07 +[2025-04-29 11:50:17] (step=0026075) Train Loss: 5.2770, Train Steps/Sec: 1.07 +[2025-04-29 11:50:40] (step=0026100) Train Loss: 5.2652, Train Steps/Sec: 1.07 +[2025-04-29 11:51:04] (step=0026125) Train Loss: 5.2955, Train Steps/Sec: 1.06 +[2025-04-29 11:51:27] (step=0026150) Train Loss: 5.3228, Train Steps/Sec: 1.07 +[2025-04-29 11:51:50] (step=0026175) Train Loss: 5.2817, Train Steps/Sec: 1.06 +[2025-04-29 11:52:14] (step=0026200) Train Loss: 5.2573, Train Steps/Sec: 1.07 +[2025-04-29 11:52:37] (step=0026225) Train Loss: 5.2813, Train Steps/Sec: 1.07 +[2025-04-29 11:53:01] (step=0026250) Train Loss: 5.2766, Train Steps/Sec: 1.07 +[2025-04-29 11:53:24] (step=0026275) Train Loss: 5.3456, Train Steps/Sec: 1.06 +[2025-04-29 11:53:48] (step=0026300) Train Loss: 5.2838, Train Steps/Sec: 1.06 +[2025-04-29 11:54:11] (step=0026325) Train Loss: 5.3173, Train Steps/Sec: 1.07 +[2025-04-29 11:54:35] (step=0026350) Train Loss: 5.2516, Train Steps/Sec: 1.07 +[2025-04-29 11:54:58] (step=0026375) Train Loss: 5.2762, Train Steps/Sec: 1.07 +[2025-04-29 11:55:21] (step=0026400) Train Loss: 5.2340, Train Steps/Sec: 1.06 +[2025-04-29 11:55:45] (step=0026425) Train Loss: 5.2510, Train Steps/Sec: 1.07 +[2025-04-29 11:56:08] (step=0026450) Train Loss: 5.2929, Train Steps/Sec: 1.07 +[2025-04-29 11:56:32] (step=0026475) Train Loss: 5.2657, Train Steps/Sec: 1.06 +[2025-04-29 11:56:55] (step=0026500) Train Loss: 5.2866, Train Steps/Sec: 1.07 +[2025-04-29 11:57:19] (step=0026525) Train Loss: 5.2708, Train Steps/Sec: 1.06 +[2025-04-29 11:57:42] (step=0026550) Train Loss: 5.2754, Train Steps/Sec: 1.07 +[2025-04-29 11:58:05] (step=0026575) Train Loss: 5.2770, Train Steps/Sec: 1.07 +[2025-04-29 11:58:29] (step=0026600) Train Loss: 5.2246, Train Steps/Sec: 1.06 +[2025-04-29 11:58:52] (step=0026625) Train Loss: 5.2626, Train Steps/Sec: 1.07 +[2025-04-29 11:59:16] (step=0026650) Train Loss: 5.2913, Train Steps/Sec: 1.07 +[2025-04-29 11:59:39] (step=0026675) Train Loss: 5.3242, Train Steps/Sec: 1.07 +[2025-04-29 12:00:03] (step=0026700) Train Loss: 5.2605, Train Steps/Sec: 1.06 +[2025-04-29 12:00:26] (step=0026725) Train Loss: 5.2805, Train Steps/Sec: 1.07 +[2025-04-29 12:00:49] (step=0026750) Train Loss: 5.2565, Train Steps/Sec: 1.07 +[2025-04-29 12:01:13] (step=0026775) Train Loss: 5.2697, Train Steps/Sec: 1.07 +[2025-04-29 12:01:36] (step=0026800) Train Loss: 5.2607, Train Steps/Sec: 1.07 +[2025-04-29 12:02:00] (step=0026825) Train Loss: 5.2546, Train Steps/Sec: 1.07 +[2025-04-29 12:02:23] (step=0026850) Train Loss: 5.2884, Train Steps/Sec: 1.07 +[2025-04-29 12:02:46] (step=0026875) Train Loss: 5.2572, Train Steps/Sec: 1.07 +[2025-04-29 12:03:10] (step=0026900) Train Loss: 5.3019, Train Steps/Sec: 1.07 +[2025-04-29 12:03:33] (step=0026925) Train Loss: 5.2559, Train Steps/Sec: 1.07 +[2025-04-29 12:04:00] (step=0026950) Train Loss: 5.2316, Train Steps/Sec: 0.92 +[2025-04-29 12:04:31] (step=0026975) Train Loss: 5.2600, Train Steps/Sec: 0.81 +[2025-04-29 12:05:02] (step=0027000) Train Loss: 5.2509, Train Steps/Sec: 0.80 +[2025-04-29 12:05:29] (step=0027025) Train Loss: 5.2088, Train Steps/Sec: 0.92 +[2025-04-29 12:05:57] (step=0027050) Train Loss: 5.2285, Train Steps/Sec: 0.89 +[2025-04-29 12:06:21] (step=0027075) Train Loss: 5.2704, Train Steps/Sec: 1.07 +[2025-04-29 12:06:44] (step=0027100) Train Loss: 5.2582, Train Steps/Sec: 1.07 +[2025-04-29 12:07:07] (step=0027125) Train Loss: 5.2979, Train Steps/Sec: 1.07 +[2025-04-29 12:07:31] (step=0027150) Train Loss: 5.2284, Train Steps/Sec: 1.07 +[2025-04-29 12:07:54] (step=0027175) Train Loss: 5.2541, Train Steps/Sec: 1.07 +[2025-04-29 12:08:18] (step=0027200) Train Loss: 5.2388, Train Steps/Sec: 1.07 +[2025-04-29 12:08:41] (step=0027225) Train Loss: 5.2613, Train Steps/Sec: 1.07 +[2025-04-29 12:09:04] (step=0027250) Train Loss: 5.2659, Train Steps/Sec: 1.07 +[2025-04-29 12:09:28] (step=0027275) Train Loss: 5.2496, Train Steps/Sec: 1.07 +[2025-04-29 12:09:51] (step=0027300) Train Loss: 5.2516, Train Steps/Sec: 1.07 +[2025-04-29 12:10:15] (step=0027325) Train Loss: 5.2693, Train Steps/Sec: 1.07 +[2025-04-29 12:10:38] (step=0027350) Train Loss: 5.2416, Train Steps/Sec: 1.07 +[2025-04-29 12:11:01] (step=0027375) Train Loss: 5.2760, Train Steps/Sec: 1.07 +[2025-04-29 12:11:25] (step=0027400) Train Loss: 5.2560, Train Steps/Sec: 1.07 +[2025-04-29 12:11:48] (step=0027425) Train Loss: 5.2685, Train Steps/Sec: 1.07 +[2025-04-29 12:12:11] (step=0027450) Train Loss: 5.2554, Train Steps/Sec: 1.07 +[2025-04-29 12:12:35] (step=0027475) Train Loss: 5.2610, Train Steps/Sec: 1.07 +[2025-04-29 12:12:58] (step=0027500) Train Loss: 5.2417, Train Steps/Sec: 1.07 +[2025-04-29 12:13:21] (step=0027525) Train Loss: 5.2157, Train Steps/Sec: 1.07 +[2025-04-29 12:13:45] (step=0027550) Train Loss: 5.1802, Train Steps/Sec: 1.07 +[2025-04-29 12:14:08] (step=0027575) Train Loss: 5.2745, Train Steps/Sec: 1.07 +[2025-04-29 12:14:32] (step=0027600) Train Loss: 5.2562, Train Steps/Sec: 1.06 +[2025-04-29 12:14:55] (step=0027625) Train Loss: 5.2708, Train Steps/Sec: 1.07 +[2025-04-29 12:15:19] (step=0027650) Train Loss: 5.2308, Train Steps/Sec: 1.07 +[2025-04-29 12:15:42] (step=0027675) Train Loss: 5.2290, Train Steps/Sec: 1.07 +[2025-04-29 12:16:05] (step=0027700) Train Loss: 5.2487, Train Steps/Sec: 1.07 +[2025-04-29 12:16:29] (step=0027725) Train Loss: 5.2307, Train Steps/Sec: 1.07 +[2025-04-29 12:16:52] (step=0027750) Train Loss: 5.2414, Train Steps/Sec: 1.07 +[2025-04-29 12:17:15] (step=0027775) Train Loss: 5.2383, Train Steps/Sec: 1.07 +[2025-04-29 12:17:39] (step=0027800) Train Loss: 5.2432, Train Steps/Sec: 1.07 +[2025-04-29 12:18:02] (step=0027825) Train Loss: 5.2462, Train Steps/Sec: 1.07 +[2025-04-29 12:18:31] (step=0027850) Train Loss: 5.2338, Train Steps/Sec: 0.86 +[2025-04-29 12:18:55] (step=0027875) Train Loss: 5.2566, Train Steps/Sec: 1.07 +[2025-04-29 12:19:18] (step=0027900) Train Loss: 5.2359, Train Steps/Sec: 1.07 +[2025-04-29 12:19:42] (step=0027925) Train Loss: 5.2635, Train Steps/Sec: 1.07 +[2025-04-29 12:20:05] (step=0027950) Train Loss: 5.2605, Train Steps/Sec: 1.07 +[2025-04-29 12:20:28] (step=0027975) Train Loss: 5.2318, Train Steps/Sec: 1.07 +[2025-04-29 12:20:52] (step=0028000) Train Loss: 5.2126, Train Steps/Sec: 1.07 +[2025-04-29 12:20:52] text_config is None. Initializing the text config with default values (`OPTConfig`). +[2025-04-29 12:27:00] Finish Eval in 28000 steps... +[2025-04-29 12:27:20] Saved checkpoint to checkpoint/X2I_700k_3mask_200k_LLava_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_filtered_objects_3mask_888_context/003-GPT-XL/checkpoints/0028000.pt +[2025-04-29 12:27:22] Removed old checkpoint: checkpoint/X2I_700k_3mask_200k_LLava_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_filtered_objects_3mask_888_context/003-GPT-XL/checkpoints/0026000.pt +[2025-04-29 12:27:46] (step=0028025) Train Loss: 5.2164, Train Steps/Sec: 0.06 +[2025-04-29 12:28:09] (step=0028050) Train Loss: 5.2797, Train Steps/Sec: 1.06 +[2025-04-29 12:28:33] (step=0028075) Train Loss: 5.2229, Train Steps/Sec: 1.07 +[2025-04-29 12:28:56] (step=0028100) Train Loss: 5.2149, Train Steps/Sec: 1.07 +[2025-04-29 12:29:19] (step=0028125) Train Loss: 5.2393, Train Steps/Sec: 1.07 +[2025-04-29 12:29:43] (step=0028150) Train Loss: 5.2575, Train Steps/Sec: 1.07 +[2025-04-29 12:30:06] (step=0028175) Train Loss: 5.2131, Train Steps/Sec: 1.06 +[2025-04-29 12:30:30] (step=0028200) Train Loss: 5.1978, Train Steps/Sec: 1.06 +[2025-04-29 12:30:53] (step=0028225) Train Loss: 5.1848, Train Steps/Sec: 1.07 +[2025-04-29 12:31:17] (step=0028250) Train Loss: 5.2455, Train Steps/Sec: 1.07 +[2025-04-29 12:31:40] (step=0028275) Train Loss: 5.2056, Train Steps/Sec: 1.07 +[2025-04-29 12:32:04] (step=0028300) Train Loss: 5.2758, Train Steps/Sec: 1.07 +[2025-04-29 12:32:27] (step=0028325) Train Loss: 5.2491, Train Steps/Sec: 1.07 +[2025-04-29 12:32:50] (step=0028350) Train Loss: 5.2551, Train Steps/Sec: 1.07 +[2025-04-29 12:33:14] (step=0028375) Train Loss: 5.1932, Train Steps/Sec: 1.07 +[2025-04-29 12:33:37] (step=0028400) Train Loss: 5.2493, Train Steps/Sec: 1.06 +[2025-04-29 12:34:01] (step=0028425) Train Loss: 5.2325, Train Steps/Sec: 1.07 +[2025-04-29 12:34:24] (step=0028450) Train Loss: 5.2675, Train Steps/Sec: 1.07 +[2025-04-29 12:34:48] (step=0028475) Train Loss: 5.2004, Train Steps/Sec: 1.07 +[2025-04-29 12:35:11] (step=0028500) Train Loss: 5.2629, Train Steps/Sec: 1.07 +[2025-04-29 12:35:34] (step=0028525) Train Loss: 5.2039, Train Steps/Sec: 1.07 +[2025-04-29 12:35:58] (step=0028550) Train Loss: 5.2420, Train Steps/Sec: 1.06 +[2025-04-29 12:36:21] (step=0028575) Train Loss: 5.1878, Train Steps/Sec: 1.07 +[2025-04-29 12:36:45] (step=0028600) Train Loss: 5.2227, Train Steps/Sec: 1.07 +[2025-04-29 12:37:08] (step=0028625) Train Loss: 5.2067, Train Steps/Sec: 1.07 +[2025-04-29 12:37:32] (step=0028650) Train Loss: 5.1911, Train Steps/Sec: 1.07 +[2025-04-29 12:37:55] (step=0028675) Train Loss: 5.2262, Train Steps/Sec: 1.07 +[2025-04-29 12:38:18] (step=0028700) Train Loss: 5.2520, Train Steps/Sec: 1.07 +[2025-04-29 12:38:42] (step=0028725) Train Loss: 5.2292, Train Steps/Sec: 1.07 +[2025-04-29 12:39:05] (step=0028750) Train Loss: 5.2110, Train Steps/Sec: 1.07 +[2025-04-29 12:39:32] (step=0028775) Train Loss: 5.2226, Train Steps/Sec: 0.92 +[2025-04-29 12:40:03] (step=0028800) Train Loss: 5.2192, Train Steps/Sec: 0.81 +[2025-04-29 12:40:30] (step=0028825) Train Loss: 5.2061, Train Steps/Sec: 0.92 +[2025-04-29 12:41:01] (step=0028850) Train Loss: 5.1933, Train Steps/Sec: 0.82 +[2025-04-29 12:41:29] (step=0028875) Train Loss: 5.2101, Train Steps/Sec: 0.89 +[2025-04-29 12:41:52] (step=0028900) Train Loss: 5.2026, Train Steps/Sec: 1.07 +[2025-04-29 12:42:16] (step=0028925) Train Loss: 5.1843, Train Steps/Sec: 1.07 +[2025-04-29 12:42:39] (step=0028950) Train Loss: 5.2237, Train Steps/Sec: 1.07 +[2025-04-29 12:43:02] (step=0028975) Train Loss: 5.2441, Train Steps/Sec: 1.07 +[2025-04-29 12:43:26] (step=0029000) Train Loss: 5.2493, Train Steps/Sec: 1.06 +[2025-04-29 12:43:49] (step=0029025) Train Loss: 5.2147, Train Steps/Sec: 1.07 +[2025-04-29 12:44:13] (step=0029050) Train Loss: 5.2493, Train Steps/Sec: 1.07 +[2025-04-29 12:44:36] (step=0029075) Train Loss: 5.2300, Train Steps/Sec: 1.07 +[2025-04-29 12:45:00] (step=0029100) Train Loss: 5.2137, Train Steps/Sec: 1.07 +[2025-04-29 12:45:23] (step=0029125) Train Loss: 5.2318, Train Steps/Sec: 1.06 +[2025-04-29 12:45:46] (step=0029150) Train Loss: 5.1863, Train Steps/Sec: 1.07 +[2025-04-29 12:46:10] (step=0029175) Train Loss: 5.2216, Train Steps/Sec: 1.07 +[2025-04-29 12:46:33] (step=0029200) Train Loss: 5.2199, Train Steps/Sec: 1.07 +[2025-04-29 12:46:57] (step=0029225) Train Loss: 5.2136, Train Steps/Sec: 1.07 +[2025-04-29 12:47:20] (step=0029250) Train Loss: 5.2190, Train Steps/Sec: 1.07 +[2025-04-29 12:47:44] (step=0029275) Train Loss: 5.2155, Train Steps/Sec: 1.07 +[2025-04-29 12:48:07] (step=0029300) Train Loss: 5.1693, Train Steps/Sec: 1.07 +[2025-04-29 12:48:30] (step=0029325) Train Loss: 5.1781, Train Steps/Sec: 1.07 +[2025-04-29 12:48:54] (step=0029350) Train Loss: 5.2320, Train Steps/Sec: 1.07 +[2025-04-29 12:49:17] (step=0029375) Train Loss: 5.2181, Train Steps/Sec: 1.07 +[2025-04-29 12:49:40] (step=0029400) Train Loss: 5.1900, Train Steps/Sec: 1.07 +[2025-04-29 12:50:04] (step=0029425) Train Loss: 5.2287, Train Steps/Sec: 1.07 +[2025-04-29 12:50:27] (step=0029450) Train Loss: 5.2059, Train Steps/Sec: 1.07 +[2025-04-29 12:50:51] (step=0029475) Train Loss: 5.1989, Train Steps/Sec: 1.07 +[2025-04-29 12:51:14] (step=0029500) Train Loss: 5.1925, Train Steps/Sec: 1.07 +[2025-04-29 12:51:37] (step=0029525) Train Loss: 5.1753, Train Steps/Sec: 1.07 +[2025-04-29 12:52:01] (step=0029550) Train Loss: 5.1792, Train Steps/Sec: 1.07 +[2025-04-29 12:52:24] (step=0029575) Train Loss: 5.1775, Train Steps/Sec: 1.07 +[2025-04-29 12:52:47] (step=0029600) Train Loss: 5.2124, Train Steps/Sec: 1.07 +[2025-04-29 12:53:11] (step=0029625) Train Loss: 5.2125, Train Steps/Sec: 1.07 +[2025-04-29 12:53:34] (step=0029650) Train Loss: 5.1764, Train Steps/Sec: 1.07 +[2025-04-29 12:53:58] (step=0029675) Train Loss: 5.2036, Train Steps/Sec: 1.07 +[2025-04-29 12:54:21] (step=0029700) Train Loss: 5.2207, Train Steps/Sec: 1.07 +[2025-04-29 12:54:44] (step=0029725) Train Loss: 5.2124, Train Steps/Sec: 1.07 +[2025-04-29 12:55:08] (step=0029750) Train Loss: 5.2169, Train Steps/Sec: 1.07 +[2025-04-29 12:55:31] (step=0029775) Train Loss: 5.2114, Train Steps/Sec: 1.06 +[2025-04-29 12:55:55] (step=0029800) Train Loss: 5.2340, Train Steps/Sec: 1.06 +[2025-04-29 12:56:23] (step=0029825) Train Loss: 5.2423, Train Steps/Sec: 0.88 +[2025-04-29 12:56:47] (step=0029850) Train Loss: 5.1694, Train Steps/Sec: 1.06 +[2025-04-29 12:57:10] (step=0029875) Train Loss: 5.1956, Train Steps/Sec: 1.07 +[2025-04-29 12:57:34] (step=0029900) Train Loss: 5.1593, Train Steps/Sec: 1.07 +[2025-04-29 12:57:57] (step=0029925) Train Loss: 5.1910, Train Steps/Sec: 1.07 +[2025-04-29 12:58:20] (step=0029950) Train Loss: 5.2130, Train Steps/Sec: 1.06 +[2025-04-29 12:58:44] (step=0029975) Train Loss: 5.2169, Train Steps/Sec: 1.06 +[2025-04-29 12:59:07] (step=0030000) Train Loss: 5.2087, Train Steps/Sec: 1.07 +[2025-04-29 12:59:07] text_config is None. Initializing the text config with default values (`OPTConfig`). +[2025-04-29 13:05:17] Finish Eval in 30000 steps... +[2025-04-29 13:05:36] Saved checkpoint to checkpoint/X2I_700k_3mask_200k_LLava_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_filtered_objects_3mask_888_context/003-GPT-XL/checkpoints/0030000.pt +[2025-04-29 13:05:38] Removed old checkpoint: checkpoint/X2I_700k_3mask_200k_LLava_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_filtered_objects_3mask_888_context/003-GPT-XL/checkpoints/0028000.pt +[2025-04-29 13:06:02] (step=0030025) Train Loss: 5.2248, Train Steps/Sec: 0.06 +[2025-04-29 13:06:25] (step=0030050) Train Loss: 5.2328, Train Steps/Sec: 1.07 +[2025-04-29 13:06:48] (step=0030075) Train Loss: 5.1814, Train Steps/Sec: 1.07 +[2025-04-29 13:07:12] (step=0030100) Train Loss: 5.2034, Train Steps/Sec: 1.07 +[2025-04-29 13:07:35] (step=0030125) Train Loss: 5.2162, Train Steps/Sec: 1.07 +[2025-04-29 13:07:58] (step=0030150) Train Loss: 5.2714, Train Steps/Sec: 1.07 +[2025-04-29 13:08:22] (step=0030175) Train Loss: 5.2001, Train Steps/Sec: 1.07 +[2025-04-29 13:08:45] (step=0030200) Train Loss: 5.1707, Train Steps/Sec: 1.07 +[2025-04-29 13:09:09] (step=0030225) Train Loss: 5.2147, Train Steps/Sec: 1.07 +[2025-04-29 13:09:32] (step=0030250) Train Loss: 5.1903, Train Steps/Sec: 1.07 +[2025-04-29 13:09:55] (step=0030275) Train Loss: 5.2096, Train Steps/Sec: 1.07 +[2025-04-29 13:10:18] (step=0030300) Train Loss: 5.1352, Train Steps/Sec: 1.07 +[2025-04-29 13:10:42] (step=0030325) Train Loss: 5.2485, Train Steps/Sec: 1.07 +[2025-04-29 13:11:05] (step=0030350) Train Loss: 5.2029, Train Steps/Sec: 1.07 +[2025-04-29 13:11:29] (step=0030375) Train Loss: 5.1775, Train Steps/Sec: 1.07 +[2025-04-29 13:11:52] (step=0030400) Train Loss: 5.1893, Train Steps/Sec: 1.07 +[2025-04-29 13:12:15] (step=0030425) Train Loss: 5.2386, Train Steps/Sec: 1.07 +[2025-04-29 13:12:39] (step=0030450) Train Loss: 5.1800, Train Steps/Sec: 1.07 +[2025-04-29 13:13:02] (step=0030475) Train Loss: 5.2027, Train Steps/Sec: 1.07 +[2025-04-29 13:13:26] (step=0030500) Train Loss: 5.1991, Train Steps/Sec: 1.06 +[2025-04-29 13:13:49] (step=0030525) Train Loss: 5.1950, Train Steps/Sec: 1.06 +[2025-04-29 13:14:12] (step=0030550) Train Loss: 5.1762, Train Steps/Sec: 1.07 +[2025-04-29 13:14:36] (step=0030575) Train Loss: 5.1982, Train Steps/Sec: 1.07 +[2025-04-29 13:14:59] (step=0030600) Train Loss: 5.1947, Train Steps/Sec: 1.06 +[2025-04-29 13:15:34] (step=0030625) Train Loss: 5.1729, Train Steps/Sec: 0.72 +[2025-04-29 13:16:01] (step=0030650) Train Loss: 5.1727, Train Steps/Sec: 0.92 +[2025-04-29 13:16:32] (step=0030675) Train Loss: 5.1740, Train Steps/Sec: 0.82 +[2025-04-29 13:17:00] (step=0030700) Train Loss: 5.1943, Train Steps/Sec: 0.88 +[2025-04-29 13:17:23] (step=0030725) Train Loss: 5.1681, Train Steps/Sec: 1.07 +[2025-04-29 13:17:47] (step=0030750) Train Loss: 5.2172, Train Steps/Sec: 1.07 +[2025-04-29 13:18:10] (step=0030775) Train Loss: 5.1802, Train Steps/Sec: 1.07 +[2025-04-29 13:18:34] (step=0030800) Train Loss: 5.1491, Train Steps/Sec: 1.06 +[2025-04-29 13:18:57] (step=0030825) Train Loss: 5.1854, Train Steps/Sec: 1.07 +[2025-04-29 13:19:20] (step=0030850) Train Loss: 5.1533, Train Steps/Sec: 1.07 +[2025-04-29 13:19:44] (step=0030875) Train Loss: 5.2202, Train Steps/Sec: 1.07 +[2025-04-29 13:20:07] (step=0030900) Train Loss: 5.1943, Train Steps/Sec: 1.07 +[2025-04-29 13:20:30] (step=0030925) Train Loss: 5.2428, Train Steps/Sec: 1.07 +[2025-04-29 13:20:54] (step=0030950) Train Loss: 5.1973, Train Steps/Sec: 1.07 +[2025-04-29 13:21:17] (step=0030975) Train Loss: 5.2051, Train Steps/Sec: 1.07 +[2025-04-29 13:21:41] (step=0031000) Train Loss: 5.1180, Train Steps/Sec: 1.07 +[2025-04-29 13:22:04] (step=0031025) Train Loss: 5.1970, Train Steps/Sec: 1.07 +[2025-04-29 13:22:27] (step=0031050) Train Loss: 5.1702, Train Steps/Sec: 1.07 +[2025-04-29 13:22:51] (step=0031075) Train Loss: 5.1655, Train Steps/Sec: 1.07 +[2025-04-29 13:23:14] (step=0031100) Train Loss: 5.1530, Train Steps/Sec: 1.07 +[2025-04-29 13:23:37] (step=0031125) Train Loss: 5.2211, Train Steps/Sec: 1.07 +[2025-04-29 13:24:01] (step=0031150) Train Loss: 5.1916, Train Steps/Sec: 1.07 +[2025-04-29 13:24:24] (step=0031175) Train Loss: 5.2369, Train Steps/Sec: 1.07 +[2025-04-29 13:24:47] (step=0031200) Train Loss: 5.1663, Train Steps/Sec: 1.07 +[2025-04-29 13:25:11] (step=0031225) Train Loss: 5.2166, Train Steps/Sec: 1.07 +[2025-04-29 13:25:34] (step=0031250) Train Loss: 5.1768, Train Steps/Sec: 1.07 +[2025-04-29 13:25:57] (step=0031275) Train Loss: 5.1858, Train Steps/Sec: 1.07 +[2025-04-29 13:26:21] (step=0031300) Train Loss: 5.1381, Train Steps/Sec: 1.07 +[2025-04-29 13:26:44] (step=0031325) Train Loss: 5.1682, Train Steps/Sec: 1.07 +[2025-04-29 13:27:08] (step=0031350) Train Loss: 5.1804, Train Steps/Sec: 1.07 +[2025-04-29 13:27:31] (step=0031375) Train Loss: 5.1651, Train Steps/Sec: 1.07 +[2025-04-29 13:27:54] (step=0031400) Train Loss: 5.2078, Train Steps/Sec: 1.07 +[2025-04-29 13:28:18] (step=0031425) Train Loss: 5.1705, Train Steps/Sec: 1.07 +[2025-04-29 13:28:41] (step=0031450) Train Loss: 5.1412, Train Steps/Sec: 1.07 +[2025-04-29 13:29:04] (step=0031475) Train Loss: 5.1621, Train Steps/Sec: 1.07 +[2025-04-29 13:29:28] (step=0031500) Train Loss: 5.1964, Train Steps/Sec: 1.07 +[2025-04-29 13:29:51] (step=0031525) Train Loss: 5.1519, Train Steps/Sec: 1.07 +[2025-04-29 13:30:14] (step=0031550) Train Loss: 5.1665, Train Steps/Sec: 1.07 +[2025-04-29 13:30:38] (step=0031575) Train Loss: 5.1658, Train Steps/Sec: 1.07 +[2025-04-29 13:31:01] (step=0031600) Train Loss: 5.1682, Train Steps/Sec: 1.07 +[2025-04-29 13:31:25] (step=0031625) Train Loss: 5.1524, Train Steps/Sec: 1.07 +[2025-04-29 13:31:48] (step=0031650) Train Loss: 5.1551, Train Steps/Sec: 1.07 +[2025-04-29 13:32:11] (step=0031675) Train Loss: 5.1682, Train Steps/Sec: 1.07 +[2025-04-29 13:32:35] (step=0031700) Train Loss: 5.1800, Train Steps/Sec: 1.07 +[2025-04-29 13:32:58] (step=0031725) Train Loss: 5.1358, Train Steps/Sec: 1.07 +[2025-04-29 13:33:21] (step=0031750) Train Loss: 5.1828, Train Steps/Sec: 1.07 +[2025-04-29 13:33:45] (step=0031775) Train Loss: 5.1580, Train Steps/Sec: 1.07 +[2025-04-29 13:34:08] (step=0031800) Train Loss: 5.1388, Train Steps/Sec: 1.07 +[2025-04-29 13:34:36] (step=0031825) Train Loss: 5.1672, Train Steps/Sec: 0.88 +[2025-04-29 13:35:00] (step=0031850) Train Loss: 5.1664, Train Steps/Sec: 1.07 +[2025-04-29 13:35:23] (step=0031875) Train Loss: 5.1319, Train Steps/Sec: 1.07 +[2025-04-29 13:35:46] (step=0031900) Train Loss: 5.1869, Train Steps/Sec: 1.07 +[2025-04-29 13:36:10] (step=0031925) Train Loss: 5.1477, Train Steps/Sec: 1.07 +[2025-04-29 13:36:33] (step=0031950) Train Loss: 5.1386, Train Steps/Sec: 1.07 +[2025-04-29 13:36:56] (step=0031975) Train Loss: 5.1773, Train Steps/Sec: 1.07 +[2025-04-29 13:37:20] (step=0032000) Train Loss: 5.1650, Train Steps/Sec: 1.07 +[2025-04-29 13:37:20] text_config is None. Initializing the text config with default values (`OPTConfig`). +[2025-04-29 13:43:27] Finish Eval in 32000 steps... +[2025-04-29 13:43:47] Saved checkpoint to checkpoint/X2I_700k_3mask_200k_LLava_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_filtered_objects_3mask_888_context/003-GPT-XL/checkpoints/0032000.pt +[2025-04-29 13:43:49] Removed old checkpoint: checkpoint/X2I_700k_3mask_200k_LLava_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_filtered_objects_3mask_888_context/003-GPT-XL/checkpoints/0030000.pt +[2025-04-29 13:44:13] (step=0032025) Train Loss: 5.1620, Train Steps/Sec: 0.06 +[2025-04-29 13:44:36] (step=0032050) Train Loss: 5.2172, Train Steps/Sec: 1.07 +[2025-04-29 13:44:59] (step=0032075) Train Loss: 5.1401, Train Steps/Sec: 1.07 +[2025-04-29 13:45:22] (step=0032100) Train Loss: 5.2276, Train Steps/Sec: 1.07 +[2025-04-29 13:45:46] (step=0032125) Train Loss: 5.1276, Train Steps/Sec: 1.07 +[2025-04-29 13:46:09] (step=0032150) Train Loss: 5.1775, Train Steps/Sec: 1.07 +[2025-04-29 13:46:32] (step=0032175) Train Loss: 5.1354, Train Steps/Sec: 1.07 +[2025-04-29 13:46:56] (step=0032200) Train Loss: 5.1328, Train Steps/Sec: 1.07 +[2025-04-29 13:47:19] (step=0032225) Train Loss: 5.1753, Train Steps/Sec: 1.07 +[2025-04-29 13:47:43] (step=0032250) Train Loss: 5.1053, Train Steps/Sec: 1.07 +[2025-04-29 13:48:06] (step=0032275) Train Loss: 5.1737, Train Steps/Sec: 1.07 +[2025-04-29 13:48:29] (step=0032300) Train Loss: 5.1911, Train Steps/Sec: 1.07 +[2025-04-29 13:48:53] (step=0032325) Train Loss: 5.1951, Train Steps/Sec: 1.07 +[2025-04-29 13:49:16] (step=0032350) Train Loss: 5.1582, Train Steps/Sec: 1.07 +[2025-04-29 13:49:39] (step=0032375) Train Loss: 5.1355, Train Steps/Sec: 1.07 +[2025-04-29 13:50:03] (step=0032400) Train Loss: 5.1706, Train Steps/Sec: 1.07 +[2025-04-29 13:50:26] (step=0032425) Train Loss: 5.1769, Train Steps/Sec: 1.06 +[2025-04-29 13:51:01] (step=0032450) Train Loss: 5.0984, Train Steps/Sec: 0.72 +[2025-04-29 13:51:24] (step=0032475) Train Loss: 5.1699, Train Steps/Sec: 1.07 +[2025-04-29 13:51:59] (step=0032500) Train Loss: 5.1469, Train Steps/Sec: 0.73 +[2025-04-29 13:52:27] (step=0032525) Train Loss: 5.1873, Train Steps/Sec: 0.88 +[2025-04-29 13:52:50] (step=0032550) Train Loss: 5.1501, Train Steps/Sec: 1.07 +[2025-04-29 13:53:14] (step=0032575) Train Loss: 5.1711, Train Steps/Sec: 1.07 +[2025-04-29 13:53:37] (step=0032600) Train Loss: 5.2123, Train Steps/Sec: 1.07 +[2025-04-29 13:54:00] (step=0032625) Train Loss: 5.1135, Train Steps/Sec: 1.07 +[2025-04-29 13:54:24] (step=0032650) Train Loss: 5.1771, Train Steps/Sec: 1.07 +[2025-04-29 13:54:47] (step=0032675) Train Loss: 5.1339, Train Steps/Sec: 1.07 +[2025-04-29 13:55:10] (step=0032700) Train Loss: 5.1596, Train Steps/Sec: 1.07 +[2025-04-29 13:55:34] (step=0032725) Train Loss: 5.1450, Train Steps/Sec: 1.07 +[2025-04-29 13:55:57] (step=0032750) Train Loss: 5.1510, Train Steps/Sec: 1.07 +[2025-04-29 13:56:20] (step=0032775) Train Loss: 5.1884, Train Steps/Sec: 1.07 +[2025-04-29 13:56:44] (step=0032800) Train Loss: 5.1756, Train Steps/Sec: 1.07 +[2025-04-29 13:57:07] (step=0032825) Train Loss: 5.1281, Train Steps/Sec: 1.07 +[2025-04-29 13:57:30] (step=0032850) Train Loss: 5.1479, Train Steps/Sec: 1.07 +[2025-04-29 13:57:54] (step=0032875) Train Loss: 5.1952, Train Steps/Sec: 1.07 +[2025-04-29 13:58:17] (step=0032900) Train Loss: 5.2000, Train Steps/Sec: 1.07 +[2025-04-29 13:58:40] (step=0032925) Train Loss: 5.1687, Train Steps/Sec: 1.07 +[2025-04-29 13:59:04] (step=0032950) Train Loss: 5.1017, Train Steps/Sec: 1.07 +[2025-04-29 13:59:27] (step=0032975) Train Loss: 5.1696, Train Steps/Sec: 1.07 +[2025-04-29 13:59:50] (step=0033000) Train Loss: 5.1596, Train Steps/Sec: 1.07 +[2025-04-29 14:00:14] (step=0033025) Train Loss: 5.1540, Train Steps/Sec: 1.07 +[2025-04-29 14:00:37] (step=0033050) Train Loss: 5.1872, Train Steps/Sec: 1.07 +[2025-04-29 14:01:00] (step=0033075) Train Loss: 5.1231, Train Steps/Sec: 1.07 +[2025-04-29 14:01:24] (step=0033100) Train Loss: 5.1321, Train Steps/Sec: 1.07 +[2025-04-29 14:01:47] (step=0033125) Train Loss: 5.0904, Train Steps/Sec: 1.07 +[2025-04-29 14:02:10] (step=0033150) Train Loss: 5.0921, Train Steps/Sec: 1.07 +[2025-04-29 14:02:34] (step=0033175) Train Loss: 5.1142, Train Steps/Sec: 1.07 +[2025-04-29 14:02:57] (step=0033200) Train Loss: 5.1168, Train Steps/Sec: 1.07 +[2025-04-29 14:03:21] (step=0033225) Train Loss: 5.1393, Train Steps/Sec: 1.07 +[2025-04-29 14:03:44] (step=0033250) Train Loss: 5.0987, Train Steps/Sec: 1.07 +[2025-04-29 14:04:07] (step=0033275) Train Loss: 5.2124, Train Steps/Sec: 1.07 +[2025-04-29 14:04:31] (step=0033300) Train Loss: 5.2076, Train Steps/Sec: 1.07 +[2025-04-29 14:04:54] (step=0033325) Train Loss: 5.1758, Train Steps/Sec: 1.07 +[2025-04-29 14:05:17] (step=0033350) Train Loss: 5.0946, Train Steps/Sec: 1.07 +[2025-04-29 14:05:41] (step=0033375) Train Loss: 5.1834, Train Steps/Sec: 1.07 +[2025-04-29 14:06:04] (step=0033400) Train Loss: 5.1310, Train Steps/Sec: 1.07 +[2025-04-29 14:06:27] (step=0033425) Train Loss: 5.1400, Train Steps/Sec: 1.07 +[2025-04-29 14:06:51] (step=0033450) Train Loss: 5.1560, Train Steps/Sec: 1.07 +[2025-04-29 14:07:14] (step=0033475) Train Loss: 5.1134, Train Steps/Sec: 1.07 +[2025-04-29 14:07:37] (step=0033500) Train Loss: 5.1390, Train Steps/Sec: 1.07 +[2025-04-29 14:08:01] (step=0033525) Train Loss: 5.1543, Train Steps/Sec: 1.07 +[2025-04-29 14:08:24] (step=0033550) Train Loss: 5.1485, Train Steps/Sec: 1.07 +[2025-04-29 14:08:48] (step=0033575) Train Loss: 5.1435, Train Steps/Sec: 1.07 +[2025-04-29 14:09:11] (step=0033600) Train Loss: 5.1294, Train Steps/Sec: 1.07 +[2025-04-29 14:09:34] (step=0033625) Train Loss: 5.1326, Train Steps/Sec: 1.07 +[2025-04-29 14:09:58] (step=0033650) Train Loss: 5.2178, Train Steps/Sec: 1.07 +[2025-04-29 14:10:21] (step=0033675) Train Loss: 5.1477, Train Steps/Sec: 1.07 +[2025-04-29 14:10:44] (step=0033700) Train Loss: 5.0970, Train Steps/Sec: 1.07 +[2025-04-29 14:11:08] (step=0033725) Train Loss: 5.1492, Train Steps/Sec: 1.07 +[2025-04-29 14:11:31] (step=0033750) Train Loss: 5.1162, Train Steps/Sec: 1.07 +[2025-04-29 14:11:54] (step=0033775) Train Loss: 5.1427, Train Steps/Sec: 1.07 +[2025-04-29 14:12:18] (step=0033800) Train Loss: 5.1856, Train Steps/Sec: 1.07 +[2025-04-29 14:12:46] (step=0033825) Train Loss: 5.1038, Train Steps/Sec: 0.88 +[2025-04-29 14:13:10] (step=0033850) Train Loss: 5.0947, Train Steps/Sec: 1.07 +[2025-04-29 14:13:33] (step=0033875) Train Loss: 5.1601, Train Steps/Sec: 1.07 +[2025-04-29 14:13:56] (step=0033900) Train Loss: 5.0992, Train Steps/Sec: 1.07 +[2025-04-29 14:14:20] (step=0033925) Train Loss: 5.1427, Train Steps/Sec: 1.07 +[2025-04-29 14:14:43] (step=0033950) Train Loss: 5.1545, Train Steps/Sec: 1.07 +[2025-04-29 14:15:06] (step=0033975) Train Loss: 5.1132, Train Steps/Sec: 1.07 +[2025-04-29 14:15:30] (step=0034000) Train Loss: 5.1677, Train Steps/Sec: 1.07 +[2025-04-29 14:15:30] text_config is None. Initializing the text config with default values (`OPTConfig`). diff --git a/CKPTS/X2I_700k_3mask_200k_LLava_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_filtered_objects_3mask_888_context/wandb/debug-internal.log b/CKPTS/X2I_700k_3mask_200k_LLava_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_filtered_objects_3mask_888_context/wandb/debug-internal.log new file mode 100644 index 0000000000000000000000000000000000000000..8660b39079613a1da2fe40dddf165f206ea7a30c --- /dev/null +++ b/CKPTS/X2I_700k_3mask_200k_LLava_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_filtered_objects_3mask_888_context/wandb/debug-internal.log @@ -0,0 +1,9 @@ +{"time":"2025-04-29T03:28:03.789661958Z","level":"INFO","msg":"stream: starting","core version":"0.19.8","symlink path":"checkpoint/X2I_700k_3mask_200k_LLava_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_filtered_objects_3mask_888_context/wandb/run-20250429_032803-lbha1z77/logs/debug-core.log"} +{"time":"2025-04-29T03:28:03.996327866Z","level":"INFO","msg":"created new stream","id":"lbha1z77"} +{"time":"2025-04-29T03:28:03.996373963Z","level":"INFO","msg":"stream: started","id":"lbha1z77"} +{"time":"2025-04-29T03:28:03.996405322Z","level":"INFO","msg":"writer: Do: started","stream_id":"lbha1z77"} +{"time":"2025-04-29T03:28:03.996438033Z","level":"INFO","msg":"handler: started","stream_id":"lbha1z77"} +{"time":"2025-04-29T03:28:03.996459228Z","level":"INFO","msg":"sender: started","stream_id":"lbha1z77"} +{"time":"2025-04-29T03:28:04.307262596Z","level":"INFO","msg":"Starting system monitor"} +{"time":"2025-04-29T10:37:34.901441196Z","level":"INFO","msg":"api: retrying HTTP error","status":502,"url":"https://api.wandb.ai/files/haozhezhao/llamagen_ti2i/lbha1z77/file_stream","body":"\n\n\n502 Server Error\n\n\n

Error: Server Error

\n

The server encountered a temporary error and could not complete your request.

Please try again in 30 seconds.

\n

\n\n"} +{"time":"2025-04-29T12:07:20.125722287Z","level":"INFO","msg":"api: retrying error","error":"Post \"https://api.wandb.ai/graphql\": context deadline exceeded (Client.Timeout exceeded while awaiting headers)"} diff --git a/CKPTS/X2I_700k_3mask_200k_LLava_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_filtered_objects_3mask_888_context/wandb/debug.log b/CKPTS/X2I_700k_3mask_200k_LLava_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_filtered_objects_3mask_888_context/wandb/debug.log new file mode 100644 index 0000000000000000000000000000000000000000..22894f56c2bb47d0ae24afe898c3ff1bb0a255a7 --- /dev/null +++ b/CKPTS/X2I_700k_3mask_200k_LLava_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_filtered_objects_3mask_888_context/wandb/debug.log @@ -0,0 +1,22 @@ +2025-04-29 03:28:03,782 INFO MainThread:3895261 [wandb_setup.py:_flush():67] Current SDK version is 0.19.8 +2025-04-29 03:28:03,782 INFO MainThread:3895261 [wandb_setup.py:_flush():67] Configure stats pid to 3895261 +2025-04-29 03:28:03,782 INFO MainThread:3895261 [wandb_setup.py:_flush():67] Loading settings from /tmp/haozhezhao/.config/wandb/settings +2025-04-29 03:28:03,782 INFO MainThread:3895261 [wandb_setup.py:_flush():67] Loading settings from /tmp/haozhezhao/MLLMG/wandb/settings +2025-04-29 03:28:03,782 INFO MainThread:3895261 [wandb_setup.py:_flush():67] Loading settings from environment variables +2025-04-29 03:28:03,782 INFO MainThread:3895261 [wandb_init.py:setup_run_log_directory():647] Logging user logs to checkpoint/X2I_700k_3mask_200k_LLava_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_filtered_objects_3mask_888_context/wandb/run-20250429_032803-lbha1z77/logs/debug.log +2025-04-29 03:28:03,782 INFO MainThread:3895261 [wandb_init.py:setup_run_log_directory():648] Logging internal logs to checkpoint/X2I_700k_3mask_200k_LLava_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_filtered_objects_3mask_888_context/wandb/run-20250429_032803-lbha1z77/logs/debug-internal.log +2025-04-29 03:28:03,782 INFO MainThread:3895261 [wandb_init.py:init():761] calling init triggers +2025-04-29 03:28:03,782 INFO MainThread:3895261 [wandb_init.py:init():766] wandb.init called with sweep_config: {} +config: {'data_path': '/tmp/haozhezhao/MLLMG/jsonl_data/X2I_Filtered_700k_segmentmask_3mask_200k_data_train.jsonl', 'cloud_save_path': '/tmp/haozhezhao/MLLMG/checkpoint', 'no_local_save': False, 'vq_model': 'VQ-16', 'vq_ckpt': '/tmp/haozhezhao/model/llamagen_t2i/vq_ds16_t2i.pt', 'codebook_size': 16384, 'codebook_embed_dim': 8, 'gpt_model': 'GPT-XL', 'gpt_ckpt': '/tmp/haozhezhao/MLLMG/checkpoint/Afteri2i_100kckpt_LLAVAT5_empty_Fixed_just_segment_pretrain_stage3_dreambench__recap_400k_t2i_400k_flux_200k_midjourney_150k_recovery_150k_grounding_100fluxseg_50samseg/005-GPT-XL/checkpoints/0078000.pt', 'gpt_type': 't2i', 'vocab_size': 16384, 'cls_token_num': 888, 'dropout_p': 0.1, 'token_dropout_p': 0.1, 'drop_path': 0.0, 'no_compile': False, 'results_dir': 'checkpoint/X2I_700k_3mask_200k_LLava_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_filtered_objects_3mask_888_context', 'dataset': 'ti2i', 'image_size': 512, 'downsample_size': 16, 'num_classes': 1000, 'epochs': 2, 'lr': 0.0003, 'weight_decay': 0.05, 'beta1': 0.9, 'beta2': 0.95, 'max_grad_norm': 1.0, 'global_batch_size': 40, 'global_seed': 0, 'num_workers': 4, 'log_every': 25, 'ckpt_every': 2000, 'gradient_accumulation_steps': 8, 'mixed_precision': 'bf16', 'val_data_path': '/tmp/haozhezhao/MLLMG/jsonl_data/multiimage_val_for_llava_X2I_threemask_dreambenplus.jsonl', 'use_vision_tower': True, 'model_name_or_path': '/tmp/haozhezhao/model/blip2-flan-t5-xl', 'image_place_holder': '', 'processor_path': None, 'do_eval': True, 'max_eval_samples': 200, 'train_text_encoder': True, 'no_left_padding': False, 'cfg_scale': 7.5, 'top_k': 16384, 'temperature': 0.9, 'top_p': 1.0, 'eval_steps': 2000, 'project_name': 'llamagen_ti2i', 'load_from_checkpoint': '/tmp/haozhezhao/MLLMG/checkpoint/FIXed_3M_1epoch_step32000.pt', 'warmup': 0.05, 'lr_decay_style': 'cosine', 'lr_decay_ratio': 0.1, 'train_iters': 500000, 'class_dropout_prob': 0.1, 'with_image_only': False, 'image_only_rate': 0.1, 'stage2': False, 'subject_driven': True, 'load_subject_embedding': None, 'reference_data_path': '/tmp/haozhezhao/MLLMG/cc12m_reference_tunnel.jsonl', 'multimodal_encoder': 'llava', 'do_recovery': True, 'no_replace': False, 'resume': False, 'dreambench_eval': True, 'find_unused_parameters': True, 'load_visual_encoder': False, 'continue_stage1': False, 'replace_subject': False, 'train_all': True, 'save_total_limit': 1, 'load_language_projection': '/tmp/haozhezhao/MLLMG/llava-v1.5-flant5_fixed-pretrain/mm_projector.bin', 'mm_vision_tower': 'openai/clip-vit-large-patch14', 'load_fixed_llamagen': True, 'unfreeze_output': False, 'fix': 'gpt-empty-fix', 'rank': 0, 'world_size': 8, 'gpu': 0, 'dist_url': 'env://', 'distributed': True, 'dist_backend': 'nccl', '_wandb': {}} +2025-04-29 03:28:03,782 INFO MainThread:3895261 [wandb_init.py:init():784] starting backend +2025-04-29 03:28:03,782 INFO MainThread:3895261 [wandb_init.py:init():788] sending inform_init request +2025-04-29 03:28:03,787 INFO MainThread:3895261 [backend.py:_multiprocessing_setup():101] multiprocessing start_methods=fork,spawn,forkserver, using: spawn +2025-04-29 03:28:03,787 INFO MainThread:3895261 [wandb_init.py:init():798] backend started and connected +2025-04-29 03:28:03,789 INFO MainThread:3895261 [wandb_init.py:init():891] updated telemetry +2025-04-29 03:28:03,790 INFO MainThread:3895261 [wandb_init.py:init():915] communicating run to backend with 90.0 second timeout +2025-04-29 03:28:04,304 INFO MainThread:3895261 [wandb_init.py:init():990] starting run threads in backend +2025-04-29 03:28:04,392 INFO MainThread:3895261 [wandb_run.py:_console_start():2375] atexit reg +2025-04-29 03:28:04,393 INFO MainThread:3895261 [wandb_run.py:_redirect():2227] redirect: wrap_raw +2025-04-29 03:28:04,393 INFO MainThread:3895261 [wandb_run.py:_redirect():2292] Wrapping output streams. +2025-04-29 03:28:04,393 INFO MainThread:3895261 [wandb_run.py:_redirect():2315] Redirects installed. +2025-04-29 03:28:04,395 INFO MainThread:3895261 [wandb_init.py:init():1032] run started, returning control to user process diff --git a/CKPTS/X2I_700k_3mask_200k_LLava_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_filtered_objects_3mask_888_context/wandb/run-20250429_024613-4f1dmylr/files/config.yaml b/CKPTS/X2I_700k_3mask_200k_LLava_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_filtered_objects_3mask_888_context/wandb/run-20250429_024613-4f1dmylr/files/config.yaml new file mode 100644 index 0000000000000000000000000000000000000000..b76d8d8589b5c44d2bc58d202a1ec963a2886f65 --- /dev/null +++ b/CKPTS/X2I_700k_3mask_200k_LLava_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_filtered_objects_3mask_888_context/wandb/run-20250429_024613-4f1dmylr/files/config.yaml @@ -0,0 +1,198 @@ +_wandb: + value: + cli_version: 0.19.8 + m: [] + python_version: 3.11.11 + t: + "1": + - 1 + - 11 + - 41 + - 49 + - 51 + - 55 + "2": + - 1 + - 11 + - 41 + - 49 + - 51 + - 55 + "3": + - 13 + - 16 + - 23 + - 55 + "4": 3.11.11 + "5": 0.19.8 + "6": 4.49.0 + "8": + - 5 + "12": 0.19.8 + "13": linux-x86_64 +beta1: + value: 0.9 +beta2: + value: 0.95 +cfg_scale: + value: 7.5 +ckpt_every: + value: 2000 +class_dropout_prob: + value: 0.1 +cloud_save_path: + value: /tmp/haozhezhao/MLLMG/checkpoint +cls_token_num: + value: 888 +codebook_embed_dim: + value: 8 +codebook_size: + value: 16384 +continue_stage1: + value: false +data_path: + value: /tmp/haozhezhao/MLLMG/jsonl_data/X2I_Filtered_700k_segmentmask_3mask_200k_data_train.jsonl +dataset: + value: ti2i +dist_backend: + value: nccl +dist_url: + value: env:// +distributed: + value: true +do_eval: + value: true +do_recovery: + value: true +downsample_size: + value: 16 +dreambench_eval: + value: true +drop_path: + value: 0 +dropout_p: + value: 0.1 +epochs: + value: 2 +eval_steps: + value: 2000 +find_unused_parameters: + value: true +fix: + value: gpt-empty-fix +global_batch_size: + value: 48 +global_seed: + value: 0 +gpt_ckpt: + value: /tmp/haozhezhao/MLLMG/checkpoint/Afteri2i_100kckpt_LLAVAT5_empty_Fixed_just_segment_pretrain_stage3_dreambench__recap_400k_t2i_400k_flux_200k_midjourney_150k_recovery_150k_grounding_100fluxseg_50samseg/005-GPT-XL/checkpoints/0078000.pt +gpt_model: + value: GPT-XL +gpt_type: + value: t2i +gpu: + value: 0 +gradient_accumulation_steps: + value: 8 +image_only_rate: + value: 0.1 +image_place_holder: + value: +image_size: + value: 512 +load_fixed_llamagen: + value: true +load_from_checkpoint: + value: /tmp/haozhezhao/MLLMG/checkpoint/FIXed_3M_1epoch_step32000.pt +load_language_projection: + value: /tmp/haozhezhao/MLLMG/llava-v1.5-flant5_fixed-pretrain/mm_projector.bin +load_subject_embedding: + value: null +load_visual_encoder: + value: false +log_every: + value: 25 +lr: + value: 0.0003 +lr_decay_ratio: + value: 0.1 +lr_decay_style: + value: cosine +max_eval_samples: + value: 200 +max_grad_norm: + value: 1 +mixed_precision: + value: bf16 +mm_vision_tower: + value: openai/clip-vit-large-patch14 +model_name_or_path: + value: /tmp/haozhezhao/model/blip2-flan-t5-xl +multimodal_encoder: + value: llava +no_compile: + value: false +no_left_padding: + value: false +no_local_save: + value: false +no_replace: + value: false +num_classes: + value: 1000 +num_workers: + value: 4 +processor_path: + value: null +project_name: + value: llamagen_ti2i +rank: + value: 0 +reference_data_path: + value: /tmp/haozhezhao/MLLMG/cc12m_reference_tunnel.jsonl +replace_subject: + value: false +results_dir: + value: checkpoint/X2I_700k_3mask_200k_LLava_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_filtered_objects_3mask_888_context +resume: + value: false +save_total_limit: + value: 1 +stage2: + value: false +subject_driven: + value: true +temperature: + value: 0.9 +token_dropout_p: + value: 0.1 +top_k: + value: 16384 +top_p: + value: 1 +train_all: + value: true +train_iters: + value: 500000 +train_text_encoder: + value: true +unfreeze_output: + value: false +use_vision_tower: + value: true +val_data_path: + value: /tmp/haozhezhao/MLLMG/jsonl_data/multiimage_val_for_llava_X2I_threemask_dreambenplus.jsonl +vocab_size: + value: 16384 +vq_ckpt: + value: /tmp/haozhezhao/model/llamagen_t2i/vq_ds16_t2i.pt +vq_model: + value: VQ-16 +warmup: + value: 0.05 +weight_decay: + value: 0.05 +with_image_only: + value: false +world_size: + value: 8 diff --git a/CKPTS/X2I_700k_3mask_200k_LLava_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_filtered_objects_3mask_888_context/wandb/run-20250429_024613-4f1dmylr/files/output.log b/CKPTS/X2I_700k_3mask_200k_LLava_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_filtered_objects_3mask_888_context/wandb/run-20250429_024613-4f1dmylr/files/output.log new file mode 100644 index 0000000000000000000000000000000000000000..4daf0945cdb2f3e3623c2ec59f4077e467a9f726 --- /dev/null +++ b/CKPTS/X2I_700k_3mask_200k_LLava_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_filtered_objects_3mask_888_context/wandb/run-20250429_024613-4f1dmylr/files/output.log @@ -0,0 +1,268 @@ +[2025-04-29 02:46:15] Training for 2 epochs... +[2025-04-29 02:46:15] Beginning epoch 0... + 0%| | 0/18929 [00:00 + main(args) + File "/tmp/haozhezhao/MLLMG/autoregressive/train/train_t2i.py", line 675, in main + _, loss = model(cond_idx=c_indices, idx=z_indices[:,:-1],pixel_values=pixel_values,cond_idx_mask= cond_attn_mask,img_mask=image_masks, targets=z_indices, valid=valid, text_input_ids=text_input_ids, text_attention_mask=text_attention_mask) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/tmp/haozhezhao/anaconda3/envs/nlp/lib/python3.11/site-packages/torch/nn/modules/module.py", line 1736, in _wrapped_call_impl + return self._call_impl(*args, **kwargs) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/tmp/haozhezhao/anaconda3/envs/nlp/lib/python3.11/site-packages/torch/nn/modules/module.py", line 1747, in _call_impl + return forward_call(*args, **kwargs) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/tmp/haozhezhao/anaconda3/envs/nlp/lib/python3.11/site-packages/torch/nn/parallel/distributed.py", line 1643, in forward + else self._run_ddp_forward(*inputs, **kwargs) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/tmp/haozhezhao/anaconda3/envs/nlp/lib/python3.11/site-packages/torch/nn/parallel/distributed.py", line 1459, in _run_ddp_forward + return self.module(*inputs, **kwargs) # type: ignore[index] + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/tmp/haozhezhao/anaconda3/envs/nlp/lib/python3.11/site-packages/torch/nn/modules/module.py", line 1736, in _wrapped_call_impl + return self._call_impl(*args, **kwargs) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/tmp/haozhezhao/anaconda3/envs/nlp/lib/python3.11/site-packages/torch/nn/modules/module.py", line 1747, in _call_impl + return forward_call(*args, **kwargs) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/tmp/haozhezhao/anaconda3/envs/nlp/lib/python3.11/site-packages/torch/_dynamo/eval_frame.py", line 465, in _fn + return fn(*args, **kwargs) + ^^^^^^^^^^^^^^^^^^^ + File "/tmp/haozhezhao/anaconda3/envs/nlp/lib/python3.11/site-packages/torch/nn/modules/module.py", line 1736, in _wrapped_call_impl + return self._call_impl(*args, **kwargs) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/tmp/haozhezhao/anaconda3/envs/nlp/lib/python3.11/site-packages/torch/nn/modules/module.py", line 1747, in _call_impl + return forward_call(*args, **kwargs) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/tmp/haozhezhao/MLLMG/autoregressive/models/empty_fix_gpt.py", line 460, in forward + def forward( + File "/tmp/haozhezhao/MLLMG/autoregressive/models/empty_fix_gpt.py", line 536, in torch_dynamo_resume_in_forward_at_477 + h = layer(h, freqs_cis, input_pos, mask, cls_token_num = rope_exclude_token_num ) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/tmp/haozhezhao/anaconda3/envs/nlp/lib/python3.11/site-packages/torch/nn/modules/module.py", line 1736, in _wrapped_call_impl + return self._call_impl(*args, **kwargs) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/tmp/haozhezhao/anaconda3/envs/nlp/lib/python3.11/site-packages/torch/nn/modules/module.py", line 1747, in _call_impl + return forward_call(*args, **kwargs) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/tmp/haozhezhao/MLLMG/autoregressive/models/empty_fix_gpt.py", line 286, in forward + def forward( + File "/tmp/haozhezhao/anaconda3/envs/nlp/lib/python3.11/site-packages/torch/nn/modules/module.py", line 1736, in _wrapped_call_impl + return self._call_impl(*args, **kwargs) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/tmp/haozhezhao/anaconda3/envs/nlp/lib/python3.11/site-packages/torch/nn/modules/module.py", line 1747, in _call_impl + return forward_call(*args, **kwargs) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/tmp/haozhezhao/anaconda3/envs/nlp/lib/python3.11/site-packages/torch/_dynamo/eval_frame.py", line 632, in _fn + return fn(*args, **kwargs) + ^^^^^^^^^^^^^^^^^^^ + File "/tmp/haozhezhao/anaconda3/envs/nlp/lib/python3.11/site-packages/torch/fx/graph_module.py", line 784, in call_wrapped + return self._wrapped_call(self, *args, **kwargs) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/tmp/haozhezhao/anaconda3/envs/nlp/lib/python3.11/site-packages/torch/fx/graph_module.py", line 361, in __call__ + raise e + File "/tmp/haozhezhao/anaconda3/envs/nlp/lib/python3.11/site-packages/torch/fx/graph_module.py", line 348, in __call__ + return super(self.cls, obj).__call__(*args, **kwargs) # type: ignore[misc] + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/tmp/haozhezhao/anaconda3/envs/nlp/lib/python3.11/site-packages/torch/nn/modules/module.py", line 1736, in _wrapped_call_impl + return self._call_impl(*args, **kwargs) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/tmp/haozhezhao/anaconda3/envs/nlp/lib/python3.11/site-packages/torch/nn/modules/module.py", line 1747, in _call_impl + return forward_call(*args, **kwargs) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File ".634", line 16, in forward + submod_1 = self.compiled_submod_1(submod_0, l_self_modules_attention_modules_wqkv_parameters_weight_, l_freqs_cis_, l_mask_, l_self_modules_attention_modules_wo_parameters_weight_, l_x_, l_self_modules_ffn_norm_parameters_weight_); submod_0 = l_self_modules_attention_modules_wqkv_parameters_weight_ = l_freqs_cis_ = l_mask_ = l_self_modules_attention_modules_wo_parameters_weight_ = l_x_ = l_self_modules_ffn_norm_parameters_weight_ = None + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/tmp/haozhezhao/anaconda3/envs/nlp/lib/python3.11/site-packages/torch/nn/modules/module.py", line 1736, in _wrapped_call_impl + return self._call_impl(*args, **kwargs) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/tmp/haozhezhao/anaconda3/envs/nlp/lib/python3.11/site-packages/torch/nn/modules/module.py", line 1747, in _call_impl + return forward_call(*args, **kwargs) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/tmp/haozhezhao/anaconda3/envs/nlp/lib/python3.11/site-packages/torch/_dynamo/backends/distributed.py", line 154, in forward + x = self.submod(*args) + ^^^^^^^^^^^^^^^^^^ + File "/tmp/haozhezhao/anaconda3/envs/nlp/lib/python3.11/site-packages/torch/_dynamo/eval_frame.py", line 632, in _fn + return fn(*args, **kwargs) + ^^^^^^^^^^^^^^^^^^^ + File "/tmp/haozhezhao/anaconda3/envs/nlp/lib/python3.11/site-packages/torch/_functorch/aot_autograd.py", line 1100, in forward + return compiled_fn(full_args) + ^^^^^^^^^^^^^^^^^^^^^^ + File "/tmp/haozhezhao/anaconda3/envs/nlp/lib/python3.11/site-packages/torch/_functorch/_aot_autograd/runtime_wrappers.py", line 308, in runtime_wrapper + all_outs = call_func_at_runtime_with_args( + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/tmp/haozhezhao/anaconda3/envs/nlp/lib/python3.11/site-packages/torch/_functorch/_aot_autograd/utils.py", line 124, in call_func_at_runtime_with_args + out = normalize_as_list(f(args)) + ^^^^^^^ + File "/tmp/haozhezhao/anaconda3/envs/nlp/lib/python3.11/site-packages/torch/_functorch/_aot_autograd/utils.py", line 98, in g + return f(*args) + ^^^^^^^^ + File "/tmp/haozhezhao/anaconda3/envs/nlp/lib/python3.11/site-packages/torch/autograd/function.py", line 575, in apply + return super().apply(*args, **kwargs) # type: ignore[misc] + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/tmp/haozhezhao/anaconda3/envs/nlp/lib/python3.11/site-packages/torch/_functorch/_aot_autograd/runtime_wrappers.py", line 1525, in forward + fw_outs = call_func_at_runtime_with_args( + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/tmp/haozhezhao/anaconda3/envs/nlp/lib/python3.11/site-packages/torch/_functorch/_aot_autograd/utils.py", line 124, in call_func_at_runtime_with_args + out = normalize_as_list(f(args)) + ^^^^^^^ + File "/tmp/haozhezhao/anaconda3/envs/nlp/lib/python3.11/site-packages/torch/_functorch/_aot_autograd/runtime_wrappers.py", line 579, in wrapper + return compiled_fn(runtime_args) + ^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/tmp/haozhezhao/anaconda3/envs/nlp/lib/python3.11/site-packages/torch/_functorch/_aot_autograd/runtime_wrappers.py", line 488, in wrapper + return compiled_fn(runtime_args) + ^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/tmp/haozhezhao/anaconda3/envs/nlp/lib/python3.11/site-packages/torch/_functorch/_aot_autograd/runtime_wrappers.py", line 667, in inner_fn + outs = compiled_fn(args) + ^^^^^^^^^^^^^^^^^ + File "/tmp/haozhezhao/anaconda3/envs/nlp/lib/python3.11/site-packages/torch/_inductor/codecache.py", line 1478, in __call__ + return self.current_callable(inputs) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/tmp/haozhezhao/anaconda3/envs/nlp/lib/python3.11/site-packages/torch/_inductor/utils.py", line 1977, in run + return model(new_inputs) + ^^^^^^^^^^^^^^^^^ + File "/tmp/torchinductor_haozhezhao/7m/c7mhmsm46efbphqxshsuj374s2ssxrin452uekumaxepik7hd5sy.py", line 516, in call + buf7 = empty_strided_cuda((6, 20, 1, 1911, 64), (2446080, 122304, 122304, 64, 1), torch.bfloat16) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ +torch.OutOfMemoryError: CUDA out of memory. Tried to allocate 28.00 MiB. GPU 0 has a total capacity of 79.15 GiB of which 21.69 MiB is free. Process 4073182 has 79.12 GiB memory in use. Of the allocated memory 75.18 GiB is allocated by PyTorch, and 2.30 GiB is reserved by PyTorch but unallocated. If reserved but unallocated memory is large try setting PYTORCH_CUDA_ALLOC_CONF=expandable_segments:True to avoid fragmentation. See documentation for Memory Management (https://pytorch.org/docs/stable/notes/cuda.html#environment-variables) +[rank0]: Traceback (most recent call last): +[rank0]: File "/tmp/haozhezhao/MLLMG/autoregressive/train/train_t2i.py", line 1139, in +[rank0]: main(args) +[rank0]: File "/tmp/haozhezhao/MLLMG/autoregressive/train/train_t2i.py", line 675, in main +[rank0]: _, loss = model(cond_idx=c_indices, idx=z_indices[:,:-1],pixel_values=pixel_values,cond_idx_mask= cond_attn_mask,img_mask=image_masks, targets=z_indices, valid=valid, text_input_ids=text_input_ids, text_attention_mask=text_attention_mask) +[rank0]: ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ +[rank0]: File "/tmp/haozhezhao/anaconda3/envs/nlp/lib/python3.11/site-packages/torch/nn/modules/module.py", line 1736, in _wrapped_call_impl +[rank0]: return self._call_impl(*args, **kwargs) +[rank0]: ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ +[rank0]: File "/tmp/haozhezhao/anaconda3/envs/nlp/lib/python3.11/site-packages/torch/nn/modules/module.py", line 1747, in _call_impl +[rank0]: return forward_call(*args, **kwargs) +[rank0]: ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ +[rank0]: File "/tmp/haozhezhao/anaconda3/envs/nlp/lib/python3.11/site-packages/torch/nn/parallel/distributed.py", line 1643, in forward +[rank0]: else self._run_ddp_forward(*inputs, **kwargs) +[rank0]: ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ +[rank0]: File "/tmp/haozhezhao/anaconda3/envs/nlp/lib/python3.11/site-packages/torch/nn/parallel/distributed.py", line 1459, in _run_ddp_forward +[rank0]: return self.module(*inputs, **kwargs) # type: ignore[index] +[rank0]: ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ +[rank0]: File "/tmp/haozhezhao/anaconda3/envs/nlp/lib/python3.11/site-packages/torch/nn/modules/module.py", line 1736, in _wrapped_call_impl +[rank0]: return self._call_impl(*args, **kwargs) +[rank0]: ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ +[rank0]: File "/tmp/haozhezhao/anaconda3/envs/nlp/lib/python3.11/site-packages/torch/nn/modules/module.py", line 1747, in _call_impl +[rank0]: return forward_call(*args, **kwargs) +[rank0]: ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ +[rank0]: File "/tmp/haozhezhao/anaconda3/envs/nlp/lib/python3.11/site-packages/torch/_dynamo/eval_frame.py", line 465, in _fn +[rank0]: return fn(*args, **kwargs) +[rank0]: ^^^^^^^^^^^^^^^^^^^ +[rank0]: File "/tmp/haozhezhao/anaconda3/envs/nlp/lib/python3.11/site-packages/torch/nn/modules/module.py", line 1736, in _wrapped_call_impl +[rank0]: return self._call_impl(*args, **kwargs) +[rank0]: ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ +[rank0]: File "/tmp/haozhezhao/anaconda3/envs/nlp/lib/python3.11/site-packages/torch/nn/modules/module.py", line 1747, in _call_impl +[rank0]: return forward_call(*args, **kwargs) +[rank0]: ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ +[rank0]: File "/tmp/haozhezhao/MLLMG/autoregressive/models/empty_fix_gpt.py", line 460, in forward +[rank0]: def forward( +[rank0]: File "/tmp/haozhezhao/MLLMG/autoregressive/models/empty_fix_gpt.py", line 536, in torch_dynamo_resume_in_forward_at_477 +[rank0]: h = layer(h, freqs_cis, input_pos, mask, cls_token_num = rope_exclude_token_num ) +[rank0]: ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ +[rank0]: File "/tmp/haozhezhao/anaconda3/envs/nlp/lib/python3.11/site-packages/torch/nn/modules/module.py", line 1736, in _wrapped_call_impl +[rank0]: return self._call_impl(*args, **kwargs) +[rank0]: ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ +[rank0]: File "/tmp/haozhezhao/anaconda3/envs/nlp/lib/python3.11/site-packages/torch/nn/modules/module.py", line 1747, in _call_impl +[rank0]: return forward_call(*args, **kwargs) +[rank0]: ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ +[rank0]: File "/tmp/haozhezhao/MLLMG/autoregressive/models/empty_fix_gpt.py", line 286, in forward +[rank0]: def forward( +[rank0]: File "/tmp/haozhezhao/anaconda3/envs/nlp/lib/python3.11/site-packages/torch/nn/modules/module.py", line 1736, in _wrapped_call_impl +[rank0]: return self._call_impl(*args, **kwargs) +[rank0]: ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ +[rank0]: File "/tmp/haozhezhao/anaconda3/envs/nlp/lib/python3.11/site-packages/torch/nn/modules/module.py", line 1747, in _call_impl +[rank0]: return forward_call(*args, **kwargs) +[rank0]: ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ +[rank0]: File "/tmp/haozhezhao/anaconda3/envs/nlp/lib/python3.11/site-packages/torch/_dynamo/eval_frame.py", line 632, in _fn +[rank0]: return fn(*args, **kwargs) +[rank0]: ^^^^^^^^^^^^^^^^^^^ +[rank0]: File "/tmp/haozhezhao/anaconda3/envs/nlp/lib/python3.11/site-packages/torch/fx/graph_module.py", line 784, in call_wrapped +[rank0]: return self._wrapped_call(self, *args, **kwargs) +[rank0]: ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ +[rank0]: File "/tmp/haozhezhao/anaconda3/envs/nlp/lib/python3.11/site-packages/torch/fx/graph_module.py", line 361, in __call__ +[rank0]: raise e +[rank0]: File "/tmp/haozhezhao/anaconda3/envs/nlp/lib/python3.11/site-packages/torch/fx/graph_module.py", line 348, in __call__ +[rank0]: return super(self.cls, obj).__call__(*args, **kwargs) # type: ignore[misc] +[rank0]: ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ +[rank0]: File "/tmp/haozhezhao/anaconda3/envs/nlp/lib/python3.11/site-packages/torch/nn/modules/module.py", line 1736, in _wrapped_call_impl +[rank0]: return self._call_impl(*args, **kwargs) +[rank0]: ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ +[rank0]: File "/tmp/haozhezhao/anaconda3/envs/nlp/lib/python3.11/site-packages/torch/nn/modules/module.py", line 1747, in _call_impl +[rank0]: return forward_call(*args, **kwargs) +[rank0]: ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ +[rank0]: File ".634", line 16, in forward +[rank0]: File "/tmp/haozhezhao/anaconda3/envs/nlp/lib/python3.11/site-packages/torch/nn/modules/module.py", line 1736, in _wrapped_call_impl +[rank0]: return self._call_impl(*args, **kwargs) +[rank0]: ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ +[rank0]: File "/tmp/haozhezhao/anaconda3/envs/nlp/lib/python3.11/site-packages/torch/nn/modules/module.py", line 1747, in _call_impl +[rank0]: return forward_call(*args, **kwargs) +[rank0]: ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ +[rank0]: File "/tmp/haozhezhao/anaconda3/envs/nlp/lib/python3.11/site-packages/torch/_dynamo/backends/distributed.py", line 154, in forward +[rank0]: x = self.submod(*args) +[rank0]: ^^^^^^^^^^^^^^^^^^ +[rank0]: File "/tmp/haozhezhao/anaconda3/envs/nlp/lib/python3.11/site-packages/torch/_dynamo/eval_frame.py", line 632, in _fn +[rank0]: return fn(*args, **kwargs) +[rank0]: ^^^^^^^^^^^^^^^^^^^ +[rank0]: File "/tmp/haozhezhao/anaconda3/envs/nlp/lib/python3.11/site-packages/torch/_functorch/aot_autograd.py", line 1100, in forward +[rank0]: return compiled_fn(full_args) +[rank0]: ^^^^^^^^^^^^^^^^^^^^^^ +[rank0]: File "/tmp/haozhezhao/anaconda3/envs/nlp/lib/python3.11/site-packages/torch/_functorch/_aot_autograd/runtime_wrappers.py", line 308, in runtime_wrapper +[rank0]: all_outs = call_func_at_runtime_with_args( +[rank0]: ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ +[rank0]: File "/tmp/haozhezhao/anaconda3/envs/nlp/lib/python3.11/site-packages/torch/_functorch/_aot_autograd/utils.py", line 124, in call_func_at_runtime_with_args +[rank0]: out = normalize_as_list(f(args)) +[rank0]: ^^^^^^^ +[rank0]: File "/tmp/haozhezhao/anaconda3/envs/nlp/lib/python3.11/site-packages/torch/_functorch/_aot_autograd/utils.py", line 98, in g +[rank0]: return f(*args) +[rank0]: ^^^^^^^^ +[rank0]: File "/tmp/haozhezhao/anaconda3/envs/nlp/lib/python3.11/site-packages/torch/autograd/function.py", line 575, in apply +[rank0]: return super().apply(*args, **kwargs) # type: ignore[misc] +[rank0]: ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ +[rank0]: File "/tmp/haozhezhao/anaconda3/envs/nlp/lib/python3.11/site-packages/torch/_functorch/_aot_autograd/runtime_wrappers.py", line 1525, in forward +[rank0]: fw_outs = call_func_at_runtime_with_args( +[rank0]: ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ +[rank0]: File "/tmp/haozhezhao/anaconda3/envs/nlp/lib/python3.11/site-packages/torch/_functorch/_aot_autograd/utils.py", line 124, in call_func_at_runtime_with_args +[rank0]: out = normalize_as_list(f(args)) +[rank0]: ^^^^^^^ +[rank0]: File "/tmp/haozhezhao/anaconda3/envs/nlp/lib/python3.11/site-packages/torch/_functorch/_aot_autograd/runtime_wrappers.py", line 579, in wrapper +[rank0]: return compiled_fn(runtime_args) +[rank0]: ^^^^^^^^^^^^^^^^^^^^^^^^^ +[rank0]: File "/tmp/haozhezhao/anaconda3/envs/nlp/lib/python3.11/site-packages/torch/_functorch/_aot_autograd/runtime_wrappers.py", line 488, in wrapper +[rank0]: return compiled_fn(runtime_args) +[rank0]: ^^^^^^^^^^^^^^^^^^^^^^^^^ +[rank0]: File "/tmp/haozhezhao/anaconda3/envs/nlp/lib/python3.11/site-packages/torch/_functorch/_aot_autograd/runtime_wrappers.py", line 667, in inner_fn +[rank0]: outs = compiled_fn(args) +[rank0]: ^^^^^^^^^^^^^^^^^ +[rank0]: File "/tmp/haozhezhao/anaconda3/envs/nlp/lib/python3.11/site-packages/torch/_inductor/codecache.py", line 1478, in __call__ +[rank0]: return self.current_callable(inputs) +[rank0]: ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ +[rank0]: File "/tmp/haozhezhao/anaconda3/envs/nlp/lib/python3.11/site-packages/torch/_inductor/utils.py", line 1977, in run +[rank0]: return model(new_inputs) +[rank0]: ^^^^^^^^^^^^^^^^^ +[rank0]: File "/tmp/torchinductor_haozhezhao/7m/c7mhmsm46efbphqxshsuj374s2ssxrin452uekumaxepik7hd5sy.py", line 516, in call +[rank0]: buf7 = empty_strided_cuda((6, 20, 1, 1911, 64), (2446080, 122304, 122304, 64, 1), torch.bfloat16) +[rank0]: ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ +[rank0]: torch.OutOfMemoryError: CUDA out of memory. Tried to allocate 28.00 MiB. GPU 0 has a total capacity of 79.15 GiB of which 21.69 MiB is free. Process 4073182 has 79.12 GiB memory in use. Of the allocated memory 75.18 GiB is allocated by PyTorch, and 2.30 GiB is reserved by PyTorch but unallocated. If reserved but unallocated memory is large try setting PYTORCH_CUDA_ALLOC_CONF=expandable_segments:True to avoid fragmentation. See documentation for Memory Management (https://pytorch.org/docs/stable/notes/cuda.html#environment-variables) diff --git a/CKPTS/X2I_700k_3mask_200k_LLava_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_filtered_objects_3mask_888_context/wandb/run-20250429_024613-4f1dmylr/files/requirements.txt b/CKPTS/X2I_700k_3mask_200k_LLava_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_filtered_objects_3mask_888_context/wandb/run-20250429_024613-4f1dmylr/files/requirements.txt new file mode 100644 index 0000000000000000000000000000000000000000..06dc78369ffff807b210006a0e79d705ffe2a7d7 --- /dev/null +++ b/CKPTS/X2I_700k_3mask_200k_LLava_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_filtered_objects_3mask_888_context/wandb/run-20250429_024613-4f1dmylr/files/requirements.txt @@ -0,0 +1,131 @@ +typing_extensions==4.12.2 +pyzmq==26.3.0 +nvidia-cufft-cu12==11.0.2.54 +triton==3.1.0 +nvidia-cublas-cu12==12.1.3.1 +psutil==7.0.0 +nvidia-cuda-cupti-cu12==12.1.105 +smmap==5.0.2 +nvidia-cuda-runtime-cu12==12.1.105 +aiohappyeyeballs==2.6.1 +asttokens==3.0.0 +huggingface-hub==0.29.3 +pyarrow==19.0.1 +fonttools==4.56.0 +python-dateutil==2.9.0.post0 +GitPython==3.1.44 +aiohttp==3.11.14 +wandb==0.19.8 +setproctitle==1.3.5 +PyYAML==6.0.2 +pydantic_core==2.27.2 +safetensors==0.5.3 +nvidia-nvjitlink-cu12==12.1.105 +aiosignal==1.3.2 +dill==0.3.8 +nvidia-cuda-nvrtc-cu12==12.1.105 +multiprocess==0.70.16 +pure_eval==0.2.3 +stack_data==0.6.3 +pydantic==2.10.6 +MarkupSafe==2.1.5 +tornado==6.4.2 +executing==2.1.0 +executing==2.2.0 +opencv-python==4.11.0.86 +nvitop==1.4.2 +multidict==6.2.0 +Jinja2==3.1.4 +torch==2.5.1+cu121 +nvidia-curand-cu12==10.3.2.106 +platformdirs==4.3.6 +six==1.17.0 +mpmath==1.3.0 +zipp==3.21.0 +packaging==24.2 +requests==2.32.3 +certifi==2025.1.31 +docker-pycreds==0.4.0 +torchvision==0.20.1+cu121 +pandas==2.2.3 +networkx==3.3 +exceptiongroup==1.2.2 +pickleshare==0.7.5 +tokenizers==0.21.1 +charset-normalizer==3.4.1 +jupyter_core==5.7.2 +wcwidth==0.2.13 +nvidia-nvtx-cu12==12.1.105 +prompt_toolkit==3.0.50 +fsspec==2024.12.0 +pillow==11.1.0 +propcache==0.3.0 +regex==2024.11.6 +ptyprocess==0.7.0 +contourpy==1.3.1 +importlib_metadata==8.6.1 +idna==3.10 +comm==0.2.2 +protobuf==5.29.3 +yarl==1.18.3 +ipython_pygments_lexers==1.1.1 +pip==25.0 +parso==0.8.4 +joblib==1.4.2 +nvidia-nccl-cu12==2.21.5 +hf_transfer==0.1.9 +Pygments==2.19.1 +decorator==5.2.1 +filelock==3.18.0 +nvidia-cusparse-cu12==12.1.0.106 +debugpy==1.8.13 +urllib3==2.3.0 +traitlets==5.14.3 +tzdata==2025.1 +matplotlib-inline==0.1.7 +matplotlib==3.10.1 +kiwisolver==1.4.8 +nest_asyncio==1.6.0 +frozenlist==1.5.0 +nvidia-ml-py==12.570.86 +transformers==4.49.0 +nltk==3.9.1 +ipykernel==6.29.5 +click==8.1.8 +gitdb==4.0.12 +pyparsing==3.2.1 +attrs==25.3.0 +jedi==0.19.2 +ipython==9.0.2 +nvidia-cudnn-cu12==9.1.0.70 +pexpect==4.9.0 +nvidia-cusolver-cu12==11.4.5.107 +numpy==2.2.4 +tqdm==4.67.1 +pytz==2025.1 +wheel==0.45.1 +sentry-sdk==2.23.1 +torchaudio==2.5.1+cu121 +jupyter_client==8.6.3 +cycler==0.12.1 +annotated-types==0.7.0 +sympy==1.13.1 +xxhash==3.5.0 +datasets==3.4.1 +setuptools==75.8.0 +typing_extensions==4.12.2 +wheel==0.43.0 +importlib_metadata==8.0.0 +backports.tarfile==1.2.0 +autocommand==2.2.2 +packaging==24.2 +tomli==2.0.1 +typeguard==4.3.0 +zipp==3.19.2 +jaraco.context==5.3.0 +jaraco.functools==4.0.1 +more-itertools==10.3.0 +platformdirs==4.2.2 +jaraco.text==3.12.1 +jaraco.collections==5.1.0 +inflect==7.3.1 diff --git a/CKPTS/X2I_700k_3mask_200k_LLava_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_filtered_objects_3mask_888_context/wandb/run-20250429_024613-4f1dmylr/files/wandb-metadata.json b/CKPTS/X2I_700k_3mask_200k_LLava_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_filtered_objects_3mask_888_context/wandb/run-20250429_024613-4f1dmylr/files/wandb-metadata.json new file mode 100644 index 0000000000000000000000000000000000000000..93b5412d043f19b6ddc9d675e2a4bd86c946f6e6 --- /dev/null +++ b/CKPTS/X2I_700k_3mask_200k_LLava_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_filtered_objects_3mask_888_context/wandb/run-20250429_024613-4f1dmylr/files/wandb-metadata.json @@ -0,0 +1,149 @@ +{ + "os": "Linux-5.15.0-1064-azure-x86_64-with-glibc2.31", + "python": "CPython 3.11.11", + "startedAt": "2025-04-29T02:46:13.907196Z", + "args": [ + "--vq-ckpt", + "/tmp/haozhezhao/model/llamagen_t2i/vq_ds16_t2i.pt", + "--data-path", + "/tmp/haozhezhao/MLLMG/jsonl_data/X2I_Filtered_700k_segmentmask_3mask_200k_data_train.jsonl", + "--dataset", + "ti2i", + "--image-size", + "512", + "--results-dir", + "checkpoint/X2I_700k_3mask_200k_LLava_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_filtered_objects_3mask_888_context", + "--cloud-save-path", + "/tmp/haozhezhao/MLLMG/checkpoint", + "--lr", + "3e-4", + "--val_data_path", + "/tmp/haozhezhao/MLLMG/jsonl_data/multiimage_val_for_llava_X2I_threemask_dreambenplus.jsonl", + "--use_vision_tower", + "--model_name_or_path", + "/tmp/haozhezhao/model/blip2-flan-t5-xl", + "--image_place_holder", + "", + "--do_eval", + "--eval_steps", + "2000", + "--max_eval_samples", + "200", + "--cfg-scale", + "7.5", + "--top-k", + "16384", + "--load_from_checkpoint", + "/tmp/haozhezhao/MLLMG/checkpoint/FIXed_3M_1epoch_step32000.pt", + "--global-batch-size", + "48", + "--num-workers", + "4", + "--warmup", + "0.05", + "--gradient-accumulation-steps", + "8", + "--train_text_encoder", + "--ckpt-every", + "2000", + "--epochs", + "2", + "--subject_driven", + "--reference_data_path", + "/tmp/haozhezhao/MLLMG/cc12m_reference_tunnel.jsonl", + "--multimodal_encoder", + "llava", + "--do_recovery", + "--find_unused_parameters", + "--cls-token-num", + "888", + "--train_all", + "--load_fixed_llamagen", + "--save_total_limit", + "1", + "--fix", + "gpt-empty-fix", + "--load_language_projection", + "/tmp/haozhezhao/MLLMG/llava-v1.5-flant5_fixed-pretrain/mm_projector.bin", + "--mm_vision_tower", + "openai/clip-vit-large-patch14", + "--gpt-ckpt", + "/tmp/haozhezhao/MLLMG/checkpoint/Afteri2i_100kckpt_LLAVAT5_empty_Fixed_just_segment_pretrain_stage3_dreambench__recap_400k_t2i_400k_flux_200k_midjourney_150k_recovery_150k_grounding_100fluxseg_50samseg/005-GPT-XL/checkpoints/0078000.pt", + "--dreambench_eval" + ], + "program": "/tmp/haozhezhao/MLLMG/autoregressive/train/train_t2i.py", + "codePath": "autoregressive/train/train_t2i.py", + "email": "mimazhe55360@gmail.com", + "root": "checkpoint/X2I_700k_3mask_200k_LLava_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_filtered_objects_3mask_888_context", + "host": "447cc403a8794092814259713c51c1df00001X", + "executable": "/tmp/haozhezhao/anaconda3/envs/nlp/bin/python", + "codePathLocal": "autoregressive/train/train_t2i.py", + "cpu_count": 96, + "cpu_count_logical": 96, + "gpu": "NVIDIA A100-SXM4-80GB", + "gpu_count": 8, + "disk": { + "/": { + "total": "133003395072", + "used": "67574145024" + } + }, + "memory": { + "total": "1902387884032" + }, + "cpu": { + "count": 96, + "countLogical": 96 + }, + "gpu_nvidia": [ + { + "name": "NVIDIA A100-SXM4-80GB", + "memoryTotal": "85899345920", + "cudaCores": 6912, + "architecture": "Ampere" + }, + { + "name": "NVIDIA A100-SXM4-80GB", + "memoryTotal": "85899345920", + "cudaCores": 6912, + "architecture": "Ampere" + }, + { + "name": "NVIDIA A100-SXM4-80GB", + "memoryTotal": "85899345920", + "cudaCores": 6912, + "architecture": "Ampere" + }, + { + "name": "NVIDIA A100-SXM4-80GB", + "memoryTotal": "85899345920", + "cudaCores": 6912, + "architecture": "Ampere" + }, + { + "name": "NVIDIA A100-SXM4-80GB", + "memoryTotal": "85899345920", + "cudaCores": 6912, + "architecture": "Ampere" + }, + { + "name": "NVIDIA A100-SXM4-80GB", + "memoryTotal": "85899345920", + "cudaCores": 6912, + "architecture": "Ampere" + }, + { + "name": "NVIDIA A100-SXM4-80GB", + "memoryTotal": "85899345920", + "cudaCores": 6912, + "architecture": "Ampere" + }, + { + "name": "NVIDIA A100-SXM4-80GB", + "memoryTotal": "85899345920", + "cudaCores": 6912, + "architecture": "Ampere" + } + ], + "cudaVersion": "12.2" +} \ No newline at end of file diff --git a/CKPTS/X2I_700k_3mask_200k_LLava_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_filtered_objects_3mask_888_context/wandb/run-20250429_024613-4f1dmylr/files/wandb-summary.json b/CKPTS/X2I_700k_3mask_200k_LLava_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_filtered_objects_3mask_888_context/wandb/run-20250429_024613-4f1dmylr/files/wandb-summary.json new file mode 100644 index 0000000000000000000000000000000000000000..d784d5c8b0b1d601f4431b841a85f0e647649a6e --- /dev/null +++ b/CKPTS/X2I_700k_3mask_200k_LLava_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_filtered_objects_3mask_888_context/wandb/run-20250429_024613-4f1dmylr/files/wandb-summary.json @@ -0,0 +1 @@ +{"_wandb":{"runtime":258}} \ No newline at end of file diff --git a/CKPTS/X2I_700k_3mask_200k_LLava_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_filtered_objects_3mask_888_context/wandb/run-20250429_024613-4f1dmylr/logs/debug-core.log b/CKPTS/X2I_700k_3mask_200k_LLava_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_filtered_objects_3mask_888_context/wandb/run-20250429_024613-4f1dmylr/logs/debug-core.log new file mode 100644 index 0000000000000000000000000000000000000000..458bb87f328d5ebe14b09a766802b4a3985a796e --- /dev/null +++ b/CKPTS/X2I_700k_3mask_200k_LLava_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_filtered_objects_3mask_888_context/wandb/run-20250429_024613-4f1dmylr/logs/debug-core.log @@ -0,0 +1,13 @@ +{"time":"2025-04-29T02:46:13.38708557Z","level":"INFO","msg":"main: starting server","port-filename":"/tmp/tmpgi0w9ah7/port-3863097.txt","pid":3863097,"log-level":0,"disable-analytics":false,"shutdown-on-parent-exit":false} +{"time":"2025-04-29T02:46:13.388436091Z","level":"INFO","msg":"Will exit if parent process dies.","ppid":3863097} +{"time":"2025-04-29T02:46:13.38842992Z","level":"INFO","msg":"server is running","addr":{"IP":"127.0.0.1","Port":34637,"Zone":""}} +{"time":"2025-04-29T02:46:13.57395921Z","level":"INFO","msg":"connection: ManageConnectionData: new connection created","id":"127.0.0.1:55566"} +{"time":"2025-04-29T02:46:13.908575877Z","level":"INFO","msg":"handleInformInit: received","streamId":"4f1dmylr","id":"127.0.0.1:55566"} +{"time":"2025-04-29T02:46:14.115786519Z","level":"INFO","msg":"handleInformInit: stream started","streamId":"4f1dmylr","id":"127.0.0.1:55566"} +{"time":"2025-04-29T02:50:32.904354143Z","level":"INFO","msg":"handleInformTeardown: server teardown initiated","id":"127.0.0.1:55566"} +{"time":"2025-04-29T02:50:32.904475205Z","level":"INFO","msg":"server is shutting down"} +{"time":"2025-04-29T02:50:32.904466879Z","level":"INFO","msg":"connection: closing","id":"127.0.0.1:55566"} +{"time":"2025-04-29T02:50:32.904667304Z","level":"INFO","msg":"connection: closed successfully","id":"127.0.0.1:55566"} +{"time":"2025-04-29T02:50:34.075545551Z","level":"INFO","msg":"handleInformTeardown: server shutdown complete","id":"127.0.0.1:55566"} +{"time":"2025-04-29T02:50:34.075574365Z","level":"INFO","msg":"connection: ManageConnectionData: connection closed","id":"127.0.0.1:55566"} +{"time":"2025-04-29T02:50:34.075584224Z","level":"INFO","msg":"server is closed"} diff --git a/CKPTS/X2I_700k_3mask_200k_LLava_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_filtered_objects_3mask_888_context/wandb/run-20250429_024613-4f1dmylr/logs/debug-internal.log b/CKPTS/X2I_700k_3mask_200k_LLava_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_filtered_objects_3mask_888_context/wandb/run-20250429_024613-4f1dmylr/logs/debug-internal.log new file mode 100644 index 0000000000000000000000000000000000000000..55a30ec1a4949b253876ac041e801e2b8e692fab --- /dev/null +++ b/CKPTS/X2I_700k_3mask_200k_LLava_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_filtered_objects_3mask_888_context/wandb/run-20250429_024613-4f1dmylr/logs/debug-internal.log @@ -0,0 +1,15 @@ +{"time":"2025-04-29T02:46:13.90876998Z","level":"INFO","msg":"stream: starting","core version":"0.19.8","symlink path":"checkpoint/X2I_700k_3mask_200k_LLava_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_filtered_objects_3mask_888_context/wandb/run-20250429_024613-4f1dmylr/logs/debug-core.log"} +{"time":"2025-04-29T02:46:14.115747285Z","level":"INFO","msg":"created new stream","id":"4f1dmylr"} +{"time":"2025-04-29T02:46:14.115780177Z","level":"INFO","msg":"stream: started","id":"4f1dmylr"} +{"time":"2025-04-29T02:46:14.11583251Z","level":"INFO","msg":"sender: started","stream_id":"4f1dmylr"} +{"time":"2025-04-29T02:46:14.115827491Z","level":"INFO","msg":"writer: Do: started","stream_id":"4f1dmylr"} +{"time":"2025-04-29T02:46:14.115897071Z","level":"INFO","msg":"handler: started","stream_id":"4f1dmylr"} +{"time":"2025-04-29T02:46:14.438974118Z","level":"INFO","msg":"Starting system monitor"} +{"time":"2025-04-29T02:50:32.904483125Z","level":"INFO","msg":"stream: closing","id":"4f1dmylr"} +{"time":"2025-04-29T02:50:32.904545306Z","level":"INFO","msg":"Stopping system monitor"} +{"time":"2025-04-29T02:50:32.905245312Z","level":"INFO","msg":"Stopped system monitor"} +{"time":"2025-04-29T02:50:33.862152382Z","level":"INFO","msg":"fileTransfer: Close: file transfer manager closed"} +{"time":"2025-04-29T02:50:34.075298359Z","level":"INFO","msg":"handler: closed","stream_id":"4f1dmylr"} +{"time":"2025-04-29T02:50:34.075349695Z","level":"INFO","msg":"writer: Close: closed","stream_id":"4f1dmylr"} +{"time":"2025-04-29T02:50:34.075368069Z","level":"INFO","msg":"sender: closed","stream_id":"4f1dmylr"} +{"time":"2025-04-29T02:50:34.075439583Z","level":"INFO","msg":"stream: closed","id":"4f1dmylr"} diff --git a/CKPTS/X2I_700k_3mask_200k_LLava_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_filtered_objects_3mask_888_context/wandb/run-20250429_024613-4f1dmylr/logs/debug.log b/CKPTS/X2I_700k_3mask_200k_LLava_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_filtered_objects_3mask_888_context/wandb/run-20250429_024613-4f1dmylr/logs/debug.log new file mode 100644 index 0000000000000000000000000000000000000000..49fed49bdc52958cb9a744f2e45d54efce471c5f --- /dev/null +++ b/CKPTS/X2I_700k_3mask_200k_LLava_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_filtered_objects_3mask_888_context/wandb/run-20250429_024613-4f1dmylr/logs/debug.log @@ -0,0 +1,23 @@ +2025-04-29 02:46:13,901 INFO MainThread:3863097 [wandb_setup.py:_flush():67] Current SDK version is 0.19.8 +2025-04-29 02:46:13,902 INFO MainThread:3863097 [wandb_setup.py:_flush():67] Configure stats pid to 3863097 +2025-04-29 02:46:13,902 INFO MainThread:3863097 [wandb_setup.py:_flush():67] Loading settings from /tmp/haozhezhao/.config/wandb/settings +2025-04-29 02:46:13,902 INFO MainThread:3863097 [wandb_setup.py:_flush():67] Loading settings from /tmp/haozhezhao/MLLMG/wandb/settings +2025-04-29 02:46:13,902 INFO MainThread:3863097 [wandb_setup.py:_flush():67] Loading settings from environment variables +2025-04-29 02:46:13,902 INFO MainThread:3863097 [wandb_init.py:setup_run_log_directory():647] Logging user logs to checkpoint/X2I_700k_3mask_200k_LLava_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_filtered_objects_3mask_888_context/wandb/run-20250429_024613-4f1dmylr/logs/debug.log +2025-04-29 02:46:13,902 INFO MainThread:3863097 [wandb_init.py:setup_run_log_directory():648] Logging internal logs to checkpoint/X2I_700k_3mask_200k_LLava_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_filtered_objects_3mask_888_context/wandb/run-20250429_024613-4f1dmylr/logs/debug-internal.log +2025-04-29 02:46:13,902 INFO MainThread:3863097 [wandb_init.py:init():761] calling init triggers +2025-04-29 02:46:13,902 INFO MainThread:3863097 [wandb_init.py:init():766] wandb.init called with sweep_config: {} +config: {'data_path': '/tmp/haozhezhao/MLLMG/jsonl_data/X2I_Filtered_700k_segmentmask_3mask_200k_data_train.jsonl', 'cloud_save_path': '/tmp/haozhezhao/MLLMG/checkpoint', 'no_local_save': False, 'vq_model': 'VQ-16', 'vq_ckpt': '/tmp/haozhezhao/model/llamagen_t2i/vq_ds16_t2i.pt', 'codebook_size': 16384, 'codebook_embed_dim': 8, 'gpt_model': 'GPT-XL', 'gpt_ckpt': '/tmp/haozhezhao/MLLMG/checkpoint/Afteri2i_100kckpt_LLAVAT5_empty_Fixed_just_segment_pretrain_stage3_dreambench__recap_400k_t2i_400k_flux_200k_midjourney_150k_recovery_150k_grounding_100fluxseg_50samseg/005-GPT-XL/checkpoints/0078000.pt', 'gpt_type': 't2i', 'vocab_size': 16384, 'cls_token_num': 888, 'dropout_p': 0.1, 'token_dropout_p': 0.1, 'drop_path': 0.0, 'no_compile': False, 'results_dir': 'checkpoint/X2I_700k_3mask_200k_LLava_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_filtered_objects_3mask_888_context', 'dataset': 'ti2i', 'image_size': 512, 'downsample_size': 16, 'num_classes': 1000, 'epochs': 2, 'lr': 0.0003, 'weight_decay': 0.05, 'beta1': 0.9, 'beta2': 0.95, 'max_grad_norm': 1.0, 'global_batch_size': 48, 'global_seed': 0, 'num_workers': 4, 'log_every': 25, 'ckpt_every': 2000, 'gradient_accumulation_steps': 8, 'mixed_precision': 'bf16', 'val_data_path': '/tmp/haozhezhao/MLLMG/jsonl_data/multiimage_val_for_llava_X2I_threemask_dreambenplus.jsonl', 'use_vision_tower': True, 'model_name_or_path': '/tmp/haozhezhao/model/blip2-flan-t5-xl', 'image_place_holder': '', 'processor_path': None, 'do_eval': True, 'max_eval_samples': 200, 'train_text_encoder': True, 'no_left_padding': False, 'cfg_scale': 7.5, 'top_k': 16384, 'temperature': 0.9, 'top_p': 1.0, 'eval_steps': 2000, 'project_name': 'llamagen_ti2i', 'load_from_checkpoint': '/tmp/haozhezhao/MLLMG/checkpoint/FIXed_3M_1epoch_step32000.pt', 'warmup': 0.05, 'lr_decay_style': 'cosine', 'lr_decay_ratio': 0.1, 'train_iters': 500000, 'class_dropout_prob': 0.1, 'with_image_only': False, 'image_only_rate': 0.1, 'stage2': False, 'subject_driven': True, 'load_subject_embedding': None, 'reference_data_path': '/tmp/haozhezhao/MLLMG/cc12m_reference_tunnel.jsonl', 'multimodal_encoder': 'llava', 'do_recovery': True, 'no_replace': False, 'resume': False, 'dreambench_eval': True, 'find_unused_parameters': True, 'load_visual_encoder': False, 'continue_stage1': False, 'replace_subject': False, 'train_all': True, 'save_total_limit': 1, 'load_language_projection': '/tmp/haozhezhao/MLLMG/llava-v1.5-flant5_fixed-pretrain/mm_projector.bin', 'mm_vision_tower': 'openai/clip-vit-large-patch14', 'load_fixed_llamagen': True, 'unfreeze_output': False, 'fix': 'gpt-empty-fix', 'rank': 0, 'world_size': 8, 'gpu': 0, 'dist_url': 'env://', 'distributed': True, 'dist_backend': 'nccl', '_wandb': {}} +2025-04-29 02:46:13,902 INFO MainThread:3863097 [wandb_init.py:init():784] starting backend +2025-04-29 02:46:13,902 INFO MainThread:3863097 [wandb_init.py:init():788] sending inform_init request +2025-04-29 02:46:13,906 INFO MainThread:3863097 [backend.py:_multiprocessing_setup():101] multiprocessing start_methods=fork,spawn,forkserver, using: spawn +2025-04-29 02:46:13,907 INFO MainThread:3863097 [wandb_init.py:init():798] backend started and connected +2025-04-29 02:46:13,910 INFO MainThread:3863097 [wandb_init.py:init():891] updated telemetry +2025-04-29 02:46:13,910 INFO MainThread:3863097 [wandb_init.py:init():915] communicating run to backend with 90.0 second timeout +2025-04-29 02:46:14,436 INFO MainThread:3863097 [wandb_init.py:init():990] starting run threads in backend +2025-04-29 02:46:14,519 INFO MainThread:3863097 [wandb_run.py:_console_start():2375] atexit reg +2025-04-29 02:46:14,519 INFO MainThread:3863097 [wandb_run.py:_redirect():2227] redirect: wrap_raw +2025-04-29 02:46:14,520 INFO MainThread:3863097 [wandb_run.py:_redirect():2292] Wrapping output streams. +2025-04-29 02:46:14,520 INFO MainThread:3863097 [wandb_run.py:_redirect():2315] Redirects installed. +2025-04-29 02:46:14,522 INFO MainThread:3863097 [wandb_init.py:init():1032] run started, returning control to user process +2025-04-29 02:50:32,903 INFO MsgRouterThr:3863097 [mailbox.py:close():129] Closing mailbox, abandoning 1 handles. diff --git a/CKPTS/X2I_700k_3mask_200k_LLava_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_filtered_objects_3mask_888_context/wandb/run-20250429_024613-4f1dmylr/run-4f1dmylr.wandb b/CKPTS/X2I_700k_3mask_200k_LLava_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_filtered_objects_3mask_888_context/wandb/run-20250429_024613-4f1dmylr/run-4f1dmylr.wandb new file mode 100644 index 0000000000000000000000000000000000000000..74e65ace121f1bee2c0ecc418f69e8739601da03 Binary files /dev/null and b/CKPTS/X2I_700k_3mask_200k_LLava_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_filtered_objects_3mask_888_context/wandb/run-20250429_024613-4f1dmylr/run-4f1dmylr.wandb differ diff --git a/CKPTS/X2I_700k_3mask_200k_LLava_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_filtered_objects_3mask_888_context/wandb/run-20250429_030813-6rsqv32i/files/output.log b/CKPTS/X2I_700k_3mask_200k_LLava_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_filtered_objects_3mask_888_context/wandb/run-20250429_030813-6rsqv32i/files/output.log new file mode 100644 index 0000000000000000000000000000000000000000..76f3748f17a32f2285391c02082c14edaccffd6c --- /dev/null +++ b/CKPTS/X2I_700k_3mask_200k_LLava_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_filtered_objects_3mask_888_context/wandb/run-20250429_030813-6rsqv32i/files/output.log @@ -0,0 +1,42 @@ +[2025-04-29 03:08:14] Training for 2 epochs... +[2025-04-29 03:08:14] Beginning epoch 0... + 0%| | 0/22715 [00:00", + "--do_eval", + "--eval_steps", + "2000", + "--max_eval_samples", + "200", + "--cfg-scale", + "7.5", + "--top-k", + "16384", + "--load_from_checkpoint", + "/tmp/haozhezhao/MLLMG/checkpoint/FIXed_3M_1epoch_step32000.pt", + "--global-batch-size", + "40", + "--num-workers", + "4", + "--warmup", + "0.05", + "--gradient-accumulation-steps", + "8", + "--train_text_encoder", + "--ckpt-every", + "2000", + "--epochs", + "2", + "--subject_driven", + "--reference_data_path", + "/tmp/haozhezhao/MLLMG/cc12m_reference_tunnel.jsonl", + "--multimodal_encoder", + "llava", + "--do_recovery", + "--find_unused_parameters", + "--cls-token-num", + "888", + "--train_all", + "--load_fixed_llamagen", + "--save_total_limit", + "1", + "--fix", + "gpt-empty-fix", + "--load_language_projection", + "/tmp/haozhezhao/MLLMG/llava-v1.5-flant5_fixed-pretrain/mm_projector.bin", + "--mm_vision_tower", + "openai/clip-vit-large-patch14", + "--gpt-ckpt", + "/tmp/haozhezhao/MLLMG/checkpoint/Afteri2i_100kckpt_LLAVAT5_empty_Fixed_just_segment_pretrain_stage3_dreambench__recap_400k_t2i_400k_flux_200k_midjourney_150k_recovery_150k_grounding_100fluxseg_50samseg/005-GPT-XL/checkpoints/0078000.pt", + "--dreambench_eval" + ], + "program": "/tmp/haozhezhao/MLLMG/autoregressive/train/train_t2i.py", + "codePath": "autoregressive/train/train_t2i.py", + "email": "mimazhe55360@gmail.com", + "root": "checkpoint/X2I_700k_3mask_200k_LLava_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_filtered_objects_3mask_888_context", + "host": "447cc403a8794092814259713c51c1df00001X", + "executable": "/tmp/haozhezhao/anaconda3/envs/nlp/bin/python", + "codePathLocal": "autoregressive/train/train_t2i.py", + "cpu_count": 96, + "cpu_count_logical": 96, + "gpu": "NVIDIA A100-SXM4-80GB", + "gpu_count": 8, + "disk": { + "/": { + "total": "133003395072", + "used": "67542601728" + } + }, + "memory": { + "total": "1902387884032" + }, + "cpu": { + "count": 96, + "countLogical": 96 + }, + "gpu_nvidia": [ + { + "name": "NVIDIA A100-SXM4-80GB", + "memoryTotal": "85899345920", + "cudaCores": 6912, + "architecture": "Ampere" + }, + { + "name": "NVIDIA A100-SXM4-80GB", + "memoryTotal": "85899345920", + "cudaCores": 6912, + "architecture": "Ampere" + }, + { + "name": "NVIDIA A100-SXM4-80GB", + "memoryTotal": "85899345920", + "cudaCores": 6912, + "architecture": "Ampere" + }, + { + "name": "NVIDIA A100-SXM4-80GB", + "memoryTotal": "85899345920", + "cudaCores": 6912, + "architecture": "Ampere" + }, + { + "name": "NVIDIA A100-SXM4-80GB", + "memoryTotal": "85899345920", + "cudaCores": 6912, + "architecture": "Ampere" + }, + { + "name": "NVIDIA A100-SXM4-80GB", + "memoryTotal": "85899345920", + "cudaCores": 6912, + "architecture": "Ampere" + }, + { + "name": "NVIDIA A100-SXM4-80GB", + "memoryTotal": "85899345920", + "cudaCores": 6912, + "architecture": "Ampere" + }, + { + "name": "NVIDIA A100-SXM4-80GB", + "memoryTotal": "85899345920", + "cudaCores": 6912, + "architecture": "Ampere" + } + ], + "cudaVersion": "12.2" +} \ No newline at end of file diff --git a/CKPTS/X2I_700k_3mask_200k_LLava_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_filtered_objects_3mask_888_context/wandb/run-20250429_030813-6rsqv32i/logs/debug-core.log b/CKPTS/X2I_700k_3mask_200k_LLava_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_filtered_objects_3mask_888_context/wandb/run-20250429_030813-6rsqv32i/logs/debug-core.log new file mode 100644 index 0000000000000000000000000000000000000000..48f2317127fc0cb86d75a2db07541254499272a8 --- /dev/null +++ b/CKPTS/X2I_700k_3mask_200k_LLava_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_filtered_objects_3mask_888_context/wandb/run-20250429_030813-6rsqv32i/logs/debug-core.log @@ -0,0 +1,7 @@ +{"time":"2025-04-29T03:08:12.594596139Z","level":"INFO","msg":"main: starting server","port-filename":"/tmp/tmp_0783ot_/port-3879246.txt","pid":3879246,"log-level":0,"disable-analytics":false,"shutdown-on-parent-exit":false} +{"time":"2025-04-29T03:08:12.596224773Z","level":"INFO","msg":"Will exit if parent process dies.","ppid":3879246} +{"time":"2025-04-29T03:08:12.596164916Z","level":"INFO","msg":"server is running","addr":{"IP":"127.0.0.1","Port":37089,"Zone":""}} +{"time":"2025-04-29T03:08:12.781101034Z","level":"INFO","msg":"connection: ManageConnectionData: new connection created","id":"127.0.0.1:54642"} +{"time":"2025-04-29T03:08:13.249145558Z","level":"INFO","msg":"handleInformInit: received","streamId":"6rsqv32i","id":"127.0.0.1:54642"} +{"time":"2025-04-29T03:08:13.456044024Z","level":"INFO","msg":"handleInformInit: stream started","streamId":"6rsqv32i","id":"127.0.0.1:54642"} +{"time":"2025-04-29T03:20:00.106514123Z","level":"INFO","msg":"Parent process exited, terminating service process."} diff --git a/CKPTS/X2I_700k_3mask_200k_LLava_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_filtered_objects_3mask_888_context/wandb/run-20250429_030813-6rsqv32i/logs/debug-internal.log b/CKPTS/X2I_700k_3mask_200k_LLava_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_filtered_objects_3mask_888_context/wandb/run-20250429_030813-6rsqv32i/logs/debug-internal.log new file mode 100644 index 0000000000000000000000000000000000000000..db5090eac381c5d5750c3c411c6e5bbb5a56432d --- /dev/null +++ b/CKPTS/X2I_700k_3mask_200k_LLava_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_filtered_objects_3mask_888_context/wandb/run-20250429_030813-6rsqv32i/logs/debug-internal.log @@ -0,0 +1,8 @@ +{"time":"2025-04-29T03:08:13.249377592Z","level":"INFO","msg":"stream: starting","core version":"0.19.8","symlink path":"checkpoint/X2I_700k_3mask_200k_LLava_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_filtered_objects_3mask_888_context/wandb/run-20250429_030813-6rsqv32i/logs/debug-core.log"} +{"time":"2025-04-29T03:08:13.456000543Z","level":"INFO","msg":"created new stream","id":"6rsqv32i"} +{"time":"2025-04-29T03:08:13.456038364Z","level":"INFO","msg":"stream: started","id":"6rsqv32i"} +{"time":"2025-04-29T03:08:13.457320774Z","level":"INFO","msg":"handler: started","stream_id":"6rsqv32i"} +{"time":"2025-04-29T03:08:13.457351311Z","level":"INFO","msg":"writer: Do: started","stream_id":"6rsqv32i"} +{"time":"2025-04-29T03:08:13.457513635Z","level":"INFO","msg":"sender: started","stream_id":"6rsqv32i"} +{"time":"2025-04-29T03:08:13.819282333Z","level":"INFO","msg":"Starting system monitor"} +{"time":"2025-04-29T03:18:46.554266679Z","level":"INFO","msg":"api: retrying error","error":"Post \"https://api.wandb.ai/graphql\": net/http: request canceled (Client.Timeout exceeded while awaiting headers)"} diff --git a/CKPTS/X2I_700k_3mask_200k_LLava_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_filtered_objects_3mask_888_context/wandb/run-20250429_030813-6rsqv32i/logs/debug.log b/CKPTS/X2I_700k_3mask_200k_LLava_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_filtered_objects_3mask_888_context/wandb/run-20250429_030813-6rsqv32i/logs/debug.log new file mode 100644 index 0000000000000000000000000000000000000000..e37caeca0b632d72e2acc3e4abcb6177421052c9 --- /dev/null +++ b/CKPTS/X2I_700k_3mask_200k_LLava_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_filtered_objects_3mask_888_context/wandb/run-20250429_030813-6rsqv32i/logs/debug.log @@ -0,0 +1,22 @@ +2025-04-29 03:08:13,242 INFO MainThread:3879246 [wandb_setup.py:_flush():67] Current SDK version is 0.19.8 +2025-04-29 03:08:13,242 INFO MainThread:3879246 [wandb_setup.py:_flush():67] Configure stats pid to 3879246 +2025-04-29 03:08:13,242 INFO MainThread:3879246 [wandb_setup.py:_flush():67] Loading settings from /tmp/haozhezhao/.config/wandb/settings +2025-04-29 03:08:13,242 INFO MainThread:3879246 [wandb_setup.py:_flush():67] Loading settings from /tmp/haozhezhao/MLLMG/wandb/settings +2025-04-29 03:08:13,242 INFO MainThread:3879246 [wandb_setup.py:_flush():67] Loading settings from environment variables +2025-04-29 03:08:13,242 INFO MainThread:3879246 [wandb_init.py:setup_run_log_directory():647] Logging user logs to checkpoint/X2I_700k_3mask_200k_LLava_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_filtered_objects_3mask_888_context/wandb/run-20250429_030813-6rsqv32i/logs/debug.log +2025-04-29 03:08:13,242 INFO MainThread:3879246 [wandb_init.py:setup_run_log_directory():648] Logging internal logs to checkpoint/X2I_700k_3mask_200k_LLava_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_filtered_objects_3mask_888_context/wandb/run-20250429_030813-6rsqv32i/logs/debug-internal.log +2025-04-29 03:08:13,242 INFO MainThread:3879246 [wandb_init.py:init():761] calling init triggers +2025-04-29 03:08:13,242 INFO MainThread:3879246 [wandb_init.py:init():766] wandb.init called with sweep_config: {} +config: {'data_path': '/tmp/haozhezhao/MLLMG/jsonl_data/X2I_Filtered_700k_segmentmask_3mask_200k_data_train.jsonl', 'cloud_save_path': '/tmp/haozhezhao/MLLMG/checkpoint', 'no_local_save': False, 'vq_model': 'VQ-16', 'vq_ckpt': '/tmp/haozhezhao/model/llamagen_t2i/vq_ds16_t2i.pt', 'codebook_size': 16384, 'codebook_embed_dim': 8, 'gpt_model': 'GPT-XL', 'gpt_ckpt': '/tmp/haozhezhao/MLLMG/checkpoint/Afteri2i_100kckpt_LLAVAT5_empty_Fixed_just_segment_pretrain_stage3_dreambench__recap_400k_t2i_400k_flux_200k_midjourney_150k_recovery_150k_grounding_100fluxseg_50samseg/005-GPT-XL/checkpoints/0078000.pt', 'gpt_type': 't2i', 'vocab_size': 16384, 'cls_token_num': 888, 'dropout_p': 0.1, 'token_dropout_p': 0.1, 'drop_path': 0.0, 'no_compile': False, 'results_dir': 'checkpoint/X2I_700k_3mask_200k_LLava_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_filtered_objects_3mask_888_context', 'dataset': 'ti2i', 'image_size': 512, 'downsample_size': 16, 'num_classes': 1000, 'epochs': 2, 'lr': 0.0003, 'weight_decay': 0.05, 'beta1': 0.9, 'beta2': 0.95, 'max_grad_norm': 1.0, 'global_batch_size': 40, 'global_seed': 0, 'num_workers': 4, 'log_every': 25, 'ckpt_every': 2000, 'gradient_accumulation_steps': 8, 'mixed_precision': 'bf16', 'val_data_path': '/tmp/haozhezhao/MLLMG/jsonl_data/multiimage_val_for_llava_X2I_threemask_dreambenplus.jsonl', 'use_vision_tower': True, 'model_name_or_path': '/tmp/haozhezhao/model/blip2-flan-t5-xl', 'image_place_holder': '', 'processor_path': None, 'do_eval': True, 'max_eval_samples': 200, 'train_text_encoder': True, 'no_left_padding': False, 'cfg_scale': 7.5, 'top_k': 16384, 'temperature': 0.9, 'top_p': 1.0, 'eval_steps': 2000, 'project_name': 'llamagen_ti2i', 'load_from_checkpoint': '/tmp/haozhezhao/MLLMG/checkpoint/FIXed_3M_1epoch_step32000.pt', 'warmup': 0.05, 'lr_decay_style': 'cosine', 'lr_decay_ratio': 0.1, 'train_iters': 500000, 'class_dropout_prob': 0.1, 'with_image_only': False, 'image_only_rate': 0.1, 'stage2': False, 'subject_driven': True, 'load_subject_embedding': None, 'reference_data_path': '/tmp/haozhezhao/MLLMG/cc12m_reference_tunnel.jsonl', 'multimodal_encoder': 'llava', 'do_recovery': True, 'no_replace': False, 'resume': False, 'dreambench_eval': True, 'find_unused_parameters': True, 'load_visual_encoder': False, 'continue_stage1': False, 'replace_subject': False, 'train_all': True, 'save_total_limit': 1, 'load_language_projection': '/tmp/haozhezhao/MLLMG/llava-v1.5-flant5_fixed-pretrain/mm_projector.bin', 'mm_vision_tower': 'openai/clip-vit-large-patch14', 'load_fixed_llamagen': True, 'unfreeze_output': False, 'fix': 'gpt-empty-fix', 'rank': 0, 'world_size': 8, 'gpu': 0, 'dist_url': 'env://', 'distributed': True, 'dist_backend': 'nccl', '_wandb': {}} +2025-04-29 03:08:13,242 INFO MainThread:3879246 [wandb_init.py:init():784] starting backend +2025-04-29 03:08:13,242 INFO MainThread:3879246 [wandb_init.py:init():788] sending inform_init request +2025-04-29 03:08:13,247 INFO MainThread:3879246 [backend.py:_multiprocessing_setup():101] multiprocessing start_methods=fork,spawn,forkserver, using: spawn +2025-04-29 03:08:13,247 INFO MainThread:3879246 [wandb_init.py:init():798] backend started and connected +2025-04-29 03:08:13,249 INFO MainThread:3879246 [wandb_init.py:init():891] updated telemetry +2025-04-29 03:08:13,249 INFO MainThread:3879246 [wandb_init.py:init():915] communicating run to backend with 90.0 second timeout +2025-04-29 03:08:13,816 INFO MainThread:3879246 [wandb_init.py:init():990] starting run threads in backend +2025-04-29 03:08:13,900 INFO MainThread:3879246 [wandb_run.py:_console_start():2375] atexit reg +2025-04-29 03:08:13,900 INFO MainThread:3879246 [wandb_run.py:_redirect():2227] redirect: wrap_raw +2025-04-29 03:08:13,900 INFO MainThread:3879246 [wandb_run.py:_redirect():2292] Wrapping output streams. +2025-04-29 03:08:13,900 INFO MainThread:3879246 [wandb_run.py:_redirect():2315] Redirects installed. +2025-04-29 03:08:13,902 INFO MainThread:3879246 [wandb_init.py:init():1032] run started, returning control to user process diff --git a/CKPTS/X2I_700k_3mask_200k_LLava_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_filtered_objects_3mask_888_context/wandb/run-20250429_030813-6rsqv32i/run-6rsqv32i.wandb b/CKPTS/X2I_700k_3mask_200k_LLava_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_filtered_objects_3mask_888_context/wandb/run-20250429_030813-6rsqv32i/run-6rsqv32i.wandb new file mode 100644 index 0000000000000000000000000000000000000000..94ca5c4e79b796acefc19060ab978336a4973101 --- /dev/null +++ b/CKPTS/X2I_700k_3mask_200k_LLava_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_filtered_objects_3mask_888_context/wandb/run-20250429_030813-6rsqv32i/run-6rsqv32i.wandb @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:56f9412b8c2e4c188210fbb2b7822334ca8fe2ef244d39c5f8bc3ab5ec64bd9a +size 262144 diff --git a/CKPTS/X2I_700k_3mask_200k_LLava_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_filtered_objects_3mask_888_context/wandb/run-20250429_032803-lbha1z77/files/output.log b/CKPTS/X2I_700k_3mask_200k_LLava_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_filtered_objects_3mask_888_context/wandb/run-20250429_032803-lbha1z77/files/output.log new file mode 100644 index 0000000000000000000000000000000000000000..4335ddea3fc8db759a697f71f2f7faa4effb6f1e --- /dev/null +++ b/CKPTS/X2I_700k_3mask_200k_LLava_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_filtered_objects_3mask_888_context/wandb/run-20250429_032803-lbha1z77/files/output.log @@ -0,0 +1,1638 @@ +[2025-04-29 03:28:04] Training for 2 epochs... +[2025-04-29 03:28:04] Beginning epoch 0... + 0%| | 0/22715 [00:00 +tokenizer length after expend 32101 +tokenizer length before expend 32100 + /tmp/haozhezhao/anaconda3/envs/nlp/lib/python3.11/contextlib.py:105: FutureWarning: `torch.backends.cuda.sdp_kernel()` is deprecated. In the future, this context manager will be removed. Please see `torch.nn.attention.sdpa_kernel()` for the new context manager, with updated signature. + self.gen = func(*args, **kwds) | 0/5 [00:00 +tokenizer length after expend 32101 +tokenizer length before expend 32100 +100%|████████████████████████████████████████████████████████████████████████████████████████████████| 5/5 [04:57<00:00, 59.59s/it] +[2025-04-29 04:48:07] Finish Eval in 4000 steps...██████████████████████████████████████████| 5/5 [04:57<00:00, 59.37s/it] +processing left-padding... +processing left-padding... +processing left-padding... +processing left-padding... +processing left-padding... +[2025-04-29 04:48:26] Saved checkpoint to checkpoint/X2I_700k_3mask_200k_LLava_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_filtered_objects_3mask_888_context/003-GPT-XL/checkpoints/0004000.pt +[2025-04-29 04:48:28] Removed old checkpoint: checkpoint/X2I_700k_3mask_200k_LLava_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_filtered_objects_3mask_888_context/003-GPT-XL/checkpoints/0002000.pt + 18%|███████████████ | 4024/22715 [1:20:46<5:02:48, 1.03it/s][2025-04-29 04:48:52] (step=0004025) Train Loss: 5.8419, Train Steps/Sec: 0.06 + 18%|███████████████▏ | 4049/22715 [1:21:10<4:51:13, 1.07it/s][2025-04-29 04:49:15] (step=0004050) Train Loss: 5.8479, Train Steps/Sec: 1.07 + 18%|███████████████▏ | 4074/22715 [1:21:33<4:50:34, 1.07it/s][2025-04-29 04:49:39] (step=0004075) Train Loss: 5.8302, Train Steps/Sec: 1.07 + 18%|███████████████▎ | 4099/22715 [1:21:56<4:48:33, 1.08it/s][2025-04-29 04:50:02] (step=0004100) Train Loss: 5.8721, Train Steps/Sec: 1.07 + 18%|███████████████▍ | 4124/22715 [1:22:20<4:47:30, 1.08it/s][2025-04-29 04:50:25] (step=0004125) Train Loss: 5.8621, Train Steps/Sec: 1.07 + 18%|███████████████▌ | 4149/22715 [1:22:43<4:47:17, 1.08it/s][2025-04-29 04:50:49] (step=0004150) Train Loss: 5.8252, Train Steps/Sec: 1.07 + 18%|███████████████▌ | 4174/22715 [1:23:06<4:47:26, 1.08it/s][2025-04-29 04:51:12] (step=0004175) Train Loss: 5.8497, Train Steps/Sec: 1.07 + 18%|███████████████▋ | 4199/22715 [1:23:30<4:45:38, 1.08it/s][2025-04-29 04:51:35] (step=0004200) Train Loss: 5.8545, Train Steps/Sec: 1.07 + 19%|███████████████▊ | 4224/22715 [1:23:53<4:54:28, 1.05it/s][2025-04-29 04:51:59] (step=0004225) Train Loss: 5.8269, Train Steps/Sec: 1.07 + 19%|███████████████▉ | 4249/22715 [1:24:17<4:49:54, 1.06it/s][2025-04-29 04:52:22] (step=0004250) Train Loss: 5.8774, Train Steps/Sec: 1.07 + 19%|███████████████▉ | 4274/22715 [1:24:40<4:47:59, 1.07it/s][2025-04-29 04:52:45] (step=0004275) Train Loss: 5.8499, Train Steps/Sec: 1.07 + 19%|████████████████ | 4299/22715 [1:25:03<4:45:41, 1.07it/s][2025-04-29 04:53:09] (step=0004300) Train Loss: 5.8770, Train Steps/Sec: 1.07 + 19%|████████████████▏ | 4324/22715 [1:25:27<4:45:57, 1.07it/s][2025-04-29 04:53:32] (step=0004325) Train Loss: 5.8492, Train Steps/Sec: 1.07 + 19%|████████████████▎ | 4349/22715 [1:25:50<4:44:26, 1.08it/s][2025-04-29 04:53:55] (step=0004350) Train Loss: 5.8460, Train Steps/Sec: 1.07 + 19%|████████████████▎ | 4374/22715 [1:26:13<4:42:58, 1.08it/s][2025-04-29 04:54:19] (step=0004375) Train Loss: 5.8522, Train Steps/Sec: 1.07 + 19%|████████████████▍ | 4399/22715 [1:26:37<4:41:16, 1.09it/s][2025-04-29 04:54:42] (step=0004400) Train Loss: 5.7965, Train Steps/Sec: 1.07 + 19%|████████████████▌ | 4424/22715 [1:27:00<4:50:20, 1.05it/s][2025-04-29 04:55:06] (step=0004425) Train Loss: 5.8620, Train Steps/Sec: 1.07 + 20%|████████████████▋ | 4449/22715 [1:27:23<4:46:44, 1.06it/s][2025-04-29 04:55:29] (step=0004450) Train Loss: 5.8760, Train Steps/Sec: 1.07 + 20%|████████████████▋ | 4474/22715 [1:27:47<4:44:14, 1.07it/s][2025-04-29 04:55:52] (step=0004475) Train Loss: 5.8182, Train Steps/Sec: 1.07 + 20%|████████████████▊ | 4499/22715 [1:28:10<4:42:40, 1.07it/s][2025-04-29 04:56:16] (step=0004500) Train Loss: 5.8286, Train Steps/Sec: 1.07 + 20%|████████████████▉ | 4524/22715 [1:28:33<4:42:12, 1.07it/s][2025-04-29 04:56:39] (step=0004525) Train Loss: 5.8227, Train Steps/Sec: 1.07 + 20%|█████████████████ | 4549/22715 [1:28:57<4:40:33, 1.08it/s][2025-04-29 04:57:02] (step=0004550) Train Loss: 5.8005, Train Steps/Sec: 1.07 + 20%|█████████████████ | 4574/22715 [1:29:20<4:39:58, 1.08it/s][2025-04-29 04:57:26] (step=0004575) Train Loss: 5.8455, Train Steps/Sec: 1.07 + 20%|█████████████████▏ | 4599/22715 [1:29:43<4:39:07, 1.08it/s][2025-04-29 04:57:49] (step=0004600) Train Loss: 5.8291, Train Steps/Sec: 1.07 + 20%|█████████████████▎ | 4624/22715 [1:30:07<4:46:53, 1.05it/s][2025-04-29 04:58:12] (step=0004625) Train Loss: 5.8473, Train Steps/Sec: 1.07 + 20%|█████████████████▍ | 4649/22715 [1:30:30<4:42:42, 1.07it/s][2025-04-29 04:58:36] (step=0004650) Train Loss: 5.8307, Train Steps/Sec: 1.07 + 21%|█████████████████▍ | 4674/22715 [1:30:53<4:41:10, 1.07it/s][2025-04-29 04:58:59] (step=0004675) Train Loss: 5.8195, Train Steps/Sec: 1.07 + 21%|█████████████████▌ | 4699/22715 [1:31:17<4:39:58, 1.07it/s][2025-04-29 04:59:22] (step=0004700) Train Loss: 5.8784, Train Steps/Sec: 1.07 + 21%|█████████████████▋ | 4724/22715 [1:31:40<4:38:36, 1.08it/s][2025-04-29 04:59:46] (step=0004725) Train Loss: 5.8428, Train Steps/Sec: 1.07 + 21%|█████████████████▊ | 4749/22715 [1:32:04<4:37:26, 1.08it/s][2025-04-29 05:00:09] (step=0004750) Train Loss: 5.8260, Train Steps/Sec: 1.07 + 21%|█████████████████▊ | 4774/22715 [1:32:27<4:37:06, 1.08it/s][2025-04-29 05:00:32] (step=0004775) Train Loss: 5.8336, Train Steps/Sec: 1.07 + 21%|█████████████████▉ | 4799/22715 [1:32:50<4:36:23, 1.08it/s][2025-04-29 05:00:56] (step=0004800) Train Loss: 5.8216, Train Steps/Sec: 1.07 + 21%|██████████████████ | 4824/22715 [1:33:14<4:44:20, 1.05it/s][2025-04-29 05:01:19] (step=0004825) Train Loss: 5.8269, Train Steps/Sec: 1.07 + 21%|██████████████████▏ | 4849/22715 [1:33:37<4:40:45, 1.06it/s][2025-04-29 05:01:43] (step=0004850) Train Loss: 5.7651, Train Steps/Sec: 1.07 + 21%|██████████████████▏ | 4874/22715 [1:34:00<4:38:29, 1.07it/s][2025-04-29 05:02:06] (step=0004875) Train Loss: 5.8340, Train Steps/Sec: 1.07 + 22%|██████████████████▎ | 4899/22715 [1:34:24<4:36:26, 1.07it/s][2025-04-29 05:02:29] (step=0004900) Train Loss: 5.8161, Train Steps/Sec: 1.07 + 22%|██████████████████▍ | 4924/22715 [1:34:47<4:35:37, 1.08it/s][2025-04-29 05:02:53] (step=0004925) Train Loss: 5.7862, Train Steps/Sec: 1.07 + 22%|██████████████████▌ | 4949/22715 [1:35:10<4:34:42, 1.08it/s][2025-04-29 05:03:16] (step=0004950) Train Loss: 5.8722, Train Steps/Sec: 1.07 + 22%|██████████████████▌ | 4974/22715 [1:35:34<4:34:01, 1.08it/s][2025-04-29 05:03:39] (step=0004975) Train Loss: 5.8339, Train Steps/Sec: 1.07 + 22%|██████████████████▋ | 4999/22715 [1:35:57<4:33:18, 1.08it/s][2025-04-29 05:04:03] (step=0005000) Train Loss: 5.8255, Train Steps/Sec: 1.07 + 22%|██████████████████▊ | 5024/22715 [1:36:24<4:40:48, 1.05it/s][2025-04-29 05:04:30] (step=0005025) Train Loss: 5.8025, Train Steps/Sec: 0.93 + 22%|██████████████████▉ | 5049/22715 [1:36:51<8:38:54, 1.76s/it][2025-04-29 05:04:57] (step=0005050) Train Loss: 5.8219, Train Steps/Sec: 0.92 + 22%|██████████████████▉ | 5074/22715 [1:37:27<9:41:19, 1.98s/it][2025-04-29 05:05:33] (step=0005075) Train Loss: 5.8094, Train Steps/Sec: 0.70 + 22%|███████████████████ | 5099/22715 [1:37:54<4:38:26, 1.05it/s][2025-04-29 05:06:00] (step=0005100) Train Loss: 5.8356, Train Steps/Sec: 0.92 + 23%|███████████████████▏ | 5124/22715 [1:38:18<4:33:22, 1.07it/s][2025-04-29 05:06:23] (step=0005125) Train Loss: 5.8141, Train Steps/Sec: 1.07 + 23%|███████████████████▎ | 5149/22715 [1:38:41<4:30:48, 1.08it/s][2025-04-29 05:06:46] (step=0005150) Train Loss: 5.8313, Train Steps/Sec: 1.07 + 23%|███████████████████▎ | 5174/22715 [1:39:04<4:30:38, 1.08it/s][2025-04-29 05:07:10] (step=0005175) Train Loss: 5.7879, Train Steps/Sec: 1.07 + 23%|███████████████████▍ | 5199/22715 [1:39:28<4:32:08, 1.07it/s][2025-04-29 05:07:33] (step=0005200) Train Loss: 5.8382, Train Steps/Sec: 1.07 + 23%|███████████████████▌ | 5224/22715 [1:39:51<4:38:01, 1.05it/s][2025-04-29 05:07:57] (step=0005225) Train Loss: 5.8006, Train Steps/Sec: 1.07 + 23%|███████████████████▋ | 5249/22715 [1:40:14<4:33:33, 1.06it/s][2025-04-29 05:08:20] (step=0005250) Train Loss: 5.8410, Train Steps/Sec: 1.07 + 23%|███████████████████▋ | 5274/22715 [1:40:38<4:34:01, 1.06it/s][2025-04-29 05:08:43] (step=0005275) Train Loss: 5.8144, Train Steps/Sec: 1.07 + 23%|███████████████████▊ | 5299/22715 [1:41:01<4:30:11, 1.07it/s][2025-04-29 05:09:07] (step=0005300) Train Loss: 5.8186, Train Steps/Sec: 1.07 + 23%|███████████████████▉ | 5324/22715 [1:41:24<4:29:46, 1.07it/s][2025-04-29 05:09:30] (step=0005325) Train Loss: 5.7685, Train Steps/Sec: 1.07 + 24%|████████████████████ | 5349/22715 [1:41:48<4:30:02, 1.07it/s][2025-04-29 05:09:53] (step=0005350) Train Loss: 5.8306, Train Steps/Sec: 1.07 + 24%|████████████████████ | 5374/22715 [1:42:11<4:27:52, 1.08it/s][2025-04-29 05:10:17] (step=0005375) Train Loss: 5.8085, Train Steps/Sec: 1.07 + 24%|████████████████████▏ | 5399/22715 [1:42:34<4:25:53, 1.09it/s][2025-04-29 05:10:40] (step=0005400) Train Loss: 5.8187, Train Steps/Sec: 1.07 + 24%|████████████████████▎ | 5424/22715 [1:42:58<4:35:17, 1.05it/s][2025-04-29 05:11:03] (step=0005425) Train Loss: 5.7996, Train Steps/Sec: 1.07 + 24%|████████████████████▍ | 5449/22715 [1:43:21<4:30:34, 1.06it/s][2025-04-29 05:11:27] (step=0005450) Train Loss: 5.8233, Train Steps/Sec: 1.07 + 24%|████████████████████▍ | 5474/22715 [1:43:45<4:29:16, 1.07it/s][2025-04-29 05:11:50] (step=0005475) Train Loss: 5.8172, Train Steps/Sec: 1.07 + 24%|████████████████████▌ | 5499/22715 [1:44:08<4:29:01, 1.07it/s][2025-04-29 05:12:13] (step=0005500) Train Loss: 5.8079, Train Steps/Sec: 1.07 + 24%|████████████████████▋ | 5524/22715 [1:44:31<4:26:46, 1.07it/s][2025-04-29 05:12:37] (step=0005525) Train Loss: 5.7883, Train Steps/Sec: 1.07 + 24%|████████████████████▊ | 5549/22715 [1:44:55<4:25:26, 1.08it/s][2025-04-29 05:13:00] (step=0005550) Train Loss: 5.7907, Train Steps/Sec: 1.07 + 25%|████████████████████▊ | 5574/22715 [1:45:18<4:24:57, 1.08it/s][2025-04-29 05:13:24] (step=0005575) Train Loss: 5.8063, Train Steps/Sec: 1.07 + 25%|████████████████████▉ | 5599/22715 [1:45:41<4:24:18, 1.08it/s][2025-04-29 05:13:47] (step=0005600) Train Loss: 5.8248, Train Steps/Sec: 1.07 + 25%|█████████████████████ | 5624/22715 [1:46:05<4:31:15, 1.05it/s][2025-04-29 05:14:10] (step=0005625) Train Loss: 5.7927, Train Steps/Sec: 1.07 + 25%|█████████████████████▏ | 5649/22715 [1:46:28<4:28:24, 1.06it/s][2025-04-29 05:14:34] (step=0005650) Train Loss: 5.7976, Train Steps/Sec: 1.07 + 25%|█████████████████████▏ | 5674/22715 [1:46:52<4:25:52, 1.07it/s][2025-04-29 05:14:57] (step=0005675) Train Loss: 5.7999, Train Steps/Sec: 1.07 + 25%|█████████████████████▎ | 5699/22715 [1:47:15<4:24:39, 1.07it/s][2025-04-29 05:15:20] (step=0005700) Train Loss: 5.7964, Train Steps/Sec: 1.07 + 25%|█████████████████████▍ | 5724/22715 [1:47:38<4:23:12, 1.08it/s][2025-04-29 05:15:44] (step=0005725) Train Loss: 5.7962, Train Steps/Sec: 1.07 + 25%|█████████████████████▌ | 5749/22715 [1:48:02<4:22:16, 1.08it/s][2025-04-29 05:16:07] (step=0005750) Train Loss: 5.8085, Train Steps/Sec: 1.07 + 25%|█████████████████████▌ | 5774/22715 [1:48:25<4:21:11, 1.08it/s][2025-04-29 05:16:30] (step=0005775) Train Loss: 5.7798, Train Steps/Sec: 1.07 + 26%|█████████████████████▋ | 5799/22715 [1:48:48<4:21:12, 1.08it/s][2025-04-29 05:16:54] (step=0005800) Train Loss: 5.8068, Train Steps/Sec: 1.07 + 26%|█████████████████████▊ | 5824/22715 [1:49:17<7:54:59, 1.69s/it][2025-04-29 05:17:22] (step=0005825) Train Loss: 5.7916, Train Steps/Sec: 0.88 + 26%|█████████████████████▉ | 5849/22715 [1:49:40<4:24:47, 1.06it/s][2025-04-29 05:17:46] (step=0005850) Train Loss: 5.7875, Train Steps/Sec: 1.07 + 26%|█████████████████████▉ | 5874/22715 [1:50:03<4:22:34, 1.07it/s][2025-04-29 05:18:09] (step=0005875) Train Loss: 5.7840, Train Steps/Sec: 1.07 + 26%|██████████████████████ | 5899/22715 [1:50:27<4:20:54, 1.07it/s][2025-04-29 05:18:32] (step=0005900) Train Loss: 5.7750, Train Steps/Sec: 1.07 + 26%|██████████████████████▏ | 5924/22715 [1:50:50<4:19:54, 1.08it/s][2025-04-29 05:18:56] (step=0005925) Train Loss: 5.8125, Train Steps/Sec: 1.07 + 26%|██████████████████████▎ | 5949/22715 [1:51:13<4:19:57, 1.07it/s][2025-04-29 05:19:19] (step=0005950) Train Loss: 5.7677, Train Steps/Sec: 1.07 + 26%|██████████████████████▎ | 5974/22715 [1:51:37<4:18:25, 1.08it/s][2025-04-29 05:19:42] (step=0005975) Train Loss: 5.7858, Train Steps/Sec: 1.07 + 26%|██████████████████████▍ | 5999/22715 [1:52:00<4:17:21, 1.08it/s][2025-04-29 05:20:06] (step=0006000) Train Loss: 5.8140, Train Steps/Sec: 1.07 +You are using a model of type blip-2 to instantiate a model of type instructblip. This is not supported for all configurations of models and can yield errors. +[2025-04-29 05:20:06] text_config is None. Initializing the text config with default values (`OPTConfig`). +model_name_or_path:/tmp/haozhezhao/model/blip2-flan-t5-xl +image_place_holder: +tokenizer length after expend 32101 +tokenizer length before expend 32100 +100%|████████████████████████████████████████████████████████████████████████████████████████████████| 5/5 [05:02<00:00, 60.46s/it] +[2025-04-29 05:26:21] Finish Eval in 6000 steps...██████████████████████████████████████████| 5/5 [05:01<00:00, 60.25s/it] +processing left-padding... +processing left-padding... +processing left-padding... +processing left-padding... +processing left-padding... +[2025-04-29 05:26:42] Saved checkpoint to checkpoint/X2I_700k_3mask_200k_LLava_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_filtered_objects_3mask_888_context/003-GPT-XL/checkpoints/0006000.pt +[2025-04-29 05:26:44] Removed old checkpoint: checkpoint/X2I_700k_3mask_200k_LLava_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_filtered_objects_3mask_888_context/003-GPT-XL/checkpoints/0004000.pt + 27%|██████████████████████▌ | 6024/22715 [1:59:02<4:31:37, 1.02it/s][2025-04-29 05:27:07] (step=0006025) Train Loss: 5.7683, Train Steps/Sec: 0.06 + 27%|██████████████████████▋ | 6049/22715 [1:59:25<4:20:09, 1.07it/s][2025-04-29 05:27:31] (step=0006050) Train Loss: 5.7708, Train Steps/Sec: 1.07 + 27%|██████████████████████▋ | 6074/22715 [1:59:48<4:19:37, 1.07it/s][2025-04-29 05:27:54] (step=0006075) Train Loss: 5.7630, Train Steps/Sec: 1.07 + 27%|██████████████████████▊ | 6099/22715 [2:00:12<4:17:48, 1.07it/s][2025-04-29 05:28:17] (step=0006100) Train Loss: 5.7745, Train Steps/Sec: 1.07 + 27%|██████████████████████▉ | 6124/22715 [2:00:35<4:16:33, 1.08it/s][2025-04-29 05:28:41] (step=0006125) Train Loss: 5.7903, Train Steps/Sec: 1.07 + 27%|███████████████████████ | 6149/22715 [2:00:59<4:19:17, 1.06it/s][2025-04-29 05:29:04] (step=0006150) Train Loss: 5.7961, Train Steps/Sec: 1.07 + 27%|███████████████████████ | 6174/22715 [2:01:22<4:15:02, 1.08it/s][2025-04-29 05:29:27] (step=0006175) Train Loss: 5.7876, Train Steps/Sec: 1.07 + 27%|███████████████████████▏ | 6199/22715 [2:01:45<4:14:56, 1.08it/s][2025-04-29 05:29:51] (step=0006200) Train Loss: 5.8221, Train Steps/Sec: 1.06 + 27%|███████████████████████▎ | 6224/22715 [2:02:09<4:21:59, 1.05it/s][2025-04-29 05:30:14] (step=0006225) Train Loss: 5.7478, Train Steps/Sec: 1.07 + 28%|███████████████████████▍ | 6249/22715 [2:02:32<4:18:05, 1.06it/s][2025-04-29 05:30:38] (step=0006250) Train Loss: 5.7541, Train Steps/Sec: 1.07 + 28%|███████████████████████▍ | 6274/22715 [2:02:55<4:16:02, 1.07it/s][2025-04-29 05:31:01] (step=0006275) Train Loss: 5.7691, Train Steps/Sec: 1.07 + 28%|███████████████████████▌ | 6299/22715 [2:03:19<4:15:23, 1.07it/s][2025-04-29 05:31:24] (step=0006300) Train Loss: 5.7651, Train Steps/Sec: 1.07 + 28%|███████████████████████▋ | 6324/22715 [2:03:42<4:14:25, 1.07it/s][2025-04-29 05:31:48] (step=0006325) Train Loss: 5.7623, Train Steps/Sec: 1.07 + 28%|███████████████████████▊ | 6349/22715 [2:04:06<4:14:13, 1.07it/s][2025-04-29 05:32:11] (step=0006350) Train Loss: 5.7490, Train Steps/Sec: 1.07 + 28%|███████████████████████▊ | 6374/22715 [2:04:29<4:12:05, 1.08it/s][2025-04-29 05:32:34] (step=0006375) Train Loss: 5.7679, Train Steps/Sec: 1.07 + 28%|███████████████████████▉ | 6399/22715 [2:04:52<4:11:50, 1.08it/s][2025-04-29 05:32:58] (step=0006400) Train Loss: 5.7582, Train Steps/Sec: 1.06 + 28%|████████████████████████ | 6424/22715 [2:05:16<4:22:24, 1.03it/s][2025-04-29 05:33:21] (step=0006425) Train Loss: 5.7595, Train Steps/Sec: 1.07 + 28%|████████████████████████▏ | 6449/22715 [2:05:39<4:14:36, 1.06it/s][2025-04-29 05:33:45] (step=0006450) Train Loss: 5.7550, Train Steps/Sec: 1.07 + 29%|████████████████████████▏ | 6474/22715 [2:06:03<4:13:37, 1.07it/s][2025-04-29 05:34:08] (step=0006475) Train Loss: 5.7961, Train Steps/Sec: 1.07 + 29%|████████████████████████▎ | 6499/22715 [2:06:26<4:11:54, 1.07it/s][2025-04-29 05:34:31] (step=0006500) Train Loss: 5.7498, Train Steps/Sec: 1.07 + 29%|████████████████████████▍ | 6524/22715 [2:06:49<4:11:08, 1.07it/s][2025-04-29 05:34:55] (step=0006525) Train Loss: 5.7668, Train Steps/Sec: 1.07 + 29%|████████████████████████▌ | 6549/22715 [2:07:13<4:10:23, 1.08it/s][2025-04-29 05:35:18] (step=0006550) Train Loss: 5.7491, Train Steps/Sec: 1.07 + 29%|████████████████████████▌ | 6574/22715 [2:07:36<4:09:35, 1.08it/s][2025-04-29 05:35:42] (step=0006575) Train Loss: 5.7654, Train Steps/Sec: 1.07 + 29%|████████████████████████▋ | 6599/22715 [2:07:59<4:12:54, 1.06it/s][2025-04-29 05:36:05] (step=0006600) Train Loss: 5.7437, Train Steps/Sec: 1.06 + 29%|████████████████████████▊ | 6624/22715 [2:08:23<4:17:14, 1.04it/s][2025-04-29 05:36:29] (step=0006625) Train Loss: 5.7555, Train Steps/Sec: 1.07 + 29%|████████████████████████▉ | 6649/22715 [2:08:46<4:12:27, 1.06it/s][2025-04-29 05:36:52] (step=0006650) Train Loss: 5.7789, Train Steps/Sec: 1.07 + 29%|████████████████████████▉ | 6674/22715 [2:09:10<4:10:33, 1.07it/s][2025-04-29 05:37:15] (step=0006675) Train Loss: 5.7788, Train Steps/Sec: 1.07 + 29%|█████████████████████████ | 6699/22715 [2:09:33<4:10:41, 1.06it/s][2025-04-29 05:37:39] (step=0006700) Train Loss: 5.7270, Train Steps/Sec: 1.07 + 30%|█████████████████████████▏ | 6724/22715 [2:09:57<4:08:40, 1.07it/s][2025-04-29 05:38:02] (step=0006725) Train Loss: 5.8227, Train Steps/Sec: 1.07 + 30%|█████████████████████████▎ | 6749/22715 [2:10:20<4:07:03, 1.08it/s][2025-04-29 05:38:26] (step=0006750) Train Loss: 5.7623, Train Steps/Sec: 1.07 + 30%|█████████████████████████▎ | 6774/22715 [2:10:43<4:06:09, 1.08it/s][2025-04-29 05:38:49] (step=0006775) Train Loss: 5.7860, Train Steps/Sec: 1.07 + 30%|█████████████████████████▍ | 6799/22715 [2:11:07<4:05:55, 1.08it/s][2025-04-29 05:39:12] (step=0006800) Train Loss: 5.7637, Train Steps/Sec: 1.06 + 30%|█████████████████████████▌ | 6824/22715 [2:11:30<4:12:31, 1.05it/s][2025-04-29 05:39:36] (step=0006825) Train Loss: 5.7720, Train Steps/Sec: 1.07 + 30%|█████████████████████████▋ | 6849/22715 [2:11:57<4:08:49, 1.06it/s][2025-04-29 05:40:03] (step=0006850) Train Loss: 5.7599, Train Steps/Sec: 0.93 + 30%|█████████████████████████▋ | 6874/22715 [2:12:21<4:07:20, 1.07it/s][2025-04-29 05:40:26] (step=0006875) Train Loss: 5.7749, Train Steps/Sec: 1.07 + 30%|█████████████████████████▊ | 6899/22715 [2:12:59<4:08:31, 1.06it/s][2025-04-29 05:41:04] (step=0006900) Train Loss: 5.7879, Train Steps/Sec: 0.65 + 30%|█████████████████████████▉ | 6924/22715 [2:13:31<4:06:54, 1.07it/s][2025-04-29 05:41:36] (step=0006925) Train Loss: 5.7678, Train Steps/Sec: 0.78 + 31%|██████████████████████████ | 6949/22715 [2:13:54<4:04:54, 1.07it/s][2025-04-29 05:42:00] (step=0006950) Train Loss: 5.7947, Train Steps/Sec: 1.07 + 31%|██████████████████████████ | 6974/22715 [2:14:17<4:02:22, 1.08it/s][2025-04-29 05:42:23] (step=0006975) Train Loss: 5.7184, Train Steps/Sec: 1.07 + 31%|██████████████████████████▏ | 6999/22715 [2:14:41<4:02:19, 1.08it/s][2025-04-29 05:42:46] (step=0007000) Train Loss: 5.7606, Train Steps/Sec: 1.07 + 31%|██████████████████████████▎ | 7024/22715 [2:15:04<4:10:13, 1.05it/s][2025-04-29 05:43:10] (step=0007025) Train Loss: 5.7921, Train Steps/Sec: 1.07 + 31%|██████████████████████████▍ | 7049/22715 [2:15:28<4:06:13, 1.06it/s][2025-04-29 05:43:33] (step=0007050) Train Loss: 5.7639, Train Steps/Sec: 1.07 + 31%|██████████████████████████▍ | 7074/22715 [2:15:51<4:04:14, 1.07it/s][2025-04-29 05:43:56] (step=0007075) Train Loss: 5.8013, Train Steps/Sec: 1.07 + 31%|██████████████████████████▌ | 7099/22715 [2:16:14<4:03:14, 1.07it/s][2025-04-29 05:44:20] (step=0007100) Train Loss: 5.7403, Train Steps/Sec: 1.07 + 31%|██████████████████████████▋ | 7124/22715 [2:16:38<4:01:57, 1.07it/s][2025-04-29 05:44:43] (step=0007125) Train Loss: 5.7578, Train Steps/Sec: 1.07 + 31%|██████████████████████████▊ | 7149/22715 [2:17:01<4:01:37, 1.07it/s][2025-04-29 05:45:07] (step=0007150) Train Loss: 5.7762, Train Steps/Sec: 1.07 + 32%|██████████████████████████▊ | 7174/22715 [2:17:24<4:00:06, 1.08it/s][2025-04-29 05:45:30] (step=0007175) Train Loss: 5.8014, Train Steps/Sec: 1.07 + 32%|██████████████████████████▉ | 7199/22715 [2:17:48<3:59:17, 1.08it/s][2025-04-29 05:45:53] (step=0007200) Train Loss: 5.7654, Train Steps/Sec: 1.07 + 32%|███████████████████████████ | 7224/22715 [2:18:11<4:08:15, 1.04it/s][2025-04-29 05:46:17] (step=0007225) Train Loss: 5.7449, Train Steps/Sec: 1.07 + 32%|███████████████████████████▏ | 7249/22715 [2:18:35<4:02:32, 1.06it/s][2025-04-29 05:46:40] (step=0007250) Train Loss: 5.7375, Train Steps/Sec: 1.07 + 32%|███████████████████████████▏ | 7274/22715 [2:18:58<4:00:34, 1.07it/s][2025-04-29 05:47:03] (step=0007275) Train Loss: 5.7704, Train Steps/Sec: 1.07 + 32%|███████████████████████████▎ | 7299/22715 [2:19:21<3:59:39, 1.07it/s][2025-04-29 05:47:27] (step=0007300) Train Loss: 5.7566, Train Steps/Sec: 1.07 + 32%|███████████████████████████▍ | 7324/22715 [2:19:45<3:58:43, 1.07it/s][2025-04-29 05:47:50] (step=0007325) Train Loss: 5.7424, Train Steps/Sec: 1.07 + 32%|███████████████████████████▌ | 7349/22715 [2:20:08<3:57:20, 1.08it/s][2025-04-29 05:48:14] (step=0007350) Train Loss: 5.7205, Train Steps/Sec: 1.07 + 32%|███████████████████████████▌ | 7374/22715 [2:20:31<3:56:58, 1.08it/s][2025-04-29 05:48:37] (step=0007375) Train Loss: 5.7476, Train Steps/Sec: 1.07 + 33%|███████████████████████████▋ | 7399/22715 [2:20:55<3:56:24, 1.08it/s][2025-04-29 05:49:00] (step=0007400) Train Loss: 5.7343, Train Steps/Sec: 1.06 + 33%|███████████████████████████▊ | 7424/22715 [2:21:18<4:03:35, 1.05it/s][2025-04-29 05:49:24] (step=0007425) Train Loss: 5.7910, Train Steps/Sec: 1.07 + 33%|███████████████████████████▊ | 7449/22715 [2:21:42<3:59:13, 1.06it/s][2025-04-29 05:49:47] (step=0007450) Train Loss: 5.7537, Train Steps/Sec: 1.07 + 33%|███████████████████████████▉ | 7474/22715 [2:22:05<3:57:49, 1.07it/s][2025-04-29 05:50:10] (step=0007475) Train Loss: 5.7322, Train Steps/Sec: 1.07 + 33%|████████████████████████████ | 7499/22715 [2:22:28<3:58:49, 1.06it/s][2025-04-29 05:50:34] (step=0007500) Train Loss: 5.7615, Train Steps/Sec: 1.07 + 33%|████████████████████████████▏ | 7524/22715 [2:22:52<3:55:36, 1.07it/s][2025-04-29 05:50:57] (step=0007525) Train Loss: 5.7558, Train Steps/Sec: 1.07 + 33%|████████████████████████████▏ | 7549/22715 [2:23:15<3:54:11, 1.08it/s][2025-04-29 05:51:21] (step=0007550) Train Loss: 5.7631, Train Steps/Sec: 1.07 + 33%|████████████████████████████▎ | 7574/22715 [2:23:38<3:53:30, 1.08it/s][2025-04-29 05:51:44] (step=0007575) Train Loss: 5.7593, Train Steps/Sec: 1.07 + 33%|████████████████████████████▍ | 7599/22715 [2:24:02<3:52:26, 1.08it/s][2025-04-29 05:52:07] (step=0007600) Train Loss: 5.7422, Train Steps/Sec: 1.07 + 34%|████████████████████████████▌ | 7624/22715 [2:24:25<4:00:10, 1.05it/s][2025-04-29 05:52:31] (step=0007625) Train Loss: 5.7696, Train Steps/Sec: 1.07 + 34%|████████████████████████████▌ | 7649/22715 [2:24:49<3:56:00, 1.06it/s][2025-04-29 05:52:54] (step=0007650) Train Loss: 5.7334, Train Steps/Sec: 1.07 + 34%|████████████████████████████▋ | 7674/22715 [2:25:12<3:55:12, 1.07it/s][2025-04-29 05:53:17] (step=0007675) Train Loss: 5.7379, Train Steps/Sec: 1.07 + 34%|████████████████████████████▊ | 7699/22715 [2:25:35<3:54:24, 1.07it/s][2025-04-29 05:53:41] (step=0007700) Train Loss: 5.7010, Train Steps/Sec: 1.07 + 34%|████████████████████████████▉ | 7724/22715 [2:25:59<3:52:20, 1.08it/s][2025-04-29 05:54:04] (step=0007725) Train Loss: 5.7493, Train Steps/Sec: 1.07 + 34%|████████████████████████████▉ | 7749/22715 [2:26:22<3:51:07, 1.08it/s][2025-04-29 05:54:28] (step=0007750) Train Loss: 5.7252, Train Steps/Sec: 1.07 + 34%|█████████████████████████████ | 7774/22715 [2:26:45<3:52:11, 1.07it/s][2025-04-29 05:54:51] (step=0007775) Train Loss: 5.7779, Train Steps/Sec: 1.07 + 34%|█████████████████████████████▏ | 7799/22715 [2:27:09<3:50:13, 1.08it/s][2025-04-29 05:55:14] (step=0007800) Train Loss: 5.7351, Train Steps/Sec: 1.07 + 34%|█████████████████████████████▎ | 7824/22715 [2:27:38<7:30:20, 1.81s/it][2025-04-29 05:55:43] (step=0007825) Train Loss: 5.7014, Train Steps/Sec: 0.86 + 35%|█████████████████████████████▎ | 7849/22715 [2:28:01<3:54:59, 1.05it/s][2025-04-29 05:56:07] (step=0007850) Train Loss: 5.7300, Train Steps/Sec: 1.07 + 35%|█████████████████████████████▍ | 7874/22715 [2:28:25<3:51:30, 1.07it/s][2025-04-29 05:56:30] (step=0007875) Train Loss: 5.7403, Train Steps/Sec: 1.07 + 35%|█████████████████████████████▌ | 7899/22715 [2:28:48<3:50:00, 1.07it/s][2025-04-29 05:56:54] (step=0007900) Train Loss: 5.7498, Train Steps/Sec: 1.07 + 35%|█████████████████████████████▋ | 7924/22715 [2:29:11<3:49:48, 1.07it/s][2025-04-29 05:57:17] (step=0007925) Train Loss: 5.7240, Train Steps/Sec: 1.07 + 35%|█████████████████████████████▋ | 7949/22715 [2:29:35<3:48:28, 1.08it/s][2025-04-29 05:57:40] (step=0007950) Train Loss: 5.7253, Train Steps/Sec: 1.07 + 35%|█████████████████████████████▊ | 7974/22715 [2:29:58<3:47:57, 1.08it/s][2025-04-29 05:58:04] (step=0007975) Train Loss: 5.7284, Train Steps/Sec: 1.07 + 35%|█████████████████████████████▉ | 7999/22715 [2:30:21<3:47:07, 1.08it/s][2025-04-29 05:58:27] (step=0008000) Train Loss: 5.7196, Train Steps/Sec: 1.07 +You are using a model of type blip-2 to instantiate a model of type instructblip. This is not supported for all configurations of models and can yield errors. +[2025-04-29 05:58:27] text_config is None. Initializing the text config with default values (`OPTConfig`). +model_name_or_path:/tmp/haozhezhao/model/blip2-flan-t5-xl +image_place_holder: +tokenizer length after expend 32101 +tokenizer length before expend 32100 +100%|████████████████████████████████████████████████████████████████████████████████████████████████| 5/5 [05:01<00:00, 60.29s/it] +[2025-04-29 06:04:42] Finish Eval in 8000 steps...██████████████████████████████████████████| 5/5 [05:00<00:00, 59.95s/it] +processing left-padding... +processing left-padding... +processing left-padding... +processing left-padding... +processing left-padding... +[2025-04-29 06:05:02] Saved checkpoint to checkpoint/X2I_700k_3mask_200k_LLava_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_filtered_objects_3mask_888_context/003-GPT-XL/checkpoints/0008000.pt +[2025-04-29 06:05:04] Removed old checkpoint: checkpoint/X2I_700k_3mask_200k_LLava_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_filtered_objects_3mask_888_context/003-GPT-XL/checkpoints/0006000.pt + 35%|██████████████████████████████ | 8024/22715 [2:37:22<3:59:01, 1.02it/s][2025-04-29 06:05:28] (step=0008025) Train Loss: 5.7476, Train Steps/Sec: 0.06 + 35%|██████████████████████████████ | 8049/22715 [2:37:45<3:48:58, 1.07it/s][2025-04-29 06:05:51] (step=0008050) Train Loss: 5.7010, Train Steps/Sec: 1.08 + 36%|██████████████████████████████▏ | 8074/22715 [2:38:09<3:46:57, 1.08it/s][2025-04-29 06:06:14] (step=0008075) Train Loss: 5.6957, Train Steps/Sec: 1.07 + 36%|██████████████████████████████▎ | 8099/22715 [2:38:32<3:48:07, 1.07it/s][2025-04-29 06:06:38] (step=0008100) Train Loss: 5.7206, Train Steps/Sec: 1.07 + 36%|██████████████████████████████▍ | 8124/22715 [2:38:55<3:45:36, 1.08it/s][2025-04-29 06:07:01] (step=0008125) Train Loss: 5.6835, Train Steps/Sec: 1.07 + 36%|██████████████████████████████▍ | 8149/22715 [2:39:19<3:45:08, 1.08it/s][2025-04-29 06:07:24] (step=0008150) Train Loss: 5.6986, Train Steps/Sec: 1.07 + 36%|██████████████████████████████▌ | 8174/22715 [2:39:42<3:44:18, 1.08it/s][2025-04-29 06:07:48] (step=0008175) Train Loss: 5.6932, Train Steps/Sec: 1.07 + 36%|██████████████████████████████▋ | 8199/22715 [2:40:06<3:43:52, 1.08it/s][2025-04-29 06:08:11] (step=0008200) Train Loss: 5.7386, Train Steps/Sec: 1.07 + 36%|██████████████████████████████▊ | 8224/22715 [2:40:29<3:50:20, 1.05it/s][2025-04-29 06:08:34] (step=0008225) Train Loss: 5.7430, Train Steps/Sec: 1.07 + 36%|██████████████████████████████▊ | 8249/22715 [2:40:52<3:46:42, 1.06it/s][2025-04-29 06:08:58] (step=0008250) Train Loss: 5.7117, Train Steps/Sec: 1.07 + 36%|██████████████████████████████▉ | 8274/22715 [2:41:16<3:44:33, 1.07it/s][2025-04-29 06:09:21] (step=0008275) Train Loss: 5.7085, Train Steps/Sec: 1.07 + 37%|███████████████████████████████ | 8299/22715 [2:41:39<3:44:39, 1.07it/s][2025-04-29 06:09:44] (step=0008300) Train Loss: 5.6827, Train Steps/Sec: 1.07 + 37%|███████████████████████████████▏ | 8324/22715 [2:42:02<3:42:50, 1.08it/s][2025-04-29 06:10:08] (step=0008325) Train Loss: 5.7108, Train Steps/Sec: 1.07 + 37%|███████████████████████████████▏ | 8349/22715 [2:42:26<3:41:59, 1.08it/s][2025-04-29 06:10:31] (step=0008350) Train Loss: 5.7244, Train Steps/Sec: 1.07 + 37%|███████████████████████████████▎ | 8374/22715 [2:42:49<3:41:28, 1.08it/s][2025-04-29 06:10:55] (step=0008375) Train Loss: 5.7078, Train Steps/Sec: 1.07 + 37%|███████████████████████████████▍ | 8399/22715 [2:43:12<3:40:59, 1.08it/s][2025-04-29 06:11:18] (step=0008400) Train Loss: 5.7299, Train Steps/Sec: 1.07 + 37%|███████████████████████████████▌ | 8424/22715 [2:43:36<3:47:52, 1.05it/s][2025-04-29 06:11:41] (step=0008425) Train Loss: 5.7218, Train Steps/Sec: 1.07 + 37%|███████████████████████████████▌ | 8449/22715 [2:43:59<3:44:00, 1.06it/s][2025-04-29 06:12:05] (step=0008450) Train Loss: 5.7360, Train Steps/Sec: 1.07 + 37%|███████████████████████████████▋ | 8474/22715 [2:44:22<3:42:13, 1.07it/s][2025-04-29 06:12:28] (step=0008475) Train Loss: 5.7127, Train Steps/Sec: 1.07 + 37%|███████████████████████████████▊ | 8499/22715 [2:44:46<3:42:16, 1.07it/s][2025-04-29 06:12:51] (step=0008500) Train Loss: 5.7754, Train Steps/Sec: 1.07 + 38%|███████████████████████████████▉ | 8524/22715 [2:45:09<3:39:50, 1.08it/s][2025-04-29 06:13:15] (step=0008525) Train Loss: 5.7064, Train Steps/Sec: 1.07 + 38%|███████████████████████████████▉ | 8549/22715 [2:45:33<3:39:11, 1.08it/s][2025-04-29 06:13:38] (step=0008550) Train Loss: 5.6630, Train Steps/Sec: 1.07 + 38%|████████████████████████████████ | 8574/22715 [2:45:56<3:38:09, 1.08it/s][2025-04-29 06:14:01] (step=0008575) Train Loss: 5.7362, Train Steps/Sec: 1.07 + 38%|████████████████████████████████▏ | 8599/22715 [2:46:19<3:37:44, 1.08it/s][2025-04-29 06:14:25] (step=0008600) Train Loss: 5.7099, Train Steps/Sec: 1.07 + 38%|████████████████████████████████▎ | 8624/22715 [2:46:43<3:43:35, 1.05it/s][2025-04-29 06:14:48] (step=0008625) Train Loss: 5.6972, Train Steps/Sec: 1.07 + 38%|████████████████████████████████▎ | 8649/22715 [2:47:06<3:40:37, 1.06it/s][2025-04-29 06:15:12] (step=0008650) Train Loss: 5.7154, Train Steps/Sec: 1.07 + 38%|████████████████████████████████▍ | 8674/22715 [2:47:33<3:39:38, 1.07it/s][2025-04-29 06:15:39] (step=0008675) Train Loss: 5.6796, Train Steps/Sec: 0.92 + 38%|████████████████████████████████▌ | 8699/22715 [2:47:57<3:36:59, 1.08it/s][2025-04-29 06:16:02] (step=0008700) Train Loss: 5.6929, Train Steps/Sec: 1.07 + 38%|████████████████████████████████▋ | 8724/22715 [2:48:35<3:46:03, 1.03it/s][2025-04-29 06:16:40] (step=0008725) Train Loss: 5.7029, Train Steps/Sec: 0.65 + 39%|████████████████████████████████▋ | 8749/22715 [2:49:07<3:39:30, 1.06it/s][2025-04-29 06:17:12] (step=0008750) Train Loss: 5.6816, Train Steps/Sec: 0.78 + 39%|████████████████████████████████▊ | 8774/22715 [2:49:30<3:35:15, 1.08it/s][2025-04-29 06:17:36] (step=0008775) Train Loss: 5.6979, Train Steps/Sec: 1.07 + 39%|████████████████████████████████▉ | 8799/22715 [2:49:54<3:35:12, 1.08it/s][2025-04-29 06:17:59] (step=0008800) Train Loss: 5.7061, Train Steps/Sec: 1.06 + 39%|█████████████████████████████████ | 8824/22715 [2:50:17<3:41:47, 1.04it/s][2025-04-29 06:18:23] (step=0008825) Train Loss: 5.6726, Train Steps/Sec: 1.07 + 39%|█████████████████████████████████ | 8849/22715 [2:50:40<3:37:18, 1.06it/s][2025-04-29 06:18:46] (step=0008850) Train Loss: 5.6719, Train Steps/Sec: 1.07 + 39%|█████████████████████████████████▏ | 8874/22715 [2:51:04<3:35:58, 1.07it/s][2025-04-29 06:19:09] (step=0008875) Train Loss: 5.6925, Train Steps/Sec: 1.07 + 39%|█████████████████████████████████▎ | 8899/22715 [2:51:27<3:35:10, 1.07it/s][2025-04-29 06:19:33] (step=0008900) Train Loss: 5.6645, Train Steps/Sec: 1.07 + 39%|█████████████████████████████████▍ | 8924/22715 [2:51:51<3:33:23, 1.08it/s][2025-04-29 06:19:56] (step=0008925) Train Loss: 5.6890, Train Steps/Sec: 1.07 + 39%|█████████████████████████████████▍ | 8949/22715 [2:52:14<3:33:35, 1.07it/s][2025-04-29 06:20:19] (step=0008950) Train Loss: 5.7225, Train Steps/Sec: 1.07 + 40%|█████████████████████████████████▌ | 8974/22715 [2:52:37<3:31:46, 1.08it/s][2025-04-29 06:20:43] (step=0008975) Train Loss: 5.7335, Train Steps/Sec: 1.07 + 40%|█████████████████████████████████▋ | 8999/22715 [2:53:01<3:31:36, 1.08it/s][2025-04-29 06:21:06] (step=0009000) Train Loss: 5.7100, Train Steps/Sec: 1.07 + 40%|█████████████████████████████████▊ | 9024/22715 [2:53:24<3:38:55, 1.04it/s][2025-04-29 06:21:30] (step=0009025) Train Loss: 5.6942, Train Steps/Sec: 1.07 + 40%|█████████████████████████████████▊ | 9049/22715 [2:53:47<3:34:18, 1.06it/s][2025-04-29 06:21:53] (step=0009050) Train Loss: 5.6565, Train Steps/Sec: 1.07 + 40%|█████████████████████████████████▉ | 9074/22715 [2:54:11<3:32:56, 1.07it/s][2025-04-29 06:22:16] (step=0009075) Train Loss: 5.6747, Train Steps/Sec: 1.07 + 40%|██████████████████████████████████ | 9099/22715 [2:54:34<3:31:20, 1.07it/s][2025-04-29 06:22:40] (step=0009100) Train Loss: 5.6518, Train Steps/Sec: 1.07 + 40%|██████████████████████████████████▏ | 9124/22715 [2:54:57<3:30:08, 1.08it/s][2025-04-29 06:23:03] (step=0009125) Train Loss: 5.7103, Train Steps/Sec: 1.07 + 40%|██████████████████████████████████▏ | 9149/22715 [2:55:21<3:30:23, 1.07it/s][2025-04-29 06:23:26] (step=0009150) Train Loss: 5.6759, Train Steps/Sec: 1.07 + 40%|██████████████████████████████████▎ | 9174/22715 [2:55:44<3:29:12, 1.08it/s][2025-04-29 06:23:50] (step=0009175) Train Loss: 5.7104, Train Steps/Sec: 1.07 + 40%|██████████████████████████████████▍ | 9199/22715 [2:56:08<3:28:27, 1.08it/s][2025-04-29 06:24:13] (step=0009200) Train Loss: 5.6650, Train Steps/Sec: 1.06 + 41%|██████████████████████████████████▌ | 9224/22715 [2:56:31<3:35:21, 1.04it/s][2025-04-29 06:24:37] (step=0009225) Train Loss: 5.6941, Train Steps/Sec: 1.07 + 41%|██████████████████████████████████▌ | 9249/22715 [2:56:54<3:31:18, 1.06it/s][2025-04-29 06:25:00] (step=0009250) Train Loss: 5.6783, Train Steps/Sec: 1.07 + 41%|██████████████████████████████████▋ | 9274/22715 [2:57:18<3:30:11, 1.07it/s][2025-04-29 06:25:23] (step=0009275) Train Loss: 5.7203, Train Steps/Sec: 1.07 + 41%|██████████████████████████████████▊ | 9299/22715 [2:57:41<3:28:13, 1.07it/s][2025-04-29 06:25:47] (step=0009300) Train Loss: 5.7089, Train Steps/Sec: 1.07 + 41%|██████████████████████████████████▉ | 9324/22715 [2:58:05<3:27:04, 1.08it/s][2025-04-29 06:26:10] (step=0009325) Train Loss: 5.6509, Train Steps/Sec: 1.07 + 41%|██████████████████████████████████▉ | 9349/22715 [2:58:28<3:28:25, 1.07it/s][2025-04-29 06:26:33] (step=0009350) Train Loss: 5.6673, Train Steps/Sec: 1.07 + 41%|███████████████████████████████████ | 9374/22715 [2:58:51<3:25:19, 1.08it/s][2025-04-29 06:26:57] (step=0009375) Train Loss: 5.6835, Train Steps/Sec: 1.07 + 41%|███████████████████████████████████▏ | 9399/22715 [2:59:15<3:25:27, 1.08it/s][2025-04-29 06:27:20] (step=0009400) Train Loss: 5.7033, Train Steps/Sec: 1.06 + 41%|███████████████████████████████████▎ | 9424/22715 [2:59:38<3:32:08, 1.04it/s][2025-04-29 06:27:44] (step=0009425) Train Loss: 5.6874, Train Steps/Sec: 1.07 + 42%|███████████████████████████████████▎ | 9449/22715 [3:00:01<3:27:08, 1.07it/s][2025-04-29 06:28:07] (step=0009450) Train Loss: 5.6879, Train Steps/Sec: 1.07 + 42%|███████████████████████████████████▍ | 9474/22715 [3:00:25<3:26:18, 1.07it/s][2025-04-29 06:28:30] (step=0009475) Train Loss: 5.6940, Train Steps/Sec: 1.07 + 42%|███████████████████████████████████▌ | 9499/22715 [3:00:48<3:25:13, 1.07it/s][2025-04-29 06:28:54] (step=0009500) Train Loss: 5.6876, Train Steps/Sec: 1.07 + 42%|███████████████████████████████████▋ | 9524/22715 [3:01:12<3:23:42, 1.08it/s][2025-04-29 06:29:17] (step=0009525) Train Loss: 5.6962, Train Steps/Sec: 1.07 + 42%|███████████████████████████████████▋ | 9549/22715 [3:01:35<3:24:34, 1.07it/s][2025-04-29 06:29:40] (step=0009550) Train Loss: 5.6532, Train Steps/Sec: 1.07 + 42%|███████████████████████████████████▊ | 9574/22715 [3:01:58<3:23:37, 1.08it/s][2025-04-29 06:30:04] (step=0009575) Train Loss: 5.6856, Train Steps/Sec: 1.07 + 42%|███████████████████████████████████▉ | 9599/22715 [3:02:22<3:22:15, 1.08it/s][2025-04-29 06:30:27] (step=0009600) Train Loss: 5.6514, Train Steps/Sec: 1.06 + 42%|████████████████████████████████████ | 9624/22715 [3:02:45<3:30:05, 1.04it/s][2025-04-29 06:30:51] (step=0009625) Train Loss: 5.6477, Train Steps/Sec: 1.07 + 42%|████████████████████████████████████ | 9649/22715 [3:03:08<3:24:54, 1.06it/s][2025-04-29 06:31:14] (step=0009650) Train Loss: 5.6648, Train Steps/Sec: 1.07 + 43%|████████████████████████████████████▏ | 9674/22715 [3:03:32<3:23:13, 1.07it/s][2025-04-29 06:31:37] (step=0009675) Train Loss: 5.6626, Train Steps/Sec: 1.07 + 43%|████████████████████████████████████▎ | 9699/22715 [3:03:55<3:22:16, 1.07it/s][2025-04-29 06:32:01] (step=0009700) Train Loss: 5.6780, Train Steps/Sec: 1.07 + 43%|████████████████████████████████████▍ | 9724/22715 [3:04:19<3:21:09, 1.08it/s][2025-04-29 06:32:24] (step=0009725) Train Loss: 5.6756, Train Steps/Sec: 1.07 + 43%|████████████████████████████████████▍ | 9749/22715 [3:04:42<3:20:30, 1.08it/s][2025-04-29 06:32:47] (step=0009750) Train Loss: 5.6596, Train Steps/Sec: 1.07 + 43%|████████████████████████████████████▌ | 9774/22715 [3:05:05<3:19:55, 1.08it/s][2025-04-29 06:33:11] (step=0009775) Train Loss: 5.6764, Train Steps/Sec: 1.07 + 43%|████████████████████████████████████▋ | 9799/22715 [3:05:29<3:18:58, 1.08it/s][2025-04-29 06:33:34] (step=0009800) Train Loss: 5.7234, Train Steps/Sec: 1.07 + 43%|████████████████████████████████████▊ | 9824/22715 [3:05:57<6:09:46, 1.72s/it][2025-04-29 06:34:03] (step=0009825) Train Loss: 5.6852, Train Steps/Sec: 0.88 + 43%|████████████████████████████████████▊ | 9849/22715 [3:06:21<3:21:38, 1.06it/s][2025-04-29 06:34:26] (step=0009850) Train Loss: 5.6784, Train Steps/Sec: 1.07 + 43%|████████████████████████████████████▉ | 9874/22715 [3:06:44<3:20:03, 1.07it/s][2025-04-29 06:34:49] (step=0009875) Train Loss: 5.6849, Train Steps/Sec: 1.07 + 44%|█████████████████████████████████████ | 9899/22715 [3:07:07<3:19:29, 1.07it/s][2025-04-29 06:35:13] (step=0009900) Train Loss: 5.6862, Train Steps/Sec: 1.07 + 44%|█████████████████████████████████████▏ | 9924/22715 [3:07:31<3:18:06, 1.08it/s][2025-04-29 06:35:36] (step=0009925) Train Loss: 5.6747, Train Steps/Sec: 1.07 + 44%|█████████████████████████████████████▏ | 9949/22715 [3:07:54<3:17:13, 1.08it/s][2025-04-29 06:36:00] (step=0009950) Train Loss: 5.6979, Train Steps/Sec: 1.07 + 44%|█████████████████████████████████████▎ | 9974/22715 [3:08:17<3:17:49, 1.07it/s][2025-04-29 06:36:23] (step=0009975) Train Loss: 5.6643, Train Steps/Sec: 1.07 + 44%|█████████████████████████████████████▍ | 9999/22715 [3:08:41<3:15:57, 1.08it/s][2025-04-29 06:36:46] (step=0010000) Train Loss: 5.6813, Train Steps/Sec: 1.07 +You are using a model of type blip-2 to instantiate a model of type instructblip. This is not supported for all configurations of models and can yield errors. +[2025-04-29 06:36:46] text_config is None. Initializing the text config with default values (`OPTConfig`). +model_name_or_path:/tmp/haozhezhao/model/blip2-flan-t5-xl +image_place_holder: +tokenizer length after expend 32101 +tokenizer length before expend 32100 +100%|████████████████████████████████████████████████████████████████████████████████████████████████| 5/5 [05:00<00:00, 60.20s/it] +[2025-04-29 06:43:00] Finish Eval in 10000 steps...█████████████████████████████████████████| 5/5 [05:00<00:00, 59.67s/it] +processing left-padding... +processing left-padding... +processing left-padding... +processing left-padding... +processing left-padding... +[2025-04-29 06:43:20] Saved checkpoint to checkpoint/X2I_700k_3mask_200k_LLava_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_filtered_objects_3mask_888_context/003-GPT-XL/checkpoints/0010000.pt +[2025-04-29 06:43:23] Removed old checkpoint: checkpoint/X2I_700k_3mask_200k_LLava_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_filtered_objects_3mask_888_context/003-GPT-XL/checkpoints/0008000.pt + 44%|█████████████████████████████████████ | 10024/22715 [3:15:40<3:25:54, 1.03it/s][2025-04-29 06:43:46] (step=0010025) Train Loss: 5.6626, Train Steps/Sec: 0.06 + 44%|█████████████████████████████████████▏ | 10049/22715 [3:16:04<3:18:00, 1.07it/s][2025-04-29 06:44:09] (step=0010050) Train Loss: 5.6262, Train Steps/Sec: 1.07 + 44%|█████████████████████████████████████▎ | 10074/22715 [3:16:27<3:16:42, 1.07it/s][2025-04-29 06:44:33] (step=0010075) Train Loss: 5.6851, Train Steps/Sec: 1.07 + 44%|█████████████████████████████████████▎ | 10099/22715 [3:16:50<3:15:38, 1.07it/s][2025-04-29 06:44:56] (step=0010100) Train Loss: 5.6767, Train Steps/Sec: 1.07 + 45%|█████████████████████████████████████▍ | 10124/22715 [3:17:14<3:14:25, 1.08it/s][2025-04-29 06:45:19] (step=0010125) Train Loss: 5.6597, Train Steps/Sec: 1.07 + 45%|█████████████████████████████████████▌ | 10149/22715 [3:17:37<3:13:51, 1.08it/s][2025-04-29 06:45:42] (step=0010150) Train Loss: 5.6600, Train Steps/Sec: 1.07 + 45%|█████████████████████████████████████▌ | 10174/22715 [3:18:00<3:12:56, 1.08it/s][2025-04-29 06:46:06] (step=0010175) Train Loss: 5.6687, Train Steps/Sec: 1.07 + 45%|█████████████████████████████████████▋ | 10199/22715 [3:18:23<3:12:27, 1.08it/s][2025-04-29 06:46:29] (step=0010200) Train Loss: 5.6509, Train Steps/Sec: 1.07 + 45%|█████████████████████████████████████▊ | 10224/22715 [3:18:47<3:17:54, 1.05it/s][2025-04-29 06:46:52] (step=0010225) Train Loss: 5.6283, Train Steps/Sec: 1.07 + 45%|█████████████████████████████████████▉ | 10249/22715 [3:19:10<3:14:56, 1.07it/s][2025-04-29 06:47:16] (step=0010250) Train Loss: 5.6811, Train Steps/Sec: 1.07 + 45%|█████████████████████████████████████▉ | 10274/22715 [3:19:33<3:14:42, 1.06it/s][2025-04-29 06:47:39] (step=0010275) Train Loss: 5.6529, Train Steps/Sec: 1.07 + 45%|██████████████████████████████████████ | 10299/22715 [3:19:57<3:12:34, 1.07it/s][2025-04-29 06:48:02] (step=0010300) Train Loss: 5.6939, Train Steps/Sec: 1.07 + 45%|██████████████████████████████████████▏ | 10324/22715 [3:20:20<3:11:41, 1.08it/s][2025-04-29 06:48:26] (step=0010325) Train Loss: 5.6217, Train Steps/Sec: 1.07 + 46%|██████████████████████████████████████▎ | 10349/22715 [3:20:43<3:11:53, 1.07it/s][2025-04-29 06:48:49] (step=0010350) Train Loss: 5.6551, Train Steps/Sec: 1.07 + 46%|██████████████████████████████████████▎ | 10374/22715 [3:21:07<3:10:27, 1.08it/s][2025-04-29 06:49:12] (step=0010375) Train Loss: 5.6804, Train Steps/Sec: 1.07 + 46%|██████████████████████████████████████▍ | 10399/22715 [3:21:30<3:09:58, 1.08it/s][2025-04-29 06:49:36] (step=0010400) Train Loss: 5.6409, Train Steps/Sec: 1.07 + 46%|██████████████████████████████████████▌ | 10424/22715 [3:21:53<3:14:52, 1.05it/s][2025-04-29 06:49:59] (step=0010425) Train Loss: 5.6080, Train Steps/Sec: 1.07 + 46%|██████████████████████████████████████▋ | 10449/22715 [3:22:17<3:12:29, 1.06it/s][2025-04-29 06:50:22] (step=0010450) Train Loss: 5.6682, Train Steps/Sec: 1.07 + 46%|██████████████████████████████████████▋ | 10474/22715 [3:22:40<3:10:34, 1.07it/s][2025-04-29 06:50:46] (step=0010475) Train Loss: 5.6738, Train Steps/Sec: 1.07 + 46%|██████████████████████████████████████▊ | 10499/22715 [3:23:07<3:12:47, 1.06it/s][2025-04-29 06:51:13] (step=0010500) Train Loss: 5.6090, Train Steps/Sec: 0.93 + 46%|██████████████████████████████████████▉ | 10524/22715 [3:23:30<3:08:43, 1.08it/s][2025-04-29 06:51:36] (step=0010525) Train Loss: 5.6532, Train Steps/Sec: 1.07 + 46%|███████████████████████████████████████ | 10549/22715 [3:24:08<4:12:42, 1.25s/it][2025-04-29 06:52:14] (step=0010550) Train Loss: 5.6482, Train Steps/Sec: 0.66 + 47%|███████████████████████████████████████ | 10574/22715 [3:24:40<3:12:11, 1.05it/s][2025-04-29 06:52:46] (step=0010575) Train Loss: 5.6383, Train Steps/Sec: 0.78 + 47%|███████████████████████████████████████▏ | 10599/22715 [3:25:04<3:07:17, 1.08it/s][2025-04-29 06:53:09] (step=0010600) Train Loss: 5.6556, Train Steps/Sec: 1.07 + 47%|███████████████████████████████████████▎ | 10624/22715 [3:25:27<3:11:46, 1.05it/s][2025-04-29 06:53:33] (step=0010625) Train Loss: 5.6372, Train Steps/Sec: 1.07 + 47%|███████████████████████████████████████▍ | 10649/22715 [3:25:50<3:09:09, 1.06it/s][2025-04-29 06:53:56] (step=0010650) Train Loss: 5.6698, Train Steps/Sec: 1.07 + 47%|███████████████████████████████████████▍ | 10674/22715 [3:26:14<3:07:33, 1.07it/s][2025-04-29 06:54:19] (step=0010675) Train Loss: 5.6394, Train Steps/Sec: 1.07 + 47%|███████████████████████████████████████▌ | 10699/22715 [3:26:37<3:06:37, 1.07it/s][2025-04-29 06:54:43] (step=0010700) Train Loss: 5.6825, Train Steps/Sec: 1.07 + 47%|███████████████████████████████████████▋ | 10724/22715 [3:27:00<3:05:15, 1.08it/s][2025-04-29 06:55:06] (step=0010725) Train Loss: 5.6149, Train Steps/Sec: 1.07 + 47%|███████████████████████████████████████▋ | 10749/22715 [3:27:24<3:04:57, 1.08it/s][2025-04-29 06:55:29] (step=0010750) Train Loss: 5.6448, Train Steps/Sec: 1.07 + 47%|███████████████████████████████████████▊ | 10774/22715 [3:27:47<3:03:53, 1.08it/s][2025-04-29 06:55:53] (step=0010775) Train Loss: 5.6573, Train Steps/Sec: 1.07 + 48%|███████████████████████████████████████▉ | 10799/22715 [3:28:10<3:03:49, 1.08it/s][2025-04-29 06:56:16] (step=0010800) Train Loss: 5.6343, Train Steps/Sec: 1.07 + 48%|████████████████████████████████████████ | 10824/22715 [3:28:34<3:08:45, 1.05it/s][2025-04-29 06:56:39] (step=0010825) Train Loss: 5.6453, Train Steps/Sec: 1.07 + 48%|████████████████████████████████████████ | 10849/22715 [3:28:57<3:05:30, 1.07it/s][2025-04-29 06:57:03] (step=0010850) Train Loss: 5.6445, Train Steps/Sec: 1.07 + 48%|████████████████████████████████████████▏ | 10874/22715 [3:29:20<3:04:29, 1.07it/s][2025-04-29 06:57:26] (step=0010875) Train Loss: 5.6419, Train Steps/Sec: 1.07 + 48%|████████████████████████████████████████▎ | 10899/22715 [3:29:44<3:03:18, 1.07it/s][2025-04-29 06:57:49] (step=0010900) Train Loss: 5.6506, Train Steps/Sec: 1.07 + 48%|████████████████████████████████████████▍ | 10924/22715 [3:30:07<3:02:21, 1.08it/s][2025-04-29 06:58:13] (step=0010925) Train Loss: 5.6356, Train Steps/Sec: 1.07 + 48%|████████████████████████████████████████▍ | 10949/22715 [3:30:30<3:01:43, 1.08it/s][2025-04-29 06:58:36] (step=0010950) Train Loss: 5.6830, Train Steps/Sec: 1.07 + 48%|████████████████████████████████████████▌ | 10974/22715 [3:30:54<3:00:51, 1.08it/s][2025-04-29 06:58:59] (step=0010975) Train Loss: 5.6555, Train Steps/Sec: 1.07 + 48%|████████████████████████████████████████▋ | 10999/22715 [3:31:17<3:00:34, 1.08it/s][2025-04-29 06:59:23] (step=0011000) Train Loss: 5.6609, Train Steps/Sec: 1.07 + 49%|████████████████████████████████████████▊ | 11024/22715 [3:31:40<3:05:37, 1.05it/s][2025-04-29 06:59:46] (step=0011025) Train Loss: 5.6156, Train Steps/Sec: 1.07 + 49%|████████████████████████████████████████▊ | 11049/22715 [3:32:04<3:02:36, 1.06it/s][2025-04-29 07:00:09] (step=0011050) Train Loss: 5.6043, Train Steps/Sec: 1.07 + 49%|████████████████████████████████████████▉ | 11074/22715 [3:32:27<3:01:24, 1.07it/s][2025-04-29 07:00:33] (step=0011075) Train Loss: 5.6439, Train Steps/Sec: 1.07 + 49%|█████████████████████████████████████████ | 11099/22715 [3:32:50<3:00:16, 1.07it/s][2025-04-29 07:00:56] (step=0011100) Train Loss: 5.6384, Train Steps/Sec: 1.07 + 49%|█████████████████████████████████████████▏ | 11124/22715 [3:33:14<2:59:24, 1.08it/s][2025-04-29 07:01:19] (step=0011125) Train Loss: 5.6232, Train Steps/Sec: 1.07 + 49%|█████████████████████████████████████████▏ | 11149/22715 [3:33:37<2:58:38, 1.08it/s][2025-04-29 07:01:42] (step=0011150) Train Loss: 5.6529, Train Steps/Sec: 1.07 + 49%|█████████████████████████████████████████▎ | 11174/22715 [3:34:00<2:58:09, 1.08it/s][2025-04-29 07:02:06] (step=0011175) Train Loss: 5.6555, Train Steps/Sec: 1.07 + 49%|█████████████████████████████████████████▍ | 11199/22715 [3:34:24<2:57:37, 1.08it/s][2025-04-29 07:02:29] (step=0011200) Train Loss: 5.6780, Train Steps/Sec: 1.06 + 49%|█████████████████████████████████████████▌ | 11224/22715 [3:34:47<3:02:24, 1.05it/s][2025-04-29 07:02:53] (step=0011225) Train Loss: 5.6328, Train Steps/Sec: 1.07 + 50%|█████████████████████████████████████████▌ | 11249/22715 [3:35:11<2:59:44, 1.06it/s][2025-04-29 07:03:16] (step=0011250) Train Loss: 5.6603, Train Steps/Sec: 1.07 + 50%|█████████████████████████████████████████▋ | 11274/22715 [3:35:34<2:57:58, 1.07it/s][2025-04-29 07:03:39] (step=0011275) Train Loss: 5.6633, Train Steps/Sec: 1.07 + 50%|█████████████████████████████████████████▊ | 11299/22715 [3:35:57<2:57:03, 1.07it/s][2025-04-29 07:04:03] (step=0011300) Train Loss: 5.6483, Train Steps/Sec: 1.07 + 50%|█████████████████████████████████████████▉ | 11324/22715 [3:36:21<2:56:18, 1.08it/s][2025-04-29 07:04:26] (step=0011325) Train Loss: 5.6910, Train Steps/Sec: 1.07 + 50%|█████████████████████████████████████████▉ | 11349/22715 [3:36:44<2:55:38, 1.08it/s][2025-04-29 07:04:49] (step=0011350) Train Loss: 5.5944, Train Steps/Sec: 1.07 + 50%|██████████████████████████████████████████ | 11374/22715 [3:37:07<2:55:08, 1.08it/s][2025-04-29 07:05:13] (step=0011375) Train Loss: 5.6404, Train Steps/Sec: 1.07 + 50%|██████████████████████████████████████████▏ | 11399/22715 [3:37:31<2:54:18, 1.08it/s][2025-04-29 07:05:36] (step=0011400) Train Loss: 5.5784, Train Steps/Sec: 1.07 + 50%|██████████████████████████████████████████▏ | 11424/22715 [3:37:54<2:59:30, 1.05it/s][2025-04-29 07:05:59] (step=0011425) Train Loss: 5.6240, Train Steps/Sec: 1.07 + 50%|██████████████████████████████████████████▎ | 11449/22715 [3:38:17<2:56:49, 1.06it/s][2025-04-29 07:06:23] (step=0011450) Train Loss: 5.5962, Train Steps/Sec: 1.07 + 51%|██████████████████████████████████████████▍ | 11474/22715 [3:38:41<2:55:16, 1.07it/s][2025-04-29 07:06:46] (step=0011475) Train Loss: 5.6259, Train Steps/Sec: 1.07 + 51%|██████████████████████████████████████████▌ | 11499/22715 [3:39:04<2:54:05, 1.07it/s][2025-04-29 07:07:09] (step=0011500) Train Loss: 5.6149, Train Steps/Sec: 1.07 + 51%|██████████████████████████████████████████▌ | 11524/22715 [3:39:27<2:52:53, 1.08it/s][2025-04-29 07:07:33] (step=0011525) Train Loss: 5.6181, Train Steps/Sec: 1.07 + 51%|██████████████████████████████████████████▋ | 11549/22715 [3:39:51<2:52:16, 1.08it/s][2025-04-29 07:07:56] (step=0011550) Train Loss: 5.6109, Train Steps/Sec: 1.07 + 51%|██████████████████████████████████████████▊ | 11574/22715 [3:40:14<2:51:58, 1.08it/s][2025-04-29 07:08:19] (step=0011575) Train Loss: 5.6214, Train Steps/Sec: 1.07 + 51%|██████████████████████████████████████████▉ | 11599/22715 [3:40:37<2:51:22, 1.08it/s][2025-04-29 07:08:43] (step=0011600) Train Loss: 5.6483, Train Steps/Sec: 1.07 + 51%|██████████████████████████████████████████▉ | 11624/22715 [3:41:01<2:55:52, 1.05it/s][2025-04-29 07:09:06] (step=0011625) Train Loss: 5.6076, Train Steps/Sec: 1.07 + 51%|███████████████████████████████████████████ | 11649/22715 [3:41:24<2:53:13, 1.06it/s][2025-04-29 07:09:29] (step=0011650) Train Loss: 5.6130, Train Steps/Sec: 1.07 + 51%|███████████████████████████████████████████▏ | 11674/22715 [3:41:47<2:52:02, 1.07it/s][2025-04-29 07:09:53] (step=0011675) Train Loss: 5.6370, Train Steps/Sec: 1.07 + 52%|███████████████████████████████████████████▎ | 11699/22715 [3:42:10<2:50:59, 1.07it/s][2025-04-29 07:10:16] (step=0011700) Train Loss: 5.6049, Train Steps/Sec: 1.07 + 52%|███████████████████████████████████████████▎ | 11724/22715 [3:42:34<2:50:18, 1.08it/s][2025-04-29 07:10:39] (step=0011725) Train Loss: 5.6355, Train Steps/Sec: 1.07 + 52%|███████████████████████████████████████████▍ | 11749/22715 [3:42:57<2:49:42, 1.08it/s][2025-04-29 07:11:03] (step=0011750) Train Loss: 5.6030, Train Steps/Sec: 1.07 + 52%|███████████████████████████████████████████▌ | 11774/22715 [3:43:20<2:48:51, 1.08it/s][2025-04-29 07:11:26] (step=0011775) Train Loss: 5.6066, Train Steps/Sec: 1.07 + 52%|███████████████████████████████████████████▋ | 11799/22715 [3:43:44<2:48:39, 1.08it/s][2025-04-29 07:11:49] (step=0011800) Train Loss: 5.6152, Train Steps/Sec: 1.07 + 52%|███████████████████████████████████████████▋ | 11824/22715 [3:44:12<5:08:04, 1.70s/it][2025-04-29 07:12:18] (step=0011825) Train Loss: 5.6392, Train Steps/Sec: 0.88 + 52%|███████████████████████████████████████████▊ | 11849/22715 [3:44:36<2:50:02, 1.07it/s][2025-04-29 07:12:41] (step=0011850) Train Loss: 5.5828, Train Steps/Sec: 1.07 + 52%|███████████████████████████████████████████▉ | 11874/22715 [3:44:59<2:48:44, 1.07it/s][2025-04-29 07:13:04] (step=0011875) Train Loss: 5.5971, Train Steps/Sec: 1.07 + 52%|████████████████████████████████████████████ | 11899/22715 [3:45:22<2:48:07, 1.07it/s][2025-04-29 07:13:28] (step=0011900) Train Loss: 5.5842, Train Steps/Sec: 1.07 + 52%|████████████████████████████████████████████ | 11924/22715 [3:45:45<2:45:11, 1.09it/s][2025-04-29 07:13:51] (step=0011925) Train Loss: 5.5815, Train Steps/Sec: 1.07 + 53%|████████████████████████████████████████████▏ | 11949/22715 [3:46:09<2:46:11, 1.08it/s][2025-04-29 07:14:14] (step=0011950) Train Loss: 5.6122, Train Steps/Sec: 1.07 + 53%|████████████████████████████████████████████▎ | 11974/22715 [3:46:32<2:45:53, 1.08it/s][2025-04-29 07:14:38] (step=0011975) Train Loss: 5.6280, Train Steps/Sec: 1.07 + 53%|████████████████████████████████████████████▎ | 11999/22715 [3:46:55<2:45:07, 1.08it/s][2025-04-29 07:15:01] (step=0012000) Train Loss: 5.6161, Train Steps/Sec: 1.07 +You are using a model of type blip-2 to instantiate a model of type instructblip. This is not supported for all configurations of models and can yield errors. +[2025-04-29 07:15:01] text_config is None. Initializing the text config with default values (`OPTConfig`). +model_name_or_path:/tmp/haozhezhao/model/blip2-flan-t5-xl +image_place_holder: +tokenizer length after expend 32101 +tokenizer length before expend 32100 +100%|████████████████████████████████████████████████████████████████████████████████████████████████| 5/5 [04:53<00:00, 58.64s/it] +[2025-04-29 07:21:06] Finish Eval in 12000 steps...█████████████████████████████████████████| 5/5 [04:52<00:00, 58.38s/it] +processing left-padding... +processing left-padding... +processing left-padding... +processing left-padding... +processing left-padding... +[2025-04-29 07:21:26] Saved checkpoint to checkpoint/X2I_700k_3mask_200k_LLava_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_filtered_objects_3mask_888_context/003-GPT-XL/checkpoints/0012000.pt +[2025-04-29 07:21:28] Removed old checkpoint: checkpoint/X2I_700k_3mask_200k_LLava_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_filtered_objects_3mask_888_context/003-GPT-XL/checkpoints/0010000.pt + 53%|████████████████████████████████████████████▍ | 12024/22715 [3:53:46<2:53:11, 1.03it/s][2025-04-29 07:21:52] (step=0012025) Train Loss: 5.6102, Train Steps/Sec: 0.06 + 53%|████████████████████████████████████████████▌ | 12049/22715 [3:54:10<2:46:44, 1.07it/s][2025-04-29 07:22:15] (step=0012050) Train Loss: 5.6099, Train Steps/Sec: 1.07 + 53%|████████████████████████████████████████████▋ | 12074/22715 [3:54:33<2:45:33, 1.07it/s][2025-04-29 07:22:38] (step=0012075) Train Loss: 5.6332, Train Steps/Sec: 1.07 + 53%|████████████████████████████████████████████▋ | 12099/22715 [3:54:56<2:44:42, 1.07it/s][2025-04-29 07:23:02] (step=0012100) Train Loss: 5.5788, Train Steps/Sec: 1.07 + 53%|████████████████████████████████████████████▊ | 12124/22715 [3:55:20<2:44:01, 1.08it/s][2025-04-29 07:23:25] (step=0012125) Train Loss: 5.6247, Train Steps/Sec: 1.07 + 53%|████████████████████████████████████████████▉ | 12149/22715 [3:55:43<2:43:17, 1.08it/s][2025-04-29 07:23:48] (step=0012150) Train Loss: 5.6152, Train Steps/Sec: 1.07 + 54%|█████████████████████████████████████████████ | 12174/22715 [3:56:06<2:42:41, 1.08it/s][2025-04-29 07:24:12] (step=0012175) Train Loss: 5.6174, Train Steps/Sec: 1.07 + 54%|█████████████████████████████████████████████ | 12199/22715 [3:56:29<2:42:04, 1.08it/s][2025-04-29 07:24:35] (step=0012200) Train Loss: 5.6034, Train Steps/Sec: 1.07 + 54%|█████████████████████████████████████████████▏ | 12224/22715 [3:56:53<2:46:37, 1.05it/s][2025-04-29 07:24:58] (step=0012225) Train Loss: 5.6604, Train Steps/Sec: 1.07 + 54%|█████████████████████████████████████████████▎ | 12249/22715 [3:57:16<2:44:00, 1.06it/s][2025-04-29 07:25:22] (step=0012250) Train Loss: 5.6412, Train Steps/Sec: 1.07 + 54%|█████████████████████████████████████████████▍ | 12274/22715 [3:57:40<2:43:02, 1.07it/s][2025-04-29 07:25:45] (step=0012275) Train Loss: 5.6142, Train Steps/Sec: 1.07 + 54%|█████████████████████████████████████████████▍ | 12299/22715 [3:58:03<2:41:33, 1.07it/s][2025-04-29 07:26:08] (step=0012300) Train Loss: 5.6229, Train Steps/Sec: 1.07 + 54%|█████████████████████████████████████████████▌ | 12324/22715 [3:58:30<2:44:36, 1.05it/s][2025-04-29 07:26:35] (step=0012325) Train Loss: 5.6354, Train Steps/Sec: 0.93 + 54%|█████████████████████████████████████████████▋ | 12349/22715 [3:58:53<2:40:07, 1.08it/s][2025-04-29 07:26:59] (step=0012350) Train Loss: 5.5657, Train Steps/Sec: 1.07 + 54%|█████████████████████████████████████████████▊ | 12374/22715 [3:59:31<6:16:47, 2.19s/it][2025-04-29 07:27:37] (step=0012375) Train Loss: 5.6202, Train Steps/Sec: 0.66 + 55%|█████████████████████████████████████████████▊ | 12399/22715 [4:00:04<3:34:27, 1.25s/it][2025-04-29 07:28:09] (step=0012400) Train Loss: 5.6375, Train Steps/Sec: 0.77 + 55%|█████████████████████████████████████████████▉ | 12424/22715 [4:00:27<2:43:41, 1.05it/s][2025-04-29 07:28:33] (step=0012425) Train Loss: 5.6353, Train Steps/Sec: 1.07 + 55%|██████████████████████████████████████████████ | 12449/22715 [4:00:50<2:40:57, 1.06it/s][2025-04-29 07:28:56] (step=0012450) Train Loss: 5.6245, Train Steps/Sec: 1.07 + 55%|██████████████████████████████████████████████▏ | 12474/22715 [4:01:14<2:39:35, 1.07it/s][2025-04-29 07:29:19] (step=0012475) Train Loss: 5.6059, Train Steps/Sec: 1.07 + 55%|██████████████████████████████████████████████▏ | 12499/22715 [4:01:37<2:38:29, 1.07it/s][2025-04-29 07:29:42] (step=0012500) Train Loss: 5.5688, Train Steps/Sec: 1.07 + 55%|██████████████████████████████████████████████▎ | 12524/22715 [4:02:00<2:37:54, 1.08it/s][2025-04-29 07:30:06] (step=0012525) Train Loss: 5.6064, Train Steps/Sec: 1.07 + 55%|██████████████████████████████████████████████▍ | 12549/22715 [4:02:24<2:37:03, 1.08it/s][2025-04-29 07:30:29] (step=0012550) Train Loss: 5.6176, Train Steps/Sec: 1.07 + 55%|██████████████████████████████████████████████▍ | 12574/22715 [4:02:47<2:36:44, 1.08it/s][2025-04-29 07:30:52] (step=0012575) Train Loss: 5.6182, Train Steps/Sec: 1.07 + 55%|██████████████████████████████████████████████▌ | 12599/22715 [4:03:10<2:36:05, 1.08it/s][2025-04-29 07:31:16] (step=0012600) Train Loss: 5.6079, Train Steps/Sec: 1.07 + 56%|██████████████████████████████████████████████▋ | 12624/22715 [4:03:34<2:40:18, 1.05it/s][2025-04-29 07:31:39] (step=0012625) Train Loss: 5.5929, Train Steps/Sec: 1.07 + 56%|██████████████████████████████████████████████▊ | 12649/22715 [4:03:57<2:37:45, 1.06it/s][2025-04-29 07:32:03] (step=0012650) Train Loss: 5.5959, Train Steps/Sec: 1.07 + 56%|██████████████████████████████████████████████▊ | 12674/22715 [4:04:20<2:36:18, 1.07it/s][2025-04-29 07:32:26] (step=0012675) Train Loss: 5.6068, Train Steps/Sec: 1.07 + 56%|██████████████████████████████████████████████▉ | 12699/22715 [4:04:44<2:35:28, 1.07it/s][2025-04-29 07:32:49] (step=0012700) Train Loss: 5.5997, Train Steps/Sec: 1.07 + 56%|███████████████████████████████████████████████ | 12724/22715 [4:05:07<2:34:43, 1.08it/s][2025-04-29 07:33:12] (step=0012725) Train Loss: 5.5990, Train Steps/Sec: 1.07 + 56%|███████████████████████████████████████████████▏ | 12749/22715 [4:05:30<2:33:41, 1.08it/s][2025-04-29 07:33:36] (step=0012750) Train Loss: 5.5619, Train Steps/Sec: 1.07 + 56%|███████████████████████████████████████████████▏ | 12774/22715 [4:05:54<2:33:20, 1.08it/s][2025-04-29 07:33:59] (step=0012775) Train Loss: 5.6062, Train Steps/Sec: 1.07 + 56%|███████████████████████████████████████████████▎ | 12799/22715 [4:06:17<2:32:52, 1.08it/s][2025-04-29 07:34:23] (step=0012800) Train Loss: 5.6018, Train Steps/Sec: 1.07 + 56%|███████████████████████████████████████████████▍ | 12824/22715 [4:06:40<2:36:43, 1.05it/s][2025-04-29 07:34:46] (step=0012825) Train Loss: 5.6039, Train Steps/Sec: 1.07 + 57%|███████████████████████████████████████████████▌ | 12849/22715 [4:07:04<2:34:19, 1.07it/s][2025-04-29 07:35:09] (step=0012850) Train Loss: 5.5776, Train Steps/Sec: 1.07 + 57%|███████████████████████████████████████████████▌ | 12874/22715 [4:07:27<2:33:17, 1.07it/s][2025-04-29 07:35:32] (step=0012875) Train Loss: 5.5906, Train Steps/Sec: 1.07 + 57%|███████████████████████████████████████████████▋ | 12899/22715 [4:07:50<2:32:22, 1.07it/s][2025-04-29 07:35:56] (step=0012900) Train Loss: 5.5995, Train Steps/Sec: 1.07 + 57%|███████████████████████████████████████████████▊ | 12924/22715 [4:08:14<2:31:36, 1.08it/s][2025-04-29 07:36:19] (step=0012925) Train Loss: 5.6090, Train Steps/Sec: 1.07 + 57%|███████████████████████████████████████████████▉ | 12949/22715 [4:08:37<2:30:34, 1.08it/s][2025-04-29 07:36:42] (step=0012950) Train Loss: 5.5639, Train Steps/Sec: 1.07 + 57%|███████████████████████████████████████████████▉ | 12974/22715 [4:09:00<2:29:59, 1.08it/s][2025-04-29 07:37:06] (step=0012975) Train Loss: 5.5988, Train Steps/Sec: 1.07 + 57%|████████████████████████████████████████████████ | 12999/22715 [4:09:24<2:29:29, 1.08it/s][2025-04-29 07:37:29] (step=0013000) Train Loss: 5.5443, Train Steps/Sec: 1.07 + 57%|████████████████████████████████████████████████▏ | 13024/22715 [4:09:47<2:33:43, 1.05it/s][2025-04-29 07:37:52] (step=0013025) Train Loss: 5.5398, Train Steps/Sec: 1.07 + 57%|████████████████████████████████████████████████▎ | 13049/22715 [4:10:10<2:32:42, 1.05it/s][2025-04-29 07:38:16] (step=0013050) Train Loss: 5.6135, Train Steps/Sec: 1.07 + 58%|████████████████████████████████████████████████▎ | 13074/22715 [4:10:34<2:30:03, 1.07it/s][2025-04-29 07:38:39] (step=0013075) Train Loss: 5.5620, Train Steps/Sec: 1.07 + 58%|████████████████████████████████████████████████▍ | 13099/22715 [4:10:57<2:29:10, 1.07it/s][2025-04-29 07:39:02] (step=0013100) Train Loss: 5.5969, Train Steps/Sec: 1.07 + 58%|████████████████████████████████████████████████▌ | 13124/22715 [4:11:20<2:28:21, 1.08it/s][2025-04-29 07:39:26] (step=0013125) Train Loss: 5.6075, Train Steps/Sec: 1.07 + 58%|████████████████████████████████████████████████▌ | 13149/22715 [4:11:44<2:28:02, 1.08it/s][2025-04-29 07:39:49] (step=0013150) Train Loss: 5.5943, Train Steps/Sec: 1.07 + 58%|████████████████████████████████████████████████▋ | 13174/22715 [4:12:07<2:27:26, 1.08it/s][2025-04-29 07:40:12] (step=0013175) Train Loss: 5.5932, Train Steps/Sec: 1.07 + 58%|████████████████████████████████████████████████▊ | 13199/22715 [4:12:30<2:26:55, 1.08it/s][2025-04-29 07:40:36] (step=0013200) Train Loss: 5.5700, Train Steps/Sec: 1.07 + 58%|████████████████████████████████████████████████▉ | 13224/22715 [4:12:54<2:30:29, 1.05it/s][2025-04-29 07:40:59] (step=0013225) Train Loss: 5.5841, Train Steps/Sec: 1.07 + 58%|████████████████████████████████████████████████▉ | 13249/22715 [4:13:17<2:28:13, 1.06it/s][2025-04-29 07:41:22] (step=0013250) Train Loss: 5.6120, Train Steps/Sec: 1.07 + 58%|█████████████████████████████████████████████████ | 13274/22715 [4:13:40<2:27:00, 1.07it/s][2025-04-29 07:41:46] (step=0013275) Train Loss: 5.5631, Train Steps/Sec: 1.07 + 59%|█████████████████████████████████████████████████▏ | 13299/22715 [4:14:04<2:26:07, 1.07it/s][2025-04-29 07:42:09] (step=0013300) Train Loss: 5.6001, Train Steps/Sec: 1.07 + 59%|█████████████████████████████████████████████████▎ | 13324/22715 [4:14:27<2:25:15, 1.08it/s][2025-04-29 07:42:32] (step=0013325) Train Loss: 5.5546, Train Steps/Sec: 1.07 + 59%|█████████████████████████████████████████████████▎ | 13349/22715 [4:14:50<2:24:28, 1.08it/s][2025-04-29 07:42:56] (step=0013350) Train Loss: 5.5810, Train Steps/Sec: 1.07 + 59%|█████████████████████████████████████████████████▍ | 13374/22715 [4:15:14<2:24:04, 1.08it/s][2025-04-29 07:43:19] (step=0013375) Train Loss: 5.6021, Train Steps/Sec: 1.07 + 59%|█████████████████████████████████████████████████▌ | 13399/22715 [4:15:37<2:23:42, 1.08it/s][2025-04-29 07:43:42] (step=0013400) Train Loss: 5.5799, Train Steps/Sec: 1.07 + 59%|█████████████████████████████████████████████████▋ | 13424/22715 [4:16:00<2:27:47, 1.05it/s][2025-04-29 07:44:06] (step=0013425) Train Loss: 5.5587, Train Steps/Sec: 1.07 + 59%|█████████████████████████████████████████████████▋ | 13449/22715 [4:16:24<2:25:15, 1.06it/s][2025-04-29 07:44:29] (step=0013450) Train Loss: 5.5708, Train Steps/Sec: 1.07 + 59%|█████████████████████████████████████████████████▊ | 13474/22715 [4:16:47<2:24:17, 1.07it/s][2025-04-29 07:44:52] (step=0013475) Train Loss: 5.6145, Train Steps/Sec: 1.07 + 59%|█████████████████████████████████████████████████▉ | 13499/22715 [4:17:10<2:23:11, 1.07it/s][2025-04-29 07:45:16] (step=0013500) Train Loss: 5.5925, Train Steps/Sec: 1.07 + 60%|██████████████████████████████████████████████████ | 13524/22715 [4:17:34<2:22:13, 1.08it/s][2025-04-29 07:45:39] (step=0013525) Train Loss: 5.5827, Train Steps/Sec: 1.07 + 60%|██████████████████████████████████████████████████ | 13549/22715 [4:17:57<2:21:50, 1.08it/s][2025-04-29 07:46:02] (step=0013550) Train Loss: 5.5588, Train Steps/Sec: 1.07 + 60%|██████████████████████████████████████████████████▏ | 13574/22715 [4:18:20<2:21:12, 1.08it/s][2025-04-29 07:46:26] (step=0013575) Train Loss: 5.6055, Train Steps/Sec: 1.07 + 60%|██████████████████████████████████████████████████▎ | 13599/22715 [4:18:43<2:20:42, 1.08it/s][2025-04-29 07:46:49] (step=0013600) Train Loss: 5.5701, Train Steps/Sec: 1.07 + 60%|██████████████████████████████████████████████████▍ | 13624/22715 [4:19:07<2:24:36, 1.05it/s][2025-04-29 07:47:12] (step=0013625) Train Loss: 5.5817, Train Steps/Sec: 1.07 + 60%|██████████████████████████████████████████████████▍ | 13649/22715 [4:19:30<2:22:14, 1.06it/s][2025-04-29 07:47:36] (step=0013650) Train Loss: 5.5826, Train Steps/Sec: 1.07 + 60%|██████████████████████████████████████████████████▌ | 13674/22715 [4:19:54<2:20:48, 1.07it/s][2025-04-29 07:47:59] (step=0013675) Train Loss: 5.5967, Train Steps/Sec: 1.07 + 60%|██████████████████████████████████████████████████▋ | 13699/22715 [4:20:17<2:20:07, 1.07it/s][2025-04-29 07:48:22] (step=0013700) Train Loss: 5.5624, Train Steps/Sec: 1.07 + 60%|██████████████████████████████████████████████████▊ | 13724/22715 [4:20:40<2:18:49, 1.08it/s][2025-04-29 07:48:46] (step=0013725) Train Loss: 5.5557, Train Steps/Sec: 1.07 + 61%|██████████████████████████████████████████████████▊ | 13749/22715 [4:21:04<2:18:29, 1.08it/s][2025-04-29 07:49:09] (step=0013750) Train Loss: 5.5652, Train Steps/Sec: 1.07 + 61%|██████████████████████████████████████████████████▉ | 13774/22715 [4:21:27<2:17:38, 1.08it/s][2025-04-29 07:49:32] (step=0013775) Train Loss: 5.5830, Train Steps/Sec: 1.07 + 61%|███████████████████████████████████████████████████ | 13799/22715 [4:21:50<2:17:23, 1.08it/s][2025-04-29 07:49:56] (step=0013800) Train Loss: 5.5423, Train Steps/Sec: 1.07 + 61%|███████████████████████████████████████████████████ | 13824/22715 [4:22:19<4:14:50, 1.72s/it][2025-04-29 07:50:24] (step=0013825) Train Loss: 5.5709, Train Steps/Sec: 0.88 + 61%|███████████████████████████████████████████████████▏ | 13849/22715 [4:22:42<2:19:22, 1.06it/s][2025-04-29 07:50:48] (step=0013850) Train Loss: 5.5457, Train Steps/Sec: 1.07 + 61%|███████████████████████████████████████████████████▎ | 13874/22715 [4:23:05<2:17:34, 1.07it/s][2025-04-29 07:51:11] (step=0013875) Train Loss: 5.5809, Train Steps/Sec: 1.07 + 61%|███████████████████████████████████████████████████▍ | 13899/22715 [4:23:29<2:16:29, 1.08it/s][2025-04-29 07:51:34] (step=0013900) Train Loss: 5.5725, Train Steps/Sec: 1.07 + 61%|███████████████████████████████████████████████████▍ | 13924/22715 [4:23:52<2:16:00, 1.08it/s][2025-04-29 07:51:58] (step=0013925) Train Loss: 5.5868, Train Steps/Sec: 1.07 + 61%|███████████████████████████████████████████████████▌ | 13949/22715 [4:24:15<2:15:37, 1.08it/s][2025-04-29 07:52:21] (step=0013950) Train Loss: 5.5737, Train Steps/Sec: 1.07 + 62%|███████████████████████████████████████████████████▋ | 13974/22715 [4:24:39<2:14:44, 1.08it/s][2025-04-29 07:52:44] (step=0013975) Train Loss: 5.5902, Train Steps/Sec: 1.07 + 62%|███████████████████████████████████████████████████▊ | 13999/22715 [4:25:02<2:14:40, 1.08it/s][2025-04-29 07:53:08] (step=0014000) Train Loss: 5.5533, Train Steps/Sec: 1.07 +You are using a model of type blip-2 to instantiate a model of type instructblip. This is not supported for all configurations of models and can yield errors. +[2025-04-29 07:53:08] text_config is None. Initializing the text config with default values (`OPTConfig`). +model_name_or_path:/tmp/haozhezhao/model/blip2-flan-t5-xl +image_place_holder: +tokenizer length after expend 32101 +tokenizer length before expend 32100 +100%|████████████████████████████████████████████████████████████████████████████████████████████████| 5/5 [04:53<00:00, 58.79s/it] +[2025-04-29 07:59:14] Finish Eval in 14000 steps...█████████████████████████████████████████| 5/5 [04:53<00:00, 58.56s/it] +processing left-padding... +processing left-padding... +processing left-padding... +processing left-padding... +processing left-padding... +[2025-04-29 07:59:33] Saved checkpoint to checkpoint/X2I_700k_3mask_200k_LLava_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_filtered_objects_3mask_888_context/003-GPT-XL/checkpoints/0014000.pt +[2025-04-29 07:59:36] Removed old checkpoint: checkpoint/X2I_700k_3mask_200k_LLava_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_filtered_objects_3mask_888_context/003-GPT-XL/checkpoints/0012000.pt + 62%|███████████████████████████████████████████████████▊ | 14024/22715 [4:31:54<2:20:56, 1.03it/s][2025-04-29 07:59:59] (step=0014025) Train Loss: 5.5546, Train Steps/Sec: 0.06 + 62%|███████████████████████████████████████████████████▉ | 14049/22715 [4:32:17<2:15:33, 1.07it/s][2025-04-29 08:00:22] (step=0014050) Train Loss: 5.6075, Train Steps/Sec: 1.07 + 62%|████████████████████████████████████████████████████ | 14074/22715 [4:32:40<2:14:35, 1.07it/s][2025-04-29 08:00:46] (step=0014075) Train Loss: 5.5798, Train Steps/Sec: 1.07 + 62%|████████████████████████████████████████████████████▏ | 14099/22715 [4:33:03<2:13:43, 1.07it/s][2025-04-29 08:01:09] (step=0014100) Train Loss: 5.5966, Train Steps/Sec: 1.07 + 62%|████████████████████████████████████████████████████▏ | 14124/22715 [4:33:27<2:13:02, 1.08it/s][2025-04-29 08:01:32] (step=0014125) Train Loss: 5.5397, Train Steps/Sec: 1.07 + 62%|████████████████████████████████████████████████████▎ | 14149/22715 [4:33:54<2:21:04, 1.01it/s][2025-04-29 08:01:59] (step=0014150) Train Loss: 5.6072, Train Steps/Sec: 0.93 + 62%|████████████████████████████████████████████████████▍ | 14174/22715 [4:34:17<2:11:42, 1.08it/s][2025-04-29 08:02:22] (step=0014175) Train Loss: 5.5601, Train Steps/Sec: 1.07 + 63%|████████████████████████████████████████████████████▌ | 14199/22715 [4:34:51<2:29:14, 1.05s/it][2025-04-29 08:02:57] (step=0014200) Train Loss: 5.5875, Train Steps/Sec: 0.73 + 63%|████████████████████████████████████████████████████▌ | 14224/22715 [4:35:24<2:32:50, 1.08s/it][2025-04-29 08:03:29] (step=0014225) Train Loss: 5.5446, Train Steps/Sec: 0.78 + 63%|████████████████████████████████████████████████████▋ | 14249/22715 [4:35:51<2:12:35, 1.06it/s][2025-04-29 08:03:56] (step=0014250) Train Loss: 5.5813, Train Steps/Sec: 0.92 + 63%|████████████████████████████████████████████████████▊ | 14274/22715 [4:36:14<2:11:25, 1.07it/s][2025-04-29 08:04:19] (step=0014275) Train Loss: 5.5751, Train Steps/Sec: 1.07 + 63%|████████████████████████████████████████████████████▉ | 14299/22715 [4:36:37<2:10:31, 1.07it/s][2025-04-29 08:04:43] (step=0014300) Train Loss: 5.5394, Train Steps/Sec: 1.06 + 63%|████████████████████████████████████████████████████▉ | 14324/22715 [4:37:01<2:10:00, 1.08it/s][2025-04-29 08:05:06] (step=0014325) Train Loss: 5.5298, Train Steps/Sec: 1.07 + 63%|█████████████████████████████████████████████████████ | 14349/22715 [4:37:24<2:09:10, 1.08it/s][2025-04-29 08:05:30] (step=0014350) Train Loss: 5.5870, Train Steps/Sec: 1.07 + 63%|█████████████████████████████████████████████████████▏ | 14374/22715 [4:37:47<2:08:28, 1.08it/s][2025-04-29 08:05:53] (step=0014375) Train Loss: 5.5338, Train Steps/Sec: 1.07 + 63%|█████████████████████████████████████████████████████▏ | 14399/22715 [4:38:11<2:08:18, 1.08it/s][2025-04-29 08:06:16] (step=0014400) Train Loss: 5.5658, Train Steps/Sec: 1.07 + 63%|█████████████████████████████████████████████████████▎ | 14424/22715 [4:38:34<2:11:45, 1.05it/s][2025-04-29 08:06:40] (step=0014425) Train Loss: 5.6030, Train Steps/Sec: 1.07 + 64%|█████████████████████████████████████████████████████▍ | 14449/22715 [4:38:57<2:09:39, 1.06it/s][2025-04-29 08:07:03] (step=0014450) Train Loss: 5.5906, Train Steps/Sec: 1.07 + 64%|█████████████████████████████████████████████████████▌ | 14474/22715 [4:39:21<2:08:22, 1.07it/s][2025-04-29 08:07:26] (step=0014475) Train Loss: 5.5478, Train Steps/Sec: 1.07 + 64%|█████████████████████████████████████████████████████▌ | 14499/22715 [4:39:44<2:07:25, 1.07it/s][2025-04-29 08:07:50] (step=0014500) Train Loss: 5.5179, Train Steps/Sec: 1.07 + 64%|█████████████████████████████████████████████████████▋ | 14524/22715 [4:40:07<2:06:44, 1.08it/s][2025-04-29 08:08:13] (step=0014525) Train Loss: 5.5665, Train Steps/Sec: 1.07 + 64%|█████████████████████████████████████████████████████▊ | 14549/22715 [4:40:31<2:06:17, 1.08it/s][2025-04-29 08:08:36] (step=0014550) Train Loss: 5.5528, Train Steps/Sec: 1.07 + 64%|█████████████████████████████████████████████████████▉ | 14574/22715 [4:40:54<2:05:28, 1.08it/s][2025-04-29 08:09:00] (step=0014575) Train Loss: 5.5221, Train Steps/Sec: 1.07 + 64%|█████████████████████████████████████████████████████▉ | 14599/22715 [4:41:17<2:04:57, 1.08it/s][2025-04-29 08:09:23] (step=0014600) Train Loss: 5.5813, Train Steps/Sec: 1.07 + 64%|██████████████████████████████████████████████████████ | 14624/22715 [4:41:41<2:08:12, 1.05it/s][2025-04-29 08:09:46] (step=0014625) Train Loss: 5.5635, Train Steps/Sec: 1.07 + 64%|██████████████████████████████████████████████████████▏ | 14649/22715 [4:42:04<2:06:15, 1.06it/s][2025-04-29 08:10:10] (step=0014650) Train Loss: 5.5568, Train Steps/Sec: 1.07 + 65%|██████████████████████████████████████████████████████▎ | 14674/22715 [4:42:27<2:05:10, 1.07it/s][2025-04-29 08:10:33] (step=0014675) Train Loss: 5.5543, Train Steps/Sec: 1.07 + 65%|██████████████████████████████████████████████████████▎ | 14699/22715 [4:42:51<2:04:40, 1.07it/s][2025-04-29 08:10:56] (step=0014700) Train Loss: 5.5417, Train Steps/Sec: 1.07 + 65%|██████████████████████████████████████████████████████▍ | 14724/22715 [4:43:14<2:03:37, 1.08it/s][2025-04-29 08:11:20] (step=0014725) Train Loss: 5.5511, Train Steps/Sec: 1.07 + 65%|██████████████████████████████████████████████████████▌ | 14749/22715 [4:43:37<2:03:14, 1.08it/s][2025-04-29 08:11:43] (step=0014750) Train Loss: 5.5200, Train Steps/Sec: 1.07 + 65%|██████████████████████████████████████████████████████▋ | 14774/22715 [4:44:01<2:02:28, 1.08it/s][2025-04-29 08:12:06] (step=0014775) Train Loss: 5.5722, Train Steps/Sec: 1.07 + 65%|██████████████████████████████████████████████████████▋ | 14799/22715 [4:44:24<2:01:49, 1.08it/s][2025-04-29 08:12:30] (step=0014800) Train Loss: 5.5240, Train Steps/Sec: 1.07 + 65%|██████████████████████████████████████████████████████▊ | 14824/22715 [4:44:47<2:05:12, 1.05it/s][2025-04-29 08:12:53] (step=0014825) Train Loss: 5.5490, Train Steps/Sec: 1.07 + 65%|██████████████████████████████████████████████████████▉ | 14849/22715 [4:45:11<2:02:59, 1.07it/s][2025-04-29 08:13:16] (step=0014850) Train Loss: 5.5081, Train Steps/Sec: 1.07 + 65%|███████████████████████████████████████████████████████ | 14874/22715 [4:45:34<2:02:08, 1.07it/s][2025-04-29 08:13:40] (step=0014875) Train Loss: 5.5414, Train Steps/Sec: 1.07 + 66%|███████████████████████████████████████████████████████ | 14899/22715 [4:45:57<2:01:20, 1.07it/s][2025-04-29 08:14:03] (step=0014900) Train Loss: 5.5642, Train Steps/Sec: 1.07 + 66%|███████████████████████████████████████████████████████▏ | 14924/22715 [4:46:21<2:00:38, 1.08it/s][2025-04-29 08:14:26] (step=0014925) Train Loss: 5.5723, Train Steps/Sec: 1.07 + 66%|███████████████████████████████████████████████████████▎ | 14949/22715 [4:46:44<2:00:06, 1.08it/s][2025-04-29 08:14:50] (step=0014950) Train Loss: 5.5581, Train Steps/Sec: 1.07 + 66%|███████████████████████████████████████████████████████▎ | 14974/22715 [4:47:07<1:59:22, 1.08it/s][2025-04-29 08:15:13] (step=0014975) Train Loss: 5.5734, Train Steps/Sec: 1.07 + 66%|███████████████████████████████████████████████████████▍ | 14999/22715 [4:47:31<1:59:12, 1.08it/s][2025-04-29 08:15:36] (step=0015000) Train Loss: 5.5174, Train Steps/Sec: 1.07 + 66%|███████████████████████████████████████████████████████▌ | 15024/22715 [4:47:54<2:02:05, 1.05it/s][2025-04-29 08:16:00] (step=0015025) Train Loss: 5.5478, Train Steps/Sec: 1.07 + 66%|███████████████████████████████████████████████████████▋ | 15049/22715 [4:48:17<2:00:18, 1.06it/s][2025-04-29 08:16:23] (step=0015050) Train Loss: 5.5366, Train Steps/Sec: 1.07 + 66%|███████████████████████████████████████████████████████▋ | 15074/22715 [4:48:41<1:59:32, 1.07it/s][2025-04-29 08:16:46] (step=0015075) Train Loss: 5.5593, Train Steps/Sec: 1.07 + 66%|███████████████████████████████████████████████████████▊ | 15099/22715 [4:49:04<1:58:18, 1.07it/s][2025-04-29 08:17:10] (step=0015100) Train Loss: 5.5522, Train Steps/Sec: 1.07 + 67%|███████████████████████████████████████████████████████▉ | 15124/22715 [4:49:27<1:57:40, 1.08it/s][2025-04-29 08:17:33] (step=0015125) Train Loss: 5.5422, Train Steps/Sec: 1.07 + 67%|████████████████████████████████████████████████████████ | 15149/22715 [4:49:51<1:56:40, 1.08it/s][2025-04-29 08:17:56] (step=0015150) Train Loss: 5.5012, Train Steps/Sec: 1.07 + 67%|████████████████████████████████████████████████████████ | 15174/22715 [4:50:14<1:56:33, 1.08it/s][2025-04-29 08:18:19] (step=0015175) Train Loss: 5.5454, Train Steps/Sec: 1.07 + 67%|████████████████████████████████████████████████████████▏ | 15199/22715 [4:50:37<1:55:49, 1.08it/s][2025-04-29 08:18:43] (step=0015200) Train Loss: 5.5244, Train Steps/Sec: 1.07 + 67%|████████████████████████████████████████████████████████▎ | 15224/22715 [4:51:01<1:58:54, 1.05it/s][2025-04-29 08:19:06] (step=0015225) Train Loss: 5.5776, Train Steps/Sec: 1.07 + 67%|████████████████████████████████████████████████████████▍ | 15249/22715 [4:51:24<1:56:49, 1.07it/s][2025-04-29 08:19:29] (step=0015250) Train Loss: 5.5335, Train Steps/Sec: 1.07 + 67%|████████████████████████████████████████████████████████▍ | 15274/22715 [4:51:47<1:56:02, 1.07it/s][2025-04-29 08:19:53] (step=0015275) Train Loss: 5.5761, Train Steps/Sec: 1.07 + 67%|████████████████████████████████████████████████████████▌ | 15299/22715 [4:52:11<1:55:09, 1.07it/s][2025-04-29 08:20:16] (step=0015300) Train Loss: 5.5541, Train Steps/Sec: 1.07 + 67%|████████████████████████████████████████████████████████▋ | 15324/22715 [4:52:34<1:54:20, 1.08it/s][2025-04-29 08:20:39] (step=0015325) Train Loss: 5.5309, Train Steps/Sec: 1.07 + 68%|████████████████████████████████████████████████████████▊ | 15349/22715 [4:52:57<1:53:45, 1.08it/s][2025-04-29 08:21:03] (step=0015350) Train Loss: 5.4955, Train Steps/Sec: 1.07 + 68%|████████████████████████████████████████████████████████▊ | 15374/22715 [4:53:21<1:53:03, 1.08it/s][2025-04-29 08:21:26] (step=0015375) Train Loss: 5.5297, Train Steps/Sec: 1.07 + 68%|████████████████████████████████████████████████████████▉ | 15399/22715 [4:53:44<1:52:40, 1.08it/s][2025-04-29 08:21:50] (step=0015400) Train Loss: 5.5132, Train Steps/Sec: 1.07 + 68%|█████████████████████████████████████████████████████████ | 15424/22715 [4:54:07<1:55:52, 1.05it/s][2025-04-29 08:22:13] (step=0015425) Train Loss: 5.5043, Train Steps/Sec: 1.07 + 68%|█████████████████████████████████████████████████████████▏ | 15449/22715 [4:54:31<1:53:52, 1.06it/s][2025-04-29 08:22:36] (step=0015450) Train Loss: 5.5396, Train Steps/Sec: 1.07 + 68%|█████████████████████████████████████████████████████████▏ | 15474/22715 [4:54:54<1:53:07, 1.07it/s][2025-04-29 08:22:59] (step=0015475) Train Loss: 5.5255, Train Steps/Sec: 1.07 + 68%|█████████████████████████████████████████████████████████▎ | 15499/22715 [4:55:17<1:52:00, 1.07it/s][2025-04-29 08:23:23] (step=0015500) Train Loss: 5.5703, Train Steps/Sec: 1.07 + 68%|█████████████████████████████████████████████████████████▍ | 15524/22715 [4:55:41<1:51:17, 1.08it/s][2025-04-29 08:23:46] (step=0015525) Train Loss: 5.5352, Train Steps/Sec: 1.07 + 68%|█████████████████████████████████████████████████████████▌ | 15549/22715 [4:56:04<1:50:45, 1.08it/s][2025-04-29 08:24:09] (step=0015550) Train Loss: 5.5093, Train Steps/Sec: 1.07 + 69%|█████████████████████████████████████████████████████████▌ | 15574/22715 [4:56:27<1:50:04, 1.08it/s][2025-04-29 08:24:33] (step=0015575) Train Loss: 5.5172, Train Steps/Sec: 1.07 + 69%|█████████████████████████████████████████████████████████▋ | 15599/22715 [4:56:51<1:50:49, 1.07it/s][2025-04-29 08:24:56] (step=0015600) Train Loss: 5.5130, Train Steps/Sec: 1.07 + 69%|█████████████████████████████████████████████████████████▊ | 15624/22715 [4:57:14<1:52:14, 1.05it/s][2025-04-29 08:25:20] (step=0015625) Train Loss: 5.5154, Train Steps/Sec: 1.07 + 69%|█████████████████████████████████████████████████████████▊ | 15649/22715 [4:57:37<1:51:02, 1.06it/s][2025-04-29 08:25:43] (step=0015650) Train Loss: 5.5052, Train Steps/Sec: 1.07 + 69%|█████████████████████████████████████████████████████████▉ | 15674/22715 [4:58:01<1:49:39, 1.07it/s][2025-04-29 08:26:06] (step=0015675) Train Loss: 5.4906, Train Steps/Sec: 1.07 + 69%|██████████████████████████████████████████████████████████ | 15699/22715 [4:58:24<1:48:43, 1.08it/s][2025-04-29 08:26:30] (step=0015700) Train Loss: 5.5407, Train Steps/Sec: 1.07 + 69%|██████████████████████████████████████████████████████████▏ | 15724/22715 [4:58:47<1:48:01, 1.08it/s][2025-04-29 08:26:53] (step=0015725) Train Loss: 5.5064, Train Steps/Sec: 1.07 + 69%|██████████████████████████████████████████████████████████▏ | 15749/22715 [4:59:11<1:47:24, 1.08it/s][2025-04-29 08:27:16] (step=0015750) Train Loss: 5.5544, Train Steps/Sec: 1.07 + 69%|██████████████████████████████████████████████████████████▎ | 15774/22715 [4:59:34<1:46:47, 1.08it/s][2025-04-29 08:27:39] (step=0015775) Train Loss: 5.5401, Train Steps/Sec: 1.07 + 70%|██████████████████████████████████████████████████████████▍ | 15799/22715 [4:59:57<1:46:23, 1.08it/s][2025-04-29 08:28:03] (step=0015800) Train Loss: 5.5560, Train Steps/Sec: 1.07 + 70%|██████████████████████████████████████████████████████████▌ | 15824/22715 [5:00:26<2:03:55, 1.08s/it][2025-04-29 08:28:31] (step=0015825) Train Loss: 5.5762, Train Steps/Sec: 0.88 + 70%|██████████████████████████████████████████████████████████▌ | 15849/22715 [5:00:49<1:47:22, 1.07it/s][2025-04-29 08:28:55] (step=0015850) Train Loss: 5.5345, Train Steps/Sec: 1.07 + 70%|██████████████████████████████████████████████████████████▋ | 15874/22715 [5:01:12<1:46:38, 1.07it/s][2025-04-29 08:29:18] (step=0015875) Train Loss: 5.5260, Train Steps/Sec: 1.07 + 70%|██████████████████████████████████████████████████████████▊ | 15899/22715 [5:01:36<1:45:42, 1.07it/s][2025-04-29 08:29:41] (step=0015900) Train Loss: 5.5095, Train Steps/Sec: 1.07 + 70%|██████████████████████████████████████████████████████████▉ | 15924/22715 [5:01:59<1:44:56, 1.08it/s][2025-04-29 08:30:05] (step=0015925) Train Loss: 5.4569, Train Steps/Sec: 1.07 + 70%|██████████████████████████████████████████████████████████▉ | 15949/22715 [5:02:22<1:44:17, 1.08it/s][2025-04-29 08:30:28] (step=0015950) Train Loss: 5.5358, Train Steps/Sec: 1.07 + 70%|███████████████████████████████████████████████████████████ | 15974/22715 [5:02:49<2:04:32, 1.11s/it][2025-04-29 08:30:55] (step=0015975) Train Loss: 5.5510, Train Steps/Sec: 0.93 + 70%|███████████████████████████████████████████████████████████▏ | 15999/22715 [5:03:13<1:43:21, 1.08it/s][2025-04-29 08:31:18] (step=0016000) Train Loss: 5.5132, Train Steps/Sec: 1.07 +You are using a model of type blip-2 to instantiate a model of type instructblip. This is not supported for all configurations of models and can yield errors. +[2025-04-29 08:31:18] text_config is None. Initializing the text config with default values (`OPTConfig`). +model_name_or_path:/tmp/haozhezhao/model/blip2-flan-t5-xl +image_place_holder: +tokenizer length after expend 32101 +tokenizer length before expend 32100 +100%|████████████████████████████████████████████████████████████████████████████████████████████████| 5/5 [04:55<00:00, 59.09s/it] +[2025-04-29 08:37:27] Finish Eval in 16000 steps...█████████████████████████████████████████| 5/5 [04:55<00:00, 58.77s/it] +processing left-padding... +processing left-padding... +processing left-padding... +processing left-padding... +processing left-padding... +[2025-04-29 08:37:46] Saved checkpoint to checkpoint/X2I_700k_3mask_200k_LLava_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_filtered_objects_3mask_888_context/003-GPT-XL/checkpoints/0016000.pt +[2025-04-29 08:37:49] Removed old checkpoint: checkpoint/X2I_700k_3mask_200k_LLava_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_filtered_objects_3mask_888_context/003-GPT-XL/checkpoints/0014000.pt + 71%|███████████████████████████████████████████████████████████▎ | 16024/22715 [5:10:14<1:56:34, 1.05s/it][2025-04-29 08:38:20] (step=0016025) Train Loss: 5.4989, Train Steps/Sec: 0.06 + 71%|███████████████████████████████████████████████████████████▎ | 16049/22715 [5:10:46<4:30:14, 2.43s/it][2025-04-29 08:38:52] (step=0016050) Train Loss: 5.5173, Train Steps/Sec: 0.78 + 71%|███████████████████████████████████████████████████████████▍ | 16074/22715 [5:11:13<1:43:29, 1.07it/s][2025-04-29 08:39:19] (step=0016075) Train Loss: 5.5201, Train Steps/Sec: 0.92 + 71%|███████████████████████████████████████████████████████████▌ | 16099/22715 [5:11:37<1:42:30, 1.08it/s][2025-04-29 08:39:42] (step=0016100) Train Loss: 5.5091, Train Steps/Sec: 1.08 + 71%|███████████████████████████████████████████████████████████▋ | 16124/22715 [5:12:00<1:42:08, 1.08it/s][2025-04-29 08:40:05] (step=0016125) Train Loss: 5.5275, Train Steps/Sec: 1.07 + 71%|███████████████████████████████████████████████████████████▋ | 16149/22715 [5:12:23<1:41:15, 1.08it/s][2025-04-29 08:40:29] (step=0016150) Train Loss: 5.5176, Train Steps/Sec: 1.07 + 71%|███████████████████████████████████████████████████████████▊ | 16174/22715 [5:12:47<1:41:11, 1.08it/s][2025-04-29 08:40:52] (step=0016175) Train Loss: 5.4848, Train Steps/Sec: 1.07 + 71%|███████████████████████████████████████████████████████████▉ | 16199/22715 [5:13:10<1:40:33, 1.08it/s][2025-04-29 08:41:16] (step=0016200) Train Loss: 5.4903, Train Steps/Sec: 1.07 + 71%|███████████████████████████████████████████████████████████▉ | 16224/22715 [5:13:33<1:43:13, 1.05it/s][2025-04-29 08:41:39] (step=0016225) Train Loss: 5.5039, Train Steps/Sec: 1.07 + 72%|████████████████████████████████████████████████████████████ | 16249/22715 [5:13:57<1:41:31, 1.06it/s][2025-04-29 08:42:02] (step=0016250) Train Loss: 5.4891, Train Steps/Sec: 1.07 + 72%|████████████████████████████████████████████████████████████▏ | 16274/22715 [5:14:20<1:40:18, 1.07it/s][2025-04-29 08:42:25] (step=0016275) Train Loss: 5.5368, Train Steps/Sec: 1.07 + 72%|████████████████████████████████████████████████████████████▎ | 16299/22715 [5:14:43<1:39:43, 1.07it/s][2025-04-29 08:42:49] (step=0016300) Train Loss: 5.4952, Train Steps/Sec: 1.07 + 72%|████████████████████████████████████████████████████████████▎ | 16324/22715 [5:15:07<1:39:10, 1.07it/s][2025-04-29 08:43:12] (step=0016325) Train Loss: 5.5017, Train Steps/Sec: 1.07 + 72%|████████████████████████████████████████████████████████████▍ | 16349/22715 [5:15:30<1:38:34, 1.08it/s][2025-04-29 08:43:35] (step=0016350) Train Loss: 5.5291, Train Steps/Sec: 1.07 + 72%|████████████████████████████████████████████████████████████▌ | 16374/22715 [5:15:53<1:38:08, 1.08it/s][2025-04-29 08:43:59] (step=0016375) Train Loss: 5.5355, Train Steps/Sec: 1.07 + 72%|████████████████████████████████████████████████████████████▋ | 16399/22715 [5:16:17<1:37:17, 1.08it/s][2025-04-29 08:44:22] (step=0016400) Train Loss: 5.5142, Train Steps/Sec: 1.07 + 72%|████████████████████████████████████████████████████████████▋ | 16424/22715 [5:16:40<1:39:58, 1.05it/s][2025-04-29 08:44:46] (step=0016425) Train Loss: 5.5458, Train Steps/Sec: 1.07 + 72%|████████████████████████████████████████████████████████████▊ | 16449/22715 [5:17:03<1:38:31, 1.06it/s][2025-04-29 08:45:09] (step=0016450) Train Loss: 5.4864, Train Steps/Sec: 1.07 + 73%|████████████████████████████████████████████████████████████▉ | 16474/22715 [5:17:27<1:37:23, 1.07it/s][2025-04-29 08:45:32] (step=0016475) Train Loss: 5.5509, Train Steps/Sec: 1.07 + 73%|█████████████████████████████████████████████████████████████ | 16499/22715 [5:17:50<1:36:46, 1.07it/s][2025-04-29 08:45:56] (step=0016500) Train Loss: 5.4937, Train Steps/Sec: 1.07 + 73%|█████████████████████████████████████████████████████████████ | 16524/22715 [5:18:13<1:35:38, 1.08it/s][2025-04-29 08:46:19] (step=0016525) Train Loss: 5.5346, Train Steps/Sec: 1.07 + 73%|█████████████████████████████████████████████████████████████▏ | 16549/22715 [5:18:37<1:35:07, 1.08it/s][2025-04-29 08:46:42] (step=0016550) Train Loss: 5.4860, Train Steps/Sec: 1.07 + 73%|█████████████████████████████████████████████████████████████▎ | 16574/22715 [5:19:00<1:34:36, 1.08it/s][2025-04-29 08:47:05] (step=0016575) Train Loss: 5.4857, Train Steps/Sec: 1.07 + 73%|█████████████████████████████████████████████████████████████▍ | 16599/22715 [5:19:23<1:34:16, 1.08it/s][2025-04-29 08:47:29] (step=0016600) Train Loss: 5.4978, Train Steps/Sec: 1.07 + 73%|█████████████████████████████████████████████████████████████▍ | 16624/22715 [5:19:47<1:36:41, 1.05it/s][2025-04-29 08:47:52] (step=0016625) Train Loss: 5.4841, Train Steps/Sec: 1.07 + 73%|█████████████████████████████████████████████████████████████▌ | 16649/22715 [5:20:10<1:34:58, 1.06it/s][2025-04-29 08:48:16] (step=0016650) Train Loss: 5.5010, Train Steps/Sec: 1.07 + 73%|█████████████████████████████████████████████████████████████▋ | 16674/22715 [5:20:33<1:34:02, 1.07it/s][2025-04-29 08:48:39] (step=0016675) Train Loss: 5.4658, Train Steps/Sec: 1.07 + 74%|█████████████████████████████████████████████████████████████▊ | 16699/22715 [5:20:57<1:33:25, 1.07it/s][2025-04-29 08:49:02] (step=0016700) Train Loss: 5.4964, Train Steps/Sec: 1.07 + 74%|█████████████████████████████████████████████████████████████▊ | 16724/22715 [5:21:20<1:32:37, 1.08it/s][2025-04-29 08:49:26] (step=0016725) Train Loss: 5.5127, Train Steps/Sec: 1.07 + 74%|█████████████████████████████████████████████████████████████▉ | 16749/22715 [5:21:43<1:32:16, 1.08it/s][2025-04-29 08:49:49] (step=0016750) Train Loss: 5.4751, Train Steps/Sec: 1.07 + 74%|██████████████████████████████████████████████████████████████ | 16774/22715 [5:22:07<1:31:22, 1.08it/s][2025-04-29 08:50:12] (step=0016775) Train Loss: 5.4721, Train Steps/Sec: 1.07 + 74%|██████████████████████████████████████████████████████████████ | 16799/22715 [5:22:30<1:31:11, 1.08it/s][2025-04-29 08:50:36] (step=0016800) Train Loss: 5.5270, Train Steps/Sec: 1.07 + 74%|██████████████████████████████████████████████████████████████▏ | 16824/22715 [5:22:53<1:33:33, 1.05it/s][2025-04-29 08:50:59] (step=0016825) Train Loss: 5.4932, Train Steps/Sec: 1.07 + 74%|██████████████████████████████████████████████████████████████▎ | 16849/22715 [5:23:17<1:31:55, 1.06it/s][2025-04-29 08:51:22] (step=0016850) Train Loss: 5.5325, Train Steps/Sec: 1.07 + 74%|██████████████████████████████████████████████████████████████▍ | 16874/22715 [5:23:40<1:31:00, 1.07it/s][2025-04-29 08:51:46] (step=0016875) Train Loss: 5.5165, Train Steps/Sec: 1.07 + 74%|██████████████████████████████████████████████████████████████▍ | 16899/22715 [5:24:03<1:30:07, 1.08it/s][2025-04-29 08:52:09] (step=0016900) Train Loss: 5.4917, Train Steps/Sec: 1.07 + 75%|██████████████████████████████████████████████████████████████▌ | 16924/22715 [5:24:27<1:29:28, 1.08it/s][2025-04-29 08:52:32] (step=0016925) Train Loss: 5.4518, Train Steps/Sec: 1.07 + 75%|██████████████████████████████████████████████████████████████▋ | 16949/22715 [5:24:50<1:29:03, 1.08it/s][2025-04-29 08:52:56] (step=0016950) Train Loss: 5.4897, Train Steps/Sec: 1.07 + 75%|██████████████████████████████████████████████████████████████▊ | 16974/22715 [5:25:13<1:28:41, 1.08it/s][2025-04-29 08:53:19] (step=0016975) Train Loss: 5.4738, Train Steps/Sec: 1.07 + 75%|██████████████████████████████████████████████████████████████▊ | 16999/22715 [5:25:37<1:27:59, 1.08it/s][2025-04-29 08:53:42] (step=0017000) Train Loss: 5.5400, Train Steps/Sec: 1.07 + 75%|██████████████████████████████████████████████████████████████▉ | 17024/22715 [5:26:00<1:30:27, 1.05it/s][2025-04-29 08:54:06] (step=0017025) Train Loss: 5.5009, Train Steps/Sec: 1.07 + 75%|███████████████████████████████████████████████████████████████ | 17049/22715 [5:26:23<1:28:41, 1.06it/s][2025-04-29 08:54:29] (step=0017050) Train Loss: 5.4947, Train Steps/Sec: 1.07 + 75%|███████████████████████████████████████████████████████████████▏ | 17074/22715 [5:26:47<1:27:53, 1.07it/s][2025-04-29 08:54:52] (step=0017075) Train Loss: 5.5011, Train Steps/Sec: 1.07 + 75%|███████████████████████████████████████████████████████████████▏ | 17099/22715 [5:27:10<1:26:58, 1.08it/s][2025-04-29 08:55:16] (step=0017100) Train Loss: 5.5044, Train Steps/Sec: 1.07 + 75%|███████████████████████████████████████████████████████████████▎ | 17124/22715 [5:27:33<1:26:35, 1.08it/s][2025-04-29 08:55:39] (step=0017125) Train Loss: 5.4795, Train Steps/Sec: 1.07 + 75%|███████████████████████████████████████████████████████████████▍ | 17149/22715 [5:27:57<1:25:47, 1.08it/s][2025-04-29 08:56:02] (step=0017150) Train Loss: 5.5091, Train Steps/Sec: 1.07 + 76%|███████████████████████████████████████████████████████████████▌ | 17174/22715 [5:28:20<1:25:35, 1.08it/s][2025-04-29 08:56:26] (step=0017175) Train Loss: 5.4738, Train Steps/Sec: 1.07 + 76%|███████████████████████████████████████████████████████████████▌ | 17199/22715 [5:28:43<1:24:56, 1.08it/s][2025-04-29 08:56:49] (step=0017200) Train Loss: 5.4797, Train Steps/Sec: 1.07 + 76%|███████████████████████████████████████████████████████████████▋ | 17224/22715 [5:29:07<1:27:18, 1.05it/s][2025-04-29 08:57:12] (step=0017225) Train Loss: 5.5094, Train Steps/Sec: 1.07 + 76%|███████████████████████████████████████████████████████████████▊ | 17249/22715 [5:29:30<1:25:29, 1.07it/s][2025-04-29 08:57:36] (step=0017250) Train Loss: 5.4896, Train Steps/Sec: 1.07 + 76%|███████████████████████████████████████████████████████████████▉ | 17274/22715 [5:29:53<1:24:51, 1.07it/s][2025-04-29 08:57:59] (step=0017275) Train Loss: 5.4999, Train Steps/Sec: 1.07 + 76%|███████████████████████████████████████████████████████████████▉ | 17299/22715 [5:30:17<1:24:00, 1.07it/s][2025-04-29 08:58:22] (step=0017300) Train Loss: 5.5186, Train Steps/Sec: 1.07 + 76%|████████████████████████████████████████████████████████████████ | 17324/22715 [5:30:40<1:23:38, 1.07it/s][2025-04-29 08:58:46] (step=0017325) Train Loss: 5.5137, Train Steps/Sec: 1.07 + 76%|████████████████████████████████████████████████████████████████▏ | 17349/22715 [5:31:03<1:23:07, 1.08it/s][2025-04-29 08:59:09] (step=0017350) Train Loss: 5.4977, Train Steps/Sec: 1.07 + 76%|████████████████████████████████████████████████████████████████▏ | 17374/22715 [5:31:27<1:22:33, 1.08it/s][2025-04-29 08:59:32] (step=0017375) Train Loss: 5.4916, Train Steps/Sec: 1.07 + 77%|████████████████████████████████████████████████████████████████▎ | 17399/22715 [5:31:50<1:21:47, 1.08it/s][2025-04-29 08:59:56] (step=0017400) Train Loss: 5.5100, Train Steps/Sec: 1.07 + 77%|████████████████████████████████████████████████████████████████▍ | 17424/22715 [5:32:13<1:24:01, 1.05it/s][2025-04-29 09:00:19] (step=0017425) Train Loss: 5.4911, Train Steps/Sec: 1.07 + 77%|████████████████████████████████████████████████████████████████▌ | 17449/22715 [5:32:37<1:22:32, 1.06it/s][2025-04-29 09:00:42] (step=0017450) Train Loss: 5.5129, Train Steps/Sec: 1.07 + 77%|████████████████████████████████████████████████████████████████▌ | 17474/22715 [5:33:00<1:21:51, 1.07it/s][2025-04-29 09:01:06] (step=0017475) Train Loss: 5.5060, Train Steps/Sec: 1.07 + 77%|████████████████████████████████████████████████████████████████▋ | 17499/22715 [5:33:23<1:20:53, 1.07it/s][2025-04-29 09:01:29] (step=0017500) Train Loss: 5.4596, Train Steps/Sec: 1.07 + 77%|████████████████████████████████████████████████████████████████▊ | 17524/22715 [5:33:47<1:20:23, 1.08it/s][2025-04-29 09:01:52] (step=0017525) Train Loss: 5.5039, Train Steps/Sec: 1.07 + 77%|████████████████████████████████████████████████████████████████▉ | 17549/22715 [5:34:10<1:19:46, 1.08it/s][2025-04-29 09:02:16] (step=0017550) Train Loss: 5.4754, Train Steps/Sec: 1.07 + 77%|████████████████████████████████████████████████████████████████▉ | 17574/22715 [5:34:33<1:19:18, 1.08it/s][2025-04-29 09:02:39] (step=0017575) Train Loss: 5.4865, Train Steps/Sec: 1.07 + 77%|█████████████████████████████████████████████████████████████████ | 17599/22715 [5:34:57<1:18:45, 1.08it/s][2025-04-29 09:03:02] (step=0017600) Train Loss: 5.4845, Train Steps/Sec: 1.07 + 78%|█████████████████████████████████████████████████████████████████▏ | 17624/22715 [5:35:20<1:20:46, 1.05it/s][2025-04-29 09:03:26] (step=0017625) Train Loss: 5.4899, Train Steps/Sec: 1.07 + 78%|█████████████████████████████████████████████████████████████████▎ | 17649/22715 [5:35:43<1:19:30, 1.06it/s][2025-04-29 09:03:49] (step=0017650) Train Loss: 5.5368, Train Steps/Sec: 1.07 + 78%|█████████████████████████████████████████████████████████████████▎ | 17674/22715 [5:36:07<1:18:28, 1.07it/s][2025-04-29 09:04:12] (step=0017675) Train Loss: 5.4715, Train Steps/Sec: 1.07 + 78%|█████████████████████████████████████████████████████████████████▍ | 17699/22715 [5:36:30<1:17:54, 1.07it/s][2025-04-29 09:04:36] (step=0017700) Train Loss: 5.4872, Train Steps/Sec: 1.07 + 78%|█████████████████████████████████████████████████████████████████▌ | 17724/22715 [5:36:53<1:17:17, 1.08it/s][2025-04-29 09:04:59] (step=0017725) Train Loss: 5.4449, Train Steps/Sec: 1.07 + 78%|█████████████████████████████████████████████████████████████████▋ | 17749/22715 [5:37:17<1:16:39, 1.08it/s][2025-04-29 09:05:22] (step=0017750) Train Loss: 5.4830, Train Steps/Sec: 1.07 + 78%|█████████████████████████████████████████████████████████████████▋ | 17774/22715 [5:37:40<1:16:19, 1.08it/s][2025-04-29 09:05:46] (step=0017775) Train Loss: 5.5085, Train Steps/Sec: 1.07 + 78%|█████████████████████████████████████████████████████████████████▊ | 17799/22715 [5:38:07<2:18:02, 1.68s/it][2025-04-29 09:06:13] (step=0017800) Train Loss: 5.5013, Train Steps/Sec: 0.92 + 78%|█████████████████████████████████████████████████████████████████▉ | 17824/22715 [5:38:36<2:18:31, 1.70s/it][2025-04-29 09:06:41] (step=0017825) Train Loss: 5.4543, Train Steps/Sec: 0.88 + 79%|██████████████████████████████████████████████████████████████████ | 17849/22715 [5:39:10<1:41:51, 1.26s/it][2025-04-29 09:07:15] (step=0017850) Train Loss: 5.4765, Train Steps/Sec: 0.73 + 79%|██████████████████████████████████████████████████████████████████ | 17874/22715 [5:39:37<1:16:35, 1.05it/s][2025-04-29 09:07:42] (step=0017875) Train Loss: 5.5138, Train Steps/Sec: 0.92 + 79%|██████████████████████████████████████████████████████████████████▏ | 17899/22715 [5:40:09<1:15:42, 1.06it/s][2025-04-29 09:08:14] (step=0017900) Train Loss: 5.5145, Train Steps/Sec: 0.78 + 79%|██████████████████████████████████████████████████████████████████▎ | 17924/22715 [5:40:32<1:13:58, 1.08it/s][2025-04-29 09:08:38] (step=0017925) Train Loss: 5.4551, Train Steps/Sec: 1.07 + 79%|██████████████████████████████████████████████████████████████████▍ | 17949/22715 [5:40:56<1:13:45, 1.08it/s][2025-04-29 09:09:01] (step=0017950) Train Loss: 5.4655, Train Steps/Sec: 1.07 + 79%|██████████████████████████████████████████████████████████████████▍ | 17974/22715 [5:41:19<1:13:16, 1.08it/s][2025-04-29 09:09:24] (step=0017975) Train Loss: 5.4681, Train Steps/Sec: 1.07 + 79%|██████████████████████████████████████████████████████████████████▌ | 17999/22715 [5:41:42<1:12:41, 1.08it/s][2025-04-29 09:09:48] (step=0018000) Train Loss: 5.4585, Train Steps/Sec: 1.07 +You are using a model of type blip-2 to instantiate a model of type instructblip. This is not supported for all configurations of models and can yield errors. +[2025-04-29 09:09:48] text_config is None. Initializing the text config with default values (`OPTConfig`). +model_name_or_path:/tmp/haozhezhao/model/blip2-flan-t5-xl +image_place_holder: +tokenizer length after expend 32101 +tokenizer length before expend 32100 +100%|████████████████████████████████████████████████████████████████████████████████████████████████| 5/5 [04:54<00:00, 58.83s/it] +[2025-04-29 09:15:54] Finish Eval in 18000 steps...█████████████████████████████████████████| 5/5 [04:53<00:00, 58.58s/it] +processing left-padding... +processing left-padding... +processing left-padding... +processing left-padding... +processing left-padding... +[2025-04-29 09:16:13] Saved checkpoint to checkpoint/X2I_700k_3mask_200k_LLava_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_filtered_objects_3mask_888_context/003-GPT-XL/checkpoints/0018000.pt +[2025-04-29 09:16:16] Removed old checkpoint: checkpoint/X2I_700k_3mask_200k_LLava_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_filtered_objects_3mask_888_context/003-GPT-XL/checkpoints/0016000.pt + 79%|██████████████████████████████████████████████████████████████████▋ | 18024/22715 [5:48:33<1:16:03, 1.03it/s][2025-04-29 09:16:39] (step=0018025) Train Loss: 5.4648, Train Steps/Sec: 0.06 + 79%|██████████████████████████████████████████████████████████████████▋ | 18049/22715 [5:48:57<1:12:53, 1.07it/s][2025-04-29 09:17:02] (step=0018050) Train Loss: 5.4403, Train Steps/Sec: 1.07 + 80%|██████████████████████████████████████████████████████████████████▊ | 18074/22715 [5:49:20<1:12:15, 1.07it/s][2025-04-29 09:17:26] (step=0018075) Train Loss: 5.5071, Train Steps/Sec: 1.06 + 80%|██████████████████████████████████████████████████████████████████▉ | 18099/22715 [5:49:43<1:11:43, 1.07it/s][2025-04-29 09:17:49] (step=0018100) Train Loss: 5.4691, Train Steps/Sec: 1.07 + 80%|███████████████████████████████████████████████████████████████████ | 18124/22715 [5:50:07<1:11:15, 1.07it/s][2025-04-29 09:18:12] (step=0018125) Train Loss: 5.4943, Train Steps/Sec: 1.07 + 80%|███████████████████████████████████████████████████████████████████ | 18149/22715 [5:50:30<1:10:35, 1.08it/s][2025-04-29 09:18:36] (step=0018150) Train Loss: 5.4989, Train Steps/Sec: 1.07 + 80%|███████████████████████████████████████████████████████████████████▏ | 18174/22715 [5:50:53<1:10:06, 1.08it/s][2025-04-29 09:18:59] (step=0018175) Train Loss: 5.5170, Train Steps/Sec: 1.07 + 80%|███████████████████████████████████████████████████████████████████▎ | 18199/22715 [5:51:17<1:08:59, 1.09it/s][2025-04-29 09:19:22] (step=0018200) Train Loss: 5.5043, Train Steps/Sec: 1.07 + 80%|███████████████████████████████████████████████████████████████████▍ | 18224/22715 [5:51:40<1:11:16, 1.05it/s][2025-04-29 09:19:46] (step=0018225) Train Loss: 5.4651, Train Steps/Sec: 1.07 + 80%|███████████████████████████████████████████████████████████████████▍ | 18249/22715 [5:52:03<1:09:57, 1.06it/s][2025-04-29 09:20:09] (step=0018250) Train Loss: 5.4732, Train Steps/Sec: 1.07 + 80%|███████████████████████████████████████████████████████████████████▌ | 18274/22715 [5:52:27<1:09:15, 1.07it/s][2025-04-29 09:20:32] (step=0018275) Train Loss: 5.4628, Train Steps/Sec: 1.07 + 81%|███████████████████████████████████████████████████████████████████▋ | 18299/22715 [5:52:50<1:08:22, 1.08it/s][2025-04-29 09:20:56] (step=0018300) Train Loss: 5.4716, Train Steps/Sec: 1.07 + 81%|███████████████████████████████████████████████████████████████████▊ | 18324/22715 [5:53:13<1:08:00, 1.08it/s][2025-04-29 09:21:19] (step=0018325) Train Loss: 5.4583, Train Steps/Sec: 1.07 + 81%|███████████████████████████████████████████████████████████████████▊ | 18349/22715 [5:53:37<1:07:23, 1.08it/s][2025-04-29 09:21:42] (step=0018350) Train Loss: 5.4479, Train Steps/Sec: 1.07 + 81%|███████████████████████████████████████████████████████████████████▉ | 18374/22715 [5:54:00<1:06:53, 1.08it/s][2025-04-29 09:22:05] (step=0018375) Train Loss: 5.4735, Train Steps/Sec: 1.07 + 81%|████████████████████████████████████████████████████████████████████ | 18399/22715 [5:54:23<1:06:26, 1.08it/s][2025-04-29 09:22:29] (step=0018400) Train Loss: 5.4485, Train Steps/Sec: 1.07 + 81%|████████████████████████████████████████████████████████████████████▏ | 18424/22715 [5:54:47<1:07:59, 1.05it/s][2025-04-29 09:22:52] (step=0018425) Train Loss: 5.4465, Train Steps/Sec: 1.07 + 81%|████████████████████████████████████████████████████████████████████▏ | 18449/22715 [5:55:10<1:06:53, 1.06it/s][2025-04-29 09:23:16] (step=0018450) Train Loss: 5.4573, Train Steps/Sec: 1.07 + 81%|████████████████████████████████████████████████████████████████████▎ | 18474/22715 [5:55:33<1:06:01, 1.07it/s][2025-04-29 09:23:39] (step=0018475) Train Loss: 5.4743, Train Steps/Sec: 1.07 + 81%|████████████████████████████████████████████████████████████████████▍ | 18499/22715 [5:55:57<1:05:36, 1.07it/s][2025-04-29 09:24:02] (step=0018500) Train Loss: 5.4736, Train Steps/Sec: 1.07 + 82%|████████████████████████████████████████████████████████████████████▌ | 18524/22715 [5:56:20<1:05:01, 1.07it/s][2025-04-29 09:24:26] (step=0018525) Train Loss: 5.4825, Train Steps/Sec: 1.07 + 82%|████████████████████████████████████████████████████████████████████▌ | 18549/22715 [5:56:43<1:04:23, 1.08it/s][2025-04-29 09:24:49] (step=0018550) Train Loss: 5.4365, Train Steps/Sec: 1.07 + 82%|████████████████████████████████████████████████████████████████████▋ | 18574/22715 [5:57:07<1:03:53, 1.08it/s][2025-04-29 09:25:12] (step=0018575) Train Loss: 5.4780, Train Steps/Sec: 1.07 + 82%|████████████████████████████████████████████████████████████████████▊ | 18599/22715 [5:57:30<1:03:31, 1.08it/s][2025-04-29 09:25:36] (step=0018600) Train Loss: 5.4439, Train Steps/Sec: 1.07 + 82%|████████████████████████████████████████████████████████████████████▊ | 18624/22715 [5:57:53<1:04:53, 1.05it/s][2025-04-29 09:25:59] (step=0018625) Train Loss: 5.4172, Train Steps/Sec: 1.07 + 82%|████████████████████████████████████████████████████████████████████▉ | 18649/22715 [5:58:17<1:03:36, 1.07it/s][2025-04-29 09:26:22] (step=0018650) Train Loss: 5.4675, Train Steps/Sec: 1.07 + 82%|█████████████████████████████████████████████████████████████████████ | 18674/22715 [5:58:40<1:03:06, 1.07it/s][2025-04-29 09:26:46] (step=0018675) Train Loss: 5.4421, Train Steps/Sec: 1.07 + 82%|█████████████████████████████████████████████████████████████████████▏ | 18699/22715 [5:59:04<1:02:34, 1.07it/s][2025-04-29 09:27:09] (step=0018700) Train Loss: 5.4884, Train Steps/Sec: 1.07 + 82%|█████████████████████████████████████████████████████████████████████▏ | 18724/22715 [5:59:27<1:01:56, 1.07it/s][2025-04-29 09:27:32] (step=0018725) Train Loss: 5.4299, Train Steps/Sec: 1.07 + 83%|█████████████████████████████████████████████████████████████████████▎ | 18749/22715 [5:59:50<1:01:09, 1.08it/s][2025-04-29 09:27:56] (step=0018750) Train Loss: 5.4688, Train Steps/Sec: 1.07 + 83%|█████████████████████████████████████████████████████████████████████▍ | 18774/22715 [6:00:14<1:01:29, 1.07it/s][2025-04-29 09:28:19] (step=0018775) Train Loss: 5.4478, Train Steps/Sec: 1.07 + 83%|█████████████████████████████████████████████████████████████████████▌ | 18799/22715 [6:00:37<1:00:29, 1.08it/s][2025-04-29 09:28:43] (step=0018800) Train Loss: 5.4857, Train Steps/Sec: 1.06 + 83%|█████████████████████████████████████████████████████████████████████▌ | 18824/22715 [6:01:01<1:01:53, 1.05it/s][2025-04-29 09:29:06] (step=0018825) Train Loss: 5.4486, Train Steps/Sec: 1.07 + 83%|█████████████████████████████████████████████████████████████████████▋ | 18849/22715 [6:01:24<1:00:30, 1.06it/s][2025-04-29 09:29:29] (step=0018850) Train Loss: 5.4134, Train Steps/Sec: 1.07 + 83%|███████████████████████████████████████████████████████████████████████▍ | 18874/22715 [6:01:47<59:51, 1.07it/s][2025-04-29 09:29:53] (step=0018875) Train Loss: 5.4713, Train Steps/Sec: 1.07 + 83%|███████████████████████████████████████████████████████████████████████▌ | 18899/22715 [6:02:11<59:08, 1.08it/s][2025-04-29 09:30:16] (step=0018900) Train Loss: 5.4576, Train Steps/Sec: 1.07 + 83%|███████████████████████████████████████████████████████████████████████▋ | 18924/22715 [6:02:34<58:37, 1.08it/s][2025-04-29 09:30:39] (step=0018925) Train Loss: 5.4791, Train Steps/Sec: 1.07 + 83%|███████████████████████████████████████████████████████████████████████▋ | 18949/22715 [6:02:57<58:02, 1.08it/s][2025-04-29 09:31:03] (step=0018950) Train Loss: 5.4137, Train Steps/Sec: 1.07 + 84%|███████████████████████████████████████████████████████████████████████▊ | 18974/22715 [6:03:21<57:45, 1.08it/s][2025-04-29 09:31:26] (step=0018975) Train Loss: 5.4501, Train Steps/Sec: 1.07 + 84%|███████████████████████████████████████████████████████████████████████▉ | 18999/22715 [6:03:44<57:16, 1.08it/s][2025-04-29 09:31:49] (step=0019000) Train Loss: 5.4553, Train Steps/Sec: 1.07 + 84%|████████████████████████████████████████████████████████████████████████ | 19024/22715 [6:04:07<58:34, 1.05it/s][2025-04-29 09:32:13] (step=0019025) Train Loss: 5.4304, Train Steps/Sec: 1.07 + 84%|████████████████████████████████████████████████████████████████████████ | 19049/22715 [6:04:31<57:28, 1.06it/s][2025-04-29 09:32:36] (step=0019050) Train Loss: 5.3881, Train Steps/Sec: 1.07 + 84%|████████████████████████████████████████████████████████████████████████▏ | 19074/22715 [6:04:54<56:53, 1.07it/s][2025-04-29 09:32:59] (step=0019075) Train Loss: 5.4838, Train Steps/Sec: 1.07 + 84%|████████████████████████████████████████████████████████████████████████▎ | 19099/22715 [6:05:17<56:08, 1.07it/s][2025-04-29 09:33:23] (step=0019100) Train Loss: 5.4403, Train Steps/Sec: 1.07 + 84%|████████████████████████████████████████████████████████████████████████▍ | 19124/22715 [6:05:41<55:48, 1.07it/s][2025-04-29 09:33:46] (step=0019125) Train Loss: 5.4572, Train Steps/Sec: 1.07 + 84%|████████████████████████████████████████████████████████████████████████▍ | 19149/22715 [6:06:04<55:05, 1.08it/s][2025-04-29 09:34:09] (step=0019150) Train Loss: 5.4832, Train Steps/Sec: 1.07 + 84%|████████████████████████████████████████████████████████████████████████▌ | 19174/22715 [6:06:27<54:46, 1.08it/s][2025-04-29 09:34:33] (step=0019175) Train Loss: 5.4566, Train Steps/Sec: 1.07 + 85%|████████████████████████████████████████████████████████████████████████▋ | 19199/22715 [6:06:51<54:17, 1.08it/s][2025-04-29 09:34:56] (step=0019200) Train Loss: 5.4452, Train Steps/Sec: 1.07 + 85%|████████████████████████████████████████████████████████████████████████▊ | 19224/22715 [6:07:14<55:39, 1.05it/s][2025-04-29 09:35:20] (step=0019225) Train Loss: 5.4772, Train Steps/Sec: 1.07 + 85%|████████████████████████████████████████████████████████████████████████▉ | 19249/22715 [6:07:37<54:20, 1.06it/s][2025-04-29 09:35:43] (step=0019250) Train Loss: 5.4370, Train Steps/Sec: 1.07 + 85%|████████████████████████████████████████████████████████████████████████▉ | 19274/22715 [6:08:01<53:29, 1.07it/s][2025-04-29 09:36:06] (step=0019275) Train Loss: 5.4299, Train Steps/Sec: 1.07 + 85%|█████████████████████████████████████████████████████████████████████████ | 19299/22715 [6:08:24<53:01, 1.07it/s][2025-04-29 09:36:30] (step=0019300) Train Loss: 5.4360, Train Steps/Sec: 1.07 + 85%|█████████████████████████████████████████████████████████████████████████▏ | 19324/22715 [6:08:47<52:26, 1.08it/s][2025-04-29 09:36:53] (step=0019325) Train Loss: 5.4435, Train Steps/Sec: 1.07 + 85%|█████████████████████████████████████████████████████████████████████████▎ | 19349/22715 [6:09:11<51:59, 1.08it/s][2025-04-29 09:37:16] (step=0019350) Train Loss: 5.4418, Train Steps/Sec: 1.07 + 85%|█████████████████████████████████████████████████████████████████████████▎ | 19374/22715 [6:09:34<51:37, 1.08it/s][2025-04-29 09:37:39] (step=0019375) Train Loss: 5.4819, Train Steps/Sec: 1.07 + 85%|█████████████████████████████████████████████████████████████████████████▍ | 19399/22715 [6:09:57<51:09, 1.08it/s][2025-04-29 09:38:03] (step=0019400) Train Loss: 5.4311, Train Steps/Sec: 1.07 + 86%|█████████████████████████████████████████████████████████████████████████▌ | 19424/22715 [6:10:21<52:24, 1.05it/s][2025-04-29 09:38:26] (step=0019425) Train Loss: 5.4420, Train Steps/Sec: 1.07 + 86%|█████████████████████████████████████████████████████████████████████████▋ | 19449/22715 [6:10:44<51:09, 1.06it/s][2025-04-29 09:38:50] (step=0019450) Train Loss: 5.3994, Train Steps/Sec: 1.07 + 86%|█████████████████████████████████████████████████████████████████████████▋ | 19474/22715 [6:11:07<50:23, 1.07it/s][2025-04-29 09:39:13] (step=0019475) Train Loss: 5.4396, Train Steps/Sec: 1.07 + 86%|█████████████████████████████████████████████████████████████████████████▊ | 19499/22715 [6:11:31<49:54, 1.07it/s][2025-04-29 09:39:36] (step=0019500) Train Loss: 5.3774, Train Steps/Sec: 1.07 + 86%|█████████████████████████████████████████████████████████████████████████▉ | 19524/22715 [6:11:54<49:21, 1.08it/s][2025-04-29 09:40:00] (step=0019525) Train Loss: 5.4365, Train Steps/Sec: 1.07 + 86%|██████████████████████████████████████████████████████████████████████████ | 19549/22715 [6:12:17<48:53, 1.08it/s][2025-04-29 09:40:23] (step=0019550) Train Loss: 5.4819, Train Steps/Sec: 1.07 + 86%|██████████████████████████████████████████████████████████████████████████ | 19574/22715 [6:12:41<48:18, 1.08it/s][2025-04-29 09:40:46] (step=0019575) Train Loss: 5.4261, Train Steps/Sec: 1.07 + 86%|██████████████████████████████████████████████████████████████████████████▏ | 19599/22715 [6:13:04<48:08, 1.08it/s][2025-04-29 09:41:10] (step=0019600) Train Loss: 5.4151, Train Steps/Sec: 1.07 + 86%|██████████████████████████████████████████████████████████████████████████▎ | 19624/22715 [6:13:27<48:57, 1.05it/s][2025-04-29 09:41:33] (step=0019625) Train Loss: 5.4233, Train Steps/Sec: 1.07 + 87%|██████████████████████████████████████████████████████████████████████████▍ | 19649/22715 [6:13:54<48:11, 1.06it/s][2025-04-29 09:42:00] (step=0019650) Train Loss: 5.4292, Train Steps/Sec: 0.93 + 87%|████████████████████████████████████████████████████████████████████████▊ | 19674/22715 [6:14:27<1:35:10, 1.88s/it][2025-04-29 09:42:33] (step=0019675) Train Loss: 5.4674, Train Steps/Sec: 0.75 + 87%|██████████████████████████████████████████████████████████████████████████▌ | 19699/22715 [6:14:55<48:23, 1.04it/s][2025-04-29 09:43:00] (step=0019700) Train Loss: 5.4311, Train Steps/Sec: 0.92 + 87%|██████████████████████████████████████████████████████████████████████████▋ | 19724/22715 [6:15:27<47:46, 1.04it/s][2025-04-29 09:43:32] (step=0019725) Train Loss: 5.4307, Train Steps/Sec: 0.78 + 87%|██████████████████████████████████████████████████████████████████████████▊ | 19749/22715 [6:15:50<45:47, 1.08it/s][2025-04-29 09:43:55] (step=0019750) Train Loss: 5.4188, Train Steps/Sec: 1.07 + 87%|██████████████████████████████████████████████████████████████████████████▊ | 19774/22715 [6:16:13<45:23, 1.08it/s][2025-04-29 09:44:19] (step=0019775) Train Loss: 5.4614, Train Steps/Sec: 1.07 + 87%|██████████████████████████████████████████████████████████████████████████▉ | 19799/22715 [6:16:37<44:52, 1.08it/s][2025-04-29 09:44:42] (step=0019800) Train Loss: 5.4207, Train Steps/Sec: 1.07 + 87%|█████████████████████████████████████████████████████████████████████████▎ | 19824/22715 [6:17:05<1:21:56, 1.70s/it][2025-04-29 09:45:11] (step=0019825) Train Loss: 5.4205, Train Steps/Sec: 0.88 + 87%|███████████████████████████████████████████████████████████████████████████▏ | 19849/22715 [6:17:28<44:52, 1.06it/s][2025-04-29 09:45:34] (step=0019850) Train Loss: 5.4217, Train Steps/Sec: 1.07 + 87%|███████████████████████████████████████████████████████████████████████████▏ | 19874/22715 [6:17:52<44:15, 1.07it/s][2025-04-29 09:45:57] (step=0019875) Train Loss: 5.4642, Train Steps/Sec: 1.07 + 88%|███████████████████████████████████████████████████████████████████████████▎ | 19899/22715 [6:18:15<43:45, 1.07it/s][2025-04-29 09:46:20] (step=0019900) Train Loss: 5.4332, Train Steps/Sec: 1.07 + 88%|███████████████████████████████████████████████████████████████████████████▍ | 19924/22715 [6:18:38<43:08, 1.08it/s][2025-04-29 09:46:44] (step=0019925) Train Loss: 5.4270, Train Steps/Sec: 1.07 + 88%|███████████████████████████████████████████████████████████████████████████▌ | 19949/22715 [6:19:02<42:44, 1.08it/s][2025-04-29 09:47:07] (step=0019950) Train Loss: 5.4052, Train Steps/Sec: 1.07 + 88%|███████████████████████████████████████████████████████████████████████████▌ | 19974/22715 [6:19:25<42:18, 1.08it/s][2025-04-29 09:47:30] (step=0019975) Train Loss: 5.4391, Train Steps/Sec: 1.07 + 88%|███████████████████████████████████████████████████████████████████████████▋ | 19999/22715 [6:19:48<41:48, 1.08it/s][2025-04-29 09:47:54] (step=0020000) Train Loss: 5.4363, Train Steps/Sec: 1.07 +You are using a model of type blip-2 to instantiate a model of type instructblip. This is not supported for all configurations of models and can yield errors. +[2025-04-29 09:47:54] text_config is None. Initializing the text config with default values (`OPTConfig`). +model_name_or_path:/tmp/haozhezhao/model/blip2-flan-t5-xl +image_place_holder: +tokenizer length after expend 32101 +tokenizer length before expend 32100 +100%|████████████████████████████████████████████████████████████████████████████████████████████████| 5/5 [04:53<00:00, 58.65s/it] +[2025-04-29 09:53:59] Finish Eval in 20000 steps...█████████████████████████████████████████| 5/5 [04:52<00:00, 58.42s/it] +processing left-padding... +processing left-padding... +processing left-padding... +processing left-padding... +processing left-padding... +[2025-04-29 09:54:19] Saved checkpoint to checkpoint/X2I_700k_3mask_200k_LLava_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_filtered_objects_3mask_888_context/003-GPT-XL/checkpoints/0020000.pt +[2025-04-29 09:54:21] Removed old checkpoint: checkpoint/X2I_700k_3mask_200k_LLava_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_filtered_objects_3mask_888_context/003-GPT-XL/checkpoints/0018000.pt + 88%|███████████████████████████████████████████████████████████████████████████▊ | 20024/22715 [6:26:39<43:44, 1.03it/s][2025-04-29 09:54:45] (step=0020025) Train Loss: 5.4268, Train Steps/Sec: 0.06 + 88%|███████████████████████████████████████████████████████████████████████████▉ | 20049/22715 [6:27:02<41:45, 1.06it/s][2025-04-29 09:55:08] (step=0020050) Train Loss: 5.4171, Train Steps/Sec: 1.07 + 88%|████████████████████████████████████████████████████████████████████████████ | 20074/22715 [6:27:26<41:11, 1.07it/s][2025-04-29 09:55:31] (step=0020075) Train Loss: 5.3948, Train Steps/Sec: 1.07 + 88%|████████████████████████████████████████████████████████████████████████████ | 20099/22715 [6:27:49<40:32, 1.08it/s][2025-04-29 09:55:54] (step=0020100) Train Loss: 5.3965, Train Steps/Sec: 1.07 + 89%|████████████████████████████████████████████████████████████████████████████▏ | 20124/22715 [6:28:12<40:06, 1.08it/s][2025-04-29 09:56:18] (step=0020125) Train Loss: 5.4149, Train Steps/Sec: 1.07 + 89%|████████████████████████████████████████████████████████████████████████████▎ | 20149/22715 [6:28:36<39:43, 1.08it/s][2025-04-29 09:56:41] (step=0020150) Train Loss: 5.4199, Train Steps/Sec: 1.07 + 89%|████████████████████████████████████████████████████████████████████████████▍ | 20174/22715 [6:28:59<39:09, 1.08it/s][2025-04-29 09:57:04] (step=0020175) Train Loss: 5.4380, Train Steps/Sec: 1.07 + 89%|████████████████████████████████████████████████████████████████████████████▍ | 20199/22715 [6:29:22<38:50, 1.08it/s][2025-04-29 09:57:28] (step=0020200) Train Loss: 5.3877, Train Steps/Sec: 1.07 + 89%|████████████████████████████████████████████████████████████████████████████▌ | 20224/22715 [6:29:46<39:29, 1.05it/s][2025-04-29 09:57:51] (step=0020225) Train Loss: 5.4076, Train Steps/Sec: 1.07 + 89%|████████████████████████████████████████████████████████████████████████████▋ | 20249/22715 [6:30:09<38:38, 1.06it/s][2025-04-29 09:58:14] (step=0020250) Train Loss: 5.4203, Train Steps/Sec: 1.07 + 89%|████████████████████████████████████████████████████████████████████████████▊ | 20274/22715 [6:30:32<38:02, 1.07it/s][2025-04-29 09:58:38] (step=0020275) Train Loss: 5.4541, Train Steps/Sec: 1.07 + 89%|████████████████████████████████████████████████████████████████████████████▊ | 20299/22715 [6:30:56<37:27, 1.07it/s][2025-04-29 09:59:01] (step=0020300) Train Loss: 5.4349, Train Steps/Sec: 1.07 + 89%|████████████████████████████████████████████████████████████████████████████▉ | 20324/22715 [6:31:19<37:00, 1.08it/s][2025-04-29 09:59:24] (step=0020325) Train Loss: 5.3799, Train Steps/Sec: 1.07 + 90%|█████████████████████████████████████████████████████████████████████████████ | 20349/22715 [6:31:43<37:00, 1.07it/s][2025-04-29 09:59:48] (step=0020350) Train Loss: 5.4510, Train Steps/Sec: 1.06 + 90%|█████████████████████████████████████████████████████████████████████████████▏ | 20374/22715 [6:32:06<36:10, 1.08it/s][2025-04-29 10:00:11] (step=0020375) Train Loss: 5.4777, Train Steps/Sec: 1.07 + 90%|█████████████████████████████████████████████████████████████████████████████▏ | 20399/22715 [6:32:29<35:47, 1.08it/s][2025-04-29 10:00:35] (step=0020400) Train Loss: 5.4012, Train Steps/Sec: 1.07 + 90%|█████████████████████████████████████████████████████████████████████████████▎ | 20424/22715 [6:32:53<36:20, 1.05it/s][2025-04-29 10:00:58] (step=0020425) Train Loss: 5.3798, Train Steps/Sec: 1.07 + 90%|█████████████████████████████████████████████████████████████████████████████▍ | 20449/22715 [6:33:16<35:35, 1.06it/s][2025-04-29 10:01:22] (step=0020450) Train Loss: 5.4311, Train Steps/Sec: 1.07 + 90%|█████████████████████████████████████████████████████████████████████████████▌ | 20474/22715 [6:33:39<34:52, 1.07it/s][2025-04-29 10:01:45] (step=0020475) Train Loss: 5.3861, Train Steps/Sec: 1.07 + 90%|█████████████████████████████████████████████████████████████████████████████▌ | 20499/22715 [6:34:03<34:25, 1.07it/s][2025-04-29 10:02:08] (step=0020500) Train Loss: 5.3770, Train Steps/Sec: 1.07 + 90%|█████████████████████████████████████████████████████████████████████████████▋ | 20524/22715 [6:34:26<33:55, 1.08it/s][2025-04-29 10:02:32] (step=0020525) Train Loss: 5.4088, Train Steps/Sec: 1.07 + 90%|█████████████████████████████████████████████████████████████████████████████▊ | 20549/22715 [6:34:49<33:26, 1.08it/s][2025-04-29 10:02:55] (step=0020550) Train Loss: 5.4109, Train Steps/Sec: 1.07 + 91%|█████████████████████████████████████████████████████████████████████████████▉ | 20574/22715 [6:35:13<33:02, 1.08it/s][2025-04-29 10:03:18] (step=0020575) Train Loss: 5.3746, Train Steps/Sec: 1.07 + 91%|█████████████████████████████████████████████████████████████████████████████▉ | 20599/22715 [6:35:36<32:32, 1.08it/s][2025-04-29 10:03:42] (step=0020600) Train Loss: 5.3871, Train Steps/Sec: 1.07 + 91%|██████████████████████████████████████████████████████████████████████████████ | 20624/22715 [6:35:59<33:09, 1.05it/s][2025-04-29 10:04:05] (step=0020625) Train Loss: 5.3330, Train Steps/Sec: 1.07 + 91%|██████████████████████████████████████████████████████████████████████████████▏ | 20649/22715 [6:36:23<32:13, 1.07it/s][2025-04-29 10:04:28] (step=0020650) Train Loss: 5.4121, Train Steps/Sec: 1.07 + 91%|██████████████████████████████████████████████████████████████████████████████▎ | 20674/22715 [6:36:46<31:48, 1.07it/s][2025-04-29 10:04:52] (step=0020675) Train Loss: 5.3987, Train Steps/Sec: 1.07 + 91%|██████████████████████████████████████████████████████████████████████████████▎ | 20699/22715 [6:37:09<31:17, 1.07it/s][2025-04-29 10:05:15] (step=0020700) Train Loss: 5.4512, Train Steps/Sec: 1.07 + 91%|██████████████████████████████████████████████████████████████████████████████▍ | 20724/22715 [6:37:33<30:45, 1.08it/s][2025-04-29 10:05:38] (step=0020725) Train Loss: 5.4253, Train Steps/Sec: 1.07 + 91%|██████████████████████████████████████████████████████████████████████████████▌ | 20749/22715 [6:37:56<30:24, 1.08it/s][2025-04-29 10:06:01] (step=0020750) Train Loss: 5.3962, Train Steps/Sec: 1.07 + 91%|██████████████████████████████████████████████████████████████████████████████▋ | 20774/22715 [6:38:19<29:54, 1.08it/s][2025-04-29 10:06:25] (step=0020775) Train Loss: 5.3879, Train Steps/Sec: 1.07 + 92%|██████████████████████████████████████████████████████████████████████████████▋ | 20799/22715 [6:38:43<29:31, 1.08it/s][2025-04-29 10:06:48] (step=0020800) Train Loss: 5.4017, Train Steps/Sec: 1.07 + 92%|██████████████████████████████████████████████████████████████████████████████▊ | 20824/22715 [6:39:06<30:02, 1.05it/s][2025-04-29 10:07:11] (step=0020825) Train Loss: 5.4864, Train Steps/Sec: 1.07 + 92%|██████████████████████████████████████████████████████████████████████████████▉ | 20849/22715 [6:39:29<29:14, 1.06it/s][2025-04-29 10:07:35] (step=0020850) Train Loss: 5.4045, Train Steps/Sec: 1.07 + 92%|███████████████████████████████████████████████████████████████████████████████ | 20874/22715 [6:39:53<28:42, 1.07it/s][2025-04-29 10:07:58] (step=0020875) Train Loss: 5.3460, Train Steps/Sec: 1.07 + 92%|███████████████████████████████████████████████████████████████████████████████ | 20899/22715 [6:40:16<28:11, 1.07it/s][2025-04-29 10:08:21] (step=0020900) Train Loss: 5.4009, Train Steps/Sec: 1.07 + 92%|███████████████████████████████████████████████████████████████████████████████▏ | 20924/22715 [6:40:39<27:50, 1.07it/s][2025-04-29 10:08:45] (step=0020925) Train Loss: 5.3862, Train Steps/Sec: 1.07 + 92%|███████████████████████████████████████████████████████████████████████████████▎ | 20949/22715 [6:41:03<27:20, 1.08it/s][2025-04-29 10:09:08] (step=0020950) Train Loss: 5.4224, Train Steps/Sec: 1.07 + 92%|███████████████████████████████████████████████████████████████████████████████▍ | 20974/22715 [6:41:26<26:51, 1.08it/s][2025-04-29 10:09:31] (step=0020975) Train Loss: 5.3661, Train Steps/Sec: 1.07 + 92%|███████████████████████████████████████████████████████████████████████████████▌ | 20999/22715 [6:41:49<26:27, 1.08it/s][2025-04-29 10:09:55] (step=0021000) Train Loss: 5.3816, Train Steps/Sec: 1.07 + 93%|███████████████████████████████████████████████████████████████████████████████▌ | 21024/22715 [6:42:13<26:51, 1.05it/s][2025-04-29 10:10:18] (step=0021025) Train Loss: 5.3846, Train Steps/Sec: 1.07 + 93%|███████████████████████████████████████████████████████████████████████████████▋ | 21049/22715 [6:42:36<26:06, 1.06it/s][2025-04-29 10:10:41] (step=0021050) Train Loss: 5.4074, Train Steps/Sec: 1.07 + 93%|███████████████████████████████████████████████████████████████████████████████▊ | 21074/22715 [6:42:59<25:31, 1.07it/s][2025-04-29 10:11:05] (step=0021075) Train Loss: 5.3808, Train Steps/Sec: 1.07 + 93%|███████████████████████████████████████████████████████████████████████████████▉ | 21099/22715 [6:43:23<25:08, 1.07it/s][2025-04-29 10:11:28] (step=0021100) Train Loss: 5.3822, Train Steps/Sec: 1.07 + 93%|███████████████████████████████████████████████████████████████████████████████▉ | 21124/22715 [6:43:46<24:35, 1.08it/s][2025-04-29 10:11:51] (step=0021125) Train Loss: 5.3822, Train Steps/Sec: 1.07 + 93%|████████████████████████████████████████████████████████████████████████████████ | 21149/22715 [6:44:09<24:09, 1.08it/s][2025-04-29 10:12:15] (step=0021150) Train Loss: 5.4112, Train Steps/Sec: 1.07 + 93%|████████████████████████████████████████████████████████████████████████████████▏ | 21174/22715 [6:44:32<23:45, 1.08it/s][2025-04-29 10:12:38] (step=0021175) Train Loss: 5.3690, Train Steps/Sec: 1.07 + 93%|████████████████████████████████████████████████████████████████████████████████▎ | 21199/22715 [6:44:56<23:21, 1.08it/s][2025-04-29 10:13:01] (step=0021200) Train Loss: 5.3775, Train Steps/Sec: 1.07 + 93%|████████████████████████████████████████████████████████████████████████████████▎ | 21224/22715 [6:45:19<23:37, 1.05it/s][2025-04-29 10:13:25] (step=0021225) Train Loss: 5.3547, Train Steps/Sec: 1.07 + 94%|████████████████████████████████████████████████████████████████████████████████▍ | 21249/22715 [6:45:42<22:55, 1.07it/s][2025-04-29 10:13:48] (step=0021250) Train Loss: 5.4142, Train Steps/Sec: 1.07 + 94%|████████████████████████████████████████████████████████████████████████████████▌ | 21274/22715 [6:46:06<22:30, 1.07it/s][2025-04-29 10:14:11] (step=0021275) Train Loss: 5.4065, Train Steps/Sec: 1.07 + 94%|████████████████████████████████████████████████████████████████████████████████▋ | 21299/22715 [6:46:29<21:57, 1.07it/s][2025-04-29 10:14:35] (step=0021300) Train Loss: 5.3926, Train Steps/Sec: 1.07 + 94%|████████████████████████████████████████████████████████████████████████████████▋ | 21324/22715 [6:46:53<21:33, 1.08it/s][2025-04-29 10:14:58] (step=0021325) Train Loss: 5.3736, Train Steps/Sec: 1.07 + 94%|████████████████████████████████████████████████████████████████████████████████▊ | 21349/22715 [6:47:16<21:07, 1.08it/s][2025-04-29 10:15:21] (step=0021350) Train Loss: 5.3554, Train Steps/Sec: 1.07 + 94%|████████████████████████████████████████████████████████████████████████████████▉ | 21374/22715 [6:47:39<20:38, 1.08it/s][2025-04-29 10:15:45] (step=0021375) Train Loss: 5.3536, Train Steps/Sec: 1.07 + 94%|█████████████████████████████████████████████████████████████████████████████████ | 21399/22715 [6:48:02<20:19, 1.08it/s][2025-04-29 10:16:08] (step=0021400) Train Loss: 5.4314, Train Steps/Sec: 1.07 + 94%|█████████████████████████████████████████████████████████████████████████████████ | 21424/22715 [6:48:26<20:28, 1.05it/s][2025-04-29 10:16:31] (step=0021425) Train Loss: 5.4026, Train Steps/Sec: 1.07 + 94%|█████████████████████████████████████████████████████████████████████████████████▏ | 21449/22715 [6:48:49<19:51, 1.06it/s][2025-04-29 10:16:55] (step=0021450) Train Loss: 5.3922, Train Steps/Sec: 1.07 + 95%|█████████████████████████████████████████████████████████████████████████████████▎ | 21474/22715 [6:49:16<19:37, 1.05it/s][2025-04-29 10:17:22] (step=0021475) Train Loss: 5.3846, Train Steps/Sec: 0.93 + 95%|█████████████████████████████████████████████████████████████████████████████████▍ | 21499/22715 [6:49:46<30:13, 1.49s/it][2025-04-29 10:17:51] (step=0021500) Train Loss: 5.4061, Train Steps/Sec: 0.85 + 95%|█████████████████████████████████████████████████████████████████████████████████▍ | 21524/22715 [6:50:12<29:10, 1.47s/it][2025-04-29 10:18:18] (step=0021525) Train Loss: 5.4147, Train Steps/Sec: 0.93 + 95%|█████████████████████████████████████████████████████████████████████████████████▌ | 21549/22715 [6:50:45<24:59, 1.29s/it][2025-04-29 10:18:50] (step=0021550) Train Loss: 5.4143, Train Steps/Sec: 0.78 + 95%|█████████████████████████████████████████████████████████████████████████████████▋ | 21574/22715 [6:51:08<17:38, 1.08it/s][2025-04-29 10:19:13] (step=0021575) Train Loss: 5.3790, Train Steps/Sec: 1.07 + 95%|█████████████████████████████████████████████████████████████████████████████████▊ | 21599/22715 [6:51:31<17:10, 1.08it/s][2025-04-29 10:19:37] (step=0021600) Train Loss: 5.3788, Train Steps/Sec: 1.07 + 95%|█████████████████████████████████████████████████████████████████████████████████▊ | 21624/22715 [6:51:55<17:19, 1.05it/s][2025-04-29 10:20:00] (step=0021625) Train Loss: 5.4109, Train Steps/Sec: 1.07 + 95%|█████████████████████████████████████████████████████████████████████████████████▉ | 21649/22715 [6:52:18<16:41, 1.06it/s][2025-04-29 10:20:23] (step=0021650) Train Loss: 5.3888, Train Steps/Sec: 1.07 + 95%|██████████████████████████████████████████████████████████████████████████████████ | 21674/22715 [6:52:41<16:15, 1.07it/s][2025-04-29 10:20:47] (step=0021675) Train Loss: 5.3486, Train Steps/Sec: 1.07 + 96%|██████████████████████████████████████████████████████████████████████████████████▏ | 21699/22715 [6:53:05<15:44, 1.08it/s][2025-04-29 10:21:10] (step=0021700) Train Loss: 5.3622, Train Steps/Sec: 1.07 + 96%|██████████████████████████████████████████████████████████████████████████████████▏ | 21724/22715 [6:53:28<15:20, 1.08it/s][2025-04-29 10:21:33] (step=0021725) Train Loss: 5.3279, Train Steps/Sec: 1.07 + 96%|██████████████████████████████████████████████████████████████████████████████████▎ | 21749/22715 [6:53:51<14:53, 1.08it/s][2025-04-29 10:21:57] (step=0021750) Train Loss: 5.3784, Train Steps/Sec: 1.07 + 96%|██████████████████████████████████████████████████████████████████████████████████▍ | 21774/22715 [6:54:14<14:31, 1.08it/s][2025-04-29 10:22:20] (step=0021775) Train Loss: 5.3495, Train Steps/Sec: 1.07 + 96%|██████████████████████████████████████████████████████████████████████████████████▌ | 21799/22715 [6:54:38<14:05, 1.08it/s][2025-04-29 10:22:43] (step=0021800) Train Loss: 5.3971, Train Steps/Sec: 1.07 + 96%|██████████████████████████████████████████████████████████████████████████████████▋ | 21824/22715 [6:55:06<24:59, 1.68s/it][2025-04-29 10:23:12] (step=0021825) Train Loss: 5.3605, Train Steps/Sec: 0.88 + 96%|██████████████████████████████████████████████████████████████████████████████████▋ | 21849/22715 [6:55:29<13:32, 1.07it/s][2025-04-29 10:23:35] (step=0021850) Train Loss: 5.3797, Train Steps/Sec: 1.07 + 96%|██████████████████████████████████████████████████████████████████████████████████▊ | 21874/22715 [6:55:53<13:07, 1.07it/s][2025-04-29 10:23:58] (step=0021875) Train Loss: 5.3925, Train Steps/Sec: 1.07 + 96%|██████████████████████████████████████████████████████████████████████████████████▉ | 21899/22715 [6:56:16<12:37, 1.08it/s][2025-04-29 10:24:22] (step=0021900) Train Loss: 5.3783, Train Steps/Sec: 1.07 + 97%|███████████████████████████████████████████████████████████████████████████████████ | 21924/22715 [6:56:39<12:14, 1.08it/s][2025-04-29 10:24:45] (step=0021925) Train Loss: 5.3551, Train Steps/Sec: 1.07 + 97%|███████████████████████████████████████████████████████████████████████████████████ | 21949/22715 [6:57:03<11:48, 1.08it/s][2025-04-29 10:25:08] (step=0021950) Train Loss: 5.3834, Train Steps/Sec: 1.07 + 97%|███████████████████████████████████████████████████████████████████████████████████▏ | 21974/22715 [6:57:26<11:25, 1.08it/s][2025-04-29 10:25:32] (step=0021975) Train Loss: 5.3904, Train Steps/Sec: 1.07 + 97%|███████████████████████████████████████████████████████████████████████████████████▎ | 21999/22715 [6:57:49<11:01, 1.08it/s][2025-04-29 10:25:55] (step=0022000) Train Loss: 5.3987, Train Steps/Sec: 1.07 +You are using a model of type blip-2 to instantiate a model of type instructblip. This is not supported for all configurations of models and can yield errors. +[2025-04-29 10:25:55] text_config is None. Initializing the text config with default values (`OPTConfig`). +model_name_or_path:/tmp/haozhezhao/model/blip2-flan-t5-xl +image_place_holder: +tokenizer length after expend 32101 +tokenizer length before expend 32100 +100%|████████████████████████████████████████████████████████████████████████████████████████████████| 5/5 [04:53<00:00, 58.62s/it] +[2025-04-29 10:32:00] Finish Eval in 22000 steps...█████████████████████████████████████████| 5/5 [04:52<00:00, 58.39s/it] +processing left-padding... +processing left-padding... +processing left-padding... +processing left-padding... +processing left-padding... +[2025-04-29 10:32:20] Saved checkpoint to checkpoint/X2I_700k_3mask_200k_LLava_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_filtered_objects_3mask_888_context/003-GPT-XL/checkpoints/0022000.pt +[2025-04-29 10:32:22] Removed old checkpoint: checkpoint/X2I_700k_3mask_200k_LLava_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_filtered_objects_3mask_888_context/003-GPT-XL/checkpoints/0020000.pt + 97%|███████████████████████████████████████████████████████████████████████████████████▍ | 22024/22715 [7:04:40<11:11, 1.03it/s][2025-04-29 10:32:46] (step=0022025) Train Loss: 5.3955, Train Steps/Sec: 0.06 + 97%|███████████████████████████████████████████████████████████████████████████████████▍ | 22049/22715 [7:05:03<10:24, 1.07it/s][2025-04-29 10:33:09] (step=0022050) Train Loss: 5.4084, Train Steps/Sec: 1.07 + 97%|███████████████████████████████████████████████████████████████████████████████████▌ | 22074/22715 [7:05:27<09:59, 1.07it/s][2025-04-29 10:33:32] (step=0022075) Train Loss: 5.3696, Train Steps/Sec: 1.07 + 97%|███████████████████████████████████████████████████████████████████████████████████▋ | 22099/22715 [7:05:50<09:32, 1.08it/s][2025-04-29 10:33:56] (step=0022100) Train Loss: 5.3669, Train Steps/Sec: 1.07 + 97%|███████████████████████████████████████████████████████████████████████████████████▊ | 22124/22715 [7:06:13<09:07, 1.08it/s][2025-04-29 10:34:19] (step=0022125) Train Loss: 5.3708, Train Steps/Sec: 1.07 + 98%|███████████████████████████████████████████████████████████████████████████████████▊ | 22149/22715 [7:06:37<08:43, 1.08it/s][2025-04-29 10:34:42] (step=0022150) Train Loss: 5.3647, Train Steps/Sec: 1.07 + 98%|███████████████████████████████████████████████████████████████████████████████████▉ | 22174/22715 [7:07:00<08:19, 1.08it/s][2025-04-29 10:35:05] (step=0022175) Train Loss: 5.3857, Train Steps/Sec: 1.07 + 98%|████████████████████████████████████████████████████████████████████████████████████ | 22199/22715 [7:07:23<07:57, 1.08it/s][2025-04-29 10:35:29] (step=0022200) Train Loss: 5.3871, Train Steps/Sec: 1.07 + 98%|████████████████████████████████████████████████████████████████████████████████████▏ | 22224/22715 [7:07:47<07:47, 1.05it/s][2025-04-29 10:35:52] (step=0022225) Train Loss: 5.3834, Train Steps/Sec: 1.07 + 98%|████████████████████████████████████████████████████████████████████████████████████▏ | 22249/22715 [7:08:10<07:17, 1.06it/s][2025-04-29 10:36:15] (step=0022250) Train Loss: 5.3720, Train Steps/Sec: 1.07 + 98%|████████████████████████████████████████████████████████████████████████████████████▎ | 22274/22715 [7:08:33<06:51, 1.07it/s][2025-04-29 10:36:39] (step=0022275) Train Loss: 5.3898, Train Steps/Sec: 1.08 + 98%|████████████████████████████████████████████████████████████████████████████████████▍ | 22299/22715 [7:08:56<06:27, 1.07it/s][2025-04-29 10:37:02] (step=0022300) Train Loss: 5.3591, Train Steps/Sec: 1.07 + 98%|████████████████████████████████████████████████████████████████████████████████████▌ | 22324/22715 [7:09:20<06:02, 1.08it/s][2025-04-29 10:37:25] (step=0022325) Train Loss: 5.3790, Train Steps/Sec: 1.07 + 98%|████████████████████████████████████████████████████████████████████████████████████▌ | 22349/22715 [7:09:43<05:40, 1.08it/s][2025-04-29 10:37:49] (step=0022350) Train Loss: 5.3982, Train Steps/Sec: 1.07 + 98%|████████████████████████████████████████████████████████████████████████████████████▋ | 22374/22715 [7:10:06<05:15, 1.08it/s][2025-04-29 10:38:12] (step=0022375) Train Loss: 5.3641, Train Steps/Sec: 1.07 + 99%|████████████████████████████████████████████████████████████████████████████████████▊ | 22399/22715 [7:10:30<04:51, 1.08it/s][2025-04-29 10:38:35] (step=0022400) Train Loss: 5.3342, Train Steps/Sec: 1.07 + 99%|████████████████████████████████████████████████████████████████████████████████████▉ | 22424/22715 [7:10:53<04:36, 1.05it/s][2025-04-29 10:38:59] (step=0022425) Train Loss: 5.3685, Train Steps/Sec: 1.07 + 99%|████████████████████████████████████████████████████████████████████████████████████▉ | 22449/22715 [7:11:16<04:10, 1.06it/s][2025-04-29 10:39:22] (step=0022450) Train Loss: 5.3928, Train Steps/Sec: 1.07 + 99%|█████████████████████████████████████████████████████████████████████████████████████ | 22474/22715 [7:11:40<03:45, 1.07it/s][2025-04-29 10:39:45] (step=0022475) Train Loss: 5.3594, Train Steps/Sec: 1.07 + 99%|█████████████████████████████████████████████████████████████████████████████████████▏| 22499/22715 [7:12:03<03:21, 1.07it/s][2025-04-29 10:40:09] (step=0022500) Train Loss: 5.3335, Train Steps/Sec: 1.07 + 99%|█████████████████████████████████████████████████████████████████████████████████████▎| 22524/22715 [7:12:27<02:57, 1.08it/s][2025-04-29 10:40:32] (step=0022525) Train Loss: 5.3560, Train Steps/Sec: 1.07 + 99%|█████████████████████████████████████████████████████████████████████████████████████▎| 22549/22715 [7:12:50<02:33, 1.08it/s][2025-04-29 10:40:55] (step=0022550) Train Loss: 5.3433, Train Steps/Sec: 1.07 + 99%|█████████████████████████████████████████████████████████████████████████████████████▍| 22574/22715 [7:13:13<02:09, 1.09it/s][2025-04-29 10:41:19] (step=0022575) Train Loss: 5.3243, Train Steps/Sec: 1.08 + 99%|█████████████████████████████████████████████████████████████████████████████████████▌| 22599/22715 [7:13:37<01:47, 1.08it/s][2025-04-29 10:41:42] (step=0022600) Train Loss: 5.3299, Train Steps/Sec: 1.07 +100%|█████████████████████████████████████████████████████████████████████████████████████▋| 22624/22715 [7:14:00<01:26, 1.05it/s][2025-04-29 10:42:05] (step=0022625) Train Loss: 5.3828, Train Steps/Sec: 1.07 +100%|█████████████████████████████████████████████████████████████████████████████████████▊| 22649/22715 [7:14:23<01:02, 1.06it/s][2025-04-29 10:42:29] (step=0022650) Train Loss: 5.3508, Train Steps/Sec: 1.07 +100%|█████████████████████████████████████████████████████████████████████████████████████▊| 22674/22715 [7:14:47<00:38, 1.07it/s][2025-04-29 10:42:52] (step=0022675) Train Loss: 5.3410, Train Steps/Sec: 1.07 +100%|█████████████████████████████████████████████████████████████████████████████████████▉| 22699/22715 [7:15:10<00:14, 1.07it/s][2025-04-29 10:43:15] (step=0022700) Train Loss: 5.3587, Train Steps/Sec: 1.07 +100%|██████████████████████████████████████████████████████████████████████████████████████| 22715/22715 [7:15:26<00:00, 1.15s/it] +[2025-04-29 10:43:30] Beginning epoch 1... + 0%| | 9/22715 [00:09<6:05:14, 1.04it/s][2025-04-29 10:43:41] (step=0022725) Train Loss: 5.3310, Train Steps/Sec: 0.97 + 0%|▏ | 34/22715 [00:33<5:51:45, 1.07it/s][2025-04-29 10:44:05] (step=0022750) Train Loss: 5.3631, Train Steps/Sec: 1.07 + 0%|▏ | 59/22715 [00:56<5:50:30, 1.08it/s][2025-04-29 10:44:28] (step=0022775) Train Loss: 5.3275, Train Steps/Sec: 1.07 + 0%|▎ | 84/22715 [01:20<5:47:25, 1.09it/s][2025-04-29 10:44:52] (step=0022800) Train Loss: 5.3543, Train Steps/Sec: 1.07 + 0%|▍ | 109/22715 [01:43<5:59:28, 1.05it/s][2025-04-29 10:45:15] (step=0022825) Train Loss: 5.3428, Train Steps/Sec: 1.07 + 1%|▌ | 134/22715 [02:06<5:54:39, 1.06it/s][2025-04-29 10:45:38] (step=0022850) Train Loss: 5.3593, Train Steps/Sec: 1.07 + 1%|▌ | 159/22715 [02:30<5:52:13, 1.07it/s][2025-04-29 10:46:02] (step=0022875) Train Loss: 5.3529, Train Steps/Sec: 1.07 + 1%|▋ | 184/22715 [02:53<5:51:02, 1.07it/s][2025-04-29 10:46:25] (step=0022900) Train Loss: 5.3342, Train Steps/Sec: 1.07 + 1%|▊ | 209/22715 [03:17<5:49:05, 1.07it/s][2025-04-29 10:46:48] (step=0022925) Train Loss: 5.3419, Train Steps/Sec: 1.07 + 1%|▉ | 234/22715 [03:40<5:47:54, 1.08it/s][2025-04-29 10:47:12] (step=0022950) Train Loss: 5.3800, Train Steps/Sec: 1.07 + 1%|█ | 259/22715 [04:03<5:47:21, 1.08it/s][2025-04-29 10:47:35] (step=0022975) Train Loss: 5.3806, Train Steps/Sec: 1.07 + 1%|█ | 284/22715 [04:27<5:46:16, 1.08it/s][2025-04-29 10:47:59] (step=0023000) Train Loss: 5.3415, Train Steps/Sec: 1.07 + 1%|█▏ | 309/22715 [04:50<5:56:17, 1.05it/s][2025-04-29 10:48:22] (step=0023025) Train Loss: 5.3212, Train Steps/Sec: 1.07 + 1%|█▎ | 334/22715 [05:13<5:51:58, 1.06it/s][2025-04-29 10:48:45] (step=0023050) Train Loss: 5.3440, Train Steps/Sec: 1.07 + 2%|█▍ | 359/22715 [05:37<5:49:34, 1.07it/s][2025-04-29 10:49:09] (step=0023075) Train Loss: 5.3573, Train Steps/Sec: 1.07 + 2%|█▍ | 384/22715 [06:00<5:49:03, 1.07it/s][2025-04-29 10:49:32] (step=0023100) Train Loss: 5.3567, Train Steps/Sec: 1.07 + 2%|█▌ | 409/22715 [06:24<5:46:07, 1.07it/s][2025-04-29 10:49:55] (step=0023125) Train Loss: 5.3897, Train Steps/Sec: 1.07 + 2%|█▋ | 434/22715 [06:47<5:45:20, 1.08it/s][2025-04-29 10:50:19] (step=0023150) Train Loss: 5.2901, Train Steps/Sec: 1.07 + 2%|█▊ | 459/22715 [07:10<5:43:53, 1.08it/s][2025-04-29 10:50:42] (step=0023175) Train Loss: 5.3289, Train Steps/Sec: 1.07 + 2%|█▉ | 484/22715 [07:34<5:43:45, 1.08it/s][2025-04-29 10:51:06] (step=0023200) Train Loss: 5.3431, Train Steps/Sec: 1.06 + 2%|█▉ | 509/22715 [07:57<5:51:12, 1.05it/s][2025-04-29 10:51:29] (step=0023225) Train Loss: 5.3050, Train Steps/Sec: 1.07 + 2%|██ | 534/22715 [08:21<5:48:11, 1.06it/s][2025-04-29 10:51:52] (step=0023250) Train Loss: 5.3344, Train Steps/Sec: 1.07 + 2%|██▏ | 559/22715 [08:44<5:45:37, 1.07it/s][2025-04-29 10:52:16] (step=0023275) Train Loss: 5.3365, Train Steps/Sec: 1.07 + 3%|██▎ | 584/22715 [09:12<5:58:51, 1.03it/s][2025-04-29 10:52:44] (step=0023300) Train Loss: 5.3361, Train Steps/Sec: 0.89 + 3%|██▎ | 609/22715 [09:45<14:19:04, 2.33s/it][2025-04-29 10:53:17] (step=0023325) Train Loss: 5.3467, Train Steps/Sec: 0.77 + 3%|██▍ | 634/22715 [10:17<8:38:40, 1.41s/it][2025-04-29 10:53:54] (step=0023350) Train Loss: 5.3220, Train Steps/Sec: 0.67 + 3%|██▌ | 659/22715 [10:51<10:54:38, 1.78s/it][2025-04-29 10:54:23] (step=0023375) Train Loss: 5.3772, Train Steps/Sec: 0.86 + 3%|██▋ | 684/22715 [11:15<5:40:19, 1.08it/s][2025-04-29 10:54:46] (step=0023400) Train Loss: 5.3291, Train Steps/Sec: 1.07 + 3%|██▋ | 709/22715 [11:38<5:49:31, 1.05it/s][2025-04-29 10:55:10] (step=0023425) Train Loss: 5.3216, Train Steps/Sec: 1.07 + 3%|██▊ | 734/22715 [12:01<5:45:07, 1.06it/s][2025-04-29 10:55:33] (step=0023450) Train Loss: 5.3298, Train Steps/Sec: 1.07 + 3%|██▉ | 759/22715 [12:25<5:44:30, 1.06it/s][2025-04-29 10:55:57] (step=0023475) Train Loss: 5.3357, Train Steps/Sec: 1.07 + 3%|███ | 784/22715 [12:48<5:41:00, 1.07it/s][2025-04-29 10:56:20] (step=0023500) Train Loss: 5.3131, Train Steps/Sec: 1.07 + 4%|███▏ | 809/22715 [13:11<5:40:06, 1.07it/s][2025-04-29 10:56:43] (step=0023525) Train Loss: 5.3172, Train Steps/Sec: 1.07 + 4%|███▏ | 834/22715 [13:35<5:38:21, 1.08it/s][2025-04-29 10:57:07] (step=0023550) Train Loss: 5.3142, Train Steps/Sec: 1.07 + 4%|███▎ | 859/22715 [13:58<5:37:50, 1.08it/s][2025-04-29 10:57:30] (step=0023575) Train Loss: 5.3333, Train Steps/Sec: 1.07 + 4%|███▍ | 884/22715 [14:22<5:37:06, 1.08it/s][2025-04-29 10:57:53] (step=0023600) Train Loss: 5.3597, Train Steps/Sec: 1.06 + 4%|███▌ | 909/22715 [14:45<5:46:58, 1.05it/s][2025-04-29 10:58:17] (step=0023625) Train Loss: 5.3118, Train Steps/Sec: 1.07 + 4%|███▌ | 934/22715 [15:08<5:41:43, 1.06it/s][2025-04-29 10:58:40] (step=0023650) Train Loss: 5.3399, Train Steps/Sec: 1.07 + 4%|███▋ | 959/22715 [15:32<5:39:56, 1.07it/s][2025-04-29 10:59:04] (step=0023675) Train Loss: 5.3148, Train Steps/Sec: 1.07 + 4%|███▊ | 984/22715 [15:55<5:38:11, 1.07it/s][2025-04-29 10:59:27] (step=0023700) Train Loss: 5.2945, Train Steps/Sec: 1.07 + 4%|███▊ | 1009/22715 [16:19<5:36:57, 1.07it/s][2025-04-29 10:59:50] (step=0023725) Train Loss: 5.3206, Train Steps/Sec: 1.07 + 5%|███▉ | 1034/22715 [16:42<5:35:36, 1.08it/s][2025-04-29 11:00:14] (step=0023750) Train Loss: 5.3269, Train Steps/Sec: 1.07 + 5%|████ | 1059/22715 [17:05<5:35:15, 1.08it/s][2025-04-29 11:00:37] (step=0023775) Train Loss: 5.2823, Train Steps/Sec: 1.07 + 5%|████▏ | 1084/22715 [17:29<5:34:40, 1.08it/s][2025-04-29 11:01:01] (step=0023800) Train Loss: 5.3338, Train Steps/Sec: 1.06 + 5%|████▏ | 1109/22715 [17:58<9:32:23, 1.59s/it][2025-04-29 11:01:30] (step=0023825) Train Loss: 5.3446, Train Steps/Sec: 0.85 + 5%|████▎ | 1134/22715 [18:22<5:39:42, 1.06it/s][2025-04-29 11:01:53] (step=0023850) Train Loss: 5.3359, Train Steps/Sec: 1.07 + 5%|████▍ | 1159/22715 [18:45<5:38:27, 1.06it/s][2025-04-29 11:02:17] (step=0023875) Train Loss: 5.3156, Train Steps/Sec: 1.06 + 5%|████▌ | 1184/22715 [19:09<5:36:09, 1.07it/s][2025-04-29 11:02:41] (step=0023900) Train Loss: 5.3365, Train Steps/Sec: 1.06 + 5%|████▋ | 1209/22715 [19:32<5:35:57, 1.07it/s][2025-04-29 11:03:04] (step=0023925) Train Loss: 5.3224, Train Steps/Sec: 1.06 + 5%|████▋ | 1234/22715 [19:56<5:36:04, 1.07it/s][2025-04-29 11:03:28] (step=0023950) Train Loss: 5.3429, Train Steps/Sec: 1.06 + 6%|████▊ | 1259/22715 [20:20<5:42:17, 1.04it/s][2025-04-29 11:03:52] (step=0023975) Train Loss: 5.3303, Train Steps/Sec: 1.06 + 6%|████▉ | 1284/22715 [20:43<5:31:29, 1.08it/s][2025-04-29 11:04:15] (step=0024000) Train Loss: 5.3276, Train Steps/Sec: 1.06 +You are using a model of type blip-2 to instantiate a model of type instructblip. This is not supported for all configurations of models and can yield errors. +[2025-04-29 11:04:15] text_config is None. Initializing the text config with default values (`OPTConfig`). +model_name_or_path:/tmp/haozhezhao/model/blip2-flan-t5-xl +image_place_holder: +tokenizer length after expend 32101 +tokenizer length before expend 32100 +100%|████████████████████████████████████████████████████████████████████████████████████████████████| 5/5 [04:54<00:00, 58.87s/it] +[2025-04-29 11:10:24] Finish Eval in 24000 steps...█████████████████████████████████████████| 5/5 [04:53<00:00, 58.68s/it] +processing left-padding... +processing left-padding... +processing left-padding... +processing left-padding... +processing left-padding... +[2025-04-29 11:10:44] Saved checkpoint to checkpoint/X2I_700k_3mask_200k_LLava_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_filtered_objects_3mask_888_context/003-GPT-XL/checkpoints/0024000.pt +[2025-04-29 11:10:46] Removed old checkpoint: checkpoint/X2I_700k_3mask_200k_LLava_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_filtered_objects_3mask_888_context/003-GPT-XL/checkpoints/0022000.pt + 6%|█████ | 1309/22715 [27:38<5:49:19, 1.02it/s][2025-04-29 11:11:10] (step=0024025) Train Loss: 5.3626, Train Steps/Sec: 0.06 + 6%|█████ | 1334/22715 [28:01<5:36:10, 1.06it/s][2025-04-29 11:11:33] (step=0024050) Train Loss: 5.3224, Train Steps/Sec: 1.07 + 6%|█████▏ | 1359/22715 [28:25<5:39:16, 1.05it/s][2025-04-29 11:11:57] (step=0024075) Train Loss: 5.3315, Train Steps/Sec: 1.07 + 6%|█████▎ | 1384/22715 [28:48<5:33:18, 1.07it/s][2025-04-29 11:12:20] (step=0024100) Train Loss: 5.3309, Train Steps/Sec: 1.06 + 6%|█████▍ | 1409/22715 [29:12<5:35:53, 1.06it/s][2025-04-29 11:12:44] (step=0024125) Train Loss: 5.3038, Train Steps/Sec: 1.06 + 6%|█████▍ | 1434/22715 [29:35<5:29:24, 1.08it/s][2025-04-29 11:13:07] (step=0024150) Train Loss: 5.3084, Train Steps/Sec: 1.07 + 6%|█████▌ | 1459/22715 [29:59<5:29:12, 1.08it/s][2025-04-29 11:13:30] (step=0024175) Train Loss: 5.3069, Train Steps/Sec: 1.07 + 7%|█████▋ | 1484/22715 [30:22<5:28:17, 1.08it/s][2025-04-29 11:13:54] (step=0024200) Train Loss: 5.2831, Train Steps/Sec: 1.07 + 7%|█████▊ | 1509/22715 [30:45<5:37:40, 1.05it/s][2025-04-29 11:14:17] (step=0024225) Train Loss: 5.2939, Train Steps/Sec: 1.07 + 7%|█████▉ | 1534/22715 [31:09<5:33:07, 1.06it/s][2025-04-29 11:14:41] (step=0024250) Train Loss: 5.3085, Train Steps/Sec: 1.07 + 7%|█████▉ | 1559/22715 [31:32<5:31:01, 1.07it/s][2025-04-29 11:15:04] (step=0024275) Train Loss: 5.3424, Train Steps/Sec: 1.07 + 7%|██████ | 1584/22715 [31:56<5:29:21, 1.07it/s][2025-04-29 11:15:27] (step=0024300) Train Loss: 5.3353, Train Steps/Sec: 1.07 + 7%|██████▏ | 1609/22715 [32:19<5:27:49, 1.07it/s][2025-04-29 11:15:51] (step=0024325) Train Loss: 5.3295, Train Steps/Sec: 1.07 + 7%|██████▎ | 1634/22715 [32:42<5:26:31, 1.08it/s][2025-04-29 11:16:14] (step=0024350) Train Loss: 5.2970, Train Steps/Sec: 1.07 + 7%|██████▎ | 1659/22715 [33:06<5:26:46, 1.07it/s][2025-04-29 11:16:38] (step=0024375) Train Loss: 5.3185, Train Steps/Sec: 1.07 + 7%|██████▍ | 1684/22715 [33:29<5:24:53, 1.08it/s][2025-04-29 11:17:01] (step=0024400) Train Loss: 5.3046, Train Steps/Sec: 1.06 + 8%|██████▌ | 1709/22715 [33:53<5:34:40, 1.05it/s][2025-04-29 11:17:25] (step=0024425) Train Loss: 5.2742, Train Steps/Sec: 1.07 + 8%|██████▋ | 1734/22715 [34:16<5:30:29, 1.06it/s][2025-04-29 11:17:48] (step=0024450) Train Loss: 5.3287, Train Steps/Sec: 1.06 + 8%|██████▋ | 1759/22715 [34:40<5:29:36, 1.06it/s][2025-04-29 11:18:12] (step=0024475) Train Loss: 5.3067, Train Steps/Sec: 1.06 + 8%|██████▊ | 1784/22715 [35:03<5:25:52, 1.07it/s][2025-04-29 11:18:35] (step=0024500) Train Loss: 5.3079, Train Steps/Sec: 1.07 + 8%|██████▉ | 1809/22715 [35:27<5:26:38, 1.07it/s][2025-04-29 11:18:59] (step=0024525) Train Loss: 5.2972, Train Steps/Sec: 1.05 + 8%|███████ | 1834/22715 [35:51<5:27:08, 1.06it/s][2025-04-29 11:19:22] (step=0024550) Train Loss: 5.3579, Train Steps/Sec: 1.06 + 8%|███████ | 1859/22715 [36:14<5:23:33, 1.07it/s][2025-04-29 11:19:46] (step=0024575) Train Loss: 5.3506, Train Steps/Sec: 1.06 + 8%|███████▏ | 1884/22715 [36:38<5:23:58, 1.07it/s][2025-04-29 11:20:10] (step=0024600) Train Loss: 5.3599, Train Steps/Sec: 1.06 + 8%|███████▎ | 1909/22715 [37:01<5:34:06, 1.04it/s][2025-04-29 11:20:33] (step=0024625) Train Loss: 5.3310, Train Steps/Sec: 1.05 + 9%|███████▍ | 1934/22715 [37:25<5:28:58, 1.05it/s][2025-04-29 11:20:57] (step=0024650) Train Loss: 5.2626, Train Steps/Sec: 1.06 + 9%|███████▌ | 1959/22715 [37:49<5:30:41, 1.05it/s][2025-04-29 11:21:20] (step=0024675) Train Loss: 5.3115, Train Steps/Sec: 1.06 + 9%|███████▌ | 1984/22715 [38:12<5:23:18, 1.07it/s][2025-04-29 11:21:44] (step=0024700) Train Loss: 5.3001, Train Steps/Sec: 1.07 + 9%|███████▋ | 2009/22715 [38:36<5:22:00, 1.07it/s][2025-04-29 11:22:07] (step=0024725) Train Loss: 5.3046, Train Steps/Sec: 1.07 + 9%|███████▊ | 2034/22715 [38:59<5:21:27, 1.07it/s][2025-04-29 11:22:31] (step=0024750) Train Loss: 5.3241, Train Steps/Sec: 1.06 + 9%|███████▉ | 2059/22715 [39:23<5:20:54, 1.07it/s][2025-04-29 11:22:54] (step=0024775) Train Loss: 5.3387, Train Steps/Sec: 1.06 + 9%|███████▉ | 2084/22715 [39:46<5:20:35, 1.07it/s][2025-04-29 11:23:18] (step=0024800) Train Loss: 5.3060, Train Steps/Sec: 1.06 + 9%|████████ | 2109/22715 [40:10<5:29:18, 1.04it/s][2025-04-29 11:23:42] (step=0024825) Train Loss: 5.3179, Train Steps/Sec: 1.07 + 9%|████████▏ | 2134/22715 [40:33<5:24:17, 1.06it/s][2025-04-29 11:24:05] (step=0024850) Train Loss: 5.2771, Train Steps/Sec: 1.07 + 10%|████████▎ | 2159/22715 [40:57<5:23:00, 1.06it/s][2025-04-29 11:24:29] (step=0024875) Train Loss: 5.2433, Train Steps/Sec: 1.06 + 10%|████████▎ | 2184/22715 [41:20<5:21:36, 1.06it/s][2025-04-29 11:24:52] (step=0024900) Train Loss: 5.3349, Train Steps/Sec: 1.06 + 10%|████████▍ | 2209/22715 [41:44<5:19:01, 1.07it/s][2025-04-29 11:25:15] (step=0024925) Train Loss: 5.3182, Train Steps/Sec: 1.07 + 10%|████████▌ | 2234/22715 [42:07<5:16:48, 1.08it/s][2025-04-29 11:25:39] (step=0024950) Train Loss: 5.3138, Train Steps/Sec: 1.07 + 10%|████████▋ | 2259/22715 [42:30<5:16:43, 1.08it/s][2025-04-29 11:26:02] (step=0024975) Train Loss: 5.2866, Train Steps/Sec: 1.07 + 10%|████████▋ | 2284/22715 [42:54<5:16:46, 1.07it/s][2025-04-29 11:26:26] (step=0025000) Train Loss: 5.2903, Train Steps/Sec: 1.06 + 10%|████████▊ | 2309/22715 [43:17<5:24:28, 1.05it/s][2025-04-29 11:26:49] (step=0025025) Train Loss: 5.2670, Train Steps/Sec: 1.07 + 10%|████████▉ | 2334/22715 [43:41<5:19:51, 1.06it/s][2025-04-29 11:27:13] (step=0025050) Train Loss: 5.2971, Train Steps/Sec: 1.07 + 10%|█████████ | 2359/22715 [44:04<5:17:58, 1.07it/s][2025-04-29 11:27:36] (step=0025075) Train Loss: 5.2747, Train Steps/Sec: 1.07 + 10%|█████████▏ | 2384/22715 [44:28<5:16:51, 1.07it/s][2025-04-29 11:27:59] (step=0025100) Train Loss: 5.2715, Train Steps/Sec: 1.07 + 11%|█████████▏ | 2409/22715 [44:55<5:58:55, 1.06s/it][2025-04-29 11:28:27] (step=0025125) Train Loss: 5.3430, Train Steps/Sec: 0.92 + 11%|█████████▏ | 2434/22715 [45:26<11:28:58, 2.04s/it][2025-04-29 11:28:57] (step=0025150) Train Loss: 5.2703, Train Steps/Sec: 0.81 + 11%|█████████▍ | 2459/22715 [45:57<7:30:40, 1.33s/it][2025-04-29 11:29:32] (step=0025175) Train Loss: 5.2976, Train Steps/Sec: 0.72 + 11%|█████████▌ | 2484/22715 [46:24<5:11:52, 1.08it/s][2025-04-29 11:30:01] (step=0025200) Train Loss: 5.2657, Train Steps/Sec: 1.07 + 11%|█████████▌ | 2509/22715 [46:52<5:19:38, 1.05it/s][2025-04-29 11:30:24] (step=0025225) Train Loss: 5.3184, Train Steps/Sec: 1.07 + 11%|█████████▋ | 2534/22715 [47:16<5:17:22, 1.06it/s][2025-04-29 11:30:48] (step=0025250) Train Loss: 5.3179, Train Steps/Sec: 1.07 + 11%|█████████▊ | 2559/22715 [47:39<5:14:38, 1.07it/s][2025-04-29 11:31:11] (step=0025275) Train Loss: 5.2808, Train Steps/Sec: 1.07 + 11%|█████████▉ | 2584/22715 [48:03<5:14:23, 1.07it/s][2025-04-29 11:31:34] (step=0025300) Train Loss: 5.3077, Train Steps/Sec: 1.07 + 11%|█████████▉ | 2609/22715 [48:26<5:18:10, 1.05it/s][2025-04-29 11:31:58] (step=0025325) Train Loss: 5.3034, Train Steps/Sec: 1.06 + 12%|██████████ | 2634/22715 [48:50<5:17:28, 1.05it/s][2025-04-29 11:32:22] (step=0025350) Train Loss: 5.3178, Train Steps/Sec: 1.06 + 12%|██████████▏ | 2659/22715 [49:13<5:10:49, 1.08it/s][2025-04-29 11:32:45] (step=0025375) Train Loss: 5.2518, Train Steps/Sec: 1.07 + 12%|██████████▎ | 2684/22715 [49:37<5:09:29, 1.08it/s][2025-04-29 11:33:09] (step=0025400) Train Loss: 5.2760, Train Steps/Sec: 1.06 + 12%|██████████▍ | 2709/22715 [50:00<5:17:49, 1.05it/s][2025-04-29 11:33:32] (step=0025425) Train Loss: 5.2901, Train Steps/Sec: 1.07 + 12%|██████████▍ | 2734/22715 [50:24<5:16:38, 1.05it/s][2025-04-29 11:33:56] (step=0025450) Train Loss: 5.2834, Train Steps/Sec: 1.06 + 12%|██████████▌ | 2759/22715 [50:47<5:13:10, 1.06it/s][2025-04-29 11:34:19] (step=0025475) Train Loss: 5.2922, Train Steps/Sec: 1.07 + 12%|██████████▋ | 2784/22715 [51:11<5:10:06, 1.07it/s][2025-04-29 11:34:42] (step=0025500) Train Loss: 5.2564, Train Steps/Sec: 1.07 + 12%|██████████▊ | 2809/22715 [51:34<5:09:32, 1.07it/s][2025-04-29 11:35:06] (step=0025525) Train Loss: 5.2840, Train Steps/Sec: 1.07 + 12%|██████████▊ | 2834/22715 [51:57<5:06:53, 1.08it/s][2025-04-29 11:35:29] (step=0025550) Train Loss: 5.3201, Train Steps/Sec: 1.07 + 13%|██████████▉ | 2859/22715 [52:21<5:06:34, 1.08it/s][2025-04-29 11:35:53] (step=0025575) Train Loss: 5.3377, Train Steps/Sec: 1.07 + 13%|███████████ | 2884/22715 [52:44<5:06:23, 1.08it/s][2025-04-29 11:36:16] (step=0025600) Train Loss: 5.2707, Train Steps/Sec: 1.07 + 13%|███████████▏ | 2909/22715 [53:08<5:14:57, 1.05it/s][2025-04-29 11:36:39] (step=0025625) Train Loss: 5.2530, Train Steps/Sec: 1.07 + 13%|███████████▏ | 2934/22715 [53:31<5:11:12, 1.06it/s][2025-04-29 11:37:03] (step=0025650) Train Loss: 5.2929, Train Steps/Sec: 1.07 + 13%|███████████▎ | 2959/22715 [53:55<5:10:46, 1.06it/s][2025-04-29 11:37:26] (step=0025675) Train Loss: 5.2941, Train Steps/Sec: 1.06 + 13%|███████████▍ | 2984/22715 [54:18<5:07:17, 1.07it/s][2025-04-29 11:37:50] (step=0025700) Train Loss: 5.3233, Train Steps/Sec: 1.07 + 13%|███████████▌ | 3009/22715 [54:42<5:13:15, 1.05it/s][2025-04-29 11:38:13] (step=0025725) Train Loss: 5.2903, Train Steps/Sec: 1.06 + 13%|███████████▌ | 3034/22715 [55:05<5:05:09, 1.07it/s][2025-04-29 11:38:37] (step=0025750) Train Loss: 5.3044, Train Steps/Sec: 1.07 + 13%|███████████▋ | 3059/22715 [55:29<5:04:35, 1.08it/s][2025-04-29 11:39:00] (step=0025775) Train Loss: 5.2795, Train Steps/Sec: 1.07 + 14%|███████████▊ | 3084/22715 [55:52<5:04:21, 1.08it/s][2025-04-29 11:39:24] (step=0025800) Train Loss: 5.3064, Train Steps/Sec: 1.06 + 14%|███████████▉ | 3109/22715 [56:21<9:58:04, 1.83s/it][2025-04-29 11:39:53] (step=0025825) Train Loss: 5.2581, Train Steps/Sec: 0.85 + 14%|████████████ | 3134/22715 [56:45<5:08:06, 1.06it/s][2025-04-29 11:40:17] (step=0025850) Train Loss: 5.2914, Train Steps/Sec: 1.07 + 14%|████████████ | 3159/22715 [57:08<5:05:25, 1.07it/s][2025-04-29 11:40:40] (step=0025875) Train Loss: 5.3002, Train Steps/Sec: 1.07 + 14%|████████████▏ | 3184/22715 [57:32<5:04:16, 1.07it/s][2025-04-29 11:41:03] (step=0025900) Train Loss: 5.2849, Train Steps/Sec: 1.07 + 14%|████████████▎ | 3209/22715 [57:55<5:02:24, 1.08it/s][2025-04-29 11:41:27] (step=0025925) Train Loss: 5.2949, Train Steps/Sec: 1.07 + 14%|████████████▍ | 3234/22715 [58:18<5:01:07, 1.08it/s][2025-04-29 11:41:50] (step=0025950) Train Loss: 5.2418, Train Steps/Sec: 1.07 + 14%|████████████▍ | 3259/22715 [58:42<5:00:48, 1.08it/s][2025-04-29 11:42:14] (step=0025975) Train Loss: 5.3101, Train Steps/Sec: 1.07 + 14%|████████████▌ | 3284/22715 [59:05<5:00:24, 1.08it/s][2025-04-29 11:42:37] (step=0026000) Train Loss: 5.2828, Train Steps/Sec: 1.06 +You are using a model of type blip-2 to instantiate a model of type instructblip. This is not supported for all configurations of models and can yield errors. +[2025-04-29 11:42:37] text_config is None. Initializing the text config with default values (`OPTConfig`). +model_name_or_path:/tmp/haozhezhao/model/blip2-flan-t5-xl +image_place_holder: +tokenizer length after expend 32101 +tokenizer length before expend 32100 +100%|████████████████████████████████████████████████████████████████████████████████████████████████| 5/5 [04:55<00:00, 59.04s/it] +[2025-04-29 11:48:44] Finish Eval in 26000 steps...█████████████████████████████████████████| 5/5 [04:54<00:00, 58.78s/it] +processing left-padding... +processing left-padding... +processing left-padding... +processing left-padding... +processing left-padding... +[2025-04-29 11:49:04] Saved checkpoint to checkpoint/X2I_700k_3mask_200k_LLava_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_filtered_objects_3mask_888_context/003-GPT-XL/checkpoints/0026000.pt +[2025-04-29 11:49:07] Removed old checkpoint: checkpoint/X2I_700k_3mask_200k_LLava_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_filtered_objects_3mask_888_context/003-GPT-XL/checkpoints/0024000.pt + 15%|████████████▍ | 3309/22715 [1:05:58<5:12:58, 1.03it/s][2025-04-29 11:49:30] (step=0026025) Train Loss: 5.3020, Train Steps/Sec: 0.06 + 15%|████████████▍ | 3334/22715 [1:06:21<5:03:07, 1.07it/s][2025-04-29 11:49:53] (step=0026050) Train Loss: 5.2898, Train Steps/Sec: 1.07 + 15%|████████████▌ | 3359/22715 [1:06:45<5:02:20, 1.07it/s][2025-04-29 11:50:17] (step=0026075) Train Loss: 5.2770, Train Steps/Sec: 1.07 + 15%|████████████▋ | 3384/22715 [1:07:08<5:00:51, 1.07it/s][2025-04-29 11:50:40] (step=0026100) Train Loss: 5.2652, Train Steps/Sec: 1.07 + 15%|████████████▊ | 3409/22715 [1:07:32<5:01:36, 1.07it/s][2025-04-29 11:51:04] (step=0026125) Train Loss: 5.2955, Train Steps/Sec: 1.06 + 15%|████████████▊ | 3434/22715 [1:07:55<4:58:46, 1.08it/s][2025-04-29 11:51:27] (step=0026150) Train Loss: 5.3228, Train Steps/Sec: 1.07 + 15%|████████████▉ | 3459/22715 [1:08:19<5:00:53, 1.07it/s][2025-04-29 11:51:50] (step=0026175) Train Loss: 5.2817, Train Steps/Sec: 1.06 + 15%|█████████████ | 3484/22715 [1:08:42<4:56:27, 1.08it/s][2025-04-29 11:52:14] (step=0026200) Train Loss: 5.2573, Train Steps/Sec: 1.07 + 15%|█████████████▏ | 3509/22715 [1:09:05<5:05:42, 1.05it/s][2025-04-29 11:52:37] (step=0026225) Train Loss: 5.2813, Train Steps/Sec: 1.07 + 16%|█████████████▏ | 3534/22715 [1:09:29<5:02:05, 1.06it/s][2025-04-29 11:53:01] (step=0026250) Train Loss: 5.2766, Train Steps/Sec: 1.07 + 16%|█████████████▎ | 3559/22715 [1:09:52<5:00:53, 1.06it/s][2025-04-29 11:53:24] (step=0026275) Train Loss: 5.3456, Train Steps/Sec: 1.06 + 16%|█████████████▍ | 3584/22715 [1:10:16<5:01:43, 1.06it/s][2025-04-29 11:53:48] (step=0026300) Train Loss: 5.2838, Train Steps/Sec: 1.06 + 16%|█████████████▌ | 3609/22715 [1:10:39<4:57:04, 1.07it/s][2025-04-29 11:54:11] (step=0026325) Train Loss: 5.3173, Train Steps/Sec: 1.07 + 16%|█████████████▌ | 3634/22715 [1:11:03<4:55:37, 1.08it/s][2025-04-29 11:54:35] (step=0026350) Train Loss: 5.2516, Train Steps/Sec: 1.07 + 16%|█████████████▋ | 3659/22715 [1:11:26<4:55:30, 1.07it/s][2025-04-29 11:54:58] (step=0026375) Train Loss: 5.2762, Train Steps/Sec: 1.07 + 16%|█████████████▊ | 3684/22715 [1:11:50<4:55:12, 1.07it/s][2025-04-29 11:55:21] (step=0026400) Train Loss: 5.2340, Train Steps/Sec: 1.06 + 16%|█████████████▉ | 3709/22715 [1:12:13<5:00:13, 1.06it/s][2025-04-29 11:55:45] (step=0026425) Train Loss: 5.2510, Train Steps/Sec: 1.07 + 16%|█████████████▉ | 3734/22715 [1:12:36<4:58:06, 1.06it/s][2025-04-29 11:56:08] (step=0026450) Train Loss: 5.2929, Train Steps/Sec: 1.07 + 17%|██████████████ | 3759/22715 [1:13:00<4:58:00, 1.06it/s][2025-04-29 11:56:32] (step=0026475) Train Loss: 5.2657, Train Steps/Sec: 1.06 + 17%|██████████████▏ | 3784/22715 [1:13:23<4:54:12, 1.07it/s][2025-04-29 11:56:55] (step=0026500) Train Loss: 5.2866, Train Steps/Sec: 1.07 + 17%|██████████████▎ | 3809/22715 [1:13:47<4:53:15, 1.07it/s][2025-04-29 11:57:19] (step=0026525) Train Loss: 5.2708, Train Steps/Sec: 1.06 + 17%|██████████████▎ | 3834/22715 [1:14:10<4:53:09, 1.07it/s][2025-04-29 11:57:42] (step=0026550) Train Loss: 5.2754, Train Steps/Sec: 1.07 + 17%|██████████████▍ | 3859/22715 [1:14:34<4:51:14, 1.08it/s][2025-04-29 11:58:05] (step=0026575) Train Loss: 5.2770, Train Steps/Sec: 1.07 + 17%|██████████████▌ | 3884/22715 [1:14:57<4:51:08, 1.08it/s][2025-04-29 11:58:29] (step=0026600) Train Loss: 5.2246, Train Steps/Sec: 1.06 + 17%|██████████████▋ | 3909/22715 [1:15:21<4:59:12, 1.05it/s][2025-04-29 11:58:52] (step=0026625) Train Loss: 5.2626, Train Steps/Sec: 1.07 + 17%|██████████████▋ | 3934/22715 [1:15:44<4:54:53, 1.06it/s][2025-04-29 11:59:16] (step=0026650) Train Loss: 5.2913, Train Steps/Sec: 1.07 + 17%|██████████████▊ | 3959/22715 [1:16:07<4:53:33, 1.06it/s][2025-04-29 11:59:39] (step=0026675) Train Loss: 5.3242, Train Steps/Sec: 1.07 + 18%|██████████████▉ | 3984/22715 [1:16:31<4:57:35, 1.05it/s][2025-04-29 12:00:03] (step=0026700) Train Loss: 5.2605, Train Steps/Sec: 1.06 + 18%|███████████████ | 4009/22715 [1:16:54<4:47:54, 1.08it/s][2025-04-29 12:00:26] (step=0026725) Train Loss: 5.2805, Train Steps/Sec: 1.07 + 18%|███████████████ | 4034/22715 [1:17:18<4:49:01, 1.08it/s][2025-04-29 12:00:49] (step=0026750) Train Loss: 5.2565, Train Steps/Sec: 1.07 + 18%|███████████████▏ | 4059/22715 [1:17:41<4:48:35, 1.08it/s][2025-04-29 12:01:13] (step=0026775) Train Loss: 5.2697, Train Steps/Sec: 1.07 + 18%|███████████████▎ | 4084/22715 [1:18:04<4:47:30, 1.08it/s][2025-04-29 12:01:36] (step=0026800) Train Loss: 5.2607, Train Steps/Sec: 1.07 + 18%|███████████████▍ | 4109/22715 [1:18:28<4:55:28, 1.05it/s][2025-04-29 12:02:00] (step=0026825) Train Loss: 5.2546, Train Steps/Sec: 1.07 + 18%|███████████████▍ | 4134/22715 [1:18:51<4:51:50, 1.06it/s][2025-04-29 12:02:23] (step=0026850) Train Loss: 5.2884, Train Steps/Sec: 1.07 + 18%|███████████████▌ | 4159/22715 [1:19:14<4:49:42, 1.07it/s][2025-04-29 12:02:46] (step=0026875) Train Loss: 5.2572, Train Steps/Sec: 1.07 + 18%|███████████████▋ | 4184/22715 [1:19:38<4:47:30, 1.07it/s][2025-04-29 12:03:10] (step=0026900) Train Loss: 5.3019, Train Steps/Sec: 1.07 + 19%|███████████████▊ | 4209/22715 [1:20:01<4:46:56, 1.07it/s][2025-04-29 12:03:33] (step=0026925) Train Loss: 5.2559, Train Steps/Sec: 1.07 + 19%|███████████████▊ | 4234/22715 [1:20:28<6:09:14, 1.20s/it][2025-04-29 12:04:00] (step=0026950) Train Loss: 5.2316, Train Steps/Sec: 0.92 + 19%|███████████████▉ | 4259/22715 [1:20:59<8:43:23, 1.70s/it][2025-04-29 12:04:31] (step=0026975) Train Loss: 5.2600, Train Steps/Sec: 0.81 + 19%|███████████████▊ | 4284/22715 [1:21:30<10:32:52, 2.06s/it][2025-04-29 12:05:02] (step=0027000) Train Loss: 5.2509, Train Steps/Sec: 0.80 + 19%|████████████████ | 4309/22715 [1:21:57<4:56:00, 1.04it/s][2025-04-29 12:05:29] (step=0027025) Train Loss: 5.2088, Train Steps/Sec: 0.92 + 19%|████████████████▏ | 4334/22715 [1:22:26<4:49:24, 1.06it/s][2025-04-29 12:05:57] (step=0027050) Train Loss: 5.2285, Train Steps/Sec: 0.89 + 19%|████████████████▎ | 4359/22715 [1:22:49<4:46:28, 1.07it/s][2025-04-29 12:06:21] (step=0027075) Train Loss: 5.2704, Train Steps/Sec: 1.07 + 19%|████████████████▍ | 4384/22715 [1:23:12<4:45:17, 1.07it/s][2025-04-29 12:06:44] (step=0027100) Train Loss: 5.2582, Train Steps/Sec: 1.07 + 19%|████████████████▍ | 4409/22715 [1:23:36<4:44:31, 1.07it/s][2025-04-29 12:07:07] (step=0027125) Train Loss: 5.2979, Train Steps/Sec: 1.07 + 20%|████████████████▌ | 4434/22715 [1:23:59<4:42:42, 1.08it/s][2025-04-29 12:07:31] (step=0027150) Train Loss: 5.2284, Train Steps/Sec: 1.07 + 20%|████████████████▋ | 4459/22715 [1:24:22<4:41:58, 1.08it/s][2025-04-29 12:07:54] (step=0027175) Train Loss: 5.2541, Train Steps/Sec: 1.07 + 20%|████████████████▊ | 4484/22715 [1:24:46<4:41:35, 1.08it/s][2025-04-29 12:08:18] (step=0027200) Train Loss: 5.2388, Train Steps/Sec: 1.07 + 20%|████████████████▊ | 4509/22715 [1:25:09<4:49:36, 1.05it/s][2025-04-29 12:08:41] (step=0027225) Train Loss: 5.2613, Train Steps/Sec: 1.07 + 20%|████████████████▉ | 4534/22715 [1:25:33<4:45:53, 1.06it/s][2025-04-29 12:09:04] (step=0027250) Train Loss: 5.2659, Train Steps/Sec: 1.07 + 20%|█████████████████ | 4559/22715 [1:25:56<4:44:09, 1.06it/s][2025-04-29 12:09:28] (step=0027275) Train Loss: 5.2496, Train Steps/Sec: 1.07 + 20%|█████████████████▏ | 4584/22715 [1:26:19<4:42:18, 1.07it/s][2025-04-29 12:09:51] (step=0027300) Train Loss: 5.2516, Train Steps/Sec: 1.07 + 20%|█████████████████▏ | 4609/22715 [1:26:43<4:41:00, 1.07it/s][2025-04-29 12:10:15] (step=0027325) Train Loss: 5.2693, Train Steps/Sec: 1.07 + 20%|█████████████████▎ | 4634/22715 [1:27:06<4:39:19, 1.08it/s][2025-04-29 12:10:38] (step=0027350) Train Loss: 5.2416, Train Steps/Sec: 1.07 + 21%|█████████████████▍ | 4659/22715 [1:27:29<4:39:01, 1.08it/s][2025-04-29 12:11:01] (step=0027375) Train Loss: 5.2760, Train Steps/Sec: 1.07 + 21%|█████████████████▌ | 4684/22715 [1:27:53<4:37:07, 1.08it/s][2025-04-29 12:11:25] (step=0027400) Train Loss: 5.2560, Train Steps/Sec: 1.07 + 21%|█████████████████▌ | 4709/22715 [1:28:16<4:45:59, 1.05it/s][2025-04-29 12:11:48] (step=0027425) Train Loss: 5.2685, Train Steps/Sec: 1.07 + 21%|█████████████████▋ | 4734/22715 [1:28:40<4:42:39, 1.06it/s][2025-04-29 12:12:11] (step=0027450) Train Loss: 5.2554, Train Steps/Sec: 1.07 + 21%|█████████████████▊ | 4759/22715 [1:29:03<4:40:20, 1.07it/s][2025-04-29 12:12:35] (step=0027475) Train Loss: 5.2610, Train Steps/Sec: 1.07 + 21%|█████████████████▉ | 4784/22715 [1:29:26<4:36:46, 1.08it/s][2025-04-29 12:12:58] (step=0027500) Train Loss: 5.2417, Train Steps/Sec: 1.07 + 21%|█████████████████▉ | 4809/22715 [1:29:50<4:37:15, 1.08it/s][2025-04-29 12:13:21] (step=0027525) Train Loss: 5.2157, Train Steps/Sec: 1.07 + 21%|██████████████████ | 4834/22715 [1:30:13<4:36:58, 1.08it/s][2025-04-29 12:13:45] (step=0027550) Train Loss: 5.1802, Train Steps/Sec: 1.07 + 21%|██████████████████▏ | 4859/22715 [1:30:36<4:36:09, 1.08it/s][2025-04-29 12:14:08] (step=0027575) Train Loss: 5.2745, Train Steps/Sec: 1.07 + 22%|██████████████████▎ | 4884/22715 [1:31:00<4:36:48, 1.07it/s][2025-04-29 12:14:32] (step=0027600) Train Loss: 5.2562, Train Steps/Sec: 1.06 + 22%|██████████████████▎ | 4909/22715 [1:31:23<4:44:06, 1.04it/s][2025-04-29 12:14:55] (step=0027625) Train Loss: 5.2708, Train Steps/Sec: 1.07 + 22%|██████████████████▍ | 4934/22715 [1:31:47<4:40:01, 1.06it/s][2025-04-29 12:15:19] (step=0027650) Train Loss: 5.2308, Train Steps/Sec: 1.07 + 22%|██████████████████▌ | 4959/22715 [1:32:10<4:37:38, 1.07it/s][2025-04-29 12:15:42] (step=0027675) Train Loss: 5.2290, Train Steps/Sec: 1.07 + 22%|██████████████████▋ | 4984/22715 [1:32:34<4:36:26, 1.07it/s][2025-04-29 12:16:05] (step=0027700) Train Loss: 5.2487, Train Steps/Sec: 1.07 + 22%|██████████████████▋ | 5009/22715 [1:32:57<4:35:14, 1.07it/s][2025-04-29 12:16:29] (step=0027725) Train Loss: 5.2307, Train Steps/Sec: 1.07 + 22%|██████████████████▊ | 5034/22715 [1:33:20<4:33:19, 1.08it/s][2025-04-29 12:16:52] (step=0027750) Train Loss: 5.2414, Train Steps/Sec: 1.07 + 22%|██████████████████▉ | 5059/22715 [1:33:44<4:33:22, 1.08it/s][2025-04-29 12:17:15] (step=0027775) Train Loss: 5.2383, Train Steps/Sec: 1.07 + 22%|███████████████████ | 5084/22715 [1:34:07<4:31:35, 1.08it/s][2025-04-29 12:17:39] (step=0027800) Train Loss: 5.2432, Train Steps/Sec: 1.07 + 22%|███████████████████ | 5109/22715 [1:34:30<4:40:25, 1.05it/s][2025-04-29 12:18:02] (step=0027825) Train Loss: 5.2462, Train Steps/Sec: 1.07 + 23%|███████████████████▏ | 5134/22715 [1:35:00<4:37:19, 1.06it/s][2025-04-29 12:18:31] (step=0027850) Train Loss: 5.2338, Train Steps/Sec: 0.86 + 23%|███████████████████▎ | 5159/22715 [1:35:23<4:33:50, 1.07it/s][2025-04-29 12:18:55] (step=0027875) Train Loss: 5.2566, Train Steps/Sec: 1.07 + 23%|███████████████████▍ | 5184/22715 [1:35:46<4:34:11, 1.07it/s][2025-04-29 12:19:18] (step=0027900) Train Loss: 5.2359, Train Steps/Sec: 1.07 + 23%|███████████████████▍ | 5209/22715 [1:36:10<4:31:37, 1.07it/s][2025-04-29 12:19:42] (step=0027925) Train Loss: 5.2635, Train Steps/Sec: 1.07 + 23%|███████████████████▌ | 5234/22715 [1:36:33<4:30:35, 1.08it/s][2025-04-29 12:20:05] (step=0027950) Train Loss: 5.2605, Train Steps/Sec: 1.07 + 23%|███████████████████▋ | 5259/22715 [1:36:57<4:29:47, 1.08it/s][2025-04-29 12:20:28] (step=0027975) Train Loss: 5.2318, Train Steps/Sec: 1.07 + 23%|███████████████████▊ | 5284/22715 [1:37:20<4:29:23, 1.08it/s][2025-04-29 12:20:52] (step=0028000) Train Loss: 5.2126, Train Steps/Sec: 1.07 +You are using a model of type blip-2 to instantiate a model of type instructblip. This is not supported for all configurations of models and can yield errors. +[2025-04-29 12:20:52] text_config is None. Initializing the text config with default values (`OPTConfig`). +model_name_or_path:/tmp/haozhezhao/model/blip2-flan-t5-xl +image_place_holder: +tokenizer length after expend 32101 +tokenizer length before expend 32100 +100%|████████████████████████████████████████████████████████████████████████████████████████████████| 5/5 [04:55<00:00, 59.16s/it] +[2025-04-29 12:27:00] Finish Eval in 28000 steps...█████████████████████████████████████████| 5/5 [04:55<00:00, 58.90s/it] +processing left-padding... +processing left-padding... +processing left-padding... +processing left-padding... +processing left-padding... +[2025-04-29 12:27:20] Saved checkpoint to checkpoint/X2I_700k_3mask_200k_LLava_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_filtered_objects_3mask_888_context/003-GPT-XL/checkpoints/0028000.pt +[2025-04-29 12:27:22] Removed old checkpoint: checkpoint/X2I_700k_3mask_200k_LLava_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_filtered_objects_3mask_888_context/003-GPT-XL/checkpoints/0026000.pt + 23%|███████████████████▊ | 5309/22715 [1:44:14<4:45:47, 1.02it/s][2025-04-29 12:27:46] (step=0028025) Train Loss: 5.2164, Train Steps/Sec: 0.06 + 23%|███████████████████▉ | 5334/22715 [1:44:38<4:33:23, 1.06it/s][2025-04-29 12:28:09] (step=0028050) Train Loss: 5.2797, Train Steps/Sec: 1.06 + 24%|████████████████████ | 5359/22715 [1:45:01<4:30:51, 1.07it/s][2025-04-29 12:28:33] (step=0028075) Train Loss: 5.2229, Train Steps/Sec: 1.07 + 24%|████████████████████▏ | 5384/22715 [1:45:24<4:28:53, 1.07it/s][2025-04-29 12:28:56] (step=0028100) Train Loss: 5.2149, Train Steps/Sec: 1.07 + 24%|████████████████████▏ | 5409/22715 [1:45:48<4:28:40, 1.07it/s][2025-04-29 12:29:19] (step=0028125) Train Loss: 5.2393, Train Steps/Sec: 1.07 + 24%|████████████████████▎ | 5434/22715 [1:46:11<4:27:42, 1.08it/s][2025-04-29 12:29:43] (step=0028150) Train Loss: 5.2575, Train Steps/Sec: 1.07 + 24%|████████████████████▍ | 5459/22715 [1:46:35<4:29:19, 1.07it/s][2025-04-29 12:30:06] (step=0028175) Train Loss: 5.2131, Train Steps/Sec: 1.06 + 24%|████████████████████▌ | 5484/22715 [1:46:58<4:26:45, 1.08it/s][2025-04-29 12:30:30] (step=0028200) Train Loss: 5.1978, Train Steps/Sec: 1.06 + 24%|████████████████████▌ | 5509/22715 [1:47:22<4:35:12, 1.04it/s][2025-04-29 12:30:53] (step=0028225) Train Loss: 5.1848, Train Steps/Sec: 1.07 + 24%|████████████████████▋ | 5534/22715 [1:47:45<4:29:50, 1.06it/s][2025-04-29 12:31:17] (step=0028250) Train Loss: 5.2455, Train Steps/Sec: 1.07 + 24%|████████████████████▊ | 5559/22715 [1:48:08<4:27:56, 1.07it/s][2025-04-29 12:31:40] (step=0028275) Train Loss: 5.2056, Train Steps/Sec: 1.07 + 25%|████████████████████▉ | 5584/22715 [1:48:32<4:26:50, 1.07it/s][2025-04-29 12:32:04] (step=0028300) Train Loss: 5.2758, Train Steps/Sec: 1.07 + 25%|████████████████████▉ | 5609/22715 [1:48:55<4:26:20, 1.07it/s][2025-04-29 12:32:27] (step=0028325) Train Loss: 5.2491, Train Steps/Sec: 1.07 + 25%|█████████████████████ | 5634/22715 [1:49:19<4:24:14, 1.08it/s][2025-04-29 12:32:50] (step=0028350) Train Loss: 5.2551, Train Steps/Sec: 1.07 + 25%|█████████████████████▏ | 5659/22715 [1:49:42<4:28:04, 1.06it/s][2025-04-29 12:33:14] (step=0028375) Train Loss: 5.1932, Train Steps/Sec: 1.07 + 25%|█████████████████████▎ | 5684/22715 [1:50:05<4:24:30, 1.07it/s][2025-04-29 12:33:37] (step=0028400) Train Loss: 5.2493, Train Steps/Sec: 1.06 + 25%|█████████████████████▎ | 5709/22715 [1:50:29<4:29:46, 1.05it/s][2025-04-29 12:34:01] (step=0028425) Train Loss: 5.2325, Train Steps/Sec: 1.07 + 25%|█████████████████████▍ | 5734/22715 [1:50:52<4:27:44, 1.06it/s][2025-04-29 12:34:24] (step=0028450) Train Loss: 5.2675, Train Steps/Sec: 1.07 + 25%|█████████████████████▌ | 5759/22715 [1:51:16<4:26:18, 1.06it/s][2025-04-29 12:34:48] (step=0028475) Train Loss: 5.2004, Train Steps/Sec: 1.07 + 25%|█████████████████████▋ | 5784/22715 [1:51:39<4:23:32, 1.07it/s][2025-04-29 12:35:11] (step=0028500) Train Loss: 5.2629, Train Steps/Sec: 1.07 + 26%|█████████████████████▋ | 5809/22715 [1:52:03<4:23:45, 1.07it/s][2025-04-29 12:35:34] (step=0028525) Train Loss: 5.2039, Train Steps/Sec: 1.07 + 26%|█████████████████████▊ | 5834/22715 [1:52:26<4:22:00, 1.07it/s][2025-04-29 12:35:58] (step=0028550) Train Loss: 5.2420, Train Steps/Sec: 1.06 + 26%|█████████████████████▉ | 5859/22715 [1:52:50<4:20:37, 1.08it/s][2025-04-29 12:36:21] (step=0028575) Train Loss: 5.1878, Train Steps/Sec: 1.07 + 26%|██████████████████████ | 5884/22715 [1:53:13<4:20:04, 1.08it/s][2025-04-29 12:36:45] (step=0028600) Train Loss: 5.2227, Train Steps/Sec: 1.07 + 26%|██████████████████████ | 5909/22715 [1:53:36<4:27:04, 1.05it/s][2025-04-29 12:37:08] (step=0028625) Train Loss: 5.2067, Train Steps/Sec: 1.07 + 26%|██████████████████████▏ | 5934/22715 [1:54:00<4:23:59, 1.06it/s][2025-04-29 12:37:32] (step=0028650) Train Loss: 5.1911, Train Steps/Sec: 1.07 + 26%|██████████████████████▎ | 5959/22715 [1:54:23<4:21:52, 1.07it/s][2025-04-29 12:37:55] (step=0028675) Train Loss: 5.2262, Train Steps/Sec: 1.07 + 26%|██████████████████████▍ | 5984/22715 [1:54:47<4:20:20, 1.07it/s][2025-04-29 12:38:18] (step=0028700) Train Loss: 5.2520, Train Steps/Sec: 1.07 + 26%|██████████████████████▍ | 6009/22715 [1:55:10<4:19:50, 1.07it/s][2025-04-29 12:38:42] (step=0028725) Train Loss: 5.2292, Train Steps/Sec: 1.07 + 27%|██████████████████████▌ | 6034/22715 [1:55:33<4:18:31, 1.08it/s][2025-04-29 12:39:05] (step=0028750) Train Loss: 5.2110, Train Steps/Sec: 1.07 + 27%|██████████████████████▋ | 6059/22715 [1:56:00<5:31:24, 1.19s/it][2025-04-29 12:39:32] (step=0028775) Train Loss: 5.2226, Train Steps/Sec: 0.92 + 27%|██████████████████████▊ | 6084/22715 [1:56:31<7:53:04, 1.71s/it][2025-04-29 12:40:03] (step=0028800) Train Loss: 5.2192, Train Steps/Sec: 0.81 + 27%|██████████████████████▊ | 6109/22715 [1:56:58<6:11:02, 1.34s/it][2025-04-29 12:40:30] (step=0028825) Train Loss: 5.2061, Train Steps/Sec: 0.92 + 27%|██████████████████████▉ | 6134/22715 [1:57:29<4:28:43, 1.03it/s][2025-04-29 12:41:01] (step=0028850) Train Loss: 5.1933, Train Steps/Sec: 0.82 + 27%|███████████████████████ | 6159/22715 [1:57:57<4:17:51, 1.07it/s][2025-04-29 12:41:29] (step=0028875) Train Loss: 5.2101, Train Steps/Sec: 0.89 + 27%|███████████████████████▏ | 6184/22715 [1:58:21<4:16:33, 1.07it/s][2025-04-29 12:41:52] (step=0028900) Train Loss: 5.2026, Train Steps/Sec: 1.07 + 27%|███████████████████████▏ | 6209/22715 [1:58:44<4:15:43, 1.08it/s][2025-04-29 12:42:16] (step=0028925) Train Loss: 5.1843, Train Steps/Sec: 1.07 + 27%|███████████████████████▎ | 6234/22715 [1:59:07<4:15:04, 1.08it/s][2025-04-29 12:42:39] (step=0028950) Train Loss: 5.2237, Train Steps/Sec: 1.07 + 28%|███████████████████████▍ | 6259/22715 [1:59:31<4:14:27, 1.08it/s][2025-04-29 12:43:02] (step=0028975) Train Loss: 5.2441, Train Steps/Sec: 1.07 + 28%|███████████████████████▌ | 6284/22715 [1:59:54<4:13:38, 1.08it/s][2025-04-29 12:43:26] (step=0029000) Train Loss: 5.2493, Train Steps/Sec: 1.06 + 28%|███████████████████████▌ | 6309/22715 [2:00:18<4:21:26, 1.05it/s][2025-04-29 12:43:49] (step=0029025) Train Loss: 5.2147, Train Steps/Sec: 1.07 + 28%|███████████████████████▋ | 6334/22715 [2:00:41<4:18:28, 1.06it/s][2025-04-29 12:44:13] (step=0029050) Train Loss: 5.2493, Train Steps/Sec: 1.07 + 28%|███████████████████████▊ | 6359/22715 [2:01:04<4:15:07, 1.07it/s][2025-04-29 12:44:36] (step=0029075) Train Loss: 5.2300, Train Steps/Sec: 1.07 + 28%|███████████████████████▉ | 6384/22715 [2:01:28<4:14:01, 1.07it/s][2025-04-29 12:45:00] (step=0029100) Train Loss: 5.2137, Train Steps/Sec: 1.07 + 28%|███████████████████████▉ | 6409/22715 [2:01:51<4:13:04, 1.07it/s][2025-04-29 12:45:23] (step=0029125) Train Loss: 5.2318, Train Steps/Sec: 1.06 + 28%|████████████████████████ | 6434/22715 [2:02:15<4:12:50, 1.07it/s][2025-04-29 12:45:46] (step=0029150) Train Loss: 5.1863, Train Steps/Sec: 1.07 + 28%|████████████████████████▏ | 6459/22715 [2:02:38<4:11:11, 1.08it/s][2025-04-29 12:46:10] (step=0029175) Train Loss: 5.2216, Train Steps/Sec: 1.07 + 29%|████████████████████████▎ | 6484/22715 [2:03:01<4:11:44, 1.07it/s][2025-04-29 12:46:33] (step=0029200) Train Loss: 5.2199, Train Steps/Sec: 1.07 + 29%|████████████████████████▎ | 6509/22715 [2:03:25<4:17:30, 1.05it/s][2025-04-29 12:46:57] (step=0029225) Train Loss: 5.2136, Train Steps/Sec: 1.07 + 29%|████████████████████████▍ | 6534/22715 [2:03:48<4:14:10, 1.06it/s][2025-04-29 12:47:20] (step=0029250) Train Loss: 5.2190, Train Steps/Sec: 1.07 + 29%|████████████████████████▌ | 6559/22715 [2:04:12<4:12:40, 1.07it/s][2025-04-29 12:47:44] (step=0029275) Train Loss: 5.2155, Train Steps/Sec: 1.07 + 29%|████████████████████████▋ | 6584/22715 [2:04:35<4:11:48, 1.07it/s][2025-04-29 12:48:07] (step=0029300) Train Loss: 5.1693, Train Steps/Sec: 1.07 + 29%|████████████████████████▋ | 6609/22715 [2:04:58<4:10:02, 1.07it/s][2025-04-29 12:48:30] (step=0029325) Train Loss: 5.1781, Train Steps/Sec: 1.07 + 29%|████████████████████████▊ | 6634/22715 [2:05:22<4:08:44, 1.08it/s][2025-04-29 12:48:54] (step=0029350) Train Loss: 5.2320, Train Steps/Sec: 1.07 + 29%|████████████████████████▉ | 6659/22715 [2:05:45<4:08:07, 1.08it/s][2025-04-29 12:49:17] (step=0029375) Train Loss: 5.2181, Train Steps/Sec: 1.07 + 29%|█████████████████████████ | 6684/22715 [2:06:09<4:07:58, 1.08it/s][2025-04-29 12:49:40] (step=0029400) Train Loss: 5.1900, Train Steps/Sec: 1.07 + 30%|█████████████████████████ | 6709/22715 [2:06:32<4:14:43, 1.05it/s][2025-04-29 12:50:04] (step=0029425) Train Loss: 5.2287, Train Steps/Sec: 1.07 + 30%|█████████████████████████▏ | 6734/22715 [2:06:55<4:11:02, 1.06it/s][2025-04-29 12:50:27] (step=0029450) Train Loss: 5.2059, Train Steps/Sec: 1.07 + 30%|█████████████████████████▎ | 6759/22715 [2:07:19<4:08:29, 1.07it/s][2025-04-29 12:50:51] (step=0029475) Train Loss: 5.1989, Train Steps/Sec: 1.07 + 30%|█████████████████████████▍ | 6784/22715 [2:07:42<4:07:54, 1.07it/s][2025-04-29 12:51:14] (step=0029500) Train Loss: 5.1925, Train Steps/Sec: 1.07 + 30%|█████████████████████████▍ | 6809/22715 [2:08:05<4:05:59, 1.08it/s][2025-04-29 12:51:37] (step=0029525) Train Loss: 5.1753, Train Steps/Sec: 1.07 + 30%|█████████████████████████▌ | 6834/22715 [2:08:29<4:05:34, 1.08it/s][2025-04-29 12:52:01] (step=0029550) Train Loss: 5.1792, Train Steps/Sec: 1.07 + 30%|█████████████████████████▋ | 6859/22715 [2:08:52<4:05:11, 1.08it/s][2025-04-29 12:52:24] (step=0029575) Train Loss: 5.1775, Train Steps/Sec: 1.07 + 30%|█████████████████████████▊ | 6884/22715 [2:09:16<4:04:28, 1.08it/s][2025-04-29 12:52:47] (step=0029600) Train Loss: 5.2124, Train Steps/Sec: 1.07 + 30%|█████████████████████████▊ | 6909/22715 [2:09:39<4:10:51, 1.05it/s][2025-04-29 12:53:11] (step=0029625) Train Loss: 5.2125, Train Steps/Sec: 1.07 + 31%|█████████████████████████▉ | 6934/22715 [2:10:02<4:07:38, 1.06it/s][2025-04-29 12:53:34] (step=0029650) Train Loss: 5.1764, Train Steps/Sec: 1.07 + 31%|██████████████████████████ | 6959/22715 [2:10:26<4:05:23, 1.07it/s][2025-04-29 12:53:58] (step=0029675) Train Loss: 5.2036, Train Steps/Sec: 1.07 + 31%|██████████████████████████▏ | 6984/22715 [2:10:49<4:04:48, 1.07it/s][2025-04-29 12:54:21] (step=0029700) Train Loss: 5.2207, Train Steps/Sec: 1.07 + 31%|██████████████████████████▏ | 7009/22715 [2:11:12<4:03:05, 1.08it/s][2025-04-29 12:54:44] (step=0029725) Train Loss: 5.2124, Train Steps/Sec: 1.07 + 31%|██████████████████████████▎ | 7034/22715 [2:11:36<4:03:18, 1.07it/s][2025-04-29 12:55:08] (step=0029750) Train Loss: 5.2169, Train Steps/Sec: 1.07 + 31%|██████████████████████████▍ | 7059/22715 [2:11:59<4:04:19, 1.07it/s][2025-04-29 12:55:31] (step=0029775) Train Loss: 5.2114, Train Steps/Sec: 1.06 + 31%|██████████████████████████▌ | 7084/22715 [2:12:23<4:02:14, 1.08it/s][2025-04-29 12:55:55] (step=0029800) Train Loss: 5.2340, Train Steps/Sec: 1.06 + 31%|██████████████████████████▌ | 7109/22715 [2:12:51<7:24:03, 1.71s/it][2025-04-29 12:56:23] (step=0029825) Train Loss: 5.2423, Train Steps/Sec: 0.88 + 31%|██████████████████████████▋ | 7134/22715 [2:13:15<4:05:56, 1.06it/s][2025-04-29 12:56:47] (step=0029850) Train Loss: 5.1694, Train Steps/Sec: 1.06 + 32%|██████████████████████████▊ | 7159/22715 [2:13:38<4:03:18, 1.07it/s][2025-04-29 12:57:10] (step=0029875) Train Loss: 5.1956, Train Steps/Sec: 1.07 + 32%|██████████████████████████▉ | 7184/22715 [2:14:02<4:02:04, 1.07it/s][2025-04-29 12:57:34] (step=0029900) Train Loss: 5.1593, Train Steps/Sec: 1.07 + 32%|██████████████████████████▉ | 7209/22715 [2:14:25<4:00:37, 1.07it/s][2025-04-29 12:57:57] (step=0029925) Train Loss: 5.1910, Train Steps/Sec: 1.07 + 32%|███████████████████████████ | 7234/22715 [2:14:49<4:00:31, 1.07it/s][2025-04-29 12:58:20] (step=0029950) Train Loss: 5.2130, Train Steps/Sec: 1.06 + 32%|███████████████████████████▏ | 7259/22715 [2:15:12<3:59:18, 1.08it/s][2025-04-29 12:58:44] (step=0029975) Train Loss: 5.2169, Train Steps/Sec: 1.06 + 32%|███████████████████████████▎ | 7284/22715 [2:15:35<3:58:37, 1.08it/s][2025-04-29 12:59:07] (step=0030000) Train Loss: 5.2087, Train Steps/Sec: 1.07 +You are using a model of type blip-2 to instantiate a model of type instructblip. This is not supported for all configurations of models and can yield errors. +[2025-04-29 12:59:07] text_config is None. Initializing the text config with default values (`OPTConfig`). +model_name_or_path:/tmp/haozhezhao/model/blip2-flan-t5-xl +image_place_holder: +tokenizer length after expend 32101 +tokenizer length before expend 32100 +100%|████████████████████████████████████████████████████████████████████████████████████████████████| 5/5 [04:56<00:00, 59.34s/it] +[2025-04-29 13:05:17] Finish Eval in 30000 steps...█████████████████████████████████████████| 5/5 [04:56<00:00, 58.96s/it] +processing left-padding... +processing left-padding... +processing left-padding... +processing left-padding... +processing left-padding... +[2025-04-29 13:05:36] Saved checkpoint to checkpoint/X2I_700k_3mask_200k_LLava_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_filtered_objects_3mask_888_context/003-GPT-XL/checkpoints/0030000.pt +[2025-04-29 13:05:38] Removed old checkpoint: checkpoint/X2I_700k_3mask_200k_LLava_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_filtered_objects_3mask_888_context/003-GPT-XL/checkpoints/0028000.pt + 32%|███████████████████████████▎ | 7309/22715 [2:22:30<4:10:01, 1.03it/s][2025-04-29 13:06:02] (step=0030025) Train Loss: 5.2248, Train Steps/Sec: 0.06 + 32%|███████████████████████████▍ | 7334/22715 [2:22:53<4:00:48, 1.06it/s][2025-04-29 13:06:25] (step=0030050) Train Loss: 5.2328, Train Steps/Sec: 1.07 + 32%|███████████████████████████▌ | 7359/22715 [2:23:16<3:59:15, 1.07it/s][2025-04-29 13:06:48] (step=0030075) Train Loss: 5.1814, Train Steps/Sec: 1.07 + 33%|███████████████████████████▋ | 7384/22715 [2:23:40<3:58:14, 1.07it/s][2025-04-29 13:07:12] (step=0030100) Train Loss: 5.2034, Train Steps/Sec: 1.07 + 33%|███████████████████████████▋ | 7409/22715 [2:24:03<3:57:31, 1.07it/s][2025-04-29 13:07:35] (step=0030125) Train Loss: 5.2162, Train Steps/Sec: 1.07 + 33%|███████████████████████████▊ | 7434/22715 [2:24:27<3:56:33, 1.08it/s][2025-04-29 13:07:58] (step=0030150) Train Loss: 5.2714, Train Steps/Sec: 1.07 + 33%|███████████████████████████▉ | 7459/22715 [2:24:50<3:55:50, 1.08it/s][2025-04-29 13:08:22] (step=0030175) Train Loss: 5.2001, Train Steps/Sec: 1.07 + 33%|████████████████████████████ | 7484/22715 [2:25:13<3:55:21, 1.08it/s][2025-04-29 13:08:45] (step=0030200) Train Loss: 5.1707, Train Steps/Sec: 1.07 + 33%|████████████████████████████ | 7509/22715 [2:25:37<4:03:07, 1.04it/s][2025-04-29 13:09:09] (step=0030225) Train Loss: 5.2147, Train Steps/Sec: 1.07 + 33%|████████████████████████████▏ | 7534/22715 [2:26:00<3:57:31, 1.07it/s][2025-04-29 13:09:32] (step=0030250) Train Loss: 5.1903, Train Steps/Sec: 1.07 + 33%|████████████████████████████▎ | 7559/22715 [2:26:23<3:56:24, 1.07it/s][2025-04-29 13:09:55] (step=0030275) Train Loss: 5.2096, Train Steps/Sec: 1.07 + 33%|████████████████████████████▍ | 7584/22715 [2:26:47<3:55:23, 1.07it/s][2025-04-29 13:10:18] (step=0030300) Train Loss: 5.1352, Train Steps/Sec: 1.07 + 33%|████████████████████████████▍ | 7609/22715 [2:27:10<3:54:07, 1.08it/s][2025-04-29 13:10:42] (step=0030325) Train Loss: 5.2485, Train Steps/Sec: 1.07 + 34%|████████████████████████████▌ | 7634/22715 [2:27:33<3:52:35, 1.08it/s][2025-04-29 13:11:05] (step=0030350) Train Loss: 5.2029, Train Steps/Sec: 1.07 + 34%|████████████████████████████▋ | 7659/22715 [2:27:57<3:52:40, 1.08it/s][2025-04-29 13:11:29] (step=0030375) Train Loss: 5.1775, Train Steps/Sec: 1.07 + 34%|████████████████████████████▊ | 7684/22715 [2:28:20<3:52:10, 1.08it/s][2025-04-29 13:11:52] (step=0030400) Train Loss: 5.1893, Train Steps/Sec: 1.07 + 34%|████████████████████████████▊ | 7709/22715 [2:28:44<3:58:23, 1.05it/s][2025-04-29 13:12:15] (step=0030425) Train Loss: 5.2386, Train Steps/Sec: 1.07 + 34%|████████████████████████████▉ | 7734/22715 [2:29:07<3:55:02, 1.06it/s][2025-04-29 13:12:39] (step=0030450) Train Loss: 5.1800, Train Steps/Sec: 1.07 + 34%|█████████████████████████████ | 7759/22715 [2:29:30<3:53:31, 1.07it/s][2025-04-29 13:13:02] (step=0030475) Train Loss: 5.2027, Train Steps/Sec: 1.07 + 34%|█████████████████████████████▏ | 7784/22715 [2:29:54<3:53:17, 1.07it/s][2025-04-29 13:13:26] (step=0030500) Train Loss: 5.1991, Train Steps/Sec: 1.06 + 34%|█████████████████████████████▏ | 7809/22715 [2:30:17<3:51:45, 1.07it/s][2025-04-29 13:13:49] (step=0030525) Train Loss: 5.1950, Train Steps/Sec: 1.06 + 34%|█████████████████████████████▎ | 7834/22715 [2:30:41<3:49:54, 1.08it/s][2025-04-29 13:14:12] (step=0030550) Train Loss: 5.1762, Train Steps/Sec: 1.07 + 35%|█████████████████████████████▍ | 7859/22715 [2:31:04<3:49:48, 1.08it/s][2025-04-29 13:14:36] (step=0030575) Train Loss: 5.1982, Train Steps/Sec: 1.07 + 35%|█████████████████████████████▌ | 7884/22715 [2:31:27<3:48:58, 1.08it/s][2025-04-29 13:14:59] (step=0030600) Train Loss: 5.1947, Train Steps/Sec: 1.06 + 35%|█████████████████████████████▌ | 7909/22715 [2:32:02<6:18:21, 1.53s/it][2025-04-29 13:15:34] (step=0030625) Train Loss: 5.1729, Train Steps/Sec: 0.72 + 35%|█████████████████████████████▋ | 7934/22715 [2:32:29<8:37:17, 2.10s/it][2025-04-29 13:16:01] (step=0030650) Train Loss: 5.1727, Train Steps/Sec: 0.92 + 35%|█████████████████████████████▊ | 7959/22715 [2:33:00<3:56:31, 1.04it/s][2025-04-29 13:16:32] (step=0030675) Train Loss: 5.1740, Train Steps/Sec: 0.82 + 35%|█████████████████████████████▉ | 7984/22715 [2:33:28<3:50:51, 1.06it/s][2025-04-29 13:17:00] (step=0030700) Train Loss: 5.1943, Train Steps/Sec: 0.88 + 35%|█████████████████████████████▉ | 8009/22715 [2:33:52<3:47:51, 1.08it/s][2025-04-29 13:17:23] (step=0030725) Train Loss: 5.1681, Train Steps/Sec: 1.07 + 35%|██████████████████████████████ | 8034/22715 [2:34:15<3:46:31, 1.08it/s][2025-04-29 13:17:47] (step=0030750) Train Loss: 5.2172, Train Steps/Sec: 1.07 + 35%|██████████████████████████████▏ | 8059/22715 [2:34:38<3:46:11, 1.08it/s][2025-04-29 13:18:10] (step=0030775) Train Loss: 5.1802, Train Steps/Sec: 1.07 + 36%|██████████████████████████████▎ | 8084/22715 [2:35:02<3:45:59, 1.08it/s][2025-04-29 13:18:34] (step=0030800) Train Loss: 5.1491, Train Steps/Sec: 1.06 + 36%|██████████████████████████████▎ | 8109/22715 [2:35:25<3:52:21, 1.05it/s][2025-04-29 13:18:57] (step=0030825) Train Loss: 5.1854, Train Steps/Sec: 1.07 + 36%|██████████████████████████████▍ | 8134/22715 [2:35:49<3:48:49, 1.06it/s][2025-04-29 13:19:20] (step=0030850) Train Loss: 5.1533, Train Steps/Sec: 1.07 + 36%|██████████████████████████████▌ | 8159/22715 [2:36:12<3:47:27, 1.07it/s][2025-04-29 13:19:44] (step=0030875) Train Loss: 5.2202, Train Steps/Sec: 1.07 + 36%|██████████████████████████████▌ | 8184/22715 [2:36:35<3:46:08, 1.07it/s][2025-04-29 13:20:07] (step=0030900) Train Loss: 5.1943, Train Steps/Sec: 1.07 + 36%|██████████████████████████████▋ | 8209/22715 [2:36:59<3:45:42, 1.07it/s][2025-04-29 13:20:30] (step=0030925) Train Loss: 5.2428, Train Steps/Sec: 1.07 + 36%|██████████████████████████████▊ | 8234/22715 [2:37:22<3:44:45, 1.07it/s][2025-04-29 13:20:54] (step=0030950) Train Loss: 5.1973, Train Steps/Sec: 1.07 + 36%|██████████████████████████████▉ | 8259/22715 [2:37:45<3:42:55, 1.08it/s][2025-04-29 13:21:17] (step=0030975) Train Loss: 5.2051, Train Steps/Sec: 1.07 + 36%|██████████████████████████████▉ | 8284/22715 [2:38:09<3:41:52, 1.08it/s][2025-04-29 13:21:41] (step=0031000) Train Loss: 5.1180, Train Steps/Sec: 1.07 + 37%|███████████████████████████████ | 8309/22715 [2:38:32<3:48:38, 1.05it/s][2025-04-29 13:22:04] (step=0031025) Train Loss: 5.1970, Train Steps/Sec: 1.07 + 37%|███████████████████████████████▏ | 8334/22715 [2:38:55<3:45:50, 1.06it/s][2025-04-29 13:22:27] (step=0031050) Train Loss: 5.1702, Train Steps/Sec: 1.07 + 37%|███████████████████████████████▎ | 8359/22715 [2:39:19<3:43:26, 1.07it/s][2025-04-29 13:22:51] (step=0031075) Train Loss: 5.1655, Train Steps/Sec: 1.07 + 37%|███████████████████████████████▎ | 8384/22715 [2:39:42<3:42:57, 1.07it/s][2025-04-29 13:23:14] (step=0031100) Train Loss: 5.1530, Train Steps/Sec: 1.07 + 37%|███████████████████████████████▍ | 8409/22715 [2:40:05<3:41:40, 1.08it/s][2025-04-29 13:23:37] (step=0031125) Train Loss: 5.2211, Train Steps/Sec: 1.07 + 37%|███████████████████████████████▌ | 8434/22715 [2:40:29<3:40:37, 1.08it/s][2025-04-29 13:24:01] (step=0031150) Train Loss: 5.1916, Train Steps/Sec: 1.07 + 37%|███████████████████████████████▋ | 8459/22715 [2:40:52<3:41:16, 1.07it/s][2025-04-29 13:24:24] (step=0031175) Train Loss: 5.2369, Train Steps/Sec: 1.07 + 37%|███████████████████████████████▋ | 8484/22715 [2:41:16<3:39:25, 1.08it/s][2025-04-29 13:24:47] (step=0031200) Train Loss: 5.1663, Train Steps/Sec: 1.07 + 37%|███████████████████████████████▊ | 8509/22715 [2:41:39<3:45:40, 1.05it/s][2025-04-29 13:25:11] (step=0031225) Train Loss: 5.2166, Train Steps/Sec: 1.07 + 38%|███████████████████████████████▉ | 8534/22715 [2:42:02<3:42:42, 1.06it/s][2025-04-29 13:25:34] (step=0031250) Train Loss: 5.1768, Train Steps/Sec: 1.07 + 38%|████████████████████████████████ | 8559/22715 [2:42:26<3:40:58, 1.07it/s][2025-04-29 13:25:57] (step=0031275) Train Loss: 5.1858, Train Steps/Sec: 1.07 + 38%|████████████████████████████████ | 8584/22715 [2:42:49<3:39:34, 1.07it/s][2025-04-29 13:26:21] (step=0031300) Train Loss: 5.1381, Train Steps/Sec: 1.07 + 38%|████████████████████████████████▏ | 8609/22715 [2:43:12<3:38:57, 1.07it/s][2025-04-29 13:26:44] (step=0031325) Train Loss: 5.1682, Train Steps/Sec: 1.07 + 38%|████████████████████████████████▎ | 8634/22715 [2:43:36<3:37:21, 1.08it/s][2025-04-29 13:27:08] (step=0031350) Train Loss: 5.1804, Train Steps/Sec: 1.07 + 38%|████████████████████████████████▍ | 8659/22715 [2:43:59<3:37:17, 1.08it/s][2025-04-29 13:27:31] (step=0031375) Train Loss: 5.1651, Train Steps/Sec: 1.07 + 38%|████████████████████████████████▍ | 8684/22715 [2:44:22<3:36:05, 1.08it/s][2025-04-29 13:27:54] (step=0031400) Train Loss: 5.2078, Train Steps/Sec: 1.07 + 38%|████████████████████████████████▌ | 8709/22715 [2:44:46<3:42:27, 1.05it/s][2025-04-29 13:28:18] (step=0031425) Train Loss: 5.1705, Train Steps/Sec: 1.07 + 38%|████████████████████████████████▋ | 8734/22715 [2:45:09<3:39:03, 1.06it/s][2025-04-29 13:28:41] (step=0031450) Train Loss: 5.1412, Train Steps/Sec: 1.07 + 39%|████████████████████████████████▊ | 8759/22715 [2:45:33<3:38:20, 1.07it/s][2025-04-29 13:29:04] (step=0031475) Train Loss: 5.1621, Train Steps/Sec: 1.07 + 39%|████████████████████████████████▊ | 8784/22715 [2:45:56<3:36:34, 1.07it/s][2025-04-29 13:29:28] (step=0031500) Train Loss: 5.1964, Train Steps/Sec: 1.07 + 39%|████████████████████████████████▉ | 8809/22715 [2:46:19<3:36:07, 1.07it/s][2025-04-29 13:29:51] (step=0031525) Train Loss: 5.1519, Train Steps/Sec: 1.07 + 39%|█████████████████████████████████ | 8834/22715 [2:46:43<3:34:43, 1.08it/s][2025-04-29 13:30:14] (step=0031550) Train Loss: 5.1665, Train Steps/Sec: 1.07 + 39%|█████████████████████████████████▏ | 8859/22715 [2:47:06<3:33:53, 1.08it/s][2025-04-29 13:30:38] (step=0031575) Train Loss: 5.1658, Train Steps/Sec: 1.07 + 39%|█████████████████████████████████▏ | 8884/22715 [2:47:29<3:33:04, 1.08it/s][2025-04-29 13:31:01] (step=0031600) Train Loss: 5.1682, Train Steps/Sec: 1.07 + 39%|█████████████████████████████████▎ | 8909/22715 [2:47:53<3:39:07, 1.05it/s][2025-04-29 13:31:25] (step=0031625) Train Loss: 5.1524, Train Steps/Sec: 1.07 + 39%|█████████████████████████████████▍ | 8934/22715 [2:48:16<3:36:03, 1.06it/s][2025-04-29 13:31:48] (step=0031650) Train Loss: 5.1551, Train Steps/Sec: 1.07 + 39%|█████████████████████████████████▌ | 8959/22715 [2:48:39<3:35:22, 1.06it/s][2025-04-29 13:32:11] (step=0031675) Train Loss: 5.1682, Train Steps/Sec: 1.07 + 40%|█████████████████████████████████▌ | 8984/22715 [2:49:03<3:33:41, 1.07it/s][2025-04-29 13:32:35] (step=0031700) Train Loss: 5.1800, Train Steps/Sec: 1.07 + 40%|█████████████████████████████████▋ | 9009/22715 [2:49:26<3:33:04, 1.07it/s][2025-04-29 13:32:58] (step=0031725) Train Loss: 5.1358, Train Steps/Sec: 1.07 + 40%|█████████████████████████████████▊ | 9034/22715 [2:49:49<3:31:42, 1.08it/s][2025-04-29 13:33:21] (step=0031750) Train Loss: 5.1828, Train Steps/Sec: 1.07 + 40%|█████████████████████████████████▉ | 9059/22715 [2:50:13<3:30:20, 1.08it/s][2025-04-29 13:33:45] (step=0031775) Train Loss: 5.1580, Train Steps/Sec: 1.07 + 40%|█████████████████████████████████▉ | 9084/22715 [2:50:36<3:30:30, 1.08it/s][2025-04-29 13:34:08] (step=0031800) Train Loss: 5.1388, Train Steps/Sec: 1.07 + 40%|██████████████████████████████████ | 9109/22715 [2:51:05<6:21:42, 1.68s/it][2025-04-29 13:34:36] (step=0031825) Train Loss: 5.1672, Train Steps/Sec: 0.88 + 40%|██████████████████████████████████▏ | 9134/22715 [2:51:28<3:33:02, 1.06it/s][2025-04-29 13:35:00] (step=0031850) Train Loss: 5.1664, Train Steps/Sec: 1.07 + 40%|██████████████████████████████████▎ | 9159/22715 [2:51:51<3:31:32, 1.07it/s][2025-04-29 13:35:23] (step=0031875) Train Loss: 5.1319, Train Steps/Sec: 1.07 + 40%|██████████████████████████████████▎ | 9184/22715 [2:52:15<3:29:57, 1.07it/s][2025-04-29 13:35:46] (step=0031900) Train Loss: 5.1869, Train Steps/Sec: 1.07 + 41%|██████████████████████████████████▍ | 9209/22715 [2:52:38<3:27:46, 1.08it/s][2025-04-29 13:36:10] (step=0031925) Train Loss: 5.1477, Train Steps/Sec: 1.07 + 41%|██████████████████████████████████▌ | 9234/22715 [2:53:01<3:28:33, 1.08it/s][2025-04-29 13:36:33] (step=0031950) Train Loss: 5.1386, Train Steps/Sec: 1.07 + 41%|██████████████████████████████████▋ | 9259/22715 [2:53:25<3:27:50, 1.08it/s][2025-04-29 13:36:56] (step=0031975) Train Loss: 5.1773, Train Steps/Sec: 1.07 + 41%|██████████████████████████████████▋ | 9284/22715 [2:53:48<3:27:14, 1.08it/s][2025-04-29 13:37:20] (step=0032000) Train Loss: 5.1650, Train Steps/Sec: 1.07 +You are using a model of type blip-2 to instantiate a model of type instructblip. This is not supported for all configurations of models and can yield errors. +[2025-04-29 13:37:20] text_config is None. Initializing the text config with default values (`OPTConfig`). +model_name_or_path:/tmp/haozhezhao/model/blip2-flan-t5-xl +image_place_holder: +tokenizer length after expend 32101 +tokenizer length before expend 32100 +100%|████████████████████████████████████████████████████████████████████████████████████████████████| 5/5 [04:54<00:00, 58.93s/it] +[2025-04-29 13:43:27] Finish Eval in 32000 steps...█████████████████████████████████████████| 5/5 [04:54<00:00, 58.69s/it] +processing left-padding... +processing left-padding... +processing left-padding... +processing left-padding... +processing left-padding... +[2025-04-29 13:43:47] Saved checkpoint to checkpoint/X2I_700k_3mask_200k_LLava_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_filtered_objects_3mask_888_context/003-GPT-XL/checkpoints/0032000.pt +[2025-04-29 13:43:49] Removed old checkpoint: checkpoint/X2I_700k_3mask_200k_LLava_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_filtered_objects_3mask_888_context/003-GPT-XL/checkpoints/0030000.pt + 41%|██████████████████████████████████▊ | 9309/22715 [3:00:41<3:37:40, 1.03it/s][2025-04-29 13:44:13] (step=0032025) Train Loss: 5.1620, Train Steps/Sec: 0.06 + 41%|██████████████████████████████████▉ | 9334/22715 [3:01:04<3:29:02, 1.07it/s][2025-04-29 13:44:36] (step=0032050) Train Loss: 5.2172, Train Steps/Sec: 1.07 + 41%|███████████████████████████████████ | 9359/22715 [3:01:27<3:28:12, 1.07it/s][2025-04-29 13:44:59] (step=0032075) Train Loss: 5.1401, Train Steps/Sec: 1.07 + 41%|███████████████████████████████████ | 9384/22715 [3:01:51<3:25:52, 1.08it/s][2025-04-29 13:45:22] (step=0032100) Train Loss: 5.2276, Train Steps/Sec: 1.07 + 41%|███████████████████████████████████▏ | 9409/22715 [3:02:14<3:26:00, 1.08it/s][2025-04-29 13:45:46] (step=0032125) Train Loss: 5.1276, Train Steps/Sec: 1.07 + 42%|███████████████████████████████████▎ | 9434/22715 [3:02:37<3:25:10, 1.08it/s][2025-04-29 13:46:09] (step=0032150) Train Loss: 5.1775, Train Steps/Sec: 1.07 + 42%|███████████████████████████████████▍ | 9459/22715 [3:03:01<3:24:16, 1.08it/s][2025-04-29 13:46:32] (step=0032175) Train Loss: 5.1354, Train Steps/Sec: 1.07 + 42%|███████████████████████████████████▍ | 9484/22715 [3:03:24<3:24:13, 1.08it/s][2025-04-29 13:46:56] (step=0032200) Train Loss: 5.1328, Train Steps/Sec: 1.07 + 42%|███████████████████████████████████▌ | 9509/22715 [3:03:47<3:29:35, 1.05it/s][2025-04-29 13:47:19] (step=0032225) Train Loss: 5.1753, Train Steps/Sec: 1.07 + 42%|███████████████████████████████████▋ | 9534/22715 [3:04:11<3:26:39, 1.06it/s][2025-04-29 13:47:43] (step=0032250) Train Loss: 5.1053, Train Steps/Sec: 1.07 + 42%|███████████████████████████████████▊ | 9559/22715 [3:04:34<3:24:58, 1.07it/s][2025-04-29 13:48:06] (step=0032275) Train Loss: 5.1737, Train Steps/Sec: 1.07 + 42%|███████████████████████████████████▊ | 9584/22715 [3:04:57<3:23:51, 1.07it/s][2025-04-29 13:48:29] (step=0032300) Train Loss: 5.1911, Train Steps/Sec: 1.07 + 42%|███████████████████████████████████▉ | 9609/22715 [3:05:21<3:22:35, 1.08it/s][2025-04-29 13:48:53] (step=0032325) Train Loss: 5.1951, Train Steps/Sec: 1.07 + 42%|████████████████████████████████████ | 9634/22715 [3:05:44<3:22:08, 1.08it/s][2025-04-29 13:49:16] (step=0032350) Train Loss: 5.1582, Train Steps/Sec: 1.07 + 43%|████████████████████████████████████▏ | 9659/22715 [3:06:07<3:21:27, 1.08it/s][2025-04-29 13:49:39] (step=0032375) Train Loss: 5.1355, Train Steps/Sec: 1.07 + 43%|████████████████████████████████████▏ | 9684/22715 [3:06:31<3:20:47, 1.08it/s][2025-04-29 13:50:03] (step=0032400) Train Loss: 5.1706, Train Steps/Sec: 1.07 + 43%|████████████████████████████████████▎ | 9709/22715 [3:06:54<3:26:50, 1.05it/s][2025-04-29 13:50:26] (step=0032425) Train Loss: 5.1769, Train Steps/Sec: 1.06 + 43%|████████████████████████████████████▍ | 9734/22715 [3:07:29<5:00:18, 1.39s/it][2025-04-29 13:51:01] (step=0032450) Train Loss: 5.0984, Train Steps/Sec: 0.72 + 43%|████████████████████████████████████▌ | 9759/22715 [3:07:52<3:22:12, 1.07it/s][2025-04-29 13:51:24] (step=0032475) Train Loss: 5.1699, Train Steps/Sec: 1.07 + 43%|████████████████████████████████████▌ | 9784/22715 [3:08:27<7:20:02, 2.04s/it][2025-04-29 13:51:59] (step=0032500) Train Loss: 5.1469, Train Steps/Sec: 0.73 + 43%|████████████████████████████████████▋ | 9809/22715 [3:08:55<5:11:01, 1.45s/it][2025-04-29 13:52:27] (step=0032525) Train Loss: 5.1873, Train Steps/Sec: 0.88 + 43%|████████████████████████████████████▊ | 9834/22715 [3:09:18<3:18:24, 1.08it/s][2025-04-29 13:52:50] (step=0032550) Train Loss: 5.1501, Train Steps/Sec: 1.07 + 43%|████████████████████████████████████▉ | 9859/22715 [3:09:42<3:18:30, 1.08it/s][2025-04-29 13:53:14] (step=0032575) Train Loss: 5.1711, Train Steps/Sec: 1.07 + 44%|████████████████████████████████████▉ | 9884/22715 [3:10:05<3:18:00, 1.08it/s][2025-04-29 13:53:37] (step=0032600) Train Loss: 5.2123, Train Steps/Sec: 1.07 + 44%|█████████████████████████████████████ | 9909/22715 [3:10:28<3:23:27, 1.05it/s][2025-04-29 13:54:00] (step=0032625) Train Loss: 5.1135, Train Steps/Sec: 1.07 + 44%|█████████████████████████████████████▏ | 9934/22715 [3:10:52<3:20:18, 1.06it/s][2025-04-29 13:54:24] (step=0032650) Train Loss: 5.1771, Train Steps/Sec: 1.07 + 44%|█████████████████████████████████████▎ | 9959/22715 [3:11:15<3:19:08, 1.07it/s][2025-04-29 13:54:47] (step=0032675) Train Loss: 5.1339, Train Steps/Sec: 1.07 + 44%|█████████████████████████████████████▎ | 9984/22715 [3:11:38<3:17:37, 1.07it/s][2025-04-29 13:55:10] (step=0032700) Train Loss: 5.1596, Train Steps/Sec: 1.07 + 44%|█████████████████████████████████████ | 10009/22715 [3:12:02<3:16:34, 1.08it/s][2025-04-29 13:55:34] (step=0032725) Train Loss: 5.1450, Train Steps/Sec: 1.07 + 44%|█████████████████████████████████████ | 10034/22715 [3:12:25<3:15:57, 1.08it/s][2025-04-29 13:55:57] (step=0032750) Train Loss: 5.1510, Train Steps/Sec: 1.07 + 44%|█████████████████████████████████████▏ | 10059/22715 [3:12:49<3:15:19, 1.08it/s][2025-04-29 13:56:20] (step=0032775) Train Loss: 5.1884, Train Steps/Sec: 1.07 + 44%|█████████████████████████████████████▎ | 10084/22715 [3:13:12<3:14:27, 1.08it/s][2025-04-29 13:56:44] (step=0032800) Train Loss: 5.1756, Train Steps/Sec: 1.07 + 45%|█████████████████████████████████████▍ | 10109/22715 [3:13:35<3:19:35, 1.05it/s][2025-04-29 13:57:07] (step=0032825) Train Loss: 5.1281, Train Steps/Sec: 1.07 + 45%|█████████████████████████████████████▍ | 10134/22715 [3:13:59<3:17:15, 1.06it/s][2025-04-29 13:57:30] (step=0032850) Train Loss: 5.1479, Train Steps/Sec: 1.07 + 45%|█████████████████████████████████████▌ | 10159/22715 [3:14:22<3:15:39, 1.07it/s][2025-04-29 13:57:54] (step=0032875) Train Loss: 5.1952, Train Steps/Sec: 1.07 + 45%|█████████████████████████████████████▋ | 10184/22715 [3:14:45<3:14:30, 1.07it/s][2025-04-29 13:58:17] (step=0032900) Train Loss: 5.2000, Train Steps/Sec: 1.07 + 45%|█████████████████████████████████████▊ | 10209/22715 [3:15:09<3:13:25, 1.08it/s][2025-04-29 13:58:40] (step=0032925) Train Loss: 5.1687, Train Steps/Sec: 1.07 + 45%|█████████████████████████████████████▊ | 10234/22715 [3:15:32<3:12:35, 1.08it/s][2025-04-29 13:59:04] (step=0032950) Train Loss: 5.1017, Train Steps/Sec: 1.07 + 45%|█████████████████████████████████████▉ | 10259/22715 [3:15:55<3:12:20, 1.08it/s][2025-04-29 13:59:27] (step=0032975) Train Loss: 5.1696, Train Steps/Sec: 1.07 + 45%|██████████████████████████████████████ | 10284/22715 [3:16:19<3:11:36, 1.08it/s][2025-04-29 13:59:50] (step=0033000) Train Loss: 5.1596, Train Steps/Sec: 1.07 + 45%|██████████████████████████████████████ | 10309/22715 [3:16:42<3:16:43, 1.05it/s][2025-04-29 14:00:14] (step=0033025) Train Loss: 5.1540, Train Steps/Sec: 1.07 + 45%|██████████████████████████████████████▏ | 10334/22715 [3:17:05<3:14:43, 1.06it/s][2025-04-29 14:00:37] (step=0033050) Train Loss: 5.1872, Train Steps/Sec: 1.07 + 46%|██████████████████████████████████████▎ | 10359/22715 [3:17:29<3:12:18, 1.07it/s][2025-04-29 14:01:00] (step=0033075) Train Loss: 5.1231, Train Steps/Sec: 1.07 + 46%|██████████████████████████████████████▍ | 10384/22715 [3:17:52<3:11:39, 1.07it/s][2025-04-29 14:01:24] (step=0033100) Train Loss: 5.1321, Train Steps/Sec: 1.07 + 46%|██████████████████████████████████████▍ | 10409/22715 [3:18:15<3:10:16, 1.08it/s][2025-04-29 14:01:47] (step=0033125) Train Loss: 5.0904, Train Steps/Sec: 1.07 + 46%|██████████████████████████████████████▌ | 10434/22715 [3:18:39<3:09:49, 1.08it/s][2025-04-29 14:02:10] (step=0033150) Train Loss: 5.0921, Train Steps/Sec: 1.07 + 46%|██████████████████████████████████████▋ | 10459/22715 [3:19:02<3:09:02, 1.08it/s][2025-04-29 14:02:34] (step=0033175) Train Loss: 5.1142, Train Steps/Sec: 1.07 + 46%|██████████████████████████████████████▊ | 10484/22715 [3:19:25<3:08:42, 1.08it/s][2025-04-29 14:02:57] (step=0033200) Train Loss: 5.1168, Train Steps/Sec: 1.07 + 46%|██████████████████████████████████████▊ | 10509/22715 [3:19:49<3:13:21, 1.05it/s][2025-04-29 14:03:21] (step=0033225) Train Loss: 5.1393, Train Steps/Sec: 1.07 + 46%|██████████████████████████████████████▉ | 10534/22715 [3:20:12<3:10:48, 1.06it/s][2025-04-29 14:03:44] (step=0033250) Train Loss: 5.0987, Train Steps/Sec: 1.07 + 46%|███████████████████████████████████████ | 10559/22715 [3:20:36<3:09:43, 1.07it/s][2025-04-29 14:04:07] (step=0033275) Train Loss: 5.2124, Train Steps/Sec: 1.07 + 47%|███████████████████████████████████████▏ | 10584/22715 [3:20:59<3:08:18, 1.07it/s][2025-04-29 14:04:31] (step=0033300) Train Loss: 5.2076, Train Steps/Sec: 1.07 + 47%|███████████████████████████████████████▏ | 10609/22715 [3:21:22<3:07:59, 1.07it/s][2025-04-29 14:04:54] (step=0033325) Train Loss: 5.1758, Train Steps/Sec: 1.07 + 47%|███████████████████████████████████████▎ | 10634/22715 [3:21:46<3:06:47, 1.08it/s][2025-04-29 14:05:17] (step=0033350) Train Loss: 5.0946, Train Steps/Sec: 1.07 + 47%|███████████████████████████████████████▍ | 10659/22715 [3:22:09<3:06:07, 1.08it/s][2025-04-29 14:05:41] (step=0033375) Train Loss: 5.1834, Train Steps/Sec: 1.07 + 47%|███████████████████████████████████████▌ | 10684/22715 [3:22:32<3:05:08, 1.08it/s][2025-04-29 14:06:04] (step=0033400) Train Loss: 5.1310, Train Steps/Sec: 1.07 + 47%|███████████████████████████████████████▌ | 10709/22715 [3:22:56<3:10:30, 1.05it/s][2025-04-29 14:06:27] (step=0033425) Train Loss: 5.1400, Train Steps/Sec: 1.07 + 47%|███████████████████████████████████████▋ | 10734/22715 [3:23:19<3:07:36, 1.06it/s][2025-04-29 14:06:51] (step=0033450) Train Loss: 5.1560, Train Steps/Sec: 1.07 + 47%|███████████████████████████████████████▊ | 10759/22715 [3:23:42<3:06:42, 1.07it/s][2025-04-29 14:07:14] (step=0033475) Train Loss: 5.1134, Train Steps/Sec: 1.07 + 47%|███████████████████████████████████████▉ | 10784/22715 [3:24:06<3:05:26, 1.07it/s][2025-04-29 14:07:37] (step=0033500) Train Loss: 5.1390, Train Steps/Sec: 1.07 + 48%|███████████████████████████████████████▉ | 10809/22715 [3:24:29<3:04:35, 1.07it/s][2025-04-29 14:08:01] (step=0033525) Train Loss: 5.1543, Train Steps/Sec: 1.07 + 48%|████████████████████████████████████████ | 10834/22715 [3:24:52<3:03:19, 1.08it/s][2025-04-29 14:08:24] (step=0033550) Train Loss: 5.1485, Train Steps/Sec: 1.07 + 48%|████████████████████████████████████████▏ | 10859/22715 [3:25:16<3:02:57, 1.08it/s][2025-04-29 14:08:48] (step=0033575) Train Loss: 5.1435, Train Steps/Sec: 1.07 + 48%|████████████████████████████████████████▏ | 10884/22715 [3:25:39<3:02:29, 1.08it/s][2025-04-29 14:09:11] (step=0033600) Train Loss: 5.1294, Train Steps/Sec: 1.07 + 48%|████████████████████████████████████████▎ | 10909/22715 [3:26:03<3:07:34, 1.05it/s][2025-04-29 14:09:34] (step=0033625) Train Loss: 5.1326, Train Steps/Sec: 1.07 + 48%|████████████████████████████████████████▍ | 10934/22715 [3:26:26<3:04:51, 1.06it/s][2025-04-29 14:09:58] (step=0033650) Train Loss: 5.2178, Train Steps/Sec: 1.07 + 48%|████████████████████████████████████████▌ | 10959/22715 [3:26:49<3:03:32, 1.07it/s][2025-04-29 14:10:21] (step=0033675) Train Loss: 5.1477, Train Steps/Sec: 1.07 + 48%|████████████████████████████████████████▌ | 10984/22715 [3:27:13<3:02:09, 1.07it/s][2025-04-29 14:10:44] (step=0033700) Train Loss: 5.0970, Train Steps/Sec: 1.07 + 48%|████████████████████████████████████████▋ | 11009/22715 [3:27:36<3:02:16, 1.07it/s][2025-04-29 14:11:08] (step=0033725) Train Loss: 5.1492, Train Steps/Sec: 1.07 + 49%|████████████████████████████████████████▊ | 11034/22715 [3:27:59<3:00:19, 1.08it/s][2025-04-29 14:11:31] (step=0033750) Train Loss: 5.1162, Train Steps/Sec: 1.07 + 49%|████████████████████████████████████████▉ | 11059/22715 [3:28:23<3:00:07, 1.08it/s][2025-04-29 14:11:54] (step=0033775) Train Loss: 5.1427, Train Steps/Sec: 1.07 + 49%|████████████████████████████████████████▉ | 11084/22715 [3:28:46<2:59:11, 1.08it/s][2025-04-29 14:12:18] (step=0033800) Train Loss: 5.1856, Train Steps/Sec: 1.07 + 49%|█████████████████████████████████████████ | 11109/22715 [3:29:14<3:37:48, 1.13s/it][2025-04-29 14:12:46] (step=0033825) Train Loss: 5.1038, Train Steps/Sec: 0.88 + 49%|█████████████████████████████████████████▏ | 11134/22715 [3:29:38<3:01:47, 1.06it/s][2025-04-29 14:13:10] (step=0033850) Train Loss: 5.0947, Train Steps/Sec: 1.07 + 49%|█████████████████████████████████████████▎ | 11159/22715 [3:30:01<3:00:37, 1.07it/s][2025-04-29 14:13:33] (step=0033875) Train Loss: 5.1601, Train Steps/Sec: 1.07 + 49%|█████████████████████████████████████████▎ | 11184/22715 [3:30:24<2:59:07, 1.07it/s][2025-04-29 14:13:56] (step=0033900) Train Loss: 5.0992, Train Steps/Sec: 1.07 + 49%|█████████████████████████████████████████▍ | 11209/22715 [3:30:48<2:58:27, 1.07it/s][2025-04-29 14:14:20] (step=0033925) Train Loss: 5.1427, Train Steps/Sec: 1.07 + 49%|█████████████████████████████████████████▌ | 11234/22715 [3:31:11<2:57:29, 1.08it/s][2025-04-29 14:14:43] (step=0033950) Train Loss: 5.1545, Train Steps/Sec: 1.07 + 50%|█████████████████████████████████████████▋ | 11259/22715 [3:31:34<2:57:06, 1.08it/s][2025-04-29 14:15:06] (step=0033975) Train Loss: 5.1132, Train Steps/Sec: 1.07 + 50%|█████████████████████████████████████████▋ | 11284/22715 [3:31:58<2:56:27, 1.08it/s][2025-04-29 14:15:30] (step=0034000) Train Loss: 5.1677, Train Steps/Sec: 1.07 +You are using a model of type blip-2 to instantiate a model of type instructblip. This is not supported for all configurations of models and can yield errors. +[2025-04-29 14:15:30] text_config is None. Initializing the text config with default values (`OPTConfig`). +model_name_or_path:/tmp/haozhezhao/model/blip2-flan-t5-xl +image_place_holder: +tokenizer length after expend 32101 +tokenizer length before expend 32100 + + 60%|█████████████████████████████████████████████████████████▌ | 3/5 [02:57<01:57, 58.95s/it] +processing left-padding... +processing left-padding... +processing left-padding... +processing left-padding... diff --git a/CKPTS/X2I_700k_3mask_200k_LLava_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_filtered_objects_3mask_888_context/wandb/run-20250429_032803-lbha1z77/files/requirements.txt b/CKPTS/X2I_700k_3mask_200k_LLava_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_filtered_objects_3mask_888_context/wandb/run-20250429_032803-lbha1z77/files/requirements.txt new file mode 100644 index 0000000000000000000000000000000000000000..06dc78369ffff807b210006a0e79d705ffe2a7d7 --- /dev/null +++ b/CKPTS/X2I_700k_3mask_200k_LLava_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_filtered_objects_3mask_888_context/wandb/run-20250429_032803-lbha1z77/files/requirements.txt @@ -0,0 +1,131 @@ +typing_extensions==4.12.2 +pyzmq==26.3.0 +nvidia-cufft-cu12==11.0.2.54 +triton==3.1.0 +nvidia-cublas-cu12==12.1.3.1 +psutil==7.0.0 +nvidia-cuda-cupti-cu12==12.1.105 +smmap==5.0.2 +nvidia-cuda-runtime-cu12==12.1.105 +aiohappyeyeballs==2.6.1 +asttokens==3.0.0 +huggingface-hub==0.29.3 +pyarrow==19.0.1 +fonttools==4.56.0 +python-dateutil==2.9.0.post0 +GitPython==3.1.44 +aiohttp==3.11.14 +wandb==0.19.8 +setproctitle==1.3.5 +PyYAML==6.0.2 +pydantic_core==2.27.2 +safetensors==0.5.3 +nvidia-nvjitlink-cu12==12.1.105 +aiosignal==1.3.2 +dill==0.3.8 +nvidia-cuda-nvrtc-cu12==12.1.105 +multiprocess==0.70.16 +pure_eval==0.2.3 +stack_data==0.6.3 +pydantic==2.10.6 +MarkupSafe==2.1.5 +tornado==6.4.2 +executing==2.1.0 +executing==2.2.0 +opencv-python==4.11.0.86 +nvitop==1.4.2 +multidict==6.2.0 +Jinja2==3.1.4 +torch==2.5.1+cu121 +nvidia-curand-cu12==10.3.2.106 +platformdirs==4.3.6 +six==1.17.0 +mpmath==1.3.0 +zipp==3.21.0 +packaging==24.2 +requests==2.32.3 +certifi==2025.1.31 +docker-pycreds==0.4.0 +torchvision==0.20.1+cu121 +pandas==2.2.3 +networkx==3.3 +exceptiongroup==1.2.2 +pickleshare==0.7.5 +tokenizers==0.21.1 +charset-normalizer==3.4.1 +jupyter_core==5.7.2 +wcwidth==0.2.13 +nvidia-nvtx-cu12==12.1.105 +prompt_toolkit==3.0.50 +fsspec==2024.12.0 +pillow==11.1.0 +propcache==0.3.0 +regex==2024.11.6 +ptyprocess==0.7.0 +contourpy==1.3.1 +importlib_metadata==8.6.1 +idna==3.10 +comm==0.2.2 +protobuf==5.29.3 +yarl==1.18.3 +ipython_pygments_lexers==1.1.1 +pip==25.0 +parso==0.8.4 +joblib==1.4.2 +nvidia-nccl-cu12==2.21.5 +hf_transfer==0.1.9 +Pygments==2.19.1 +decorator==5.2.1 +filelock==3.18.0 +nvidia-cusparse-cu12==12.1.0.106 +debugpy==1.8.13 +urllib3==2.3.0 +traitlets==5.14.3 +tzdata==2025.1 +matplotlib-inline==0.1.7 +matplotlib==3.10.1 +kiwisolver==1.4.8 +nest_asyncio==1.6.0 +frozenlist==1.5.0 +nvidia-ml-py==12.570.86 +transformers==4.49.0 +nltk==3.9.1 +ipykernel==6.29.5 +click==8.1.8 +gitdb==4.0.12 +pyparsing==3.2.1 +attrs==25.3.0 +jedi==0.19.2 +ipython==9.0.2 +nvidia-cudnn-cu12==9.1.0.70 +pexpect==4.9.0 +nvidia-cusolver-cu12==11.4.5.107 +numpy==2.2.4 +tqdm==4.67.1 +pytz==2025.1 +wheel==0.45.1 +sentry-sdk==2.23.1 +torchaudio==2.5.1+cu121 +jupyter_client==8.6.3 +cycler==0.12.1 +annotated-types==0.7.0 +sympy==1.13.1 +xxhash==3.5.0 +datasets==3.4.1 +setuptools==75.8.0 +typing_extensions==4.12.2 +wheel==0.43.0 +importlib_metadata==8.0.0 +backports.tarfile==1.2.0 +autocommand==2.2.2 +packaging==24.2 +tomli==2.0.1 +typeguard==4.3.0 +zipp==3.19.2 +jaraco.context==5.3.0 +jaraco.functools==4.0.1 +more-itertools==10.3.0 +platformdirs==4.2.2 +jaraco.text==3.12.1 +jaraco.collections==5.1.0 +inflect==7.3.1 diff --git a/CKPTS/X2I_700k_3mask_200k_LLava_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_filtered_objects_3mask_888_context/wandb/run-20250429_032803-lbha1z77/files/wandb-metadata.json b/CKPTS/X2I_700k_3mask_200k_LLava_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_filtered_objects_3mask_888_context/wandb/run-20250429_032803-lbha1z77/files/wandb-metadata.json new file mode 100644 index 0000000000000000000000000000000000000000..603f8dd562e973ad5f0203809ace4255a5243330 --- /dev/null +++ b/CKPTS/X2I_700k_3mask_200k_LLava_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_filtered_objects_3mask_888_context/wandb/run-20250429_032803-lbha1z77/files/wandb-metadata.json @@ -0,0 +1,149 @@ +{ + "os": "Linux-5.15.0-1064-azure-x86_64-with-glibc2.31", + "python": "CPython 3.11.11", + "startedAt": "2025-04-29T03:28:03.788030Z", + "args": [ + "--vq-ckpt", + "/tmp/haozhezhao/model/llamagen_t2i/vq_ds16_t2i.pt", + "--data-path", + "/tmp/haozhezhao/MLLMG/jsonl_data/X2I_Filtered_700k_segmentmask_3mask_200k_data_train.jsonl", + "--dataset", + "ti2i", + "--image-size", + "512", + "--results-dir", + "checkpoint/X2I_700k_3mask_200k_LLava_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_filtered_objects_3mask_888_context", + "--cloud-save-path", + "/tmp/haozhezhao/MLLMG/checkpoint", + "--lr", + "3e-4", + "--val_data_path", + "/tmp/haozhezhao/MLLMG/jsonl_data/multiimage_val_for_llava_X2I_threemask_dreambenplus.jsonl", + "--use_vision_tower", + "--model_name_or_path", + "/tmp/haozhezhao/model/blip2-flan-t5-xl", + "--image_place_holder", + "", + "--do_eval", + "--eval_steps", + "2000", + "--max_eval_samples", + "200", + "--cfg-scale", + "7.5", + "--top-k", + "16384", + "--load_from_checkpoint", + "/tmp/haozhezhao/MLLMG/checkpoint/FIXed_3M_1epoch_step32000.pt", + "--global-batch-size", + "40", + "--num-workers", + "4", + "--warmup", + "0.05", + "--gradient-accumulation-steps", + "8", + "--train_text_encoder", + "--ckpt-every", + "2000", + "--epochs", + "2", + "--subject_driven", + "--reference_data_path", + "/tmp/haozhezhao/MLLMG/cc12m_reference_tunnel.jsonl", + "--multimodal_encoder", + "llava", + "--do_recovery", + "--find_unused_parameters", + "--cls-token-num", + "888", + "--train_all", + "--load_fixed_llamagen", + "--save_total_limit", + "1", + "--fix", + "gpt-empty-fix", + "--load_language_projection", + "/tmp/haozhezhao/MLLMG/llava-v1.5-flant5_fixed-pretrain/mm_projector.bin", + "--mm_vision_tower", + "openai/clip-vit-large-patch14", + "--gpt-ckpt", + "/tmp/haozhezhao/MLLMG/checkpoint/Afteri2i_100kckpt_LLAVAT5_empty_Fixed_just_segment_pretrain_stage3_dreambench__recap_400k_t2i_400k_flux_200k_midjourney_150k_recovery_150k_grounding_100fluxseg_50samseg/005-GPT-XL/checkpoints/0078000.pt", + "--dreambench_eval" + ], + "program": "/tmp/haozhezhao/MLLMG/autoregressive/train/train_t2i.py", + "codePath": "autoregressive/train/train_t2i.py", + "email": "mimazhe55360@gmail.com", + "root": "checkpoint/X2I_700k_3mask_200k_LLava_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_filtered_objects_3mask_888_context", + "host": "447cc403a8794092814259713c51c1df00001X", + "executable": "/tmp/haozhezhao/anaconda3/envs/nlp/bin/python", + "codePathLocal": "autoregressive/train/train_t2i.py", + "cpu_count": 96, + "cpu_count_logical": 96, + "gpu": "NVIDIA A100-SXM4-80GB", + "gpu_count": 8, + "disk": { + "/": { + "total": "133003395072", + "used": "67545722880" + } + }, + "memory": { + "total": "1902387884032" + }, + "cpu": { + "count": 96, + "countLogical": 96 + }, + "gpu_nvidia": [ + { + "name": "NVIDIA A100-SXM4-80GB", + "memoryTotal": "85899345920", + "cudaCores": 6912, + "architecture": "Ampere" + }, + { + "name": "NVIDIA A100-SXM4-80GB", + "memoryTotal": "85899345920", + "cudaCores": 6912, + "architecture": "Ampere" + }, + { + "name": "NVIDIA A100-SXM4-80GB", + "memoryTotal": "85899345920", + "cudaCores": 6912, + "architecture": "Ampere" + }, + { + "name": "NVIDIA A100-SXM4-80GB", + "memoryTotal": "85899345920", + "cudaCores": 6912, + "architecture": "Ampere" + }, + { + "name": "NVIDIA A100-SXM4-80GB", + "memoryTotal": "85899345920", + "cudaCores": 6912, + "architecture": "Ampere" + }, + { + "name": "NVIDIA A100-SXM4-80GB", + "memoryTotal": "85899345920", + "cudaCores": 6912, + "architecture": "Ampere" + }, + { + "name": "NVIDIA A100-SXM4-80GB", + "memoryTotal": "85899345920", + "cudaCores": 6912, + "architecture": "Ampere" + }, + { + "name": "NVIDIA A100-SXM4-80GB", + "memoryTotal": "85899345920", + "cudaCores": 6912, + "architecture": "Ampere" + } + ], + "cudaVersion": "12.2" +} \ No newline at end of file diff --git a/CKPTS/X2I_700k_3mask_200k_LLava_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_filtered_objects_3mask_888_context/wandb/run-20250429_032803-lbha1z77/logs/debug-core.log b/CKPTS/X2I_700k_3mask_200k_LLava_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_filtered_objects_3mask_888_context/wandb/run-20250429_032803-lbha1z77/logs/debug-core.log new file mode 100644 index 0000000000000000000000000000000000000000..e5563f85ce69798e4a12fe716f4523d8ff73e2f4 --- /dev/null +++ b/CKPTS/X2I_700k_3mask_200k_LLava_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_filtered_objects_3mask_888_context/wandb/run-20250429_032803-lbha1z77/logs/debug-core.log @@ -0,0 +1,7 @@ +{"time":"2025-04-29T03:28:03.243465735Z","level":"INFO","msg":"main: starting server","port-filename":"/tmp/tmpa28o36f0/port-3895261.txt","pid":3895261,"log-level":0,"disable-analytics":false,"shutdown-on-parent-exit":false} +{"time":"2025-04-29T03:28:03.244914199Z","level":"INFO","msg":"Will exit if parent process dies.","ppid":3895261} +{"time":"2025-04-29T03:28:03.244898659Z","level":"INFO","msg":"server is running","addr":{"IP":"127.0.0.1","Port":36167,"Zone":""}} +{"time":"2025-04-29T03:28:03.429391961Z","level":"INFO","msg":"connection: ManageConnectionData: new connection created","id":"127.0.0.1:42712"} +{"time":"2025-04-29T03:28:03.789428812Z","level":"INFO","msg":"handleInformInit: received","streamId":"lbha1z77","id":"127.0.0.1:42712"} +{"time":"2025-04-29T03:28:03.996379483Z","level":"INFO","msg":"handleInformInit: stream started","streamId":"lbha1z77","id":"127.0.0.1:42712"} +{"time":"2025-04-29T14:20:12.224308384Z","level":"INFO","msg":"Parent process exited, terminating service process."} diff --git a/CKPTS/X2I_700k_3mask_200k_LLava_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_filtered_objects_3mask_888_context/wandb/run-20250429_032803-lbha1z77/logs/debug-internal.log b/CKPTS/X2I_700k_3mask_200k_LLava_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_filtered_objects_3mask_888_context/wandb/run-20250429_032803-lbha1z77/logs/debug-internal.log new file mode 100644 index 0000000000000000000000000000000000000000..8660b39079613a1da2fe40dddf165f206ea7a30c --- /dev/null +++ b/CKPTS/X2I_700k_3mask_200k_LLava_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_filtered_objects_3mask_888_context/wandb/run-20250429_032803-lbha1z77/logs/debug-internal.log @@ -0,0 +1,9 @@ +{"time":"2025-04-29T03:28:03.789661958Z","level":"INFO","msg":"stream: starting","core version":"0.19.8","symlink path":"checkpoint/X2I_700k_3mask_200k_LLava_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_filtered_objects_3mask_888_context/wandb/run-20250429_032803-lbha1z77/logs/debug-core.log"} +{"time":"2025-04-29T03:28:03.996327866Z","level":"INFO","msg":"created new stream","id":"lbha1z77"} +{"time":"2025-04-29T03:28:03.996373963Z","level":"INFO","msg":"stream: started","id":"lbha1z77"} +{"time":"2025-04-29T03:28:03.996405322Z","level":"INFO","msg":"writer: Do: started","stream_id":"lbha1z77"} +{"time":"2025-04-29T03:28:03.996438033Z","level":"INFO","msg":"handler: started","stream_id":"lbha1z77"} +{"time":"2025-04-29T03:28:03.996459228Z","level":"INFO","msg":"sender: started","stream_id":"lbha1z77"} +{"time":"2025-04-29T03:28:04.307262596Z","level":"INFO","msg":"Starting system monitor"} +{"time":"2025-04-29T10:37:34.901441196Z","level":"INFO","msg":"api: retrying HTTP error","status":502,"url":"https://api.wandb.ai/files/haozhezhao/llamagen_ti2i/lbha1z77/file_stream","body":"\n\n\n502 Server Error\n\n\n

Error: Server Error

\n

The server encountered a temporary error and could not complete your request.

Please try again in 30 seconds.

\n

\n\n"} +{"time":"2025-04-29T12:07:20.125722287Z","level":"INFO","msg":"api: retrying error","error":"Post \"https://api.wandb.ai/graphql\": context deadline exceeded (Client.Timeout exceeded while awaiting headers)"} diff --git a/CKPTS/X2I_700k_3mask_200k_LLava_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_filtered_objects_3mask_888_context/wandb/run-20250429_032803-lbha1z77/logs/debug.log b/CKPTS/X2I_700k_3mask_200k_LLava_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_filtered_objects_3mask_888_context/wandb/run-20250429_032803-lbha1z77/logs/debug.log new file mode 100644 index 0000000000000000000000000000000000000000..22894f56c2bb47d0ae24afe898c3ff1bb0a255a7 --- /dev/null +++ b/CKPTS/X2I_700k_3mask_200k_LLava_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_filtered_objects_3mask_888_context/wandb/run-20250429_032803-lbha1z77/logs/debug.log @@ -0,0 +1,22 @@ +2025-04-29 03:28:03,782 INFO MainThread:3895261 [wandb_setup.py:_flush():67] Current SDK version is 0.19.8 +2025-04-29 03:28:03,782 INFO MainThread:3895261 [wandb_setup.py:_flush():67] Configure stats pid to 3895261 +2025-04-29 03:28:03,782 INFO MainThread:3895261 [wandb_setup.py:_flush():67] Loading settings from /tmp/haozhezhao/.config/wandb/settings +2025-04-29 03:28:03,782 INFO MainThread:3895261 [wandb_setup.py:_flush():67] Loading settings from /tmp/haozhezhao/MLLMG/wandb/settings +2025-04-29 03:28:03,782 INFO MainThread:3895261 [wandb_setup.py:_flush():67] Loading settings from environment variables +2025-04-29 03:28:03,782 INFO MainThread:3895261 [wandb_init.py:setup_run_log_directory():647] Logging user logs to checkpoint/X2I_700k_3mask_200k_LLava_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_filtered_objects_3mask_888_context/wandb/run-20250429_032803-lbha1z77/logs/debug.log +2025-04-29 03:28:03,782 INFO MainThread:3895261 [wandb_init.py:setup_run_log_directory():648] Logging internal logs to checkpoint/X2I_700k_3mask_200k_LLava_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_filtered_objects_3mask_888_context/wandb/run-20250429_032803-lbha1z77/logs/debug-internal.log +2025-04-29 03:28:03,782 INFO MainThread:3895261 [wandb_init.py:init():761] calling init triggers +2025-04-29 03:28:03,782 INFO MainThread:3895261 [wandb_init.py:init():766] wandb.init called with sweep_config: {} +config: {'data_path': '/tmp/haozhezhao/MLLMG/jsonl_data/X2I_Filtered_700k_segmentmask_3mask_200k_data_train.jsonl', 'cloud_save_path': '/tmp/haozhezhao/MLLMG/checkpoint', 'no_local_save': False, 'vq_model': 'VQ-16', 'vq_ckpt': '/tmp/haozhezhao/model/llamagen_t2i/vq_ds16_t2i.pt', 'codebook_size': 16384, 'codebook_embed_dim': 8, 'gpt_model': 'GPT-XL', 'gpt_ckpt': '/tmp/haozhezhao/MLLMG/checkpoint/Afteri2i_100kckpt_LLAVAT5_empty_Fixed_just_segment_pretrain_stage3_dreambench__recap_400k_t2i_400k_flux_200k_midjourney_150k_recovery_150k_grounding_100fluxseg_50samseg/005-GPT-XL/checkpoints/0078000.pt', 'gpt_type': 't2i', 'vocab_size': 16384, 'cls_token_num': 888, 'dropout_p': 0.1, 'token_dropout_p': 0.1, 'drop_path': 0.0, 'no_compile': False, 'results_dir': 'checkpoint/X2I_700k_3mask_200k_LLava_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_filtered_objects_3mask_888_context', 'dataset': 'ti2i', 'image_size': 512, 'downsample_size': 16, 'num_classes': 1000, 'epochs': 2, 'lr': 0.0003, 'weight_decay': 0.05, 'beta1': 0.9, 'beta2': 0.95, 'max_grad_norm': 1.0, 'global_batch_size': 40, 'global_seed': 0, 'num_workers': 4, 'log_every': 25, 'ckpt_every': 2000, 'gradient_accumulation_steps': 8, 'mixed_precision': 'bf16', 'val_data_path': '/tmp/haozhezhao/MLLMG/jsonl_data/multiimage_val_for_llava_X2I_threemask_dreambenplus.jsonl', 'use_vision_tower': True, 'model_name_or_path': '/tmp/haozhezhao/model/blip2-flan-t5-xl', 'image_place_holder': '', 'processor_path': None, 'do_eval': True, 'max_eval_samples': 200, 'train_text_encoder': True, 'no_left_padding': False, 'cfg_scale': 7.5, 'top_k': 16384, 'temperature': 0.9, 'top_p': 1.0, 'eval_steps': 2000, 'project_name': 'llamagen_ti2i', 'load_from_checkpoint': '/tmp/haozhezhao/MLLMG/checkpoint/FIXed_3M_1epoch_step32000.pt', 'warmup': 0.05, 'lr_decay_style': 'cosine', 'lr_decay_ratio': 0.1, 'train_iters': 500000, 'class_dropout_prob': 0.1, 'with_image_only': False, 'image_only_rate': 0.1, 'stage2': False, 'subject_driven': True, 'load_subject_embedding': None, 'reference_data_path': '/tmp/haozhezhao/MLLMG/cc12m_reference_tunnel.jsonl', 'multimodal_encoder': 'llava', 'do_recovery': True, 'no_replace': False, 'resume': False, 'dreambench_eval': True, 'find_unused_parameters': True, 'load_visual_encoder': False, 'continue_stage1': False, 'replace_subject': False, 'train_all': True, 'save_total_limit': 1, 'load_language_projection': '/tmp/haozhezhao/MLLMG/llava-v1.5-flant5_fixed-pretrain/mm_projector.bin', 'mm_vision_tower': 'openai/clip-vit-large-patch14', 'load_fixed_llamagen': True, 'unfreeze_output': False, 'fix': 'gpt-empty-fix', 'rank': 0, 'world_size': 8, 'gpu': 0, 'dist_url': 'env://', 'distributed': True, 'dist_backend': 'nccl', '_wandb': {}} +2025-04-29 03:28:03,782 INFO MainThread:3895261 [wandb_init.py:init():784] starting backend +2025-04-29 03:28:03,782 INFO MainThread:3895261 [wandb_init.py:init():788] sending inform_init request +2025-04-29 03:28:03,787 INFO MainThread:3895261 [backend.py:_multiprocessing_setup():101] multiprocessing start_methods=fork,spawn,forkserver, using: spawn +2025-04-29 03:28:03,787 INFO MainThread:3895261 [wandb_init.py:init():798] backend started and connected +2025-04-29 03:28:03,789 INFO MainThread:3895261 [wandb_init.py:init():891] updated telemetry +2025-04-29 03:28:03,790 INFO MainThread:3895261 [wandb_init.py:init():915] communicating run to backend with 90.0 second timeout +2025-04-29 03:28:04,304 INFO MainThread:3895261 [wandb_init.py:init():990] starting run threads in backend +2025-04-29 03:28:04,392 INFO MainThread:3895261 [wandb_run.py:_console_start():2375] atexit reg +2025-04-29 03:28:04,393 INFO MainThread:3895261 [wandb_run.py:_redirect():2227] redirect: wrap_raw +2025-04-29 03:28:04,393 INFO MainThread:3895261 [wandb_run.py:_redirect():2292] Wrapping output streams. +2025-04-29 03:28:04,393 INFO MainThread:3895261 [wandb_run.py:_redirect():2315] Redirects installed. +2025-04-29 03:28:04,395 INFO MainThread:3895261 [wandb_init.py:init():1032] run started, returning control to user process diff --git a/CKPTS/X2I_700k_3mask_200k_LLava_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_filtered_objects_3mask_888_context/wandb/run-20250429_032803-lbha1z77/run-lbha1z77.wandb b/CKPTS/X2I_700k_3mask_200k_LLava_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_filtered_objects_3mask_888_context/wandb/run-20250429_032803-lbha1z77/run-lbha1z77.wandb new file mode 100644 index 0000000000000000000000000000000000000000..2d1acd1bb5afc7e0384829c68cff5f3553480bdd --- /dev/null +++ b/CKPTS/X2I_700k_3mask_200k_LLava_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_filtered_objects_3mask_888_context/wandb/run-20250429_032803-lbha1z77/run-lbha1z77.wandb @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:91f618e417eec0719d58b86d168b683d39b32c7f34ac8fb0bf68440820976a64 +size 19038208 diff --git a/CKPTS/X2I_UltraEdiit_fourmask_LLava_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_filtered_objects_4mask_1280_context/000-GPT-XL/log.txt b/CKPTS/X2I_UltraEdiit_fourmask_LLava_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_filtered_objects_4mask_1280_context/000-GPT-XL/log.txt new file mode 100644 index 0000000000000000000000000000000000000000..87ca5c81c97ebb9d3e03b99d886503cf42b433ce --- /dev/null +++ b/CKPTS/X2I_UltraEdiit_fourmask_LLava_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_filtered_objects_4mask_1280_context/000-GPT-XL/log.txt @@ -0,0 +1,10 @@ +[2025-04-26 20:35:07] Experiment directory created at checkpoint/X2I_UltraEdiit_fourmask_LLava_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_filtered_objects_4mask_1280_context/000-GPT-XL +[2025-04-26 20:35:07] Namespace(data_path='/tmp/haozhezhao/MLLMG/jsonl_data/multiimage_training_for_llava_X2I_UltraEdiit_fourmask.jsonl', cloud_save_path='/tmp/haozhezhao/MLLMG/checkpoint', no_local_save=False, vq_model='VQ-16', vq_ckpt='/tmp/haozhezhao/model/llamagen_t2i/vq_ds16_t2i.pt', codebook_size=16384, codebook_embed_dim=8, gpt_model='GPT-XL', gpt_ckpt='/tmp/haozhezhao/MLLMG/checkpoint/Afteri2i_100kckpt_LLAVAT5_empty_Fixed_just_segment_pretrain_stage3_dreambench__recap_400k_t2i_400k_flux_200k_midjourney_150k_recovery_150k_grounding_100fluxseg_50samseg/005-GPT-XL/checkpoints/0078000.pt', gpt_type='t2i', vocab_size=16384, cls_token_num=1280, dropout_p=0.1, token_dropout_p=0.1, drop_path=0.0, no_compile=False, results_dir='checkpoint/X2I_UltraEdiit_fourmask_LLava_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_filtered_objects_4mask_1280_context', dataset='ti2i', image_size=512, downsample_size=16, num_classes=1000, epochs=1, lr=0.0001, weight_decay=0.05, beta1=0.9, beta2=0.95, max_grad_norm=1.0, global_batch_size=24, global_seed=0, num_workers=4, log_every=25, ckpt_every=2000, gradient_accumulation_steps=8, mixed_precision='bf16', val_data_path='/tmp/haozhezhao/MLLMG/jsonl_data/multiimage_val_for_llava_X2I_UltraEdiit_fourmask_dreambenplus.jsonl', use_vision_tower=True, model_name_or_path='/tmp/haozhezhao/model/blip2-flan-t5-xl', image_place_holder='', processor_path=None, do_eval=True, max_eval_samples=128, train_text_encoder=True, no_left_padding=False, cfg_scale=7.5, top_k=16384, temperature=0.9, top_p=1.0, eval_steps=2000, project_name='llamagen_ti2i', load_from_checkpoint='/tmp/haozhezhao/MLLMG/checkpoint/FIXed_3M_1epoch_step32000.pt', warmup=0.05, lr_decay_style='cosine', lr_decay_ratio=0.1, train_iters=500000, class_dropout_prob=0.1, with_image_only=False, image_only_rate=0.1, stage2=False, subject_driven=True, load_subject_embedding=None, reference_data_path='/tmp/haozhezhao/MLLMG/cc12m_reference_tunnel.jsonl', multimodal_encoder='llava', do_recovery=True, no_replace=False, resume=False, dreambench_eval=True, find_unused_parameters=True, load_visual_encoder=False, continue_stage1=False, replace_subject=False, train_all=True, save_total_limit=1, load_language_projection='/tmp/haozhezhao/MLLMG/llava-v1.5-flant5_fixed-pretrain/mm_projector.bin', mm_vision_tower='openai/clip-vit-large-patch14', load_fixed_llamagen=True, unfreeze_output=False, fix='gpt-empty-fix', rank=0, world_size=8, gpu=0, dist_url='env://', distributed=True, dist_backend='nccl') +[2025-04-26 20:35:07] Starting rank=0, seed=0, world_size=8. +[2025-04-26 20:35:07] text_config is None. Initializing the text config with default values (`OPTConfig`). +[2025-04-26 20:36:12] GPT Parameters: 2,310,680,832 +[2025-04-26 20:36:12] num decayed parameter tensors: 356, with 2,007,303,168 parameters +[2025-04-26 20:36:12] num non-decayed parameter tensors: 124, with 197,888 parameters +[2025-04-26 20:36:12] using fused AdamW: True +[2025-04-26 20:36:43] Dataset contains 2,656,708 images +[2025-04-26 20:36:43] Train iters 110696 , warmup 5534.8, len of loader 110696 diff --git a/CKPTS/X2I_UltraEdiit_fourmask_LLava_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_filtered_objects_4mask_1280_context/001-GPT-XL/log.txt b/CKPTS/X2I_UltraEdiit_fourmask_LLava_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_filtered_objects_4mask_1280_context/001-GPT-XL/log.txt new file mode 100644 index 0000000000000000000000000000000000000000..6b9523f4f5b75d37e709f39b10df6364b743b92f --- /dev/null +++ b/CKPTS/X2I_UltraEdiit_fourmask_LLava_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_filtered_objects_4mask_1280_context/001-GPT-XL/log.txt @@ -0,0 +1,25 @@ +[2025-04-26 20:51:10] Experiment directory created at checkpoint/X2I_UltraEdiit_fourmask_LLava_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_filtered_objects_4mask_1280_context/001-GPT-XL +[2025-04-26 20:51:10] Namespace(data_path='/tmp/haozhezhao/MLLMG/jsonl_data/multiimage_training_for_llava_X2I_UltraEdiit_fourmask.jsonl', cloud_save_path='/tmp/haozhezhao/MLLMG/checkpoint', no_local_save=False, vq_model='VQ-16', vq_ckpt='/tmp/haozhezhao/model/llamagen_t2i/vq_ds16_t2i.pt', codebook_size=16384, codebook_embed_dim=8, gpt_model='GPT-XL', gpt_ckpt='/tmp/haozhezhao/MLLMG/checkpoint/Afteri2i_100kckpt_LLAVAT5_empty_Fixed_just_segment_pretrain_stage3_dreambench__recap_400k_t2i_400k_flux_200k_midjourney_150k_recovery_150k_grounding_100fluxseg_50samseg/005-GPT-XL/checkpoints/0078000.pt', gpt_type='t2i', vocab_size=16384, cls_token_num=1280, dropout_p=0.1, token_dropout_p=0.1, drop_path=0.0, no_compile=False, results_dir='checkpoint/X2I_UltraEdiit_fourmask_LLava_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_filtered_objects_4mask_1280_context', dataset='ti2i', image_size=512, downsample_size=16, num_classes=1000, epochs=1, lr=0.0001, weight_decay=0.05, beta1=0.9, beta2=0.95, max_grad_norm=1.0, global_batch_size=24, global_seed=0, num_workers=4, log_every=25, ckpt_every=2000, gradient_accumulation_steps=8, mixed_precision='bf16', val_data_path='/tmp/haozhezhao/MLLMG/jsonl_data/multiimage_val_for_llava_X2I_UltraEdiit_fourmask_dreambenplus.jsonl', use_vision_tower=True, model_name_or_path='/tmp/haozhezhao/model/blip2-flan-t5-xl', image_place_holder='', processor_path=None, do_eval=True, max_eval_samples=128, train_text_encoder=True, no_left_padding=False, cfg_scale=7.5, top_k=16384, temperature=0.9, top_p=1.0, eval_steps=2000, project_name='llamagen_ti2i', load_from_checkpoint='/tmp/haozhezhao/MLLMG/checkpoint/FIXed_3M_1epoch_step32000.pt', warmup=0.05, lr_decay_style='cosine', lr_decay_ratio=0.1, train_iters=500000, class_dropout_prob=0.1, with_image_only=False, image_only_rate=0.1, stage2=False, subject_driven=True, load_subject_embedding=None, reference_data_path='/tmp/haozhezhao/MLLMG/cc12m_reference_tunnel.jsonl', multimodal_encoder='llava', do_recovery=True, no_replace=False, resume=False, dreambench_eval=True, find_unused_parameters=True, load_visual_encoder=False, continue_stage1=False, replace_subject=False, train_all=True, save_total_limit=1, load_language_projection='/tmp/haozhezhao/MLLMG/llava-v1.5-flant5_fixed-pretrain/mm_projector.bin', mm_vision_tower='openai/clip-vit-large-patch14', load_fixed_llamagen=True, unfreeze_output=False, fix='gpt-empty-fix', rank=0, world_size=8, gpu=0, dist_url='env://', distributed=True, dist_backend='nccl') +[2025-04-26 20:51:10] Starting rank=0, seed=0, world_size=8. +[2025-04-26 20:51:10] text_config is None. Initializing the text config with default values (`OPTConfig`). +[2025-04-26 20:52:16] GPT Parameters: 2,310,680,832 +[2025-04-26 20:52:16] num decayed parameter tensors: 356, with 2,007,303,168 parameters +[2025-04-26 20:52:16] num non-decayed parameter tensors: 124, with 197,888 parameters +[2025-04-26 20:52:16] using fused AdamW: True +[2025-04-26 20:52:47] Dataset contains 2,656,708 images +[2025-04-26 20:52:47] Train iters 110696 , warmup 5534.8, len of loader 110696 +[2025-04-26 20:53:17] ### LOAD pretraining weights from checkpoint: /tmp/haozhezhao/MLLMG/checkpoint/Afteri2i_100kckpt_LLAVAT5_empty_Fixed_just_segment_pretrain_stage3_dreambench__recap_400k_t2i_400k_flux_200k_midjourney_150k_recovery_150k_grounding_100fluxseg_50samseg/005-GPT-XL/checkpoints/0078000.pt +[2025-04-26 20:53:17] Initial state: steps=0, epochs=0 +[2025-04-26 20:53:17] compiling the model... (may take several minutes) +[2025-04-26 20:53:18] freeze the vit +[2025-04-26 20:53:18] ***** total param is 2310680832 ***** +[2025-04-26 20:53:18] ***** total trained param is 2007501056 ***** +[2025-04-26 20:53:20] Training for 1 epochs... +[2025-04-26 20:53:20] Beginning epoch 0... +[2025-04-26 20:57:35] (step=0000025) Train Loss: 5.1578, Train Steps/Sec: 0.10 +[2025-04-26 20:57:57] (step=0000050) Train Loss: 5.0694, Train Steps/Sec: 1.12 +[2025-04-26 20:58:20] (step=0000075) Train Loss: 4.8793, Train Steps/Sec: 1.12 +[2025-04-26 20:58:42] (step=0000100) Train Loss: 5.1843, Train Steps/Sec: 1.12 +[2025-04-26 20:59:04] (step=0000125) Train Loss: 4.8762, Train Steps/Sec: 1.12 +[2025-04-26 20:59:26] (step=0000150) Train Loss: 5.1708, Train Steps/Sec: 1.12 +[2025-04-26 20:59:49] (step=0000175) Train Loss: 5.1110, Train Steps/Sec: 1.12 diff --git a/CKPTS/X2I_UltraEdiit_fourmask_LLava_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_filtered_objects_4mask_1280_context/003-GPT-XL/checkpoints/0110000.pt b/CKPTS/X2I_UltraEdiit_fourmask_LLava_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_filtered_objects_4mask_1280_context/003-GPT-XL/checkpoints/0110000.pt new file mode 100644 index 0000000000000000000000000000000000000000..e88766a4bfe4f92b91e7a6fb840c029d453ac088 --- /dev/null +++ b/CKPTS/X2I_UltraEdiit_fourmask_LLava_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_filtered_objects_4mask_1280_context/003-GPT-XL/checkpoints/0110000.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:2e76fd988b092d7adb7453981dc94be92b54f15ef637da1aaf79075907dcd2c0 +size 17328771130 diff --git a/CKPTS/X2I_UltraEdiit_fourmask_LLava_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_filtered_objects_4mask_1280_context/003-GPT-XL/checkpoints/eval_step_10000/batch_0_cfg_7.5_topk_16384.jpg b/CKPTS/X2I_UltraEdiit_fourmask_LLava_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_filtered_objects_4mask_1280_context/003-GPT-XL/checkpoints/eval_step_10000/batch_0_cfg_7.5_topk_16384.jpg new file mode 100644 index 0000000000000000000000000000000000000000..38482629346e825a3691e682722496d6912aac28 --- /dev/null +++ b/CKPTS/X2I_UltraEdiit_fourmask_LLava_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_filtered_objects_4mask_1280_context/003-GPT-XL/checkpoints/eval_step_10000/batch_0_cfg_7.5_topk_16384.jpg @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:9a6486f7130a50b9e9ee0df23260d77eb37baf4073d3e2083b0ab1943a83770b +size 466183 diff --git a/CKPTS/X2I_UltraEdiit_fourmask_LLava_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_filtered_objects_4mask_1280_context/003-GPT-XL/checkpoints/eval_step_10000/batch_1_cfg_7.5_topk_16384.jpg b/CKPTS/X2I_UltraEdiit_fourmask_LLava_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_filtered_objects_4mask_1280_context/003-GPT-XL/checkpoints/eval_step_10000/batch_1_cfg_7.5_topk_16384.jpg new file mode 100644 index 0000000000000000000000000000000000000000..43d2a2df5760bb4f1bfd218f4e93858ebd4f85bc --- /dev/null +++ b/CKPTS/X2I_UltraEdiit_fourmask_LLava_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_filtered_objects_4mask_1280_context/003-GPT-XL/checkpoints/eval_step_10000/batch_1_cfg_7.5_topk_16384.jpg @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:1bbebb6af05e91ab32dadde077c65f2f1eb9531df5239d6a92844851757d5f69 +size 433364 diff --git a/CKPTS/X2I_UltraEdiit_fourmask_LLava_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_filtered_objects_4mask_1280_context/003-GPT-XL/checkpoints/eval_step_10000/batch_2_cfg_7.5_topk_16384.jpg b/CKPTS/X2I_UltraEdiit_fourmask_LLava_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_filtered_objects_4mask_1280_context/003-GPT-XL/checkpoints/eval_step_10000/batch_2_cfg_7.5_topk_16384.jpg new file mode 100644 index 0000000000000000000000000000000000000000..f2e21aa79f3ba5f4c967ec9de78ba139e615216e --- /dev/null +++ b/CKPTS/X2I_UltraEdiit_fourmask_LLava_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_filtered_objects_4mask_1280_context/003-GPT-XL/checkpoints/eval_step_10000/batch_2_cfg_7.5_topk_16384.jpg @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:f9eea78c6cbc33e186e008dbb8927bc122f4cd7ce3f5ccd715bed3f1a9f5083d +size 413994 diff --git a/CKPTS/X2I_UltraEdiit_fourmask_LLava_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_filtered_objects_4mask_1280_context/003-GPT-XL/checkpoints/eval_step_10000/batch_3_cfg_7.5_topk_16384.jpg b/CKPTS/X2I_UltraEdiit_fourmask_LLava_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_filtered_objects_4mask_1280_context/003-GPT-XL/checkpoints/eval_step_10000/batch_3_cfg_7.5_topk_16384.jpg new file mode 100644 index 0000000000000000000000000000000000000000..f684a78612ef13b12d7ca9677b3f4e946fba2863 --- /dev/null +++ b/CKPTS/X2I_UltraEdiit_fourmask_LLava_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_filtered_objects_4mask_1280_context/003-GPT-XL/checkpoints/eval_step_10000/batch_3_cfg_7.5_topk_16384.jpg @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:bd9b4d5240830824909f976e4a93f119e55f1986455c6053389f80856c6c3366 +size 358087 diff --git a/CKPTS/X2I_UltraEdiit_fourmask_LLava_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_filtered_objects_4mask_1280_context/003-GPT-XL/checkpoints/eval_step_10000/batch_4_cfg_7.5_topk_16384.jpg b/CKPTS/X2I_UltraEdiit_fourmask_LLava_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_filtered_objects_4mask_1280_context/003-GPT-XL/checkpoints/eval_step_10000/batch_4_cfg_7.5_topk_16384.jpg new file mode 100644 index 0000000000000000000000000000000000000000..533cc289caabf856834e79c0dbc91cb921062ae1 --- /dev/null +++ b/CKPTS/X2I_UltraEdiit_fourmask_LLava_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_filtered_objects_4mask_1280_context/003-GPT-XL/checkpoints/eval_step_10000/batch_4_cfg_7.5_topk_16384.jpg @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:2d7c9804555388ae3b2b3f6effe57475a5b1c40dd36d1b4f2d98b061930765db +size 360961 diff --git a/CKPTS/X2I_UltraEdiit_fourmask_LLava_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_filtered_objects_4mask_1280_context/003-GPT-XL/checkpoints/eval_step_100000/batch_0_cfg_7.5_topk_16384.jpg b/CKPTS/X2I_UltraEdiit_fourmask_LLava_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_filtered_objects_4mask_1280_context/003-GPT-XL/checkpoints/eval_step_100000/batch_0_cfg_7.5_topk_16384.jpg new file mode 100644 index 0000000000000000000000000000000000000000..f80208e158cdd1da7f0479a5817f3919001257a1 --- /dev/null +++ b/CKPTS/X2I_UltraEdiit_fourmask_LLava_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_filtered_objects_4mask_1280_context/003-GPT-XL/checkpoints/eval_step_100000/batch_0_cfg_7.5_topk_16384.jpg @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:d9e02bc107870e986cbe7ad5c3f0587b74b61df921007133668ef4732410fe27 +size 424618 diff --git a/CKPTS/X2I_UltraEdiit_fourmask_LLava_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_filtered_objects_4mask_1280_context/003-GPT-XL/checkpoints/eval_step_100000/batch_1_cfg_7.5_topk_16384.jpg b/CKPTS/X2I_UltraEdiit_fourmask_LLava_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_filtered_objects_4mask_1280_context/003-GPT-XL/checkpoints/eval_step_100000/batch_1_cfg_7.5_topk_16384.jpg new file mode 100644 index 0000000000000000000000000000000000000000..9a21ec515eff961270f59ca1fce6e28e151050ff --- /dev/null +++ b/CKPTS/X2I_UltraEdiit_fourmask_LLava_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_filtered_objects_4mask_1280_context/003-GPT-XL/checkpoints/eval_step_100000/batch_1_cfg_7.5_topk_16384.jpg @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:b4b3b6be8c90c7f38719a7c42293161cf4038a238d3b42173755828125531b99 +size 434557 diff --git a/CKPTS/X2I_UltraEdiit_fourmask_LLava_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_filtered_objects_4mask_1280_context/003-GPT-XL/checkpoints/eval_step_100000/batch_2_cfg_7.5_topk_16384.jpg b/CKPTS/X2I_UltraEdiit_fourmask_LLava_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_filtered_objects_4mask_1280_context/003-GPT-XL/checkpoints/eval_step_100000/batch_2_cfg_7.5_topk_16384.jpg new file mode 100644 index 0000000000000000000000000000000000000000..d649655819b36019d1c49fc3540220c39ffebdd9 --- /dev/null +++ b/CKPTS/X2I_UltraEdiit_fourmask_LLava_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_filtered_objects_4mask_1280_context/003-GPT-XL/checkpoints/eval_step_100000/batch_2_cfg_7.5_topk_16384.jpg @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:dd587ca6ebbee45ebd518b56556b39da7e2116bd5262c61fb44cb9ec46be6fb3 +size 430356 diff --git a/CKPTS/X2I_UltraEdiit_fourmask_LLava_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_filtered_objects_4mask_1280_context/003-GPT-XL/checkpoints/eval_step_100000/batch_3_cfg_7.5_topk_16384.jpg b/CKPTS/X2I_UltraEdiit_fourmask_LLava_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_filtered_objects_4mask_1280_context/003-GPT-XL/checkpoints/eval_step_100000/batch_3_cfg_7.5_topk_16384.jpg new file mode 100644 index 0000000000000000000000000000000000000000..665e306bc929ae41990c2eea673e28ceaf87280a --- /dev/null +++ b/CKPTS/X2I_UltraEdiit_fourmask_LLava_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_filtered_objects_4mask_1280_context/003-GPT-XL/checkpoints/eval_step_100000/batch_3_cfg_7.5_topk_16384.jpg @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:4c15f18b012956da5066590ea803a22d11165f4f7af5ea9d793f09c937c0a9e9 +size 382674 diff --git a/CKPTS/X2I_UltraEdiit_fourmask_LLava_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_filtered_objects_4mask_1280_context/003-GPT-XL/checkpoints/eval_step_100000/batch_4_cfg_7.5_topk_16384.jpg b/CKPTS/X2I_UltraEdiit_fourmask_LLava_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_filtered_objects_4mask_1280_context/003-GPT-XL/checkpoints/eval_step_100000/batch_4_cfg_7.5_topk_16384.jpg new file mode 100644 index 0000000000000000000000000000000000000000..6cca42ce88a3cbc139eb6c3699d24c82a75667f7 --- /dev/null +++ b/CKPTS/X2I_UltraEdiit_fourmask_LLava_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_filtered_objects_4mask_1280_context/003-GPT-XL/checkpoints/eval_step_100000/batch_4_cfg_7.5_topk_16384.jpg @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:2a49204b0678295302f74b5d2e27aa4b10048df91bef17b9877ce12404fcd555 +size 338820 diff --git a/CKPTS/X2I_UltraEdiit_fourmask_LLava_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_filtered_objects_4mask_1280_context/003-GPT-XL/checkpoints/eval_step_102000/batch_0_cfg_7.5_topk_16384.jpg b/CKPTS/X2I_UltraEdiit_fourmask_LLava_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_filtered_objects_4mask_1280_context/003-GPT-XL/checkpoints/eval_step_102000/batch_0_cfg_7.5_topk_16384.jpg new file mode 100644 index 0000000000000000000000000000000000000000..4628f3e018e74072d5946d9816f2d3923f491550 --- /dev/null +++ b/CKPTS/X2I_UltraEdiit_fourmask_LLava_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_filtered_objects_4mask_1280_context/003-GPT-XL/checkpoints/eval_step_102000/batch_0_cfg_7.5_topk_16384.jpg @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:359b7874a9032b98041762c97afd8f362793ce4974459bf79487eab4e91d1723 +size 443974 diff --git a/CKPTS/X2I_UltraEdiit_fourmask_LLava_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_filtered_objects_4mask_1280_context/003-GPT-XL/checkpoints/eval_step_102000/batch_1_cfg_7.5_topk_16384.jpg b/CKPTS/X2I_UltraEdiit_fourmask_LLava_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_filtered_objects_4mask_1280_context/003-GPT-XL/checkpoints/eval_step_102000/batch_1_cfg_7.5_topk_16384.jpg new file mode 100644 index 0000000000000000000000000000000000000000..03d29cfa197eb3bef2c68c640e76ce9d5b77f3b6 --- /dev/null +++ b/CKPTS/X2I_UltraEdiit_fourmask_LLava_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_filtered_objects_4mask_1280_context/003-GPT-XL/checkpoints/eval_step_102000/batch_1_cfg_7.5_topk_16384.jpg @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:f43177f7145da046d2dc683e5a6888372fbbb64ebeb5caf16cadebf4927d5599 +size 429883 diff --git a/CKPTS/X2I_UltraEdiit_fourmask_LLava_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_filtered_objects_4mask_1280_context/003-GPT-XL/checkpoints/eval_step_102000/batch_2_cfg_7.5_topk_16384.jpg b/CKPTS/X2I_UltraEdiit_fourmask_LLava_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_filtered_objects_4mask_1280_context/003-GPT-XL/checkpoints/eval_step_102000/batch_2_cfg_7.5_topk_16384.jpg new file mode 100644 index 0000000000000000000000000000000000000000..4ddc6018cab89373b6ff6d04bb894efc87b834e7 --- /dev/null +++ b/CKPTS/X2I_UltraEdiit_fourmask_LLava_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_filtered_objects_4mask_1280_context/003-GPT-XL/checkpoints/eval_step_102000/batch_2_cfg_7.5_topk_16384.jpg @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:c12441fa98fec95379362389695db341f40e2106976fc240fbad56e1dbc9a3f8 +size 433368 diff --git a/CKPTS/X2I_UltraEdiit_fourmask_LLava_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_filtered_objects_4mask_1280_context/003-GPT-XL/checkpoints/eval_step_102000/batch_3_cfg_7.5_topk_16384.jpg b/CKPTS/X2I_UltraEdiit_fourmask_LLava_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_filtered_objects_4mask_1280_context/003-GPT-XL/checkpoints/eval_step_102000/batch_3_cfg_7.5_topk_16384.jpg new file mode 100644 index 0000000000000000000000000000000000000000..b27aa50d27ab6860d15fd22c03e39ea7e340f151 --- /dev/null +++ b/CKPTS/X2I_UltraEdiit_fourmask_LLava_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_filtered_objects_4mask_1280_context/003-GPT-XL/checkpoints/eval_step_102000/batch_3_cfg_7.5_topk_16384.jpg @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:d13aa81170599f41eed36e142479a0c0b021bce02606081c359e4e068f5b3e1a +size 401797 diff --git a/CKPTS/X2I_UltraEdiit_fourmask_LLava_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_filtered_objects_4mask_1280_context/003-GPT-XL/checkpoints/eval_step_102000/batch_4_cfg_7.5_topk_16384.jpg b/CKPTS/X2I_UltraEdiit_fourmask_LLava_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_filtered_objects_4mask_1280_context/003-GPT-XL/checkpoints/eval_step_102000/batch_4_cfg_7.5_topk_16384.jpg new file mode 100644 index 0000000000000000000000000000000000000000..7d5529974aa449a20a627406d71e93f47c409020 --- /dev/null +++ b/CKPTS/X2I_UltraEdiit_fourmask_LLava_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_filtered_objects_4mask_1280_context/003-GPT-XL/checkpoints/eval_step_102000/batch_4_cfg_7.5_topk_16384.jpg @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:c993544a15a14c8be6c965dd2e75d68e68c64c7e9f89ead5412985e570d2ffb2 +size 346382 diff --git a/CKPTS/X2I_UltraEdiit_fourmask_LLava_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_filtered_objects_4mask_1280_context/003-GPT-XL/checkpoints/eval_step_104000/batch_0_cfg_7.5_topk_16384.jpg b/CKPTS/X2I_UltraEdiit_fourmask_LLava_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_filtered_objects_4mask_1280_context/003-GPT-XL/checkpoints/eval_step_104000/batch_0_cfg_7.5_topk_16384.jpg new file mode 100644 index 0000000000000000000000000000000000000000..cad77959ba6a22c07d19aaed8cf5e557aba97853 --- /dev/null +++ b/CKPTS/X2I_UltraEdiit_fourmask_LLava_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_filtered_objects_4mask_1280_context/003-GPT-XL/checkpoints/eval_step_104000/batch_0_cfg_7.5_topk_16384.jpg @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:838217d9db006fa3be8c1c85724dcc3696030740c3c2ca2d0cca7e2cc7421648 +size 447544 diff --git a/CKPTS/X2I_UltraEdiit_fourmask_LLava_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_filtered_objects_4mask_1280_context/003-GPT-XL/checkpoints/eval_step_104000/batch_1_cfg_7.5_topk_16384.jpg b/CKPTS/X2I_UltraEdiit_fourmask_LLava_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_filtered_objects_4mask_1280_context/003-GPT-XL/checkpoints/eval_step_104000/batch_1_cfg_7.5_topk_16384.jpg new file mode 100644 index 0000000000000000000000000000000000000000..3a50b27894f243f0ec752a5c26e4ca3ca96544d6 --- /dev/null +++ b/CKPTS/X2I_UltraEdiit_fourmask_LLava_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_filtered_objects_4mask_1280_context/003-GPT-XL/checkpoints/eval_step_104000/batch_1_cfg_7.5_topk_16384.jpg @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:83e805c40f80ebec42b9d324c9f69a89b58c10cd0515aac8026ed931b1b0683d +size 416863 diff --git a/CKPTS/X2I_UltraEdiit_fourmask_LLava_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_filtered_objects_4mask_1280_context/003-GPT-XL/checkpoints/eval_step_104000/batch_2_cfg_7.5_topk_16384.jpg b/CKPTS/X2I_UltraEdiit_fourmask_LLava_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_filtered_objects_4mask_1280_context/003-GPT-XL/checkpoints/eval_step_104000/batch_2_cfg_7.5_topk_16384.jpg new file mode 100644 index 0000000000000000000000000000000000000000..dfccffc4b73284c1c909dce8d94eb84eedbc3d75 --- /dev/null +++ b/CKPTS/X2I_UltraEdiit_fourmask_LLava_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_filtered_objects_4mask_1280_context/003-GPT-XL/checkpoints/eval_step_104000/batch_2_cfg_7.5_topk_16384.jpg @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:be4e328d3e0781e515b95837438e988f577d2cc33ac1c0cb9c67c02641f33b8d +size 421452 diff --git a/CKPTS/X2I_UltraEdiit_fourmask_LLava_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_filtered_objects_4mask_1280_context/003-GPT-XL/checkpoints/eval_step_104000/batch_3_cfg_7.5_topk_16384.jpg b/CKPTS/X2I_UltraEdiit_fourmask_LLava_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_filtered_objects_4mask_1280_context/003-GPT-XL/checkpoints/eval_step_104000/batch_3_cfg_7.5_topk_16384.jpg new file mode 100644 index 0000000000000000000000000000000000000000..4bed45f1ed75d78b6a47b772e6d489507a2044f9 --- /dev/null +++ b/CKPTS/X2I_UltraEdiit_fourmask_LLava_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_filtered_objects_4mask_1280_context/003-GPT-XL/checkpoints/eval_step_104000/batch_3_cfg_7.5_topk_16384.jpg @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:7e28e1a48fba4bab35ba9e776e17d00e20d0388c8a865e8b8d785dfe2418a11b +size 407024 diff --git a/CKPTS/X2I_UltraEdiit_fourmask_LLava_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_filtered_objects_4mask_1280_context/003-GPT-XL/checkpoints/eval_step_104000/batch_4_cfg_7.5_topk_16384.jpg b/CKPTS/X2I_UltraEdiit_fourmask_LLava_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_filtered_objects_4mask_1280_context/003-GPT-XL/checkpoints/eval_step_104000/batch_4_cfg_7.5_topk_16384.jpg new file mode 100644 index 0000000000000000000000000000000000000000..23fd4bb5a8852e218f05603bf35c607865778662 --- /dev/null +++ b/CKPTS/X2I_UltraEdiit_fourmask_LLava_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_filtered_objects_4mask_1280_context/003-GPT-XL/checkpoints/eval_step_104000/batch_4_cfg_7.5_topk_16384.jpg @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:da2f37c9bf87b534ef9eab7d440c7ee678a6a0cce09d9dc40b8b911717894b02 +size 345071 diff --git a/CKPTS/X2I_UltraEdiit_fourmask_LLava_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_filtered_objects_4mask_1280_context/003-GPT-XL/checkpoints/eval_step_106000/batch_0_cfg_7.5_topk_16384.jpg b/CKPTS/X2I_UltraEdiit_fourmask_LLava_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_filtered_objects_4mask_1280_context/003-GPT-XL/checkpoints/eval_step_106000/batch_0_cfg_7.5_topk_16384.jpg new file mode 100644 index 0000000000000000000000000000000000000000..5772d40b3a02be5f2aa3ae2c4ac864b4727d5be5 --- /dev/null +++ b/CKPTS/X2I_UltraEdiit_fourmask_LLava_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_filtered_objects_4mask_1280_context/003-GPT-XL/checkpoints/eval_step_106000/batch_0_cfg_7.5_topk_16384.jpg @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:f5c163bc9fca2730fb88f4463d59b47f8169895c8c5dc24c33057fc9e0cb16d7 +size 445761 diff --git a/CKPTS/X2I_UltraEdiit_fourmask_LLava_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_filtered_objects_4mask_1280_context/003-GPT-XL/checkpoints/eval_step_106000/batch_1_cfg_7.5_topk_16384.jpg b/CKPTS/X2I_UltraEdiit_fourmask_LLava_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_filtered_objects_4mask_1280_context/003-GPT-XL/checkpoints/eval_step_106000/batch_1_cfg_7.5_topk_16384.jpg new file mode 100644 index 0000000000000000000000000000000000000000..328879a338c0914ad63adc03290b270986404638 --- /dev/null +++ b/CKPTS/X2I_UltraEdiit_fourmask_LLava_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_filtered_objects_4mask_1280_context/003-GPT-XL/checkpoints/eval_step_106000/batch_1_cfg_7.5_topk_16384.jpg @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:44b252a9a935c64c7fb7ad158f88e92b8640a9abdae26386eb05b77d1439c81f +size 439810 diff --git a/CKPTS/X2I_UltraEdiit_fourmask_LLava_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_filtered_objects_4mask_1280_context/003-GPT-XL/checkpoints/eval_step_106000/batch_2_cfg_7.5_topk_16384.jpg b/CKPTS/X2I_UltraEdiit_fourmask_LLava_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_filtered_objects_4mask_1280_context/003-GPT-XL/checkpoints/eval_step_106000/batch_2_cfg_7.5_topk_16384.jpg new file mode 100644 index 0000000000000000000000000000000000000000..8809f0a4bbefd129574626cf89c03c61a1754532 --- /dev/null +++ b/CKPTS/X2I_UltraEdiit_fourmask_LLava_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_filtered_objects_4mask_1280_context/003-GPT-XL/checkpoints/eval_step_106000/batch_2_cfg_7.5_topk_16384.jpg @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:3eb8d71ffb437295b91faba33eb1a86dd65475a0e0358d51af68ac7ab6215fe4 +size 425176 diff --git a/CKPTS/X2I_UltraEdiit_fourmask_LLava_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_filtered_objects_4mask_1280_context/003-GPT-XL/checkpoints/eval_step_106000/batch_3_cfg_7.5_topk_16384.jpg b/CKPTS/X2I_UltraEdiit_fourmask_LLava_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_filtered_objects_4mask_1280_context/003-GPT-XL/checkpoints/eval_step_106000/batch_3_cfg_7.5_topk_16384.jpg new file mode 100644 index 0000000000000000000000000000000000000000..e3f19a5e6764583800ec3917236eb2a7eaa1332a --- /dev/null +++ b/CKPTS/X2I_UltraEdiit_fourmask_LLava_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_filtered_objects_4mask_1280_context/003-GPT-XL/checkpoints/eval_step_106000/batch_3_cfg_7.5_topk_16384.jpg @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:7adf63e5767b7a869bf0af2b300d0a3b5226f19c62046d3ff374b546120a0d4c +size 387837 diff --git a/CKPTS/X2I_UltraEdiit_fourmask_LLava_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_filtered_objects_4mask_1280_context/003-GPT-XL/checkpoints/eval_step_106000/batch_4_cfg_7.5_topk_16384.jpg b/CKPTS/X2I_UltraEdiit_fourmask_LLava_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_filtered_objects_4mask_1280_context/003-GPT-XL/checkpoints/eval_step_106000/batch_4_cfg_7.5_topk_16384.jpg new file mode 100644 index 0000000000000000000000000000000000000000..c49992653e620d6df86c0c27dc69340e0db53681 --- /dev/null +++ b/CKPTS/X2I_UltraEdiit_fourmask_LLava_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_filtered_objects_4mask_1280_context/003-GPT-XL/checkpoints/eval_step_106000/batch_4_cfg_7.5_topk_16384.jpg @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:a7eb2e193bbbd256af94a954f7e26c0bb56b5a00fc52ead0b15c72d17ae71d9e +size 344219 diff --git a/CKPTS/X2I_UltraEdiit_fourmask_LLava_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_filtered_objects_4mask_1280_context/003-GPT-XL/checkpoints/eval_step_108000/batch_0_cfg_7.5_topk_16384.jpg b/CKPTS/X2I_UltraEdiit_fourmask_LLava_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_filtered_objects_4mask_1280_context/003-GPT-XL/checkpoints/eval_step_108000/batch_0_cfg_7.5_topk_16384.jpg new file mode 100644 index 0000000000000000000000000000000000000000..22c06e45023a26eba8fe62f490f0e3d852a20bca --- /dev/null +++ b/CKPTS/X2I_UltraEdiit_fourmask_LLava_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_filtered_objects_4mask_1280_context/003-GPT-XL/checkpoints/eval_step_108000/batch_0_cfg_7.5_topk_16384.jpg @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:ef6104f6cab76471f0a2032a608822eac78c711b8cad23f228b39f0679e8dfcc +size 447344 diff --git a/CKPTS/X2I_UltraEdiit_fourmask_LLava_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_filtered_objects_4mask_1280_context/003-GPT-XL/checkpoints/eval_step_108000/batch_1_cfg_7.5_topk_16384.jpg b/CKPTS/X2I_UltraEdiit_fourmask_LLava_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_filtered_objects_4mask_1280_context/003-GPT-XL/checkpoints/eval_step_108000/batch_1_cfg_7.5_topk_16384.jpg new file mode 100644 index 0000000000000000000000000000000000000000..2fd37923f1eed87a59c5d87d11c22d871d2072ce --- /dev/null +++ b/CKPTS/X2I_UltraEdiit_fourmask_LLava_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_filtered_objects_4mask_1280_context/003-GPT-XL/checkpoints/eval_step_108000/batch_1_cfg_7.5_topk_16384.jpg @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:28e54c89ab371731655552e0b432b483bffb45f181c913e24ca8f69de7f87e83 +size 429874 diff --git a/CKPTS/X2I_UltraEdiit_fourmask_LLava_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_filtered_objects_4mask_1280_context/003-GPT-XL/checkpoints/eval_step_108000/batch_2_cfg_7.5_topk_16384.jpg b/CKPTS/X2I_UltraEdiit_fourmask_LLava_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_filtered_objects_4mask_1280_context/003-GPT-XL/checkpoints/eval_step_108000/batch_2_cfg_7.5_topk_16384.jpg new file mode 100644 index 0000000000000000000000000000000000000000..905bca0db415aa4294bd8b0cde0fcfbd5d57f632 --- /dev/null +++ b/CKPTS/X2I_UltraEdiit_fourmask_LLava_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_filtered_objects_4mask_1280_context/003-GPT-XL/checkpoints/eval_step_108000/batch_2_cfg_7.5_topk_16384.jpg @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:92bd2417589a7c06763378b908168ed66eb8a04a1cc960f0eea207aa933a63be +size 418569 diff --git a/CKPTS/X2I_UltraEdiit_fourmask_LLava_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_filtered_objects_4mask_1280_context/003-GPT-XL/checkpoints/eval_step_108000/batch_3_cfg_7.5_topk_16384.jpg b/CKPTS/X2I_UltraEdiit_fourmask_LLava_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_filtered_objects_4mask_1280_context/003-GPT-XL/checkpoints/eval_step_108000/batch_3_cfg_7.5_topk_16384.jpg new file mode 100644 index 0000000000000000000000000000000000000000..0e21a961a34870955532d665829fe835ec271a44 --- /dev/null +++ b/CKPTS/X2I_UltraEdiit_fourmask_LLava_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_filtered_objects_4mask_1280_context/003-GPT-XL/checkpoints/eval_step_108000/batch_3_cfg_7.5_topk_16384.jpg @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:a764737e0ce971bcb3ac0ab3a7bf5c30d942e28451f953867defe7e9c47c8101 +size 378954 diff --git a/CKPTS/X2I_UltraEdiit_fourmask_LLava_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_filtered_objects_4mask_1280_context/003-GPT-XL/checkpoints/eval_step_108000/batch_4_cfg_7.5_topk_16384.jpg b/CKPTS/X2I_UltraEdiit_fourmask_LLava_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_filtered_objects_4mask_1280_context/003-GPT-XL/checkpoints/eval_step_108000/batch_4_cfg_7.5_topk_16384.jpg new file mode 100644 index 0000000000000000000000000000000000000000..297f91a47ea8d535760ff868f793fbba1537586c --- /dev/null +++ b/CKPTS/X2I_UltraEdiit_fourmask_LLava_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_filtered_objects_4mask_1280_context/003-GPT-XL/checkpoints/eval_step_108000/batch_4_cfg_7.5_topk_16384.jpg @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:674a4bf66c564d3296055950a47c3a59bb9aa9a1d66d00007931e24365e2fa52 +size 345333 diff --git a/CKPTS/X2I_UltraEdiit_fourmask_LLava_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_filtered_objects_4mask_1280_context/003-GPT-XL/checkpoints/eval_step_110000/batch_0_cfg_7.5_topk_16384.jpg b/CKPTS/X2I_UltraEdiit_fourmask_LLava_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_filtered_objects_4mask_1280_context/003-GPT-XL/checkpoints/eval_step_110000/batch_0_cfg_7.5_topk_16384.jpg new file mode 100644 index 0000000000000000000000000000000000000000..9f44a970a5718e0aeaed4c060922c79cb1a95fde --- /dev/null +++ b/CKPTS/X2I_UltraEdiit_fourmask_LLava_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_filtered_objects_4mask_1280_context/003-GPT-XL/checkpoints/eval_step_110000/batch_0_cfg_7.5_topk_16384.jpg @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:445aeda01fddd83eab7fb3d9df3d14e23d0ae22e2ae230d385564c3501e33c7a +size 442817 diff --git a/CKPTS/X2I_UltraEdiit_fourmask_LLava_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_filtered_objects_4mask_1280_context/003-GPT-XL/checkpoints/eval_step_110000/batch_1_cfg_7.5_topk_16384.jpg b/CKPTS/X2I_UltraEdiit_fourmask_LLava_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_filtered_objects_4mask_1280_context/003-GPT-XL/checkpoints/eval_step_110000/batch_1_cfg_7.5_topk_16384.jpg new file mode 100644 index 0000000000000000000000000000000000000000..39b2150d1f8a21581d0efb6617f3798064619f67 --- /dev/null +++ b/CKPTS/X2I_UltraEdiit_fourmask_LLava_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_filtered_objects_4mask_1280_context/003-GPT-XL/checkpoints/eval_step_110000/batch_1_cfg_7.5_topk_16384.jpg @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:4b7cf8c07ea4e4c33dce0a5dd1619a267ba85f202f705a5dfcd1706e1d7da0af +size 435611 diff --git a/CKPTS/X2I_UltraEdiit_fourmask_LLava_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_filtered_objects_4mask_1280_context/003-GPT-XL/checkpoints/eval_step_110000/batch_2_cfg_7.5_topk_16384.jpg b/CKPTS/X2I_UltraEdiit_fourmask_LLava_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_filtered_objects_4mask_1280_context/003-GPT-XL/checkpoints/eval_step_110000/batch_2_cfg_7.5_topk_16384.jpg new file mode 100644 index 0000000000000000000000000000000000000000..7ce9056994251c14fbae1c7ab43b212d6ba7a40d --- /dev/null +++ b/CKPTS/X2I_UltraEdiit_fourmask_LLava_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_filtered_objects_4mask_1280_context/003-GPT-XL/checkpoints/eval_step_110000/batch_2_cfg_7.5_topk_16384.jpg @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:401eed49f52325f6a66bb7c455e361174cd7c114a411af47b1cf8257fa1807d5 +size 422368 diff --git a/CKPTS/X2I_UltraEdiit_fourmask_LLava_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_filtered_objects_4mask_1280_context/003-GPT-XL/checkpoints/eval_step_110000/batch_3_cfg_7.5_topk_16384.jpg b/CKPTS/X2I_UltraEdiit_fourmask_LLava_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_filtered_objects_4mask_1280_context/003-GPT-XL/checkpoints/eval_step_110000/batch_3_cfg_7.5_topk_16384.jpg new file mode 100644 index 0000000000000000000000000000000000000000..bdd67b2d88f9502e17c903cd6ed9803948122b44 --- /dev/null +++ b/CKPTS/X2I_UltraEdiit_fourmask_LLava_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_filtered_objects_4mask_1280_context/003-GPT-XL/checkpoints/eval_step_110000/batch_3_cfg_7.5_topk_16384.jpg @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:ed5e1e7f0adcaf5d5f303df152875bb08a4510ba2aa17c1cf42d01654235fed7 +size 402524 diff --git a/CKPTS/X2I_UltraEdiit_fourmask_LLava_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_filtered_objects_4mask_1280_context/003-GPT-XL/checkpoints/eval_step_110000/batch_4_cfg_7.5_topk_16384.jpg b/CKPTS/X2I_UltraEdiit_fourmask_LLava_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_filtered_objects_4mask_1280_context/003-GPT-XL/checkpoints/eval_step_110000/batch_4_cfg_7.5_topk_16384.jpg new file mode 100644 index 0000000000000000000000000000000000000000..c26a1f857cf9cc306b0e0c342ea89392fd935202 --- /dev/null +++ b/CKPTS/X2I_UltraEdiit_fourmask_LLava_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_filtered_objects_4mask_1280_context/003-GPT-XL/checkpoints/eval_step_110000/batch_4_cfg_7.5_topk_16384.jpg @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:3a9fe26366ca311a85b9dbc9ecc7298075d772784ba252a1d813bd83170dc661 +size 344638 diff --git a/CKPTS/X2I_UltraEdiit_fourmask_LLava_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_filtered_objects_4mask_1280_context/003-GPT-XL/checkpoints/eval_step_12000/batch_0_cfg_7.5_topk_16384.jpg b/CKPTS/X2I_UltraEdiit_fourmask_LLava_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_filtered_objects_4mask_1280_context/003-GPT-XL/checkpoints/eval_step_12000/batch_0_cfg_7.5_topk_16384.jpg new file mode 100644 index 0000000000000000000000000000000000000000..1affdcc01859fb454c64b38e2d4950b66162e562 --- /dev/null +++ b/CKPTS/X2I_UltraEdiit_fourmask_LLava_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_filtered_objects_4mask_1280_context/003-GPT-XL/checkpoints/eval_step_12000/batch_0_cfg_7.5_topk_16384.jpg @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:7156a3f2b061519f53f8759ea14851ddfb025a75eba27b6de52745f16931591b +size 454031 diff --git a/CKPTS/X2I_UltraEdiit_fourmask_LLava_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_filtered_objects_4mask_1280_context/003-GPT-XL/checkpoints/eval_step_12000/batch_1_cfg_7.5_topk_16384.jpg b/CKPTS/X2I_UltraEdiit_fourmask_LLava_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_filtered_objects_4mask_1280_context/003-GPT-XL/checkpoints/eval_step_12000/batch_1_cfg_7.5_topk_16384.jpg new file mode 100644 index 0000000000000000000000000000000000000000..56e61872f47a7418f99f939239e259a17ebe593b --- /dev/null +++ b/CKPTS/X2I_UltraEdiit_fourmask_LLava_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_filtered_objects_4mask_1280_context/003-GPT-XL/checkpoints/eval_step_12000/batch_1_cfg_7.5_topk_16384.jpg @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:028faa4030b11c5ba5dd327d8b903462b32cd034364c2eaefee1e73949af44b7 +size 412934 diff --git a/CKPTS/X2I_UltraEdiit_fourmask_LLava_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_filtered_objects_4mask_1280_context/003-GPT-XL/checkpoints/eval_step_12000/batch_2_cfg_7.5_topk_16384.jpg b/CKPTS/X2I_UltraEdiit_fourmask_LLava_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_filtered_objects_4mask_1280_context/003-GPT-XL/checkpoints/eval_step_12000/batch_2_cfg_7.5_topk_16384.jpg new file mode 100644 index 0000000000000000000000000000000000000000..04e73108c75dab66afc4b4ddf5cd8fafd2e2bedf --- /dev/null +++ b/CKPTS/X2I_UltraEdiit_fourmask_LLava_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_filtered_objects_4mask_1280_context/003-GPT-XL/checkpoints/eval_step_12000/batch_2_cfg_7.5_topk_16384.jpg @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:44886fc45de359a51a162df2a25f0c6445176c8fff85b79b7bf842396e65d03e +size 413643 diff --git a/CKPTS/X2I_UltraEdiit_fourmask_LLava_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_filtered_objects_4mask_1280_context/003-GPT-XL/checkpoints/eval_step_12000/batch_3_cfg_7.5_topk_16384.jpg b/CKPTS/X2I_UltraEdiit_fourmask_LLava_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_filtered_objects_4mask_1280_context/003-GPT-XL/checkpoints/eval_step_12000/batch_3_cfg_7.5_topk_16384.jpg new file mode 100644 index 0000000000000000000000000000000000000000..aa8a5b92e7e364f6c5d16f8496837c6ba3dd80b5 --- /dev/null +++ b/CKPTS/X2I_UltraEdiit_fourmask_LLava_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_filtered_objects_4mask_1280_context/003-GPT-XL/checkpoints/eval_step_12000/batch_3_cfg_7.5_topk_16384.jpg @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:e7d94d7e51723852b03f4f9ce0c7df17130887161544a1d36b5aad7f3ba806ad +size 374997 diff --git a/CKPTS/X2I_UltraEdiit_fourmask_LLava_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_filtered_objects_4mask_1280_context/003-GPT-XL/checkpoints/eval_step_12000/batch_4_cfg_7.5_topk_16384.jpg b/CKPTS/X2I_UltraEdiit_fourmask_LLava_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_filtered_objects_4mask_1280_context/003-GPT-XL/checkpoints/eval_step_12000/batch_4_cfg_7.5_topk_16384.jpg new file mode 100644 index 0000000000000000000000000000000000000000..eb9aa9ba6661f701aa2d2417aaa3ec36681fcace --- /dev/null +++ b/CKPTS/X2I_UltraEdiit_fourmask_LLava_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_filtered_objects_4mask_1280_context/003-GPT-XL/checkpoints/eval_step_12000/batch_4_cfg_7.5_topk_16384.jpg @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:d38150357422db9c163ef061b73bf07dbc3f9baab8c256454e17dab1768711da +size 362978 diff --git a/CKPTS/X2I_UltraEdiit_fourmask_LLava_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_filtered_objects_4mask_1280_context/003-GPT-XL/checkpoints/eval_step_14000/batch_0_cfg_7.5_topk_16384.jpg b/CKPTS/X2I_UltraEdiit_fourmask_LLava_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_filtered_objects_4mask_1280_context/003-GPT-XL/checkpoints/eval_step_14000/batch_0_cfg_7.5_topk_16384.jpg new file mode 100644 index 0000000000000000000000000000000000000000..776264a535aa1a473c6145df5ed851763beff0c5 --- /dev/null +++ b/CKPTS/X2I_UltraEdiit_fourmask_LLava_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_filtered_objects_4mask_1280_context/003-GPT-XL/checkpoints/eval_step_14000/batch_0_cfg_7.5_topk_16384.jpg @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:a8e55580b34afdd0b45a4e589e043b6eb3ff9a8de7876395c5e976e283ec380a +size 456961 diff --git a/CKPTS/X2I_UltraEdiit_fourmask_LLava_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_filtered_objects_4mask_1280_context/003-GPT-XL/checkpoints/eval_step_14000/batch_1_cfg_7.5_topk_16384.jpg b/CKPTS/X2I_UltraEdiit_fourmask_LLava_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_filtered_objects_4mask_1280_context/003-GPT-XL/checkpoints/eval_step_14000/batch_1_cfg_7.5_topk_16384.jpg new file mode 100644 index 0000000000000000000000000000000000000000..00e4fae181eefd7143600b54365bfa929bdca907 --- /dev/null +++ b/CKPTS/X2I_UltraEdiit_fourmask_LLava_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_filtered_objects_4mask_1280_context/003-GPT-XL/checkpoints/eval_step_14000/batch_1_cfg_7.5_topk_16384.jpg @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:3caf0bb90ecba45d822287cd26b7805be0761e7e94b54444a6d26bc4bca04c58 +size 414139 diff --git a/CKPTS/X2I_UltraEdiit_fourmask_LLava_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_filtered_objects_4mask_1280_context/003-GPT-XL/checkpoints/eval_step_14000/batch_2_cfg_7.5_topk_16384.jpg b/CKPTS/X2I_UltraEdiit_fourmask_LLava_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_filtered_objects_4mask_1280_context/003-GPT-XL/checkpoints/eval_step_14000/batch_2_cfg_7.5_topk_16384.jpg new file mode 100644 index 0000000000000000000000000000000000000000..9d30b9d6caa480ec179f64a4f8abbe072f8096aa --- /dev/null +++ b/CKPTS/X2I_UltraEdiit_fourmask_LLava_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_filtered_objects_4mask_1280_context/003-GPT-XL/checkpoints/eval_step_14000/batch_2_cfg_7.5_topk_16384.jpg @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:adaf7cca3b740ab12753e7665f322fd3fbbb9ee71495418d5c8ab6c4ccbdbd7c +size 403259 diff --git a/CKPTS/X2I_UltraEdiit_fourmask_LLava_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_filtered_objects_4mask_1280_context/003-GPT-XL/checkpoints/eval_step_14000/batch_3_cfg_7.5_topk_16384.jpg b/CKPTS/X2I_UltraEdiit_fourmask_LLava_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_filtered_objects_4mask_1280_context/003-GPT-XL/checkpoints/eval_step_14000/batch_3_cfg_7.5_topk_16384.jpg new file mode 100644 index 0000000000000000000000000000000000000000..c885d1e89d1cce038a5985c97a1f43e5b8566f2a --- /dev/null +++ b/CKPTS/X2I_UltraEdiit_fourmask_LLava_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_filtered_objects_4mask_1280_context/003-GPT-XL/checkpoints/eval_step_14000/batch_3_cfg_7.5_topk_16384.jpg @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:7cddf33fd520b54bd446dc4dc85d73cbde8c420265d29fc110f5dd753a9ac098 +size 393777 diff --git a/CKPTS/X2I_UltraEdiit_fourmask_LLava_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_filtered_objects_4mask_1280_context/003-GPT-XL/checkpoints/eval_step_14000/batch_4_cfg_7.5_topk_16384.jpg b/CKPTS/X2I_UltraEdiit_fourmask_LLava_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_filtered_objects_4mask_1280_context/003-GPT-XL/checkpoints/eval_step_14000/batch_4_cfg_7.5_topk_16384.jpg new file mode 100644 index 0000000000000000000000000000000000000000..b138b56f0bb3cf98f84aeb541e74162fcdc82a36 --- /dev/null +++ b/CKPTS/X2I_UltraEdiit_fourmask_LLava_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_filtered_objects_4mask_1280_context/003-GPT-XL/checkpoints/eval_step_14000/batch_4_cfg_7.5_topk_16384.jpg @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:854d751715ee1bb140eadffbb95057b946b74daada600759a1a93f46476a2267 +size 357838 diff --git a/CKPTS/X2I_UltraEdiit_fourmask_LLava_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_filtered_objects_4mask_1280_context/003-GPT-XL/checkpoints/eval_step_16000/batch_0_cfg_7.5_topk_16384.jpg b/CKPTS/X2I_UltraEdiit_fourmask_LLava_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_filtered_objects_4mask_1280_context/003-GPT-XL/checkpoints/eval_step_16000/batch_0_cfg_7.5_topk_16384.jpg new file mode 100644 index 0000000000000000000000000000000000000000..35a985be4ea520420a2a543260fb27ca85043480 --- /dev/null +++ b/CKPTS/X2I_UltraEdiit_fourmask_LLava_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_filtered_objects_4mask_1280_context/003-GPT-XL/checkpoints/eval_step_16000/batch_0_cfg_7.5_topk_16384.jpg @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:34b7143f6738b6124a282b7ee74697877940bbe232cc7d9c26c56aa3639405fb +size 454876 diff --git a/CKPTS/X2I_UltraEdiit_fourmask_LLava_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_filtered_objects_4mask_1280_context/003-GPT-XL/checkpoints/eval_step_16000/batch_1_cfg_7.5_topk_16384.jpg b/CKPTS/X2I_UltraEdiit_fourmask_LLava_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_filtered_objects_4mask_1280_context/003-GPT-XL/checkpoints/eval_step_16000/batch_1_cfg_7.5_topk_16384.jpg new file mode 100644 index 0000000000000000000000000000000000000000..1f5d7a419d9d4e5f39f418bb3434d6a5dc7dad6d --- /dev/null +++ b/CKPTS/X2I_UltraEdiit_fourmask_LLava_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_filtered_objects_4mask_1280_context/003-GPT-XL/checkpoints/eval_step_16000/batch_1_cfg_7.5_topk_16384.jpg @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:12d6f1535383e297e60a33ccc18d3fd926b311cca7ebe17680676e9abb62f9d8 +size 411659 diff --git a/CKPTS/X2I_UltraEdiit_fourmask_LLava_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_filtered_objects_4mask_1280_context/003-GPT-XL/checkpoints/eval_step_16000/batch_2_cfg_7.5_topk_16384.jpg b/CKPTS/X2I_UltraEdiit_fourmask_LLava_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_filtered_objects_4mask_1280_context/003-GPT-XL/checkpoints/eval_step_16000/batch_2_cfg_7.5_topk_16384.jpg new file mode 100644 index 0000000000000000000000000000000000000000..65d970929a6e12d99f41ad7a58446bf59d87b578 --- /dev/null +++ b/CKPTS/X2I_UltraEdiit_fourmask_LLava_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_filtered_objects_4mask_1280_context/003-GPT-XL/checkpoints/eval_step_16000/batch_2_cfg_7.5_topk_16384.jpg @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:39fbf50b8b32801f333c444c39dd91356ac995e714b763864339e1f253ed9f59 +size 422738 diff --git a/CKPTS/X2I_UltraEdiit_fourmask_LLava_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_filtered_objects_4mask_1280_context/003-GPT-XL/checkpoints/eval_step_16000/batch_3_cfg_7.5_topk_16384.jpg b/CKPTS/X2I_UltraEdiit_fourmask_LLava_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_filtered_objects_4mask_1280_context/003-GPT-XL/checkpoints/eval_step_16000/batch_3_cfg_7.5_topk_16384.jpg new file mode 100644 index 0000000000000000000000000000000000000000..02d8f9905214228461549ccf866742257b41e565 --- /dev/null +++ b/CKPTS/X2I_UltraEdiit_fourmask_LLava_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_filtered_objects_4mask_1280_context/003-GPT-XL/checkpoints/eval_step_16000/batch_3_cfg_7.5_topk_16384.jpg @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:59c4ef3d59eab1d86db8f2f3037d3bd144fc4a658c6637a011de819a21c7e3a0 +size 415947 diff --git a/CKPTS/X2I_UltraEdiit_fourmask_LLava_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_filtered_objects_4mask_1280_context/003-GPT-XL/checkpoints/eval_step_16000/batch_4_cfg_7.5_topk_16384.jpg b/CKPTS/X2I_UltraEdiit_fourmask_LLava_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_filtered_objects_4mask_1280_context/003-GPT-XL/checkpoints/eval_step_16000/batch_4_cfg_7.5_topk_16384.jpg new file mode 100644 index 0000000000000000000000000000000000000000..2fdb293f699a6ccd073c164783158763a1c6daea --- /dev/null +++ b/CKPTS/X2I_UltraEdiit_fourmask_LLava_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_filtered_objects_4mask_1280_context/003-GPT-XL/checkpoints/eval_step_16000/batch_4_cfg_7.5_topk_16384.jpg @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:143ddd62ff64c930735c83185253d65bcf55b07a0d1bb583e6e5273a3155acb9 +size 354994 diff --git a/CKPTS/X2I_UltraEdiit_fourmask_LLava_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_filtered_objects_4mask_1280_context/003-GPT-XL/checkpoints/eval_step_18000/batch_0_cfg_7.5_topk_16384.jpg b/CKPTS/X2I_UltraEdiit_fourmask_LLava_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_filtered_objects_4mask_1280_context/003-GPT-XL/checkpoints/eval_step_18000/batch_0_cfg_7.5_topk_16384.jpg new file mode 100644 index 0000000000000000000000000000000000000000..54ceeabdde0010bde3994361a91b56aa1ef94930 --- /dev/null +++ b/CKPTS/X2I_UltraEdiit_fourmask_LLava_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_filtered_objects_4mask_1280_context/003-GPT-XL/checkpoints/eval_step_18000/batch_0_cfg_7.5_topk_16384.jpg @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:415ac8d4089bf70c1c6c869b6c4821689c7b8bb356f27bd1907c6a1ecd6f90b3 +size 434276 diff --git a/CKPTS/X2I_UltraEdiit_fourmask_LLava_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_filtered_objects_4mask_1280_context/003-GPT-XL/checkpoints/eval_step_18000/batch_1_cfg_7.5_topk_16384.jpg b/CKPTS/X2I_UltraEdiit_fourmask_LLava_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_filtered_objects_4mask_1280_context/003-GPT-XL/checkpoints/eval_step_18000/batch_1_cfg_7.5_topk_16384.jpg new file mode 100644 index 0000000000000000000000000000000000000000..8cb842e1f4cda919ab80f1d0c64a196437ccb913 --- /dev/null +++ b/CKPTS/X2I_UltraEdiit_fourmask_LLava_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_filtered_objects_4mask_1280_context/003-GPT-XL/checkpoints/eval_step_18000/batch_1_cfg_7.5_topk_16384.jpg @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:ba0ca71fbdc878af1737a0b8e36fdc1d529b23b1f2f53a23a5d68e8ef4cd1410 +size 418739 diff --git a/CKPTS/X2I_UltraEdiit_fourmask_LLava_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_filtered_objects_4mask_1280_context/003-GPT-XL/checkpoints/eval_step_18000/batch_2_cfg_7.5_topk_16384.jpg b/CKPTS/X2I_UltraEdiit_fourmask_LLava_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_filtered_objects_4mask_1280_context/003-GPT-XL/checkpoints/eval_step_18000/batch_2_cfg_7.5_topk_16384.jpg new file mode 100644 index 0000000000000000000000000000000000000000..9b93b0242fc19f91774418e9ac0d48ca6ff8d7f0 --- /dev/null +++ b/CKPTS/X2I_UltraEdiit_fourmask_LLava_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_filtered_objects_4mask_1280_context/003-GPT-XL/checkpoints/eval_step_18000/batch_2_cfg_7.5_topk_16384.jpg @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:d69c2e5e7bb54d17bc66ae5daff55166fbc16b5c6d633b84565f67f75d282394 +size 423930 diff --git a/CKPTS/X2I_UltraEdiit_fourmask_LLava_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_filtered_objects_4mask_1280_context/003-GPT-XL/checkpoints/eval_step_18000/batch_3_cfg_7.5_topk_16384.jpg b/CKPTS/X2I_UltraEdiit_fourmask_LLava_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_filtered_objects_4mask_1280_context/003-GPT-XL/checkpoints/eval_step_18000/batch_3_cfg_7.5_topk_16384.jpg new file mode 100644 index 0000000000000000000000000000000000000000..d093d02b86dd8b1a12bc7b8db437cc3d382725ac --- /dev/null +++ b/CKPTS/X2I_UltraEdiit_fourmask_LLava_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_filtered_objects_4mask_1280_context/003-GPT-XL/checkpoints/eval_step_18000/batch_3_cfg_7.5_topk_16384.jpg @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:38dd2eb019eb9c405ca891f5e8779e82cf1378a63ac64bbdd02279db7ef702fe +size 398894 diff --git a/CKPTS/X2I_UltraEdiit_fourmask_LLava_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_filtered_objects_4mask_1280_context/003-GPT-XL/checkpoints/eval_step_18000/batch_4_cfg_7.5_topk_16384.jpg b/CKPTS/X2I_UltraEdiit_fourmask_LLava_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_filtered_objects_4mask_1280_context/003-GPT-XL/checkpoints/eval_step_18000/batch_4_cfg_7.5_topk_16384.jpg new file mode 100644 index 0000000000000000000000000000000000000000..ad7e50f6e66dda442caa70ba92e25cc426687872 --- /dev/null +++ b/CKPTS/X2I_UltraEdiit_fourmask_LLava_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_filtered_objects_4mask_1280_context/003-GPT-XL/checkpoints/eval_step_18000/batch_4_cfg_7.5_topk_16384.jpg @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:dd682614031a6ffe996c0dca579946552e52e2338efbf6d7ce8d5650a4056fa4 +size 349480 diff --git a/CKPTS/X2I_UltraEdiit_fourmask_LLava_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_filtered_objects_4mask_1280_context/003-GPT-XL/checkpoints/eval_step_2000/batch_0_cfg_7.5_topk_16384.jpg b/CKPTS/X2I_UltraEdiit_fourmask_LLava_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_filtered_objects_4mask_1280_context/003-GPT-XL/checkpoints/eval_step_2000/batch_0_cfg_7.5_topk_16384.jpg new file mode 100644 index 0000000000000000000000000000000000000000..4af8027bf3013e02e40e8900b11d0009264e89c8 --- /dev/null +++ b/CKPTS/X2I_UltraEdiit_fourmask_LLava_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_filtered_objects_4mask_1280_context/003-GPT-XL/checkpoints/eval_step_2000/batch_0_cfg_7.5_topk_16384.jpg @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:cae7a1883218e617f2b618f989bfb75b70bc5fca87fa337ed0b33deed82934ae +size 418935 diff --git a/CKPTS/X2I_UltraEdiit_fourmask_LLava_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_filtered_objects_4mask_1280_context/003-GPT-XL/checkpoints/eval_step_2000/batch_1_cfg_7.5_topk_16384.jpg b/CKPTS/X2I_UltraEdiit_fourmask_LLava_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_filtered_objects_4mask_1280_context/003-GPT-XL/checkpoints/eval_step_2000/batch_1_cfg_7.5_topk_16384.jpg new file mode 100644 index 0000000000000000000000000000000000000000..4b2a33a17bf62c6454c51fc80056146f0d340bfa --- /dev/null +++ b/CKPTS/X2I_UltraEdiit_fourmask_LLava_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_filtered_objects_4mask_1280_context/003-GPT-XL/checkpoints/eval_step_2000/batch_1_cfg_7.5_topk_16384.jpg @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:c21cbd76585b8ded83389c1d25214f4af8d16b2cf31726973e1268b5e830e2e2 +size 405633 diff --git a/CKPTS/X2I_UltraEdiit_fourmask_LLava_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_filtered_objects_4mask_1280_context/003-GPT-XL/checkpoints/eval_step_2000/batch_2_cfg_7.5_topk_16384.jpg b/CKPTS/X2I_UltraEdiit_fourmask_LLava_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_filtered_objects_4mask_1280_context/003-GPT-XL/checkpoints/eval_step_2000/batch_2_cfg_7.5_topk_16384.jpg new file mode 100644 index 0000000000000000000000000000000000000000..b7800b956979e51665504cfd724b4b65b6c6e074 --- /dev/null +++ b/CKPTS/X2I_UltraEdiit_fourmask_LLava_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_filtered_objects_4mask_1280_context/003-GPT-XL/checkpoints/eval_step_2000/batch_2_cfg_7.5_topk_16384.jpg @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:95013c1f4674032912e779bdf041c4c18517a60b88a5881b53e76b94893449e5 +size 415691 diff --git a/CKPTS/X2I_UltraEdiit_fourmask_LLava_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_filtered_objects_4mask_1280_context/003-GPT-XL/checkpoints/eval_step_2000/batch_3_cfg_7.5_topk_16384.jpg b/CKPTS/X2I_UltraEdiit_fourmask_LLava_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_filtered_objects_4mask_1280_context/003-GPT-XL/checkpoints/eval_step_2000/batch_3_cfg_7.5_topk_16384.jpg new file mode 100644 index 0000000000000000000000000000000000000000..092dd304f932dfbf3cc15a8d2153a79eab509661 --- /dev/null +++ b/CKPTS/X2I_UltraEdiit_fourmask_LLava_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_filtered_objects_4mask_1280_context/003-GPT-XL/checkpoints/eval_step_2000/batch_3_cfg_7.5_topk_16384.jpg @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:13871632f74d6f042921367c5e846e66082eec5d9e932bbcd0fdc7ebb6c28c62 +size 390146 diff --git a/CKPTS/X2I_UltraEdiit_fourmask_LLava_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_filtered_objects_4mask_1280_context/003-GPT-XL/checkpoints/eval_step_2000/batch_4_cfg_7.5_topk_16384.jpg b/CKPTS/X2I_UltraEdiit_fourmask_LLava_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_filtered_objects_4mask_1280_context/003-GPT-XL/checkpoints/eval_step_2000/batch_4_cfg_7.5_topk_16384.jpg new file mode 100644 index 0000000000000000000000000000000000000000..2f2769561b67ff8156fd41d7ecefdef325f6c1e7 --- /dev/null +++ b/CKPTS/X2I_UltraEdiit_fourmask_LLava_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_filtered_objects_4mask_1280_context/003-GPT-XL/checkpoints/eval_step_2000/batch_4_cfg_7.5_topk_16384.jpg @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:2ab4dddb81b9d6627e2d135e78d8bc28ad910683f543506629182e346c40ae31 +size 356052 diff --git a/CKPTS/X2I_UltraEdiit_fourmask_LLava_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_filtered_objects_4mask_1280_context/003-GPT-XL/checkpoints/eval_step_20000/batch_0_cfg_7.5_topk_16384.jpg b/CKPTS/X2I_UltraEdiit_fourmask_LLava_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_filtered_objects_4mask_1280_context/003-GPT-XL/checkpoints/eval_step_20000/batch_0_cfg_7.5_topk_16384.jpg new file mode 100644 index 0000000000000000000000000000000000000000..4dfa62a399b52869cc57f671724442ee7f98bdc1 --- /dev/null +++ b/CKPTS/X2I_UltraEdiit_fourmask_LLava_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_filtered_objects_4mask_1280_context/003-GPT-XL/checkpoints/eval_step_20000/batch_0_cfg_7.5_topk_16384.jpg @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:80e5e7366e98d1db9e3b9ec763aac4e546a6a02e6ef03e418738f4fbf0d7c816 +size 425168 diff --git a/CKPTS/X2I_UltraEdiit_fourmask_LLava_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_filtered_objects_4mask_1280_context/003-GPT-XL/checkpoints/eval_step_20000/batch_1_cfg_7.5_topk_16384.jpg b/CKPTS/X2I_UltraEdiit_fourmask_LLava_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_filtered_objects_4mask_1280_context/003-GPT-XL/checkpoints/eval_step_20000/batch_1_cfg_7.5_topk_16384.jpg new file mode 100644 index 0000000000000000000000000000000000000000..f88acee0f98722737aeb14aaf59bb6759a6609c2 --- /dev/null +++ b/CKPTS/X2I_UltraEdiit_fourmask_LLava_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_filtered_objects_4mask_1280_context/003-GPT-XL/checkpoints/eval_step_20000/batch_1_cfg_7.5_topk_16384.jpg @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:d5f7c04531e834a85cbe8695a51b1af121d042e852bb3d0cc4916e02e7c751ce +size 421994 diff --git a/CKPTS/X2I_UltraEdiit_fourmask_LLava_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_filtered_objects_4mask_1280_context/003-GPT-XL/checkpoints/eval_step_20000/batch_2_cfg_7.5_topk_16384.jpg b/CKPTS/X2I_UltraEdiit_fourmask_LLava_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_filtered_objects_4mask_1280_context/003-GPT-XL/checkpoints/eval_step_20000/batch_2_cfg_7.5_topk_16384.jpg new file mode 100644 index 0000000000000000000000000000000000000000..50a9836db702e55b36f48cb0d59c958eb2d4d644 --- /dev/null +++ b/CKPTS/X2I_UltraEdiit_fourmask_LLava_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_filtered_objects_4mask_1280_context/003-GPT-XL/checkpoints/eval_step_20000/batch_2_cfg_7.5_topk_16384.jpg @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:fe6ad19da3f9d8dcd185d185f047de54e66680d699a43d994888ee73553c5f29 +size 432920 diff --git a/CKPTS/X2I_UltraEdiit_fourmask_LLava_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_filtered_objects_4mask_1280_context/003-GPT-XL/checkpoints/eval_step_20000/batch_3_cfg_7.5_topk_16384.jpg b/CKPTS/X2I_UltraEdiit_fourmask_LLava_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_filtered_objects_4mask_1280_context/003-GPT-XL/checkpoints/eval_step_20000/batch_3_cfg_7.5_topk_16384.jpg new file mode 100644 index 0000000000000000000000000000000000000000..4eb18b4dff08e738a3295521119df4e46608b687 --- /dev/null +++ b/CKPTS/X2I_UltraEdiit_fourmask_LLava_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_filtered_objects_4mask_1280_context/003-GPT-XL/checkpoints/eval_step_20000/batch_3_cfg_7.5_topk_16384.jpg @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:2e4ae1dcdda48f597ef1eebfd3a1e61cff2b229c7df48913c24ae636f572e92c +size 416886 diff --git a/CKPTS/X2I_UltraEdiit_fourmask_LLava_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_filtered_objects_4mask_1280_context/003-GPT-XL/checkpoints/eval_step_20000/batch_4_cfg_7.5_topk_16384.jpg b/CKPTS/X2I_UltraEdiit_fourmask_LLava_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_filtered_objects_4mask_1280_context/003-GPT-XL/checkpoints/eval_step_20000/batch_4_cfg_7.5_topk_16384.jpg new file mode 100644 index 0000000000000000000000000000000000000000..0f0286a14baf9b5c1310512ffc667d050073a00a --- /dev/null +++ b/CKPTS/X2I_UltraEdiit_fourmask_LLava_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_filtered_objects_4mask_1280_context/003-GPT-XL/checkpoints/eval_step_20000/batch_4_cfg_7.5_topk_16384.jpg @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:d3dabefb3dc6c79c560af058ae4277378c83d8f7cb23adb73ff58cd35846b901 +size 345056 diff --git a/CKPTS/X2I_UltraEdiit_fourmask_LLava_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_filtered_objects_4mask_1280_context/003-GPT-XL/checkpoints/eval_step_22000/batch_0_cfg_7.5_topk_16384.jpg b/CKPTS/X2I_UltraEdiit_fourmask_LLava_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_filtered_objects_4mask_1280_context/003-GPT-XL/checkpoints/eval_step_22000/batch_0_cfg_7.5_topk_16384.jpg new file mode 100644 index 0000000000000000000000000000000000000000..97882df86737cac8c5dfd4660d504362ede33589 --- /dev/null +++ b/CKPTS/X2I_UltraEdiit_fourmask_LLava_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_filtered_objects_4mask_1280_context/003-GPT-XL/checkpoints/eval_step_22000/batch_0_cfg_7.5_topk_16384.jpg @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:c46bba7ffd38513bad1eed327f6e846fb163a0b8b945a1fcbc087984ffd92edb +size 445409 diff --git a/CKPTS/X2I_UltraEdiit_fourmask_LLava_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_filtered_objects_4mask_1280_context/003-GPT-XL/checkpoints/eval_step_22000/batch_1_cfg_7.5_topk_16384.jpg b/CKPTS/X2I_UltraEdiit_fourmask_LLava_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_filtered_objects_4mask_1280_context/003-GPT-XL/checkpoints/eval_step_22000/batch_1_cfg_7.5_topk_16384.jpg new file mode 100644 index 0000000000000000000000000000000000000000..d2e827dc3a573f108aae5309aa8674470768eed3 --- /dev/null +++ b/CKPTS/X2I_UltraEdiit_fourmask_LLava_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_filtered_objects_4mask_1280_context/003-GPT-XL/checkpoints/eval_step_22000/batch_1_cfg_7.5_topk_16384.jpg @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:74fbb6d7a731a195f25e4464cc42733818fdd37b3624cc99591b895a1a4b7260 +size 431485 diff --git a/CKPTS/X2I_UltraEdiit_fourmask_LLava_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_filtered_objects_4mask_1280_context/003-GPT-XL/checkpoints/eval_step_22000/batch_2_cfg_7.5_topk_16384.jpg b/CKPTS/X2I_UltraEdiit_fourmask_LLava_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_filtered_objects_4mask_1280_context/003-GPT-XL/checkpoints/eval_step_22000/batch_2_cfg_7.5_topk_16384.jpg new file mode 100644 index 0000000000000000000000000000000000000000..7c3e2350b1475fba127f9d8a1a785d9980a2ffb3 --- /dev/null +++ b/CKPTS/X2I_UltraEdiit_fourmask_LLava_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_filtered_objects_4mask_1280_context/003-GPT-XL/checkpoints/eval_step_22000/batch_2_cfg_7.5_topk_16384.jpg @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:53ff05d7872022021046bf3f7fdc69fa41f6187edebedd78b789aacb3f91c3c5 +size 428254 diff --git a/CKPTS/X2I_UltraEdiit_fourmask_LLava_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_filtered_objects_4mask_1280_context/003-GPT-XL/checkpoints/eval_step_22000/batch_3_cfg_7.5_topk_16384.jpg b/CKPTS/X2I_UltraEdiit_fourmask_LLava_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_filtered_objects_4mask_1280_context/003-GPT-XL/checkpoints/eval_step_22000/batch_3_cfg_7.5_topk_16384.jpg new file mode 100644 index 0000000000000000000000000000000000000000..7c1e2e6619a47054f19e2ea25d5b3b4cf1a9d1aa --- /dev/null +++ b/CKPTS/X2I_UltraEdiit_fourmask_LLava_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_filtered_objects_4mask_1280_context/003-GPT-XL/checkpoints/eval_step_22000/batch_3_cfg_7.5_topk_16384.jpg @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:c1b928d2f832c593faa0dc35f8e6b4063c6b0f58586d3059775e3e2f8bc113cf +size 410025 diff --git a/CKPTS/X2I_UltraEdiit_fourmask_LLava_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_filtered_objects_4mask_1280_context/003-GPT-XL/checkpoints/eval_step_22000/batch_4_cfg_7.5_topk_16384.jpg b/CKPTS/X2I_UltraEdiit_fourmask_LLava_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_filtered_objects_4mask_1280_context/003-GPT-XL/checkpoints/eval_step_22000/batch_4_cfg_7.5_topk_16384.jpg new file mode 100644 index 0000000000000000000000000000000000000000..73ea6e33a041f7157325ab31f09427c81775d6df --- /dev/null +++ b/CKPTS/X2I_UltraEdiit_fourmask_LLava_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_filtered_objects_4mask_1280_context/003-GPT-XL/checkpoints/eval_step_22000/batch_4_cfg_7.5_topk_16384.jpg @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:75088da710142ef83161210599b5cfb618fe2f3a86c70b93f6ad05bf61589274 +size 343085 diff --git a/CKPTS/X2I_UltraEdiit_fourmask_LLava_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_filtered_objects_4mask_1280_context/003-GPT-XL/checkpoints/eval_step_24000/batch_0_cfg_7.5_topk_16384.jpg b/CKPTS/X2I_UltraEdiit_fourmask_LLava_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_filtered_objects_4mask_1280_context/003-GPT-XL/checkpoints/eval_step_24000/batch_0_cfg_7.5_topk_16384.jpg new file mode 100644 index 0000000000000000000000000000000000000000..eef59ddfd9e2c45d5e415b195598ea9c7e19f5bd --- /dev/null +++ b/CKPTS/X2I_UltraEdiit_fourmask_LLava_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_filtered_objects_4mask_1280_context/003-GPT-XL/checkpoints/eval_step_24000/batch_0_cfg_7.5_topk_16384.jpg @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:befdfd266b140a436473bf4940e859c945248402139aa47e8e439b02c73bbd6b +size 435341 diff --git a/CKPTS/X2I_UltraEdiit_fourmask_LLava_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_filtered_objects_4mask_1280_context/003-GPT-XL/checkpoints/eval_step_24000/batch_1_cfg_7.5_topk_16384.jpg b/CKPTS/X2I_UltraEdiit_fourmask_LLava_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_filtered_objects_4mask_1280_context/003-GPT-XL/checkpoints/eval_step_24000/batch_1_cfg_7.5_topk_16384.jpg new file mode 100644 index 0000000000000000000000000000000000000000..2e759d8b907ed451a13f9d7748c858c5c8b69609 --- /dev/null +++ b/CKPTS/X2I_UltraEdiit_fourmask_LLava_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_filtered_objects_4mask_1280_context/003-GPT-XL/checkpoints/eval_step_24000/batch_1_cfg_7.5_topk_16384.jpg @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:56412debf21188c96d3664fc3884936a4255956e66bec9d050f7c25af3733852 +size 422029 diff --git a/CKPTS/X2I_UltraEdiit_fourmask_LLava_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_filtered_objects_4mask_1280_context/003-GPT-XL/checkpoints/eval_step_24000/batch_2_cfg_7.5_topk_16384.jpg b/CKPTS/X2I_UltraEdiit_fourmask_LLava_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_filtered_objects_4mask_1280_context/003-GPT-XL/checkpoints/eval_step_24000/batch_2_cfg_7.5_topk_16384.jpg new file mode 100644 index 0000000000000000000000000000000000000000..2f93cb2b2eb7175e5bfe65c0d19386e2ba6afc0f --- /dev/null +++ b/CKPTS/X2I_UltraEdiit_fourmask_LLava_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_filtered_objects_4mask_1280_context/003-GPT-XL/checkpoints/eval_step_24000/batch_2_cfg_7.5_topk_16384.jpg @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:f3bce2c921e126143e013b0c112af0b7759cb94b339aa0645017ac82c1a4aa44 +size 405757 diff --git a/CKPTS/X2I_UltraEdiit_fourmask_LLava_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_filtered_objects_4mask_1280_context/003-GPT-XL/checkpoints/eval_step_24000/batch_3_cfg_7.5_topk_16384.jpg b/CKPTS/X2I_UltraEdiit_fourmask_LLava_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_filtered_objects_4mask_1280_context/003-GPT-XL/checkpoints/eval_step_24000/batch_3_cfg_7.5_topk_16384.jpg new file mode 100644 index 0000000000000000000000000000000000000000..d0bc09168caefc0b3f6ef83d372e45d7664b76af --- /dev/null +++ b/CKPTS/X2I_UltraEdiit_fourmask_LLava_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_filtered_objects_4mask_1280_context/003-GPT-XL/checkpoints/eval_step_24000/batch_3_cfg_7.5_topk_16384.jpg @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:5391da79f67cb308839d45918dfe988fd83f4b303ca062ce0ae7040f1bbff898 +size 376689 diff --git a/CKPTS/X2I_UltraEdiit_fourmask_LLava_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_filtered_objects_4mask_1280_context/003-GPT-XL/checkpoints/eval_step_24000/batch_4_cfg_7.5_topk_16384.jpg b/CKPTS/X2I_UltraEdiit_fourmask_LLava_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_filtered_objects_4mask_1280_context/003-GPT-XL/checkpoints/eval_step_24000/batch_4_cfg_7.5_topk_16384.jpg new file mode 100644 index 0000000000000000000000000000000000000000..8ea91a6d8e4fe884118b1a372c2fee3f809d115e --- /dev/null +++ b/CKPTS/X2I_UltraEdiit_fourmask_LLava_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_filtered_objects_4mask_1280_context/003-GPT-XL/checkpoints/eval_step_24000/batch_4_cfg_7.5_topk_16384.jpg @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:0735ba3ca1ab4a85972c40cabbb428f7989146692faf8b18377996f9a04b5dbf +size 341219 diff --git a/CKPTS/X2I_UltraEdiit_fourmask_LLava_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_filtered_objects_4mask_1280_context/003-GPT-XL/checkpoints/eval_step_26000/batch_0_cfg_7.5_topk_16384.jpg b/CKPTS/X2I_UltraEdiit_fourmask_LLava_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_filtered_objects_4mask_1280_context/003-GPT-XL/checkpoints/eval_step_26000/batch_0_cfg_7.5_topk_16384.jpg new file mode 100644 index 0000000000000000000000000000000000000000..23393398b37807ca0a027b151cc2e40693c80ac9 --- /dev/null +++ b/CKPTS/X2I_UltraEdiit_fourmask_LLava_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_filtered_objects_4mask_1280_context/003-GPT-XL/checkpoints/eval_step_26000/batch_0_cfg_7.5_topk_16384.jpg @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:f8a3a0972d61ca05f9ed7d8e55ac043095c36a4e75448f726ec119d486ecd12e +size 451611 diff --git a/CKPTS/X2I_UltraEdiit_fourmask_LLava_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_filtered_objects_4mask_1280_context/003-GPT-XL/checkpoints/eval_step_26000/batch_1_cfg_7.5_topk_16384.jpg b/CKPTS/X2I_UltraEdiit_fourmask_LLava_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_filtered_objects_4mask_1280_context/003-GPT-XL/checkpoints/eval_step_26000/batch_1_cfg_7.5_topk_16384.jpg new file mode 100644 index 0000000000000000000000000000000000000000..b00d0f6c065f145afdc95fc049a37c514e4246ef --- /dev/null +++ b/CKPTS/X2I_UltraEdiit_fourmask_LLava_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_filtered_objects_4mask_1280_context/003-GPT-XL/checkpoints/eval_step_26000/batch_1_cfg_7.5_topk_16384.jpg @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:672c9a2b96ce0a86c7476846bd99f61047ca58e71403dfd297ac2c5c8352decf +size 435085 diff --git a/CKPTS/X2I_UltraEdiit_fourmask_LLava_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_filtered_objects_4mask_1280_context/003-GPT-XL/checkpoints/eval_step_26000/batch_2_cfg_7.5_topk_16384.jpg b/CKPTS/X2I_UltraEdiit_fourmask_LLava_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_filtered_objects_4mask_1280_context/003-GPT-XL/checkpoints/eval_step_26000/batch_2_cfg_7.5_topk_16384.jpg new file mode 100644 index 0000000000000000000000000000000000000000..bef78154cbd241d3ea036a9a04dc8c885414ace7 --- /dev/null +++ b/CKPTS/X2I_UltraEdiit_fourmask_LLava_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_filtered_objects_4mask_1280_context/003-GPT-XL/checkpoints/eval_step_26000/batch_2_cfg_7.5_topk_16384.jpg @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:a60522d32487e163a8d399133ffcb5e9296aec53ba00763d4e99472b193ef14c +size 435739 diff --git a/CKPTS/X2I_UltraEdiit_fourmask_LLava_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_filtered_objects_4mask_1280_context/003-GPT-XL/checkpoints/eval_step_26000/batch_3_cfg_7.5_topk_16384.jpg b/CKPTS/X2I_UltraEdiit_fourmask_LLava_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_filtered_objects_4mask_1280_context/003-GPT-XL/checkpoints/eval_step_26000/batch_3_cfg_7.5_topk_16384.jpg new file mode 100644 index 0000000000000000000000000000000000000000..b25b3d205031c1a4b0fc12cdf0d8d69f1c39fe50 --- /dev/null +++ b/CKPTS/X2I_UltraEdiit_fourmask_LLava_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_filtered_objects_4mask_1280_context/003-GPT-XL/checkpoints/eval_step_26000/batch_3_cfg_7.5_topk_16384.jpg @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:ed9d4fb483416a0e7debaa598806d3bb2407ec1e9d55c4d272f0bc753ed280cd +size 407590 diff --git a/CKPTS/X2I_UltraEdiit_fourmask_LLava_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_filtered_objects_4mask_1280_context/003-GPT-XL/checkpoints/eval_step_26000/batch_4_cfg_7.5_topk_16384.jpg b/CKPTS/X2I_UltraEdiit_fourmask_LLava_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_filtered_objects_4mask_1280_context/003-GPT-XL/checkpoints/eval_step_26000/batch_4_cfg_7.5_topk_16384.jpg new file mode 100644 index 0000000000000000000000000000000000000000..c4330151434629357e0814e46bc84dcc11785564 --- /dev/null +++ b/CKPTS/X2I_UltraEdiit_fourmask_LLava_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_filtered_objects_4mask_1280_context/003-GPT-XL/checkpoints/eval_step_26000/batch_4_cfg_7.5_topk_16384.jpg @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:cf910fbbf8c891c7550000f6c926e75380c02883222ecba9bc43f66d08d7034c +size 344127 diff --git a/CKPTS/X2I_UltraEdiit_fourmask_LLava_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_filtered_objects_4mask_1280_context/003-GPT-XL/checkpoints/eval_step_28000/batch_0_cfg_7.5_topk_16384.jpg b/CKPTS/X2I_UltraEdiit_fourmask_LLava_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_filtered_objects_4mask_1280_context/003-GPT-XL/checkpoints/eval_step_28000/batch_0_cfg_7.5_topk_16384.jpg new file mode 100644 index 0000000000000000000000000000000000000000..f9255c7fc34e8d0ca675fa29d1821babb42ce7d8 --- /dev/null +++ b/CKPTS/X2I_UltraEdiit_fourmask_LLava_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_filtered_objects_4mask_1280_context/003-GPT-XL/checkpoints/eval_step_28000/batch_0_cfg_7.5_topk_16384.jpg @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:8c81da180b2c0de160c3aae5626358ec694d9b2e964f0cf1175ef82e9edf4fe8 +size 434661 diff --git a/CKPTS/X2I_UltraEdiit_fourmask_LLava_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_filtered_objects_4mask_1280_context/003-GPT-XL/checkpoints/eval_step_28000/batch_1_cfg_7.5_topk_16384.jpg b/CKPTS/X2I_UltraEdiit_fourmask_LLava_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_filtered_objects_4mask_1280_context/003-GPT-XL/checkpoints/eval_step_28000/batch_1_cfg_7.5_topk_16384.jpg new file mode 100644 index 0000000000000000000000000000000000000000..2e9572efa0ad6088c96d95e1038930effc917c50 --- /dev/null +++ b/CKPTS/X2I_UltraEdiit_fourmask_LLava_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_filtered_objects_4mask_1280_context/003-GPT-XL/checkpoints/eval_step_28000/batch_1_cfg_7.5_topk_16384.jpg @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:07fc40cd7248d4f4142aca4c9e26375d3845ae3f738d405f13cde054c79e0106 +size 425950 diff --git a/CKPTS/X2I_UltraEdiit_fourmask_LLava_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_filtered_objects_4mask_1280_context/003-GPT-XL/checkpoints/eval_step_28000/batch_2_cfg_7.5_topk_16384.jpg b/CKPTS/X2I_UltraEdiit_fourmask_LLava_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_filtered_objects_4mask_1280_context/003-GPT-XL/checkpoints/eval_step_28000/batch_2_cfg_7.5_topk_16384.jpg new file mode 100644 index 0000000000000000000000000000000000000000..e14cfa070116f01abea69ec11e49e22b9f43ee11 --- /dev/null +++ b/CKPTS/X2I_UltraEdiit_fourmask_LLava_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_filtered_objects_4mask_1280_context/003-GPT-XL/checkpoints/eval_step_28000/batch_2_cfg_7.5_topk_16384.jpg @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:0a3954157c88fa7b8c48926407e6973d6e6a0aeea085e54e07b68e4b812de230 +size 418687 diff --git a/CKPTS/X2I_UltraEdiit_fourmask_LLava_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_filtered_objects_4mask_1280_context/003-GPT-XL/checkpoints/eval_step_28000/batch_3_cfg_7.5_topk_16384.jpg b/CKPTS/X2I_UltraEdiit_fourmask_LLava_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_filtered_objects_4mask_1280_context/003-GPT-XL/checkpoints/eval_step_28000/batch_3_cfg_7.5_topk_16384.jpg new file mode 100644 index 0000000000000000000000000000000000000000..a0d30c30dfe9fe3c7ce65fb1c058d7dbed9a5e7c --- /dev/null +++ b/CKPTS/X2I_UltraEdiit_fourmask_LLava_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_filtered_objects_4mask_1280_context/003-GPT-XL/checkpoints/eval_step_28000/batch_3_cfg_7.5_topk_16384.jpg @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:15a327582f863c5393a2fa80405f628900de2038ffc3d75cae88dde0832e55fa +size 396832 diff --git a/CKPTS/X2I_UltraEdiit_fourmask_LLava_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_filtered_objects_4mask_1280_context/003-GPT-XL/checkpoints/eval_step_28000/batch_4_cfg_7.5_topk_16384.jpg b/CKPTS/X2I_UltraEdiit_fourmask_LLava_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_filtered_objects_4mask_1280_context/003-GPT-XL/checkpoints/eval_step_28000/batch_4_cfg_7.5_topk_16384.jpg new file mode 100644 index 0000000000000000000000000000000000000000..de4564b352643f6b0454302962ddfaa36f8c997d --- /dev/null +++ b/CKPTS/X2I_UltraEdiit_fourmask_LLava_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_filtered_objects_4mask_1280_context/003-GPT-XL/checkpoints/eval_step_28000/batch_4_cfg_7.5_topk_16384.jpg @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:b7dedba66aecfe3793c54b0104c4eeeaff95ec33ba73cd4352c951d068e666cb +size 344649 diff --git a/CKPTS/X2I_UltraEdiit_fourmask_LLava_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_filtered_objects_4mask_1280_context/003-GPT-XL/checkpoints/eval_step_30000/batch_0_cfg_7.5_topk_16384.jpg b/CKPTS/X2I_UltraEdiit_fourmask_LLava_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_filtered_objects_4mask_1280_context/003-GPT-XL/checkpoints/eval_step_30000/batch_0_cfg_7.5_topk_16384.jpg new file mode 100644 index 0000000000000000000000000000000000000000..3c5e5d846e2e648df6ea08c38f49cb84d05575b6 --- /dev/null +++ b/CKPTS/X2I_UltraEdiit_fourmask_LLava_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_filtered_objects_4mask_1280_context/003-GPT-XL/checkpoints/eval_step_30000/batch_0_cfg_7.5_topk_16384.jpg @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:6dde179ca58ca57e7ea7914fa81d27c9a765a1fb8c6244d7efd2cf0e06d1a9a7 +size 443146 diff --git a/CKPTS/X2I_UltraEdiit_fourmask_LLava_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_filtered_objects_4mask_1280_context/003-GPT-XL/checkpoints/eval_step_30000/batch_1_cfg_7.5_topk_16384.jpg b/CKPTS/X2I_UltraEdiit_fourmask_LLava_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_filtered_objects_4mask_1280_context/003-GPT-XL/checkpoints/eval_step_30000/batch_1_cfg_7.5_topk_16384.jpg new file mode 100644 index 0000000000000000000000000000000000000000..4ca480ed1b61ed8c499c272ab6a00590a813faea --- /dev/null +++ b/CKPTS/X2I_UltraEdiit_fourmask_LLava_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_filtered_objects_4mask_1280_context/003-GPT-XL/checkpoints/eval_step_30000/batch_1_cfg_7.5_topk_16384.jpg @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:800a462af514bc6cdae373ea97007e40fff278975f066d7d234d241f49460f86 +size 428421 diff --git a/CKPTS/X2I_UltraEdiit_fourmask_LLava_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_filtered_objects_4mask_1280_context/003-GPT-XL/checkpoints/eval_step_30000/batch_2_cfg_7.5_topk_16384.jpg b/CKPTS/X2I_UltraEdiit_fourmask_LLava_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_filtered_objects_4mask_1280_context/003-GPT-XL/checkpoints/eval_step_30000/batch_2_cfg_7.5_topk_16384.jpg new file mode 100644 index 0000000000000000000000000000000000000000..192cc0e20fb54ec7198cea9f562b3fb16eb1a30d --- /dev/null +++ b/CKPTS/X2I_UltraEdiit_fourmask_LLava_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_filtered_objects_4mask_1280_context/003-GPT-XL/checkpoints/eval_step_30000/batch_2_cfg_7.5_topk_16384.jpg @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:28a12de11e43f87426fdbc7ebc6a11dec74cf75335d7be5020007ce560d38e79 +size 413759 diff --git a/CKPTS/X2I_UltraEdiit_fourmask_LLava_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_filtered_objects_4mask_1280_context/003-GPT-XL/checkpoints/eval_step_30000/batch_3_cfg_7.5_topk_16384.jpg b/CKPTS/X2I_UltraEdiit_fourmask_LLava_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_filtered_objects_4mask_1280_context/003-GPT-XL/checkpoints/eval_step_30000/batch_3_cfg_7.5_topk_16384.jpg new file mode 100644 index 0000000000000000000000000000000000000000..8263c965ddae07583d46809a0892ae9be0cf88a3 --- /dev/null +++ b/CKPTS/X2I_UltraEdiit_fourmask_LLava_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_filtered_objects_4mask_1280_context/003-GPT-XL/checkpoints/eval_step_30000/batch_3_cfg_7.5_topk_16384.jpg @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:bfc77d97fc7b7ea7a91436413b7066d65b951fca8c0ed42d98628eaebca3b783 +size 397433 diff --git a/CKPTS/X2I_UltraEdiit_fourmask_LLava_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_filtered_objects_4mask_1280_context/003-GPT-XL/checkpoints/eval_step_30000/batch_4_cfg_7.5_topk_16384.jpg b/CKPTS/X2I_UltraEdiit_fourmask_LLava_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_filtered_objects_4mask_1280_context/003-GPT-XL/checkpoints/eval_step_30000/batch_4_cfg_7.5_topk_16384.jpg new file mode 100644 index 0000000000000000000000000000000000000000..d372d1669ba1ca4f4363d984016cc407906bca0d --- /dev/null +++ b/CKPTS/X2I_UltraEdiit_fourmask_LLava_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_filtered_objects_4mask_1280_context/003-GPT-XL/checkpoints/eval_step_30000/batch_4_cfg_7.5_topk_16384.jpg @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:6cd3d5143af8a0bc9aa37a7957873ff87423eed0c9d867a91cb9f15e72e0ebac +size 341376 diff --git a/CKPTS/X2I_UltraEdiit_fourmask_LLava_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_filtered_objects_4mask_1280_context/003-GPT-XL/checkpoints/eval_step_32000/batch_0_cfg_7.5_topk_16384.jpg b/CKPTS/X2I_UltraEdiit_fourmask_LLava_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_filtered_objects_4mask_1280_context/003-GPT-XL/checkpoints/eval_step_32000/batch_0_cfg_7.5_topk_16384.jpg new file mode 100644 index 0000000000000000000000000000000000000000..68e43f6365c9b776050f61213ab1eab0c8362af3 --- /dev/null +++ b/CKPTS/X2I_UltraEdiit_fourmask_LLava_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_filtered_objects_4mask_1280_context/003-GPT-XL/checkpoints/eval_step_32000/batch_0_cfg_7.5_topk_16384.jpg @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:ca24bed9d6af847a8cad92cee5c0478c9f703376a5edba12d4a066bdf2a23db2 +size 433071 diff --git a/CKPTS/X2I_UltraEdiit_fourmask_LLava_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_filtered_objects_4mask_1280_context/003-GPT-XL/checkpoints/eval_step_32000/batch_1_cfg_7.5_topk_16384.jpg b/CKPTS/X2I_UltraEdiit_fourmask_LLava_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_filtered_objects_4mask_1280_context/003-GPT-XL/checkpoints/eval_step_32000/batch_1_cfg_7.5_topk_16384.jpg new file mode 100644 index 0000000000000000000000000000000000000000..fd9cfc2f5eb5c63674c8a6677d5ea8b102b1340b --- /dev/null +++ b/CKPTS/X2I_UltraEdiit_fourmask_LLava_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_filtered_objects_4mask_1280_context/003-GPT-XL/checkpoints/eval_step_32000/batch_1_cfg_7.5_topk_16384.jpg @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:3403301a0a2123d7f0b31198846367b904b7aa3390ee3727fd78606b0c18f908 +size 437824 diff --git a/CKPTS/X2I_UltraEdiit_fourmask_LLava_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_filtered_objects_4mask_1280_context/003-GPT-XL/checkpoints/eval_step_32000/batch_2_cfg_7.5_topk_16384.jpg b/CKPTS/X2I_UltraEdiit_fourmask_LLava_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_filtered_objects_4mask_1280_context/003-GPT-XL/checkpoints/eval_step_32000/batch_2_cfg_7.5_topk_16384.jpg new file mode 100644 index 0000000000000000000000000000000000000000..ce325cd0ca15c0209ed4333c37c883e27ce00a2d --- /dev/null +++ b/CKPTS/X2I_UltraEdiit_fourmask_LLava_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_filtered_objects_4mask_1280_context/003-GPT-XL/checkpoints/eval_step_32000/batch_2_cfg_7.5_topk_16384.jpg @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:9f50259bd50104b936812e51164b092ad195bd3744d12c9bc545cec3804642bc +size 413865 diff --git a/CKPTS/X2I_UltraEdiit_fourmask_LLava_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_filtered_objects_4mask_1280_context/003-GPT-XL/checkpoints/eval_step_32000/batch_3_cfg_7.5_topk_16384.jpg b/CKPTS/X2I_UltraEdiit_fourmask_LLava_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_filtered_objects_4mask_1280_context/003-GPT-XL/checkpoints/eval_step_32000/batch_3_cfg_7.5_topk_16384.jpg new file mode 100644 index 0000000000000000000000000000000000000000..1c1bb694729955024fb1e40a2b3e66d2a799112e --- /dev/null +++ b/CKPTS/X2I_UltraEdiit_fourmask_LLava_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_filtered_objects_4mask_1280_context/003-GPT-XL/checkpoints/eval_step_32000/batch_3_cfg_7.5_topk_16384.jpg @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:8347bf4c3c0b8189cdd7516a5411309839f6a59dee1a28ca113ea23a7b424dc0 +size 394246 diff --git a/CKPTS/X2I_UltraEdiit_fourmask_LLava_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_filtered_objects_4mask_1280_context/003-GPT-XL/checkpoints/eval_step_32000/batch_4_cfg_7.5_topk_16384.jpg b/CKPTS/X2I_UltraEdiit_fourmask_LLava_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_filtered_objects_4mask_1280_context/003-GPT-XL/checkpoints/eval_step_32000/batch_4_cfg_7.5_topk_16384.jpg new file mode 100644 index 0000000000000000000000000000000000000000..882a7b248d56084cd5884a33ed68bc41f88669f7 --- /dev/null +++ b/CKPTS/X2I_UltraEdiit_fourmask_LLava_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_filtered_objects_4mask_1280_context/003-GPT-XL/checkpoints/eval_step_32000/batch_4_cfg_7.5_topk_16384.jpg @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:734af251e60d32ea1b8002f09e93ef0bb1e4c86308cc866dd0a086532230c65a +size 340391 diff --git a/CKPTS/X2I_UltraEdiit_fourmask_LLava_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_filtered_objects_4mask_1280_context/003-GPT-XL/checkpoints/eval_step_34000/batch_0_cfg_7.5_topk_16384.jpg b/CKPTS/X2I_UltraEdiit_fourmask_LLava_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_filtered_objects_4mask_1280_context/003-GPT-XL/checkpoints/eval_step_34000/batch_0_cfg_7.5_topk_16384.jpg new file mode 100644 index 0000000000000000000000000000000000000000..f5596915b735434c6e7290140d53ec33b90d0e13 --- /dev/null +++ b/CKPTS/X2I_UltraEdiit_fourmask_LLava_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_filtered_objects_4mask_1280_context/003-GPT-XL/checkpoints/eval_step_34000/batch_0_cfg_7.5_topk_16384.jpg @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:ef1268cc1120607fe42319eb0ea1a569ca0679e2aab5723b49cc275c43f7a717 +size 432964 diff --git a/CKPTS/X2I_UltraEdiit_fourmask_LLava_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_filtered_objects_4mask_1280_context/003-GPT-XL/checkpoints/eval_step_34000/batch_1_cfg_7.5_topk_16384.jpg b/CKPTS/X2I_UltraEdiit_fourmask_LLava_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_filtered_objects_4mask_1280_context/003-GPT-XL/checkpoints/eval_step_34000/batch_1_cfg_7.5_topk_16384.jpg new file mode 100644 index 0000000000000000000000000000000000000000..d8c45bcaa501342c90eaa44ddf9c037f5529c9e7 --- /dev/null +++ b/CKPTS/X2I_UltraEdiit_fourmask_LLava_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_filtered_objects_4mask_1280_context/003-GPT-XL/checkpoints/eval_step_34000/batch_1_cfg_7.5_topk_16384.jpg @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:45589116b7e47d02098c1a74619e6a6e4651f9f4326bf652bce0ddfa2ed659b5 +size 410776 diff --git a/CKPTS/X2I_UltraEdiit_fourmask_LLava_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_filtered_objects_4mask_1280_context/003-GPT-XL/checkpoints/eval_step_34000/batch_2_cfg_7.5_topk_16384.jpg b/CKPTS/X2I_UltraEdiit_fourmask_LLava_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_filtered_objects_4mask_1280_context/003-GPT-XL/checkpoints/eval_step_34000/batch_2_cfg_7.5_topk_16384.jpg new file mode 100644 index 0000000000000000000000000000000000000000..7cbefc718322203e94498887a56e890fd8fd1467 --- /dev/null +++ b/CKPTS/X2I_UltraEdiit_fourmask_LLava_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_filtered_objects_4mask_1280_context/003-GPT-XL/checkpoints/eval_step_34000/batch_2_cfg_7.5_topk_16384.jpg @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:f63bc1405e4f85e890f2cc5a078df754974688b0a7a62f85bc4c5bfc80a6331f +size 413114 diff --git a/CKPTS/X2I_UltraEdiit_fourmask_LLava_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_filtered_objects_4mask_1280_context/003-GPT-XL/checkpoints/eval_step_34000/batch_3_cfg_7.5_topk_16384.jpg b/CKPTS/X2I_UltraEdiit_fourmask_LLava_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_filtered_objects_4mask_1280_context/003-GPT-XL/checkpoints/eval_step_34000/batch_3_cfg_7.5_topk_16384.jpg new file mode 100644 index 0000000000000000000000000000000000000000..8ef8d37db20ba496559c7ec8129d62635618008a --- /dev/null +++ b/CKPTS/X2I_UltraEdiit_fourmask_LLava_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_filtered_objects_4mask_1280_context/003-GPT-XL/checkpoints/eval_step_34000/batch_3_cfg_7.5_topk_16384.jpg @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:0db25f03b500173a9a9412892d81efaca8cf8bb115c865daa153eacf691c503f +size 405652 diff --git a/CKPTS/X2I_UltraEdiit_fourmask_LLava_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_filtered_objects_4mask_1280_context/003-GPT-XL/checkpoints/eval_step_34000/batch_4_cfg_7.5_topk_16384.jpg b/CKPTS/X2I_UltraEdiit_fourmask_LLava_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_filtered_objects_4mask_1280_context/003-GPT-XL/checkpoints/eval_step_34000/batch_4_cfg_7.5_topk_16384.jpg new file mode 100644 index 0000000000000000000000000000000000000000..7f0930b1f1f1f5dc1b36dad604528d57167f3205 --- /dev/null +++ b/CKPTS/X2I_UltraEdiit_fourmask_LLava_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_filtered_objects_4mask_1280_context/003-GPT-XL/checkpoints/eval_step_34000/batch_4_cfg_7.5_topk_16384.jpg @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:18fe08a08589da81674b5e6805639ddbcf68cec60c3c53245af969464ec560d6 +size 341431 diff --git a/CKPTS/X2I_UltraEdiit_fourmask_LLava_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_filtered_objects_4mask_1280_context/003-GPT-XL/checkpoints/eval_step_36000/batch_0_cfg_7.5_topk_16384.jpg b/CKPTS/X2I_UltraEdiit_fourmask_LLava_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_filtered_objects_4mask_1280_context/003-GPT-XL/checkpoints/eval_step_36000/batch_0_cfg_7.5_topk_16384.jpg new file mode 100644 index 0000000000000000000000000000000000000000..45e912993d086bb0a7c8adefe78a4609fd69c33d --- /dev/null +++ b/CKPTS/X2I_UltraEdiit_fourmask_LLava_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_filtered_objects_4mask_1280_context/003-GPT-XL/checkpoints/eval_step_36000/batch_0_cfg_7.5_topk_16384.jpg @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:8d35fb34f2047d2a11bb5d15c6b9041be10f0e19999ab80c4b30cc6dff69a5df +size 447353 diff --git a/CKPTS/X2I_UltraEdiit_fourmask_LLava_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_filtered_objects_4mask_1280_context/003-GPT-XL/checkpoints/eval_step_36000/batch_1_cfg_7.5_topk_16384.jpg b/CKPTS/X2I_UltraEdiit_fourmask_LLava_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_filtered_objects_4mask_1280_context/003-GPT-XL/checkpoints/eval_step_36000/batch_1_cfg_7.5_topk_16384.jpg new file mode 100644 index 0000000000000000000000000000000000000000..c55c72ff5b02b5720cd3a4eace5a9bc65fa598ad --- /dev/null +++ b/CKPTS/X2I_UltraEdiit_fourmask_LLava_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_filtered_objects_4mask_1280_context/003-GPT-XL/checkpoints/eval_step_36000/batch_1_cfg_7.5_topk_16384.jpg @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:46d14c7ec8c15b0030911358c0018f71cc8029b48e39c735d102d3ea3d14bcb4 +size 421283 diff --git a/CKPTS/X2I_UltraEdiit_fourmask_LLava_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_filtered_objects_4mask_1280_context/003-GPT-XL/checkpoints/eval_step_36000/batch_2_cfg_7.5_topk_16384.jpg b/CKPTS/X2I_UltraEdiit_fourmask_LLava_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_filtered_objects_4mask_1280_context/003-GPT-XL/checkpoints/eval_step_36000/batch_2_cfg_7.5_topk_16384.jpg new file mode 100644 index 0000000000000000000000000000000000000000..a15e246c78b4509e1248df77886366d791fd9df0 --- /dev/null +++ b/CKPTS/X2I_UltraEdiit_fourmask_LLava_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_filtered_objects_4mask_1280_context/003-GPT-XL/checkpoints/eval_step_36000/batch_2_cfg_7.5_topk_16384.jpg @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:d2302a5c0faacd8316ff6ef5eaf7763cbf178a64e8a41b0c396d7d0658cbf4af +size 426591 diff --git a/CKPTS/X2I_UltraEdiit_fourmask_LLava_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_filtered_objects_4mask_1280_context/003-GPT-XL/checkpoints/eval_step_36000/batch_3_cfg_7.5_topk_16384.jpg b/CKPTS/X2I_UltraEdiit_fourmask_LLava_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_filtered_objects_4mask_1280_context/003-GPT-XL/checkpoints/eval_step_36000/batch_3_cfg_7.5_topk_16384.jpg new file mode 100644 index 0000000000000000000000000000000000000000..393ef4dcb21727dfb73346d8ff501e995d177e18 --- /dev/null +++ b/CKPTS/X2I_UltraEdiit_fourmask_LLava_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_filtered_objects_4mask_1280_context/003-GPT-XL/checkpoints/eval_step_36000/batch_3_cfg_7.5_topk_16384.jpg @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:a6d70bd2de34022631ff88d2a2e2a906aa4de8ec46ab4e457a67250211f01610 +size 416038 diff --git a/CKPTS/X2I_UltraEdiit_fourmask_LLava_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_filtered_objects_4mask_1280_context/003-GPT-XL/checkpoints/eval_step_36000/batch_4_cfg_7.5_topk_16384.jpg b/CKPTS/X2I_UltraEdiit_fourmask_LLava_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_filtered_objects_4mask_1280_context/003-GPT-XL/checkpoints/eval_step_36000/batch_4_cfg_7.5_topk_16384.jpg new file mode 100644 index 0000000000000000000000000000000000000000..2e31a298818c6e906cefdfcdbea38a5a5004e2c0 --- /dev/null +++ b/CKPTS/X2I_UltraEdiit_fourmask_LLava_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_filtered_objects_4mask_1280_context/003-GPT-XL/checkpoints/eval_step_36000/batch_4_cfg_7.5_topk_16384.jpg @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:f6847dc83925cd91298e09e043924fef629075aca65e3d89ecf15ef8ee4b211b +size 347038 diff --git a/CKPTS/X2I_UltraEdiit_fourmask_LLava_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_filtered_objects_4mask_1280_context/003-GPT-XL/checkpoints/eval_step_38000/batch_0_cfg_7.5_topk_16384.jpg b/CKPTS/X2I_UltraEdiit_fourmask_LLava_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_filtered_objects_4mask_1280_context/003-GPT-XL/checkpoints/eval_step_38000/batch_0_cfg_7.5_topk_16384.jpg new file mode 100644 index 0000000000000000000000000000000000000000..e3a6b1311ee3c8d344a080c6beb702a50e1f7b87 --- /dev/null +++ b/CKPTS/X2I_UltraEdiit_fourmask_LLava_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_filtered_objects_4mask_1280_context/003-GPT-XL/checkpoints/eval_step_38000/batch_0_cfg_7.5_topk_16384.jpg @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:8dc9162373f0ec617a5e6a9cbc79bdca0debbb2cd44974af81568d6f93cc4ec9 +size 429020 diff --git a/CKPTS/X2I_UltraEdiit_fourmask_LLava_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_filtered_objects_4mask_1280_context/003-GPT-XL/checkpoints/eval_step_38000/batch_1_cfg_7.5_topk_16384.jpg b/CKPTS/X2I_UltraEdiit_fourmask_LLava_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_filtered_objects_4mask_1280_context/003-GPT-XL/checkpoints/eval_step_38000/batch_1_cfg_7.5_topk_16384.jpg new file mode 100644 index 0000000000000000000000000000000000000000..d54df46bc6574d8020f2de283fd2dd9f4f117beb --- /dev/null +++ b/CKPTS/X2I_UltraEdiit_fourmask_LLava_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_filtered_objects_4mask_1280_context/003-GPT-XL/checkpoints/eval_step_38000/batch_1_cfg_7.5_topk_16384.jpg @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:1a9fa69596038d9969090b3b9c08ab78508710124ad27d5e4ea79fa32b324660 +size 410267 diff --git a/CKPTS/X2I_UltraEdiit_fourmask_LLava_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_filtered_objects_4mask_1280_context/003-GPT-XL/checkpoints/eval_step_38000/batch_2_cfg_7.5_topk_16384.jpg b/CKPTS/X2I_UltraEdiit_fourmask_LLava_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_filtered_objects_4mask_1280_context/003-GPT-XL/checkpoints/eval_step_38000/batch_2_cfg_7.5_topk_16384.jpg new file mode 100644 index 0000000000000000000000000000000000000000..f5d66cdf761b53d94738c1732e178f727ba121c5 --- /dev/null +++ b/CKPTS/X2I_UltraEdiit_fourmask_LLava_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_filtered_objects_4mask_1280_context/003-GPT-XL/checkpoints/eval_step_38000/batch_2_cfg_7.5_topk_16384.jpg @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:24a194ec112899271a0638d23fef3cc04e276aa1d7d46ce107024d503957e65e +size 432507 diff --git a/CKPTS/X2I_UltraEdiit_fourmask_LLava_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_filtered_objects_4mask_1280_context/003-GPT-XL/checkpoints/eval_step_38000/batch_3_cfg_7.5_topk_16384.jpg b/CKPTS/X2I_UltraEdiit_fourmask_LLava_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_filtered_objects_4mask_1280_context/003-GPT-XL/checkpoints/eval_step_38000/batch_3_cfg_7.5_topk_16384.jpg new file mode 100644 index 0000000000000000000000000000000000000000..7435eda4e79f60703ba405f22a3c2129d992a63e --- /dev/null +++ b/CKPTS/X2I_UltraEdiit_fourmask_LLava_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_filtered_objects_4mask_1280_context/003-GPT-XL/checkpoints/eval_step_38000/batch_3_cfg_7.5_topk_16384.jpg @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:c900db7a80d1ba044c7f6e8792649e82f93b0cc2288ea4b42522ad9b65fa271d +size 379125 diff --git a/CKPTS/X2I_UltraEdiit_fourmask_LLava_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_filtered_objects_4mask_1280_context/003-GPT-XL/checkpoints/eval_step_38000/batch_4_cfg_7.5_topk_16384.jpg b/CKPTS/X2I_UltraEdiit_fourmask_LLava_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_filtered_objects_4mask_1280_context/003-GPT-XL/checkpoints/eval_step_38000/batch_4_cfg_7.5_topk_16384.jpg new file mode 100644 index 0000000000000000000000000000000000000000..5b53ea42b48125483f552ed339d6c55a25de531b --- /dev/null +++ b/CKPTS/X2I_UltraEdiit_fourmask_LLava_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_filtered_objects_4mask_1280_context/003-GPT-XL/checkpoints/eval_step_38000/batch_4_cfg_7.5_topk_16384.jpg @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:b9d678993e23952f28de6f2b6d4b37de929554d91d5c59b7059709ab3bf2a11f +size 353617 diff --git a/CKPTS/X2I_UltraEdiit_fourmask_LLava_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_filtered_objects_4mask_1280_context/003-GPT-XL/checkpoints/eval_step_4000/batch_0_cfg_7.5_topk_16384.jpg b/CKPTS/X2I_UltraEdiit_fourmask_LLava_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_filtered_objects_4mask_1280_context/003-GPT-XL/checkpoints/eval_step_4000/batch_0_cfg_7.5_topk_16384.jpg new file mode 100644 index 0000000000000000000000000000000000000000..253d3e37fc2bcbf9beccfd73d766cec3f8e93b96 --- /dev/null +++ b/CKPTS/X2I_UltraEdiit_fourmask_LLava_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_filtered_objects_4mask_1280_context/003-GPT-XL/checkpoints/eval_step_4000/batch_0_cfg_7.5_topk_16384.jpg @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:ac33d805ad965a337ee3351ae71f9e6b2e88e494b4608493b827c82a9ff404ed +size 446687 diff --git a/CKPTS/X2I_UltraEdiit_fourmask_LLava_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_filtered_objects_4mask_1280_context/003-GPT-XL/checkpoints/eval_step_4000/batch_1_cfg_7.5_topk_16384.jpg b/CKPTS/X2I_UltraEdiit_fourmask_LLava_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_filtered_objects_4mask_1280_context/003-GPT-XL/checkpoints/eval_step_4000/batch_1_cfg_7.5_topk_16384.jpg new file mode 100644 index 0000000000000000000000000000000000000000..989ba8e48fd1d98694cf5ed61fc8d767c1cbe18b --- /dev/null +++ b/CKPTS/X2I_UltraEdiit_fourmask_LLava_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_filtered_objects_4mask_1280_context/003-GPT-XL/checkpoints/eval_step_4000/batch_1_cfg_7.5_topk_16384.jpg @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:120141f153086a1af7e71fca2c5310449c07a0602a3b6b64e76b8f613939d2a3 +size 414604 diff --git a/CKPTS/X2I_UltraEdiit_fourmask_LLava_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_filtered_objects_4mask_1280_context/003-GPT-XL/checkpoints/eval_step_4000/batch_2_cfg_7.5_topk_16384.jpg b/CKPTS/X2I_UltraEdiit_fourmask_LLava_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_filtered_objects_4mask_1280_context/003-GPT-XL/checkpoints/eval_step_4000/batch_2_cfg_7.5_topk_16384.jpg new file mode 100644 index 0000000000000000000000000000000000000000..110c7d0ed5bfd981b89d02c2659ebace987ce4e8 --- /dev/null +++ b/CKPTS/X2I_UltraEdiit_fourmask_LLava_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_filtered_objects_4mask_1280_context/003-GPT-XL/checkpoints/eval_step_4000/batch_2_cfg_7.5_topk_16384.jpg @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:94e37e373dfb53fa309d80c615969a0209237a66d13a05c9148cbeeb3fd604ab +size 429857 diff --git a/CKPTS/X2I_UltraEdiit_fourmask_LLava_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_filtered_objects_4mask_1280_context/003-GPT-XL/checkpoints/eval_step_4000/batch_3_cfg_7.5_topk_16384.jpg b/CKPTS/X2I_UltraEdiit_fourmask_LLava_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_filtered_objects_4mask_1280_context/003-GPT-XL/checkpoints/eval_step_4000/batch_3_cfg_7.5_topk_16384.jpg new file mode 100644 index 0000000000000000000000000000000000000000..fac55c9a12d6e968f1f5584ffc8c4da6edf1818a --- /dev/null +++ b/CKPTS/X2I_UltraEdiit_fourmask_LLava_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_filtered_objects_4mask_1280_context/003-GPT-XL/checkpoints/eval_step_4000/batch_3_cfg_7.5_topk_16384.jpg @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:2d978a705dddd3316c02af31cd23c6b6ce78c86bc38c06e1c1451bf689f8a449 +size 409901 diff --git a/CKPTS/X2I_UltraEdiit_fourmask_LLava_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_filtered_objects_4mask_1280_context/003-GPT-XL/checkpoints/eval_step_4000/batch_4_cfg_7.5_topk_16384.jpg b/CKPTS/X2I_UltraEdiit_fourmask_LLava_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_filtered_objects_4mask_1280_context/003-GPT-XL/checkpoints/eval_step_4000/batch_4_cfg_7.5_topk_16384.jpg new file mode 100644 index 0000000000000000000000000000000000000000..eedfcc20c5ec91c697b8bcd11c90cfec45ca309f --- /dev/null +++ b/CKPTS/X2I_UltraEdiit_fourmask_LLava_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_filtered_objects_4mask_1280_context/003-GPT-XL/checkpoints/eval_step_4000/batch_4_cfg_7.5_topk_16384.jpg @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:71cfaf59eabac563fbf7d6fe41bb82c92b52120abd0e2f0f3ee77c0b0a35ff8f +size 377744 diff --git a/CKPTS/X2I_UltraEdiit_fourmask_LLava_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_filtered_objects_4mask_1280_context/003-GPT-XL/checkpoints/eval_step_40000/batch_0_cfg_7.5_topk_16384.jpg b/CKPTS/X2I_UltraEdiit_fourmask_LLava_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_filtered_objects_4mask_1280_context/003-GPT-XL/checkpoints/eval_step_40000/batch_0_cfg_7.5_topk_16384.jpg new file mode 100644 index 0000000000000000000000000000000000000000..69f30ddc87be2b518c7f6d54abd5b4c46058cffc --- /dev/null +++ b/CKPTS/X2I_UltraEdiit_fourmask_LLava_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_filtered_objects_4mask_1280_context/003-GPT-XL/checkpoints/eval_step_40000/batch_0_cfg_7.5_topk_16384.jpg @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:bcbb58773bf0cca583fa3645e2a52d330200a765a71a58c91a349baefe99f364 +size 432344 diff --git a/CKPTS/X2I_UltraEdiit_fourmask_LLava_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_filtered_objects_4mask_1280_context/003-GPT-XL/checkpoints/eval_step_40000/batch_1_cfg_7.5_topk_16384.jpg b/CKPTS/X2I_UltraEdiit_fourmask_LLava_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_filtered_objects_4mask_1280_context/003-GPT-XL/checkpoints/eval_step_40000/batch_1_cfg_7.5_topk_16384.jpg new file mode 100644 index 0000000000000000000000000000000000000000..a66a1e0bdc15f86ce1037a84544d7ccf7675684a --- /dev/null +++ b/CKPTS/X2I_UltraEdiit_fourmask_LLava_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_filtered_objects_4mask_1280_context/003-GPT-XL/checkpoints/eval_step_40000/batch_1_cfg_7.5_topk_16384.jpg @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:90646ed6f50b44b9794b50e4b7a92e862ed0b3b6c0a0a891f61516dc482dbcc6 +size 417270 diff --git a/CKPTS/X2I_UltraEdiit_fourmask_LLava_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_filtered_objects_4mask_1280_context/003-GPT-XL/checkpoints/eval_step_40000/batch_2_cfg_7.5_topk_16384.jpg b/CKPTS/X2I_UltraEdiit_fourmask_LLava_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_filtered_objects_4mask_1280_context/003-GPT-XL/checkpoints/eval_step_40000/batch_2_cfg_7.5_topk_16384.jpg new file mode 100644 index 0000000000000000000000000000000000000000..e66249146dfe597fb869cb0bfe1f8a9bc9b9131e --- /dev/null +++ b/CKPTS/X2I_UltraEdiit_fourmask_LLava_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_filtered_objects_4mask_1280_context/003-GPT-XL/checkpoints/eval_step_40000/batch_2_cfg_7.5_topk_16384.jpg @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:e9710eb33132b5fe93c1c70fd705aa4da43497071f05aed0e89455bc81daa950 +size 429878 diff --git a/CKPTS/X2I_UltraEdiit_fourmask_LLava_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_filtered_objects_4mask_1280_context/003-GPT-XL/checkpoints/eval_step_40000/batch_3_cfg_7.5_topk_16384.jpg b/CKPTS/X2I_UltraEdiit_fourmask_LLava_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_filtered_objects_4mask_1280_context/003-GPT-XL/checkpoints/eval_step_40000/batch_3_cfg_7.5_topk_16384.jpg new file mode 100644 index 0000000000000000000000000000000000000000..57678c90a491d09d16fbb1d4e6265a0b6c383be5 --- /dev/null +++ b/CKPTS/X2I_UltraEdiit_fourmask_LLava_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_filtered_objects_4mask_1280_context/003-GPT-XL/checkpoints/eval_step_40000/batch_3_cfg_7.5_topk_16384.jpg @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:7c02f96a24e06f0de54635a9573a3a4642878235b5d5ca150f54f3053fdc90e4 +size 407447 diff --git a/CKPTS/X2I_UltraEdiit_fourmask_LLava_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_filtered_objects_4mask_1280_context/003-GPT-XL/checkpoints/eval_step_40000/batch_4_cfg_7.5_topk_16384.jpg b/CKPTS/X2I_UltraEdiit_fourmask_LLava_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_filtered_objects_4mask_1280_context/003-GPT-XL/checkpoints/eval_step_40000/batch_4_cfg_7.5_topk_16384.jpg new file mode 100644 index 0000000000000000000000000000000000000000..76a5b951444644d645b0fd499d0c01b546b1f472 --- /dev/null +++ b/CKPTS/X2I_UltraEdiit_fourmask_LLava_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_filtered_objects_4mask_1280_context/003-GPT-XL/checkpoints/eval_step_40000/batch_4_cfg_7.5_topk_16384.jpg @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:ccd9a5fd25021dbbc10e8aae8a2f9176b84b717a6071921eeb957576ea894dd1 +size 353646 diff --git a/CKPTS/X2I_UltraEdiit_fourmask_LLava_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_filtered_objects_4mask_1280_context/003-GPT-XL/checkpoints/eval_step_42000/batch_0_cfg_7.5_topk_16384.jpg b/CKPTS/X2I_UltraEdiit_fourmask_LLava_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_filtered_objects_4mask_1280_context/003-GPT-XL/checkpoints/eval_step_42000/batch_0_cfg_7.5_topk_16384.jpg new file mode 100644 index 0000000000000000000000000000000000000000..8fd65530717d9563849d4ca3ae1c1bec66a9e4ce --- /dev/null +++ b/CKPTS/X2I_UltraEdiit_fourmask_LLava_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_filtered_objects_4mask_1280_context/003-GPT-XL/checkpoints/eval_step_42000/batch_0_cfg_7.5_topk_16384.jpg @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:81f53d24ddbdba549ba68b7feb51b6bf79b396655353b729e340841750a5ee49 +size 450677 diff --git a/CKPTS/X2I_UltraEdiit_fourmask_LLava_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_filtered_objects_4mask_1280_context/003-GPT-XL/checkpoints/eval_step_42000/batch_1_cfg_7.5_topk_16384.jpg b/CKPTS/X2I_UltraEdiit_fourmask_LLava_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_filtered_objects_4mask_1280_context/003-GPT-XL/checkpoints/eval_step_42000/batch_1_cfg_7.5_topk_16384.jpg new file mode 100644 index 0000000000000000000000000000000000000000..3877ba78c69e3fe385264076dcc0433827df9154 --- /dev/null +++ b/CKPTS/X2I_UltraEdiit_fourmask_LLava_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_filtered_objects_4mask_1280_context/003-GPT-XL/checkpoints/eval_step_42000/batch_1_cfg_7.5_topk_16384.jpg @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:177b91edc35d3ee07129c432c10e1d063260cb379a6c1685dddcadaa5bc586ed +size 417734 diff --git a/CKPTS/X2I_UltraEdiit_fourmask_LLava_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_filtered_objects_4mask_1280_context/003-GPT-XL/checkpoints/eval_step_42000/batch_2_cfg_7.5_topk_16384.jpg b/CKPTS/X2I_UltraEdiit_fourmask_LLava_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_filtered_objects_4mask_1280_context/003-GPT-XL/checkpoints/eval_step_42000/batch_2_cfg_7.5_topk_16384.jpg new file mode 100644 index 0000000000000000000000000000000000000000..d89001dce1181e2a6c21c2d59c542bb566d5bb6b --- /dev/null +++ b/CKPTS/X2I_UltraEdiit_fourmask_LLava_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_filtered_objects_4mask_1280_context/003-GPT-XL/checkpoints/eval_step_42000/batch_2_cfg_7.5_topk_16384.jpg @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:052d97946bd7c966e7487dd3856997ed5274b2df9025831f05173387d245d14e +size 421392 diff --git a/CKPTS/X2I_UltraEdiit_fourmask_LLava_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_filtered_objects_4mask_1280_context/003-GPT-XL/checkpoints/eval_step_42000/batch_3_cfg_7.5_topk_16384.jpg b/CKPTS/X2I_UltraEdiit_fourmask_LLava_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_filtered_objects_4mask_1280_context/003-GPT-XL/checkpoints/eval_step_42000/batch_3_cfg_7.5_topk_16384.jpg new file mode 100644 index 0000000000000000000000000000000000000000..a590a7af6a44ca8ebc1db93aeec832ff60c1ff63 --- /dev/null +++ b/CKPTS/X2I_UltraEdiit_fourmask_LLava_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_filtered_objects_4mask_1280_context/003-GPT-XL/checkpoints/eval_step_42000/batch_3_cfg_7.5_topk_16384.jpg @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:827b5da1b2a847be993cf2674f29b1cd0580dc17f6ee5b5b3d25f988f1d46a3f +size 391050 diff --git a/CKPTS/X2I_UltraEdiit_fourmask_LLava_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_filtered_objects_4mask_1280_context/003-GPT-XL/checkpoints/eval_step_42000/batch_4_cfg_7.5_topk_16384.jpg b/CKPTS/X2I_UltraEdiit_fourmask_LLava_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_filtered_objects_4mask_1280_context/003-GPT-XL/checkpoints/eval_step_42000/batch_4_cfg_7.5_topk_16384.jpg new file mode 100644 index 0000000000000000000000000000000000000000..f8f020d1bb5fa7099be10e33101c6dd13c1ddb41 --- /dev/null +++ b/CKPTS/X2I_UltraEdiit_fourmask_LLava_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_filtered_objects_4mask_1280_context/003-GPT-XL/checkpoints/eval_step_42000/batch_4_cfg_7.5_topk_16384.jpg @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:91c947662c00c3e56fd19cf910d8fa199451c13d77a4eb4e9c6d484bf006d55a +size 348387 diff --git a/CKPTS/X2I_UltraEdiit_fourmask_LLava_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_filtered_objects_4mask_1280_context/003-GPT-XL/checkpoints/eval_step_44000/batch_0_cfg_7.5_topk_16384.jpg b/CKPTS/X2I_UltraEdiit_fourmask_LLava_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_filtered_objects_4mask_1280_context/003-GPT-XL/checkpoints/eval_step_44000/batch_0_cfg_7.5_topk_16384.jpg new file mode 100644 index 0000000000000000000000000000000000000000..a71178719021c2dfff0ef5d8a9b23e28d2c04918 --- /dev/null +++ b/CKPTS/X2I_UltraEdiit_fourmask_LLava_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_filtered_objects_4mask_1280_context/003-GPT-XL/checkpoints/eval_step_44000/batch_0_cfg_7.5_topk_16384.jpg @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:3587c846bab52572c8d0b866db34ff58821cc64500180684820e3e91729b63fa +size 440537 diff --git a/CKPTS/X2I_UltraEdiit_fourmask_LLava_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_filtered_objects_4mask_1280_context/003-GPT-XL/checkpoints/eval_step_44000/batch_1_cfg_7.5_topk_16384.jpg b/CKPTS/X2I_UltraEdiit_fourmask_LLava_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_filtered_objects_4mask_1280_context/003-GPT-XL/checkpoints/eval_step_44000/batch_1_cfg_7.5_topk_16384.jpg new file mode 100644 index 0000000000000000000000000000000000000000..e02f4e99814c4c577567dcd74ddd4f65973d79ea --- /dev/null +++ b/CKPTS/X2I_UltraEdiit_fourmask_LLava_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_filtered_objects_4mask_1280_context/003-GPT-XL/checkpoints/eval_step_44000/batch_1_cfg_7.5_topk_16384.jpg @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:d93d4609bacd1c7825f2e2336b4bae9d076e85b21e1a4b5032d8c2c3a6c158f8 +size 428422 diff --git a/CKPTS/X2I_UltraEdiit_fourmask_LLava_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_filtered_objects_4mask_1280_context/003-GPT-XL/checkpoints/eval_step_44000/batch_2_cfg_7.5_topk_16384.jpg b/CKPTS/X2I_UltraEdiit_fourmask_LLava_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_filtered_objects_4mask_1280_context/003-GPT-XL/checkpoints/eval_step_44000/batch_2_cfg_7.5_topk_16384.jpg new file mode 100644 index 0000000000000000000000000000000000000000..b0ce11d1f3d00461f3cadef9405fe7d62ad11f5e --- /dev/null +++ b/CKPTS/X2I_UltraEdiit_fourmask_LLava_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_filtered_objects_4mask_1280_context/003-GPT-XL/checkpoints/eval_step_44000/batch_2_cfg_7.5_topk_16384.jpg @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:98956129d367fa832ccda2a4fd3e0ccf4e8c9ea8e48620dd1d51ac09ecfc45cd +size 430903 diff --git a/CKPTS/X2I_UltraEdiit_fourmask_LLava_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_filtered_objects_4mask_1280_context/003-GPT-XL/checkpoints/eval_step_44000/batch_3_cfg_7.5_topk_16384.jpg b/CKPTS/X2I_UltraEdiit_fourmask_LLava_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_filtered_objects_4mask_1280_context/003-GPT-XL/checkpoints/eval_step_44000/batch_3_cfg_7.5_topk_16384.jpg new file mode 100644 index 0000000000000000000000000000000000000000..d6a9fd4a927fe043d3f3dc36ec51584834f0739b --- /dev/null +++ b/CKPTS/X2I_UltraEdiit_fourmask_LLava_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_filtered_objects_4mask_1280_context/003-GPT-XL/checkpoints/eval_step_44000/batch_3_cfg_7.5_topk_16384.jpg @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:890b8a9c8730161231d252bdae83d171728b4f08ad9164a5b486995e2f4937e4 +size 403855 diff --git a/CKPTS/X2I_UltraEdiit_fourmask_LLava_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_filtered_objects_4mask_1280_context/003-GPT-XL/checkpoints/eval_step_44000/batch_4_cfg_7.5_topk_16384.jpg b/CKPTS/X2I_UltraEdiit_fourmask_LLava_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_filtered_objects_4mask_1280_context/003-GPT-XL/checkpoints/eval_step_44000/batch_4_cfg_7.5_topk_16384.jpg new file mode 100644 index 0000000000000000000000000000000000000000..4887c704a1b096f2d9891eb6fd11e1b1eeb8159f --- /dev/null +++ b/CKPTS/X2I_UltraEdiit_fourmask_LLava_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_filtered_objects_4mask_1280_context/003-GPT-XL/checkpoints/eval_step_44000/batch_4_cfg_7.5_topk_16384.jpg @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:8d92bae0eeaa0adf7ae487f14e11841275f03a2a07863ed32598ab12e18fab2e +size 350582 diff --git a/CKPTS/X2I_UltraEdiit_fourmask_LLava_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_filtered_objects_4mask_1280_context/003-GPT-XL/checkpoints/eval_step_46000/batch_0_cfg_7.5_topk_16384.jpg b/CKPTS/X2I_UltraEdiit_fourmask_LLava_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_filtered_objects_4mask_1280_context/003-GPT-XL/checkpoints/eval_step_46000/batch_0_cfg_7.5_topk_16384.jpg new file mode 100644 index 0000000000000000000000000000000000000000..8b18b0a1e671dc576330e6f9cf945f20a26ee9d7 --- /dev/null +++ b/CKPTS/X2I_UltraEdiit_fourmask_LLava_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_filtered_objects_4mask_1280_context/003-GPT-XL/checkpoints/eval_step_46000/batch_0_cfg_7.5_topk_16384.jpg @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:969a2cb742f9871af813f937317a81224b0f229fd47d091a2c341cf665157e09 +size 442858 diff --git a/CKPTS/X2I_UltraEdiit_fourmask_LLava_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_filtered_objects_4mask_1280_context/003-GPT-XL/checkpoints/eval_step_46000/batch_1_cfg_7.5_topk_16384.jpg b/CKPTS/X2I_UltraEdiit_fourmask_LLava_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_filtered_objects_4mask_1280_context/003-GPT-XL/checkpoints/eval_step_46000/batch_1_cfg_7.5_topk_16384.jpg new file mode 100644 index 0000000000000000000000000000000000000000..1944058672621bb7bc6a584d69d20fc28c06968c --- /dev/null +++ b/CKPTS/X2I_UltraEdiit_fourmask_LLava_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_filtered_objects_4mask_1280_context/003-GPT-XL/checkpoints/eval_step_46000/batch_1_cfg_7.5_topk_16384.jpg @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:a5fffcc197bacbf3b10b941793ec3310c823fe492f8191600843ddcd7811db79 +size 426841 diff --git a/CKPTS/X2I_UltraEdiit_fourmask_LLava_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_filtered_objects_4mask_1280_context/003-GPT-XL/checkpoints/eval_step_46000/batch_2_cfg_7.5_topk_16384.jpg b/CKPTS/X2I_UltraEdiit_fourmask_LLava_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_filtered_objects_4mask_1280_context/003-GPT-XL/checkpoints/eval_step_46000/batch_2_cfg_7.5_topk_16384.jpg new file mode 100644 index 0000000000000000000000000000000000000000..7349fa0453a60bbaebafa63a87825ba9487c42a1 --- /dev/null +++ b/CKPTS/X2I_UltraEdiit_fourmask_LLava_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_filtered_objects_4mask_1280_context/003-GPT-XL/checkpoints/eval_step_46000/batch_2_cfg_7.5_topk_16384.jpg @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:23cfe587eec2d1bf04bb65121e337b42b91447d0637c42579ce24debda955aba +size 434309 diff --git a/CKPTS/X2I_UltraEdiit_fourmask_LLava_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_filtered_objects_4mask_1280_context/003-GPT-XL/checkpoints/eval_step_46000/batch_3_cfg_7.5_topk_16384.jpg b/CKPTS/X2I_UltraEdiit_fourmask_LLava_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_filtered_objects_4mask_1280_context/003-GPT-XL/checkpoints/eval_step_46000/batch_3_cfg_7.5_topk_16384.jpg new file mode 100644 index 0000000000000000000000000000000000000000..350a2739e3579f27a312653def8e06c95b0fc577 --- /dev/null +++ b/CKPTS/X2I_UltraEdiit_fourmask_LLava_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_filtered_objects_4mask_1280_context/003-GPT-XL/checkpoints/eval_step_46000/batch_3_cfg_7.5_topk_16384.jpg @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:fd8c22dd1d3d9607b811697348e3fe1163a8f4233305c5a8e5eaf53dcec59835 +size 395890 diff --git a/CKPTS/X2I_UltraEdiit_fourmask_LLava_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_filtered_objects_4mask_1280_context/003-GPT-XL/checkpoints/eval_step_46000/batch_4_cfg_7.5_topk_16384.jpg b/CKPTS/X2I_UltraEdiit_fourmask_LLava_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_filtered_objects_4mask_1280_context/003-GPT-XL/checkpoints/eval_step_46000/batch_4_cfg_7.5_topk_16384.jpg new file mode 100644 index 0000000000000000000000000000000000000000..3cce7a0d81a9b6ae5b85565e0fac632d36073e84 --- /dev/null +++ b/CKPTS/X2I_UltraEdiit_fourmask_LLava_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_filtered_objects_4mask_1280_context/003-GPT-XL/checkpoints/eval_step_46000/batch_4_cfg_7.5_topk_16384.jpg @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:01005152e0746950cd24a062eacfb7736902319e2e6befacfddb1aaccea015d6 +size 343672 diff --git a/CKPTS/X2I_UltraEdiit_fourmask_LLava_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_filtered_objects_4mask_1280_context/003-GPT-XL/checkpoints/eval_step_48000/batch_0_cfg_7.5_topk_16384.jpg b/CKPTS/X2I_UltraEdiit_fourmask_LLava_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_filtered_objects_4mask_1280_context/003-GPT-XL/checkpoints/eval_step_48000/batch_0_cfg_7.5_topk_16384.jpg new file mode 100644 index 0000000000000000000000000000000000000000..498a0568be84252c3a0d1aadcc333d9002234230 --- /dev/null +++ b/CKPTS/X2I_UltraEdiit_fourmask_LLava_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_filtered_objects_4mask_1280_context/003-GPT-XL/checkpoints/eval_step_48000/batch_0_cfg_7.5_topk_16384.jpg @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:bc5709204586f47a4632f2f432dee5f06fa4a0d1f0d9efdb36b9902d93938fc5 +size 448357 diff --git a/CKPTS/X2I_UltraEdiit_fourmask_LLava_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_filtered_objects_4mask_1280_context/003-GPT-XL/checkpoints/eval_step_48000/batch_1_cfg_7.5_topk_16384.jpg b/CKPTS/X2I_UltraEdiit_fourmask_LLava_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_filtered_objects_4mask_1280_context/003-GPT-XL/checkpoints/eval_step_48000/batch_1_cfg_7.5_topk_16384.jpg new file mode 100644 index 0000000000000000000000000000000000000000..0ee17b2d1fdea6024e25cd2efc97726e2004bd08 --- /dev/null +++ b/CKPTS/X2I_UltraEdiit_fourmask_LLava_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_filtered_objects_4mask_1280_context/003-GPT-XL/checkpoints/eval_step_48000/batch_1_cfg_7.5_topk_16384.jpg @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:56a8df046b0b8161ef8f88570204f5b99c35ffd9d80b8a2727344decda19eb83 +size 416584 diff --git a/CKPTS/X2I_UltraEdiit_fourmask_LLava_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_filtered_objects_4mask_1280_context/003-GPT-XL/checkpoints/eval_step_48000/batch_2_cfg_7.5_topk_16384.jpg b/CKPTS/X2I_UltraEdiit_fourmask_LLava_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_filtered_objects_4mask_1280_context/003-GPT-XL/checkpoints/eval_step_48000/batch_2_cfg_7.5_topk_16384.jpg new file mode 100644 index 0000000000000000000000000000000000000000..dba3d3533d6b74e3b8d73a79115fe609d3a22c99 --- /dev/null +++ b/CKPTS/X2I_UltraEdiit_fourmask_LLava_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_filtered_objects_4mask_1280_context/003-GPT-XL/checkpoints/eval_step_48000/batch_2_cfg_7.5_topk_16384.jpg @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:68c81bf83a411fdf690b7b9aa135cc62dc30bda3a3e829fd1b933a8bb4463593 +size 429258 diff --git a/CKPTS/X2I_UltraEdiit_fourmask_LLava_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_filtered_objects_4mask_1280_context/003-GPT-XL/checkpoints/eval_step_48000/batch_3_cfg_7.5_topk_16384.jpg b/CKPTS/X2I_UltraEdiit_fourmask_LLava_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_filtered_objects_4mask_1280_context/003-GPT-XL/checkpoints/eval_step_48000/batch_3_cfg_7.5_topk_16384.jpg new file mode 100644 index 0000000000000000000000000000000000000000..4f2e64a3b1e12881b247a67815c22835fe35f2dd --- /dev/null +++ b/CKPTS/X2I_UltraEdiit_fourmask_LLava_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_filtered_objects_4mask_1280_context/003-GPT-XL/checkpoints/eval_step_48000/batch_3_cfg_7.5_topk_16384.jpg @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:0f77b9a6c71e6a20fdb2983dd31de1323ea958973a2c3ecaa0a5f500fc0ca285 +size 384574 diff --git a/CKPTS/X2I_UltraEdiit_fourmask_LLava_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_filtered_objects_4mask_1280_context/003-GPT-XL/checkpoints/eval_step_48000/batch_4_cfg_7.5_topk_16384.jpg b/CKPTS/X2I_UltraEdiit_fourmask_LLava_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_filtered_objects_4mask_1280_context/003-GPT-XL/checkpoints/eval_step_48000/batch_4_cfg_7.5_topk_16384.jpg new file mode 100644 index 0000000000000000000000000000000000000000..2ce04918e345d75caf5e92473a5a64873a4ec7cc --- /dev/null +++ b/CKPTS/X2I_UltraEdiit_fourmask_LLava_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_filtered_objects_4mask_1280_context/003-GPT-XL/checkpoints/eval_step_48000/batch_4_cfg_7.5_topk_16384.jpg @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:ad1e74f7d2f3ca6b79416a898bde5000d8a048c6ca9dbc6d9359f878a3ebc7ca +size 345140 diff --git a/CKPTS/X2I_UltraEdiit_fourmask_LLava_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_filtered_objects_4mask_1280_context/003-GPT-XL/checkpoints/eval_step_50000/batch_0_cfg_7.5_topk_16384.jpg b/CKPTS/X2I_UltraEdiit_fourmask_LLava_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_filtered_objects_4mask_1280_context/003-GPT-XL/checkpoints/eval_step_50000/batch_0_cfg_7.5_topk_16384.jpg new file mode 100644 index 0000000000000000000000000000000000000000..6ddd050dd329528538873bce5e112a29b2e9f33a --- /dev/null +++ b/CKPTS/X2I_UltraEdiit_fourmask_LLava_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_filtered_objects_4mask_1280_context/003-GPT-XL/checkpoints/eval_step_50000/batch_0_cfg_7.5_topk_16384.jpg @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:f4bf242d7d24f3e6005f650a6d64cc5b6471c3a4c12063415d5e18fbd17cd60f +size 441958 diff --git a/CKPTS/X2I_UltraEdiit_fourmask_LLava_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_filtered_objects_4mask_1280_context/003-GPT-XL/checkpoints/eval_step_50000/batch_1_cfg_7.5_topk_16384.jpg b/CKPTS/X2I_UltraEdiit_fourmask_LLava_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_filtered_objects_4mask_1280_context/003-GPT-XL/checkpoints/eval_step_50000/batch_1_cfg_7.5_topk_16384.jpg new file mode 100644 index 0000000000000000000000000000000000000000..3a82ea20ce3391d030a366b5cb3264ca69110179 --- /dev/null +++ b/CKPTS/X2I_UltraEdiit_fourmask_LLava_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_filtered_objects_4mask_1280_context/003-GPT-XL/checkpoints/eval_step_50000/batch_1_cfg_7.5_topk_16384.jpg @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:af9d5e70bf6d68390a6f5609c4fbd32dad0121ce83fdaa55d5a500420bc88330 +size 436302 diff --git a/CKPTS/X2I_UltraEdiit_fourmask_LLava_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_filtered_objects_4mask_1280_context/003-GPT-XL/checkpoints/eval_step_50000/batch_2_cfg_7.5_topk_16384.jpg b/CKPTS/X2I_UltraEdiit_fourmask_LLava_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_filtered_objects_4mask_1280_context/003-GPT-XL/checkpoints/eval_step_50000/batch_2_cfg_7.5_topk_16384.jpg new file mode 100644 index 0000000000000000000000000000000000000000..ecdb50713618c19fc91ad01b038342a61bb876f2 --- /dev/null +++ b/CKPTS/X2I_UltraEdiit_fourmask_LLava_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_filtered_objects_4mask_1280_context/003-GPT-XL/checkpoints/eval_step_50000/batch_2_cfg_7.5_topk_16384.jpg @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:ed00c2fc2f3bc6a69a505c698ab3d096d02ce503d48554e3f9b1dcd73a2f2631 +size 428860 diff --git a/CKPTS/X2I_UltraEdiit_fourmask_LLava_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_filtered_objects_4mask_1280_context/003-GPT-XL/checkpoints/eval_step_50000/batch_3_cfg_7.5_topk_16384.jpg b/CKPTS/X2I_UltraEdiit_fourmask_LLava_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_filtered_objects_4mask_1280_context/003-GPT-XL/checkpoints/eval_step_50000/batch_3_cfg_7.5_topk_16384.jpg new file mode 100644 index 0000000000000000000000000000000000000000..6cd53cc1f8f896581ea000e7f51f314f05a0601a --- /dev/null +++ b/CKPTS/X2I_UltraEdiit_fourmask_LLava_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_filtered_objects_4mask_1280_context/003-GPT-XL/checkpoints/eval_step_50000/batch_3_cfg_7.5_topk_16384.jpg @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:3e67c97d30120224855b7ee11913fc34cdd614c7cb3e60c829d5faed0e9a201f +size 399502 diff --git a/CKPTS/X2I_UltraEdiit_fourmask_LLava_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_filtered_objects_4mask_1280_context/003-GPT-XL/checkpoints/eval_step_50000/batch_4_cfg_7.5_topk_16384.jpg b/CKPTS/X2I_UltraEdiit_fourmask_LLava_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_filtered_objects_4mask_1280_context/003-GPT-XL/checkpoints/eval_step_50000/batch_4_cfg_7.5_topk_16384.jpg new file mode 100644 index 0000000000000000000000000000000000000000..ad91f481d9e3461bf718217ff1dada63bb1353bc --- /dev/null +++ b/CKPTS/X2I_UltraEdiit_fourmask_LLava_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_filtered_objects_4mask_1280_context/003-GPT-XL/checkpoints/eval_step_50000/batch_4_cfg_7.5_topk_16384.jpg @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:afc2c2d3e6858c9bd15e7e24e50e6f7ce4979a7808e194536fc61f011643fa64 +size 346153 diff --git a/CKPTS/X2I_UltraEdiit_fourmask_LLava_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_filtered_objects_4mask_1280_context/003-GPT-XL/checkpoints/eval_step_52000/batch_0_cfg_7.5_topk_16384.jpg b/CKPTS/X2I_UltraEdiit_fourmask_LLava_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_filtered_objects_4mask_1280_context/003-GPT-XL/checkpoints/eval_step_52000/batch_0_cfg_7.5_topk_16384.jpg new file mode 100644 index 0000000000000000000000000000000000000000..d63aea74b08e285d0381feb104c7facbb0784ca1 --- /dev/null +++ b/CKPTS/X2I_UltraEdiit_fourmask_LLava_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_filtered_objects_4mask_1280_context/003-GPT-XL/checkpoints/eval_step_52000/batch_0_cfg_7.5_topk_16384.jpg @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:69f900dbca0cc7af921dd731399528c58b932aea26cb0a17bebd903df668b3ae +size 443923 diff --git a/CKPTS/X2I_UltraEdiit_fourmask_LLava_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_filtered_objects_4mask_1280_context/003-GPT-XL/checkpoints/eval_step_52000/batch_1_cfg_7.5_topk_16384.jpg b/CKPTS/X2I_UltraEdiit_fourmask_LLava_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_filtered_objects_4mask_1280_context/003-GPT-XL/checkpoints/eval_step_52000/batch_1_cfg_7.5_topk_16384.jpg new file mode 100644 index 0000000000000000000000000000000000000000..5451cd484ce38ffbaea33443f41894ec0ab71489 --- /dev/null +++ b/CKPTS/X2I_UltraEdiit_fourmask_LLava_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_filtered_objects_4mask_1280_context/003-GPT-XL/checkpoints/eval_step_52000/batch_1_cfg_7.5_topk_16384.jpg @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:99efd9b6567259dd613aec7f3f99769e233161865bdab381315a26979eb519d7 +size 434011 diff --git a/CKPTS/X2I_UltraEdiit_fourmask_LLava_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_filtered_objects_4mask_1280_context/003-GPT-XL/checkpoints/eval_step_52000/batch_2_cfg_7.5_topk_16384.jpg b/CKPTS/X2I_UltraEdiit_fourmask_LLava_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_filtered_objects_4mask_1280_context/003-GPT-XL/checkpoints/eval_step_52000/batch_2_cfg_7.5_topk_16384.jpg new file mode 100644 index 0000000000000000000000000000000000000000..4003f92ce02be7559b2a9ecfbb8a54484ad3eff0 --- /dev/null +++ b/CKPTS/X2I_UltraEdiit_fourmask_LLava_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_filtered_objects_4mask_1280_context/003-GPT-XL/checkpoints/eval_step_52000/batch_2_cfg_7.5_topk_16384.jpg @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:889e0d015c1404979478f56a6bdadff5972e969ca6c95798845328bb3ae5883f +size 434626 diff --git a/CKPTS/X2I_UltraEdiit_fourmask_LLava_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_filtered_objects_4mask_1280_context/003-GPT-XL/checkpoints/eval_step_52000/batch_3_cfg_7.5_topk_16384.jpg b/CKPTS/X2I_UltraEdiit_fourmask_LLava_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_filtered_objects_4mask_1280_context/003-GPT-XL/checkpoints/eval_step_52000/batch_3_cfg_7.5_topk_16384.jpg new file mode 100644 index 0000000000000000000000000000000000000000..cf1741b708b66f685d6f6f89787fe327aaf09d86 --- /dev/null +++ b/CKPTS/X2I_UltraEdiit_fourmask_LLava_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_filtered_objects_4mask_1280_context/003-GPT-XL/checkpoints/eval_step_52000/batch_3_cfg_7.5_topk_16384.jpg @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:d960ff12e704ea1c3b1348cb79b6dc4d37445ba806b70789387acc1471e3aad8 +size 382269 diff --git a/CKPTS/X2I_UltraEdiit_fourmask_LLava_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_filtered_objects_4mask_1280_context/003-GPT-XL/checkpoints/eval_step_52000/batch_4_cfg_7.5_topk_16384.jpg b/CKPTS/X2I_UltraEdiit_fourmask_LLava_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_filtered_objects_4mask_1280_context/003-GPT-XL/checkpoints/eval_step_52000/batch_4_cfg_7.5_topk_16384.jpg new file mode 100644 index 0000000000000000000000000000000000000000..fa6e019e285e09b99bf641e4ced2d1f6e6abb592 --- /dev/null +++ b/CKPTS/X2I_UltraEdiit_fourmask_LLava_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_filtered_objects_4mask_1280_context/003-GPT-XL/checkpoints/eval_step_52000/batch_4_cfg_7.5_topk_16384.jpg @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:f872b8401e13ef5ade6ef1761e0be3faec55984c6813a29f8bdc07fdb9514ef9 +size 338165 diff --git a/CKPTS/X2I_UltraEdiit_fourmask_LLava_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_filtered_objects_4mask_1280_context/003-GPT-XL/checkpoints/eval_step_54000/batch_0_cfg_7.5_topk_16384.jpg b/CKPTS/X2I_UltraEdiit_fourmask_LLava_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_filtered_objects_4mask_1280_context/003-GPT-XL/checkpoints/eval_step_54000/batch_0_cfg_7.5_topk_16384.jpg new file mode 100644 index 0000000000000000000000000000000000000000..6829cee25322d86d80e34c2abb1a8dae7ef9a42e --- /dev/null +++ b/CKPTS/X2I_UltraEdiit_fourmask_LLava_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_filtered_objects_4mask_1280_context/003-GPT-XL/checkpoints/eval_step_54000/batch_0_cfg_7.5_topk_16384.jpg @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:e1ae80186604eee378adc29cef9c98e687bc7df4a0797659891f82134450e07f +size 449133 diff --git a/CKPTS/X2I_UltraEdiit_fourmask_LLava_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_filtered_objects_4mask_1280_context/003-GPT-XL/checkpoints/eval_step_54000/batch_1_cfg_7.5_topk_16384.jpg b/CKPTS/X2I_UltraEdiit_fourmask_LLava_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_filtered_objects_4mask_1280_context/003-GPT-XL/checkpoints/eval_step_54000/batch_1_cfg_7.5_topk_16384.jpg new file mode 100644 index 0000000000000000000000000000000000000000..26a0f8897edbcf885f94c6a2ac5c1abf6c577a18 --- /dev/null +++ b/CKPTS/X2I_UltraEdiit_fourmask_LLava_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_filtered_objects_4mask_1280_context/003-GPT-XL/checkpoints/eval_step_54000/batch_1_cfg_7.5_topk_16384.jpg @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:6f9dd1d4152fd24acdcc952a82f7ac027c0b497f7f4ea361b4c4e7c050b6f5e5 +size 434875 diff --git a/CKPTS/X2I_UltraEdiit_fourmask_LLava_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_filtered_objects_4mask_1280_context/003-GPT-XL/checkpoints/eval_step_54000/batch_2_cfg_7.5_topk_16384.jpg b/CKPTS/X2I_UltraEdiit_fourmask_LLava_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_filtered_objects_4mask_1280_context/003-GPT-XL/checkpoints/eval_step_54000/batch_2_cfg_7.5_topk_16384.jpg new file mode 100644 index 0000000000000000000000000000000000000000..53e7711d7265b2db29bcbcc6893a31262ae8b660 --- /dev/null +++ b/CKPTS/X2I_UltraEdiit_fourmask_LLava_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_filtered_objects_4mask_1280_context/003-GPT-XL/checkpoints/eval_step_54000/batch_2_cfg_7.5_topk_16384.jpg @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:7aada5b175e0142886d553b984c759aaee935e7a4244b1d6e976914f09dfb4c9 +size 417876 diff --git a/CKPTS/X2I_UltraEdiit_fourmask_LLava_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_filtered_objects_4mask_1280_context/003-GPT-XL/checkpoints/eval_step_54000/batch_3_cfg_7.5_topk_16384.jpg b/CKPTS/X2I_UltraEdiit_fourmask_LLava_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_filtered_objects_4mask_1280_context/003-GPT-XL/checkpoints/eval_step_54000/batch_3_cfg_7.5_topk_16384.jpg new file mode 100644 index 0000000000000000000000000000000000000000..3ccfb69c93c37e3956853ff08e6e8964b672b5cd --- /dev/null +++ b/CKPTS/X2I_UltraEdiit_fourmask_LLava_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_filtered_objects_4mask_1280_context/003-GPT-XL/checkpoints/eval_step_54000/batch_3_cfg_7.5_topk_16384.jpg @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:9ab63e74b91c58cebe6a5ea5db9fbbc6710acf1af0490ea0cea0c2ae7c5924cd +size 387395 diff --git a/CKPTS/X2I_UltraEdiit_fourmask_LLava_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_filtered_objects_4mask_1280_context/003-GPT-XL/checkpoints/eval_step_54000/batch_4_cfg_7.5_topk_16384.jpg b/CKPTS/X2I_UltraEdiit_fourmask_LLava_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_filtered_objects_4mask_1280_context/003-GPT-XL/checkpoints/eval_step_54000/batch_4_cfg_7.5_topk_16384.jpg new file mode 100644 index 0000000000000000000000000000000000000000..7405be234620cbb733f20db48b370334048024c3 --- /dev/null +++ b/CKPTS/X2I_UltraEdiit_fourmask_LLava_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_filtered_objects_4mask_1280_context/003-GPT-XL/checkpoints/eval_step_54000/batch_4_cfg_7.5_topk_16384.jpg @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:7b2816744bc7298e940ff26387f92112ff9185125d4fe16221d158d4798c3f91 +size 347260 diff --git a/CKPTS/X2I_UltraEdiit_fourmask_LLava_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_filtered_objects_4mask_1280_context/003-GPT-XL/checkpoints/eval_step_56000/batch_0_cfg_7.5_topk_16384.jpg b/CKPTS/X2I_UltraEdiit_fourmask_LLava_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_filtered_objects_4mask_1280_context/003-GPT-XL/checkpoints/eval_step_56000/batch_0_cfg_7.5_topk_16384.jpg new file mode 100644 index 0000000000000000000000000000000000000000..d705de8c8ad940dd10698d35044ff667a2a53e82 --- /dev/null +++ b/CKPTS/X2I_UltraEdiit_fourmask_LLava_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_filtered_objects_4mask_1280_context/003-GPT-XL/checkpoints/eval_step_56000/batch_0_cfg_7.5_topk_16384.jpg @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:e761cfebbd304e1dc17dd7c7702afc163324669c77d72e16b742eb146be18abd +size 431052 diff --git a/CKPTS/X2I_UltraEdiit_fourmask_LLava_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_filtered_objects_4mask_1280_context/003-GPT-XL/checkpoints/eval_step_56000/batch_1_cfg_7.5_topk_16384.jpg b/CKPTS/X2I_UltraEdiit_fourmask_LLava_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_filtered_objects_4mask_1280_context/003-GPT-XL/checkpoints/eval_step_56000/batch_1_cfg_7.5_topk_16384.jpg new file mode 100644 index 0000000000000000000000000000000000000000..1df09a875f7334f2d298684300071175aa244af0 --- /dev/null +++ b/CKPTS/X2I_UltraEdiit_fourmask_LLava_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_filtered_objects_4mask_1280_context/003-GPT-XL/checkpoints/eval_step_56000/batch_1_cfg_7.5_topk_16384.jpg @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:9a2da64f555a1366c85280b3dc0fd1262a2813bb31153bce58b42d2e740ac944 +size 423821 diff --git a/CKPTS/X2I_UltraEdiit_fourmask_LLava_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_filtered_objects_4mask_1280_context/003-GPT-XL/checkpoints/eval_step_56000/batch_2_cfg_7.5_topk_16384.jpg b/CKPTS/X2I_UltraEdiit_fourmask_LLava_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_filtered_objects_4mask_1280_context/003-GPT-XL/checkpoints/eval_step_56000/batch_2_cfg_7.5_topk_16384.jpg new file mode 100644 index 0000000000000000000000000000000000000000..9a19ff08d6e6c8ce936ae1fd387eff5ee89b778f --- /dev/null +++ b/CKPTS/X2I_UltraEdiit_fourmask_LLava_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_filtered_objects_4mask_1280_context/003-GPT-XL/checkpoints/eval_step_56000/batch_2_cfg_7.5_topk_16384.jpg @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:458c5ba292637934d854f2ecd59737d1dee2d968e3533aa03374dabc8b2d217d +size 422971 diff --git a/CKPTS/X2I_UltraEdiit_fourmask_LLava_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_filtered_objects_4mask_1280_context/003-GPT-XL/checkpoints/eval_step_56000/batch_3_cfg_7.5_topk_16384.jpg b/CKPTS/X2I_UltraEdiit_fourmask_LLava_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_filtered_objects_4mask_1280_context/003-GPT-XL/checkpoints/eval_step_56000/batch_3_cfg_7.5_topk_16384.jpg new file mode 100644 index 0000000000000000000000000000000000000000..46a1cdb21e7ebc3194cf3d62b394aa5163093c9a --- /dev/null +++ b/CKPTS/X2I_UltraEdiit_fourmask_LLava_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_filtered_objects_4mask_1280_context/003-GPT-XL/checkpoints/eval_step_56000/batch_3_cfg_7.5_topk_16384.jpg @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:7d2be654b5e595146aa67c6040ea77653fbd0e8e181e9f7547a0e928872af22f +size 375394 diff --git a/CKPTS/X2I_UltraEdiit_fourmask_LLava_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_filtered_objects_4mask_1280_context/003-GPT-XL/checkpoints/eval_step_56000/batch_4_cfg_7.5_topk_16384.jpg b/CKPTS/X2I_UltraEdiit_fourmask_LLava_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_filtered_objects_4mask_1280_context/003-GPT-XL/checkpoints/eval_step_56000/batch_4_cfg_7.5_topk_16384.jpg new file mode 100644 index 0000000000000000000000000000000000000000..88d2d82d461099f4078f8e508f4c2833a51f73b4 --- /dev/null +++ b/CKPTS/X2I_UltraEdiit_fourmask_LLava_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_filtered_objects_4mask_1280_context/003-GPT-XL/checkpoints/eval_step_56000/batch_4_cfg_7.5_topk_16384.jpg @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:95b084645e56de2b7534e048a329f5da7566d27e49b2c8c28188a6f4092ea57f +size 348303 diff --git a/CKPTS/X2I_UltraEdiit_fourmask_LLava_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_filtered_objects_4mask_1280_context/003-GPT-XL/checkpoints/eval_step_58000/batch_0_cfg_7.5_topk_16384.jpg b/CKPTS/X2I_UltraEdiit_fourmask_LLava_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_filtered_objects_4mask_1280_context/003-GPT-XL/checkpoints/eval_step_58000/batch_0_cfg_7.5_topk_16384.jpg new file mode 100644 index 0000000000000000000000000000000000000000..f906c1b0f0cf3c272165061ea0674f54806142ad --- /dev/null +++ b/CKPTS/X2I_UltraEdiit_fourmask_LLava_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_filtered_objects_4mask_1280_context/003-GPT-XL/checkpoints/eval_step_58000/batch_0_cfg_7.5_topk_16384.jpg @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:45c9807efec0e4fb858721ce4196ac5e894f5e1d62b8706e94323880489c992c +size 444148 diff --git a/CKPTS/X2I_UltraEdiit_fourmask_LLava_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_filtered_objects_4mask_1280_context/003-GPT-XL/checkpoints/eval_step_58000/batch_1_cfg_7.5_topk_16384.jpg b/CKPTS/X2I_UltraEdiit_fourmask_LLava_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_filtered_objects_4mask_1280_context/003-GPT-XL/checkpoints/eval_step_58000/batch_1_cfg_7.5_topk_16384.jpg new file mode 100644 index 0000000000000000000000000000000000000000..29ea3a5891ca74176da0e5cc2a5995332c4e8c4d --- /dev/null +++ b/CKPTS/X2I_UltraEdiit_fourmask_LLava_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_filtered_objects_4mask_1280_context/003-GPT-XL/checkpoints/eval_step_58000/batch_1_cfg_7.5_topk_16384.jpg @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:b0c0afdb462265593d4d713604238e31efda96f7a7165bef6be0fa6f29b21c1a +size 433329 diff --git a/CKPTS/X2I_UltraEdiit_fourmask_LLava_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_filtered_objects_4mask_1280_context/003-GPT-XL/checkpoints/eval_step_58000/batch_2_cfg_7.5_topk_16384.jpg b/CKPTS/X2I_UltraEdiit_fourmask_LLava_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_filtered_objects_4mask_1280_context/003-GPT-XL/checkpoints/eval_step_58000/batch_2_cfg_7.5_topk_16384.jpg new file mode 100644 index 0000000000000000000000000000000000000000..bb09b2e50c3e3510323a0621348188abe9889e69 --- /dev/null +++ b/CKPTS/X2I_UltraEdiit_fourmask_LLava_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_filtered_objects_4mask_1280_context/003-GPT-XL/checkpoints/eval_step_58000/batch_2_cfg_7.5_topk_16384.jpg @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:758d672a9235845784332e14512db5d3ba03da5a6b93e048e4811a39d4f76b6f +size 428662 diff --git a/CKPTS/X2I_UltraEdiit_fourmask_LLava_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_filtered_objects_4mask_1280_context/003-GPT-XL/checkpoints/eval_step_58000/batch_3_cfg_7.5_topk_16384.jpg b/CKPTS/X2I_UltraEdiit_fourmask_LLava_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_filtered_objects_4mask_1280_context/003-GPT-XL/checkpoints/eval_step_58000/batch_3_cfg_7.5_topk_16384.jpg new file mode 100644 index 0000000000000000000000000000000000000000..3aa3b14f3cf9a38645cd32c4586aca8fd5e17515 --- /dev/null +++ b/CKPTS/X2I_UltraEdiit_fourmask_LLava_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_filtered_objects_4mask_1280_context/003-GPT-XL/checkpoints/eval_step_58000/batch_3_cfg_7.5_topk_16384.jpg @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:9dbfa11088498a8a6079c32d5aca4ae06f339f771435ccd7b6553bf64d00ab22 +size 386095 diff --git a/CKPTS/X2I_UltraEdiit_fourmask_LLava_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_filtered_objects_4mask_1280_context/003-GPT-XL/checkpoints/eval_step_58000/batch_4_cfg_7.5_topk_16384.jpg b/CKPTS/X2I_UltraEdiit_fourmask_LLava_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_filtered_objects_4mask_1280_context/003-GPT-XL/checkpoints/eval_step_58000/batch_4_cfg_7.5_topk_16384.jpg new file mode 100644 index 0000000000000000000000000000000000000000..e8f102d83ba508d0190aa6fb821bd5000f3521bc --- /dev/null +++ b/CKPTS/X2I_UltraEdiit_fourmask_LLava_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_filtered_objects_4mask_1280_context/003-GPT-XL/checkpoints/eval_step_58000/batch_4_cfg_7.5_topk_16384.jpg @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:d0250f11992a419b59dae5eecfa9c74162ba29fcbb29e9f560c7d260f470f944 +size 342999 diff --git a/CKPTS/X2I_UltraEdiit_fourmask_LLava_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_filtered_objects_4mask_1280_context/003-GPT-XL/checkpoints/eval_step_6000/batch_0_cfg_7.5_topk_16384.jpg b/CKPTS/X2I_UltraEdiit_fourmask_LLava_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_filtered_objects_4mask_1280_context/003-GPT-XL/checkpoints/eval_step_6000/batch_0_cfg_7.5_topk_16384.jpg new file mode 100644 index 0000000000000000000000000000000000000000..e85a37cd3207cf6139b861f1344e21f52111e5d7 --- /dev/null +++ b/CKPTS/X2I_UltraEdiit_fourmask_LLava_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_filtered_objects_4mask_1280_context/003-GPT-XL/checkpoints/eval_step_6000/batch_0_cfg_7.5_topk_16384.jpg @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:2e458bc35ebf67aa4b900a9a392d0b7a917025cd36d8dfd489f906ecf0de9078 +size 468367 diff --git a/CKPTS/X2I_UltraEdiit_fourmask_LLava_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_filtered_objects_4mask_1280_context/003-GPT-XL/checkpoints/eval_step_6000/batch_1_cfg_7.5_topk_16384.jpg b/CKPTS/X2I_UltraEdiit_fourmask_LLava_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_filtered_objects_4mask_1280_context/003-GPT-XL/checkpoints/eval_step_6000/batch_1_cfg_7.5_topk_16384.jpg new file mode 100644 index 0000000000000000000000000000000000000000..c566cc4ae378da4c189428d7cab01e03f93be12b --- /dev/null +++ b/CKPTS/X2I_UltraEdiit_fourmask_LLava_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_filtered_objects_4mask_1280_context/003-GPT-XL/checkpoints/eval_step_6000/batch_1_cfg_7.5_topk_16384.jpg @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:fb5d2f3b8879cf56a9a0840aa5529780583f86e32dbe1d0766b7f668f31ab5a4 +size 395871 diff --git a/CKPTS/X2I_UltraEdiit_fourmask_LLava_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_filtered_objects_4mask_1280_context/003-GPT-XL/checkpoints/eval_step_6000/batch_2_cfg_7.5_topk_16384.jpg b/CKPTS/X2I_UltraEdiit_fourmask_LLava_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_filtered_objects_4mask_1280_context/003-GPT-XL/checkpoints/eval_step_6000/batch_2_cfg_7.5_topk_16384.jpg new file mode 100644 index 0000000000000000000000000000000000000000..8f638b57ab70140f312aa178bdf2571453d7ceb0 --- /dev/null +++ b/CKPTS/X2I_UltraEdiit_fourmask_LLava_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_filtered_objects_4mask_1280_context/003-GPT-XL/checkpoints/eval_step_6000/batch_2_cfg_7.5_topk_16384.jpg @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:62479cca3c3f586acc28e0bd115aeab6da66baf550edc3f84a147da5aba10629 +size 407105 diff --git a/CKPTS/X2I_UltraEdiit_fourmask_LLava_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_filtered_objects_4mask_1280_context/003-GPT-XL/checkpoints/eval_step_6000/batch_3_cfg_7.5_topk_16384.jpg b/CKPTS/X2I_UltraEdiit_fourmask_LLava_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_filtered_objects_4mask_1280_context/003-GPT-XL/checkpoints/eval_step_6000/batch_3_cfg_7.5_topk_16384.jpg new file mode 100644 index 0000000000000000000000000000000000000000..d8d3f64b4797a90e5522045505a46faacf7cca0d --- /dev/null +++ b/CKPTS/X2I_UltraEdiit_fourmask_LLava_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_filtered_objects_4mask_1280_context/003-GPT-XL/checkpoints/eval_step_6000/batch_3_cfg_7.5_topk_16384.jpg @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:e87fe320806bb076dd44691af2c1970b12e1f35a9f2d20de6d0ad632fb6440db +size 403363 diff --git a/CKPTS/X2I_UltraEdiit_fourmask_LLava_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_filtered_objects_4mask_1280_context/003-GPT-XL/checkpoints/eval_step_6000/batch_4_cfg_7.5_topk_16384.jpg b/CKPTS/X2I_UltraEdiit_fourmask_LLava_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_filtered_objects_4mask_1280_context/003-GPT-XL/checkpoints/eval_step_6000/batch_4_cfg_7.5_topk_16384.jpg new file mode 100644 index 0000000000000000000000000000000000000000..bf977e4181422a631dc2555324bbee49fe416401 --- /dev/null +++ b/CKPTS/X2I_UltraEdiit_fourmask_LLava_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_filtered_objects_4mask_1280_context/003-GPT-XL/checkpoints/eval_step_6000/batch_4_cfg_7.5_topk_16384.jpg @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:9f162d998de67334a4b6845daedc7ac80a89bd69252c61ad7625aca048907383 +size 364136 diff --git a/CKPTS/X2I_UltraEdiit_fourmask_LLava_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_filtered_objects_4mask_1280_context/003-GPT-XL/checkpoints/eval_step_60000/batch_0_cfg_7.5_topk_16384.jpg b/CKPTS/X2I_UltraEdiit_fourmask_LLava_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_filtered_objects_4mask_1280_context/003-GPT-XL/checkpoints/eval_step_60000/batch_0_cfg_7.5_topk_16384.jpg new file mode 100644 index 0000000000000000000000000000000000000000..9af5e98a8d9a6dffab6615be6343bb027a430547 --- /dev/null +++ b/CKPTS/X2I_UltraEdiit_fourmask_LLava_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_filtered_objects_4mask_1280_context/003-GPT-XL/checkpoints/eval_step_60000/batch_0_cfg_7.5_topk_16384.jpg @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:b6fdd13630ffaa7767c08ab2adbc45bf36e22ccc9eed59bee9d7a59bc83a1ae6 +size 436813 diff --git a/CKPTS/X2I_UltraEdiit_fourmask_LLava_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_filtered_objects_4mask_1280_context/003-GPT-XL/checkpoints/eval_step_60000/batch_1_cfg_7.5_topk_16384.jpg b/CKPTS/X2I_UltraEdiit_fourmask_LLava_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_filtered_objects_4mask_1280_context/003-GPT-XL/checkpoints/eval_step_60000/batch_1_cfg_7.5_topk_16384.jpg new file mode 100644 index 0000000000000000000000000000000000000000..62beb42ea50c545d43c2bbfd07c9d1491cb0e15a --- /dev/null +++ b/CKPTS/X2I_UltraEdiit_fourmask_LLava_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_filtered_objects_4mask_1280_context/003-GPT-XL/checkpoints/eval_step_60000/batch_1_cfg_7.5_topk_16384.jpg @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:131a5e3a294bedc9df0738d1fbc5f731e723b3c61427e6e5b5810653602c70ac +size 429906 diff --git a/CKPTS/X2I_UltraEdiit_fourmask_LLava_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_filtered_objects_4mask_1280_context/003-GPT-XL/checkpoints/eval_step_60000/batch_2_cfg_7.5_topk_16384.jpg b/CKPTS/X2I_UltraEdiit_fourmask_LLava_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_filtered_objects_4mask_1280_context/003-GPT-XL/checkpoints/eval_step_60000/batch_2_cfg_7.5_topk_16384.jpg new file mode 100644 index 0000000000000000000000000000000000000000..c4b8d2dd0fe5744644c0f690480425fe23ec5b00 --- /dev/null +++ b/CKPTS/X2I_UltraEdiit_fourmask_LLava_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_filtered_objects_4mask_1280_context/003-GPT-XL/checkpoints/eval_step_60000/batch_2_cfg_7.5_topk_16384.jpg @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:3b61d8c069fa36e79decc6530faf5e49f7adb4f3949d779cbfe9e92937c2e60c +size 430921 diff --git a/CKPTS/X2I_UltraEdiit_fourmask_LLava_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_filtered_objects_4mask_1280_context/003-GPT-XL/checkpoints/eval_step_60000/batch_3_cfg_7.5_topk_16384.jpg b/CKPTS/X2I_UltraEdiit_fourmask_LLava_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_filtered_objects_4mask_1280_context/003-GPT-XL/checkpoints/eval_step_60000/batch_3_cfg_7.5_topk_16384.jpg new file mode 100644 index 0000000000000000000000000000000000000000..944bdaedce6e9fb2c4de83c7a58419c45a85d3b8 --- /dev/null +++ b/CKPTS/X2I_UltraEdiit_fourmask_LLava_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_filtered_objects_4mask_1280_context/003-GPT-XL/checkpoints/eval_step_60000/batch_3_cfg_7.5_topk_16384.jpg @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:25b86efb69d5dc0dad327c10b7700271208c84e6befdfe88694a0b9eaeb80b3d +size 394321 diff --git a/CKPTS/X2I_UltraEdiit_fourmask_LLava_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_filtered_objects_4mask_1280_context/003-GPT-XL/checkpoints/eval_step_60000/batch_4_cfg_7.5_topk_16384.jpg b/CKPTS/X2I_UltraEdiit_fourmask_LLava_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_filtered_objects_4mask_1280_context/003-GPT-XL/checkpoints/eval_step_60000/batch_4_cfg_7.5_topk_16384.jpg new file mode 100644 index 0000000000000000000000000000000000000000..06fd8f20fecd7016f27af0d968ec85164e91e01d --- /dev/null +++ b/CKPTS/X2I_UltraEdiit_fourmask_LLava_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_filtered_objects_4mask_1280_context/003-GPT-XL/checkpoints/eval_step_60000/batch_4_cfg_7.5_topk_16384.jpg @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:e3cda00de2b09647f0dd2e9afce5f6ff02648db055f235760029f3cb170c0dbd +size 349328 diff --git a/CKPTS/X2I_UltraEdiit_fourmask_LLava_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_filtered_objects_4mask_1280_context/003-GPT-XL/checkpoints/eval_step_62000/batch_0_cfg_7.5_topk_16384.jpg b/CKPTS/X2I_UltraEdiit_fourmask_LLava_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_filtered_objects_4mask_1280_context/003-GPT-XL/checkpoints/eval_step_62000/batch_0_cfg_7.5_topk_16384.jpg new file mode 100644 index 0000000000000000000000000000000000000000..877ee6ce2b49961a34023b4f3779071b74b05c40 --- /dev/null +++ b/CKPTS/X2I_UltraEdiit_fourmask_LLava_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_filtered_objects_4mask_1280_context/003-GPT-XL/checkpoints/eval_step_62000/batch_0_cfg_7.5_topk_16384.jpg @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:0d58c4404ac13768972425b3b57bd44f7ba45d08763df9669e896259ea8920ac +size 440446 diff --git a/CKPTS/X2I_UltraEdiit_fourmask_LLava_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_filtered_objects_4mask_1280_context/003-GPT-XL/checkpoints/eval_step_62000/batch_1_cfg_7.5_topk_16384.jpg b/CKPTS/X2I_UltraEdiit_fourmask_LLava_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_filtered_objects_4mask_1280_context/003-GPT-XL/checkpoints/eval_step_62000/batch_1_cfg_7.5_topk_16384.jpg new file mode 100644 index 0000000000000000000000000000000000000000..2328a3db3db660ce35925a147a5c067711eb2fe8 --- /dev/null +++ b/CKPTS/X2I_UltraEdiit_fourmask_LLava_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_filtered_objects_4mask_1280_context/003-GPT-XL/checkpoints/eval_step_62000/batch_1_cfg_7.5_topk_16384.jpg @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:6e08a8ca7a8cb28d98544d8975d1e7b190798f73d5403e7f75619931a2d4307f +size 435711 diff --git a/CKPTS/X2I_UltraEdiit_fourmask_LLava_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_filtered_objects_4mask_1280_context/003-GPT-XL/checkpoints/eval_step_62000/batch_2_cfg_7.5_topk_16384.jpg b/CKPTS/X2I_UltraEdiit_fourmask_LLava_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_filtered_objects_4mask_1280_context/003-GPT-XL/checkpoints/eval_step_62000/batch_2_cfg_7.5_topk_16384.jpg new file mode 100644 index 0000000000000000000000000000000000000000..bff7a083876a92daabf4381049a1cf9f8a02441d --- /dev/null +++ b/CKPTS/X2I_UltraEdiit_fourmask_LLava_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_filtered_objects_4mask_1280_context/003-GPT-XL/checkpoints/eval_step_62000/batch_2_cfg_7.5_topk_16384.jpg @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:4f7d32a3849a26fa69211d9dfae159ccf7211d98e19a0643505db20e976fa251 +size 423161 diff --git a/CKPTS/X2I_UltraEdiit_fourmask_LLava_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_filtered_objects_4mask_1280_context/003-GPT-XL/checkpoints/eval_step_62000/batch_3_cfg_7.5_topk_16384.jpg b/CKPTS/X2I_UltraEdiit_fourmask_LLava_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_filtered_objects_4mask_1280_context/003-GPT-XL/checkpoints/eval_step_62000/batch_3_cfg_7.5_topk_16384.jpg new file mode 100644 index 0000000000000000000000000000000000000000..533cdbb740143e6c464f929d36836465bcba5646 --- /dev/null +++ b/CKPTS/X2I_UltraEdiit_fourmask_LLava_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_filtered_objects_4mask_1280_context/003-GPT-XL/checkpoints/eval_step_62000/batch_3_cfg_7.5_topk_16384.jpg @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:65663df97e5eb158d892d33a351a50633f2a08c015db367532eaca15a55e631d +size 381964 diff --git a/CKPTS/X2I_UltraEdiit_fourmask_LLava_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_filtered_objects_4mask_1280_context/003-GPT-XL/checkpoints/eval_step_62000/batch_4_cfg_7.5_topk_16384.jpg b/CKPTS/X2I_UltraEdiit_fourmask_LLava_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_filtered_objects_4mask_1280_context/003-GPT-XL/checkpoints/eval_step_62000/batch_4_cfg_7.5_topk_16384.jpg new file mode 100644 index 0000000000000000000000000000000000000000..6b8497967190d010660153dc13d83013c19d8ea5 --- /dev/null +++ b/CKPTS/X2I_UltraEdiit_fourmask_LLava_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_filtered_objects_4mask_1280_context/003-GPT-XL/checkpoints/eval_step_62000/batch_4_cfg_7.5_topk_16384.jpg @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:e9d9c0544c69107221b84d2f4ef01b52446d8ee54efd48e50c6d4d43cd587821 +size 343596 diff --git a/CKPTS/X2I_UltraEdiit_fourmask_LLava_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_filtered_objects_4mask_1280_context/003-GPT-XL/checkpoints/eval_step_64000/batch_0_cfg_7.5_topk_16384.jpg b/CKPTS/X2I_UltraEdiit_fourmask_LLava_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_filtered_objects_4mask_1280_context/003-GPT-XL/checkpoints/eval_step_64000/batch_0_cfg_7.5_topk_16384.jpg new file mode 100644 index 0000000000000000000000000000000000000000..718a52dd14395930cee1ec6546cb43ea4cd19e5b --- /dev/null +++ b/CKPTS/X2I_UltraEdiit_fourmask_LLava_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_filtered_objects_4mask_1280_context/003-GPT-XL/checkpoints/eval_step_64000/batch_0_cfg_7.5_topk_16384.jpg @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:83707b47a2ab53eaf569a1eab1d40c20d680332f3d452ad3451ccb58ead6b591 +size 436981 diff --git a/CKPTS/X2I_UltraEdiit_fourmask_LLava_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_filtered_objects_4mask_1280_context/003-GPT-XL/checkpoints/eval_step_64000/batch_1_cfg_7.5_topk_16384.jpg b/CKPTS/X2I_UltraEdiit_fourmask_LLava_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_filtered_objects_4mask_1280_context/003-GPT-XL/checkpoints/eval_step_64000/batch_1_cfg_7.5_topk_16384.jpg new file mode 100644 index 0000000000000000000000000000000000000000..06e121d7910e34f19fad38c73cf0cae40fb18ee2 --- /dev/null +++ b/CKPTS/X2I_UltraEdiit_fourmask_LLava_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_filtered_objects_4mask_1280_context/003-GPT-XL/checkpoints/eval_step_64000/batch_1_cfg_7.5_topk_16384.jpg @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:4b2922f4794f50ed196f246b72b30819a2c982649229d82307916947894fe198 +size 423253 diff --git a/CKPTS/X2I_UltraEdiit_fourmask_LLava_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_filtered_objects_4mask_1280_context/003-GPT-XL/checkpoints/eval_step_64000/batch_2_cfg_7.5_topk_16384.jpg b/CKPTS/X2I_UltraEdiit_fourmask_LLava_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_filtered_objects_4mask_1280_context/003-GPT-XL/checkpoints/eval_step_64000/batch_2_cfg_7.5_topk_16384.jpg new file mode 100644 index 0000000000000000000000000000000000000000..b180fd5ab729938e5a6e706bb3172996490739c4 --- /dev/null +++ b/CKPTS/X2I_UltraEdiit_fourmask_LLava_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_filtered_objects_4mask_1280_context/003-GPT-XL/checkpoints/eval_step_64000/batch_2_cfg_7.5_topk_16384.jpg @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:3783eb97e1fccd14df7adcb63fd3b87709daea96c8609787b2dfb087bb618829 +size 420204 diff --git a/CKPTS/X2I_UltraEdiit_fourmask_LLava_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_filtered_objects_4mask_1280_context/003-GPT-XL/checkpoints/eval_step_64000/batch_3_cfg_7.5_topk_16384.jpg b/CKPTS/X2I_UltraEdiit_fourmask_LLava_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_filtered_objects_4mask_1280_context/003-GPT-XL/checkpoints/eval_step_64000/batch_3_cfg_7.5_topk_16384.jpg new file mode 100644 index 0000000000000000000000000000000000000000..4d8ea812abb7b3012ce1d2ea322b463f7ad99110 --- /dev/null +++ b/CKPTS/X2I_UltraEdiit_fourmask_LLava_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_filtered_objects_4mask_1280_context/003-GPT-XL/checkpoints/eval_step_64000/batch_3_cfg_7.5_topk_16384.jpg @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:61716b1b95967f1f15493118cc7de13afad83e2c7b4a5a7174300ba0fe471c18 +size 390982 diff --git a/CKPTS/X2I_UltraEdiit_fourmask_LLava_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_filtered_objects_4mask_1280_context/003-GPT-XL/checkpoints/eval_step_64000/batch_4_cfg_7.5_topk_16384.jpg b/CKPTS/X2I_UltraEdiit_fourmask_LLava_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_filtered_objects_4mask_1280_context/003-GPT-XL/checkpoints/eval_step_64000/batch_4_cfg_7.5_topk_16384.jpg new file mode 100644 index 0000000000000000000000000000000000000000..602db5509a182f1af1149f7d62ff40b7d975e826 --- /dev/null +++ b/CKPTS/X2I_UltraEdiit_fourmask_LLava_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_filtered_objects_4mask_1280_context/003-GPT-XL/checkpoints/eval_step_64000/batch_4_cfg_7.5_topk_16384.jpg @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:08eb42efcdc23d25a5111aab210fc3136e28906ba55df27b429568cdc42f11e6 +size 347067 diff --git a/CKPTS/X2I_UltraEdiit_fourmask_LLava_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_filtered_objects_4mask_1280_context/003-GPT-XL/checkpoints/eval_step_66000/batch_0_cfg_7.5_topk_16384.jpg b/CKPTS/X2I_UltraEdiit_fourmask_LLava_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_filtered_objects_4mask_1280_context/003-GPT-XL/checkpoints/eval_step_66000/batch_0_cfg_7.5_topk_16384.jpg new file mode 100644 index 0000000000000000000000000000000000000000..7d6044961f3b7511d0b867ee9d87bf74add1f69c --- /dev/null +++ b/CKPTS/X2I_UltraEdiit_fourmask_LLava_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_filtered_objects_4mask_1280_context/003-GPT-XL/checkpoints/eval_step_66000/batch_0_cfg_7.5_topk_16384.jpg @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:1f7cdb05d96af3703534e3afe99ec7a798f4600e10990a88f5218a5871905d9c +size 437791 diff --git a/CKPTS/X2I_UltraEdiit_fourmask_LLava_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_filtered_objects_4mask_1280_context/003-GPT-XL/checkpoints/eval_step_66000/batch_1_cfg_7.5_topk_16384.jpg b/CKPTS/X2I_UltraEdiit_fourmask_LLava_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_filtered_objects_4mask_1280_context/003-GPT-XL/checkpoints/eval_step_66000/batch_1_cfg_7.5_topk_16384.jpg new file mode 100644 index 0000000000000000000000000000000000000000..7ab39f7d36146f5cc3de76a2a2706cdbdd901aa4 --- /dev/null +++ b/CKPTS/X2I_UltraEdiit_fourmask_LLava_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_filtered_objects_4mask_1280_context/003-GPT-XL/checkpoints/eval_step_66000/batch_1_cfg_7.5_topk_16384.jpg @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:364ad1f00a6f5fd13aba6efe4fa31d22f9754c64693e41af804e8805283aae44 +size 432685 diff --git a/CKPTS/X2I_UltraEdiit_fourmask_LLava_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_filtered_objects_4mask_1280_context/003-GPT-XL/checkpoints/eval_step_66000/batch_2_cfg_7.5_topk_16384.jpg b/CKPTS/X2I_UltraEdiit_fourmask_LLava_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_filtered_objects_4mask_1280_context/003-GPT-XL/checkpoints/eval_step_66000/batch_2_cfg_7.5_topk_16384.jpg new file mode 100644 index 0000000000000000000000000000000000000000..c3fe6f9950e4fc53cf00ba56c840fe717e5f473f --- /dev/null +++ b/CKPTS/X2I_UltraEdiit_fourmask_LLava_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_filtered_objects_4mask_1280_context/003-GPT-XL/checkpoints/eval_step_66000/batch_2_cfg_7.5_topk_16384.jpg @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:f7328f8a422cc482ddc2bc1ae370286640565e8ea1b0a72e0e2b77bb8d57deef +size 432563 diff --git a/CKPTS/X2I_UltraEdiit_fourmask_LLava_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_filtered_objects_4mask_1280_context/003-GPT-XL/checkpoints/eval_step_66000/batch_3_cfg_7.5_topk_16384.jpg b/CKPTS/X2I_UltraEdiit_fourmask_LLava_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_filtered_objects_4mask_1280_context/003-GPT-XL/checkpoints/eval_step_66000/batch_3_cfg_7.5_topk_16384.jpg new file mode 100644 index 0000000000000000000000000000000000000000..0f6c709e3ecffd36c2b4328fee380ef80ec8f899 --- /dev/null +++ b/CKPTS/X2I_UltraEdiit_fourmask_LLava_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_filtered_objects_4mask_1280_context/003-GPT-XL/checkpoints/eval_step_66000/batch_3_cfg_7.5_topk_16384.jpg @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:1d8bc6d55ae71ae7e97397b6e0466715698da76bddf55632d04eb5df177cb5b3 +size 388884 diff --git a/CKPTS/X2I_UltraEdiit_fourmask_LLava_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_filtered_objects_4mask_1280_context/003-GPT-XL/checkpoints/eval_step_66000/batch_4_cfg_7.5_topk_16384.jpg b/CKPTS/X2I_UltraEdiit_fourmask_LLava_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_filtered_objects_4mask_1280_context/003-GPT-XL/checkpoints/eval_step_66000/batch_4_cfg_7.5_topk_16384.jpg new file mode 100644 index 0000000000000000000000000000000000000000..8580c1790080dc6076a378235f1f9bad48dcc6a3 --- /dev/null +++ b/CKPTS/X2I_UltraEdiit_fourmask_LLava_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_filtered_objects_4mask_1280_context/003-GPT-XL/checkpoints/eval_step_66000/batch_4_cfg_7.5_topk_16384.jpg @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:b129b017946256d5186b1a54d4470b1ef4073242e0f361bb95e129c4d05fe506 +size 342361 diff --git a/CKPTS/X2I_UltraEdiit_fourmask_LLava_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_filtered_objects_4mask_1280_context/003-GPT-XL/checkpoints/eval_step_68000/batch_0_cfg_7.5_topk_16384.jpg b/CKPTS/X2I_UltraEdiit_fourmask_LLava_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_filtered_objects_4mask_1280_context/003-GPT-XL/checkpoints/eval_step_68000/batch_0_cfg_7.5_topk_16384.jpg new file mode 100644 index 0000000000000000000000000000000000000000..a76f0b927610347681f4ba364e3e2417f35cfa5a --- /dev/null +++ b/CKPTS/X2I_UltraEdiit_fourmask_LLava_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_filtered_objects_4mask_1280_context/003-GPT-XL/checkpoints/eval_step_68000/batch_0_cfg_7.5_topk_16384.jpg @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:194f2b6c33741632bad6c679161f408ee1acb94d174e0d37550c7fa05f962ef1 +size 443216 diff --git a/CKPTS/X2I_UltraEdiit_fourmask_LLava_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_filtered_objects_4mask_1280_context/003-GPT-XL/checkpoints/eval_step_68000/batch_1_cfg_7.5_topk_16384.jpg b/CKPTS/X2I_UltraEdiit_fourmask_LLava_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_filtered_objects_4mask_1280_context/003-GPT-XL/checkpoints/eval_step_68000/batch_1_cfg_7.5_topk_16384.jpg new file mode 100644 index 0000000000000000000000000000000000000000..beefcdbefad58caf7dae43e5038ea5dcb1eff9ad --- /dev/null +++ b/CKPTS/X2I_UltraEdiit_fourmask_LLava_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_filtered_objects_4mask_1280_context/003-GPT-XL/checkpoints/eval_step_68000/batch_1_cfg_7.5_topk_16384.jpg @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:cc1c6961d7cad8539188e4963b2f0bc96db6ff692b672e11277ed3e66edb87b0 +size 425771 diff --git a/CKPTS/X2I_UltraEdiit_fourmask_LLava_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_filtered_objects_4mask_1280_context/003-GPT-XL/checkpoints/eval_step_68000/batch_2_cfg_7.5_topk_16384.jpg b/CKPTS/X2I_UltraEdiit_fourmask_LLava_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_filtered_objects_4mask_1280_context/003-GPT-XL/checkpoints/eval_step_68000/batch_2_cfg_7.5_topk_16384.jpg new file mode 100644 index 0000000000000000000000000000000000000000..577dc5fca62609cb314c2de04758146f73b8ca25 --- /dev/null +++ b/CKPTS/X2I_UltraEdiit_fourmask_LLava_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_filtered_objects_4mask_1280_context/003-GPT-XL/checkpoints/eval_step_68000/batch_2_cfg_7.5_topk_16384.jpg @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:bd4aa59c19ac654ed0e4f1682ee4ece356e0e3be26608bb8ff93834455df1987 +size 423110 diff --git a/CKPTS/X2I_UltraEdiit_fourmask_LLava_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_filtered_objects_4mask_1280_context/003-GPT-XL/checkpoints/eval_step_68000/batch_3_cfg_7.5_topk_16384.jpg b/CKPTS/X2I_UltraEdiit_fourmask_LLava_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_filtered_objects_4mask_1280_context/003-GPT-XL/checkpoints/eval_step_68000/batch_3_cfg_7.5_topk_16384.jpg new file mode 100644 index 0000000000000000000000000000000000000000..1ee4e0ba62f0ff2d5b03876d5c3cfde1b6a02943 --- /dev/null +++ b/CKPTS/X2I_UltraEdiit_fourmask_LLava_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_filtered_objects_4mask_1280_context/003-GPT-XL/checkpoints/eval_step_68000/batch_3_cfg_7.5_topk_16384.jpg @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:0e07a2741c0f2e29c91d9b60498720a4f43577390d6312e60759f986952e0b94 +size 399263 diff --git a/CKPTS/X2I_UltraEdiit_fourmask_LLava_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_filtered_objects_4mask_1280_context/003-GPT-XL/checkpoints/eval_step_68000/batch_4_cfg_7.5_topk_16384.jpg b/CKPTS/X2I_UltraEdiit_fourmask_LLava_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_filtered_objects_4mask_1280_context/003-GPT-XL/checkpoints/eval_step_68000/batch_4_cfg_7.5_topk_16384.jpg new file mode 100644 index 0000000000000000000000000000000000000000..0b3020e57e44c007558b65254873919a5e727e9a --- /dev/null +++ b/CKPTS/X2I_UltraEdiit_fourmask_LLava_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_filtered_objects_4mask_1280_context/003-GPT-XL/checkpoints/eval_step_68000/batch_4_cfg_7.5_topk_16384.jpg @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:70880777e40de3e86b28653d91d40a670937f4bab32c39b5548b723e9c3e162d +size 348633 diff --git a/CKPTS/X2I_UltraEdiit_fourmask_LLava_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_filtered_objects_4mask_1280_context/003-GPT-XL/checkpoints/eval_step_70000/batch_0_cfg_7.5_topk_16384.jpg b/CKPTS/X2I_UltraEdiit_fourmask_LLava_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_filtered_objects_4mask_1280_context/003-GPT-XL/checkpoints/eval_step_70000/batch_0_cfg_7.5_topk_16384.jpg new file mode 100644 index 0000000000000000000000000000000000000000..e906953ae8172d2634d816a7ae960ef6f8f20de5 --- /dev/null +++ b/CKPTS/X2I_UltraEdiit_fourmask_LLava_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_filtered_objects_4mask_1280_context/003-GPT-XL/checkpoints/eval_step_70000/batch_0_cfg_7.5_topk_16384.jpg @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:8906e5375ccfc239bcfb3f9a7e80cf09d2c5dcae9e681b65a680fa583115db61 +size 446422 diff --git a/CKPTS/X2I_UltraEdiit_fourmask_LLava_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_filtered_objects_4mask_1280_context/003-GPT-XL/checkpoints/eval_step_70000/batch_1_cfg_7.5_topk_16384.jpg b/CKPTS/X2I_UltraEdiit_fourmask_LLava_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_filtered_objects_4mask_1280_context/003-GPT-XL/checkpoints/eval_step_70000/batch_1_cfg_7.5_topk_16384.jpg new file mode 100644 index 0000000000000000000000000000000000000000..aa13837e358be52d16ebe2f5d9b9e7e06bb667a7 --- /dev/null +++ b/CKPTS/X2I_UltraEdiit_fourmask_LLava_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_filtered_objects_4mask_1280_context/003-GPT-XL/checkpoints/eval_step_70000/batch_1_cfg_7.5_topk_16384.jpg @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:92574fee929fcc092f7d585aeb153db43c5a3ebb61b66b24f0657c3b64edca99 +size 433250 diff --git a/CKPTS/X2I_UltraEdiit_fourmask_LLava_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_filtered_objects_4mask_1280_context/003-GPT-XL/checkpoints/eval_step_70000/batch_2_cfg_7.5_topk_16384.jpg b/CKPTS/X2I_UltraEdiit_fourmask_LLava_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_filtered_objects_4mask_1280_context/003-GPT-XL/checkpoints/eval_step_70000/batch_2_cfg_7.5_topk_16384.jpg new file mode 100644 index 0000000000000000000000000000000000000000..bcd6cec3f7e270f23455e8753445751446ddce28 --- /dev/null +++ b/CKPTS/X2I_UltraEdiit_fourmask_LLava_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_filtered_objects_4mask_1280_context/003-GPT-XL/checkpoints/eval_step_70000/batch_2_cfg_7.5_topk_16384.jpg @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:9e56899fc7fff4ddea332f852a3850a10f9f3c62d8a16e3ab38d550da0d0ab5f +size 417822 diff --git a/CKPTS/X2I_UltraEdiit_fourmask_LLava_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_filtered_objects_4mask_1280_context/003-GPT-XL/checkpoints/eval_step_70000/batch_3_cfg_7.5_topk_16384.jpg b/CKPTS/X2I_UltraEdiit_fourmask_LLava_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_filtered_objects_4mask_1280_context/003-GPT-XL/checkpoints/eval_step_70000/batch_3_cfg_7.5_topk_16384.jpg new file mode 100644 index 0000000000000000000000000000000000000000..63603f7a45c212ff04f970cf05582738dc3f243b --- /dev/null +++ b/CKPTS/X2I_UltraEdiit_fourmask_LLava_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_filtered_objects_4mask_1280_context/003-GPT-XL/checkpoints/eval_step_70000/batch_3_cfg_7.5_topk_16384.jpg @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:55626e02e48a0e8f375ff8cf149e3bb1464417e79a4e3c84d1f7aa172d1a68e8 +size 381312 diff --git a/CKPTS/X2I_UltraEdiit_fourmask_LLava_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_filtered_objects_4mask_1280_context/003-GPT-XL/checkpoints/eval_step_70000/batch_4_cfg_7.5_topk_16384.jpg b/CKPTS/X2I_UltraEdiit_fourmask_LLava_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_filtered_objects_4mask_1280_context/003-GPT-XL/checkpoints/eval_step_70000/batch_4_cfg_7.5_topk_16384.jpg new file mode 100644 index 0000000000000000000000000000000000000000..cdece8cbefc2a9e2c150eac4f7c312604a172cef --- /dev/null +++ b/CKPTS/X2I_UltraEdiit_fourmask_LLava_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_filtered_objects_4mask_1280_context/003-GPT-XL/checkpoints/eval_step_70000/batch_4_cfg_7.5_topk_16384.jpg @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:30403feff68056c71d28e9926bbe480db293a25490a8ea56f026a497e41364c4 +size 343410 diff --git a/CKPTS/X2I_UltraEdiit_fourmask_LLava_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_filtered_objects_4mask_1280_context/003-GPT-XL/checkpoints/eval_step_72000/batch_0_cfg_7.5_topk_16384.jpg b/CKPTS/X2I_UltraEdiit_fourmask_LLava_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_filtered_objects_4mask_1280_context/003-GPT-XL/checkpoints/eval_step_72000/batch_0_cfg_7.5_topk_16384.jpg new file mode 100644 index 0000000000000000000000000000000000000000..4e79dc9da2a62a72ead444b3afedaee002bdaf0c --- /dev/null +++ b/CKPTS/X2I_UltraEdiit_fourmask_LLava_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_filtered_objects_4mask_1280_context/003-GPT-XL/checkpoints/eval_step_72000/batch_0_cfg_7.5_topk_16384.jpg @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:d51bf793e3b269b257c5ed3bdcf5230ff884d5cb13f03a18216f24d8480ff4ec +size 422160 diff --git a/CKPTS/X2I_UltraEdiit_fourmask_LLava_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_filtered_objects_4mask_1280_context/003-GPT-XL/checkpoints/eval_step_72000/batch_1_cfg_7.5_topk_16384.jpg b/CKPTS/X2I_UltraEdiit_fourmask_LLava_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_filtered_objects_4mask_1280_context/003-GPT-XL/checkpoints/eval_step_72000/batch_1_cfg_7.5_topk_16384.jpg new file mode 100644 index 0000000000000000000000000000000000000000..50a2ea833117ac1d4eeaf7055e916a6c4c90588f --- /dev/null +++ b/CKPTS/X2I_UltraEdiit_fourmask_LLava_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_filtered_objects_4mask_1280_context/003-GPT-XL/checkpoints/eval_step_72000/batch_1_cfg_7.5_topk_16384.jpg @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:a7ab8a5cd900f1babb7a680e912ed1dc292726249714ebf13a1e6a8c798ca988 +size 428477 diff --git a/CKPTS/X2I_UltraEdiit_fourmask_LLava_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_filtered_objects_4mask_1280_context/003-GPT-XL/checkpoints/eval_step_72000/batch_2_cfg_7.5_topk_16384.jpg b/CKPTS/X2I_UltraEdiit_fourmask_LLava_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_filtered_objects_4mask_1280_context/003-GPT-XL/checkpoints/eval_step_72000/batch_2_cfg_7.5_topk_16384.jpg new file mode 100644 index 0000000000000000000000000000000000000000..79593314e37470a7896a0898466ab618eadaff95 --- /dev/null +++ b/CKPTS/X2I_UltraEdiit_fourmask_LLava_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_filtered_objects_4mask_1280_context/003-GPT-XL/checkpoints/eval_step_72000/batch_2_cfg_7.5_topk_16384.jpg @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:db9647f5c5067f98c752525b596d3f829220ebda6bb7962c7ffc18124c365931 +size 418547 diff --git a/CKPTS/X2I_UltraEdiit_fourmask_LLava_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_filtered_objects_4mask_1280_context/003-GPT-XL/checkpoints/eval_step_72000/batch_3_cfg_7.5_topk_16384.jpg b/CKPTS/X2I_UltraEdiit_fourmask_LLava_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_filtered_objects_4mask_1280_context/003-GPT-XL/checkpoints/eval_step_72000/batch_3_cfg_7.5_topk_16384.jpg new file mode 100644 index 0000000000000000000000000000000000000000..a464868151d1dd5110f11d8e25133805cb9885cd --- /dev/null +++ b/CKPTS/X2I_UltraEdiit_fourmask_LLava_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_filtered_objects_4mask_1280_context/003-GPT-XL/checkpoints/eval_step_72000/batch_3_cfg_7.5_topk_16384.jpg @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:b944703ccc8f9a808504659a7831d3179328c4bf8de0211e4b00b3346ebb55c3 +size 399026 diff --git a/CKPTS/X2I_UltraEdiit_fourmask_LLava_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_filtered_objects_4mask_1280_context/003-GPT-XL/checkpoints/eval_step_72000/batch_4_cfg_7.5_topk_16384.jpg b/CKPTS/X2I_UltraEdiit_fourmask_LLava_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_filtered_objects_4mask_1280_context/003-GPT-XL/checkpoints/eval_step_72000/batch_4_cfg_7.5_topk_16384.jpg new file mode 100644 index 0000000000000000000000000000000000000000..3eeacce1c6eabba4ba40cc13631f63f472f834d7 --- /dev/null +++ b/CKPTS/X2I_UltraEdiit_fourmask_LLava_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_filtered_objects_4mask_1280_context/003-GPT-XL/checkpoints/eval_step_72000/batch_4_cfg_7.5_topk_16384.jpg @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:06e721b727e1dc74508493dcb2501c7f25f239278c8d443233321f3525c8494f +size 345011 diff --git a/CKPTS/X2I_UltraEdiit_fourmask_LLava_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_filtered_objects_4mask_1280_context/003-GPT-XL/checkpoints/eval_step_74000/batch_0_cfg_7.5_topk_16384.jpg b/CKPTS/X2I_UltraEdiit_fourmask_LLava_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_filtered_objects_4mask_1280_context/003-GPT-XL/checkpoints/eval_step_74000/batch_0_cfg_7.5_topk_16384.jpg new file mode 100644 index 0000000000000000000000000000000000000000..143c0e5a27391f18afe94fd69b3d1e42a4e6b0e6 --- /dev/null +++ b/CKPTS/X2I_UltraEdiit_fourmask_LLava_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_filtered_objects_4mask_1280_context/003-GPT-XL/checkpoints/eval_step_74000/batch_0_cfg_7.5_topk_16384.jpg @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:611ac0f033aeb318d7680aa43378d78a7fa142d12e30b1a030571e3325f7416c +size 429000 diff --git a/CKPTS/X2I_UltraEdiit_fourmask_LLava_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_filtered_objects_4mask_1280_context/003-GPT-XL/checkpoints/eval_step_74000/batch_1_cfg_7.5_topk_16384.jpg b/CKPTS/X2I_UltraEdiit_fourmask_LLava_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_filtered_objects_4mask_1280_context/003-GPT-XL/checkpoints/eval_step_74000/batch_1_cfg_7.5_topk_16384.jpg new file mode 100644 index 0000000000000000000000000000000000000000..5a14dc71200a49e85fb12426fa9a264c99db974e --- /dev/null +++ b/CKPTS/X2I_UltraEdiit_fourmask_LLava_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_filtered_objects_4mask_1280_context/003-GPT-XL/checkpoints/eval_step_74000/batch_1_cfg_7.5_topk_16384.jpg @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:aebd5ffaab7948f066fe2b5e1715e3d4ad867e6103b3367c1c3220fc7f71f72f +size 428882 diff --git a/CKPTS/X2I_UltraEdiit_fourmask_LLava_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_filtered_objects_4mask_1280_context/003-GPT-XL/checkpoints/eval_step_74000/batch_2_cfg_7.5_topk_16384.jpg b/CKPTS/X2I_UltraEdiit_fourmask_LLava_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_filtered_objects_4mask_1280_context/003-GPT-XL/checkpoints/eval_step_74000/batch_2_cfg_7.5_topk_16384.jpg new file mode 100644 index 0000000000000000000000000000000000000000..da67ad0f71b1449fe2a220720a992a6ca4659dfe --- /dev/null +++ b/CKPTS/X2I_UltraEdiit_fourmask_LLava_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_filtered_objects_4mask_1280_context/003-GPT-XL/checkpoints/eval_step_74000/batch_2_cfg_7.5_topk_16384.jpg @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:d59c8641f3c77300b42ebb135cfadf0c1811f49564cfb87867bde59138921c6d +size 431049 diff --git a/CKPTS/X2I_UltraEdiit_fourmask_LLava_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_filtered_objects_4mask_1280_context/003-GPT-XL/checkpoints/eval_step_74000/batch_3_cfg_7.5_topk_16384.jpg b/CKPTS/X2I_UltraEdiit_fourmask_LLava_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_filtered_objects_4mask_1280_context/003-GPT-XL/checkpoints/eval_step_74000/batch_3_cfg_7.5_topk_16384.jpg new file mode 100644 index 0000000000000000000000000000000000000000..ae5e49040243134a507187e995eaacd1449f0dbf --- /dev/null +++ b/CKPTS/X2I_UltraEdiit_fourmask_LLava_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_filtered_objects_4mask_1280_context/003-GPT-XL/checkpoints/eval_step_74000/batch_3_cfg_7.5_topk_16384.jpg @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:7639502319cea9f4993dd4d555b2f30c26cad0a6089cb442246a712b4645802f +size 375594 diff --git a/CKPTS/X2I_UltraEdiit_fourmask_LLava_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_filtered_objects_4mask_1280_context/003-GPT-XL/checkpoints/eval_step_74000/batch_4_cfg_7.5_topk_16384.jpg b/CKPTS/X2I_UltraEdiit_fourmask_LLava_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_filtered_objects_4mask_1280_context/003-GPT-XL/checkpoints/eval_step_74000/batch_4_cfg_7.5_topk_16384.jpg new file mode 100644 index 0000000000000000000000000000000000000000..c462f0244bb1453d10e57aef764f334accbe0fd1 --- /dev/null +++ b/CKPTS/X2I_UltraEdiit_fourmask_LLava_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_filtered_objects_4mask_1280_context/003-GPT-XL/checkpoints/eval_step_74000/batch_4_cfg_7.5_topk_16384.jpg @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:3964e46db6c86e72720374578b72b93c32704e471fd5a3ab204fa837f7a5b982 +size 339901 diff --git a/CKPTS/X2I_UltraEdiit_fourmask_LLava_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_filtered_objects_4mask_1280_context/003-GPT-XL/checkpoints/eval_step_76000/batch_0_cfg_7.5_topk_16384.jpg b/CKPTS/X2I_UltraEdiit_fourmask_LLava_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_filtered_objects_4mask_1280_context/003-GPT-XL/checkpoints/eval_step_76000/batch_0_cfg_7.5_topk_16384.jpg new file mode 100644 index 0000000000000000000000000000000000000000..acfca69ed4a0b3b0d65182395e0babf516abd6ac --- /dev/null +++ b/CKPTS/X2I_UltraEdiit_fourmask_LLava_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_filtered_objects_4mask_1280_context/003-GPT-XL/checkpoints/eval_step_76000/batch_0_cfg_7.5_topk_16384.jpg @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:1b6579608d4188f9654bdbdc5476eaf72cd30a52e832af917d1624ba24ab0ce5 +size 429368 diff --git a/CKPTS/X2I_UltraEdiit_fourmask_LLava_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_filtered_objects_4mask_1280_context/003-GPT-XL/checkpoints/eval_step_76000/batch_1_cfg_7.5_topk_16384.jpg b/CKPTS/X2I_UltraEdiit_fourmask_LLava_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_filtered_objects_4mask_1280_context/003-GPT-XL/checkpoints/eval_step_76000/batch_1_cfg_7.5_topk_16384.jpg new file mode 100644 index 0000000000000000000000000000000000000000..7c561a2fc73b55e3e153682dfc1bf8b583ca0518 --- /dev/null +++ b/CKPTS/X2I_UltraEdiit_fourmask_LLava_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_filtered_objects_4mask_1280_context/003-GPT-XL/checkpoints/eval_step_76000/batch_1_cfg_7.5_topk_16384.jpg @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:f58b8a421d2f03eca0c58e48cfb5684e565046d9abb6af4129d4260002d3031c +size 413935 diff --git a/CKPTS/X2I_UltraEdiit_fourmask_LLava_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_filtered_objects_4mask_1280_context/003-GPT-XL/checkpoints/eval_step_76000/batch_2_cfg_7.5_topk_16384.jpg b/CKPTS/X2I_UltraEdiit_fourmask_LLava_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_filtered_objects_4mask_1280_context/003-GPT-XL/checkpoints/eval_step_76000/batch_2_cfg_7.5_topk_16384.jpg new file mode 100644 index 0000000000000000000000000000000000000000..dd9dd90bb0eda0b12cac07953c55b426599cd48a --- /dev/null +++ b/CKPTS/X2I_UltraEdiit_fourmask_LLava_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_filtered_objects_4mask_1280_context/003-GPT-XL/checkpoints/eval_step_76000/batch_2_cfg_7.5_topk_16384.jpg @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:0a7606b2101d69db6ebf5ebbff2df852d4035d63309f2a6240d17ab96dd71cf9 +size 422872 diff --git a/CKPTS/X2I_UltraEdiit_fourmask_LLava_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_filtered_objects_4mask_1280_context/003-GPT-XL/checkpoints/eval_step_76000/batch_3_cfg_7.5_topk_16384.jpg b/CKPTS/X2I_UltraEdiit_fourmask_LLava_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_filtered_objects_4mask_1280_context/003-GPT-XL/checkpoints/eval_step_76000/batch_3_cfg_7.5_topk_16384.jpg new file mode 100644 index 0000000000000000000000000000000000000000..16f6337a99b0d340617df176fb00ebe09dea3860 --- /dev/null +++ b/CKPTS/X2I_UltraEdiit_fourmask_LLava_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_filtered_objects_4mask_1280_context/003-GPT-XL/checkpoints/eval_step_76000/batch_3_cfg_7.5_topk_16384.jpg @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:f4a6075539dcaf42b751f9693044b70cc33768eac63446ff910a1ad82c9e25b2 +size 399609 diff --git a/CKPTS/X2I_UltraEdiit_fourmask_LLava_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_filtered_objects_4mask_1280_context/003-GPT-XL/checkpoints/eval_step_76000/batch_4_cfg_7.5_topk_16384.jpg b/CKPTS/X2I_UltraEdiit_fourmask_LLava_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_filtered_objects_4mask_1280_context/003-GPT-XL/checkpoints/eval_step_76000/batch_4_cfg_7.5_topk_16384.jpg new file mode 100644 index 0000000000000000000000000000000000000000..e88e9f9b02987096f6dea27b72c89fe65c795e5a --- /dev/null +++ b/CKPTS/X2I_UltraEdiit_fourmask_LLava_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_filtered_objects_4mask_1280_context/003-GPT-XL/checkpoints/eval_step_76000/batch_4_cfg_7.5_topk_16384.jpg @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:8e1532240fba6b7e5672ba87c9d817a6f4e52b9e360378614b5bb2b5a677e9d1 +size 341728 diff --git a/CKPTS/X2I_UltraEdiit_fourmask_LLava_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_filtered_objects_4mask_1280_context/003-GPT-XL/checkpoints/eval_step_78000/batch_0_cfg_7.5_topk_16384.jpg b/CKPTS/X2I_UltraEdiit_fourmask_LLava_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_filtered_objects_4mask_1280_context/003-GPT-XL/checkpoints/eval_step_78000/batch_0_cfg_7.5_topk_16384.jpg new file mode 100644 index 0000000000000000000000000000000000000000..f7e5c044720681ccd9fa610b71c48b70c7e67afc --- /dev/null +++ b/CKPTS/X2I_UltraEdiit_fourmask_LLava_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_filtered_objects_4mask_1280_context/003-GPT-XL/checkpoints/eval_step_78000/batch_0_cfg_7.5_topk_16384.jpg @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:6c447525d4c3a5277694fe5cd2364b9c9f19426f9bc4c72f276b73deff80e3df +size 439648 diff --git a/CKPTS/X2I_UltraEdiit_fourmask_LLava_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_filtered_objects_4mask_1280_context/003-GPT-XL/checkpoints/eval_step_78000/batch_1_cfg_7.5_topk_16384.jpg b/CKPTS/X2I_UltraEdiit_fourmask_LLava_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_filtered_objects_4mask_1280_context/003-GPT-XL/checkpoints/eval_step_78000/batch_1_cfg_7.5_topk_16384.jpg new file mode 100644 index 0000000000000000000000000000000000000000..debfc2471f715d4f455d9c07918c39667aaeb517 --- /dev/null +++ b/CKPTS/X2I_UltraEdiit_fourmask_LLava_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_filtered_objects_4mask_1280_context/003-GPT-XL/checkpoints/eval_step_78000/batch_1_cfg_7.5_topk_16384.jpg @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:00e6ed329636303e45c655ecb81e93bae7205e797485b619ebeb9d51f877ce0e +size 436138 diff --git a/CKPTS/X2I_UltraEdiit_fourmask_LLava_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_filtered_objects_4mask_1280_context/003-GPT-XL/checkpoints/eval_step_78000/batch_2_cfg_7.5_topk_16384.jpg b/CKPTS/X2I_UltraEdiit_fourmask_LLava_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_filtered_objects_4mask_1280_context/003-GPT-XL/checkpoints/eval_step_78000/batch_2_cfg_7.5_topk_16384.jpg new file mode 100644 index 0000000000000000000000000000000000000000..3c64fd19e8f6bea6c605bdfd13918b91aa5f9bbb --- /dev/null +++ b/CKPTS/X2I_UltraEdiit_fourmask_LLava_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_filtered_objects_4mask_1280_context/003-GPT-XL/checkpoints/eval_step_78000/batch_2_cfg_7.5_topk_16384.jpg @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:39f049b1d2e7cfa8587ab2b043b7c46ec28f30241f42dcc1dc1ad89b1412a0e5 +size 427310 diff --git a/CKPTS/X2I_UltraEdiit_fourmask_LLava_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_filtered_objects_4mask_1280_context/003-GPT-XL/checkpoints/eval_step_78000/batch_3_cfg_7.5_topk_16384.jpg b/CKPTS/X2I_UltraEdiit_fourmask_LLava_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_filtered_objects_4mask_1280_context/003-GPT-XL/checkpoints/eval_step_78000/batch_3_cfg_7.5_topk_16384.jpg new file mode 100644 index 0000000000000000000000000000000000000000..ab017eeb82c0514f6659b836f0b558677c36df85 --- /dev/null +++ b/CKPTS/X2I_UltraEdiit_fourmask_LLava_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_filtered_objects_4mask_1280_context/003-GPT-XL/checkpoints/eval_step_78000/batch_3_cfg_7.5_topk_16384.jpg @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:6b1eab2a6a1f5e5cc638aa61dc4e216c289dfdbd73ee4394a06c77dabba982ae +size 390889 diff --git a/CKPTS/X2I_UltraEdiit_fourmask_LLava_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_filtered_objects_4mask_1280_context/003-GPT-XL/checkpoints/eval_step_78000/batch_4_cfg_7.5_topk_16384.jpg b/CKPTS/X2I_UltraEdiit_fourmask_LLava_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_filtered_objects_4mask_1280_context/003-GPT-XL/checkpoints/eval_step_78000/batch_4_cfg_7.5_topk_16384.jpg new file mode 100644 index 0000000000000000000000000000000000000000..72015789b6708cffbfaf2435fb0ae70064e5d16f --- /dev/null +++ b/CKPTS/X2I_UltraEdiit_fourmask_LLava_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_filtered_objects_4mask_1280_context/003-GPT-XL/checkpoints/eval_step_78000/batch_4_cfg_7.5_topk_16384.jpg @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:436012495e9396a19bb0306de38d23d5ab8c33f08a1274542e8f3401244b066d +size 339246 diff --git a/CKPTS/X2I_UltraEdiit_fourmask_LLava_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_filtered_objects_4mask_1280_context/003-GPT-XL/checkpoints/eval_step_8000/batch_0_cfg_7.5_topk_16384.jpg b/CKPTS/X2I_UltraEdiit_fourmask_LLava_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_filtered_objects_4mask_1280_context/003-GPT-XL/checkpoints/eval_step_8000/batch_0_cfg_7.5_topk_16384.jpg new file mode 100644 index 0000000000000000000000000000000000000000..1fece4afdf04a1df4254ba14c2a367408fe22a7f --- /dev/null +++ b/CKPTS/X2I_UltraEdiit_fourmask_LLava_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_filtered_objects_4mask_1280_context/003-GPT-XL/checkpoints/eval_step_8000/batch_0_cfg_7.5_topk_16384.jpg @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:ef16f5632873d583acb1090f05fac1a180e76c1b0c6a2707155882bc7f364a98 +size 467659 diff --git a/CKPTS/X2I_UltraEdiit_fourmask_LLava_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_filtered_objects_4mask_1280_context/003-GPT-XL/checkpoints/eval_step_8000/batch_1_cfg_7.5_topk_16384.jpg b/CKPTS/X2I_UltraEdiit_fourmask_LLava_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_filtered_objects_4mask_1280_context/003-GPT-XL/checkpoints/eval_step_8000/batch_1_cfg_7.5_topk_16384.jpg new file mode 100644 index 0000000000000000000000000000000000000000..46953427a83e7af6ccc82cf8c288f633c9188c3b --- /dev/null +++ b/CKPTS/X2I_UltraEdiit_fourmask_LLava_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_filtered_objects_4mask_1280_context/003-GPT-XL/checkpoints/eval_step_8000/batch_1_cfg_7.5_topk_16384.jpg @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:6b98889a4ab3917e6b5a83ff3936df0326fed2dc8aabad11bcc185f359e4363c +size 436670 diff --git a/CKPTS/X2I_UltraEdiit_fourmask_LLava_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_filtered_objects_4mask_1280_context/003-GPT-XL/checkpoints/eval_step_8000/batch_2_cfg_7.5_topk_16384.jpg b/CKPTS/X2I_UltraEdiit_fourmask_LLava_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_filtered_objects_4mask_1280_context/003-GPT-XL/checkpoints/eval_step_8000/batch_2_cfg_7.5_topk_16384.jpg new file mode 100644 index 0000000000000000000000000000000000000000..903c0c6ed6d78420cfa1755139e5ed82418765b4 --- /dev/null +++ b/CKPTS/X2I_UltraEdiit_fourmask_LLava_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_filtered_objects_4mask_1280_context/003-GPT-XL/checkpoints/eval_step_8000/batch_2_cfg_7.5_topk_16384.jpg @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:9330d1195b711236fadf87f15d395301bbbbf92d857656d507889a13fcbd0d2c +size 415178 diff --git a/CKPTS/X2I_UltraEdiit_fourmask_LLava_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_filtered_objects_4mask_1280_context/003-GPT-XL/checkpoints/eval_step_8000/batch_3_cfg_7.5_topk_16384.jpg b/CKPTS/X2I_UltraEdiit_fourmask_LLava_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_filtered_objects_4mask_1280_context/003-GPT-XL/checkpoints/eval_step_8000/batch_3_cfg_7.5_topk_16384.jpg new file mode 100644 index 0000000000000000000000000000000000000000..0bcf15fa7013eefcfbac2fa125424b7f26e07019 --- /dev/null +++ b/CKPTS/X2I_UltraEdiit_fourmask_LLava_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_filtered_objects_4mask_1280_context/003-GPT-XL/checkpoints/eval_step_8000/batch_3_cfg_7.5_topk_16384.jpg @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:7b069eea277835f94ed09a775ec9ce0cc7cf37790756f1cb94424254f1bd1b3b +size 423448 diff --git a/CKPTS/X2I_UltraEdiit_fourmask_LLava_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_filtered_objects_4mask_1280_context/003-GPT-XL/checkpoints/eval_step_8000/batch_4_cfg_7.5_topk_16384.jpg b/CKPTS/X2I_UltraEdiit_fourmask_LLava_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_filtered_objects_4mask_1280_context/003-GPT-XL/checkpoints/eval_step_8000/batch_4_cfg_7.5_topk_16384.jpg new file mode 100644 index 0000000000000000000000000000000000000000..d4ce96cf48188e6246054076f374f3a09ebfc503 --- /dev/null +++ b/CKPTS/X2I_UltraEdiit_fourmask_LLava_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_filtered_objects_4mask_1280_context/003-GPT-XL/checkpoints/eval_step_8000/batch_4_cfg_7.5_topk_16384.jpg @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:f7351d2af06391ad1495a73fc855c23495e413d0f5be820975be27d657b5bb49 +size 347862 diff --git a/CKPTS/X2I_UltraEdiit_fourmask_LLava_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_filtered_objects_4mask_1280_context/003-GPT-XL/checkpoints/eval_step_80000/batch_0_cfg_7.5_topk_16384.jpg b/CKPTS/X2I_UltraEdiit_fourmask_LLava_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_filtered_objects_4mask_1280_context/003-GPT-XL/checkpoints/eval_step_80000/batch_0_cfg_7.5_topk_16384.jpg new file mode 100644 index 0000000000000000000000000000000000000000..53bdd915d21b2fa6e047f54ee9f04ead9b3c8b95 --- /dev/null +++ b/CKPTS/X2I_UltraEdiit_fourmask_LLava_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_filtered_objects_4mask_1280_context/003-GPT-XL/checkpoints/eval_step_80000/batch_0_cfg_7.5_topk_16384.jpg @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:16bed57dc6e27fb91c07eaa03f0374dc0e5e06db1c1a651a372bc26fda9d523c +size 420556 diff --git a/CKPTS/X2I_UltraEdiit_fourmask_LLava_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_filtered_objects_4mask_1280_context/003-GPT-XL/checkpoints/eval_step_80000/batch_1_cfg_7.5_topk_16384.jpg b/CKPTS/X2I_UltraEdiit_fourmask_LLava_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_filtered_objects_4mask_1280_context/003-GPT-XL/checkpoints/eval_step_80000/batch_1_cfg_7.5_topk_16384.jpg new file mode 100644 index 0000000000000000000000000000000000000000..ddb75346abd54cd95eb3a159cff15bb3c1af8118 --- /dev/null +++ b/CKPTS/X2I_UltraEdiit_fourmask_LLava_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_filtered_objects_4mask_1280_context/003-GPT-XL/checkpoints/eval_step_80000/batch_1_cfg_7.5_topk_16384.jpg @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:df3ffea8fb0737b7481b67ed1afbce7b37a01ef558c3fc5951f9e600441be6fa +size 422509 diff --git a/CKPTS/X2I_UltraEdiit_fourmask_LLava_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_filtered_objects_4mask_1280_context/003-GPT-XL/checkpoints/eval_step_80000/batch_2_cfg_7.5_topk_16384.jpg b/CKPTS/X2I_UltraEdiit_fourmask_LLava_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_filtered_objects_4mask_1280_context/003-GPT-XL/checkpoints/eval_step_80000/batch_2_cfg_7.5_topk_16384.jpg new file mode 100644 index 0000000000000000000000000000000000000000..d472d3291cc982dbdd9bed8f82d70a66155325ff --- /dev/null +++ b/CKPTS/X2I_UltraEdiit_fourmask_LLava_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_filtered_objects_4mask_1280_context/003-GPT-XL/checkpoints/eval_step_80000/batch_2_cfg_7.5_topk_16384.jpg @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:483745fcca35131d1b3b435f2400cee3f46989859f2219c91c9e22c11c5ef54d +size 431508 diff --git a/CKPTS/X2I_UltraEdiit_fourmask_LLava_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_filtered_objects_4mask_1280_context/003-GPT-XL/checkpoints/eval_step_80000/batch_3_cfg_7.5_topk_16384.jpg b/CKPTS/X2I_UltraEdiit_fourmask_LLava_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_filtered_objects_4mask_1280_context/003-GPT-XL/checkpoints/eval_step_80000/batch_3_cfg_7.5_topk_16384.jpg new file mode 100644 index 0000000000000000000000000000000000000000..7b6ebb23efe64551627f3c87915d41911d43c540 --- /dev/null +++ b/CKPTS/X2I_UltraEdiit_fourmask_LLava_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_filtered_objects_4mask_1280_context/003-GPT-XL/checkpoints/eval_step_80000/batch_3_cfg_7.5_topk_16384.jpg @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:5dc6b5b3257cb5c15dc8a544cc431f260fd841d2e40ccf81e513e4f156ed67ce +size 386576 diff --git a/CKPTS/X2I_UltraEdiit_fourmask_LLava_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_filtered_objects_4mask_1280_context/003-GPT-XL/checkpoints/eval_step_80000/batch_4_cfg_7.5_topk_16384.jpg b/CKPTS/X2I_UltraEdiit_fourmask_LLava_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_filtered_objects_4mask_1280_context/003-GPT-XL/checkpoints/eval_step_80000/batch_4_cfg_7.5_topk_16384.jpg new file mode 100644 index 0000000000000000000000000000000000000000..be822114b8477fb744b4803abeabed67b80b32f3 --- /dev/null +++ b/CKPTS/X2I_UltraEdiit_fourmask_LLava_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_filtered_objects_4mask_1280_context/003-GPT-XL/checkpoints/eval_step_80000/batch_4_cfg_7.5_topk_16384.jpg @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:9eb40d6ebfd3d2f3c233f0e2df4bfa62813d4142d5cab7a817cf3ce1eedc063e +size 351294 diff --git a/CKPTS/X2I_UltraEdiit_fourmask_LLava_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_filtered_objects_4mask_1280_context/003-GPT-XL/checkpoints/eval_step_82000/batch_0_cfg_7.5_topk_16384.jpg b/CKPTS/X2I_UltraEdiit_fourmask_LLava_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_filtered_objects_4mask_1280_context/003-GPT-XL/checkpoints/eval_step_82000/batch_0_cfg_7.5_topk_16384.jpg new file mode 100644 index 0000000000000000000000000000000000000000..e35197c9f2cda55264314c3abf9da1688024b591 --- /dev/null +++ b/CKPTS/X2I_UltraEdiit_fourmask_LLava_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_filtered_objects_4mask_1280_context/003-GPT-XL/checkpoints/eval_step_82000/batch_0_cfg_7.5_topk_16384.jpg @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:0500264ecc95c181436f85f00632feea37692a55c834f32709bb99784db76aad +size 464477 diff --git a/CKPTS/X2I_UltraEdiit_fourmask_LLava_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_filtered_objects_4mask_1280_context/003-GPT-XL/checkpoints/eval_step_82000/batch_1_cfg_7.5_topk_16384.jpg b/CKPTS/X2I_UltraEdiit_fourmask_LLava_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_filtered_objects_4mask_1280_context/003-GPT-XL/checkpoints/eval_step_82000/batch_1_cfg_7.5_topk_16384.jpg new file mode 100644 index 0000000000000000000000000000000000000000..3fec88d1f5b7ab39e228a04edc3a3f0e3966c2c1 --- /dev/null +++ b/CKPTS/X2I_UltraEdiit_fourmask_LLava_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_filtered_objects_4mask_1280_context/003-GPT-XL/checkpoints/eval_step_82000/batch_1_cfg_7.5_topk_16384.jpg @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:30db0a119d8c4816e990feab831d1f6447b47a48d17d0e9019ce4b997299d7f7 +size 430930 diff --git a/CKPTS/X2I_UltraEdiit_fourmask_LLava_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_filtered_objects_4mask_1280_context/003-GPT-XL/checkpoints/eval_step_82000/batch_2_cfg_7.5_topk_16384.jpg b/CKPTS/X2I_UltraEdiit_fourmask_LLava_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_filtered_objects_4mask_1280_context/003-GPT-XL/checkpoints/eval_step_82000/batch_2_cfg_7.5_topk_16384.jpg new file mode 100644 index 0000000000000000000000000000000000000000..7c00124fba95d757b4b0dea788e2e5b2aa49084c --- /dev/null +++ b/CKPTS/X2I_UltraEdiit_fourmask_LLava_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_filtered_objects_4mask_1280_context/003-GPT-XL/checkpoints/eval_step_82000/batch_2_cfg_7.5_topk_16384.jpg @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:15e66581b26f3fcfe63fbf676425227b3aa747209ae6b41795950e50417c8f70 +size 423184 diff --git a/CKPTS/X2I_UltraEdiit_fourmask_LLava_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_filtered_objects_4mask_1280_context/003-GPT-XL/checkpoints/eval_step_82000/batch_3_cfg_7.5_topk_16384.jpg b/CKPTS/X2I_UltraEdiit_fourmask_LLava_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_filtered_objects_4mask_1280_context/003-GPT-XL/checkpoints/eval_step_82000/batch_3_cfg_7.5_topk_16384.jpg new file mode 100644 index 0000000000000000000000000000000000000000..908c3d6eb588d4e00b6f051408583a6026956c9f --- /dev/null +++ b/CKPTS/X2I_UltraEdiit_fourmask_LLava_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_filtered_objects_4mask_1280_context/003-GPT-XL/checkpoints/eval_step_82000/batch_3_cfg_7.5_topk_16384.jpg @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:4f279ffbd6ba958e556f5ef9b1257ea2518d6e63b21302436935c59f40d06cb6 +size 399811 diff --git a/CKPTS/X2I_UltraEdiit_fourmask_LLava_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_filtered_objects_4mask_1280_context/003-GPT-XL/checkpoints/eval_step_82000/batch_4_cfg_7.5_topk_16384.jpg b/CKPTS/X2I_UltraEdiit_fourmask_LLava_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_filtered_objects_4mask_1280_context/003-GPT-XL/checkpoints/eval_step_82000/batch_4_cfg_7.5_topk_16384.jpg new file mode 100644 index 0000000000000000000000000000000000000000..d4dd6469be8565fadf63ea7504af37b0336eaa22 --- /dev/null +++ b/CKPTS/X2I_UltraEdiit_fourmask_LLava_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_filtered_objects_4mask_1280_context/003-GPT-XL/checkpoints/eval_step_82000/batch_4_cfg_7.5_topk_16384.jpg @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:bf5dd19de8495aaec97d84f2d67fb454c00d477dbf655a6fc579f809af031a5f +size 340939 diff --git a/CKPTS/X2I_UltraEdiit_fourmask_LLava_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_filtered_objects_4mask_1280_context/003-GPT-XL/checkpoints/eval_step_84000/batch_0_cfg_7.5_topk_16384.jpg b/CKPTS/X2I_UltraEdiit_fourmask_LLava_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_filtered_objects_4mask_1280_context/003-GPT-XL/checkpoints/eval_step_84000/batch_0_cfg_7.5_topk_16384.jpg new file mode 100644 index 0000000000000000000000000000000000000000..cbdd9dc1bfb2d2056b35311c56397a86c80170af --- /dev/null +++ b/CKPTS/X2I_UltraEdiit_fourmask_LLava_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_filtered_objects_4mask_1280_context/003-GPT-XL/checkpoints/eval_step_84000/batch_0_cfg_7.5_topk_16384.jpg @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:03268a54317730eb90f22110b02ffc389b21896fe7b592185c463f3e1d324048 +size 450308 diff --git a/CKPTS/X2I_UltraEdiit_fourmask_LLava_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_filtered_objects_4mask_1280_context/003-GPT-XL/checkpoints/eval_step_84000/batch_1_cfg_7.5_topk_16384.jpg b/CKPTS/X2I_UltraEdiit_fourmask_LLava_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_filtered_objects_4mask_1280_context/003-GPT-XL/checkpoints/eval_step_84000/batch_1_cfg_7.5_topk_16384.jpg new file mode 100644 index 0000000000000000000000000000000000000000..4c0870b39cb84ea586ae35c37e363be47f459785 --- /dev/null +++ b/CKPTS/X2I_UltraEdiit_fourmask_LLava_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_filtered_objects_4mask_1280_context/003-GPT-XL/checkpoints/eval_step_84000/batch_1_cfg_7.5_topk_16384.jpg @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:b057de147e6553863f10dfc7ce6e5ef555a82b3f96f22b6afe4afeacc1733ba9 +size 421848 diff --git a/CKPTS/X2I_UltraEdiit_fourmask_LLava_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_filtered_objects_4mask_1280_context/003-GPT-XL/checkpoints/eval_step_84000/batch_2_cfg_7.5_topk_16384.jpg b/CKPTS/X2I_UltraEdiit_fourmask_LLava_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_filtered_objects_4mask_1280_context/003-GPT-XL/checkpoints/eval_step_84000/batch_2_cfg_7.5_topk_16384.jpg new file mode 100644 index 0000000000000000000000000000000000000000..e80c848064b3913f60ea621570e2cdba2db2917d --- /dev/null +++ b/CKPTS/X2I_UltraEdiit_fourmask_LLava_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_filtered_objects_4mask_1280_context/003-GPT-XL/checkpoints/eval_step_84000/batch_2_cfg_7.5_topk_16384.jpg @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:34c0ec636bd6b07ff561f24ed77ad5a27e06ec93ff835288edb343ebd947f531 +size 416414 diff --git a/CKPTS/X2I_UltraEdiit_fourmask_LLava_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_filtered_objects_4mask_1280_context/003-GPT-XL/checkpoints/eval_step_84000/batch_3_cfg_7.5_topk_16384.jpg b/CKPTS/X2I_UltraEdiit_fourmask_LLava_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_filtered_objects_4mask_1280_context/003-GPT-XL/checkpoints/eval_step_84000/batch_3_cfg_7.5_topk_16384.jpg new file mode 100644 index 0000000000000000000000000000000000000000..051398aeac68c5062243a57b6687e252244cc5bd --- /dev/null +++ b/CKPTS/X2I_UltraEdiit_fourmask_LLava_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_filtered_objects_4mask_1280_context/003-GPT-XL/checkpoints/eval_step_84000/batch_3_cfg_7.5_topk_16384.jpg @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:f28f495c3484b9f91d0c95c6f829d3259737bdeebfa3327111e1a1791fc156d2 +size 394162 diff --git a/CKPTS/X2I_UltraEdiit_fourmask_LLava_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_filtered_objects_4mask_1280_context/003-GPT-XL/checkpoints/eval_step_84000/batch_4_cfg_7.5_topk_16384.jpg b/CKPTS/X2I_UltraEdiit_fourmask_LLava_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_filtered_objects_4mask_1280_context/003-GPT-XL/checkpoints/eval_step_84000/batch_4_cfg_7.5_topk_16384.jpg new file mode 100644 index 0000000000000000000000000000000000000000..5def425ae0d741a588f2614f9532c37c48925f18 --- /dev/null +++ b/CKPTS/X2I_UltraEdiit_fourmask_LLava_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_filtered_objects_4mask_1280_context/003-GPT-XL/checkpoints/eval_step_84000/batch_4_cfg_7.5_topk_16384.jpg @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:2f8ceef78e81fc3c8de6e88912d05cfa8294f2fc2328b7fdb4fe0b5992c36ed1 +size 343109 diff --git a/CKPTS/X2I_UltraEdiit_fourmask_LLava_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_filtered_objects_4mask_1280_context/003-GPT-XL/checkpoints/eval_step_86000/batch_0_cfg_7.5_topk_16384.jpg b/CKPTS/X2I_UltraEdiit_fourmask_LLava_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_filtered_objects_4mask_1280_context/003-GPT-XL/checkpoints/eval_step_86000/batch_0_cfg_7.5_topk_16384.jpg new file mode 100644 index 0000000000000000000000000000000000000000..19c274d22bf5acf6c2d3de266ce523a30fbcfa43 --- /dev/null +++ b/CKPTS/X2I_UltraEdiit_fourmask_LLava_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_filtered_objects_4mask_1280_context/003-GPT-XL/checkpoints/eval_step_86000/batch_0_cfg_7.5_topk_16384.jpg @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:63e3ba5edb105e630e2973656fd3e2230ffb856367653213bd636d5c07a8cc86 +size 427229 diff --git a/CKPTS/X2I_UltraEdiit_fourmask_LLava_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_filtered_objects_4mask_1280_context/003-GPT-XL/checkpoints/eval_step_86000/batch_1_cfg_7.5_topk_16384.jpg b/CKPTS/X2I_UltraEdiit_fourmask_LLava_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_filtered_objects_4mask_1280_context/003-GPT-XL/checkpoints/eval_step_86000/batch_1_cfg_7.5_topk_16384.jpg new file mode 100644 index 0000000000000000000000000000000000000000..581fabcbbe6d668d660c5b01b72995b9844b3285 --- /dev/null +++ b/CKPTS/X2I_UltraEdiit_fourmask_LLava_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_filtered_objects_4mask_1280_context/003-GPT-XL/checkpoints/eval_step_86000/batch_1_cfg_7.5_topk_16384.jpg @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:7070afe8ba1a7b9de3bb96deaaf22841f18c07c494b1fb748aaab59b3734fa98 +size 439075 diff --git a/CKPTS/X2I_UltraEdiit_fourmask_LLava_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_filtered_objects_4mask_1280_context/003-GPT-XL/checkpoints/eval_step_86000/batch_2_cfg_7.5_topk_16384.jpg b/CKPTS/X2I_UltraEdiit_fourmask_LLava_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_filtered_objects_4mask_1280_context/003-GPT-XL/checkpoints/eval_step_86000/batch_2_cfg_7.5_topk_16384.jpg new file mode 100644 index 0000000000000000000000000000000000000000..f763f64f55c35065b83eb58ce716fbb1edb820da --- /dev/null +++ b/CKPTS/X2I_UltraEdiit_fourmask_LLava_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_filtered_objects_4mask_1280_context/003-GPT-XL/checkpoints/eval_step_86000/batch_2_cfg_7.5_topk_16384.jpg @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:3214470f73edd3b45707beff0003d8e6bacc833dd0e715d5f057a9ce4bc4ddcd +size 433020 diff --git a/CKPTS/X2I_UltraEdiit_fourmask_LLava_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_filtered_objects_4mask_1280_context/003-GPT-XL/checkpoints/eval_step_86000/batch_3_cfg_7.5_topk_16384.jpg b/CKPTS/X2I_UltraEdiit_fourmask_LLava_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_filtered_objects_4mask_1280_context/003-GPT-XL/checkpoints/eval_step_86000/batch_3_cfg_7.5_topk_16384.jpg new file mode 100644 index 0000000000000000000000000000000000000000..739f8d29c0e816bf2e2f43bce2e3c5a8b410b869 --- /dev/null +++ b/CKPTS/X2I_UltraEdiit_fourmask_LLava_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_filtered_objects_4mask_1280_context/003-GPT-XL/checkpoints/eval_step_86000/batch_3_cfg_7.5_topk_16384.jpg @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:f584220d63fa5c95c3caf61219a1ecc523c4893d4d45bb6abb6463ac4c3948ac +size 400707 diff --git a/CKPTS/X2I_UltraEdiit_fourmask_LLava_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_filtered_objects_4mask_1280_context/003-GPT-XL/checkpoints/eval_step_86000/batch_4_cfg_7.5_topk_16384.jpg b/CKPTS/X2I_UltraEdiit_fourmask_LLava_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_filtered_objects_4mask_1280_context/003-GPT-XL/checkpoints/eval_step_86000/batch_4_cfg_7.5_topk_16384.jpg new file mode 100644 index 0000000000000000000000000000000000000000..e34b02f4f946020c7b05b3c3b940b620fec5d1f2 --- /dev/null +++ b/CKPTS/X2I_UltraEdiit_fourmask_LLava_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_filtered_objects_4mask_1280_context/003-GPT-XL/checkpoints/eval_step_86000/batch_4_cfg_7.5_topk_16384.jpg @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:600ad763e94e35a63413b2bcf1a837d60633a479131d0b02edf56d3a5d6f7967 +size 349655 diff --git a/CKPTS/X2I_UltraEdiit_fourmask_LLava_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_filtered_objects_4mask_1280_context/003-GPT-XL/checkpoints/eval_step_88000/batch_0_cfg_7.5_topk_16384.jpg b/CKPTS/X2I_UltraEdiit_fourmask_LLava_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_filtered_objects_4mask_1280_context/003-GPT-XL/checkpoints/eval_step_88000/batch_0_cfg_7.5_topk_16384.jpg new file mode 100644 index 0000000000000000000000000000000000000000..4faf02caaa59299209bd5fd62bfef637476459e0 --- /dev/null +++ b/CKPTS/X2I_UltraEdiit_fourmask_LLava_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_filtered_objects_4mask_1280_context/003-GPT-XL/checkpoints/eval_step_88000/batch_0_cfg_7.5_topk_16384.jpg @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:d622498f2647998c3a991f6ee4e89e9e954b586bc77d3c660e3f8a75a39dada9 +size 432717 diff --git a/CKPTS/X2I_UltraEdiit_fourmask_LLava_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_filtered_objects_4mask_1280_context/003-GPT-XL/checkpoints/eval_step_88000/batch_1_cfg_7.5_topk_16384.jpg b/CKPTS/X2I_UltraEdiit_fourmask_LLava_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_filtered_objects_4mask_1280_context/003-GPT-XL/checkpoints/eval_step_88000/batch_1_cfg_7.5_topk_16384.jpg new file mode 100644 index 0000000000000000000000000000000000000000..bb5f0dc5cdf822feb10cfa1320644ec7eb85121b --- /dev/null +++ b/CKPTS/X2I_UltraEdiit_fourmask_LLava_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_filtered_objects_4mask_1280_context/003-GPT-XL/checkpoints/eval_step_88000/batch_1_cfg_7.5_topk_16384.jpg @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:a84dfa4077fdc430464a7c1fcbd801dc6eb3a06791836793ce89d79ec395821e +size 424354 diff --git a/CKPTS/X2I_UltraEdiit_fourmask_LLava_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_filtered_objects_4mask_1280_context/003-GPT-XL/checkpoints/eval_step_88000/batch_2_cfg_7.5_topk_16384.jpg b/CKPTS/X2I_UltraEdiit_fourmask_LLava_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_filtered_objects_4mask_1280_context/003-GPT-XL/checkpoints/eval_step_88000/batch_2_cfg_7.5_topk_16384.jpg new file mode 100644 index 0000000000000000000000000000000000000000..ec2151decac66f1e0c0d160c6409ad00d6422ae6 --- /dev/null +++ b/CKPTS/X2I_UltraEdiit_fourmask_LLava_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_filtered_objects_4mask_1280_context/003-GPT-XL/checkpoints/eval_step_88000/batch_2_cfg_7.5_topk_16384.jpg @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:a9447f797152ef6a887651d7fd4c548c97f494fcd62c736742b4f533d6dfb06b +size 421776 diff --git a/CKPTS/X2I_UltraEdiit_fourmask_LLava_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_filtered_objects_4mask_1280_context/003-GPT-XL/checkpoints/eval_step_88000/batch_3_cfg_7.5_topk_16384.jpg b/CKPTS/X2I_UltraEdiit_fourmask_LLava_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_filtered_objects_4mask_1280_context/003-GPT-XL/checkpoints/eval_step_88000/batch_3_cfg_7.5_topk_16384.jpg new file mode 100644 index 0000000000000000000000000000000000000000..f4629ace47ada642df74f10b46f090102d266bd1 --- /dev/null +++ b/CKPTS/X2I_UltraEdiit_fourmask_LLava_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_filtered_objects_4mask_1280_context/003-GPT-XL/checkpoints/eval_step_88000/batch_3_cfg_7.5_topk_16384.jpg @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:9b509a5146b2ad6acb62167f098f504d9b480386caf981f1849434b5d76532ac +size 384743 diff --git a/CKPTS/X2I_UltraEdiit_fourmask_LLava_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_filtered_objects_4mask_1280_context/003-GPT-XL/checkpoints/eval_step_88000/batch_4_cfg_7.5_topk_16384.jpg b/CKPTS/X2I_UltraEdiit_fourmask_LLava_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_filtered_objects_4mask_1280_context/003-GPT-XL/checkpoints/eval_step_88000/batch_4_cfg_7.5_topk_16384.jpg new file mode 100644 index 0000000000000000000000000000000000000000..186411cbca10dd7a543f9d96ae0413abaa68f9f4 --- /dev/null +++ b/CKPTS/X2I_UltraEdiit_fourmask_LLava_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_filtered_objects_4mask_1280_context/003-GPT-XL/checkpoints/eval_step_88000/batch_4_cfg_7.5_topk_16384.jpg @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:e85bd70a857e86023bd92a11c91547fcd121171d6cbdc9c60397cfefc8d46e0f +size 348383 diff --git a/CKPTS/X2I_UltraEdiit_fourmask_LLava_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_filtered_objects_4mask_1280_context/003-GPT-XL/checkpoints/eval_step_90000/batch_0_cfg_7.5_topk_16384.jpg b/CKPTS/X2I_UltraEdiit_fourmask_LLava_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_filtered_objects_4mask_1280_context/003-GPT-XL/checkpoints/eval_step_90000/batch_0_cfg_7.5_topk_16384.jpg new file mode 100644 index 0000000000000000000000000000000000000000..6f9fbeea8e8acbc2bc8c9c013c9251763c0139da --- /dev/null +++ b/CKPTS/X2I_UltraEdiit_fourmask_LLava_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_filtered_objects_4mask_1280_context/003-GPT-XL/checkpoints/eval_step_90000/batch_0_cfg_7.5_topk_16384.jpg @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:4530b657bff994c545af6b2e3abc04f626665217692e188d4ed04f1dd595cf16 +size 437532 diff --git a/CKPTS/X2I_UltraEdiit_fourmask_LLava_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_filtered_objects_4mask_1280_context/003-GPT-XL/checkpoints/eval_step_90000/batch_1_cfg_7.5_topk_16384.jpg b/CKPTS/X2I_UltraEdiit_fourmask_LLava_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_filtered_objects_4mask_1280_context/003-GPT-XL/checkpoints/eval_step_90000/batch_1_cfg_7.5_topk_16384.jpg new file mode 100644 index 0000000000000000000000000000000000000000..714e007baa0bb96ed847fc2695ab029a9b99fbbd --- /dev/null +++ b/CKPTS/X2I_UltraEdiit_fourmask_LLava_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_filtered_objects_4mask_1280_context/003-GPT-XL/checkpoints/eval_step_90000/batch_1_cfg_7.5_topk_16384.jpg @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:1bd5418ca2e73d3ed53ee63414760befac39ac50b35b7bd89def33863d86ddc7 +size 419487 diff --git a/CKPTS/X2I_UltraEdiit_fourmask_LLava_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_filtered_objects_4mask_1280_context/003-GPT-XL/checkpoints/eval_step_90000/batch_2_cfg_7.5_topk_16384.jpg b/CKPTS/X2I_UltraEdiit_fourmask_LLava_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_filtered_objects_4mask_1280_context/003-GPT-XL/checkpoints/eval_step_90000/batch_2_cfg_7.5_topk_16384.jpg new file mode 100644 index 0000000000000000000000000000000000000000..f18e151c1a2825265d6fe5b72a2b054d66b5b87c --- /dev/null +++ b/CKPTS/X2I_UltraEdiit_fourmask_LLava_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_filtered_objects_4mask_1280_context/003-GPT-XL/checkpoints/eval_step_90000/batch_2_cfg_7.5_topk_16384.jpg @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:82df93981e6f731e0bca43d4dde1f78078b4938d6f70664cba3b5b76e0149d3b +size 438372 diff --git a/CKPTS/X2I_UltraEdiit_fourmask_LLava_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_filtered_objects_4mask_1280_context/003-GPT-XL/checkpoints/eval_step_90000/batch_3_cfg_7.5_topk_16384.jpg b/CKPTS/X2I_UltraEdiit_fourmask_LLava_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_filtered_objects_4mask_1280_context/003-GPT-XL/checkpoints/eval_step_90000/batch_3_cfg_7.5_topk_16384.jpg new file mode 100644 index 0000000000000000000000000000000000000000..8dd1fde159805d5a77331361a556641928e9f582 --- /dev/null +++ b/CKPTS/X2I_UltraEdiit_fourmask_LLava_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_filtered_objects_4mask_1280_context/003-GPT-XL/checkpoints/eval_step_90000/batch_3_cfg_7.5_topk_16384.jpg @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:4c26c07791485092028a8e3f7a2c770be4aa9ae36a766d774dcf440e4d3a5a89 +size 396493 diff --git a/CKPTS/X2I_UltraEdiit_fourmask_LLava_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_filtered_objects_4mask_1280_context/003-GPT-XL/checkpoints/eval_step_90000/batch_4_cfg_7.5_topk_16384.jpg b/CKPTS/X2I_UltraEdiit_fourmask_LLava_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_filtered_objects_4mask_1280_context/003-GPT-XL/checkpoints/eval_step_90000/batch_4_cfg_7.5_topk_16384.jpg new file mode 100644 index 0000000000000000000000000000000000000000..0b1456ff6e6f7bb07d0e994ea885a7571b6ba136 --- /dev/null +++ b/CKPTS/X2I_UltraEdiit_fourmask_LLava_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_filtered_objects_4mask_1280_context/003-GPT-XL/checkpoints/eval_step_90000/batch_4_cfg_7.5_topk_16384.jpg @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:d431d49e1b908252f38616c6f301c24fadd2c2a8384007eca8ccc013f886cc93 +size 342643 diff --git a/CKPTS/X2I_UltraEdiit_fourmask_LLava_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_filtered_objects_4mask_1280_context/003-GPT-XL/checkpoints/eval_step_92000/batch_0_cfg_7.5_topk_16384.jpg b/CKPTS/X2I_UltraEdiit_fourmask_LLava_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_filtered_objects_4mask_1280_context/003-GPT-XL/checkpoints/eval_step_92000/batch_0_cfg_7.5_topk_16384.jpg new file mode 100644 index 0000000000000000000000000000000000000000..55ce998502d9c540bf5ba34a3235391e9813eff4 --- /dev/null +++ b/CKPTS/X2I_UltraEdiit_fourmask_LLava_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_filtered_objects_4mask_1280_context/003-GPT-XL/checkpoints/eval_step_92000/batch_0_cfg_7.5_topk_16384.jpg @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:cacc9a0866a0697fe70041ddd83b1b5aea891d56633b5e358e2d5aa338c93735 +size 436889 diff --git a/CKPTS/X2I_UltraEdiit_fourmask_LLava_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_filtered_objects_4mask_1280_context/003-GPT-XL/checkpoints/eval_step_92000/batch_1_cfg_7.5_topk_16384.jpg b/CKPTS/X2I_UltraEdiit_fourmask_LLava_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_filtered_objects_4mask_1280_context/003-GPT-XL/checkpoints/eval_step_92000/batch_1_cfg_7.5_topk_16384.jpg new file mode 100644 index 0000000000000000000000000000000000000000..340fe3e000ed871b9556c520542fd40deb3c92bb --- /dev/null +++ b/CKPTS/X2I_UltraEdiit_fourmask_LLava_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_filtered_objects_4mask_1280_context/003-GPT-XL/checkpoints/eval_step_92000/batch_1_cfg_7.5_topk_16384.jpg @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:b8fa74e0b91b2b42035b85e15b5a2d00a165c4c12894a47f098113ee8f5aedc3 +size 435215 diff --git a/CKPTS/X2I_UltraEdiit_fourmask_LLava_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_filtered_objects_4mask_1280_context/003-GPT-XL/checkpoints/eval_step_92000/batch_2_cfg_7.5_topk_16384.jpg b/CKPTS/X2I_UltraEdiit_fourmask_LLava_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_filtered_objects_4mask_1280_context/003-GPT-XL/checkpoints/eval_step_92000/batch_2_cfg_7.5_topk_16384.jpg new file mode 100644 index 0000000000000000000000000000000000000000..3e142a9faf6eefd01c69dce966eacc37f3dc9454 --- /dev/null +++ b/CKPTS/X2I_UltraEdiit_fourmask_LLava_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_filtered_objects_4mask_1280_context/003-GPT-XL/checkpoints/eval_step_92000/batch_2_cfg_7.5_topk_16384.jpg @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:382d67bbf5aee762bd47952571a0c9e04d224167b3ff861f0fe60d7742fb81d0 +size 420558 diff --git a/CKPTS/X2I_UltraEdiit_fourmask_LLava_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_filtered_objects_4mask_1280_context/003-GPT-XL/checkpoints/eval_step_92000/batch_3_cfg_7.5_topk_16384.jpg b/CKPTS/X2I_UltraEdiit_fourmask_LLava_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_filtered_objects_4mask_1280_context/003-GPT-XL/checkpoints/eval_step_92000/batch_3_cfg_7.5_topk_16384.jpg new file mode 100644 index 0000000000000000000000000000000000000000..2e0d03a5f2b689d06a790bc3848ffc3dc73ad602 --- /dev/null +++ b/CKPTS/X2I_UltraEdiit_fourmask_LLava_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_filtered_objects_4mask_1280_context/003-GPT-XL/checkpoints/eval_step_92000/batch_3_cfg_7.5_topk_16384.jpg @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:fb0d566c861f685ff4ace9e6230434d7e990aa89e9842366ecf769de30dcf5c9 +size 391234 diff --git a/CKPTS/X2I_UltraEdiit_fourmask_LLava_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_filtered_objects_4mask_1280_context/003-GPT-XL/checkpoints/eval_step_92000/batch_4_cfg_7.5_topk_16384.jpg b/CKPTS/X2I_UltraEdiit_fourmask_LLava_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_filtered_objects_4mask_1280_context/003-GPT-XL/checkpoints/eval_step_92000/batch_4_cfg_7.5_topk_16384.jpg new file mode 100644 index 0000000000000000000000000000000000000000..0a521c262d3f785d305a5a2e02709169503c6fe0 --- /dev/null +++ b/CKPTS/X2I_UltraEdiit_fourmask_LLava_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_filtered_objects_4mask_1280_context/003-GPT-XL/checkpoints/eval_step_92000/batch_4_cfg_7.5_topk_16384.jpg @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:8627d19d4da4a47688bce053986b520364bad9ed62b088bd2e2aad33b533ae23 +size 353679 diff --git a/CKPTS/X2I_UltraEdiit_fourmask_LLava_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_filtered_objects_4mask_1280_context/003-GPT-XL/checkpoints/eval_step_94000/batch_0_cfg_7.5_topk_16384.jpg b/CKPTS/X2I_UltraEdiit_fourmask_LLava_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_filtered_objects_4mask_1280_context/003-GPT-XL/checkpoints/eval_step_94000/batch_0_cfg_7.5_topk_16384.jpg new file mode 100644 index 0000000000000000000000000000000000000000..f07f913a8b92f6c4d920d3abac9454b2fac5dd00 --- /dev/null +++ b/CKPTS/X2I_UltraEdiit_fourmask_LLava_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_filtered_objects_4mask_1280_context/003-GPT-XL/checkpoints/eval_step_94000/batch_0_cfg_7.5_topk_16384.jpg @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:1d6f7557588258c70f60a6e28602ed0ee4f1191f13624e2b1365c39888019699 +size 460019 diff --git a/CKPTS/X2I_UltraEdiit_fourmask_LLava_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_filtered_objects_4mask_1280_context/003-GPT-XL/checkpoints/eval_step_94000/batch_1_cfg_7.5_topk_16384.jpg b/CKPTS/X2I_UltraEdiit_fourmask_LLava_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_filtered_objects_4mask_1280_context/003-GPT-XL/checkpoints/eval_step_94000/batch_1_cfg_7.5_topk_16384.jpg new file mode 100644 index 0000000000000000000000000000000000000000..56c2373e83d2b61e2bb00fae6a8804100348b8ae --- /dev/null +++ b/CKPTS/X2I_UltraEdiit_fourmask_LLava_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_filtered_objects_4mask_1280_context/003-GPT-XL/checkpoints/eval_step_94000/batch_1_cfg_7.5_topk_16384.jpg @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:f9033285d1a85a7bec652bc6c6efdf6b6d60c1d337a934ddc26a1b5b2f293807 +size 412074 diff --git a/CKPTS/X2I_UltraEdiit_fourmask_LLava_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_filtered_objects_4mask_1280_context/003-GPT-XL/checkpoints/eval_step_94000/batch_2_cfg_7.5_topk_16384.jpg b/CKPTS/X2I_UltraEdiit_fourmask_LLava_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_filtered_objects_4mask_1280_context/003-GPT-XL/checkpoints/eval_step_94000/batch_2_cfg_7.5_topk_16384.jpg new file mode 100644 index 0000000000000000000000000000000000000000..09a4636fb218dfb1c6019671da099d247505a11e --- /dev/null +++ b/CKPTS/X2I_UltraEdiit_fourmask_LLava_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_filtered_objects_4mask_1280_context/003-GPT-XL/checkpoints/eval_step_94000/batch_2_cfg_7.5_topk_16384.jpg @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:4a616231b78d941359e33a34393e385aa363ac6bafadec09877b5cae217f8b96 +size 427246 diff --git a/CKPTS/X2I_UltraEdiit_fourmask_LLava_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_filtered_objects_4mask_1280_context/003-GPT-XL/checkpoints/eval_step_94000/batch_3_cfg_7.5_topk_16384.jpg b/CKPTS/X2I_UltraEdiit_fourmask_LLava_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_filtered_objects_4mask_1280_context/003-GPT-XL/checkpoints/eval_step_94000/batch_3_cfg_7.5_topk_16384.jpg new file mode 100644 index 0000000000000000000000000000000000000000..13bc0445703ef18fe9063440869f6a8866fbe694 --- /dev/null +++ b/CKPTS/X2I_UltraEdiit_fourmask_LLava_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_filtered_objects_4mask_1280_context/003-GPT-XL/checkpoints/eval_step_94000/batch_3_cfg_7.5_topk_16384.jpg @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:0a27ff18f3460b36afe07af07cb886d664283e6ead2d3a4738e12821e1cf2867 +size 380343 diff --git a/CKPTS/X2I_UltraEdiit_fourmask_LLava_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_filtered_objects_4mask_1280_context/003-GPT-XL/checkpoints/eval_step_94000/batch_4_cfg_7.5_topk_16384.jpg b/CKPTS/X2I_UltraEdiit_fourmask_LLava_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_filtered_objects_4mask_1280_context/003-GPT-XL/checkpoints/eval_step_94000/batch_4_cfg_7.5_topk_16384.jpg new file mode 100644 index 0000000000000000000000000000000000000000..412c43e4fd465c6909a594977cf43b8ec508e955 --- /dev/null +++ b/CKPTS/X2I_UltraEdiit_fourmask_LLava_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_filtered_objects_4mask_1280_context/003-GPT-XL/checkpoints/eval_step_94000/batch_4_cfg_7.5_topk_16384.jpg @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:c18a7289ea15ec92b29253802bd2d5ff77eb8297aaf0e8a4a9d0f715c6c45f75 +size 346089 diff --git a/CKPTS/X2I_UltraEdiit_fourmask_LLava_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_filtered_objects_4mask_1280_context/003-GPT-XL/checkpoints/eval_step_96000/batch_0_cfg_7.5_topk_16384.jpg b/CKPTS/X2I_UltraEdiit_fourmask_LLava_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_filtered_objects_4mask_1280_context/003-GPT-XL/checkpoints/eval_step_96000/batch_0_cfg_7.5_topk_16384.jpg new file mode 100644 index 0000000000000000000000000000000000000000..d15d476ca07fe2eb930e4878fb773e6af6f8aee3 --- /dev/null +++ b/CKPTS/X2I_UltraEdiit_fourmask_LLava_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_filtered_objects_4mask_1280_context/003-GPT-XL/checkpoints/eval_step_96000/batch_0_cfg_7.5_topk_16384.jpg @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:8702bc0d5861516640289313847770c11d9d6f55277081ea0c685201c43969af +size 445604 diff --git a/CKPTS/X2I_UltraEdiit_fourmask_LLava_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_filtered_objects_4mask_1280_context/003-GPT-XL/checkpoints/eval_step_96000/batch_1_cfg_7.5_topk_16384.jpg b/CKPTS/X2I_UltraEdiit_fourmask_LLava_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_filtered_objects_4mask_1280_context/003-GPT-XL/checkpoints/eval_step_96000/batch_1_cfg_7.5_topk_16384.jpg new file mode 100644 index 0000000000000000000000000000000000000000..1988612992aec5f20b1ad47f19e2c18f5eb8edac --- /dev/null +++ b/CKPTS/X2I_UltraEdiit_fourmask_LLava_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_filtered_objects_4mask_1280_context/003-GPT-XL/checkpoints/eval_step_96000/batch_1_cfg_7.5_topk_16384.jpg @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:d78edd59679a005536395869c065dbf6e6e3c24fd768ece7a3796b1573389a4e +size 429774 diff --git a/CKPTS/X2I_UltraEdiit_fourmask_LLava_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_filtered_objects_4mask_1280_context/003-GPT-XL/checkpoints/eval_step_96000/batch_2_cfg_7.5_topk_16384.jpg b/CKPTS/X2I_UltraEdiit_fourmask_LLava_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_filtered_objects_4mask_1280_context/003-GPT-XL/checkpoints/eval_step_96000/batch_2_cfg_7.5_topk_16384.jpg new file mode 100644 index 0000000000000000000000000000000000000000..ed8dfc1aaa99bbac4a58f33aa8e6bc5cceda6492 --- /dev/null +++ b/CKPTS/X2I_UltraEdiit_fourmask_LLava_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_filtered_objects_4mask_1280_context/003-GPT-XL/checkpoints/eval_step_96000/batch_2_cfg_7.5_topk_16384.jpg @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:ae2ec1b816ba3fd4ecca9dd0cad73665ec48a29ea277e2e21d172c0dc8b6cff2 +size 424745 diff --git a/CKPTS/X2I_UltraEdiit_fourmask_LLava_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_filtered_objects_4mask_1280_context/003-GPT-XL/checkpoints/eval_step_96000/batch_3_cfg_7.5_topk_16384.jpg b/CKPTS/X2I_UltraEdiit_fourmask_LLava_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_filtered_objects_4mask_1280_context/003-GPT-XL/checkpoints/eval_step_96000/batch_3_cfg_7.5_topk_16384.jpg new file mode 100644 index 0000000000000000000000000000000000000000..9aa8988efb7337aaeb41c79b4a6a1451b86504e5 --- /dev/null +++ b/CKPTS/X2I_UltraEdiit_fourmask_LLava_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_filtered_objects_4mask_1280_context/003-GPT-XL/checkpoints/eval_step_96000/batch_3_cfg_7.5_topk_16384.jpg @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:9afb8773595a75a6157da95ea5a7e6c360073e5484c61d0de835484d96c9fdfa +size 397041 diff --git a/CKPTS/X2I_UltraEdiit_fourmask_LLava_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_filtered_objects_4mask_1280_context/003-GPT-XL/checkpoints/eval_step_96000/batch_4_cfg_7.5_topk_16384.jpg b/CKPTS/X2I_UltraEdiit_fourmask_LLava_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_filtered_objects_4mask_1280_context/003-GPT-XL/checkpoints/eval_step_96000/batch_4_cfg_7.5_topk_16384.jpg new file mode 100644 index 0000000000000000000000000000000000000000..f3d281670205821cb351a4da52464c48d9ed330f --- /dev/null +++ b/CKPTS/X2I_UltraEdiit_fourmask_LLava_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_filtered_objects_4mask_1280_context/003-GPT-XL/checkpoints/eval_step_96000/batch_4_cfg_7.5_topk_16384.jpg @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:13626dac974fd0bb21f2c6ddda864f38cdc48bb05aed852d15096952152d16c4 +size 347119 diff --git a/CKPTS/X2I_UltraEdiit_fourmask_LLava_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_filtered_objects_4mask_1280_context/003-GPT-XL/checkpoints/eval_step_98000/batch_0_cfg_7.5_topk_16384.jpg b/CKPTS/X2I_UltraEdiit_fourmask_LLava_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_filtered_objects_4mask_1280_context/003-GPT-XL/checkpoints/eval_step_98000/batch_0_cfg_7.5_topk_16384.jpg new file mode 100644 index 0000000000000000000000000000000000000000..1dc702193d83291a318332bd6e51677072a3f97e --- /dev/null +++ b/CKPTS/X2I_UltraEdiit_fourmask_LLava_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_filtered_objects_4mask_1280_context/003-GPT-XL/checkpoints/eval_step_98000/batch_0_cfg_7.5_topk_16384.jpg @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:097dffcd872d621ac3ef04b903aaad8bb71efc6cb879318c22f5e719e9239bd1 +size 437056 diff --git a/CKPTS/X2I_UltraEdiit_fourmask_LLava_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_filtered_objects_4mask_1280_context/003-GPT-XL/checkpoints/eval_step_98000/batch_1_cfg_7.5_topk_16384.jpg b/CKPTS/X2I_UltraEdiit_fourmask_LLava_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_filtered_objects_4mask_1280_context/003-GPT-XL/checkpoints/eval_step_98000/batch_1_cfg_7.5_topk_16384.jpg new file mode 100644 index 0000000000000000000000000000000000000000..681b2bd1a1fc0d49ddb59ec3adb740813981eec0 --- /dev/null +++ b/CKPTS/X2I_UltraEdiit_fourmask_LLava_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_filtered_objects_4mask_1280_context/003-GPT-XL/checkpoints/eval_step_98000/batch_1_cfg_7.5_topk_16384.jpg @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:ac844ed6647b61653effac5d090880dd87fa77eff22806862706e214143675e2 +size 436196 diff --git a/CKPTS/X2I_UltraEdiit_fourmask_LLava_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_filtered_objects_4mask_1280_context/003-GPT-XL/checkpoints/eval_step_98000/batch_2_cfg_7.5_topk_16384.jpg b/CKPTS/X2I_UltraEdiit_fourmask_LLava_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_filtered_objects_4mask_1280_context/003-GPT-XL/checkpoints/eval_step_98000/batch_2_cfg_7.5_topk_16384.jpg new file mode 100644 index 0000000000000000000000000000000000000000..4a1c7a09b532301f33962f263bfc49c9fc123c70 --- /dev/null +++ b/CKPTS/X2I_UltraEdiit_fourmask_LLava_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_filtered_objects_4mask_1280_context/003-GPT-XL/checkpoints/eval_step_98000/batch_2_cfg_7.5_topk_16384.jpg @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:86b5488b9756aba676f155c529d8d8fce5a73ad71def9cb7d8eb5a3912a8143b +size 439239 diff --git a/CKPTS/X2I_UltraEdiit_fourmask_LLava_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_filtered_objects_4mask_1280_context/003-GPT-XL/checkpoints/eval_step_98000/batch_3_cfg_7.5_topk_16384.jpg b/CKPTS/X2I_UltraEdiit_fourmask_LLava_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_filtered_objects_4mask_1280_context/003-GPT-XL/checkpoints/eval_step_98000/batch_3_cfg_7.5_topk_16384.jpg new file mode 100644 index 0000000000000000000000000000000000000000..1772773e1bcf368011570aa67dd93b8d79af0fec --- /dev/null +++ b/CKPTS/X2I_UltraEdiit_fourmask_LLava_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_filtered_objects_4mask_1280_context/003-GPT-XL/checkpoints/eval_step_98000/batch_3_cfg_7.5_topk_16384.jpg @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:6105dbd6b0ac713341c03ef5c550548423d66ad70b745599162ce131457d7fbc +size 394854 diff --git a/CKPTS/X2I_UltraEdiit_fourmask_LLava_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_filtered_objects_4mask_1280_context/003-GPT-XL/checkpoints/eval_step_98000/batch_4_cfg_7.5_topk_16384.jpg b/CKPTS/X2I_UltraEdiit_fourmask_LLava_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_filtered_objects_4mask_1280_context/003-GPT-XL/checkpoints/eval_step_98000/batch_4_cfg_7.5_topk_16384.jpg new file mode 100644 index 0000000000000000000000000000000000000000..759ff0fd15b3540804d8181c4d3b29edf5fedde8 --- /dev/null +++ b/CKPTS/X2I_UltraEdiit_fourmask_LLava_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_filtered_objects_4mask_1280_context/003-GPT-XL/checkpoints/eval_step_98000/batch_4_cfg_7.5_topk_16384.jpg @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:3ad780ad109ab0fee76688ddc28f4c5f1a3d19bf9fd25f622f915f5da9e7d3b9 +size 343233 diff --git a/CKPTS/X2I_UltraEdiit_fourmask_LLava_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_filtered_objects_4mask_1280_context/003-GPT-XL/log.txt b/CKPTS/X2I_UltraEdiit_fourmask_LLava_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_filtered_objects_4mask_1280_context/003-GPT-XL/log.txt new file mode 100644 index 0000000000000000000000000000000000000000..cc11401ac260093b4b073fef93ac0526636ccdd3 --- /dev/null +++ b/CKPTS/X2I_UltraEdiit_fourmask_LLava_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_filtered_objects_4mask_1280_context/003-GPT-XL/log.txt @@ -0,0 +1,4665 @@ +[2025-04-26 21:01:18] Experiment directory created at checkpoint/X2I_UltraEdiit_fourmask_LLava_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_filtered_objects_4mask_1280_context/003-GPT-XL +[2025-04-26 21:01:18] Namespace(data_path='/tmp/haozhezhao/MLLMG/jsonl_data/multiimage_training_for_llava_X2I_UltraEdiit_fourmask.jsonl', cloud_save_path='/tmp/haozhezhao/MLLMG/checkpoint', no_local_save=False, vq_model='VQ-16', vq_ckpt='/tmp/haozhezhao/model/llamagen_t2i/vq_ds16_t2i.pt', codebook_size=16384, codebook_embed_dim=8, gpt_model='GPT-XL', gpt_ckpt='/tmp/haozhezhao/MLLMG/checkpoint/Afteri2i_100kckpt_LLAVAT5_empty_Fixed_just_segment_pretrain_stage3_dreambench__recap_400k_t2i_400k_flux_200k_midjourney_150k_recovery_150k_grounding_100fluxseg_50samseg/005-GPT-XL/checkpoints/0078000.pt', gpt_type='t2i', vocab_size=16384, cls_token_num=1280, dropout_p=0.1, token_dropout_p=0.1, drop_path=0.0, no_compile=False, results_dir='checkpoint/X2I_UltraEdiit_fourmask_LLava_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_filtered_objects_4mask_1280_context', dataset='ti2i', image_size=512, downsample_size=16, num_classes=1000, epochs=1, lr=0.0001, weight_decay=0.05, beta1=0.9, beta2=0.95, max_grad_norm=1.0, global_batch_size=24, global_seed=0, num_workers=4, log_every=25, ckpt_every=2000, gradient_accumulation_steps=8, mixed_precision='bf16', val_data_path='/tmp/haozhezhao/MLLMG/jsonl_data/multiimage_val_for_llava_X2I_UltraEdiit_fourmask_dreambenplus.jsonl', use_vision_tower=True, model_name_or_path='/tmp/haozhezhao/model/blip2-flan-t5-xl', image_place_holder='', processor_path=None, do_eval=True, max_eval_samples=128, train_text_encoder=True, no_left_padding=False, cfg_scale=7.5, top_k=16384, temperature=0.9, top_p=1.0, eval_steps=2000, project_name='llamagen_ti2i', load_from_checkpoint='/tmp/haozhezhao/MLLMG/checkpoint/FIXed_3M_1epoch_step32000.pt', warmup=0.05, lr_decay_style='cosine', lr_decay_ratio=0.1, train_iters=500000, class_dropout_prob=0.1, with_image_only=False, image_only_rate=0.1, stage2=False, subject_driven=True, load_subject_embedding=None, reference_data_path='/tmp/haozhezhao/MLLMG/cc12m_reference_tunnel.jsonl', multimodal_encoder='llava', do_recovery=True, no_replace=False, resume=False, dreambench_eval=True, find_unused_parameters=True, load_visual_encoder=False, continue_stage1=False, replace_subject=False, train_all=True, save_total_limit=1, load_language_projection='/tmp/haozhezhao/MLLMG/llava-v1.5-flant5_fixed-pretrain/mm_projector.bin', mm_vision_tower='openai/clip-vit-large-patch14', load_fixed_llamagen=True, unfreeze_output=False, fix='gpt-empty-fix', rank=0, world_size=8, gpu=0, dist_url='env://', distributed=True, dist_backend='nccl') +[2025-04-26 21:01:18] Starting rank=0, seed=0, world_size=8. +[2025-04-26 21:01:18] text_config is None. Initializing the text config with default values (`OPTConfig`). +[2025-04-26 21:02:23] GPT Parameters: 2,310,680,832 +[2025-04-26 21:02:23] num decayed parameter tensors: 356, with 2,007,303,168 parameters +[2025-04-26 21:02:23] num non-decayed parameter tensors: 124, with 197,888 parameters +[2025-04-26 21:02:23] using fused AdamW: True +[2025-04-26 21:02:56] Dataset contains 2,656,708 images +[2025-04-26 21:02:56] Train iters 110696 , warmup 5534.8, len of loader 110696 +[2025-04-26 21:03:17] ### LOAD pretraining weights from checkpoint: /tmp/haozhezhao/MLLMG/checkpoint/Afteri2i_100kckpt_LLAVAT5_empty_Fixed_just_segment_pretrain_stage3_dreambench__recap_400k_t2i_400k_flux_200k_midjourney_150k_recovery_150k_grounding_100fluxseg_50samseg/005-GPT-XL/checkpoints/0078000.pt +[2025-04-26 21:03:17] Initial state: steps=0, epochs=0 +[2025-04-26 21:03:17] compiling the model... (may take several minutes) +[2025-04-26 21:03:17] freeze the vit +[2025-04-26 21:03:17] ***** total param is 2310680832 ***** +[2025-04-26 21:03:17] ***** total trained param is 2007501056 ***** +[2025-04-26 21:03:24] Training for 1 epochs... +[2025-04-26 21:03:24] Beginning epoch 0... +[2025-04-26 21:08:21] (step=0000025) Train Loss: 6.7529, Train Steps/Sec: 0.08 +[2025-04-26 21:08:48] (step=0000050) Train Loss: 6.8104, Train Steps/Sec: 0.91 +[2025-04-26 21:09:11] (step=0000075) Train Loss: 6.7157, Train Steps/Sec: 1.11 +[2025-04-26 21:09:34] (step=0000100) Train Loss: 6.7176, Train Steps/Sec: 1.07 +[2025-04-26 21:09:57] (step=0000125) Train Loss: 6.7114, Train Steps/Sec: 1.11 +[2025-04-26 21:10:19] (step=0000150) Train Loss: 6.8432, Train Steps/Sec: 1.12 +[2025-04-26 21:10:42] (step=0000175) Train Loss: 6.6707, Train Steps/Sec: 1.12 +[2025-04-26 21:11:04] (step=0000200) Train Loss: 6.7134, Train Steps/Sec: 1.11 +[2025-04-26 21:11:26] (step=0000225) Train Loss: 6.7255, Train Steps/Sec: 1.12 +[2025-04-26 21:11:49] (step=0000250) Train Loss: 6.7367, Train Steps/Sec: 1.12 +[2025-04-26 21:12:11] (step=0000275) Train Loss: 6.6841, Train Steps/Sec: 1.12 +[2025-04-26 21:12:33] (step=0000300) Train Loss: 6.6545, Train Steps/Sec: 1.12 +[2025-04-26 21:12:56] (step=0000325) Train Loss: 6.7202, Train Steps/Sec: 1.12 +[2025-04-26 21:13:18] (step=0000350) Train Loss: 6.7349, Train Steps/Sec: 1.12 +[2025-04-26 21:13:41] (step=0000375) Train Loss: 6.6640, Train Steps/Sec: 1.12 +[2025-04-26 21:14:03] (step=0000400) Train Loss: 6.6282, Train Steps/Sec: 1.11 +[2025-04-26 21:14:25] (step=0000425) Train Loss: 6.7078, Train Steps/Sec: 1.12 +[2025-04-26 21:14:48] (step=0000450) Train Loss: 6.6944, Train Steps/Sec: 1.12 +[2025-04-26 21:15:10] (step=0000475) Train Loss: 6.7030, Train Steps/Sec: 1.12 +[2025-04-26 21:15:32] (step=0000500) Train Loss: 6.6896, Train Steps/Sec: 1.12 +[2025-04-26 21:15:55] (step=0000525) Train Loss: 6.5311, Train Steps/Sec: 1.12 +[2025-04-26 21:16:17] (step=0000550) Train Loss: 6.7427, Train Steps/Sec: 1.12 +[2025-04-26 21:16:39] (step=0000575) Train Loss: 6.6732, Train Steps/Sec: 1.12 +[2025-04-26 21:17:02] (step=0000600) Train Loss: 6.6913, Train Steps/Sec: 1.12 +[2025-04-26 21:17:24] (step=0000625) Train Loss: 6.7470, Train Steps/Sec: 1.12 +[2025-04-26 21:17:47] (step=0000650) Train Loss: 6.6397, Train Steps/Sec: 1.12 +[2025-04-26 21:18:09] (step=0000675) Train Loss: 6.7086, Train Steps/Sec: 1.12 +[2025-04-26 21:18:31] (step=0000700) Train Loss: 6.6428, Train Steps/Sec: 1.12 +[2025-04-26 21:18:54] (step=0000725) Train Loss: 6.6054, Train Steps/Sec: 1.12 +[2025-04-26 21:19:16] (step=0000750) Train Loss: 6.6141, Train Steps/Sec: 1.12 +[2025-04-26 21:19:38] (step=0000775) Train Loss: 6.6391, Train Steps/Sec: 1.12 +[2025-04-26 21:20:01] (step=0000800) Train Loss: 6.5635, Train Steps/Sec: 1.12 +[2025-04-26 21:20:23] (step=0000825) Train Loss: 6.6364, Train Steps/Sec: 1.12 +[2025-04-26 21:20:45] (step=0000850) Train Loss: 6.5850, Train Steps/Sec: 1.12 +[2025-04-26 21:21:08] (step=0000875) Train Loss: 6.6453, Train Steps/Sec: 1.12 +[2025-04-26 21:21:30] (step=0000900) Train Loss: 6.5588, Train Steps/Sec: 1.12 +[2025-04-26 21:21:52] (step=0000925) Train Loss: 6.5758, Train Steps/Sec: 1.12 +[2025-04-26 21:22:15] (step=0000950) Train Loss: 6.5626, Train Steps/Sec: 1.12 +[2025-04-26 21:22:37] (step=0000975) Train Loss: 6.6427, Train Steps/Sec: 1.12 +[2025-04-26 21:22:59] (step=0001000) Train Loss: 6.5863, Train Steps/Sec: 1.11 +[2025-04-26 21:23:22] (step=0001025) Train Loss: 6.6233, Train Steps/Sec: 1.12 +[2025-04-26 21:23:44] (step=0001050) Train Loss: 6.5604, Train Steps/Sec: 1.12 +[2025-04-26 21:24:06] (step=0001075) Train Loss: 6.6190, Train Steps/Sec: 1.12 +[2025-04-26 21:24:29] (step=0001100) Train Loss: 6.5717, Train Steps/Sec: 1.12 +[2025-04-26 21:24:51] (step=0001125) Train Loss: 6.5264, Train Steps/Sec: 1.12 +[2025-04-26 21:25:13] (step=0001150) Train Loss: 6.5451, Train Steps/Sec: 1.12 +[2025-04-26 21:25:36] (step=0001175) Train Loss: 6.5642, Train Steps/Sec: 1.12 +[2025-04-26 21:25:58] (step=0001200) Train Loss: 6.5527, Train Steps/Sec: 1.12 +[2025-04-26 21:26:20] (step=0001225) Train Loss: 6.5606, Train Steps/Sec: 1.12 +[2025-04-26 21:26:43] (step=0001250) Train Loss: 6.5663, Train Steps/Sec: 1.12 +[2025-04-26 21:27:05] (step=0001275) Train Loss: 6.5347, Train Steps/Sec: 1.12 +[2025-04-26 21:27:27] (step=0001300) Train Loss: 6.5207, Train Steps/Sec: 1.12 +[2025-04-26 21:27:50] (step=0001325) Train Loss: 6.5558, Train Steps/Sec: 1.12 +[2025-04-26 21:28:12] (step=0001350) Train Loss: 6.5376, Train Steps/Sec: 1.12 +[2025-04-26 21:28:34] (step=0001375) Train Loss: 6.4822, Train Steps/Sec: 1.12 +[2025-04-26 21:29:02] (step=0001400) Train Loss: 6.5158, Train Steps/Sec: 0.89 +[2025-04-26 21:29:25] (step=0001425) Train Loss: 6.5361, Train Steps/Sec: 1.12 +[2025-04-26 21:29:47] (step=0001450) Train Loss: 6.4931, Train Steps/Sec: 1.12 +[2025-04-26 21:30:09] (step=0001475) Train Loss: 6.5149, Train Steps/Sec: 1.12 +[2025-04-26 21:30:32] (step=0001500) Train Loss: 6.5004, Train Steps/Sec: 1.12 +[2025-04-26 21:30:54] (step=0001525) Train Loss: 6.4929, Train Steps/Sec: 1.12 +[2025-04-26 21:31:17] (step=0001550) Train Loss: 6.3915, Train Steps/Sec: 1.12 +[2025-04-26 21:31:39] (step=0001575) Train Loss: 6.4802, Train Steps/Sec: 1.12 +[2025-04-26 21:32:01] (step=0001600) Train Loss: 6.4339, Train Steps/Sec: 1.11 +[2025-04-26 21:32:24] (step=0001625) Train Loss: 6.4666, Train Steps/Sec: 1.12 +[2025-04-26 21:32:46] (step=0001650) Train Loss: 6.4769, Train Steps/Sec: 1.12 +[2025-04-26 21:33:08] (step=0001675) Train Loss: 6.5604, Train Steps/Sec: 1.12 +[2025-04-26 21:33:31] (step=0001700) Train Loss: 6.4665, Train Steps/Sec: 1.12 +[2025-04-26 21:33:53] (step=0001725) Train Loss: 6.4187, Train Steps/Sec: 1.12 +[2025-04-26 21:34:15] (step=0001750) Train Loss: 6.4122, Train Steps/Sec: 1.12 +[2025-04-26 21:34:38] (step=0001775) Train Loss: 6.4295, Train Steps/Sec: 1.12 +[2025-04-26 21:35:00] (step=0001800) Train Loss: 6.4605, Train Steps/Sec: 1.11 +[2025-04-26 21:35:23] (step=0001825) Train Loss: 6.4944, Train Steps/Sec: 1.12 +[2025-04-26 21:35:45] (step=0001850) Train Loss: 6.4391, Train Steps/Sec: 1.12 +[2025-04-26 21:36:07] (step=0001875) Train Loss: 6.4838, Train Steps/Sec: 1.12 +[2025-04-26 21:36:30] (step=0001900) Train Loss: 6.4389, Train Steps/Sec: 1.12 +[2025-04-26 21:36:52] (step=0001925) Train Loss: 6.4390, Train Steps/Sec: 1.12 +[2025-04-26 21:37:14] (step=0001950) Train Loss: 6.4384, Train Steps/Sec: 1.12 +[2025-04-26 21:37:36] (step=0001975) Train Loss: 6.4260, Train Steps/Sec: 1.12 +[2025-04-26 21:37:59] (step=0002000) Train Loss: 6.4519, Train Steps/Sec: 1.11 +[2025-04-26 21:37:59] text_config is None. Initializing the text config with default values (`OPTConfig`). +[2025-04-26 21:44:04] Finish Eval in 2000 steps... +[2025-04-26 21:44:22] Saved checkpoint to checkpoint/X2I_UltraEdiit_fourmask_LLava_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_filtered_objects_4mask_1280_context/003-GPT-XL/checkpoints/0002000.pt +[2025-04-26 21:44:45] (step=0002025) Train Loss: 6.4037, Train Steps/Sec: 0.06 +[2025-04-26 21:45:08] (step=0002050) Train Loss: 6.4489, Train Steps/Sec: 1.10 +[2025-04-26 21:45:30] (step=0002075) Train Loss: 6.4062, Train Steps/Sec: 1.11 +[2025-04-26 21:45:52] (step=0002100) Train Loss: 6.4107, Train Steps/Sec: 1.12 +[2025-04-26 21:46:15] (step=0002125) Train Loss: 6.4249, Train Steps/Sec: 1.12 +[2025-04-26 21:46:37] (step=0002150) Train Loss: 6.4624, Train Steps/Sec: 1.12 +[2025-04-26 21:47:00] (step=0002175) Train Loss: 6.4005, Train Steps/Sec: 1.12 +[2025-04-26 21:47:22] (step=0002200) Train Loss: 6.4313, Train Steps/Sec: 1.11 +[2025-04-26 21:47:45] (step=0002225) Train Loss: 6.3851, Train Steps/Sec: 1.12 +[2025-04-26 21:48:07] (step=0002250) Train Loss: 6.5035, Train Steps/Sec: 1.12 +[2025-04-26 21:48:30] (step=0002275) Train Loss: 6.3781, Train Steps/Sec: 1.11 +[2025-04-26 21:48:52] (step=0002300) Train Loss: 6.3891, Train Steps/Sec: 1.12 +[2025-04-26 21:49:14] (step=0002325) Train Loss: 6.4321, Train Steps/Sec: 1.12 +[2025-04-26 21:49:37] (step=0002350) Train Loss: 6.3842, Train Steps/Sec: 1.12 +[2025-04-26 21:49:59] (step=0002375) Train Loss: 6.3493, Train Steps/Sec: 1.12 +[2025-04-26 21:50:48] (step=0002400) Train Loss: 6.3958, Train Steps/Sec: 0.51 +[2025-04-26 21:51:11] (step=0002425) Train Loss: 6.3943, Train Steps/Sec: 1.12 +[2025-04-26 21:51:33] (step=0002450) Train Loss: 6.3996, Train Steps/Sec: 1.12 +[2025-04-26 21:51:55] (step=0002475) Train Loss: 6.3698, Train Steps/Sec: 1.12 +[2025-04-26 21:52:18] (step=0002500) Train Loss: 6.3621, Train Steps/Sec: 1.12 +[2025-04-26 21:52:40] (step=0002525) Train Loss: 6.4460, Train Steps/Sec: 1.12 +[2025-04-26 21:53:02] (step=0002550) Train Loss: 6.3762, Train Steps/Sec: 1.12 +[2025-04-26 21:53:25] (step=0002575) Train Loss: 6.3286, Train Steps/Sec: 1.12 +[2025-04-26 21:53:47] (step=0002600) Train Loss: 6.3366, Train Steps/Sec: 1.12 +[2025-04-26 21:54:10] (step=0002625) Train Loss: 6.3467, Train Steps/Sec: 1.12 +[2025-04-26 21:54:32] (step=0002650) Train Loss: 6.3487, Train Steps/Sec: 1.12 +[2025-04-26 21:55:02] (step=0002675) Train Loss: 6.4098, Train Steps/Sec: 0.84 +[2025-04-26 21:55:24] (step=0002700) Train Loss: 6.4376, Train Steps/Sec: 1.12 +[2025-04-26 21:55:53] (step=0002725) Train Loss: 6.3796, Train Steps/Sec: 0.86 +[2025-04-26 21:56:16] (step=0002750) Train Loss: 6.3958, Train Steps/Sec: 1.12 +[2025-04-26 21:56:38] (step=0002775) Train Loss: 6.4719, Train Steps/Sec: 1.12 +[2025-04-26 21:57:00] (step=0002800) Train Loss: 6.3812, Train Steps/Sec: 1.12 +[2025-04-26 21:57:23] (step=0002825) Train Loss: 6.4326, Train Steps/Sec: 1.12 +[2025-04-26 21:57:45] (step=0002850) Train Loss: 6.3608, Train Steps/Sec: 1.12 +[2025-04-26 21:58:07] (step=0002875) Train Loss: 6.3491, Train Steps/Sec: 1.12 +[2025-04-26 21:58:30] (step=0002900) Train Loss: 6.4223, Train Steps/Sec: 1.12 +[2025-04-26 21:58:52] (step=0002925) Train Loss: 6.3232, Train Steps/Sec: 1.12 +[2025-04-26 21:59:14] (step=0002950) Train Loss: 6.3208, Train Steps/Sec: 1.12 +[2025-04-26 21:59:37] (step=0002975) Train Loss: 6.4128, Train Steps/Sec: 1.12 +[2025-04-26 21:59:59] (step=0003000) Train Loss: 6.3682, Train Steps/Sec: 1.11 +[2025-04-26 22:00:22] (step=0003025) Train Loss: 6.3998, Train Steps/Sec: 1.12 +[2025-04-26 22:00:44] (step=0003050) Train Loss: 6.3772, Train Steps/Sec: 1.12 +[2025-04-26 22:01:06] (step=0003075) Train Loss: 6.4068, Train Steps/Sec: 1.12 +[2025-04-26 22:01:29] (step=0003100) Train Loss: 6.3846, Train Steps/Sec: 1.12 +[2025-04-26 22:01:51] (step=0003125) Train Loss: 6.3608, Train Steps/Sec: 1.12 +[2025-04-26 22:02:13] (step=0003150) Train Loss: 6.3604, Train Steps/Sec: 1.12 +[2025-04-26 22:02:36] (step=0003175) Train Loss: 6.2782, Train Steps/Sec: 1.12 +[2025-04-26 22:02:58] (step=0003200) Train Loss: 6.2867, Train Steps/Sec: 1.11 +[2025-04-26 22:03:20] (step=0003225) Train Loss: 6.2728, Train Steps/Sec: 1.12 +[2025-04-26 22:03:43] (step=0003250) Train Loss: 6.3278, Train Steps/Sec: 1.12 +[2025-04-26 22:04:05] (step=0003275) Train Loss: 6.3442, Train Steps/Sec: 1.12 +[2025-04-26 22:04:28] (step=0003300) Train Loss: 6.3784, Train Steps/Sec: 1.12 +[2025-04-26 22:04:50] (step=0003325) Train Loss: 6.4433, Train Steps/Sec: 1.12 +[2025-04-26 22:05:12] (step=0003350) Train Loss: 6.3682, Train Steps/Sec: 1.11 +[2025-04-26 22:05:35] (step=0003375) Train Loss: 6.3709, Train Steps/Sec: 1.12 +[2025-04-26 22:05:57] (step=0003400) Train Loss: 6.3651, Train Steps/Sec: 1.11 +[2025-04-26 22:06:19] (step=0003425) Train Loss: 6.3374, Train Steps/Sec: 1.12 +[2025-04-26 22:06:42] (step=0003450) Train Loss: 6.3382, Train Steps/Sec: 1.12 +[2025-04-26 22:07:04] (step=0003475) Train Loss: 6.3642, Train Steps/Sec: 1.12 +[2025-04-26 22:07:27] (step=0003500) Train Loss: 6.2961, Train Steps/Sec: 1.12 +[2025-04-26 22:07:49] (step=0003525) Train Loss: 6.2519, Train Steps/Sec: 1.12 +[2025-04-26 22:08:11] (step=0003550) Train Loss: 6.3639, Train Steps/Sec: 1.12 +[2025-04-26 22:08:34] (step=0003575) Train Loss: 6.3173, Train Steps/Sec: 1.12 +[2025-04-26 22:08:56] (step=0003600) Train Loss: 6.3451, Train Steps/Sec: 1.11 +[2025-04-26 22:09:18] (step=0003625) Train Loss: 6.3184, Train Steps/Sec: 1.12 +[2025-04-26 22:09:41] (step=0003650) Train Loss: 6.2676, Train Steps/Sec: 1.12 +[2025-04-26 22:10:03] (step=0003675) Train Loss: 6.3075, Train Steps/Sec: 1.12 +[2025-04-26 22:10:25] (step=0003700) Train Loss: 6.3201, Train Steps/Sec: 1.12 +[2025-04-26 22:10:48] (step=0003725) Train Loss: 6.3114, Train Steps/Sec: 1.12 +[2025-04-26 22:11:10] (step=0003750) Train Loss: 6.3259, Train Steps/Sec: 1.12 +[2025-04-26 22:11:33] (step=0003775) Train Loss: 6.3102, Train Steps/Sec: 1.12 +[2025-04-26 22:11:55] (step=0003800) Train Loss: 6.3069, Train Steps/Sec: 1.11 +[2025-04-26 22:12:17] (step=0003825) Train Loss: 6.3990, Train Steps/Sec: 1.12 +[2025-04-26 22:12:40] (step=0003850) Train Loss: 6.3595, Train Steps/Sec: 1.12 +[2025-04-26 22:13:02] (step=0003875) Train Loss: 6.3234, Train Steps/Sec: 1.12 +[2025-04-26 22:13:24] (step=0003900) Train Loss: 6.3122, Train Steps/Sec: 1.12 +[2025-04-26 22:13:47] (step=0003925) Train Loss: 6.3040, Train Steps/Sec: 1.12 +[2025-04-26 22:14:09] (step=0003950) Train Loss: 6.3210, Train Steps/Sec: 1.12 +[2025-04-26 22:14:31] (step=0003975) Train Loss: 6.3216, Train Steps/Sec: 1.12 +[2025-04-26 22:14:54] (step=0004000) Train Loss: 6.3398, Train Steps/Sec: 1.11 +[2025-04-26 22:14:54] text_config is None. Initializing the text config with default values (`OPTConfig`). +[2025-04-26 22:21:02] Finish Eval in 4000 steps... +[2025-04-26 22:21:21] Saved checkpoint to checkpoint/X2I_UltraEdiit_fourmask_LLava_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_filtered_objects_4mask_1280_context/003-GPT-XL/checkpoints/0004000.pt +[2025-04-26 22:21:23] Removed old checkpoint: checkpoint/X2I_UltraEdiit_fourmask_LLava_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_filtered_objects_4mask_1280_context/003-GPT-XL/checkpoints/0002000.pt +[2025-04-26 22:21:46] (step=0004025) Train Loss: 6.2964, Train Steps/Sec: 0.06 +[2025-04-26 22:22:08] (step=0004050) Train Loss: 6.3278, Train Steps/Sec: 1.12 +[2025-04-26 22:22:30] (step=0004075) Train Loss: 6.3072, Train Steps/Sec: 1.12 +[2025-04-26 22:22:53] (step=0004100) Train Loss: 6.2894, Train Steps/Sec: 1.12 +[2025-04-26 22:23:15] (step=0004125) Train Loss: 6.3272, Train Steps/Sec: 1.12 +[2025-04-26 22:23:37] (step=0004150) Train Loss: 6.2906, Train Steps/Sec: 1.12 +[2025-04-26 22:24:00] (step=0004175) Train Loss: 6.3075, Train Steps/Sec: 1.12 +[2025-04-26 22:24:22] (step=0004200) Train Loss: 6.2586, Train Steps/Sec: 1.11 +[2025-04-26 22:24:44] (step=0004225) Train Loss: 6.3069, Train Steps/Sec: 1.12 +[2025-04-26 22:25:07] (step=0004250) Train Loss: 6.3023, Train Steps/Sec: 1.12 +[2025-04-26 22:25:29] (step=0004275) Train Loss: 6.3411, Train Steps/Sec: 1.12 +[2025-04-26 22:25:52] (step=0004300) Train Loss: 6.3057, Train Steps/Sec: 1.12 +[2025-04-26 22:26:14] (step=0004325) Train Loss: 6.2551, Train Steps/Sec: 1.12 +[2025-04-26 22:26:36] (step=0004350) Train Loss: 6.2859, Train Steps/Sec: 1.12 +[2025-04-26 22:26:59] (step=0004375) Train Loss: 6.2846, Train Steps/Sec: 1.12 +[2025-04-26 22:27:21] (step=0004400) Train Loss: 6.3353, Train Steps/Sec: 1.12 +[2025-04-26 22:27:43] (step=0004425) Train Loss: 6.3273, Train Steps/Sec: 1.12 +[2025-04-26 22:28:06] (step=0004450) Train Loss: 6.3061, Train Steps/Sec: 1.12 +[2025-04-26 22:28:28] (step=0004475) Train Loss: 6.2885, Train Steps/Sec: 1.12 +[2025-04-26 22:28:51] (step=0004500) Train Loss: 6.2773, Train Steps/Sec: 1.12 +[2025-04-26 22:29:13] (step=0004525) Train Loss: 6.2515, Train Steps/Sec: 1.12 +[2025-04-26 22:29:35] (step=0004550) Train Loss: 6.2671, Train Steps/Sec: 1.12 +[2025-04-26 22:29:58] (step=0004575) Train Loss: 6.3157, Train Steps/Sec: 1.11 +[2025-04-26 22:30:20] (step=0004600) Train Loss: 6.3046, Train Steps/Sec: 1.11 +[2025-04-26 22:30:42] (step=0004625) Train Loss: 6.2857, Train Steps/Sec: 1.12 +[2025-04-26 22:31:05] (step=0004650) Train Loss: 6.3340, Train Steps/Sec: 1.12 +[2025-04-26 22:31:27] (step=0004675) Train Loss: 6.2850, Train Steps/Sec: 1.12 +[2025-04-26 22:31:49] (step=0004700) Train Loss: 6.2601, Train Steps/Sec: 1.12 +[2025-04-26 22:32:12] (step=0004725) Train Loss: 6.2891, Train Steps/Sec: 1.12 +[2025-04-26 22:32:34] (step=0004750) Train Loss: 6.3079, Train Steps/Sec: 1.12 +[2025-04-26 22:32:56] (step=0004775) Train Loss: 6.3800, Train Steps/Sec: 1.12 +[2025-04-26 22:33:19] (step=0004800) Train Loss: 6.2954, Train Steps/Sec: 1.11 +[2025-04-26 22:33:41] (step=0004825) Train Loss: 6.2441, Train Steps/Sec: 1.12 +[2025-04-26 22:34:04] (step=0004850) Train Loss: 6.3170, Train Steps/Sec: 1.12 +[2025-04-26 22:34:26] (step=0004875) Train Loss: 6.3163, Train Steps/Sec: 1.12 +[2025-04-26 22:34:48] (step=0004900) Train Loss: 6.3055, Train Steps/Sec: 1.12 +[2025-04-26 22:35:11] (step=0004925) Train Loss: 6.3080, Train Steps/Sec: 1.12 +[2025-04-26 22:35:33] (step=0004950) Train Loss: 6.2524, Train Steps/Sec: 1.12 +[2025-04-26 22:35:55] (step=0004975) Train Loss: 6.3135, Train Steps/Sec: 1.12 +[2025-04-26 22:36:18] (step=0005000) Train Loss: 6.2795, Train Steps/Sec: 1.11 +[2025-04-26 22:36:40] (step=0005025) Train Loss: 6.3269, Train Steps/Sec: 1.12 +[2025-04-26 22:37:03] (step=0005050) Train Loss: 6.2596, Train Steps/Sec: 1.12 +[2025-04-26 22:37:25] (step=0005075) Train Loss: 6.2862, Train Steps/Sec: 1.12 +[2025-04-26 22:37:47] (step=0005100) Train Loss: 6.3035, Train Steps/Sec: 1.12 +[2025-04-26 22:38:10] (step=0005125) Train Loss: 6.2684, Train Steps/Sec: 1.12 +[2025-04-26 22:38:32] (step=0005150) Train Loss: 6.2551, Train Steps/Sec: 1.12 +[2025-04-26 22:38:54] (step=0005175) Train Loss: 6.3131, Train Steps/Sec: 1.12 +[2025-04-26 22:39:17] (step=0005200) Train Loss: 6.2411, Train Steps/Sec: 1.11 +[2025-04-26 22:39:39] (step=0005225) Train Loss: 6.2727, Train Steps/Sec: 1.12 +[2025-04-26 22:40:02] (step=0005250) Train Loss: 6.2894, Train Steps/Sec: 1.12 +[2025-04-26 22:40:24] (step=0005275) Train Loss: 6.3140, Train Steps/Sec: 1.12 +[2025-04-26 22:40:46] (step=0005300) Train Loss: 6.2501, Train Steps/Sec: 1.12 +[2025-04-26 22:41:09] (step=0005325) Train Loss: 6.2478, Train Steps/Sec: 1.11 +[2025-04-26 22:41:31] (step=0005350) Train Loss: 6.2941, Train Steps/Sec: 1.11 +[2025-04-26 22:41:53] (step=0005375) Train Loss: 6.3197, Train Steps/Sec: 1.12 +[2025-04-26 22:42:16] (step=0005400) Train Loss: 6.2895, Train Steps/Sec: 1.11 +[2025-04-26 22:42:38] (step=0005425) Train Loss: 6.3199, Train Steps/Sec: 1.12 +[2025-04-26 22:43:01] (step=0005450) Train Loss: 6.3395, Train Steps/Sec: 1.12 +[2025-04-26 22:43:23] (step=0005475) Train Loss: 6.2418, Train Steps/Sec: 1.12 +[2025-04-26 22:43:45] (step=0005500) Train Loss: 6.2254, Train Steps/Sec: 1.12 +[2025-04-26 22:44:08] (step=0005525) Train Loss: 6.2337, Train Steps/Sec: 1.12 +[2025-04-26 22:44:30] (step=0005550) Train Loss: 6.3453, Train Steps/Sec: 1.12 +[2025-04-26 22:44:52] (step=0005575) Train Loss: 6.2590, Train Steps/Sec: 1.12 +[2025-04-26 22:45:15] (step=0005600) Train Loss: 6.2499, Train Steps/Sec: 1.11 +[2025-04-26 22:45:37] (step=0005625) Train Loss: 6.2323, Train Steps/Sec: 1.12 +[2025-04-26 22:46:00] (step=0005650) Train Loss: 6.3066, Train Steps/Sec: 1.12 +[2025-04-26 22:46:22] (step=0005675) Train Loss: 6.2206, Train Steps/Sec: 1.12 +[2025-04-26 22:46:44] (step=0005700) Train Loss: 6.2536, Train Steps/Sec: 1.11 +[2025-04-26 22:47:07] (step=0005725) Train Loss: 6.2365, Train Steps/Sec: 1.12 +[2025-04-26 22:47:29] (step=0005750) Train Loss: 6.2544, Train Steps/Sec: 1.12 +[2025-04-26 22:47:51] (step=0005775) Train Loss: 6.2794, Train Steps/Sec: 1.12 +[2025-04-26 22:48:19] (step=0005800) Train Loss: 6.2641, Train Steps/Sec: 0.91 +[2025-04-26 22:48:41] (step=0005825) Train Loss: 6.2891, Train Steps/Sec: 1.12 +[2025-04-26 22:49:04] (step=0005850) Train Loss: 6.2363, Train Steps/Sec: 1.12 +[2025-04-26 22:49:26] (step=0005875) Train Loss: 6.2938, Train Steps/Sec: 1.12 +[2025-04-26 22:49:48] (step=0005900) Train Loss: 6.2996, Train Steps/Sec: 1.12 +[2025-04-26 22:50:11] (step=0005925) Train Loss: 6.2419, Train Steps/Sec: 1.12 +[2025-04-26 22:50:33] (step=0005950) Train Loss: 6.2123, Train Steps/Sec: 1.12 +[2025-04-26 22:50:55] (step=0005975) Train Loss: 6.1992, Train Steps/Sec: 1.12 +[2025-04-26 22:51:18] (step=0006000) Train Loss: 6.2881, Train Steps/Sec: 1.11 +[2025-04-26 22:51:18] text_config is None. Initializing the text config with default values (`OPTConfig`). +[2025-04-26 22:57:26] Finish Eval in 6000 steps... +[2025-04-26 22:57:45] Saved checkpoint to checkpoint/X2I_UltraEdiit_fourmask_LLava_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_filtered_objects_4mask_1280_context/003-GPT-XL/checkpoints/0006000.pt +[2025-04-26 22:57:47] Removed old checkpoint: checkpoint/X2I_UltraEdiit_fourmask_LLava_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_filtered_objects_4mask_1280_context/003-GPT-XL/checkpoints/0004000.pt +[2025-04-26 22:58:10] (step=0006025) Train Loss: 6.2863, Train Steps/Sec: 0.06 +[2025-04-26 22:58:32] (step=0006050) Train Loss: 6.2622, Train Steps/Sec: 1.12 +[2025-04-26 22:58:55] (step=0006075) Train Loss: 6.2294, Train Steps/Sec: 1.12 +[2025-04-26 22:59:17] (step=0006100) Train Loss: 6.2250, Train Steps/Sec: 1.12 +[2025-04-26 22:59:39] (step=0006125) Train Loss: 6.2449, Train Steps/Sec: 1.12 +[2025-04-26 23:00:02] (step=0006150) Train Loss: 6.2267, Train Steps/Sec: 1.12 +[2025-04-26 23:00:24] (step=0006175) Train Loss: 6.1763, Train Steps/Sec: 1.12 +[2025-04-26 23:00:47] (step=0006200) Train Loss: 6.2821, Train Steps/Sec: 1.11 +[2025-04-26 23:01:09] (step=0006225) Train Loss: 6.2271, Train Steps/Sec: 1.12 +[2025-04-26 23:01:31] (step=0006250) Train Loss: 6.2157, Train Steps/Sec: 1.12 +[2025-04-26 23:01:54] (step=0006275) Train Loss: 6.2284, Train Steps/Sec: 1.12 +[2025-04-26 23:02:16] (step=0006300) Train Loss: 6.2214, Train Steps/Sec: 1.12 +[2025-04-26 23:02:39] (step=0006325) Train Loss: 6.2760, Train Steps/Sec: 1.11 +[2025-04-26 23:03:01] (step=0006350) Train Loss: 6.2449, Train Steps/Sec: 1.12 +[2025-04-26 23:03:23] (step=0006375) Train Loss: 6.2642, Train Steps/Sec: 1.11 +[2025-04-26 23:03:46] (step=0006400) Train Loss: 6.2256, Train Steps/Sec: 1.10 +[2025-04-26 23:04:09] (step=0006425) Train Loss: 6.2250, Train Steps/Sec: 1.11 +[2025-04-26 23:04:31] (step=0006450) Train Loss: 6.2806, Train Steps/Sec: 1.11 +[2025-04-26 23:04:54] (step=0006475) Train Loss: 6.2353, Train Steps/Sec: 1.11 +[2025-04-26 23:05:16] (step=0006500) Train Loss: 6.2234, Train Steps/Sec: 1.11 +[2025-04-26 23:05:39] (step=0006525) Train Loss: 6.2629, Train Steps/Sec: 1.11 +[2025-04-26 23:06:01] (step=0006550) Train Loss: 6.2606, Train Steps/Sec: 1.12 +[2025-04-26 23:06:24] (step=0006575) Train Loss: 6.2083, Train Steps/Sec: 1.12 +[2025-04-26 23:06:46] (step=0006600) Train Loss: 6.2291, Train Steps/Sec: 1.11 +[2025-04-26 23:07:08] (step=0006625) Train Loss: 6.1996, Train Steps/Sec: 1.12 +[2025-04-26 23:07:31] (step=0006650) Train Loss: 6.2627, Train Steps/Sec: 1.12 +[2025-04-26 23:07:53] (step=0006675) Train Loss: 6.1968, Train Steps/Sec: 1.12 +[2025-04-26 23:08:16] (step=0006700) Train Loss: 6.2226, Train Steps/Sec: 1.12 +[2025-04-26 23:08:54] (step=0006725) Train Loss: 6.2141, Train Steps/Sec: 0.66 +[2025-04-26 23:09:16] (step=0006750) Train Loss: 6.2369, Train Steps/Sec: 1.12 +[2025-04-26 23:09:38] (step=0006775) Train Loss: 6.2367, Train Steps/Sec: 1.12 +[2025-04-26 23:10:01] (step=0006800) Train Loss: 6.2484, Train Steps/Sec: 1.11 +[2025-04-26 23:10:23] (step=0006825) Train Loss: 6.3009, Train Steps/Sec: 1.12 +[2025-04-26 23:10:46] (step=0006850) Train Loss: 6.2284, Train Steps/Sec: 1.12 +[2025-04-26 23:11:08] (step=0006875) Train Loss: 6.2302, Train Steps/Sec: 1.12 +[2025-04-26 23:11:30] (step=0006900) Train Loss: 6.3052, Train Steps/Sec: 1.12 +[2025-04-26 23:11:58] (step=0006925) Train Loss: 6.2216, Train Steps/Sec: 0.91 +[2025-04-26 23:12:20] (step=0006950) Train Loss: 6.1920, Train Steps/Sec: 1.12 +[2025-04-26 23:12:48] (step=0006975) Train Loss: 6.2626, Train Steps/Sec: 0.91 +[2025-04-26 23:13:10] (step=0007000) Train Loss: 6.2151, Train Steps/Sec: 1.11 +[2025-04-26 23:13:33] (step=0007025) Train Loss: 6.2561, Train Steps/Sec: 1.12 +[2025-04-26 23:13:55] (step=0007050) Train Loss: 6.1687, Train Steps/Sec: 1.12 +[2025-04-26 23:14:17] (step=0007075) Train Loss: 6.2426, Train Steps/Sec: 1.12 +[2025-04-26 23:14:40] (step=0007100) Train Loss: 6.3134, Train Steps/Sec: 1.12 +[2025-04-26 23:15:02] (step=0007125) Train Loss: 6.2441, Train Steps/Sec: 1.12 +[2025-04-26 23:15:25] (step=0007150) Train Loss: 6.2642, Train Steps/Sec: 1.12 +[2025-04-26 23:15:47] (step=0007175) Train Loss: 6.2138, Train Steps/Sec: 1.12 +[2025-04-26 23:16:09] (step=0007200) Train Loss: 6.2657, Train Steps/Sec: 1.11 +[2025-04-26 23:16:32] (step=0007225) Train Loss: 6.2653, Train Steps/Sec: 1.12 +[2025-04-26 23:16:54] (step=0007250) Train Loss: 6.2618, Train Steps/Sec: 1.12 +[2025-04-26 23:17:17] (step=0007275) Train Loss: 6.2298, Train Steps/Sec: 1.12 +[2025-04-26 23:17:39] (step=0007300) Train Loss: 6.3074, Train Steps/Sec: 1.12 +[2025-04-26 23:18:01] (step=0007325) Train Loss: 6.2395, Train Steps/Sec: 1.12 +[2025-04-26 23:18:24] (step=0007350) Train Loss: 6.2542, Train Steps/Sec: 1.12 +[2025-04-26 23:18:46] (step=0007375) Train Loss: 6.1842, Train Steps/Sec: 1.12 +[2025-04-26 23:19:09] (step=0007400) Train Loss: 6.2553, Train Steps/Sec: 1.11 +[2025-04-26 23:19:31] (step=0007425) Train Loss: 6.1726, Train Steps/Sec: 1.12 +[2025-04-26 23:19:53] (step=0007450) Train Loss: 6.2697, Train Steps/Sec: 1.12 +[2025-04-26 23:20:16] (step=0007475) Train Loss: 6.1879, Train Steps/Sec: 1.12 +[2025-04-26 23:20:38] (step=0007500) Train Loss: 6.1982, Train Steps/Sec: 1.12 +[2025-04-26 23:21:00] (step=0007525) Train Loss: 6.2835, Train Steps/Sec: 1.12 +[2025-04-26 23:21:23] (step=0007550) Train Loss: 6.2300, Train Steps/Sec: 1.12 +[2025-04-26 23:21:45] (step=0007575) Train Loss: 6.2375, Train Steps/Sec: 1.12 +[2025-04-26 23:22:08] (step=0007600) Train Loss: 6.2076, Train Steps/Sec: 1.11 +[2025-04-26 23:22:30] (step=0007625) Train Loss: 6.1928, Train Steps/Sec: 1.12 +[2025-04-26 23:22:52] (step=0007650) Train Loss: 6.2257, Train Steps/Sec: 1.12 +[2025-04-26 23:23:15] (step=0007675) Train Loss: 6.2520, Train Steps/Sec: 1.12 +[2025-04-26 23:23:37] (step=0007700) Train Loss: 6.1485, Train Steps/Sec: 1.12 +[2025-04-26 23:23:59] (step=0007725) Train Loss: 6.1636, Train Steps/Sec: 1.12 +[2025-04-26 23:24:22] (step=0007750) Train Loss: 6.1898, Train Steps/Sec: 1.12 +[2025-04-26 23:24:44] (step=0007775) Train Loss: 6.1900, Train Steps/Sec: 1.12 +[2025-04-26 23:25:07] (step=0007800) Train Loss: 6.2152, Train Steps/Sec: 1.11 +[2025-04-26 23:25:29] (step=0007825) Train Loss: 6.2039, Train Steps/Sec: 1.12 +[2025-04-26 23:25:52] (step=0007850) Train Loss: 6.2275, Train Steps/Sec: 1.11 +[2025-04-26 23:26:14] (step=0007875) Train Loss: 6.1943, Train Steps/Sec: 1.12 +[2025-04-26 23:26:36] (step=0007900) Train Loss: 6.2097, Train Steps/Sec: 1.12 +[2025-04-26 23:26:59] (step=0007925) Train Loss: 6.1858, Train Steps/Sec: 1.12 +[2025-04-26 23:27:21] (step=0007950) Train Loss: 6.2481, Train Steps/Sec: 1.12 +[2025-04-26 23:27:43] (step=0007975) Train Loss: 6.2204, Train Steps/Sec: 1.12 +[2025-04-26 23:28:06] (step=0008000) Train Loss: 6.1445, Train Steps/Sec: 1.11 +[2025-04-26 23:28:06] text_config is None. Initializing the text config with default values (`OPTConfig`). +[2025-04-26 23:34:13] Finish Eval in 8000 steps... +[2025-04-26 23:34:32] Saved checkpoint to checkpoint/X2I_UltraEdiit_fourmask_LLava_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_filtered_objects_4mask_1280_context/003-GPT-XL/checkpoints/0008000.pt +[2025-04-26 23:34:34] Removed old checkpoint: checkpoint/X2I_UltraEdiit_fourmask_LLava_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_filtered_objects_4mask_1280_context/003-GPT-XL/checkpoints/0006000.pt +[2025-04-26 23:34:56] (step=0008025) Train Loss: 6.2186, Train Steps/Sec: 0.06 +[2025-04-26 23:35:19] (step=0008050) Train Loss: 6.1772, Train Steps/Sec: 1.12 +[2025-04-26 23:35:41] (step=0008075) Train Loss: 6.2167, Train Steps/Sec: 1.12 +[2025-04-26 23:36:03] (step=0008100) Train Loss: 6.1606, Train Steps/Sec: 1.12 +[2025-04-26 23:36:26] (step=0008125) Train Loss: 6.2719, Train Steps/Sec: 1.12 +[2025-04-26 23:36:48] (step=0008150) Train Loss: 6.2460, Train Steps/Sec: 1.12 +[2025-04-26 23:37:10] (step=0008175) Train Loss: 6.2599, Train Steps/Sec: 1.12 +[2025-04-26 23:37:33] (step=0008200) Train Loss: 6.2424, Train Steps/Sec: 1.11 +[2025-04-26 23:37:55] (step=0008225) Train Loss: 6.2016, Train Steps/Sec: 1.12 +[2025-04-26 23:38:17] (step=0008250) Train Loss: 6.2074, Train Steps/Sec: 1.12 +[2025-04-26 23:38:40] (step=0008275) Train Loss: 6.2573, Train Steps/Sec: 1.12 +[2025-04-26 23:39:02] (step=0008300) Train Loss: 6.2007, Train Steps/Sec: 1.12 +[2025-04-26 23:39:24] (step=0008325) Train Loss: 6.2107, Train Steps/Sec: 1.12 +[2025-04-26 23:39:47] (step=0008350) Train Loss: 6.1790, Train Steps/Sec: 1.12 +[2025-04-26 23:40:09] (step=0008375) Train Loss: 6.1734, Train Steps/Sec: 1.12 +[2025-04-26 23:40:32] (step=0008400) Train Loss: 6.2087, Train Steps/Sec: 1.12 +[2025-04-26 23:40:54] (step=0008425) Train Loss: 6.1879, Train Steps/Sec: 1.12 +[2025-04-26 23:41:16] (step=0008450) Train Loss: 6.1751, Train Steps/Sec: 1.12 +[2025-04-26 23:41:39] (step=0008475) Train Loss: 6.1864, Train Steps/Sec: 1.12 +[2025-04-26 23:42:01] (step=0008500) Train Loss: 6.2424, Train Steps/Sec: 1.12 +[2025-04-26 23:42:23] (step=0008525) Train Loss: 6.2782, Train Steps/Sec: 1.12 +[2025-04-26 23:42:46] (step=0008550) Train Loss: 6.1880, Train Steps/Sec: 1.12 +[2025-04-26 23:43:08] (step=0008575) Train Loss: 6.1688, Train Steps/Sec: 1.12 +[2025-04-26 23:43:31] (step=0008600) Train Loss: 6.1768, Train Steps/Sec: 1.11 +[2025-04-26 23:43:53] (step=0008625) Train Loss: 6.2435, Train Steps/Sec: 1.12 +[2025-04-26 23:44:15] (step=0008650) Train Loss: 6.1780, Train Steps/Sec: 1.12 +[2025-04-26 23:44:38] (step=0008675) Train Loss: 6.1856, Train Steps/Sec: 1.12 +[2025-04-26 23:45:00] (step=0008700) Train Loss: 6.2038, Train Steps/Sec: 1.12 +[2025-04-26 23:45:22] (step=0008725) Train Loss: 6.1764, Train Steps/Sec: 1.12 +[2025-04-26 23:45:45] (step=0008750) Train Loss: 6.1919, Train Steps/Sec: 1.12 +[2025-04-26 23:46:07] (step=0008775) Train Loss: 6.1731, Train Steps/Sec: 1.11 +[2025-04-26 23:46:30] (step=0008800) Train Loss: 6.2395, Train Steps/Sec: 1.11 +[2025-04-26 23:46:52] (step=0008825) Train Loss: 6.1680, Train Steps/Sec: 1.12 +[2025-04-26 23:47:14] (step=0008850) Train Loss: 6.1746, Train Steps/Sec: 1.12 +[2025-04-26 23:47:37] (step=0008875) Train Loss: 6.1984, Train Steps/Sec: 1.12 +[2025-04-26 23:47:59] (step=0008900) Train Loss: 6.2257, Train Steps/Sec: 1.12 +[2025-04-26 23:48:21] (step=0008925) Train Loss: 6.1384, Train Steps/Sec: 1.12 +[2025-04-26 23:48:44] (step=0008950) Train Loss: 6.2580, Train Steps/Sec: 1.12 +[2025-04-26 23:49:06] (step=0008975) Train Loss: 6.1882, Train Steps/Sec: 1.12 +[2025-04-26 23:49:29] (step=0009000) Train Loss: 6.1978, Train Steps/Sec: 1.11 +[2025-04-26 23:49:51] (step=0009025) Train Loss: 6.2823, Train Steps/Sec: 1.12 +[2025-04-26 23:50:13] (step=0009050) Train Loss: 6.1656, Train Steps/Sec: 1.12 +[2025-04-26 23:50:36] (step=0009075) Train Loss: 6.1798, Train Steps/Sec: 1.12 +[2025-04-26 23:50:58] (step=0009100) Train Loss: 6.2140, Train Steps/Sec: 1.12 +[2025-04-26 23:51:20] (step=0009125) Train Loss: 6.2370, Train Steps/Sec: 1.12 +[2025-04-26 23:51:43] (step=0009150) Train Loss: 6.1746, Train Steps/Sec: 1.12 +[2025-04-26 23:52:05] (step=0009175) Train Loss: 6.2138, Train Steps/Sec: 1.12 +[2025-04-26 23:52:28] (step=0009200) Train Loss: 6.2456, Train Steps/Sec: 1.11 +[2025-04-26 23:52:50] (step=0009225) Train Loss: 6.1869, Train Steps/Sec: 1.12 +[2025-04-26 23:53:12] (step=0009250) Train Loss: 6.1351, Train Steps/Sec: 1.12 +[2025-04-26 23:53:35] (step=0009275) Train Loss: 6.1717, Train Steps/Sec: 1.12 +[2025-04-26 23:53:57] (step=0009300) Train Loss: 6.1988, Train Steps/Sec: 1.12 +[2025-04-26 23:54:19] (step=0009325) Train Loss: 6.1131, Train Steps/Sec: 1.12 +[2025-04-26 23:54:42] (step=0009350) Train Loss: 6.1538, Train Steps/Sec: 1.12 +[2025-04-26 23:55:04] (step=0009375) Train Loss: 6.1440, Train Steps/Sec: 1.12 +[2025-04-26 23:55:27] (step=0009400) Train Loss: 6.2038, Train Steps/Sec: 1.11 +[2025-04-26 23:55:49] (step=0009425) Train Loss: 6.1830, Train Steps/Sec: 1.12 +[2025-04-26 23:56:11] (step=0009450) Train Loss: 6.1893, Train Steps/Sec: 1.11 +[2025-04-26 23:56:34] (step=0009475) Train Loss: 6.1902, Train Steps/Sec: 1.12 +[2025-04-26 23:56:56] (step=0009500) Train Loss: 6.2341, Train Steps/Sec: 1.12 +[2025-04-26 23:57:19] (step=0009525) Train Loss: 6.1710, Train Steps/Sec: 1.12 +[2025-04-26 23:57:41] (step=0009550) Train Loss: 6.1807, Train Steps/Sec: 1.12 +[2025-04-26 23:58:03] (step=0009575) Train Loss: 6.1398, Train Steps/Sec: 1.12 +[2025-04-26 23:58:26] (step=0009600) Train Loss: 6.1379, Train Steps/Sec: 1.12 +[2025-04-26 23:58:48] (step=0009625) Train Loss: 6.1992, Train Steps/Sec: 1.12 +[2025-04-26 23:59:10] (step=0009650) Train Loss: 6.1831, Train Steps/Sec: 1.12 +[2025-04-26 23:59:33] (step=0009675) Train Loss: 6.1850, Train Steps/Sec: 1.12 +[2025-04-26 23:59:55] (step=0009700) Train Loss: 6.1971, Train Steps/Sec: 1.12 +[2025-04-27 00:00:17] (step=0009725) Train Loss: 6.2124, Train Steps/Sec: 1.12 +[2025-04-27 00:00:40] (step=0009750) Train Loss: 6.1820, Train Steps/Sec: 1.12 +[2025-04-27 00:01:02] (step=0009775) Train Loss: 6.1737, Train Steps/Sec: 1.12 +[2025-04-27 00:01:24] (step=0009800) Train Loss: 6.1444, Train Steps/Sec: 1.12 +[2025-04-27 00:01:47] (step=0009825) Train Loss: 6.1401, Train Steps/Sec: 1.12 +[2025-04-27 00:02:09] (step=0009850) Train Loss: 6.1416, Train Steps/Sec: 1.12 +[2025-04-27 00:02:31] (step=0009875) Train Loss: 6.2357, Train Steps/Sec: 1.12 +[2025-04-27 00:02:54] (step=0009900) Train Loss: 6.1321, Train Steps/Sec: 1.12 +[2025-04-27 00:03:16] (step=0009925) Train Loss: 6.2106, Train Steps/Sec: 1.12 +[2025-04-27 00:03:39] (step=0009950) Train Loss: 6.2060, Train Steps/Sec: 1.12 +[2025-04-27 00:04:01] (step=0009975) Train Loss: 6.1741, Train Steps/Sec: 1.12 +[2025-04-27 00:04:23] (step=0010000) Train Loss: 6.2019, Train Steps/Sec: 1.11 +[2025-04-27 00:04:23] text_config is None. Initializing the text config with default values (`OPTConfig`). +[2025-04-27 00:10:32] Finish Eval in 10000 steps... +[2025-04-27 00:10:51] Saved checkpoint to checkpoint/X2I_UltraEdiit_fourmask_LLava_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_filtered_objects_4mask_1280_context/003-GPT-XL/checkpoints/0010000.pt +[2025-04-27 00:10:53] Removed old checkpoint: checkpoint/X2I_UltraEdiit_fourmask_LLava_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_filtered_objects_4mask_1280_context/003-GPT-XL/checkpoints/0008000.pt +[2025-04-27 00:11:15] (step=0010025) Train Loss: 6.1804, Train Steps/Sec: 0.06 +[2025-04-27 00:11:38] (step=0010050) Train Loss: 6.0934, Train Steps/Sec: 1.12 +[2025-04-27 00:12:00] (step=0010075) Train Loss: 6.2040, Train Steps/Sec: 1.12 +[2025-04-27 00:12:22] (step=0010100) Train Loss: 6.1897, Train Steps/Sec: 1.12 +[2025-04-27 00:12:45] (step=0010125) Train Loss: 6.1158, Train Steps/Sec: 1.12 +[2025-04-27 00:13:07] (step=0010150) Train Loss: 6.1947, Train Steps/Sec: 1.12 +[2025-04-27 00:13:29] (step=0010175) Train Loss: 6.1638, Train Steps/Sec: 1.12 +[2025-04-27 00:13:52] (step=0010200) Train Loss: 6.1699, Train Steps/Sec: 1.12 +[2025-04-27 00:14:21] (step=0010225) Train Loss: 6.0947, Train Steps/Sec: 0.85 +[2025-04-27 00:14:43] (step=0010250) Train Loss: 6.1865, Train Steps/Sec: 1.12 +[2025-04-27 00:15:06] (step=0010275) Train Loss: 6.1648, Train Steps/Sec: 1.12 +[2025-04-27 00:15:28] (step=0010300) Train Loss: 6.1437, Train Steps/Sec: 1.11 +[2025-04-27 00:15:51] (step=0010325) Train Loss: 6.1661, Train Steps/Sec: 1.12 +[2025-04-27 00:16:13] (step=0010350) Train Loss: 6.1890, Train Steps/Sec: 1.12 +[2025-04-27 00:16:35] (step=0010375) Train Loss: 6.1922, Train Steps/Sec: 1.12 +[2025-04-27 00:16:58] (step=0010400) Train Loss: 6.2179, Train Steps/Sec: 1.11 +[2025-04-27 00:17:20] (step=0010425) Train Loss: 6.2665, Train Steps/Sec: 1.12 +[2025-04-27 00:17:42] (step=0010450) Train Loss: 6.1188, Train Steps/Sec: 1.12 +[2025-04-27 00:18:05] (step=0010475) Train Loss: 6.1856, Train Steps/Sec: 1.12 +[2025-04-27 00:18:27] (step=0010500) Train Loss: 6.2059, Train Steps/Sec: 1.12 +[2025-04-27 00:18:49] (step=0010525) Train Loss: 6.2282, Train Steps/Sec: 1.12 +[2025-04-27 00:19:12] (step=0010550) Train Loss: 6.2230, Train Steps/Sec: 1.12 +[2025-04-27 00:19:34] (step=0010575) Train Loss: 6.0977, Train Steps/Sec: 1.12 +[2025-04-27 00:19:57] (step=0010600) Train Loss: 6.1637, Train Steps/Sec: 1.11 +[2025-04-27 00:20:19] (step=0010625) Train Loss: 6.1360, Train Steps/Sec: 1.12 +[2025-04-27 00:20:41] (step=0010650) Train Loss: 6.1337, Train Steps/Sec: 1.12 +[2025-04-27 00:21:04] (step=0010675) Train Loss: 6.1835, Train Steps/Sec: 1.12 +[2025-04-27 00:21:26] (step=0010700) Train Loss: 6.1405, Train Steps/Sec: 1.10 +[2025-04-27 00:21:49] (step=0010725) Train Loss: 6.2023, Train Steps/Sec: 1.12 +[2025-04-27 00:22:11] (step=0010750) Train Loss: 6.1453, Train Steps/Sec: 1.12 +[2025-04-27 00:22:33] (step=0010775) Train Loss: 6.1355, Train Steps/Sec: 1.12 +[2025-04-27 00:22:56] (step=0010800) Train Loss: 6.1925, Train Steps/Sec: 1.11 +[2025-04-27 00:23:18] (step=0010825) Train Loss: 6.1506, Train Steps/Sec: 1.12 +[2025-04-27 00:23:41] (step=0010850) Train Loss: 6.1025, Train Steps/Sec: 1.12 +[2025-04-27 00:24:03] (step=0010875) Train Loss: 6.1400, Train Steps/Sec: 1.12 +[2025-04-27 00:24:25] (step=0010900) Train Loss: 6.2093, Train Steps/Sec: 1.12 +[2025-04-27 00:24:48] (step=0010925) Train Loss: 6.1673, Train Steps/Sec: 1.12 +[2025-04-27 00:25:10] (step=0010950) Train Loss: 6.1925, Train Steps/Sec: 1.12 +[2025-04-27 00:25:32] (step=0010975) Train Loss: 6.1679, Train Steps/Sec: 1.12 +[2025-04-27 00:25:55] (step=0011000) Train Loss: 6.1114, Train Steps/Sec: 1.11 +[2025-04-27 00:26:17] (step=0011025) Train Loss: 6.1532, Train Steps/Sec: 1.12 +[2025-04-27 00:27:00] (step=0011050) Train Loss: 6.1413, Train Steps/Sec: 0.59 +[2025-04-27 00:27:22] (step=0011075) Train Loss: 6.1825, Train Steps/Sec: 1.12 +[2025-04-27 00:27:44] (step=0011100) Train Loss: 6.0836, Train Steps/Sec: 1.12 +[2025-04-27 00:28:07] (step=0011125) Train Loss: 6.1163, Train Steps/Sec: 1.12 +[2025-04-27 00:28:29] (step=0011150) Train Loss: 6.2097, Train Steps/Sec: 1.12 +[2025-04-27 00:28:51] (step=0011175) Train Loss: 6.1999, Train Steps/Sec: 1.12 +[2025-04-27 00:29:19] (step=0011200) Train Loss: 6.2131, Train Steps/Sec: 0.91 +[2025-04-27 00:29:41] (step=0011225) Train Loss: 6.1498, Train Steps/Sec: 1.12 +[2025-04-27 00:30:08] (step=0011250) Train Loss: 6.1797, Train Steps/Sec: 0.92 +[2025-04-27 00:30:31] (step=0011275) Train Loss: 6.1378, Train Steps/Sec: 1.12 +[2025-04-27 00:30:53] (step=0011300) Train Loss: 6.1711, Train Steps/Sec: 1.12 +[2025-04-27 00:31:15] (step=0011325) Train Loss: 6.1595, Train Steps/Sec: 1.12 +[2025-04-27 00:31:38] (step=0011350) Train Loss: 6.1687, Train Steps/Sec: 1.11 +[2025-04-27 00:32:00] (step=0011375) Train Loss: 6.1264, Train Steps/Sec: 1.12 +[2025-04-27 00:32:23] (step=0011400) Train Loss: 6.2009, Train Steps/Sec: 1.11 +[2025-04-27 00:32:45] (step=0011425) Train Loss: 6.1524, Train Steps/Sec: 1.12 +[2025-04-27 00:33:08] (step=0011450) Train Loss: 6.1863, Train Steps/Sec: 1.12 +[2025-04-27 00:33:30] (step=0011475) Train Loss: 6.1403, Train Steps/Sec: 1.12 +[2025-04-27 00:33:52] (step=0011500) Train Loss: 6.1082, Train Steps/Sec: 1.12 +[2025-04-27 00:34:15] (step=0011525) Train Loss: 6.0971, Train Steps/Sec: 1.12 +[2025-04-27 00:34:37] (step=0011550) Train Loss: 6.1674, Train Steps/Sec: 1.12 +[2025-04-27 00:34:59] (step=0011575) Train Loss: 6.1779, Train Steps/Sec: 1.12 +[2025-04-27 00:35:22] (step=0011600) Train Loss: 6.1627, Train Steps/Sec: 1.11 +[2025-04-27 00:35:44] (step=0011625) Train Loss: 6.1255, Train Steps/Sec: 1.12 +[2025-04-27 00:36:07] (step=0011650) Train Loss: 6.1608, Train Steps/Sec: 1.12 +[2025-04-27 00:36:29] (step=0011675) Train Loss: 6.1971, Train Steps/Sec: 1.12 +[2025-04-27 00:36:51] (step=0011700) Train Loss: 6.1749, Train Steps/Sec: 1.12 +[2025-04-27 00:37:14] (step=0011725) Train Loss: 6.1544, Train Steps/Sec: 1.12 +[2025-04-27 00:37:36] (step=0011750) Train Loss: 6.1103, Train Steps/Sec: 1.12 +[2025-04-27 00:37:58] (step=0011775) Train Loss: 6.2335, Train Steps/Sec: 1.12 +[2025-04-27 00:38:21] (step=0011800) Train Loss: 6.1485, Train Steps/Sec: 1.11 +[2025-04-27 00:38:43] (step=0011825) Train Loss: 6.1077, Train Steps/Sec: 1.12 +[2025-04-27 00:39:05] (step=0011850) Train Loss: 6.1457, Train Steps/Sec: 1.12 +[2025-04-27 00:39:28] (step=0011875) Train Loss: 6.1432, Train Steps/Sec: 1.12 +[2025-04-27 00:39:50] (step=0011900) Train Loss: 6.1250, Train Steps/Sec: 1.12 +[2025-04-27 00:40:12] (step=0011925) Train Loss: 6.1634, Train Steps/Sec: 1.12 +[2025-04-27 00:40:35] (step=0011950) Train Loss: 6.1627, Train Steps/Sec: 1.12 +[2025-04-27 00:40:57] (step=0011975) Train Loss: 6.1477, Train Steps/Sec: 1.12 +[2025-04-27 00:41:20] (step=0012000) Train Loss: 6.1416, Train Steps/Sec: 1.12 +[2025-04-27 00:41:20] text_config is None. Initializing the text config with default values (`OPTConfig`). +[2025-04-27 00:47:26] Finish Eval in 12000 steps... +[2025-04-27 00:47:46] Saved checkpoint to checkpoint/X2I_UltraEdiit_fourmask_LLava_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_filtered_objects_4mask_1280_context/003-GPT-XL/checkpoints/0012000.pt +[2025-04-27 00:47:48] Removed old checkpoint: checkpoint/X2I_UltraEdiit_fourmask_LLava_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_filtered_objects_4mask_1280_context/003-GPT-XL/checkpoints/0010000.pt +[2025-04-27 00:48:10] (step=0012025) Train Loss: 6.1299, Train Steps/Sec: 0.06 +[2025-04-27 00:48:33] (step=0012050) Train Loss: 6.1623, Train Steps/Sec: 1.12 +[2025-04-27 00:48:55] (step=0012075) Train Loss: 6.1050, Train Steps/Sec: 1.12 +[2025-04-27 00:49:17] (step=0012100) Train Loss: 6.1223, Train Steps/Sec: 1.12 +[2025-04-27 00:49:40] (step=0012125) Train Loss: 6.1452, Train Steps/Sec: 1.12 +[2025-04-27 00:50:02] (step=0012150) Train Loss: 6.1565, Train Steps/Sec: 1.12 +[2025-04-27 00:50:24] (step=0012175) Train Loss: 6.1869, Train Steps/Sec: 1.12 +[2025-04-27 00:50:47] (step=0012200) Train Loss: 6.1185, Train Steps/Sec: 1.11 +[2025-04-27 00:51:09] (step=0012225) Train Loss: 6.1121, Train Steps/Sec: 1.12 +[2025-04-27 00:51:32] (step=0012250) Train Loss: 6.0968, Train Steps/Sec: 1.12 +[2025-04-27 00:51:54] (step=0012275) Train Loss: 6.1237, Train Steps/Sec: 1.12 +[2025-04-27 00:52:16] (step=0012300) Train Loss: 6.1436, Train Steps/Sec: 1.12 +[2025-04-27 00:52:39] (step=0012325) Train Loss: 6.1315, Train Steps/Sec: 1.12 +[2025-04-27 00:53:01] (step=0012350) Train Loss: 6.0971, Train Steps/Sec: 1.12 +[2025-04-27 00:53:23] (step=0012375) Train Loss: 6.1289, Train Steps/Sec: 1.12 +[2025-04-27 00:53:46] (step=0012400) Train Loss: 6.1283, Train Steps/Sec: 1.12 +[2025-04-27 00:54:08] (step=0012425) Train Loss: 6.1752, Train Steps/Sec: 1.12 +[2025-04-27 00:54:31] (step=0012450) Train Loss: 6.1276, Train Steps/Sec: 1.12 +[2025-04-27 00:54:53] (step=0012475) Train Loss: 6.1435, Train Steps/Sec: 1.12 +[2025-04-27 00:55:15] (step=0012500) Train Loss: 6.1084, Train Steps/Sec: 1.12 +[2025-04-27 00:55:38] (step=0012525) Train Loss: 6.1778, Train Steps/Sec: 1.12 +[2025-04-27 00:56:00] (step=0012550) Train Loss: 6.1054, Train Steps/Sec: 1.12 +[2025-04-27 00:56:22] (step=0012575) Train Loss: 6.1609, Train Steps/Sec: 1.12 +[2025-04-27 00:56:45] (step=0012600) Train Loss: 6.1276, Train Steps/Sec: 1.11 +[2025-04-27 00:57:07] (step=0012625) Train Loss: 6.2570, Train Steps/Sec: 1.12 +[2025-04-27 00:57:29] (step=0012650) Train Loss: 6.0886, Train Steps/Sec: 1.12 +[2025-04-27 00:57:52] (step=0012675) Train Loss: 6.1545, Train Steps/Sec: 1.12 +[2025-04-27 00:58:14] (step=0012700) Train Loss: 6.1164, Train Steps/Sec: 1.12 +[2025-04-27 00:58:37] (step=0012725) Train Loss: 6.0929, Train Steps/Sec: 1.12 +[2025-04-27 00:58:59] (step=0012750) Train Loss: 6.1327, Train Steps/Sec: 1.12 +[2025-04-27 00:59:21] (step=0012775) Train Loss: 6.1621, Train Steps/Sec: 1.12 +[2025-04-27 00:59:44] (step=0012800) Train Loss: 6.1154, Train Steps/Sec: 1.12 +[2025-04-27 01:00:06] (step=0012825) Train Loss: 6.0943, Train Steps/Sec: 1.12 +[2025-04-27 01:00:28] (step=0012850) Train Loss: 6.1357, Train Steps/Sec: 1.12 +[2025-04-27 01:00:51] (step=0012875) Train Loss: 6.0476, Train Steps/Sec: 1.12 +[2025-04-27 01:01:13] (step=0012900) Train Loss: 6.1633, Train Steps/Sec: 1.12 +[2025-04-27 01:01:36] (step=0012925) Train Loss: 6.1268, Train Steps/Sec: 1.12 +[2025-04-27 01:01:58] (step=0012950) Train Loss: 6.0752, Train Steps/Sec: 1.12 +[2025-04-27 01:02:20] (step=0012975) Train Loss: 6.1316, Train Steps/Sec: 1.12 +[2025-04-27 01:02:43] (step=0013000) Train Loss: 6.1158, Train Steps/Sec: 1.11 +[2025-04-27 01:03:05] (step=0013025) Train Loss: 6.0956, Train Steps/Sec: 1.12 +[2025-04-27 01:03:27] (step=0013050) Train Loss: 6.0899, Train Steps/Sec: 1.12 +[2025-04-27 01:03:50] (step=0013075) Train Loss: 6.0800, Train Steps/Sec: 1.12 +[2025-04-27 01:04:12] (step=0013100) Train Loss: 6.0516, Train Steps/Sec: 1.12 +[2025-04-27 01:04:34] (step=0013125) Train Loss: 6.0976, Train Steps/Sec: 1.12 +[2025-04-27 01:04:57] (step=0013150) Train Loss: 6.1257, Train Steps/Sec: 1.12 +[2025-04-27 01:05:19] (step=0013175) Train Loss: 6.1248, Train Steps/Sec: 1.12 +[2025-04-27 01:05:42] (step=0013200) Train Loss: 6.1227, Train Steps/Sec: 1.12 +[2025-04-27 01:06:04] (step=0013225) Train Loss: 6.0392, Train Steps/Sec: 1.12 +[2025-04-27 01:06:26] (step=0013250) Train Loss: 6.1242, Train Steps/Sec: 1.12 +[2025-04-27 01:06:49] (step=0013275) Train Loss: 6.0973, Train Steps/Sec: 1.10 +[2025-04-27 01:07:11] (step=0013300) Train Loss: 6.1151, Train Steps/Sec: 1.12 +[2025-04-27 01:07:34] (step=0013325) Train Loss: 6.1187, Train Steps/Sec: 1.12 +[2025-04-27 01:07:56] (step=0013350) Train Loss: 6.1397, Train Steps/Sec: 1.12 +[2025-04-27 01:08:18] (step=0013375) Train Loss: 6.0894, Train Steps/Sec: 1.12 +[2025-04-27 01:08:41] (step=0013400) Train Loss: 6.1662, Train Steps/Sec: 1.12 +[2025-04-27 01:09:03] (step=0013425) Train Loss: 6.1208, Train Steps/Sec: 1.12 +[2025-04-27 01:09:26] (step=0013450) Train Loss: 6.1432, Train Steps/Sec: 1.12 +[2025-04-27 01:09:48] (step=0013475) Train Loss: 6.0549, Train Steps/Sec: 1.12 +[2025-04-27 01:10:10] (step=0013500) Train Loss: 6.0870, Train Steps/Sec: 1.12 +[2025-04-27 01:10:33] (step=0013525) Train Loss: 6.0915, Train Steps/Sec: 1.12 +[2025-04-27 01:10:55] (step=0013550) Train Loss: 6.0999, Train Steps/Sec: 1.12 +[2025-04-27 01:11:17] (step=0013575) Train Loss: 6.1139, Train Steps/Sec: 1.12 +[2025-04-27 01:11:40] (step=0013600) Train Loss: 6.1332, Train Steps/Sec: 1.12 +[2025-04-27 01:12:02] (step=0013625) Train Loss: 6.0852, Train Steps/Sec: 1.12 +[2025-04-27 01:12:24] (step=0013650) Train Loss: 6.1050, Train Steps/Sec: 1.12 +[2025-04-27 01:12:47] (step=0013675) Train Loss: 6.1376, Train Steps/Sec: 1.12 +[2025-04-27 01:13:09] (step=0013700) Train Loss: 6.1209, Train Steps/Sec: 1.12 +[2025-04-27 01:13:31] (step=0013725) Train Loss: 6.0896, Train Steps/Sec: 1.12 +[2025-04-27 01:13:54] (step=0013750) Train Loss: 6.0444, Train Steps/Sec: 1.12 +[2025-04-27 01:14:16] (step=0013775) Train Loss: 6.1085, Train Steps/Sec: 1.12 +[2025-04-27 01:14:38] (step=0013800) Train Loss: 6.1145, Train Steps/Sec: 1.12 +[2025-04-27 01:15:01] (step=0013825) Train Loss: 6.1240, Train Steps/Sec: 1.12 +[2025-04-27 01:15:23] (step=0013850) Train Loss: 6.0972, Train Steps/Sec: 1.12 +[2025-04-27 01:15:45] (step=0013875) Train Loss: 6.1266, Train Steps/Sec: 1.12 +[2025-04-27 01:16:08] (step=0013900) Train Loss: 6.1704, Train Steps/Sec: 1.12 +[2025-04-27 01:16:30] (step=0013925) Train Loss: 6.0268, Train Steps/Sec: 1.12 +[2025-04-27 01:16:53] (step=0013950) Train Loss: 6.0798, Train Steps/Sec: 1.12 +[2025-04-27 01:17:15] (step=0013975) Train Loss: 6.1048, Train Steps/Sec: 1.12 +[2025-04-27 01:17:37] (step=0014000) Train Loss: 6.0873, Train Steps/Sec: 1.12 +[2025-04-27 01:17:37] text_config is None. Initializing the text config with default values (`OPTConfig`). +[2025-04-27 01:23:45] Finish Eval in 14000 steps... +[2025-04-27 01:24:04] Saved checkpoint to checkpoint/X2I_UltraEdiit_fourmask_LLava_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_filtered_objects_4mask_1280_context/003-GPT-XL/checkpoints/0014000.pt +[2025-04-27 01:24:06] Removed old checkpoint: checkpoint/X2I_UltraEdiit_fourmask_LLava_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_filtered_objects_4mask_1280_context/003-GPT-XL/checkpoints/0012000.pt +[2025-04-27 01:24:29] (step=0014025) Train Loss: 6.0562, Train Steps/Sec: 0.06 +[2025-04-27 01:24:51] (step=0014050) Train Loss: 6.1414, Train Steps/Sec: 1.12 +[2025-04-27 01:25:14] (step=0014075) Train Loss: 6.1442, Train Steps/Sec: 1.12 +[2025-04-27 01:25:36] (step=0014100) Train Loss: 6.1105, Train Steps/Sec: 1.12 +[2025-04-27 01:25:58] (step=0014125) Train Loss: 6.1153, Train Steps/Sec: 1.12 +[2025-04-27 01:26:21] (step=0014150) Train Loss: 6.0646, Train Steps/Sec: 1.12 +[2025-04-27 01:26:43] (step=0014175) Train Loss: 6.1055, Train Steps/Sec: 1.12 +[2025-04-27 01:27:05] (step=0014200) Train Loss: 6.1276, Train Steps/Sec: 1.12 +[2025-04-27 01:27:28] (step=0014225) Train Loss: 6.1511, Train Steps/Sec: 1.12 +[2025-04-27 01:27:50] (step=0014250) Train Loss: 6.0543, Train Steps/Sec: 1.12 +[2025-04-27 01:28:12] (step=0014275) Train Loss: 6.0797, Train Steps/Sec: 1.12 +[2025-04-27 01:28:35] (step=0014300) Train Loss: 6.1078, Train Steps/Sec: 1.12 +[2025-04-27 01:28:57] (step=0014325) Train Loss: 6.0549, Train Steps/Sec: 1.12 +[2025-04-27 01:29:19] (step=0014350) Train Loss: 6.1485, Train Steps/Sec: 1.12 +[2025-04-27 01:29:42] (step=0014375) Train Loss: 6.0924, Train Steps/Sec: 1.12 +[2025-04-27 01:30:04] (step=0014400) Train Loss: 6.0661, Train Steps/Sec: 1.11 +[2025-04-27 01:30:26] (step=0014425) Train Loss: 6.0936, Train Steps/Sec: 1.12 +[2025-04-27 01:30:49] (step=0014450) Train Loss: 6.0994, Train Steps/Sec: 1.12 +[2025-04-27 01:31:11] (step=0014475) Train Loss: 6.1111, Train Steps/Sec: 1.12 +[2025-04-27 01:31:33] (step=0014500) Train Loss: 6.1018, Train Steps/Sec: 1.12 +[2025-04-27 01:31:56] (step=0014525) Train Loss: 6.0495, Train Steps/Sec: 1.12 +[2025-04-27 01:32:18] (step=0014550) Train Loss: 6.1786, Train Steps/Sec: 1.12 +[2025-04-27 01:32:41] (step=0014575) Train Loss: 6.1120, Train Steps/Sec: 1.12 +[2025-04-27 01:33:03] (step=0014600) Train Loss: 6.1076, Train Steps/Sec: 1.11 +[2025-04-27 01:33:30] (step=0014625) Train Loss: 6.1094, Train Steps/Sec: 0.91 +[2025-04-27 01:33:53] (step=0014650) Train Loss: 6.0458, Train Steps/Sec: 1.12 +[2025-04-27 01:34:15] (step=0014675) Train Loss: 6.0871, Train Steps/Sec: 1.12 +[2025-04-27 01:34:37] (step=0014700) Train Loss: 6.1138, Train Steps/Sec: 1.12 +[2025-04-27 01:35:00] (step=0014725) Train Loss: 6.0893, Train Steps/Sec: 1.12 +[2025-04-27 01:35:22] (step=0014750) Train Loss: 6.1121, Train Steps/Sec: 1.12 +[2025-04-27 01:35:45] (step=0014775) Train Loss: 6.1131, Train Steps/Sec: 1.12 +[2025-04-27 01:36:07] (step=0014800) Train Loss: 6.1081, Train Steps/Sec: 1.12 +[2025-04-27 01:36:29] (step=0014825) Train Loss: 6.1170, Train Steps/Sec: 1.12 +[2025-04-27 01:36:52] (step=0014850) Train Loss: 6.1596, Train Steps/Sec: 1.12 +[2025-04-27 01:37:14] (step=0014875) Train Loss: 6.1337, Train Steps/Sec: 1.12 +[2025-04-27 01:37:36] (step=0014900) Train Loss: 6.1442, Train Steps/Sec: 1.12 +[2025-04-27 01:37:59] (step=0014925) Train Loss: 6.1756, Train Steps/Sec: 1.12 +[2025-04-27 01:38:21] (step=0014950) Train Loss: 6.0803, Train Steps/Sec: 1.12 +[2025-04-27 01:38:43] (step=0014975) Train Loss: 6.0509, Train Steps/Sec: 1.12 +[2025-04-27 01:39:06] (step=0015000) Train Loss: 6.0728, Train Steps/Sec: 1.12 +[2025-04-27 01:39:28] (step=0015025) Train Loss: 6.1645, Train Steps/Sec: 1.12 +[2025-04-27 01:39:50] (step=0015050) Train Loss: 6.1137, Train Steps/Sec: 1.12 +[2025-04-27 01:40:13] (step=0015075) Train Loss: 6.1229, Train Steps/Sec: 1.12 +[2025-04-27 01:40:35] (step=0015100) Train Loss: 6.0835, Train Steps/Sec: 1.12 +[2025-04-27 01:40:57] (step=0015125) Train Loss: 6.0864, Train Steps/Sec: 1.12 +[2025-04-27 01:41:20] (step=0015150) Train Loss: 6.1318, Train Steps/Sec: 1.12 +[2025-04-27 01:41:42] (step=0015175) Train Loss: 6.0789, Train Steps/Sec: 1.12 +[2025-04-27 01:42:04] (step=0015200) Train Loss: 6.1044, Train Steps/Sec: 1.12 +[2025-04-27 01:42:27] (step=0015225) Train Loss: 6.1337, Train Steps/Sec: 1.12 +[2025-04-27 01:42:49] (step=0015250) Train Loss: 6.0789, Train Steps/Sec: 1.12 +[2025-04-27 01:43:11] (step=0015275) Train Loss: 6.0628, Train Steps/Sec: 1.12 +[2025-04-27 01:43:34] (step=0015300) Train Loss: 6.0544, Train Steps/Sec: 1.12 +[2025-04-27 01:43:56] (step=0015325) Train Loss: 6.1339, Train Steps/Sec: 1.12 +[2025-04-27 01:44:18] (step=0015350) Train Loss: 6.0963, Train Steps/Sec: 1.12 +[2025-04-27 01:44:55] (step=0015375) Train Loss: 6.0122, Train Steps/Sec: 0.67 +[2025-04-27 01:45:23] (step=0015400) Train Loss: 6.0854, Train Steps/Sec: 0.90 +[2025-04-27 01:45:46] (step=0015425) Train Loss: 6.0612, Train Steps/Sec: 1.12 +[2025-04-27 01:46:08] (step=0015450) Train Loss: 6.0580, Train Steps/Sec: 1.12 +[2025-04-27 01:46:35] (step=0015475) Train Loss: 6.0434, Train Steps/Sec: 0.92 +[2025-04-27 01:47:02] (step=0015500) Train Loss: 6.1113, Train Steps/Sec: 0.93 +[2025-04-27 01:47:25] (step=0015525) Train Loss: 6.1056, Train Steps/Sec: 1.12 +[2025-04-27 01:47:47] (step=0015550) Train Loss: 6.1003, Train Steps/Sec: 1.12 +[2025-04-27 01:48:09] (step=0015575) Train Loss: 6.1463, Train Steps/Sec: 1.12 +[2025-04-27 01:48:32] (step=0015600) Train Loss: 6.0799, Train Steps/Sec: 1.12 +[2025-04-27 01:48:54] (step=0015625) Train Loss: 6.0913, Train Steps/Sec: 1.12 +[2025-04-27 01:49:16] (step=0015650) Train Loss: 6.1104, Train Steps/Sec: 1.12 +[2025-04-27 01:49:39] (step=0015675) Train Loss: 6.1006, Train Steps/Sec: 1.12 +[2025-04-27 01:50:01] (step=0015700) Train Loss: 6.0872, Train Steps/Sec: 1.12 +[2025-04-27 01:50:23] (step=0015725) Train Loss: 6.0669, Train Steps/Sec: 1.12 +[2025-04-27 01:50:46] (step=0015750) Train Loss: 6.0921, Train Steps/Sec: 1.12 +[2025-04-27 01:51:08] (step=0015775) Train Loss: 5.9975, Train Steps/Sec: 1.12 +[2025-04-27 01:51:31] (step=0015800) Train Loss: 6.0806, Train Steps/Sec: 1.12 +[2025-04-27 01:51:53] (step=0015825) Train Loss: 6.0898, Train Steps/Sec: 1.12 +[2025-04-27 01:52:15] (step=0015850) Train Loss: 6.0789, Train Steps/Sec: 1.12 +[2025-04-27 01:52:38] (step=0015875) Train Loss: 6.0330, Train Steps/Sec: 1.12 +[2025-04-27 01:53:00] (step=0015900) Train Loss: 6.1404, Train Steps/Sec: 1.12 +[2025-04-27 01:53:22] (step=0015925) Train Loss: 6.1283, Train Steps/Sec: 1.12 +[2025-04-27 01:53:45] (step=0015950) Train Loss: 6.1368, Train Steps/Sec: 1.12 +[2025-04-27 01:54:07] (step=0015975) Train Loss: 6.1057, Train Steps/Sec: 1.12 +[2025-04-27 01:54:29] (step=0016000) Train Loss: 6.0916, Train Steps/Sec: 1.11 +[2025-04-27 01:54:29] text_config is None. Initializing the text config with default values (`OPTConfig`). +[2025-04-27 02:00:38] Finish Eval in 16000 steps... +[2025-04-27 02:00:57] Saved checkpoint to checkpoint/X2I_UltraEdiit_fourmask_LLava_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_filtered_objects_4mask_1280_context/003-GPT-XL/checkpoints/0016000.pt +[2025-04-27 02:00:59] Removed old checkpoint: checkpoint/X2I_UltraEdiit_fourmask_LLava_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_filtered_objects_4mask_1280_context/003-GPT-XL/checkpoints/0014000.pt +[2025-04-27 02:01:21] (step=0016025) Train Loss: 6.1124, Train Steps/Sec: 0.06 +[2025-04-27 02:01:44] (step=0016050) Train Loss: 6.1041, Train Steps/Sec: 1.12 +[2025-04-27 02:02:06] (step=0016075) Train Loss: 6.0960, Train Steps/Sec: 1.12 +[2025-04-27 02:02:28] (step=0016100) Train Loss: 6.0806, Train Steps/Sec: 1.12 +[2025-04-27 02:02:51] (step=0016125) Train Loss: 6.0537, Train Steps/Sec: 1.12 +[2025-04-27 02:03:13] (step=0016150) Train Loss: 6.0664, Train Steps/Sec: 1.12 +[2025-04-27 02:03:35] (step=0016175) Train Loss: 6.0893, Train Steps/Sec: 1.12 +[2025-04-27 02:03:58] (step=0016200) Train Loss: 6.0300, Train Steps/Sec: 1.11 +[2025-04-27 02:04:20] (step=0016225) Train Loss: 6.0890, Train Steps/Sec: 1.12 +[2025-04-27 02:04:42] (step=0016250) Train Loss: 6.0990, Train Steps/Sec: 1.12 +[2025-04-27 02:05:05] (step=0016275) Train Loss: 6.1133, Train Steps/Sec: 1.12 +[2025-04-27 02:05:27] (step=0016300) Train Loss: 6.0539, Train Steps/Sec: 1.12 +[2025-04-27 02:05:50] (step=0016325) Train Loss: 6.1702, Train Steps/Sec: 1.12 +[2025-04-27 02:06:12] (step=0016350) Train Loss: 6.1367, Train Steps/Sec: 1.12 +[2025-04-27 02:06:34] (step=0016375) Train Loss: 6.0733, Train Steps/Sec: 1.12 +[2025-04-27 02:06:57] (step=0016400) Train Loss: 6.1358, Train Steps/Sec: 1.11 +[2025-04-27 02:07:19] (step=0016425) Train Loss: 6.0781, Train Steps/Sec: 1.12 +[2025-04-27 02:07:41] (step=0016450) Train Loss: 6.1309, Train Steps/Sec: 1.12 +[2025-04-27 02:08:04] (step=0016475) Train Loss: 6.1061, Train Steps/Sec: 1.12 +[2025-04-27 02:08:26] (step=0016500) Train Loss: 6.1064, Train Steps/Sec: 1.12 +[2025-04-27 02:08:48] (step=0016525) Train Loss: 6.1254, Train Steps/Sec: 1.12 +[2025-04-27 02:09:11] (step=0016550) Train Loss: 6.0575, Train Steps/Sec: 1.12 +[2025-04-27 02:09:33] (step=0016575) Train Loss: 6.0622, Train Steps/Sec: 1.12 +[2025-04-27 02:09:56] (step=0016600) Train Loss: 6.0466, Train Steps/Sec: 1.11 +[2025-04-27 02:10:18] (step=0016625) Train Loss: 6.1212, Train Steps/Sec: 1.12 +[2025-04-27 02:10:40] (step=0016650) Train Loss: 6.0926, Train Steps/Sec: 1.12 +[2025-04-27 02:11:03] (step=0016675) Train Loss: 6.0318, Train Steps/Sec: 1.12 +[2025-04-27 02:11:25] (step=0016700) Train Loss: 6.0590, Train Steps/Sec: 1.12 +[2025-04-27 02:11:47] (step=0016725) Train Loss: 6.1127, Train Steps/Sec: 1.12 +[2025-04-27 02:12:10] (step=0016750) Train Loss: 6.0631, Train Steps/Sec: 1.12 +[2025-04-27 02:12:32] (step=0016775) Train Loss: 6.0829, Train Steps/Sec: 1.12 +[2025-04-27 02:12:54] (step=0016800) Train Loss: 6.0432, Train Steps/Sec: 1.11 +[2025-04-27 02:13:17] (step=0016825) Train Loss: 6.0650, Train Steps/Sec: 1.12 +[2025-04-27 02:13:39] (step=0016850) Train Loss: 6.0700, Train Steps/Sec: 1.12 +[2025-04-27 02:14:02] (step=0016875) Train Loss: 6.0963, Train Steps/Sec: 1.12 +[2025-04-27 02:14:24] (step=0016900) Train Loss: 6.1048, Train Steps/Sec: 1.12 +[2025-04-27 02:14:46] (step=0016925) Train Loss: 6.1113, Train Steps/Sec: 1.12 +[2025-04-27 02:15:09] (step=0016950) Train Loss: 6.0995, Train Steps/Sec: 1.12 +[2025-04-27 02:15:31] (step=0016975) Train Loss: 6.0022, Train Steps/Sec: 1.12 +[2025-04-27 02:15:53] (step=0017000) Train Loss: 6.1580, Train Steps/Sec: 1.11 +[2025-04-27 02:16:16] (step=0017025) Train Loss: 6.1035, Train Steps/Sec: 1.12 +[2025-04-27 02:16:38] (step=0017050) Train Loss: 6.1473, Train Steps/Sec: 1.12 +[2025-04-27 02:17:00] (step=0017075) Train Loss: 6.0496, Train Steps/Sec: 1.12 +[2025-04-27 02:17:23] (step=0017100) Train Loss: 6.0831, Train Steps/Sec: 1.12 +[2025-04-27 02:17:45] (step=0017125) Train Loss: 6.1341, Train Steps/Sec: 1.12 +[2025-04-27 02:18:08] (step=0017150) Train Loss: 6.0335, Train Steps/Sec: 1.12 +[2025-04-27 02:18:30] (step=0017175) Train Loss: 6.0124, Train Steps/Sec: 1.12 +[2025-04-27 02:18:52] (step=0017200) Train Loss: 6.1200, Train Steps/Sec: 1.11 +[2025-04-27 02:19:15] (step=0017225) Train Loss: 6.0233, Train Steps/Sec: 1.12 +[2025-04-27 02:19:37] (step=0017250) Train Loss: 6.0627, Train Steps/Sec: 1.12 +[2025-04-27 02:19:59] (step=0017275) Train Loss: 6.0995, Train Steps/Sec: 1.12 +[2025-04-27 02:20:22] (step=0017300) Train Loss: 6.0261, Train Steps/Sec: 1.12 +[2025-04-27 02:20:44] (step=0017325) Train Loss: 6.0161, Train Steps/Sec: 1.12 +[2025-04-27 02:21:06] (step=0017350) Train Loss: 6.0406, Train Steps/Sec: 1.12 +[2025-04-27 02:21:29] (step=0017375) Train Loss: 6.0063, Train Steps/Sec: 1.12 +[2025-04-27 02:21:51] (step=0017400) Train Loss: 6.0467, Train Steps/Sec: 1.11 +[2025-04-27 02:22:14] (step=0017425) Train Loss: 6.0926, Train Steps/Sec: 1.12 +[2025-04-27 02:22:36] (step=0017450) Train Loss: 6.0556, Train Steps/Sec: 1.12 +[2025-04-27 02:22:58] (step=0017475) Train Loss: 5.9977, Train Steps/Sec: 1.12 +[2025-04-27 02:23:21] (step=0017500) Train Loss: 6.0642, Train Steps/Sec: 1.12 +[2025-04-27 02:23:43] (step=0017525) Train Loss: 6.0977, Train Steps/Sec: 1.12 +[2025-04-27 02:24:05] (step=0017550) Train Loss: 6.0742, Train Steps/Sec: 1.12 +[2025-04-27 02:24:28] (step=0017575) Train Loss: 6.0824, Train Steps/Sec: 1.12 +[2025-04-27 02:24:50] (step=0017600) Train Loss: 6.0264, Train Steps/Sec: 1.11 +[2025-04-27 02:25:13] (step=0017625) Train Loss: 6.0727, Train Steps/Sec: 1.12 +[2025-04-27 02:25:35] (step=0017650) Train Loss: 6.0556, Train Steps/Sec: 1.12 +[2025-04-27 02:25:57] (step=0017675) Train Loss: 5.9822, Train Steps/Sec: 1.12 +[2025-04-27 02:26:20] (step=0017700) Train Loss: 6.0432, Train Steps/Sec: 1.12 +[2025-04-27 02:26:42] (step=0017725) Train Loss: 6.0821, Train Steps/Sec: 1.12 +[2025-04-27 02:27:04] (step=0017750) Train Loss: 6.0582, Train Steps/Sec: 1.12 +[2025-04-27 02:27:27] (step=0017775) Train Loss: 6.0812, Train Steps/Sec: 1.12 +[2025-04-27 02:27:49] (step=0017800) Train Loss: 6.0760, Train Steps/Sec: 1.11 +[2025-04-27 02:28:12] (step=0017825) Train Loss: 6.0625, Train Steps/Sec: 1.12 +[2025-04-27 02:28:34] (step=0017850) Train Loss: 6.0301, Train Steps/Sec: 1.12 +[2025-04-27 02:28:56] (step=0017875) Train Loss: 6.0714, Train Steps/Sec: 1.12 +[2025-04-27 02:29:19] (step=0017900) Train Loss: 6.0525, Train Steps/Sec: 1.12 +[2025-04-27 02:29:41] (step=0017925) Train Loss: 6.0116, Train Steps/Sec: 1.12 +[2025-04-27 02:30:03] (step=0017950) Train Loss: 6.0398, Train Steps/Sec: 1.12 +[2025-04-27 02:30:26] (step=0017975) Train Loss: 6.0584, Train Steps/Sec: 1.12 +[2025-04-27 02:30:48] (step=0018000) Train Loss: 6.0731, Train Steps/Sec: 1.11 +[2025-04-27 02:30:48] text_config is None. Initializing the text config with default values (`OPTConfig`). +[2025-04-27 02:36:57] Finish Eval in 18000 steps... +[2025-04-27 02:37:16] Saved checkpoint to checkpoint/X2I_UltraEdiit_fourmask_LLava_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_filtered_objects_4mask_1280_context/003-GPT-XL/checkpoints/0018000.pt +[2025-04-27 02:37:19] Removed old checkpoint: checkpoint/X2I_UltraEdiit_fourmask_LLava_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_filtered_objects_4mask_1280_context/003-GPT-XL/checkpoints/0016000.pt +[2025-04-27 02:37:41] (step=0018025) Train Loss: 6.0224, Train Steps/Sec: 0.06 +[2025-04-27 02:38:03] (step=0018050) Train Loss: 6.0157, Train Steps/Sec: 1.12 +[2025-04-27 02:38:26] (step=0018075) Train Loss: 6.0786, Train Steps/Sec: 1.12 +[2025-04-27 02:38:48] (step=0018100) Train Loss: 6.0685, Train Steps/Sec: 1.12 +[2025-04-27 02:39:10] (step=0018125) Train Loss: 6.1247, Train Steps/Sec: 1.12 +[2025-04-27 02:39:33] (step=0018150) Train Loss: 6.0746, Train Steps/Sec: 1.12 +[2025-04-27 02:39:55] (step=0018175) Train Loss: 6.0298, Train Steps/Sec: 1.12 +[2025-04-27 02:40:17] (step=0018200) Train Loss: 6.0326, Train Steps/Sec: 1.12 +[2025-04-27 02:40:40] (step=0018225) Train Loss: 6.1006, Train Steps/Sec: 1.12 +[2025-04-27 02:41:02] (step=0018250) Train Loss: 6.0143, Train Steps/Sec: 1.12 +[2025-04-27 02:41:24] (step=0018275) Train Loss: 6.0168, Train Steps/Sec: 1.12 +[2025-04-27 02:41:47] (step=0018300) Train Loss: 6.0495, Train Steps/Sec: 1.12 +[2025-04-27 02:42:09] (step=0018325) Train Loss: 6.1018, Train Steps/Sec: 1.12 +[2025-04-27 02:42:31] (step=0018350) Train Loss: 6.0922, Train Steps/Sec: 1.12 +[2025-04-27 02:42:54] (step=0018375) Train Loss: 6.0285, Train Steps/Sec: 1.12 +[2025-04-27 02:43:16] (step=0018400) Train Loss: 6.0540, Train Steps/Sec: 1.12 +[2025-04-27 02:43:38] (step=0018425) Train Loss: 6.0862, Train Steps/Sec: 1.12 +[2025-04-27 02:44:01] (step=0018450) Train Loss: 6.0698, Train Steps/Sec: 1.12 +[2025-04-27 02:44:23] (step=0018475) Train Loss: 6.0122, Train Steps/Sec: 1.12 +[2025-04-27 02:44:45] (step=0018500) Train Loss: 6.0453, Train Steps/Sec: 1.12 +[2025-04-27 02:45:08] (step=0018525) Train Loss: 6.0491, Train Steps/Sec: 1.12 +[2025-04-27 02:45:30] (step=0018550) Train Loss: 6.0260, Train Steps/Sec: 1.12 +[2025-04-27 02:45:52] (step=0018575) Train Loss: 6.0623, Train Steps/Sec: 1.12 +[2025-04-27 02:46:15] (step=0018600) Train Loss: 6.0388, Train Steps/Sec: 1.11 +[2025-04-27 02:46:37] (step=0018625) Train Loss: 6.0774, Train Steps/Sec: 1.12 +[2025-04-27 02:46:59] (step=0018650) Train Loss: 6.0422, Train Steps/Sec: 1.12 +[2025-04-27 02:47:22] (step=0018675) Train Loss: 6.0103, Train Steps/Sec: 1.12 +[2025-04-27 02:47:44] (step=0018700) Train Loss: 5.9635, Train Steps/Sec: 1.12 +[2025-04-27 02:48:06] (step=0018725) Train Loss: 6.1013, Train Steps/Sec: 1.12 +[2025-04-27 02:48:29] (step=0018750) Train Loss: 6.0372, Train Steps/Sec: 1.12 +[2025-04-27 02:48:51] (step=0018775) Train Loss: 6.0179, Train Steps/Sec: 1.12 +[2025-04-27 02:49:14] (step=0018800) Train Loss: 6.0566, Train Steps/Sec: 1.12 +[2025-04-27 02:49:36] (step=0018825) Train Loss: 5.9799, Train Steps/Sec: 1.12 +[2025-04-27 02:49:58] (step=0018850) Train Loss: 6.0722, Train Steps/Sec: 1.12 +[2025-04-27 02:50:21] (step=0018875) Train Loss: 6.0879, Train Steps/Sec: 1.12 +[2025-04-27 02:50:43] (step=0018900) Train Loss: 5.9990, Train Steps/Sec: 1.12 +[2025-04-27 02:51:05] (step=0018925) Train Loss: 6.0497, Train Steps/Sec: 1.12 +[2025-04-27 02:51:27] (step=0018950) Train Loss: 6.0621, Train Steps/Sec: 1.12 +[2025-04-27 02:51:50] (step=0018975) Train Loss: 6.0909, Train Steps/Sec: 1.12 +[2025-04-27 02:52:12] (step=0019000) Train Loss: 6.0431, Train Steps/Sec: 1.12 +[2025-04-27 02:52:40] (step=0019025) Train Loss: 6.0058, Train Steps/Sec: 0.91 +[2025-04-27 02:53:02] (step=0019050) Train Loss: 6.0589, Train Steps/Sec: 1.12 +[2025-04-27 02:53:24] (step=0019075) Train Loss: 6.0470, Train Steps/Sec: 1.12 +[2025-04-27 02:53:47] (step=0019100) Train Loss: 6.0424, Train Steps/Sec: 1.12 +[2025-04-27 02:54:09] (step=0019125) Train Loss: 6.0258, Train Steps/Sec: 1.12 +[2025-04-27 02:54:31] (step=0019150) Train Loss: 6.0617, Train Steps/Sec: 1.12 +[2025-04-27 02:54:53] (step=0019175) Train Loss: 6.0232, Train Steps/Sec: 1.12 +[2025-04-27 02:55:16] (step=0019200) Train Loss: 6.0788, Train Steps/Sec: 1.12 +[2025-04-27 02:55:38] (step=0019225) Train Loss: 6.0446, Train Steps/Sec: 1.12 +[2025-04-27 02:56:01] (step=0019250) Train Loss: 6.0622, Train Steps/Sec: 1.12 +[2025-04-27 02:56:23] (step=0019275) Train Loss: 6.0546, Train Steps/Sec: 1.12 +[2025-04-27 02:56:45] (step=0019300) Train Loss: 6.0101, Train Steps/Sec: 1.12 +[2025-04-27 02:57:08] (step=0019325) Train Loss: 5.9827, Train Steps/Sec: 1.12 +[2025-04-27 02:57:30] (step=0019350) Train Loss: 6.0317, Train Steps/Sec: 1.12 +[2025-04-27 02:57:52] (step=0019375) Train Loss: 6.0059, Train Steps/Sec: 1.12 +[2025-04-27 02:58:15] (step=0019400) Train Loss: 6.0662, Train Steps/Sec: 1.12 +[2025-04-27 02:58:37] (step=0019425) Train Loss: 5.9902, Train Steps/Sec: 1.12 +[2025-04-27 02:58:59] (step=0019450) Train Loss: 6.0413, Train Steps/Sec: 1.12 +[2025-04-27 02:59:22] (step=0019475) Train Loss: 6.1079, Train Steps/Sec: 1.12 +[2025-04-27 02:59:44] (step=0019500) Train Loss: 5.9832, Train Steps/Sec: 1.12 +[2025-04-27 03:00:06] (step=0019525) Train Loss: 5.9543, Train Steps/Sec: 1.12 +[2025-04-27 03:00:29] (step=0019550) Train Loss: 6.0190, Train Steps/Sec: 1.12 +[2025-04-27 03:00:51] (step=0019575) Train Loss: 6.0108, Train Steps/Sec: 1.12 +[2025-04-27 03:01:13] (step=0019600) Train Loss: 6.0698, Train Steps/Sec: 1.12 +[2025-04-27 03:01:36] (step=0019625) Train Loss: 6.0659, Train Steps/Sec: 1.12 +[2025-04-27 03:01:58] (step=0019650) Train Loss: 6.0727, Train Steps/Sec: 1.12 +[2025-04-27 03:02:20] (step=0019675) Train Loss: 6.0446, Train Steps/Sec: 1.12 +[2025-04-27 03:02:53] (step=0019700) Train Loss: 6.0191, Train Steps/Sec: 0.77 +[2025-04-27 03:03:25] (step=0019725) Train Loss: 6.0261, Train Steps/Sec: 0.77 +[2025-04-27 03:03:57] (step=0019750) Train Loss: 6.0444, Train Steps/Sec: 0.78 +[2025-04-27 03:04:19] (step=0019775) Train Loss: 6.0021, Train Steps/Sec: 1.12 +[2025-04-27 03:04:42] (step=0019800) Train Loss: 6.0507, Train Steps/Sec: 1.11 +[2025-04-27 03:05:04] (step=0019825) Train Loss: 6.0419, Train Steps/Sec: 1.12 +[2025-04-27 03:05:27] (step=0019850) Train Loss: 5.9829, Train Steps/Sec: 1.12 +[2025-04-27 03:05:49] (step=0019875) Train Loss: 5.9921, Train Steps/Sec: 1.12 +[2025-04-27 03:06:11] (step=0019900) Train Loss: 6.0617, Train Steps/Sec: 1.12 +[2025-04-27 03:06:34] (step=0019925) Train Loss: 6.0471, Train Steps/Sec: 1.12 +[2025-04-27 03:06:56] (step=0019950) Train Loss: 5.9850, Train Steps/Sec: 1.12 +[2025-04-27 03:07:18] (step=0019975) Train Loss: 6.1023, Train Steps/Sec: 1.12 +[2025-04-27 03:07:41] (step=0020000) Train Loss: 6.0145, Train Steps/Sec: 1.12 +[2025-04-27 03:07:41] text_config is None. Initializing the text config with default values (`OPTConfig`). +[2025-04-27 03:13:48] Finish Eval in 20000 steps... +[2025-04-27 03:14:08] Saved checkpoint to checkpoint/X2I_UltraEdiit_fourmask_LLava_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_filtered_objects_4mask_1280_context/003-GPT-XL/checkpoints/0020000.pt +[2025-04-27 03:14:10] Removed old checkpoint: checkpoint/X2I_UltraEdiit_fourmask_LLava_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_filtered_objects_4mask_1280_context/003-GPT-XL/checkpoints/0018000.pt +[2025-04-27 03:14:32] (step=0020025) Train Loss: 6.0319, Train Steps/Sec: 0.06 +[2025-04-27 03:14:54] (step=0020050) Train Loss: 5.9930, Train Steps/Sec: 1.12 +[2025-04-27 03:15:17] (step=0020075) Train Loss: 6.0231, Train Steps/Sec: 1.12 +[2025-04-27 03:15:39] (step=0020100) Train Loss: 5.9838, Train Steps/Sec: 1.12 +[2025-04-27 03:16:01] (step=0020125) Train Loss: 6.0056, Train Steps/Sec: 1.12 +[2025-04-27 03:16:24] (step=0020150) Train Loss: 6.0620, Train Steps/Sec: 1.12 +[2025-04-27 03:16:46] (step=0020175) Train Loss: 5.9766, Train Steps/Sec: 1.12 +[2025-04-27 03:17:08] (step=0020200) Train Loss: 6.0516, Train Steps/Sec: 1.12 +[2025-04-27 03:17:31] (step=0020225) Train Loss: 6.0437, Train Steps/Sec: 1.12 +[2025-04-27 03:17:53] (step=0020250) Train Loss: 6.0522, Train Steps/Sec: 1.12 +[2025-04-27 03:18:15] (step=0020275) Train Loss: 6.0215, Train Steps/Sec: 1.12 +[2025-04-27 03:18:38] (step=0020300) Train Loss: 6.0014, Train Steps/Sec: 1.12 +[2025-04-27 03:19:00] (step=0020325) Train Loss: 5.9982, Train Steps/Sec: 1.12 +[2025-04-27 03:19:22] (step=0020350) Train Loss: 5.9689, Train Steps/Sec: 1.12 +[2025-04-27 03:19:45] (step=0020375) Train Loss: 6.0009, Train Steps/Sec: 1.12 +[2025-04-27 03:20:07] (step=0020400) Train Loss: 6.0356, Train Steps/Sec: 1.12 +[2025-04-27 03:20:29] (step=0020425) Train Loss: 6.0862, Train Steps/Sec: 1.12 +[2025-04-27 03:20:52] (step=0020450) Train Loss: 6.0228, Train Steps/Sec: 1.12 +[2025-04-27 03:21:14] (step=0020475) Train Loss: 5.9909, Train Steps/Sec: 1.11 +[2025-04-27 03:21:36] (step=0020500) Train Loss: 5.9842, Train Steps/Sec: 1.12 +[2025-04-27 03:21:59] (step=0020525) Train Loss: 6.0423, Train Steps/Sec: 1.12 +[2025-04-27 03:22:21] (step=0020550) Train Loss: 6.0420, Train Steps/Sec: 1.12 +[2025-04-27 03:22:43] (step=0020575) Train Loss: 6.0317, Train Steps/Sec: 1.12 +[2025-04-27 03:23:06] (step=0020600) Train Loss: 6.0613, Train Steps/Sec: 1.11 +[2025-04-27 03:23:28] (step=0020625) Train Loss: 6.0248, Train Steps/Sec: 1.12 +[2025-04-27 03:23:51] (step=0020650) Train Loss: 5.9820, Train Steps/Sec: 1.12 +[2025-04-27 03:24:13] (step=0020675) Train Loss: 6.0020, Train Steps/Sec: 1.12 +[2025-04-27 03:24:35] (step=0020700) Train Loss: 6.0474, Train Steps/Sec: 1.12 +[2025-04-27 03:24:58] (step=0020725) Train Loss: 5.9784, Train Steps/Sec: 1.12 +[2025-04-27 03:25:20] (step=0020750) Train Loss: 5.9967, Train Steps/Sec: 1.12 +[2025-04-27 03:25:42] (step=0020775) Train Loss: 6.0049, Train Steps/Sec: 1.12 +[2025-04-27 03:26:05] (step=0020800) Train Loss: 6.0087, Train Steps/Sec: 1.12 +[2025-04-27 03:26:27] (step=0020825) Train Loss: 6.0302, Train Steps/Sec: 1.12 +[2025-04-27 03:26:49] (step=0020850) Train Loss: 6.0428, Train Steps/Sec: 1.12 +[2025-04-27 03:27:12] (step=0020875) Train Loss: 6.0269, Train Steps/Sec: 1.12 +[2025-04-27 03:27:34] (step=0020900) Train Loss: 6.0555, Train Steps/Sec: 1.12 +[2025-04-27 03:27:56] (step=0020925) Train Loss: 6.0481, Train Steps/Sec: 1.12 +[2025-04-27 03:28:19] (step=0020950) Train Loss: 6.0222, Train Steps/Sec: 1.12 +[2025-04-27 03:28:41] (step=0020975) Train Loss: 6.0475, Train Steps/Sec: 1.12 +[2025-04-27 03:29:03] (step=0021000) Train Loss: 5.9656, Train Steps/Sec: 1.11 +[2025-04-27 03:29:26] (step=0021025) Train Loss: 5.9892, Train Steps/Sec: 1.12 +[2025-04-27 03:29:48] (step=0021050) Train Loss: 5.9835, Train Steps/Sec: 1.12 +[2025-04-27 03:30:10] (step=0021075) Train Loss: 6.0623, Train Steps/Sec: 1.12 +[2025-04-27 03:30:33] (step=0021100) Train Loss: 6.0381, Train Steps/Sec: 1.12 +[2025-04-27 03:30:55] (step=0021125) Train Loss: 6.0414, Train Steps/Sec: 1.12 +[2025-04-27 03:31:17] (step=0021150) Train Loss: 6.0218, Train Steps/Sec: 1.12 +[2025-04-27 03:31:40] (step=0021175) Train Loss: 5.9799, Train Steps/Sec: 1.12 +[2025-04-27 03:32:02] (step=0021200) Train Loss: 6.0368, Train Steps/Sec: 1.12 +[2025-04-27 03:32:24] (step=0021225) Train Loss: 6.0716, Train Steps/Sec: 1.12 +[2025-04-27 03:32:47] (step=0021250) Train Loss: 6.0819, Train Steps/Sec: 1.12 +[2025-04-27 03:33:09] (step=0021275) Train Loss: 6.0225, Train Steps/Sec: 1.12 +[2025-04-27 03:33:31] (step=0021300) Train Loss: 6.0759, Train Steps/Sec: 1.12 +[2025-04-27 03:33:54] (step=0021325) Train Loss: 6.1121, Train Steps/Sec: 1.12 +[2025-04-27 03:34:16] (step=0021350) Train Loss: 6.0333, Train Steps/Sec: 1.12 +[2025-04-27 03:34:38] (step=0021375) Train Loss: 6.0260, Train Steps/Sec: 1.12 +[2025-04-27 03:35:01] (step=0021400) Train Loss: 6.0445, Train Steps/Sec: 1.12 +[2025-04-27 03:35:23] (step=0021425) Train Loss: 5.9888, Train Steps/Sec: 1.12 +[2025-04-27 03:35:45] (step=0021450) Train Loss: 6.0609, Train Steps/Sec: 1.12 +[2025-04-27 03:36:08] (step=0021475) Train Loss: 5.9951, Train Steps/Sec: 1.12 +[2025-04-27 03:36:30] (step=0021500) Train Loss: 6.0180, Train Steps/Sec: 1.12 +[2025-04-27 03:36:52] (step=0021525) Train Loss: 6.0397, Train Steps/Sec: 1.12 +[2025-04-27 03:37:15] (step=0021550) Train Loss: 6.0239, Train Steps/Sec: 1.12 +[2025-04-27 03:37:37] (step=0021575) Train Loss: 6.0673, Train Steps/Sec: 1.12 +[2025-04-27 03:37:59] (step=0021600) Train Loss: 6.0339, Train Steps/Sec: 1.11 +[2025-04-27 03:38:22] (step=0021625) Train Loss: 6.0203, Train Steps/Sec: 1.12 +[2025-04-27 03:38:44] (step=0021650) Train Loss: 6.0359, Train Steps/Sec: 1.12 +[2025-04-27 03:39:06] (step=0021675) Train Loss: 5.9715, Train Steps/Sec: 1.12 +[2025-04-27 03:39:29] (step=0021700) Train Loss: 6.0134, Train Steps/Sec: 1.12 +[2025-04-27 03:39:51] (step=0021725) Train Loss: 6.0291, Train Steps/Sec: 1.12 +[2025-04-27 03:40:13] (step=0021750) Train Loss: 5.9506, Train Steps/Sec: 1.12 +[2025-04-27 03:40:36] (step=0021775) Train Loss: 6.0053, Train Steps/Sec: 1.12 +[2025-04-27 03:40:58] (step=0021800) Train Loss: 5.9994, Train Steps/Sec: 1.12 +[2025-04-27 03:41:20] (step=0021825) Train Loss: 6.0082, Train Steps/Sec: 1.12 +[2025-04-27 03:41:43] (step=0021850) Train Loss: 6.0595, Train Steps/Sec: 1.12 +[2025-04-27 03:42:05] (step=0021875) Train Loss: 5.9878, Train Steps/Sec: 1.12 +[2025-04-27 03:42:28] (step=0021900) Train Loss: 6.0359, Train Steps/Sec: 1.12 +[2025-04-27 03:42:50] (step=0021925) Train Loss: 6.0074, Train Steps/Sec: 1.12 +[2025-04-27 03:43:12] (step=0021950) Train Loss: 5.9964, Train Steps/Sec: 1.12 +[2025-04-27 03:43:34] (step=0021975) Train Loss: 5.9899, Train Steps/Sec: 1.12 +[2025-04-27 03:43:57] (step=0022000) Train Loss: 6.0329, Train Steps/Sec: 1.12 +[2025-04-27 03:43:57] text_config is None. Initializing the text config with default values (`OPTConfig`). +[2025-04-27 03:50:05] Finish Eval in 22000 steps... +[2025-04-27 03:50:24] Saved checkpoint to checkpoint/X2I_UltraEdiit_fourmask_LLava_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_filtered_objects_4mask_1280_context/003-GPT-XL/checkpoints/0022000.pt +[2025-04-27 03:50:26] Removed old checkpoint: checkpoint/X2I_UltraEdiit_fourmask_LLava_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_filtered_objects_4mask_1280_context/003-GPT-XL/checkpoints/0020000.pt +[2025-04-27 03:50:49] (step=0022025) Train Loss: 6.0017, Train Steps/Sec: 0.06 +[2025-04-27 03:51:11] (step=0022050) Train Loss: 6.0122, Train Steps/Sec: 1.12 +[2025-04-27 03:51:34] (step=0022075) Train Loss: 6.0326, Train Steps/Sec: 1.12 +[2025-04-27 03:51:56] (step=0022100) Train Loss: 5.9934, Train Steps/Sec: 1.12 +[2025-04-27 03:52:18] (step=0022125) Train Loss: 6.0388, Train Steps/Sec: 1.12 +[2025-04-27 03:52:41] (step=0022150) Train Loss: 5.9830, Train Steps/Sec: 1.12 +[2025-04-27 03:53:03] (step=0022175) Train Loss: 5.9664, Train Steps/Sec: 1.12 +[2025-04-27 03:53:26] (step=0022200) Train Loss: 6.0647, Train Steps/Sec: 1.11 +[2025-04-27 03:53:48] (step=0022225) Train Loss: 6.0424, Train Steps/Sec: 1.12 +[2025-04-27 03:54:10] (step=0022250) Train Loss: 6.0339, Train Steps/Sec: 1.12 +[2025-04-27 03:54:33] (step=0022275) Train Loss: 5.9947, Train Steps/Sec: 1.12 +[2025-04-27 03:54:55] (step=0022300) Train Loss: 5.9996, Train Steps/Sec: 1.12 +[2025-04-27 03:55:17] (step=0022325) Train Loss: 6.0910, Train Steps/Sec: 1.12 +[2025-04-27 03:55:40] (step=0022350) Train Loss: 5.9500, Train Steps/Sec: 1.12 +[2025-04-27 03:56:02] (step=0022375) Train Loss: 6.0511, Train Steps/Sec: 1.11 +[2025-04-27 03:56:24] (step=0022400) Train Loss: 5.9695, Train Steps/Sec: 1.12 +[2025-04-27 03:56:47] (step=0022425) Train Loss: 6.0302, Train Steps/Sec: 1.12 +[2025-04-27 03:57:09] (step=0022450) Train Loss: 5.9774, Train Steps/Sec: 1.12 +[2025-04-27 03:57:32] (step=0022475) Train Loss: 6.0618, Train Steps/Sec: 1.12 +[2025-04-27 03:57:54] (step=0022500) Train Loss: 6.0761, Train Steps/Sec: 1.12 +[2025-04-27 03:58:16] (step=0022525) Train Loss: 6.0243, Train Steps/Sec: 1.12 +[2025-04-27 03:58:39] (step=0022550) Train Loss: 6.0209, Train Steps/Sec: 1.12 +[2025-04-27 03:59:01] (step=0022575) Train Loss: 6.0679, Train Steps/Sec: 1.12 +[2025-04-27 03:59:23] (step=0022600) Train Loss: 6.0740, Train Steps/Sec: 1.12 +[2025-04-27 03:59:46] (step=0022625) Train Loss: 6.0052, Train Steps/Sec: 1.12 +[2025-04-27 04:00:08] (step=0022650) Train Loss: 5.9554, Train Steps/Sec: 1.12 +[2025-04-27 04:00:30] (step=0022675) Train Loss: 6.0176, Train Steps/Sec: 1.12 +[2025-04-27 04:00:52] (step=0022700) Train Loss: 5.9969, Train Steps/Sec: 1.12 +[2025-04-27 04:01:15] (step=0022725) Train Loss: 5.9804, Train Steps/Sec: 1.12 +[2025-04-27 04:01:37] (step=0022750) Train Loss: 5.9639, Train Steps/Sec: 1.12 +[2025-04-27 04:01:59] (step=0022775) Train Loss: 5.9914, Train Steps/Sec: 1.12 +[2025-04-27 04:02:22] (step=0022800) Train Loss: 6.0202, Train Steps/Sec: 1.12 +[2025-04-27 04:02:44] (step=0022825) Train Loss: 6.0384, Train Steps/Sec: 1.12 +[2025-04-27 04:03:06] (step=0022850) Train Loss: 5.9454, Train Steps/Sec: 1.12 +[2025-04-27 04:03:29] (step=0022875) Train Loss: 6.0141, Train Steps/Sec: 1.12 +[2025-04-27 04:03:51] (step=0022900) Train Loss: 5.9586, Train Steps/Sec: 1.12 +[2025-04-27 04:04:14] (step=0022925) Train Loss: 6.0276, Train Steps/Sec: 1.12 +[2025-04-27 04:04:36] (step=0022950) Train Loss: 5.9864, Train Steps/Sec: 1.12 +[2025-04-27 04:04:58] (step=0022975) Train Loss: 6.0175, Train Steps/Sec: 1.12 +[2025-04-27 04:05:21] (step=0023000) Train Loss: 5.9951, Train Steps/Sec: 1.11 +[2025-04-27 04:05:43] (step=0023025) Train Loss: 5.9958, Train Steps/Sec: 1.12 +[2025-04-27 04:06:05] (step=0023050) Train Loss: 6.0236, Train Steps/Sec: 1.12 +[2025-04-27 04:06:28] (step=0023075) Train Loss: 5.9899, Train Steps/Sec: 1.12 +[2025-04-27 04:06:50] (step=0023100) Train Loss: 5.9009, Train Steps/Sec: 1.12 +[2025-04-27 04:07:12] (step=0023125) Train Loss: 5.9816, Train Steps/Sec: 1.12 +[2025-04-27 04:07:35] (step=0023150) Train Loss: 5.9893, Train Steps/Sec: 1.12 +[2025-04-27 04:07:57] (step=0023175) Train Loss: 6.0734, Train Steps/Sec: 1.12 +[2025-04-27 04:08:19] (step=0023200) Train Loss: 6.0186, Train Steps/Sec: 1.12 +[2025-04-27 04:08:42] (step=0023225) Train Loss: 5.9453, Train Steps/Sec: 1.12 +[2025-04-27 04:09:04] (step=0023250) Train Loss: 6.0434, Train Steps/Sec: 1.12 +[2025-04-27 04:09:26] (step=0023275) Train Loss: 6.0130, Train Steps/Sec: 1.12 +[2025-04-27 04:09:49] (step=0023300) Train Loss: 6.0280, Train Steps/Sec: 1.12 +[2025-04-27 04:10:11] (step=0023325) Train Loss: 6.0405, Train Steps/Sec: 1.12 +[2025-04-27 04:10:33] (step=0023350) Train Loss: 6.0086, Train Steps/Sec: 1.12 +[2025-04-27 04:10:56] (step=0023375) Train Loss: 6.0293, Train Steps/Sec: 1.12 +[2025-04-27 04:11:18] (step=0023400) Train Loss: 6.0027, Train Steps/Sec: 1.12 +[2025-04-27 04:11:45] (step=0023425) Train Loss: 6.0553, Train Steps/Sec: 0.91 +[2025-04-27 04:12:08] (step=0023450) Train Loss: 6.0537, Train Steps/Sec: 1.12 +[2025-04-27 04:12:30] (step=0023475) Train Loss: 5.9991, Train Steps/Sec: 1.12 +[2025-04-27 04:12:53] (step=0023500) Train Loss: 5.9712, Train Steps/Sec: 1.12 +[2025-04-27 04:13:15] (step=0023525) Train Loss: 5.9490, Train Steps/Sec: 1.12 +[2025-04-27 04:13:37] (step=0023550) Train Loss: 6.0133, Train Steps/Sec: 1.12 +[2025-04-27 04:14:00] (step=0023575) Train Loss: 5.9807, Train Steps/Sec: 1.12 +[2025-04-27 04:14:22] (step=0023600) Train Loss: 6.0017, Train Steps/Sec: 1.12 +[2025-04-27 04:14:44] (step=0023625) Train Loss: 5.9955, Train Steps/Sec: 1.12 +[2025-04-27 04:15:07] (step=0023650) Train Loss: 6.0046, Train Steps/Sec: 1.12 +[2025-04-27 04:15:29] (step=0023675) Train Loss: 6.0034, Train Steps/Sec: 1.12 +[2025-04-27 04:15:51] (step=0023700) Train Loss: 6.0164, Train Steps/Sec: 1.12 +[2025-04-27 04:16:14] (step=0023725) Train Loss: 5.9016, Train Steps/Sec: 1.12 +[2025-04-27 04:16:36] (step=0023750) Train Loss: 5.9990, Train Steps/Sec: 1.12 +[2025-04-27 04:16:58] (step=0023775) Train Loss: 6.0222, Train Steps/Sec: 1.12 +[2025-04-27 04:17:21] (step=0023800) Train Loss: 5.9663, Train Steps/Sec: 1.12 +[2025-04-27 04:17:43] (step=0023825) Train Loss: 5.9656, Train Steps/Sec: 1.12 +[2025-04-27 04:18:06] (step=0023850) Train Loss: 5.9906, Train Steps/Sec: 1.12 +[2025-04-27 04:18:28] (step=0023875) Train Loss: 5.9936, Train Steps/Sec: 1.12 +[2025-04-27 04:18:50] (step=0023900) Train Loss: 5.9600, Train Steps/Sec: 1.12 +[2025-04-27 04:19:13] (step=0023925) Train Loss: 5.9502, Train Steps/Sec: 1.12 +[2025-04-27 04:19:35] (step=0023950) Train Loss: 6.0297, Train Steps/Sec: 1.12 +[2025-04-27 04:19:57] (step=0023975) Train Loss: 5.9836, Train Steps/Sec: 1.12 +[2025-04-27 04:20:24] (step=0024000) Train Loss: 6.0087, Train Steps/Sec: 0.92 +[2025-04-27 04:20:25] text_config is None. Initializing the text config with default values (`OPTConfig`). +[2025-04-27 04:26:31] Finish Eval in 24000 steps... +[2025-04-27 04:26:50] Saved checkpoint to checkpoint/X2I_UltraEdiit_fourmask_LLava_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_filtered_objects_4mask_1280_context/003-GPT-XL/checkpoints/0024000.pt +[2025-04-27 04:26:52] Removed old checkpoint: checkpoint/X2I_UltraEdiit_fourmask_LLava_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_filtered_objects_4mask_1280_context/003-GPT-XL/checkpoints/0022000.pt +[2025-04-27 04:27:20] (step=0024025) Train Loss: 5.9901, Train Steps/Sec: 0.06 +[2025-04-27 04:28:02] (step=0024050) Train Loss: 5.9992, Train Steps/Sec: 0.59 +[2025-04-27 04:28:24] (step=0024075) Train Loss: 5.9726, Train Steps/Sec: 1.12 +[2025-04-27 04:28:47] (step=0024100) Train Loss: 6.0037, Train Steps/Sec: 1.12 +[2025-04-27 04:29:09] (step=0024125) Train Loss: 6.0201, Train Steps/Sec: 1.12 +[2025-04-27 04:29:31] (step=0024150) Train Loss: 5.9679, Train Steps/Sec: 1.12 +[2025-04-27 04:29:54] (step=0024175) Train Loss: 5.9927, Train Steps/Sec: 1.12 +[2025-04-27 04:30:16] (step=0024200) Train Loss: 6.0627, Train Steps/Sec: 1.12 +[2025-04-27 04:30:38] (step=0024225) Train Loss: 5.9653, Train Steps/Sec: 1.12 +[2025-04-27 04:31:01] (step=0024250) Train Loss: 5.9385, Train Steps/Sec: 1.12 +[2025-04-27 04:31:23] (step=0024275) Train Loss: 6.0092, Train Steps/Sec: 1.12 +[2025-04-27 04:31:45] (step=0024300) Train Loss: 5.9626, Train Steps/Sec: 1.12 +[2025-04-27 04:32:08] (step=0024325) Train Loss: 6.0000, Train Steps/Sec: 1.12 +[2025-04-27 04:32:30] (step=0024350) Train Loss: 5.9767, Train Steps/Sec: 1.12 +[2025-04-27 04:32:52] (step=0024375) Train Loss: 5.9598, Train Steps/Sec: 1.12 +[2025-04-27 04:33:15] (step=0024400) Train Loss: 6.0131, Train Steps/Sec: 1.11 +[2025-04-27 04:33:37] (step=0024425) Train Loss: 6.0045, Train Steps/Sec: 1.12 +[2025-04-27 04:34:00] (step=0024450) Train Loss: 5.9481, Train Steps/Sec: 1.12 +[2025-04-27 04:34:22] (step=0024475) Train Loss: 5.9468, Train Steps/Sec: 1.12 +[2025-04-27 04:34:44] (step=0024500) Train Loss: 5.9933, Train Steps/Sec: 1.12 +[2025-04-27 04:35:06] (step=0024525) Train Loss: 5.9812, Train Steps/Sec: 1.12 +[2025-04-27 04:35:29] (step=0024550) Train Loss: 5.9872, Train Steps/Sec: 1.12 +[2025-04-27 04:35:51] (step=0024575) Train Loss: 5.9876, Train Steps/Sec: 1.12 +[2025-04-27 04:36:14] (step=0024600) Train Loss: 5.9409, Train Steps/Sec: 1.11 +[2025-04-27 04:36:36] (step=0024625) Train Loss: 5.9135, Train Steps/Sec: 1.12 +[2025-04-27 04:36:58] (step=0024650) Train Loss: 5.9946, Train Steps/Sec: 1.12 +[2025-04-27 04:37:21] (step=0024675) Train Loss: 6.0174, Train Steps/Sec: 1.12 +[2025-04-27 04:37:43] (step=0024700) Train Loss: 6.0043, Train Steps/Sec: 1.12 +[2025-04-27 04:38:05] (step=0024725) Train Loss: 6.0135, Train Steps/Sec: 1.12 +[2025-04-27 04:38:28] (step=0024750) Train Loss: 5.9700, Train Steps/Sec: 1.12 +[2025-04-27 04:38:50] (step=0024775) Train Loss: 5.9271, Train Steps/Sec: 1.12 +[2025-04-27 04:39:13] (step=0024800) Train Loss: 5.9919, Train Steps/Sec: 1.11 +[2025-04-27 04:39:35] (step=0024825) Train Loss: 6.0373, Train Steps/Sec: 1.12 +[2025-04-27 04:39:57] (step=0024850) Train Loss: 5.9283, Train Steps/Sec: 1.12 +[2025-04-27 04:40:20] (step=0024875) Train Loss: 5.9933, Train Steps/Sec: 1.12 +[2025-04-27 04:40:42] (step=0024900) Train Loss: 5.9589, Train Steps/Sec: 1.12 +[2025-04-27 04:41:04] (step=0024925) Train Loss: 5.9748, Train Steps/Sec: 1.12 +[2025-04-27 04:41:27] (step=0024950) Train Loss: 5.9660, Train Steps/Sec: 1.12 +[2025-04-27 04:41:49] (step=0024975) Train Loss: 6.0128, Train Steps/Sec: 1.12 +[2025-04-27 04:42:11] (step=0025000) Train Loss: 6.0024, Train Steps/Sec: 1.12 +[2025-04-27 04:42:34] (step=0025025) Train Loss: 5.9648, Train Steps/Sec: 1.12 +[2025-04-27 04:42:56] (step=0025050) Train Loss: 5.8989, Train Steps/Sec: 1.12 +[2025-04-27 04:43:18] (step=0025075) Train Loss: 5.9793, Train Steps/Sec: 1.12 +[2025-04-27 04:43:41] (step=0025100) Train Loss: 5.9914, Train Steps/Sec: 1.12 +[2025-04-27 04:44:03] (step=0025125) Train Loss: 5.9868, Train Steps/Sec: 1.12 +[2025-04-27 04:44:25] (step=0025150) Train Loss: 5.8950, Train Steps/Sec: 1.12 +[2025-04-27 04:44:48] (step=0025175) Train Loss: 6.0262, Train Steps/Sec: 1.12 +[2025-04-27 04:45:10] (step=0025200) Train Loss: 5.9055, Train Steps/Sec: 1.12 +[2025-04-27 04:45:32] (step=0025225) Train Loss: 5.9880, Train Steps/Sec: 1.12 +[2025-04-27 04:45:55] (step=0025250) Train Loss: 5.9560, Train Steps/Sec: 1.12 +[2025-04-27 04:46:17] (step=0025275) Train Loss: 5.9094, Train Steps/Sec: 1.12 +[2025-04-27 04:46:39] (step=0025300) Train Loss: 5.9595, Train Steps/Sec: 1.12 +[2025-04-27 04:47:02] (step=0025325) Train Loss: 5.9443, Train Steps/Sec: 1.12 +[2025-04-27 04:47:24] (step=0025350) Train Loss: 5.9705, Train Steps/Sec: 1.12 +[2025-04-27 04:47:46] (step=0025375) Train Loss: 6.0289, Train Steps/Sec: 1.12 +[2025-04-27 04:48:09] (step=0025400) Train Loss: 5.9633, Train Steps/Sec: 1.12 +[2025-04-27 04:48:31] (step=0025425) Train Loss: 6.0048, Train Steps/Sec: 1.12 +[2025-04-27 04:48:53] (step=0025450) Train Loss: 5.9611, Train Steps/Sec: 1.12 +[2025-04-27 04:49:16] (step=0025475) Train Loss: 5.9820, Train Steps/Sec: 1.12 +[2025-04-27 04:49:38] (step=0025500) Train Loss: 5.9313, Train Steps/Sec: 1.12 +[2025-04-27 04:50:00] (step=0025525) Train Loss: 5.9853, Train Steps/Sec: 1.12 +[2025-04-27 04:50:23] (step=0025550) Train Loss: 5.9564, Train Steps/Sec: 1.12 +[2025-04-27 04:50:45] (step=0025575) Train Loss: 6.0570, Train Steps/Sec: 1.12 +[2025-04-27 04:51:07] (step=0025600) Train Loss: 5.9960, Train Steps/Sec: 1.11 +[2025-04-27 04:51:30] (step=0025625) Train Loss: 5.9491, Train Steps/Sec: 1.12 +[2025-04-27 04:51:52] (step=0025650) Train Loss: 5.9249, Train Steps/Sec: 1.12 +[2025-04-27 04:52:14] (step=0025675) Train Loss: 5.9751, Train Steps/Sec: 1.12 +[2025-04-27 04:52:37] (step=0025700) Train Loss: 5.9811, Train Steps/Sec: 1.12 +[2025-04-27 04:52:59] (step=0025725) Train Loss: 5.9672, Train Steps/Sec: 1.12 +[2025-04-27 04:53:21] (step=0025750) Train Loss: 5.9692, Train Steps/Sec: 1.12 +[2025-04-27 04:53:44] (step=0025775) Train Loss: 5.9547, Train Steps/Sec: 1.12 +[2025-04-27 04:54:06] (step=0025800) Train Loss: 5.9650, Train Steps/Sec: 1.12 +[2025-04-27 04:54:28] (step=0025825) Train Loss: 5.9397, Train Steps/Sec: 1.12 +[2025-04-27 04:54:51] (step=0025850) Train Loss: 6.0198, Train Steps/Sec: 1.12 +[2025-04-27 04:55:13] (step=0025875) Train Loss: 6.0176, Train Steps/Sec: 1.12 +[2025-04-27 04:55:35] (step=0025900) Train Loss: 5.9177, Train Steps/Sec: 1.12 +[2025-04-27 04:55:58] (step=0025925) Train Loss: 5.9429, Train Steps/Sec: 1.12 +[2025-04-27 04:56:20] (step=0025950) Train Loss: 6.0261, Train Steps/Sec: 1.12 +[2025-04-27 04:56:42] (step=0025975) Train Loss: 5.9616, Train Steps/Sec: 1.12 +[2025-04-27 04:57:05] (step=0026000) Train Loss: 5.9657, Train Steps/Sec: 1.11 +[2025-04-27 04:57:05] text_config is None. Initializing the text config with default values (`OPTConfig`). +[2025-04-27 05:03:13] Finish Eval in 26000 steps... +[2025-04-27 05:03:32] Saved checkpoint to checkpoint/X2I_UltraEdiit_fourmask_LLava_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_filtered_objects_4mask_1280_context/003-GPT-XL/checkpoints/0026000.pt +[2025-04-27 05:03:34] Removed old checkpoint: checkpoint/X2I_UltraEdiit_fourmask_LLava_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_filtered_objects_4mask_1280_context/003-GPT-XL/checkpoints/0024000.pt +[2025-04-27 05:03:57] (step=0026025) Train Loss: 6.0213, Train Steps/Sec: 0.06 +[2025-04-27 05:04:19] (step=0026050) Train Loss: 6.0009, Train Steps/Sec: 1.12 +[2025-04-27 05:04:42] (step=0026075) Train Loss: 5.9480, Train Steps/Sec: 1.12 +[2025-04-27 05:05:04] (step=0026100) Train Loss: 5.9998, Train Steps/Sec: 1.12 +[2025-04-27 05:05:26] (step=0026125) Train Loss: 5.9703, Train Steps/Sec: 1.12 +[2025-04-27 05:05:49] (step=0026150) Train Loss: 5.9717, Train Steps/Sec: 1.12 +[2025-04-27 05:06:11] (step=0026175) Train Loss: 5.9451, Train Steps/Sec: 1.11 +[2025-04-27 05:06:33] (step=0026200) Train Loss: 6.0389, Train Steps/Sec: 1.11 +[2025-04-27 05:06:56] (step=0026225) Train Loss: 5.9507, Train Steps/Sec: 1.12 +[2025-04-27 05:07:18] (step=0026250) Train Loss: 5.9718, Train Steps/Sec: 1.12 +[2025-04-27 05:07:41] (step=0026275) Train Loss: 5.9407, Train Steps/Sec: 1.12 +[2025-04-27 05:08:03] (step=0026300) Train Loss: 5.9975, Train Steps/Sec: 1.12 +[2025-04-27 05:08:25] (step=0026325) Train Loss: 6.0209, Train Steps/Sec: 1.12 +[2025-04-27 05:08:48] (step=0026350) Train Loss: 5.9944, Train Steps/Sec: 1.12 +[2025-04-27 05:09:10] (step=0026375) Train Loss: 5.9834, Train Steps/Sec: 1.12 +[2025-04-27 05:09:32] (step=0026400) Train Loss: 6.0093, Train Steps/Sec: 1.11 +[2025-04-27 05:09:55] (step=0026425) Train Loss: 5.9326, Train Steps/Sec: 1.12 +[2025-04-27 05:10:17] (step=0026450) Train Loss: 5.9625, Train Steps/Sec: 1.12 +[2025-04-27 05:10:39] (step=0026475) Train Loss: 5.9818, Train Steps/Sec: 1.12 +[2025-04-27 05:11:02] (step=0026500) Train Loss: 6.0021, Train Steps/Sec: 1.12 +[2025-04-27 05:11:24] (step=0026525) Train Loss: 5.9601, Train Steps/Sec: 1.12 +[2025-04-27 05:11:46] (step=0026550) Train Loss: 5.9738, Train Steps/Sec: 1.12 +[2025-04-27 05:12:09] (step=0026575) Train Loss: 5.9452, Train Steps/Sec: 1.12 +[2025-04-27 05:12:31] (step=0026600) Train Loss: 6.0132, Train Steps/Sec: 1.11 +[2025-04-27 05:12:54] (step=0026625) Train Loss: 5.9961, Train Steps/Sec: 1.12 +[2025-04-27 05:13:16] (step=0026650) Train Loss: 5.9577, Train Steps/Sec: 1.12 +[2025-04-27 05:13:38] (step=0026675) Train Loss: 6.0203, Train Steps/Sec: 1.12 +[2025-04-27 05:14:01] (step=0026700) Train Loss: 5.8888, Train Steps/Sec: 1.12 +[2025-04-27 05:14:23] (step=0026725) Train Loss: 6.0009, Train Steps/Sec: 1.12 +[2025-04-27 05:14:45] (step=0026750) Train Loss: 5.9705, Train Steps/Sec: 1.12 +[2025-04-27 05:15:08] (step=0026775) Train Loss: 5.9025, Train Steps/Sec: 1.12 +[2025-04-27 05:15:30] (step=0026800) Train Loss: 5.9646, Train Steps/Sec: 1.11 +[2025-04-27 05:15:52] (step=0026825) Train Loss: 5.9788, Train Steps/Sec: 1.12 +[2025-04-27 05:16:15] (step=0026850) Train Loss: 6.0318, Train Steps/Sec: 1.12 +[2025-04-27 05:16:37] (step=0026875) Train Loss: 6.0197, Train Steps/Sec: 1.12 +[2025-04-27 05:16:59] (step=0026900) Train Loss: 6.0363, Train Steps/Sec: 1.12 +[2025-04-27 05:17:22] (step=0026925) Train Loss: 5.9419, Train Steps/Sec: 1.12 +[2025-04-27 05:17:44] (step=0026950) Train Loss: 5.9247, Train Steps/Sec: 1.12 +[2025-04-27 05:18:06] (step=0026975) Train Loss: 5.9355, Train Steps/Sec: 1.12 +[2025-04-27 05:18:29] (step=0027000) Train Loss: 5.9529, Train Steps/Sec: 1.11 +[2025-04-27 05:18:51] (step=0027025) Train Loss: 5.8630, Train Steps/Sec: 1.12 +[2025-04-27 05:19:14] (step=0027050) Train Loss: 5.9836, Train Steps/Sec: 1.12 +[2025-04-27 05:19:36] (step=0027075) Train Loss: 5.9260, Train Steps/Sec: 1.12 +[2025-04-27 05:19:58] (step=0027100) Train Loss: 5.9440, Train Steps/Sec: 1.12 +[2025-04-27 05:20:21] (step=0027125) Train Loss: 5.9308, Train Steps/Sec: 1.12 +[2025-04-27 05:20:43] (step=0027150) Train Loss: 5.9846, Train Steps/Sec: 1.12 +[2025-04-27 05:21:05] (step=0027175) Train Loss: 5.9849, Train Steps/Sec: 1.12 +[2025-04-27 05:21:28] (step=0027200) Train Loss: 6.0299, Train Steps/Sec: 1.11 +[2025-04-27 05:21:50] (step=0027225) Train Loss: 5.9732, Train Steps/Sec: 1.12 +[2025-04-27 05:22:12] (step=0027250) Train Loss: 5.9767, Train Steps/Sec: 1.12 +[2025-04-27 05:22:35] (step=0027275) Train Loss: 5.9951, Train Steps/Sec: 1.12 +[2025-04-27 05:22:57] (step=0027300) Train Loss: 6.0032, Train Steps/Sec: 1.12 +[2025-04-27 05:23:20] (step=0027325) Train Loss: 5.9400, Train Steps/Sec: 1.12 +[2025-04-27 05:23:42] (step=0027350) Train Loss: 6.0271, Train Steps/Sec: 1.12 +[2025-04-27 05:24:04] (step=0027375) Train Loss: 6.0099, Train Steps/Sec: 1.12 +[2025-04-27 05:24:27] (step=0027400) Train Loss: 5.9997, Train Steps/Sec: 1.11 +[2025-04-27 05:24:49] (step=0027425) Train Loss: 5.8848, Train Steps/Sec: 1.12 +[2025-04-27 05:25:11] (step=0027450) Train Loss: 5.9380, Train Steps/Sec: 1.12 +[2025-04-27 05:25:34] (step=0027475) Train Loss: 5.9792, Train Steps/Sec: 1.12 +[2025-04-27 05:25:56] (step=0027500) Train Loss: 5.9510, Train Steps/Sec: 1.12 +[2025-04-27 05:26:18] (step=0027525) Train Loss: 6.0163, Train Steps/Sec: 1.12 +[2025-04-27 05:26:41] (step=0027550) Train Loss: 5.9872, Train Steps/Sec: 1.12 +[2025-04-27 05:27:03] (step=0027575) Train Loss: 5.9507, Train Steps/Sec: 1.12 +[2025-04-27 05:27:26] (step=0027600) Train Loss: 6.0138, Train Steps/Sec: 1.11 +[2025-04-27 05:27:48] (step=0027625) Train Loss: 5.9893, Train Steps/Sec: 1.12 +[2025-04-27 05:28:10] (step=0027650) Train Loss: 5.9907, Train Steps/Sec: 1.12 +[2025-04-27 05:28:33] (step=0027675) Train Loss: 6.0040, Train Steps/Sec: 1.12 +[2025-04-27 05:28:55] (step=0027700) Train Loss: 5.9808, Train Steps/Sec: 1.12 +[2025-04-27 05:29:17] (step=0027725) Train Loss: 5.9444, Train Steps/Sec: 1.12 +[2025-04-27 05:29:40] (step=0027750) Train Loss: 5.9672, Train Steps/Sec: 1.12 +[2025-04-27 05:30:02] (step=0027775) Train Loss: 6.0021, Train Steps/Sec: 1.12 +[2025-04-27 05:30:24] (step=0027800) Train Loss: 6.0012, Train Steps/Sec: 1.11 +[2025-04-27 05:30:52] (step=0027825) Train Loss: 5.9896, Train Steps/Sec: 0.91 +[2025-04-27 05:31:14] (step=0027850) Train Loss: 6.0014, Train Steps/Sec: 1.12 +[2025-04-27 05:31:37] (step=0027875) Train Loss: 5.9006, Train Steps/Sec: 1.12 +[2025-04-27 05:31:59] (step=0027900) Train Loss: 5.9942, Train Steps/Sec: 1.12 +[2025-04-27 05:32:21] (step=0027925) Train Loss: 5.9802, Train Steps/Sec: 1.12 +[2025-04-27 05:32:44] (step=0027950) Train Loss: 5.9469, Train Steps/Sec: 1.12 +[2025-04-27 05:33:06] (step=0027975) Train Loss: 6.0242, Train Steps/Sec: 1.12 +[2025-04-27 05:33:28] (step=0028000) Train Loss: 5.9704, Train Steps/Sec: 1.12 +[2025-04-27 05:33:28] text_config is None. Initializing the text config with default values (`OPTConfig`). +[2025-04-27 05:39:35] Finish Eval in 28000 steps... +[2025-04-27 05:39:55] Saved checkpoint to checkpoint/X2I_UltraEdiit_fourmask_LLava_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_filtered_objects_4mask_1280_context/003-GPT-XL/checkpoints/0028000.pt +[2025-04-27 05:39:57] Removed old checkpoint: checkpoint/X2I_UltraEdiit_fourmask_LLava_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_filtered_objects_4mask_1280_context/003-GPT-XL/checkpoints/0026000.pt +[2025-04-27 05:40:19] (step=0028025) Train Loss: 5.9524, Train Steps/Sec: 0.06 +[2025-04-27 05:40:41] (step=0028050) Train Loss: 5.9922, Train Steps/Sec: 1.12 +[2025-04-27 05:41:04] (step=0028075) Train Loss: 5.9886, Train Steps/Sec: 1.12 +[2025-04-27 05:41:26] (step=0028100) Train Loss: 5.9120, Train Steps/Sec: 1.12 +[2025-04-27 05:41:48] (step=0028125) Train Loss: 5.9613, Train Steps/Sec: 1.12 +[2025-04-27 05:42:11] (step=0028150) Train Loss: 5.9898, Train Steps/Sec: 1.12 +[2025-04-27 05:42:33] (step=0028175) Train Loss: 5.9541, Train Steps/Sec: 1.12 +[2025-04-27 05:42:55] (step=0028200) Train Loss: 5.8745, Train Steps/Sec: 1.12 +[2025-04-27 05:43:18] (step=0028225) Train Loss: 5.9450, Train Steps/Sec: 1.12 +[2025-04-27 05:43:40] (step=0028250) Train Loss: 5.9820, Train Steps/Sec: 1.12 +[2025-04-27 05:44:07] (step=0028275) Train Loss: 5.9357, Train Steps/Sec: 0.92 +[2025-04-27 05:44:34] (step=0028300) Train Loss: 5.9925, Train Steps/Sec: 0.92 +[2025-04-27 05:44:56] (step=0028325) Train Loss: 5.9208, Train Steps/Sec: 1.12 +[2025-04-27 05:45:19] (step=0028350) Train Loss: 5.9240, Train Steps/Sec: 1.12 +[2025-04-27 05:46:01] (step=0028375) Train Loss: 5.9688, Train Steps/Sec: 0.59 +[2025-04-27 05:46:24] (step=0028400) Train Loss: 6.0068, Train Steps/Sec: 1.12 +[2025-04-27 05:46:46] (step=0028425) Train Loss: 5.9415, Train Steps/Sec: 1.12 +[2025-04-27 05:47:08] (step=0028450) Train Loss: 6.0259, Train Steps/Sec: 1.12 +[2025-04-27 05:47:30] (step=0028475) Train Loss: 5.9302, Train Steps/Sec: 1.12 +[2025-04-27 05:47:53] (step=0028500) Train Loss: 5.9244, Train Steps/Sec: 1.12 +[2025-04-27 05:48:15] (step=0028525) Train Loss: 5.9709, Train Steps/Sec: 1.12 +[2025-04-27 05:48:37] (step=0028550) Train Loss: 5.9418, Train Steps/Sec: 1.12 +[2025-04-27 05:49:00] (step=0028575) Train Loss: 5.9702, Train Steps/Sec: 1.12 +[2025-04-27 05:49:22] (step=0028600) Train Loss: 5.9469, Train Steps/Sec: 1.11 +[2025-04-27 05:49:45] (step=0028625) Train Loss: 5.9887, Train Steps/Sec: 1.12 +[2025-04-27 05:50:07] (step=0028650) Train Loss: 6.0052, Train Steps/Sec: 1.12 +[2025-04-27 05:50:29] (step=0028675) Train Loss: 5.9485, Train Steps/Sec: 1.12 +[2025-04-27 05:50:52] (step=0028700) Train Loss: 5.9754, Train Steps/Sec: 1.12 +[2025-04-27 05:51:14] (step=0028725) Train Loss: 5.9627, Train Steps/Sec: 1.12 +[2025-04-27 05:51:36] (step=0028750) Train Loss: 5.8926, Train Steps/Sec: 1.12 +[2025-04-27 05:51:59] (step=0028775) Train Loss: 5.9175, Train Steps/Sec: 1.12 +[2025-04-27 05:52:21] (step=0028800) Train Loss: 5.9382, Train Steps/Sec: 1.12 +[2025-04-27 05:52:43] (step=0028825) Train Loss: 5.9209, Train Steps/Sec: 1.12 +[2025-04-27 05:53:06] (step=0028850) Train Loss: 5.9417, Train Steps/Sec: 1.12 +[2025-04-27 05:53:28] (step=0028875) Train Loss: 5.9136, Train Steps/Sec: 1.12 +[2025-04-27 05:53:50] (step=0028900) Train Loss: 5.9747, Train Steps/Sec: 1.12 +[2025-04-27 05:54:13] (step=0028925) Train Loss: 5.9959, Train Steps/Sec: 1.12 +[2025-04-27 05:54:35] (step=0028950) Train Loss: 5.9354, Train Steps/Sec: 1.12 +[2025-04-27 05:54:57] (step=0028975) Train Loss: 5.9493, Train Steps/Sec: 1.12 +[2025-04-27 05:55:20] (step=0029000) Train Loss: 5.9853, Train Steps/Sec: 1.11 +[2025-04-27 05:55:42] (step=0029025) Train Loss: 6.0035, Train Steps/Sec: 1.12 +[2025-04-27 05:56:04] (step=0029050) Train Loss: 5.9381, Train Steps/Sec: 1.12 +[2025-04-27 05:56:27] (step=0029075) Train Loss: 5.8981, Train Steps/Sec: 1.12 +[2025-04-27 05:56:49] (step=0029100) Train Loss: 5.9344, Train Steps/Sec: 1.12 +[2025-04-27 05:57:12] (step=0029125) Train Loss: 5.9714, Train Steps/Sec: 1.12 +[2025-04-27 05:57:34] (step=0029150) Train Loss: 5.9647, Train Steps/Sec: 1.12 +[2025-04-27 05:57:56] (step=0029175) Train Loss: 5.9535, Train Steps/Sec: 1.12 +[2025-04-27 05:58:19] (step=0029200) Train Loss: 5.9645, Train Steps/Sec: 1.12 +[2025-04-27 05:58:41] (step=0029225) Train Loss: 5.9774, Train Steps/Sec: 1.12 +[2025-04-27 05:59:03] (step=0029250) Train Loss: 5.9227, Train Steps/Sec: 1.12 +[2025-04-27 05:59:26] (step=0029275) Train Loss: 5.8993, Train Steps/Sec: 1.12 +[2025-04-27 05:59:48] (step=0029300) Train Loss: 5.9319, Train Steps/Sec: 1.12 +[2025-04-27 06:00:10] (step=0029325) Train Loss: 5.8672, Train Steps/Sec: 1.12 +[2025-04-27 06:00:33] (step=0029350) Train Loss: 5.9332, Train Steps/Sec: 1.12 +[2025-04-27 06:00:55] (step=0029375) Train Loss: 5.9178, Train Steps/Sec: 1.12 +[2025-04-27 06:01:17] (step=0029400) Train Loss: 5.8886, Train Steps/Sec: 1.11 +[2025-04-27 06:01:40] (step=0029425) Train Loss: 5.9060, Train Steps/Sec: 1.12 +[2025-04-27 06:02:02] (step=0029450) Train Loss: 5.9906, Train Steps/Sec: 1.12 +[2025-04-27 06:02:24] (step=0029475) Train Loss: 5.9811, Train Steps/Sec: 1.12 +[2025-04-27 06:02:47] (step=0029500) Train Loss: 5.9477, Train Steps/Sec: 1.12 +[2025-04-27 06:03:09] (step=0029525) Train Loss: 5.9562, Train Steps/Sec: 1.12 +[2025-04-27 06:03:31] (step=0029550) Train Loss: 5.8950, Train Steps/Sec: 1.12 +[2025-04-27 06:03:54] (step=0029575) Train Loss: 5.9240, Train Steps/Sec: 1.12 +[2025-04-27 06:04:16] (step=0029600) Train Loss: 5.9431, Train Steps/Sec: 1.12 +[2025-04-27 06:04:38] (step=0029625) Train Loss: 5.9347, Train Steps/Sec: 1.12 +[2025-04-27 06:05:01] (step=0029650) Train Loss: 5.8375, Train Steps/Sec: 1.12 +[2025-04-27 06:05:23] (step=0029675) Train Loss: 5.9202, Train Steps/Sec: 1.12 +[2025-04-27 06:05:45] (step=0029700) Train Loss: 5.9225, Train Steps/Sec: 1.12 +[2025-04-27 06:06:08] (step=0029725) Train Loss: 5.9233, Train Steps/Sec: 1.12 +[2025-04-27 06:06:30] (step=0029750) Train Loss: 5.9356, Train Steps/Sec: 1.12 +[2025-04-27 06:06:52] (step=0029775) Train Loss: 5.9415, Train Steps/Sec: 1.12 +[2025-04-27 06:07:15] (step=0029800) Train Loss: 5.9775, Train Steps/Sec: 1.12 +[2025-04-27 06:07:37] (step=0029825) Train Loss: 6.0007, Train Steps/Sec: 1.12 +[2025-04-27 06:07:59] (step=0029850) Train Loss: 5.8922, Train Steps/Sec: 1.12 +[2025-04-27 06:08:22] (step=0029875) Train Loss: 5.9272, Train Steps/Sec: 1.12 +[2025-04-27 06:08:44] (step=0029900) Train Loss: 5.9051, Train Steps/Sec: 1.12 +[2025-04-27 06:09:06] (step=0029925) Train Loss: 5.9718, Train Steps/Sec: 1.12 +[2025-04-27 06:09:29] (step=0029950) Train Loss: 5.9474, Train Steps/Sec: 1.12 +[2025-04-27 06:09:51] (step=0029975) Train Loss: 5.9660, Train Steps/Sec: 1.11 +[2025-04-27 06:10:14] (step=0030000) Train Loss: 5.9675, Train Steps/Sec: 1.11 +[2025-04-27 06:10:14] text_config is None. Initializing the text config with default values (`OPTConfig`). +[2025-04-27 06:16:21] Finish Eval in 30000 steps... +[2025-04-27 06:16:41] Saved checkpoint to checkpoint/X2I_UltraEdiit_fourmask_LLava_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_filtered_objects_4mask_1280_context/003-GPT-XL/checkpoints/0030000.pt +[2025-04-27 06:16:43] Removed old checkpoint: checkpoint/X2I_UltraEdiit_fourmask_LLava_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_filtered_objects_4mask_1280_context/003-GPT-XL/checkpoints/0028000.pt +[2025-04-27 06:17:05] (step=0030025) Train Loss: 5.9794, Train Steps/Sec: 0.06 +[2025-04-27 06:17:28] (step=0030050) Train Loss: 5.9925, Train Steps/Sec: 1.12 +[2025-04-27 06:17:50] (step=0030075) Train Loss: 5.9350, Train Steps/Sec: 1.12 +[2025-04-27 06:18:12] (step=0030100) Train Loss: 5.9932, Train Steps/Sec: 1.12 +[2025-04-27 06:18:34] (step=0030125) Train Loss: 5.9421, Train Steps/Sec: 1.12 +[2025-04-27 06:18:57] (step=0030150) Train Loss: 5.9626, Train Steps/Sec: 1.12 +[2025-04-27 06:19:19] (step=0030175) Train Loss: 5.9365, Train Steps/Sec: 1.12 +[2025-04-27 06:19:42] (step=0030200) Train Loss: 5.9442, Train Steps/Sec: 1.11 +[2025-04-27 06:20:04] (step=0030225) Train Loss: 5.9771, Train Steps/Sec: 1.11 +[2025-04-27 06:20:26] (step=0030250) Train Loss: 5.8949, Train Steps/Sec: 1.12 +[2025-04-27 06:20:49] (step=0030275) Train Loss: 6.0196, Train Steps/Sec: 1.12 +[2025-04-27 06:21:11] (step=0030300) Train Loss: 5.9784, Train Steps/Sec: 1.12 +[2025-04-27 06:21:33] (step=0030325) Train Loss: 5.9627, Train Steps/Sec: 1.12 +[2025-04-27 06:21:56] (step=0030350) Train Loss: 5.9548, Train Steps/Sec: 1.12 +[2025-04-27 06:22:18] (step=0030375) Train Loss: 5.9799, Train Steps/Sec: 1.12 +[2025-04-27 06:22:40] (step=0030400) Train Loss: 5.8935, Train Steps/Sec: 1.11 +[2025-04-27 06:23:03] (step=0030425) Train Loss: 5.9629, Train Steps/Sec: 1.12 +[2025-04-27 06:23:25] (step=0030450) Train Loss: 5.8692, Train Steps/Sec: 1.12 +[2025-04-27 06:23:47] (step=0030475) Train Loss: 5.9449, Train Steps/Sec: 1.12 +[2025-04-27 06:24:10] (step=0030500) Train Loss: 5.8719, Train Steps/Sec: 1.12 +[2025-04-27 06:24:32] (step=0030525) Train Loss: 5.9071, Train Steps/Sec: 1.12 +[2025-04-27 06:24:54] (step=0030550) Train Loss: 5.9811, Train Steps/Sec: 1.12 +[2025-04-27 06:25:17] (step=0030575) Train Loss: 5.9040, Train Steps/Sec: 1.12 +[2025-04-27 06:25:39] (step=0030600) Train Loss: 5.8865, Train Steps/Sec: 1.11 +[2025-04-27 06:26:02] (step=0030625) Train Loss: 5.9304, Train Steps/Sec: 1.12 +[2025-04-27 06:26:24] (step=0030650) Train Loss: 5.9030, Train Steps/Sec: 1.12 +[2025-04-27 06:26:46] (step=0030675) Train Loss: 5.8797, Train Steps/Sec: 1.12 +[2025-04-27 06:27:09] (step=0030700) Train Loss: 5.8798, Train Steps/Sec: 1.12 +[2025-04-27 06:27:31] (step=0030725) Train Loss: 5.9719, Train Steps/Sec: 1.12 +[2025-04-27 06:27:53] (step=0030750) Train Loss: 5.9243, Train Steps/Sec: 1.12 +[2025-04-27 06:28:16] (step=0030775) Train Loss: 5.9510, Train Steps/Sec: 1.12 +[2025-04-27 06:28:38] (step=0030800) Train Loss: 5.9283, Train Steps/Sec: 1.12 +[2025-04-27 06:29:00] (step=0030825) Train Loss: 5.8951, Train Steps/Sec: 1.12 +[2025-04-27 06:29:23] (step=0030850) Train Loss: 5.9631, Train Steps/Sec: 1.12 +[2025-04-27 06:29:45] (step=0030875) Train Loss: 5.9411, Train Steps/Sec: 1.12 +[2025-04-27 06:30:07] (step=0030900) Train Loss: 5.8908, Train Steps/Sec: 1.12 +[2025-04-27 06:30:30] (step=0030925) Train Loss: 5.9564, Train Steps/Sec: 1.12 +[2025-04-27 06:30:52] (step=0030950) Train Loss: 5.9855, Train Steps/Sec: 1.12 +[2025-04-27 06:31:14] (step=0030975) Train Loss: 5.8587, Train Steps/Sec: 1.12 +[2025-04-27 06:31:37] (step=0031000) Train Loss: 5.8753, Train Steps/Sec: 1.11 +[2025-04-27 06:31:59] (step=0031025) Train Loss: 5.9637, Train Steps/Sec: 1.12 +[2025-04-27 06:32:21] (step=0031050) Train Loss: 5.9178, Train Steps/Sec: 1.12 +[2025-04-27 06:32:44] (step=0031075) Train Loss: 5.9138, Train Steps/Sec: 1.12 +[2025-04-27 06:33:06] (step=0031100) Train Loss: 5.9048, Train Steps/Sec: 1.12 +[2025-04-27 06:33:28] (step=0031125) Train Loss: 5.9960, Train Steps/Sec: 1.12 +[2025-04-27 06:33:51] (step=0031150) Train Loss: 5.9257, Train Steps/Sec: 1.12 +[2025-04-27 06:34:13] (step=0031175) Train Loss: 5.9037, Train Steps/Sec: 1.12 +[2025-04-27 06:34:35] (step=0031200) Train Loss: 5.9814, Train Steps/Sec: 1.12 +[2025-04-27 06:34:58] (step=0031225) Train Loss: 5.9289, Train Steps/Sec: 1.12 +[2025-04-27 06:35:20] (step=0031250) Train Loss: 5.9869, Train Steps/Sec: 1.12 +[2025-04-27 06:35:42] (step=0031275) Train Loss: 5.9298, Train Steps/Sec: 1.12 +[2025-04-27 06:36:05] (step=0031300) Train Loss: 5.9372, Train Steps/Sec: 1.12 +[2025-04-27 06:36:27] (step=0031325) Train Loss: 5.9204, Train Steps/Sec: 1.12 +[2025-04-27 06:36:49] (step=0031350) Train Loss: 5.9112, Train Steps/Sec: 1.12 +[2025-04-27 06:37:12] (step=0031375) Train Loss: 5.9100, Train Steps/Sec: 1.12 +[2025-04-27 06:37:34] (step=0031400) Train Loss: 5.8562, Train Steps/Sec: 1.11 +[2025-04-27 06:37:57] (step=0031425) Train Loss: 5.9711, Train Steps/Sec: 1.12 +[2025-04-27 06:38:19] (step=0031450) Train Loss: 5.9260, Train Steps/Sec: 1.12 +[2025-04-27 06:38:41] (step=0031475) Train Loss: 5.9274, Train Steps/Sec: 1.12 +[2025-04-27 06:39:03] (step=0031500) Train Loss: 5.9075, Train Steps/Sec: 1.12 +[2025-04-27 06:39:26] (step=0031525) Train Loss: 5.9507, Train Steps/Sec: 1.12 +[2025-04-27 06:39:48] (step=0031550) Train Loss: 5.9273, Train Steps/Sec: 1.12 +[2025-04-27 06:40:10] (step=0031575) Train Loss: 5.9084, Train Steps/Sec: 1.12 +[2025-04-27 06:40:33] (step=0031600) Train Loss: 5.8337, Train Steps/Sec: 1.11 +[2025-04-27 06:40:55] (step=0031625) Train Loss: 5.8747, Train Steps/Sec: 1.12 +[2025-04-27 06:41:18] (step=0031650) Train Loss: 5.9105, Train Steps/Sec: 1.12 +[2025-04-27 06:41:40] (step=0031675) Train Loss: 5.9273, Train Steps/Sec: 1.12 +[2025-04-27 06:42:02] (step=0031700) Train Loss: 5.9116, Train Steps/Sec: 1.12 +[2025-04-27 06:42:25] (step=0031725) Train Loss: 5.9404, Train Steps/Sec: 1.12 +[2025-04-27 06:42:47] (step=0031750) Train Loss: 5.9233, Train Steps/Sec: 1.12 +[2025-04-27 06:43:09] (step=0031775) Train Loss: 5.9642, Train Steps/Sec: 1.12 +[2025-04-27 06:43:32] (step=0031800) Train Loss: 5.9430, Train Steps/Sec: 1.12 +[2025-04-27 06:43:54] (step=0031825) Train Loss: 5.9106, Train Steps/Sec: 1.12 +[2025-04-27 06:44:16] (step=0031850) Train Loss: 5.9079, Train Steps/Sec: 1.12 +[2025-04-27 06:44:38] (step=0031875) Train Loss: 5.8988, Train Steps/Sec: 1.12 +[2025-04-27 06:45:01] (step=0031900) Train Loss: 5.8514, Train Steps/Sec: 1.12 +[2025-04-27 06:45:23] (step=0031925) Train Loss: 5.9318, Train Steps/Sec: 1.12 +[2025-04-27 06:45:45] (step=0031950) Train Loss: 5.9315, Train Steps/Sec: 1.12 +[2025-04-27 06:46:08] (step=0031975) Train Loss: 5.8701, Train Steps/Sec: 1.12 +[2025-04-27 06:46:30] (step=0032000) Train Loss: 5.9243, Train Steps/Sec: 1.12 +[2025-04-27 06:46:30] text_config is None. Initializing the text config with default values (`OPTConfig`). +[2025-04-27 06:52:38] Finish Eval in 32000 steps... +[2025-04-27 06:52:57] Saved checkpoint to checkpoint/X2I_UltraEdiit_fourmask_LLava_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_filtered_objects_4mask_1280_context/003-GPT-XL/checkpoints/0032000.pt +[2025-04-27 06:52:59] Removed old checkpoint: checkpoint/X2I_UltraEdiit_fourmask_LLava_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_filtered_objects_4mask_1280_context/003-GPT-XL/checkpoints/0030000.pt +[2025-04-27 06:53:22] (step=0032025) Train Loss: 5.8780, Train Steps/Sec: 0.06 +[2025-04-27 06:53:44] (step=0032050) Train Loss: 5.9092, Train Steps/Sec: 1.12 +[2025-04-27 06:54:06] (step=0032075) Train Loss: 5.9585, Train Steps/Sec: 1.12 +[2025-04-27 06:54:29] (step=0032100) Train Loss: 5.9574, Train Steps/Sec: 1.12 +[2025-04-27 06:54:51] (step=0032125) Train Loss: 5.9136, Train Steps/Sec: 1.12 +[2025-04-27 06:55:13] (step=0032150) Train Loss: 5.9636, Train Steps/Sec: 1.12 +[2025-04-27 06:55:36] (step=0032175) Train Loss: 5.9369, Train Steps/Sec: 1.12 +[2025-04-27 06:55:58] (step=0032200) Train Loss: 5.8829, Train Steps/Sec: 1.12 +[2025-04-27 06:56:26] (step=0032225) Train Loss: 5.9316, Train Steps/Sec: 0.91 +[2025-04-27 06:56:48] (step=0032250) Train Loss: 5.8904, Train Steps/Sec: 1.12 +[2025-04-27 06:57:10] (step=0032275) Train Loss: 5.9508, Train Steps/Sec: 1.12 +[2025-04-27 06:57:32] (step=0032300) Train Loss: 5.9065, Train Steps/Sec: 1.12 +[2025-04-27 06:57:55] (step=0032325) Train Loss: 5.9025, Train Steps/Sec: 1.12 +[2025-04-27 06:58:17] (step=0032350) Train Loss: 5.9105, Train Steps/Sec: 1.12 +[2025-04-27 06:58:39] (step=0032375) Train Loss: 5.8377, Train Steps/Sec: 1.12 +[2025-04-27 06:59:02] (step=0032400) Train Loss: 5.9348, Train Steps/Sec: 1.12 +[2025-04-27 06:59:24] (step=0032425) Train Loss: 5.9100, Train Steps/Sec: 1.12 +[2025-04-27 06:59:46] (step=0032450) Train Loss: 5.9330, Train Steps/Sec: 1.12 +[2025-04-27 07:00:09] (step=0032475) Train Loss: 5.8879, Train Steps/Sec: 1.12 +[2025-04-27 07:00:31] (step=0032500) Train Loss: 5.9883, Train Steps/Sec: 1.12 +[2025-04-27 07:00:58] (step=0032525) Train Loss: 5.9211, Train Steps/Sec: 0.93 +[2025-04-27 07:01:20] (step=0032550) Train Loss: 5.8420, Train Steps/Sec: 1.11 +[2025-04-27 07:01:47] (step=0032575) Train Loss: 5.9489, Train Steps/Sec: 0.93 +[2025-04-27 07:02:10] (step=0032600) Train Loss: 5.8761, Train Steps/Sec: 1.11 +[2025-04-27 07:02:32] (step=0032625) Train Loss: 5.8357, Train Steps/Sec: 1.12 +[2025-04-27 07:02:54] (step=0032650) Train Loss: 5.9089, Train Steps/Sec: 1.12 +[2025-04-27 07:03:17] (step=0032675) Train Loss: 5.9074, Train Steps/Sec: 1.12 +[2025-04-27 07:03:59] (step=0032700) Train Loss: 5.9196, Train Steps/Sec: 0.59 +[2025-04-27 07:04:21] (step=0032725) Train Loss: 5.9385, Train Steps/Sec: 1.12 +[2025-04-27 07:04:44] (step=0032750) Train Loss: 5.9260, Train Steps/Sec: 1.12 +[2025-04-27 07:05:06] (step=0032775) Train Loss: 5.9205, Train Steps/Sec: 1.12 +[2025-04-27 07:05:28] (step=0032800) Train Loss: 5.9240, Train Steps/Sec: 1.11 +[2025-04-27 07:05:51] (step=0032825) Train Loss: 5.8951, Train Steps/Sec: 1.12 +[2025-04-27 07:06:13] (step=0032850) Train Loss: 5.8784, Train Steps/Sec: 1.12 +[2025-04-27 07:06:35] (step=0032875) Train Loss: 5.9186, Train Steps/Sec: 1.12 +[2025-04-27 07:06:58] (step=0032900) Train Loss: 5.9249, Train Steps/Sec: 1.12 +[2025-04-27 07:07:20] (step=0032925) Train Loss: 5.9545, Train Steps/Sec: 1.12 +[2025-04-27 07:07:42] (step=0032950) Train Loss: 5.8904, Train Steps/Sec: 1.12 +[2025-04-27 07:08:05] (step=0032975) Train Loss: 5.9222, Train Steps/Sec: 1.12 +[2025-04-27 07:08:27] (step=0033000) Train Loss: 5.9331, Train Steps/Sec: 1.12 +[2025-04-27 07:08:49] (step=0033025) Train Loss: 5.8067, Train Steps/Sec: 1.12 +[2025-04-27 07:09:12] (step=0033050) Train Loss: 5.9677, Train Steps/Sec: 1.12 +[2025-04-27 07:09:34] (step=0033075) Train Loss: 5.9730, Train Steps/Sec: 1.12 +[2025-04-27 07:09:56] (step=0033100) Train Loss: 5.9071, Train Steps/Sec: 1.12 +[2025-04-27 07:10:19] (step=0033125) Train Loss: 5.9323, Train Steps/Sec: 1.12 +[2025-04-27 07:10:41] (step=0033150) Train Loss: 5.9386, Train Steps/Sec: 1.12 +[2025-04-27 07:11:03] (step=0033175) Train Loss: 5.9198, Train Steps/Sec: 1.12 +[2025-04-27 07:11:26] (step=0033200) Train Loss: 5.8328, Train Steps/Sec: 1.12 +[2025-04-27 07:11:48] (step=0033225) Train Loss: 5.8746, Train Steps/Sec: 1.12 +[2025-04-27 07:12:10] (step=0033250) Train Loss: 5.9271, Train Steps/Sec: 1.12 +[2025-04-27 07:12:33] (step=0033275) Train Loss: 5.9293, Train Steps/Sec: 1.12 +[2025-04-27 07:12:55] (step=0033300) Train Loss: 5.9515, Train Steps/Sec: 1.12 +[2025-04-27 07:13:17] (step=0033325) Train Loss: 5.8866, Train Steps/Sec: 1.12 +[2025-04-27 07:13:40] (step=0033350) Train Loss: 5.9100, Train Steps/Sec: 1.12 +[2025-04-27 07:14:02] (step=0033375) Train Loss: 5.8542, Train Steps/Sec: 1.12 +[2025-04-27 07:14:24] (step=0033400) Train Loss: 5.9030, Train Steps/Sec: 1.12 +[2025-04-27 07:14:47] (step=0033425) Train Loss: 5.8882, Train Steps/Sec: 1.12 +[2025-04-27 07:15:09] (step=0033450) Train Loss: 5.8799, Train Steps/Sec: 1.12 +[2025-04-27 07:15:31] (step=0033475) Train Loss: 5.9888, Train Steps/Sec: 1.11 +[2025-04-27 07:15:54] (step=0033500) Train Loss: 5.8629, Train Steps/Sec: 1.12 +[2025-04-27 07:16:16] (step=0033525) Train Loss: 5.8951, Train Steps/Sec: 1.12 +[2025-04-27 07:16:38] (step=0033550) Train Loss: 5.9157, Train Steps/Sec: 1.12 +[2025-04-27 07:17:01] (step=0033575) Train Loss: 5.8491, Train Steps/Sec: 1.12 +[2025-04-27 07:17:23] (step=0033600) Train Loss: 5.9008, Train Steps/Sec: 1.11 +[2025-04-27 07:17:45] (step=0033625) Train Loss: 5.9354, Train Steps/Sec: 1.12 +[2025-04-27 07:18:08] (step=0033650) Train Loss: 5.9262, Train Steps/Sec: 1.12 +[2025-04-27 07:18:30] (step=0033675) Train Loss: 5.8387, Train Steps/Sec: 1.12 +[2025-04-27 07:18:52] (step=0033700) Train Loss: 5.9911, Train Steps/Sec: 1.12 +[2025-04-27 07:19:15] (step=0033725) Train Loss: 5.9863, Train Steps/Sec: 1.12 +[2025-04-27 07:19:37] (step=0033750) Train Loss: 5.9202, Train Steps/Sec: 1.12 +[2025-04-27 07:19:59] (step=0033775) Train Loss: 5.9247, Train Steps/Sec: 1.12 +[2025-04-27 07:20:22] (step=0033800) Train Loss: 5.8545, Train Steps/Sec: 1.11 +[2025-04-27 07:20:44] (step=0033825) Train Loss: 5.9205, Train Steps/Sec: 1.12 +[2025-04-27 07:21:07] (step=0033850) Train Loss: 5.9053, Train Steps/Sec: 1.12 +[2025-04-27 07:21:29] (step=0033875) Train Loss: 5.8777, Train Steps/Sec: 1.12 +[2025-04-27 07:21:51] (step=0033900) Train Loss: 5.8924, Train Steps/Sec: 1.12 +[2025-04-27 07:22:14] (step=0033925) Train Loss: 5.9701, Train Steps/Sec: 1.12 +[2025-04-27 07:22:36] (step=0033950) Train Loss: 5.9150, Train Steps/Sec: 1.12 +[2025-04-27 07:22:58] (step=0033975) Train Loss: 5.9348, Train Steps/Sec: 1.12 +[2025-04-27 07:23:21] (step=0034000) Train Loss: 5.9628, Train Steps/Sec: 1.12 +[2025-04-27 07:23:21] text_config is None. Initializing the text config with default values (`OPTConfig`). +[2025-04-27 07:29:29] Finish Eval in 34000 steps... +[2025-04-27 07:29:48] Saved checkpoint to checkpoint/X2I_UltraEdiit_fourmask_LLava_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_filtered_objects_4mask_1280_context/003-GPT-XL/checkpoints/0034000.pt +[2025-04-27 07:29:50] Removed old checkpoint: checkpoint/X2I_UltraEdiit_fourmask_LLava_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_filtered_objects_4mask_1280_context/003-GPT-XL/checkpoints/0032000.pt +[2025-04-27 07:30:13] (step=0034025) Train Loss: 5.8748, Train Steps/Sec: 0.06 +[2025-04-27 07:30:35] (step=0034050) Train Loss: 5.9991, Train Steps/Sec: 1.12 +[2025-04-27 07:30:57] (step=0034075) Train Loss: 5.8893, Train Steps/Sec: 1.12 +[2025-04-27 07:31:20] (step=0034100) Train Loss: 5.9605, Train Steps/Sec: 1.12 +[2025-04-27 07:31:42] (step=0034125) Train Loss: 5.9311, Train Steps/Sec: 1.12 +[2025-04-27 07:32:04] (step=0034150) Train Loss: 5.9065, Train Steps/Sec: 1.12 +[2025-04-27 07:32:26] (step=0034175) Train Loss: 5.9619, Train Steps/Sec: 1.12 +[2025-04-27 07:32:49] (step=0034200) Train Loss: 5.9195, Train Steps/Sec: 1.12 +[2025-04-27 07:33:11] (step=0034225) Train Loss: 5.9370, Train Steps/Sec: 1.12 +[2025-04-27 07:33:34] (step=0034250) Train Loss: 5.8952, Train Steps/Sec: 1.12 +[2025-04-27 07:33:56] (step=0034275) Train Loss: 5.8829, Train Steps/Sec: 1.12 +[2025-04-27 07:34:18] (step=0034300) Train Loss: 5.9048, Train Steps/Sec: 1.12 +[2025-04-27 07:34:40] (step=0034325) Train Loss: 5.9566, Train Steps/Sec: 1.12 +[2025-04-27 07:35:03] (step=0034350) Train Loss: 5.9103, Train Steps/Sec: 1.12 +[2025-04-27 07:35:25] (step=0034375) Train Loss: 5.8570, Train Steps/Sec: 1.12 +[2025-04-27 07:35:47] (step=0034400) Train Loss: 5.9379, Train Steps/Sec: 1.11 +[2025-04-27 07:36:10] (step=0034425) Train Loss: 5.8920, Train Steps/Sec: 1.12 +[2025-04-27 07:36:32] (step=0034450) Train Loss: 5.9443, Train Steps/Sec: 1.11 +[2025-04-27 07:36:55] (step=0034475) Train Loss: 5.9701, Train Steps/Sec: 1.12 +[2025-04-27 07:37:17] (step=0034500) Train Loss: 5.8841, Train Steps/Sec: 1.12 +[2025-04-27 07:37:39] (step=0034525) Train Loss: 5.8905, Train Steps/Sec: 1.12 +[2025-04-27 07:38:01] (step=0034550) Train Loss: 5.8829, Train Steps/Sec: 1.12 +[2025-04-27 07:38:24] (step=0034575) Train Loss: 5.9348, Train Steps/Sec: 1.12 +[2025-04-27 07:38:46] (step=0034600) Train Loss: 5.8771, Train Steps/Sec: 1.11 +[2025-04-27 07:39:09] (step=0034625) Train Loss: 5.8376, Train Steps/Sec: 1.12 +[2025-04-27 07:39:31] (step=0034650) Train Loss: 5.9063, Train Steps/Sec: 1.12 +[2025-04-27 07:39:53] (step=0034675) Train Loss: 5.8400, Train Steps/Sec: 1.12 +[2025-04-27 07:40:16] (step=0034700) Train Loss: 5.9418, Train Steps/Sec: 1.12 +[2025-04-27 07:40:38] (step=0034725) Train Loss: 5.8576, Train Steps/Sec: 1.12 +[2025-04-27 07:41:00] (step=0034750) Train Loss: 5.9099, Train Steps/Sec: 1.12 +[2025-04-27 07:41:23] (step=0034775) Train Loss: 5.8636, Train Steps/Sec: 1.12 +[2025-04-27 07:41:45] (step=0034800) Train Loss: 5.9211, Train Steps/Sec: 1.12 +[2025-04-27 07:42:07] (step=0034825) Train Loss: 5.8795, Train Steps/Sec: 1.12 +[2025-04-27 07:42:30] (step=0034850) Train Loss: 5.9088, Train Steps/Sec: 1.12 +[2025-04-27 07:42:52] (step=0034875) Train Loss: 5.9331, Train Steps/Sec: 1.12 +[2025-04-27 07:43:14] (step=0034900) Train Loss: 5.9097, Train Steps/Sec: 1.12 +[2025-04-27 07:43:37] (step=0034925) Train Loss: 5.9184, Train Steps/Sec: 1.12 +[2025-04-27 07:43:59] (step=0034950) Train Loss: 6.0011, Train Steps/Sec: 1.12 +[2025-04-27 07:44:21] (step=0034975) Train Loss: 5.9369, Train Steps/Sec: 1.12 +[2025-04-27 07:44:44] (step=0035000) Train Loss: 5.9623, Train Steps/Sec: 1.12 +[2025-04-27 07:45:06] (step=0035025) Train Loss: 5.9646, Train Steps/Sec: 1.12 +[2025-04-27 07:45:28] (step=0035050) Train Loss: 5.8805, Train Steps/Sec: 1.12 +[2025-04-27 07:45:51] (step=0035075) Train Loss: 5.9409, Train Steps/Sec: 1.11 +[2025-04-27 07:46:13] (step=0035100) Train Loss: 5.9395, Train Steps/Sec: 1.12 +[2025-04-27 07:46:35] (step=0035125) Train Loss: 5.9030, Train Steps/Sec: 1.12 +[2025-04-27 07:46:58] (step=0035150) Train Loss: 5.9827, Train Steps/Sec: 1.12 +[2025-04-27 07:47:20] (step=0035175) Train Loss: 5.9801, Train Steps/Sec: 1.12 +[2025-04-27 07:47:42] (step=0035200) Train Loss: 5.9522, Train Steps/Sec: 1.12 +[2025-04-27 07:48:05] (step=0035225) Train Loss: 5.8972, Train Steps/Sec: 1.12 +[2025-04-27 07:48:27] (step=0035250) Train Loss: 5.8915, Train Steps/Sec: 1.12 +[2025-04-27 07:48:49] (step=0035275) Train Loss: 5.8562, Train Steps/Sec: 1.12 +[2025-04-27 07:49:12] (step=0035300) Train Loss: 5.9557, Train Steps/Sec: 1.12 +[2025-04-27 07:49:34] (step=0035325) Train Loss: 5.9047, Train Steps/Sec: 1.12 +[2025-04-27 07:49:57] (step=0035350) Train Loss: 5.9244, Train Steps/Sec: 1.12 +[2025-04-27 07:50:19] (step=0035375) Train Loss: 5.8832, Train Steps/Sec: 1.12 +[2025-04-27 07:50:41] (step=0035400) Train Loss: 5.9131, Train Steps/Sec: 1.12 +[2025-04-27 07:51:04] (step=0035425) Train Loss: 5.9146, Train Steps/Sec: 1.12 +[2025-04-27 07:51:26] (step=0035450) Train Loss: 5.9586, Train Steps/Sec: 1.12 +[2025-04-27 07:51:48] (step=0035475) Train Loss: 5.9430, Train Steps/Sec: 1.12 +[2025-04-27 07:52:11] (step=0035500) Train Loss: 5.9872, Train Steps/Sec: 1.12 +[2025-04-27 07:52:33] (step=0035525) Train Loss: 5.8978, Train Steps/Sec: 1.12 +[2025-04-27 07:52:55] (step=0035550) Train Loss: 5.9399, Train Steps/Sec: 1.12 +[2025-04-27 07:53:18] (step=0035575) Train Loss: 5.8550, Train Steps/Sec: 1.12 +[2025-04-27 07:53:40] (step=0035600) Train Loss: 5.9245, Train Steps/Sec: 1.12 +[2025-04-27 07:54:02] (step=0035625) Train Loss: 5.9235, Train Steps/Sec: 1.12 +[2025-04-27 07:54:25] (step=0035650) Train Loss: 5.9068, Train Steps/Sec: 1.12 +[2025-04-27 07:54:47] (step=0035675) Train Loss: 5.8909, Train Steps/Sec: 1.12 +[2025-04-27 07:55:09] (step=0035700) Train Loss: 5.9012, Train Steps/Sec: 1.12 +[2025-04-27 07:55:32] (step=0035725) Train Loss: 5.9413, Train Steps/Sec: 1.12 +[2025-04-27 07:55:54] (step=0035750) Train Loss: 5.8568, Train Steps/Sec: 1.12 +[2025-04-27 07:56:16] (step=0035775) Train Loss: 5.9464, Train Steps/Sec: 1.12 +[2025-04-27 07:56:39] (step=0035800) Train Loss: 5.9195, Train Steps/Sec: 1.12 +[2025-04-27 07:57:01] (step=0035825) Train Loss: 5.8914, Train Steps/Sec: 1.12 +[2025-04-27 07:57:24] (step=0035850) Train Loss: 5.9127, Train Steps/Sec: 1.12 +[2025-04-27 07:57:46] (step=0035875) Train Loss: 5.8864, Train Steps/Sec: 1.12 +[2025-04-27 07:58:08] (step=0035900) Train Loss: 5.9341, Train Steps/Sec: 1.12 +[2025-04-27 07:58:30] (step=0035925) Train Loss: 5.8822, Train Steps/Sec: 1.12 +[2025-04-27 07:58:53] (step=0035950) Train Loss: 5.9004, Train Steps/Sec: 1.12 +[2025-04-27 07:59:15] (step=0035975) Train Loss: 5.9062, Train Steps/Sec: 1.12 +[2025-04-27 07:59:38] (step=0036000) Train Loss: 5.8767, Train Steps/Sec: 1.12 +[2025-04-27 07:59:38] text_config is None. Initializing the text config with default values (`OPTConfig`). +[2025-04-27 08:05:44] Finish Eval in 36000 steps... +[2025-04-27 08:06:03] Saved checkpoint to checkpoint/X2I_UltraEdiit_fourmask_LLava_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_filtered_objects_4mask_1280_context/003-GPT-XL/checkpoints/0036000.pt +[2025-04-27 08:06:05] Removed old checkpoint: checkpoint/X2I_UltraEdiit_fourmask_LLava_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_filtered_objects_4mask_1280_context/003-GPT-XL/checkpoints/0034000.pt +[2025-04-27 08:06:28] (step=0036025) Train Loss: 5.9054, Train Steps/Sec: 0.06 +[2025-04-27 08:06:50] (step=0036050) Train Loss: 5.9168, Train Steps/Sec: 1.12 +[2025-04-27 08:07:12] (step=0036075) Train Loss: 5.8957, Train Steps/Sec: 1.12 +[2025-04-27 08:07:35] (step=0036100) Train Loss: 5.8885, Train Steps/Sec: 1.12 +[2025-04-27 08:07:57] (step=0036125) Train Loss: 5.9333, Train Steps/Sec: 1.12 +[2025-04-27 08:08:19] (step=0036150) Train Loss: 5.9329, Train Steps/Sec: 1.12 +[2025-04-27 08:08:42] (step=0036175) Train Loss: 5.8893, Train Steps/Sec: 1.12 +[2025-04-27 08:09:04] (step=0036200) Train Loss: 5.9318, Train Steps/Sec: 1.12 +[2025-04-27 08:09:26] (step=0036225) Train Loss: 5.8900, Train Steps/Sec: 1.12 +[2025-04-27 08:09:49] (step=0036250) Train Loss: 5.9515, Train Steps/Sec: 1.12 +[2025-04-27 08:10:11] (step=0036275) Train Loss: 5.8844, Train Steps/Sec: 1.12 +[2025-04-27 08:10:33] (step=0036300) Train Loss: 5.8723, Train Steps/Sec: 1.12 +[2025-04-27 08:10:56] (step=0036325) Train Loss: 5.8977, Train Steps/Sec: 1.12 +[2025-04-27 08:11:18] (step=0036350) Train Loss: 5.9080, Train Steps/Sec: 1.11 +[2025-04-27 08:11:41] (step=0036375) Train Loss: 5.8309, Train Steps/Sec: 1.12 +[2025-04-27 08:12:03] (step=0036400) Train Loss: 5.8908, Train Steps/Sec: 1.12 +[2025-04-27 08:12:25] (step=0036425) Train Loss: 5.9047, Train Steps/Sec: 1.12 +[2025-04-27 08:12:48] (step=0036450) Train Loss: 5.9678, Train Steps/Sec: 1.12 +[2025-04-27 08:13:10] (step=0036475) Train Loss: 5.8826, Train Steps/Sec: 1.12 +[2025-04-27 08:13:32] (step=0036500) Train Loss: 5.9219, Train Steps/Sec: 1.12 +[2025-04-27 08:13:54] (step=0036525) Train Loss: 5.8258, Train Steps/Sec: 1.12 +[2025-04-27 08:14:17] (step=0036550) Train Loss: 5.9249, Train Steps/Sec: 1.12 +[2025-04-27 08:14:39] (step=0036575) Train Loss: 5.8534, Train Steps/Sec: 1.12 +[2025-04-27 08:15:01] (step=0036600) Train Loss: 5.8691, Train Steps/Sec: 1.12 +[2025-04-27 08:15:29] (step=0036625) Train Loss: 5.8788, Train Steps/Sec: 0.91 +[2025-04-27 08:15:51] (step=0036650) Train Loss: 5.8979, Train Steps/Sec: 1.12 +[2025-04-27 08:16:14] (step=0036675) Train Loss: 5.9174, Train Steps/Sec: 1.12 +[2025-04-27 08:16:36] (step=0036700) Train Loss: 5.8538, Train Steps/Sec: 1.12 +[2025-04-27 08:16:58] (step=0036725) Train Loss: 5.8681, Train Steps/Sec: 1.12 +[2025-04-27 08:17:20] (step=0036750) Train Loss: 5.9342, Train Steps/Sec: 1.12 +[2025-04-27 08:17:43] (step=0036775) Train Loss: 5.8258, Train Steps/Sec: 1.12 +[2025-04-27 08:18:10] (step=0036800) Train Loss: 5.8871, Train Steps/Sec: 0.92 +[2025-04-27 08:18:32] (step=0036825) Train Loss: 5.9094, Train Steps/Sec: 1.12 +[2025-04-27 08:18:59] (step=0036850) Train Loss: 5.8226, Train Steps/Sec: 0.93 +[2025-04-27 08:19:22] (step=0036875) Train Loss: 5.9130, Train Steps/Sec: 1.12 +[2025-04-27 08:19:44] (step=0036900) Train Loss: 5.8419, Train Steps/Sec: 1.12 +[2025-04-27 08:20:06] (step=0036925) Train Loss: 5.8587, Train Steps/Sec: 1.12 +[2025-04-27 08:20:29] (step=0036950) Train Loss: 5.8645, Train Steps/Sec: 1.12 +[2025-04-27 08:20:51] (step=0036975) Train Loss: 5.8939, Train Steps/Sec: 1.12 +[2025-04-27 08:21:13] (step=0037000) Train Loss: 5.9451, Train Steps/Sec: 1.12 +[2025-04-27 08:21:50] (step=0037025) Train Loss: 5.9055, Train Steps/Sec: 0.68 +[2025-04-27 08:22:18] (step=0037050) Train Loss: 5.8643, Train Steps/Sec: 0.90 +[2025-04-27 08:22:40] (step=0037075) Train Loss: 5.8616, Train Steps/Sec: 1.12 +[2025-04-27 08:23:03] (step=0037100) Train Loss: 5.9064, Train Steps/Sec: 1.12 +[2025-04-27 08:23:25] (step=0037125) Train Loss: 5.8713, Train Steps/Sec: 1.12 +[2025-04-27 08:23:47] (step=0037150) Train Loss: 5.9057, Train Steps/Sec: 1.12 +[2025-04-27 08:24:10] (step=0037175) Train Loss: 5.9435, Train Steps/Sec: 1.12 +[2025-04-27 08:24:32] (step=0037200) Train Loss: 5.8732, Train Steps/Sec: 1.12 +[2025-04-27 08:24:54] (step=0037225) Train Loss: 5.8673, Train Steps/Sec: 1.12 +[2025-04-27 08:25:17] (step=0037250) Train Loss: 5.9176, Train Steps/Sec: 1.12 +[2025-04-27 08:25:39] (step=0037275) Train Loss: 5.9064, Train Steps/Sec: 1.12 +[2025-04-27 08:26:01] (step=0037300) Train Loss: 5.8174, Train Steps/Sec: 1.12 +[2025-04-27 08:26:24] (step=0037325) Train Loss: 5.8546, Train Steps/Sec: 1.12 +[2025-04-27 08:26:46] (step=0037350) Train Loss: 5.8864, Train Steps/Sec: 1.12 +[2025-04-27 08:27:08] (step=0037375) Train Loss: 5.9636, Train Steps/Sec: 1.12 +[2025-04-27 08:27:31] (step=0037400) Train Loss: 5.8853, Train Steps/Sec: 1.12 +[2025-04-27 08:27:53] (step=0037425) Train Loss: 5.8840, Train Steps/Sec: 1.12 +[2025-04-27 08:28:15] (step=0037450) Train Loss: 5.8710, Train Steps/Sec: 1.12 +[2025-04-27 08:28:38] (step=0037475) Train Loss: 5.9179, Train Steps/Sec: 1.12 +[2025-04-27 08:29:00] (step=0037500) Train Loss: 5.8596, Train Steps/Sec: 1.12 +[2025-04-27 08:29:22] (step=0037525) Train Loss: 5.8703, Train Steps/Sec: 1.12 +[2025-04-27 08:29:44] (step=0037550) Train Loss: 5.8431, Train Steps/Sec: 1.12 +[2025-04-27 08:30:07] (step=0037575) Train Loss: 5.9178, Train Steps/Sec: 1.12 +[2025-04-27 08:30:29] (step=0037600) Train Loss: 5.8849, Train Steps/Sec: 1.11 +[2025-04-27 08:30:52] (step=0037625) Train Loss: 5.8734, Train Steps/Sec: 1.12 +[2025-04-27 08:31:14] (step=0037650) Train Loss: 5.8870, Train Steps/Sec: 1.12 +[2025-04-27 08:31:36] (step=0037675) Train Loss: 5.9073, Train Steps/Sec: 1.12 +[2025-04-27 08:31:59] (step=0037700) Train Loss: 5.9240, Train Steps/Sec: 1.12 +[2025-04-27 08:32:21] (step=0037725) Train Loss: 5.8360, Train Steps/Sec: 1.12 +[2025-04-27 08:32:43] (step=0037750) Train Loss: 5.8196, Train Steps/Sec: 1.12 +[2025-04-27 08:33:06] (step=0037775) Train Loss: 5.9395, Train Steps/Sec: 1.12 +[2025-04-27 08:33:28] (step=0037800) Train Loss: 5.8615, Train Steps/Sec: 1.12 +[2025-04-27 08:33:50] (step=0037825) Train Loss: 5.8549, Train Steps/Sec: 1.12 +[2025-04-27 08:34:13] (step=0037850) Train Loss: 5.8736, Train Steps/Sec: 1.12 +[2025-04-27 08:34:35] (step=0037875) Train Loss: 5.8524, Train Steps/Sec: 1.12 +[2025-04-27 08:34:57] (step=0037900) Train Loss: 5.9079, Train Steps/Sec: 1.12 +[2025-04-27 08:35:20] (step=0037925) Train Loss: 5.8772, Train Steps/Sec: 1.12 +[2025-04-27 08:35:42] (step=0037950) Train Loss: 5.8751, Train Steps/Sec: 1.12 +[2025-04-27 08:36:04] (step=0037975) Train Loss: 5.8979, Train Steps/Sec: 1.12 +[2025-04-27 08:36:27] (step=0038000) Train Loss: 5.8520, Train Steps/Sec: 1.11 +[2025-04-27 08:36:27] text_config is None. Initializing the text config with default values (`OPTConfig`). +[2025-04-27 08:42:35] Finish Eval in 38000 steps... +[2025-04-27 08:42:55] Saved checkpoint to checkpoint/X2I_UltraEdiit_fourmask_LLava_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_filtered_objects_4mask_1280_context/003-GPT-XL/checkpoints/0038000.pt +[2025-04-27 08:42:57] Removed old checkpoint: checkpoint/X2I_UltraEdiit_fourmask_LLava_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_filtered_objects_4mask_1280_context/003-GPT-XL/checkpoints/0036000.pt +[2025-04-27 08:43:19] (step=0038025) Train Loss: 5.8474, Train Steps/Sec: 0.06 +[2025-04-27 08:43:41] (step=0038050) Train Loss: 5.8810, Train Steps/Sec: 1.12 +[2025-04-27 08:44:04] (step=0038075) Train Loss: 5.8478, Train Steps/Sec: 1.12 +[2025-04-27 08:44:26] (step=0038100) Train Loss: 5.8821, Train Steps/Sec: 1.12 +[2025-04-27 08:44:48] (step=0038125) Train Loss: 5.8513, Train Steps/Sec: 1.12 +[2025-04-27 08:45:11] (step=0038150) Train Loss: 5.8453, Train Steps/Sec: 1.12 +[2025-04-27 08:45:33] (step=0038175) Train Loss: 5.9101, Train Steps/Sec: 1.12 +[2025-04-27 08:45:55] (step=0038200) Train Loss: 5.8740, Train Steps/Sec: 1.12 +[2025-04-27 08:46:18] (step=0038225) Train Loss: 5.9316, Train Steps/Sec: 1.11 +[2025-04-27 08:46:40] (step=0038250) Train Loss: 5.9175, Train Steps/Sec: 1.11 +[2025-04-27 08:47:03] (step=0038275) Train Loss: 5.8189, Train Steps/Sec: 1.12 +[2025-04-27 08:47:25] (step=0038300) Train Loss: 5.8846, Train Steps/Sec: 1.12 +[2025-04-27 08:47:47] (step=0038325) Train Loss: 5.8771, Train Steps/Sec: 1.12 +[2025-04-27 08:48:10] (step=0038350) Train Loss: 5.9141, Train Steps/Sec: 1.12 +[2025-04-27 08:48:32] (step=0038375) Train Loss: 5.8968, Train Steps/Sec: 1.12 +[2025-04-27 08:48:54] (step=0038400) Train Loss: 5.8371, Train Steps/Sec: 1.11 +[2025-04-27 08:49:17] (step=0038425) Train Loss: 5.8292, Train Steps/Sec: 1.12 +[2025-04-27 08:49:39] (step=0038450) Train Loss: 5.8554, Train Steps/Sec: 1.12 +[2025-04-27 08:50:01] (step=0038475) Train Loss: 5.8755, Train Steps/Sec: 1.12 +[2025-04-27 08:50:24] (step=0038500) Train Loss: 5.8692, Train Steps/Sec: 1.12 +[2025-04-27 08:50:46] (step=0038525) Train Loss: 5.8169, Train Steps/Sec: 1.12 +[2025-04-27 08:51:09] (step=0038550) Train Loss: 5.8419, Train Steps/Sec: 1.12 +[2025-04-27 08:51:31] (step=0038575) Train Loss: 5.8307, Train Steps/Sec: 1.12 +[2025-04-27 08:51:53] (step=0038600) Train Loss: 5.9335, Train Steps/Sec: 1.11 +[2025-04-27 08:52:16] (step=0038625) Train Loss: 5.8898, Train Steps/Sec: 1.12 +[2025-04-27 08:52:38] (step=0038650) Train Loss: 5.8799, Train Steps/Sec: 1.12 +[2025-04-27 08:53:00] (step=0038675) Train Loss: 5.8949, Train Steps/Sec: 1.12 +[2025-04-27 08:53:23] (step=0038700) Train Loss: 5.8902, Train Steps/Sec: 1.12 +[2025-04-27 08:53:45] (step=0038725) Train Loss: 5.9144, Train Steps/Sec: 1.12 +[2025-04-27 08:54:07] (step=0038750) Train Loss: 5.9047, Train Steps/Sec: 1.12 +[2025-04-27 08:54:30] (step=0038775) Train Loss: 5.8257, Train Steps/Sec: 1.12 +[2025-04-27 08:54:52] (step=0038800) Train Loss: 5.9062, Train Steps/Sec: 1.11 +[2025-04-27 08:55:14] (step=0038825) Train Loss: 5.8924, Train Steps/Sec: 1.12 +[2025-04-27 08:55:37] (step=0038850) Train Loss: 5.8536, Train Steps/Sec: 1.12 +[2025-04-27 08:55:59] (step=0038875) Train Loss: 5.8023, Train Steps/Sec: 1.12 +[2025-04-27 08:56:22] (step=0038900) Train Loss: 5.8616, Train Steps/Sec: 1.12 +[2025-04-27 08:56:44] (step=0038925) Train Loss: 5.9045, Train Steps/Sec: 1.12 +[2025-04-27 08:57:06] (step=0038950) Train Loss: 5.8987, Train Steps/Sec: 1.12 +[2025-04-27 08:57:29] (step=0038975) Train Loss: 5.9251, Train Steps/Sec: 1.12 +[2025-04-27 08:57:51] (step=0039000) Train Loss: 5.9266, Train Steps/Sec: 1.11 +[2025-04-27 08:58:13] (step=0039025) Train Loss: 5.8465, Train Steps/Sec: 1.12 +[2025-04-27 08:58:36] (step=0039050) Train Loss: 5.8760, Train Steps/Sec: 1.12 +[2025-04-27 08:58:58] (step=0039075) Train Loss: 5.8560, Train Steps/Sec: 1.12 +[2025-04-27 08:59:20] (step=0039100) Train Loss: 5.9058, Train Steps/Sec: 1.12 +[2025-04-27 08:59:43] (step=0039125) Train Loss: 5.8840, Train Steps/Sec: 1.12 +[2025-04-27 09:00:05] (step=0039150) Train Loss: 5.8594, Train Steps/Sec: 1.12 +[2025-04-27 09:00:27] (step=0039175) Train Loss: 5.8564, Train Steps/Sec: 1.12 +[2025-04-27 09:00:50] (step=0039200) Train Loss: 5.8613, Train Steps/Sec: 1.11 +[2025-04-27 09:01:12] (step=0039225) Train Loss: 5.8879, Train Steps/Sec: 1.12 +[2025-04-27 09:01:34] (step=0039250) Train Loss: 5.8766, Train Steps/Sec: 1.12 +[2025-04-27 09:01:57] (step=0039275) Train Loss: 5.8577, Train Steps/Sec: 1.12 +[2025-04-27 09:02:19] (step=0039300) Train Loss: 5.9746, Train Steps/Sec: 1.12 +[2025-04-27 09:02:41] (step=0039325) Train Loss: 5.9384, Train Steps/Sec: 1.12 +[2025-04-27 09:03:04] (step=0039350) Train Loss: 5.8779, Train Steps/Sec: 1.12 +[2025-04-27 09:03:26] (step=0039375) Train Loss: 5.8664, Train Steps/Sec: 1.12 +[2025-04-27 09:03:48] (step=0039400) Train Loss: 5.7833, Train Steps/Sec: 1.11 +[2025-04-27 09:04:11] (step=0039425) Train Loss: 5.8260, Train Steps/Sec: 1.12 +[2025-04-27 09:04:33] (step=0039450) Train Loss: 5.8953, Train Steps/Sec: 1.12 +[2025-04-27 09:04:55] (step=0039475) Train Loss: 5.9052, Train Steps/Sec: 1.12 +[2025-04-27 09:05:18] (step=0039500) Train Loss: 5.9044, Train Steps/Sec: 1.12 +[2025-04-27 09:05:40] (step=0039525) Train Loss: 5.8151, Train Steps/Sec: 1.12 +[2025-04-27 09:06:02] (step=0039550) Train Loss: 5.8866, Train Steps/Sec: 1.12 +[2025-04-27 09:06:25] (step=0039575) Train Loss: 5.9259, Train Steps/Sec: 1.12 +[2025-04-27 09:06:47] (step=0039600) Train Loss: 5.8711, Train Steps/Sec: 1.12 +[2025-04-27 09:07:10] (step=0039625) Train Loss: 5.8156, Train Steps/Sec: 1.12 +[2025-04-27 09:07:32] (step=0039650) Train Loss: 5.8775, Train Steps/Sec: 1.12 +[2025-04-27 09:07:54] (step=0039675) Train Loss: 5.8296, Train Steps/Sec: 1.12 +[2025-04-27 09:08:16] (step=0039700) Train Loss: 5.9460, Train Steps/Sec: 1.12 +[2025-04-27 09:08:39] (step=0039725) Train Loss: 5.8665, Train Steps/Sec: 1.12 +[2025-04-27 09:09:01] (step=0039750) Train Loss: 5.7933, Train Steps/Sec: 1.12 +[2025-04-27 09:09:24] (step=0039775) Train Loss: 5.8691, Train Steps/Sec: 1.12 +[2025-04-27 09:09:46] (step=0039800) Train Loss: 5.8602, Train Steps/Sec: 1.12 +[2025-04-27 09:10:08] (step=0039825) Train Loss: 5.9218, Train Steps/Sec: 1.12 +[2025-04-27 09:10:31] (step=0039850) Train Loss: 5.9743, Train Steps/Sec: 1.12 +[2025-04-27 09:10:53] (step=0039875) Train Loss: 5.8544, Train Steps/Sec: 1.12 +[2025-04-27 09:11:15] (step=0039900) Train Loss: 5.8429, Train Steps/Sec: 1.12 +[2025-04-27 09:11:38] (step=0039925) Train Loss: 5.9170, Train Steps/Sec: 1.12 +[2025-04-27 09:12:00] (step=0039950) Train Loss: 5.8360, Train Steps/Sec: 1.12 +[2025-04-27 09:12:22] (step=0039975) Train Loss: 5.8082, Train Steps/Sec: 1.12 +[2025-04-27 09:12:45] (step=0040000) Train Loss: 5.8571, Train Steps/Sec: 1.11 +[2025-04-27 09:12:45] text_config is None. Initializing the text config with default values (`OPTConfig`). +[2025-04-27 09:18:53] Finish Eval in 40000 steps... +[2025-04-27 09:19:12] Saved checkpoint to checkpoint/X2I_UltraEdiit_fourmask_LLava_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_filtered_objects_4mask_1280_context/003-GPT-XL/checkpoints/0040000.pt +[2025-04-27 09:19:14] Removed old checkpoint: checkpoint/X2I_UltraEdiit_fourmask_LLava_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_filtered_objects_4mask_1280_context/003-GPT-XL/checkpoints/0038000.pt +[2025-04-27 09:19:37] (step=0040025) Train Loss: 5.8694, Train Steps/Sec: 0.06 +[2025-04-27 09:19:59] (step=0040050) Train Loss: 5.9151, Train Steps/Sec: 1.12 +[2025-04-27 09:20:21] (step=0040075) Train Loss: 5.8016, Train Steps/Sec: 1.12 +[2025-04-27 09:20:44] (step=0040100) Train Loss: 5.9272, Train Steps/Sec: 1.12 +[2025-04-27 09:21:06] (step=0040125) Train Loss: 5.8374, Train Steps/Sec: 1.12 +[2025-04-27 09:21:28] (step=0040150) Train Loss: 5.9271, Train Steps/Sec: 1.11 +[2025-04-27 09:21:51] (step=0040175) Train Loss: 5.8495, Train Steps/Sec: 1.12 +[2025-04-27 09:22:13] (step=0040200) Train Loss: 5.9265, Train Steps/Sec: 1.11 +[2025-04-27 09:22:35] (step=0040225) Train Loss: 5.9238, Train Steps/Sec: 1.12 +[2025-04-27 09:22:58] (step=0040250) Train Loss: 5.8567, Train Steps/Sec: 1.12 +[2025-04-27 09:23:20] (step=0040275) Train Loss: 5.8310, Train Steps/Sec: 1.12 +[2025-04-27 09:23:42] (step=0040300) Train Loss: 5.8119, Train Steps/Sec: 1.12 +[2025-04-27 09:24:05] (step=0040325) Train Loss: 5.8620, Train Steps/Sec: 1.12 +[2025-04-27 09:24:27] (step=0040350) Train Loss: 5.8932, Train Steps/Sec: 1.12 +[2025-04-27 09:24:49] (step=0040375) Train Loss: 5.9260, Train Steps/Sec: 1.12 +[2025-04-27 09:25:12] (step=0040400) Train Loss: 5.8629, Train Steps/Sec: 1.12 +[2025-04-27 09:25:34] (step=0040425) Train Loss: 5.8828, Train Steps/Sec: 1.12 +[2025-04-27 09:25:56] (step=0040450) Train Loss: 5.9289, Train Steps/Sec: 1.12 +[2025-04-27 09:26:19] (step=0040475) Train Loss: 5.8689, Train Steps/Sec: 1.12 +[2025-04-27 09:26:41] (step=0040500) Train Loss: 5.8557, Train Steps/Sec: 1.12 +[2025-04-27 09:27:03] (step=0040525) Train Loss: 5.8864, Train Steps/Sec: 1.12 +[2025-04-27 09:27:26] (step=0040550) Train Loss: 5.8648, Train Steps/Sec: 1.12 +[2025-04-27 09:27:48] (step=0040575) Train Loss: 5.8974, Train Steps/Sec: 1.12 +[2025-04-27 09:28:10] (step=0040600) Train Loss: 5.8579, Train Steps/Sec: 1.11 +[2025-04-27 09:28:33] (step=0040625) Train Loss: 5.8884, Train Steps/Sec: 1.12 +[2025-04-27 09:28:55] (step=0040650) Train Loss: 5.9110, Train Steps/Sec: 1.12 +[2025-04-27 09:29:17] (step=0040675) Train Loss: 5.8622, Train Steps/Sec: 1.12 +[2025-04-27 09:29:40] (step=0040700) Train Loss: 5.8962, Train Steps/Sec: 1.12 +[2025-04-27 09:30:02] (step=0040725) Train Loss: 5.8917, Train Steps/Sec: 1.12 +[2025-04-27 09:30:24] (step=0040750) Train Loss: 5.8723, Train Steps/Sec: 1.12 +[2025-04-27 09:30:47] (step=0040775) Train Loss: 5.7867, Train Steps/Sec: 1.12 +[2025-04-27 09:31:09] (step=0040800) Train Loss: 5.8418, Train Steps/Sec: 1.12 +[2025-04-27 09:31:31] (step=0040825) Train Loss: 5.8711, Train Steps/Sec: 1.12 +[2025-04-27 09:31:54] (step=0040850) Train Loss: 5.8872, Train Steps/Sec: 1.12 +[2025-04-27 09:32:16] (step=0040875) Train Loss: 5.8744, Train Steps/Sec: 1.12 +[2025-04-27 09:32:38] (step=0040900) Train Loss: 5.8982, Train Steps/Sec: 1.12 +[2025-04-27 09:33:01] (step=0040925) Train Loss: 5.9591, Train Steps/Sec: 1.12 +[2025-04-27 09:33:23] (step=0040950) Train Loss: 5.7693, Train Steps/Sec: 1.12 +[2025-04-27 09:33:45] (step=0040975) Train Loss: 5.8233, Train Steps/Sec: 1.12 +[2025-04-27 09:34:08] (step=0041000) Train Loss: 5.8705, Train Steps/Sec: 1.12 +[2025-04-27 09:34:35] (step=0041025) Train Loss: 5.8776, Train Steps/Sec: 0.91 +[2025-04-27 09:35:02] (step=0041050) Train Loss: 5.8488, Train Steps/Sec: 0.93 +[2025-04-27 09:35:24] (step=0041075) Train Loss: 5.8486, Train Steps/Sec: 1.12 +[2025-04-27 09:35:47] (step=0041100) Train Loss: 5.8776, Train Steps/Sec: 1.12 +[2025-04-27 09:36:14] (step=0041125) Train Loss: 5.8817, Train Steps/Sec: 0.93 +[2025-04-27 09:36:36] (step=0041150) Train Loss: 5.8038, Train Steps/Sec: 1.12 +[2025-04-27 09:36:58] (step=0041175) Train Loss: 5.8812, Train Steps/Sec: 1.12 +[2025-04-27 09:37:21] (step=0041200) Train Loss: 5.8525, Train Steps/Sec: 1.12 +[2025-04-27 09:37:43] (step=0041225) Train Loss: 5.9068, Train Steps/Sec: 1.12 +[2025-04-27 09:38:05] (step=0041250) Train Loss: 5.8711, Train Steps/Sec: 1.12 +[2025-04-27 09:38:28] (step=0041275) Train Loss: 5.8500, Train Steps/Sec: 1.12 +[2025-04-27 09:38:50] (step=0041300) Train Loss: 5.8938, Train Steps/Sec: 1.12 +[2025-04-27 09:39:12] (step=0041325) Train Loss: 5.8382, Train Steps/Sec: 1.12 +[2025-04-27 09:39:44] (step=0041350) Train Loss: 5.9090, Train Steps/Sec: 0.78 +[2025-04-27 09:40:17] (step=0041375) Train Loss: 5.9219, Train Steps/Sec: 0.77 +[2025-04-27 09:40:39] (step=0041400) Train Loss: 5.8639, Train Steps/Sec: 1.11 +[2025-04-27 09:41:01] (step=0041425) Train Loss: 5.8749, Train Steps/Sec: 1.12 +[2025-04-27 09:41:24] (step=0041450) Train Loss: 5.8374, Train Steps/Sec: 1.12 +[2025-04-27 09:41:46] (step=0041475) Train Loss: 5.8837, Train Steps/Sec: 1.12 +[2025-04-27 09:42:08] (step=0041500) Train Loss: 5.8397, Train Steps/Sec: 1.12 +[2025-04-27 09:42:31] (step=0041525) Train Loss: 5.8521, Train Steps/Sec: 1.12 +[2025-04-27 09:42:53] (step=0041550) Train Loss: 5.9077, Train Steps/Sec: 1.12 +[2025-04-27 09:43:15] (step=0041575) Train Loss: 5.8805, Train Steps/Sec: 1.12 +[2025-04-27 09:43:38] (step=0041600) Train Loss: 5.8455, Train Steps/Sec: 1.12 +[2025-04-27 09:44:00] (step=0041625) Train Loss: 5.7876, Train Steps/Sec: 1.12 +[2025-04-27 09:44:22] (step=0041650) Train Loss: 5.8443, Train Steps/Sec: 1.12 +[2025-04-27 09:44:45] (step=0041675) Train Loss: 5.8460, Train Steps/Sec: 1.12 +[2025-04-27 09:45:07] (step=0041700) Train Loss: 5.9755, Train Steps/Sec: 1.12 +[2025-04-27 09:45:29] (step=0041725) Train Loss: 5.8437, Train Steps/Sec: 1.12 +[2025-04-27 09:45:52] (step=0041750) Train Loss: 5.9234, Train Steps/Sec: 1.12 +[2025-04-27 09:46:14] (step=0041775) Train Loss: 5.8555, Train Steps/Sec: 1.12 +[2025-04-27 09:46:36] (step=0041800) Train Loss: 5.9066, Train Steps/Sec: 1.11 +[2025-04-27 09:46:59] (step=0041825) Train Loss: 5.9010, Train Steps/Sec: 1.12 +[2025-04-27 09:47:21] (step=0041850) Train Loss: 5.9204, Train Steps/Sec: 1.12 +[2025-04-27 09:47:43] (step=0041875) Train Loss: 5.8899, Train Steps/Sec: 1.12 +[2025-04-27 09:48:06] (step=0041900) Train Loss: 5.8648, Train Steps/Sec: 1.12 +[2025-04-27 09:48:28] (step=0041925) Train Loss: 5.8011, Train Steps/Sec: 1.12 +[2025-04-27 09:48:50] (step=0041950) Train Loss: 5.8391, Train Steps/Sec: 1.12 +[2025-04-27 09:49:13] (step=0041975) Train Loss: 5.8392, Train Steps/Sec: 1.12 +[2025-04-27 09:49:35] (step=0042000) Train Loss: 5.8959, Train Steps/Sec: 1.12 +[2025-04-27 09:49:35] text_config is None. Initializing the text config with default values (`OPTConfig`). +[2025-04-27 09:56:02] Finish Eval in 42000 steps... +[2025-04-27 09:56:24] Saved checkpoint to checkpoint/X2I_UltraEdiit_fourmask_LLava_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_filtered_objects_4mask_1280_context/003-GPT-XL/checkpoints/0042000.pt +[2025-04-27 09:56:27] Removed old checkpoint: checkpoint/X2I_UltraEdiit_fourmask_LLava_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_filtered_objects_4mask_1280_context/003-GPT-XL/checkpoints/0040000.pt +[2025-04-27 09:56:49] (step=0042025) Train Loss: 5.8824, Train Steps/Sec: 0.06 +[2025-04-27 09:57:12] (step=0042050) Train Loss: 5.8154, Train Steps/Sec: 1.09 +[2025-04-27 09:57:35] (step=0042075) Train Loss: 5.9001, Train Steps/Sec: 1.12 +[2025-04-27 09:57:57] (step=0042100) Train Loss: 5.9237, Train Steps/Sec: 1.12 +[2025-04-27 09:58:19] (step=0042125) Train Loss: 5.8363, Train Steps/Sec: 1.12 +[2025-04-27 09:58:42] (step=0042150) Train Loss: 5.7919, Train Steps/Sec: 1.12 +[2025-04-27 09:59:04] (step=0042175) Train Loss: 5.9020, Train Steps/Sec: 1.12 +[2025-04-27 09:59:27] (step=0042200) Train Loss: 5.8871, Train Steps/Sec: 1.11 +[2025-04-27 09:59:49] (step=0042225) Train Loss: 5.9024, Train Steps/Sec: 1.11 +[2025-04-27 10:00:11] (step=0042250) Train Loss: 5.9021, Train Steps/Sec: 1.12 +[2025-04-27 10:00:34] (step=0042275) Train Loss: 5.8520, Train Steps/Sec: 1.11 +[2025-04-27 10:00:56] (step=0042300) Train Loss: 5.8108, Train Steps/Sec: 1.12 +[2025-04-27 10:01:19] (step=0042325) Train Loss: 5.8923, Train Steps/Sec: 1.12 +[2025-04-27 10:01:41] (step=0042350) Train Loss: 5.9194, Train Steps/Sec: 1.11 +[2025-04-27 10:02:04] (step=0042375) Train Loss: 5.8624, Train Steps/Sec: 1.11 +[2025-04-27 10:02:26] (step=0042400) Train Loss: 5.8829, Train Steps/Sec: 1.10 +[2025-04-27 10:02:49] (step=0042425) Train Loss: 5.8410, Train Steps/Sec: 1.11 +[2025-04-27 10:03:11] (step=0042450) Train Loss: 5.8517, Train Steps/Sec: 1.11 +[2025-04-27 10:03:34] (step=0042475) Train Loss: 5.8804, Train Steps/Sec: 1.11 +[2025-04-27 10:03:56] (step=0042500) Train Loss: 5.8574, Train Steps/Sec: 1.12 +[2025-04-27 10:04:18] (step=0042525) Train Loss: 5.8626, Train Steps/Sec: 1.12 +[2025-04-27 10:04:41] (step=0042550) Train Loss: 5.8692, Train Steps/Sec: 1.12 +[2025-04-27 10:05:03] (step=0042575) Train Loss: 5.9332, Train Steps/Sec: 1.12 +[2025-04-27 10:05:26] (step=0042600) Train Loss: 5.8781, Train Steps/Sec: 1.11 +[2025-04-27 10:05:48] (step=0042625) Train Loss: 5.8606, Train Steps/Sec: 1.11 +[2025-04-27 10:06:10] (step=0042650) Train Loss: 5.7922, Train Steps/Sec: 1.12 +[2025-04-27 10:06:33] (step=0042675) Train Loss: 5.8096, Train Steps/Sec: 1.12 +[2025-04-27 10:06:55] (step=0042700) Train Loss: 5.7826, Train Steps/Sec: 1.12 +[2025-04-27 10:07:17] (step=0042725) Train Loss: 5.8463, Train Steps/Sec: 1.12 +[2025-04-27 10:07:40] (step=0042750) Train Loss: 5.8804, Train Steps/Sec: 1.12 +[2025-04-27 10:08:02] (step=0042775) Train Loss: 5.8643, Train Steps/Sec: 1.12 +[2025-04-27 10:08:25] (step=0042800) Train Loss: 5.7893, Train Steps/Sec: 1.11 +[2025-04-27 10:08:47] (step=0042825) Train Loss: 5.8519, Train Steps/Sec: 1.12 +[2025-04-27 10:09:09] (step=0042850) Train Loss: 5.8428, Train Steps/Sec: 1.12 +[2025-04-27 10:09:32] (step=0042875) Train Loss: 5.8890, Train Steps/Sec: 1.12 +[2025-04-27 10:09:54] (step=0042900) Train Loss: 5.8034, Train Steps/Sec: 1.12 +[2025-04-27 10:10:16] (step=0042925) Train Loss: 5.8932, Train Steps/Sec: 1.12 +[2025-04-27 10:10:38] (step=0042950) Train Loss: 5.8606, Train Steps/Sec: 1.12 +[2025-04-27 10:11:01] (step=0042975) Train Loss: 5.8512, Train Steps/Sec: 1.12 +[2025-04-27 10:11:23] (step=0043000) Train Loss: 5.7795, Train Steps/Sec: 1.12 +[2025-04-27 10:11:46] (step=0043025) Train Loss: 5.8279, Train Steps/Sec: 1.12 +[2025-04-27 10:12:08] (step=0043050) Train Loss: 5.8498, Train Steps/Sec: 1.12 +[2025-04-27 10:12:30] (step=0043075) Train Loss: 5.8675, Train Steps/Sec: 1.12 +[2025-04-27 10:12:52] (step=0043100) Train Loss: 5.8528, Train Steps/Sec: 1.12 +[2025-04-27 10:13:15] (step=0043125) Train Loss: 5.8991, Train Steps/Sec: 1.12 +[2025-04-27 10:13:37] (step=0043150) Train Loss: 5.8579, Train Steps/Sec: 1.12 +[2025-04-27 10:13:59] (step=0043175) Train Loss: 5.8585, Train Steps/Sec: 1.12 +[2025-04-27 10:14:22] (step=0043200) Train Loss: 5.7799, Train Steps/Sec: 1.11 +[2025-04-27 10:14:44] (step=0043225) Train Loss: 5.8778, Train Steps/Sec: 1.12 +[2025-04-27 10:15:06] (step=0043250) Train Loss: 5.8280, Train Steps/Sec: 1.12 +[2025-04-27 10:15:29] (step=0043275) Train Loss: 5.8248, Train Steps/Sec: 1.12 +[2025-04-27 10:15:51] (step=0043300) Train Loss: 5.8637, Train Steps/Sec: 1.12 +[2025-04-27 10:16:13] (step=0043325) Train Loss: 5.8435, Train Steps/Sec: 1.12 +[2025-04-27 10:16:36] (step=0043350) Train Loss: 5.8471, Train Steps/Sec: 1.12 +[2025-04-27 10:16:58] (step=0043375) Train Loss: 5.8254, Train Steps/Sec: 1.12 +[2025-04-27 10:17:20] (step=0043400) Train Loss: 5.8449, Train Steps/Sec: 1.12 +[2025-04-27 10:17:43] (step=0043425) Train Loss: 5.8331, Train Steps/Sec: 1.12 +[2025-04-27 10:18:05] (step=0043450) Train Loss: 5.8370, Train Steps/Sec: 1.12 +[2025-04-27 10:18:27] (step=0043475) Train Loss: 5.8046, Train Steps/Sec: 1.12 +[2025-04-27 10:18:50] (step=0043500) Train Loss: 5.8592, Train Steps/Sec: 1.12 +[2025-04-27 10:19:12] (step=0043525) Train Loss: 5.8800, Train Steps/Sec: 1.12 +[2025-04-27 10:19:35] (step=0043550) Train Loss: 5.9135, Train Steps/Sec: 1.12 +[2025-04-27 10:19:57] (step=0043575) Train Loss: 5.8626, Train Steps/Sec: 1.12 +[2025-04-27 10:20:19] (step=0043600) Train Loss: 5.9251, Train Steps/Sec: 1.11 +[2025-04-27 10:20:42] (step=0043625) Train Loss: 5.7884, Train Steps/Sec: 1.12 +[2025-04-27 10:21:04] (step=0043650) Train Loss: 5.8518, Train Steps/Sec: 1.12 +[2025-04-27 10:21:26] (step=0043675) Train Loss: 5.8425, Train Steps/Sec: 1.12 +[2025-04-27 10:21:49] (step=0043700) Train Loss: 5.8809, Train Steps/Sec: 1.12 +[2025-04-27 10:22:11] (step=0043725) Train Loss: 5.8595, Train Steps/Sec: 1.12 +[2025-04-27 10:22:33] (step=0043750) Train Loss: 5.8669, Train Steps/Sec: 1.12 +[2025-04-27 10:22:56] (step=0043775) Train Loss: 5.8286, Train Steps/Sec: 1.12 +[2025-04-27 10:23:18] (step=0043800) Train Loss: 5.8904, Train Steps/Sec: 1.11 +[2025-04-27 10:23:40] (step=0043825) Train Loss: 5.7997, Train Steps/Sec: 1.12 +[2025-04-27 10:24:03] (step=0043850) Train Loss: 5.8477, Train Steps/Sec: 1.12 +[2025-04-27 10:24:25] (step=0043875) Train Loss: 5.7913, Train Steps/Sec: 1.12 +[2025-04-27 10:24:47] (step=0043900) Train Loss: 5.8447, Train Steps/Sec: 1.12 +[2025-04-27 10:25:10] (step=0043925) Train Loss: 5.7923, Train Steps/Sec: 1.12 +[2025-04-27 10:25:32] (step=0043950) Train Loss: 5.9140, Train Steps/Sec: 1.12 +[2025-04-27 10:25:54] (step=0043975) Train Loss: 5.8561, Train Steps/Sec: 1.12 +[2025-04-27 10:26:17] (step=0044000) Train Loss: 5.8351, Train Steps/Sec: 1.11 +[2025-04-27 10:26:17] text_config is None. Initializing the text config with default values (`OPTConfig`). +[2025-04-27 10:32:27] Finish Eval in 44000 steps... +[2025-04-27 10:32:47] Saved checkpoint to checkpoint/X2I_UltraEdiit_fourmask_LLava_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_filtered_objects_4mask_1280_context/003-GPT-XL/checkpoints/0044000.pt +[2025-04-27 10:32:50] Removed old checkpoint: checkpoint/X2I_UltraEdiit_fourmask_LLava_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_filtered_objects_4mask_1280_context/003-GPT-XL/checkpoints/0042000.pt +[2025-04-27 10:33:13] (step=0044025) Train Loss: 5.8251, Train Steps/Sec: 0.06 +[2025-04-27 10:33:35] (step=0044050) Train Loss: 5.7978, Train Steps/Sec: 1.12 +[2025-04-27 10:33:57] (step=0044075) Train Loss: 5.8632, Train Steps/Sec: 1.12 +[2025-04-27 10:34:20] (step=0044100) Train Loss: 5.9039, Train Steps/Sec: 1.12 +[2025-04-27 10:34:42] (step=0044125) Train Loss: 5.8423, Train Steps/Sec: 1.12 +[2025-04-27 10:35:04] (step=0044150) Train Loss: 5.8249, Train Steps/Sec: 1.12 +[2025-04-27 10:35:27] (step=0044175) Train Loss: 5.7924, Train Steps/Sec: 1.12 +[2025-04-27 10:35:49] (step=0044200) Train Loss: 5.8078, Train Steps/Sec: 1.12 +[2025-04-27 10:36:11] (step=0044225) Train Loss: 5.9150, Train Steps/Sec: 1.12 +[2025-04-27 10:36:34] (step=0044250) Train Loss: 5.7911, Train Steps/Sec: 1.12 +[2025-04-27 10:36:56] (step=0044275) Train Loss: 5.7089, Train Steps/Sec: 1.12 +[2025-04-27 10:37:18] (step=0044300) Train Loss: 5.8582, Train Steps/Sec: 1.12 +[2025-04-27 10:37:41] (step=0044325) Train Loss: 5.8230, Train Steps/Sec: 1.12 +[2025-04-27 10:38:03] (step=0044350) Train Loss: 5.8367, Train Steps/Sec: 1.12 +[2025-04-27 10:38:25] (step=0044375) Train Loss: 5.8179, Train Steps/Sec: 1.12 +[2025-04-27 10:38:48] (step=0044400) Train Loss: 5.8105, Train Steps/Sec: 1.12 +[2025-04-27 10:39:10] (step=0044425) Train Loss: 5.9106, Train Steps/Sec: 1.12 +[2025-04-27 10:39:32] (step=0044450) Train Loss: 5.8752, Train Steps/Sec: 1.12 +[2025-04-27 10:39:54] (step=0044475) Train Loss: 5.8485, Train Steps/Sec: 1.12 +[2025-04-27 10:40:17] (step=0044500) Train Loss: 5.7759, Train Steps/Sec: 1.12 +[2025-04-27 10:40:39] (step=0044525) Train Loss: 5.8913, Train Steps/Sec: 1.12 +[2025-04-27 10:41:01] (step=0044550) Train Loss: 5.9238, Train Steps/Sec: 1.12 +[2025-04-27 10:41:24] (step=0044575) Train Loss: 5.8381, Train Steps/Sec: 1.12 +[2025-04-27 10:41:46] (step=0044600) Train Loss: 5.7485, Train Steps/Sec: 1.12 +[2025-04-27 10:42:08] (step=0044625) Train Loss: 5.8872, Train Steps/Sec: 1.12 +[2025-04-27 10:42:31] (step=0044650) Train Loss: 5.8409, Train Steps/Sec: 1.12 +[2025-04-27 10:42:53] (step=0044675) Train Loss: 5.7854, Train Steps/Sec: 1.12 +[2025-04-27 10:43:15] (step=0044700) Train Loss: 5.8466, Train Steps/Sec: 1.12 +[2025-04-27 10:43:38] (step=0044725) Train Loss: 5.9249, Train Steps/Sec: 1.12 +[2025-04-27 10:44:00] (step=0044750) Train Loss: 5.7852, Train Steps/Sec: 1.12 +[2025-04-27 10:44:22] (step=0044775) Train Loss: 5.8545, Train Steps/Sec: 1.12 +[2025-04-27 10:44:45] (step=0044800) Train Loss: 5.7478, Train Steps/Sec: 1.11 +[2025-04-27 10:45:07] (step=0044825) Train Loss: 5.7863, Train Steps/Sec: 1.12 +[2025-04-27 10:45:29] (step=0044850) Train Loss: 5.8420, Train Steps/Sec: 1.12 +[2025-04-27 10:45:52] (step=0044875) Train Loss: 5.8765, Train Steps/Sec: 1.12 +[2025-04-27 10:46:14] (step=0044900) Train Loss: 5.8184, Train Steps/Sec: 1.12 +[2025-04-27 10:46:36] (step=0044925) Train Loss: 5.8313, Train Steps/Sec: 1.12 +[2025-04-27 10:46:59] (step=0044950) Train Loss: 5.8611, Train Steps/Sec: 1.12 +[2025-04-27 10:47:21] (step=0044975) Train Loss: 5.8886, Train Steps/Sec: 1.12 +[2025-04-27 10:47:43] (step=0045000) Train Loss: 5.8739, Train Steps/Sec: 1.12 +[2025-04-27 10:48:06] (step=0045025) Train Loss: 5.8961, Train Steps/Sec: 1.12 +[2025-04-27 10:48:28] (step=0045050) Train Loss: 5.8775, Train Steps/Sec: 1.12 +[2025-04-27 10:48:50] (step=0045075) Train Loss: 5.8421, Train Steps/Sec: 1.12 +[2025-04-27 10:49:12] (step=0045100) Train Loss: 5.9003, Train Steps/Sec: 1.12 +[2025-04-27 10:49:35] (step=0045125) Train Loss: 5.8494, Train Steps/Sec: 1.12 +[2025-04-27 10:49:57] (step=0045150) Train Loss: 5.7969, Train Steps/Sec: 1.12 +[2025-04-27 10:50:19] (step=0045175) Train Loss: 5.8462, Train Steps/Sec: 1.12 +[2025-04-27 10:50:42] (step=0045200) Train Loss: 5.8106, Train Steps/Sec: 1.12 +[2025-04-27 10:51:04] (step=0045225) Train Loss: 5.8579, Train Steps/Sec: 1.12 +[2025-04-27 10:51:26] (step=0045250) Train Loss: 5.8235, Train Steps/Sec: 1.12 +[2025-04-27 10:51:49] (step=0045275) Train Loss: 5.8773, Train Steps/Sec: 1.12 +[2025-04-27 10:52:16] (step=0045300) Train Loss: 5.8133, Train Steps/Sec: 0.92 +[2025-04-27 10:52:38] (step=0045325) Train Loss: 5.8128, Train Steps/Sec: 1.12 +[2025-04-27 10:53:00] (step=0045350) Train Loss: 5.8213, Train Steps/Sec: 1.12 +[2025-04-27 10:53:23] (step=0045375) Train Loss: 5.8343, Train Steps/Sec: 1.12 +[2025-04-27 10:53:50] (step=0045400) Train Loss: 5.8803, Train Steps/Sec: 0.92 +[2025-04-27 10:54:17] (step=0045425) Train Loss: 5.7267, Train Steps/Sec: 0.91 +[2025-04-27 10:54:40] (step=0045450) Train Loss: 5.9024, Train Steps/Sec: 1.12 +[2025-04-27 10:55:02] (step=0045475) Train Loss: 5.8717, Train Steps/Sec: 1.12 +[2025-04-27 10:55:24] (step=0045500) Train Loss: 5.8034, Train Steps/Sec: 1.12 +[2025-04-27 10:55:47] (step=0045525) Train Loss: 5.8120, Train Steps/Sec: 1.12 +[2025-04-27 10:56:09] (step=0045550) Train Loss: 5.8851, Train Steps/Sec: 1.12 +[2025-04-27 10:56:31] (step=0045575) Train Loss: 5.8565, Train Steps/Sec: 1.12 +[2025-04-27 10:56:54] (step=0045600) Train Loss: 5.7946, Train Steps/Sec: 1.11 +[2025-04-27 10:57:16] (step=0045625) Train Loss: 5.8258, Train Steps/Sec: 1.12 +[2025-04-27 10:57:38] (step=0045650) Train Loss: 5.8553, Train Steps/Sec: 1.12 +[2025-04-27 10:58:10] (step=0045675) Train Loss: 5.8630, Train Steps/Sec: 0.80 +[2025-04-27 10:58:42] (step=0045700) Train Loss: 5.8784, Train Steps/Sec: 0.77 +[2025-04-27 10:59:04] (step=0045725) Train Loss: 5.8680, Train Steps/Sec: 1.12 +[2025-04-27 10:59:26] (step=0045750) Train Loss: 5.8417, Train Steps/Sec: 1.12 +[2025-04-27 10:59:49] (step=0045775) Train Loss: 5.8875, Train Steps/Sec: 1.12 +[2025-04-27 11:00:11] (step=0045800) Train Loss: 5.8468, Train Steps/Sec: 1.11 +[2025-04-27 11:00:33] (step=0045825) Train Loss: 5.7998, Train Steps/Sec: 1.12 +[2025-04-27 11:00:56] (step=0045850) Train Loss: 5.8189, Train Steps/Sec: 1.12 +[2025-04-27 11:01:18] (step=0045875) Train Loss: 5.8535, Train Steps/Sec: 1.12 +[2025-04-27 11:01:40] (step=0045900) Train Loss: 5.8831, Train Steps/Sec: 1.12 +[2025-04-27 11:02:03] (step=0045925) Train Loss: 5.8442, Train Steps/Sec: 1.12 +[2025-04-27 11:02:25] (step=0045950) Train Loss: 5.8563, Train Steps/Sec: 1.12 +[2025-04-27 11:02:47] (step=0045975) Train Loss: 5.8177, Train Steps/Sec: 1.12 +[2025-04-27 11:03:10] (step=0046000) Train Loss: 5.8056, Train Steps/Sec: 1.12 +[2025-04-27 11:03:10] text_config is None. Initializing the text config with default values (`OPTConfig`). +[2025-04-27 11:09:18] Finish Eval in 46000 steps... +[2025-04-27 11:09:37] Saved checkpoint to checkpoint/X2I_UltraEdiit_fourmask_LLava_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_filtered_objects_4mask_1280_context/003-GPT-XL/checkpoints/0046000.pt +[2025-04-27 11:09:39] Removed old checkpoint: checkpoint/X2I_UltraEdiit_fourmask_LLava_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_filtered_objects_4mask_1280_context/003-GPT-XL/checkpoints/0044000.pt +[2025-04-27 11:10:01] (step=0046025) Train Loss: 5.8416, Train Steps/Sec: 0.06 +[2025-04-27 11:10:24] (step=0046050) Train Loss: 5.8559, Train Steps/Sec: 1.12 +[2025-04-27 11:10:46] (step=0046075) Train Loss: 5.8362, Train Steps/Sec: 1.12 +[2025-04-27 11:11:08] (step=0046100) Train Loss: 5.8156, Train Steps/Sec: 1.12 +[2025-04-27 11:11:31] (step=0046125) Train Loss: 5.8111, Train Steps/Sec: 1.12 +[2025-04-27 11:11:53] (step=0046150) Train Loss: 5.8051, Train Steps/Sec: 1.12 +[2025-04-27 11:12:15] (step=0046175) Train Loss: 5.7943, Train Steps/Sec: 1.12 +[2025-04-27 11:12:38] (step=0046200) Train Loss: 5.8174, Train Steps/Sec: 1.11 +[2025-04-27 11:13:00] (step=0046225) Train Loss: 5.8287, Train Steps/Sec: 1.12 +[2025-04-27 11:13:22] (step=0046250) Train Loss: 5.8522, Train Steps/Sec: 1.12 +[2025-04-27 11:13:45] (step=0046275) Train Loss: 5.8166, Train Steps/Sec: 1.12 +[2025-04-27 11:14:07] (step=0046300) Train Loss: 5.8138, Train Steps/Sec: 1.12 +[2025-04-27 11:14:29] (step=0046325) Train Loss: 5.8671, Train Steps/Sec: 1.12 +[2025-04-27 11:14:52] (step=0046350) Train Loss: 5.8603, Train Steps/Sec: 1.12 +[2025-04-27 11:15:14] (step=0046375) Train Loss: 5.8490, Train Steps/Sec: 1.12 +[2025-04-27 11:15:36] (step=0046400) Train Loss: 5.8233, Train Steps/Sec: 1.11 +[2025-04-27 11:15:59] (step=0046425) Train Loss: 5.7743, Train Steps/Sec: 1.12 +[2025-04-27 11:16:21] (step=0046450) Train Loss: 5.8252, Train Steps/Sec: 1.12 +[2025-04-27 11:16:43] (step=0046475) Train Loss: 5.8781, Train Steps/Sec: 1.12 +[2025-04-27 11:17:06] (step=0046500) Train Loss: 5.8656, Train Steps/Sec: 1.12 +[2025-04-27 11:17:28] (step=0046525) Train Loss: 5.7939, Train Steps/Sec: 1.11 +[2025-04-27 11:17:51] (step=0046550) Train Loss: 5.8665, Train Steps/Sec: 1.12 +[2025-04-27 11:18:13] (step=0046575) Train Loss: 5.8348, Train Steps/Sec: 1.12 +[2025-04-27 11:18:35] (step=0046600) Train Loss: 5.8790, Train Steps/Sec: 1.12 +[2025-04-27 11:18:58] (step=0046625) Train Loss: 5.8414, Train Steps/Sec: 1.12 +[2025-04-27 11:19:20] (step=0046650) Train Loss: 5.8508, Train Steps/Sec: 1.12 +[2025-04-27 11:19:42] (step=0046675) Train Loss: 5.8673, Train Steps/Sec: 1.12 +[2025-04-27 11:20:05] (step=0046700) Train Loss: 5.8546, Train Steps/Sec: 1.12 +[2025-04-27 11:20:27] (step=0046725) Train Loss: 5.9066, Train Steps/Sec: 1.12 +[2025-04-27 11:20:49] (step=0046750) Train Loss: 5.8451, Train Steps/Sec: 1.12 +[2025-04-27 11:21:12] (step=0046775) Train Loss: 5.8236, Train Steps/Sec: 1.12 +[2025-04-27 11:21:34] (step=0046800) Train Loss: 5.7969, Train Steps/Sec: 1.12 +[2025-04-27 11:21:56] (step=0046825) Train Loss: 5.8352, Train Steps/Sec: 1.12 +[2025-04-27 11:22:19] (step=0046850) Train Loss: 5.8318, Train Steps/Sec: 1.12 +[2025-04-27 11:22:41] (step=0046875) Train Loss: 5.8679, Train Steps/Sec: 1.12 +[2025-04-27 11:23:03] (step=0046900) Train Loss: 5.8561, Train Steps/Sec: 1.12 +[2025-04-27 11:23:26] (step=0046925) Train Loss: 5.8392, Train Steps/Sec: 1.12 +[2025-04-27 11:23:48] (step=0046950) Train Loss: 5.8197, Train Steps/Sec: 1.12 +[2025-04-27 11:24:10] (step=0046975) Train Loss: 5.8643, Train Steps/Sec: 1.12 +[2025-04-27 11:24:33] (step=0047000) Train Loss: 5.8273, Train Steps/Sec: 1.12 +[2025-04-27 11:24:55] (step=0047025) Train Loss: 5.8548, Train Steps/Sec: 1.12 +[2025-04-27 11:25:17] (step=0047050) Train Loss: 5.8400, Train Steps/Sec: 1.12 +[2025-04-27 11:25:40] (step=0047075) Train Loss: 5.8553, Train Steps/Sec: 1.12 +[2025-04-27 11:26:02] (step=0047100) Train Loss: 5.9012, Train Steps/Sec: 1.12 +[2025-04-27 11:26:24] (step=0047125) Train Loss: 5.7677, Train Steps/Sec: 1.12 +[2025-04-27 11:26:47] (step=0047150) Train Loss: 5.7956, Train Steps/Sec: 1.12 +[2025-04-27 11:27:09] (step=0047175) Train Loss: 5.8051, Train Steps/Sec: 1.12 +[2025-04-27 11:27:32] (step=0047200) Train Loss: 5.8697, Train Steps/Sec: 1.11 +[2025-04-27 11:27:54] (step=0047225) Train Loss: 5.7726, Train Steps/Sec: 1.12 +[2025-04-27 11:28:16] (step=0047250) Train Loss: 5.8691, Train Steps/Sec: 1.12 +[2025-04-27 11:28:39] (step=0047275) Train Loss: 5.7965, Train Steps/Sec: 1.12 +[2025-04-27 11:29:01] (step=0047300) Train Loss: 5.8873, Train Steps/Sec: 1.12 +[2025-04-27 11:29:23] (step=0047325) Train Loss: 5.8407, Train Steps/Sec: 1.12 +[2025-04-27 11:29:46] (step=0047350) Train Loss: 5.8695, Train Steps/Sec: 1.12 +[2025-04-27 11:30:08] (step=0047375) Train Loss: 5.7754, Train Steps/Sec: 1.12 +[2025-04-27 11:30:30] (step=0047400) Train Loss: 5.8162, Train Steps/Sec: 1.11 +[2025-04-27 11:30:53] (step=0047425) Train Loss: 5.8494, Train Steps/Sec: 1.12 +[2025-04-27 11:31:15] (step=0047450) Train Loss: 5.7741, Train Steps/Sec: 1.12 +[2025-04-27 11:31:38] (step=0047475) Train Loss: 5.8112, Train Steps/Sec: 1.12 +[2025-04-27 11:32:00] (step=0047500) Train Loss: 5.7527, Train Steps/Sec: 1.12 +[2025-04-27 11:32:22] (step=0047525) Train Loss: 5.8203, Train Steps/Sec: 1.12 +[2025-04-27 11:32:45] (step=0047550) Train Loss: 5.7804, Train Steps/Sec: 1.12 +[2025-04-27 11:33:07] (step=0047575) Train Loss: 5.8647, Train Steps/Sec: 1.12 +[2025-04-27 11:33:29] (step=0047600) Train Loss: 5.7774, Train Steps/Sec: 1.11 +[2025-04-27 11:33:52] (step=0047625) Train Loss: 5.8650, Train Steps/Sec: 1.12 +[2025-04-27 11:34:14] (step=0047650) Train Loss: 5.9065, Train Steps/Sec: 1.12 +[2025-04-27 11:34:36] (step=0047675) Train Loss: 5.7967, Train Steps/Sec: 1.12 +[2025-04-27 11:34:59] (step=0047700) Train Loss: 5.8055, Train Steps/Sec: 1.12 +[2025-04-27 11:35:21] (step=0047725) Train Loss: 5.7982, Train Steps/Sec: 1.12 +[2025-04-27 11:35:43] (step=0047750) Train Loss: 5.8353, Train Steps/Sec: 1.12 +[2025-04-27 11:36:06] (step=0047775) Train Loss: 5.7870, Train Steps/Sec: 1.12 +[2025-04-27 11:36:28] (step=0047800) Train Loss: 5.8349, Train Steps/Sec: 1.12 +[2025-04-27 11:36:50] (step=0047825) Train Loss: 5.8156, Train Steps/Sec: 1.12 +[2025-04-27 11:37:13] (step=0047850) Train Loss: 5.8131, Train Steps/Sec: 1.12 +[2025-04-27 11:37:35] (step=0047875) Train Loss: 5.8180, Train Steps/Sec: 1.12 +[2025-04-27 11:37:57] (step=0047900) Train Loss: 5.7896, Train Steps/Sec: 1.12 +[2025-04-27 11:38:20] (step=0047925) Train Loss: 5.7484, Train Steps/Sec: 1.12 +[2025-04-27 11:38:42] (step=0047950) Train Loss: 5.8704, Train Steps/Sec: 1.12 +[2025-04-27 11:39:04] (step=0047975) Train Loss: 5.8445, Train Steps/Sec: 1.12 +[2025-04-27 11:39:27] (step=0048000) Train Loss: 5.7777, Train Steps/Sec: 1.12 +[2025-04-27 11:39:27] text_config is None. Initializing the text config with default values (`OPTConfig`). +[2025-04-27 11:45:36] Finish Eval in 48000 steps... +[2025-04-27 11:45:55] Saved checkpoint to checkpoint/X2I_UltraEdiit_fourmask_LLava_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_filtered_objects_4mask_1280_context/003-GPT-XL/checkpoints/0048000.pt +[2025-04-27 11:45:57] Removed old checkpoint: checkpoint/X2I_UltraEdiit_fourmask_LLava_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_filtered_objects_4mask_1280_context/003-GPT-XL/checkpoints/0046000.pt +[2025-04-27 11:46:20] (step=0048025) Train Loss: 5.8620, Train Steps/Sec: 0.06 +[2025-04-27 11:46:42] (step=0048050) Train Loss: 5.8776, Train Steps/Sec: 1.12 +[2025-04-27 11:47:04] (step=0048075) Train Loss: 5.8011, Train Steps/Sec: 1.12 +[2025-04-27 11:47:27] (step=0048100) Train Loss: 5.8558, Train Steps/Sec: 1.12 +[2025-04-27 11:47:49] (step=0048125) Train Loss: 5.9142, Train Steps/Sec: 1.12 +[2025-04-27 11:48:11] (step=0048150) Train Loss: 5.8290, Train Steps/Sec: 1.12 +[2025-04-27 11:48:34] (step=0048175) Train Loss: 5.8427, Train Steps/Sec: 1.12 +[2025-04-27 11:48:56] (step=0048200) Train Loss: 5.8734, Train Steps/Sec: 1.12 +[2025-04-27 11:49:18] (step=0048225) Train Loss: 5.8898, Train Steps/Sec: 1.12 +[2025-04-27 11:49:41] (step=0048250) Train Loss: 5.7715, Train Steps/Sec: 1.12 +[2025-04-27 11:50:03] (step=0048275) Train Loss: 5.8195, Train Steps/Sec: 1.12 +[2025-04-27 11:50:25] (step=0048300) Train Loss: 5.8449, Train Steps/Sec: 1.12 +[2025-04-27 11:50:47] (step=0048325) Train Loss: 5.8006, Train Steps/Sec: 1.12 +[2025-04-27 11:51:10] (step=0048350) Train Loss: 5.8292, Train Steps/Sec: 1.12 +[2025-04-27 11:51:32] (step=0048375) Train Loss: 5.8179, Train Steps/Sec: 1.12 +[2025-04-27 11:51:54] (step=0048400) Train Loss: 5.7734, Train Steps/Sec: 1.12 +[2025-04-27 11:52:17] (step=0048425) Train Loss: 5.8546, Train Steps/Sec: 1.10 +[2025-04-27 11:52:39] (step=0048450) Train Loss: 5.8485, Train Steps/Sec: 1.12 +[2025-04-27 11:53:02] (step=0048475) Train Loss: 5.8306, Train Steps/Sec: 1.12 +[2025-04-27 11:53:24] (step=0048500) Train Loss: 5.7978, Train Steps/Sec: 1.12 +[2025-04-27 11:53:46] (step=0048525) Train Loss: 5.8071, Train Steps/Sec: 1.12 +[2025-04-27 11:54:09] (step=0048550) Train Loss: 5.8294, Train Steps/Sec: 1.12 +[2025-04-27 11:54:31] (step=0048575) Train Loss: 5.8188, Train Steps/Sec: 1.12 +[2025-04-27 11:54:53] (step=0048600) Train Loss: 5.8220, Train Steps/Sec: 1.12 +[2025-04-27 11:55:16] (step=0048625) Train Loss: 5.8622, Train Steps/Sec: 1.12 +[2025-04-27 11:55:38] (step=0048650) Train Loss: 5.8044, Train Steps/Sec: 1.12 +[2025-04-27 11:56:00] (step=0048675) Train Loss: 5.8428, Train Steps/Sec: 1.12 +[2025-04-27 11:56:22] (step=0048700) Train Loss: 5.8284, Train Steps/Sec: 1.12 +[2025-04-27 11:56:45] (step=0048725) Train Loss: 5.8628, Train Steps/Sec: 1.12 +[2025-04-27 11:57:07] (step=0048750) Train Loss: 5.7929, Train Steps/Sec: 1.12 +[2025-04-27 11:57:29] (step=0048775) Train Loss: 5.8308, Train Steps/Sec: 1.12 +[2025-04-27 11:57:52] (step=0048800) Train Loss: 5.7855, Train Steps/Sec: 1.12 +[2025-04-27 11:58:14] (step=0048825) Train Loss: 5.7973, Train Steps/Sec: 1.12 +[2025-04-27 11:58:36] (step=0048850) Train Loss: 5.7700, Train Steps/Sec: 1.12 +[2025-04-27 11:58:59] (step=0048875) Train Loss: 5.8744, Train Steps/Sec: 1.12 +[2025-04-27 11:59:21] (step=0048900) Train Loss: 5.8351, Train Steps/Sec: 1.12 +[2025-04-27 11:59:43] (step=0048925) Train Loss: 5.7933, Train Steps/Sec: 1.12 +[2025-04-27 12:00:05] (step=0048950) Train Loss: 5.7890, Train Steps/Sec: 1.12 +[2025-04-27 12:00:28] (step=0048975) Train Loss: 5.8110, Train Steps/Sec: 1.12 +[2025-04-27 12:00:50] (step=0049000) Train Loss: 5.7610, Train Steps/Sec: 1.12 +[2025-04-27 12:01:13] (step=0049025) Train Loss: 5.8518, Train Steps/Sec: 1.12 +[2025-04-27 12:01:35] (step=0049050) Train Loss: 5.8827, Train Steps/Sec: 1.12 +[2025-04-27 12:01:57] (step=0049075) Train Loss: 5.8588, Train Steps/Sec: 1.12 +[2025-04-27 12:02:19] (step=0049100) Train Loss: 5.8275, Train Steps/Sec: 1.12 +[2025-04-27 12:02:42] (step=0049125) Train Loss: 5.8426, Train Steps/Sec: 1.12 +[2025-04-27 12:03:04] (step=0049150) Train Loss: 5.8909, Train Steps/Sec: 1.12 +[2025-04-27 12:03:26] (step=0049175) Train Loss: 5.8546, Train Steps/Sec: 1.12 +[2025-04-27 12:03:49] (step=0049200) Train Loss: 5.8607, Train Steps/Sec: 1.12 +[2025-04-27 12:04:11] (step=0049225) Train Loss: 5.8349, Train Steps/Sec: 1.12 +[2025-04-27 12:04:33] (step=0049250) Train Loss: 5.8225, Train Steps/Sec: 1.12 +[2025-04-27 12:04:56] (step=0049275) Train Loss: 5.8545, Train Steps/Sec: 1.12 +[2025-04-27 12:05:18] (step=0049300) Train Loss: 5.7511, Train Steps/Sec: 1.12 +[2025-04-27 12:05:40] (step=0049325) Train Loss: 5.8370, Train Steps/Sec: 1.12 +[2025-04-27 12:06:03] (step=0049350) Train Loss: 5.8323, Train Steps/Sec: 1.12 +[2025-04-27 12:06:25] (step=0049375) Train Loss: 5.8022, Train Steps/Sec: 1.12 +[2025-04-27 12:06:47] (step=0049400) Train Loss: 5.8785, Train Steps/Sec: 1.12 +[2025-04-27 12:07:10] (step=0049425) Train Loss: 5.7835, Train Steps/Sec: 1.12 +[2025-04-27 12:07:32] (step=0049450) Train Loss: 5.8241, Train Steps/Sec: 1.12 +[2025-04-27 12:07:54] (step=0049475) Train Loss: 5.8468, Train Steps/Sec: 1.12 +[2025-04-27 12:08:17] (step=0049500) Train Loss: 5.8565, Train Steps/Sec: 1.12 +[2025-04-27 12:08:39] (step=0049525) Train Loss: 5.7854, Train Steps/Sec: 1.12 +[2025-04-27 12:09:06] (step=0049550) Train Loss: 5.8192, Train Steps/Sec: 0.93 +[2025-04-27 12:09:28] (step=0049575) Train Loss: 5.8180, Train Steps/Sec: 1.12 +[2025-04-27 12:09:50] (step=0049600) Train Loss: 5.8607, Train Steps/Sec: 1.12 +[2025-04-27 12:10:13] (step=0049625) Train Loss: 5.8605, Train Steps/Sec: 1.12 +[2025-04-27 12:10:40] (step=0049650) Train Loss: 5.8208, Train Steps/Sec: 0.93 +[2025-04-27 12:11:02] (step=0049675) Train Loss: 5.8851, Train Steps/Sec: 1.12 +[2025-04-27 12:11:24] (step=0049700) Train Loss: 5.8484, Train Steps/Sec: 1.12 +[2025-04-27 12:11:47] (step=0049725) Train Loss: 5.7450, Train Steps/Sec: 1.12 +[2025-04-27 12:12:09] (step=0049750) Train Loss: 5.8500, Train Steps/Sec: 1.12 +[2025-04-27 12:12:31] (step=0049775) Train Loss: 5.8935, Train Steps/Sec: 1.12 +[2025-04-27 12:12:54] (step=0049800) Train Loss: 5.8149, Train Steps/Sec: 1.12 +[2025-04-27 12:13:21] (step=0049825) Train Loss: 5.8320, Train Steps/Sec: 0.91 +[2025-04-27 12:13:43] (step=0049850) Train Loss: 5.8376, Train Steps/Sec: 1.12 +[2025-04-27 12:14:06] (step=0049875) Train Loss: 5.8137, Train Steps/Sec: 1.12 +[2025-04-27 12:14:28] (step=0049900) Train Loss: 5.7979, Train Steps/Sec: 1.12 +[2025-04-27 12:14:50] (step=0049925) Train Loss: 5.8670, Train Steps/Sec: 1.12 +[2025-04-27 12:15:13] (step=0049950) Train Loss: 5.7946, Train Steps/Sec: 1.12 +[2025-04-27 12:15:35] (step=0049975) Train Loss: 5.8140, Train Steps/Sec: 1.12 +[2025-04-27 12:16:06] (step=0050000) Train Loss: 5.8324, Train Steps/Sec: 0.81 +[2025-04-27 12:16:06] text_config is None. Initializing the text config with default values (`OPTConfig`). +[2025-04-27 12:22:15] Finish Eval in 50000 steps... +[2025-04-27 12:22:34] Saved checkpoint to checkpoint/X2I_UltraEdiit_fourmask_LLava_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_filtered_objects_4mask_1280_context/003-GPT-XL/checkpoints/0050000.pt +[2025-04-27 12:22:36] Removed old checkpoint: checkpoint/X2I_UltraEdiit_fourmask_LLava_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_filtered_objects_4mask_1280_context/003-GPT-XL/checkpoints/0048000.pt +[2025-04-27 12:23:08] (step=0050025) Train Loss: 5.8653, Train Steps/Sec: 0.06 +[2025-04-27 12:23:31] (step=0050050) Train Loss: 5.7756, Train Steps/Sec: 1.12 +[2025-04-27 12:23:53] (step=0050075) Train Loss: 5.8337, Train Steps/Sec: 1.12 +[2025-04-27 12:24:15] (step=0050100) Train Loss: 5.7787, Train Steps/Sec: 1.12 +[2025-04-27 12:24:38] (step=0050125) Train Loss: 5.9065, Train Steps/Sec: 1.12 +[2025-04-27 12:25:00] (step=0050150) Train Loss: 5.7701, Train Steps/Sec: 1.12 +[2025-04-27 12:25:22] (step=0050175) Train Loss: 5.7830, Train Steps/Sec: 1.12 +[2025-04-27 12:25:45] (step=0050200) Train Loss: 5.7792, Train Steps/Sec: 1.11 +[2025-04-27 12:26:07] (step=0050225) Train Loss: 5.8056, Train Steps/Sec: 1.12 +[2025-04-27 12:26:29] (step=0050250) Train Loss: 5.8495, Train Steps/Sec: 1.12 +[2025-04-27 12:26:52] (step=0050275) Train Loss: 5.7627, Train Steps/Sec: 1.12 +[2025-04-27 12:27:14] (step=0050300) Train Loss: 5.8159, Train Steps/Sec: 1.12 +[2025-04-27 12:27:37] (step=0050325) Train Loss: 5.8549, Train Steps/Sec: 1.11 +[2025-04-27 12:27:59] (step=0050350) Train Loss: 5.8331, Train Steps/Sec: 1.12 +[2025-04-27 12:28:21] (step=0050375) Train Loss: 5.8433, Train Steps/Sec: 1.12 +[2025-04-27 12:28:44] (step=0050400) Train Loss: 5.8115, Train Steps/Sec: 1.12 +[2025-04-27 12:29:06] (step=0050425) Train Loss: 5.7438, Train Steps/Sec: 1.12 +[2025-04-27 12:29:28] (step=0050450) Train Loss: 5.8081, Train Steps/Sec: 1.12 +[2025-04-27 12:29:50] (step=0050475) Train Loss: 5.8680, Train Steps/Sec: 1.12 +[2025-04-27 12:30:13] (step=0050500) Train Loss: 5.8609, Train Steps/Sec: 1.12 +[2025-04-27 12:30:35] (step=0050525) Train Loss: 5.8412, Train Steps/Sec: 1.12 +[2025-04-27 12:30:57] (step=0050550) Train Loss: 5.8251, Train Steps/Sec: 1.12 +[2025-04-27 12:31:20] (step=0050575) Train Loss: 5.7646, Train Steps/Sec: 1.12 +[2025-04-27 12:31:42] (step=0050600) Train Loss: 5.8506, Train Steps/Sec: 1.11 +[2025-04-27 12:32:05] (step=0050625) Train Loss: 5.8510, Train Steps/Sec: 1.12 +[2025-04-27 12:32:27] (step=0050650) Train Loss: 5.8196, Train Steps/Sec: 1.12 +[2025-04-27 12:32:49] (step=0050675) Train Loss: 5.7585, Train Steps/Sec: 1.12 +[2025-04-27 12:33:12] (step=0050700) Train Loss: 5.8042, Train Steps/Sec: 1.12 +[2025-04-27 12:33:34] (step=0050725) Train Loss: 5.8326, Train Steps/Sec: 1.12 +[2025-04-27 12:33:56] (step=0050750) Train Loss: 5.8422, Train Steps/Sec: 1.12 +[2025-04-27 12:34:18] (step=0050775) Train Loss: 5.7564, Train Steps/Sec: 1.12 +[2025-04-27 12:34:41] (step=0050800) Train Loss: 5.8148, Train Steps/Sec: 1.12 +[2025-04-27 12:35:03] (step=0050825) Train Loss: 5.7933, Train Steps/Sec: 1.12 +[2025-04-27 12:35:25] (step=0050850) Train Loss: 5.8680, Train Steps/Sec: 1.12 +[2025-04-27 12:35:48] (step=0050875) Train Loss: 5.7729, Train Steps/Sec: 1.12 +[2025-04-27 12:36:10] (step=0050900) Train Loss: 5.8294, Train Steps/Sec: 1.12 +[2025-04-27 12:36:32] (step=0050925) Train Loss: 5.8134, Train Steps/Sec: 1.12 +[2025-04-27 12:36:55] (step=0050950) Train Loss: 5.8275, Train Steps/Sec: 1.12 +[2025-04-27 12:37:17] (step=0050975) Train Loss: 5.8262, Train Steps/Sec: 1.12 +[2025-04-27 12:37:39] (step=0051000) Train Loss: 5.8605, Train Steps/Sec: 1.11 +[2025-04-27 12:38:02] (step=0051025) Train Loss: 5.7406, Train Steps/Sec: 1.12 +[2025-04-27 12:38:24] (step=0051050) Train Loss: 5.8594, Train Steps/Sec: 1.12 +[2025-04-27 12:38:46] (step=0051075) Train Loss: 5.8230, Train Steps/Sec: 1.12 +[2025-04-27 12:39:09] (step=0051100) Train Loss: 5.8033, Train Steps/Sec: 1.12 +[2025-04-27 12:39:31] (step=0051125) Train Loss: 5.7866, Train Steps/Sec: 1.12 +[2025-04-27 12:39:53] (step=0051150) Train Loss: 5.7969, Train Steps/Sec: 1.12 +[2025-04-27 12:40:16] (step=0051175) Train Loss: 5.8134, Train Steps/Sec: 1.12 +[2025-04-27 12:40:38] (step=0051200) Train Loss: 5.7648, Train Steps/Sec: 1.11 +[2025-04-27 12:41:00] (step=0051225) Train Loss: 5.7778, Train Steps/Sec: 1.12 +[2025-04-27 12:41:23] (step=0051250) Train Loss: 5.8544, Train Steps/Sec: 1.12 +[2025-04-27 12:41:45] (step=0051275) Train Loss: 5.7872, Train Steps/Sec: 1.12 +[2025-04-27 12:42:07] (step=0051300) Train Loss: 5.8653, Train Steps/Sec: 1.12 +[2025-04-27 12:42:30] (step=0051325) Train Loss: 5.7729, Train Steps/Sec: 1.12 +[2025-04-27 12:42:52] (step=0051350) Train Loss: 5.7932, Train Steps/Sec: 1.12 +[2025-04-27 12:43:14] (step=0051375) Train Loss: 5.7564, Train Steps/Sec: 1.12 +[2025-04-27 12:43:37] (step=0051400) Train Loss: 5.8480, Train Steps/Sec: 1.11 +[2025-04-27 12:43:59] (step=0051425) Train Loss: 5.7932, Train Steps/Sec: 1.12 +[2025-04-27 12:44:21] (step=0051450) Train Loss: 5.8157, Train Steps/Sec: 1.12 +[2025-04-27 12:44:44] (step=0051475) Train Loss: 5.8744, Train Steps/Sec: 1.12 +[2025-04-27 12:45:06] (step=0051500) Train Loss: 5.8522, Train Steps/Sec: 1.12 +[2025-04-27 12:45:28] (step=0051525) Train Loss: 5.7804, Train Steps/Sec: 1.12 +[2025-04-27 12:45:51] (step=0051550) Train Loss: 5.8312, Train Steps/Sec: 1.12 +[2025-04-27 12:46:13] (step=0051575) Train Loss: 5.7996, Train Steps/Sec: 1.12 +[2025-04-27 12:46:35] (step=0051600) Train Loss: 5.8068, Train Steps/Sec: 1.12 +[2025-04-27 12:46:58] (step=0051625) Train Loss: 5.7449, Train Steps/Sec: 1.12 +[2025-04-27 12:47:20] (step=0051650) Train Loss: 5.8036, Train Steps/Sec: 1.12 +[2025-04-27 12:47:42] (step=0051675) Train Loss: 5.8697, Train Steps/Sec: 1.12 +[2025-04-27 12:48:05] (step=0051700) Train Loss: 5.7883, Train Steps/Sec: 1.12 +[2025-04-27 12:48:27] (step=0051725) Train Loss: 5.7937, Train Steps/Sec: 1.12 +[2025-04-27 12:48:49] (step=0051750) Train Loss: 5.8208, Train Steps/Sec: 1.12 +[2025-04-27 12:49:12] (step=0051775) Train Loss: 5.7605, Train Steps/Sec: 1.12 +[2025-04-27 12:49:34] (step=0051800) Train Loss: 5.7830, Train Steps/Sec: 1.12 +[2025-04-27 12:49:56] (step=0051825) Train Loss: 5.7771, Train Steps/Sec: 1.12 +[2025-04-27 12:50:19] (step=0051850) Train Loss: 5.7913, Train Steps/Sec: 1.12 +[2025-04-27 12:50:41] (step=0051875) Train Loss: 5.8828, Train Steps/Sec: 1.12 +[2025-04-27 12:51:03] (step=0051900) Train Loss: 5.7846, Train Steps/Sec: 1.12 +[2025-04-27 12:51:26] (step=0051925) Train Loss: 5.8163, Train Steps/Sec: 1.12 +[2025-04-27 12:51:48] (step=0051950) Train Loss: 5.8479, Train Steps/Sec: 1.12 +[2025-04-27 12:52:10] (step=0051975) Train Loss: 5.8151, Train Steps/Sec: 1.12 +[2025-04-27 12:52:33] (step=0052000) Train Loss: 5.8805, Train Steps/Sec: 1.12 +[2025-04-27 12:52:33] text_config is None. Initializing the text config with default values (`OPTConfig`). +[2025-04-27 12:58:42] Finish Eval in 52000 steps... +[2025-04-27 12:59:02] Saved checkpoint to checkpoint/X2I_UltraEdiit_fourmask_LLava_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_filtered_objects_4mask_1280_context/003-GPT-XL/checkpoints/0052000.pt +[2025-04-27 12:59:04] Removed old checkpoint: checkpoint/X2I_UltraEdiit_fourmask_LLava_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_filtered_objects_4mask_1280_context/003-GPT-XL/checkpoints/0050000.pt +[2025-04-27 12:59:26] (step=0052025) Train Loss: 5.7631, Train Steps/Sec: 0.06 +[2025-04-27 12:59:48] (step=0052050) Train Loss: 5.8168, Train Steps/Sec: 1.12 +[2025-04-27 13:00:11] (step=0052075) Train Loss: 5.8613, Train Steps/Sec: 1.12 +[2025-04-27 13:00:33] (step=0052100) Train Loss: 5.8383, Train Steps/Sec: 1.12 +[2025-04-27 13:00:55] (step=0052125) Train Loss: 5.8050, Train Steps/Sec: 1.12 +[2025-04-27 13:01:17] (step=0052150) Train Loss: 5.8656, Train Steps/Sec: 1.12 +[2025-04-27 13:01:40] (step=0052175) Train Loss: 5.7717, Train Steps/Sec: 1.13 +[2025-04-27 13:02:02] (step=0052200) Train Loss: 5.8569, Train Steps/Sec: 1.12 +[2025-04-27 13:02:25] (step=0052225) Train Loss: 5.8101, Train Steps/Sec: 1.11 +[2025-04-27 13:02:47] (step=0052250) Train Loss: 5.8467, Train Steps/Sec: 1.12 +[2025-04-27 13:03:09] (step=0052275) Train Loss: 5.8442, Train Steps/Sec: 1.12 +[2025-04-27 13:03:31] (step=0052300) Train Loss: 5.7810, Train Steps/Sec: 1.12 +[2025-04-27 13:03:54] (step=0052325) Train Loss: 5.8323, Train Steps/Sec: 1.12 +[2025-04-27 13:04:16] (step=0052350) Train Loss: 5.7931, Train Steps/Sec: 1.12 +[2025-04-27 13:04:38] (step=0052375) Train Loss: 5.7956, Train Steps/Sec: 1.12 +[2025-04-27 13:05:01] (step=0052400) Train Loss: 5.8602, Train Steps/Sec: 1.12 +[2025-04-27 13:05:23] (step=0052425) Train Loss: 5.8046, Train Steps/Sec: 1.12 +[2025-04-27 13:05:45] (step=0052450) Train Loss: 5.8219, Train Steps/Sec: 1.12 +[2025-04-27 13:06:07] (step=0052475) Train Loss: 5.8417, Train Steps/Sec: 1.12 +[2025-04-27 13:06:30] (step=0052500) Train Loss: 5.7979, Train Steps/Sec: 1.12 +[2025-04-27 13:06:52] (step=0052525) Train Loss: 5.7926, Train Steps/Sec: 1.12 +[2025-04-27 13:07:14] (step=0052550) Train Loss: 5.7560, Train Steps/Sec: 1.12 +[2025-04-27 13:07:36] (step=0052575) Train Loss: 5.7791, Train Steps/Sec: 1.12 +[2025-04-27 13:07:59] (step=0052600) Train Loss: 5.8128, Train Steps/Sec: 1.12 +[2025-04-27 13:08:21] (step=0052625) Train Loss: 5.7915, Train Steps/Sec: 1.12 +[2025-04-27 13:08:43] (step=0052650) Train Loss: 5.8365, Train Steps/Sec: 1.12 +[2025-04-27 13:09:06] (step=0052675) Train Loss: 5.7933, Train Steps/Sec: 1.12 +[2025-04-27 13:09:28] (step=0052700) Train Loss: 5.7884, Train Steps/Sec: 1.12 +[2025-04-27 13:09:50] (step=0052725) Train Loss: 5.7967, Train Steps/Sec: 1.12 +[2025-04-27 13:10:12] (step=0052750) Train Loss: 5.8262, Train Steps/Sec: 1.12 +[2025-04-27 13:10:35] (step=0052775) Train Loss: 5.8482, Train Steps/Sec: 1.12 +[2025-04-27 13:10:57] (step=0052800) Train Loss: 5.8174, Train Steps/Sec: 1.12 +[2025-04-27 13:11:19] (step=0052825) Train Loss: 5.7195, Train Steps/Sec: 1.12 +[2025-04-27 13:11:42] (step=0052850) Train Loss: 5.8238, Train Steps/Sec: 1.12 +[2025-04-27 13:12:04] (step=0052875) Train Loss: 5.8091, Train Steps/Sec: 1.12 +[2025-04-27 13:12:26] (step=0052900) Train Loss: 5.8630, Train Steps/Sec: 1.12 +[2025-04-27 13:12:49] (step=0052925) Train Loss: 5.8625, Train Steps/Sec: 1.12 +[2025-04-27 13:13:11] (step=0052950) Train Loss: 5.8413, Train Steps/Sec: 1.12 +[2025-04-27 13:13:33] (step=0052975) Train Loss: 5.8414, Train Steps/Sec: 1.12 +[2025-04-27 13:13:56] (step=0053000) Train Loss: 5.8397, Train Steps/Sec: 1.12 +[2025-04-27 13:14:18] (step=0053025) Train Loss: 5.7212, Train Steps/Sec: 1.12 +[2025-04-27 13:14:40] (step=0053050) Train Loss: 5.7422, Train Steps/Sec: 1.12 +[2025-04-27 13:15:02] (step=0053075) Train Loss: 5.8462, Train Steps/Sec: 1.12 +[2025-04-27 13:15:25] (step=0053100) Train Loss: 5.8384, Train Steps/Sec: 1.13 +[2025-04-27 13:15:47] (step=0053125) Train Loss: 5.8886, Train Steps/Sec: 1.12 +[2025-04-27 13:16:09] (step=0053150) Train Loss: 5.7617, Train Steps/Sec: 1.12 +[2025-04-27 13:16:31] (step=0053175) Train Loss: 5.7533, Train Steps/Sec: 1.12 +[2025-04-27 13:16:54] (step=0053200) Train Loss: 5.7798, Train Steps/Sec: 1.12 +[2025-04-27 13:17:16] (step=0053225) Train Loss: 5.7703, Train Steps/Sec: 1.12 +[2025-04-27 13:17:38] (step=0053250) Train Loss: 5.7948, Train Steps/Sec: 1.12 +[2025-04-27 13:18:01] (step=0053275) Train Loss: 5.8027, Train Steps/Sec: 1.12 +[2025-04-27 13:18:23] (step=0053300) Train Loss: 5.8050, Train Steps/Sec: 1.12 +[2025-04-27 13:18:45] (step=0053325) Train Loss: 5.7738, Train Steps/Sec: 1.12 +[2025-04-27 13:19:07] (step=0053350) Train Loss: 5.8381, Train Steps/Sec: 1.12 +[2025-04-27 13:19:30] (step=0053375) Train Loss: 5.7908, Train Steps/Sec: 1.12 +[2025-04-27 13:19:52] (step=0053400) Train Loss: 5.7841, Train Steps/Sec: 1.12 +[2025-04-27 13:20:14] (step=0053425) Train Loss: 5.8420, Train Steps/Sec: 1.12 +[2025-04-27 13:20:37] (step=0053450) Train Loss: 5.8162, Train Steps/Sec: 1.12 +[2025-04-27 13:20:59] (step=0053475) Train Loss: 5.7983, Train Steps/Sec: 1.12 +[2025-04-27 13:21:21] (step=0053500) Train Loss: 5.8713, Train Steps/Sec: 1.12 +[2025-04-27 13:21:43] (step=0053525) Train Loss: 5.7537, Train Steps/Sec: 1.12 +[2025-04-27 13:22:06] (step=0053550) Train Loss: 5.7240, Train Steps/Sec: 1.12 +[2025-04-27 13:22:28] (step=0053575) Train Loss: 5.8256, Train Steps/Sec: 1.12 +[2025-04-27 13:22:50] (step=0053600) Train Loss: 5.8174, Train Steps/Sec: 1.12 +[2025-04-27 13:23:13] (step=0053625) Train Loss: 5.8753, Train Steps/Sec: 1.12 +[2025-04-27 13:23:35] (step=0053650) Train Loss: 5.7424, Train Steps/Sec: 1.12 +[2025-04-27 13:23:57] (step=0053675) Train Loss: 5.8367, Train Steps/Sec: 1.12 +[2025-04-27 13:24:19] (step=0053700) Train Loss: 5.7372, Train Steps/Sec: 1.13 +[2025-04-27 13:24:42] (step=0053725) Train Loss: 5.7795, Train Steps/Sec: 1.12 +[2025-04-27 13:25:04] (step=0053750) Train Loss: 5.8005, Train Steps/Sec: 1.12 +[2025-04-27 13:25:26] (step=0053775) Train Loss: 5.7903, Train Steps/Sec: 1.12 +[2025-04-27 13:25:53] (step=0053800) Train Loss: 5.7296, Train Steps/Sec: 0.93 +[2025-04-27 13:26:15] (step=0053825) Train Loss: 5.8310, Train Steps/Sec: 1.12 +[2025-04-27 13:26:37] (step=0053850) Train Loss: 5.8460, Train Steps/Sec: 1.13 +[2025-04-27 13:27:00] (step=0053875) Train Loss: 5.8339, Train Steps/Sec: 1.12 +[2025-04-27 13:27:22] (step=0053900) Train Loss: 5.7667, Train Steps/Sec: 1.13 +[2025-04-27 13:27:49] (step=0053925) Train Loss: 5.8106, Train Steps/Sec: 0.93 +[2025-04-27 13:28:11] (step=0053950) Train Loss: 5.7747, Train Steps/Sec: 1.12 +[2025-04-27 13:28:33] (step=0053975) Train Loss: 5.7797, Train Steps/Sec: 1.12 +[2025-04-27 13:28:56] (step=0054000) Train Loss: 5.9028, Train Steps/Sec: 1.12 +[2025-04-27 13:28:56] text_config is None. Initializing the text config with default values (`OPTConfig`). +[2025-04-27 13:35:05] Finish Eval in 54000 steps... +[2025-04-27 13:35:24] Saved checkpoint to checkpoint/X2I_UltraEdiit_fourmask_LLava_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_filtered_objects_4mask_1280_context/003-GPT-XL/checkpoints/0054000.pt +[2025-04-27 13:35:25] Removed old checkpoint: checkpoint/X2I_UltraEdiit_fourmask_LLava_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_filtered_objects_4mask_1280_context/003-GPT-XL/checkpoints/0052000.pt +[2025-04-27 13:35:48] (step=0054025) Train Loss: 5.7993, Train Steps/Sec: 0.06 +[2025-04-27 13:36:10] (step=0054050) Train Loss: 5.8045, Train Steps/Sec: 1.13 +[2025-04-27 13:36:32] (step=0054075) Train Loss: 5.7795, Train Steps/Sec: 1.13 +[2025-04-27 13:36:55] (step=0054100) Train Loss: 5.7540, Train Steps/Sec: 1.12 +[2025-04-27 13:37:17] (step=0054125) Train Loss: 5.7212, Train Steps/Sec: 1.12 +[2025-04-27 13:37:40] (step=0054150) Train Loss: 5.7862, Train Steps/Sec: 1.12 +[2025-04-27 13:38:02] (step=0054175) Train Loss: 5.7633, Train Steps/Sec: 1.12 +[2025-04-27 13:38:24] (step=0054200) Train Loss: 5.7641, Train Steps/Sec: 1.12 +[2025-04-27 13:38:52] (step=0054225) Train Loss: 5.8516, Train Steps/Sec: 0.91 +[2025-04-27 13:39:14] (step=0054250) Train Loss: 5.7948, Train Steps/Sec: 1.12 +[2025-04-27 13:39:36] (step=0054275) Train Loss: 5.8668, Train Steps/Sec: 1.12 +[2025-04-27 13:40:02] (step=0054300) Train Loss: 5.7804, Train Steps/Sec: 0.96 +[2025-04-27 13:40:29] (step=0054325) Train Loss: 5.8224, Train Steps/Sec: 0.93 +[2025-04-27 13:40:56] (step=0054350) Train Loss: 5.8056, Train Steps/Sec: 0.93 +[2025-04-27 13:41:24] (step=0054375) Train Loss: 5.8844, Train Steps/Sec: 0.91 +[2025-04-27 13:41:46] (step=0054400) Train Loss: 5.8464, Train Steps/Sec: 1.12 +[2025-04-27 13:42:09] (step=0054425) Train Loss: 5.7792, Train Steps/Sec: 1.12 +[2025-04-27 13:42:31] (step=0054450) Train Loss: 5.7713, Train Steps/Sec: 1.12 +[2025-04-27 13:42:53] (step=0054475) Train Loss: 5.7977, Train Steps/Sec: 1.12 +[2025-04-27 13:43:15] (step=0054500) Train Loss: 5.7966, Train Steps/Sec: 1.12 +[2025-04-27 13:43:38] (step=0054525) Train Loss: 5.7542, Train Steps/Sec: 1.12 +[2025-04-27 13:44:00] (step=0054550) Train Loss: 5.8237, Train Steps/Sec: 1.12 +[2025-04-27 13:44:22] (step=0054575) Train Loss: 5.8258, Train Steps/Sec: 1.12 +[2025-04-27 13:44:45] (step=0054600) Train Loss: 5.8108, Train Steps/Sec: 1.11 +[2025-04-27 13:45:07] (step=0054625) Train Loss: 5.7913, Train Steps/Sec: 1.12 +[2025-04-27 13:45:29] (step=0054650) Train Loss: 5.8631, Train Steps/Sec: 1.12 +[2025-04-27 13:45:52] (step=0054675) Train Loss: 5.8213, Train Steps/Sec: 1.12 +[2025-04-27 13:46:14] (step=0054700) Train Loss: 5.8105, Train Steps/Sec: 1.12 +[2025-04-27 13:46:36] (step=0054725) Train Loss: 5.7459, Train Steps/Sec: 1.12 +[2025-04-27 13:46:59] (step=0054750) Train Loss: 5.7718, Train Steps/Sec: 1.12 +[2025-04-27 13:47:21] (step=0054775) Train Loss: 5.8489, Train Steps/Sec: 1.12 +[2025-04-27 13:47:43] (step=0054800) Train Loss: 5.8394, Train Steps/Sec: 1.11 +[2025-04-27 13:48:06] (step=0054825) Train Loss: 5.7946, Train Steps/Sec: 1.12 +[2025-04-27 13:48:28] (step=0054850) Train Loss: 5.8388, Train Steps/Sec: 1.12 +[2025-04-27 13:48:50] (step=0054875) Train Loss: 5.8450, Train Steps/Sec: 1.12 +[2025-04-27 13:49:13] (step=0054900) Train Loss: 5.8255, Train Steps/Sec: 1.12 +[2025-04-27 13:49:35] (step=0054925) Train Loss: 5.8241, Train Steps/Sec: 1.12 +[2025-04-27 13:49:57] (step=0054950) Train Loss: 5.8129, Train Steps/Sec: 1.12 +[2025-04-27 13:50:20] (step=0054975) Train Loss: 5.7316, Train Steps/Sec: 1.12 +[2025-04-27 13:50:42] (step=0055000) Train Loss: 5.8142, Train Steps/Sec: 1.11 +[2025-04-27 13:51:04] (step=0055025) Train Loss: 5.8332, Train Steps/Sec: 1.12 +[2025-04-27 13:51:27] (step=0055050) Train Loss: 5.7974, Train Steps/Sec: 1.12 +[2025-04-27 13:51:49] (step=0055075) Train Loss: 5.7856, Train Steps/Sec: 1.12 +[2025-04-27 13:52:11] (step=0055100) Train Loss: 5.7918, Train Steps/Sec: 1.12 +[2025-04-27 13:52:34] (step=0055125) Train Loss: 5.8146, Train Steps/Sec: 1.12 +[2025-04-27 13:52:56] (step=0055150) Train Loss: 5.6463, Train Steps/Sec: 1.12 +[2025-04-27 13:53:18] (step=0055175) Train Loss: 5.8319, Train Steps/Sec: 1.12 +[2025-04-27 13:53:41] (step=0055200) Train Loss: 5.7919, Train Steps/Sec: 1.12 +[2025-04-27 13:54:03] (step=0055225) Train Loss: 5.7622, Train Steps/Sec: 1.12 +[2025-04-27 13:54:25] (step=0055250) Train Loss: 5.7679, Train Steps/Sec: 1.12 +[2025-04-27 13:54:48] (step=0055275) Train Loss: 5.7535, Train Steps/Sec: 1.12 +[2025-04-27 13:55:10] (step=0055300) Train Loss: 5.7608, Train Steps/Sec: 1.12 +[2025-04-27 13:55:32] (step=0055325) Train Loss: 5.8030, Train Steps/Sec: 1.12 +[2025-04-27 13:55:55] (step=0055350) Train Loss: 5.8261, Train Steps/Sec: 1.12 +[2025-04-27 13:56:17] (step=0055375) Train Loss: 5.8021, Train Steps/Sec: 1.12 +[2025-04-27 13:56:39] (step=0055400) Train Loss: 5.7660, Train Steps/Sec: 1.12 +[2025-04-27 13:57:02] (step=0055425) Train Loss: 5.7797, Train Steps/Sec: 1.12 +[2025-04-27 13:57:24] (step=0055450) Train Loss: 5.7545, Train Steps/Sec: 1.12 +[2025-04-27 13:57:46] (step=0055475) Train Loss: 5.8011, Train Steps/Sec: 1.12 +[2025-04-27 13:58:09] (step=0055500) Train Loss: 5.7928, Train Steps/Sec: 1.12 +[2025-04-27 13:58:31] (step=0055525) Train Loss: 5.7790, Train Steps/Sec: 1.12 +[2025-04-27 13:58:53] (step=0055550) Train Loss: 5.7472, Train Steps/Sec: 1.12 +[2025-04-27 13:59:15] (step=0055575) Train Loss: 5.7803, Train Steps/Sec: 1.12 +[2025-04-27 13:59:38] (step=0055600) Train Loss: 5.8024, Train Steps/Sec: 1.11 +[2025-04-27 14:00:00] (step=0055625) Train Loss: 5.7645, Train Steps/Sec: 1.12 +[2025-04-27 14:00:22] (step=0055650) Train Loss: 5.8414, Train Steps/Sec: 1.12 +[2025-04-27 14:00:45] (step=0055675) Train Loss: 5.7996, Train Steps/Sec: 1.12 +[2025-04-27 14:01:07] (step=0055700) Train Loss: 5.7602, Train Steps/Sec: 1.12 +[2025-04-27 14:01:29] (step=0055725) Train Loss: 5.7983, Train Steps/Sec: 1.12 +[2025-04-27 14:01:52] (step=0055750) Train Loss: 5.8281, Train Steps/Sec: 1.12 +[2025-04-27 14:02:14] (step=0055775) Train Loss: 5.8336, Train Steps/Sec: 1.12 +[2025-04-27 14:02:36] (step=0055800) Train Loss: 5.7761, Train Steps/Sec: 1.12 +[2025-04-27 14:02:59] (step=0055825) Train Loss: 5.7860, Train Steps/Sec: 1.12 +[2025-04-27 14:03:21] (step=0055850) Train Loss: 5.8751, Train Steps/Sec: 1.12 +[2025-04-27 14:03:43] (step=0055875) Train Loss: 5.8026, Train Steps/Sec: 1.12 +[2025-04-27 14:04:06] (step=0055900) Train Loss: 5.7985, Train Steps/Sec: 1.12 +[2025-04-27 14:04:28] (step=0055925) Train Loss: 5.7922, Train Steps/Sec: 1.12 +[2025-04-27 14:04:50] (step=0055950) Train Loss: 5.8307, Train Steps/Sec: 1.12 +[2025-04-27 14:05:13] (step=0055975) Train Loss: 5.8380, Train Steps/Sec: 1.12 +[2025-04-27 14:05:35] (step=0056000) Train Loss: 5.7695, Train Steps/Sec: 1.12 +[2025-04-27 14:05:35] text_config is None. Initializing the text config with default values (`OPTConfig`). +[2025-04-27 14:11:43] Finish Eval in 56000 steps... +[2025-04-27 14:12:02] Saved checkpoint to checkpoint/X2I_UltraEdiit_fourmask_LLava_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_filtered_objects_4mask_1280_context/003-GPT-XL/checkpoints/0056000.pt +[2025-04-27 14:12:04] Removed old checkpoint: checkpoint/X2I_UltraEdiit_fourmask_LLava_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_filtered_objects_4mask_1280_context/003-GPT-XL/checkpoints/0054000.pt +[2025-04-27 14:12:26] (step=0056025) Train Loss: 5.8289, Train Steps/Sec: 0.06 +[2025-04-27 14:12:49] (step=0056050) Train Loss: 5.8056, Train Steps/Sec: 1.11 +[2025-04-27 14:13:11] (step=0056075) Train Loss: 5.8062, Train Steps/Sec: 1.12 +[2025-04-27 14:13:34] (step=0056100) Train Loss: 5.8100, Train Steps/Sec: 1.12 +[2025-04-27 14:13:56] (step=0056125) Train Loss: 5.7554, Train Steps/Sec: 1.12 +[2025-04-27 14:14:18] (step=0056150) Train Loss: 5.6926, Train Steps/Sec: 1.12 +[2025-04-27 14:14:40] (step=0056175) Train Loss: 5.7688, Train Steps/Sec: 1.12 +[2025-04-27 14:15:03] (step=0056200) Train Loss: 5.7003, Train Steps/Sec: 1.12 +[2025-04-27 14:15:25] (step=0056225) Train Loss: 5.7772, Train Steps/Sec: 1.12 +[2025-04-27 14:15:47] (step=0056250) Train Loss: 5.8121, Train Steps/Sec: 1.12 +[2025-04-27 14:16:10] (step=0056275) Train Loss: 5.7703, Train Steps/Sec: 1.12 +[2025-04-27 14:16:32] (step=0056300) Train Loss: 5.7627, Train Steps/Sec: 1.12 +[2025-04-27 14:16:54] (step=0056325) Train Loss: 5.7750, Train Steps/Sec: 1.12 +[2025-04-27 14:17:16] (step=0056350) Train Loss: 5.7984, Train Steps/Sec: 1.12 +[2025-04-27 14:17:39] (step=0056375) Train Loss: 5.8157, Train Steps/Sec: 1.12 +[2025-04-27 14:18:01] (step=0056400) Train Loss: 5.7628, Train Steps/Sec: 1.12 +[2025-04-27 14:18:23] (step=0056425) Train Loss: 5.7460, Train Steps/Sec: 1.12 +[2025-04-27 14:18:46] (step=0056450) Train Loss: 5.8653, Train Steps/Sec: 1.12 +[2025-04-27 14:19:08] (step=0056475) Train Loss: 5.7945, Train Steps/Sec: 1.12 +[2025-04-27 14:19:30] (step=0056500) Train Loss: 5.7887, Train Steps/Sec: 1.12 +[2025-04-27 14:19:52] (step=0056525) Train Loss: 5.7188, Train Steps/Sec: 1.12 +[2025-04-27 14:20:15] (step=0056550) Train Loss: 5.8649, Train Steps/Sec: 1.12 +[2025-04-27 14:20:37] (step=0056575) Train Loss: 5.7891, Train Steps/Sec: 1.12 +[2025-04-27 14:20:59] (step=0056600) Train Loss: 5.8471, Train Steps/Sec: 1.12 +[2025-04-27 14:21:22] (step=0056625) Train Loss: 5.7715, Train Steps/Sec: 1.12 +[2025-04-27 14:21:44] (step=0056650) Train Loss: 5.8381, Train Steps/Sec: 1.12 +[2025-04-27 14:22:06] (step=0056675) Train Loss: 5.7744, Train Steps/Sec: 1.12 +[2025-04-27 14:22:29] (step=0056700) Train Loss: 5.7159, Train Steps/Sec: 1.12 +[2025-04-27 14:22:51] (step=0056725) Train Loss: 5.7230, Train Steps/Sec: 1.12 +[2025-04-27 14:23:13] (step=0056750) Train Loss: 5.8204, Train Steps/Sec: 1.12 +[2025-04-27 14:23:36] (step=0056775) Train Loss: 5.7692, Train Steps/Sec: 1.12 +[2025-04-27 14:23:58] (step=0056800) Train Loss: 5.8437, Train Steps/Sec: 1.12 +[2025-04-27 14:24:20] (step=0056825) Train Loss: 5.8385, Train Steps/Sec: 1.12 +[2025-04-27 14:24:43] (step=0056850) Train Loss: 5.7802, Train Steps/Sec: 1.12 +[2025-04-27 14:25:05] (step=0056875) Train Loss: 5.8371, Train Steps/Sec: 1.12 +[2025-04-27 14:25:27] (step=0056900) Train Loss: 5.8682, Train Steps/Sec: 1.12 +[2025-04-27 14:25:49] (step=0056925) Train Loss: 5.7540, Train Steps/Sec: 1.12 +[2025-04-27 14:26:12] (step=0056950) Train Loss: 5.7955, Train Steps/Sec: 1.12 +[2025-04-27 14:26:34] (step=0056975) Train Loss: 5.7868, Train Steps/Sec: 1.12 +[2025-04-27 14:26:56] (step=0057000) Train Loss: 5.8225, Train Steps/Sec: 1.12 +[2025-04-27 14:27:19] (step=0057025) Train Loss: 5.7458, Train Steps/Sec: 1.12 +[2025-04-27 14:27:41] (step=0057050) Train Loss: 5.7793, Train Steps/Sec: 1.12 +[2025-04-27 14:28:03] (step=0057075) Train Loss: 5.8635, Train Steps/Sec: 1.12 +[2025-04-27 14:28:26] (step=0057100) Train Loss: 5.7832, Train Steps/Sec: 1.12 +[2025-04-27 14:28:48] (step=0057125) Train Loss: 5.8243, Train Steps/Sec: 1.12 +[2025-04-27 14:29:10] (step=0057150) Train Loss: 5.7258, Train Steps/Sec: 1.12 +[2025-04-27 14:29:32] (step=0057175) Train Loss: 5.7780, Train Steps/Sec: 1.12 +[2025-04-27 14:29:55] (step=0057200) Train Loss: 5.7611, Train Steps/Sec: 1.12 +[2025-04-27 14:30:17] (step=0057225) Train Loss: 5.8122, Train Steps/Sec: 1.12 +[2025-04-27 14:30:39] (step=0057250) Train Loss: 5.8352, Train Steps/Sec: 1.12 +[2025-04-27 14:31:02] (step=0057275) Train Loss: 5.8399, Train Steps/Sec: 1.12 +[2025-04-27 14:31:24] (step=0057300) Train Loss: 5.8726, Train Steps/Sec: 1.12 +[2025-04-27 14:31:46] (step=0057325) Train Loss: 5.8442, Train Steps/Sec: 1.12 +[2025-04-27 14:32:09] (step=0057350) Train Loss: 5.8296, Train Steps/Sec: 1.12 +[2025-04-27 14:32:31] (step=0057375) Train Loss: 5.8085, Train Steps/Sec: 1.12 +[2025-04-27 14:32:53] (step=0057400) Train Loss: 5.8122, Train Steps/Sec: 1.12 +[2025-04-27 14:33:15] (step=0057425) Train Loss: 5.7464, Train Steps/Sec: 1.13 +[2025-04-27 14:33:38] (step=0057450) Train Loss: 5.7569, Train Steps/Sec: 1.12 +[2025-04-27 14:34:00] (step=0057475) Train Loss: 5.7616, Train Steps/Sec: 1.12 +[2025-04-27 14:34:22] (step=0057500) Train Loss: 5.7575, Train Steps/Sec: 1.12 +[2025-04-27 14:34:45] (step=0057525) Train Loss: 5.7522, Train Steps/Sec: 1.12 +[2025-04-27 14:35:07] (step=0057550) Train Loss: 5.8239, Train Steps/Sec: 1.13 +[2025-04-27 14:35:29] (step=0057575) Train Loss: 5.8705, Train Steps/Sec: 1.12 +[2025-04-27 14:35:51] (step=0057600) Train Loss: 5.7653, Train Steps/Sec: 1.12 +[2025-04-27 14:36:14] (step=0057625) Train Loss: 5.7647, Train Steps/Sec: 1.12 +[2025-04-27 14:36:36] (step=0057650) Train Loss: 5.7852, Train Steps/Sec: 1.12 +[2025-04-27 14:36:58] (step=0057675) Train Loss: 5.7867, Train Steps/Sec: 1.12 +[2025-04-27 14:37:21] (step=0057700) Train Loss: 5.7638, Train Steps/Sec: 1.12 +[2025-04-27 14:37:43] (step=0057725) Train Loss: 5.7539, Train Steps/Sec: 1.12 +[2025-04-27 14:38:05] (step=0057750) Train Loss: 5.8046, Train Steps/Sec: 1.12 +[2025-04-27 14:38:28] (step=0057775) Train Loss: 5.8314, Train Steps/Sec: 1.12 +[2025-04-27 14:38:50] (step=0057800) Train Loss: 5.7602, Train Steps/Sec: 1.12 +[2025-04-27 14:39:12] (step=0057825) Train Loss: 5.7690, Train Steps/Sec: 1.12 +[2025-04-27 14:39:34] (step=0057850) Train Loss: 5.8320, Train Steps/Sec: 1.12 +[2025-04-27 14:39:57] (step=0057875) Train Loss: 5.8325, Train Steps/Sec: 1.12 +[2025-04-27 14:40:19] (step=0057900) Train Loss: 5.8511, Train Steps/Sec: 1.12 +[2025-04-27 14:40:41] (step=0057925) Train Loss: 5.7939, Train Steps/Sec: 1.12 +[2025-04-27 14:41:04] (step=0057950) Train Loss: 5.7412, Train Steps/Sec: 1.12 +[2025-04-27 14:41:26] (step=0057975) Train Loss: 5.8578, Train Steps/Sec: 1.12 +[2025-04-27 14:41:48] (step=0058000) Train Loss: 5.7512, Train Steps/Sec: 1.12 +[2025-04-27 14:41:48] text_config is None. Initializing the text config with default values (`OPTConfig`). +[2025-04-27 14:47:56] Finish Eval in 58000 steps... +[2025-04-27 14:48:16] Saved checkpoint to checkpoint/X2I_UltraEdiit_fourmask_LLava_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_filtered_objects_4mask_1280_context/003-GPT-XL/checkpoints/0058000.pt +[2025-04-27 14:48:18] Removed old checkpoint: checkpoint/X2I_UltraEdiit_fourmask_LLava_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_filtered_objects_4mask_1280_context/003-GPT-XL/checkpoints/0056000.pt +[2025-04-27 14:48:40] (step=0058025) Train Loss: 5.8519, Train Steps/Sec: 0.06 +[2025-04-27 14:49:03] (step=0058050) Train Loss: 5.8059, Train Steps/Sec: 1.12 +[2025-04-27 14:49:29] (step=0058075) Train Loss: 5.7056, Train Steps/Sec: 0.93 +[2025-04-27 14:49:52] (step=0058100) Train Loss: 5.7506, Train Steps/Sec: 1.12 +[2025-04-27 14:50:14] (step=0058125) Train Loss: 5.8088, Train Steps/Sec: 1.12 +[2025-04-27 14:50:36] (step=0058150) Train Loss: 5.7521, Train Steps/Sec: 1.12 +[2025-04-27 14:50:59] (step=0058175) Train Loss: 5.7651, Train Steps/Sec: 1.12 +[2025-04-27 14:51:26] (step=0058200) Train Loss: 5.8518, Train Steps/Sec: 0.93 +[2025-04-27 14:51:48] (step=0058225) Train Loss: 5.7997, Train Steps/Sec: 1.12 +[2025-04-27 14:52:10] (step=0058250) Train Loss: 5.8301, Train Steps/Sec: 1.12 +[2025-04-27 14:52:32] (step=0058275) Train Loss: 5.7840, Train Steps/Sec: 1.12 +[2025-04-27 14:52:55] (step=0058300) Train Loss: 5.7047, Train Steps/Sec: 1.12 +[2025-04-27 14:53:17] (step=0058325) Train Loss: 5.7322, Train Steps/Sec: 1.12 +[2025-04-27 14:53:39] (step=0058350) Train Loss: 5.7718, Train Steps/Sec: 1.12 +[2025-04-27 14:54:02] (step=0058375) Train Loss: 5.7703, Train Steps/Sec: 1.12 +[2025-04-27 14:54:24] (step=0058400) Train Loss: 5.7163, Train Steps/Sec: 1.12 +[2025-04-27 14:54:46] (step=0058425) Train Loss: 5.8399, Train Steps/Sec: 1.12 +[2025-04-27 14:55:09] (step=0058450) Train Loss: 5.8523, Train Steps/Sec: 1.12 +[2025-04-27 14:55:31] (step=0058475) Train Loss: 5.8346, Train Steps/Sec: 1.12 +[2025-04-27 14:55:53] (step=0058500) Train Loss: 5.7876, Train Steps/Sec: 1.12 +[2025-04-27 14:56:16] (step=0058525) Train Loss: 5.7028, Train Steps/Sec: 1.12 +[2025-04-27 14:56:38] (step=0058550) Train Loss: 5.7430, Train Steps/Sec: 1.12 +[2025-04-27 14:57:00] (step=0058575) Train Loss: 5.7867, Train Steps/Sec: 1.12 +[2025-04-27 14:57:23] (step=0058600) Train Loss: 5.8207, Train Steps/Sec: 1.10 +[2025-04-27 14:57:51] (step=0058625) Train Loss: 5.7623, Train Steps/Sec: 0.91 +[2025-04-27 14:58:21] (step=0058650) Train Loss: 5.7390, Train Steps/Sec: 0.82 +[2025-04-27 14:58:48] (step=0058675) Train Loss: 5.8145, Train Steps/Sec: 0.93 +[2025-04-27 14:59:16] (step=0058700) Train Loss: 5.8313, Train Steps/Sec: 0.90 +[2025-04-27 14:59:38] (step=0058725) Train Loss: 5.7542, Train Steps/Sec: 1.12 +[2025-04-27 15:00:00] (step=0058750) Train Loss: 5.8093, Train Steps/Sec: 1.12 +[2025-04-27 15:00:23] (step=0058775) Train Loss: 5.7157, Train Steps/Sec: 1.12 +[2025-04-27 15:00:45] (step=0058800) Train Loss: 5.8387, Train Steps/Sec: 1.12 +[2025-04-27 15:01:07] (step=0058825) Train Loss: 5.7493, Train Steps/Sec: 1.12 +[2025-04-27 15:01:30] (step=0058850) Train Loss: 5.7528, Train Steps/Sec: 1.12 +[2025-04-27 15:01:52] (step=0058875) Train Loss: 5.7829, Train Steps/Sec: 1.12 +[2025-04-27 15:02:14] (step=0058900) Train Loss: 5.8513, Train Steps/Sec: 1.12 +[2025-04-27 15:02:37] (step=0058925) Train Loss: 5.7874, Train Steps/Sec: 1.12 +[2025-04-27 15:02:59] (step=0058950) Train Loss: 5.7713, Train Steps/Sec: 1.12 +[2025-04-27 15:03:21] (step=0058975) Train Loss: 5.7406, Train Steps/Sec: 1.12 +[2025-04-27 15:03:44] (step=0059000) Train Loss: 5.8598, Train Steps/Sec: 1.12 +[2025-04-27 15:04:06] (step=0059025) Train Loss: 5.7772, Train Steps/Sec: 1.12 +[2025-04-27 15:04:28] (step=0059050) Train Loss: 5.7474, Train Steps/Sec: 1.12 +[2025-04-27 15:04:51] (step=0059075) Train Loss: 5.7897, Train Steps/Sec: 1.12 +[2025-04-27 15:05:13] (step=0059100) Train Loss: 5.7803, Train Steps/Sec: 1.12 +[2025-04-27 15:05:35] (step=0059125) Train Loss: 5.7661, Train Steps/Sec: 1.12 +[2025-04-27 15:05:58] (step=0059150) Train Loss: 5.7059, Train Steps/Sec: 1.12 +[2025-04-27 15:06:20] (step=0059175) Train Loss: 5.6871, Train Steps/Sec: 1.12 +[2025-04-27 15:06:42] (step=0059200) Train Loss: 5.8231, Train Steps/Sec: 1.12 +[2025-04-27 15:07:05] (step=0059225) Train Loss: 5.7870, Train Steps/Sec: 1.12 +[2025-04-27 15:07:27] (step=0059250) Train Loss: 5.8231, Train Steps/Sec: 1.12 +[2025-04-27 15:07:49] (step=0059275) Train Loss: 5.7853, Train Steps/Sec: 1.12 +[2025-04-27 15:08:12] (step=0059300) Train Loss: 5.7373, Train Steps/Sec: 1.12 +[2025-04-27 15:08:34] (step=0059325) Train Loss: 5.8618, Train Steps/Sec: 1.12 +[2025-04-27 15:08:56] (step=0059350) Train Loss: 5.7357, Train Steps/Sec: 1.12 +[2025-04-27 15:09:19] (step=0059375) Train Loss: 5.7579, Train Steps/Sec: 1.12 +[2025-04-27 15:09:41] (step=0059400) Train Loss: 5.7611, Train Steps/Sec: 1.12 +[2025-04-27 15:10:03] (step=0059425) Train Loss: 5.7475, Train Steps/Sec: 1.12 +[2025-04-27 15:10:26] (step=0059450) Train Loss: 5.7525, Train Steps/Sec: 1.12 +[2025-04-27 15:10:48] (step=0059475) Train Loss: 5.7253, Train Steps/Sec: 1.12 +[2025-04-27 15:11:10] (step=0059500) Train Loss: 5.7642, Train Steps/Sec: 1.12 +[2025-04-27 15:11:33] (step=0059525) Train Loss: 5.7888, Train Steps/Sec: 1.12 +[2025-04-27 15:11:55] (step=0059550) Train Loss: 5.7348, Train Steps/Sec: 1.12 +[2025-04-27 15:12:17] (step=0059575) Train Loss: 5.7846, Train Steps/Sec: 1.12 +[2025-04-27 15:12:40] (step=0059600) Train Loss: 5.7790, Train Steps/Sec: 1.12 +[2025-04-27 15:13:02] (step=0059625) Train Loss: 5.8215, Train Steps/Sec: 1.12 +[2025-04-27 15:13:24] (step=0059650) Train Loss: 5.7836, Train Steps/Sec: 1.12 +[2025-04-27 15:13:46] (step=0059675) Train Loss: 5.7723, Train Steps/Sec: 1.12 +[2025-04-27 15:14:09] (step=0059700) Train Loss: 5.7902, Train Steps/Sec: 1.12 +[2025-04-27 15:14:31] (step=0059725) Train Loss: 5.7162, Train Steps/Sec: 1.12 +[2025-04-27 15:14:53] (step=0059750) Train Loss: 5.7036, Train Steps/Sec: 1.12 +[2025-04-27 15:15:16] (step=0059775) Train Loss: 5.7370, Train Steps/Sec: 1.12 +[2025-04-27 15:15:38] (step=0059800) Train Loss: 5.7596, Train Steps/Sec: 1.12 +[2025-04-27 15:16:00] (step=0059825) Train Loss: 5.7548, Train Steps/Sec: 1.12 +[2025-04-27 15:16:23] (step=0059850) Train Loss: 5.7396, Train Steps/Sec: 1.12 +[2025-04-27 15:16:45] (step=0059875) Train Loss: 5.7476, Train Steps/Sec: 1.12 +[2025-04-27 15:17:07] (step=0059900) Train Loss: 5.7858, Train Steps/Sec: 1.12 +[2025-04-27 15:17:30] (step=0059925) Train Loss: 5.7376, Train Steps/Sec: 1.12 +[2025-04-27 15:17:52] (step=0059950) Train Loss: 5.7353, Train Steps/Sec: 1.12 +[2025-04-27 15:18:14] (step=0059975) Train Loss: 5.7653, Train Steps/Sec: 1.12 +[2025-04-27 15:18:36] (step=0060000) Train Loss: 5.7907, Train Steps/Sec: 1.12 +[2025-04-27 15:18:37] text_config is None. Initializing the text config with default values (`OPTConfig`). +[2025-04-27 15:24:52] Finish Eval in 60000 steps... +[2025-04-27 15:25:12] Saved checkpoint to checkpoint/X2I_UltraEdiit_fourmask_LLava_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_filtered_objects_4mask_1280_context/003-GPT-XL/checkpoints/0060000.pt +[2025-04-27 15:25:15] Removed old checkpoint: checkpoint/X2I_UltraEdiit_fourmask_LLava_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_filtered_objects_4mask_1280_context/003-GPT-XL/checkpoints/0058000.pt +[2025-04-27 15:25:37] (step=0060025) Train Loss: 5.7344, Train Steps/Sec: 0.06 +[2025-04-27 15:26:00] (step=0060050) Train Loss: 5.7604, Train Steps/Sec: 1.11 +[2025-04-27 15:26:22] (step=0060075) Train Loss: 5.7831, Train Steps/Sec: 1.12 +[2025-04-27 15:26:45] (step=0060100) Train Loss: 5.7995, Train Steps/Sec: 1.12 +[2025-04-27 15:27:07] (step=0060125) Train Loss: 5.7997, Train Steps/Sec: 1.11 +[2025-04-27 15:27:29] (step=0060150) Train Loss: 5.7676, Train Steps/Sec: 1.12 +[2025-04-27 15:27:52] (step=0060175) Train Loss: 5.7983, Train Steps/Sec: 1.12 +[2025-04-27 15:28:14] (step=0060200) Train Loss: 5.7304, Train Steps/Sec: 1.11 +[2025-04-27 15:28:37] (step=0060225) Train Loss: 5.7218, Train Steps/Sec: 1.11 +[2025-04-27 15:28:59] (step=0060250) Train Loss: 5.7814, Train Steps/Sec: 1.11 +[2025-04-27 15:29:21] (step=0060275) Train Loss: 5.7788, Train Steps/Sec: 1.12 +[2025-04-27 15:29:44] (step=0060300) Train Loss: 5.7936, Train Steps/Sec: 1.11 +[2025-04-27 15:30:06] (step=0060325) Train Loss: 5.8051, Train Steps/Sec: 1.11 +[2025-04-27 15:30:29] (step=0060350) Train Loss: 5.7688, Train Steps/Sec: 1.12 +[2025-04-27 15:30:51] (step=0060375) Train Loss: 5.7273, Train Steps/Sec: 1.12 +[2025-04-27 15:31:14] (step=0060400) Train Loss: 5.7312, Train Steps/Sec: 1.11 +[2025-04-27 15:31:36] (step=0060425) Train Loss: 5.7730, Train Steps/Sec: 1.12 +[2025-04-27 15:31:58] (step=0060450) Train Loss: 5.7267, Train Steps/Sec: 1.12 +[2025-04-27 15:32:21] (step=0060475) Train Loss: 5.7414, Train Steps/Sec: 1.12 +[2025-04-27 15:32:43] (step=0060500) Train Loss: 5.7235, Train Steps/Sec: 1.10 +[2025-04-27 15:33:06] (step=0060525) Train Loss: 5.7769, Train Steps/Sec: 1.12 +[2025-04-27 15:33:28] (step=0060550) Train Loss: 5.7516, Train Steps/Sec: 1.11 +[2025-04-27 15:33:51] (step=0060575) Train Loss: 5.7268, Train Steps/Sec: 1.12 +[2025-04-27 15:34:13] (step=0060600) Train Loss: 5.8066, Train Steps/Sec: 1.11 +[2025-04-27 15:34:35] (step=0060625) Train Loss: 5.7575, Train Steps/Sec: 1.12 +[2025-04-27 15:34:58] (step=0060650) Train Loss: 5.8001, Train Steps/Sec: 1.12 +[2025-04-27 15:35:20] (step=0060675) Train Loss: 5.7328, Train Steps/Sec: 1.12 +[2025-04-27 15:35:42] (step=0060700) Train Loss: 5.7971, Train Steps/Sec: 1.12 +[2025-04-27 15:36:05] (step=0060725) Train Loss: 5.8110, Train Steps/Sec: 1.12 +[2025-04-27 15:36:27] (step=0060750) Train Loss: 5.7600, Train Steps/Sec: 1.12 +[2025-04-27 15:36:50] (step=0060775) Train Loss: 5.7736, Train Steps/Sec: 1.12 +[2025-04-27 15:37:12] (step=0060800) Train Loss: 5.7659, Train Steps/Sec: 1.11 +[2025-04-27 15:37:34] (step=0060825) Train Loss: 5.7854, Train Steps/Sec: 1.12 +[2025-04-27 15:37:57] (step=0060850) Train Loss: 5.8069, Train Steps/Sec: 1.12 +[2025-04-27 15:38:19] (step=0060875) Train Loss: 5.7845, Train Steps/Sec: 1.12 +[2025-04-27 15:38:42] (step=0060900) Train Loss: 5.7972, Train Steps/Sec: 1.12 +[2025-04-27 15:39:04] (step=0060925) Train Loss: 5.7704, Train Steps/Sec: 1.12 +[2025-04-27 15:39:26] (step=0060950) Train Loss: 5.7591, Train Steps/Sec: 1.12 +[2025-04-27 15:39:49] (step=0060975) Train Loss: 5.7906, Train Steps/Sec: 1.12 +[2025-04-27 15:40:11] (step=0061000) Train Loss: 5.7385, Train Steps/Sec: 1.11 +[2025-04-27 15:40:34] (step=0061025) Train Loss: 5.7861, Train Steps/Sec: 1.12 +[2025-04-27 15:40:56] (step=0061050) Train Loss: 5.7856, Train Steps/Sec: 1.12 +[2025-04-27 15:41:18] (step=0061075) Train Loss: 5.7643, Train Steps/Sec: 1.12 +[2025-04-27 15:41:41] (step=0061100) Train Loss: 5.9311, Train Steps/Sec: 1.11 +[2025-04-27 15:42:03] (step=0061125) Train Loss: 5.7453, Train Steps/Sec: 1.11 +[2025-04-27 15:42:26] (step=0061150) Train Loss: 5.7855, Train Steps/Sec: 1.12 +[2025-04-27 15:42:48] (step=0061175) Train Loss: 5.7420, Train Steps/Sec: 1.12 +[2025-04-27 15:43:10] (step=0061200) Train Loss: 5.7269, Train Steps/Sec: 1.12 +[2025-04-27 15:43:33] (step=0061225) Train Loss: 5.7836, Train Steps/Sec: 1.12 +[2025-04-27 15:43:55] (step=0061250) Train Loss: 5.7578, Train Steps/Sec: 1.12 +[2025-04-27 15:44:17] (step=0061275) Train Loss: 5.8153, Train Steps/Sec: 1.12 +[2025-04-27 15:44:40] (step=0061300) Train Loss: 5.8085, Train Steps/Sec: 1.12 +[2025-04-27 15:45:02] (step=0061325) Train Loss: 5.7838, Train Steps/Sec: 1.12 +[2025-04-27 15:45:24] (step=0061350) Train Loss: 5.7496, Train Steps/Sec: 1.12 +[2025-04-27 15:45:47] (step=0061375) Train Loss: 5.8375, Train Steps/Sec: 1.12 +[2025-04-27 15:46:09] (step=0061400) Train Loss: 5.8632, Train Steps/Sec: 1.12 +[2025-04-27 15:46:32] (step=0061425) Train Loss: 5.7898, Train Steps/Sec: 1.12 +[2025-04-27 15:46:54] (step=0061450) Train Loss: 5.7313, Train Steps/Sec: 1.12 +[2025-04-27 15:47:16] (step=0061475) Train Loss: 5.7839, Train Steps/Sec: 1.12 +[2025-04-27 15:47:38] (step=0061500) Train Loss: 5.8047, Train Steps/Sec: 1.12 +[2025-04-27 15:48:01] (step=0061525) Train Loss: 5.7391, Train Steps/Sec: 1.12 +[2025-04-27 15:48:23] (step=0061550) Train Loss: 5.7552, Train Steps/Sec: 1.12 +[2025-04-27 15:48:45] (step=0061575) Train Loss: 5.7652, Train Steps/Sec: 1.12 +[2025-04-27 15:49:08] (step=0061600) Train Loss: 5.7994, Train Steps/Sec: 1.12 +[2025-04-27 15:49:30] (step=0061625) Train Loss: 5.8085, Train Steps/Sec: 1.12 +[2025-04-27 15:49:53] (step=0061650) Train Loss: 5.7458, Train Steps/Sec: 1.12 +[2025-04-27 15:50:15] (step=0061675) Train Loss: 5.8260, Train Steps/Sec: 1.12 +[2025-04-27 15:50:37] (step=0061700) Train Loss: 5.7984, Train Steps/Sec: 1.12 +[2025-04-27 15:51:00] (step=0061725) Train Loss: 5.7607, Train Steps/Sec: 1.11 +[2025-04-27 15:51:22] (step=0061750) Train Loss: 5.8463, Train Steps/Sec: 1.12 +[2025-04-27 15:51:45] (step=0061775) Train Loss: 5.7481, Train Steps/Sec: 1.12 +[2025-04-27 15:52:07] (step=0061800) Train Loss: 5.7433, Train Steps/Sec: 1.11 +[2025-04-27 15:52:29] (step=0061825) Train Loss: 5.7468, Train Steps/Sec: 1.11 +[2025-04-27 15:52:52] (step=0061850) Train Loss: 5.7823, Train Steps/Sec: 1.11 +[2025-04-27 15:53:14] (step=0061875) Train Loss: 5.7694, Train Steps/Sec: 1.12 +[2025-04-27 15:53:37] (step=0061900) Train Loss: 5.7816, Train Steps/Sec: 1.12 +[2025-04-27 15:53:59] (step=0061925) Train Loss: 5.8340, Train Steps/Sec: 1.12 +[2025-04-27 15:54:21] (step=0061950) Train Loss: 5.7952, Train Steps/Sec: 1.12 +[2025-04-27 15:54:43] (step=0061975) Train Loss: 5.7756, Train Steps/Sec: 1.12 +[2025-04-27 15:55:06] (step=0062000) Train Loss: 5.7679, Train Steps/Sec: 1.12 +[2025-04-27 15:55:06] text_config is None. Initializing the text config with default values (`OPTConfig`). +[2025-04-27 16:01:23] Finish Eval in 62000 steps... +[2025-04-27 16:01:44] Saved checkpoint to checkpoint/X2I_UltraEdiit_fourmask_LLava_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_filtered_objects_4mask_1280_context/003-GPT-XL/checkpoints/0062000.pt +[2025-04-27 16:01:46] Removed old checkpoint: checkpoint/X2I_UltraEdiit_fourmask_LLava_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_filtered_objects_4mask_1280_context/003-GPT-XL/checkpoints/0060000.pt +[2025-04-27 16:02:08] (step=0062025) Train Loss: 5.7849, Train Steps/Sec: 0.06 +[2025-04-27 16:02:31] (step=0062050) Train Loss: 5.7468, Train Steps/Sec: 1.11 +[2025-04-27 16:02:53] (step=0062075) Train Loss: 5.8140, Train Steps/Sec: 1.12 +[2025-04-27 16:03:16] (step=0062100) Train Loss: 5.7962, Train Steps/Sec: 1.12 +[2025-04-27 16:03:38] (step=0062125) Train Loss: 5.7314, Train Steps/Sec: 1.12 +[2025-04-27 16:04:01] (step=0062150) Train Loss: 5.7185, Train Steps/Sec: 1.11 +[2025-04-27 16:04:23] (step=0062175) Train Loss: 5.7561, Train Steps/Sec: 1.11 +[2025-04-27 16:04:46] (step=0062200) Train Loss: 5.7795, Train Steps/Sec: 1.11 +[2025-04-27 16:05:08] (step=0062225) Train Loss: 5.7751, Train Steps/Sec: 1.11 +[2025-04-27 16:05:31] (step=0062250) Train Loss: 5.7562, Train Steps/Sec: 1.11 +[2025-04-27 16:05:53] (step=0062275) Train Loss: 5.7677, Train Steps/Sec: 1.11 +[2025-04-27 16:06:15] (step=0062300) Train Loss: 5.7618, Train Steps/Sec: 1.12 +[2025-04-27 16:06:42] (step=0062325) Train Loss: 5.7755, Train Steps/Sec: 0.93 +[2025-04-27 16:07:05] (step=0062350) Train Loss: 5.8036, Train Steps/Sec: 1.12 +[2025-04-27 16:07:27] (step=0062375) Train Loss: 5.7485, Train Steps/Sec: 1.12 +[2025-04-27 16:07:50] (step=0062400) Train Loss: 5.7851, Train Steps/Sec: 1.10 +[2025-04-27 16:08:12] (step=0062425) Train Loss: 5.7490, Train Steps/Sec: 1.12 +[2025-04-27 16:08:34] (step=0062450) Train Loss: 5.7490, Train Steps/Sec: 1.12 +[2025-04-27 16:09:01] (step=0062475) Train Loss: 5.7376, Train Steps/Sec: 0.92 +[2025-04-27 16:09:24] (step=0062500) Train Loss: 5.8654, Train Steps/Sec: 1.12 +[2025-04-27 16:09:46] (step=0062525) Train Loss: 5.7737, Train Steps/Sec: 1.12 +[2025-04-27 16:10:08] (step=0062550) Train Loss: 5.7323, Train Steps/Sec: 1.12 +[2025-04-27 16:10:31] (step=0062575) Train Loss: 5.7977, Train Steps/Sec: 1.12 +[2025-04-27 16:10:53] (step=0062600) Train Loss: 5.7715, Train Steps/Sec: 1.12 +[2025-04-27 16:11:16] (step=0062625) Train Loss: 5.7743, Train Steps/Sec: 1.12 +[2025-04-27 16:11:38] (step=0062650) Train Loss: 5.7606, Train Steps/Sec: 1.12 +[2025-04-27 16:12:00] (step=0062675) Train Loss: 5.7651, Train Steps/Sec: 1.12 +[2025-04-27 16:12:23] (step=0062700) Train Loss: 5.7807, Train Steps/Sec: 1.12 +[2025-04-27 16:12:45] (step=0062725) Train Loss: 5.8207, Train Steps/Sec: 1.12 +[2025-04-27 16:13:07] (step=0062750) Train Loss: 5.7255, Train Steps/Sec: 1.12 +[2025-04-27 16:13:30] (step=0062775) Train Loss: 5.7111, Train Steps/Sec: 1.12 +[2025-04-27 16:13:52] (step=0062800) Train Loss: 5.7597, Train Steps/Sec: 1.11 +[2025-04-27 16:14:14] (step=0062825) Train Loss: 5.7796, Train Steps/Sec: 1.12 +[2025-04-27 16:14:37] (step=0062850) Train Loss: 5.8206, Train Steps/Sec: 1.12 +[2025-04-27 16:14:59] (step=0062875) Train Loss: 5.6717, Train Steps/Sec: 1.12 +[2025-04-27 16:15:21] (step=0062900) Train Loss: 5.6923, Train Steps/Sec: 1.12 +[2025-04-27 16:15:44] (step=0062925) Train Loss: 5.7892, Train Steps/Sec: 1.12 +[2025-04-27 16:16:06] (step=0062950) Train Loss: 5.8051, Train Steps/Sec: 1.12 +[2025-04-27 16:16:37] (step=0062975) Train Loss: 5.7864, Train Steps/Sec: 0.82 +[2025-04-27 16:17:03] (step=0063000) Train Loss: 5.8109, Train Steps/Sec: 0.94 +[2025-04-27 16:17:36] (step=0063025) Train Loss: 5.8343, Train Steps/Sec: 0.76 +[2025-04-27 16:17:59] (step=0063050) Train Loss: 5.7449, Train Steps/Sec: 1.12 +[2025-04-27 16:18:21] (step=0063075) Train Loss: 5.7464, Train Steps/Sec: 1.11 +[2025-04-27 16:18:43] (step=0063100) Train Loss: 5.7515, Train Steps/Sec: 1.12 +[2025-04-27 16:19:06] (step=0063125) Train Loss: 5.7185, Train Steps/Sec: 1.12 +[2025-04-27 16:19:28] (step=0063150) Train Loss: 5.7386, Train Steps/Sec: 1.12 +[2025-04-27 16:19:51] (step=0063175) Train Loss: 5.7571, Train Steps/Sec: 1.12 +[2025-04-27 16:20:13] (step=0063200) Train Loss: 5.7604, Train Steps/Sec: 1.11 +[2025-04-27 16:20:35] (step=0063225) Train Loss: 5.7015, Train Steps/Sec: 1.12 +[2025-04-27 16:20:58] (step=0063250) Train Loss: 5.7440, Train Steps/Sec: 1.12 +[2025-04-27 16:21:20] (step=0063275) Train Loss: 5.7387, Train Steps/Sec: 1.12 +[2025-04-27 16:21:42] (step=0063300) Train Loss: 5.7462, Train Steps/Sec: 1.12 +[2025-04-27 16:22:05] (step=0063325) Train Loss: 5.7127, Train Steps/Sec: 1.12 +[2025-04-27 16:22:27] (step=0063350) Train Loss: 5.7658, Train Steps/Sec: 1.12 +[2025-04-27 16:22:49] (step=0063375) Train Loss: 5.7850, Train Steps/Sec: 1.12 +[2025-04-27 16:23:12] (step=0063400) Train Loss: 5.7415, Train Steps/Sec: 1.12 +[2025-04-27 16:23:34] (step=0063425) Train Loss: 5.7504, Train Steps/Sec: 1.12 +[2025-04-27 16:23:56] (step=0063450) Train Loss: 5.7515, Train Steps/Sec: 1.12 +[2025-04-27 16:24:19] (step=0063475) Train Loss: 5.8003, Train Steps/Sec: 1.12 +[2025-04-27 16:24:41] (step=0063500) Train Loss: 5.7063, Train Steps/Sec: 1.12 +[2025-04-27 16:25:03] (step=0063525) Train Loss: 5.8486, Train Steps/Sec: 1.12 +[2025-04-27 16:25:26] (step=0063550) Train Loss: 5.7835, Train Steps/Sec: 1.12 +[2025-04-27 16:25:48] (step=0063575) Train Loss: 5.7700, Train Steps/Sec: 1.12 +[2025-04-27 16:26:11] (step=0063600) Train Loss: 5.7493, Train Steps/Sec: 1.12 +[2025-04-27 16:26:33] (step=0063625) Train Loss: 5.7623, Train Steps/Sec: 1.12 +[2025-04-27 16:26:55] (step=0063650) Train Loss: 5.7592, Train Steps/Sec: 1.12 +[2025-04-27 16:27:17] (step=0063675) Train Loss: 5.8028, Train Steps/Sec: 1.12 +[2025-04-27 16:27:40] (step=0063700) Train Loss: 5.8158, Train Steps/Sec: 1.12 +[2025-04-27 16:28:02] (step=0063725) Train Loss: 5.7741, Train Steps/Sec: 1.12 +[2025-04-27 16:28:24] (step=0063750) Train Loss: 5.7778, Train Steps/Sec: 1.12 +[2025-04-27 16:28:47] (step=0063775) Train Loss: 5.8168, Train Steps/Sec: 1.12 +[2025-04-27 16:29:09] (step=0063800) Train Loss: 5.7099, Train Steps/Sec: 1.11 +[2025-04-27 16:29:32] (step=0063825) Train Loss: 5.7562, Train Steps/Sec: 1.12 +[2025-04-27 16:29:54] (step=0063850) Train Loss: 5.7583, Train Steps/Sec: 1.12 +[2025-04-27 16:30:16] (step=0063875) Train Loss: 5.7657, Train Steps/Sec: 1.12 +[2025-04-27 16:30:39] (step=0063900) Train Loss: 5.6851, Train Steps/Sec: 1.12 +[2025-04-27 16:31:01] (step=0063925) Train Loss: 5.7762, Train Steps/Sec: 1.12 +[2025-04-27 16:31:23] (step=0063950) Train Loss: 5.7864, Train Steps/Sec: 1.12 +[2025-04-27 16:31:46] (step=0063975) Train Loss: 5.7642, Train Steps/Sec: 1.12 +[2025-04-27 16:32:08] (step=0064000) Train Loss: 5.8009, Train Steps/Sec: 1.11 +[2025-04-27 16:32:08] text_config is None. Initializing the text config with default values (`OPTConfig`). +[2025-04-27 16:38:31] Finish Eval in 64000 steps... +[2025-04-27 16:38:51] Saved checkpoint to checkpoint/X2I_UltraEdiit_fourmask_LLava_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_filtered_objects_4mask_1280_context/003-GPT-XL/checkpoints/0064000.pt +[2025-04-27 16:38:53] Removed old checkpoint: checkpoint/X2I_UltraEdiit_fourmask_LLava_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_filtered_objects_4mask_1280_context/003-GPT-XL/checkpoints/0062000.pt +[2025-04-27 16:39:16] (step=0064025) Train Loss: 5.8236, Train Steps/Sec: 0.06 +[2025-04-27 16:39:38] (step=0064050) Train Loss: 5.7731, Train Steps/Sec: 1.12 +[2025-04-27 16:40:00] (step=0064075) Train Loss: 5.7783, Train Steps/Sec: 1.12 +[2025-04-27 16:40:22] (step=0064100) Train Loss: 5.8387, Train Steps/Sec: 1.12 +[2025-04-27 16:40:45] (step=0064125) Train Loss: 5.7503, Train Steps/Sec: 1.12 +[2025-04-27 16:41:07] (step=0064150) Train Loss: 5.7619, Train Steps/Sec: 1.12 +[2025-04-27 16:41:30] (step=0064175) Train Loss: 5.7831, Train Steps/Sec: 1.12 +[2025-04-27 16:41:52] (step=0064200) Train Loss: 5.8300, Train Steps/Sec: 1.12 +[2025-04-27 16:42:14] (step=0064225) Train Loss: 5.7570, Train Steps/Sec: 1.12 +[2025-04-27 16:42:36] (step=0064250) Train Loss: 5.8014, Train Steps/Sec: 1.12 +[2025-04-27 16:42:59] (step=0064275) Train Loss: 5.7853, Train Steps/Sec: 1.12 +[2025-04-27 16:43:22] (step=0064300) Train Loss: 5.7902, Train Steps/Sec: 1.09 +[2025-04-27 16:43:44] (step=0064325) Train Loss: 5.8110, Train Steps/Sec: 1.12 +[2025-04-27 16:44:06] (step=0064350) Train Loss: 5.7959, Train Steps/Sec: 1.12 +[2025-04-27 16:44:29] (step=0064375) Train Loss: 5.7330, Train Steps/Sec: 1.12 +[2025-04-27 16:44:51] (step=0064400) Train Loss: 5.7844, Train Steps/Sec: 1.12 +[2025-04-27 16:45:13] (step=0064425) Train Loss: 5.7352, Train Steps/Sec: 1.12 +[2025-04-27 16:45:36] (step=0064450) Train Loss: 5.7401, Train Steps/Sec: 1.12 +[2025-04-27 16:45:58] (step=0064475) Train Loss: 5.8244, Train Steps/Sec: 1.12 +[2025-04-27 16:46:20] (step=0064500) Train Loss: 5.8563, Train Steps/Sec: 1.12 +[2025-04-27 16:46:42] (step=0064525) Train Loss: 5.7654, Train Steps/Sec: 1.12 +[2025-04-27 16:47:05] (step=0064550) Train Loss: 5.7478, Train Steps/Sec: 1.12 +[2025-04-27 16:47:27] (step=0064575) Train Loss: 5.7938, Train Steps/Sec: 1.12 +[2025-04-27 16:47:49] (step=0064600) Train Loss: 5.7252, Train Steps/Sec: 1.12 +[2025-04-27 16:48:12] (step=0064625) Train Loss: 5.6685, Train Steps/Sec: 1.12 +[2025-04-27 16:48:34] (step=0064650) Train Loss: 5.7458, Train Steps/Sec: 1.12 +[2025-04-27 16:48:56] (step=0064675) Train Loss: 5.7520, Train Steps/Sec: 1.12 +[2025-04-27 16:49:19] (step=0064700) Train Loss: 5.7685, Train Steps/Sec: 1.12 +[2025-04-27 16:49:41] (step=0064725) Train Loss: 5.7923, Train Steps/Sec: 1.12 +[2025-04-27 16:50:03] (step=0064750) Train Loss: 5.8430, Train Steps/Sec: 1.12 +[2025-04-27 16:50:25] (step=0064775) Train Loss: 5.7164, Train Steps/Sec: 1.13 +[2025-04-27 16:50:48] (step=0064800) Train Loss: 5.7685, Train Steps/Sec: 1.12 +[2025-04-27 16:51:10] (step=0064825) Train Loss: 5.7588, Train Steps/Sec: 1.12 +[2025-04-27 16:51:32] (step=0064850) Train Loss: 5.8585, Train Steps/Sec: 1.12 +[2025-04-27 16:51:55] (step=0064875) Train Loss: 5.7571, Train Steps/Sec: 1.12 +[2025-04-27 16:52:17] (step=0064900) Train Loss: 5.7059, Train Steps/Sec: 1.12 +[2025-04-27 16:52:39] (step=0064925) Train Loss: 5.7625, Train Steps/Sec: 1.12 +[2025-04-27 16:53:01] (step=0064950) Train Loss: 5.7666, Train Steps/Sec: 1.12 +[2025-04-27 16:53:24] (step=0064975) Train Loss: 5.6663, Train Steps/Sec: 1.12 +[2025-04-27 16:53:46] (step=0065000) Train Loss: 5.7398, Train Steps/Sec: 1.12 +[2025-04-27 16:54:08] (step=0065025) Train Loss: 5.7139, Train Steps/Sec: 1.12 +[2025-04-27 16:54:31] (step=0065050) Train Loss: 5.7711, Train Steps/Sec: 1.12 +[2025-04-27 16:54:53] (step=0065075) Train Loss: 5.7995, Train Steps/Sec: 1.12 +[2025-04-27 16:55:15] (step=0065100) Train Loss: 5.7179, Train Steps/Sec: 1.12 +[2025-04-27 16:55:38] (step=0065125) Train Loss: 5.7962, Train Steps/Sec: 1.12 +[2025-04-27 16:56:00] (step=0065150) Train Loss: 5.7755, Train Steps/Sec: 1.12 +[2025-04-27 16:56:22] (step=0065175) Train Loss: 5.7072, Train Steps/Sec: 1.12 +[2025-04-27 16:56:45] (step=0065200) Train Loss: 5.7031, Train Steps/Sec: 1.12 +[2025-04-27 16:57:07] (step=0065225) Train Loss: 5.7400, Train Steps/Sec: 1.12 +[2025-04-27 16:57:29] (step=0065250) Train Loss: 5.7622, Train Steps/Sec: 1.12 +[2025-04-27 16:57:51] (step=0065275) Train Loss: 5.7448, Train Steps/Sec: 1.12 +[2025-04-27 16:58:14] (step=0065300) Train Loss: 5.7653, Train Steps/Sec: 1.12 +[2025-04-27 16:58:36] (step=0065325) Train Loss: 5.8096, Train Steps/Sec: 1.12 +[2025-04-27 16:58:58] (step=0065350) Train Loss: 5.7688, Train Steps/Sec: 1.12 +[2025-04-27 16:59:21] (step=0065375) Train Loss: 5.7737, Train Steps/Sec: 1.12 +[2025-04-27 16:59:43] (step=0065400) Train Loss: 5.7429, Train Steps/Sec: 1.12 +[2025-04-27 17:00:05] (step=0065425) Train Loss: 5.8250, Train Steps/Sec: 1.12 +[2025-04-27 17:00:28] (step=0065450) Train Loss: 5.7000, Train Steps/Sec: 1.12 +[2025-04-27 17:00:50] (step=0065475) Train Loss: 5.8099, Train Steps/Sec: 1.12 +[2025-04-27 17:01:12] (step=0065500) Train Loss: 5.8023, Train Steps/Sec: 1.12 +[2025-04-27 17:01:35] (step=0065525) Train Loss: 5.8064, Train Steps/Sec: 1.12 +[2025-04-27 17:01:57] (step=0065550) Train Loss: 5.7611, Train Steps/Sec: 1.12 +[2025-04-27 17:02:19] (step=0065575) Train Loss: 5.7641, Train Steps/Sec: 1.12 +[2025-04-27 17:02:41] (step=0065600) Train Loss: 5.7722, Train Steps/Sec: 1.12 +[2025-04-27 17:03:04] (step=0065625) Train Loss: 5.7229, Train Steps/Sec: 1.12 +[2025-04-27 17:03:26] (step=0065650) Train Loss: 5.7628, Train Steps/Sec: 1.12 +[2025-04-27 17:03:48] (step=0065675) Train Loss: 5.7431, Train Steps/Sec: 1.12 +[2025-04-27 17:04:11] (step=0065700) Train Loss: 5.6945, Train Steps/Sec: 1.12 +[2025-04-27 17:04:33] (step=0065725) Train Loss: 5.8078, Train Steps/Sec: 1.12 +[2025-04-27 17:04:55] (step=0065750) Train Loss: 5.7606, Train Steps/Sec: 1.12 +[2025-04-27 17:05:17] (step=0065775) Train Loss: 5.7167, Train Steps/Sec: 1.12 +[2025-04-27 17:05:40] (step=0065800) Train Loss: 5.7902, Train Steps/Sec: 1.12 +[2025-04-27 17:06:02] (step=0065825) Train Loss: 5.7732, Train Steps/Sec: 1.11 +[2025-04-27 17:06:25] (step=0065850) Train Loss: 5.7592, Train Steps/Sec: 1.12 +[2025-04-27 17:06:47] (step=0065875) Train Loss: 5.6994, Train Steps/Sec: 1.12 +[2025-04-27 17:07:09] (step=0065900) Train Loss: 5.7150, Train Steps/Sec: 1.12 +[2025-04-27 17:07:31] (step=0065925) Train Loss: 5.7667, Train Steps/Sec: 1.12 +[2025-04-27 17:07:54] (step=0065950) Train Loss: 5.7466, Train Steps/Sec: 1.12 +[2025-04-27 17:08:16] (step=0065975) Train Loss: 5.7648, Train Steps/Sec: 1.12 +[2025-04-27 17:08:38] (step=0066000) Train Loss: 5.7544, Train Steps/Sec: 1.12 +[2025-04-27 17:08:38] text_config is None. Initializing the text config with default values (`OPTConfig`). +[2025-04-27 17:14:58] Finish Eval in 66000 steps... +[2025-04-27 17:15:18] Saved checkpoint to checkpoint/X2I_UltraEdiit_fourmask_LLava_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_filtered_objects_4mask_1280_context/003-GPT-XL/checkpoints/0066000.pt +[2025-04-27 17:15:20] Removed old checkpoint: checkpoint/X2I_UltraEdiit_fourmask_LLava_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_filtered_objects_4mask_1280_context/003-GPT-XL/checkpoints/0064000.pt +[2025-04-27 17:15:43] (step=0066025) Train Loss: 5.7493, Train Steps/Sec: 0.06 +[2025-04-27 17:16:05] (step=0066050) Train Loss: 5.7910, Train Steps/Sec: 1.12 +[2025-04-27 17:16:28] (step=0066075) Train Loss: 5.7336, Train Steps/Sec: 1.12 +[2025-04-27 17:16:50] (step=0066100) Train Loss: 5.7509, Train Steps/Sec: 1.12 +[2025-04-27 17:17:12] (step=0066125) Train Loss: 5.7552, Train Steps/Sec: 1.12 +[2025-04-27 17:17:34] (step=0066150) Train Loss: 5.6799, Train Steps/Sec: 1.12 +[2025-04-27 17:17:57] (step=0066175) Train Loss: 5.7296, Train Steps/Sec: 1.12 +[2025-04-27 17:18:19] (step=0066200) Train Loss: 5.7656, Train Steps/Sec: 1.12 +[2025-04-27 17:18:42] (step=0066225) Train Loss: 5.7875, Train Steps/Sec: 1.12 +[2025-04-27 17:19:04] (step=0066250) Train Loss: 5.7311, Train Steps/Sec: 1.12 +[2025-04-27 17:19:26] (step=0066275) Train Loss: 5.7400, Train Steps/Sec: 1.12 +[2025-04-27 17:19:49] (step=0066300) Train Loss: 5.7605, Train Steps/Sec: 1.12 +[2025-04-27 17:20:11] (step=0066325) Train Loss: 5.8007, Train Steps/Sec: 1.12 +[2025-04-27 17:20:33] (step=0066350) Train Loss: 5.7957, Train Steps/Sec: 1.12 +[2025-04-27 17:20:56] (step=0066375) Train Loss: 5.7241, Train Steps/Sec: 1.12 +[2025-04-27 17:21:18] (step=0066400) Train Loss: 5.7466, Train Steps/Sec: 1.12 +[2025-04-27 17:21:40] (step=0066425) Train Loss: 5.8021, Train Steps/Sec: 1.12 +[2025-04-27 17:22:03] (step=0066450) Train Loss: 5.7209, Train Steps/Sec: 1.12 +[2025-04-27 17:22:25] (step=0066475) Train Loss: 5.7719, Train Steps/Sec: 1.12 +[2025-04-27 17:22:47] (step=0066500) Train Loss: 5.7879, Train Steps/Sec: 1.12 +[2025-04-27 17:23:10] (step=0066525) Train Loss: 5.7381, Train Steps/Sec: 1.12 +[2025-04-27 17:23:32] (step=0066550) Train Loss: 5.7753, Train Steps/Sec: 1.12 +[2025-04-27 17:23:59] (step=0066575) Train Loss: 5.7429, Train Steps/Sec: 0.93 +[2025-04-27 17:24:21] (step=0066600) Train Loss: 5.7274, Train Steps/Sec: 1.12 +[2025-04-27 17:24:43] (step=0066625) Train Loss: 5.7660, Train Steps/Sec: 1.12 +[2025-04-27 17:25:06] (step=0066650) Train Loss: 5.7113, Train Steps/Sec: 1.12 +[2025-04-27 17:25:28] (step=0066675) Train Loss: 5.7964, Train Steps/Sec: 1.12 +[2025-04-27 17:25:50] (step=0066700) Train Loss: 5.7528, Train Steps/Sec: 1.12 +[2025-04-27 17:26:12] (step=0066725) Train Loss: 5.7495, Train Steps/Sec: 1.12 +[2025-04-27 17:26:40] (step=0066750) Train Loss: 5.7939, Train Steps/Sec: 0.93 +[2025-04-27 17:27:02] (step=0066775) Train Loss: 5.7285, Train Steps/Sec: 1.12 +[2025-04-27 17:27:24] (step=0066800) Train Loss: 5.7746, Train Steps/Sec: 1.12 +[2025-04-27 17:27:47] (step=0066825) Train Loss: 5.7017, Train Steps/Sec: 1.12 +[2025-04-27 17:28:09] (step=0066850) Train Loss: 5.7074, Train Steps/Sec: 1.12 +[2025-04-27 17:28:31] (step=0066875) Train Loss: 5.8048, Train Steps/Sec: 1.12 +[2025-04-27 17:28:54] (step=0066900) Train Loss: 5.7790, Train Steps/Sec: 1.12 +[2025-04-27 17:29:16] (step=0066925) Train Loss: 5.8089, Train Steps/Sec: 1.12 +[2025-04-27 17:29:38] (step=0066950) Train Loss: 5.7355, Train Steps/Sec: 1.12 +[2025-04-27 17:30:00] (step=0066975) Train Loss: 5.7345, Train Steps/Sec: 1.12 +[2025-04-27 17:30:23] (step=0067000) Train Loss: 5.7640, Train Steps/Sec: 1.12 +[2025-04-27 17:30:45] (step=0067025) Train Loss: 5.7251, Train Steps/Sec: 1.12 +[2025-04-27 17:31:07] (step=0067050) Train Loss: 5.7479, Train Steps/Sec: 1.12 +[2025-04-27 17:31:30] (step=0067075) Train Loss: 5.7227, Train Steps/Sec: 1.12 +[2025-04-27 17:31:52] (step=0067100) Train Loss: 5.7378, Train Steps/Sec: 1.12 +[2025-04-27 17:32:14] (step=0067125) Train Loss: 5.7598, Train Steps/Sec: 1.12 +[2025-04-27 17:32:37] (step=0067150) Train Loss: 5.6923, Train Steps/Sec: 1.12 +[2025-04-27 17:32:59] (step=0067175) Train Loss: 5.7084, Train Steps/Sec: 1.12 +[2025-04-27 17:33:21] (step=0067200) Train Loss: 5.7806, Train Steps/Sec: 1.12 +[2025-04-27 17:33:44] (step=0067225) Train Loss: 5.7828, Train Steps/Sec: 1.12 +[2025-04-27 17:34:06] (step=0067250) Train Loss: 5.7608, Train Steps/Sec: 1.12 +[2025-04-27 17:34:28] (step=0067275) Train Loss: 5.7505, Train Steps/Sec: 1.12 +[2025-04-27 17:34:55] (step=0067300) Train Loss: 5.7716, Train Steps/Sec: 0.96 +[2025-04-27 17:35:26] (step=0067325) Train Loss: 5.7723, Train Steps/Sec: 0.81 +[2025-04-27 17:35:53] (step=0067350) Train Loss: 5.7964, Train Steps/Sec: 0.91 +[2025-04-27 17:36:15] (step=0067375) Train Loss: 5.8184, Train Steps/Sec: 1.12 +[2025-04-27 17:36:38] (step=0067400) Train Loss: 5.7927, Train Steps/Sec: 1.12 +[2025-04-27 17:37:00] (step=0067425) Train Loss: 5.7378, Train Steps/Sec: 1.12 +[2025-04-27 17:37:27] (step=0067450) Train Loss: 5.7308, Train Steps/Sec: 0.91 +[2025-04-27 17:37:50] (step=0067475) Train Loss: 5.7467, Train Steps/Sec: 1.12 +[2025-04-27 17:38:12] (step=0067500) Train Loss: 5.7464, Train Steps/Sec: 1.12 +[2025-04-27 17:38:34] (step=0067525) Train Loss: 5.7944, Train Steps/Sec: 1.12 +[2025-04-27 17:38:57] (step=0067550) Train Loss: 5.7935, Train Steps/Sec: 1.12 +[2025-04-27 17:39:19] (step=0067575) Train Loss: 5.7861, Train Steps/Sec: 1.12 +[2025-04-27 17:39:41] (step=0067600) Train Loss: 5.6957, Train Steps/Sec: 1.12 +[2025-04-27 17:40:04] (step=0067625) Train Loss: 5.7342, Train Steps/Sec: 1.12 +[2025-04-27 17:40:26] (step=0067650) Train Loss: 5.7345, Train Steps/Sec: 1.12 +[2025-04-27 17:40:48] (step=0067675) Train Loss: 5.8059, Train Steps/Sec: 1.12 +[2025-04-27 17:41:10] (step=0067700) Train Loss: 5.6982, Train Steps/Sec: 1.12 +[2025-04-27 17:41:33] (step=0067725) Train Loss: 5.7323, Train Steps/Sec: 1.12 +[2025-04-27 17:41:55] (step=0067750) Train Loss: 5.7328, Train Steps/Sec: 1.12 +[2025-04-27 17:42:17] (step=0067775) Train Loss: 5.7475, Train Steps/Sec: 1.12 +[2025-04-27 17:42:40] (step=0067800) Train Loss: 5.7930, Train Steps/Sec: 1.12 +[2025-04-27 17:43:02] (step=0067825) Train Loss: 5.7570, Train Steps/Sec: 1.12 +[2025-04-27 17:43:24] (step=0067850) Train Loss: 5.7218, Train Steps/Sec: 1.12 +[2025-04-27 17:43:46] (step=0067875) Train Loss: 5.7603, Train Steps/Sec: 1.12 +[2025-04-27 17:44:09] (step=0067900) Train Loss: 5.7319, Train Steps/Sec: 1.12 +[2025-04-27 17:44:31] (step=0067925) Train Loss: 5.7260, Train Steps/Sec: 1.12 +[2025-04-27 17:44:53] (step=0067950) Train Loss: 5.7465, Train Steps/Sec: 1.12 +[2025-04-27 17:45:16] (step=0067975) Train Loss: 5.7541, Train Steps/Sec: 1.12 +[2025-04-27 17:45:38] (step=0068000) Train Loss: 5.8193, Train Steps/Sec: 1.12 +[2025-04-27 17:45:38] text_config is None. Initializing the text config with default values (`OPTConfig`). +[2025-04-27 17:51:58] Finish Eval in 68000 steps... +[2025-04-27 17:52:18] Saved checkpoint to checkpoint/X2I_UltraEdiit_fourmask_LLava_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_filtered_objects_4mask_1280_context/003-GPT-XL/checkpoints/0068000.pt +[2025-04-27 17:52:20] Removed old checkpoint: checkpoint/X2I_UltraEdiit_fourmask_LLava_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_filtered_objects_4mask_1280_context/003-GPT-XL/checkpoints/0066000.pt +[2025-04-27 17:52:43] (step=0068025) Train Loss: 5.7897, Train Steps/Sec: 0.06 +[2025-04-27 17:53:05] (step=0068050) Train Loss: 5.7264, Train Steps/Sec: 1.12 +[2025-04-27 17:53:27] (step=0068075) Train Loss: 5.7102, Train Steps/Sec: 1.12 +[2025-04-27 17:53:50] (step=0068100) Train Loss: 5.8192, Train Steps/Sec: 1.12 +[2025-04-27 17:54:12] (step=0068125) Train Loss: 5.7023, Train Steps/Sec: 1.12 +[2025-04-27 17:54:35] (step=0068150) Train Loss: 5.7768, Train Steps/Sec: 1.12 +[2025-04-27 17:54:57] (step=0068175) Train Loss: 5.7650, Train Steps/Sec: 1.12 +[2025-04-27 17:55:19] (step=0068200) Train Loss: 5.7337, Train Steps/Sec: 1.12 +[2025-04-27 17:55:42] (step=0068225) Train Loss: 5.7147, Train Steps/Sec: 1.12 +[2025-04-27 17:56:04] (step=0068250) Train Loss: 5.6666, Train Steps/Sec: 1.12 +[2025-04-27 17:56:26] (step=0068275) Train Loss: 5.7900, Train Steps/Sec: 1.12 +[2025-04-27 17:56:48] (step=0068300) Train Loss: 5.7424, Train Steps/Sec: 1.13 +[2025-04-27 17:57:11] (step=0068325) Train Loss: 5.7320, Train Steps/Sec: 1.12 +[2025-04-27 17:57:33] (step=0068350) Train Loss: 5.7284, Train Steps/Sec: 1.12 +[2025-04-27 17:57:55] (step=0068375) Train Loss: 5.7486, Train Steps/Sec: 1.12 +[2025-04-27 17:58:17] (step=0068400) Train Loss: 5.6980, Train Steps/Sec: 1.12 +[2025-04-27 17:58:40] (step=0068425) Train Loss: 5.7812, Train Steps/Sec: 1.12 +[2025-04-27 17:59:02] (step=0068450) Train Loss: 5.7079, Train Steps/Sec: 1.12 +[2025-04-27 17:59:24] (step=0068475) Train Loss: 5.7727, Train Steps/Sec: 1.12 +[2025-04-27 17:59:46] (step=0068500) Train Loss: 5.7567, Train Steps/Sec: 1.12 +[2025-04-27 18:00:09] (step=0068525) Train Loss: 5.7080, Train Steps/Sec: 1.12 +[2025-04-27 18:00:31] (step=0068550) Train Loss: 5.7637, Train Steps/Sec: 1.12 +[2025-04-27 18:00:53] (step=0068575) Train Loss: 5.7026, Train Steps/Sec: 1.12 +[2025-04-27 18:01:16] (step=0068600) Train Loss: 5.8155, Train Steps/Sec: 1.12 +[2025-04-27 18:01:38] (step=0068625) Train Loss: 5.7846, Train Steps/Sec: 1.12 +[2025-04-27 18:02:00] (step=0068650) Train Loss: 5.7280, Train Steps/Sec: 1.13 +[2025-04-27 18:02:22] (step=0068675) Train Loss: 5.7887, Train Steps/Sec: 1.12 +[2025-04-27 18:02:45] (step=0068700) Train Loss: 5.7409, Train Steps/Sec: 1.12 +[2025-04-27 18:03:07] (step=0068725) Train Loss: 5.6965, Train Steps/Sec: 1.12 +[2025-04-27 18:03:29] (step=0068750) Train Loss: 5.7537, Train Steps/Sec: 1.12 +[2025-04-27 18:03:52] (step=0068775) Train Loss: 5.6965, Train Steps/Sec: 1.12 +[2025-04-27 18:04:14] (step=0068800) Train Loss: 5.7998, Train Steps/Sec: 1.12 +[2025-04-27 18:04:36] (step=0068825) Train Loss: 5.6756, Train Steps/Sec: 1.12 +[2025-04-27 18:04:58] (step=0068850) Train Loss: 5.7876, Train Steps/Sec: 1.12 +[2025-04-27 18:05:21] (step=0068875) Train Loss: 5.8342, Train Steps/Sec: 1.12 +[2025-04-27 18:05:43] (step=0068900) Train Loss: 5.7272, Train Steps/Sec: 1.13 +[2025-04-27 18:06:05] (step=0068925) Train Loss: 5.7128, Train Steps/Sec: 1.13 +[2025-04-27 18:06:27] (step=0068950) Train Loss: 5.7007, Train Steps/Sec: 1.12 +[2025-04-27 18:06:50] (step=0068975) Train Loss: 5.8050, Train Steps/Sec: 1.13 +[2025-04-27 18:07:12] (step=0069000) Train Loss: 5.7134, Train Steps/Sec: 1.12 +[2025-04-27 18:07:34] (step=0069025) Train Loss: 5.8004, Train Steps/Sec: 1.13 +[2025-04-27 18:07:56] (step=0069050) Train Loss: 5.7390, Train Steps/Sec: 1.12 +[2025-04-27 18:08:19] (step=0069075) Train Loss: 5.7418, Train Steps/Sec: 1.12 +[2025-04-27 18:08:41] (step=0069100) Train Loss: 5.8079, Train Steps/Sec: 1.12 +[2025-04-27 18:09:03] (step=0069125) Train Loss: 5.7736, Train Steps/Sec: 1.13 +[2025-04-27 18:09:25] (step=0069150) Train Loss: 5.7290, Train Steps/Sec: 1.12 +[2025-04-27 18:09:48] (step=0069175) Train Loss: 5.7348, Train Steps/Sec: 1.12 +[2025-04-27 18:10:10] (step=0069200) Train Loss: 5.7389, Train Steps/Sec: 1.12 +[2025-04-27 18:10:32] (step=0069225) Train Loss: 5.6673, Train Steps/Sec: 1.13 +[2025-04-27 18:10:55] (step=0069250) Train Loss: 5.7965, Train Steps/Sec: 1.12 +[2025-04-27 18:11:17] (step=0069275) Train Loss: 5.7417, Train Steps/Sec: 1.12 +[2025-04-27 18:11:39] (step=0069300) Train Loss: 5.7141, Train Steps/Sec: 1.12 +[2025-04-27 18:12:01] (step=0069325) Train Loss: 5.7526, Train Steps/Sec: 1.12 +[2025-04-27 18:12:24] (step=0069350) Train Loss: 5.8276, Train Steps/Sec: 1.13 +[2025-04-27 18:12:46] (step=0069375) Train Loss: 5.7238, Train Steps/Sec: 1.12 +[2025-04-27 18:13:08] (step=0069400) Train Loss: 5.7763, Train Steps/Sec: 1.12 +[2025-04-27 18:13:31] (step=0069425) Train Loss: 5.7821, Train Steps/Sec: 1.12 +[2025-04-27 18:13:53] (step=0069450) Train Loss: 5.7973, Train Steps/Sec: 1.12 +[2025-04-27 18:14:15] (step=0069475) Train Loss: 5.7290, Train Steps/Sec: 1.12 +[2025-04-27 18:14:37] (step=0069500) Train Loss: 5.7711, Train Steps/Sec: 1.12 +[2025-04-27 18:15:00] (step=0069525) Train Loss: 5.7610, Train Steps/Sec: 1.13 +[2025-04-27 18:15:22] (step=0069550) Train Loss: 5.7397, Train Steps/Sec: 1.12 +[2025-04-27 18:15:44] (step=0069575) Train Loss: 5.7663, Train Steps/Sec: 1.12 +[2025-04-27 18:16:07] (step=0069600) Train Loss: 5.7835, Train Steps/Sec: 1.12 +[2025-04-27 18:16:29] (step=0069625) Train Loss: 5.6902, Train Steps/Sec: 1.12 +[2025-04-27 18:16:51] (step=0069650) Train Loss: 5.7882, Train Steps/Sec: 1.12 +[2025-04-27 18:17:13] (step=0069675) Train Loss: 5.6815, Train Steps/Sec: 1.12 +[2025-04-27 18:17:36] (step=0069700) Train Loss: 5.7482, Train Steps/Sec: 1.12 +[2025-04-27 18:17:58] (step=0069725) Train Loss: 5.7828, Train Steps/Sec: 1.12 +[2025-04-27 18:18:20] (step=0069750) Train Loss: 5.7409, Train Steps/Sec: 1.12 +[2025-04-27 18:18:43] (step=0069775) Train Loss: 5.6648, Train Steps/Sec: 1.12 +[2025-04-27 18:19:05] (step=0069800) Train Loss: 5.7995, Train Steps/Sec: 1.11 +[2025-04-27 18:19:27] (step=0069825) Train Loss: 5.7686, Train Steps/Sec: 1.12 +[2025-04-27 18:19:50] (step=0069850) Train Loss: 5.7451, Train Steps/Sec: 1.12 +[2025-04-27 18:20:12] (step=0069875) Train Loss: 5.7720, Train Steps/Sec: 1.12 +[2025-04-27 18:20:34] (step=0069900) Train Loss: 5.7113, Train Steps/Sec: 1.12 +[2025-04-27 18:20:56] (step=0069925) Train Loss: 5.6982, Train Steps/Sec: 1.12 +[2025-04-27 18:21:19] (step=0069950) Train Loss: 5.7595, Train Steps/Sec: 1.12 +[2025-04-27 18:21:41] (step=0069975) Train Loss: 5.7736, Train Steps/Sec: 1.12 +[2025-04-27 18:22:03] (step=0070000) Train Loss: 5.7053, Train Steps/Sec: 1.12 +[2025-04-27 18:22:03] text_config is None. Initializing the text config with default values (`OPTConfig`). +[2025-04-27 18:28:22] Finish Eval in 70000 steps... +[2025-04-27 18:28:43] Saved checkpoint to checkpoint/X2I_UltraEdiit_fourmask_LLava_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_filtered_objects_4mask_1280_context/003-GPT-XL/checkpoints/0070000.pt +[2025-04-27 18:28:45] Removed old checkpoint: checkpoint/X2I_UltraEdiit_fourmask_LLava_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_filtered_objects_4mask_1280_context/003-GPT-XL/checkpoints/0068000.pt +[2025-04-27 18:29:08] (step=0070025) Train Loss: 5.7028, Train Steps/Sec: 0.06 +[2025-04-27 18:29:30] (step=0070050) Train Loss: 5.8009, Train Steps/Sec: 1.12 +[2025-04-27 18:29:52] (step=0070075) Train Loss: 5.7069, Train Steps/Sec: 1.12 +[2025-04-27 18:30:14] (step=0070100) Train Loss: 5.7105, Train Steps/Sec: 1.12 +[2025-04-27 18:30:37] (step=0070125) Train Loss: 5.8008, Train Steps/Sec: 1.12 +[2025-04-27 18:30:59] (step=0070150) Train Loss: 5.7598, Train Steps/Sec: 1.12 +[2025-04-27 18:31:21] (step=0070175) Train Loss: 5.6955, Train Steps/Sec: 1.12 +[2025-04-27 18:31:44] (step=0070200) Train Loss: 5.7582, Train Steps/Sec: 1.12 +[2025-04-27 18:32:06] (step=0070225) Train Loss: 5.7420, Train Steps/Sec: 1.12 +[2025-04-27 18:32:28] (step=0070250) Train Loss: 5.7596, Train Steps/Sec: 1.12 +[2025-04-27 18:32:50] (step=0070275) Train Loss: 5.7119, Train Steps/Sec: 1.12 +[2025-04-27 18:33:13] (step=0070300) Train Loss: 5.7568, Train Steps/Sec: 1.12 +[2025-04-27 18:33:35] (step=0070325) Train Loss: 5.7608, Train Steps/Sec: 1.12 +[2025-04-27 18:33:57] (step=0070350) Train Loss: 5.7527, Train Steps/Sec: 1.12 +[2025-04-27 18:34:20] (step=0070375) Train Loss: 5.7280, Train Steps/Sec: 1.12 +[2025-04-27 18:34:42] (step=0070400) Train Loss: 5.7472, Train Steps/Sec: 1.12 +[2025-04-27 18:35:04] (step=0070425) Train Loss: 5.8042, Train Steps/Sec: 1.12 +[2025-04-27 18:35:27] (step=0070450) Train Loss: 5.7715, Train Steps/Sec: 1.12 +[2025-04-27 18:35:49] (step=0070475) Train Loss: 5.7702, Train Steps/Sec: 1.12 +[2025-04-27 18:36:11] (step=0070500) Train Loss: 5.8133, Train Steps/Sec: 1.12 +[2025-04-27 18:36:34] (step=0070525) Train Loss: 5.7727, Train Steps/Sec: 1.12 +[2025-04-27 18:36:56] (step=0070550) Train Loss: 5.6694, Train Steps/Sec: 1.12 +[2025-04-27 18:37:18] (step=0070575) Train Loss: 5.7688, Train Steps/Sec: 1.12 +[2025-04-27 18:37:41] (step=0070600) Train Loss: 5.7314, Train Steps/Sec: 1.12 +[2025-04-27 18:38:03] (step=0070625) Train Loss: 5.8011, Train Steps/Sec: 1.12 +[2025-04-27 18:38:25] (step=0070650) Train Loss: 5.8194, Train Steps/Sec: 1.12 +[2025-04-27 18:38:48] (step=0070675) Train Loss: 5.7389, Train Steps/Sec: 1.12 +[2025-04-27 18:39:10] (step=0070700) Train Loss: 5.7040, Train Steps/Sec: 1.12 +[2025-04-27 18:39:32] (step=0070725) Train Loss: 5.7279, Train Steps/Sec: 1.12 +[2025-04-27 18:39:54] (step=0070750) Train Loss: 5.7617, Train Steps/Sec: 1.12 +[2025-04-27 18:40:17] (step=0070775) Train Loss: 5.7676, Train Steps/Sec: 1.12 +[2025-04-27 18:40:39] (step=0070800) Train Loss: 5.7442, Train Steps/Sec: 1.12 +[2025-04-27 18:41:06] (step=0070825) Train Loss: 5.7646, Train Steps/Sec: 0.94 +[2025-04-27 18:41:28] (step=0070850) Train Loss: 5.7242, Train Steps/Sec: 1.12 +[2025-04-27 18:41:50] (step=0070875) Train Loss: 5.7253, Train Steps/Sec: 1.12 +[2025-04-27 18:42:13] (step=0070900) Train Loss: 5.7293, Train Steps/Sec: 1.12 +[2025-04-27 18:42:35] (step=0070925) Train Loss: 5.7248, Train Steps/Sec: 1.12 +[2025-04-27 18:42:57] (step=0070950) Train Loss: 5.7474, Train Steps/Sec: 1.12 +[2025-04-27 18:43:20] (step=0070975) Train Loss: 5.7027, Train Steps/Sec: 1.12 +[2025-04-27 18:43:42] (step=0071000) Train Loss: 5.6776, Train Steps/Sec: 1.12 +[2025-04-27 18:44:09] (step=0071025) Train Loss: 5.7592, Train Steps/Sec: 0.92 +[2025-04-27 18:44:32] (step=0071050) Train Loss: 5.6859, Train Steps/Sec: 1.12 +[2025-04-27 18:44:54] (step=0071075) Train Loss: 5.7517, Train Steps/Sec: 1.12 +[2025-04-27 18:45:16] (step=0071100) Train Loss: 5.7736, Train Steps/Sec: 1.12 +[2025-04-27 18:45:38] (step=0071125) Train Loss: 5.7502, Train Steps/Sec: 1.12 +[2025-04-27 18:46:01] (step=0071150) Train Loss: 5.7146, Train Steps/Sec: 1.12 +[2025-04-27 18:46:23] (step=0071175) Train Loss: 5.6955, Train Steps/Sec: 1.12 +[2025-04-27 18:46:45] (step=0071200) Train Loss: 5.6876, Train Steps/Sec: 1.11 +[2025-04-27 18:47:08] (step=0071225) Train Loss: 5.7820, Train Steps/Sec: 1.12 +[2025-04-27 18:47:30] (step=0071250) Train Loss: 5.7298, Train Steps/Sec: 1.12 +[2025-04-27 18:47:52] (step=0071275) Train Loss: 5.7708, Train Steps/Sec: 1.12 +[2025-04-27 18:48:15] (step=0071300) Train Loss: 5.7604, Train Steps/Sec: 1.12 +[2025-04-27 18:48:37] (step=0071325) Train Loss: 5.7145, Train Steps/Sec: 1.12 +[2025-04-27 18:48:59] (step=0071350) Train Loss: 5.7147, Train Steps/Sec: 1.12 +[2025-04-27 18:49:21] (step=0071375) Train Loss: 5.7400, Train Steps/Sec: 1.12 +[2025-04-27 18:49:44] (step=0071400) Train Loss: 5.6741, Train Steps/Sec: 1.11 +[2025-04-27 18:50:06] (step=0071425) Train Loss: 5.7642, Train Steps/Sec: 1.12 +[2025-04-27 18:50:28] (step=0071450) Train Loss: 5.7298, Train Steps/Sec: 1.12 +[2025-04-27 18:50:51] (step=0071475) Train Loss: 5.7392, Train Steps/Sec: 1.12 +[2025-04-27 18:51:13] (step=0071500) Train Loss: 5.7823, Train Steps/Sec: 1.12 +[2025-04-27 18:51:35] (step=0071525) Train Loss: 5.6913, Train Steps/Sec: 1.12 +[2025-04-27 18:51:58] (step=0071550) Train Loss: 5.8002, Train Steps/Sec: 1.12 +[2025-04-27 18:52:20] (step=0071575) Train Loss: 5.7080, Train Steps/Sec: 1.12 +[2025-04-27 18:52:42] (step=0071600) Train Loss: 5.7649, Train Steps/Sec: 1.11 +[2025-04-27 18:53:08] (step=0071625) Train Loss: 5.7800, Train Steps/Sec: 0.96 +[2025-04-27 18:53:40] (step=0071650) Train Loss: 5.7631, Train Steps/Sec: 0.80 +[2025-04-27 18:54:02] (step=0071675) Train Loss: 5.6343, Train Steps/Sec: 1.12 +[2025-04-27 18:54:30] (step=0071700) Train Loss: 5.7202, Train Steps/Sec: 0.90 +[2025-04-27 18:54:52] (step=0071725) Train Loss: 5.7995, Train Steps/Sec: 1.12 +[2025-04-27 18:55:14] (step=0071750) Train Loss: 5.7438, Train Steps/Sec: 1.12 +[2025-04-27 18:55:36] (step=0071775) Train Loss: 5.7041, Train Steps/Sec: 1.12 +[2025-04-27 18:55:59] (step=0071800) Train Loss: 5.7477, Train Steps/Sec: 1.12 +[2025-04-27 18:56:21] (step=0071825) Train Loss: 5.7196, Train Steps/Sec: 1.12 +[2025-04-27 18:56:48] (step=0071850) Train Loss: 5.8030, Train Steps/Sec: 0.92 +[2025-04-27 18:57:11] (step=0071875) Train Loss: 5.7486, Train Steps/Sec: 1.12 +[2025-04-27 18:57:33] (step=0071900) Train Loss: 5.7364, Train Steps/Sec: 1.12 +[2025-04-27 18:57:55] (step=0071925) Train Loss: 5.7185, Train Steps/Sec: 1.12 +[2025-04-27 18:58:18] (step=0071950) Train Loss: 5.7453, Train Steps/Sec: 1.12 +[2025-04-27 18:58:40] (step=0071975) Train Loss: 5.7713, Train Steps/Sec: 1.12 +[2025-04-27 18:59:02] (step=0072000) Train Loss: 5.7667, Train Steps/Sec: 1.11 +[2025-04-27 18:59:02] text_config is None. Initializing the text config with default values (`OPTConfig`). +[2025-04-27 19:05:22] Finish Eval in 72000 steps... +[2025-04-27 19:05:41] Saved checkpoint to checkpoint/X2I_UltraEdiit_fourmask_LLava_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_filtered_objects_4mask_1280_context/003-GPT-XL/checkpoints/0072000.pt +[2025-04-27 19:05:43] Removed old checkpoint: checkpoint/X2I_UltraEdiit_fourmask_LLava_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_filtered_objects_4mask_1280_context/003-GPT-XL/checkpoints/0070000.pt +[2025-04-27 19:06:06] (step=0072025) Train Loss: 5.7760, Train Steps/Sec: 0.06 +[2025-04-27 19:06:28] (step=0072050) Train Loss: 5.7930, Train Steps/Sec: 1.12 +[2025-04-27 19:06:51] (step=0072075) Train Loss: 5.7698, Train Steps/Sec: 1.12 +[2025-04-27 19:07:13] (step=0072100) Train Loss: 5.7274, Train Steps/Sec: 1.12 +[2025-04-27 19:07:35] (step=0072125) Train Loss: 5.7864, Train Steps/Sec: 1.12 +[2025-04-27 19:07:57] (step=0072150) Train Loss: 5.7677, Train Steps/Sec: 1.12 +[2025-04-27 19:08:20] (step=0072175) Train Loss: 5.7257, Train Steps/Sec: 1.12 +[2025-04-27 19:08:42] (step=0072200) Train Loss: 5.6819, Train Steps/Sec: 1.12 +[2025-04-27 19:09:04] (step=0072225) Train Loss: 5.7252, Train Steps/Sec: 1.12 +[2025-04-27 19:09:27] (step=0072250) Train Loss: 5.6863, Train Steps/Sec: 1.12 +[2025-04-27 19:09:49] (step=0072275) Train Loss: 5.7034, Train Steps/Sec: 1.12 +[2025-04-27 19:10:11] (step=0072300) Train Loss: 5.7038, Train Steps/Sec: 1.12 +[2025-04-27 19:10:33] (step=0072325) Train Loss: 5.8282, Train Steps/Sec: 1.12 +[2025-04-27 19:10:56] (step=0072350) Train Loss: 5.7618, Train Steps/Sec: 1.12 +[2025-04-27 19:11:18] (step=0072375) Train Loss: 5.7333, Train Steps/Sec: 1.12 +[2025-04-27 19:11:40] (step=0072400) Train Loss: 5.7145, Train Steps/Sec: 1.12 +[2025-04-27 19:12:02] (step=0072425) Train Loss: 5.7108, Train Steps/Sec: 1.12 +[2025-04-27 19:12:25] (step=0072450) Train Loss: 5.7554, Train Steps/Sec: 1.12 +[2025-04-27 19:12:47] (step=0072475) Train Loss: 5.7535, Train Steps/Sec: 1.12 +[2025-04-27 19:13:09] (step=0072500) Train Loss: 5.7891, Train Steps/Sec: 1.12 +[2025-04-27 19:13:32] (step=0072525) Train Loss: 5.7557, Train Steps/Sec: 1.12 +[2025-04-27 19:13:54] (step=0072550) Train Loss: 5.6986, Train Steps/Sec: 1.12 +[2025-04-27 19:14:17] (step=0072575) Train Loss: 5.6911, Train Steps/Sec: 1.10 +[2025-04-27 19:14:39] (step=0072600) Train Loss: 5.7664, Train Steps/Sec: 1.12 +[2025-04-27 19:15:01] (step=0072625) Train Loss: 5.7254, Train Steps/Sec: 1.12 +[2025-04-27 19:15:23] (step=0072650) Train Loss: 5.7423, Train Steps/Sec: 1.12 +[2025-04-27 19:15:46] (step=0072675) Train Loss: 5.7783, Train Steps/Sec: 1.12 +[2025-04-27 19:16:08] (step=0072700) Train Loss: 5.7569, Train Steps/Sec: 1.12 +[2025-04-27 19:16:30] (step=0072725) Train Loss: 5.7509, Train Steps/Sec: 1.12 +[2025-04-27 19:16:53] (step=0072750) Train Loss: 5.6785, Train Steps/Sec: 1.12 +[2025-04-27 19:17:15] (step=0072775) Train Loss: 5.7755, Train Steps/Sec: 1.12 +[2025-04-27 19:17:37] (step=0072800) Train Loss: 5.6921, Train Steps/Sec: 1.12 +[2025-04-27 19:17:59] (step=0072825) Train Loss: 5.7098, Train Steps/Sec: 1.12 +[2025-04-27 19:18:22] (step=0072850) Train Loss: 5.7552, Train Steps/Sec: 1.12 +[2025-04-27 19:18:44] (step=0072875) Train Loss: 5.7057, Train Steps/Sec: 1.12 +[2025-04-27 19:19:06] (step=0072900) Train Loss: 5.7526, Train Steps/Sec: 1.12 +[2025-04-27 19:19:29] (step=0072925) Train Loss: 5.7120, Train Steps/Sec: 1.12 +[2025-04-27 19:19:51] (step=0072950) Train Loss: 5.7929, Train Steps/Sec: 1.12 +[2025-04-27 19:20:13] (step=0072975) Train Loss: 5.7757, Train Steps/Sec: 1.12 +[2025-04-27 19:20:36] (step=0073000) Train Loss: 5.6968, Train Steps/Sec: 1.12 +[2025-04-27 19:20:58] (step=0073025) Train Loss: 5.7167, Train Steps/Sec: 1.13 +[2025-04-27 19:21:20] (step=0073050) Train Loss: 5.7088, Train Steps/Sec: 1.12 +[2025-04-27 19:21:42] (step=0073075) Train Loss: 5.7270, Train Steps/Sec: 1.12 +[2025-04-27 19:22:05] (step=0073100) Train Loss: 5.6994, Train Steps/Sec: 1.11 +[2025-04-27 19:22:27] (step=0073125) Train Loss: 5.7565, Train Steps/Sec: 1.12 +[2025-04-27 19:22:49] (step=0073150) Train Loss: 5.7112, Train Steps/Sec: 1.12 +[2025-04-27 19:23:12] (step=0073175) Train Loss: 5.7605, Train Steps/Sec: 1.12 +[2025-04-27 19:23:34] (step=0073200) Train Loss: 5.7109, Train Steps/Sec: 1.11 +[2025-04-27 19:23:56] (step=0073225) Train Loss: 5.7892, Train Steps/Sec: 1.13 +[2025-04-27 19:24:19] (step=0073250) Train Loss: 5.7307, Train Steps/Sec: 1.12 +[2025-04-27 19:24:41] (step=0073275) Train Loss: 5.7243, Train Steps/Sec: 1.12 +[2025-04-27 19:25:03] (step=0073300) Train Loss: 5.7862, Train Steps/Sec: 1.12 +[2025-04-27 19:25:26] (step=0073325) Train Loss: 5.7194, Train Steps/Sec: 1.12 +[2025-04-27 19:25:48] (step=0073350) Train Loss: 5.8119, Train Steps/Sec: 1.12 +[2025-04-27 19:26:10] (step=0073375) Train Loss: 5.7047, Train Steps/Sec: 1.12 +[2025-04-27 19:26:33] (step=0073400) Train Loss: 5.7521, Train Steps/Sec: 1.12 +[2025-04-27 19:26:55] (step=0073425) Train Loss: 5.7159, Train Steps/Sec: 1.12 +[2025-04-27 19:27:17] (step=0073450) Train Loss: 5.6808, Train Steps/Sec: 1.12 +[2025-04-27 19:27:40] (step=0073475) Train Loss: 5.6852, Train Steps/Sec: 1.12 +[2025-04-27 19:28:02] (step=0073500) Train Loss: 5.7072, Train Steps/Sec: 1.12 +[2025-04-27 19:28:24] (step=0073525) Train Loss: 5.6907, Train Steps/Sec: 1.12 +[2025-04-27 19:28:47] (step=0073550) Train Loss: 5.7365, Train Steps/Sec: 1.12 +[2025-04-27 19:29:09] (step=0073575) Train Loss: 5.7058, Train Steps/Sec: 1.12 +[2025-04-27 19:29:31] (step=0073600) Train Loss: 5.7637, Train Steps/Sec: 1.11 +[2025-04-27 19:29:54] (step=0073625) Train Loss: 5.7006, Train Steps/Sec: 1.12 +[2025-04-27 19:30:16] (step=0073650) Train Loss: 5.7122, Train Steps/Sec: 1.12 +[2025-04-27 19:30:38] (step=0073675) Train Loss: 5.7070, Train Steps/Sec: 1.12 +[2025-04-27 19:31:01] (step=0073700) Train Loss: 5.7880, Train Steps/Sec: 1.12 +[2025-04-27 19:31:23] (step=0073725) Train Loss: 5.7524, Train Steps/Sec: 1.12 +[2025-04-27 19:31:45] (step=0073750) Train Loss: 5.7361, Train Steps/Sec: 1.12 +[2025-04-27 19:32:07] (step=0073775) Train Loss: 5.7128, Train Steps/Sec: 1.12 +[2025-04-27 19:32:30] (step=0073800) Train Loss: 5.7960, Train Steps/Sec: 1.12 +[2025-04-27 19:32:52] (step=0073825) Train Loss: 5.7617, Train Steps/Sec: 1.12 +[2025-04-27 19:33:14] (step=0073850) Train Loss: 5.7813, Train Steps/Sec: 1.12 +[2025-04-27 19:33:37] (step=0073875) Train Loss: 5.7240, Train Steps/Sec: 1.12 +[2025-04-27 19:33:59] (step=0073900) Train Loss: 5.7395, Train Steps/Sec: 1.12 +[2025-04-27 19:34:21] (step=0073925) Train Loss: 5.7057, Train Steps/Sec: 1.12 +[2025-04-27 19:34:44] (step=0073950) Train Loss: 5.7378, Train Steps/Sec: 1.12 +[2025-04-27 19:35:06] (step=0073975) Train Loss: 5.7402, Train Steps/Sec: 1.12 +[2025-04-27 19:35:28] (step=0074000) Train Loss: 5.6978, Train Steps/Sec: 1.11 +[2025-04-27 19:35:28] text_config is None. Initializing the text config with default values (`OPTConfig`). +[2025-04-27 19:41:47] Finish Eval in 74000 steps... +[2025-04-27 19:42:08] Saved checkpoint to checkpoint/X2I_UltraEdiit_fourmask_LLava_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_filtered_objects_4mask_1280_context/003-GPT-XL/checkpoints/0074000.pt +[2025-04-27 19:42:10] Removed old checkpoint: checkpoint/X2I_UltraEdiit_fourmask_LLava_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_filtered_objects_4mask_1280_context/003-GPT-XL/checkpoints/0072000.pt +[2025-04-27 19:42:33] (step=0074025) Train Loss: 5.7721, Train Steps/Sec: 0.06 +[2025-04-27 19:42:55] (step=0074050) Train Loss: 5.6573, Train Steps/Sec: 1.12 +[2025-04-27 19:43:17] (step=0074075) Train Loss: 5.7614, Train Steps/Sec: 1.12 +[2025-04-27 19:43:39] (step=0074100) Train Loss: 5.7687, Train Steps/Sec: 1.12 +[2025-04-27 19:44:02] (step=0074125) Train Loss: 5.7464, Train Steps/Sec: 1.12 +[2025-04-27 19:44:24] (step=0074150) Train Loss: 5.7295, Train Steps/Sec: 1.12 +[2025-04-27 19:44:46] (step=0074175) Train Loss: 5.7339, Train Steps/Sec: 1.12 +[2025-04-27 19:45:09] (step=0074200) Train Loss: 5.6535, Train Steps/Sec: 1.12 +[2025-04-27 19:45:31] (step=0074225) Train Loss: 5.6877, Train Steps/Sec: 1.12 +[2025-04-27 19:45:53] (step=0074250) Train Loss: 5.7601, Train Steps/Sec: 1.12 +[2025-04-27 19:46:15] (step=0074275) Train Loss: 5.7443, Train Steps/Sec: 1.12 +[2025-04-27 19:46:38] (step=0074300) Train Loss: 5.7373, Train Steps/Sec: 1.12 +[2025-04-27 19:47:00] (step=0074325) Train Loss: 5.7036, Train Steps/Sec: 1.13 +[2025-04-27 19:47:22] (step=0074350) Train Loss: 5.7590, Train Steps/Sec: 1.12 +[2025-04-27 19:47:45] (step=0074375) Train Loss: 5.7624, Train Steps/Sec: 1.12 +[2025-04-27 19:48:07] (step=0074400) Train Loss: 5.7275, Train Steps/Sec: 1.11 +[2025-04-27 19:48:29] (step=0074425) Train Loss: 5.7070, Train Steps/Sec: 1.12 +[2025-04-27 19:48:52] (step=0074450) Train Loss: 5.6965, Train Steps/Sec: 1.12 +[2025-04-27 19:49:14] (step=0074475) Train Loss: 5.7359, Train Steps/Sec: 1.10 +[2025-04-27 19:49:37] (step=0074500) Train Loss: 5.7461, Train Steps/Sec: 1.12 +[2025-04-27 19:49:59] (step=0074525) Train Loss: 5.7630, Train Steps/Sec: 1.12 +[2025-04-27 19:50:21] (step=0074550) Train Loss: 5.6933, Train Steps/Sec: 1.12 +[2025-04-27 19:50:44] (step=0074575) Train Loss: 5.7472, Train Steps/Sec: 1.12 +[2025-04-27 19:51:06] (step=0074600) Train Loss: 5.6954, Train Steps/Sec: 1.12 +[2025-04-27 19:51:28] (step=0074625) Train Loss: 5.6965, Train Steps/Sec: 1.12 +[2025-04-27 19:51:51] (step=0074650) Train Loss: 5.7242, Train Steps/Sec: 1.12 +[2025-04-27 19:52:13] (step=0074675) Train Loss: 5.7934, Train Steps/Sec: 1.12 +[2025-04-27 19:52:35] (step=0074700) Train Loss: 5.6675, Train Steps/Sec: 1.12 +[2025-04-27 19:52:58] (step=0074725) Train Loss: 5.7213, Train Steps/Sec: 1.12 +[2025-04-27 19:53:20] (step=0074750) Train Loss: 5.7610, Train Steps/Sec: 1.12 +[2025-04-27 19:53:42] (step=0074775) Train Loss: 5.7227, Train Steps/Sec: 1.12 +[2025-04-27 19:54:05] (step=0074800) Train Loss: 5.7354, Train Steps/Sec: 1.12 +[2025-04-27 19:54:27] (step=0074825) Train Loss: 5.7442, Train Steps/Sec: 1.12 +[2025-04-27 19:54:49] (step=0074850) Train Loss: 5.7098, Train Steps/Sec: 1.12 +[2025-04-27 19:55:12] (step=0074875) Train Loss: 5.7777, Train Steps/Sec: 1.12 +[2025-04-27 19:55:34] (step=0074900) Train Loss: 5.7104, Train Steps/Sec: 1.12 +[2025-04-27 19:55:56] (step=0074925) Train Loss: 5.7372, Train Steps/Sec: 1.12 +[2025-04-27 19:56:18] (step=0074950) Train Loss: 5.7109, Train Steps/Sec: 1.12 +[2025-04-27 19:56:41] (step=0074975) Train Loss: 5.7632, Train Steps/Sec: 1.12 +[2025-04-27 19:57:03] (step=0075000) Train Loss: 5.7678, Train Steps/Sec: 1.11 +[2025-04-27 19:57:26] (step=0075025) Train Loss: 5.7229, Train Steps/Sec: 1.12 +[2025-04-27 19:57:48] (step=0075050) Train Loss: 5.6992, Train Steps/Sec: 1.12 +[2025-04-27 19:58:15] (step=0075075) Train Loss: 5.7934, Train Steps/Sec: 0.93 +[2025-04-27 19:58:37] (step=0075100) Train Loss: 5.6244, Train Steps/Sec: 1.12 +[2025-04-27 19:58:59] (step=0075125) Train Loss: 5.8105, Train Steps/Sec: 1.12 +[2025-04-27 19:59:22] (step=0075150) Train Loss: 5.7188, Train Steps/Sec: 1.12 +[2025-04-27 19:59:44] (step=0075175) Train Loss: 5.7058, Train Steps/Sec: 1.12 +[2025-04-27 20:00:06] (step=0075200) Train Loss: 5.7529, Train Steps/Sec: 1.12 +[2025-04-27 20:00:29] (step=0075225) Train Loss: 5.7394, Train Steps/Sec: 1.12 +[2025-04-27 20:00:51] (step=0075250) Train Loss: 5.7660, Train Steps/Sec: 1.12 +[2025-04-27 20:01:13] (step=0075275) Train Loss: 5.7507, Train Steps/Sec: 1.12 +[2025-04-27 20:01:40] (step=0075300) Train Loss: 5.7227, Train Steps/Sec: 0.93 +[2025-04-27 20:02:03] (step=0075325) Train Loss: 5.6992, Train Steps/Sec: 1.12 +[2025-04-27 20:02:25] (step=0075350) Train Loss: 5.7038, Train Steps/Sec: 1.12 +[2025-04-27 20:02:47] (step=0075375) Train Loss: 5.7150, Train Steps/Sec: 1.12 +[2025-04-27 20:03:10] (step=0075400) Train Loss: 5.7224, Train Steps/Sec: 1.12 +[2025-04-27 20:03:32] (step=0075425) Train Loss: 5.6826, Train Steps/Sec: 1.12 +[2025-04-27 20:03:54] (step=0075450) Train Loss: 5.7195, Train Steps/Sec: 1.12 +[2025-04-27 20:04:16] (step=0075475) Train Loss: 5.7410, Train Steps/Sec: 1.12 +[2025-04-27 20:04:39] (step=0075500) Train Loss: 5.7243, Train Steps/Sec: 1.12 +[2025-04-27 20:05:01] (step=0075525) Train Loss: 5.7159, Train Steps/Sec: 1.12 +[2025-04-27 20:05:23] (step=0075550) Train Loss: 5.7429, Train Steps/Sec: 1.12 +[2025-04-27 20:05:45] (step=0075575) Train Loss: 5.6991, Train Steps/Sec: 1.12 +[2025-04-27 20:06:08] (step=0075600) Train Loss: 5.6775, Train Steps/Sec: 1.11 +[2025-04-27 20:06:30] (step=0075625) Train Loss: 5.7091, Train Steps/Sec: 1.12 +[2025-04-27 20:06:52] (step=0075650) Train Loss: 5.7268, Train Steps/Sec: 1.13 +[2025-04-27 20:07:15] (step=0075675) Train Loss: 5.6942, Train Steps/Sec: 1.12 +[2025-04-27 20:07:37] (step=0075700) Train Loss: 5.6816, Train Steps/Sec: 1.12 +[2025-04-27 20:07:59] (step=0075725) Train Loss: 5.7684, Train Steps/Sec: 1.12 +[2025-04-27 20:08:22] (step=0075750) Train Loss: 5.7469, Train Steps/Sec: 1.12 +[2025-04-27 20:08:44] (step=0075775) Train Loss: 5.7208, Train Steps/Sec: 1.12 +[2025-04-27 20:09:06] (step=0075800) Train Loss: 5.6633, Train Steps/Sec: 1.11 +[2025-04-27 20:09:29] (step=0075825) Train Loss: 5.7462, Train Steps/Sec: 1.12 +[2025-04-27 20:09:51] (step=0075850) Train Loss: 5.7390, Train Steps/Sec: 1.12 +[2025-04-27 20:10:13] (step=0075875) Train Loss: 5.6789, Train Steps/Sec: 1.12 +[2025-04-27 20:10:36] (step=0075900) Train Loss: 5.7237, Train Steps/Sec: 1.12 +[2025-04-27 20:10:58] (step=0075925) Train Loss: 5.7162, Train Steps/Sec: 1.12 +[2025-04-27 20:11:20] (step=0075950) Train Loss: 5.7140, Train Steps/Sec: 1.12 +[2025-04-27 20:11:51] (step=0075975) Train Loss: 5.6973, Train Steps/Sec: 0.81 +[2025-04-27 20:12:18] (step=0076000) Train Loss: 5.7591, Train Steps/Sec: 0.94 +[2025-04-27 20:12:18] text_config is None. Initializing the text config with default values (`OPTConfig`). +[2025-04-27 20:18:36] Finish Eval in 76000 steps... +[2025-04-27 20:18:56] Saved checkpoint to checkpoint/X2I_UltraEdiit_fourmask_LLava_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_filtered_objects_4mask_1280_context/003-GPT-XL/checkpoints/0076000.pt +[2025-04-27 20:18:58] Removed old checkpoint: checkpoint/X2I_UltraEdiit_fourmask_LLava_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_filtered_objects_4mask_1280_context/003-GPT-XL/checkpoints/0074000.pt +[2025-04-27 20:19:25] (step=0076025) Train Loss: 5.6696, Train Steps/Sec: 0.06 +[2025-04-27 20:19:48] (step=0076050) Train Loss: 5.6943, Train Steps/Sec: 1.12 +[2025-04-27 20:20:10] (step=0076075) Train Loss: 5.6934, Train Steps/Sec: 1.12 +[2025-04-27 20:20:32] (step=0076100) Train Loss: 5.7573, Train Steps/Sec: 1.12 +[2025-04-27 20:20:55] (step=0076125) Train Loss: 5.7233, Train Steps/Sec: 1.12 +[2025-04-27 20:21:17] (step=0076150) Train Loss: 5.7097, Train Steps/Sec: 1.12 +[2025-04-27 20:21:39] (step=0076175) Train Loss: 5.7543, Train Steps/Sec: 1.12 +[2025-04-27 20:22:02] (step=0076200) Train Loss: 5.7857, Train Steps/Sec: 1.12 +[2025-04-27 20:22:24] (step=0076225) Train Loss: 5.7199, Train Steps/Sec: 1.12 +[2025-04-27 20:22:51] (step=0076250) Train Loss: 5.7736, Train Steps/Sec: 0.92 +[2025-04-27 20:23:13] (step=0076275) Train Loss: 5.6506, Train Steps/Sec: 1.12 +[2025-04-27 20:23:35] (step=0076300) Train Loss: 5.7243, Train Steps/Sec: 1.12 +[2025-04-27 20:23:58] (step=0076325) Train Loss: 5.7546, Train Steps/Sec: 1.12 +[2025-04-27 20:24:20] (step=0076350) Train Loss: 5.6752, Train Steps/Sec: 1.12 +[2025-04-27 20:24:43] (step=0076375) Train Loss: 5.7136, Train Steps/Sec: 1.10 +[2025-04-27 20:25:05] (step=0076400) Train Loss: 5.6707, Train Steps/Sec: 1.12 +[2025-04-27 20:25:27] (step=0076425) Train Loss: 5.7918, Train Steps/Sec: 1.12 +[2025-04-27 20:25:50] (step=0076450) Train Loss: 5.7957, Train Steps/Sec: 1.12 +[2025-04-27 20:26:12] (step=0076475) Train Loss: 5.6796, Train Steps/Sec: 1.12 +[2025-04-27 20:26:34] (step=0076500) Train Loss: 5.6525, Train Steps/Sec: 1.12 +[2025-04-27 20:26:56] (step=0076525) Train Loss: 5.6823, Train Steps/Sec: 1.12 +[2025-04-27 20:27:19] (step=0076550) Train Loss: 5.7147, Train Steps/Sec: 1.12 +[2025-04-27 20:27:41] (step=0076575) Train Loss: 5.7535, Train Steps/Sec: 1.12 +[2025-04-27 20:28:04] (step=0076600) Train Loss: 5.7049, Train Steps/Sec: 1.12 +[2025-04-27 20:28:26] (step=0076625) Train Loss: 5.7371, Train Steps/Sec: 1.12 +[2025-04-27 20:28:48] (step=0076650) Train Loss: 5.7015, Train Steps/Sec: 1.12 +[2025-04-27 20:29:10] (step=0076675) Train Loss: 5.7297, Train Steps/Sec: 1.12 +[2025-04-27 20:29:33] (step=0076700) Train Loss: 5.8067, Train Steps/Sec: 1.12 +[2025-04-27 20:29:55] (step=0076725) Train Loss: 5.7075, Train Steps/Sec: 1.12 +[2025-04-27 20:30:17] (step=0076750) Train Loss: 5.7099, Train Steps/Sec: 1.12 +[2025-04-27 20:30:40] (step=0076775) Train Loss: 5.7005, Train Steps/Sec: 1.12 +[2025-04-27 20:31:02] (step=0076800) Train Loss: 5.7753, Train Steps/Sec: 1.12 +[2025-04-27 20:31:24] (step=0076825) Train Loss: 5.7228, Train Steps/Sec: 1.12 +[2025-04-27 20:31:47] (step=0076850) Train Loss: 5.7663, Train Steps/Sec: 1.12 +[2025-04-27 20:32:09] (step=0076875) Train Loss: 5.7125, Train Steps/Sec: 1.12 +[2025-04-27 20:32:31] (step=0076900) Train Loss: 5.7393, Train Steps/Sec: 1.12 +[2025-04-27 20:32:54] (step=0076925) Train Loss: 5.7744, Train Steps/Sec: 1.12 +[2025-04-27 20:33:16] (step=0076950) Train Loss: 5.6764, Train Steps/Sec: 1.12 +[2025-04-27 20:33:38] (step=0076975) Train Loss: 5.7356, Train Steps/Sec: 1.12 +[2025-04-27 20:34:01] (step=0077000) Train Loss: 5.7037, Train Steps/Sec: 1.12 +[2025-04-27 20:34:23] (step=0077025) Train Loss: 5.7430, Train Steps/Sec: 1.12 +[2025-04-27 20:34:45] (step=0077050) Train Loss: 5.6990, Train Steps/Sec: 1.12 +[2025-04-27 20:35:08] (step=0077075) Train Loss: 5.6896, Train Steps/Sec: 1.12 +[2025-04-27 20:35:30] (step=0077100) Train Loss: 5.7429, Train Steps/Sec: 1.12 +[2025-04-27 20:35:52] (step=0077125) Train Loss: 5.7996, Train Steps/Sec: 1.12 +[2025-04-27 20:36:14] (step=0077150) Train Loss: 5.7497, Train Steps/Sec: 1.12 +[2025-04-27 20:36:37] (step=0077175) Train Loss: 5.7268, Train Steps/Sec: 1.12 +[2025-04-27 20:36:59] (step=0077200) Train Loss: 5.7813, Train Steps/Sec: 1.12 +[2025-04-27 20:37:21] (step=0077225) Train Loss: 5.7120, Train Steps/Sec: 1.12 +[2025-04-27 20:37:44] (step=0077250) Train Loss: 5.7619, Train Steps/Sec: 1.12 +[2025-04-27 20:38:06] (step=0077275) Train Loss: 5.7927, Train Steps/Sec: 1.12 +[2025-04-27 20:38:28] (step=0077300) Train Loss: 5.7105, Train Steps/Sec: 1.12 +[2025-04-27 20:38:51] (step=0077325) Train Loss: 5.6451, Train Steps/Sec: 1.12 +[2025-04-27 20:39:13] (step=0077350) Train Loss: 5.7461, Train Steps/Sec: 1.12 +[2025-04-27 20:39:35] (step=0077375) Train Loss: 5.7718, Train Steps/Sec: 1.12 +[2025-04-27 20:39:58] (step=0077400) Train Loss: 5.7640, Train Steps/Sec: 1.11 +[2025-04-27 20:40:20] (step=0077425) Train Loss: 5.6955, Train Steps/Sec: 1.12 +[2025-04-27 20:40:42] (step=0077450) Train Loss: 5.7781, Train Steps/Sec: 1.12 +[2025-04-27 20:41:05] (step=0077475) Train Loss: 5.7168, Train Steps/Sec: 1.12 +[2025-04-27 20:41:27] (step=0077500) Train Loss: 5.6614, Train Steps/Sec: 1.12 +[2025-04-27 20:41:49] (step=0077525) Train Loss: 5.7237, Train Steps/Sec: 1.12 +[2025-04-27 20:42:12] (step=0077550) Train Loss: 5.6897, Train Steps/Sec: 1.12 +[2025-04-27 20:42:34] (step=0077575) Train Loss: 5.7029, Train Steps/Sec: 1.12 +[2025-04-27 20:42:56] (step=0077600) Train Loss: 5.7288, Train Steps/Sec: 1.12 +[2025-04-27 20:43:19] (step=0077625) Train Loss: 5.6730, Train Steps/Sec: 1.12 +[2025-04-27 20:43:41] (step=0077650) Train Loss: 5.7057, Train Steps/Sec: 1.12 +[2025-04-27 20:44:03] (step=0077675) Train Loss: 5.7060, Train Steps/Sec: 1.12 +[2025-04-27 20:44:25] (step=0077700) Train Loss: 5.6839, Train Steps/Sec: 1.12 +[2025-04-27 20:44:48] (step=0077725) Train Loss: 5.7517, Train Steps/Sec: 1.12 +[2025-04-27 20:45:10] (step=0077750) Train Loss: 5.6762, Train Steps/Sec: 1.12 +[2025-04-27 20:45:32] (step=0077775) Train Loss: 5.7888, Train Steps/Sec: 1.12 +[2025-04-27 20:45:55] (step=0077800) Train Loss: 5.7860, Train Steps/Sec: 1.12 +[2025-04-27 20:46:17] (step=0077825) Train Loss: 5.7246, Train Steps/Sec: 1.12 +[2025-04-27 20:46:39] (step=0077850) Train Loss: 5.7439, Train Steps/Sec: 1.12 +[2025-04-27 20:47:02] (step=0077875) Train Loss: 5.7032, Train Steps/Sec: 1.12 +[2025-04-27 20:47:24] (step=0077900) Train Loss: 5.7598, Train Steps/Sec: 1.12 +[2025-04-27 20:47:46] (step=0077925) Train Loss: 5.7616, Train Steps/Sec: 1.12 +[2025-04-27 20:48:09] (step=0077950) Train Loss: 5.7475, Train Steps/Sec: 1.12 +[2025-04-27 20:48:31] (step=0077975) Train Loss: 5.6824, Train Steps/Sec: 1.12 +[2025-04-27 20:48:53] (step=0078000) Train Loss: 5.7386, Train Steps/Sec: 1.11 +[2025-04-27 20:48:53] text_config is None. Initializing the text config with default values (`OPTConfig`). +[2025-04-27 20:55:10] Finish Eval in 78000 steps... +[2025-04-27 20:55:30] Saved checkpoint to checkpoint/X2I_UltraEdiit_fourmask_LLava_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_filtered_objects_4mask_1280_context/003-GPT-XL/checkpoints/0078000.pt +[2025-04-27 20:55:32] Removed old checkpoint: checkpoint/X2I_UltraEdiit_fourmask_LLava_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_filtered_objects_4mask_1280_context/003-GPT-XL/checkpoints/0076000.pt +[2025-04-27 20:55:55] (step=0078025) Train Loss: 5.7780, Train Steps/Sec: 0.06 +[2025-04-27 20:56:17] (step=0078050) Train Loss: 5.6752, Train Steps/Sec: 1.12 +[2025-04-27 20:56:39] (step=0078075) Train Loss: 5.6859, Train Steps/Sec: 1.12 +[2025-04-27 20:57:01] (step=0078100) Train Loss: 5.7633, Train Steps/Sec: 1.12 +[2025-04-27 20:57:24] (step=0078125) Train Loss: 5.7159, Train Steps/Sec: 1.12 +[2025-04-27 20:57:46] (step=0078150) Train Loss: 5.7515, Train Steps/Sec: 1.12 +[2025-04-27 20:58:08] (step=0078175) Train Loss: 5.7067, Train Steps/Sec: 1.13 +[2025-04-27 20:58:30] (step=0078200) Train Loss: 5.7095, Train Steps/Sec: 1.12 +[2025-04-27 20:58:53] (step=0078225) Train Loss: 5.7550, Train Steps/Sec: 1.12 +[2025-04-27 20:59:15] (step=0078250) Train Loss: 5.6855, Train Steps/Sec: 1.13 +[2025-04-27 20:59:38] (step=0078275) Train Loss: 5.6456, Train Steps/Sec: 1.11 +[2025-04-27 21:00:00] (step=0078300) Train Loss: 5.7888, Train Steps/Sec: 1.12 +[2025-04-27 21:00:22] (step=0078325) Train Loss: 5.7384, Train Steps/Sec: 1.12 +[2025-04-27 21:00:44] (step=0078350) Train Loss: 5.6986, Train Steps/Sec: 1.12 +[2025-04-27 21:01:07] (step=0078375) Train Loss: 5.7102, Train Steps/Sec: 1.12 +[2025-04-27 21:01:29] (step=0078400) Train Loss: 5.7824, Train Steps/Sec: 1.12 +[2025-04-27 21:01:51] (step=0078425) Train Loss: 5.7604, Train Steps/Sec: 1.12 +[2025-04-27 21:02:14] (step=0078450) Train Loss: 5.6498, Train Steps/Sec: 1.12 +[2025-04-27 21:02:36] (step=0078475) Train Loss: 5.7302, Train Steps/Sec: 1.12 +[2025-04-27 21:02:58] (step=0078500) Train Loss: 5.6521, Train Steps/Sec: 1.12 +[2025-04-27 21:03:21] (step=0078525) Train Loss: 5.7017, Train Steps/Sec: 1.12 +[2025-04-27 21:03:43] (step=0078550) Train Loss: 5.7102, Train Steps/Sec: 1.12 +[2025-04-27 21:04:05] (step=0078575) Train Loss: 5.7726, Train Steps/Sec: 1.12 +[2025-04-27 21:04:28] (step=0078600) Train Loss: 5.7319, Train Steps/Sec: 1.12 +[2025-04-27 21:04:50] (step=0078625) Train Loss: 5.7254, Train Steps/Sec: 1.12 +[2025-04-27 21:05:12] (step=0078650) Train Loss: 5.7103, Train Steps/Sec: 1.12 +[2025-04-27 21:05:34] (step=0078675) Train Loss: 5.7113, Train Steps/Sec: 1.12 +[2025-04-27 21:05:57] (step=0078700) Train Loss: 5.7402, Train Steps/Sec: 1.12 +[2025-04-27 21:06:19] (step=0078725) Train Loss: 5.7180, Train Steps/Sec: 1.12 +[2025-04-27 21:06:41] (step=0078750) Train Loss: 5.6692, Train Steps/Sec: 1.12 +[2025-04-27 21:07:03] (step=0078775) Train Loss: 5.7363, Train Steps/Sec: 1.12 +[2025-04-27 21:07:26] (step=0078800) Train Loss: 5.7168, Train Steps/Sec: 1.12 +[2025-04-27 21:07:48] (step=0078825) Train Loss: 5.7787, Train Steps/Sec: 1.12 +[2025-04-27 21:08:10] (step=0078850) Train Loss: 5.7033, Train Steps/Sec: 1.12 +[2025-04-27 21:08:33] (step=0078875) Train Loss: 5.7221, Train Steps/Sec: 1.12 +[2025-04-27 21:08:55] (step=0078900) Train Loss: 5.7370, Train Steps/Sec: 1.12 +[2025-04-27 21:09:17] (step=0078925) Train Loss: 5.6703, Train Steps/Sec: 1.12 +[2025-04-27 21:09:40] (step=0078950) Train Loss: 5.7563, Train Steps/Sec: 1.12 +[2025-04-27 21:10:02] (step=0078975) Train Loss: 5.7335, Train Steps/Sec: 1.12 +[2025-04-27 21:10:24] (step=0079000) Train Loss: 5.7567, Train Steps/Sec: 1.12 +[2025-04-27 21:10:47] (step=0079025) Train Loss: 5.7492, Train Steps/Sec: 1.13 +[2025-04-27 21:11:09] (step=0079050) Train Loss: 5.7394, Train Steps/Sec: 1.12 +[2025-04-27 21:11:31] (step=0079075) Train Loss: 5.7095, Train Steps/Sec: 1.12 +[2025-04-27 21:11:53] (step=0079100) Train Loss: 5.7487, Train Steps/Sec: 1.12 +[2025-04-27 21:12:16] (step=0079125) Train Loss: 5.7204, Train Steps/Sec: 1.12 +[2025-04-27 21:12:38] (step=0079150) Train Loss: 5.7512, Train Steps/Sec: 1.12 +[2025-04-27 21:13:00] (step=0079175) Train Loss: 5.6861, Train Steps/Sec: 1.12 +[2025-04-27 21:13:23] (step=0079200) Train Loss: 5.6972, Train Steps/Sec: 1.11 +[2025-04-27 21:13:45] (step=0079225) Train Loss: 5.7041, Train Steps/Sec: 1.12 +[2025-04-27 21:14:07] (step=0079250) Train Loss: 5.6990, Train Steps/Sec: 1.12 +[2025-04-27 21:14:30] (step=0079275) Train Loss: 5.7544, Train Steps/Sec: 1.12 +[2025-04-27 21:14:52] (step=0079300) Train Loss: 5.7192, Train Steps/Sec: 1.12 +[2025-04-27 21:15:19] (step=0079325) Train Loss: 5.6975, Train Steps/Sec: 0.94 +[2025-04-27 21:15:41] (step=0079350) Train Loss: 5.7374, Train Steps/Sec: 1.12 +[2025-04-27 21:16:03] (step=0079375) Train Loss: 5.7054, Train Steps/Sec: 1.12 +[2025-04-27 21:16:26] (step=0079400) Train Loss: 5.8172, Train Steps/Sec: 1.12 +[2025-04-27 21:16:48] (step=0079425) Train Loss: 5.7287, Train Steps/Sec: 1.13 +[2025-04-27 21:17:10] (step=0079450) Train Loss: 5.6929, Train Steps/Sec: 1.12 +[2025-04-27 21:17:32] (step=0079475) Train Loss: 5.6777, Train Steps/Sec: 1.12 +[2025-04-27 21:17:55] (step=0079500) Train Loss: 5.7285, Train Steps/Sec: 1.12 +[2025-04-27 21:18:17] (step=0079525) Train Loss: 5.6441, Train Steps/Sec: 1.13 +[2025-04-27 21:18:39] (step=0079550) Train Loss: 5.7539, Train Steps/Sec: 1.12 +[2025-04-27 21:19:06] (step=0079575) Train Loss: 5.7067, Train Steps/Sec: 0.93 +[2025-04-27 21:19:29] (step=0079600) Train Loss: 5.7267, Train Steps/Sec: 1.12 +[2025-04-27 21:19:51] (step=0079625) Train Loss: 5.7300, Train Steps/Sec: 1.12 +[2025-04-27 21:20:13] (step=0079650) Train Loss: 5.7583, Train Steps/Sec: 1.12 +[2025-04-27 21:20:35] (step=0079675) Train Loss: 5.6433, Train Steps/Sec: 1.12 +[2025-04-27 21:20:58] (step=0079700) Train Loss: 5.6668, Train Steps/Sec: 1.12 +[2025-04-27 21:21:20] (step=0079725) Train Loss: 5.6938, Train Steps/Sec: 1.13 +[2025-04-27 21:21:42] (step=0079750) Train Loss: 5.7003, Train Steps/Sec: 1.13 +[2025-04-27 21:22:04] (step=0079775) Train Loss: 5.6471, Train Steps/Sec: 1.12 +[2025-04-27 21:22:27] (step=0079800) Train Loss: 5.7392, Train Steps/Sec: 1.12 +[2025-04-27 21:22:49] (step=0079825) Train Loss: 5.7002, Train Steps/Sec: 1.12 +[2025-04-27 21:23:11] (step=0079850) Train Loss: 5.6756, Train Steps/Sec: 1.12 +[2025-04-27 21:23:34] (step=0079875) Train Loss: 5.7165, Train Steps/Sec: 1.12 +[2025-04-27 21:23:56] (step=0079900) Train Loss: 5.7446, Train Steps/Sec: 1.12 +[2025-04-27 21:24:18] (step=0079925) Train Loss: 5.6686, Train Steps/Sec: 1.12 +[2025-04-27 21:24:40] (step=0079950) Train Loss: 5.7316, Train Steps/Sec: 1.12 +[2025-04-27 21:25:03] (step=0079975) Train Loss: 5.7983, Train Steps/Sec: 1.12 +[2025-04-27 21:25:25] (step=0080000) Train Loss: 5.7237, Train Steps/Sec: 1.12 +[2025-04-27 21:25:25] text_config is None. Initializing the text config with default values (`OPTConfig`). +[2025-04-27 21:31:45] Finish Eval in 80000 steps... +[2025-04-27 21:32:05] Saved checkpoint to checkpoint/X2I_UltraEdiit_fourmask_LLava_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_filtered_objects_4mask_1280_context/003-GPT-XL/checkpoints/0080000.pt +[2025-04-27 21:32:07] Removed old checkpoint: checkpoint/X2I_UltraEdiit_fourmask_LLava_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_filtered_objects_4mask_1280_context/003-GPT-XL/checkpoints/0078000.pt +[2025-04-27 21:32:30] (step=0080025) Train Loss: 5.7411, Train Steps/Sec: 0.06 +[2025-04-27 21:32:52] (step=0080050) Train Loss: 5.7015, Train Steps/Sec: 1.12 +[2025-04-27 21:33:14] (step=0080075) Train Loss: 5.7247, Train Steps/Sec: 1.12 +[2025-04-27 21:33:36] (step=0080100) Train Loss: 5.7823, Train Steps/Sec: 1.12 +[2025-04-27 21:33:59] (step=0080125) Train Loss: 5.7032, Train Steps/Sec: 1.12 +[2025-04-27 21:34:21] (step=0080150) Train Loss: 5.7284, Train Steps/Sec: 1.12 +[2025-04-27 21:34:43] (step=0080175) Train Loss: 5.6436, Train Steps/Sec: 1.12 +[2025-04-27 21:35:06] (step=0080200) Train Loss: 5.7453, Train Steps/Sec: 1.09 +[2025-04-27 21:35:28] (step=0080225) Train Loss: 5.7208, Train Steps/Sec: 1.12 +[2025-04-27 21:35:51] (step=0080250) Train Loss: 5.7474, Train Steps/Sec: 1.12 +[2025-04-27 21:36:13] (step=0080275) Train Loss: 5.6310, Train Steps/Sec: 1.12 +[2025-04-27 21:36:43] (step=0080300) Train Loss: 5.6718, Train Steps/Sec: 0.82 +[2025-04-27 21:37:10] (step=0080325) Train Loss: 5.7636, Train Steps/Sec: 0.94 +[2025-04-27 21:37:37] (step=0080350) Train Loss: 5.7279, Train Steps/Sec: 0.91 +[2025-04-27 21:38:00] (step=0080375) Train Loss: 5.6776, Train Steps/Sec: 1.12 +[2025-04-27 21:38:22] (step=0080400) Train Loss: 5.7098, Train Steps/Sec: 1.12 +[2025-04-27 21:38:44] (step=0080425) Train Loss: 5.7267, Train Steps/Sec: 1.12 +[2025-04-27 21:39:07] (step=0080450) Train Loss: 5.7492, Train Steps/Sec: 1.12 +[2025-04-27 21:39:29] (step=0080475) Train Loss: 5.7493, Train Steps/Sec: 1.12 +[2025-04-27 21:39:51] (step=0080500) Train Loss: 5.7365, Train Steps/Sec: 1.12 +[2025-04-27 21:40:14] (step=0080525) Train Loss: 5.7479, Train Steps/Sec: 1.12 +[2025-04-27 21:40:36] (step=0080550) Train Loss: 5.7392, Train Steps/Sec: 1.12 +[2025-04-27 21:40:58] (step=0080575) Train Loss: 5.6386, Train Steps/Sec: 1.12 +[2025-04-27 21:41:21] (step=0080600) Train Loss: 5.7757, Train Steps/Sec: 1.12 +[2025-04-27 21:41:43] (step=0080625) Train Loss: 5.7486, Train Steps/Sec: 1.12 +[2025-04-27 21:42:10] (step=0080650) Train Loss: 5.6948, Train Steps/Sec: 0.92 +[2025-04-27 21:42:32] (step=0080675) Train Loss: 5.7412, Train Steps/Sec: 1.12 +[2025-04-27 21:42:55] (step=0080700) Train Loss: 5.7273, Train Steps/Sec: 1.12 +[2025-04-27 21:43:17] (step=0080725) Train Loss: 5.7458, Train Steps/Sec: 1.12 +[2025-04-27 21:43:39] (step=0080750) Train Loss: 5.7475, Train Steps/Sec: 1.12 +[2025-04-27 21:44:01] (step=0080775) Train Loss: 5.7737, Train Steps/Sec: 1.12 +[2025-04-27 21:44:24] (step=0080800) Train Loss: 5.7249, Train Steps/Sec: 1.12 +[2025-04-27 21:44:46] (step=0080825) Train Loss: 5.7064, Train Steps/Sec: 1.12 +[2025-04-27 21:45:08] (step=0080850) Train Loss: 5.7084, Train Steps/Sec: 1.12 +[2025-04-27 21:45:31] (step=0080875) Train Loss: 5.6962, Train Steps/Sec: 1.12 +[2025-04-27 21:45:53] (step=0080900) Train Loss: 5.7035, Train Steps/Sec: 1.12 +[2025-04-27 21:46:15] (step=0080925) Train Loss: 5.6863, Train Steps/Sec: 1.12 +[2025-04-27 21:46:38] (step=0080950) Train Loss: 5.7624, Train Steps/Sec: 1.12 +[2025-04-27 21:47:00] (step=0080975) Train Loss: 5.7487, Train Steps/Sec: 1.12 +[2025-04-27 21:47:22] (step=0081000) Train Loss: 5.7172, Train Steps/Sec: 1.12 +[2025-04-27 21:47:44] (step=0081025) Train Loss: 5.7207, Train Steps/Sec: 1.12 +[2025-04-27 21:48:07] (step=0081050) Train Loss: 5.7074, Train Steps/Sec: 1.12 +[2025-04-27 21:48:29] (step=0081075) Train Loss: 5.7157, Train Steps/Sec: 1.12 +[2025-04-27 21:48:51] (step=0081100) Train Loss: 5.7350, Train Steps/Sec: 1.12 +[2025-04-27 21:49:14] (step=0081125) Train Loss: 5.7097, Train Steps/Sec: 1.12 +[2025-04-27 21:49:36] (step=0081150) Train Loss: 5.7220, Train Steps/Sec: 1.12 +[2025-04-27 21:49:58] (step=0081175) Train Loss: 5.7145, Train Steps/Sec: 1.12 +[2025-04-27 21:50:21] (step=0081200) Train Loss: 5.7599, Train Steps/Sec: 1.12 +[2025-04-27 21:50:43] (step=0081225) Train Loss: 5.6874, Train Steps/Sec: 1.12 +[2025-04-27 21:51:05] (step=0081250) Train Loss: 5.6677, Train Steps/Sec: 1.12 +[2025-04-27 21:51:27] (step=0081275) Train Loss: 5.7177, Train Steps/Sec: 1.12 +[2025-04-27 21:51:50] (step=0081300) Train Loss: 5.6862, Train Steps/Sec: 1.12 +[2025-04-27 21:52:12] (step=0081325) Train Loss: 5.7284, Train Steps/Sec: 1.12 +[2025-04-27 21:52:34] (step=0081350) Train Loss: 5.6993, Train Steps/Sec: 1.12 +[2025-04-27 21:52:57] (step=0081375) Train Loss: 5.7459, Train Steps/Sec: 1.12 +[2025-04-27 21:53:19] (step=0081400) Train Loss: 5.7236, Train Steps/Sec: 1.12 +[2025-04-27 21:53:41] (step=0081425) Train Loss: 5.6397, Train Steps/Sec: 1.12 +[2025-04-27 21:54:04] (step=0081450) Train Loss: 5.7623, Train Steps/Sec: 1.12 +[2025-04-27 21:54:26] (step=0081475) Train Loss: 5.7378, Train Steps/Sec: 1.12 +[2025-04-27 21:54:48] (step=0081500) Train Loss: 5.7191, Train Steps/Sec: 1.12 +[2025-04-27 21:55:11] (step=0081525) Train Loss: 5.7210, Train Steps/Sec: 1.12 +[2025-04-27 21:55:33] (step=0081550) Train Loss: 5.6772, Train Steps/Sec: 1.12 +[2025-04-27 21:55:55] (step=0081575) Train Loss: 5.6470, Train Steps/Sec: 1.12 +[2025-04-27 21:56:18] (step=0081600) Train Loss: 5.6866, Train Steps/Sec: 1.12 +[2025-04-27 21:56:40] (step=0081625) Train Loss: 5.7508, Train Steps/Sec: 1.12 +[2025-04-27 21:57:02] (step=0081650) Train Loss: 5.7055, Train Steps/Sec: 1.12 +[2025-04-27 21:57:24] (step=0081675) Train Loss: 5.7767, Train Steps/Sec: 1.12 +[2025-04-27 21:57:47] (step=0081700) Train Loss: 5.6001, Train Steps/Sec: 1.12 +[2025-04-27 21:58:09] (step=0081725) Train Loss: 5.7170, Train Steps/Sec: 1.12 +[2025-04-27 21:58:31] (step=0081750) Train Loss: 5.7292, Train Steps/Sec: 1.12 +[2025-04-27 21:58:54] (step=0081775) Train Loss: 5.7452, Train Steps/Sec: 1.12 +[2025-04-27 21:59:16] (step=0081800) Train Loss: 5.6708, Train Steps/Sec: 1.12 +[2025-04-27 21:59:38] (step=0081825) Train Loss: 5.7133, Train Steps/Sec: 1.12 +[2025-04-27 22:00:01] (step=0081850) Train Loss: 5.7478, Train Steps/Sec: 1.12 +[2025-04-27 22:00:23] (step=0081875) Train Loss: 5.7593, Train Steps/Sec: 1.12 +[2025-04-27 22:00:45] (step=0081900) Train Loss: 5.7281, Train Steps/Sec: 1.12 +[2025-04-27 22:01:07] (step=0081925) Train Loss: 5.7103, Train Steps/Sec: 1.12 +[2025-04-27 22:01:30] (step=0081950) Train Loss: 5.7950, Train Steps/Sec: 1.12 +[2025-04-27 22:01:52] (step=0081975) Train Loss: 5.7054, Train Steps/Sec: 1.12 +[2025-04-27 22:02:14] (step=0082000) Train Loss: 5.7378, Train Steps/Sec: 1.12 +[2025-04-27 22:02:14] text_config is None. Initializing the text config with default values (`OPTConfig`). +[2025-04-27 22:08:33] Finish Eval in 82000 steps... +[2025-04-27 22:08:53] Saved checkpoint to checkpoint/X2I_UltraEdiit_fourmask_LLava_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_filtered_objects_4mask_1280_context/003-GPT-XL/checkpoints/0082000.pt +[2025-04-27 22:08:55] Removed old checkpoint: checkpoint/X2I_UltraEdiit_fourmask_LLava_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_filtered_objects_4mask_1280_context/003-GPT-XL/checkpoints/0080000.pt +[2025-04-27 22:09:18] (step=0082025) Train Loss: 5.6959, Train Steps/Sec: 0.06 +[2025-04-27 22:09:40] (step=0082050) Train Loss: 5.7004, Train Steps/Sec: 1.12 +[2025-04-27 22:10:02] (step=0082075) Train Loss: 5.7105, Train Steps/Sec: 1.12 +[2025-04-27 22:10:25] (step=0082100) Train Loss: 5.7161, Train Steps/Sec: 1.10 +[2025-04-27 22:10:47] (step=0082125) Train Loss: 5.7298, Train Steps/Sec: 1.12 +[2025-04-27 22:11:09] (step=0082150) Train Loss: 5.7614, Train Steps/Sec: 1.12 +[2025-04-27 22:11:32] (step=0082175) Train Loss: 5.6564, Train Steps/Sec: 1.12 +[2025-04-27 22:11:54] (step=0082200) Train Loss: 5.7258, Train Steps/Sec: 1.12 +[2025-04-27 22:12:16] (step=0082225) Train Loss: 5.7487, Train Steps/Sec: 1.12 +[2025-04-27 22:12:38] (step=0082250) Train Loss: 5.7056, Train Steps/Sec: 1.12 +[2025-04-27 22:13:01] (step=0082275) Train Loss: 5.7602, Train Steps/Sec: 1.12 +[2025-04-27 22:13:23] (step=0082300) Train Loss: 5.7621, Train Steps/Sec: 1.12 +[2025-04-27 22:13:45] (step=0082325) Train Loss: 5.6342, Train Steps/Sec: 1.12 +[2025-04-27 22:14:08] (step=0082350) Train Loss: 5.7166, Train Steps/Sec: 1.12 +[2025-04-27 22:14:30] (step=0082375) Train Loss: 5.7221, Train Steps/Sec: 1.12 +[2025-04-27 22:14:52] (step=0082400) Train Loss: 5.7253, Train Steps/Sec: 1.12 +[2025-04-27 22:15:14] (step=0082425) Train Loss: 5.7038, Train Steps/Sec: 1.12 +[2025-04-27 22:15:37] (step=0082450) Train Loss: 5.7428, Train Steps/Sec: 1.12 +[2025-04-27 22:15:59] (step=0082475) Train Loss: 5.7849, Train Steps/Sec: 1.12 +[2025-04-27 22:16:21] (step=0082500) Train Loss: 5.6996, Train Steps/Sec: 1.12 +[2025-04-27 22:16:43] (step=0082525) Train Loss: 5.6924, Train Steps/Sec: 1.12 +[2025-04-27 22:17:06] (step=0082550) Train Loss: 5.7301, Train Steps/Sec: 1.12 +[2025-04-27 22:17:28] (step=0082575) Train Loss: 5.7365, Train Steps/Sec: 1.12 +[2025-04-27 22:17:50] (step=0082600) Train Loss: 5.7343, Train Steps/Sec: 1.12 +[2025-04-27 22:18:13] (step=0082625) Train Loss: 5.6706, Train Steps/Sec: 1.12 +[2025-04-27 22:18:35] (step=0082650) Train Loss: 5.7887, Train Steps/Sec: 1.12 +[2025-04-27 22:18:57] (step=0082675) Train Loss: 5.7197, Train Steps/Sec: 1.12 +[2025-04-27 22:19:19] (step=0082700) Train Loss: 5.6927, Train Steps/Sec: 1.13 +[2025-04-27 22:19:42] (step=0082725) Train Loss: 5.6701, Train Steps/Sec: 1.12 +[2025-04-27 22:20:04] (step=0082750) Train Loss: 5.7175, Train Steps/Sec: 1.12 +[2025-04-27 22:20:26] (step=0082775) Train Loss: 5.6631, Train Steps/Sec: 1.12 +[2025-04-27 22:20:49] (step=0082800) Train Loss: 5.7600, Train Steps/Sec: 1.12 +[2025-04-27 22:21:11] (step=0082825) Train Loss: 5.6903, Train Steps/Sec: 1.12 +[2025-04-27 22:21:33] (step=0082850) Train Loss: 5.7082, Train Steps/Sec: 1.13 +[2025-04-27 22:21:55] (step=0082875) Train Loss: 5.7143, Train Steps/Sec: 1.12 +[2025-04-27 22:22:18] (step=0082900) Train Loss: 5.6657, Train Steps/Sec: 1.12 +[2025-04-27 22:22:40] (step=0082925) Train Loss: 5.7258, Train Steps/Sec: 1.12 +[2025-04-27 22:23:02] (step=0082950) Train Loss: 5.7621, Train Steps/Sec: 1.12 +[2025-04-27 22:23:24] (step=0082975) Train Loss: 5.7599, Train Steps/Sec: 1.13 +[2025-04-27 22:23:47] (step=0083000) Train Loss: 5.7075, Train Steps/Sec: 1.11 +[2025-04-27 22:24:09] (step=0083025) Train Loss: 5.6711, Train Steps/Sec: 1.12 +[2025-04-27 22:24:32] (step=0083050) Train Loss: 5.7285, Train Steps/Sec: 1.12 +[2025-04-27 22:24:54] (step=0083075) Train Loss: 5.7017, Train Steps/Sec: 1.12 +[2025-04-27 22:25:16] (step=0083100) Train Loss: 5.7485, Train Steps/Sec: 1.13 +[2025-04-27 22:25:38] (step=0083125) Train Loss: 5.7712, Train Steps/Sec: 1.12 +[2025-04-27 22:26:01] (step=0083150) Train Loss: 5.6508, Train Steps/Sec: 1.12 +[2025-04-27 22:26:23] (step=0083175) Train Loss: 5.7643, Train Steps/Sec: 1.13 +[2025-04-27 22:26:45] (step=0083200) Train Loss: 5.7061, Train Steps/Sec: 1.12 +[2025-04-27 22:27:07] (step=0083225) Train Loss: 5.6824, Train Steps/Sec: 1.12 +[2025-04-27 22:27:30] (step=0083250) Train Loss: 5.7328, Train Steps/Sec: 1.12 +[2025-04-27 22:27:52] (step=0083275) Train Loss: 5.7143, Train Steps/Sec: 1.12 +[2025-04-27 22:28:14] (step=0083300) Train Loss: 5.6227, Train Steps/Sec: 1.12 +[2025-04-27 22:28:37] (step=0083325) Train Loss: 5.7180, Train Steps/Sec: 1.12 +[2025-04-27 22:28:59] (step=0083350) Train Loss: 5.7718, Train Steps/Sec: 1.12 +[2025-04-27 22:29:21] (step=0083375) Train Loss: 5.7194, Train Steps/Sec: 1.12 +[2025-04-27 22:29:44] (step=0083400) Train Loss: 5.7223, Train Steps/Sec: 1.12 +[2025-04-27 22:30:06] (step=0083425) Train Loss: 5.7415, Train Steps/Sec: 1.12 +[2025-04-27 22:30:28] (step=0083450) Train Loss: 5.6701, Train Steps/Sec: 1.12 +[2025-04-27 22:30:50] (step=0083475) Train Loss: 5.7291, Train Steps/Sec: 1.12 +[2025-04-27 22:31:13] (step=0083500) Train Loss: 5.7073, Train Steps/Sec: 1.13 +[2025-04-27 22:31:35] (step=0083525) Train Loss: 5.7555, Train Steps/Sec: 1.12 +[2025-04-27 22:31:57] (step=0083550) Train Loss: 5.7056, Train Steps/Sec: 1.12 +[2025-04-27 22:32:24] (step=0083575) Train Loss: 5.7000, Train Steps/Sec: 0.94 +[2025-04-27 22:32:46] (step=0083600) Train Loss: 5.7301, Train Steps/Sec: 1.12 +[2025-04-27 22:33:08] (step=0083625) Train Loss: 5.7505, Train Steps/Sec: 1.12 +[2025-04-27 22:33:31] (step=0083650) Train Loss: 5.6828, Train Steps/Sec: 1.12 +[2025-04-27 22:33:53] (step=0083675) Train Loss: 5.7185, Train Steps/Sec: 1.12 +[2025-04-27 22:34:15] (step=0083700) Train Loss: 5.6956, Train Steps/Sec: 1.12 +[2025-04-27 22:34:38] (step=0083725) Train Loss: 5.6945, Train Steps/Sec: 1.12 +[2025-04-27 22:35:00] (step=0083750) Train Loss: 5.7020, Train Steps/Sec: 1.12 +[2025-04-27 22:35:22] (step=0083775) Train Loss: 5.6849, Train Steps/Sec: 1.12 +[2025-04-27 22:35:45] (step=0083800) Train Loss: 5.7239, Train Steps/Sec: 1.12 +[2025-04-27 22:36:07] (step=0083825) Train Loss: 5.7326, Train Steps/Sec: 1.12 +[2025-04-27 22:36:34] (step=0083850) Train Loss: 5.7449, Train Steps/Sec: 0.92 +[2025-04-27 22:36:56] (step=0083875) Train Loss: 5.7893, Train Steps/Sec: 1.12 +[2025-04-27 22:37:18] (step=0083900) Train Loss: 5.7186, Train Steps/Sec: 1.12 +[2025-04-27 22:37:41] (step=0083925) Train Loss: 5.7331, Train Steps/Sec: 1.13 +[2025-04-27 22:38:03] (step=0083950) Train Loss: 5.7431, Train Steps/Sec: 1.12 +[2025-04-27 22:38:25] (step=0083975) Train Loss: 5.6727, Train Steps/Sec: 1.12 +[2025-04-27 22:38:48] (step=0084000) Train Loss: 5.7709, Train Steps/Sec: 1.12 +[2025-04-27 22:38:48] text_config is None. Initializing the text config with default values (`OPTConfig`). +[2025-04-27 22:45:06] Finish Eval in 84000 steps... +[2025-04-27 22:45:26] Saved checkpoint to checkpoint/X2I_UltraEdiit_fourmask_LLava_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_filtered_objects_4mask_1280_context/003-GPT-XL/checkpoints/0084000.pt +[2025-04-27 22:45:29] Removed old checkpoint: checkpoint/X2I_UltraEdiit_fourmask_LLava_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_filtered_objects_4mask_1280_context/003-GPT-XL/checkpoints/0082000.pt +[2025-04-27 22:45:51] (step=0084025) Train Loss: 5.6930, Train Steps/Sec: 0.06 +[2025-04-27 22:46:13] (step=0084050) Train Loss: 5.7144, Train Steps/Sec: 1.13 +[2025-04-27 22:46:35] (step=0084075) Train Loss: 5.7187, Train Steps/Sec: 1.13 +[2025-04-27 22:46:58] (step=0084100) Train Loss: 5.6925, Train Steps/Sec: 1.12 +[2025-04-27 22:47:20] (step=0084125) Train Loss: 5.7306, Train Steps/Sec: 1.12 +[2025-04-27 22:47:42] (step=0084150) Train Loss: 5.7394, Train Steps/Sec: 1.12 +[2025-04-27 22:48:05] (step=0084175) Train Loss: 5.7025, Train Steps/Sec: 1.12 +[2025-04-27 22:48:27] (step=0084200) Train Loss: 5.7623, Train Steps/Sec: 1.12 +[2025-04-27 22:48:49] (step=0084225) Train Loss: 5.6858, Train Steps/Sec: 1.12 +[2025-04-27 22:49:11] (step=0084250) Train Loss: 5.7613, Train Steps/Sec: 1.12 +[2025-04-27 22:49:34] (step=0084275) Train Loss: 5.7183, Train Steps/Sec: 1.12 +[2025-04-27 22:49:56] (step=0084300) Train Loss: 5.7136, Train Steps/Sec: 1.12 +[2025-04-27 22:50:18] (step=0084325) Train Loss: 5.6908, Train Steps/Sec: 1.13 +[2025-04-27 22:50:40] (step=0084350) Train Loss: 5.7028, Train Steps/Sec: 1.12 +[2025-04-27 22:51:03] (step=0084375) Train Loss: 5.7411, Train Steps/Sec: 1.12 +[2025-04-27 22:51:25] (step=0084400) Train Loss: 5.6990, Train Steps/Sec: 1.12 +[2025-04-27 22:51:47] (step=0084425) Train Loss: 5.6942, Train Steps/Sec: 1.12 +[2025-04-27 22:52:09] (step=0084450) Train Loss: 5.7551, Train Steps/Sec: 1.13 +[2025-04-27 22:52:32] (step=0084475) Train Loss: 5.6724, Train Steps/Sec: 1.12 +[2025-04-27 22:52:54] (step=0084500) Train Loss: 5.7127, Train Steps/Sec: 1.12 +[2025-04-27 22:53:16] (step=0084525) Train Loss: 5.7217, Train Steps/Sec: 1.12 +[2025-04-27 22:53:38] (step=0084550) Train Loss: 5.7166, Train Steps/Sec: 1.12 +[2025-04-27 22:54:01] (step=0084575) Train Loss: 5.6808, Train Steps/Sec: 1.12 +[2025-04-27 22:54:23] (step=0084600) Train Loss: 5.7758, Train Steps/Sec: 1.12 +[2025-04-27 22:54:54] (step=0084625) Train Loss: 5.7479, Train Steps/Sec: 0.81 +[2025-04-27 22:55:21] (step=0084650) Train Loss: 5.6793, Train Steps/Sec: 0.94 +[2025-04-27 22:55:48] (step=0084675) Train Loss: 5.7269, Train Steps/Sec: 0.91 +[2025-04-27 22:56:10] (step=0084700) Train Loss: 5.7340, Train Steps/Sec: 1.12 +[2025-04-27 22:56:33] (step=0084725) Train Loss: 5.7289, Train Steps/Sec: 1.12 +[2025-04-27 22:56:55] (step=0084750) Train Loss: 5.7107, Train Steps/Sec: 1.12 +[2025-04-27 22:57:17] (step=0084775) Train Loss: 5.6654, Train Steps/Sec: 1.12 +[2025-04-27 22:57:40] (step=0084800) Train Loss: 5.7324, Train Steps/Sec: 1.12 +[2025-04-27 22:58:02] (step=0084825) Train Loss: 5.6362, Train Steps/Sec: 1.12 +[2025-04-27 22:58:24] (step=0084850) Train Loss: 5.6456, Train Steps/Sec: 1.12 +[2025-04-27 22:58:47] (step=0084875) Train Loss: 5.7357, Train Steps/Sec: 1.12 +[2025-04-27 22:59:09] (step=0084900) Train Loss: 5.6527, Train Steps/Sec: 1.12 +[2025-04-27 22:59:31] (step=0084925) Train Loss: 5.7034, Train Steps/Sec: 1.12 +[2025-04-27 22:59:53] (step=0084950) Train Loss: 5.7067, Train Steps/Sec: 1.12 +[2025-04-27 23:00:16] (step=0084975) Train Loss: 5.6913, Train Steps/Sec: 1.12 +[2025-04-27 23:00:38] (step=0085000) Train Loss: 5.6940, Train Steps/Sec: 1.12 +[2025-04-27 23:01:00] (step=0085025) Train Loss: 5.7046, Train Steps/Sec: 1.12 +[2025-04-27 23:01:27] (step=0085050) Train Loss: 5.6951, Train Steps/Sec: 0.92 +[2025-04-27 23:01:50] (step=0085075) Train Loss: 5.6815, Train Steps/Sec: 1.12 +[2025-04-27 23:02:12] (step=0085100) Train Loss: 5.7035, Train Steps/Sec: 1.12 +[2025-04-27 23:02:34] (step=0085125) Train Loss: 5.7365, Train Steps/Sec: 1.12 +[2025-04-27 23:02:57] (step=0085150) Train Loss: 5.7258, Train Steps/Sec: 1.12 +[2025-04-27 23:03:19] (step=0085175) Train Loss: 5.6694, Train Steps/Sec: 1.12 +[2025-04-27 23:03:41] (step=0085200) Train Loss: 5.7110, Train Steps/Sec: 1.12 +[2025-04-27 23:04:04] (step=0085225) Train Loss: 5.6965, Train Steps/Sec: 1.12 +[2025-04-27 23:04:26] (step=0085250) Train Loss: 5.6834, Train Steps/Sec: 1.12 +[2025-04-27 23:04:48] (step=0085275) Train Loss: 5.7014, Train Steps/Sec: 1.12 +[2025-04-27 23:05:10] (step=0085300) Train Loss: 5.6944, Train Steps/Sec: 1.12 +[2025-04-27 23:05:33] (step=0085325) Train Loss: 5.7178, Train Steps/Sec: 1.12 +[2025-04-27 23:05:55] (step=0085350) Train Loss: 5.6632, Train Steps/Sec: 1.12 +[2025-04-27 23:06:17] (step=0085375) Train Loss: 5.7104, Train Steps/Sec: 1.12 +[2025-04-27 23:06:40] (step=0085400) Train Loss: 5.6982, Train Steps/Sec: 1.12 +[2025-04-27 23:07:02] (step=0085425) Train Loss: 5.6735, Train Steps/Sec: 1.12 +[2025-04-27 23:07:24] (step=0085450) Train Loss: 5.6979, Train Steps/Sec: 1.12 +[2025-04-27 23:07:46] (step=0085475) Train Loss: 5.6812, Train Steps/Sec: 1.12 +[2025-04-27 23:08:09] (step=0085500) Train Loss: 5.7029, Train Steps/Sec: 1.12 +[2025-04-27 23:08:31] (step=0085525) Train Loss: 5.6764, Train Steps/Sec: 1.12 +[2025-04-27 23:08:53] (step=0085550) Train Loss: 5.7288, Train Steps/Sec: 1.12 +[2025-04-27 23:09:16] (step=0085575) Train Loss: 5.7521, Train Steps/Sec: 1.12 +[2025-04-27 23:09:38] (step=0085600) Train Loss: 5.6973, Train Steps/Sec: 1.12 +[2025-04-27 23:10:00] (step=0085625) Train Loss: 5.6637, Train Steps/Sec: 1.12 +[2025-04-27 23:10:22] (step=0085650) Train Loss: 5.7091, Train Steps/Sec: 1.12 +[2025-04-27 23:10:45] (step=0085675) Train Loss: 5.7254, Train Steps/Sec: 1.12 +[2025-04-27 23:11:07] (step=0085700) Train Loss: 5.7016, Train Steps/Sec: 1.12 +[2025-04-27 23:11:29] (step=0085725) Train Loss: 5.6822, Train Steps/Sec: 1.12 +[2025-04-27 23:11:52] (step=0085750) Train Loss: 5.7104, Train Steps/Sec: 1.12 +[2025-04-27 23:12:14] (step=0085775) Train Loss: 5.6568, Train Steps/Sec: 1.12 +[2025-04-27 23:12:36] (step=0085800) Train Loss: 5.6148, Train Steps/Sec: 1.12 +[2025-04-27 23:12:58] (step=0085825) Train Loss: 5.7155, Train Steps/Sec: 1.12 +[2025-04-27 23:13:21] (step=0085850) Train Loss: 5.7031, Train Steps/Sec: 1.12 +[2025-04-27 23:13:43] (step=0085875) Train Loss: 5.6866, Train Steps/Sec: 1.12 +[2025-04-27 23:14:05] (step=0085900) Train Loss: 5.7289, Train Steps/Sec: 1.12 +[2025-04-27 23:14:28] (step=0085925) Train Loss: 5.7308, Train Steps/Sec: 1.12 +[2025-04-27 23:14:50] (step=0085950) Train Loss: 5.6454, Train Steps/Sec: 1.12 +[2025-04-27 23:15:12] (step=0085975) Train Loss: 5.6625, Train Steps/Sec: 1.12 +[2025-04-27 23:15:35] (step=0086000) Train Loss: 5.6339, Train Steps/Sec: 1.12 +[2025-04-27 23:15:35] text_config is None. Initializing the text config with default values (`OPTConfig`). +[2025-04-27 23:21:53] Finish Eval in 86000 steps... +[2025-04-27 23:22:12] Saved checkpoint to checkpoint/X2I_UltraEdiit_fourmask_LLava_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_filtered_objects_4mask_1280_context/003-GPT-XL/checkpoints/0086000.pt +[2025-04-27 23:22:14] Removed old checkpoint: checkpoint/X2I_UltraEdiit_fourmask_LLava_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_filtered_objects_4mask_1280_context/003-GPT-XL/checkpoints/0084000.pt +[2025-04-27 23:22:37] (step=0086025) Train Loss: 5.6778, Train Steps/Sec: 0.06 +[2025-04-27 23:22:59] (step=0086050) Train Loss: 5.7297, Train Steps/Sec: 1.12 +[2025-04-27 23:23:21] (step=0086075) Train Loss: 5.6614, Train Steps/Sec: 1.13 +[2025-04-27 23:23:44] (step=0086100) Train Loss: 5.6788, Train Steps/Sec: 1.12 +[2025-04-27 23:24:06] (step=0086125) Train Loss: 5.7083, Train Steps/Sec: 1.12 +[2025-04-27 23:24:28] (step=0086150) Train Loss: 5.7718, Train Steps/Sec: 1.12 +[2025-04-27 23:24:50] (step=0086175) Train Loss: 5.6889, Train Steps/Sec: 1.12 +[2025-04-27 23:25:13] (step=0086200) Train Loss: 5.6899, Train Steps/Sec: 1.12 +[2025-04-27 23:25:35] (step=0086225) Train Loss: 5.7252, Train Steps/Sec: 1.12 +[2025-04-27 23:25:57] (step=0086250) Train Loss: 5.7197, Train Steps/Sec: 1.12 +[2025-04-27 23:26:20] (step=0086275) Train Loss: 5.7082, Train Steps/Sec: 1.12 +[2025-04-27 23:26:42] (step=0086300) Train Loss: 5.6963, Train Steps/Sec: 1.12 +[2025-04-27 23:27:04] (step=0086325) Train Loss: 5.6967, Train Steps/Sec: 1.12 +[2025-04-27 23:27:26] (step=0086350) Train Loss: 5.6815, Train Steps/Sec: 1.12 +[2025-04-27 23:27:49] (step=0086375) Train Loss: 5.7694, Train Steps/Sec: 1.12 +[2025-04-27 23:28:11] (step=0086400) Train Loss: 5.7587, Train Steps/Sec: 1.12 +[2025-04-27 23:28:33] (step=0086425) Train Loss: 5.6572, Train Steps/Sec: 1.12 +[2025-04-27 23:28:55] (step=0086450) Train Loss: 5.7088, Train Steps/Sec: 1.12 +[2025-04-27 23:29:18] (step=0086475) Train Loss: 5.6921, Train Steps/Sec: 1.12 +[2025-04-27 23:29:40] (step=0086500) Train Loss: 5.7787, Train Steps/Sec: 1.12 +[2025-04-27 23:30:02] (step=0086525) Train Loss: 5.7279, Train Steps/Sec: 1.12 +[2025-04-27 23:30:25] (step=0086550) Train Loss: 5.7070, Train Steps/Sec: 1.10 +[2025-04-27 23:30:47] (step=0086575) Train Loss: 5.7498, Train Steps/Sec: 1.12 +[2025-04-27 23:31:09] (step=0086600) Train Loss: 5.6886, Train Steps/Sec: 1.12 +[2025-04-27 23:31:32] (step=0086625) Train Loss: 5.7292, Train Steps/Sec: 1.12 +[2025-04-27 23:31:54] (step=0086650) Train Loss: 5.6942, Train Steps/Sec: 1.12 +[2025-04-27 23:32:16] (step=0086675) Train Loss: 5.6816, Train Steps/Sec: 1.12 +[2025-04-27 23:32:39] (step=0086700) Train Loss: 5.7893, Train Steps/Sec: 1.12 +[2025-04-27 23:33:01] (step=0086725) Train Loss: 5.6266, Train Steps/Sec: 1.12 +[2025-04-27 23:33:23] (step=0086750) Train Loss: 5.7318, Train Steps/Sec: 1.12 +[2025-04-27 23:33:45] (step=0086775) Train Loss: 5.6662, Train Steps/Sec: 1.12 +[2025-04-27 23:34:08] (step=0086800) Train Loss: 5.7433, Train Steps/Sec: 1.12 +[2025-04-27 23:34:30] (step=0086825) Train Loss: 5.7196, Train Steps/Sec: 1.12 +[2025-04-27 23:34:52] (step=0086850) Train Loss: 5.6787, Train Steps/Sec: 1.12 +[2025-04-27 23:35:15] (step=0086875) Train Loss: 5.6985, Train Steps/Sec: 1.12 +[2025-04-27 23:35:37] (step=0086900) Train Loss: 5.7095, Train Steps/Sec: 1.12 +[2025-04-27 23:35:59] (step=0086925) Train Loss: 5.6857, Train Steps/Sec: 1.12 +[2025-04-27 23:36:21] (step=0086950) Train Loss: 5.7469, Train Steps/Sec: 1.12 +[2025-04-27 23:36:44] (step=0086975) Train Loss: 5.6917, Train Steps/Sec: 1.12 +[2025-04-27 23:37:06] (step=0087000) Train Loss: 5.7376, Train Steps/Sec: 1.12 +[2025-04-27 23:37:28] (step=0087025) Train Loss: 5.6663, Train Steps/Sec: 1.12 +[2025-04-27 23:37:51] (step=0087050) Train Loss: 5.6541, Train Steps/Sec: 1.12 +[2025-04-27 23:38:13] (step=0087075) Train Loss: 5.6086, Train Steps/Sec: 1.12 +[2025-04-27 23:38:35] (step=0087100) Train Loss: 5.6115, Train Steps/Sec: 1.12 +[2025-04-27 23:38:57] (step=0087125) Train Loss: 5.7014, Train Steps/Sec: 1.12 +[2025-04-27 23:39:20] (step=0087150) Train Loss: 5.6933, Train Steps/Sec: 1.12 +[2025-04-27 23:39:42] (step=0087175) Train Loss: 5.7770, Train Steps/Sec: 1.12 +[2025-04-27 23:40:04] (step=0087200) Train Loss: 5.6982, Train Steps/Sec: 1.12 +[2025-04-27 23:40:27] (step=0087225) Train Loss: 5.7127, Train Steps/Sec: 1.12 +[2025-04-27 23:40:49] (step=0087250) Train Loss: 5.6595, Train Steps/Sec: 1.12 +[2025-04-27 23:41:11] (step=0087275) Train Loss: 5.6859, Train Steps/Sec: 1.12 +[2025-04-27 23:41:34] (step=0087300) Train Loss: 5.6539, Train Steps/Sec: 1.12 +[2025-04-27 23:41:56] (step=0087325) Train Loss: 5.7166, Train Steps/Sec: 1.12 +[2025-04-27 23:42:18] (step=0087350) Train Loss: 5.7141, Train Steps/Sec: 1.12 +[2025-04-27 23:42:41] (step=0087375) Train Loss: 5.6517, Train Steps/Sec: 1.12 +[2025-04-27 23:43:03] (step=0087400) Train Loss: 5.7341, Train Steps/Sec: 1.12 +[2025-04-27 23:43:25] (step=0087425) Train Loss: 5.7334, Train Steps/Sec: 1.12 +[2025-04-27 23:43:47] (step=0087450) Train Loss: 5.5675, Train Steps/Sec: 1.12 +[2025-04-27 23:44:10] (step=0087475) Train Loss: 5.7574, Train Steps/Sec: 1.12 +[2025-04-27 23:44:32] (step=0087500) Train Loss: 5.6816, Train Steps/Sec: 1.12 +[2025-04-27 23:44:54] (step=0087525) Train Loss: 5.7332, Train Steps/Sec: 1.12 +[2025-04-27 23:45:17] (step=0087550) Train Loss: 5.6187, Train Steps/Sec: 1.12 +[2025-04-27 23:45:39] (step=0087575) Train Loss: 5.7431, Train Steps/Sec: 1.12 +[2025-04-27 23:46:01] (step=0087600) Train Loss: 5.6970, Train Steps/Sec: 1.12 +[2025-04-27 23:46:24] (step=0087625) Train Loss: 5.7150, Train Steps/Sec: 1.12 +[2025-04-27 23:46:46] (step=0087650) Train Loss: 5.7127, Train Steps/Sec: 1.13 +[2025-04-27 23:47:08] (step=0087675) Train Loss: 5.7622, Train Steps/Sec: 1.12 +[2025-04-27 23:47:30] (step=0087700) Train Loss: 5.7315, Train Steps/Sec: 1.12 +[2025-04-27 23:47:53] (step=0087725) Train Loss: 5.7446, Train Steps/Sec: 1.12 +[2025-04-27 23:48:15] (step=0087750) Train Loss: 5.7517, Train Steps/Sec: 1.12 +[2025-04-27 23:48:37] (step=0087775) Train Loss: 5.6449, Train Steps/Sec: 1.12 +[2025-04-27 23:49:00] (step=0087800) Train Loss: 5.6657, Train Steps/Sec: 1.12 +[2025-04-27 23:49:22] (step=0087825) Train Loss: 5.7055, Train Steps/Sec: 1.12 +[2025-04-27 23:49:49] (step=0087850) Train Loss: 5.6516, Train Steps/Sec: 0.93 +[2025-04-27 23:50:11] (step=0087875) Train Loss: 5.6452, Train Steps/Sec: 1.12 +[2025-04-27 23:50:33] (step=0087900) Train Loss: 5.6504, Train Steps/Sec: 1.12 +[2025-04-27 23:50:56] (step=0087925) Train Loss: 5.6880, Train Steps/Sec: 1.12 +[2025-04-27 23:51:18] (step=0087950) Train Loss: 5.7783, Train Steps/Sec: 1.12 +[2025-04-27 23:51:40] (step=0087975) Train Loss: 5.7572, Train Steps/Sec: 1.12 +[2025-04-27 23:52:03] (step=0088000) Train Loss: 5.6709, Train Steps/Sec: 1.12 +[2025-04-27 23:52:03] text_config is None. Initializing the text config with default values (`OPTConfig`). +[2025-04-27 23:58:22] Finish Eval in 88000 steps... +[2025-04-27 23:58:42] Saved checkpoint to checkpoint/X2I_UltraEdiit_fourmask_LLava_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_filtered_objects_4mask_1280_context/003-GPT-XL/checkpoints/0088000.pt +[2025-04-27 23:58:44] Removed old checkpoint: checkpoint/X2I_UltraEdiit_fourmask_LLava_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_filtered_objects_4mask_1280_context/003-GPT-XL/checkpoints/0086000.pt +[2025-04-27 23:59:07] (step=0088025) Train Loss: 5.7294, Train Steps/Sec: 0.06 +[2025-04-27 23:59:29] (step=0088050) Train Loss: 5.7266, Train Steps/Sec: 1.12 +[2025-04-27 23:59:51] (step=0088075) Train Loss: 5.7064, Train Steps/Sec: 1.12 +[2025-04-28 00:00:13] (step=0088100) Train Loss: 5.6797, Train Steps/Sec: 1.12 +[2025-04-28 00:00:41] (step=0088125) Train Loss: 5.6539, Train Steps/Sec: 0.92 +[2025-04-28 00:01:03] (step=0088150) Train Loss: 5.6855, Train Steps/Sec: 1.12 +[2025-04-28 00:01:25] (step=0088175) Train Loss: 5.7131, Train Steps/Sec: 1.12 +[2025-04-28 00:01:47] (step=0088200) Train Loss: 5.6872, Train Steps/Sec: 1.12 +[2025-04-28 00:02:10] (step=0088225) Train Loss: 5.7532, Train Steps/Sec: 1.12 +[2025-04-28 00:02:32] (step=0088250) Train Loss: 5.6946, Train Steps/Sec: 1.12 +[2025-04-28 00:02:54] (step=0088275) Train Loss: 5.7084, Train Steps/Sec: 1.12 +[2025-04-28 00:03:16] (step=0088300) Train Loss: 5.7004, Train Steps/Sec: 1.12 +[2025-04-28 00:03:39] (step=0088325) Train Loss: 5.7864, Train Steps/Sec: 1.12 +[2025-04-28 00:04:01] (step=0088350) Train Loss: 5.6601, Train Steps/Sec: 1.12 +[2025-04-28 00:04:23] (step=0088375) Train Loss: 5.6689, Train Steps/Sec: 1.12 +[2025-04-28 00:04:46] (step=0088400) Train Loss: 5.7123, Train Steps/Sec: 1.12 +[2025-04-28 00:05:08] (step=0088425) Train Loss: 5.7291, Train Steps/Sec: 1.12 +[2025-04-28 00:05:30] (step=0088450) Train Loss: 5.6779, Train Steps/Sec: 1.11 +[2025-04-28 00:05:53] (step=0088475) Train Loss: 5.7649, Train Steps/Sec: 1.12 +[2025-04-28 00:06:15] (step=0088500) Train Loss: 5.7453, Train Steps/Sec: 1.12 +[2025-04-28 00:06:37] (step=0088525) Train Loss: 5.6938, Train Steps/Sec: 1.12 +[2025-04-28 00:06:59] (step=0088550) Train Loss: 5.7757, Train Steps/Sec: 1.12 +[2025-04-28 00:07:22] (step=0088575) Train Loss: 5.7305, Train Steps/Sec: 1.12 +[2025-04-28 00:07:44] (step=0088600) Train Loss: 5.7315, Train Steps/Sec: 1.11 +[2025-04-28 00:08:06] (step=0088625) Train Loss: 5.6790, Train Steps/Sec: 1.12 +[2025-04-28 00:08:29] (step=0088650) Train Loss: 5.6662, Train Steps/Sec: 1.12 +[2025-04-28 00:08:51] (step=0088675) Train Loss: 5.6666, Train Steps/Sec: 1.12 +[2025-04-28 00:09:13] (step=0088700) Train Loss: 5.6861, Train Steps/Sec: 1.12 +[2025-04-28 00:09:35] (step=0088725) Train Loss: 5.6688, Train Steps/Sec: 1.12 +[2025-04-28 00:09:58] (step=0088750) Train Loss: 5.7686, Train Steps/Sec: 1.12 +[2025-04-28 00:10:20] (step=0088775) Train Loss: 5.7248, Train Steps/Sec: 1.12 +[2025-04-28 00:10:42] (step=0088800) Train Loss: 5.7078, Train Steps/Sec: 1.12 +[2025-04-28 00:11:05] (step=0088825) Train Loss: 5.7366, Train Steps/Sec: 1.12 +[2025-04-28 00:11:27] (step=0088850) Train Loss: 5.6987, Train Steps/Sec: 1.12 +[2025-04-28 00:11:49] (step=0088875) Train Loss: 5.7016, Train Steps/Sec: 1.12 +[2025-04-28 00:12:11] (step=0088900) Train Loss: 5.6910, Train Steps/Sec: 1.12 +[2025-04-28 00:12:34] (step=0088925) Train Loss: 5.7167, Train Steps/Sec: 1.12 +[2025-04-28 00:13:04] (step=0088950) Train Loss: 5.6573, Train Steps/Sec: 0.81 +[2025-04-28 00:13:31] (step=0088975) Train Loss: 5.6901, Train Steps/Sec: 0.94 +[2025-04-28 00:13:59] (step=0089000) Train Loss: 5.6843, Train Steps/Sec: 0.91 +[2025-04-28 00:14:21] (step=0089025) Train Loss: 5.7196, Train Steps/Sec: 1.12 +[2025-04-28 00:14:43] (step=0089050) Train Loss: 5.7204, Train Steps/Sec: 1.12 +[2025-04-28 00:15:05] (step=0089075) Train Loss: 5.7048, Train Steps/Sec: 1.13 +[2025-04-28 00:15:28] (step=0089100) Train Loss: 5.6859, Train Steps/Sec: 1.12 +[2025-04-28 00:15:50] (step=0089125) Train Loss: 5.6174, Train Steps/Sec: 1.12 +[2025-04-28 00:16:12] (step=0089150) Train Loss: 5.6527, Train Steps/Sec: 1.12 +[2025-04-28 00:16:34] (step=0089175) Train Loss: 5.6826, Train Steps/Sec: 1.12 +[2025-04-28 00:16:57] (step=0089200) Train Loss: 5.7478, Train Steps/Sec: 1.12 +[2025-04-28 00:17:19] (step=0089225) Train Loss: 5.6360, Train Steps/Sec: 1.12 +[2025-04-28 00:17:41] (step=0089250) Train Loss: 5.7251, Train Steps/Sec: 1.12 +[2025-04-28 00:18:04] (step=0089275) Train Loss: 5.6973, Train Steps/Sec: 1.12 +[2025-04-28 00:18:26] (step=0089300) Train Loss: 5.6856, Train Steps/Sec: 1.12 +[2025-04-28 00:18:48] (step=0089325) Train Loss: 5.6942, Train Steps/Sec: 1.12 +[2025-04-28 00:19:10] (step=0089350) Train Loss: 5.7047, Train Steps/Sec: 1.12 +[2025-04-28 00:19:33] (step=0089375) Train Loss: 5.6975, Train Steps/Sec: 1.12 +[2025-04-28 00:19:55] (step=0089400) Train Loss: 5.6230, Train Steps/Sec: 1.12 +[2025-04-28 00:20:17] (step=0089425) Train Loss: 5.7248, Train Steps/Sec: 1.12 +[2025-04-28 00:20:45] (step=0089450) Train Loss: 5.7345, Train Steps/Sec: 0.92 +[2025-04-28 00:21:07] (step=0089475) Train Loss: 5.7596, Train Steps/Sec: 1.13 +[2025-04-28 00:21:29] (step=0089500) Train Loss: 5.7198, Train Steps/Sec: 1.12 +[2025-04-28 00:21:51] (step=0089525) Train Loss: 5.7153, Train Steps/Sec: 1.12 +[2025-04-28 00:22:14] (step=0089550) Train Loss: 5.6707, Train Steps/Sec: 1.12 +[2025-04-28 00:22:36] (step=0089575) Train Loss: 5.7229, Train Steps/Sec: 1.12 +[2025-04-28 00:22:58] (step=0089600) Train Loss: 5.6959, Train Steps/Sec: 1.12 +[2025-04-28 00:23:20] (step=0089625) Train Loss: 5.6657, Train Steps/Sec: 1.12 +[2025-04-28 00:23:43] (step=0089650) Train Loss: 5.6923, Train Steps/Sec: 1.12 +[2025-04-28 00:24:05] (step=0089675) Train Loss: 5.7061, Train Steps/Sec: 1.12 +[2025-04-28 00:24:27] (step=0089700) Train Loss: 5.7874, Train Steps/Sec: 1.12 +[2025-04-28 00:24:49] (step=0089725) Train Loss: 5.7079, Train Steps/Sec: 1.12 +[2025-04-28 00:25:12] (step=0089750) Train Loss: 5.7393, Train Steps/Sec: 1.12 +[2025-04-28 00:25:34] (step=0089775) Train Loss: 5.7192, Train Steps/Sec: 1.12 +[2025-04-28 00:25:56] (step=0089800) Train Loss: 5.6587, Train Steps/Sec: 1.12 +[2025-04-28 00:26:19] (step=0089825) Train Loss: 5.6806, Train Steps/Sec: 1.12 +[2025-04-28 00:26:41] (step=0089850) Train Loss: 5.6442, Train Steps/Sec: 1.12 +[2025-04-28 00:27:03] (step=0089875) Train Loss: 5.6562, Train Steps/Sec: 1.12 +[2025-04-28 00:27:26] (step=0089900) Train Loss: 5.7224, Train Steps/Sec: 1.12 +[2025-04-28 00:27:48] (step=0089925) Train Loss: 5.7055, Train Steps/Sec: 1.12 +[2025-04-28 00:28:10] (step=0089950) Train Loss: 5.6883, Train Steps/Sec: 1.12 +[2025-04-28 00:28:32] (step=0089975) Train Loss: 5.6880, Train Steps/Sec: 1.12 +[2025-04-28 00:28:55] (step=0090000) Train Loss: 5.7202, Train Steps/Sec: 1.12 +[2025-04-28 00:28:55] text_config is None. Initializing the text config with default values (`OPTConfig`). +[2025-04-28 00:35:14] Finish Eval in 90000 steps... +[2025-04-28 00:35:34] Saved checkpoint to checkpoint/X2I_UltraEdiit_fourmask_LLava_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_filtered_objects_4mask_1280_context/003-GPT-XL/checkpoints/0090000.pt +[2025-04-28 00:35:36] Removed old checkpoint: checkpoint/X2I_UltraEdiit_fourmask_LLava_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_filtered_objects_4mask_1280_context/003-GPT-XL/checkpoints/0088000.pt +[2025-04-28 00:35:58] (step=0090025) Train Loss: 5.6885, Train Steps/Sec: 0.06 +[2025-04-28 00:36:21] (step=0090050) Train Loss: 5.7443, Train Steps/Sec: 1.12 +[2025-04-28 00:36:43] (step=0090075) Train Loss: 5.6504, Train Steps/Sec: 1.12 +[2025-04-28 00:37:05] (step=0090100) Train Loss: 5.7389, Train Steps/Sec: 1.12 +[2025-04-28 00:37:28] (step=0090125) Train Loss: 5.6784, Train Steps/Sec: 1.12 +[2025-04-28 00:37:50] (step=0090150) Train Loss: 5.7046, Train Steps/Sec: 1.12 +[2025-04-28 00:38:12] (step=0090175) Train Loss: 5.6435, Train Steps/Sec: 1.12 +[2025-04-28 00:38:35] (step=0090200) Train Loss: 5.6938, Train Steps/Sec: 1.12 +[2025-04-28 00:38:57] (step=0090225) Train Loss: 5.6632, Train Steps/Sec: 1.12 +[2025-04-28 00:39:19] (step=0090250) Train Loss: 5.6851, Train Steps/Sec: 1.12 +[2025-04-28 00:39:42] (step=0090275) Train Loss: 5.6276, Train Steps/Sec: 1.12 +[2025-04-28 00:40:04] (step=0090300) Train Loss: 5.6471, Train Steps/Sec: 1.12 +[2025-04-28 00:40:26] (step=0090325) Train Loss: 5.6571, Train Steps/Sec: 1.12 +[2025-04-28 00:40:49] (step=0090350) Train Loss: 5.7344, Train Steps/Sec: 1.10 +[2025-04-28 00:41:11] (step=0090375) Train Loss: 5.7017, Train Steps/Sec: 1.12 +[2025-04-28 00:41:34] (step=0090400) Train Loss: 5.6685, Train Steps/Sec: 1.12 +[2025-04-28 00:41:56] (step=0090425) Train Loss: 5.7942, Train Steps/Sec: 1.12 +[2025-04-28 00:42:18] (step=0090450) Train Loss: 5.7237, Train Steps/Sec: 1.12 +[2025-04-28 00:42:41] (step=0090475) Train Loss: 5.6767, Train Steps/Sec: 1.12 +[2025-04-28 00:43:03] (step=0090500) Train Loss: 5.7121, Train Steps/Sec: 1.12 +[2025-04-28 00:43:25] (step=0090525) Train Loss: 5.7396, Train Steps/Sec: 1.12 +[2025-04-28 00:43:48] (step=0090550) Train Loss: 5.7267, Train Steps/Sec: 1.12 +[2025-04-28 00:44:10] (step=0090575) Train Loss: 5.6392, Train Steps/Sec: 1.12 +[2025-04-28 00:44:32] (step=0090600) Train Loss: 5.7545, Train Steps/Sec: 1.12 +[2025-04-28 00:44:55] (step=0090625) Train Loss: 5.6585, Train Steps/Sec: 1.12 +[2025-04-28 00:45:17] (step=0090650) Train Loss: 5.6555, Train Steps/Sec: 1.12 +[2025-04-28 00:45:39] (step=0090675) Train Loss: 5.7466, Train Steps/Sec: 1.12 +[2025-04-28 00:46:01] (step=0090700) Train Loss: 5.6684, Train Steps/Sec: 1.12 +[2025-04-28 00:46:24] (step=0090725) Train Loss: 5.6406, Train Steps/Sec: 1.12 +[2025-04-28 00:46:46] (step=0090750) Train Loss: 5.6526, Train Steps/Sec: 1.12 +[2025-04-28 00:47:08] (step=0090775) Train Loss: 5.7470, Train Steps/Sec: 1.12 +[2025-04-28 00:47:31] (step=0090800) Train Loss: 5.6691, Train Steps/Sec: 1.12 +[2025-04-28 00:47:53] (step=0090825) Train Loss: 5.6375, Train Steps/Sec: 1.12 +[2025-04-28 00:48:15] (step=0090850) Train Loss: 5.7053, Train Steps/Sec: 1.12 +[2025-04-28 00:48:38] (step=0090875) Train Loss: 5.6599, Train Steps/Sec: 1.12 +[2025-04-28 00:49:00] (step=0090900) Train Loss: 5.7138, Train Steps/Sec: 1.12 +[2025-04-28 00:49:22] (step=0090925) Train Loss: 5.6693, Train Steps/Sec: 1.12 +[2025-04-28 00:49:45] (step=0090950) Train Loss: 5.7932, Train Steps/Sec: 1.12 +[2025-04-28 00:50:07] (step=0090975) Train Loss: 5.6900, Train Steps/Sec: 1.12 +[2025-04-28 00:50:29] (step=0091000) Train Loss: 5.7629, Train Steps/Sec: 1.12 +[2025-04-28 00:50:52] (step=0091025) Train Loss: 5.7294, Train Steps/Sec: 1.12 +[2025-04-28 00:51:14] (step=0091050) Train Loss: 5.6976, Train Steps/Sec: 1.12 +[2025-04-28 00:51:36] (step=0091075) Train Loss: 5.6914, Train Steps/Sec: 1.12 +[2025-04-28 00:51:58] (step=0091100) Train Loss: 5.6975, Train Steps/Sec: 1.12 +[2025-04-28 00:52:21] (step=0091125) Train Loss: 5.7786, Train Steps/Sec: 1.12 +[2025-04-28 00:52:43] (step=0091150) Train Loss: 5.6120, Train Steps/Sec: 1.12 +[2025-04-28 00:53:05] (step=0091175) Train Loss: 5.7638, Train Steps/Sec: 1.12 +[2025-04-28 00:53:28] (step=0091200) Train Loss: 5.7098, Train Steps/Sec: 1.12 +[2025-04-28 00:53:50] (step=0091225) Train Loss: 5.6783, Train Steps/Sec: 1.12 +[2025-04-28 00:54:12] (step=0091250) Train Loss: 5.7445, Train Steps/Sec: 1.12 +[2025-04-28 00:54:35] (step=0091275) Train Loss: 5.6933, Train Steps/Sec: 1.12 +[2025-04-28 00:54:57] (step=0091300) Train Loss: 5.7063, Train Steps/Sec: 1.12 +[2025-04-28 00:55:19] (step=0091325) Train Loss: 5.7635, Train Steps/Sec: 1.12 +[2025-04-28 00:55:41] (step=0091350) Train Loss: 5.7082, Train Steps/Sec: 1.12 +[2025-04-28 00:56:04] (step=0091375) Train Loss: 5.6910, Train Steps/Sec: 1.12 +[2025-04-28 00:56:26] (step=0091400) Train Loss: 5.8075, Train Steps/Sec: 1.12 +[2025-04-28 00:56:49] (step=0091425) Train Loss: 5.7414, Train Steps/Sec: 1.12 +[2025-04-28 00:57:11] (step=0091450) Train Loss: 5.6526, Train Steps/Sec: 1.12 +[2025-04-28 00:57:33] (step=0091475) Train Loss: 5.6707, Train Steps/Sec: 1.12 +[2025-04-28 00:57:55] (step=0091500) Train Loss: 5.6779, Train Steps/Sec: 1.12 +[2025-04-28 00:58:18] (step=0091525) Train Loss: 5.6514, Train Steps/Sec: 1.12 +[2025-04-28 00:58:40] (step=0091550) Train Loss: 5.6789, Train Steps/Sec: 1.12 +[2025-04-28 00:59:02] (step=0091575) Train Loss: 5.6942, Train Steps/Sec: 1.12 +[2025-04-28 00:59:25] (step=0091600) Train Loss: 5.7347, Train Steps/Sec: 1.12 +[2025-04-28 00:59:47] (step=0091625) Train Loss: 5.7033, Train Steps/Sec: 1.12 +[2025-04-28 01:00:09] (step=0091650) Train Loss: 5.6451, Train Steps/Sec: 1.12 +[2025-04-28 01:00:32] (step=0091675) Train Loss: 5.6340, Train Steps/Sec: 1.12 +[2025-04-28 01:00:54] (step=0091700) Train Loss: 5.6715, Train Steps/Sec: 1.12 +[2025-04-28 01:01:16] (step=0091725) Train Loss: 5.6150, Train Steps/Sec: 1.12 +[2025-04-28 01:01:39] (step=0091750) Train Loss: 5.8065, Train Steps/Sec: 1.12 +[2025-04-28 01:02:01] (step=0091775) Train Loss: 5.7151, Train Steps/Sec: 1.12 +[2025-04-28 01:02:23] (step=0091800) Train Loss: 5.6624, Train Steps/Sec: 1.12 +[2025-04-28 01:02:46] (step=0091825) Train Loss: 5.7143, Train Steps/Sec: 1.12 +[2025-04-28 01:03:08] (step=0091850) Train Loss: 5.6553, Train Steps/Sec: 1.12 +[2025-04-28 01:03:30] (step=0091875) Train Loss: 5.6677, Train Steps/Sec: 1.12 +[2025-04-28 01:03:53] (step=0091900) Train Loss: 5.7056, Train Steps/Sec: 1.12 +[2025-04-28 01:04:15] (step=0091925) Train Loss: 5.7193, Train Steps/Sec: 1.12 +[2025-04-28 01:04:37] (step=0091950) Train Loss: 5.6999, Train Steps/Sec: 1.12 +[2025-04-28 01:05:00] (step=0091975) Train Loss: 5.6834, Train Steps/Sec: 1.12 +[2025-04-28 01:05:22] (step=0092000) Train Loss: 5.7076, Train Steps/Sec: 1.12 +[2025-04-28 01:05:22] text_config is None. Initializing the text config with default values (`OPTConfig`). +[2025-04-28 01:11:40] Finish Eval in 92000 steps... +[2025-04-28 01:11:59] Saved checkpoint to checkpoint/X2I_UltraEdiit_fourmask_LLava_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_filtered_objects_4mask_1280_context/003-GPT-XL/checkpoints/0092000.pt +[2025-04-28 01:12:01] Removed old checkpoint: checkpoint/X2I_UltraEdiit_fourmask_LLava_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_filtered_objects_4mask_1280_context/003-GPT-XL/checkpoints/0090000.pt +[2025-04-28 01:12:24] (step=0092025) Train Loss: 5.6565, Train Steps/Sec: 0.06 +[2025-04-28 01:12:46] (step=0092050) Train Loss: 5.6869, Train Steps/Sec: 1.12 +[2025-04-28 01:13:08] (step=0092075) Train Loss: 5.7229, Train Steps/Sec: 1.12 +[2025-04-28 01:13:35] (step=0092100) Train Loss: 5.6278, Train Steps/Sec: 0.93 +[2025-04-28 01:13:57] (step=0092125) Train Loss: 5.8197, Train Steps/Sec: 1.12 +[2025-04-28 01:14:20] (step=0092150) Train Loss: 5.7098, Train Steps/Sec: 1.12 +[2025-04-28 01:14:42] (step=0092175) Train Loss: 5.6568, Train Steps/Sec: 1.12 +[2025-04-28 01:15:04] (step=0092200) Train Loss: 5.6617, Train Steps/Sec: 1.12 +[2025-04-28 01:15:27] (step=0092225) Train Loss: 5.7796, Train Steps/Sec: 1.12 +[2025-04-28 01:15:49] (step=0092250) Train Loss: 5.6753, Train Steps/Sec: 1.12 +[2025-04-28 01:16:12] (step=0092275) Train Loss: 5.7019, Train Steps/Sec: 1.12 +[2025-04-28 01:16:34] (step=0092300) Train Loss: 5.6507, Train Steps/Sec: 1.12 +[2025-04-28 01:16:56] (step=0092325) Train Loss: 5.6815, Train Steps/Sec: 1.12 +[2025-04-28 01:17:18] (step=0092350) Train Loss: 5.6913, Train Steps/Sec: 1.12 +[2025-04-28 01:17:41] (step=0092375) Train Loss: 5.7307, Train Steps/Sec: 1.13 +[2025-04-28 01:18:08] (step=0092400) Train Loss: 5.6658, Train Steps/Sec: 0.92 +[2025-04-28 01:18:30] (step=0092425) Train Loss: 5.6755, Train Steps/Sec: 1.12 +[2025-04-28 01:18:52] (step=0092450) Train Loss: 5.6396, Train Steps/Sec: 1.12 +[2025-04-28 01:19:15] (step=0092475) Train Loss: 5.6025, Train Steps/Sec: 1.12 +[2025-04-28 01:19:37] (step=0092500) Train Loss: 5.6244, Train Steps/Sec: 1.12 +[2025-04-28 01:19:59] (step=0092525) Train Loss: 5.7422, Train Steps/Sec: 1.12 +[2025-04-28 01:20:22] (step=0092550) Train Loss: 5.6400, Train Steps/Sec: 1.12 +[2025-04-28 01:20:44] (step=0092575) Train Loss: 5.6868, Train Steps/Sec: 1.12 +[2025-04-28 01:21:06] (step=0092600) Train Loss: 5.7238, Train Steps/Sec: 1.12 +[2025-04-28 01:21:28] (step=0092625) Train Loss: 5.6674, Train Steps/Sec: 1.12 +[2025-04-28 01:21:51] (step=0092650) Train Loss: 5.7250, Train Steps/Sec: 1.12 +[2025-04-28 01:22:13] (step=0092675) Train Loss: 5.6559, Train Steps/Sec: 1.12 +[2025-04-28 01:22:35] (step=0092700) Train Loss: 5.6414, Train Steps/Sec: 1.12 +[2025-04-28 01:22:58] (step=0092725) Train Loss: 5.6994, Train Steps/Sec: 1.12 +[2025-04-28 01:23:20] (step=0092750) Train Loss: 5.6047, Train Steps/Sec: 1.12 +[2025-04-28 01:23:42] (step=0092775) Train Loss: 5.7008, Train Steps/Sec: 1.12 +[2025-04-28 01:24:04] (step=0092800) Train Loss: 5.6411, Train Steps/Sec: 1.12 +[2025-04-28 01:24:27] (step=0092825) Train Loss: 5.6756, Train Steps/Sec: 1.12 +[2025-04-28 01:24:49] (step=0092850) Train Loss: 5.6824, Train Steps/Sec: 1.12 +[2025-04-28 01:25:11] (step=0092875) Train Loss: 5.7048, Train Steps/Sec: 1.12 +[2025-04-28 01:25:34] (step=0092900) Train Loss: 5.7817, Train Steps/Sec: 1.12 +[2025-04-28 01:25:56] (step=0092925) Train Loss: 5.7415, Train Steps/Sec: 1.12 +[2025-04-28 01:26:18] (step=0092950) Train Loss: 5.6841, Train Steps/Sec: 1.12 +[2025-04-28 01:26:40] (step=0092975) Train Loss: 5.6600, Train Steps/Sec: 1.12 +[2025-04-28 01:27:03] (step=0093000) Train Loss: 5.6771, Train Steps/Sec: 1.12 +[2025-04-28 01:27:25] (step=0093025) Train Loss: 5.6780, Train Steps/Sec: 1.12 +[2025-04-28 01:27:47] (step=0093050) Train Loss: 5.6731, Train Steps/Sec: 1.12 +[2025-04-28 01:28:10] (step=0093075) Train Loss: 5.7408, Train Steps/Sec: 1.12 +[2025-04-28 01:28:32] (step=0093100) Train Loss: 5.7415, Train Steps/Sec: 1.12 +[2025-04-28 01:28:54] (step=0093125) Train Loss: 5.6853, Train Steps/Sec: 1.12 +[2025-04-28 01:29:16] (step=0093150) Train Loss: 5.7227, Train Steps/Sec: 1.12 +[2025-04-28 01:29:39] (step=0093175) Train Loss: 5.6878, Train Steps/Sec: 1.12 +[2025-04-28 01:30:01] (step=0093200) Train Loss: 5.6709, Train Steps/Sec: 1.12 +[2025-04-28 01:30:23] (step=0093225) Train Loss: 5.6878, Train Steps/Sec: 1.12 +[2025-04-28 01:30:46] (step=0093250) Train Loss: 5.6325, Train Steps/Sec: 1.12 +[2025-04-28 01:31:16] (step=0093275) Train Loss: 5.6809, Train Steps/Sec: 0.82 +[2025-04-28 01:31:39] (step=0093300) Train Loss: 5.6762, Train Steps/Sec: 1.12 +[2025-04-28 01:32:05] (step=0093325) Train Loss: 5.6710, Train Steps/Sec: 0.94 +[2025-04-28 01:32:33] (step=0093350) Train Loss: 5.6285, Train Steps/Sec: 0.91 +[2025-04-28 01:32:55] (step=0093375) Train Loss: 5.7136, Train Steps/Sec: 1.12 +[2025-04-28 01:33:17] (step=0093400) Train Loss: 5.6827, Train Steps/Sec: 1.12 +[2025-04-28 01:33:40] (step=0093425) Train Loss: 5.7446, Train Steps/Sec: 1.12 +[2025-04-28 01:34:02] (step=0093450) Train Loss: 5.7183, Train Steps/Sec: 1.12 +[2025-04-28 01:34:24] (step=0093475) Train Loss: 5.6875, Train Steps/Sec: 1.12 +[2025-04-28 01:34:46] (step=0093500) Train Loss: 5.5967, Train Steps/Sec: 1.12 +[2025-04-28 01:35:09] (step=0093525) Train Loss: 5.7827, Train Steps/Sec: 1.12 +[2025-04-28 01:35:31] (step=0093550) Train Loss: 5.6466, Train Steps/Sec: 1.12 +[2025-04-28 01:35:53] (step=0093575) Train Loss: 5.6218, Train Steps/Sec: 1.12 +[2025-04-28 01:36:16] (step=0093600) Train Loss: 5.6942, Train Steps/Sec: 1.12 +[2025-04-28 01:36:38] (step=0093625) Train Loss: 5.6305, Train Steps/Sec: 1.12 +[2025-04-28 01:37:00] (step=0093650) Train Loss: 5.7153, Train Steps/Sec: 1.12 +[2025-04-28 01:37:23] (step=0093675) Train Loss: 5.6634, Train Steps/Sec: 1.12 +[2025-04-28 01:37:45] (step=0093700) Train Loss: 5.6470, Train Steps/Sec: 1.12 +[2025-04-28 01:38:07] (step=0093725) Train Loss: 5.6665, Train Steps/Sec: 1.12 +[2025-04-28 01:38:29] (step=0093750) Train Loss: 5.6987, Train Steps/Sec: 1.12 +[2025-04-28 01:38:52] (step=0093775) Train Loss: 5.6488, Train Steps/Sec: 1.12 +[2025-04-28 01:39:14] (step=0093800) Train Loss: 5.6664, Train Steps/Sec: 1.12 +[2025-04-28 01:39:36] (step=0093825) Train Loss: 5.6825, Train Steps/Sec: 1.12 +[2025-04-28 01:40:04] (step=0093850) Train Loss: 5.6860, Train Steps/Sec: 0.92 +[2025-04-28 01:40:26] (step=0093875) Train Loss: 5.7626, Train Steps/Sec: 1.12 +[2025-04-28 01:40:48] (step=0093900) Train Loss: 5.7313, Train Steps/Sec: 1.12 +[2025-04-28 01:41:10] (step=0093925) Train Loss: 5.6901, Train Steps/Sec: 1.12 +[2025-04-28 01:41:33] (step=0093950) Train Loss: 5.6110, Train Steps/Sec: 1.12 +[2025-04-28 01:41:55] (step=0093975) Train Loss: 5.6854, Train Steps/Sec: 1.12 +[2025-04-28 01:42:17] (step=0094000) Train Loss: 5.6774, Train Steps/Sec: 1.12 +[2025-04-28 01:42:17] text_config is None. Initializing the text config with default values (`OPTConfig`). +[2025-04-28 01:48:35] Finish Eval in 94000 steps... +[2025-04-28 01:48:55] Saved checkpoint to checkpoint/X2I_UltraEdiit_fourmask_LLava_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_filtered_objects_4mask_1280_context/003-GPT-XL/checkpoints/0094000.pt +[2025-04-28 01:48:57] Removed old checkpoint: checkpoint/X2I_UltraEdiit_fourmask_LLava_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_filtered_objects_4mask_1280_context/003-GPT-XL/checkpoints/0092000.pt +[2025-04-28 01:49:19] (step=0094025) Train Loss: 5.6328, Train Steps/Sec: 0.06 +[2025-04-28 01:49:42] (step=0094050) Train Loss: 5.7090, Train Steps/Sec: 1.12 +[2025-04-28 01:50:04] (step=0094075) Train Loss: 5.6303, Train Steps/Sec: 1.12 +[2025-04-28 01:50:26] (step=0094100) Train Loss: 5.7241, Train Steps/Sec: 1.12 +[2025-04-28 01:50:49] (step=0094125) Train Loss: 5.6866, Train Steps/Sec: 1.12 +[2025-04-28 01:51:11] (step=0094150) Train Loss: 5.6186, Train Steps/Sec: 1.12 +[2025-04-28 01:51:33] (step=0094175) Train Loss: 5.7162, Train Steps/Sec: 1.12 +[2025-04-28 01:51:56] (step=0094200) Train Loss: 5.6589, Train Steps/Sec: 1.11 +[2025-04-28 01:52:18] (step=0094225) Train Loss: 5.6063, Train Steps/Sec: 1.12 +[2025-04-28 01:52:41] (step=0094250) Train Loss: 5.6669, Train Steps/Sec: 1.12 +[2025-04-28 01:53:03] (step=0094275) Train Loss: 5.6841, Train Steps/Sec: 1.12 +[2025-04-28 01:53:25] (step=0094300) Train Loss: 5.6363, Train Steps/Sec: 1.12 +[2025-04-28 01:53:47] (step=0094325) Train Loss: 5.7129, Train Steps/Sec: 1.12 +[2025-04-28 01:54:10] (step=0094350) Train Loss: 5.6910, Train Steps/Sec: 1.12 +[2025-04-28 01:54:32] (step=0094375) Train Loss: 5.6583, Train Steps/Sec: 1.12 +[2025-04-28 01:54:54] (step=0094400) Train Loss: 5.7083, Train Steps/Sec: 1.12 +[2025-04-28 01:55:17] (step=0094425) Train Loss: 5.6972, Train Steps/Sec: 1.12 +[2025-04-28 01:55:39] (step=0094450) Train Loss: 5.6410, Train Steps/Sec: 1.12 +[2025-04-28 01:56:01] (step=0094475) Train Loss: 5.7020, Train Steps/Sec: 1.12 +[2025-04-28 01:56:24] (step=0094500) Train Loss: 5.7281, Train Steps/Sec: 1.12 +[2025-04-28 01:56:46] (step=0094525) Train Loss: 5.6374, Train Steps/Sec: 1.12 +[2025-04-28 01:57:08] (step=0094550) Train Loss: 5.7088, Train Steps/Sec: 1.12 +[2025-04-28 01:57:30] (step=0094575) Train Loss: 5.6711, Train Steps/Sec: 1.12 +[2025-04-28 01:57:53] (step=0094600) Train Loss: 5.6573, Train Steps/Sec: 1.12 +[2025-04-28 01:58:15] (step=0094625) Train Loss: 5.7184, Train Steps/Sec: 1.12 +[2025-04-28 01:58:37] (step=0094650) Train Loss: 5.7352, Train Steps/Sec: 1.12 +[2025-04-28 01:59:00] (step=0094675) Train Loss: 5.7308, Train Steps/Sec: 1.12 +[2025-04-28 01:59:22] (step=0094700) Train Loss: 5.6472, Train Steps/Sec: 1.12 +[2025-04-28 01:59:44] (step=0094725) Train Loss: 5.7084, Train Steps/Sec: 1.12 +[2025-04-28 02:00:06] (step=0094750) Train Loss: 5.6631, Train Steps/Sec: 1.12 +[2025-04-28 02:00:29] (step=0094775) Train Loss: 5.6913, Train Steps/Sec: 1.12 +[2025-04-28 02:00:51] (step=0094800) Train Loss: 5.7390, Train Steps/Sec: 1.12 +[2025-04-28 02:01:13] (step=0094825) Train Loss: 5.6667, Train Steps/Sec: 1.12 +[2025-04-28 02:01:36] (step=0094850) Train Loss: 5.6912, Train Steps/Sec: 1.12 +[2025-04-28 02:01:58] (step=0094875) Train Loss: 5.6806, Train Steps/Sec: 1.12 +[2025-04-28 02:02:20] (step=0094900) Train Loss: 5.6721, Train Steps/Sec: 1.12 +[2025-04-28 02:02:43] (step=0094925) Train Loss: 5.6349, Train Steps/Sec: 1.12 +[2025-04-28 02:03:05] (step=0094950) Train Loss: 5.6830, Train Steps/Sec: 1.12 +[2025-04-28 02:03:27] (step=0094975) Train Loss: 5.7297, Train Steps/Sec: 1.12 +[2025-04-28 02:03:50] (step=0095000) Train Loss: 5.6576, Train Steps/Sec: 1.11 +[2025-04-28 02:04:12] (step=0095025) Train Loss: 5.6612, Train Steps/Sec: 1.12 +[2025-04-28 02:04:35] (step=0095050) Train Loss: 5.7610, Train Steps/Sec: 1.12 +[2025-04-28 02:04:57] (step=0095075) Train Loss: 5.6623, Train Steps/Sec: 1.12 +[2025-04-28 02:05:19] (step=0095100) Train Loss: 5.7058, Train Steps/Sec: 1.12 +[2025-04-28 02:05:41] (step=0095125) Train Loss: 5.7150, Train Steps/Sec: 1.12 +[2025-04-28 02:06:04] (step=0095150) Train Loss: 5.6434, Train Steps/Sec: 1.12 +[2025-04-28 02:06:26] (step=0095175) Train Loss: 5.7724, Train Steps/Sec: 1.12 +[2025-04-28 02:06:48] (step=0095200) Train Loss: 5.6577, Train Steps/Sec: 1.12 +[2025-04-28 02:07:11] (step=0095225) Train Loss: 5.6778, Train Steps/Sec: 1.12 +[2025-04-28 02:07:33] (step=0095250) Train Loss: 5.7470, Train Steps/Sec: 1.12 +[2025-04-28 02:07:55] (step=0095275) Train Loss: 5.6746, Train Steps/Sec: 1.12 +[2025-04-28 02:08:18] (step=0095300) Train Loss: 5.7368, Train Steps/Sec: 1.12 +[2025-04-28 02:08:40] (step=0095325) Train Loss: 5.6722, Train Steps/Sec: 1.12 +[2025-04-28 02:09:02] (step=0095350) Train Loss: 5.7102, Train Steps/Sec: 1.12 +[2025-04-28 02:09:25] (step=0095375) Train Loss: 5.7302, Train Steps/Sec: 1.12 +[2025-04-28 02:09:47] (step=0095400) Train Loss: 5.6148, Train Steps/Sec: 1.12 +[2025-04-28 02:10:09] (step=0095425) Train Loss: 5.7232, Train Steps/Sec: 1.12 +[2025-04-28 02:10:32] (step=0095450) Train Loss: 5.6650, Train Steps/Sec: 1.12 +[2025-04-28 02:10:54] (step=0095475) Train Loss: 5.6448, Train Steps/Sec: 1.12 +[2025-04-28 02:11:16] (step=0095500) Train Loss: 5.6736, Train Steps/Sec: 1.12 +[2025-04-28 02:11:39] (step=0095525) Train Loss: 5.6602, Train Steps/Sec: 1.12 +[2025-04-28 02:12:01] (step=0095550) Train Loss: 5.7284, Train Steps/Sec: 1.12 +[2025-04-28 02:12:23] (step=0095575) Train Loss: 5.7370, Train Steps/Sec: 1.12 +[2025-04-28 02:12:46] (step=0095600) Train Loss: 5.7008, Train Steps/Sec: 1.12 +[2025-04-28 02:13:08] (step=0095625) Train Loss: 5.6994, Train Steps/Sec: 1.12 +[2025-04-28 02:13:30] (step=0095650) Train Loss: 5.5852, Train Steps/Sec: 1.12 +[2025-04-28 02:13:53] (step=0095675) Train Loss: 5.6343, Train Steps/Sec: 1.12 +[2025-04-28 02:14:15] (step=0095700) Train Loss: 5.6603, Train Steps/Sec: 1.12 +[2025-04-28 02:14:37] (step=0095725) Train Loss: 5.7015, Train Steps/Sec: 1.12 +[2025-04-28 02:15:00] (step=0095750) Train Loss: 5.6721, Train Steps/Sec: 1.12 +[2025-04-28 02:15:22] (step=0095775) Train Loss: 5.6584, Train Steps/Sec: 1.12 +[2025-04-28 02:15:44] (step=0095800) Train Loss: 5.7048, Train Steps/Sec: 1.12 +[2025-04-28 02:16:07] (step=0095825) Train Loss: 5.6500, Train Steps/Sec: 1.12 +[2025-04-28 02:16:29] (step=0095850) Train Loss: 5.5986, Train Steps/Sec: 1.12 +[2025-04-28 02:16:51] (step=0095875) Train Loss: 5.7032, Train Steps/Sec: 1.12 +[2025-04-28 02:17:14] (step=0095900) Train Loss: 5.6644, Train Steps/Sec: 1.12 +[2025-04-28 02:17:36] (step=0095925) Train Loss: 5.6752, Train Steps/Sec: 1.12 +[2025-04-28 02:17:58] (step=0095950) Train Loss: 5.6920, Train Steps/Sec: 1.12 +[2025-04-28 02:18:21] (step=0095975) Train Loss: 5.7754, Train Steps/Sec: 1.12 +[2025-04-28 02:18:43] (step=0096000) Train Loss: 5.6544, Train Steps/Sec: 1.12 +[2025-04-28 02:18:43] text_config is None. Initializing the text config with default values (`OPTConfig`). +[2025-04-28 02:25:00] Finish Eval in 96000 steps... +[2025-04-28 02:25:20] Saved checkpoint to checkpoint/X2I_UltraEdiit_fourmask_LLava_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_filtered_objects_4mask_1280_context/003-GPT-XL/checkpoints/0096000.pt +[2025-04-28 02:25:22] Removed old checkpoint: checkpoint/X2I_UltraEdiit_fourmask_LLava_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_filtered_objects_4mask_1280_context/003-GPT-XL/checkpoints/0094000.pt +[2025-04-28 02:25:44] (step=0096025) Train Loss: 5.6806, Train Steps/Sec: 0.06 +[2025-04-28 02:26:07] (step=0096050) Train Loss: 5.7286, Train Steps/Sec: 1.12 +[2025-04-28 02:26:29] (step=0096075) Train Loss: 5.6890, Train Steps/Sec: 1.10 +[2025-04-28 02:26:52] (step=0096100) Train Loss: 5.7228, Train Steps/Sec: 1.12 +[2025-04-28 02:27:14] (step=0096125) Train Loss: 5.7554, Train Steps/Sec: 1.12 +[2025-04-28 02:27:36] (step=0096150) Train Loss: 5.6759, Train Steps/Sec: 1.12 +[2025-04-28 02:27:58] (step=0096175) Train Loss: 5.6807, Train Steps/Sec: 1.12 +[2025-04-28 02:28:21] (step=0096200) Train Loss: 5.6753, Train Steps/Sec: 1.12 +[2025-04-28 02:28:43] (step=0096225) Train Loss: 5.6639, Train Steps/Sec: 1.12 +[2025-04-28 02:29:05] (step=0096250) Train Loss: 5.6562, Train Steps/Sec: 1.12 +[2025-04-28 02:29:28] (step=0096275) Train Loss: 5.7223, Train Steps/Sec: 1.12 +[2025-04-28 02:29:50] (step=0096300) Train Loss: 5.6409, Train Steps/Sec: 1.12 +[2025-04-28 02:30:12] (step=0096325) Train Loss: 5.6944, Train Steps/Sec: 1.12 +[2025-04-28 02:30:39] (step=0096350) Train Loss: 5.7074, Train Steps/Sec: 0.94 +[2025-04-28 02:31:01] (step=0096375) Train Loss: 5.6625, Train Steps/Sec: 1.12 +[2025-04-28 02:31:24] (step=0096400) Train Loss: 5.6956, Train Steps/Sec: 1.12 +[2025-04-28 02:31:46] (step=0096425) Train Loss: 5.6629, Train Steps/Sec: 1.12 +[2025-04-28 02:32:08] (step=0096450) Train Loss: 5.6805, Train Steps/Sec: 1.12 +[2025-04-28 02:32:30] (step=0096475) Train Loss: 5.6630, Train Steps/Sec: 1.12 +[2025-04-28 02:32:53] (step=0096500) Train Loss: 5.6758, Train Steps/Sec: 1.12 +[2025-04-28 02:33:15] (step=0096525) Train Loss: 5.6460, Train Steps/Sec: 1.12 +[2025-04-28 02:33:37] (step=0096550) Train Loss: 5.6458, Train Steps/Sec: 1.12 +[2025-04-28 02:34:00] (step=0096575) Train Loss: 5.7129, Train Steps/Sec: 1.12 +[2025-04-28 02:34:22] (step=0096600) Train Loss: 5.6223, Train Steps/Sec: 1.12 +[2025-04-28 02:34:44] (step=0096625) Train Loss: 5.7280, Train Steps/Sec: 1.12 +[2025-04-28 02:35:11] (step=0096650) Train Loss: 5.6753, Train Steps/Sec: 0.93 +[2025-04-28 02:35:33] (step=0096675) Train Loss: 5.7137, Train Steps/Sec: 1.12 +[2025-04-28 02:35:56] (step=0096700) Train Loss: 5.6738, Train Steps/Sec: 1.12 +[2025-04-28 02:36:18] (step=0096725) Train Loss: 5.6542, Train Steps/Sec: 1.12 +[2025-04-28 02:36:40] (step=0096750) Train Loss: 5.6881, Train Steps/Sec: 1.12 +[2025-04-28 02:37:03] (step=0096775) Train Loss: 5.7009, Train Steps/Sec: 1.12 +[2025-04-28 02:37:25] (step=0096800) Train Loss: 5.7230, Train Steps/Sec: 1.12 +[2025-04-28 02:37:47] (step=0096825) Train Loss: 5.6088, Train Steps/Sec: 1.12 +[2025-04-28 02:38:09] (step=0096850) Train Loss: 5.6684, Train Steps/Sec: 1.12 +[2025-04-28 02:38:32] (step=0096875) Train Loss: 5.6491, Train Steps/Sec: 1.12 +[2025-04-28 02:38:54] (step=0096900) Train Loss: 5.7184, Train Steps/Sec: 1.12 +[2025-04-28 02:39:16] (step=0096925) Train Loss: 5.7054, Train Steps/Sec: 1.12 +[2025-04-28 02:39:39] (step=0096950) Train Loss: 5.6911, Train Steps/Sec: 1.12 +[2025-04-28 02:40:01] (step=0096975) Train Loss: 5.6724, Train Steps/Sec: 1.12 +[2025-04-28 02:40:23] (step=0097000) Train Loss: 5.6680, Train Steps/Sec: 1.12 +[2025-04-28 02:40:45] (step=0097025) Train Loss: 5.7212, Train Steps/Sec: 1.12 +[2025-04-28 02:41:08] (step=0097050) Train Loss: 5.7542, Train Steps/Sec: 1.12 +[2025-04-28 02:41:30] (step=0097075) Train Loss: 5.6960, Train Steps/Sec: 1.12 +[2025-04-28 02:41:52] (step=0097100) Train Loss: 5.6460, Train Steps/Sec: 1.12 +[2025-04-28 02:42:14] (step=0097125) Train Loss: 5.6137, Train Steps/Sec: 1.12 +[2025-04-28 02:42:37] (step=0097150) Train Loss: 5.7263, Train Steps/Sec: 1.12 +[2025-04-28 02:42:59] (step=0097175) Train Loss: 5.6948, Train Steps/Sec: 1.12 +[2025-04-28 02:43:21] (step=0097200) Train Loss: 5.6863, Train Steps/Sec: 1.12 +[2025-04-28 02:43:44] (step=0097225) Train Loss: 5.6661, Train Steps/Sec: 1.12 +[2025-04-28 02:44:06] (step=0097250) Train Loss: 5.6434, Train Steps/Sec: 1.13 +[2025-04-28 02:44:28] (step=0097275) Train Loss: 5.6233, Train Steps/Sec: 1.12 +[2025-04-28 02:44:50] (step=0097300) Train Loss: 5.7897, Train Steps/Sec: 1.12 +[2025-04-28 02:45:13] (step=0097325) Train Loss: 5.6049, Train Steps/Sec: 1.13 +[2025-04-28 02:45:35] (step=0097350) Train Loss: 5.6211, Train Steps/Sec: 1.12 +[2025-04-28 02:45:57] (step=0097375) Train Loss: 5.6811, Train Steps/Sec: 1.12 +[2025-04-28 02:46:19] (step=0097400) Train Loss: 5.6838, Train Steps/Sec: 1.12 +[2025-04-28 02:46:42] (step=0097425) Train Loss: 5.6669, Train Steps/Sec: 1.12 +[2025-04-28 02:47:04] (step=0097450) Train Loss: 5.6717, Train Steps/Sec: 1.12 +[2025-04-28 02:47:26] (step=0097475) Train Loss: 5.6957, Train Steps/Sec: 1.12 +[2025-04-28 02:47:49] (step=0097500) Train Loss: 5.7033, Train Steps/Sec: 1.12 +[2025-04-28 02:48:11] (step=0097525) Train Loss: 5.5787, Train Steps/Sec: 1.12 +[2025-04-28 02:48:33] (step=0097550) Train Loss: 5.7249, Train Steps/Sec: 1.12 +[2025-04-28 02:48:55] (step=0097575) Train Loss: 5.7181, Train Steps/Sec: 1.12 +[2025-04-28 02:49:22] (step=0097600) Train Loss: 5.6781, Train Steps/Sec: 0.96 +[2025-04-28 02:49:48] (step=0097625) Train Loss: 5.6591, Train Steps/Sec: 0.93 +[2025-04-28 02:50:15] (step=0097650) Train Loss: 5.6378, Train Steps/Sec: 0.94 +[2025-04-28 02:50:42] (step=0097675) Train Loss: 5.6691, Train Steps/Sec: 0.91 +[2025-04-28 02:51:05] (step=0097700) Train Loss: 5.6803, Train Steps/Sec: 1.12 +[2025-04-28 02:51:27] (step=0097725) Train Loss: 5.6957, Train Steps/Sec: 1.12 +[2025-04-28 02:51:49] (step=0097750) Train Loss: 5.6755, Train Steps/Sec: 1.12 +[2025-04-28 02:52:11] (step=0097775) Train Loss: 5.6517, Train Steps/Sec: 1.12 +[2025-04-28 02:52:34] (step=0097800) Train Loss: 5.7008, Train Steps/Sec: 1.12 +[2025-04-28 02:52:56] (step=0097825) Train Loss: 5.7344, Train Steps/Sec: 1.12 +[2025-04-28 02:53:18] (step=0097850) Train Loss: 5.7254, Train Steps/Sec: 1.12 +[2025-04-28 02:53:41] (step=0097875) Train Loss: 5.6669, Train Steps/Sec: 1.12 +[2025-04-28 02:54:03] (step=0097900) Train Loss: 5.6258, Train Steps/Sec: 1.12 +[2025-04-28 02:54:25] (step=0097925) Train Loss: 5.7080, Train Steps/Sec: 1.12 +[2025-04-28 02:54:48] (step=0097950) Train Loss: 5.6701, Train Steps/Sec: 1.12 +[2025-04-28 02:55:10] (step=0097975) Train Loss: 5.6450, Train Steps/Sec: 1.12 +[2025-04-28 02:55:32] (step=0098000) Train Loss: 5.6770, Train Steps/Sec: 1.12 +[2025-04-28 02:55:32] text_config is None. Initializing the text config with default values (`OPTConfig`). +[2025-04-28 03:01:55] Finish Eval in 98000 steps... +[2025-04-28 03:02:15] Saved checkpoint to checkpoint/X2I_UltraEdiit_fourmask_LLava_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_filtered_objects_4mask_1280_context/003-GPT-XL/checkpoints/0098000.pt +[2025-04-28 03:02:17] Removed old checkpoint: checkpoint/X2I_UltraEdiit_fourmask_LLava_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_filtered_objects_4mask_1280_context/003-GPT-XL/checkpoints/0096000.pt +[2025-04-28 03:02:39] (step=0098025) Train Loss: 5.6641, Train Steps/Sec: 0.06 +[2025-04-28 03:03:01] (step=0098050) Train Loss: 5.6866, Train Steps/Sec: 1.12 +[2025-04-28 03:03:24] (step=0098075) Train Loss: 5.6637, Train Steps/Sec: 1.12 +[2025-04-28 03:03:46] (step=0098100) Train Loss: 5.6378, Train Steps/Sec: 1.12 +[2025-04-28 03:04:08] (step=0098125) Train Loss: 5.7048, Train Steps/Sec: 1.12 +[2025-04-28 03:04:30] (step=0098150) Train Loss: 5.7012, Train Steps/Sec: 1.13 +[2025-04-28 03:04:53] (step=0098175) Train Loss: 5.6602, Train Steps/Sec: 1.12 +[2025-04-28 03:05:15] (step=0098200) Train Loss: 5.6779, Train Steps/Sec: 1.12 +[2025-04-28 03:05:37] (step=0098225) Train Loss: 5.6544, Train Steps/Sec: 1.12 +[2025-04-28 03:06:05] (step=0098250) Train Loss: 5.6997, Train Steps/Sec: 0.92 +[2025-04-28 03:06:27] (step=0098275) Train Loss: 5.6872, Train Steps/Sec: 1.12 +[2025-04-28 03:06:49] (step=0098300) Train Loss: 5.6479, Train Steps/Sec: 1.12 +[2025-04-28 03:07:11] (step=0098325) Train Loss: 5.7382, Train Steps/Sec: 1.12 +[2025-04-28 03:07:34] (step=0098350) Train Loss: 5.6529, Train Steps/Sec: 1.12 +[2025-04-28 03:07:56] (step=0098375) Train Loss: 5.6830, Train Steps/Sec: 1.12 +[2025-04-28 03:08:18] (step=0098400) Train Loss: 5.7248, Train Steps/Sec: 1.12 +[2025-04-28 03:08:41] (step=0098425) Train Loss: 5.6550, Train Steps/Sec: 1.12 +[2025-04-28 03:09:03] (step=0098450) Train Loss: 5.7588, Train Steps/Sec: 1.12 +[2025-04-28 03:09:25] (step=0098475) Train Loss: 5.6737, Train Steps/Sec: 1.12 +[2025-04-28 03:09:47] (step=0098500) Train Loss: 5.6557, Train Steps/Sec: 1.12 +[2025-04-28 03:10:10] (step=0098525) Train Loss: 5.6318, Train Steps/Sec: 1.12 +[2025-04-28 03:10:32] (step=0098550) Train Loss: 5.6720, Train Steps/Sec: 1.12 +[2025-04-28 03:10:54] (step=0098575) Train Loss: 5.6744, Train Steps/Sec: 1.12 +[2025-04-28 03:11:17] (step=0098600) Train Loss: 5.6161, Train Steps/Sec: 1.12 +[2025-04-28 03:11:39] (step=0098625) Train Loss: 5.7121, Train Steps/Sec: 1.12 +[2025-04-28 03:12:01] (step=0098650) Train Loss: 5.6572, Train Steps/Sec: 1.12 +[2025-04-28 03:12:24] (step=0098675) Train Loss: 5.7682, Train Steps/Sec: 1.12 +[2025-04-28 03:12:46] (step=0098700) Train Loss: 5.7138, Train Steps/Sec: 1.12 +[2025-04-28 03:13:08] (step=0098725) Train Loss: 5.6636, Train Steps/Sec: 1.12 +[2025-04-28 03:13:30] (step=0098750) Train Loss: 5.6824, Train Steps/Sec: 1.12 +[2025-04-28 03:13:53] (step=0098775) Train Loss: 5.6300, Train Steps/Sec: 1.12 +[2025-04-28 03:14:15] (step=0098800) Train Loss: 5.6832, Train Steps/Sec: 1.12 +[2025-04-28 03:14:37] (step=0098825) Train Loss: 5.6926, Train Steps/Sec: 1.12 +[2025-04-28 03:15:00] (step=0098850) Train Loss: 5.6563, Train Steps/Sec: 1.13 +[2025-04-28 03:15:22] (step=0098875) Train Loss: 5.6867, Train Steps/Sec: 1.12 +[2025-04-28 03:15:44] (step=0098900) Train Loss: 5.7025, Train Steps/Sec: 1.12 +[2025-04-28 03:16:06] (step=0098925) Train Loss: 5.6525, Train Steps/Sec: 1.12 +[2025-04-28 03:16:29] (step=0098950) Train Loss: 5.7354, Train Steps/Sec: 1.12 +[2025-04-28 03:16:51] (step=0098975) Train Loss: 5.7048, Train Steps/Sec: 1.12 +[2025-04-28 03:17:13] (step=0099000) Train Loss: 5.6546, Train Steps/Sec: 1.12 +[2025-04-28 03:17:36] (step=0099025) Train Loss: 5.6579, Train Steps/Sec: 1.12 +[2025-04-28 03:17:58] (step=0099050) Train Loss: 5.7574, Train Steps/Sec: 1.12 +[2025-04-28 03:18:20] (step=0099075) Train Loss: 5.6845, Train Steps/Sec: 1.12 +[2025-04-28 03:18:42] (step=0099100) Train Loss: 5.6532, Train Steps/Sec: 1.12 +[2025-04-28 03:19:05] (step=0099125) Train Loss: 5.6318, Train Steps/Sec: 1.12 +[2025-04-28 03:19:27] (step=0099150) Train Loss: 5.7035, Train Steps/Sec: 1.12 +[2025-04-28 03:19:49] (step=0099175) Train Loss: 5.6758, Train Steps/Sec: 1.12 +[2025-04-28 03:20:12] (step=0099200) Train Loss: 5.6373, Train Steps/Sec: 1.12 +[2025-04-28 03:20:34] (step=0099225) Train Loss: 5.7275, Train Steps/Sec: 1.12 +[2025-04-28 03:20:56] (step=0099250) Train Loss: 5.6223, Train Steps/Sec: 1.12 +[2025-04-28 03:21:18] (step=0099275) Train Loss: 5.6952, Train Steps/Sec: 1.12 +[2025-04-28 03:21:41] (step=0099300) Train Loss: 5.7340, Train Steps/Sec: 1.12 +[2025-04-28 03:22:03] (step=0099325) Train Loss: 5.6813, Train Steps/Sec: 1.12 +[2025-04-28 03:22:25] (step=0099350) Train Loss: 5.6191, Train Steps/Sec: 1.12 +[2025-04-28 03:22:48] (step=0099375) Train Loss: 5.6968, Train Steps/Sec: 1.12 +[2025-04-28 03:23:10] (step=0099400) Train Loss: 5.6797, Train Steps/Sec: 1.12 +[2025-04-28 03:23:32] (step=0099425) Train Loss: 5.6799, Train Steps/Sec: 1.12 +[2025-04-28 03:23:54] (step=0099450) Train Loss: 5.6238, Train Steps/Sec: 1.12 +[2025-04-28 03:24:17] (step=0099475) Train Loss: 5.6483, Train Steps/Sec: 1.12 +[2025-04-28 03:24:39] (step=0099500) Train Loss: 5.6906, Train Steps/Sec: 1.12 +[2025-04-28 03:25:01] (step=0099525) Train Loss: 5.6607, Train Steps/Sec: 1.12 +[2025-04-28 03:25:24] (step=0099550) Train Loss: 5.6534, Train Steps/Sec: 1.12 +[2025-04-28 03:25:46] (step=0099575) Train Loss: 5.6461, Train Steps/Sec: 1.12 +[2025-04-28 03:26:08] (step=0099600) Train Loss: 5.6843, Train Steps/Sec: 1.12 +[2025-04-28 03:26:31] (step=0099625) Train Loss: 5.6255, Train Steps/Sec: 1.13 +[2025-04-28 03:26:53] (step=0099650) Train Loss: 5.6169, Train Steps/Sec: 1.12 +[2025-04-28 03:27:15] (step=0099675) Train Loss: 5.6341, Train Steps/Sec: 1.12 +[2025-04-28 03:27:37] (step=0099700) Train Loss: 5.6921, Train Steps/Sec: 1.12 +[2025-04-28 03:28:00] (step=0099725) Train Loss: 5.7276, Train Steps/Sec: 1.12 +[2025-04-28 03:28:22] (step=0099750) Train Loss: 5.6612, Train Steps/Sec: 1.12 +[2025-04-28 03:28:44] (step=0099775) Train Loss: 5.6342, Train Steps/Sec: 1.12 +[2025-04-28 03:29:07] (step=0099800) Train Loss: 5.6309, Train Steps/Sec: 1.12 +[2025-04-28 03:29:29] (step=0099825) Train Loss: 5.7062, Train Steps/Sec: 1.12 +[2025-04-28 03:29:51] (step=0099850) Train Loss: 5.6889, Train Steps/Sec: 1.12 +[2025-04-28 03:30:14] (step=0099875) Train Loss: 5.6950, Train Steps/Sec: 1.12 +[2025-04-28 03:30:36] (step=0099900) Train Loss: 5.6346, Train Steps/Sec: 1.12 +[2025-04-28 03:30:58] (step=0099925) Train Loss: 5.6356, Train Steps/Sec: 1.12 +[2025-04-28 03:31:20] (step=0099950) Train Loss: 5.6926, Train Steps/Sec: 1.12 +[2025-04-28 03:31:43] (step=0099975) Train Loss: 5.6845, Train Steps/Sec: 1.12 +[2025-04-28 03:32:05] (step=0100000) Train Loss: 5.7195, Train Steps/Sec: 1.12 +[2025-04-28 03:32:05] text_config is None. Initializing the text config with default values (`OPTConfig`). +[2025-04-28 03:38:22] Finish Eval in 100000 steps... +[2025-04-28 03:38:41] Saved checkpoint to checkpoint/X2I_UltraEdiit_fourmask_LLava_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_filtered_objects_4mask_1280_context/003-GPT-XL/checkpoints/0100000.pt +[2025-04-28 03:38:43] Removed old checkpoint: checkpoint/X2I_UltraEdiit_fourmask_LLava_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_filtered_objects_4mask_1280_context/003-GPT-XL/checkpoints/0098000.pt +[2025-04-28 03:39:06] (step=0100025) Train Loss: 5.6511, Train Steps/Sec: 0.06 +[2025-04-28 03:39:28] (step=0100050) Train Loss: 5.6591, Train Steps/Sec: 1.12 +[2025-04-28 03:39:50] (step=0100075) Train Loss: 5.6294, Train Steps/Sec: 1.12 +[2025-04-28 03:40:12] (step=0100100) Train Loss: 5.7044, Train Steps/Sec: 1.12 +[2025-04-28 03:40:35] (step=0100125) Train Loss: 5.7240, Train Steps/Sec: 1.12 +[2025-04-28 03:40:57] (step=0100150) Train Loss: 5.6056, Train Steps/Sec: 1.12 +[2025-04-28 03:41:19] (step=0100175) Train Loss: 5.6583, Train Steps/Sec: 1.12 +[2025-04-28 03:41:42] (step=0100200) Train Loss: 5.7007, Train Steps/Sec: 1.12 +[2025-04-28 03:42:04] (step=0100225) Train Loss: 5.6820, Train Steps/Sec: 1.12 +[2025-04-28 03:42:26] (step=0100250) Train Loss: 5.6584, Train Steps/Sec: 1.12 +[2025-04-28 03:42:48] (step=0100275) Train Loss: 5.7000, Train Steps/Sec: 1.13 +[2025-04-28 03:43:11] (step=0100300) Train Loss: 5.7081, Train Steps/Sec: 1.12 +[2025-04-28 03:43:33] (step=0100325) Train Loss: 5.6880, Train Steps/Sec: 1.12 +[2025-04-28 03:43:55] (step=0100350) Train Loss: 5.6872, Train Steps/Sec: 1.12 +[2025-04-28 03:44:17] (step=0100375) Train Loss: 5.6555, Train Steps/Sec: 1.12 +[2025-04-28 03:44:40] (step=0100400) Train Loss: 5.6029, Train Steps/Sec: 1.12 +[2025-04-28 03:45:02] (step=0100425) Train Loss: 5.6558, Train Steps/Sec: 1.13 +[2025-04-28 03:45:24] (step=0100450) Train Loss: 5.7090, Train Steps/Sec: 1.12 +[2025-04-28 03:45:46] (step=0100475) Train Loss: 5.6122, Train Steps/Sec: 1.12 +[2025-04-28 03:46:09] (step=0100500) Train Loss: 5.5975, Train Steps/Sec: 1.12 +[2025-04-28 03:46:31] (step=0100525) Train Loss: 5.7055, Train Steps/Sec: 1.11 +[2025-04-28 03:46:54] (step=0100550) Train Loss: 5.6064, Train Steps/Sec: 1.12 +[2025-04-28 03:47:16] (step=0100575) Train Loss: 5.7554, Train Steps/Sec: 1.12 +[2025-04-28 03:47:43] (step=0100600) Train Loss: 5.6328, Train Steps/Sec: 0.93 +[2025-04-28 03:48:05] (step=0100625) Train Loss: 5.6381, Train Steps/Sec: 1.12 +[2025-04-28 03:48:27] (step=0100650) Train Loss: 5.6511, Train Steps/Sec: 1.12 +[2025-04-28 03:48:50] (step=0100675) Train Loss: 5.6411, Train Steps/Sec: 1.12 +[2025-04-28 03:49:12] (step=0100700) Train Loss: 5.6792, Train Steps/Sec: 1.12 +[2025-04-28 03:49:34] (step=0100725) Train Loss: 5.6769, Train Steps/Sec: 1.12 +[2025-04-28 03:49:57] (step=0100750) Train Loss: 5.6729, Train Steps/Sec: 1.12 +[2025-04-28 03:50:19] (step=0100775) Train Loss: 5.6243, Train Steps/Sec: 1.12 +[2025-04-28 03:50:41] (step=0100800) Train Loss: 5.6476, Train Steps/Sec: 1.12 +[2025-04-28 03:51:03] (step=0100825) Train Loss: 5.6813, Train Steps/Sec: 1.12 +[2025-04-28 03:51:26] (step=0100850) Train Loss: 5.7674, Train Steps/Sec: 1.12 +[2025-04-28 03:51:48] (step=0100875) Train Loss: 5.6877, Train Steps/Sec: 1.12 +[2025-04-28 03:52:10] (step=0100900) Train Loss: 5.7132, Train Steps/Sec: 1.12 +[2025-04-28 03:52:37] (step=0100925) Train Loss: 5.6587, Train Steps/Sec: 0.92 +[2025-04-28 03:53:00] (step=0100950) Train Loss: 5.6957, Train Steps/Sec: 1.12 +[2025-04-28 03:53:22] (step=0100975) Train Loss: 5.6311, Train Steps/Sec: 1.13 +[2025-04-28 03:53:44] (step=0101000) Train Loss: 5.7214, Train Steps/Sec: 1.12 +[2025-04-28 03:54:07] (step=0101025) Train Loss: 5.7446, Train Steps/Sec: 1.12 +[2025-04-28 03:54:29] (step=0101050) Train Loss: 5.6871, Train Steps/Sec: 1.12 +[2025-04-28 03:54:51] (step=0101075) Train Loss: 5.7434, Train Steps/Sec: 1.12 +[2025-04-28 03:55:13] (step=0101100) Train Loss: 5.6773, Train Steps/Sec: 1.12 +[2025-04-28 03:55:36] (step=0101125) Train Loss: 5.6628, Train Steps/Sec: 1.12 +[2025-04-28 03:55:58] (step=0101150) Train Loss: 5.7078, Train Steps/Sec: 1.12 +[2025-04-28 03:56:20] (step=0101175) Train Loss: 5.6605, Train Steps/Sec: 1.12 +[2025-04-28 03:56:43] (step=0101200) Train Loss: 5.5937, Train Steps/Sec: 1.12 +[2025-04-28 03:57:05] (step=0101225) Train Loss: 5.7106, Train Steps/Sec: 1.12 +[2025-04-28 03:57:27] (step=0101250) Train Loss: 5.6526, Train Steps/Sec: 1.12 +[2025-04-28 03:57:49] (step=0101275) Train Loss: 5.6474, Train Steps/Sec: 1.12 +[2025-04-28 03:58:12] (step=0101300) Train Loss: 5.6524, Train Steps/Sec: 1.12 +[2025-04-28 03:58:34] (step=0101325) Train Loss: 5.6013, Train Steps/Sec: 1.12 +[2025-04-28 03:58:56] (step=0101350) Train Loss: 5.6242, Train Steps/Sec: 1.12 +[2025-04-28 03:59:19] (step=0101375) Train Loss: 5.6229, Train Steps/Sec: 1.12 +[2025-04-28 03:59:41] (step=0101400) Train Loss: 5.6965, Train Steps/Sec: 1.11 +[2025-04-28 04:00:03] (step=0101425) Train Loss: 5.6164, Train Steps/Sec: 1.12 +[2025-04-28 04:00:26] (step=0101450) Train Loss: 5.7047, Train Steps/Sec: 1.12 +[2025-04-28 04:00:48] (step=0101475) Train Loss: 5.6603, Train Steps/Sec: 1.12 +[2025-04-28 04:01:10] (step=0101500) Train Loss: 5.6765, Train Steps/Sec: 1.13 +[2025-04-28 04:01:32] (step=0101525) Train Loss: 5.6951, Train Steps/Sec: 1.12 +[2025-04-28 04:01:55] (step=0101550) Train Loss: 5.6594, Train Steps/Sec: 1.12 +[2025-04-28 04:02:17] (step=0101575) Train Loss: 5.6170, Train Steps/Sec: 1.12 +[2025-04-28 04:02:39] (step=0101600) Train Loss: 5.6542, Train Steps/Sec: 1.12 +[2025-04-28 04:03:02] (step=0101625) Train Loss: 5.6737, Train Steps/Sec: 1.12 +[2025-04-28 04:03:24] (step=0101650) Train Loss: 5.6582, Train Steps/Sec: 1.12 +[2025-04-28 04:03:46] (step=0101675) Train Loss: 5.6690, Train Steps/Sec: 1.12 +[2025-04-28 04:04:08] (step=0101700) Train Loss: 5.6396, Train Steps/Sec: 1.12 +[2025-04-28 04:04:31] (step=0101725) Train Loss: 5.6636, Train Steps/Sec: 1.12 +[2025-04-28 04:04:53] (step=0101750) Train Loss: 5.6638, Train Steps/Sec: 1.12 +[2025-04-28 04:05:15] (step=0101775) Train Loss: 5.6327, Train Steps/Sec: 1.12 +[2025-04-28 04:05:38] (step=0101800) Train Loss: 5.6865, Train Steps/Sec: 1.12 +[2025-04-28 04:06:00] (step=0101825) Train Loss: 5.5983, Train Steps/Sec: 1.12 +[2025-04-28 04:06:22] (step=0101850) Train Loss: 5.6999, Train Steps/Sec: 1.12 +[2025-04-28 04:06:45] (step=0101875) Train Loss: 5.6517, Train Steps/Sec: 1.12 +[2025-04-28 04:07:07] (step=0101900) Train Loss: 5.6498, Train Steps/Sec: 1.12 +[2025-04-28 04:07:29] (step=0101925) Train Loss: 5.7594, Train Steps/Sec: 1.12 +[2025-04-28 04:08:00] (step=0101950) Train Loss: 5.6108, Train Steps/Sec: 0.82 +[2025-04-28 04:08:32] (step=0101975) Train Loss: 5.6536, Train Steps/Sec: 0.79 +[2025-04-28 04:08:54] (step=0102000) Train Loss: 5.6247, Train Steps/Sec: 1.12 +[2025-04-28 04:08:54] text_config is None. Initializing the text config with default values (`OPTConfig`). +[2025-04-28 04:15:12] Finish Eval in 102000 steps... +[2025-04-28 04:15:31] Saved checkpoint to checkpoint/X2I_UltraEdiit_fourmask_LLava_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_filtered_objects_4mask_1280_context/003-GPT-XL/checkpoints/0102000.pt +[2025-04-28 04:15:33] Removed old checkpoint: checkpoint/X2I_UltraEdiit_fourmask_LLava_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_filtered_objects_4mask_1280_context/003-GPT-XL/checkpoints/0100000.pt +[2025-04-28 04:15:56] (step=0102025) Train Loss: 5.6695, Train Steps/Sec: 0.06 +[2025-04-28 04:16:18] (step=0102050) Train Loss: 5.7094, Train Steps/Sec: 1.12 +[2025-04-28 04:16:40] (step=0102075) Train Loss: 5.6780, Train Steps/Sec: 1.12 +[2025-04-28 04:17:02] (step=0102100) Train Loss: 5.6988, Train Steps/Sec: 1.12 +[2025-04-28 04:17:25] (step=0102125) Train Loss: 5.6840, Train Steps/Sec: 1.13 +[2025-04-28 04:17:47] (step=0102150) Train Loss: 5.6552, Train Steps/Sec: 1.12 +[2025-04-28 04:18:09] (step=0102175) Train Loss: 5.6527, Train Steps/Sec: 1.12 +[2025-04-28 04:18:31] (step=0102200) Train Loss: 5.7136, Train Steps/Sec: 1.12 +[2025-04-28 04:18:54] (step=0102225) Train Loss: 5.7066, Train Steps/Sec: 1.12 +[2025-04-28 04:19:16] (step=0102250) Train Loss: 5.6224, Train Steps/Sec: 1.12 +[2025-04-28 04:19:38] (step=0102275) Train Loss: 5.6394, Train Steps/Sec: 1.12 +[2025-04-28 04:20:00] (step=0102300) Train Loss: 5.6300, Train Steps/Sec: 1.12 +[2025-04-28 04:20:23] (step=0102325) Train Loss: 5.6929, Train Steps/Sec: 1.13 +[2025-04-28 04:20:45] (step=0102350) Train Loss: 5.5800, Train Steps/Sec: 1.12 +[2025-04-28 04:21:07] (step=0102375) Train Loss: 5.6901, Train Steps/Sec: 1.12 +[2025-04-28 04:21:29] (step=0102400) Train Loss: 5.7106, Train Steps/Sec: 1.12 +[2025-04-28 04:21:52] (step=0102425) Train Loss: 5.6833, Train Steps/Sec: 1.10 +[2025-04-28 04:22:15] (step=0102450) Train Loss: 5.6636, Train Steps/Sec: 1.12 +[2025-04-28 04:22:37] (step=0102475) Train Loss: 5.6992, Train Steps/Sec: 1.12 +[2025-04-28 04:22:59] (step=0102500) Train Loss: 5.6804, Train Steps/Sec: 1.12 +[2025-04-28 04:23:21] (step=0102525) Train Loss: 5.5894, Train Steps/Sec: 1.12 +[2025-04-28 04:23:44] (step=0102550) Train Loss: 5.7648, Train Steps/Sec: 1.12 +[2025-04-28 04:24:06] (step=0102575) Train Loss: 5.6593, Train Steps/Sec: 1.12 +[2025-04-28 04:24:28] (step=0102600) Train Loss: 5.6976, Train Steps/Sec: 1.12 +[2025-04-28 04:24:51] (step=0102625) Train Loss: 5.6759, Train Steps/Sec: 1.12 +[2025-04-28 04:25:18] (step=0102650) Train Loss: 5.7008, Train Steps/Sec: 0.92 +[2025-04-28 04:25:40] (step=0102675) Train Loss: 5.6179, Train Steps/Sec: 1.12 +[2025-04-28 04:26:02] (step=0102700) Train Loss: 5.6266, Train Steps/Sec: 1.13 +[2025-04-28 04:26:24] (step=0102725) Train Loss: 5.6885, Train Steps/Sec: 1.12 +[2025-04-28 04:26:47] (step=0102750) Train Loss: 5.7082, Train Steps/Sec: 1.13 +[2025-04-28 04:27:09] (step=0102775) Train Loss: 5.6584, Train Steps/Sec: 1.12 +[2025-04-28 04:27:31] (step=0102800) Train Loss: 5.6590, Train Steps/Sec: 1.12 +[2025-04-28 04:27:54] (step=0102825) Train Loss: 5.6984, Train Steps/Sec: 1.12 +[2025-04-28 04:28:16] (step=0102850) Train Loss: 5.6681, Train Steps/Sec: 1.12 +[2025-04-28 04:28:38] (step=0102875) Train Loss: 5.7023, Train Steps/Sec: 1.12 +[2025-04-28 04:29:01] (step=0102900) Train Loss: 5.6146, Train Steps/Sec: 1.12 +[2025-04-28 04:29:23] (step=0102925) Train Loss: 5.6445, Train Steps/Sec: 1.12 +[2025-04-28 04:29:45] (step=0102950) Train Loss: 5.6958, Train Steps/Sec: 1.12 +[2025-04-28 04:30:07] (step=0102975) Train Loss: 5.6946, Train Steps/Sec: 1.12 +[2025-04-28 04:30:30] (step=0103000) Train Loss: 5.6230, Train Steps/Sec: 1.12 +[2025-04-28 04:30:52] (step=0103025) Train Loss: 5.6614, Train Steps/Sec: 1.12 +[2025-04-28 04:31:14] (step=0103050) Train Loss: 5.7380, Train Steps/Sec: 1.12 +[2025-04-28 04:31:37] (step=0103075) Train Loss: 5.7132, Train Steps/Sec: 1.12 +[2025-04-28 04:31:59] (step=0103100) Train Loss: 5.6151, Train Steps/Sec: 1.12 +[2025-04-28 04:32:21] (step=0103125) Train Loss: 5.6593, Train Steps/Sec: 1.12 +[2025-04-28 04:32:44] (step=0103150) Train Loss: 5.6724, Train Steps/Sec: 1.12 +[2025-04-28 04:33:06] (step=0103175) Train Loss: 5.7030, Train Steps/Sec: 1.12 +[2025-04-28 04:33:28] (step=0103200) Train Loss: 5.7043, Train Steps/Sec: 1.12 +[2025-04-28 04:33:50] (step=0103225) Train Loss: 5.6568, Train Steps/Sec: 1.13 +[2025-04-28 04:34:13] (step=0103250) Train Loss: 5.6681, Train Steps/Sec: 1.12 +[2025-04-28 04:34:35] (step=0103275) Train Loss: 5.6422, Train Steps/Sec: 1.12 +[2025-04-28 04:34:57] (step=0103300) Train Loss: 5.6954, Train Steps/Sec: 1.12 +[2025-04-28 04:35:20] (step=0103325) Train Loss: 5.6809, Train Steps/Sec: 1.12 +[2025-04-28 04:35:42] (step=0103350) Train Loss: 5.6553, Train Steps/Sec: 1.12 +[2025-04-28 04:36:04] (step=0103375) Train Loss: 5.6475, Train Steps/Sec: 1.12 +[2025-04-28 04:36:26] (step=0103400) Train Loss: 5.5749, Train Steps/Sec: 1.12 +[2025-04-28 04:36:49] (step=0103425) Train Loss: 5.6366, Train Steps/Sec: 1.12 +[2025-04-28 04:37:11] (step=0103450) Train Loss: 5.6298, Train Steps/Sec: 1.12 +[2025-04-28 04:37:33] (step=0103475) Train Loss: 5.7032, Train Steps/Sec: 1.12 +[2025-04-28 04:37:56] (step=0103500) Train Loss: 5.6550, Train Steps/Sec: 1.12 +[2025-04-28 04:38:18] (step=0103525) Train Loss: 5.6697, Train Steps/Sec: 1.12 +[2025-04-28 04:38:40] (step=0103550) Train Loss: 5.6298, Train Steps/Sec: 1.12 +[2025-04-28 04:39:02] (step=0103575) Train Loss: 5.6551, Train Steps/Sec: 1.12 +[2025-04-28 04:39:25] (step=0103600) Train Loss: 5.7166, Train Steps/Sec: 1.12 +[2025-04-28 04:39:47] (step=0103625) Train Loss: 5.7148, Train Steps/Sec: 1.12 +[2025-04-28 04:40:09] (step=0103650) Train Loss: 5.6383, Train Steps/Sec: 1.12 +[2025-04-28 04:40:32] (step=0103675) Train Loss: 5.6079, Train Steps/Sec: 1.13 +[2025-04-28 04:40:54] (step=0103700) Train Loss: 5.6269, Train Steps/Sec: 1.12 +[2025-04-28 04:41:16] (step=0103725) Train Loss: 5.6319, Train Steps/Sec: 1.12 +[2025-04-28 04:41:38] (step=0103750) Train Loss: 5.6479, Train Steps/Sec: 1.12 +[2025-04-28 04:42:01] (step=0103775) Train Loss: 5.6134, Train Steps/Sec: 1.12 +[2025-04-28 04:42:23] (step=0103800) Train Loss: 5.6975, Train Steps/Sec: 1.12 +[2025-04-28 04:42:45] (step=0103825) Train Loss: 5.6719, Train Steps/Sec: 1.12 +[2025-04-28 04:43:08] (step=0103850) Train Loss: 5.6731, Train Steps/Sec: 1.12 +[2025-04-28 04:43:30] (step=0103875) Train Loss: 5.7238, Train Steps/Sec: 1.12 +[2025-04-28 04:43:52] (step=0103900) Train Loss: 5.6306, Train Steps/Sec: 1.12 +[2025-04-28 04:44:14] (step=0103925) Train Loss: 5.6715, Train Steps/Sec: 1.12 +[2025-04-28 04:44:37] (step=0103950) Train Loss: 5.6783, Train Steps/Sec: 1.12 +[2025-04-28 04:44:59] (step=0103975) Train Loss: 5.6428, Train Steps/Sec: 1.12 +[2025-04-28 04:45:21] (step=0104000) Train Loss: 5.7248, Train Steps/Sec: 1.12 +[2025-04-28 04:45:21] text_config is None. Initializing the text config with default values (`OPTConfig`). +[2025-04-28 04:51:40] Finish Eval in 104000 steps... +[2025-04-28 04:52:00] Saved checkpoint to checkpoint/X2I_UltraEdiit_fourmask_LLava_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_filtered_objects_4mask_1280_context/003-GPT-XL/checkpoints/0104000.pt +[2025-04-28 04:52:02] Removed old checkpoint: checkpoint/X2I_UltraEdiit_fourmask_LLava_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_filtered_objects_4mask_1280_context/003-GPT-XL/checkpoints/0102000.pt +[2025-04-28 04:52:24] (step=0104025) Train Loss: 5.6286, Train Steps/Sec: 0.06 +[2025-04-28 04:52:46] (step=0104050) Train Loss: 5.6815, Train Steps/Sec: 1.12 +[2025-04-28 04:53:09] (step=0104075) Train Loss: 5.6616, Train Steps/Sec: 1.12 +[2025-04-28 04:53:31] (step=0104100) Train Loss: 5.6505, Train Steps/Sec: 1.12 +[2025-04-28 04:53:53] (step=0104125) Train Loss: 5.6988, Train Steps/Sec: 1.12 +[2025-04-28 04:54:15] (step=0104150) Train Loss: 5.6185, Train Steps/Sec: 1.12 +[2025-04-28 04:54:38] (step=0104175) Train Loss: 5.6522, Train Steps/Sec: 1.12 +[2025-04-28 04:55:00] (step=0104200) Train Loss: 5.6767, Train Steps/Sec: 1.12 +[2025-04-28 04:55:22] (step=0104225) Train Loss: 5.7029, Train Steps/Sec: 1.13 +[2025-04-28 04:55:45] (step=0104250) Train Loss: 5.5823, Train Steps/Sec: 1.12 +[2025-04-28 04:56:07] (step=0104275) Train Loss: 5.6710, Train Steps/Sec: 1.12 +[2025-04-28 04:56:29] (step=0104300) Train Loss: 5.6820, Train Steps/Sec: 1.12 +[2025-04-28 04:56:51] (step=0104325) Train Loss: 5.6094, Train Steps/Sec: 1.12 +[2025-04-28 04:57:14] (step=0104350) Train Loss: 5.7068, Train Steps/Sec: 1.12 +[2025-04-28 04:57:36] (step=0104375) Train Loss: 5.7071, Train Steps/Sec: 1.12 +[2025-04-28 04:57:59] (step=0104400) Train Loss: 5.6844, Train Steps/Sec: 1.12 +[2025-04-28 04:58:21] (step=0104425) Train Loss: 5.6802, Train Steps/Sec: 1.12 +[2025-04-28 04:58:43] (step=0104450) Train Loss: 5.7649, Train Steps/Sec: 1.12 +[2025-04-28 04:59:05] (step=0104475) Train Loss: 5.6328, Train Steps/Sec: 1.12 +[2025-04-28 04:59:28] (step=0104500) Train Loss: 5.6158, Train Steps/Sec: 1.12 +[2025-04-28 04:59:50] (step=0104525) Train Loss: 5.6232, Train Steps/Sec: 1.12 +[2025-04-28 05:00:12] (step=0104550) Train Loss: 5.6202, Train Steps/Sec: 1.12 +[2025-04-28 05:00:35] (step=0104575) Train Loss: 5.6900, Train Steps/Sec: 1.12 +[2025-04-28 05:00:57] (step=0104600) Train Loss: 5.6840, Train Steps/Sec: 1.12 +[2025-04-28 05:01:19] (step=0104625) Train Loss: 5.6924, Train Steps/Sec: 1.12 +[2025-04-28 05:01:42] (step=0104650) Train Loss: 5.7199, Train Steps/Sec: 1.12 +[2025-04-28 05:02:04] (step=0104675) Train Loss: 5.6982, Train Steps/Sec: 1.12 +[2025-04-28 05:02:26] (step=0104700) Train Loss: 5.6402, Train Steps/Sec: 1.12 +[2025-04-28 05:02:48] (step=0104725) Train Loss: 5.6147, Train Steps/Sec: 1.12 +[2025-04-28 05:03:11] (step=0104750) Train Loss: 5.6616, Train Steps/Sec: 1.12 +[2025-04-28 05:03:33] (step=0104775) Train Loss: 5.7037, Train Steps/Sec: 1.12 +[2025-04-28 05:03:55] (step=0104800) Train Loss: 5.7020, Train Steps/Sec: 1.12 +[2025-04-28 05:04:17] (step=0104825) Train Loss: 5.6442, Train Steps/Sec: 1.12 +[2025-04-28 05:04:40] (step=0104850) Train Loss: 5.6295, Train Steps/Sec: 1.12 +[2025-04-28 05:05:06] (step=0104875) Train Loss: 5.6307, Train Steps/Sec: 0.94 +[2025-04-28 05:05:29] (step=0104900) Train Loss: 5.6720, Train Steps/Sec: 1.12 +[2025-04-28 05:05:51] (step=0104925) Train Loss: 5.6253, Train Steps/Sec: 1.12 +[2025-04-28 05:06:13] (step=0104950) Train Loss: 5.6822, Train Steps/Sec: 1.12 +[2025-04-28 05:06:36] (step=0104975) Train Loss: 5.6185, Train Steps/Sec: 1.12 +[2025-04-28 05:06:58] (step=0105000) Train Loss: 5.7470, Train Steps/Sec: 1.12 +[2025-04-28 05:07:20] (step=0105025) Train Loss: 5.6415, Train Steps/Sec: 1.12 +[2025-04-28 05:07:42] (step=0105050) Train Loss: 5.6485, Train Steps/Sec: 1.12 +[2025-04-28 05:08:05] (step=0105075) Train Loss: 5.7041, Train Steps/Sec: 1.12 +[2025-04-28 05:08:27] (step=0105100) Train Loss: 5.6191, Train Steps/Sec: 1.12 +[2025-04-28 05:08:49] (step=0105125) Train Loss: 5.7136, Train Steps/Sec: 1.12 +[2025-04-28 05:09:12] (step=0105150) Train Loss: 5.6208, Train Steps/Sec: 1.12 +[2025-04-28 05:09:34] (step=0105175) Train Loss: 5.6519, Train Steps/Sec: 1.12 +[2025-04-28 05:10:01] (step=0105200) Train Loss: 5.7018, Train Steps/Sec: 0.92 +[2025-04-28 05:10:23] (step=0105225) Train Loss: 5.7527, Train Steps/Sec: 1.12 +[2025-04-28 05:10:46] (step=0105250) Train Loss: 5.5952, Train Steps/Sec: 1.12 +[2025-04-28 05:11:08] (step=0105275) Train Loss: 5.6289, Train Steps/Sec: 1.12 +[2025-04-28 05:11:30] (step=0105300) Train Loss: 5.6615, Train Steps/Sec: 1.12 +[2025-04-28 05:11:52] (step=0105325) Train Loss: 5.6667, Train Steps/Sec: 1.12 +[2025-04-28 05:12:15] (step=0105350) Train Loss: 5.7322, Train Steps/Sec: 1.12 +[2025-04-28 05:12:37] (step=0105375) Train Loss: 5.6177, Train Steps/Sec: 1.12 +[2025-04-28 05:12:59] (step=0105400) Train Loss: 5.6619, Train Steps/Sec: 1.12 +[2025-04-28 05:13:22] (step=0105425) Train Loss: 5.6395, Train Steps/Sec: 1.13 +[2025-04-28 05:13:44] (step=0105450) Train Loss: 5.6864, Train Steps/Sec: 1.12 +[2025-04-28 05:14:06] (step=0105475) Train Loss: 5.6774, Train Steps/Sec: 1.12 +[2025-04-28 05:14:28] (step=0105500) Train Loss: 5.6698, Train Steps/Sec: 1.12 +[2025-04-28 05:14:51] (step=0105525) Train Loss: 5.6273, Train Steps/Sec: 1.12 +[2025-04-28 05:15:13] (step=0105550) Train Loss: 5.6333, Train Steps/Sec: 1.12 +[2025-04-28 05:15:35] (step=0105575) Train Loss: 5.5996, Train Steps/Sec: 1.12 +[2025-04-28 05:15:58] (step=0105600) Train Loss: 5.6513, Train Steps/Sec: 1.12 +[2025-04-28 05:16:20] (step=0105625) Train Loss: 5.6297, Train Steps/Sec: 1.12 +[2025-04-28 05:16:42] (step=0105650) Train Loss: 5.6239, Train Steps/Sec: 1.12 +[2025-04-28 05:17:05] (step=0105675) Train Loss: 5.6696, Train Steps/Sec: 1.12 +[2025-04-28 05:17:27] (step=0105700) Train Loss: 5.7294, Train Steps/Sec: 1.12 +[2025-04-28 05:17:49] (step=0105725) Train Loss: 5.5749, Train Steps/Sec: 1.12 +[2025-04-28 05:18:12] (step=0105750) Train Loss: 5.7510, Train Steps/Sec: 1.12 +[2025-04-28 05:18:34] (step=0105775) Train Loss: 5.6422, Train Steps/Sec: 1.12 +[2025-04-28 05:18:56] (step=0105800) Train Loss: 5.7241, Train Steps/Sec: 1.12 +[2025-04-28 05:19:19] (step=0105825) Train Loss: 5.7239, Train Steps/Sec: 1.12 +[2025-04-28 05:19:41] (step=0105850) Train Loss: 5.6135, Train Steps/Sec: 1.12 +[2025-04-28 05:20:03] (step=0105875) Train Loss: 5.6584, Train Steps/Sec: 1.12 +[2025-04-28 05:20:25] (step=0105900) Train Loss: 5.7054, Train Steps/Sec: 1.12 +[2025-04-28 05:20:48] (step=0105925) Train Loss: 5.6962, Train Steps/Sec: 1.12 +[2025-04-28 05:21:10] (step=0105950) Train Loss: 5.7337, Train Steps/Sec: 1.12 +[2025-04-28 05:21:32] (step=0105975) Train Loss: 5.6620, Train Steps/Sec: 1.12 +[2025-04-28 05:21:55] (step=0106000) Train Loss: 5.6149, Train Steps/Sec: 1.12 +[2025-04-28 05:21:55] text_config is None. Initializing the text config with default values (`OPTConfig`). +[2025-04-28 05:28:13] Finish Eval in 106000 steps... +[2025-04-28 05:28:33] Saved checkpoint to checkpoint/X2I_UltraEdiit_fourmask_LLava_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_filtered_objects_4mask_1280_context/003-GPT-XL/checkpoints/0106000.pt +[2025-04-28 05:28:35] Removed old checkpoint: checkpoint/X2I_UltraEdiit_fourmask_LLava_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_filtered_objects_4mask_1280_context/003-GPT-XL/checkpoints/0104000.pt +[2025-04-28 05:28:57] (step=0106025) Train Loss: 5.6616, Train Steps/Sec: 0.06 +[2025-04-28 05:29:20] (step=0106050) Train Loss: 5.6860, Train Steps/Sec: 1.12 +[2025-04-28 05:29:42] (step=0106075) Train Loss: 5.7142, Train Steps/Sec: 1.12 +[2025-04-28 05:30:04] (step=0106100) Train Loss: 5.6275, Train Steps/Sec: 1.12 +[2025-04-28 05:30:27] (step=0106125) Train Loss: 5.6484, Train Steps/Sec: 1.12 +[2025-04-28 05:30:49] (step=0106150) Train Loss: 5.6182, Train Steps/Sec: 1.12 +[2025-04-28 05:31:11] (step=0106175) Train Loss: 5.6594, Train Steps/Sec: 1.12 +[2025-04-28 05:31:33] (step=0106200) Train Loss: 5.6233, Train Steps/Sec: 1.12 +[2025-04-28 05:31:56] (step=0106225) Train Loss: 5.6929, Train Steps/Sec: 1.12 +[2025-04-28 05:32:18] (step=0106250) Train Loss: 5.6785, Train Steps/Sec: 1.11 +[2025-04-28 05:32:49] (step=0106275) Train Loss: 5.6600, Train Steps/Sec: 0.81 +[2025-04-28 05:33:16] (step=0106300) Train Loss: 5.6614, Train Steps/Sec: 0.94 +[2025-04-28 05:33:43] (step=0106325) Train Loss: 5.6518, Train Steps/Sec: 0.91 +[2025-04-28 05:34:05] (step=0106350) Train Loss: 5.6222, Train Steps/Sec: 1.12 +[2025-04-28 05:34:28] (step=0106375) Train Loss: 5.5722, Train Steps/Sec: 1.12 +[2025-04-28 05:34:50] (step=0106400) Train Loss: 5.6212, Train Steps/Sec: 1.12 +[2025-04-28 05:35:12] (step=0106425) Train Loss: 5.6859, Train Steps/Sec: 1.12 +[2025-04-28 05:35:34] (step=0106450) Train Loss: 5.6593, Train Steps/Sec: 1.13 +[2025-04-28 05:35:57] (step=0106475) Train Loss: 5.6533, Train Steps/Sec: 1.12 +[2025-04-28 05:36:19] (step=0106500) Train Loss: 5.6748, Train Steps/Sec: 1.12 +[2025-04-28 05:36:41] (step=0106525) Train Loss: 5.5965, Train Steps/Sec: 1.12 +[2025-04-28 05:37:03] (step=0106550) Train Loss: 5.6909, Train Steps/Sec: 1.12 +[2025-04-28 05:37:26] (step=0106575) Train Loss: 5.6592, Train Steps/Sec: 1.12 +[2025-04-28 05:37:48] (step=0106600) Train Loss: 5.6932, Train Steps/Sec: 1.12 +[2025-04-28 05:38:10] (step=0106625) Train Loss: 5.6847, Train Steps/Sec: 1.13 +[2025-04-28 05:38:33] (step=0106650) Train Loss: 5.7112, Train Steps/Sec: 1.12 +[2025-04-28 05:38:55] (step=0106675) Train Loss: 5.6570, Train Steps/Sec: 1.12 +[2025-04-28 05:39:17] (step=0106700) Train Loss: 5.6682, Train Steps/Sec: 1.12 +[2025-04-28 05:39:39] (step=0106725) Train Loss: 5.6847, Train Steps/Sec: 1.12 +[2025-04-28 05:40:02] (step=0106750) Train Loss: 5.7051, Train Steps/Sec: 1.13 +[2025-04-28 05:40:24] (step=0106775) Train Loss: 5.7418, Train Steps/Sec: 1.12 +[2025-04-28 05:40:46] (step=0106800) Train Loss: 5.6276, Train Steps/Sec: 1.12 +[2025-04-28 05:41:09] (step=0106825) Train Loss: 5.6928, Train Steps/Sec: 1.12 +[2025-04-28 05:41:31] (step=0106850) Train Loss: 5.7028, Train Steps/Sec: 1.12 +[2025-04-28 05:41:53] (step=0106875) Train Loss: 5.6421, Train Steps/Sec: 1.12 +[2025-04-28 05:42:15] (step=0106900) Train Loss: 5.7371, Train Steps/Sec: 1.12 +[2025-04-28 05:42:38] (step=0106925) Train Loss: 5.5983, Train Steps/Sec: 1.12 +[2025-04-28 05:43:00] (step=0106950) Train Loss: 5.6492, Train Steps/Sec: 1.13 +[2025-04-28 05:43:22] (step=0106975) Train Loss: 5.6312, Train Steps/Sec: 1.12 +[2025-04-28 05:43:45] (step=0107000) Train Loss: 5.6815, Train Steps/Sec: 1.12 +[2025-04-28 05:44:07] (step=0107025) Train Loss: 5.6344, Train Steps/Sec: 1.12 +[2025-04-28 05:44:29] (step=0107050) Train Loss: 5.7710, Train Steps/Sec: 1.12 +[2025-04-28 05:44:56] (step=0107075) Train Loss: 5.5905, Train Steps/Sec: 0.92 +[2025-04-28 05:45:19] (step=0107100) Train Loss: 5.6663, Train Steps/Sec: 1.12 +[2025-04-28 05:45:41] (step=0107125) Train Loss: 5.6756, Train Steps/Sec: 1.12 +[2025-04-28 05:46:03] (step=0107150) Train Loss: 5.6508, Train Steps/Sec: 1.12 +[2025-04-28 05:46:25] (step=0107175) Train Loss: 5.6479, Train Steps/Sec: 1.12 +[2025-04-28 05:46:48] (step=0107200) Train Loss: 5.7219, Train Steps/Sec: 1.12 +[2025-04-28 05:47:10] (step=0107225) Train Loss: 5.6614, Train Steps/Sec: 1.12 +[2025-04-28 05:47:32] (step=0107250) Train Loss: 5.6385, Train Steps/Sec: 1.12 +[2025-04-28 05:47:55] (step=0107275) Train Loss: 5.6041, Train Steps/Sec: 1.12 +[2025-04-28 05:48:17] (step=0107300) Train Loss: 5.7071, Train Steps/Sec: 1.12 +[2025-04-28 05:48:39] (step=0107325) Train Loss: 5.6523, Train Steps/Sec: 1.12 +[2025-04-28 05:49:01] (step=0107350) Train Loss: 5.7322, Train Steps/Sec: 1.12 +[2025-04-28 05:49:24] (step=0107375) Train Loss: 5.6573, Train Steps/Sec: 1.12 +[2025-04-28 05:49:46] (step=0107400) Train Loss: 5.6853, Train Steps/Sec: 1.12 +[2025-04-28 05:50:08] (step=0107425) Train Loss: 5.7120, Train Steps/Sec: 1.12 +[2025-04-28 05:50:31] (step=0107450) Train Loss: 5.6531, Train Steps/Sec: 1.12 +[2025-04-28 05:50:53] (step=0107475) Train Loss: 5.6696, Train Steps/Sec: 1.12 +[2025-04-28 05:51:15] (step=0107500) Train Loss: 5.6240, Train Steps/Sec: 1.12 +[2025-04-28 05:51:37] (step=0107525) Train Loss: 5.6913, Train Steps/Sec: 1.12 +[2025-04-28 05:52:00] (step=0107550) Train Loss: 5.6039, Train Steps/Sec: 1.12 +[2025-04-28 05:52:22] (step=0107575) Train Loss: 5.6692, Train Steps/Sec: 1.12 +[2025-04-28 05:52:44] (step=0107600) Train Loss: 5.6389, Train Steps/Sec: 1.12 +[2025-04-28 05:53:07] (step=0107625) Train Loss: 5.6716, Train Steps/Sec: 1.12 +[2025-04-28 05:53:29] (step=0107650) Train Loss: 5.6596, Train Steps/Sec: 1.12 +[2025-04-28 05:53:51] (step=0107675) Train Loss: 5.6890, Train Steps/Sec: 1.12 +[2025-04-28 05:54:14] (step=0107700) Train Loss: 5.6765, Train Steps/Sec: 1.12 +[2025-04-28 05:54:36] (step=0107725) Train Loss: 5.6471, Train Steps/Sec: 1.12 +[2025-04-28 05:54:58] (step=0107750) Train Loss: 5.6349, Train Steps/Sec: 1.12 +[2025-04-28 05:55:20] (step=0107775) Train Loss: 5.7140, Train Steps/Sec: 1.12 +[2025-04-28 05:55:43] (step=0107800) Train Loss: 5.6696, Train Steps/Sec: 1.12 +[2025-04-28 05:56:05] (step=0107825) Train Loss: 5.6253, Train Steps/Sec: 1.12 +[2025-04-28 05:56:27] (step=0107850) Train Loss: 5.6703, Train Steps/Sec: 1.12 +[2025-04-28 05:56:50] (step=0107875) Train Loss: 5.6673, Train Steps/Sec: 1.12 +[2025-04-28 05:57:12] (step=0107900) Train Loss: 5.7142, Train Steps/Sec: 1.12 +[2025-04-28 05:57:34] (step=0107925) Train Loss: 5.7046, Train Steps/Sec: 1.12 +[2025-04-28 05:57:56] (step=0107950) Train Loss: 5.6355, Train Steps/Sec: 1.12 +[2025-04-28 05:58:19] (step=0107975) Train Loss: 5.6673, Train Steps/Sec: 1.12 +[2025-04-28 05:58:41] (step=0108000) Train Loss: 5.7060, Train Steps/Sec: 1.12 +[2025-04-28 05:58:41] text_config is None. Initializing the text config with default values (`OPTConfig`). +[2025-04-28 06:05:00] Finish Eval in 108000 steps... +[2025-04-28 06:05:20] Saved checkpoint to checkpoint/X2I_UltraEdiit_fourmask_LLava_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_filtered_objects_4mask_1280_context/003-GPT-XL/checkpoints/0108000.pt +[2025-04-28 06:05:22] Removed old checkpoint: checkpoint/X2I_UltraEdiit_fourmask_LLava_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_filtered_objects_4mask_1280_context/003-GPT-XL/checkpoints/0106000.pt +[2025-04-28 06:05:44] (step=0108025) Train Loss: 5.7117, Train Steps/Sec: 0.06 +[2025-04-28 06:06:06] (step=0108050) Train Loss: 5.7686, Train Steps/Sec: 1.12 +[2025-04-28 06:06:29] (step=0108075) Train Loss: 5.6358, Train Steps/Sec: 1.12 +[2025-04-28 06:06:51] (step=0108100) Train Loss: 5.6790, Train Steps/Sec: 1.12 +[2025-04-28 06:07:13] (step=0108125) Train Loss: 5.6760, Train Steps/Sec: 1.12 +[2025-04-28 06:07:36] (step=0108150) Train Loss: 5.6635, Train Steps/Sec: 1.10 +[2025-04-28 06:07:58] (step=0108175) Train Loss: 5.6564, Train Steps/Sec: 1.12 +[2025-04-28 06:08:21] (step=0108200) Train Loss: 5.6676, Train Steps/Sec: 1.11 +[2025-04-28 06:08:43] (step=0108225) Train Loss: 5.6975, Train Steps/Sec: 1.12 +[2025-04-28 06:09:05] (step=0108250) Train Loss: 5.7219, Train Steps/Sec: 1.12 +[2025-04-28 06:09:28] (step=0108275) Train Loss: 5.6147, Train Steps/Sec: 1.12 +[2025-04-28 06:09:50] (step=0108300) Train Loss: 5.7275, Train Steps/Sec: 1.12 +[2025-04-28 06:10:12] (step=0108325) Train Loss: 5.6888, Train Steps/Sec: 1.12 +[2025-04-28 06:10:35] (step=0108350) Train Loss: 5.6635, Train Steps/Sec: 1.12 +[2025-04-28 06:10:57] (step=0108375) Train Loss: 5.6532, Train Steps/Sec: 1.12 +[2025-04-28 06:11:19] (step=0108400) Train Loss: 5.7016, Train Steps/Sec: 1.12 +[2025-04-28 06:11:41] (step=0108425) Train Loss: 5.6836, Train Steps/Sec: 1.12 +[2025-04-28 06:12:04] (step=0108450) Train Loss: 5.6696, Train Steps/Sec: 1.13 +[2025-04-28 06:12:26] (step=0108475) Train Loss: 5.6215, Train Steps/Sec: 1.12 +[2025-04-28 06:12:48] (step=0108500) Train Loss: 5.6226, Train Steps/Sec: 1.12 +[2025-04-28 06:13:11] (step=0108525) Train Loss: 5.6640, Train Steps/Sec: 1.12 +[2025-04-28 06:13:33] (step=0108550) Train Loss: 5.6623, Train Steps/Sec: 1.12 +[2025-04-28 06:13:55] (step=0108575) Train Loss: 5.7157, Train Steps/Sec: 1.12 +[2025-04-28 06:14:18] (step=0108600) Train Loss: 5.6495, Train Steps/Sec: 1.11 +[2025-04-28 06:14:40] (step=0108625) Train Loss: 5.7214, Train Steps/Sec: 1.12 +[2025-04-28 06:15:02] (step=0108650) Train Loss: 5.6718, Train Steps/Sec: 1.12 +[2025-04-28 06:15:24] (step=0108675) Train Loss: 5.5886, Train Steps/Sec: 1.12 +[2025-04-28 06:15:47] (step=0108700) Train Loss: 5.5783, Train Steps/Sec: 1.12 +[2025-04-28 06:16:09] (step=0108725) Train Loss: 5.5424, Train Steps/Sec: 1.12 +[2025-04-28 06:16:31] (step=0108750) Train Loss: 5.6489, Train Steps/Sec: 1.12 +[2025-04-28 06:16:54] (step=0108775) Train Loss: 5.6288, Train Steps/Sec: 1.12 +[2025-04-28 06:17:16] (step=0108800) Train Loss: 5.6587, Train Steps/Sec: 1.11 +[2025-04-28 06:17:38] (step=0108825) Train Loss: 5.6620, Train Steps/Sec: 1.12 +[2025-04-28 06:18:01] (step=0108850) Train Loss: 5.6392, Train Steps/Sec: 1.12 +[2025-04-28 06:18:23] (step=0108875) Train Loss: 5.5899, Train Steps/Sec: 1.12 +[2025-04-28 06:18:45] (step=0108900) Train Loss: 5.6960, Train Steps/Sec: 1.12 +[2025-04-28 06:19:08] (step=0108925) Train Loss: 5.6765, Train Steps/Sec: 1.12 +[2025-04-28 06:19:30] (step=0108950) Train Loss: 5.7514, Train Steps/Sec: 1.12 +[2025-04-28 06:19:52] (step=0108975) Train Loss: 5.7057, Train Steps/Sec: 1.12 +[2025-04-28 06:20:15] (step=0109000) Train Loss: 5.6216, Train Steps/Sec: 1.12 +[2025-04-28 06:20:37] (step=0109025) Train Loss: 5.6664, Train Steps/Sec: 1.12 +[2025-04-28 06:20:59] (step=0109050) Train Loss: 5.6447, Train Steps/Sec: 1.12 +[2025-04-28 06:21:21] (step=0109075) Train Loss: 5.6742, Train Steps/Sec: 1.12 +[2025-04-28 06:21:44] (step=0109100) Train Loss: 5.6565, Train Steps/Sec: 1.12 +[2025-04-28 06:22:11] (step=0109125) Train Loss: 5.6200, Train Steps/Sec: 0.93 +[2025-04-28 06:22:33] (step=0109150) Train Loss: 5.6786, Train Steps/Sec: 1.12 +[2025-04-28 06:22:55] (step=0109175) Train Loss: 5.7064, Train Steps/Sec: 1.12 +[2025-04-28 06:23:18] (step=0109200) Train Loss: 5.6454, Train Steps/Sec: 1.12 +[2025-04-28 06:23:40] (step=0109225) Train Loss: 5.6551, Train Steps/Sec: 1.12 +[2025-04-28 06:24:02] (step=0109250) Train Loss: 5.7165, Train Steps/Sec: 1.12 +[2025-04-28 06:24:24] (step=0109275) Train Loss: 5.7141, Train Steps/Sec: 1.12 +[2025-04-28 06:24:47] (step=0109300) Train Loss: 5.5816, Train Steps/Sec: 1.12 +[2025-04-28 06:25:09] (step=0109325) Train Loss: 5.6663, Train Steps/Sec: 1.12 +[2025-04-28 06:25:31] (step=0109350) Train Loss: 5.7142, Train Steps/Sec: 1.12 +[2025-04-28 06:25:54] (step=0109375) Train Loss: 5.7033, Train Steps/Sec: 1.12 +[2025-04-28 06:26:16] (step=0109400) Train Loss: 5.5975, Train Steps/Sec: 1.12 +[2025-04-28 06:26:38] (step=0109425) Train Loss: 5.6214, Train Steps/Sec: 1.12 +[2025-04-28 06:27:00] (step=0109450) Train Loss: 5.6980, Train Steps/Sec: 1.12 +[2025-04-28 06:27:27] (step=0109475) Train Loss: 5.6878, Train Steps/Sec: 0.92 +[2025-04-28 06:27:50] (step=0109500) Train Loss: 5.5591, Train Steps/Sec: 1.12 +[2025-04-28 06:28:12] (step=0109525) Train Loss: 5.6242, Train Steps/Sec: 1.12 +[2025-04-28 06:28:34] (step=0109550) Train Loss: 5.6647, Train Steps/Sec: 1.12 +[2025-04-28 06:28:57] (step=0109575) Train Loss: 5.6834, Train Steps/Sec: 1.12 +[2025-04-28 06:29:19] (step=0109600) Train Loss: 5.6525, Train Steps/Sec: 1.12 +[2025-04-28 06:29:41] (step=0109625) Train Loss: 5.6323, Train Steps/Sec: 1.12 +[2025-04-28 06:30:04] (step=0109650) Train Loss: 5.6584, Train Steps/Sec: 1.12 +[2025-04-28 06:30:26] (step=0109675) Train Loss: 5.6350, Train Steps/Sec: 1.12 +[2025-04-28 06:30:48] (step=0109700) Train Loss: 5.6594, Train Steps/Sec: 1.12 +[2025-04-28 06:31:10] (step=0109725) Train Loss: 5.7304, Train Steps/Sec: 1.12 +[2025-04-28 06:31:33] (step=0109750) Train Loss: 5.6889, Train Steps/Sec: 1.12 +[2025-04-28 06:31:55] (step=0109775) Train Loss: 5.6489, Train Steps/Sec: 1.12 +[2025-04-28 06:32:17] (step=0109800) Train Loss: 5.6170, Train Steps/Sec: 1.12 +[2025-04-28 06:32:40] (step=0109825) Train Loss: 5.6850, Train Steps/Sec: 1.12 +[2025-04-28 06:33:02] (step=0109850) Train Loss: 5.6664, Train Steps/Sec: 1.12 +[2025-04-28 06:33:24] (step=0109875) Train Loss: 5.6091, Train Steps/Sec: 1.12 +[2025-04-28 06:33:46] (step=0109900) Train Loss: 5.6328, Train Steps/Sec: 1.12 +[2025-04-28 06:34:09] (step=0109925) Train Loss: 5.6571, Train Steps/Sec: 1.12 +[2025-04-28 06:34:31] (step=0109950) Train Loss: 5.6161, Train Steps/Sec: 1.12 +[2025-04-28 06:34:53] (step=0109975) Train Loss: 5.6925, Train Steps/Sec: 1.12 +[2025-04-28 06:35:16] (step=0110000) Train Loss: 5.6601, Train Steps/Sec: 1.12 +[2025-04-28 06:35:16] text_config is None. Initializing the text config with default values (`OPTConfig`). +[2025-04-28 06:41:34] Finish Eval in 110000 steps... +[2025-04-28 06:41:53] Saved checkpoint to checkpoint/X2I_UltraEdiit_fourmask_LLava_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_filtered_objects_4mask_1280_context/003-GPT-XL/checkpoints/0110000.pt +[2025-04-28 06:41:55] Removed old checkpoint: checkpoint/X2I_UltraEdiit_fourmask_LLava_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_filtered_objects_4mask_1280_context/003-GPT-XL/checkpoints/0108000.pt +[2025-04-28 06:42:17] (step=0110025) Train Loss: 5.7012, Train Steps/Sec: 0.06 +[2025-04-28 06:42:40] (step=0110050) Train Loss: 5.6505, Train Steps/Sec: 1.10 +[2025-04-28 06:43:02] (step=0110075) Train Loss: 5.6027, Train Steps/Sec: 1.12 +[2025-04-28 06:43:25] (step=0110100) Train Loss: 5.6725, Train Steps/Sec: 1.12 +[2025-04-28 06:43:47] (step=0110125) Train Loss: 5.7205, Train Steps/Sec: 1.12 +[2025-04-28 06:44:09] (step=0110150) Train Loss: 5.6553, Train Steps/Sec: 1.12 +[2025-04-28 06:44:31] (step=0110175) Train Loss: 5.6895, Train Steps/Sec: 1.12 +[2025-04-28 06:44:54] (step=0110200) Train Loss: 5.7215, Train Steps/Sec: 1.12 +[2025-04-28 06:45:16] (step=0110225) Train Loss: 5.6276, Train Steps/Sec: 1.12 +[2025-04-28 06:45:38] (step=0110250) Train Loss: 5.6122, Train Steps/Sec: 1.12 +[2025-04-28 06:46:01] (step=0110275) Train Loss: 5.7012, Train Steps/Sec: 1.12 +[2025-04-28 06:46:23] (step=0110300) Train Loss: 5.6694, Train Steps/Sec: 1.12 +[2025-04-28 06:46:45] (step=0110325) Train Loss: 5.6489, Train Steps/Sec: 1.12 +[2025-04-28 06:47:07] (step=0110350) Train Loss: 5.6281, Train Steps/Sec: 1.12 +[2025-04-28 06:47:30] (step=0110375) Train Loss: 5.6736, Train Steps/Sec: 1.12 +[2025-04-28 06:47:52] (step=0110400) Train Loss: 5.6578, Train Steps/Sec: 1.12 +[2025-04-28 06:48:14] (step=0110425) Train Loss: 5.6692, Train Steps/Sec: 1.12 +[2025-04-28 06:48:37] (step=0110450) Train Loss: 5.6487, Train Steps/Sec: 1.12 +[2025-04-28 06:48:59] (step=0110475) Train Loss: 5.6544, Train Steps/Sec: 1.12 +[2025-04-28 06:49:21] (step=0110500) Train Loss: 5.6840, Train Steps/Sec: 1.12 +[2025-04-28 06:49:43] (step=0110525) Train Loss: 5.5940, Train Steps/Sec: 1.12 +[2025-04-28 06:50:06] (step=0110550) Train Loss: 5.6951, Train Steps/Sec: 1.12 +[2025-04-28 06:50:28] (step=0110575) Train Loss: 5.6126, Train Steps/Sec: 1.12 +[2025-04-28 06:51:03] (step=0110600) Train Loss: 5.6076, Train Steps/Sec: 0.71 +[2025-04-28 06:51:25] (step=0110625) Train Loss: 5.6658, Train Steps/Sec: 1.12 +[2025-04-28 06:51:53] (step=0110650) Train Loss: 5.6951, Train Steps/Sec: 0.91 +[2025-04-28 06:52:15] (step=0110675) Train Loss: 5.6242, Train Steps/Sec: 1.12 +[2025-04-28 06:52:36] Done! diff --git a/CKPTS/X2I_UltraEdiit_fourmask_LLava_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_filtered_objects_4mask_1280_context/wandb/debug-internal.log b/CKPTS/X2I_UltraEdiit_fourmask_LLava_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_filtered_objects_4mask_1280_context/wandb/debug-internal.log new file mode 100644 index 0000000000000000000000000000000000000000..b84e7a539e16b615eccb2774280b7e984207c8bd --- /dev/null +++ b/CKPTS/X2I_UltraEdiit_fourmask_LLava_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_filtered_objects_4mask_1280_context/wandb/debug-internal.log @@ -0,0 +1,15 @@ +{"time":"2025-04-26T21:03:23.738486816Z","level":"INFO","msg":"stream: starting","core version":"0.19.8","symlink path":"checkpoint/X2I_UltraEdiit_fourmask_LLava_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_filtered_objects_4mask_1280_context/wandb/run-20250426_210323-vtrxdt92/logs/debug-core.log"} +{"time":"2025-04-26T21:03:23.945252394Z","level":"INFO","msg":"created new stream","id":"vtrxdt92"} +{"time":"2025-04-26T21:03:23.945284534Z","level":"INFO","msg":"stream: started","id":"vtrxdt92"} +{"time":"2025-04-26T21:03:23.945335685Z","level":"INFO","msg":"writer: Do: started","stream_id":"vtrxdt92"} +{"time":"2025-04-26T21:03:23.945496626Z","level":"INFO","msg":"handler: started","stream_id":"vtrxdt92"} +{"time":"2025-04-26T21:03:23.945514981Z","level":"INFO","msg":"sender: started","stream_id":"vtrxdt92"} +{"time":"2025-04-26T21:03:24.215962364Z","level":"INFO","msg":"Starting system monitor"} +{"time":"2025-04-28T06:52:42.747611793Z","level":"INFO","msg":"stream: closing","id":"vtrxdt92"} +{"time":"2025-04-28T06:52:42.747654202Z","level":"INFO","msg":"Stopping system monitor"} +{"time":"2025-04-28T06:52:42.748621161Z","level":"INFO","msg":"Stopped system monitor"} +{"time":"2025-04-28T06:52:44.085866147Z","level":"INFO","msg":"fileTransfer: Close: file transfer manager closed"} +{"time":"2025-04-28T06:52:44.290568215Z","level":"INFO","msg":"handler: closed","stream_id":"vtrxdt92"} +{"time":"2025-04-28T06:52:44.290611926Z","level":"INFO","msg":"sender: closed","stream_id":"vtrxdt92"} +{"time":"2025-04-28T06:52:44.290607003Z","level":"INFO","msg":"writer: Close: closed","stream_id":"vtrxdt92"} +{"time":"2025-04-28T06:52:44.290714634Z","level":"INFO","msg":"stream: closed","id":"vtrxdt92"} diff --git a/CKPTS/X2I_UltraEdiit_fourmask_LLava_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_filtered_objects_4mask_1280_context/wandb/debug.log b/CKPTS/X2I_UltraEdiit_fourmask_LLava_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_filtered_objects_4mask_1280_context/wandb/debug.log new file mode 100644 index 0000000000000000000000000000000000000000..d9a06495e04d3aebe3d21430ae11c2af92cdf189 --- /dev/null +++ b/CKPTS/X2I_UltraEdiit_fourmask_LLava_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_filtered_objects_4mask_1280_context/wandb/debug.log @@ -0,0 +1,23 @@ +2025-04-26 21:03:23,731 INFO MainThread:3723769 [wandb_setup.py:_flush():67] Current SDK version is 0.19.8 +2025-04-26 21:03:23,731 INFO MainThread:3723769 [wandb_setup.py:_flush():67] Configure stats pid to 3723769 +2025-04-26 21:03:23,731 INFO MainThread:3723769 [wandb_setup.py:_flush():67] Loading settings from /tmp/haozhezhao/.config/wandb/settings +2025-04-26 21:03:23,731 INFO MainThread:3723769 [wandb_setup.py:_flush():67] Loading settings from /tmp/haozhezhao/MLLMG/wandb/settings +2025-04-26 21:03:23,731 INFO MainThread:3723769 [wandb_setup.py:_flush():67] Loading settings from environment variables +2025-04-26 21:03:23,731 INFO MainThread:3723769 [wandb_init.py:setup_run_log_directory():647] Logging user logs to checkpoint/X2I_UltraEdiit_fourmask_LLava_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_filtered_objects_4mask_1280_context/wandb/run-20250426_210323-vtrxdt92/logs/debug.log +2025-04-26 21:03:23,731 INFO MainThread:3723769 [wandb_init.py:setup_run_log_directory():648] Logging internal logs to checkpoint/X2I_UltraEdiit_fourmask_LLava_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_filtered_objects_4mask_1280_context/wandb/run-20250426_210323-vtrxdt92/logs/debug-internal.log +2025-04-26 21:03:23,731 INFO MainThread:3723769 [wandb_init.py:init():761] calling init triggers +2025-04-26 21:03:23,731 INFO MainThread:3723769 [wandb_init.py:init():766] wandb.init called with sweep_config: {} +config: {'data_path': '/tmp/haozhezhao/MLLMG/jsonl_data/multiimage_training_for_llava_X2I_UltraEdiit_fourmask.jsonl', 'cloud_save_path': '/tmp/haozhezhao/MLLMG/checkpoint', 'no_local_save': False, 'vq_model': 'VQ-16', 'vq_ckpt': '/tmp/haozhezhao/model/llamagen_t2i/vq_ds16_t2i.pt', 'codebook_size': 16384, 'codebook_embed_dim': 8, 'gpt_model': 'GPT-XL', 'gpt_ckpt': '/tmp/haozhezhao/MLLMG/checkpoint/Afteri2i_100kckpt_LLAVAT5_empty_Fixed_just_segment_pretrain_stage3_dreambench__recap_400k_t2i_400k_flux_200k_midjourney_150k_recovery_150k_grounding_100fluxseg_50samseg/005-GPT-XL/checkpoints/0078000.pt', 'gpt_type': 't2i', 'vocab_size': 16384, 'cls_token_num': 1280, 'dropout_p': 0.1, 'token_dropout_p': 0.1, 'drop_path': 0.0, 'no_compile': False, 'results_dir': 'checkpoint/X2I_UltraEdiit_fourmask_LLava_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_filtered_objects_4mask_1280_context', 'dataset': 'ti2i', 'image_size': 512, 'downsample_size': 16, 'num_classes': 1000, 'epochs': 1, 'lr': 0.0001, 'weight_decay': 0.05, 'beta1': 0.9, 'beta2': 0.95, 'max_grad_norm': 1.0, 'global_batch_size': 24, 'global_seed': 0, 'num_workers': 4, 'log_every': 25, 'ckpt_every': 2000, 'gradient_accumulation_steps': 8, 'mixed_precision': 'bf16', 'val_data_path': '/tmp/haozhezhao/MLLMG/jsonl_data/multiimage_val_for_llava_X2I_UltraEdiit_fourmask_dreambenplus.jsonl', 'use_vision_tower': True, 'model_name_or_path': '/tmp/haozhezhao/model/blip2-flan-t5-xl', 'image_place_holder': '', 'processor_path': None, 'do_eval': True, 'max_eval_samples': 128, 'train_text_encoder': True, 'no_left_padding': False, 'cfg_scale': 7.5, 'top_k': 16384, 'temperature': 0.9, 'top_p': 1.0, 'eval_steps': 2000, 'project_name': 'llamagen_ti2i', 'load_from_checkpoint': '/tmp/haozhezhao/MLLMG/checkpoint/FIXed_3M_1epoch_step32000.pt', 'warmup': 0.05, 'lr_decay_style': 'cosine', 'lr_decay_ratio': 0.1, 'train_iters': 500000, 'class_dropout_prob': 0.1, 'with_image_only': False, 'image_only_rate': 0.1, 'stage2': False, 'subject_driven': True, 'load_subject_embedding': None, 'reference_data_path': '/tmp/haozhezhao/MLLMG/cc12m_reference_tunnel.jsonl', 'multimodal_encoder': 'llava', 'do_recovery': True, 'no_replace': False, 'resume': False, 'dreambench_eval': True, 'find_unused_parameters': True, 'load_visual_encoder': False, 'continue_stage1': False, 'replace_subject': False, 'train_all': True, 'save_total_limit': 1, 'load_language_projection': '/tmp/haozhezhao/MLLMG/llava-v1.5-flant5_fixed-pretrain/mm_projector.bin', 'mm_vision_tower': 'openai/clip-vit-large-patch14', 'load_fixed_llamagen': True, 'unfreeze_output': False, 'fix': 'gpt-empty-fix', 'rank': 0, 'world_size': 8, 'gpu': 0, 'dist_url': 'env://', 'distributed': True, 'dist_backend': 'nccl', '_wandb': {}} +2025-04-26 21:03:23,732 INFO MainThread:3723769 [wandb_init.py:init():784] starting backend +2025-04-26 21:03:23,732 INFO MainThread:3723769 [wandb_init.py:init():788] sending inform_init request +2025-04-26 21:03:23,736 INFO MainThread:3723769 [backend.py:_multiprocessing_setup():101] multiprocessing start_methods=fork,spawn,forkserver, using: spawn +2025-04-26 21:03:23,736 INFO MainThread:3723769 [wandb_init.py:init():798] backend started and connected +2025-04-26 21:03:23,738 INFO MainThread:3723769 [wandb_init.py:init():891] updated telemetry +2025-04-26 21:03:23,739 INFO MainThread:3723769 [wandb_init.py:init():915] communicating run to backend with 90.0 second timeout +2025-04-26 21:03:24,213 INFO MainThread:3723769 [wandb_init.py:init():990] starting run threads in backend +2025-04-26 21:03:24,296 INFO MainThread:3723769 [wandb_run.py:_console_start():2375] atexit reg +2025-04-26 21:03:24,296 INFO MainThread:3723769 [wandb_run.py:_redirect():2227] redirect: wrap_raw +2025-04-26 21:03:24,297 INFO MainThread:3723769 [wandb_run.py:_redirect():2292] Wrapping output streams. +2025-04-26 21:03:24,297 INFO MainThread:3723769 [wandb_run.py:_redirect():2315] Redirects installed. +2025-04-26 21:03:24,298 INFO MainThread:3723769 [wandb_init.py:init():1032] run started, returning control to user process +2025-04-28 06:52:42,746 INFO MsgRouterThr:3723769 [mailbox.py:close():129] Closing mailbox, abandoning 1 handles. diff --git a/CKPTS/X2I_UltraEdiit_fourmask_LLava_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_filtered_objects_4mask_1280_context/wandb/run-20250426_205319-m68pf6i5/files/output.log b/CKPTS/X2I_UltraEdiit_fourmask_LLava_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_filtered_objects_4mask_1280_context/wandb/run-20250426_205319-m68pf6i5/files/output.log new file mode 100644 index 0000000000000000000000000000000000000000..b3d4dadb6531a361f9b8d82be2096adcda03bf04 --- /dev/null +++ b/CKPTS/X2I_UltraEdiit_fourmask_LLava_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_filtered_objects_4mask_1280_context/wandb/run-20250426_205319-m68pf6i5/files/output.log @@ -0,0 +1,33 @@ +[2025-04-26 20:53:20] Training for 1 epochs... +[2025-04-26 20:53:20] Beginning epoch 0... + 0%| | 0/110696 [00:00", + "--do_eval", + "--eval_steps", + "2000", + "--max_eval_samples", + "128", + "--cfg-scale", + "7.5", + "--top-k", + "16384", + "--load_from_checkpoint", + "/tmp/haozhezhao/MLLMG/checkpoint/FIXed_3M_1epoch_step32000.pt", + "--global-batch-size", + "24", + "--num-workers", + "4", + "--warmup", + "0.05", + "--gradient-accumulation-steps", + "8", + "--train_text_encoder", + "--ckpt-every", + "2000", + "--epochs", + "1", + "--subject_driven", + "--reference_data_path", + "/tmp/haozhezhao/MLLMG/cc12m_reference_tunnel.jsonl", + "--multimodal_encoder", + "llava", + "--do_recovery", + "--find_unused_parameters", + "--cls-token-num", + "1280", + "--train_all", + "--load_fixed_llamagen", + "--save_total_limit", + "1", + "--fix", + "gpt-empty-fix", + "--load_language_projection", + "/tmp/haozhezhao/MLLMG/llava-v1.5-flant5_fixed-pretrain/mm_projector.bin", + "--mm_vision_tower", + "openai/clip-vit-large-patch14", + "--gpt-ckpt", + "/tmp/haozhezhao/MLLMG/checkpoint/Afteri2i_100kckpt_LLAVAT5_empty_Fixed_just_segment_pretrain_stage3_dreambench__recap_400k_t2i_400k_flux_200k_midjourney_150k_recovery_150k_grounding_100fluxseg_50samseg/005-GPT-XL/checkpoints/0078000.pt", + "--dreambench_eval" + ], + "program": "/tmp/haozhezhao/MLLMG/autoregressive/train/train_t2i.py", + "codePath": "autoregressive/train/train_t2i.py", + "email": "mimazhe55360@gmail.com", + "root": "checkpoint/X2I_UltraEdiit_fourmask_LLava_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_filtered_objects_4mask_1280_context", + "host": "447cc403a8794092814259713c51c1df00001X", + "executable": "/tmp/haozhezhao/anaconda3/envs/nlp/bin/python", + "codePathLocal": "autoregressive/train/train_t2i.py", + "cpu_count": 96, + "cpu_count_logical": 96, + "gpu": "NVIDIA A100-SXM4-80GB", + "gpu_count": 8, + "disk": { + "/": { + "total": "133003395072", + "used": "65761198080" + } + }, + "memory": { + "total": "1902387884032" + }, + "cpu": { + "count": 96, + "countLogical": 96 + }, + "gpu_nvidia": [ + { + "name": "NVIDIA A100-SXM4-80GB", + "memoryTotal": "85899345920", + "cudaCores": 6912, + "architecture": "Ampere" + }, + { + "name": "NVIDIA A100-SXM4-80GB", + "memoryTotal": "85899345920", + "cudaCores": 6912, + "architecture": "Ampere" + }, + { + "name": "NVIDIA A100-SXM4-80GB", + "memoryTotal": "85899345920", + "cudaCores": 6912, + "architecture": "Ampere" + }, + { + "name": "NVIDIA A100-SXM4-80GB", + "memoryTotal": "85899345920", + "cudaCores": 6912, + "architecture": "Ampere" + }, + { + "name": "NVIDIA A100-SXM4-80GB", + "memoryTotal": "85899345920", + "cudaCores": 6912, + "architecture": "Ampere" + }, + { + "name": "NVIDIA A100-SXM4-80GB", + "memoryTotal": "85899345920", + "cudaCores": 6912, + "architecture": "Ampere" + }, + { + "name": "NVIDIA A100-SXM4-80GB", + "memoryTotal": "85899345920", + "cudaCores": 6912, + "architecture": "Ampere" + }, + { + "name": "NVIDIA A100-SXM4-80GB", + "memoryTotal": "85899345920", + "cudaCores": 6912, + "architecture": "Ampere" + } + ], + "cudaVersion": "12.2" +} \ No newline at end of file diff --git a/CKPTS/X2I_UltraEdiit_fourmask_LLava_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_filtered_objects_4mask_1280_context/wandb/run-20250426_205319-m68pf6i5/logs/debug-core.log b/CKPTS/X2I_UltraEdiit_fourmask_LLava_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_filtered_objects_4mask_1280_context/wandb/run-20250426_205319-m68pf6i5/logs/debug-core.log new file mode 100644 index 0000000000000000000000000000000000000000..787fcc0b52b5bbb55281fe2ae2ef632951e2beee --- /dev/null +++ b/CKPTS/X2I_UltraEdiit_fourmask_LLava_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_filtered_objects_4mask_1280_context/wandb/run-20250426_205319-m68pf6i5/logs/debug-core.log @@ -0,0 +1,7 @@ +{"time":"2025-04-26T20:53:19.006490062Z","level":"INFO","msg":"main: starting server","port-filename":"/tmp/tmp76itc16p/port-3707059.txt","pid":3707059,"log-level":0,"disable-analytics":false,"shutdown-on-parent-exit":false} +{"time":"2025-04-26T20:53:19.007669247Z","level":"INFO","msg":"Will exit if parent process dies.","ppid":3707059} +{"time":"2025-04-26T20:53:19.007659949Z","level":"INFO","msg":"server is running","addr":{"IP":"127.0.0.1","Port":41263,"Zone":""}} +{"time":"2025-04-26T20:53:19.194400551Z","level":"INFO","msg":"connection: ManageConnectionData: new connection created","id":"127.0.0.1:36638"} +{"time":"2025-04-26T20:53:19.904818111Z","level":"INFO","msg":"handleInformInit: received","streamId":"m68pf6i5","id":"127.0.0.1:36638"} +{"time":"2025-04-26T20:53:20.127052732Z","level":"INFO","msg":"handleInformInit: stream started","streamId":"m68pf6i5","id":"127.0.0.1:36638"} +{"time":"2025-04-26T21:00:08.585187211Z","level":"INFO","msg":"Parent process exited, terminating service process."} diff --git a/CKPTS/X2I_UltraEdiit_fourmask_LLava_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_filtered_objects_4mask_1280_context/wandb/run-20250426_205319-m68pf6i5/logs/debug-internal.log b/CKPTS/X2I_UltraEdiit_fourmask_LLava_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_filtered_objects_4mask_1280_context/wandb/run-20250426_205319-m68pf6i5/logs/debug-internal.log new file mode 100644 index 0000000000000000000000000000000000000000..744c09b85d3f90e14037dfa4ce40b073133f517e --- /dev/null +++ b/CKPTS/X2I_UltraEdiit_fourmask_LLava_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_filtered_objects_4mask_1280_context/wandb/run-20250426_205319-m68pf6i5/logs/debug-internal.log @@ -0,0 +1,7 @@ +{"time":"2025-04-26T20:53:19.905183544Z","level":"INFO","msg":"stream: starting","core version":"0.19.8","symlink path":"checkpoint/X2I_UltraEdiit_fourmask_LLava_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_filtered_objects_4mask_1280_context/wandb/run-20250426_205319-m68pf6i5/logs/debug-core.log"} +{"time":"2025-04-26T20:53:20.12699255Z","level":"INFO","msg":"created new stream","id":"m68pf6i5"} +{"time":"2025-04-26T20:53:20.127046651Z","level":"INFO","msg":"stream: started","id":"m68pf6i5"} +{"time":"2025-04-26T20:53:20.127071653Z","level":"INFO","msg":"writer: Do: started","stream_id":"m68pf6i5"} +{"time":"2025-04-26T20:53:20.127088424Z","level":"INFO","msg":"sender: started","stream_id":"m68pf6i5"} +{"time":"2025-04-26T20:53:20.127118861Z","level":"INFO","msg":"handler: started","stream_id":"m68pf6i5"} +{"time":"2025-04-26T20:53:20.534700416Z","level":"INFO","msg":"Starting system monitor"} diff --git a/CKPTS/X2I_UltraEdiit_fourmask_LLava_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_filtered_objects_4mask_1280_context/wandb/run-20250426_205319-m68pf6i5/logs/debug.log b/CKPTS/X2I_UltraEdiit_fourmask_LLava_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_filtered_objects_4mask_1280_context/wandb/run-20250426_205319-m68pf6i5/logs/debug.log new file mode 100644 index 0000000000000000000000000000000000000000..a9b5b0421e09540f2ce94a2053784e3ed509180a --- /dev/null +++ b/CKPTS/X2I_UltraEdiit_fourmask_LLava_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_filtered_objects_4mask_1280_context/wandb/run-20250426_205319-m68pf6i5/logs/debug.log @@ -0,0 +1,22 @@ +2025-04-26 20:53:19,884 INFO MainThread:3707059 [wandb_setup.py:_flush():67] Current SDK version is 0.19.8 +2025-04-26 20:53:19,884 INFO MainThread:3707059 [wandb_setup.py:_flush():67] Configure stats pid to 3707059 +2025-04-26 20:53:19,884 INFO MainThread:3707059 [wandb_setup.py:_flush():67] Loading settings from /tmp/haozhezhao/.config/wandb/settings +2025-04-26 20:53:19,884 INFO MainThread:3707059 [wandb_setup.py:_flush():67] Loading settings from /tmp/haozhezhao/MLLMG/wandb/settings +2025-04-26 20:53:19,884 INFO MainThread:3707059 [wandb_setup.py:_flush():67] Loading settings from environment variables +2025-04-26 20:53:19,884 INFO MainThread:3707059 [wandb_init.py:setup_run_log_directory():647] Logging user logs to checkpoint/X2I_UltraEdiit_fourmask_LLava_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_filtered_objects_4mask_1280_context/wandb/run-20250426_205319-m68pf6i5/logs/debug.log +2025-04-26 20:53:19,884 INFO MainThread:3707059 [wandb_init.py:setup_run_log_directory():648] Logging internal logs to checkpoint/X2I_UltraEdiit_fourmask_LLava_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_filtered_objects_4mask_1280_context/wandb/run-20250426_205319-m68pf6i5/logs/debug-internal.log +2025-04-26 20:53:19,884 INFO MainThread:3707059 [wandb_init.py:init():761] calling init triggers +2025-04-26 20:53:19,884 INFO MainThread:3707059 [wandb_init.py:init():766] wandb.init called with sweep_config: {} +config: {'data_path': '/tmp/haozhezhao/MLLMG/jsonl_data/multiimage_training_for_llava_X2I_UltraEdiit_fourmask.jsonl', 'cloud_save_path': '/tmp/haozhezhao/MLLMG/checkpoint', 'no_local_save': False, 'vq_model': 'VQ-16', 'vq_ckpt': '/tmp/haozhezhao/model/llamagen_t2i/vq_ds16_t2i.pt', 'codebook_size': 16384, 'codebook_embed_dim': 8, 'gpt_model': 'GPT-XL', 'gpt_ckpt': '/tmp/haozhezhao/MLLMG/checkpoint/Afteri2i_100kckpt_LLAVAT5_empty_Fixed_just_segment_pretrain_stage3_dreambench__recap_400k_t2i_400k_flux_200k_midjourney_150k_recovery_150k_grounding_100fluxseg_50samseg/005-GPT-XL/checkpoints/0078000.pt', 'gpt_type': 't2i', 'vocab_size': 16384, 'cls_token_num': 1280, 'dropout_p': 0.1, 'token_dropout_p': 0.1, 'drop_path': 0.0, 'no_compile': False, 'results_dir': 'checkpoint/X2I_UltraEdiit_fourmask_LLava_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_filtered_objects_4mask_1280_context', 'dataset': 'ti2i', 'image_size': 512, 'downsample_size': 16, 'num_classes': 1000, 'epochs': 1, 'lr': 0.0001, 'weight_decay': 0.05, 'beta1': 0.9, 'beta2': 0.95, 'max_grad_norm': 1.0, 'global_batch_size': 24, 'global_seed': 0, 'num_workers': 4, 'log_every': 25, 'ckpt_every': 2000, 'gradient_accumulation_steps': 8, 'mixed_precision': 'bf16', 'val_data_path': '/tmp/haozhezhao/MLLMG/jsonl_data/multiimage_val_for_llava_X2I_UltraEdiit_fourmask_dreambenplus.jsonl', 'use_vision_tower': True, 'model_name_or_path': '/tmp/haozhezhao/model/blip2-flan-t5-xl', 'image_place_holder': '', 'processor_path': None, 'do_eval': True, 'max_eval_samples': 128, 'train_text_encoder': True, 'no_left_padding': False, 'cfg_scale': 7.5, 'top_k': 16384, 'temperature': 0.9, 'top_p': 1.0, 'eval_steps': 2000, 'project_name': 'llamagen_ti2i', 'load_from_checkpoint': '/tmp/haozhezhao/MLLMG/checkpoint/FIXed_3M_1epoch_step32000.pt', 'warmup': 0.05, 'lr_decay_style': 'cosine', 'lr_decay_ratio': 0.1, 'train_iters': 500000, 'class_dropout_prob': 0.1, 'with_image_only': False, 'image_only_rate': 0.1, 'stage2': False, 'subject_driven': True, 'load_subject_embedding': None, 'reference_data_path': '/tmp/haozhezhao/MLLMG/cc12m_reference_tunnel.jsonl', 'multimodal_encoder': 'llava', 'do_recovery': True, 'no_replace': False, 'resume': False, 'dreambench_eval': True, 'find_unused_parameters': True, 'load_visual_encoder': False, 'continue_stage1': False, 'replace_subject': False, 'train_all': True, 'save_total_limit': 1, 'load_language_projection': '/tmp/haozhezhao/MLLMG/llava-v1.5-flant5_fixed-pretrain/mm_projector.bin', 'mm_vision_tower': 'openai/clip-vit-large-patch14', 'load_fixed_llamagen': True, 'unfreeze_output': False, 'fix': 'gpt-empty-fix', 'rank': 0, 'world_size': 8, 'gpu': 0, 'dist_url': 'env://', 'distributed': True, 'dist_backend': 'nccl', '_wandb': {}} +2025-04-26 20:53:19,884 INFO MainThread:3707059 [wandb_init.py:init():784] starting backend +2025-04-26 20:53:19,884 INFO MainThread:3707059 [wandb_init.py:init():788] sending inform_init request +2025-04-26 20:53:19,899 INFO MainThread:3707059 [backend.py:_multiprocessing_setup():101] multiprocessing start_methods=fork,spawn,forkserver, using: spawn +2025-04-26 20:53:19,899 INFO MainThread:3707059 [wandb_init.py:init():798] backend started and connected +2025-04-26 20:53:19,901 INFO MainThread:3707059 [wandb_init.py:init():891] updated telemetry +2025-04-26 20:53:19,902 INFO MainThread:3707059 [wandb_init.py:init():915] communicating run to backend with 90.0 second timeout +2025-04-26 20:53:20,532 INFO MainThread:3707059 [wandb_init.py:init():990] starting run threads in backend +2025-04-26 20:53:20,615 INFO MainThread:3707059 [wandb_run.py:_console_start():2375] atexit reg +2025-04-26 20:53:20,615 INFO MainThread:3707059 [wandb_run.py:_redirect():2227] redirect: wrap_raw +2025-04-26 20:53:20,616 INFO MainThread:3707059 [wandb_run.py:_redirect():2292] Wrapping output streams. +2025-04-26 20:53:20,616 INFO MainThread:3707059 [wandb_run.py:_redirect():2315] Redirects installed. +2025-04-26 20:53:20,618 INFO MainThread:3707059 [wandb_init.py:init():1032] run started, returning control to user process diff --git a/CKPTS/X2I_UltraEdiit_fourmask_LLava_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_filtered_objects_4mask_1280_context/wandb/run-20250426_205319-m68pf6i5/run-m68pf6i5.wandb b/CKPTS/X2I_UltraEdiit_fourmask_LLava_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_filtered_objects_4mask_1280_context/wandb/run-20250426_205319-m68pf6i5/run-m68pf6i5.wandb new file mode 100644 index 0000000000000000000000000000000000000000..77767e92ca4db1e5866ba368c98c638b4c6db279 --- /dev/null +++ b/CKPTS/X2I_UltraEdiit_fourmask_LLava_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_filtered_objects_4mask_1280_context/wandb/run-20250426_205319-m68pf6i5/run-m68pf6i5.wandb @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:ccd704742c2034fb9014c5ef1f09a7d776d26868c3ac7b9fa7341b8f4e5e6ab9 +size 131072 diff --git a/CKPTS/X2I_UltraEdiit_fourmask_LLava_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_filtered_objects_4mask_1280_context/wandb/run-20250426_210323-vtrxdt92/files/config.yaml b/CKPTS/X2I_UltraEdiit_fourmask_LLava_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_filtered_objects_4mask_1280_context/wandb/run-20250426_210323-vtrxdt92/files/config.yaml new file mode 100644 index 0000000000000000000000000000000000000000..de510c69f4dd48cba041501aafce4ec60ae22b8b --- /dev/null +++ b/CKPTS/X2I_UltraEdiit_fourmask_LLava_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_filtered_objects_4mask_1280_context/wandb/run-20250426_210323-vtrxdt92/files/config.yaml @@ -0,0 +1,199 @@ +_wandb: + value: + cli_version: 0.19.8 + m: [] + python_version: 3.11.11 + t: + "1": + - 1 + - 11 + - 41 + - 49 + - 51 + - 55 + "2": + - 1 + - 11 + - 41 + - 49 + - 51 + - 55 + "3": + - 13 + - 16 + - 23 + - 55 + - 61 + "4": 3.11.11 + "5": 0.19.8 + "6": 4.49.0 + "8": + - 5 + "12": 0.19.8 + "13": linux-x86_64 +beta1: + value: 0.9 +beta2: + value: 0.95 +cfg_scale: + value: 7.5 +ckpt_every: + value: 2000 +class_dropout_prob: + value: 0.1 +cloud_save_path: + value: /tmp/haozhezhao/MLLMG/checkpoint +cls_token_num: + value: 1280 +codebook_embed_dim: + value: 8 +codebook_size: + value: 16384 +continue_stage1: + value: false +data_path: + value: /tmp/haozhezhao/MLLMG/jsonl_data/multiimage_training_for_llava_X2I_UltraEdiit_fourmask.jsonl +dataset: + value: ti2i +dist_backend: + value: nccl +dist_url: + value: env:// +distributed: + value: true +do_eval: + value: true +do_recovery: + value: true +downsample_size: + value: 16 +dreambench_eval: + value: true +drop_path: + value: 0 +dropout_p: + value: 0.1 +epochs: + value: 1 +eval_steps: + value: 2000 +find_unused_parameters: + value: true +fix: + value: gpt-empty-fix +global_batch_size: + value: 24 +global_seed: + value: 0 +gpt_ckpt: + value: /tmp/haozhezhao/MLLMG/checkpoint/Afteri2i_100kckpt_LLAVAT5_empty_Fixed_just_segment_pretrain_stage3_dreambench__recap_400k_t2i_400k_flux_200k_midjourney_150k_recovery_150k_grounding_100fluxseg_50samseg/005-GPT-XL/checkpoints/0078000.pt +gpt_model: + value: GPT-XL +gpt_type: + value: t2i +gpu: + value: 0 +gradient_accumulation_steps: + value: 8 +image_only_rate: + value: 0.1 +image_place_holder: + value: +image_size: + value: 512 +load_fixed_llamagen: + value: true +load_from_checkpoint: + value: /tmp/haozhezhao/MLLMG/checkpoint/FIXed_3M_1epoch_step32000.pt +load_language_projection: + value: /tmp/haozhezhao/MLLMG/llava-v1.5-flant5_fixed-pretrain/mm_projector.bin +load_subject_embedding: + value: null +load_visual_encoder: + value: false +log_every: + value: 25 +lr: + value: 0.0001 +lr_decay_ratio: + value: 0.1 +lr_decay_style: + value: cosine +max_eval_samples: + value: 128 +max_grad_norm: + value: 1 +mixed_precision: + value: bf16 +mm_vision_tower: + value: openai/clip-vit-large-patch14 +model_name_or_path: + value: /tmp/haozhezhao/model/blip2-flan-t5-xl +multimodal_encoder: + value: llava +no_compile: + value: false +no_left_padding: + value: false +no_local_save: + value: false +no_replace: + value: false +num_classes: + value: 1000 +num_workers: + value: 4 +processor_path: + value: null +project_name: + value: llamagen_ti2i +rank: + value: 0 +reference_data_path: + value: /tmp/haozhezhao/MLLMG/cc12m_reference_tunnel.jsonl +replace_subject: + value: false +results_dir: + value: checkpoint/X2I_UltraEdiit_fourmask_LLava_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_filtered_objects_4mask_1280_context +resume: + value: false +save_total_limit: + value: 1 +stage2: + value: false +subject_driven: + value: true +temperature: + value: 0.9 +token_dropout_p: + value: 0.1 +top_k: + value: 16384 +top_p: + value: 1 +train_all: + value: true +train_iters: + value: 500000 +train_text_encoder: + value: true +unfreeze_output: + value: false +use_vision_tower: + value: true +val_data_path: + value: /tmp/haozhezhao/MLLMG/jsonl_data/multiimage_val_for_llava_X2I_UltraEdiit_fourmask_dreambenplus.jsonl +vocab_size: + value: 16384 +vq_ckpt: + value: /tmp/haozhezhao/model/llamagen_t2i/vq_ds16_t2i.pt +vq_model: + value: VQ-16 +warmup: + value: 0.05 +weight_decay: + value: 0.05 +with_image_only: + value: false +world_size: + value: 8 diff --git a/CKPTS/X2I_UltraEdiit_fourmask_LLava_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_filtered_objects_4mask_1280_context/wandb/run-20250426_210323-vtrxdt92/files/output.log b/CKPTS/X2I_UltraEdiit_fourmask_LLava_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_filtered_objects_4mask_1280_context/wandb/run-20250426_210323-vtrxdt92/files/output.log new file mode 100644 index 0000000000000000000000000000000000000000..8f125b8be2278323649c7a8d3a00d1e09247f27f --- /dev/null +++ b/CKPTS/X2I_UltraEdiit_fourmask_LLava_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_filtered_objects_4mask_1280_context/wandb/run-20250426_210323-vtrxdt92/files/output.log @@ -0,0 +1,5280 @@ +[2025-04-26 21:03:24] Training for 1 epochs... +[2025-04-26 21:03:24] Beginning epoch 0... + 0%| | 0/110696 [00:00 +tokenizer length after expend 32101 +tokenizer length before expend 32100 + /tmp/haozhezhao/anaconda3/envs/nlp/lib/python3.11/contextlib.py:105: FutureWarning: `torch.backends.cuda.sdp_kernel()` is deprecated. In the future, this context manager will be removed. Please see `torch.nn.attention.sdpa_kernel()` for the new context manager, with updated signature. + self.gen = func(*args, **kwds) | 0/5 [00:00 +tokenizer length after expend 32101 +tokenizer length before expend 32100 +100%|█████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████| 5/5 [04:57<00:00, 59.44s/it] +[2025-04-26 22:21:02] Finish Eval in 4000 steps...███████████████████████████████████████████████████████████████████████████████████████| 5/5 [04:56<00:00, 59.24s/it] +processing left-padding... +processing left-padding... +processing left-padding... +processing left-padding... +processing left-padding... +[2025-04-26 22:21:21] Saved checkpoint to checkpoint/X2I_UltraEdiit_fourmask_LLava_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_filtered_objects_4mask_1280_context/003-GPT-XL/checkpoints/0004000.pt +[2025-04-26 22:21:23] Removed old checkpoint: checkpoint/X2I_UltraEdiit_fourmask_LLava_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_filtered_objects_4mask_1280_context/003-GPT-XL/checkpoints/0002000.pt + 4%|████▋ | 4024/110696 [1:18:20<27:45:52, 1.07it/s][2025-04-26 22:21:46] (step=0004025) Train Loss: 6.2964, Train Steps/Sec: 0.06 + 4%|████▋ | 4049/110696 [1:18:43<26:45:46, 1.11it/s][2025-04-26 22:22:08] (step=0004050) Train Loss: 6.3278, Train Steps/Sec: 1.12 + 4%|████▋ | 4074/110696 [1:19:05<26:30:28, 1.12it/s][2025-04-26 22:22:30] (step=0004075) Train Loss: 6.3072, Train Steps/Sec: 1.12 + 4%|████▋ | 4099/110696 [1:19:27<26:33:09, 1.12it/s][2025-04-26 22:22:53] (step=0004100) Train Loss: 6.2894, Train Steps/Sec: 1.12 + 4%|████▊ | 4124/110696 [1:19:50<26:22:43, 1.12it/s][2025-04-26 22:23:15] (step=0004125) Train Loss: 6.3272, Train Steps/Sec: 1.12 + 4%|████▊ | 4149/110696 [1:20:12<26:13:13, 1.13it/s][2025-04-26 22:23:37] (step=0004150) Train Loss: 6.2906, Train Steps/Sec: 1.12 + 4%|████▊ | 4174/110696 [1:20:34<26:11:57, 1.13it/s][2025-04-26 22:24:00] (step=0004175) Train Loss: 6.3075, Train Steps/Sec: 1.12 + 4%|████▊ | 4199/110696 [1:20:57<26:17:12, 1.13it/s][2025-04-26 22:24:22] (step=0004200) Train Loss: 6.2586, Train Steps/Sec: 1.11 + 4%|████▉ | 4224/110696 [1:21:19<27:02:02, 1.09it/s][2025-04-26 22:24:44] (step=0004225) Train Loss: 6.3069, Train Steps/Sec: 1.12 + 4%|████▉ | 4249/110696 [1:21:41<26:42:23, 1.11it/s][2025-04-26 22:25:07] (step=0004250) Train Loss: 6.3023, Train Steps/Sec: 1.12 + 4%|████▉ | 4274/110696 [1:22:04<26:28:28, 1.12it/s][2025-04-26 22:25:29] (step=0004275) Train Loss: 6.3411, Train Steps/Sec: 1.12 + 4%|████▉ | 4299/110696 [1:22:26<26:28:12, 1.12it/s][2025-04-26 22:25:52] (step=0004300) Train Loss: 6.3057, Train Steps/Sec: 1.12 + 4%|████▉ | 4324/110696 [1:22:49<26:13:46, 1.13it/s][2025-04-26 22:26:14] (step=0004325) Train Loss: 6.2551, Train Steps/Sec: 1.12 + 4%|█████ | 4349/110696 [1:23:11<26:16:59, 1.12it/s][2025-04-26 22:26:36] (step=0004350) Train Loss: 6.2859, Train Steps/Sec: 1.12 + 4%|█████ | 4374/110696 [1:23:33<26:13:54, 1.13it/s][2025-04-26 22:26:59] (step=0004375) Train Loss: 6.2846, Train Steps/Sec: 1.12 + 4%|█████ | 4399/110696 [1:23:56<26:05:02, 1.13it/s][2025-04-26 22:27:21] (step=0004400) Train Loss: 6.3353, Train Steps/Sec: 1.12 + 4%|█████ | 4424/110696 [1:24:18<26:57:26, 1.10it/s][2025-04-26 22:27:43] (step=0004425) Train Loss: 6.3273, Train Steps/Sec: 1.12 + 4%|█████▏ | 4449/110696 [1:24:40<26:44:09, 1.10it/s][2025-04-26 22:28:06] (step=0004450) Train Loss: 6.3061, Train Steps/Sec: 1.12 + 4%|█████▏ | 4474/110696 [1:25:03<26:26:07, 1.12it/s][2025-04-26 22:28:28] (step=0004475) Train Loss: 6.2885, Train Steps/Sec: 1.12 + 4%|█████▏ | 4499/110696 [1:25:25<26:21:01, 1.12it/s][2025-04-26 22:28:51] (step=0004500) Train Loss: 6.2773, Train Steps/Sec: 1.12 + 4%|█████▏ | 4524/110696 [1:25:47<26:15:15, 1.12it/s][2025-04-26 22:29:13] (step=0004525) Train Loss: 6.2515, Train Steps/Sec: 1.12 + 4%|█████▎ | 4549/110696 [1:26:10<26:09:29, 1.13it/s][2025-04-26 22:29:35] (step=0004550) Train Loss: 6.2671, Train Steps/Sec: 1.12 + 4%|█████▎ | 4574/110696 [1:26:32<26:26:48, 1.11it/s][2025-04-26 22:29:58] (step=0004575) Train Loss: 6.3157, Train Steps/Sec: 1.11 + 4%|█████▎ | 4599/110696 [1:26:55<26:05:12, 1.13it/s][2025-04-26 22:30:20] (step=0004600) Train Loss: 6.3046, Train Steps/Sec: 1.11 + 4%|█████▎ | 4624/110696 [1:27:17<26:57:46, 1.09it/s][2025-04-26 22:30:42] (step=0004625) Train Loss: 6.2857, Train Steps/Sec: 1.12 + 4%|█████▍ | 4649/110696 [1:27:39<26:33:08, 1.11it/s][2025-04-26 22:31:05] (step=0004650) Train Loss: 6.3340, Train Steps/Sec: 1.12 + 4%|█████▍ | 4674/110696 [1:28:02<26:24:56, 1.11it/s][2025-04-26 22:31:27] (step=0004675) Train Loss: 6.2850, Train Steps/Sec: 1.12 + 4%|█████▍ | 4699/110696 [1:28:24<26:15:42, 1.12it/s][2025-04-26 22:31:49] (step=0004700) Train Loss: 6.2601, Train Steps/Sec: 1.12 + 4%|█████▍ | 4724/110696 [1:28:46<26:09:07, 1.13it/s][2025-04-26 22:32:12] (step=0004725) Train Loss: 6.2891, Train Steps/Sec: 1.12 + 4%|█████▍ | 4749/110696 [1:29:09<26:09:00, 1.13it/s][2025-04-26 22:32:34] (step=0004750) Train Loss: 6.3079, Train Steps/Sec: 1.12 + 4%|█████▌ | 4774/110696 [1:29:31<26:04:37, 1.13it/s][2025-04-26 22:32:56] (step=0004775) Train Loss: 6.3800, Train Steps/Sec: 1.12 + 4%|█████▌ | 4799/110696 [1:29:53<26:02:55, 1.13it/s][2025-04-26 22:33:19] (step=0004800) Train Loss: 6.2954, Train Steps/Sec: 1.11 + 4%|█████▌ | 4824/110696 [1:30:16<26:45:23, 1.10it/s][2025-04-26 22:33:41] (step=0004825) Train Loss: 6.2441, Train Steps/Sec: 1.12 + 4%|█████▌ | 4849/110696 [1:30:38<26:30:21, 1.11it/s][2025-04-26 22:34:04] (step=0004850) Train Loss: 6.3170, Train Steps/Sec: 1.12 + 4%|█████▋ | 4874/110696 [1:31:01<26:26:48, 1.11it/s][2025-04-26 22:34:26] (step=0004875) Train Loss: 6.3163, Train Steps/Sec: 1.12 + 4%|█████▋ | 4899/110696 [1:31:23<26:14:52, 1.12it/s][2025-04-26 22:34:48] (step=0004900) Train Loss: 6.3055, Train Steps/Sec: 1.12 + 4%|█████▋ | 4924/110696 [1:31:45<26:10:06, 1.12it/s][2025-04-26 22:35:11] (step=0004925) Train Loss: 6.3080, Train Steps/Sec: 1.12 + 4%|█████▋ | 4949/110696 [1:32:08<26:07:00, 1.12it/s][2025-04-26 22:35:33] (step=0004950) Train Loss: 6.2524, Train Steps/Sec: 1.12 + 4%|█████▊ | 4974/110696 [1:32:30<25:59:17, 1.13it/s][2025-04-26 22:35:55] (step=0004975) Train Loss: 6.3135, Train Steps/Sec: 1.12 + 5%|█████▊ | 4999/110696 [1:32:52<26:05:16, 1.13it/s][2025-04-26 22:36:18] (step=0005000) Train Loss: 6.2795, Train Steps/Sec: 1.11 + 5%|█████▊ | 5024/110696 [1:33:15<26:42:51, 1.10it/s][2025-04-26 22:36:40] (step=0005025) Train Loss: 6.3269, Train Steps/Sec: 1.12 + 5%|█████▊ | 5049/110696 [1:33:37<26:27:03, 1.11it/s][2025-04-26 22:37:03] (step=0005050) Train Loss: 6.2596, Train Steps/Sec: 1.12 + 5%|█████▊ | 5074/110696 [1:34:00<26:17:07, 1.12it/s][2025-04-26 22:37:25] (step=0005075) Train Loss: 6.2862, Train Steps/Sec: 1.12 + 5%|█████▉ | 5099/110696 [1:34:22<26:13:11, 1.12it/s][2025-04-26 22:37:47] (step=0005100) Train Loss: 6.3035, Train Steps/Sec: 1.12 + 5%|█████▉ | 5124/110696 [1:34:44<26:05:13, 1.12it/s][2025-04-26 22:38:10] (step=0005125) Train Loss: 6.2684, Train Steps/Sec: 1.12 + 5%|█████▉ | 5149/110696 [1:35:07<25:58:50, 1.13it/s][2025-04-26 22:38:32] (step=0005150) Train Loss: 6.2551, Train Steps/Sec: 1.12 + 5%|█████▉ | 5174/110696 [1:35:29<25:58:58, 1.13it/s][2025-04-26 22:38:54] (step=0005175) Train Loss: 6.3131, Train Steps/Sec: 1.12 + 5%|██████ | 5199/110696 [1:35:51<25:57:46, 1.13it/s][2025-04-26 22:39:17] (step=0005200) Train Loss: 6.2411, Train Steps/Sec: 1.11 + 5%|██████ | 5224/110696 [1:36:14<26:47:24, 1.09it/s][2025-04-26 22:39:39] (step=0005225) Train Loss: 6.2727, Train Steps/Sec: 1.12 + 5%|██████ | 5249/110696 [1:36:36<26:22:01, 1.11it/s][2025-04-26 22:40:02] (step=0005250) Train Loss: 6.2894, Train Steps/Sec: 1.12 + 5%|██████ | 5274/110696 [1:36:59<26:14:24, 1.12it/s][2025-04-26 22:40:24] (step=0005275) Train Loss: 6.3140, Train Steps/Sec: 1.12 + 5%|██████▏ | 5299/110696 [1:37:21<26:05:52, 1.12it/s][2025-04-26 22:40:46] (step=0005300) Train Loss: 6.2501, Train Steps/Sec: 1.12 + 5%|██████▏ | 5324/110696 [1:37:43<26:12:21, 1.12it/s][2025-04-26 22:41:09] (step=0005325) Train Loss: 6.2478, Train Steps/Sec: 1.11 + 5%|██████▏ | 5349/110696 [1:38:06<26:03:21, 1.12it/s][2025-04-26 22:41:31] (step=0005350) Train Loss: 6.2941, Train Steps/Sec: 1.11 + 5%|██████▏ | 5374/110696 [1:38:28<25:59:21, 1.13it/s][2025-04-26 22:41:53] (step=0005375) Train Loss: 6.3197, Train Steps/Sec: 1.12 + 5%|██████▏ | 5399/110696 [1:38:50<25:58:41, 1.13it/s][2025-04-26 22:42:16] (step=0005400) Train Loss: 6.2895, Train Steps/Sec: 1.11 + 5%|██████▎ | 5424/110696 [1:39:13<26:45:32, 1.09it/s][2025-04-26 22:42:38] (step=0005425) Train Loss: 6.3199, Train Steps/Sec: 1.12 + 5%|██████▎ | 5449/110696 [1:39:35<26:19:39, 1.11it/s][2025-04-26 22:43:01] (step=0005450) Train Loss: 6.3395, Train Steps/Sec: 1.12 + 5%|██████▎ | 5474/110696 [1:39:58<26:14:31, 1.11it/s][2025-04-26 22:43:23] (step=0005475) Train Loss: 6.2418, Train Steps/Sec: 1.12 + 5%|██████▎ | 5499/110696 [1:40:20<26:09:07, 1.12it/s][2025-04-26 22:43:45] (step=0005500) Train Loss: 6.2254, Train Steps/Sec: 1.12 + 5%|██████▍ | 5524/110696 [1:40:42<25:55:53, 1.13it/s][2025-04-26 22:44:08] (step=0005525) Train Loss: 6.2337, Train Steps/Sec: 1.12 + 5%|██████▍ | 5549/110696 [1:41:05<26:00:53, 1.12it/s][2025-04-26 22:44:30] (step=0005550) Train Loss: 6.3453, Train Steps/Sec: 1.12 + 5%|██████▍ | 5574/110696 [1:41:27<26:00:03, 1.12it/s][2025-04-26 22:44:52] (step=0005575) Train Loss: 6.2590, Train Steps/Sec: 1.12 + 5%|██████▍ | 5599/110696 [1:41:49<25:52:16, 1.13it/s][2025-04-26 22:45:15] (step=0005600) Train Loss: 6.2499, Train Steps/Sec: 1.11 + 5%|██████▌ | 5624/110696 [1:42:12<26:39:49, 1.09it/s][2025-04-26 22:45:37] (step=0005625) Train Loss: 6.2323, Train Steps/Sec: 1.12 + 5%|██████▌ | 5649/110696 [1:42:34<26:20:28, 1.11it/s][2025-04-26 22:46:00] (step=0005650) Train Loss: 6.3066, Train Steps/Sec: 1.12 + 5%|██████▌ | 5674/110696 [1:42:57<26:06:34, 1.12it/s][2025-04-26 22:46:22] (step=0005675) Train Loss: 6.2206, Train Steps/Sec: 1.12 + 5%|██████▌ | 5699/110696 [1:43:19<26:03:05, 1.12it/s][2025-04-26 22:46:44] (step=0005700) Train Loss: 6.2536, Train Steps/Sec: 1.11 + 5%|██████▌ | 5724/110696 [1:43:41<25:57:27, 1.12it/s][2025-04-26 22:47:07] (step=0005725) Train Loss: 6.2365, Train Steps/Sec: 1.12 + 5%|██████▋ | 5749/110696 [1:44:04<25:55:29, 1.12it/s][2025-04-26 22:47:29] (step=0005750) Train Loss: 6.2544, Train Steps/Sec: 1.12 + 5%|██████▋ | 5774/110696 [1:44:26<25:50:59, 1.13it/s][2025-04-26 22:47:51] (step=0005775) Train Loss: 6.2794, Train Steps/Sec: 1.12 + 5%|██████▋ | 5799/110696 [1:44:53<40:53:29, 1.40s/it][2025-04-26 22:48:19] (step=0005800) Train Loss: 6.2641, Train Steps/Sec: 0.91 + 5%|██████▋ | 5824/110696 [1:45:16<26:35:26, 1.10it/s][2025-04-26 22:48:41] (step=0005825) Train Loss: 6.2891, Train Steps/Sec: 1.12 + 5%|██████▊ | 5849/110696 [1:45:38<26:12:07, 1.11it/s][2025-04-26 22:49:04] (step=0005850) Train Loss: 6.2363, Train Steps/Sec: 1.12 + 5%|██████▊ | 5874/110696 [1:46:01<26:08:12, 1.11it/s][2025-04-26 22:49:26] (step=0005875) Train Loss: 6.2938, Train Steps/Sec: 1.12 + 5%|██████▊ | 5899/110696 [1:46:23<26:05:24, 1.12it/s][2025-04-26 22:49:48] (step=0005900) Train Loss: 6.2996, Train Steps/Sec: 1.12 + 5%|██████▊ | 5924/110696 [1:46:45<25:58:09, 1.12it/s][2025-04-26 22:50:11] (step=0005925) Train Loss: 6.2419, Train Steps/Sec: 1.12 + 5%|██████▉ | 5949/110696 [1:47:08<25:55:51, 1.12it/s][2025-04-26 22:50:33] (step=0005950) Train Loss: 6.2123, Train Steps/Sec: 1.12 + 5%|██████▉ | 5974/110696 [1:47:30<25:44:41, 1.13it/s][2025-04-26 22:50:55] (step=0005975) Train Loss: 6.1992, Train Steps/Sec: 1.12 + 5%|██████▉ | 5999/110696 [1:47:52<25:43:10, 1.13it/s][2025-04-26 22:51:18] (step=0006000) Train Loss: 6.2881, Train Steps/Sec: 1.11 +You are using a model of type blip-2 to instantiate a model of type instructblip. This is not supported for all configurations of models and can yield errors. +[2025-04-26 22:51:18] text_config is None. Initializing the text config with default values (`OPTConfig`). +model_name_or_path:/tmp/haozhezhao/model/blip2-flan-t5-xl +image_place_holder: +tokenizer length after expend 32101 +tokenizer length before expend 32100 +100%|█████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████| 5/5 [04:56<00:00, 59.27s/it] +[2025-04-26 22:57:26] Finish Eval in 6000 steps...███████████████████████████████████████████████████████████████████████████████████████| 5/5 [04:55<00:00, 59.11s/it] +processing left-padding... +processing left-padding... +processing left-padding... +processing left-padding... +processing left-padding... +[2025-04-26 22:57:45] Saved checkpoint to checkpoint/X2I_UltraEdiit_fourmask_LLava_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_filtered_objects_4mask_1280_context/003-GPT-XL/checkpoints/0006000.pt +[2025-04-26 22:57:47] Removed old checkpoint: checkpoint/X2I_UltraEdiit_fourmask_LLava_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_filtered_objects_4mask_1280_context/003-GPT-XL/checkpoints/0004000.pt + 5%|██████▉ | 6024/110696 [1:54:44<27:17:38, 1.07it/s][2025-04-26 22:58:10] (step=0006025) Train Loss: 6.2863, Train Steps/Sec: 0.06 + 5%|██████▉ | 6049/110696 [1:55:07<26:12:44, 1.11it/s][2025-04-26 22:58:32] (step=0006050) Train Loss: 6.2622, Train Steps/Sec: 1.12 + 5%|███████ | 6074/110696 [1:55:29<26:06:18, 1.11it/s][2025-04-26 22:58:55] (step=0006075) Train Loss: 6.2294, Train Steps/Sec: 1.12 + 6%|███████ | 6099/110696 [1:55:52<25:51:36, 1.12it/s][2025-04-26 22:59:17] (step=0006100) Train Loss: 6.2250, Train Steps/Sec: 1.12 + 6%|███████ | 6124/110696 [1:56:14<25:54:30, 1.12it/s][2025-04-26 22:59:39] (step=0006125) Train Loss: 6.2449, Train Steps/Sec: 1.12 + 6%|███████ | 6149/110696 [1:56:36<25:47:28, 1.13it/s][2025-04-26 23:00:02] (step=0006150) Train Loss: 6.2267, Train Steps/Sec: 1.12 + 6%|███████▏ | 6174/110696 [1:56:59<25:44:23, 1.13it/s][2025-04-26 23:00:24] (step=0006175) Train Loss: 6.1763, Train Steps/Sec: 1.12 + 6%|███████▏ | 6199/110696 [1:57:21<25:43:39, 1.13it/s][2025-04-26 23:00:47] (step=0006200) Train Loss: 6.2821, Train Steps/Sec: 1.11 + 6%|███████▏ | 6224/110696 [1:57:44<26:31:42, 1.09it/s][2025-04-26 23:01:09] (step=0006225) Train Loss: 6.2271, Train Steps/Sec: 1.12 + 6%|███████▏ | 6249/110696 [1:58:06<26:13:55, 1.11it/s][2025-04-26 23:01:31] (step=0006250) Train Loss: 6.2157, Train Steps/Sec: 1.12 + 6%|███████▎ | 6274/110696 [1:58:28<25:57:48, 1.12it/s][2025-04-26 23:01:54] (step=0006275) Train Loss: 6.2284, Train Steps/Sec: 1.12 + 6%|███████▎ | 6299/110696 [1:58:51<25:56:20, 1.12it/s][2025-04-26 23:02:16] (step=0006300) Train Loss: 6.2214, Train Steps/Sec: 1.12 + 6%|███████▎ | 6324/110696 [1:59:13<25:56:00, 1.12it/s][2025-04-26 23:02:39] (step=0006325) Train Loss: 6.2760, Train Steps/Sec: 1.11 + 6%|███████▎ | 6349/110696 [1:59:36<25:45:31, 1.13it/s][2025-04-26 23:03:01] (step=0006350) Train Loss: 6.2449, Train Steps/Sec: 1.12 + 6%|███████▎ | 6374/110696 [1:59:58<25:52:29, 1.12it/s][2025-04-26 23:03:23] (step=0006375) Train Loss: 6.2642, Train Steps/Sec: 1.11 + 6%|███████▍ | 6399/110696 [2:00:21<25:47:37, 1.12it/s][2025-04-26 23:03:46] (step=0006400) Train Loss: 6.2256, Train Steps/Sec: 1.10 + 6%|███████▍ | 6424/110696 [2:00:43<26:42:31, 1.08it/s][2025-04-26 23:04:09] (step=0006425) Train Loss: 6.2250, Train Steps/Sec: 1.11 + 6%|███████▍ | 6449/110696 [2:01:06<26:21:24, 1.10it/s][2025-04-26 23:04:31] (step=0006450) Train Loss: 6.2806, Train Steps/Sec: 1.11 + 6%|███████▍ | 6474/110696 [2:01:28<26:05:57, 1.11it/s][2025-04-26 23:04:54] (step=0006475) Train Loss: 6.2353, Train Steps/Sec: 1.11 + 6%|███████▌ | 6499/110696 [2:01:51<25:57:21, 1.12it/s][2025-04-26 23:05:16] (step=0006500) Train Loss: 6.2234, Train Steps/Sec: 1.11 + 6%|███████▌ | 6524/110696 [2:02:13<25:48:28, 1.12it/s][2025-04-26 23:05:39] (step=0006525) Train Loss: 6.2629, Train Steps/Sec: 1.11 + 6%|███████▌ | 6549/110696 [2:02:36<25:41:32, 1.13it/s][2025-04-26 23:06:01] (step=0006550) Train Loss: 6.2606, Train Steps/Sec: 1.12 + 6%|███████▌ | 6574/110696 [2:02:58<25:42:24, 1.13it/s][2025-04-26 23:06:24] (step=0006575) Train Loss: 6.2083, Train Steps/Sec: 1.12 + 6%|███████▋ | 6599/110696 [2:03:21<25:35:42, 1.13it/s][2025-04-26 23:06:46] (step=0006600) Train Loss: 6.2291, Train Steps/Sec: 1.11 + 6%|███████▋ | 6624/110696 [2:03:43<26:26:35, 1.09it/s][2025-04-26 23:07:08] (step=0006625) Train Loss: 6.1996, Train Steps/Sec: 1.12 + 6%|███████▋ | 6649/110696 [2:04:05<26:10:14, 1.10it/s][2025-04-26 23:07:31] (step=0006650) Train Loss: 6.2627, Train Steps/Sec: 1.12 + 6%|███████▋ | 6674/110696 [2:04:28<25:56:29, 1.11it/s][2025-04-26 23:07:53] (step=0006675) Train Loss: 6.1968, Train Steps/Sec: 1.12 + 6%|███████▋ | 6699/110696 [2:04:50<25:49:14, 1.12it/s][2025-04-26 23:08:16] (step=0006700) Train Loss: 6.2226, Train Steps/Sec: 1.12 + 6%|███████▊ | 6724/110696 [2:05:28<37:13:57, 1.29s/it][2025-04-26 23:08:54] (step=0006725) Train Loss: 6.2141, Train Steps/Sec: 0.66 + 6%|███████▊ | 6749/110696 [2:05:51<25:53:03, 1.12it/s][2025-04-26 23:09:16] (step=0006750) Train Loss: 6.2369, Train Steps/Sec: 1.12 + 6%|███████▊ | 6774/110696 [2:06:13<25:37:46, 1.13it/s][2025-04-26 23:09:38] (step=0006775) Train Loss: 6.2367, Train Steps/Sec: 1.12 + 6%|███████▊ | 6799/110696 [2:06:35<25:37:03, 1.13it/s][2025-04-26 23:10:01] (step=0006800) Train Loss: 6.2484, Train Steps/Sec: 1.11 + 6%|███████▉ | 6824/110696 [2:06:58<26:26:33, 1.09it/s][2025-04-26 23:10:23] (step=0006825) Train Loss: 6.3009, Train Steps/Sec: 1.12 + 6%|███████▉ | 6849/110696 [2:07:20<26:09:05, 1.10it/s][2025-04-26 23:10:46] (step=0006850) Train Loss: 6.2284, Train Steps/Sec: 1.12 + 6%|███████▉ | 6874/110696 [2:07:43<25:49:55, 1.12it/s][2025-04-26 23:11:08] (step=0006875) Train Loss: 6.2302, Train Steps/Sec: 1.12 + 6%|███████▉ | 6899/110696 [2:08:05<25:45:48, 1.12it/s][2025-04-26 23:11:30] (step=0006900) Train Loss: 6.3052, Train Steps/Sec: 1.12 + 6%|████████ | 6924/110696 [2:08:33<30:54:06, 1.07s/it][2025-04-26 23:11:58] (step=0006925) Train Loss: 6.2216, Train Steps/Sec: 0.91 + 6%|████████ | 6949/110696 [2:08:55<25:37:33, 1.12it/s][2025-04-26 23:12:20] (step=0006950) Train Loss: 6.1920, Train Steps/Sec: 1.12 + 6%|████████ | 6974/110696 [2:09:23<40:53:26, 1.42s/it][2025-04-26 23:12:48] (step=0006975) Train Loss: 6.2626, Train Steps/Sec: 0.91 + 6%|████████ | 6999/110696 [2:09:45<25:32:47, 1.13it/s][2025-04-26 23:13:10] (step=0007000) Train Loss: 6.2151, Train Steps/Sec: 1.11 + 6%|████████ | 7024/110696 [2:10:07<26:20:12, 1.09it/s][2025-04-26 23:13:33] (step=0007025) Train Loss: 6.2561, Train Steps/Sec: 1.12 + 6%|████████▏ | 7049/110696 [2:10:30<25:52:27, 1.11it/s][2025-04-26 23:13:55] (step=0007050) Train Loss: 6.1687, Train Steps/Sec: 1.12 + 6%|████████▏ | 7074/110696 [2:10:52<26:03:06, 1.10it/s][2025-04-26 23:14:17] (step=0007075) Train Loss: 6.2426, Train Steps/Sec: 1.12 + 6%|████████▏ | 7099/110696 [2:11:15<25:47:23, 1.12it/s][2025-04-26 23:14:40] (step=0007100) Train Loss: 6.3134, Train Steps/Sec: 1.12 + 6%|████████▏ | 7124/110696 [2:11:37<25:41:05, 1.12it/s][2025-04-26 23:15:02] (step=0007125) Train Loss: 6.2441, Train Steps/Sec: 1.12 + 6%|████████▎ | 7149/110696 [2:11:59<25:41:52, 1.12it/s][2025-04-26 23:15:25] (step=0007150) Train Loss: 6.2642, Train Steps/Sec: 1.12 + 6%|████████▎ | 7174/110696 [2:12:22<25:27:26, 1.13it/s][2025-04-26 23:15:47] (step=0007175) Train Loss: 6.2138, Train Steps/Sec: 1.12 + 7%|████████▎ | 7199/110696 [2:12:44<25:28:27, 1.13it/s][2025-04-26 23:16:09] (step=0007200) Train Loss: 6.2657, Train Steps/Sec: 1.11 + 7%|████████▎ | 7224/110696 [2:13:07<26:18:24, 1.09it/s][2025-04-26 23:16:32] (step=0007225) Train Loss: 6.2653, Train Steps/Sec: 1.12 + 7%|████████▍ | 7249/110696 [2:13:29<25:58:55, 1.11it/s][2025-04-26 23:16:54] (step=0007250) Train Loss: 6.2618, Train Steps/Sec: 1.12 + 7%|████████▍ | 7274/110696 [2:13:51<25:46:04, 1.11it/s][2025-04-26 23:17:17] (step=0007275) Train Loss: 6.2298, Train Steps/Sec: 1.12 + 7%|████████▍ | 7299/110696 [2:14:14<25:38:14, 1.12it/s][2025-04-26 23:17:39] (step=0007300) Train Loss: 6.3074, Train Steps/Sec: 1.12 + 7%|████████▍ | 7324/110696 [2:14:36<25:30:02, 1.13it/s][2025-04-26 23:18:01] (step=0007325) Train Loss: 6.2395, Train Steps/Sec: 1.12 + 7%|████████▍ | 7349/110696 [2:14:58<25:29:37, 1.13it/s][2025-04-26 23:18:24] (step=0007350) Train Loss: 6.2542, Train Steps/Sec: 1.12 + 7%|████████▌ | 7374/110696 [2:15:21<25:28:26, 1.13it/s][2025-04-26 23:18:46] (step=0007375) Train Loss: 6.1842, Train Steps/Sec: 1.12 + 7%|████████▌ | 7399/110696 [2:15:43<25:27:04, 1.13it/s][2025-04-26 23:19:09] (step=0007400) Train Loss: 6.2553, Train Steps/Sec: 1.11 + 7%|████████▌ | 7424/110696 [2:16:05<26:08:36, 1.10it/s][2025-04-26 23:19:31] (step=0007425) Train Loss: 6.1726, Train Steps/Sec: 1.12 + 7%|████████▌ | 7449/110696 [2:16:28<25:53:55, 1.11it/s][2025-04-26 23:19:53] (step=0007450) Train Loss: 6.2697, Train Steps/Sec: 1.12 + 7%|████████▋ | 7474/110696 [2:16:50<25:49:49, 1.11it/s][2025-04-26 23:20:16] (step=0007475) Train Loss: 6.1879, Train Steps/Sec: 1.12 + 7%|████████▋ | 7499/110696 [2:17:13<25:34:22, 1.12it/s][2025-04-26 23:20:38] (step=0007500) Train Loss: 6.1982, Train Steps/Sec: 1.12 + 7%|████████▋ | 7524/110696 [2:17:35<25:31:57, 1.12it/s][2025-04-26 23:21:00] (step=0007525) Train Loss: 6.2835, Train Steps/Sec: 1.12 + 7%|████████▋ | 7549/110696 [2:17:57<25:25:17, 1.13it/s][2025-04-26 23:21:23] (step=0007550) Train Loss: 6.2300, Train Steps/Sec: 1.12 + 7%|████████▊ | 7574/110696 [2:18:20<25:20:36, 1.13it/s][2025-04-26 23:21:45] (step=0007575) Train Loss: 6.2375, Train Steps/Sec: 1.12 + 7%|████████▊ | 7599/110696 [2:18:42<25:23:05, 1.13it/s][2025-04-26 23:22:08] (step=0007600) Train Loss: 6.2076, Train Steps/Sec: 1.11 + 7%|████████▊ | 7624/110696 [2:19:05<26:07:40, 1.10it/s][2025-04-26 23:22:30] (step=0007625) Train Loss: 6.1928, Train Steps/Sec: 1.12 + 7%|████████▊ | 7649/110696 [2:19:27<25:49:51, 1.11it/s][2025-04-26 23:22:52] (step=0007650) Train Loss: 6.2257, Train Steps/Sec: 1.12 + 7%|████████▊ | 7674/110696 [2:19:49<25:46:39, 1.11it/s][2025-04-26 23:23:15] (step=0007675) Train Loss: 6.2520, Train Steps/Sec: 1.12 + 7%|████████▉ | 7699/110696 [2:20:12<25:29:56, 1.12it/s][2025-04-26 23:23:37] (step=0007700) Train Loss: 6.1485, Train Steps/Sec: 1.12 + 7%|████████▉ | 7724/110696 [2:20:34<25:36:59, 1.12it/s][2025-04-26 23:23:59] (step=0007725) Train Loss: 6.1636, Train Steps/Sec: 1.12 + 7%|████████▉ | 7749/110696 [2:20:57<25:26:31, 1.12it/s][2025-04-26 23:24:22] (step=0007750) Train Loss: 6.1898, Train Steps/Sec: 1.12 + 7%|████████▉ | 7774/110696 [2:21:19<25:23:08, 1.13it/s][2025-04-26 23:24:44] (step=0007775) Train Loss: 6.1900, Train Steps/Sec: 1.12 + 7%|█████████ | 7799/110696 [2:21:41<25:32:29, 1.12it/s][2025-04-26 23:25:07] (step=0007800) Train Loss: 6.2152, Train Steps/Sec: 1.11 + 7%|█████████ | 7824/110696 [2:22:04<26:08:09, 1.09it/s][2025-04-26 23:25:29] (step=0007825) Train Loss: 6.2039, Train Steps/Sec: 1.12 + 7%|█████████ | 7849/110696 [2:22:26<27:03:33, 1.06it/s][2025-04-26 23:25:52] (step=0007850) Train Loss: 6.2275, Train Steps/Sec: 1.11 + 7%|█████████ | 7874/110696 [2:22:49<25:37:06, 1.11it/s][2025-04-26 23:26:14] (step=0007875) Train Loss: 6.1943, Train Steps/Sec: 1.12 + 7%|█████████▏ | 7899/110696 [2:23:11<25:31:55, 1.12it/s][2025-04-26 23:26:36] (step=0007900) Train Loss: 6.2097, Train Steps/Sec: 1.12 + 7%|█████████▏ | 7924/110696 [2:23:33<25:25:10, 1.12it/s][2025-04-26 23:26:59] (step=0007925) Train Loss: 6.1858, Train Steps/Sec: 1.12 + 7%|█████████▏ | 7949/110696 [2:23:56<25:22:55, 1.12it/s][2025-04-26 23:27:21] (step=0007950) Train Loss: 6.2481, Train Steps/Sec: 1.12 + 7%|█████████▏ | 7974/110696 [2:24:18<25:21:54, 1.12it/s][2025-04-26 23:27:43] (step=0007975) Train Loss: 6.2204, Train Steps/Sec: 1.12 + 7%|█████████▏ | 7999/110696 [2:24:41<25:19:00, 1.13it/s][2025-04-26 23:28:06] (step=0008000) Train Loss: 6.1445, Train Steps/Sec: 1.11 +You are using a model of type blip-2 to instantiate a model of type instructblip. This is not supported for all configurations of models and can yield errors. +[2025-04-26 23:28:06] text_config is None. Initializing the text config with default values (`OPTConfig`). +model_name_or_path:/tmp/haozhezhao/model/blip2-flan-t5-xl +image_place_holder: +tokenizer length after expend 32101 +tokenizer length before expend 32100 +100%|█████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████| 5/5 [04:56<00:00, 59.21s/it] +[2025-04-26 23:34:13] Finish Eval in 8000 steps...███████████████████████████████████████████████████████████████████████████████████████| 5/5 [04:55<00:00, 59.02s/it] +processing left-padding... +processing left-padding... +processing left-padding... +processing left-padding... +processing left-padding... +[2025-04-26 23:34:32] Saved checkpoint to checkpoint/X2I_UltraEdiit_fourmask_LLava_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_filtered_objects_4mask_1280_context/003-GPT-XL/checkpoints/0008000.pt +[2025-04-26 23:34:34] Removed old checkpoint: checkpoint/X2I_UltraEdiit_fourmask_LLava_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_filtered_objects_4mask_1280_context/003-GPT-XL/checkpoints/0006000.pt + 7%|█████████▎ | 8024/110696 [2:31:31<26:39:43, 1.07it/s][2025-04-26 23:34:56] (step=0008025) Train Loss: 6.2186, Train Steps/Sec: 0.06 + 7%|█████████▎ | 8049/110696 [2:31:53<25:41:52, 1.11it/s][2025-04-26 23:35:19] (step=0008050) Train Loss: 6.1772, Train Steps/Sec: 1.12 + 7%|█████████▎ | 8074/110696 [2:32:16<25:30:32, 1.12it/s][2025-04-26 23:35:41] (step=0008075) Train Loss: 6.2167, Train Steps/Sec: 1.12 + 7%|█████████▎ | 8099/110696 [2:32:38<25:21:41, 1.12it/s][2025-04-26 23:36:03] (step=0008100) Train Loss: 6.1606, Train Steps/Sec: 1.12 + 7%|█████████▍ | 8124/110696 [2:33:00<25:19:08, 1.13it/s][2025-04-26 23:36:26] (step=0008125) Train Loss: 6.2719, Train Steps/Sec: 1.12 + 7%|█████████▍ | 8149/110696 [2:33:23<25:20:48, 1.12it/s][2025-04-26 23:36:48] (step=0008150) Train Loss: 6.2460, Train Steps/Sec: 1.12 + 7%|█████████▍ | 8174/110696 [2:33:45<25:15:38, 1.13it/s][2025-04-26 23:37:10] (step=0008175) Train Loss: 6.2599, Train Steps/Sec: 1.12 + 7%|█████████▍ | 8199/110696 [2:34:07<25:14:12, 1.13it/s][2025-04-26 23:37:33] (step=0008200) Train Loss: 6.2424, Train Steps/Sec: 1.11 + 7%|█████████▌ | 8224/110696 [2:34:30<26:00:15, 1.09it/s][2025-04-26 23:37:55] (step=0008225) Train Loss: 6.2016, Train Steps/Sec: 1.12 + 7%|█████████▌ | 8249/110696 [2:34:52<25:36:17, 1.11it/s][2025-04-26 23:38:17] (step=0008250) Train Loss: 6.2074, Train Steps/Sec: 1.12 + 7%|█████████▌ | 8274/110696 [2:35:14<25:29:25, 1.12it/s][2025-04-26 23:38:40] (step=0008275) Train Loss: 6.2573, Train Steps/Sec: 1.12 + 7%|█████████▌ | 8299/110696 [2:35:37<25:24:40, 1.12it/s][2025-04-26 23:39:02] (step=0008300) Train Loss: 6.2007, Train Steps/Sec: 1.12 + 8%|█████████▋ | 8324/110696 [2:35:59<25:10:49, 1.13it/s][2025-04-26 23:39:24] (step=0008325) Train Loss: 6.2107, Train Steps/Sec: 1.12 + 8%|█████████▋ | 8349/110696 [2:36:21<25:09:32, 1.13it/s][2025-04-26 23:39:47] (step=0008350) Train Loss: 6.1790, Train Steps/Sec: 1.12 + 8%|█████████▋ | 8374/110696 [2:36:44<25:18:34, 1.12it/s][2025-04-26 23:40:09] (step=0008375) Train Loss: 6.1734, Train Steps/Sec: 1.12 + 8%|█████████▋ | 8399/110696 [2:37:06<25:04:08, 1.13it/s][2025-04-26 23:40:32] (step=0008400) Train Loss: 6.2087, Train Steps/Sec: 1.12 + 8%|█████████▋ | 8424/110696 [2:37:29<25:56:09, 1.10it/s][2025-04-26 23:40:54] (step=0008425) Train Loss: 6.1879, Train Steps/Sec: 1.12 + 8%|█████████▊ | 8449/110696 [2:37:51<25:34:36, 1.11it/s][2025-04-26 23:41:16] (step=0008450) Train Loss: 6.1751, Train Steps/Sec: 1.12 + 8%|█████████▊ | 8474/110696 [2:38:13<25:24:03, 1.12it/s][2025-04-26 23:41:39] (step=0008475) Train Loss: 6.1864, Train Steps/Sec: 1.12 + 8%|█████████▊ | 8499/110696 [2:38:36<25:25:27, 1.12it/s][2025-04-26 23:42:01] (step=0008500) Train Loss: 6.2424, Train Steps/Sec: 1.12 + 8%|█████████▊ | 8524/110696 [2:38:58<25:16:31, 1.12it/s][2025-04-26 23:42:23] (step=0008525) Train Loss: 6.2782, Train Steps/Sec: 1.12 + 8%|█████████▉ | 8549/110696 [2:39:20<25:08:17, 1.13it/s][2025-04-26 23:42:46] (step=0008550) Train Loss: 6.1880, Train Steps/Sec: 1.12 + 8%|█████████▉ | 8574/110696 [2:39:43<25:10:49, 1.13it/s][2025-04-26 23:43:08] (step=0008575) Train Loss: 6.1688, Train Steps/Sec: 1.12 + 8%|█████████▉ | 8599/110696 [2:40:05<25:08:25, 1.13it/s][2025-04-26 23:43:31] (step=0008600) Train Loss: 6.1768, Train Steps/Sec: 1.11 + 8%|█████████▉ | 8624/110696 [2:40:28<25:53:17, 1.10it/s][2025-04-26 23:43:53] (step=0008625) Train Loss: 6.2435, Train Steps/Sec: 1.12 + 8%|██████████ | 8649/110696 [2:40:50<25:29:36, 1.11it/s][2025-04-26 23:44:15] (step=0008650) Train Loss: 6.1780, Train Steps/Sec: 1.12 + 8%|██████████ | 8674/110696 [2:41:12<25:25:53, 1.11it/s][2025-04-26 23:44:38] (step=0008675) Train Loss: 6.1856, Train Steps/Sec: 1.12 + 8%|██████████ | 8699/110696 [2:41:35<25:16:21, 1.12it/s][2025-04-26 23:45:00] (step=0008700) Train Loss: 6.2038, Train Steps/Sec: 1.12 + 8%|██████████ | 8724/110696 [2:41:57<25:12:01, 1.12it/s][2025-04-26 23:45:22] (step=0008725) Train Loss: 6.1764, Train Steps/Sec: 1.12 + 8%|██████████ | 8749/110696 [2:42:19<25:06:46, 1.13it/s][2025-04-26 23:45:45] (step=0008750) Train Loss: 6.1919, Train Steps/Sec: 1.12 + 8%|██████████▏ | 8774/110696 [2:42:42<25:15:15, 1.12it/s][2025-04-26 23:46:07] (step=0008775) Train Loss: 6.1731, Train Steps/Sec: 1.11 + 8%|██████████▏ | 8799/110696 [2:43:04<25:04:50, 1.13it/s][2025-04-26 23:46:30] (step=0008800) Train Loss: 6.2395, Train Steps/Sec: 1.11 + 8%|██████████▏ | 8824/110696 [2:43:27<25:50:43, 1.09it/s][2025-04-26 23:46:52] (step=0008825) Train Loss: 6.1680, Train Steps/Sec: 1.12 + 8%|██████████▏ | 8849/110696 [2:43:49<25:29:02, 1.11it/s][2025-04-26 23:47:14] (step=0008850) Train Loss: 6.1746, Train Steps/Sec: 1.12 + 8%|██████████▎ | 8874/110696 [2:44:11<25:20:40, 1.12it/s][2025-04-26 23:47:37] (step=0008875) Train Loss: 6.1984, Train Steps/Sec: 1.12 + 8%|██████████▎ | 8899/110696 [2:44:34<25:13:21, 1.12it/s][2025-04-26 23:47:59] (step=0008900) Train Loss: 6.2257, Train Steps/Sec: 1.12 + 8%|██████████▎ | 8924/110696 [2:44:56<25:15:49, 1.12it/s][2025-04-26 23:48:21] (step=0008925) Train Loss: 6.1384, Train Steps/Sec: 1.12 + 8%|██████████▎ | 8949/110696 [2:45:18<25:08:56, 1.12it/s][2025-04-26 23:48:44] (step=0008950) Train Loss: 6.2580, Train Steps/Sec: 1.12 + 8%|██████████▍ | 8974/110696 [2:45:41<24:58:54, 1.13it/s][2025-04-26 23:49:06] (step=0008975) Train Loss: 6.1882, Train Steps/Sec: 1.12 + 8%|██████████▍ | 8999/110696 [2:46:03<25:02:41, 1.13it/s][2025-04-26 23:49:29] (step=0009000) Train Loss: 6.1978, Train Steps/Sec: 1.11 + 8%|██████████▍ | 9024/110696 [2:46:26<25:41:30, 1.10it/s][2025-04-26 23:49:51] (step=0009025) Train Loss: 6.2823, Train Steps/Sec: 1.12 + 8%|██████████▍ | 9049/110696 [2:46:48<25:26:11, 1.11it/s][2025-04-26 23:50:13] (step=0009050) Train Loss: 6.1656, Train Steps/Sec: 1.12 + 8%|██████████▍ | 9074/110696 [2:47:10<25:12:32, 1.12it/s][2025-04-26 23:50:36] (step=0009075) Train Loss: 6.1798, Train Steps/Sec: 1.12 + 8%|██████████▌ | 9099/110696 [2:47:33<25:10:09, 1.12it/s][2025-04-26 23:50:58] (step=0009100) Train Loss: 6.2140, Train Steps/Sec: 1.12 + 8%|██████████▌ | 9124/110696 [2:47:55<25:12:26, 1.12it/s][2025-04-26 23:51:20] (step=0009125) Train Loss: 6.2370, Train Steps/Sec: 1.12 + 8%|██████████▌ | 9149/110696 [2:48:17<25:04:32, 1.12it/s][2025-04-26 23:51:43] (step=0009150) Train Loss: 6.1746, Train Steps/Sec: 1.12 + 8%|██████████▌ | 9174/110696 [2:48:40<25:02:14, 1.13it/s][2025-04-26 23:52:05] (step=0009175) Train Loss: 6.2138, Train Steps/Sec: 1.12 + 8%|██████████▋ | 9199/110696 [2:49:02<24:58:18, 1.13it/s][2025-04-26 23:52:28] (step=0009200) Train Loss: 6.2456, Train Steps/Sec: 1.11 + 8%|██████████▋ | 9224/110696 [2:49:25<25:40:23, 1.10it/s][2025-04-26 23:52:50] (step=0009225) Train Loss: 6.1869, Train Steps/Sec: 1.12 + 8%|██████████▋ | 9249/110696 [2:49:47<25:23:11, 1.11it/s][2025-04-26 23:53:12] (step=0009250) Train Loss: 6.1351, Train Steps/Sec: 1.12 + 8%|██████████▋ | 9274/110696 [2:50:09<25:19:23, 1.11it/s][2025-04-26 23:53:35] (step=0009275) Train Loss: 6.1717, Train Steps/Sec: 1.12 + 8%|██████████▊ | 9299/110696 [2:50:32<25:05:56, 1.12it/s][2025-04-26 23:53:57] (step=0009300) Train Loss: 6.1988, Train Steps/Sec: 1.12 + 8%|██████████▊ | 9324/110696 [2:50:54<25:05:57, 1.12it/s][2025-04-26 23:54:19] (step=0009325) Train Loss: 6.1131, Train Steps/Sec: 1.12 + 8%|██████████▊ | 9349/110696 [2:51:16<24:59:25, 1.13it/s][2025-04-26 23:54:42] (step=0009350) Train Loss: 6.1538, Train Steps/Sec: 1.12 + 8%|██████████▊ | 9374/110696 [2:51:39<24:58:47, 1.13it/s][2025-04-26 23:55:04] (step=0009375) Train Loss: 6.1440, Train Steps/Sec: 1.12 + 8%|██████████▊ | 9399/110696 [2:52:01<24:57:58, 1.13it/s][2025-04-26 23:55:27] (step=0009400) Train Loss: 6.2038, Train Steps/Sec: 1.11 + 9%|██████████▉ | 9424/110696 [2:52:24<25:43:57, 1.09it/s][2025-04-26 23:55:49] (step=0009425) Train Loss: 6.1830, Train Steps/Sec: 1.12 + 9%|██████████▉ | 9449/110696 [2:52:46<25:44:16, 1.09it/s][2025-04-26 23:56:11] (step=0009450) Train Loss: 6.1893, Train Steps/Sec: 1.11 + 9%|██████████▉ | 9474/110696 [2:53:08<25:11:12, 1.12it/s][2025-04-26 23:56:34] (step=0009475) Train Loss: 6.1902, Train Steps/Sec: 1.12 + 9%|██████████▉ | 9499/110696 [2:53:31<25:21:48, 1.11it/s][2025-04-26 23:56:56] (step=0009500) Train Loss: 6.2341, Train Steps/Sec: 1.12 + 9%|███████████ | 9524/110696 [2:53:53<25:01:53, 1.12it/s][2025-04-26 23:57:19] (step=0009525) Train Loss: 6.1710, Train Steps/Sec: 1.12 + 9%|███████████ | 9549/110696 [2:54:15<24:53:08, 1.13it/s][2025-04-26 23:57:41] (step=0009550) Train Loss: 6.1807, Train Steps/Sec: 1.12 + 9%|███████████ | 9574/110696 [2:54:38<24:53:08, 1.13it/s][2025-04-26 23:58:03] (step=0009575) Train Loss: 6.1398, Train Steps/Sec: 1.12 + 9%|███████████ | 9599/110696 [2:55:00<24:49:41, 1.13it/s][2025-04-26 23:58:26] (step=0009600) Train Loss: 6.1379, Train Steps/Sec: 1.12 + 9%|███████████▏ | 9624/110696 [2:55:23<25:48:52, 1.09it/s][2025-04-26 23:58:48] (step=0009625) Train Loss: 6.1992, Train Steps/Sec: 1.12 + 9%|███████████▏ | 9649/110696 [2:55:45<25:24:55, 1.10it/s][2025-04-26 23:59:10] (step=0009650) Train Loss: 6.1831, Train Steps/Sec: 1.12 + 9%|███████████▏ | 9674/110696 [2:56:07<25:10:29, 1.11it/s][2025-04-26 23:59:33] (step=0009675) Train Loss: 6.1850, Train Steps/Sec: 1.12 + 9%|███████████▏ | 9699/110696 [2:56:30<25:04:20, 1.12it/s][2025-04-26 23:59:55] (step=0009700) Train Loss: 6.1971, Train Steps/Sec: 1.12 + 9%|███████████▏ | 9724/110696 [2:56:52<25:00:18, 1.12it/s][2025-04-27 00:00:17] (step=0009725) Train Loss: 6.2124, Train Steps/Sec: 1.12 + 9%|███████████▎ | 9749/110696 [2:57:14<24:53:18, 1.13it/s][2025-04-27 00:00:40] (step=0009750) Train Loss: 6.1820, Train Steps/Sec: 1.12 + 9%|███████████▎ | 9774/110696 [2:57:37<24:49:31, 1.13it/s][2025-04-27 00:01:02] (step=0009775) Train Loss: 6.1737, Train Steps/Sec: 1.12 + 9%|███████████▎ | 9799/110696 [2:57:59<24:46:53, 1.13it/s][2025-04-27 00:01:24] (step=0009800) Train Loss: 6.1444, Train Steps/Sec: 1.12 + 9%|███████████▎ | 9824/110696 [2:58:21<25:32:58, 1.10it/s][2025-04-27 00:01:47] (step=0009825) Train Loss: 6.1401, Train Steps/Sec: 1.12 + 9%|███████████▍ | 9849/110696 [2:58:44<25:19:26, 1.11it/s][2025-04-27 00:02:09] (step=0009850) Train Loss: 6.1416, Train Steps/Sec: 1.12 + 9%|███████████▍ | 9874/110696 [2:59:06<25:00:57, 1.12it/s][2025-04-27 00:02:31] (step=0009875) Train Loss: 6.2357, Train Steps/Sec: 1.12 + 9%|███████████▍ | 9899/110696 [2:59:28<24:59:34, 1.12it/s][2025-04-27 00:02:54] (step=0009900) Train Loss: 6.1321, Train Steps/Sec: 1.12 + 9%|███████████▍ | 9924/110696 [2:59:51<24:55:54, 1.12it/s][2025-04-27 00:03:16] (step=0009925) Train Loss: 6.2106, Train Steps/Sec: 1.12 + 9%|███████████▌ | 9949/110696 [3:00:13<24:55:14, 1.12it/s][2025-04-27 00:03:39] (step=0009950) Train Loss: 6.2060, Train Steps/Sec: 1.12 + 9%|███████████▌ | 9974/110696 [3:00:36<24:45:53, 1.13it/s][2025-04-27 00:04:01] (step=0009975) Train Loss: 6.1741, Train Steps/Sec: 1.12 + 9%|███████████▌ | 9999/110696 [3:00:58<24:45:10, 1.13it/s][2025-04-27 00:04:23] (step=0010000) Train Loss: 6.2019, Train Steps/Sec: 1.11 +You are using a model of type blip-2 to instantiate a model of type instructblip. This is not supported for all configurations of models and can yield errors. +[2025-04-27 00:04:23] text_config is None. Initializing the text config with default values (`OPTConfig`). +model_name_or_path:/tmp/haozhezhao/model/blip2-flan-t5-xl +image_place_holder: +tokenizer length after expend 32101 +tokenizer length before expend 32100 +100%|█████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████| 5/5 [04:57<00:00, 59.55s/it] +[2025-04-27 00:10:32] Finish Eval in 10000 steps...██████████████████████████████████████████████████████████████████████████████████████| 5/5 [04:57<00:00, 59.25s/it] +processing left-padding... +processing left-padding... +processing left-padding... +processing left-padding... +processing left-padding... +[2025-04-27 00:10:51] Saved checkpoint to checkpoint/X2I_UltraEdiit_fourmask_LLava_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_filtered_objects_4mask_1280_context/003-GPT-XL/checkpoints/0010000.pt +[2025-04-27 00:10:53] Removed old checkpoint: checkpoint/X2I_UltraEdiit_fourmask_LLava_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_filtered_objects_4mask_1280_context/003-GPT-XL/checkpoints/0008000.pt + 9%|███████████▌ | 10024/110696 [3:07:50<26:10:15, 1.07it/s][2025-04-27 00:11:15] (step=0010025) Train Loss: 6.1804, Train Steps/Sec: 0.06 + 9%|███████████▌ | 10049/110696 [3:08:12<25:14:41, 1.11it/s][2025-04-27 00:11:38] (step=0010050) Train Loss: 6.0934, Train Steps/Sec: 1.12 + 9%|███████████▌ | 10074/110696 [3:08:35<25:04:53, 1.11it/s][2025-04-27 00:12:00] (step=0010075) Train Loss: 6.2040, Train Steps/Sec: 1.12 + 9%|███████████▌ | 10099/110696 [3:08:57<24:56:29, 1.12it/s][2025-04-27 00:12:22] (step=0010100) Train Loss: 6.1897, Train Steps/Sec: 1.12 + 9%|███████████▌ | 10124/110696 [3:09:19<24:49:37, 1.13it/s][2025-04-27 00:12:45] (step=0010125) Train Loss: 6.1158, Train Steps/Sec: 1.12 + 9%|███████████▋ | 10149/110696 [3:09:42<24:56:57, 1.12it/s][2025-04-27 00:13:07] (step=0010150) Train Loss: 6.1947, Train Steps/Sec: 1.12 + 9%|███████████▋ | 10174/110696 [3:10:04<24:40:45, 1.13it/s][2025-04-27 00:13:29] (step=0010175) Train Loss: 6.1638, Train Steps/Sec: 1.12 + 9%|███████████▋ | 10199/110696 [3:10:26<24:43:25, 1.13it/s][2025-04-27 00:13:52] (step=0010200) Train Loss: 6.1699, Train Steps/Sec: 1.12 + 9%|███████████▋ | 10224/110696 [3:10:56<25:30:44, 1.09it/s][2025-04-27 00:14:21] (step=0010225) Train Loss: 6.0947, Train Steps/Sec: 0.85 + 9%|███████████▊ | 10249/110696 [3:11:18<25:05:38, 1.11it/s][2025-04-27 00:14:43] (step=0010250) Train Loss: 6.1865, Train Steps/Sec: 1.12 + 9%|███████████▊ | 10274/110696 [3:11:40<25:01:51, 1.11it/s][2025-04-27 00:15:06] (step=0010275) Train Loss: 6.1648, Train Steps/Sec: 1.12 + 9%|███████████▊ | 10299/110696 [3:12:03<24:50:05, 1.12it/s][2025-04-27 00:15:28] (step=0010300) Train Loss: 6.1437, Train Steps/Sec: 1.11 + 9%|███████████▊ | 10324/110696 [3:12:25<24:48:38, 1.12it/s][2025-04-27 00:15:51] (step=0010325) Train Loss: 6.1661, Train Steps/Sec: 1.12 + 9%|███████████▊ | 10349/110696 [3:12:48<24:39:10, 1.13it/s][2025-04-27 00:16:13] (step=0010350) Train Loss: 6.1890, Train Steps/Sec: 1.12 + 9%|███████████▉ | 10374/110696 [3:13:10<24:39:34, 1.13it/s][2025-04-27 00:16:35] (step=0010375) Train Loss: 6.1922, Train Steps/Sec: 1.12 + 9%|███████████▉ | 10399/110696 [3:13:32<24:41:50, 1.13it/s][2025-04-27 00:16:58] (step=0010400) Train Loss: 6.2179, Train Steps/Sec: 1.11 + 9%|███████████▉ | 10424/110696 [3:13:55<25:29:11, 1.09it/s][2025-04-27 00:17:20] (step=0010425) Train Loss: 6.2665, Train Steps/Sec: 1.12 + 9%|███████████▉ | 10449/110696 [3:14:17<25:07:50, 1.11it/s][2025-04-27 00:17:42] (step=0010450) Train Loss: 6.1188, Train Steps/Sec: 1.12 + 9%|████████████ | 10474/110696 [3:14:39<25:00:13, 1.11it/s][2025-04-27 00:18:05] (step=0010475) Train Loss: 6.1856, Train Steps/Sec: 1.12 + 9%|████████████ | 10499/110696 [3:15:02<24:49:46, 1.12it/s][2025-04-27 00:18:27] (step=0010500) Train Loss: 6.2059, Train Steps/Sec: 1.12 + 10%|████████████ | 10524/110696 [3:15:24<24:48:11, 1.12it/s][2025-04-27 00:18:49] (step=0010525) Train Loss: 6.2282, Train Steps/Sec: 1.12 + 10%|████████████ | 10549/110696 [3:15:46<24:38:03, 1.13it/s][2025-04-27 00:19:12] (step=0010550) Train Loss: 6.2230, Train Steps/Sec: 1.12 + 10%|████████████▏ | 10574/110696 [3:16:09<24:42:54, 1.13it/s][2025-04-27 00:19:34] (step=0010575) Train Loss: 6.0977, Train Steps/Sec: 1.12 + 10%|████████████▏ | 10599/110696 [3:16:31<24:35:38, 1.13it/s][2025-04-27 00:19:57] (step=0010600) Train Loss: 6.1637, Train Steps/Sec: 1.11 + 10%|████████████▏ | 10624/110696 [3:16:54<25:26:46, 1.09it/s][2025-04-27 00:20:19] (step=0010625) Train Loss: 6.1360, Train Steps/Sec: 1.12 + 10%|████████████▏ | 10649/110696 [3:17:16<25:06:27, 1.11it/s][2025-04-27 00:20:41] (step=0010650) Train Loss: 6.1337, Train Steps/Sec: 1.12 + 10%|████████████▏ | 10674/110696 [3:17:38<24:52:49, 1.12it/s][2025-04-27 00:21:04] (step=0010675) Train Loss: 6.1835, Train Steps/Sec: 1.12 + 10%|████████████▎ | 10699/110696 [3:18:01<25:25:27, 1.09it/s][2025-04-27 00:21:26] (step=0010700) Train Loss: 6.1405, Train Steps/Sec: 1.10 + 10%|████████████▎ | 10724/110696 [3:18:23<24:43:24, 1.12it/s][2025-04-27 00:21:49] (step=0010725) Train Loss: 6.2023, Train Steps/Sec: 1.12 + 10%|████████████▎ | 10749/110696 [3:18:46<24:39:48, 1.13it/s][2025-04-27 00:22:11] (step=0010750) Train Loss: 6.1453, Train Steps/Sec: 1.12 + 10%|████████████▎ | 10774/110696 [3:19:08<24:33:53, 1.13it/s][2025-04-27 00:22:33] (step=0010775) Train Loss: 6.1355, Train Steps/Sec: 1.12 + 10%|████████████▍ | 10799/110696 [3:19:30<24:34:11, 1.13it/s][2025-04-27 00:22:56] (step=0010800) Train Loss: 6.1925, Train Steps/Sec: 1.11 + 10%|████████████▍ | 10824/110696 [3:19:53<25:28:36, 1.09it/s][2025-04-27 00:23:18] (step=0010825) Train Loss: 6.1506, Train Steps/Sec: 1.12 + 10%|████████████▍ | 10849/110696 [3:20:15<25:03:58, 1.11it/s][2025-04-27 00:23:41] (step=0010850) Train Loss: 6.1025, Train Steps/Sec: 1.12 + 10%|████████████▍ | 10874/110696 [3:20:38<24:49:40, 1.12it/s][2025-04-27 00:24:03] (step=0010875) Train Loss: 6.1400, Train Steps/Sec: 1.12 + 10%|████████████▌ | 10899/110696 [3:21:00<24:43:10, 1.12it/s][2025-04-27 00:24:25] (step=0010900) Train Loss: 6.2093, Train Steps/Sec: 1.12 + 10%|████████████▌ | 10924/110696 [3:21:22<24:42:01, 1.12it/s][2025-04-27 00:24:48] (step=0010925) Train Loss: 6.1673, Train Steps/Sec: 1.12 + 10%|████████████▌ | 10949/110696 [3:21:45<24:39:46, 1.12it/s][2025-04-27 00:25:10] (step=0010950) Train Loss: 6.1925, Train Steps/Sec: 1.12 + 10%|████████████▌ | 10974/110696 [3:22:07<24:30:34, 1.13it/s][2025-04-27 00:25:32] (step=0010975) Train Loss: 6.1679, Train Steps/Sec: 1.12 + 10%|████████████▌ | 10999/110696 [3:22:29<24:33:36, 1.13it/s][2025-04-27 00:25:55] (step=0011000) Train Loss: 6.1114, Train Steps/Sec: 1.11 + 10%|████████████▋ | 11024/110696 [3:22:52<25:18:26, 1.09it/s][2025-04-27 00:26:17] (step=0011025) Train Loss: 6.1532, Train Steps/Sec: 1.12 + 10%|████████████▋ | 11049/110696 [3:23:34<68:06:57, 2.46s/it][2025-04-27 00:27:00] (step=0011050) Train Loss: 6.1413, Train Steps/Sec: 0.59 + 10%|████████████▋ | 11074/110696 [3:23:57<24:51:14, 1.11it/s][2025-04-27 00:27:22] (step=0011075) Train Loss: 6.1825, Train Steps/Sec: 1.12 + 10%|████████████▋ | 11099/110696 [3:24:19<24:46:21, 1.12it/s][2025-04-27 00:27:44] (step=0011100) Train Loss: 6.0836, Train Steps/Sec: 1.12 + 10%|████████████▊ | 11124/110696 [3:24:41<24:37:30, 1.12it/s][2025-04-27 00:28:07] (step=0011125) Train Loss: 6.1163, Train Steps/Sec: 1.12 + 10%|████████████▊ | 11149/110696 [3:25:04<24:33:38, 1.13it/s][2025-04-27 00:28:29] (step=0011150) Train Loss: 6.2097, Train Steps/Sec: 1.12 + 10%|████████████▊ | 11174/110696 [3:25:26<24:32:20, 1.13it/s][2025-04-27 00:28:51] (step=0011175) Train Loss: 6.1999, Train Steps/Sec: 1.12 + 10%|████████████▊ | 11199/110696 [3:25:53<24:42:15, 1.12it/s][2025-04-27 00:29:19] (step=0011200) Train Loss: 6.2131, Train Steps/Sec: 0.91 + 10%|████████████▉ | 11224/110696 [3:26:16<25:10:54, 1.10it/s][2025-04-27 00:29:41] (step=0011225) Train Loss: 6.1498, Train Steps/Sec: 1.12 + 10%|████████████▉ | 11249/110696 [3:26:43<25:00:47, 1.10it/s][2025-04-27 00:30:08] (step=0011250) Train Loss: 6.1797, Train Steps/Sec: 0.92 + 10%|████████████▉ | 11274/110696 [3:27:05<24:48:36, 1.11it/s][2025-04-27 00:30:31] (step=0011275) Train Loss: 6.1378, Train Steps/Sec: 1.12 + 10%|████████████▉ | 11299/110696 [3:27:28<24:37:38, 1.12it/s][2025-04-27 00:30:53] (step=0011300) Train Loss: 6.1711, Train Steps/Sec: 1.12 + 10%|████████████▉ | 11324/110696 [3:27:50<24:32:33, 1.12it/s][2025-04-27 00:31:15] (step=0011325) Train Loss: 6.1595, Train Steps/Sec: 1.12 + 10%|█████████████ | 11349/110696 [3:28:13<24:34:50, 1.12it/s][2025-04-27 00:31:38] (step=0011350) Train Loss: 6.1687, Train Steps/Sec: 1.11 + 10%|█████████████ | 11374/110696 [3:28:35<24:34:48, 1.12it/s][2025-04-27 00:32:00] (step=0011375) Train Loss: 6.1264, Train Steps/Sec: 1.12 + 10%|█████████████ | 11399/110696 [3:28:57<24:22:36, 1.13it/s][2025-04-27 00:32:23] (step=0011400) Train Loss: 6.2009, Train Steps/Sec: 1.11 + 10%|█████████████ | 11424/110696 [3:29:20<25:14:49, 1.09it/s][2025-04-27 00:32:45] (step=0011425) Train Loss: 6.1524, Train Steps/Sec: 1.12 + 10%|█████████████▏ | 11449/110696 [3:29:42<24:47:29, 1.11it/s][2025-04-27 00:33:08] (step=0011450) Train Loss: 6.1863, Train Steps/Sec: 1.12 + 10%|█████████████▏ | 11474/110696 [3:30:05<24:41:29, 1.12it/s][2025-04-27 00:33:30] (step=0011475) Train Loss: 6.1403, Train Steps/Sec: 1.12 + 10%|█████████████▏ | 11499/110696 [3:30:27<24:37:22, 1.12it/s][2025-04-27 00:33:52] (step=0011500) Train Loss: 6.1082, Train Steps/Sec: 1.12 + 10%|█████████████▏ | 11524/110696 [3:30:49<24:27:03, 1.13it/s][2025-04-27 00:34:15] (step=0011525) Train Loss: 6.0971, Train Steps/Sec: 1.12 + 10%|█████████████▎ | 11549/110696 [3:31:12<24:29:31, 1.12it/s][2025-04-27 00:34:37] (step=0011550) Train Loss: 6.1674, Train Steps/Sec: 1.12 + 10%|█████████████▎ | 11574/110696 [3:31:34<24:25:30, 1.13it/s][2025-04-27 00:34:59] (step=0011575) Train Loss: 6.1779, Train Steps/Sec: 1.12 + 10%|█████████████▎ | 11599/110696 [3:31:56<24:21:17, 1.13it/s][2025-04-27 00:35:22] (step=0011600) Train Loss: 6.1627, Train Steps/Sec: 1.11 + 11%|█████████████▎ | 11624/110696 [3:32:19<25:06:38, 1.10it/s][2025-04-27 00:35:44] (step=0011625) Train Loss: 6.1255, Train Steps/Sec: 1.12 + 11%|█████████████▎ | 11649/110696 [3:32:41<24:54:39, 1.10it/s][2025-04-27 00:36:07] (step=0011650) Train Loss: 6.1608, Train Steps/Sec: 1.12 + 11%|█████████████▍ | 11674/110696 [3:33:04<24:35:53, 1.12it/s][2025-04-27 00:36:29] (step=0011675) Train Loss: 6.1971, Train Steps/Sec: 1.12 + 11%|█████████████▍ | 11699/110696 [3:33:26<24:31:39, 1.12it/s][2025-04-27 00:36:51] (step=0011700) Train Loss: 6.1749, Train Steps/Sec: 1.12 + 11%|█████████████▍ | 11724/110696 [3:33:48<24:21:03, 1.13it/s][2025-04-27 00:37:14] (step=0011725) Train Loss: 6.1544, Train Steps/Sec: 1.12 + 11%|█████████████▍ | 11749/110696 [3:34:11<24:21:44, 1.13it/s][2025-04-27 00:37:36] (step=0011750) Train Loss: 6.1103, Train Steps/Sec: 1.12 + 11%|█████████████▌ | 11774/110696 [3:34:33<24:21:04, 1.13it/s][2025-04-27 00:37:58] (step=0011775) Train Loss: 6.2335, Train Steps/Sec: 1.12 + 11%|█████████████▌ | 11799/110696 [3:34:55<24:20:46, 1.13it/s][2025-04-27 00:38:21] (step=0011800) Train Loss: 6.1485, Train Steps/Sec: 1.11 + 11%|█████████████▌ | 11824/110696 [3:35:18<25:04:49, 1.10it/s][2025-04-27 00:38:43] (step=0011825) Train Loss: 6.1077, Train Steps/Sec: 1.12 + 11%|█████████████▌ | 11849/110696 [3:35:40<24:48:36, 1.11it/s][2025-04-27 00:39:05] (step=0011850) Train Loss: 6.1457, Train Steps/Sec: 1.12 + 11%|█████████████▌ | 11874/110696 [3:36:02<24:37:52, 1.11it/s][2025-04-27 00:39:28] (step=0011875) Train Loss: 6.1432, Train Steps/Sec: 1.12 + 11%|█████████████▋ | 11899/110696 [3:36:25<24:31:42, 1.12it/s][2025-04-27 00:39:50] (step=0011900) Train Loss: 6.1250, Train Steps/Sec: 1.12 + 11%|█████████████▋ | 11924/110696 [3:36:47<24:22:49, 1.13it/s][2025-04-27 00:40:12] (step=0011925) Train Loss: 6.1634, Train Steps/Sec: 1.12 + 11%|█████████████▋ | 11949/110696 [3:37:09<24:23:31, 1.12it/s][2025-04-27 00:40:35] (step=0011950) Train Loss: 6.1627, Train Steps/Sec: 1.12 + 11%|█████████████▋ | 11974/110696 [3:37:32<24:15:55, 1.13it/s][2025-04-27 00:40:57] (step=0011975) Train Loss: 6.1477, Train Steps/Sec: 1.12 + 11%|█████████████▊ | 11999/110696 [3:37:54<24:14:45, 1.13it/s][2025-04-27 00:41:20] (step=0012000) Train Loss: 6.1416, Train Steps/Sec: 1.12 +You are using a model of type blip-2 to instantiate a model of type instructblip. This is not supported for all configurations of models and can yield errors. +[2025-04-27 00:41:20] text_config is None. Initializing the text config with default values (`OPTConfig`). +model_name_or_path:/tmp/haozhezhao/model/blip2-flan-t5-xl +image_place_holder: +tokenizer length after expend 32101 +tokenizer length before expend 32100 +100%|█████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████| 5/5 [04:56<00:00, 59.28s/it] +[2025-04-27 00:47:26] Finish Eval in 12000 steps...██████████████████████████████████████████████████████████████████████████████████████| 5/5 [04:56<00:00, 59.10s/it] +processing left-padding... +processing left-padding... +processing left-padding... +processing left-padding... +processing left-padding... +[2025-04-27 00:47:46] Saved checkpoint to checkpoint/X2I_UltraEdiit_fourmask_LLava_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_filtered_objects_4mask_1280_context/003-GPT-XL/checkpoints/0012000.pt +[2025-04-27 00:47:48] Removed old checkpoint: checkpoint/X2I_UltraEdiit_fourmask_LLava_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_filtered_objects_4mask_1280_context/003-GPT-XL/checkpoints/0010000.pt + 11%|█████████████▊ | 12024/110696 [3:44:45<25:36:03, 1.07it/s][2025-04-27 00:48:10] (step=0012025) Train Loss: 6.1299, Train Steps/Sec: 0.06 + 11%|█████████████▊ | 12049/110696 [3:45:07<24:42:37, 1.11it/s][2025-04-27 00:48:33] (step=0012050) Train Loss: 6.1623, Train Steps/Sec: 1.12 + 11%|█████████████▊ | 12074/110696 [3:45:30<24:27:11, 1.12it/s][2025-04-27 00:48:55] (step=0012075) Train Loss: 6.1050, Train Steps/Sec: 1.12 + 11%|█████████████▉ | 12099/110696 [3:45:52<24:22:06, 1.12it/s][2025-04-27 00:49:17] (step=0012100) Train Loss: 6.1223, Train Steps/Sec: 1.12 + 11%|█████████████▉ | 12124/110696 [3:46:14<24:25:24, 1.12it/s][2025-04-27 00:49:40] (step=0012125) Train Loss: 6.1452, Train Steps/Sec: 1.12 + 11%|█████████████▉ | 12149/110696 [3:46:37<24:20:47, 1.12it/s][2025-04-27 00:50:02] (step=0012150) Train Loss: 6.1565, Train Steps/Sec: 1.12 + 11%|█████████████▉ | 12174/110696 [3:46:59<24:17:34, 1.13it/s][2025-04-27 00:50:24] (step=0012175) Train Loss: 6.1869, Train Steps/Sec: 1.12 + 11%|█████████████▉ | 12199/110696 [3:47:22<24:13:03, 1.13it/s][2025-04-27 00:50:47] (step=0012200) Train Loss: 6.1185, Train Steps/Sec: 1.11 + 11%|██████████████ | 12224/110696 [3:47:44<24:56:18, 1.10it/s][2025-04-27 00:51:09] (step=0012225) Train Loss: 6.1121, Train Steps/Sec: 1.12 + 11%|██████████████ | 12249/110696 [3:48:06<24:42:25, 1.11it/s][2025-04-27 00:51:32] (step=0012250) Train Loss: 6.0968, Train Steps/Sec: 1.12 + 11%|██████████████ | 12274/110696 [3:48:29<24:40:51, 1.11it/s][2025-04-27 00:51:54] (step=0012275) Train Loss: 6.1237, Train Steps/Sec: 1.12 + 11%|██████████████ | 12299/110696 [3:48:51<24:22:42, 1.12it/s][2025-04-27 00:52:16] (step=0012300) Train Loss: 6.1436, Train Steps/Sec: 1.12 + 11%|██████████████▏ | 12324/110696 [3:49:13<24:18:05, 1.12it/s][2025-04-27 00:52:39] (step=0012325) Train Loss: 6.1315, Train Steps/Sec: 1.12 + 11%|██████████████▏ | 12349/110696 [3:49:36<24:14:10, 1.13it/s][2025-04-27 00:53:01] (step=0012350) Train Loss: 6.0971, Train Steps/Sec: 1.12 + 11%|██████████████▏ | 12374/110696 [3:49:58<24:16:54, 1.12it/s][2025-04-27 00:53:23] (step=0012375) Train Loss: 6.1289, Train Steps/Sec: 1.12 + 11%|██████████████▏ | 12399/110696 [3:50:20<24:06:39, 1.13it/s][2025-04-27 00:53:46] (step=0012400) Train Loss: 6.1283, Train Steps/Sec: 1.12 + 11%|██████████████▎ | 12424/110696 [3:50:43<25:02:55, 1.09it/s][2025-04-27 00:54:08] (step=0012425) Train Loss: 6.1752, Train Steps/Sec: 1.12 + 11%|██████████████▎ | 12449/110696 [3:51:05<24:33:51, 1.11it/s][2025-04-27 00:54:31] (step=0012450) Train Loss: 6.1276, Train Steps/Sec: 1.12 + 11%|██████████████▎ | 12474/110696 [3:51:28<24:24:53, 1.12it/s][2025-04-27 00:54:53] (step=0012475) Train Loss: 6.1435, Train Steps/Sec: 1.12 + 11%|██████████████▎ | 12499/110696 [3:51:50<24:22:08, 1.12it/s][2025-04-27 00:55:15] (step=0012500) Train Loss: 6.1084, Train Steps/Sec: 1.12 + 11%|██████████████▎ | 12524/110696 [3:52:12<24:14:55, 1.12it/s][2025-04-27 00:55:38] (step=0012525) Train Loss: 6.1778, Train Steps/Sec: 1.12 + 11%|██████████████▍ | 12549/110696 [3:52:35<24:15:15, 1.12it/s][2025-04-27 00:56:00] (step=0012550) Train Loss: 6.1054, Train Steps/Sec: 1.12 + 11%|██████████████▍ | 12574/110696 [3:52:57<24:08:54, 1.13it/s][2025-04-27 00:56:22] (step=0012575) Train Loss: 6.1609, Train Steps/Sec: 1.12 + 11%|██████████████▍ | 12599/110696 [3:53:19<24:10:15, 1.13it/s][2025-04-27 00:56:45] (step=0012600) Train Loss: 6.1276, Train Steps/Sec: 1.11 + 11%|██████████████▍ | 12624/110696 [3:53:42<24:56:02, 1.09it/s][2025-04-27 00:57:07] (step=0012625) Train Loss: 6.2570, Train Steps/Sec: 1.12 + 11%|██████████████▌ | 12649/110696 [3:54:04<24:37:23, 1.11it/s][2025-04-27 00:57:29] (step=0012650) Train Loss: 6.0886, Train Steps/Sec: 1.12 + 11%|██████████████▌ | 12674/110696 [3:54:26<24:27:15, 1.11it/s][2025-04-27 00:57:52] (step=0012675) Train Loss: 6.1545, Train Steps/Sec: 1.12 + 11%|██████████████▌ | 12699/110696 [3:54:49<24:17:29, 1.12it/s][2025-04-27 00:58:14] (step=0012700) Train Loss: 6.1164, Train Steps/Sec: 1.12 + 11%|██████████████▌ | 12724/110696 [3:55:11<24:13:15, 1.12it/s][2025-04-27 00:58:37] (step=0012725) Train Loss: 6.0929, Train Steps/Sec: 1.12 + 12%|██████████████▋ | 12749/110696 [3:55:34<24:09:16, 1.13it/s][2025-04-27 00:58:59] (step=0012750) Train Loss: 6.1327, Train Steps/Sec: 1.12 + 12%|██████████████▋ | 12774/110696 [3:55:56<24:21:31, 1.12it/s][2025-04-27 00:59:21] (step=0012775) Train Loss: 6.1621, Train Steps/Sec: 1.12 + 12%|██████████████▋ | 12799/110696 [3:56:18<24:09:41, 1.13it/s][2025-04-27 00:59:44] (step=0012800) Train Loss: 6.1154, Train Steps/Sec: 1.12 + 12%|██████████████▋ | 12824/110696 [3:56:41<24:54:21, 1.09it/s][2025-04-27 01:00:06] (step=0012825) Train Loss: 6.0943, Train Steps/Sec: 1.12 + 12%|██████████████▋ | 12849/110696 [3:57:03<24:28:58, 1.11it/s][2025-04-27 01:00:28] (step=0012850) Train Loss: 6.1357, Train Steps/Sec: 1.12 + 12%|██████████████▊ | 12874/110696 [3:57:25<24:22:40, 1.11it/s][2025-04-27 01:00:51] (step=0012875) Train Loss: 6.0476, Train Steps/Sec: 1.12 + 12%|██████████████▊ | 12899/110696 [3:57:48<24:14:46, 1.12it/s][2025-04-27 01:01:13] (step=0012900) Train Loss: 6.1633, Train Steps/Sec: 1.12 + 12%|██████████████▊ | 12924/110696 [3:58:10<24:09:27, 1.12it/s][2025-04-27 01:01:36] (step=0012925) Train Loss: 6.1268, Train Steps/Sec: 1.12 + 12%|██████████████▊ | 12949/110696 [3:58:32<24:05:17, 1.13it/s][2025-04-27 01:01:58] (step=0012950) Train Loss: 6.0752, Train Steps/Sec: 1.12 + 12%|██████████████▉ | 12974/110696 [3:58:55<24:05:48, 1.13it/s][2025-04-27 01:02:20] (step=0012975) Train Loss: 6.1316, Train Steps/Sec: 1.12 + 12%|██████████████▉ | 12999/110696 [3:59:17<24:02:56, 1.13it/s][2025-04-27 01:02:43] (step=0013000) Train Loss: 6.1158, Train Steps/Sec: 1.11 + 12%|██████████████▉ | 13024/110696 [3:59:40<24:40:52, 1.10it/s][2025-04-27 01:03:05] (step=0013025) Train Loss: 6.0956, Train Steps/Sec: 1.12 + 12%|██████████████▉ | 13049/110696 [4:00:02<24:22:46, 1.11it/s][2025-04-27 01:03:27] (step=0013050) Train Loss: 6.0899, Train Steps/Sec: 1.12 + 12%|██████████████▉ | 13074/110696 [4:00:24<24:22:30, 1.11it/s][2025-04-27 01:03:50] (step=0013075) Train Loss: 6.0800, Train Steps/Sec: 1.12 + 12%|███████████████ | 13099/110696 [4:00:47<24:08:59, 1.12it/s][2025-04-27 01:04:12] (step=0013100) Train Loss: 6.0516, Train Steps/Sec: 1.12 + 12%|███████████████ | 13124/110696 [4:01:09<24:07:45, 1.12it/s][2025-04-27 01:04:34] (step=0013125) Train Loss: 6.0976, Train Steps/Sec: 1.12 + 12%|███████████████ | 13149/110696 [4:01:31<24:06:23, 1.12it/s][2025-04-27 01:04:57] (step=0013150) Train Loss: 6.1257, Train Steps/Sec: 1.12 + 12%|███████████████ | 13174/110696 [4:01:54<24:01:47, 1.13it/s][2025-04-27 01:05:19] (step=0013175) Train Loss: 6.1248, Train Steps/Sec: 1.12 + 12%|███████████████▏ | 13199/110696 [4:02:16<23:59:48, 1.13it/s][2025-04-27 01:05:42] (step=0013200) Train Loss: 6.1227, Train Steps/Sec: 1.12 + 12%|███████████████▏ | 13224/110696 [4:02:38<24:41:12, 1.10it/s][2025-04-27 01:06:04] (step=0013225) Train Loss: 6.0392, Train Steps/Sec: 1.12 + 12%|███████████████▏ | 13249/110696 [4:03:01<24:30:11, 1.10it/s][2025-04-27 01:06:26] (step=0013250) Train Loss: 6.1242, Train Steps/Sec: 1.12 + 12%|███████████████▏ | 13274/110696 [4:03:24<26:27:49, 1.02it/s][2025-04-27 01:06:49] (step=0013275) Train Loss: 6.0973, Train Steps/Sec: 1.10 + 12%|███████████████▎ | 13299/110696 [4:03:46<24:06:34, 1.12it/s][2025-04-27 01:07:11] (step=0013300) Train Loss: 6.1151, Train Steps/Sec: 1.12 + 12%|███████████████▎ | 13324/110696 [4:04:08<24:06:21, 1.12it/s][2025-04-27 01:07:34] (step=0013325) Train Loss: 6.1187, Train Steps/Sec: 1.12 + 12%|███████████████▎ | 13349/110696 [4:04:31<24:01:29, 1.13it/s][2025-04-27 01:07:56] (step=0013350) Train Loss: 6.1397, Train Steps/Sec: 1.12 + 12%|███████████████▎ | 13374/110696 [4:04:53<23:58:26, 1.13it/s][2025-04-27 01:08:18] (step=0013375) Train Loss: 6.0894, Train Steps/Sec: 1.12 + 12%|███████████████▎ | 13399/110696 [4:05:15<23:55:14, 1.13it/s][2025-04-27 01:08:41] (step=0013400) Train Loss: 6.1662, Train Steps/Sec: 1.12 + 12%|███████████████▍ | 13424/110696 [4:05:38<24:45:13, 1.09it/s][2025-04-27 01:09:03] (step=0013425) Train Loss: 6.1208, Train Steps/Sec: 1.12 + 12%|███████████████▍ | 13449/110696 [4:06:00<24:24:33, 1.11it/s][2025-04-27 01:09:26] (step=0013450) Train Loss: 6.1432, Train Steps/Sec: 1.12 + 12%|███████████████▍ | 13474/110696 [4:06:23<24:13:28, 1.11it/s][2025-04-27 01:09:48] (step=0013475) Train Loss: 6.0549, Train Steps/Sec: 1.12 + 12%|███████████████▍ | 13499/110696 [4:06:45<24:03:48, 1.12it/s][2025-04-27 01:10:10] (step=0013500) Train Loss: 6.0870, Train Steps/Sec: 1.12 + 12%|███████████████▌ | 13524/110696 [4:07:07<23:59:27, 1.13it/s][2025-04-27 01:10:33] (step=0013525) Train Loss: 6.0915, Train Steps/Sec: 1.12 + 12%|███████████████▌ | 13549/110696 [4:07:30<23:53:31, 1.13it/s][2025-04-27 01:10:55] (step=0013550) Train Loss: 6.0999, Train Steps/Sec: 1.12 + 12%|███████████████▌ | 13574/110696 [4:07:52<23:53:48, 1.13it/s][2025-04-27 01:11:17] (step=0013575) Train Loss: 6.1139, Train Steps/Sec: 1.12 + 12%|███████████████▌ | 13599/110696 [4:08:14<23:50:46, 1.13it/s][2025-04-27 01:11:40] (step=0013600) Train Loss: 6.1332, Train Steps/Sec: 1.12 + 12%|███████████████▋ | 13624/110696 [4:08:37<24:39:33, 1.09it/s][2025-04-27 01:12:02] (step=0013625) Train Loss: 6.0852, Train Steps/Sec: 1.12 + 12%|███████████████▋ | 13649/110696 [4:08:59<24:19:13, 1.11it/s][2025-04-27 01:12:24] (step=0013650) Train Loss: 6.1050, Train Steps/Sec: 1.12 + 12%|███████████████▋ | 13674/110696 [4:09:21<24:07:00, 1.12it/s][2025-04-27 01:12:47] (step=0013675) Train Loss: 6.1376, Train Steps/Sec: 1.12 + 12%|███████████████▋ | 13699/110696 [4:09:44<24:00:05, 1.12it/s][2025-04-27 01:13:09] (step=0013700) Train Loss: 6.1209, Train Steps/Sec: 1.12 + 12%|███████████████▋ | 13724/110696 [4:10:06<23:55:28, 1.13it/s][2025-04-27 01:13:31] (step=0013725) Train Loss: 6.0896, Train Steps/Sec: 1.12 + 12%|███████████████▊ | 13749/110696 [4:10:28<23:51:57, 1.13it/s][2025-04-27 01:13:54] (step=0013750) Train Loss: 6.0444, Train Steps/Sec: 1.12 + 12%|███████████████▊ | 13774/110696 [4:10:51<23:51:37, 1.13it/s][2025-04-27 01:14:16] (step=0013775) Train Loss: 6.1085, Train Steps/Sec: 1.12 + 12%|███████████████▊ | 13799/110696 [4:11:13<23:49:10, 1.13it/s][2025-04-27 01:14:38] (step=0013800) Train Loss: 6.1145, Train Steps/Sec: 1.12 + 12%|███████████████▊ | 13824/110696 [4:11:35<24:34:11, 1.10it/s][2025-04-27 01:15:01] (step=0013825) Train Loss: 6.1240, Train Steps/Sec: 1.12 + 13%|███████████████▉ | 13849/110696 [4:11:58<24:14:35, 1.11it/s][2025-04-27 01:15:23] (step=0013850) Train Loss: 6.0972, Train Steps/Sec: 1.12 + 13%|███████████████▉ | 13874/110696 [4:12:20<24:04:05, 1.12it/s][2025-04-27 01:15:45] (step=0013875) Train Loss: 6.1266, Train Steps/Sec: 1.12 + 13%|███████████████▉ | 13899/110696 [4:12:42<23:58:41, 1.12it/s][2025-04-27 01:16:08] (step=0013900) Train Loss: 6.1704, Train Steps/Sec: 1.12 + 13%|███████████████▉ | 13924/110696 [4:13:05<23:55:02, 1.12it/s][2025-04-27 01:16:30] (step=0013925) Train Loss: 6.0268, Train Steps/Sec: 1.12 + 13%|████████████████ | 13949/110696 [4:13:27<23:49:06, 1.13it/s][2025-04-27 01:16:53] (step=0013950) Train Loss: 6.0798, Train Steps/Sec: 1.12 + 13%|████████████████ | 13974/110696 [4:13:50<23:50:20, 1.13it/s][2025-04-27 01:17:15] (step=0013975) Train Loss: 6.1048, Train Steps/Sec: 1.12 + 13%|████████████████ | 13999/110696 [4:14:12<23:48:36, 1.13it/s][2025-04-27 01:17:37] (step=0014000) Train Loss: 6.0873, Train Steps/Sec: 1.12 +You are using a model of type blip-2 to instantiate a model of type instructblip. This is not supported for all configurations of models and can yield errors. +[2025-04-27 01:17:37] text_config is None. Initializing the text config with default values (`OPTConfig`). +model_name_or_path:/tmp/haozhezhao/model/blip2-flan-t5-xl +image_place_holder: +tokenizer length after expend 32101 +tokenizer length before expend 32100 +100%|█████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████| 5/5 [04:55<00:00, 59.16s/it] +[2025-04-27 01:23:45] Finish Eval in 14000 steps...██████████████████████████████████████████████████████████████████████████████████████| 5/5 [04:55<00:00, 58.96s/it] +processing left-padding... +processing left-padding... +processing left-padding... +processing left-padding... +processing left-padding... +[2025-04-27 01:24:04] Saved checkpoint to checkpoint/X2I_UltraEdiit_fourmask_LLava_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_filtered_objects_4mask_1280_context/003-GPT-XL/checkpoints/0014000.pt +[2025-04-27 01:24:06] Removed old checkpoint: checkpoint/X2I_UltraEdiit_fourmask_LLava_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_filtered_objects_4mask_1280_context/003-GPT-XL/checkpoints/0012000.pt + 13%|████████████████ | 14024/110696 [4:21:04<25:04:50, 1.07it/s][2025-04-27 01:24:29] (step=0014025) Train Loss: 6.0562, Train Steps/Sec: 0.06 + 13%|████████████████ | 14049/110696 [4:21:26<24:11:24, 1.11it/s][2025-04-27 01:24:51] (step=0014050) Train Loss: 6.1414, Train Steps/Sec: 1.12 + 13%|████████████████▏ | 14074/110696 [4:21:48<24:04:17, 1.11it/s][2025-04-27 01:25:14] (step=0014075) Train Loss: 6.1442, Train Steps/Sec: 1.12 + 13%|████████████████▏ | 14099/110696 [4:22:11<23:56:43, 1.12it/s][2025-04-27 01:25:36] (step=0014100) Train Loss: 6.1105, Train Steps/Sec: 1.12 + 13%|████████████████▏ | 14124/110696 [4:22:33<23:45:36, 1.13it/s][2025-04-27 01:25:58] (step=0014125) Train Loss: 6.1153, Train Steps/Sec: 1.12 + 13%|████████████████▏ | 14149/110696 [4:22:55<23:46:25, 1.13it/s][2025-04-27 01:26:21] (step=0014150) Train Loss: 6.0646, Train Steps/Sec: 1.12 + 13%|████████████████▎ | 14174/110696 [4:23:18<23:44:57, 1.13it/s][2025-04-27 01:26:43] (step=0014175) Train Loss: 6.1055, Train Steps/Sec: 1.12 + 13%|████████████████▎ | 14199/110696 [4:23:40<23:41:20, 1.13it/s][2025-04-27 01:27:05] (step=0014200) Train Loss: 6.1276, Train Steps/Sec: 1.12 + 13%|████████████████▎ | 14224/110696 [4:24:02<24:29:31, 1.09it/s][2025-04-27 01:27:28] (step=0014225) Train Loss: 6.1511, Train Steps/Sec: 1.12 + 13%|████████████████▎ | 14249/110696 [4:24:25<24:07:24, 1.11it/s][2025-04-27 01:27:50] (step=0014250) Train Loss: 6.0543, Train Steps/Sec: 1.12 + 13%|████████████████▍ | 14274/110696 [4:24:47<23:59:54, 1.12it/s][2025-04-27 01:28:12] (step=0014275) Train Loss: 6.0797, Train Steps/Sec: 1.12 + 13%|████████████████▍ | 14299/110696 [4:25:09<23:55:56, 1.12it/s][2025-04-27 01:28:35] (step=0014300) Train Loss: 6.1078, Train Steps/Sec: 1.12 + 13%|████████████████▍ | 14324/110696 [4:25:32<23:49:54, 1.12it/s][2025-04-27 01:28:57] (step=0014325) Train Loss: 6.0549, Train Steps/Sec: 1.12 + 13%|████████████████▍ | 14349/110696 [4:25:54<23:44:56, 1.13it/s][2025-04-27 01:29:19] (step=0014350) Train Loss: 6.1485, Train Steps/Sec: 1.12 + 13%|████████████████▍ | 14374/110696 [4:26:16<23:37:59, 1.13it/s][2025-04-27 01:29:42] (step=0014375) Train Loss: 6.0924, Train Steps/Sec: 1.12 + 13%|████████████████▌ | 14399/110696 [4:26:39<23:39:11, 1.13it/s][2025-04-27 01:30:04] (step=0014400) Train Loss: 6.0661, Train Steps/Sec: 1.11 + 13%|████████████████▌ | 14424/110696 [4:27:01<24:20:31, 1.10it/s][2025-04-27 01:30:26] (step=0014425) Train Loss: 6.0936, Train Steps/Sec: 1.12 + 13%|████████████████▌ | 14449/110696 [4:27:23<24:07:31, 1.11it/s][2025-04-27 01:30:49] (step=0014450) Train Loss: 6.0994, Train Steps/Sec: 1.12 + 13%|████████████████▌ | 14474/110696 [4:27:46<23:52:21, 1.12it/s][2025-04-27 01:31:11] (step=0014475) Train Loss: 6.1111, Train Steps/Sec: 1.12 + 13%|████████████████▋ | 14499/110696 [4:28:08<23:52:00, 1.12it/s][2025-04-27 01:31:33] (step=0014500) Train Loss: 6.1018, Train Steps/Sec: 1.12 + 13%|████████████████▋ | 14524/110696 [4:28:30<23:48:30, 1.12it/s][2025-04-27 01:31:56] (step=0014525) Train Loss: 6.0495, Train Steps/Sec: 1.12 + 13%|████████████████▋ | 14549/110696 [4:28:53<23:41:23, 1.13it/s][2025-04-27 01:32:18] (step=0014550) Train Loss: 6.1786, Train Steps/Sec: 1.12 + 13%|████████████████▋ | 14574/110696 [4:29:15<23:36:25, 1.13it/s][2025-04-27 01:32:41] (step=0014575) Train Loss: 6.1120, Train Steps/Sec: 1.12 + 13%|████████████████▋ | 14599/110696 [4:29:38<23:43:16, 1.13it/s][2025-04-27 01:33:03] (step=0014600) Train Loss: 6.1076, Train Steps/Sec: 1.11 + 13%|████████████████▊ | 14624/110696 [4:30:05<24:19:01, 1.10it/s][2025-04-27 01:33:30] (step=0014625) Train Loss: 6.1094, Train Steps/Sec: 0.91 + 13%|████████████████▊ | 14649/110696 [4:30:27<24:01:25, 1.11it/s][2025-04-27 01:33:53] (step=0014650) Train Loss: 6.0458, Train Steps/Sec: 1.12 + 13%|████████████████▊ | 14674/110696 [4:30:50<23:58:23, 1.11it/s][2025-04-27 01:34:15] (step=0014675) Train Loss: 6.0871, Train Steps/Sec: 1.12 + 13%|████████████████▊ | 14699/110696 [4:31:12<23:48:23, 1.12it/s][2025-04-27 01:34:37] (step=0014700) Train Loss: 6.1138, Train Steps/Sec: 1.12 + 13%|████████████████▉ | 14724/110696 [4:31:34<23:45:37, 1.12it/s][2025-04-27 01:35:00] (step=0014725) Train Loss: 6.0893, Train Steps/Sec: 1.12 + 13%|████████████████▉ | 14749/110696 [4:31:57<23:46:31, 1.12it/s][2025-04-27 01:35:22] (step=0014750) Train Loss: 6.1121, Train Steps/Sec: 1.12 + 13%|████████████████▉ | 14774/110696 [4:32:19<23:38:39, 1.13it/s][2025-04-27 01:35:45] (step=0014775) Train Loss: 6.1131, Train Steps/Sec: 1.12 + 13%|████████████████▉ | 14799/110696 [4:32:41<23:31:35, 1.13it/s][2025-04-27 01:36:07] (step=0014800) Train Loss: 6.1081, Train Steps/Sec: 1.12 + 13%|█████████████████ | 14824/110696 [4:33:04<24:17:25, 1.10it/s][2025-04-27 01:36:29] (step=0014825) Train Loss: 6.1170, Train Steps/Sec: 1.12 + 13%|█████████████████ | 14849/110696 [4:33:26<24:01:11, 1.11it/s][2025-04-27 01:36:52] (step=0014850) Train Loss: 6.1596, Train Steps/Sec: 1.12 + 13%|█████████████████ | 14874/110696 [4:33:49<23:52:30, 1.11it/s][2025-04-27 01:37:14] (step=0014875) Train Loss: 6.1337, Train Steps/Sec: 1.12 + 13%|█████████████████ | 14899/110696 [4:34:11<23:46:02, 1.12it/s][2025-04-27 01:37:36] (step=0014900) Train Loss: 6.1442, Train Steps/Sec: 1.12 + 13%|█████████████████ | 14924/110696 [4:34:33<23:40:29, 1.12it/s][2025-04-27 01:37:59] (step=0014925) Train Loss: 6.1756, Train Steps/Sec: 1.12 + 14%|█████████████████▏ | 14949/110696 [4:34:56<23:37:45, 1.13it/s][2025-04-27 01:38:21] (step=0014950) Train Loss: 6.0803, Train Steps/Sec: 1.12 + 14%|█████████████████▏ | 14974/110696 [4:35:18<23:34:35, 1.13it/s][2025-04-27 01:38:43] (step=0014975) Train Loss: 6.0509, Train Steps/Sec: 1.12 + 14%|█████████████████▏ | 14999/110696 [4:35:40<23:33:26, 1.13it/s][2025-04-27 01:39:06] (step=0015000) Train Loss: 6.0728, Train Steps/Sec: 1.12 + 14%|█████████████████▏ | 15024/110696 [4:36:03<24:14:44, 1.10it/s][2025-04-27 01:39:28] (step=0015025) Train Loss: 6.1645, Train Steps/Sec: 1.12 + 14%|█████████████████▎ | 15049/110696 [4:36:25<24:01:18, 1.11it/s][2025-04-27 01:39:50] (step=0015050) Train Loss: 6.1137, Train Steps/Sec: 1.12 + 14%|█████████████████▎ | 15074/110696 [4:36:47<23:48:54, 1.12it/s][2025-04-27 01:40:13] (step=0015075) Train Loss: 6.1229, Train Steps/Sec: 1.12 + 14%|█████████████████▎ | 15099/110696 [4:37:10<23:44:01, 1.12it/s][2025-04-27 01:40:35] (step=0015100) Train Loss: 6.0835, Train Steps/Sec: 1.12 + 14%|█████████████████▎ | 15124/110696 [4:37:32<23:34:57, 1.13it/s][2025-04-27 01:40:57] (step=0015125) Train Loss: 6.0864, Train Steps/Sec: 1.12 + 14%|█████████████████▍ | 15149/110696 [4:37:54<23:35:34, 1.12it/s][2025-04-27 01:41:20] (step=0015150) Train Loss: 6.1318, Train Steps/Sec: 1.12 + 14%|█████████████████▍ | 15174/110696 [4:38:17<23:27:11, 1.13it/s][2025-04-27 01:41:42] (step=0015175) Train Loss: 6.0789, Train Steps/Sec: 1.12 + 14%|█████████████████▍ | 15199/110696 [4:38:39<23:30:08, 1.13it/s][2025-04-27 01:42:04] (step=0015200) Train Loss: 6.1044, Train Steps/Sec: 1.12 + 14%|█████████████████▍ | 15224/110696 [4:39:01<24:13:16, 1.09it/s][2025-04-27 01:42:27] (step=0015225) Train Loss: 6.1337, Train Steps/Sec: 1.12 + 14%|█████████████████▍ | 15249/110696 [4:39:24<23:53:24, 1.11it/s][2025-04-27 01:42:49] (step=0015250) Train Loss: 6.0789, Train Steps/Sec: 1.12 + 14%|█████████████████▌ | 15274/110696 [4:39:46<23:45:39, 1.12it/s][2025-04-27 01:43:11] (step=0015275) Train Loss: 6.0628, Train Steps/Sec: 1.12 + 14%|█████████████████▌ | 15299/110696 [4:40:08<23:36:24, 1.12it/s][2025-04-27 01:43:34] (step=0015300) Train Loss: 6.0544, Train Steps/Sec: 1.12 + 14%|█████████████████▌ | 15324/110696 [4:40:31<23:32:15, 1.13it/s][2025-04-27 01:43:56] (step=0015325) Train Loss: 6.1339, Train Steps/Sec: 1.12 + 14%|█████████████████▌ | 15349/110696 [4:40:53<23:27:46, 1.13it/s][2025-04-27 01:44:18] (step=0015350) Train Loss: 6.0963, Train Steps/Sec: 1.12 + 14%|█████████████████▋ | 15374/110696 [4:41:30<49:25:05, 1.87s/it][2025-04-27 01:44:55] (step=0015375) Train Loss: 6.0122, Train Steps/Sec: 0.67 + 14%|█████████████████▋ | 15399/110696 [4:41:58<23:48:30, 1.11it/s][2025-04-27 01:45:23] (step=0015400) Train Loss: 6.0854, Train Steps/Sec: 0.90 + 14%|█████████████████▋ | 15424/110696 [4:42:20<24:07:55, 1.10it/s][2025-04-27 01:45:46] (step=0015425) Train Loss: 6.0612, Train Steps/Sec: 1.12 + 14%|█████████████████▋ | 15449/110696 [4:42:43<23:54:10, 1.11it/s][2025-04-27 01:46:08] (step=0015450) Train Loss: 6.0580, Train Steps/Sec: 1.12 + 14%|█████████████████▊ | 15474/110696 [4:43:10<23:51:47, 1.11it/s][2025-04-27 01:46:35] (step=0015475) Train Loss: 6.0434, Train Steps/Sec: 0.92 + 14%|█████████████████▊ | 15499/110696 [4:43:37<24:16:48, 1.09it/s][2025-04-27 01:47:02] (step=0015500) Train Loss: 6.1113, Train Steps/Sec: 0.93 + 14%|█████████████████▊ | 15524/110696 [4:43:59<23:32:28, 1.12it/s][2025-04-27 01:47:25] (step=0015525) Train Loss: 6.1056, Train Steps/Sec: 1.12 + 14%|█████████████████▊ | 15549/110696 [4:44:22<23:31:42, 1.12it/s][2025-04-27 01:47:47] (step=0015550) Train Loss: 6.1003, Train Steps/Sec: 1.12 + 14%|█████████████████▊ | 15574/110696 [4:44:44<23:22:59, 1.13it/s][2025-04-27 01:48:09] (step=0015575) Train Loss: 6.1463, Train Steps/Sec: 1.12 + 14%|█████████████████▉ | 15599/110696 [4:45:06<23:22:56, 1.13it/s][2025-04-27 01:48:32] (step=0015600) Train Loss: 6.0799, Train Steps/Sec: 1.12 + 14%|█████████████████▉ | 15624/110696 [4:45:29<24:04:20, 1.10it/s][2025-04-27 01:48:54] (step=0015625) Train Loss: 6.0913, Train Steps/Sec: 1.12 + 14%|█████████████████▉ | 15649/110696 [4:45:51<23:45:35, 1.11it/s][2025-04-27 01:49:16] (step=0015650) Train Loss: 6.1104, Train Steps/Sec: 1.12 + 14%|█████████████████▉ | 15674/110696 [4:46:13<23:38:07, 1.12it/s][2025-04-27 01:49:39] (step=0015675) Train Loss: 6.1006, Train Steps/Sec: 1.12 + 14%|██████████████████ | 15699/110696 [4:46:36<23:35:46, 1.12it/s][2025-04-27 01:50:01] (step=0015700) Train Loss: 6.0872, Train Steps/Sec: 1.12 + 14%|██████████████████ | 15724/110696 [4:46:58<23:28:34, 1.12it/s][2025-04-27 01:50:23] (step=0015725) Train Loss: 6.0669, Train Steps/Sec: 1.12 + 14%|██████████████████ | 15749/110696 [4:47:20<23:23:17, 1.13it/s][2025-04-27 01:50:46] (step=0015750) Train Loss: 6.0921, Train Steps/Sec: 1.12 + 14%|██████████████████ | 15774/110696 [4:47:43<23:24:56, 1.13it/s][2025-04-27 01:51:08] (step=0015775) Train Loss: 5.9975, Train Steps/Sec: 1.12 + 14%|██████████████████▏ | 15799/110696 [4:48:05<23:19:19, 1.13it/s][2025-04-27 01:51:31] (step=0015800) Train Loss: 6.0806, Train Steps/Sec: 1.12 + 14%|██████████████████▏ | 15824/110696 [4:48:28<24:06:48, 1.09it/s][2025-04-27 01:51:53] (step=0015825) Train Loss: 6.0898, Train Steps/Sec: 1.12 + 14%|██████████████████▏ | 15849/110696 [4:48:50<23:44:04, 1.11it/s][2025-04-27 01:52:15] (step=0015850) Train Loss: 6.0789, Train Steps/Sec: 1.12 + 14%|██████████████████▏ | 15874/110696 [4:49:12<23:35:21, 1.12it/s][2025-04-27 01:52:38] (step=0015875) Train Loss: 6.0330, Train Steps/Sec: 1.12 + 14%|██████████████████▏ | 15899/110696 [4:49:35<23:28:06, 1.12it/s][2025-04-27 01:53:00] (step=0015900) Train Loss: 6.1404, Train Steps/Sec: 1.12 + 14%|██████████████████▎ | 15924/110696 [4:49:57<23:21:14, 1.13it/s][2025-04-27 01:53:22] (step=0015925) Train Loss: 6.1283, Train Steps/Sec: 1.12 + 14%|██████████████████▎ | 15949/110696 [4:50:19<23:24:57, 1.12it/s][2025-04-27 01:53:45] (step=0015950) Train Loss: 6.1368, Train Steps/Sec: 1.12 + 14%|██████████████████▎ | 15974/110696 [4:50:42<23:18:40, 1.13it/s][2025-04-27 01:54:07] (step=0015975) Train Loss: 6.1057, Train Steps/Sec: 1.12 + 14%|██████████████████▎ | 15999/110696 [4:51:04<23:22:45, 1.13it/s][2025-04-27 01:54:29] (step=0016000) Train Loss: 6.0916, Train Steps/Sec: 1.11 +You are using a model of type blip-2 to instantiate a model of type instructblip. This is not supported for all configurations of models and can yield errors. +[2025-04-27 01:54:29] text_config is None. Initializing the text config with default values (`OPTConfig`). +model_name_or_path:/tmp/haozhezhao/model/blip2-flan-t5-xl +image_place_holder: +tokenizer length after expend 32101 +tokenizer length before expend 32100 +100%|█████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████| 5/5 [04:56<00:00, 59.31s/it] +[2025-04-27 02:00:38] Finish Eval in 16000 steps...██████████████████████████████████████████████████████████████████████████████████████| 5/5 [04:56<00:00, 59.13s/it] +processing left-padding... +processing left-padding... +processing left-padding... +processing left-padding... +processing left-padding... +[2025-04-27 02:00:57] Saved checkpoint to checkpoint/X2I_UltraEdiit_fourmask_LLava_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_filtered_objects_4mask_1280_context/003-GPT-XL/checkpoints/0016000.pt +[2025-04-27 02:00:59] Removed old checkpoint: checkpoint/X2I_UltraEdiit_fourmask_LLava_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_filtered_objects_4mask_1280_context/003-GPT-XL/checkpoints/0014000.pt + 14%|██████████████████▍ | 16024/110696 [4:57:56<24:40:11, 1.07it/s][2025-04-27 02:01:21] (step=0016025) Train Loss: 6.1124, Train Steps/Sec: 0.06 + 14%|██████████████████▍ | 16049/110696 [4:58:18<23:44:00, 1.11it/s][2025-04-27 02:01:44] (step=0016050) Train Loss: 6.1041, Train Steps/Sec: 1.12 + 15%|██████████████████▍ | 16074/110696 [4:58:41<23:33:06, 1.12it/s][2025-04-27 02:02:06] (step=0016075) Train Loss: 6.0960, Train Steps/Sec: 1.12 + 15%|██████████████████▍ | 16099/110696 [4:59:03<23:29:11, 1.12it/s][2025-04-27 02:02:28] (step=0016100) Train Loss: 6.0806, Train Steps/Sec: 1.12 + 15%|██████████████████▍ | 16124/110696 [4:59:25<23:22:04, 1.12it/s][2025-04-27 02:02:51] (step=0016125) Train Loss: 6.0537, Train Steps/Sec: 1.12 + 15%|██████████████████▌ | 16149/110696 [4:59:48<23:20:02, 1.13it/s][2025-04-27 02:03:13] (step=0016150) Train Loss: 6.0664, Train Steps/Sec: 1.12 + 15%|██████████████████▌ | 16174/110696 [5:00:10<23:17:07, 1.13it/s][2025-04-27 02:03:35] (step=0016175) Train Loss: 6.0893, Train Steps/Sec: 1.12 + 15%|██████████████████▌ | 16199/110696 [5:00:32<23:17:04, 1.13it/s][2025-04-27 02:03:58] (step=0016200) Train Loss: 6.0300, Train Steps/Sec: 1.11 + 15%|██████████████████▌ | 16224/110696 [5:00:55<24:01:37, 1.09it/s][2025-04-27 02:04:20] (step=0016225) Train Loss: 6.0890, Train Steps/Sec: 1.12 + 15%|██████████████████▋ | 16249/110696 [5:01:17<23:36:02, 1.11it/s][2025-04-27 02:04:42] (step=0016250) Train Loss: 6.0990, Train Steps/Sec: 1.12 + 15%|██████████████████▋ | 16274/110696 [5:01:39<23:39:12, 1.11it/s][2025-04-27 02:05:05] (step=0016275) Train Loss: 6.1133, Train Steps/Sec: 1.12 + 15%|██████████████████▋ | 16299/110696 [5:02:02<23:27:40, 1.12it/s][2025-04-27 02:05:27] (step=0016300) Train Loss: 6.0539, Train Steps/Sec: 1.12 + 15%|██████████████████▋ | 16324/110696 [5:02:24<23:18:04, 1.13it/s][2025-04-27 02:05:50] (step=0016325) Train Loss: 6.1702, Train Steps/Sec: 1.12 + 15%|██████████████████▊ | 16349/110696 [5:02:47<23:16:51, 1.13it/s][2025-04-27 02:06:12] (step=0016350) Train Loss: 6.1367, Train Steps/Sec: 1.12 + 15%|██████████████████▊ | 16374/110696 [5:03:09<23:10:24, 1.13it/s][2025-04-27 02:06:34] (step=0016375) Train Loss: 6.0733, Train Steps/Sec: 1.12 + 15%|██████████████████▊ | 16399/110696 [5:03:31<23:07:34, 1.13it/s][2025-04-27 02:06:57] (step=0016400) Train Loss: 6.1358, Train Steps/Sec: 1.11 + 15%|██████████████████▊ | 16424/110696 [5:03:54<23:52:55, 1.10it/s][2025-04-27 02:07:19] (step=0016425) Train Loss: 6.0781, Train Steps/Sec: 1.12 + 15%|██████████████████▊ | 16449/110696 [5:04:16<23:40:42, 1.11it/s][2025-04-27 02:07:41] (step=0016450) Train Loss: 6.1309, Train Steps/Sec: 1.12 + 15%|██████████████████▉ | 16474/110696 [5:04:38<23:29:47, 1.11it/s][2025-04-27 02:08:04] (step=0016475) Train Loss: 6.1061, Train Steps/Sec: 1.12 + 15%|██████████████████▉ | 16499/110696 [5:05:01<23:14:09, 1.13it/s][2025-04-27 02:08:26] (step=0016500) Train Loss: 6.1064, Train Steps/Sec: 1.12 + 15%|██████████████████▉ | 16524/110696 [5:05:23<23:15:53, 1.12it/s][2025-04-27 02:08:48] (step=0016525) Train Loss: 6.1254, Train Steps/Sec: 1.12 + 15%|██████████████████▉ | 16549/110696 [5:05:45<23:13:31, 1.13it/s][2025-04-27 02:09:11] (step=0016550) Train Loss: 6.0575, Train Steps/Sec: 1.12 + 15%|███████████████████ | 16574/110696 [5:06:08<23:12:13, 1.13it/s][2025-04-27 02:09:33] (step=0016575) Train Loss: 6.0622, Train Steps/Sec: 1.12 + 15%|███████████████████ | 16599/110696 [5:06:30<23:06:04, 1.13it/s][2025-04-27 02:09:56] (step=0016600) Train Loss: 6.0466, Train Steps/Sec: 1.11 + 15%|███████████████████ | 16624/110696 [5:06:53<23:52:12, 1.09it/s][2025-04-27 02:10:18] (step=0016625) Train Loss: 6.1212, Train Steps/Sec: 1.12 + 15%|███████████████████ | 16649/110696 [5:07:15<23:41:53, 1.10it/s][2025-04-27 02:10:40] (step=0016650) Train Loss: 6.0926, Train Steps/Sec: 1.12 + 15%|███████████████████▏ | 16674/110696 [5:07:37<23:23:54, 1.12it/s][2025-04-27 02:11:03] (step=0016675) Train Loss: 6.0318, Train Steps/Sec: 1.12 + 15%|███████████████████▏ | 16699/110696 [5:08:00<23:15:31, 1.12it/s][2025-04-27 02:11:25] (step=0016700) Train Loss: 6.0590, Train Steps/Sec: 1.12 + 15%|███████████████████▏ | 16724/110696 [5:08:22<23:07:53, 1.13it/s][2025-04-27 02:11:47] (step=0016725) Train Loss: 6.1127, Train Steps/Sec: 1.12 + 15%|███████████████████▏ | 16749/110696 [5:08:44<23:11:20, 1.13it/s][2025-04-27 02:12:10] (step=0016750) Train Loss: 6.0631, Train Steps/Sec: 1.12 + 15%|███████████████████▏ | 16774/110696 [5:09:07<23:07:51, 1.13it/s][2025-04-27 02:12:32] (step=0016775) Train Loss: 6.0829, Train Steps/Sec: 1.12 + 15%|███████████████████▎ | 16799/110696 [5:09:29<23:04:54, 1.13it/s][2025-04-27 02:12:54] (step=0016800) Train Loss: 6.0432, Train Steps/Sec: 1.11 + 15%|███████████████████▎ | 16824/110696 [5:09:51<23:49:18, 1.09it/s][2025-04-27 02:13:17] (step=0016825) Train Loss: 6.0650, Train Steps/Sec: 1.12 + 15%|███████████████████▎ | 16849/110696 [5:10:14<23:22:46, 1.12it/s][2025-04-27 02:13:39] (step=0016850) Train Loss: 6.0700, Train Steps/Sec: 1.12 + 15%|███████████████████▎ | 16874/110696 [5:10:36<23:25:14, 1.11it/s][2025-04-27 02:14:02] (step=0016875) Train Loss: 6.0963, Train Steps/Sec: 1.12 + 15%|███████████████████▍ | 16899/110696 [5:10:59<23:30:42, 1.11it/s][2025-04-27 02:14:24] (step=0016900) Train Loss: 6.1048, Train Steps/Sec: 1.12 + 15%|███████████████████▍ | 16924/110696 [5:11:21<23:12:39, 1.12it/s][2025-04-27 02:14:46] (step=0016925) Train Loss: 6.1113, Train Steps/Sec: 1.12 + 15%|███████████████████▍ | 16949/110696 [5:11:43<23:13:59, 1.12it/s][2025-04-27 02:15:09] (step=0016950) Train Loss: 6.0995, Train Steps/Sec: 1.12 + 15%|███████████████████▍ | 16974/110696 [5:12:06<23:04:22, 1.13it/s][2025-04-27 02:15:31] (step=0016975) Train Loss: 6.0022, Train Steps/Sec: 1.12 + 15%|███████████████████▌ | 16999/110696 [5:12:28<22:59:45, 1.13it/s][2025-04-27 02:15:53] (step=0017000) Train Loss: 6.1580, Train Steps/Sec: 1.11 + 15%|███████████████████▌ | 17024/110696 [5:12:50<23:46:26, 1.09it/s][2025-04-27 02:16:16] (step=0017025) Train Loss: 6.1035, Train Steps/Sec: 1.12 + 15%|███████████████████▌ | 17049/110696 [5:13:13<23:29:29, 1.11it/s][2025-04-27 02:16:38] (step=0017050) Train Loss: 6.1473, Train Steps/Sec: 1.12 + 15%|███████████████████▌ | 17074/110696 [5:13:35<23:17:19, 1.12it/s][2025-04-27 02:17:00] (step=0017075) Train Loss: 6.0496, Train Steps/Sec: 1.12 + 15%|███████████████████▌ | 17099/110696 [5:13:58<23:17:56, 1.12it/s][2025-04-27 02:17:23] (step=0017100) Train Loss: 6.0831, Train Steps/Sec: 1.12 + 15%|███████████████████▋ | 17124/110696 [5:14:20<23:06:11, 1.13it/s][2025-04-27 02:17:45] (step=0017125) Train Loss: 6.1341, Train Steps/Sec: 1.12 + 15%|███████████████████▋ | 17149/110696 [5:14:42<23:10:43, 1.12it/s][2025-04-27 02:18:08] (step=0017150) Train Loss: 6.0335, Train Steps/Sec: 1.12 + 16%|███████████████████▋ | 17174/110696 [5:15:05<23:02:23, 1.13it/s][2025-04-27 02:18:30] (step=0017175) Train Loss: 6.0124, Train Steps/Sec: 1.12 + 16%|███████████████████▋ | 17199/110696 [5:15:27<23:03:18, 1.13it/s][2025-04-27 02:18:52] (step=0017200) Train Loss: 6.1200, Train Steps/Sec: 1.11 + 16%|███████████████████▊ | 17224/110696 [5:15:49<23:46:56, 1.09it/s][2025-04-27 02:19:15] (step=0017225) Train Loss: 6.0233, Train Steps/Sec: 1.12 + 16%|███████████████████▊ | 17249/110696 [5:16:12<23:23:45, 1.11it/s][2025-04-27 02:19:37] (step=0017250) Train Loss: 6.0627, Train Steps/Sec: 1.12 + 16%|███████████████████▊ | 17274/110696 [5:16:34<23:19:01, 1.11it/s][2025-04-27 02:19:59] (step=0017275) Train Loss: 6.0995, Train Steps/Sec: 1.12 + 16%|███████████████████▊ | 17299/110696 [5:16:56<23:10:25, 1.12it/s][2025-04-27 02:20:22] (step=0017300) Train Loss: 6.0261, Train Steps/Sec: 1.12 + 16%|███████████████████▉ | 17324/110696 [5:17:19<23:02:29, 1.13it/s][2025-04-27 02:20:44] (step=0017325) Train Loss: 6.0161, Train Steps/Sec: 1.12 + 16%|███████████████████▉ | 17349/110696 [5:17:41<23:02:27, 1.13it/s][2025-04-27 02:21:06] (step=0017350) Train Loss: 6.0406, Train Steps/Sec: 1.12 + 16%|███████████████████▉ | 17374/110696 [5:18:03<23:00:30, 1.13it/s][2025-04-27 02:21:29] (step=0017375) Train Loss: 6.0063, Train Steps/Sec: 1.12 + 16%|███████████████████▉ | 17399/110696 [5:18:26<23:02:09, 1.13it/s][2025-04-27 02:21:51] (step=0017400) Train Loss: 6.0467, Train Steps/Sec: 1.11 + 16%|███████████████████▉ | 17424/110696 [5:18:48<23:44:13, 1.09it/s][2025-04-27 02:22:14] (step=0017425) Train Loss: 6.0926, Train Steps/Sec: 1.12 + 16%|████████████████████ | 17449/110696 [5:19:11<23:21:48, 1.11it/s][2025-04-27 02:22:36] (step=0017450) Train Loss: 6.0556, Train Steps/Sec: 1.12 + 16%|████████████████████ | 17474/110696 [5:19:33<23:12:02, 1.12it/s][2025-04-27 02:22:58] (step=0017475) Train Loss: 5.9977, Train Steps/Sec: 1.12 + 16%|████████████████████ | 17499/110696 [5:19:55<23:04:24, 1.12it/s][2025-04-27 02:23:21] (step=0017500) Train Loss: 6.0642, Train Steps/Sec: 1.12 + 16%|████████████████████ | 17524/110696 [5:20:18<23:02:26, 1.12it/s][2025-04-27 02:23:43] (step=0017525) Train Loss: 6.0977, Train Steps/Sec: 1.12 + 16%|████████████████████▏ | 17549/110696 [5:20:40<22:57:34, 1.13it/s][2025-04-27 02:24:05] (step=0017550) Train Loss: 6.0742, Train Steps/Sec: 1.12 + 16%|████████████████████▏ | 17574/110696 [5:21:02<22:52:22, 1.13it/s][2025-04-27 02:24:28] (step=0017575) Train Loss: 6.0824, Train Steps/Sec: 1.12 + 16%|████████████████████▏ | 17599/110696 [5:21:25<22:54:49, 1.13it/s][2025-04-27 02:24:50] (step=0017600) Train Loss: 6.0264, Train Steps/Sec: 1.11 + 16%|████████████████████▏ | 17624/110696 [5:21:47<23:36:37, 1.09it/s][2025-04-27 02:25:13] (step=0017625) Train Loss: 6.0727, Train Steps/Sec: 1.12 + 16%|████████████████████▏ | 17649/110696 [5:22:10<23:16:56, 1.11it/s][2025-04-27 02:25:35] (step=0017650) Train Loss: 6.0556, Train Steps/Sec: 1.12 + 16%|████████████████████▎ | 17674/110696 [5:22:32<23:10:32, 1.11it/s][2025-04-27 02:25:57] (step=0017675) Train Loss: 5.9822, Train Steps/Sec: 1.12 + 16%|████████████████████▎ | 17699/110696 [5:22:54<23:03:44, 1.12it/s][2025-04-27 02:26:20] (step=0017700) Train Loss: 6.0432, Train Steps/Sec: 1.12 + 16%|████████████████████▎ | 17724/110696 [5:23:17<22:57:34, 1.12it/s][2025-04-27 02:26:42] (step=0017725) Train Loss: 6.0821, Train Steps/Sec: 1.12 + 16%|████████████████████▎ | 17749/110696 [5:23:39<22:54:25, 1.13it/s][2025-04-27 02:27:04] (step=0017750) Train Loss: 6.0582, Train Steps/Sec: 1.12 + 16%|████████████████████▍ | 17774/110696 [5:24:01<22:54:28, 1.13it/s][2025-04-27 02:27:27] (step=0017775) Train Loss: 6.0812, Train Steps/Sec: 1.12 + 16%|████████████████████▍ | 17799/110696 [5:24:24<22:53:47, 1.13it/s][2025-04-27 02:27:49] (step=0017800) Train Loss: 6.0760, Train Steps/Sec: 1.11 + 16%|████████████████████▍ | 17824/110696 [5:24:46<23:34:41, 1.09it/s][2025-04-27 02:28:12] (step=0017825) Train Loss: 6.0625, Train Steps/Sec: 1.12 + 16%|████████████████████▍ | 17849/110696 [5:25:09<23:15:22, 1.11it/s][2025-04-27 02:28:34] (step=0017850) Train Loss: 6.0301, Train Steps/Sec: 1.12 + 16%|████████████████████▌ | 17874/110696 [5:25:31<23:07:38, 1.11it/s][2025-04-27 02:28:56] (step=0017875) Train Loss: 6.0714, Train Steps/Sec: 1.12 + 16%|████████████████████▌ | 17899/110696 [5:25:53<23:02:51, 1.12it/s][2025-04-27 02:29:19] (step=0017900) Train Loss: 6.0525, Train Steps/Sec: 1.12 + 16%|████████████████████▌ | 17924/110696 [5:26:16<22:51:13, 1.13it/s][2025-04-27 02:29:41] (step=0017925) Train Loss: 6.0116, Train Steps/Sec: 1.12 + 16%|████████████████████▌ | 17949/110696 [5:26:38<22:51:57, 1.13it/s][2025-04-27 02:30:03] (step=0017950) Train Loss: 6.0398, Train Steps/Sec: 1.12 + 16%|████████████████████▌ | 17974/110696 [5:27:00<22:47:20, 1.13it/s][2025-04-27 02:30:26] (step=0017975) Train Loss: 6.0584, Train Steps/Sec: 1.12 + 16%|████████████████████▋ | 17999/110696 [5:27:23<22:46:45, 1.13it/s][2025-04-27 02:30:48] (step=0018000) Train Loss: 6.0731, Train Steps/Sec: 1.11 +You are using a model of type blip-2 to instantiate a model of type instructblip. This is not supported for all configurations of models and can yield errors. +[2025-04-27 02:30:48] text_config is None. Initializing the text config with default values (`OPTConfig`). +model_name_or_path:/tmp/haozhezhao/model/blip2-flan-t5-xl +image_place_holder: +tokenizer length after expend 32101 +tokenizer length before expend 32100 +100%|█████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████| 5/5 [04:57<00:00, 59.51s/it] +[2025-04-27 02:36:57] Finish Eval in 18000 steps...██████████████████████████████████████████████████████████████████████████████████████| 5/5 [04:57<00:00, 59.22s/it] +processing left-padding... +processing left-padding... +processing left-padding... +processing left-padding... +processing left-padding... +[2025-04-27 02:37:16] Saved checkpoint to checkpoint/X2I_UltraEdiit_fourmask_LLava_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_filtered_objects_4mask_1280_context/003-GPT-XL/checkpoints/0018000.pt +[2025-04-27 02:37:19] Removed old checkpoint: checkpoint/X2I_UltraEdiit_fourmask_LLava_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_filtered_objects_4mask_1280_context/003-GPT-XL/checkpoints/0016000.pt + 16%|████████████████████▋ | 18024/110696 [5:34:16<24:05:08, 1.07it/s][2025-04-27 02:37:41] (step=0018025) Train Loss: 6.0224, Train Steps/Sec: 0.06 + 16%|████████████████████▋ | 18049/110696 [5:34:38<23:10:43, 1.11it/s][2025-04-27 02:38:03] (step=0018050) Train Loss: 6.0157, Train Steps/Sec: 1.12 + 16%|████████████████████▋ | 18074/110696 [5:35:00<23:04:06, 1.12it/s][2025-04-27 02:38:26] (step=0018075) Train Loss: 6.0786, Train Steps/Sec: 1.12 + 16%|████████████████████▊ | 18099/110696 [5:35:23<22:53:15, 1.12it/s][2025-04-27 02:38:48] (step=0018100) Train Loss: 6.0685, Train Steps/Sec: 1.12 + 16%|████████████████████▊ | 18124/110696 [5:35:45<22:50:31, 1.13it/s][2025-04-27 02:39:10] (step=0018125) Train Loss: 6.1247, Train Steps/Sec: 1.12 + 16%|████████████████████▊ | 18149/110696 [5:36:07<22:51:18, 1.12it/s][2025-04-27 02:39:33] (step=0018150) Train Loss: 6.0746, Train Steps/Sec: 1.12 + 16%|████████████████████▊ | 18174/110696 [5:36:30<22:46:15, 1.13it/s][2025-04-27 02:39:55] (step=0018175) Train Loss: 6.0298, Train Steps/Sec: 1.12 + 16%|████████████████████▉ | 18199/110696 [5:36:52<22:41:36, 1.13it/s][2025-04-27 02:40:17] (step=0018200) Train Loss: 6.0326, Train Steps/Sec: 1.12 + 16%|████████████████████▉ | 18224/110696 [5:37:14<23:22:18, 1.10it/s][2025-04-27 02:40:40] (step=0018225) Train Loss: 6.1006, Train Steps/Sec: 1.12 + 16%|████████████████████▉ | 18249/110696 [5:37:37<23:11:13, 1.11it/s][2025-04-27 02:41:02] (step=0018250) Train Loss: 6.0143, Train Steps/Sec: 1.12 + 17%|████████████████████▉ | 18274/110696 [5:37:59<23:01:17, 1.12it/s][2025-04-27 02:41:24] (step=0018275) Train Loss: 6.0168, Train Steps/Sec: 1.12 + 17%|████████████████████▉ | 18299/110696 [5:38:21<22:50:30, 1.12it/s][2025-04-27 02:41:47] (step=0018300) Train Loss: 6.0495, Train Steps/Sec: 1.12 + 17%|█████████████████████ | 18324/110696 [5:38:43<22:45:48, 1.13it/s][2025-04-27 02:42:09] (step=0018325) Train Loss: 6.1018, Train Steps/Sec: 1.12 + 17%|█████████████████████ | 18349/110696 [5:39:06<22:50:26, 1.12it/s][2025-04-27 02:42:31] (step=0018350) Train Loss: 6.0922, Train Steps/Sec: 1.12 + 17%|█████████████████████ | 18374/110696 [5:39:28<22:43:11, 1.13it/s][2025-04-27 02:42:54] (step=0018375) Train Loss: 6.0285, Train Steps/Sec: 1.12 + 17%|█████████████████████ | 18399/110696 [5:39:51<22:42:15, 1.13it/s][2025-04-27 02:43:16] (step=0018400) Train Loss: 6.0540, Train Steps/Sec: 1.12 + 17%|█████████████████████▏ | 18424/110696 [5:40:13<23:20:45, 1.10it/s][2025-04-27 02:43:38] (step=0018425) Train Loss: 6.0862, Train Steps/Sec: 1.12 + 17%|█████████████████████▏ | 18449/110696 [5:40:35<23:06:10, 1.11it/s][2025-04-27 02:44:01] (step=0018450) Train Loss: 6.0698, Train Steps/Sec: 1.12 + 17%|█████████████████████▏ | 18474/110696 [5:40:58<22:50:19, 1.12it/s][2025-04-27 02:44:23] (step=0018475) Train Loss: 6.0122, Train Steps/Sec: 1.12 + 17%|█████████████████████▏ | 18499/110696 [5:41:20<22:49:45, 1.12it/s][2025-04-27 02:44:45] (step=0018500) Train Loss: 6.0453, Train Steps/Sec: 1.12 + 17%|█████████████████████▎ | 18524/110696 [5:41:42<22:50:07, 1.12it/s][2025-04-27 02:45:08] (step=0018525) Train Loss: 6.0491, Train Steps/Sec: 1.12 + 17%|█████████████████████▎ | 18549/110696 [5:42:05<22:44:45, 1.13it/s][2025-04-27 02:45:30] (step=0018550) Train Loss: 6.0260, Train Steps/Sec: 1.12 + 17%|█████████████████████▎ | 18574/110696 [5:42:27<22:38:56, 1.13it/s][2025-04-27 02:45:52] (step=0018575) Train Loss: 6.0623, Train Steps/Sec: 1.12 + 17%|█████████████████████▎ | 18599/110696 [5:42:49<22:34:54, 1.13it/s][2025-04-27 02:46:15] (step=0018600) Train Loss: 6.0388, Train Steps/Sec: 1.11 + 17%|█████████████████████▎ | 18624/110696 [5:43:12<23:18:41, 1.10it/s][2025-04-27 02:46:37] (step=0018625) Train Loss: 6.0774, Train Steps/Sec: 1.12 + 17%|█████████████████████▍ | 18649/110696 [5:43:34<22:57:43, 1.11it/s][2025-04-27 02:46:59] (step=0018650) Train Loss: 6.0422, Train Steps/Sec: 1.12 + 17%|█████████████████████▍ | 18674/110696 [5:43:56<22:54:54, 1.12it/s][2025-04-27 02:47:22] (step=0018675) Train Loss: 6.0103, Train Steps/Sec: 1.12 + 17%|█████████████████████▍ | 18699/110696 [5:44:19<22:44:51, 1.12it/s][2025-04-27 02:47:44] (step=0018700) Train Loss: 5.9635, Train Steps/Sec: 1.12 + 17%|█████████████████████▍ | 18724/110696 [5:44:41<22:38:07, 1.13it/s][2025-04-27 02:48:06] (step=0018725) Train Loss: 6.1013, Train Steps/Sec: 1.12 + 17%|█████████████████████▌ | 18749/110696 [5:45:03<22:39:59, 1.13it/s][2025-04-27 02:48:29] (step=0018750) Train Loss: 6.0372, Train Steps/Sec: 1.12 + 17%|█████████████████████▌ | 18774/110696 [5:45:26<22:34:44, 1.13it/s][2025-04-27 02:48:51] (step=0018775) Train Loss: 6.0179, Train Steps/Sec: 1.12 + 17%|█████████████████████▌ | 18799/110696 [5:45:48<22:32:05, 1.13it/s][2025-04-27 02:49:14] (step=0018800) Train Loss: 6.0566, Train Steps/Sec: 1.12 + 17%|█████████████████████▌ | 18824/110696 [5:46:10<23:17:01, 1.10it/s][2025-04-27 02:49:36] (step=0018825) Train Loss: 5.9799, Train Steps/Sec: 1.12 + 17%|█████████████████████▋ | 18849/110696 [5:46:33<23:02:17, 1.11it/s][2025-04-27 02:49:58] (step=0018850) Train Loss: 6.0722, Train Steps/Sec: 1.12 + 17%|█████████████████████▋ | 18874/110696 [5:46:55<22:50:45, 1.12it/s][2025-04-27 02:50:21] (step=0018875) Train Loss: 6.0879, Train Steps/Sec: 1.12 + 17%|█████████████████████▋ | 18899/110696 [5:47:17<22:47:20, 1.12it/s][2025-04-27 02:50:43] (step=0018900) Train Loss: 5.9990, Train Steps/Sec: 1.12 + 17%|█████████████████████▋ | 18924/110696 [5:47:40<22:38:44, 1.13it/s][2025-04-27 02:51:05] (step=0018925) Train Loss: 6.0497, Train Steps/Sec: 1.12 + 17%|█████████████████████▋ | 18949/110696 [5:48:02<22:33:14, 1.13it/s][2025-04-27 02:51:27] (step=0018950) Train Loss: 6.0621, Train Steps/Sec: 1.12 + 17%|█████████████████████▊ | 18974/110696 [5:48:24<22:33:18, 1.13it/s][2025-04-27 02:51:50] (step=0018975) Train Loss: 6.0909, Train Steps/Sec: 1.12 + 17%|█████████████████████▊ | 18999/110696 [5:48:47<22:28:59, 1.13it/s][2025-04-27 02:52:12] (step=0019000) Train Loss: 6.0431, Train Steps/Sec: 1.12 + 17%|█████████████████████▊ | 19024/110696 [5:49:14<23:23:13, 1.09it/s][2025-04-27 02:52:40] (step=0019025) Train Loss: 6.0058, Train Steps/Sec: 0.91 + 17%|█████████████████████▊ | 19049/110696 [5:49:37<22:56:53, 1.11it/s][2025-04-27 02:53:02] (step=0019050) Train Loss: 6.0589, Train Steps/Sec: 1.12 + 17%|█████████████████████▉ | 19074/110696 [5:49:59<22:48:51, 1.12it/s][2025-04-27 02:53:24] (step=0019075) Train Loss: 6.0470, Train Steps/Sec: 1.12 + 17%|█████████████████████▉ | 19099/110696 [5:50:21<22:39:25, 1.12it/s][2025-04-27 02:53:47] (step=0019100) Train Loss: 6.0424, Train Steps/Sec: 1.12 + 17%|█████████████████████▉ | 19124/110696 [5:50:43<22:35:57, 1.13it/s][2025-04-27 02:54:09] (step=0019125) Train Loss: 6.0258, Train Steps/Sec: 1.12 + 17%|█████████████████████▉ | 19149/110696 [5:51:06<22:29:27, 1.13it/s][2025-04-27 02:54:31] (step=0019150) Train Loss: 6.0617, Train Steps/Sec: 1.12 + 17%|█████████████████████▉ | 19174/110696 [5:51:28<22:25:23, 1.13it/s][2025-04-27 02:54:53] (step=0019175) Train Loss: 6.0232, Train Steps/Sec: 1.12 + 17%|██████████████████████ | 19199/110696 [5:51:50<22:31:01, 1.13it/s][2025-04-27 02:55:16] (step=0019200) Train Loss: 6.0788, Train Steps/Sec: 1.12 + 17%|██████████████████████ | 19224/110696 [5:52:13<23:16:10, 1.09it/s][2025-04-27 02:55:38] (step=0019225) Train Loss: 6.0446, Train Steps/Sec: 1.12 + 17%|██████████████████████ | 19249/110696 [5:52:35<22:54:55, 1.11it/s][2025-04-27 02:56:01] (step=0019250) Train Loss: 6.0622, Train Steps/Sec: 1.12 + 17%|██████████████████████ | 19274/110696 [5:52:58<22:42:50, 1.12it/s][2025-04-27 02:56:23] (step=0019275) Train Loss: 6.0546, Train Steps/Sec: 1.12 + 17%|██████████████████████▏ | 19299/110696 [5:53:20<22:39:53, 1.12it/s][2025-04-27 02:56:45] (step=0019300) Train Loss: 6.0101, Train Steps/Sec: 1.12 + 17%|██████████████████████▏ | 19324/110696 [5:53:42<22:33:16, 1.13it/s][2025-04-27 02:57:08] (step=0019325) Train Loss: 5.9827, Train Steps/Sec: 1.12 + 17%|██████████████████████▏ | 19349/110696 [5:54:05<22:27:35, 1.13it/s][2025-04-27 02:57:30] (step=0019350) Train Loss: 6.0317, Train Steps/Sec: 1.12 + 18%|██████████████████████▏ | 19374/110696 [5:54:27<22:25:54, 1.13it/s][2025-04-27 02:57:52] (step=0019375) Train Loss: 6.0059, Train Steps/Sec: 1.12 + 18%|██████████████████████▎ | 19399/110696 [5:54:49<22:25:20, 1.13it/s][2025-04-27 02:58:15] (step=0019400) Train Loss: 6.0662, Train Steps/Sec: 1.12 + 18%|██████████████████████▎ | 19424/110696 [5:55:12<23:13:41, 1.09it/s][2025-04-27 02:58:37] (step=0019425) Train Loss: 5.9902, Train Steps/Sec: 1.12 + 18%|██████████████████████▎ | 19449/110696 [5:55:34<22:47:53, 1.11it/s][2025-04-27 02:58:59] (step=0019450) Train Loss: 6.0413, Train Steps/Sec: 1.12 + 18%|██████████████████████▎ | 19474/110696 [5:55:56<22:43:11, 1.12it/s][2025-04-27 02:59:22] (step=0019475) Train Loss: 6.1079, Train Steps/Sec: 1.12 + 18%|██████████████████████▎ | 19499/110696 [5:56:19<22:38:10, 1.12it/s][2025-04-27 02:59:44] (step=0019500) Train Loss: 5.9832, Train Steps/Sec: 1.12 + 18%|██████████████████████▍ | 19524/110696 [5:56:41<22:33:04, 1.12it/s][2025-04-27 03:00:06] (step=0019525) Train Loss: 5.9543, Train Steps/Sec: 1.12 + 18%|██████████████████████▍ | 19549/110696 [5:57:03<22:23:54, 1.13it/s][2025-04-27 03:00:29] (step=0019550) Train Loss: 6.0190, Train Steps/Sec: 1.12 + 18%|██████████████████████▍ | 19574/110696 [5:57:26<22:29:11, 1.13it/s][2025-04-27 03:00:51] (step=0019575) Train Loss: 6.0108, Train Steps/Sec: 1.12 + 18%|██████████████████████▍ | 19599/110696 [5:57:48<22:20:01, 1.13it/s][2025-04-27 03:01:13] (step=0019600) Train Loss: 6.0698, Train Steps/Sec: 1.12 + 18%|██████████████████████▌ | 19624/110696 [5:58:10<23:08:48, 1.09it/s][2025-04-27 03:01:36] (step=0019625) Train Loss: 6.0659, Train Steps/Sec: 1.12 + 18%|██████████████████████▌ | 19649/110696 [5:58:33<22:51:48, 1.11it/s][2025-04-27 03:01:58] (step=0019650) Train Loss: 6.0727, Train Steps/Sec: 1.12 + 18%|██████████████████████▌ | 19674/110696 [5:58:55<22:41:51, 1.11it/s][2025-04-27 03:02:20] (step=0019675) Train Loss: 6.0446, Train Steps/Sec: 1.12 + 18%|██████████████████████▌ | 19699/110696 [5:59:27<57:22:06, 2.27s/it][2025-04-27 03:02:53] (step=0019700) Train Loss: 6.0191, Train Steps/Sec: 0.77 + 18%|██████████████████████▋ | 19724/110696 [6:00:00<24:53:27, 1.02it/s][2025-04-27 03:03:25] (step=0019725) Train Loss: 6.0261, Train Steps/Sec: 0.77 + 18%|██████████████████████▋ | 19749/110696 [6:00:32<24:32:52, 1.03it/s][2025-04-27 03:03:57] (step=0019750) Train Loss: 6.0444, Train Steps/Sec: 0.78 + 18%|██████████████████████▋ | 19774/110696 [6:00:54<22:21:59, 1.13it/s][2025-04-27 03:04:19] (step=0019775) Train Loss: 6.0021, Train Steps/Sec: 1.12 + 18%|██████████████████████▋ | 19799/110696 [6:01:16<22:28:18, 1.12it/s][2025-04-27 03:04:42] (step=0019800) Train Loss: 6.0507, Train Steps/Sec: 1.11 + 18%|██████████████████████▋ | 19824/110696 [6:01:39<22:58:12, 1.10it/s][2025-04-27 03:05:04] (step=0019825) Train Loss: 6.0419, Train Steps/Sec: 1.12 + 18%|██████████████████████▊ | 19849/110696 [6:02:01<22:42:28, 1.11it/s][2025-04-27 03:05:27] (step=0019850) Train Loss: 5.9829, Train Steps/Sec: 1.12 + 18%|██████████████████████▊ | 19874/110696 [6:02:24<22:43:51, 1.11it/s][2025-04-27 03:05:49] (step=0019875) Train Loss: 5.9921, Train Steps/Sec: 1.12 + 18%|██████████████████████▊ | 19899/110696 [6:02:46<22:24:29, 1.13it/s][2025-04-27 03:06:11] (step=0019900) Train Loss: 6.0617, Train Steps/Sec: 1.12 + 18%|██████████████████████▊ | 19924/110696 [6:03:08<22:22:50, 1.13it/s][2025-04-27 03:06:34] (step=0019925) Train Loss: 6.0471, Train Steps/Sec: 1.12 + 18%|██████████████████████▉ | 19949/110696 [6:03:31<22:23:22, 1.13it/s][2025-04-27 03:06:56] (step=0019950) Train Loss: 5.9850, Train Steps/Sec: 1.12 + 18%|██████████████████████▉ | 19974/110696 [6:03:53<22:21:40, 1.13it/s][2025-04-27 03:07:18] (step=0019975) Train Loss: 6.1023, Train Steps/Sec: 1.12 + 18%|██████████████████████▉ | 19999/110696 [6:04:15<22:19:06, 1.13it/s][2025-04-27 03:07:41] (step=0020000) Train Loss: 6.0145, Train Steps/Sec: 1.12 +You are using a model of type blip-2 to instantiate a model of type instructblip. This is not supported for all configurations of models and can yield errors. +[2025-04-27 03:07:41] text_config is None. Initializing the text config with default values (`OPTConfig`). +model_name_or_path:/tmp/haozhezhao/model/blip2-flan-t5-xl +image_place_holder: +tokenizer length after expend 32101 +tokenizer length before expend 32100 +100%|█████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████| 5/5 [04:56<00:00, 59.24s/it] +[2025-04-27 03:13:48] Finish Eval in 20000 steps...██████████████████████████████████████████████████████████████████████████████████████| 5/5 [04:55<00:00, 59.05s/it] +processing left-padding... +processing left-padding... +processing left-padding... +processing left-padding... +processing left-padding... +[2025-04-27 03:14:08] Saved checkpoint to checkpoint/X2I_UltraEdiit_fourmask_LLava_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_filtered_objects_4mask_1280_context/003-GPT-XL/checkpoints/0020000.pt +[2025-04-27 03:14:10] Removed old checkpoint: checkpoint/X2I_UltraEdiit_fourmask_LLava_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_filtered_objects_4mask_1280_context/003-GPT-XL/checkpoints/0018000.pt + 18%|██████████████████████▉ | 20024/110696 [6:11:07<23:30:48, 1.07it/s][2025-04-27 03:14:32] (step=0020025) Train Loss: 6.0319, Train Steps/Sec: 0.06 + 18%|███████████████████████ | 20049/110696 [6:11:29<22:42:11, 1.11it/s][2025-04-27 03:14:54] (step=0020050) Train Loss: 5.9930, Train Steps/Sec: 1.12 + 18%|███████████████████████ | 20074/110696 [6:11:51<22:33:53, 1.12it/s][2025-04-27 03:15:17] (step=0020075) Train Loss: 6.0231, Train Steps/Sec: 1.12 + 18%|███████████████████████ | 20099/110696 [6:12:14<22:22:39, 1.12it/s][2025-04-27 03:15:39] (step=0020100) Train Loss: 5.9838, Train Steps/Sec: 1.12 + 18%|███████████████████████ | 20124/110696 [6:12:36<22:18:23, 1.13it/s][2025-04-27 03:16:01] (step=0020125) Train Loss: 6.0056, Train Steps/Sec: 1.12 + 18%|███████████████████████ | 20149/110696 [6:12:58<22:18:17, 1.13it/s][2025-04-27 03:16:24] (step=0020150) Train Loss: 6.0620, Train Steps/Sec: 1.12 + 18%|███████████████████████▏ | 20174/110696 [6:13:21<22:09:09, 1.14it/s][2025-04-27 03:16:46] (step=0020175) Train Loss: 5.9766, Train Steps/Sec: 1.12 + 18%|███████████████████████▏ | 20199/110696 [6:13:43<22:17:01, 1.13it/s][2025-04-27 03:17:08] (step=0020200) Train Loss: 6.0516, Train Steps/Sec: 1.12 + 18%|███████████████████████▏ | 20224/110696 [6:14:05<22:52:40, 1.10it/s][2025-04-27 03:17:31] (step=0020225) Train Loss: 6.0437, Train Steps/Sec: 1.12 + 18%|███████████████████████▏ | 20249/110696 [6:14:28<22:40:48, 1.11it/s][2025-04-27 03:17:53] (step=0020250) Train Loss: 6.0522, Train Steps/Sec: 1.12 + 18%|███████████████████████▎ | 20274/110696 [6:14:50<22:26:51, 1.12it/s][2025-04-27 03:18:15] (step=0020275) Train Loss: 6.0215, Train Steps/Sec: 1.12 + 18%|███████████████████████▎ | 20299/110696 [6:15:12<22:25:09, 1.12it/s][2025-04-27 03:18:38] (step=0020300) Train Loss: 6.0014, Train Steps/Sec: 1.12 + 18%|███████████████████████▎ | 20324/110696 [6:15:34<22:16:50, 1.13it/s][2025-04-27 03:19:00] (step=0020325) Train Loss: 5.9982, Train Steps/Sec: 1.12 + 18%|███████████████████████▎ | 20349/110696 [6:15:57<22:14:20, 1.13it/s][2025-04-27 03:19:22] (step=0020350) Train Loss: 5.9689, Train Steps/Sec: 1.12 + 18%|███████████████████████▎ | 20374/110696 [6:16:19<22:12:18, 1.13it/s][2025-04-27 03:19:45] (step=0020375) Train Loss: 6.0009, Train Steps/Sec: 1.12 + 18%|███████████████████████▍ | 20399/110696 [6:16:41<22:10:21, 1.13it/s][2025-04-27 03:20:07] (step=0020400) Train Loss: 6.0356, Train Steps/Sec: 1.12 + 18%|███████████████████████▍ | 20424/110696 [6:17:04<22:59:21, 1.09it/s][2025-04-27 03:20:29] (step=0020425) Train Loss: 6.0862, Train Steps/Sec: 1.12 + 18%|███████████████████████▍ | 20449/110696 [6:17:26<22:38:50, 1.11it/s][2025-04-27 03:20:52] (step=0020450) Train Loss: 6.0228, Train Steps/Sec: 1.12 + 18%|███████████████████████▍ | 20474/110696 [6:17:49<22:28:20, 1.12it/s][2025-04-27 03:21:14] (step=0020475) Train Loss: 5.9909, Train Steps/Sec: 1.11 + 19%|███████████████████████▌ | 20499/110696 [6:18:11<22:19:41, 1.12it/s][2025-04-27 03:21:36] (step=0020500) Train Loss: 5.9842, Train Steps/Sec: 1.12 + 19%|███████████████████████▌ | 20524/110696 [6:18:33<22:14:25, 1.13it/s][2025-04-27 03:21:59] (step=0020525) Train Loss: 6.0423, Train Steps/Sec: 1.12 + 19%|███████████████████████▌ | 20549/110696 [6:18:56<22:13:06, 1.13it/s][2025-04-27 03:22:21] (step=0020550) Train Loss: 6.0420, Train Steps/Sec: 1.12 + 19%|███████████████████████▌ | 20574/110696 [6:19:18<22:10:30, 1.13it/s][2025-04-27 03:22:43] (step=0020575) Train Loss: 6.0317, Train Steps/Sec: 1.12 + 19%|███████████████████████▋ | 20599/110696 [6:19:40<22:12:25, 1.13it/s][2025-04-27 03:23:06] (step=0020600) Train Loss: 6.0613, Train Steps/Sec: 1.11 + 19%|███████████████████████▋ | 20624/110696 [6:20:03<22:52:24, 1.09it/s][2025-04-27 03:23:28] (step=0020625) Train Loss: 6.0248, Train Steps/Sec: 1.12 + 19%|███████████████████████▋ | 20649/110696 [6:20:25<22:35:30, 1.11it/s][2025-04-27 03:23:51] (step=0020650) Train Loss: 5.9820, Train Steps/Sec: 1.12 + 19%|███████████████████████▋ | 20674/110696 [6:20:48<22:21:34, 1.12it/s][2025-04-27 03:24:13] (step=0020675) Train Loss: 6.0020, Train Steps/Sec: 1.12 + 19%|███████████████████████▋ | 20699/110696 [6:21:10<22:15:13, 1.12it/s][2025-04-27 03:24:35] (step=0020700) Train Loss: 6.0474, Train Steps/Sec: 1.12 + 19%|███████████████████████▊ | 20724/110696 [6:21:32<22:19:37, 1.12it/s][2025-04-27 03:24:58] (step=0020725) Train Loss: 5.9784, Train Steps/Sec: 1.12 + 19%|███████████████████████▊ | 20749/110696 [6:21:55<22:12:18, 1.13it/s][2025-04-27 03:25:20] (step=0020750) Train Loss: 5.9967, Train Steps/Sec: 1.12 + 19%|███████████████████████▊ | 20774/110696 [6:22:17<22:02:25, 1.13it/s][2025-04-27 03:25:42] (step=0020775) Train Loss: 6.0049, Train Steps/Sec: 1.12 + 19%|███████████████████████▊ | 20799/110696 [6:22:39<22:03:07, 1.13it/s][2025-04-27 03:26:05] (step=0020800) Train Loss: 6.0087, Train Steps/Sec: 1.12 + 19%|███████████████████████▉ | 20824/110696 [6:23:02<22:46:49, 1.10it/s][2025-04-27 03:26:27] (step=0020825) Train Loss: 6.0302, Train Steps/Sec: 1.12 + 19%|███████████████████████▉ | 20849/110696 [6:23:24<22:33:55, 1.11it/s][2025-04-27 03:26:49] (step=0020850) Train Loss: 6.0428, Train Steps/Sec: 1.12 + 19%|███████████████████████▉ | 20874/110696 [6:23:46<22:19:41, 1.12it/s][2025-04-27 03:27:12] (step=0020875) Train Loss: 6.0269, Train Steps/Sec: 1.12 + 19%|███████████████████████▉ | 20899/110696 [6:24:09<22:14:01, 1.12it/s][2025-04-27 03:27:34] (step=0020900) Train Loss: 6.0555, Train Steps/Sec: 1.12 + 19%|████████████████████████ | 20924/110696 [6:24:31<22:09:12, 1.13it/s][2025-04-27 03:27:56] (step=0020925) Train Loss: 6.0481, Train Steps/Sec: 1.12 + 19%|████████████████████████ | 20949/110696 [6:24:53<22:04:14, 1.13it/s][2025-04-27 03:28:19] (step=0020950) Train Loss: 6.0222, Train Steps/Sec: 1.12 + 19%|████████████████████████ | 20974/110696 [6:25:16<22:05:42, 1.13it/s][2025-04-27 03:28:41] (step=0020975) Train Loss: 6.0475, Train Steps/Sec: 1.12 + 19%|████████████████████████ | 20999/110696 [6:25:38<22:02:15, 1.13it/s][2025-04-27 03:29:03] (step=0021000) Train Loss: 5.9656, Train Steps/Sec: 1.11 + 19%|████████████████████████ | 21024/110696 [6:26:00<22:42:31, 1.10it/s][2025-04-27 03:29:26] (step=0021025) Train Loss: 5.9892, Train Steps/Sec: 1.12 + 19%|████████████████████████▏ | 21049/110696 [6:26:23<22:21:59, 1.11it/s][2025-04-27 03:29:48] (step=0021050) Train Loss: 5.9835, Train Steps/Sec: 1.12 + 19%|████████████████████████▏ | 21074/110696 [6:26:45<22:20:13, 1.11it/s][2025-04-27 03:30:10] (step=0021075) Train Loss: 6.0623, Train Steps/Sec: 1.12 + 19%|████████████████████████▏ | 21099/110696 [6:27:07<22:10:14, 1.12it/s][2025-04-27 03:30:33] (step=0021100) Train Loss: 6.0381, Train Steps/Sec: 1.12 + 19%|████████████████████████▏ | 21124/110696 [6:27:30<22:07:52, 1.12it/s][2025-04-27 03:30:55] (step=0021125) Train Loss: 6.0414, Train Steps/Sec: 1.12 + 19%|████████████████████████▎ | 21149/110696 [6:27:52<22:03:10, 1.13it/s][2025-04-27 03:31:17] (step=0021150) Train Loss: 6.0218, Train Steps/Sec: 1.12 + 19%|████████████████████████▎ | 21174/110696 [6:28:14<22:01:19, 1.13it/s][2025-04-27 03:31:40] (step=0021175) Train Loss: 5.9799, Train Steps/Sec: 1.12 + 19%|████████████████████████▎ | 21199/110696 [6:28:37<21:55:58, 1.13it/s][2025-04-27 03:32:02] (step=0021200) Train Loss: 6.0368, Train Steps/Sec: 1.12 + 19%|████████████████████████▎ | 21224/110696 [6:28:59<22:39:38, 1.10it/s][2025-04-27 03:32:24] (step=0021225) Train Loss: 6.0716, Train Steps/Sec: 1.12 + 19%|████████████████████████▍ | 21249/110696 [6:29:21<22:21:19, 1.11it/s][2025-04-27 03:32:47] (step=0021250) Train Loss: 6.0819, Train Steps/Sec: 1.12 + 19%|████████████████████████▍ | 21274/110696 [6:29:43<22:10:39, 1.12it/s][2025-04-27 03:33:09] (step=0021275) Train Loss: 6.0225, Train Steps/Sec: 1.12 + 19%|████████████████████████▍ | 21299/110696 [6:30:06<22:15:33, 1.12it/s][2025-04-27 03:33:31] (step=0021300) Train Loss: 6.0759, Train Steps/Sec: 1.12 + 19%|████████████████████████▍ | 21324/110696 [6:30:28<22:03:43, 1.13it/s][2025-04-27 03:33:54] (step=0021325) Train Loss: 6.1121, Train Steps/Sec: 1.12 + 19%|████████████████████████▍ | 21349/110696 [6:30:50<21:59:44, 1.13it/s][2025-04-27 03:34:16] (step=0021350) Train Loss: 6.0333, Train Steps/Sec: 1.12 + 19%|████████████████████████▌ | 21374/110696 [6:31:13<21:54:02, 1.13it/s][2025-04-27 03:34:38] (step=0021375) Train Loss: 6.0260, Train Steps/Sec: 1.12 + 19%|████████████████████████▌ | 21399/110696 [6:31:35<21:53:10, 1.13it/s][2025-04-27 03:35:01] (step=0021400) Train Loss: 6.0445, Train Steps/Sec: 1.12 + 19%|████████████████████████▌ | 21424/110696 [6:31:57<22:34:33, 1.10it/s][2025-04-27 03:35:23] (step=0021425) Train Loss: 5.9888, Train Steps/Sec: 1.12 + 19%|████████████████████████▌ | 21449/110696 [6:32:20<22:18:54, 1.11it/s][2025-04-27 03:35:45] (step=0021450) Train Loss: 6.0609, Train Steps/Sec: 1.12 + 19%|████████████████████████▋ | 21474/110696 [6:32:42<22:12:35, 1.12it/s][2025-04-27 03:36:08] (step=0021475) Train Loss: 5.9951, Train Steps/Sec: 1.12 + 19%|████████████████████████▋ | 21499/110696 [6:33:04<22:02:02, 1.12it/s][2025-04-27 03:36:30] (step=0021500) Train Loss: 6.0180, Train Steps/Sec: 1.12 + 19%|████████████████████████▋ | 21524/110696 [6:33:27<22:00:08, 1.13it/s][2025-04-27 03:36:52] (step=0021525) Train Loss: 6.0397, Train Steps/Sec: 1.12 + 19%|████████████████████████▋ | 21549/110696 [6:33:49<21:57:11, 1.13it/s][2025-04-27 03:37:15] (step=0021550) Train Loss: 6.0239, Train Steps/Sec: 1.12 + 19%|████████████████████████▊ | 21574/110696 [6:34:12<21:55:30, 1.13it/s][2025-04-27 03:37:37] (step=0021575) Train Loss: 6.0673, Train Steps/Sec: 1.12 + 20%|████████████████████████▊ | 21599/110696 [6:34:34<21:56:26, 1.13it/s][2025-04-27 03:37:59] (step=0021600) Train Loss: 6.0339, Train Steps/Sec: 1.11 + 20%|████████████████████████▊ | 21624/110696 [6:34:56<22:39:02, 1.09it/s][2025-04-27 03:38:22] (step=0021625) Train Loss: 6.0203, Train Steps/Sec: 1.12 + 20%|████████████████████████▊ | 21649/110696 [6:35:19<22:15:15, 1.11it/s][2025-04-27 03:38:44] (step=0021650) Train Loss: 6.0359, Train Steps/Sec: 1.12 + 20%|████████████████████████▊ | 21674/110696 [6:35:41<22:06:44, 1.12it/s][2025-04-27 03:39:06] (step=0021675) Train Loss: 5.9715, Train Steps/Sec: 1.12 + 20%|████████████████████████▉ | 21699/110696 [6:36:03<22:01:55, 1.12it/s][2025-04-27 03:39:29] (step=0021700) Train Loss: 6.0134, Train Steps/Sec: 1.12 + 20%|████████████████████████▉ | 21724/110696 [6:36:26<21:58:44, 1.12it/s][2025-04-27 03:39:51] (step=0021725) Train Loss: 6.0291, Train Steps/Sec: 1.12 + 20%|████████████████████████▉ | 21749/110696 [6:36:48<21:51:04, 1.13it/s][2025-04-27 03:40:13] (step=0021750) Train Loss: 5.9506, Train Steps/Sec: 1.12 + 20%|████████████████████████▉ | 21774/110696 [6:37:10<21:51:43, 1.13it/s][2025-04-27 03:40:36] (step=0021775) Train Loss: 6.0053, Train Steps/Sec: 1.12 + 20%|█████████████████████████ | 21799/110696 [6:37:33<21:49:13, 1.13it/s][2025-04-27 03:40:58] (step=0021800) Train Loss: 5.9994, Train Steps/Sec: 1.12 + 20%|█████████████████████████ | 21824/110696 [6:37:55<22:40:56, 1.09it/s][2025-04-27 03:41:20] (step=0021825) Train Loss: 6.0082, Train Steps/Sec: 1.12 + 20%|█████████████████████████ | 21849/110696 [6:38:17<22:13:07, 1.11it/s][2025-04-27 03:41:43] (step=0021850) Train Loss: 6.0595, Train Steps/Sec: 1.12 + 20%|█████████████████████████ | 21874/110696 [6:38:40<22:08:41, 1.11it/s][2025-04-27 03:42:05] (step=0021875) Train Loss: 5.9878, Train Steps/Sec: 1.12 + 20%|█████████████████████████ | 21899/110696 [6:39:02<21:57:21, 1.12it/s][2025-04-27 03:42:28] (step=0021900) Train Loss: 6.0359, Train Steps/Sec: 1.12 + 20%|█████████████████████████▏ | 21924/110696 [6:39:24<21:52:50, 1.13it/s][2025-04-27 03:42:50] (step=0021925) Train Loss: 6.0074, Train Steps/Sec: 1.12 + 20%|█████████████████████████▏ | 21949/110696 [6:39:47<21:52:50, 1.13it/s][2025-04-27 03:43:12] (step=0021950) Train Loss: 5.9964, Train Steps/Sec: 1.12 + 20%|█████████████████████████▏ | 21974/110696 [6:40:09<21:48:35, 1.13it/s][2025-04-27 03:43:34] (step=0021975) Train Loss: 5.9899, Train Steps/Sec: 1.12 + 20%|█████████████████████████▏ | 21999/110696 [6:40:31<21:46:50, 1.13it/s][2025-04-27 03:43:57] (step=0022000) Train Loss: 6.0329, Train Steps/Sec: 1.12 +You are using a model of type blip-2 to instantiate a model of type instructblip. This is not supported for all configurations of models and can yield errors. +[2025-04-27 03:43:57] text_config is None. Initializing the text config with default values (`OPTConfig`). +model_name_or_path:/tmp/haozhezhao/model/blip2-flan-t5-xl +image_place_holder: +tokenizer length after expend 32101 +tokenizer length before expend 32100 +100%|████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████| 5/5 [04:57<00:00, 59.42s/it] +[2025-04-27 03:50:05] Finish Eval in 22000 steps...█████████████████████████████████████████████████████████████████████████████████████████████| 5/5 [04:56<00:00, 59.24s/it] +processing left-padding... +processing left-padding... +processing left-padding... +processing left-padding... +processing left-padding... +[2025-04-27 03:50:24] Saved checkpoint to checkpoint/X2I_UltraEdiit_fourmask_LLava_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_filtered_objects_4mask_1280_context/003-GPT-XL/checkpoints/0022000.pt +[2025-04-27 03:50:26] Removed old checkpoint: checkpoint/X2I_UltraEdiit_fourmask_LLava_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_filtered_objects_4mask_1280_context/003-GPT-XL/checkpoints/0020000.pt + 20%|█████████████████████████▎ | 22024/110696 [6:47:24<23:00:47, 1.07it/s][2025-04-27 03:50:49] (step=0022025) Train Loss: 6.0017, Train Steps/Sec: 0.06 + 20%|█████████████████████████▎ | 22049/110696 [6:47:46<22:12:11, 1.11it/s][2025-04-27 03:51:11] (step=0022050) Train Loss: 6.0122, Train Steps/Sec: 1.12 + 20%|█████████████████████████▎ | 22074/110696 [6:48:08<22:03:12, 1.12it/s][2025-04-27 03:51:34] (step=0022075) Train Loss: 6.0326, Train Steps/Sec: 1.12 + 20%|█████████████████████████▎ | 22099/110696 [6:48:31<21:53:12, 1.12it/s][2025-04-27 03:51:56] (step=0022100) Train Loss: 5.9934, Train Steps/Sec: 1.12 + 20%|█████████████████████████▍ | 22124/110696 [6:48:53<21:53:03, 1.12it/s][2025-04-27 03:52:18] (step=0022125) Train Loss: 6.0388, Train Steps/Sec: 1.12 + 20%|█████████████████████████▍ | 22149/110696 [6:49:15<21:51:09, 1.13it/s][2025-04-27 03:52:41] (step=0022150) Train Loss: 5.9830, Train Steps/Sec: 1.12 + 20%|█████████████████████████▍ | 22174/110696 [6:49:38<21:46:51, 1.13it/s][2025-04-27 03:53:03] (step=0022175) Train Loss: 5.9664, Train Steps/Sec: 1.12 + 20%|█████████████████████████▍ | 22199/110696 [6:50:00<21:45:02, 1.13it/s][2025-04-27 03:53:26] (step=0022200) Train Loss: 6.0647, Train Steps/Sec: 1.11 + 20%|█████████████████████████▍ | 22224/110696 [6:50:23<22:27:27, 1.09it/s][2025-04-27 03:53:48] (step=0022225) Train Loss: 6.0424, Train Steps/Sec: 1.12 + 20%|█████████████████████████▌ | 22249/110696 [6:50:45<22:10:30, 1.11it/s][2025-04-27 03:54:10] (step=0022250) Train Loss: 6.0339, Train Steps/Sec: 1.12 + 20%|█████████████████████████▌ | 22274/110696 [6:51:07<21:59:44, 1.12it/s][2025-04-27 03:54:33] (step=0022275) Train Loss: 5.9947, Train Steps/Sec: 1.12 + 20%|█████████████████████████▌ | 22299/110696 [6:51:29<21:54:03, 1.12it/s][2025-04-27 03:54:55] (step=0022300) Train Loss: 5.9996, Train Steps/Sec: 1.12 + 20%|█████████████████████████▌ | 22324/110696 [6:51:52<21:45:20, 1.13it/s][2025-04-27 03:55:17] (step=0022325) Train Loss: 6.0910, Train Steps/Sec: 1.12 + 20%|█████████████████████████▋ | 22349/110696 [6:52:14<21:48:32, 1.13it/s][2025-04-27 03:55:40] (step=0022350) Train Loss: 5.9500, Train Steps/Sec: 1.12 + 20%|█████████████████████████▋ | 22374/110696 [6:52:37<21:46:16, 1.13it/s][2025-04-27 03:56:02] (step=0022375) Train Loss: 6.0511, Train Steps/Sec: 1.11 + 20%|█████████████████████████▋ | 22399/110696 [6:52:59<21:44:27, 1.13it/s][2025-04-27 03:56:24] (step=0022400) Train Loss: 5.9695, Train Steps/Sec: 1.12 + 20%|█████████████████████████▋ | 22424/110696 [6:53:21<22:23:09, 1.10it/s][2025-04-27 03:56:47] (step=0022425) Train Loss: 6.0302, Train Steps/Sec: 1.12 + 20%|█████████████████████████▊ | 22449/110696 [6:53:44<22:07:30, 1.11it/s][2025-04-27 03:57:09] (step=0022450) Train Loss: 5.9774, Train Steps/Sec: 1.12 + 20%|█████████████████████████▊ | 22474/110696 [6:54:06<21:55:56, 1.12it/s][2025-04-27 03:57:32] (step=0022475) Train Loss: 6.0618, Train Steps/Sec: 1.12 + 20%|█████████████████████████▊ | 22499/110696 [6:54:28<21:54:30, 1.12it/s][2025-04-27 03:57:54] (step=0022500) Train Loss: 6.0761, Train Steps/Sec: 1.12 + 20%|█████████████████████████▊ | 22524/110696 [6:54:51<21:48:51, 1.12it/s][2025-04-27 03:58:16] (step=0022525) Train Loss: 6.0243, Train Steps/Sec: 1.12 + 20%|█████████████████████████▊ | 22549/110696 [6:55:13<21:45:24, 1.13it/s][2025-04-27 03:58:39] (step=0022550) Train Loss: 6.0209, Train Steps/Sec: 1.12 + 20%|█████████████████████████▉ | 22574/110696 [6:55:36<21:38:25, 1.13it/s][2025-04-27 03:59:01] (step=0022575) Train Loss: 6.0679, Train Steps/Sec: 1.12 + 20%|█████████████████████████▉ | 22599/110696 [6:55:58<21:35:47, 1.13it/s][2025-04-27 03:59:23] (step=0022600) Train Loss: 6.0740, Train Steps/Sec: 1.12 + 20%|█████████████████████████▉ | 22624/110696 [6:56:20<22:15:53, 1.10it/s][2025-04-27 03:59:46] (step=0022625) Train Loss: 6.0052, Train Steps/Sec: 1.12 + 20%|█████████████████████████▉ | 22649/110696 [6:56:42<22:04:00, 1.11it/s][2025-04-27 04:00:08] (step=0022650) Train Loss: 5.9554, Train Steps/Sec: 1.12 + 20%|██████████████████████████ | 22674/110696 [6:57:05<21:55:49, 1.11it/s][2025-04-27 04:00:30] (step=0022675) Train Loss: 6.0176, Train Steps/Sec: 1.12 + 21%|██████████████████████████ | 22699/110696 [6:57:27<21:45:00, 1.12it/s][2025-04-27 04:00:52] (step=0022700) Train Loss: 5.9969, Train Steps/Sec: 1.12 + 21%|██████████████████████████ | 22724/110696 [6:57:49<21:42:11, 1.13it/s][2025-04-27 04:01:15] (step=0022725) Train Loss: 5.9804, Train Steps/Sec: 1.12 + 21%|██████████████████████████ | 22749/110696 [6:58:12<21:39:36, 1.13it/s][2025-04-27 04:01:37] (step=0022750) Train Loss: 5.9639, Train Steps/Sec: 1.12 + 21%|██████████████████████████▏ | 22774/110696 [6:58:34<21:32:54, 1.13it/s][2025-04-27 04:01:59] (step=0022775) Train Loss: 5.9914, Train Steps/Sec: 1.12 + 21%|██████████████████████████▏ | 22799/110696 [6:58:56<21:33:50, 1.13it/s][2025-04-27 04:02:22] (step=0022800) Train Loss: 6.0202, Train Steps/Sec: 1.12 + 21%|██████████████████████████▏ | 22824/110696 [6:59:19<22:16:24, 1.10it/s][2025-04-27 04:02:44] (step=0022825) Train Loss: 6.0384, Train Steps/Sec: 1.12 + 21%|██████████████████████████▏ | 22849/110696 [6:59:41<22:01:08, 1.11it/s][2025-04-27 04:03:06] (step=0022850) Train Loss: 5.9454, Train Steps/Sec: 1.12 + 21%|██████████████████████████▏ | 22874/110696 [7:00:03<21:56:54, 1.11it/s][2025-04-27 04:03:29] (step=0022875) Train Loss: 6.0141, Train Steps/Sec: 1.12 + 21%|██████████████████████████▎ | 22899/110696 [7:00:26<21:44:22, 1.12it/s][2025-04-27 04:03:51] (step=0022900) Train Loss: 5.9586, Train Steps/Sec: 1.12 + 21%|██████████████████████████▎ | 22924/110696 [7:00:48<21:40:28, 1.12it/s][2025-04-27 04:04:14] (step=0022925) Train Loss: 6.0276, Train Steps/Sec: 1.12 + 21%|██████████████████████████▎ | 22949/110696 [7:01:11<21:37:59, 1.13it/s][2025-04-27 04:04:36] (step=0022950) Train Loss: 5.9864, Train Steps/Sec: 1.12 + 21%|██████████████████████████▎ | 22974/110696 [7:01:33<21:35:21, 1.13it/s][2025-04-27 04:04:58] (step=0022975) Train Loss: 6.0175, Train Steps/Sec: 1.12 + 21%|██████████████████████████▍ | 22999/110696 [7:01:55<21:34:32, 1.13it/s][2025-04-27 04:05:21] (step=0023000) Train Loss: 5.9951, Train Steps/Sec: 1.11 + 21%|██████████████████████████▍ | 23024/110696 [7:02:18<22:17:35, 1.09it/s][2025-04-27 04:05:43] (step=0023025) Train Loss: 5.9958, Train Steps/Sec: 1.12 + 21%|██████████████████████████▍ | 23049/110696 [7:02:40<21:57:28, 1.11it/s][2025-04-27 04:06:05] (step=0023050) Train Loss: 6.0236, Train Steps/Sec: 1.12 + 21%|██████████████████████████▍ | 23074/110696 [7:03:02<21:45:31, 1.12it/s][2025-04-27 04:06:28] (step=0023075) Train Loss: 5.9899, Train Steps/Sec: 1.12 + 21%|██████████████████████████▌ | 23099/110696 [7:03:25<21:42:05, 1.12it/s][2025-04-27 04:06:50] (step=0023100) Train Loss: 5.9009, Train Steps/Sec: 1.12 + 21%|██████████████████████████▌ | 23124/110696 [7:03:47<21:40:34, 1.12it/s][2025-04-27 04:07:12] (step=0023125) Train Loss: 5.9816, Train Steps/Sec: 1.12 + 21%|██████████████████████████▌ | 23149/110696 [7:04:09<21:33:12, 1.13it/s][2025-04-27 04:07:35] (step=0023150) Train Loss: 5.9893, Train Steps/Sec: 1.12 + 21%|██████████████████████████▌ | 23174/110696 [7:04:32<21:30:32, 1.13it/s][2025-04-27 04:07:57] (step=0023175) Train Loss: 6.0734, Train Steps/Sec: 1.12 + 21%|██████████████████████████▌ | 23199/110696 [7:04:54<21:28:13, 1.13it/s][2025-04-27 04:08:19] (step=0023200) Train Loss: 6.0186, Train Steps/Sec: 1.12 + 21%|██████████████████████████▋ | 23224/110696 [7:05:16<22:06:13, 1.10it/s][2025-04-27 04:08:42] (step=0023225) Train Loss: 5.9453, Train Steps/Sec: 1.12 + 21%|██████████████████████████▋ | 23249/110696 [7:05:39<21:56:29, 1.11it/s][2025-04-27 04:09:04] (step=0023250) Train Loss: 6.0434, Train Steps/Sec: 1.12 + 21%|██████████████████████████▋ | 23274/110696 [7:06:01<21:40:31, 1.12it/s][2025-04-27 04:09:26] (step=0023275) Train Loss: 6.0130, Train Steps/Sec: 1.12 + 21%|██████████████████████████▋ | 23299/110696 [7:06:23<21:35:14, 1.12it/s][2025-04-27 04:09:49] (step=0023300) Train Loss: 6.0280, Train Steps/Sec: 1.12 + 21%|██████████████████████████▊ | 23324/110696 [7:06:46<21:37:12, 1.12it/s][2025-04-27 04:10:11] (step=0023325) Train Loss: 6.0405, Train Steps/Sec: 1.12 + 21%|██████████████████████████▊ | 23349/110696 [7:07:08<21:28:37, 1.13it/s][2025-04-27 04:10:33] (step=0023350) Train Loss: 6.0086, Train Steps/Sec: 1.12 + 21%|██████████████████████████▊ | 23374/110696 [7:07:30<21:25:44, 1.13it/s][2025-04-27 04:10:56] (step=0023375) Train Loss: 6.0293, Train Steps/Sec: 1.12 + 21%|██████████████████████████▊ | 23399/110696 [7:07:53<21:25:55, 1.13it/s][2025-04-27 04:11:18] (step=0023400) Train Loss: 6.0027, Train Steps/Sec: 1.12 + 21%|██████████████████████████▊ | 23424/110696 [7:08:20<23:36:20, 1.03it/s][2025-04-27 04:11:45] (step=0023425) Train Loss: 6.0553, Train Steps/Sec: 0.91 + 21%|██████████████████████████▉ | 23449/110696 [7:08:42<21:48:50, 1.11it/s][2025-04-27 04:12:08] (step=0023450) Train Loss: 6.0537, Train Steps/Sec: 1.12 + 21%|██████████████████████████▉ | 23474/110696 [7:09:05<21:43:59, 1.11it/s][2025-04-27 04:12:30] (step=0023475) Train Loss: 5.9991, Train Steps/Sec: 1.12 + 21%|██████████████████████████▉ | 23499/110696 [7:09:27<21:34:49, 1.12it/s][2025-04-27 04:12:53] (step=0023500) Train Loss: 5.9712, Train Steps/Sec: 1.12 + 21%|██████████████████████████▉ | 23524/110696 [7:09:50<21:34:17, 1.12it/s][2025-04-27 04:13:15] (step=0023525) Train Loss: 5.9490, Train Steps/Sec: 1.12 + 21%|███████████████████████████ | 23549/110696 [7:10:12<21:33:10, 1.12it/s][2025-04-27 04:13:37] (step=0023550) Train Loss: 6.0133, Train Steps/Sec: 1.12 + 21%|███████████████████████████ | 23574/110696 [7:10:34<21:26:41, 1.13it/s][2025-04-27 04:14:00] (step=0023575) Train Loss: 5.9807, Train Steps/Sec: 1.12 + 21%|███████████████████████████ | 23599/110696 [7:10:57<21:22:52, 1.13it/s][2025-04-27 04:14:22] (step=0023600) Train Loss: 6.0017, Train Steps/Sec: 1.12 + 21%|███████████████████████████ | 23624/110696 [7:11:19<22:01:56, 1.10it/s][2025-04-27 04:14:44] (step=0023625) Train Loss: 5.9955, Train Steps/Sec: 1.12 + 21%|███████████████████████████▏ | 23649/110696 [7:11:41<21:43:31, 1.11it/s][2025-04-27 04:15:07] (step=0023650) Train Loss: 6.0046, Train Steps/Sec: 1.12 + 21%|███████████████████████████▏ | 23674/110696 [7:12:04<21:44:01, 1.11it/s][2025-04-27 04:15:29] (step=0023675) Train Loss: 6.0034, Train Steps/Sec: 1.12 + 21%|███████████████████████████▏ | 23699/110696 [7:12:26<21:34:49, 1.12it/s][2025-04-27 04:15:51] (step=0023700) Train Loss: 6.0164, Train Steps/Sec: 1.12 + 21%|███████████████████████████▏ | 23724/110696 [7:12:48<21:24:19, 1.13it/s][2025-04-27 04:16:14] (step=0023725) Train Loss: 5.9016, Train Steps/Sec: 1.12 + 21%|███████████████████████████▏ | 23749/110696 [7:13:11<21:24:03, 1.13it/s][2025-04-27 04:16:36] (step=0023750) Train Loss: 5.9990, Train Steps/Sec: 1.12 + 21%|███████████████████████████▎ | 23774/110696 [7:13:33<21:24:36, 1.13it/s][2025-04-27 04:16:58] (step=0023775) Train Loss: 6.0222, Train Steps/Sec: 1.12 + 21%|███████████████████████████▎ | 23799/110696 [7:13:55<21:27:28, 1.12it/s][2025-04-27 04:17:21] (step=0023800) Train Loss: 5.9663, Train Steps/Sec: 1.12 + 22%|███████████████████████████▎ | 23824/110696 [7:14:18<22:07:24, 1.09it/s][2025-04-27 04:17:43] (step=0023825) Train Loss: 5.9656, Train Steps/Sec: 1.12 + 22%|███████████████████████████▎ | 23849/110696 [7:14:40<21:53:17, 1.10it/s][2025-04-27 04:18:06] (step=0023850) Train Loss: 5.9906, Train Steps/Sec: 1.12 + 22%|███████████████████████████▍ | 23874/110696 [7:15:03<21:43:17, 1.11it/s][2025-04-27 04:18:28] (step=0023875) Train Loss: 5.9936, Train Steps/Sec: 1.12 + 22%|███████████████████████████▍ | 23899/110696 [7:15:25<21:31:27, 1.12it/s][2025-04-27 04:18:50] (step=0023900) Train Loss: 5.9600, Train Steps/Sec: 1.12 + 22%|███████████████████████████▍ | 23924/110696 [7:15:47<21:36:44, 1.12it/s][2025-04-27 04:19:13] (step=0023925) Train Loss: 5.9502, Train Steps/Sec: 1.12 + 22%|███████████████████████████▍ | 23949/110696 [7:16:10<21:25:35, 1.12it/s][2025-04-27 04:19:35] (step=0023950) Train Loss: 6.0297, Train Steps/Sec: 1.12 + 22%|███████████████████████████▌ | 23974/110696 [7:16:32<21:21:05, 1.13it/s][2025-04-27 04:19:57] (step=0023975) Train Loss: 5.9836, Train Steps/Sec: 1.12 + 22%|███████████████████████████▌ | 23999/110696 [7:16:59<55:10:49, 2.29s/it][2025-04-27 04:20:24] (step=0024000) Train Loss: 6.0087, Train Steps/Sec: 0.92 +You are using a model of type blip-2 to instantiate a model of type instructblip. This is not supported for all configurations of models and can yield errors. +[2025-04-27 04:20:25] text_config is None. Initializing the text config with default values (`OPTConfig`). +model_name_or_path:/tmp/haozhezhao/model/blip2-flan-t5-xl +image_place_holder: +tokenizer length after expend 32101 +tokenizer length before expend 32100 +100%|████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████| 5/5 [04:56<00:00, 59.20s/it] +[2025-04-27 04:26:31] Finish Eval in 24000 steps...█████████████████████████████████████████████████████████████████████████████████████████████| 5/5 [04:55<00:00, 58.98s/it] +processing left-padding... +processing left-padding... +processing left-padding... +processing left-padding... +processing left-padding... +[2025-04-27 04:26:50] Saved checkpoint to checkpoint/X2I_UltraEdiit_fourmask_LLava_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_filtered_objects_4mask_1280_context/003-GPT-XL/checkpoints/0024000.pt +[2025-04-27 04:26:52] Removed old checkpoint: checkpoint/X2I_UltraEdiit_fourmask_LLava_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_filtered_objects_4mask_1280_context/003-GPT-XL/checkpoints/0022000.pt + 22%|███████████████████████████▌ | 24024/110696 [7:23:54<23:12:46, 1.04it/s][2025-04-27 04:27:20] (step=0024025) Train Loss: 5.9901, Train Steps/Sec: 0.06 + 22%|███████████████████████████▌ | 24049/110696 [7:24:37<26:19:45, 1.09s/it][2025-04-27 04:28:02] (step=0024050) Train Loss: 5.9992, Train Steps/Sec: 0.59 + 22%|███████████████████████████▌ | 24074/110696 [7:24:59<21:33:22, 1.12it/s][2025-04-27 04:28:24] (step=0024075) Train Loss: 5.9726, Train Steps/Sec: 1.12 + 22%|███████████████████████████▋ | 24099/110696 [7:25:21<21:22:59, 1.12it/s][2025-04-27 04:28:47] (step=0024100) Train Loss: 6.0037, Train Steps/Sec: 1.12 + 22%|███████████████████████████▋ | 24124/110696 [7:25:44<21:23:41, 1.12it/s][2025-04-27 04:29:09] (step=0024125) Train Loss: 6.0201, Train Steps/Sec: 1.12 + 22%|███████████████████████████▋ | 24149/110696 [7:26:06<21:20:39, 1.13it/s][2025-04-27 04:29:31] (step=0024150) Train Loss: 5.9679, Train Steps/Sec: 1.12 + 22%|███████████████████████████▋ | 24174/110696 [7:26:28<21:15:28, 1.13it/s][2025-04-27 04:29:54] (step=0024175) Train Loss: 5.9927, Train Steps/Sec: 1.12 + 22%|███████████████████████████▊ | 24199/110696 [7:26:51<21:16:09, 1.13it/s][2025-04-27 04:30:16] (step=0024200) Train Loss: 6.0627, Train Steps/Sec: 1.12 + 22%|███████████████████████████▊ | 24224/110696 [7:27:13<21:39:28, 1.11it/s][2025-04-27 04:30:38] (step=0024225) Train Loss: 5.9653, Train Steps/Sec: 1.12 + 22%|███████████████████████████▊ | 24249/110696 [7:27:35<21:41:39, 1.11it/s][2025-04-27 04:31:01] (step=0024250) Train Loss: 5.9385, Train Steps/Sec: 1.12 + 22%|███████████████████████████▊ | 24274/110696 [7:27:58<21:28:00, 1.12it/s][2025-04-27 04:31:23] (step=0024275) Train Loss: 6.0092, Train Steps/Sec: 1.12 + 22%|███████████████████████████▉ | 24299/110696 [7:28:20<21:26:32, 1.12it/s][2025-04-27 04:31:45] (step=0024300) Train Loss: 5.9626, Train Steps/Sec: 1.12 + 22%|███████████████████████████▉ | 24324/110696 [7:28:42<21:17:47, 1.13it/s][2025-04-27 04:32:08] (step=0024325) Train Loss: 6.0000, Train Steps/Sec: 1.12 + 22%|███████████████████████████▉ | 24349/110696 [7:29:05<21:17:06, 1.13it/s][2025-04-27 04:32:30] (step=0024350) Train Loss: 5.9767, Train Steps/Sec: 1.12 + 22%|███████████████████████████▉ | 24374/110696 [7:29:27<21:14:06, 1.13it/s][2025-04-27 04:32:52] (step=0024375) Train Loss: 5.9598, Train Steps/Sec: 1.12 + 22%|███████████████████████████▉ | 24399/110696 [7:29:49<21:11:21, 1.13it/s][2025-04-27 04:33:15] (step=0024400) Train Loss: 6.0131, Train Steps/Sec: 1.11 + 22%|████████████████████████████ | 24424/110696 [7:30:12<21:50:02, 1.10it/s][2025-04-27 04:33:37] (step=0024425) Train Loss: 6.0045, Train Steps/Sec: 1.12 + 22%|████████████████████████████ | 24449/110696 [7:30:34<21:39:51, 1.11it/s][2025-04-27 04:34:00] (step=0024450) Train Loss: 5.9481, Train Steps/Sec: 1.12 + 22%|████████████████████████████ | 24474/110696 [7:30:56<21:29:06, 1.11it/s][2025-04-27 04:34:22] (step=0024475) Train Loss: 5.9468, Train Steps/Sec: 1.12 + 22%|████████████████████████████ | 24499/110696 [7:31:19<21:24:43, 1.12it/s][2025-04-27 04:34:44] (step=0024500) Train Loss: 5.9933, Train Steps/Sec: 1.12 + 22%|████████████████████████████▏ | 24524/110696 [7:31:41<21:14:58, 1.13it/s][2025-04-27 04:35:06] (step=0024525) Train Loss: 5.9812, Train Steps/Sec: 1.12 + 22%|████████████████████████████▏ | 24549/110696 [7:32:03<21:15:22, 1.13it/s][2025-04-27 04:35:29] (step=0024550) Train Loss: 5.9872, Train Steps/Sec: 1.12 + 22%|████████████████████████████▏ | 24574/110696 [7:32:26<21:12:03, 1.13it/s][2025-04-27 04:35:51] (step=0024575) Train Loss: 5.9876, Train Steps/Sec: 1.12 + 22%|████████████████████████████▏ | 24599/110696 [7:32:48<21:12:39, 1.13it/s][2025-04-27 04:36:14] (step=0024600) Train Loss: 5.9409, Train Steps/Sec: 1.11 + 22%|████████████████████████████▎ | 24624/110696 [7:33:11<21:50:42, 1.09it/s][2025-04-27 04:36:36] (step=0024625) Train Loss: 5.9135, Train Steps/Sec: 1.12 + 22%|████████████████████████████▎ | 24649/110696 [7:33:33<21:38:20, 1.10it/s][2025-04-27 04:36:58] (step=0024650) Train Loss: 5.9946, Train Steps/Sec: 1.12 + 22%|████████████████████████████▎ | 24674/110696 [7:33:55<21:18:51, 1.12it/s][2025-04-27 04:37:21] (step=0024675) Train Loss: 6.0174, Train Steps/Sec: 1.12 + 22%|████████████████████████████▎ | 24699/110696 [7:34:18<21:20:14, 1.12it/s][2025-04-27 04:37:43] (step=0024700) Train Loss: 6.0043, Train Steps/Sec: 1.12 + 22%|████████████████████████████▎ | 24724/110696 [7:34:40<21:11:36, 1.13it/s][2025-04-27 04:38:05] (step=0024725) Train Loss: 6.0135, Train Steps/Sec: 1.12 + 22%|████████████████████████████▍ | 24749/110696 [7:35:02<21:17:52, 1.12it/s][2025-04-27 04:38:28] (step=0024750) Train Loss: 5.9700, Train Steps/Sec: 1.12 + 22%|████████████████████████████▍ | 24774/110696 [7:35:25<21:10:08, 1.13it/s][2025-04-27 04:38:50] (step=0024775) Train Loss: 5.9271, Train Steps/Sec: 1.12 + 22%|████████████████████████████▍ | 24799/110696 [7:35:47<21:07:58, 1.13it/s][2025-04-27 04:39:13] (step=0024800) Train Loss: 5.9919, Train Steps/Sec: 1.11 + 22%|████████████████████████████▍ | 24824/110696 [7:36:10<21:44:10, 1.10it/s][2025-04-27 04:39:35] (step=0024825) Train Loss: 6.0373, Train Steps/Sec: 1.12 + 22%|████████████████████████████▌ | 24849/110696 [7:36:32<21:34:51, 1.10it/s][2025-04-27 04:39:57] (step=0024850) Train Loss: 5.9283, Train Steps/Sec: 1.12 + 22%|████████████████████████████▌ | 24874/110696 [7:36:54<21:19:21, 1.12it/s][2025-04-27 04:40:20] (step=0024875) Train Loss: 5.9933, Train Steps/Sec: 1.12 + 22%|████████████████████████████▌ | 24899/110696 [7:37:17<21:17:28, 1.12it/s][2025-04-27 04:40:42] (step=0024900) Train Loss: 5.9589, Train Steps/Sec: 1.12 + 23%|████████████████████████████▌ | 24924/110696 [7:37:39<21:13:17, 1.12it/s][2025-04-27 04:41:04] (step=0024925) Train Loss: 5.9748, Train Steps/Sec: 1.12 + 23%|████████████████████████████▌ | 24949/110696 [7:38:01<21:11:06, 1.12it/s][2025-04-27 04:41:27] (step=0024950) Train Loss: 5.9660, Train Steps/Sec: 1.12 + 23%|████████████████████████████▋ | 24974/110696 [7:38:24<20:59:30, 1.13it/s][2025-04-27 04:41:49] (step=0024975) Train Loss: 6.0128, Train Steps/Sec: 1.12 + 23%|████████████████████████████▋ | 24999/110696 [7:38:46<21:04:00, 1.13it/s][2025-04-27 04:42:11] (step=0025000) Train Loss: 6.0024, Train Steps/Sec: 1.12 + 23%|████████████████████████████▋ | 25024/110696 [7:39:08<21:40:07, 1.10it/s][2025-04-27 04:42:34] (step=0025025) Train Loss: 5.9648, Train Steps/Sec: 1.12 + 23%|████████████████████████████▋ | 25049/110696 [7:39:31<21:26:29, 1.11it/s][2025-04-27 04:42:56] (step=0025050) Train Loss: 5.8989, Train Steps/Sec: 1.12 + 23%|████████████████████████████▊ | 25074/110696 [7:39:53<21:18:32, 1.12it/s][2025-04-27 04:43:18] (step=0025075) Train Loss: 5.9793, Train Steps/Sec: 1.12 + 23%|████████████████████████████▊ | 25099/110696 [7:40:15<21:06:12, 1.13it/s][2025-04-27 04:43:41] (step=0025100) Train Loss: 5.9914, Train Steps/Sec: 1.12 + 23%|████████████████████████████▊ | 25124/110696 [7:40:38<21:14:17, 1.12it/s][2025-04-27 04:44:03] (step=0025125) Train Loss: 5.9868, Train Steps/Sec: 1.12 + 23%|████████████████████████████▊ | 25149/110696 [7:41:00<21:05:34, 1.13it/s][2025-04-27 04:44:25] (step=0025150) Train Loss: 5.8950, Train Steps/Sec: 1.12 + 23%|████████████████████████████▉ | 25174/110696 [7:41:22<21:00:07, 1.13it/s][2025-04-27 04:44:48] (step=0025175) Train Loss: 6.0262, Train Steps/Sec: 1.12 + 23%|████████████████████████████▉ | 25199/110696 [7:41:45<20:57:51, 1.13it/s][2025-04-27 04:45:10] (step=0025200) Train Loss: 5.9055, Train Steps/Sec: 1.12 + 23%|████████████████████████████▉ | 25224/110696 [7:42:07<21:37:36, 1.10it/s][2025-04-27 04:45:32] (step=0025225) Train Loss: 5.9880, Train Steps/Sec: 1.12 + 23%|████████████████████████████▉ | 25249/110696 [7:42:29<21:21:55, 1.11it/s][2025-04-27 04:45:55] (step=0025250) Train Loss: 5.9560, Train Steps/Sec: 1.12 + 23%|████████████████████████████▉ | 25274/110696 [7:42:52<21:10:24, 1.12it/s][2025-04-27 04:46:17] (step=0025275) Train Loss: 5.9094, Train Steps/Sec: 1.12 + 23%|█████████████████████████████ | 25299/110696 [7:43:14<21:06:52, 1.12it/s][2025-04-27 04:46:39] (step=0025300) Train Loss: 5.9595, Train Steps/Sec: 1.12 + 23%|█████████████████████████████ | 25324/110696 [7:43:36<21:01:07, 1.13it/s][2025-04-27 04:47:02] (step=0025325) Train Loss: 5.9443, Train Steps/Sec: 1.12 + 23%|█████████████████████████████ | 25349/110696 [7:43:59<21:02:28, 1.13it/s][2025-04-27 04:47:24] (step=0025350) Train Loss: 5.9705, Train Steps/Sec: 1.12 + 23%|█████████████████████████████ | 25374/110696 [7:44:21<21:00:03, 1.13it/s][2025-04-27 04:47:46] (step=0025375) Train Loss: 6.0289, Train Steps/Sec: 1.12 + 23%|█████████████████████████████▏ | 25399/110696 [7:44:43<20:57:05, 1.13it/s][2025-04-27 04:48:09] (step=0025400) Train Loss: 5.9633, Train Steps/Sec: 1.12 + 23%|█████████████████████████████▏ | 25424/110696 [7:45:06<21:34:38, 1.10it/s][2025-04-27 04:48:31] (step=0025425) Train Loss: 6.0048, Train Steps/Sec: 1.12 + 23%|█████████████████████████████▏ | 25449/110696 [7:45:28<21:24:13, 1.11it/s][2025-04-27 04:48:53] (step=0025450) Train Loss: 5.9611, Train Steps/Sec: 1.12 + 23%|█████████████████████████████▏ | 25474/110696 [7:45:50<21:11:58, 1.12it/s][2025-04-27 04:49:16] (step=0025475) Train Loss: 5.9820, Train Steps/Sec: 1.12 + 23%|█████████████████████████████▎ | 25499/110696 [7:46:13<21:05:07, 1.12it/s][2025-04-27 04:49:38] (step=0025500) Train Loss: 5.9313, Train Steps/Sec: 1.12 + 23%|█████████████████████████████▎ | 25524/110696 [7:46:35<20:59:07, 1.13it/s][2025-04-27 04:50:00] (step=0025525) Train Loss: 5.9853, Train Steps/Sec: 1.12 + 23%|█████████████████████████████▎ | 25549/110696 [7:46:57<21:02:54, 1.12it/s][2025-04-27 04:50:23] (step=0025550) Train Loss: 5.9564, Train Steps/Sec: 1.12 + 23%|█████████████████████████████▎ | 25574/110696 [7:47:20<20:55:05, 1.13it/s][2025-04-27 04:50:45] (step=0025575) Train Loss: 6.0570, Train Steps/Sec: 1.12 + 23%|█████████████████████████████▎ | 25599/110696 [7:47:42<20:56:52, 1.13it/s][2025-04-27 04:51:07] (step=0025600) Train Loss: 5.9960, Train Steps/Sec: 1.11 + 23%|█████████████████████████████▍ | 25624/110696 [7:48:04<21:32:09, 1.10it/s][2025-04-27 04:51:30] (step=0025625) Train Loss: 5.9491, Train Steps/Sec: 1.12 + 23%|█████████████████████████████▍ | 25649/110696 [7:48:27<21:16:49, 1.11it/s][2025-04-27 04:51:52] (step=0025650) Train Loss: 5.9249, Train Steps/Sec: 1.12 + 23%|█████████████████████████████▍ | 25674/110696 [7:48:49<21:08:53, 1.12it/s][2025-04-27 04:52:14] (step=0025675) Train Loss: 5.9751, Train Steps/Sec: 1.12 + 23%|█████████████████████████████▍ | 25699/110696 [7:49:11<21:01:58, 1.12it/s][2025-04-27 04:52:37] (step=0025700) Train Loss: 5.9811, Train Steps/Sec: 1.12 + 23%|█████████████████████████████▌ | 25724/110696 [7:49:34<20:58:12, 1.13it/s][2025-04-27 04:52:59] (step=0025725) Train Loss: 5.9672, Train Steps/Sec: 1.12 + 23%|█████████████████████████████▌ | 25749/110696 [7:49:56<20:57:31, 1.13it/s][2025-04-27 04:53:21] (step=0025750) Train Loss: 5.9692, Train Steps/Sec: 1.12 + 23%|█████████████████████████████▌ | 25774/110696 [7:50:18<20:48:18, 1.13it/s][2025-04-27 04:53:44] (step=0025775) Train Loss: 5.9547, Train Steps/Sec: 1.12 + 23%|█████████████████████████████▌ | 25799/110696 [7:50:41<20:50:28, 1.13it/s][2025-04-27 04:54:06] (step=0025800) Train Loss: 5.9650, Train Steps/Sec: 1.12 + 23%|█████████████████████████████▋ | 25824/110696 [7:51:03<21:32:03, 1.09it/s][2025-04-27 04:54:28] (step=0025825) Train Loss: 5.9397, Train Steps/Sec: 1.12 + 23%|█████████████████████████████▋ | 25849/110696 [7:51:25<21:13:40, 1.11it/s][2025-04-27 04:54:51] (step=0025850) Train Loss: 6.0198, Train Steps/Sec: 1.12 + 23%|█████████████████████████████▋ | 25874/110696 [7:51:48<21:04:43, 1.12it/s][2025-04-27 04:55:13] (step=0025875) Train Loss: 6.0176, Train Steps/Sec: 1.12 + 23%|█████████████████████████████▋ | 25899/110696 [7:52:10<20:56:56, 1.12it/s][2025-04-27 04:55:35] (step=0025900) Train Loss: 5.9177, Train Steps/Sec: 1.12 + 23%|█████████████████████████████▋ | 25924/110696 [7:52:32<20:53:34, 1.13it/s][2025-04-27 04:55:58] (step=0025925) Train Loss: 5.9429, Train Steps/Sec: 1.12 + 23%|█████████████████████████████▊ | 25949/110696 [7:52:55<20:52:22, 1.13it/s][2025-04-27 04:56:20] (step=0025950) Train Loss: 6.0261, Train Steps/Sec: 1.12 + 23%|█████████████████████████████▊ | 25974/110696 [7:53:17<20:53:51, 1.13it/s][2025-04-27 04:56:42] (step=0025975) Train Loss: 5.9616, Train Steps/Sec: 1.12 + 23%|█████████████████████████████▊ | 25999/110696 [7:53:39<20:50:53, 1.13it/s][2025-04-27 04:57:05] (step=0026000) Train Loss: 5.9657, Train Steps/Sec: 1.11 +You are using a model of type blip-2 to instantiate a model of type instructblip. This is not supported for all configurations of models and can yield errors. +[2025-04-27 04:57:05] text_config is None. Initializing the text config with default values (`OPTConfig`). +model_name_or_path:/tmp/haozhezhao/model/blip2-flan-t5-xl +image_place_holder: +tokenizer length after expend 32101 +tokenizer length before expend 32100 +100%|████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████| 5/5 [04:57<00:00, 59.42s/it] +[2025-04-27 05:03:13] Finish Eval in 26000 steps...█████████████████████████████████████████████████████████████████████████████████████████████| 5/5 [04:56<00:00, 59.12s/it] +processing left-padding... +processing left-padding... +processing left-padding... +processing left-padding... +processing left-padding... +[2025-04-27 05:03:32] Saved checkpoint to checkpoint/X2I_UltraEdiit_fourmask_LLava_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_filtered_objects_4mask_1280_context/003-GPT-XL/checkpoints/0026000.pt +[2025-04-27 05:03:34] Removed old checkpoint: checkpoint/X2I_UltraEdiit_fourmask_LLava_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_filtered_objects_4mask_1280_context/003-GPT-XL/checkpoints/0024000.pt + 24%|█████████████████████████████▊ | 26024/110696 [8:00:32<21:57:09, 1.07it/s][2025-04-27 05:03:57] (step=0026025) Train Loss: 6.0213, Train Steps/Sec: 0.06 + 24%|█████████████████████████████▉ | 26049/110696 [8:00:54<21:12:03, 1.11it/s][2025-04-27 05:04:19] (step=0026050) Train Loss: 6.0009, Train Steps/Sec: 1.12 + 24%|█████████████████████████████▉ | 26074/110696 [8:01:16<21:00:06, 1.12it/s][2025-04-27 05:04:42] (step=0026075) Train Loss: 5.9480, Train Steps/Sec: 1.12 + 24%|█████████████████████████████▉ | 26099/110696 [8:01:39<21:00:44, 1.12it/s][2025-04-27 05:05:04] (step=0026100) Train Loss: 5.9998, Train Steps/Sec: 1.12 + 24%|█████████████████████████████▉ | 26124/110696 [8:02:01<20:53:37, 1.12it/s][2025-04-27 05:05:26] (step=0026125) Train Loss: 5.9703, Train Steps/Sec: 1.12 + 24%|██████████████████████████████ | 26149/110696 [8:02:23<20:52:23, 1.13it/s][2025-04-27 05:05:49] (step=0026150) Train Loss: 5.9717, Train Steps/Sec: 1.12 + 24%|██████████████████████████████ | 26174/110696 [8:02:46<20:48:32, 1.13it/s][2025-04-27 05:06:11] (step=0026175) Train Loss: 5.9451, Train Steps/Sec: 1.11 + 24%|██████████████████████████████ | 26199/110696 [8:03:08<20:48:07, 1.13it/s][2025-04-27 05:06:33] (step=0026200) Train Loss: 6.0389, Train Steps/Sec: 1.11 + 24%|██████████████████████████████ | 26224/110696 [8:03:30<21:30:12, 1.09it/s][2025-04-27 05:06:56] (step=0026225) Train Loss: 5.9507, Train Steps/Sec: 1.12 + 24%|██████████████████████████████ | 26249/110696 [8:03:53<21:13:14, 1.11it/s][2025-04-27 05:07:18] (step=0026250) Train Loss: 5.9718, Train Steps/Sec: 1.12 + 24%|██████████████████████████████▏ | 26274/110696 [8:04:15<21:04:55, 1.11it/s][2025-04-27 05:07:41] (step=0026275) Train Loss: 5.9407, Train Steps/Sec: 1.12 + 24%|██████████████████████████████▏ | 26299/110696 [8:04:37<20:56:35, 1.12it/s][2025-04-27 05:08:03] (step=0026300) Train Loss: 5.9975, Train Steps/Sec: 1.12 + 24%|██████████████████████████████▏ | 26324/110696 [8:05:00<20:49:42, 1.13it/s][2025-04-27 05:08:25] (step=0026325) Train Loss: 6.0209, Train Steps/Sec: 1.12 + 24%|██████████████████████████████▏ | 26349/110696 [8:05:22<20:46:22, 1.13it/s][2025-04-27 05:08:48] (step=0026350) Train Loss: 5.9944, Train Steps/Sec: 1.12 + 24%|██████████████████████████████▎ | 26374/110696 [8:05:45<20:44:11, 1.13it/s][2025-04-27 05:09:10] (step=0026375) Train Loss: 5.9834, Train Steps/Sec: 1.12 + 24%|██████████████████████████████▎ | 26399/110696 [8:06:07<20:46:41, 1.13it/s][2025-04-27 05:09:32] (step=0026400) Train Loss: 6.0093, Train Steps/Sec: 1.11 + 24%|██████████████████████████████▎ | 26424/110696 [8:06:29<21:21:52, 1.10it/s][2025-04-27 05:09:55] (step=0026425) Train Loss: 5.9326, Train Steps/Sec: 1.12 + 24%|██████████████████████████████▎ | 26449/110696 [8:06:52<21:02:27, 1.11it/s][2025-04-27 05:10:17] (step=0026450) Train Loss: 5.9625, Train Steps/Sec: 1.12 + 24%|██████████████████████████████▎ | 26474/110696 [8:07:14<20:57:11, 1.12it/s][2025-04-27 05:10:39] (step=0026475) Train Loss: 5.9818, Train Steps/Sec: 1.12 + 24%|██████████████████████████████▍ | 26499/110696 [8:07:36<20:52:50, 1.12it/s][2025-04-27 05:11:02] (step=0026500) Train Loss: 6.0021, Train Steps/Sec: 1.12 + 24%|██████████████████████████████▍ | 26524/110696 [8:07:59<20:45:48, 1.13it/s][2025-04-27 05:11:24] (step=0026525) Train Loss: 5.9601, Train Steps/Sec: 1.12 + 24%|██████████████████████████████▍ | 26549/110696 [8:08:21<20:45:08, 1.13it/s][2025-04-27 05:11:46] (step=0026550) Train Loss: 5.9738, Train Steps/Sec: 1.12 + 24%|██████████████████████████████▍ | 26574/110696 [8:08:43<20:44:25, 1.13it/s][2025-04-27 05:12:09] (step=0026575) Train Loss: 5.9452, Train Steps/Sec: 1.12 + 24%|██████████████████████████████▌ | 26599/110696 [8:09:06<20:44:15, 1.13it/s][2025-04-27 05:12:31] (step=0026600) Train Loss: 6.0132, Train Steps/Sec: 1.11 + 24%|██████████████████████████████▌ | 26624/110696 [8:09:28<21:15:06, 1.10it/s][2025-04-27 05:12:54] (step=0026625) Train Loss: 5.9961, Train Steps/Sec: 1.12 + 24%|██████████████████████████████▌ | 26649/110696 [8:09:51<21:01:56, 1.11it/s][2025-04-27 05:13:16] (step=0026650) Train Loss: 5.9577, Train Steps/Sec: 1.12 + 24%|██████████████████████████████▌ | 26674/110696 [8:10:13<20:52:27, 1.12it/s][2025-04-27 05:13:38] (step=0026675) Train Loss: 6.0203, Train Steps/Sec: 1.12 + 24%|██████████████████████████████▋ | 26699/110696 [8:10:35<20:48:39, 1.12it/s][2025-04-27 05:14:01] (step=0026700) Train Loss: 5.8888, Train Steps/Sec: 1.12 + 24%|██████████████████████████████▋ | 26724/110696 [8:10:58<20:44:23, 1.12it/s][2025-04-27 05:14:23] (step=0026725) Train Loss: 6.0009, Train Steps/Sec: 1.12 + 24%|██████████████████████████████▋ | 26749/110696 [8:11:20<20:44:41, 1.12it/s][2025-04-27 05:14:45] (step=0026750) Train Loss: 5.9705, Train Steps/Sec: 1.12 + 24%|██████████████████████████████▋ | 26774/110696 [8:11:42<20:40:34, 1.13it/s][2025-04-27 05:15:08] (step=0026775) Train Loss: 5.9025, Train Steps/Sec: 1.12 + 24%|██████████████████████████████▋ | 26799/110696 [8:12:05<20:34:45, 1.13it/s][2025-04-27 05:15:30] (step=0026800) Train Loss: 5.9646, Train Steps/Sec: 1.11 + 24%|██████████████████████████████▊ | 26824/110696 [8:12:27<21:12:20, 1.10it/s][2025-04-27 05:15:52] (step=0026825) Train Loss: 5.9788, Train Steps/Sec: 1.12 + 24%|██████████████████████████████▊ | 26849/110696 [8:12:49<21:00:15, 1.11it/s][2025-04-27 05:16:15] (step=0026850) Train Loss: 6.0318, Train Steps/Sec: 1.12 + 24%|██████████████████████████████▊ | 26874/110696 [8:13:12<20:55:57, 1.11it/s][2025-04-27 05:16:37] (step=0026875) Train Loss: 6.0197, Train Steps/Sec: 1.12 + 24%|██████████████████████████████▊ | 26899/110696 [8:13:34<20:45:17, 1.12it/s][2025-04-27 05:16:59] (step=0026900) Train Loss: 6.0363, Train Steps/Sec: 1.12 + 24%|██████████████████████████████▉ | 26924/110696 [8:13:56<20:41:25, 1.12it/s][2025-04-27 05:17:22] (step=0026925) Train Loss: 5.9419, Train Steps/Sec: 1.12 + 24%|██████████████████████████████▉ | 26949/110696 [8:14:19<20:33:42, 1.13it/s][2025-04-27 05:17:44] (step=0026950) Train Loss: 5.9247, Train Steps/Sec: 1.12 + 24%|██████████████████████████████▉ | 26974/110696 [8:14:41<20:36:04, 1.13it/s][2025-04-27 05:18:06] (step=0026975) Train Loss: 5.9355, Train Steps/Sec: 1.12 + 24%|██████████████████████████████▉ | 26999/110696 [8:15:03<20:32:46, 1.13it/s][2025-04-27 05:18:29] (step=0027000) Train Loss: 5.9529, Train Steps/Sec: 1.11 + 24%|███████████████████████████████ | 27024/110696 [8:15:26<21:08:28, 1.10it/s][2025-04-27 05:18:51] (step=0027025) Train Loss: 5.8630, Train Steps/Sec: 1.12 + 24%|███████████████████████████████ | 27049/110696 [8:15:48<20:58:56, 1.11it/s][2025-04-27 05:19:14] (step=0027050) Train Loss: 5.9836, Train Steps/Sec: 1.12 + 24%|███████████████████████████████ | 27074/110696 [8:16:11<20:48:33, 1.12it/s][2025-04-27 05:19:36] (step=0027075) Train Loss: 5.9260, Train Steps/Sec: 1.12 + 24%|███████████████████████████████ | 27099/110696 [8:16:33<20:46:06, 1.12it/s][2025-04-27 05:19:58] (step=0027100) Train Loss: 5.9440, Train Steps/Sec: 1.12 + 25%|███████████████████████████████ | 27124/110696 [8:16:55<20:37:25, 1.13it/s][2025-04-27 05:20:21] (step=0027125) Train Loss: 5.9308, Train Steps/Sec: 1.12 + 25%|███████████████████████████████▏ | 27149/110696 [8:17:18<20:38:32, 1.12it/s][2025-04-27 05:20:43] (step=0027150) Train Loss: 5.9846, Train Steps/Sec: 1.12 + 25%|███████████████████████████████▏ | 27174/110696 [8:17:40<20:37:59, 1.12it/s][2025-04-27 05:21:05] (step=0027175) Train Loss: 5.9849, Train Steps/Sec: 1.12 + 25%|███████████████████████████████▏ | 27199/110696 [8:18:02<20:31:27, 1.13it/s][2025-04-27 05:21:28] (step=0027200) Train Loss: 6.0299, Train Steps/Sec: 1.11 + 25%|███████████████████████████████▏ | 27224/110696 [8:18:25<21:09:28, 1.10it/s][2025-04-27 05:21:50] (step=0027225) Train Loss: 5.9732, Train Steps/Sec: 1.12 + 25%|███████████████████████████████▎ | 27249/110696 [8:18:47<20:55:11, 1.11it/s][2025-04-27 05:22:12] (step=0027250) Train Loss: 5.9767, Train Steps/Sec: 1.12 + 25%|███████████████████████████████▎ | 27274/110696 [8:19:09<20:44:12, 1.12it/s][2025-04-27 05:22:35] (step=0027275) Train Loss: 5.9951, Train Steps/Sec: 1.12 + 25%|███████████████████████████████▎ | 27299/110696 [8:19:32<20:36:44, 1.12it/s][2025-04-27 05:22:57] (step=0027300) Train Loss: 6.0032, Train Steps/Sec: 1.12 + 25%|███████████████████████████████▎ | 27324/110696 [8:19:54<20:35:29, 1.12it/s][2025-04-27 05:23:20] (step=0027325) Train Loss: 5.9400, Train Steps/Sec: 1.12 + 25%|███████████████████████████████▍ | 27349/110696 [8:20:16<20:31:55, 1.13it/s][2025-04-27 05:23:42] (step=0027350) Train Loss: 6.0271, Train Steps/Sec: 1.12 + 25%|███████████████████████████████▍ | 27374/110696 [8:20:39<20:29:36, 1.13it/s][2025-04-27 05:24:04] (step=0027375) Train Loss: 6.0099, Train Steps/Sec: 1.12 + 25%|███████████████████████████████▍ | 27399/110696 [8:21:01<20:37:24, 1.12it/s][2025-04-27 05:24:27] (step=0027400) Train Loss: 5.9997, Train Steps/Sec: 1.11 + 25%|███████████████████████████████▍ | 27424/110696 [8:21:24<21:04:58, 1.10it/s][2025-04-27 05:24:49] (step=0027425) Train Loss: 5.8848, Train Steps/Sec: 1.12 + 25%|███████████████████████████████▍ | 27449/110696 [8:21:46<20:57:58, 1.10it/s][2025-04-27 05:25:11] (step=0027450) Train Loss: 5.9380, Train Steps/Sec: 1.12 + 25%|███████████████████████████████▌ | 27474/110696 [8:22:08<20:42:23, 1.12it/s][2025-04-27 05:25:34] (step=0027475) Train Loss: 5.9792, Train Steps/Sec: 1.12 + 25%|███████████████████████████████▌ | 27499/110696 [8:22:31<20:39:10, 1.12it/s][2025-04-27 05:25:56] (step=0027500) Train Loss: 5.9510, Train Steps/Sec: 1.12 + 25%|███████████████████████████████▌ | 27524/110696 [8:22:53<20:31:58, 1.13it/s][2025-04-27 05:26:18] (step=0027525) Train Loss: 6.0163, Train Steps/Sec: 1.12 + 25%|███████████████████████████████▌ | 27549/110696 [8:23:15<20:32:37, 1.12it/s][2025-04-27 05:26:41] (step=0027550) Train Loss: 5.9872, Train Steps/Sec: 1.12 + 25%|███████████████████████████████▋ | 27574/110696 [8:23:38<20:28:46, 1.13it/s][2025-04-27 05:27:03] (step=0027575) Train Loss: 5.9507, Train Steps/Sec: 1.12 + 25%|███████████████████████████████▋ | 27599/110696 [8:24:00<20:24:29, 1.13it/s][2025-04-27 05:27:26] (step=0027600) Train Loss: 6.0138, Train Steps/Sec: 1.11 + 25%|███████████████████████████████▋ | 27624/110696 [8:24:22<21:02:11, 1.10it/s][2025-04-27 05:27:48] (step=0027625) Train Loss: 5.9893, Train Steps/Sec: 1.12 + 25%|███████████████████████████████▋ | 27649/110696 [8:24:45<20:46:29, 1.11it/s][2025-04-27 05:28:10] (step=0027650) Train Loss: 5.9907, Train Steps/Sec: 1.12 + 25%|███████████████████████████████▊ | 27674/110696 [8:25:07<20:40:46, 1.12it/s][2025-04-27 05:28:33] (step=0027675) Train Loss: 6.0040, Train Steps/Sec: 1.12 + 25%|███████████████████████████████▊ | 27699/110696 [8:25:30<20:33:19, 1.12it/s][2025-04-27 05:28:55] (step=0027700) Train Loss: 5.9808, Train Steps/Sec: 1.12 + 25%|███████████████████████████████▊ | 27724/110696 [8:25:52<20:33:04, 1.12it/s][2025-04-27 05:29:17] (step=0027725) Train Loss: 5.9444, Train Steps/Sec: 1.12 + 25%|███████████████████████████████▊ | 27749/110696 [8:26:14<20:28:35, 1.13it/s][2025-04-27 05:29:40] (step=0027750) Train Loss: 5.9672, Train Steps/Sec: 1.12 + 25%|███████████████████████████████▊ | 27774/110696 [8:26:37<20:21:56, 1.13it/s][2025-04-27 05:30:02] (step=0027775) Train Loss: 6.0021, Train Steps/Sec: 1.12 + 25%|███████████████████████████████▉ | 27799/110696 [8:26:59<20:22:29, 1.13it/s][2025-04-27 05:30:24] (step=0027800) Train Loss: 6.0012, Train Steps/Sec: 1.11 + 25%|███████████████████████████████▉ | 27824/110696 [8:27:26<33:07:06, 1.44s/it][2025-04-27 05:30:52] (step=0027825) Train Loss: 5.9896, Train Steps/Sec: 0.91 + 25%|███████████████████████████████▉ | 27849/110696 [8:27:49<20:53:52, 1.10it/s][2025-04-27 05:31:14] (step=0027850) Train Loss: 6.0014, Train Steps/Sec: 1.12 + 25%|███████████████████████████████▉ | 27874/110696 [8:28:11<20:38:37, 1.11it/s][2025-04-27 05:31:37] (step=0027875) Train Loss: 5.9006, Train Steps/Sec: 1.12 + 25%|████████████████████████████████ | 27899/110696 [8:28:34<20:28:42, 1.12it/s][2025-04-27 05:31:59] (step=0027900) Train Loss: 5.9942, Train Steps/Sec: 1.12 + 25%|████████████████████████████████ | 27924/110696 [8:28:56<20:29:37, 1.12it/s][2025-04-27 05:32:21] (step=0027925) Train Loss: 5.9802, Train Steps/Sec: 1.12 + 25%|████████████████████████████████ | 27949/110696 [8:29:18<20:20:59, 1.13it/s][2025-04-27 05:32:44] (step=0027950) Train Loss: 5.9469, Train Steps/Sec: 1.12 + 25%|████████████████████████████████ | 27974/110696 [8:29:41<20:23:08, 1.13it/s][2025-04-27 05:33:06] (step=0027975) Train Loss: 6.0242, Train Steps/Sec: 1.12 + 25%|████████████████████████████████ | 27999/110696 [8:30:03<20:18:45, 1.13it/s][2025-04-27 05:33:28] (step=0028000) Train Loss: 5.9704, Train Steps/Sec: 1.12 +You are using a model of type blip-2 to instantiate a model of type instructblip. This is not supported for all configurations of models and can yield errors. +[2025-04-27 05:33:28] text_config is None. Initializing the text config with default values (`OPTConfig`). +model_name_or_path:/tmp/haozhezhao/model/blip2-flan-t5-xl +image_place_holder: +tokenizer length after expend 32101 +tokenizer length before expend 32100 +100%|████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████| 5/5 [04:56<00:00, 59.25s/it] +[2025-04-27 05:39:35] Finish Eval in 28000 steps...█████████████████████████████████████████████████████████████████████████████████████████████| 5/5 [04:55<00:00, 59.10s/it] +processing left-padding... +processing left-padding... +processing left-padding... +processing left-padding... +processing left-padding... +[2025-04-27 05:39:55] Saved checkpoint to checkpoint/X2I_UltraEdiit_fourmask_LLava_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_filtered_objects_4mask_1280_context/003-GPT-XL/checkpoints/0028000.pt +[2025-04-27 05:39:57] Removed old checkpoint: checkpoint/X2I_UltraEdiit_fourmask_LLava_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_filtered_objects_4mask_1280_context/003-GPT-XL/checkpoints/0026000.pt + 25%|████████████████████████████████▏ | 28024/110696 [8:36:54<21:25:57, 1.07it/s][2025-04-27 05:40:19] (step=0028025) Train Loss: 5.9524, Train Steps/Sec: 0.06 + 25%|████████████████████████████████▏ | 28049/110696 [8:37:16<20:40:47, 1.11it/s][2025-04-27 05:40:41] (step=0028050) Train Loss: 5.9922, Train Steps/Sec: 1.12 + 25%|████████████████████████████████▏ | 28074/110696 [8:37:38<20:32:59, 1.12it/s][2025-04-27 05:41:04] (step=0028075) Train Loss: 5.9886, Train Steps/Sec: 1.12 + 25%|████████████████████████████████▏ | 28099/110696 [8:38:01<20:28:50, 1.12it/s][2025-04-27 05:41:26] (step=0028100) Train Loss: 5.9120, Train Steps/Sec: 1.12 + 25%|████████████████████████████████▎ | 28124/110696 [8:38:23<20:20:21, 1.13it/s][2025-04-27 05:41:48] (step=0028125) Train Loss: 5.9613, Train Steps/Sec: 1.12 + 25%|████████████████████████████████▎ | 28149/110696 [8:38:45<20:16:40, 1.13it/s][2025-04-27 05:42:11] (step=0028150) Train Loss: 5.9898, Train Steps/Sec: 1.12 + 25%|████████████████████████████████▎ | 28174/110696 [8:39:08<20:13:59, 1.13it/s][2025-04-27 05:42:33] (step=0028175) Train Loss: 5.9541, Train Steps/Sec: 1.12 + 25%|████████████████████████████████▎ | 28199/110696 [8:39:30<20:17:11, 1.13it/s][2025-04-27 05:42:55] (step=0028200) Train Loss: 5.8745, Train Steps/Sec: 1.12 + 25%|████████████████████████████████▍ | 28224/110696 [8:39:52<20:52:42, 1.10it/s][2025-04-27 05:43:18] (step=0028225) Train Loss: 5.9450, Train Steps/Sec: 1.12 + 26%|████████████████████████████████▍ | 28249/110696 [8:40:15<20:37:10, 1.11it/s][2025-04-27 05:43:40] (step=0028250) Train Loss: 5.9820, Train Steps/Sec: 1.12 + 26%|████████████████████████████████▍ | 28274/110696 [8:40:42<20:31:57, 1.12it/s][2025-04-27 05:44:07] (step=0028275) Train Loss: 5.9357, Train Steps/Sec: 0.92 + 26%|████████████████████████████████▍ | 28299/110696 [8:41:09<23:05:10, 1.01s/it][2025-04-27 05:44:34] (step=0028300) Train Loss: 5.9925, Train Steps/Sec: 0.92 + 26%|████████████████████████████████▍ | 28324/110696 [8:41:31<20:25:00, 1.12it/s][2025-04-27 05:44:56] (step=0028325) Train Loss: 5.9208, Train Steps/Sec: 1.12 + 26%|████████████████████████████████▌ | 28349/110696 [8:41:53<20:15:33, 1.13it/s][2025-04-27 05:45:19] (step=0028350) Train Loss: 5.9240, Train Steps/Sec: 1.12 + 26%|████████████████████████████████▌ | 28374/110696 [8:42:36<22:10:52, 1.03it/s][2025-04-27 05:46:01] (step=0028375) Train Loss: 5.9688, Train Steps/Sec: 0.59 + 26%|████████████████████████████████▌ | 28399/110696 [8:42:58<20:13:48, 1.13it/s][2025-04-27 05:46:24] (step=0028400) Train Loss: 6.0068, Train Steps/Sec: 1.12 + 26%|████████████████████████████████▌ | 28424/110696 [8:43:21<20:52:02, 1.10it/s][2025-04-27 05:46:46] (step=0028425) Train Loss: 5.9415, Train Steps/Sec: 1.12 + 26%|████████████████████████████████▋ | 28449/110696 [8:43:43<20:36:42, 1.11it/s][2025-04-27 05:47:08] (step=0028450) Train Loss: 6.0259, Train Steps/Sec: 1.12 + 26%|████████████████████████████████▋ | 28474/110696 [8:44:05<20:24:28, 1.12it/s][2025-04-27 05:47:30] (step=0028475) Train Loss: 5.9302, Train Steps/Sec: 1.12 + 26%|████████████████████████████████▋ | 28499/110696 [8:44:27<20:16:58, 1.13it/s][2025-04-27 05:47:53] (step=0028500) Train Loss: 5.9244, Train Steps/Sec: 1.12 + 26%|████████████████████████████████▋ | 28524/110696 [8:44:50<20:19:25, 1.12it/s][2025-04-27 05:48:15] (step=0028525) Train Loss: 5.9709, Train Steps/Sec: 1.12 + 26%|████████████████████████████████▊ | 28549/110696 [8:45:12<20:15:51, 1.13it/s][2025-04-27 05:48:37] (step=0028550) Train Loss: 5.9418, Train Steps/Sec: 1.12 + 26%|████████████████████████████████▊ | 28574/110696 [8:45:34<20:16:44, 1.12it/s][2025-04-27 05:49:00] (step=0028575) Train Loss: 5.9702, Train Steps/Sec: 1.12 + 26%|████████████████████████████████▊ | 28599/110696 [8:45:57<20:13:01, 1.13it/s][2025-04-27 05:49:22] (step=0028600) Train Loss: 5.9469, Train Steps/Sec: 1.11 + 26%|████████████████████████████████▊ | 28624/110696 [8:46:19<20:50:16, 1.09it/s][2025-04-27 05:49:45] (step=0028625) Train Loss: 5.9887, Train Steps/Sec: 1.12 + 26%|████████████████████████████████▊ | 28649/110696 [8:46:41<20:31:48, 1.11it/s][2025-04-27 05:50:07] (step=0028650) Train Loss: 6.0052, Train Steps/Sec: 1.12 + 26%|████████████████████████████████▉ | 28674/110696 [8:47:04<20:25:27, 1.12it/s][2025-04-27 05:50:29] (step=0028675) Train Loss: 5.9485, Train Steps/Sec: 1.12 + 26%|████████████████████████████████▉ | 28699/110696 [8:47:26<20:18:06, 1.12it/s][2025-04-27 05:50:52] (step=0028700) Train Loss: 5.9754, Train Steps/Sec: 1.12 + 26%|████████████████████████████████▉ | 28724/110696 [8:47:49<20:12:07, 1.13it/s][2025-04-27 05:51:14] (step=0028725) Train Loss: 5.9627, Train Steps/Sec: 1.12 + 26%|████████████████████████████████▉ | 28749/110696 [8:48:11<20:11:14, 1.13it/s][2025-04-27 05:51:36] (step=0028750) Train Loss: 5.8926, Train Steps/Sec: 1.12 + 26%|█████████████████████████████████ | 28774/110696 [8:48:33<20:10:13, 1.13it/s][2025-04-27 05:51:59] (step=0028775) Train Loss: 5.9175, Train Steps/Sec: 1.12 + 26%|█████████████████████████████████ | 28799/110696 [8:48:56<20:01:52, 1.14it/s][2025-04-27 05:52:21] (step=0028800) Train Loss: 5.9382, Train Steps/Sec: 1.12 + 26%|█████████████████████████████████ | 28824/110696 [8:49:18<20:42:57, 1.10it/s][2025-04-27 05:52:43] (step=0028825) Train Loss: 5.9209, Train Steps/Sec: 1.12 + 26%|█████████████████████████████████ | 28849/110696 [8:49:40<20:28:45, 1.11it/s][2025-04-27 05:53:06] (step=0028850) Train Loss: 5.9417, Train Steps/Sec: 1.12 + 26%|█████████████████████████████████▏ | 28874/110696 [8:50:03<20:20:50, 1.12it/s][2025-04-27 05:53:28] (step=0028875) Train Loss: 5.9136, Train Steps/Sec: 1.12 + 26%|█████████████████████████████████▏ | 28899/110696 [8:50:25<20:15:30, 1.12it/s][2025-04-27 05:53:50] (step=0028900) Train Loss: 5.9747, Train Steps/Sec: 1.12 + 26%|█████████████████████████████████▏ | 28924/110696 [8:50:47<20:13:23, 1.12it/s][2025-04-27 05:54:13] (step=0028925) Train Loss: 5.9959, Train Steps/Sec: 1.12 + 26%|█████████████████████████████████▏ | 28949/110696 [8:51:10<20:07:48, 1.13it/s][2025-04-27 05:54:35] (step=0028950) Train Loss: 5.9354, Train Steps/Sec: 1.12 + 26%|█████████████████████████████████▏ | 28974/110696 [8:51:32<20:04:07, 1.13it/s][2025-04-27 05:54:57] (step=0028975) Train Loss: 5.9493, Train Steps/Sec: 1.12 + 26%|█████████████████████████████████▎ | 28999/110696 [8:51:54<20:06:11, 1.13it/s][2025-04-27 05:55:20] (step=0029000) Train Loss: 5.9853, Train Steps/Sec: 1.11 + 26%|█████████████████████████████████▎ | 29024/110696 [8:52:17<20:38:52, 1.10it/s][2025-04-27 05:55:42] (step=0029025) Train Loss: 6.0035, Train Steps/Sec: 1.12 + 26%|█████████████████████████████████▎ | 29049/110696 [8:52:39<20:26:04, 1.11it/s][2025-04-27 05:56:04] (step=0029050) Train Loss: 5.9381, Train Steps/Sec: 1.12 + 26%|█████████████████████████████████▎ | 29074/110696 [8:53:01<20:15:52, 1.12it/s][2025-04-27 05:56:27] (step=0029075) Train Loss: 5.8981, Train Steps/Sec: 1.12 + 26%|█████████████████████████████████▍ | 29099/110696 [8:53:24<20:16:44, 1.12it/s][2025-04-27 05:56:49] (step=0029100) Train Loss: 5.9344, Train Steps/Sec: 1.12 + 26%|█████████████████████████████████▍ | 29124/110696 [8:53:46<20:13:48, 1.12it/s][2025-04-27 05:57:12] (step=0029125) Train Loss: 5.9714, Train Steps/Sec: 1.12 + 26%|█████████████████████████████████▍ | 29149/110696 [8:54:09<20:09:34, 1.12it/s][2025-04-27 05:57:34] (step=0029150) Train Loss: 5.9647, Train Steps/Sec: 1.12 + 26%|█████████████████████████████████▍ | 29174/110696 [8:54:31<20:05:52, 1.13it/s][2025-04-27 05:57:56] (step=0029175) Train Loss: 5.9535, Train Steps/Sec: 1.12 + 26%|█████████████████████████████████▍ | 29199/110696 [8:54:53<19:59:48, 1.13it/s][2025-04-27 05:58:19] (step=0029200) Train Loss: 5.9645, Train Steps/Sec: 1.12 + 26%|█████████████████████████████████▌ | 29224/110696 [8:55:16<20:41:04, 1.09it/s][2025-04-27 05:58:41] (step=0029225) Train Loss: 5.9774, Train Steps/Sec: 1.12 + 26%|█████████████████████████████████▌ | 29249/110696 [8:55:38<20:21:15, 1.11it/s][2025-04-27 05:59:03] (step=0029250) Train Loss: 5.9227, Train Steps/Sec: 1.12 + 26%|█████████████████████████████████▌ | 29274/110696 [8:56:00<20:16:42, 1.12it/s][2025-04-27 05:59:26] (step=0029275) Train Loss: 5.8993, Train Steps/Sec: 1.12 + 26%|█████████████████████████████████▌ | 29299/110696 [8:56:23<20:12:28, 1.12it/s][2025-04-27 05:59:48] (step=0029300) Train Loss: 5.9319, Train Steps/Sec: 1.12 + 26%|█████████████████████████████████▋ | 29324/110696 [8:56:45<20:00:49, 1.13it/s][2025-04-27 06:00:10] (step=0029325) Train Loss: 5.8672, Train Steps/Sec: 1.12 + 27%|█████████████████████████████████▋ | 29349/110696 [8:57:07<20:04:12, 1.13it/s][2025-04-27 06:00:33] (step=0029350) Train Loss: 5.9332, Train Steps/Sec: 1.12 + 27%|█████████████████████████████████▋ | 29374/110696 [8:57:30<19:59:52, 1.13it/s][2025-04-27 06:00:55] (step=0029375) Train Loss: 5.9178, Train Steps/Sec: 1.12 + 27%|█████████████████████████████████▋ | 29399/110696 [8:57:52<20:00:48, 1.13it/s][2025-04-27 06:01:17] (step=0029400) Train Loss: 5.8886, Train Steps/Sec: 1.11 + 27%|█████████████████████████████████▊ | 29424/110696 [8:58:14<20:36:47, 1.10it/s][2025-04-27 06:01:40] (step=0029425) Train Loss: 5.9060, Train Steps/Sec: 1.12 + 27%|█████████████████████████████████▊ | 29449/110696 [8:58:37<20:17:02, 1.11it/s][2025-04-27 06:02:02] (step=0029450) Train Loss: 5.9906, Train Steps/Sec: 1.12 + 27%|█████████████████████████████████▊ | 29474/110696 [8:58:59<20:11:22, 1.12it/s][2025-04-27 06:02:24] (step=0029475) Train Loss: 5.9811, Train Steps/Sec: 1.12 + 27%|█████████████████████████████████▊ | 29499/110696 [8:59:21<20:05:02, 1.12it/s][2025-04-27 06:02:47] (step=0029500) Train Loss: 5.9477, Train Steps/Sec: 1.12 + 27%|█████████████████████████████████▊ | 29524/110696 [8:59:44<20:06:40, 1.12it/s][2025-04-27 06:03:09] (step=0029525) Train Loss: 5.9562, Train Steps/Sec: 1.12 + 27%|█████████████████████████████████▉ | 29549/110696 [9:00:06<20:01:11, 1.13it/s][2025-04-27 06:03:31] (step=0029550) Train Loss: 5.8950, Train Steps/Sec: 1.12 + 27%|█████████████████████████████████▉ | 29574/110696 [9:00:28<19:55:23, 1.13it/s][2025-04-27 06:03:54] (step=0029575) Train Loss: 5.9240, Train Steps/Sec: 1.12 + 27%|█████████████████████████████████▉ | 29599/110696 [9:00:51<19:56:03, 1.13it/s][2025-04-27 06:04:16] (step=0029600) Train Loss: 5.9431, Train Steps/Sec: 1.12 + 27%|█████████████████████████████████▉ | 29624/110696 [9:01:13<20:29:04, 1.10it/s][2025-04-27 06:04:38] (step=0029625) Train Loss: 5.9347, Train Steps/Sec: 1.12 + 27%|██████████████████████████████████ | 29649/110696 [9:01:35<20:15:47, 1.11it/s][2025-04-27 06:05:01] (step=0029650) Train Loss: 5.8375, Train Steps/Sec: 1.12 + 27%|██████████████████████████████████ | 29674/110696 [9:01:58<20:13:48, 1.11it/s][2025-04-27 06:05:23] (step=0029675) Train Loss: 5.9202, Train Steps/Sec: 1.12 + 27%|██████████████████████████████████ | 29699/110696 [9:02:20<20:03:25, 1.12it/s][2025-04-27 06:05:45] (step=0029700) Train Loss: 5.9225, Train Steps/Sec: 1.12 + 27%|██████████████████████████████████ | 29724/110696 [9:02:42<19:57:20, 1.13it/s][2025-04-27 06:06:08] (step=0029725) Train Loss: 5.9233, Train Steps/Sec: 1.12 + 27%|██████████████████████████████████▏ | 29749/110696 [9:03:05<19:51:42, 1.13it/s][2025-04-27 06:06:30] (step=0029750) Train Loss: 5.9356, Train Steps/Sec: 1.12 + 27%|██████████████████████████████████▏ | 29774/110696 [9:03:27<19:55:37, 1.13it/s][2025-04-27 06:06:52] (step=0029775) Train Loss: 5.9415, Train Steps/Sec: 1.12 + 27%|██████████████████████████████████▏ | 29799/110696 [9:03:49<19:53:01, 1.13it/s][2025-04-27 06:07:15] (step=0029800) Train Loss: 5.9775, Train Steps/Sec: 1.12 + 27%|██████████████████████████████████▏ | 29824/110696 [9:04:12<20:27:08, 1.10it/s][2025-04-27 06:07:37] (step=0029825) Train Loss: 6.0007, Train Steps/Sec: 1.12 + 27%|██████████████████████████████████▏ | 29849/110696 [9:04:34<20:17:28, 1.11it/s][2025-04-27 06:07:59] (step=0029850) Train Loss: 5.8922, Train Steps/Sec: 1.12 + 27%|██████████████████████████████████▎ | 29874/110696 [9:04:56<20:11:29, 1.11it/s][2025-04-27 06:08:22] (step=0029875) Train Loss: 5.9272, Train Steps/Sec: 1.12 + 27%|██████████████████████████████████▎ | 29899/110696 [9:05:19<20:03:40, 1.12it/s][2025-04-27 06:08:44] (step=0029900) Train Loss: 5.9051, Train Steps/Sec: 1.12 + 27%|██████████████████████████████████▎ | 29924/110696 [9:05:41<19:59:10, 1.12it/s][2025-04-27 06:09:06] (step=0029925) Train Loss: 5.9718, Train Steps/Sec: 1.12 + 27%|██████████████████████████████████▎ | 29949/110696 [9:06:03<19:55:02, 1.13it/s][2025-04-27 06:09:29] (step=0029950) Train Loss: 5.9474, Train Steps/Sec: 1.12 + 27%|██████████████████████████████████▍ | 29974/110696 [9:06:26<19:53:55, 1.13it/s][2025-04-27 06:09:51] (step=0029975) Train Loss: 5.9660, Train Steps/Sec: 1.11 + 27%|██████████████████████████████████▍ | 29999/110696 [9:06:48<19:51:55, 1.13it/s][2025-04-27 06:10:14] (step=0030000) Train Loss: 5.9675, Train Steps/Sec: 1.11 +You are using a model of type blip-2 to instantiate a model of type instructblip. This is not supported for all configurations of models and can yield errors. +[2025-04-27 06:10:14] text_config is None. Initializing the text config with default values (`OPTConfig`). +model_name_or_path:/tmp/haozhezhao/model/blip2-flan-t5-xl +image_place_holder: +tokenizer length after expend 32101 +tokenizer length before expend 32100 +100%|████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████| 5/5 [04:56<00:00, 59.25s/it] +[2025-04-27 06:16:21] Finish Eval in 30000 steps...█████████████████████████████████████████████████████████████████████████████████████████████| 5/5 [04:55<00:00, 59.05s/it] +processing left-padding... +processing left-padding... +processing left-padding... +processing left-padding... +processing left-padding... +[2025-04-27 06:16:41] Saved checkpoint to checkpoint/X2I_UltraEdiit_fourmask_LLava_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_filtered_objects_4mask_1280_context/003-GPT-XL/checkpoints/0030000.pt +[2025-04-27 06:16:43] Removed old checkpoint: checkpoint/X2I_UltraEdiit_fourmask_LLava_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_filtered_objects_4mask_1280_context/003-GPT-XL/checkpoints/0028000.pt + 27%|██████████████████████████████████▍ | 30024/110696 [9:13:40<21:00:10, 1.07it/s][2025-04-27 06:17:05] (step=0030025) Train Loss: 5.9794, Train Steps/Sec: 0.06 + 27%|██████████████████████████████████▍ | 30049/110696 [9:14:02<20:09:40, 1.11it/s][2025-04-27 06:17:28] (step=0030050) Train Loss: 5.9925, Train Steps/Sec: 1.12 + 27%|██████████████████████████████████▌ | 30074/110696 [9:14:24<20:01:20, 1.12it/s][2025-04-27 06:17:50] (step=0030075) Train Loss: 5.9350, Train Steps/Sec: 1.12 + 27%|██████████████████████████████████▌ | 30099/110696 [9:14:47<19:52:06, 1.13it/s][2025-04-27 06:18:12] (step=0030100) Train Loss: 5.9932, Train Steps/Sec: 1.12 + 27%|██████████████████████████████████▌ | 30124/110696 [9:15:09<19:54:18, 1.12it/s][2025-04-27 06:18:34] (step=0030125) Train Loss: 5.9421, Train Steps/Sec: 1.12 + 27%|██████████████████████████████████▌ | 30149/110696 [9:15:31<19:50:09, 1.13it/s][2025-04-27 06:18:57] (step=0030150) Train Loss: 5.9626, Train Steps/Sec: 1.12 + 27%|██████████████████████████████████▌ | 30174/110696 [9:15:54<19:48:19, 1.13it/s][2025-04-27 06:19:19] (step=0030175) Train Loss: 5.9365, Train Steps/Sec: 1.12 + 27%|██████████████████████████████████▋ | 30199/110696 [9:16:16<19:49:53, 1.13it/s][2025-04-27 06:19:42] (step=0030200) Train Loss: 5.9442, Train Steps/Sec: 1.11 + 27%|██████████████████████████████████▋ | 30224/110696 [9:16:39<20:24:36, 1.10it/s][2025-04-27 06:20:04] (step=0030225) Train Loss: 5.9771, Train Steps/Sec: 1.11 + 27%|██████████████████████████████████▋ | 30249/110696 [9:17:01<20:03:47, 1.11it/s][2025-04-27 06:20:26] (step=0030250) Train Loss: 5.8949, Train Steps/Sec: 1.12 + 27%|██████████████████████████████████▋ | 30274/110696 [9:17:23<19:57:14, 1.12it/s][2025-04-27 06:20:49] (step=0030275) Train Loss: 6.0196, Train Steps/Sec: 1.12 + 27%|██████████████████████████████████▊ | 30299/110696 [9:17:46<19:55:41, 1.12it/s][2025-04-27 06:21:11] (step=0030300) Train Loss: 5.9784, Train Steps/Sec: 1.12 + 27%|██████████████████████████████████▊ | 30324/110696 [9:18:08<19:49:26, 1.13it/s][2025-04-27 06:21:33] (step=0030325) Train Loss: 5.9627, Train Steps/Sec: 1.12 + 27%|██████████████████████████████████▊ | 30349/110696 [9:18:30<19:52:06, 1.12it/s][2025-04-27 06:21:56] (step=0030350) Train Loss: 5.9548, Train Steps/Sec: 1.12 + 27%|██████████████████████████████████▊ | 30374/110696 [9:18:53<19:49:03, 1.13it/s][2025-04-27 06:22:18] (step=0030375) Train Loss: 5.9799, Train Steps/Sec: 1.12 + 27%|██████████████████████████████████▉ | 30399/110696 [9:19:15<19:43:39, 1.13it/s][2025-04-27 06:22:40] (step=0030400) Train Loss: 5.8935, Train Steps/Sec: 1.11 + 27%|██████████████████████████████████▉ | 30424/110696 [9:19:38<20:25:23, 1.09it/s][2025-04-27 06:23:03] (step=0030425) Train Loss: 5.9629, Train Steps/Sec: 1.12 + 28%|██████████████████████████████████▉ | 30449/110696 [9:20:00<20:01:01, 1.11it/s][2025-04-27 06:23:25] (step=0030450) Train Loss: 5.8692, Train Steps/Sec: 1.12 + 28%|██████████████████████████████████▉ | 30474/110696 [9:20:22<19:55:07, 1.12it/s][2025-04-27 06:23:47] (step=0030475) Train Loss: 5.9449, Train Steps/Sec: 1.12 + 28%|██████████████████████████████████▉ | 30499/110696 [9:20:44<19:52:53, 1.12it/s][2025-04-27 06:24:10] (step=0030500) Train Loss: 5.8719, Train Steps/Sec: 1.12 + 28%|███████████████████████████████████ | 30524/110696 [9:21:07<19:42:25, 1.13it/s][2025-04-27 06:24:32] (step=0030525) Train Loss: 5.9071, Train Steps/Sec: 1.12 + 28%|███████████████████████████████████ | 30549/110696 [9:21:29<19:47:54, 1.12it/s][2025-04-27 06:24:54] (step=0030550) Train Loss: 5.9811, Train Steps/Sec: 1.12 + 28%|███████████████████████████████████ | 30574/110696 [9:21:51<19:42:39, 1.13it/s][2025-04-27 06:25:17] (step=0030575) Train Loss: 5.9040, Train Steps/Sec: 1.12 + 28%|███████████████████████████████████ | 30599/110696 [9:22:14<19:45:25, 1.13it/s][2025-04-27 06:25:39] (step=0030600) Train Loss: 5.8865, Train Steps/Sec: 1.11 + 28%|███████████████████████████████████▏ | 30624/110696 [9:22:36<20:23:49, 1.09it/s][2025-04-27 06:26:02] (step=0030625) Train Loss: 5.9304, Train Steps/Sec: 1.12 + 28%|███████████████████████████████████▏ | 30649/110696 [9:22:59<20:00:57, 1.11it/s][2025-04-27 06:26:24] (step=0030650) Train Loss: 5.9030, Train Steps/Sec: 1.12 + 28%|███████████████████████████████████▏ | 30674/110696 [9:23:21<19:52:41, 1.12it/s][2025-04-27 06:26:46] (step=0030675) Train Loss: 5.8797, Train Steps/Sec: 1.12 + 28%|███████████████████████████████████▏ | 30699/110696 [9:23:43<19:52:18, 1.12it/s][2025-04-27 06:27:09] (step=0030700) Train Loss: 5.8798, Train Steps/Sec: 1.12 + 28%|███████████████████████████████████▏ | 30724/110696 [9:24:06<19:42:56, 1.13it/s][2025-04-27 06:27:31] (step=0030725) Train Loss: 5.9719, Train Steps/Sec: 1.12 + 28%|███████████████████████████████████▎ | 30749/110696 [9:24:28<19:40:02, 1.13it/s][2025-04-27 06:27:53] (step=0030750) Train Loss: 5.9243, Train Steps/Sec: 1.12 + 28%|███████████████████████████████████▎ | 30774/110696 [9:24:50<19:38:57, 1.13it/s][2025-04-27 06:28:16] (step=0030775) Train Loss: 5.9510, Train Steps/Sec: 1.12 + 28%|███████████████████████████████████▎ | 30799/110696 [9:25:13<19:37:26, 1.13it/s][2025-04-27 06:28:38] (step=0030800) Train Loss: 5.9283, Train Steps/Sec: 1.12 + 28%|███████████████████████████████████▎ | 30824/110696 [9:25:35<20:14:52, 1.10it/s][2025-04-27 06:29:00] (step=0030825) Train Loss: 5.8951, Train Steps/Sec: 1.12 + 28%|███████████████████████████████████▍ | 30849/110696 [9:25:57<20:01:04, 1.11it/s][2025-04-27 06:29:23] (step=0030850) Train Loss: 5.9631, Train Steps/Sec: 1.12 + 28%|███████████████████████████████████▍ | 30874/110696 [9:26:20<19:54:45, 1.11it/s][2025-04-27 06:29:45] (step=0030875) Train Loss: 5.9411, Train Steps/Sec: 1.12 + 28%|███████████████████████████████████▍ | 30899/110696 [9:26:42<19:43:39, 1.12it/s][2025-04-27 06:30:07] (step=0030900) Train Loss: 5.8908, Train Steps/Sec: 1.12 + 28%|███████████████████████████████████▍ | 30924/110696 [9:27:04<19:44:53, 1.12it/s][2025-04-27 06:30:30] (step=0030925) Train Loss: 5.9564, Train Steps/Sec: 1.12 + 28%|███████████████████████████████████▌ | 30949/110696 [9:27:27<19:47:16, 1.12it/s][2025-04-27 06:30:52] (step=0030950) Train Loss: 5.9855, Train Steps/Sec: 1.12 + 28%|███████████████████████████████████▌ | 30974/110696 [9:27:49<19:36:14, 1.13it/s][2025-04-27 06:31:14] (step=0030975) Train Loss: 5.8587, Train Steps/Sec: 1.12 + 28%|███████████████████████████████████▌ | 30999/110696 [9:28:11<19:36:45, 1.13it/s][2025-04-27 06:31:37] (step=0031000) Train Loss: 5.8753, Train Steps/Sec: 1.11 + 28%|███████████████████████████████████▌ | 31024/110696 [9:28:34<20:12:44, 1.09it/s][2025-04-27 06:31:59] (step=0031025) Train Loss: 5.9637, Train Steps/Sec: 1.12 + 28%|███████████████████████████████████▌ | 31049/110696 [9:28:56<19:58:02, 1.11it/s][2025-04-27 06:32:21] (step=0031050) Train Loss: 5.9178, Train Steps/Sec: 1.12 + 28%|███████████████████████████████████▋ | 31074/110696 [9:29:18<19:48:28, 1.12it/s][2025-04-27 06:32:44] (step=0031075) Train Loss: 5.9138, Train Steps/Sec: 1.12 + 28%|███████████████████████████████████▋ | 31099/110696 [9:29:41<19:49:05, 1.12it/s][2025-04-27 06:33:06] (step=0031100) Train Loss: 5.9048, Train Steps/Sec: 1.12 + 28%|███████████████████████████████████▋ | 31124/110696 [9:30:03<19:36:26, 1.13it/s][2025-04-27 06:33:28] (step=0031125) Train Loss: 5.9960, Train Steps/Sec: 1.12 + 28%|███████████████████████████████████▋ | 31149/110696 [9:30:25<19:38:10, 1.13it/s][2025-04-27 06:33:51] (step=0031150) Train Loss: 5.9257, Train Steps/Sec: 1.12 + 28%|███████████████████████████████████▊ | 31174/110696 [9:30:48<19:31:59, 1.13it/s][2025-04-27 06:34:13] (step=0031175) Train Loss: 5.9037, Train Steps/Sec: 1.12 + 28%|███████████████████████████████████▊ | 31199/110696 [9:31:10<19:30:09, 1.13it/s][2025-04-27 06:34:35] (step=0031200) Train Loss: 5.9814, Train Steps/Sec: 1.12 + 28%|███████████████████████████████████▊ | 31224/110696 [9:31:32<20:07:13, 1.10it/s][2025-04-27 06:34:58] (step=0031225) Train Loss: 5.9289, Train Steps/Sec: 1.12 + 28%|███████████████████████████████████▊ | 31249/110696 [9:31:55<19:51:09, 1.11it/s][2025-04-27 06:35:20] (step=0031250) Train Loss: 5.9869, Train Steps/Sec: 1.12 + 28%|███████████████████████████████████▉ | 31274/110696 [9:32:17<19:46:04, 1.12it/s][2025-04-27 06:35:42] (step=0031275) Train Loss: 5.9298, Train Steps/Sec: 1.12 + 28%|███████████████████████████████████▉ | 31299/110696 [9:32:39<19:41:28, 1.12it/s][2025-04-27 06:36:05] (step=0031300) Train Loss: 5.9372, Train Steps/Sec: 1.12 + 28%|███████████████████████████████████▉ | 31324/110696 [9:33:02<19:37:46, 1.12it/s][2025-04-27 06:36:27] (step=0031325) Train Loss: 5.9204, Train Steps/Sec: 1.12 + 28%|███████████████████████████████████▉ | 31349/110696 [9:33:24<19:33:41, 1.13it/s][2025-04-27 06:36:49] (step=0031350) Train Loss: 5.9112, Train Steps/Sec: 1.12 + 28%|███████████████████████████████████▉ | 31374/110696 [9:33:46<19:28:44, 1.13it/s][2025-04-27 06:37:12] (step=0031375) Train Loss: 5.9100, Train Steps/Sec: 1.12 + 28%|████████████████████████████████████ | 31399/110696 [9:34:09<19:29:07, 1.13it/s][2025-04-27 06:37:34] (step=0031400) Train Loss: 5.8562, Train Steps/Sec: 1.11 + 28%|████████████████████████████████████ | 31424/110696 [9:34:31<20:05:58, 1.10it/s][2025-04-27 06:37:57] (step=0031425) Train Loss: 5.9711, Train Steps/Sec: 1.12 + 28%|████████████████████████████████████ | 31449/110696 [9:34:53<19:47:42, 1.11it/s][2025-04-27 06:38:19] (step=0031450) Train Loss: 5.9260, Train Steps/Sec: 1.12 + 28%|████████████████████████████████████ | 31474/110696 [9:35:16<19:41:14, 1.12it/s][2025-04-27 06:38:41] (step=0031475) Train Loss: 5.9274, Train Steps/Sec: 1.12 + 28%|████████████████████████████████████▏ | 31499/110696 [9:35:38<19:34:55, 1.12it/s][2025-04-27 06:39:03] (step=0031500) Train Loss: 5.9075, Train Steps/Sec: 1.12 + 28%|████████████████████████████████████▏ | 31524/110696 [9:36:00<19:34:28, 1.12it/s][2025-04-27 06:39:26] (step=0031525) Train Loss: 5.9507, Train Steps/Sec: 1.12 + 29%|████████████████████████████████████▏ | 31549/110696 [9:36:23<19:27:46, 1.13it/s][2025-04-27 06:39:48] (step=0031550) Train Loss: 5.9273, Train Steps/Sec: 1.12 + 29%|████████████████████████████████████▏ | 31574/110696 [9:36:45<19:32:17, 1.12it/s][2025-04-27 06:40:10] (step=0031575) Train Loss: 5.9084, Train Steps/Sec: 1.12 + 29%|████████████████████████████████████▎ | 31599/110696 [9:37:07<19:28:39, 1.13it/s][2025-04-27 06:40:33] (step=0031600) Train Loss: 5.8337, Train Steps/Sec: 1.11 + 29%|████████████████████████████████████▎ | 31624/110696 [9:37:30<19:58:21, 1.10it/s][2025-04-27 06:40:55] (step=0031625) Train Loss: 5.8747, Train Steps/Sec: 1.12 + 29%|████████████████████████████████████▎ | 31649/110696 [9:37:52<19:53:42, 1.10it/s][2025-04-27 06:41:18] (step=0031650) Train Loss: 5.9105, Train Steps/Sec: 1.12 + 29%|████████████████████████████████████▎ | 31674/110696 [9:38:15<19:38:47, 1.12it/s][2025-04-27 06:41:40] (step=0031675) Train Loss: 5.9273, Train Steps/Sec: 1.12 + 29%|████████████████████████████████████▎ | 31699/110696 [9:38:37<19:37:04, 1.12it/s][2025-04-27 06:42:02] (step=0031700) Train Loss: 5.9116, Train Steps/Sec: 1.12 + 29%|████████████████████████████████████▍ | 31724/110696 [9:38:59<19:28:54, 1.13it/s][2025-04-27 06:42:25] (step=0031725) Train Loss: 5.9404, Train Steps/Sec: 1.12 + 29%|████████████████████████████████████▍ | 31749/110696 [9:39:21<19:22:37, 1.13it/s][2025-04-27 06:42:47] (step=0031750) Train Loss: 5.9233, Train Steps/Sec: 1.12 + 29%|████████████████████████████████████▍ | 31774/110696 [9:39:44<19:22:57, 1.13it/s][2025-04-27 06:43:09] (step=0031775) Train Loss: 5.9642, Train Steps/Sec: 1.12 + 29%|████████████████████████████████████▍ | 31799/110696 [9:40:06<19:19:00, 1.13it/s][2025-04-27 06:43:32] (step=0031800) Train Loss: 5.9430, Train Steps/Sec: 1.12 + 29%|████████████████████████████████████▌ | 31824/110696 [9:40:28<19:55:45, 1.10it/s][2025-04-27 06:43:54] (step=0031825) Train Loss: 5.9106, Train Steps/Sec: 1.12 + 29%|████████████████████████████████████▌ | 31849/110696 [9:40:51<19:42:58, 1.11it/s][2025-04-27 06:44:16] (step=0031850) Train Loss: 5.9079, Train Steps/Sec: 1.12 + 29%|████████████████████████████████████▌ | 31874/110696 [9:41:13<19:38:30, 1.11it/s][2025-04-27 06:44:38] (step=0031875) Train Loss: 5.8988, Train Steps/Sec: 1.12 + 29%|████████████████████████████████████▌ | 31899/110696 [9:41:35<19:28:24, 1.12it/s][2025-04-27 06:45:01] (step=0031900) Train Loss: 5.8514, Train Steps/Sec: 1.12 + 29%|████████████████████████████████████▋ | 31924/110696 [9:41:58<19:27:13, 1.12it/s][2025-04-27 06:45:23] (step=0031925) Train Loss: 5.9318, Train Steps/Sec: 1.12 + 29%|████████████████████████████████████▋ | 31949/110696 [9:42:20<19:20:08, 1.13it/s][2025-04-27 06:45:45] (step=0031950) Train Loss: 5.9315, Train Steps/Sec: 1.12 + 29%|████████████████████████████████████▋ | 31974/110696 [9:42:42<19:27:52, 1.12it/s][2025-04-27 06:46:08] (step=0031975) Train Loss: 5.8701, Train Steps/Sec: 1.12 + 29%|████████████████████████████████████▋ | 31999/110696 [9:43:05<19:21:01, 1.13it/s][2025-04-27 06:46:30] (step=0032000) Train Loss: 5.9243, Train Steps/Sec: 1.12 +You are using a model of type blip-2 to instantiate a model of type instructblip. This is not supported for all configurations of models and can yield errors. +[2025-04-27 06:46:30] text_config is None. Initializing the text config with default values (`OPTConfig`). +model_name_or_path:/tmp/haozhezhao/model/blip2-flan-t5-xl +image_place_holder: +tokenizer length after expend 32101 +tokenizer length before expend 32100 +100%|████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████| 5/5 [04:57<00:00, 59.45s/it] +[2025-04-27 06:52:38] Finish Eval in 32000 steps...█████████████████████████████████████████████████████████████████████████████████████████████| 5/5 [04:56<00:00, 59.27s/it] +processing left-padding... +processing left-padding... +processing left-padding... +processing left-padding... +processing left-padding... +[2025-04-27 06:52:57] Saved checkpoint to checkpoint/X2I_UltraEdiit_fourmask_LLava_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_filtered_objects_4mask_1280_context/003-GPT-XL/checkpoints/0032000.pt +[2025-04-27 06:52:59] Removed old checkpoint: checkpoint/X2I_UltraEdiit_fourmask_LLava_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_filtered_objects_4mask_1280_context/003-GPT-XL/checkpoints/0030000.pt + 29%|████████████████████████████████████▋ | 32024/110696 [9:49:56<20:27:14, 1.07it/s][2025-04-27 06:53:22] (step=0032025) Train Loss: 5.8780, Train Steps/Sec: 0.06 + 29%|████████████████████████████████████▊ | 32049/110696 [9:50:19<19:40:21, 1.11it/s][2025-04-27 06:53:44] (step=0032050) Train Loss: 5.9092, Train Steps/Sec: 1.12 + 29%|████████████████████████████████████▊ | 32074/110696 [9:50:41<19:32:03, 1.12it/s][2025-04-27 06:54:06] (step=0032075) Train Loss: 5.9585, Train Steps/Sec: 1.12 + 29%|████████████████████████████████████▊ | 32099/110696 [9:51:03<19:28:48, 1.12it/s][2025-04-27 06:54:29] (step=0032100) Train Loss: 5.9574, Train Steps/Sec: 1.12 + 29%|████████████████████████████████████▊ | 32124/110696 [9:51:26<19:23:20, 1.13it/s][2025-04-27 06:54:51] (step=0032125) Train Loss: 5.9136, Train Steps/Sec: 1.12 + 29%|████████████████████████████████████▉ | 32149/110696 [9:51:48<19:20:06, 1.13it/s][2025-04-27 06:55:13] (step=0032150) Train Loss: 5.9636, Train Steps/Sec: 1.12 + 29%|████████████████████████████████████▉ | 32174/110696 [9:52:10<19:19:20, 1.13it/s][2025-04-27 06:55:36] (step=0032175) Train Loss: 5.9369, Train Steps/Sec: 1.12 + 29%|████████████████████████████████████▉ | 32199/110696 [9:52:33<19:17:16, 1.13it/s][2025-04-27 06:55:58] (step=0032200) Train Loss: 5.8829, Train Steps/Sec: 1.12 + 29%|████████████████████████████████████▉ | 32224/110696 [9:53:00<25:35:47, 1.17s/it][2025-04-27 06:56:26] (step=0032225) Train Loss: 5.9316, Train Steps/Sec: 0.91 + 29%|████████████████████████████████████▉ | 32249/110696 [9:53:22<19:35:20, 1.11it/s][2025-04-27 06:56:48] (step=0032250) Train Loss: 5.8904, Train Steps/Sec: 1.12 + 29%|█████████████████████████████████████ | 32274/110696 [9:53:45<19:23:52, 1.12it/s][2025-04-27 06:57:10] (step=0032275) Train Loss: 5.9508, Train Steps/Sec: 1.12 + 29%|█████████████████████████████████████ | 32299/110696 [9:54:07<19:24:48, 1.12it/s][2025-04-27 06:57:32] (step=0032300) Train Loss: 5.9065, Train Steps/Sec: 1.12 + 29%|█████████████████████████████████████ | 32324/110696 [9:54:29<19:18:12, 1.13it/s][2025-04-27 06:57:55] (step=0032325) Train Loss: 5.9025, Train Steps/Sec: 1.12 + 29%|█████████████████████████████████████ | 32349/110696 [9:54:52<19:14:24, 1.13it/s][2025-04-27 06:58:17] (step=0032350) Train Loss: 5.9105, Train Steps/Sec: 1.12 + 29%|█████████████████████████████████████▏ | 32374/110696 [9:55:14<19:17:45, 1.13it/s][2025-04-27 06:58:39] (step=0032375) Train Loss: 5.8377, Train Steps/Sec: 1.12 + 29%|█████████████████████████████████████▏ | 32399/110696 [9:55:36<19:13:47, 1.13it/s][2025-04-27 06:59:02] (step=0032400) Train Loss: 5.9348, Train Steps/Sec: 1.12 + 29%|█████████████████████████████████████▏ | 32424/110696 [9:55:59<19:46:36, 1.10it/s][2025-04-27 06:59:24] (step=0032425) Train Loss: 5.9100, Train Steps/Sec: 1.12 + 29%|█████████████████████████████████████▏ | 32449/110696 [9:56:21<19:30:29, 1.11it/s][2025-04-27 06:59:46] (step=0032450) Train Loss: 5.9330, Train Steps/Sec: 1.12 + 29%|█████████████████████████████████████▎ | 32474/110696 [9:56:43<19:27:44, 1.12it/s][2025-04-27 07:00:09] (step=0032475) Train Loss: 5.8879, Train Steps/Sec: 1.12 + 29%|█████████████████████████████████████▎ | 32499/110696 [9:57:06<19:19:22, 1.12it/s][2025-04-27 07:00:31] (step=0032500) Train Loss: 5.9883, Train Steps/Sec: 1.12 + 29%|█████████████████████████████████████▎ | 32524/110696 [9:57:28<19:18:37, 1.12it/s][2025-04-27 07:00:58] (step=0032525) Train Loss: 5.9211, Train Steps/Sec: 0.93 + 29%|█████████████████████████████████████▎ | 32549/110696 [9:57:55<19:15:16, 1.13it/s][2025-04-27 07:01:20] (step=0032550) Train Loss: 5.8420, Train Steps/Sec: 1.11 + 29%|█████████████████████████████████████▎ | 32574/110696 [9:58:22<19:26:28, 1.12it/s][2025-04-27 07:01:47] (step=0032575) Train Loss: 5.9489, Train Steps/Sec: 0.93 + 29%|█████████████████████████████████████▍ | 32599/110696 [9:58:44<19:18:21, 1.12it/s][2025-04-27 07:02:10] (step=0032600) Train Loss: 5.8761, Train Steps/Sec: 1.11 + 29%|█████████████████████████████████████▍ | 32624/110696 [9:59:07<19:47:51, 1.10it/s][2025-04-27 07:02:32] (step=0032625) Train Loss: 5.8357, Train Steps/Sec: 1.12 + 29%|█████████████████████████████████████▍ | 32649/110696 [9:59:29<19:32:19, 1.11it/s][2025-04-27 07:02:54] (step=0032650) Train Loss: 5.9089, Train Steps/Sec: 1.12 + 30%|█████████████████████████████████████▍ | 32674/110696 [9:59:51<19:18:26, 1.12it/s][2025-04-27 07:03:17] (step=0032675) Train Loss: 5.9074, Train Steps/Sec: 1.12 + 30%|█████████████████████████████████████▏ | 32699/110696 [10:00:28<21:24:01, 1.01it/s][2025-04-27 07:03:59] (step=0032700) Train Loss: 5.9196, Train Steps/Sec: 0.59 + 30%|█████████████████████████████████████▏ | 32724/110696 [10:00:56<19:15:38, 1.12it/s][2025-04-27 07:04:21] (step=0032725) Train Loss: 5.9385, Train Steps/Sec: 1.12 + 30%|█████████████████████████████████████▎ | 32749/110696 [10:01:18<19:09:54, 1.13it/s][2025-04-27 07:04:44] (step=0032750) Train Loss: 5.9260, Train Steps/Sec: 1.12 + 30%|█████████████████████████████████████▎ | 32774/110696 [10:01:40<19:05:07, 1.13it/s][2025-04-27 07:05:06] (step=0032775) Train Loss: 5.9205, Train Steps/Sec: 1.12 + 30%|█████████████████████████████████████▎ | 32799/110696 [10:02:03<19:10:56, 1.13it/s][2025-04-27 07:05:28] (step=0032800) Train Loss: 5.9240, Train Steps/Sec: 1.11 + 30%|█████████████████████████████████████▎ | 32824/110696 [10:02:25<19:43:21, 1.10it/s][2025-04-27 07:05:51] (step=0032825) Train Loss: 5.8951, Train Steps/Sec: 1.12 + 30%|█████████████████████████████████████▍ | 32849/110696 [10:02:48<19:32:24, 1.11it/s][2025-04-27 07:06:13] (step=0032850) Train Loss: 5.8784, Train Steps/Sec: 1.12 + 30%|█████████████████████████████████████▍ | 32874/110696 [10:03:10<19:26:53, 1.11it/s][2025-04-27 07:06:35] (step=0032875) Train Loss: 5.9186, Train Steps/Sec: 1.12 + 30%|█████████████████████████████████████▍ | 32899/110696 [10:03:32<19:13:31, 1.12it/s][2025-04-27 07:06:58] (step=0032900) Train Loss: 5.9249, Train Steps/Sec: 1.12 + 30%|█████████████████████████████████████▍ | 32924/110696 [10:03:55<19:16:04, 1.12it/s][2025-04-27 07:07:20] (step=0032925) Train Loss: 5.9545, Train Steps/Sec: 1.12 + 30%|█████████████████████████████████████▌ | 32949/110696 [10:04:17<19:09:41, 1.13it/s][2025-04-27 07:07:42] (step=0032950) Train Loss: 5.8904, Train Steps/Sec: 1.12 + 30%|█████████████████████████████████████▌ | 32974/110696 [10:04:39<19:07:18, 1.13it/s][2025-04-27 07:08:05] (step=0032975) Train Loss: 5.9222, Train Steps/Sec: 1.12 + 30%|█████████████████████████████████████▌ | 32999/110696 [10:05:01<19:02:39, 1.13it/s][2025-04-27 07:08:27] (step=0033000) Train Loss: 5.9331, Train Steps/Sec: 1.12 + 30%|█████████████████████████████████████▌ | 33024/110696 [10:05:24<19:39:01, 1.10it/s][2025-04-27 07:08:49] (step=0033025) Train Loss: 5.8067, Train Steps/Sec: 1.12 + 30%|█████████████████████████████████████▌ | 33049/110696 [10:05:46<19:20:28, 1.12it/s][2025-04-27 07:09:12] (step=0033050) Train Loss: 5.9677, Train Steps/Sec: 1.12 + 30%|█████████████████████████████████████▋ | 33074/110696 [10:06:09<19:22:04, 1.11it/s][2025-04-27 07:09:34] (step=0033075) Train Loss: 5.9730, Train Steps/Sec: 1.12 + 30%|█████████████████████████████████████▋ | 33099/110696 [10:06:31<19:15:42, 1.12it/s][2025-04-27 07:09:56] (step=0033100) Train Loss: 5.9071, Train Steps/Sec: 1.12 + 30%|█████████████████████████████████████▋ | 33124/110696 [10:06:53<19:09:59, 1.12it/s][2025-04-27 07:10:19] (step=0033125) Train Loss: 5.9323, Train Steps/Sec: 1.12 + 30%|█████████████████████████████████████▋ | 33149/110696 [10:07:16<19:05:29, 1.13it/s][2025-04-27 07:10:41] (step=0033150) Train Loss: 5.9386, Train Steps/Sec: 1.12 + 30%|█████████████████████████████████████▊ | 33174/110696 [10:07:38<19:03:54, 1.13it/s][2025-04-27 07:11:03] (step=0033175) Train Loss: 5.9198, Train Steps/Sec: 1.12 + 30%|█████████████████████████████████████▊ | 33199/110696 [10:08:00<19:00:20, 1.13it/s][2025-04-27 07:11:26] (step=0033200) Train Loss: 5.8328, Train Steps/Sec: 1.12 + 30%|█████████████████████████████████████▊ | 33224/110696 [10:08:23<19:41:45, 1.09it/s][2025-04-27 07:11:48] (step=0033225) Train Loss: 5.8746, Train Steps/Sec: 1.12 + 30%|█████████████████████████████████████▊ | 33249/110696 [10:08:45<19:23:54, 1.11it/s][2025-04-27 07:12:10] (step=0033250) Train Loss: 5.9271, Train Steps/Sec: 1.12 + 30%|█████████████████████████████████████▊ | 33274/110696 [10:09:07<19:12:41, 1.12it/s][2025-04-27 07:12:33] (step=0033275) Train Loss: 5.9293, Train Steps/Sec: 1.12 + 30%|█████████████████████████████████████▉ | 33299/110696 [10:09:30<19:10:01, 1.12it/s][2025-04-27 07:12:55] (step=0033300) Train Loss: 5.9515, Train Steps/Sec: 1.12 + 30%|█████████████████████████████████████▉ | 33324/110696 [10:09:52<19:02:44, 1.13it/s][2025-04-27 07:13:17] (step=0033325) Train Loss: 5.8866, Train Steps/Sec: 1.12 + 30%|█████████████████████████████████████▉ | 33349/110696 [10:10:14<19:02:51, 1.13it/s][2025-04-27 07:13:40] (step=0033350) Train Loss: 5.9100, Train Steps/Sec: 1.12 + 30%|█████████████████████████████████████▉ | 33374/110696 [10:10:37<19:02:14, 1.13it/s][2025-04-27 07:14:02] (step=0033375) Train Loss: 5.8542, Train Steps/Sec: 1.12 + 30%|██████████████████████████████████████ | 33399/110696 [10:10:59<18:58:57, 1.13it/s][2025-04-27 07:14:24] (step=0033400) Train Loss: 5.9030, Train Steps/Sec: 1.12 + 30%|██████████████████████████████████████ | 33424/110696 [10:11:21<19:34:12, 1.10it/s][2025-04-27 07:14:47] (step=0033425) Train Loss: 5.8882, Train Steps/Sec: 1.12 + 30%|██████████████████████████████████████ | 33449/110696 [10:11:44<19:22:53, 1.11it/s][2025-04-27 07:15:09] (step=0033450) Train Loss: 5.8799, Train Steps/Sec: 1.12 + 30%|██████████████████████████████████████ | 33474/110696 [10:12:06<19:11:35, 1.12it/s][2025-04-27 07:15:31] (step=0033475) Train Loss: 5.9888, Train Steps/Sec: 1.11 + 30%|██████████████████████████████████████▏ | 33499/110696 [10:12:28<19:16:35, 1.11it/s][2025-04-27 07:15:54] (step=0033500) Train Loss: 5.8629, Train Steps/Sec: 1.12 + 30%|██████████████████████████████████████▏ | 33524/110696 [10:12:51<19:03:22, 1.12it/s][2025-04-27 07:16:16] (step=0033525) Train Loss: 5.8951, Train Steps/Sec: 1.12 + 30%|██████████████████████████████████████▏ | 33549/110696 [10:13:13<19:02:48, 1.13it/s][2025-04-27 07:16:38] (step=0033550) Train Loss: 5.9157, Train Steps/Sec: 1.12 + 30%|██████████████████████████████████████▏ | 33574/110696 [10:13:35<19:02:12, 1.13it/s][2025-04-27 07:17:01] (step=0033575) Train Loss: 5.8491, Train Steps/Sec: 1.12 + 30%|██████████████████████████████████████▏ | 33599/110696 [10:13:58<18:55:19, 1.13it/s][2025-04-27 07:17:23] (step=0033600) Train Loss: 5.9008, Train Steps/Sec: 1.11 + 30%|██████████████████████████████████████▎ | 33624/110696 [10:14:20<19:32:09, 1.10it/s][2025-04-27 07:17:45] (step=0033625) Train Loss: 5.9354, Train Steps/Sec: 1.12 + 30%|██████████████████████████████████████▎ | 33649/110696 [10:14:42<19:16:12, 1.11it/s][2025-04-27 07:18:08] (step=0033650) Train Loss: 5.9262, Train Steps/Sec: 1.12 + 30%|██████████████████████████████████████▎ | 33674/110696 [10:15:05<19:12:21, 1.11it/s][2025-04-27 07:18:30] (step=0033675) Train Loss: 5.8387, Train Steps/Sec: 1.12 + 30%|██████████████████████████████████████▎ | 33699/110696 [10:15:27<19:02:37, 1.12it/s][2025-04-27 07:18:52] (step=0033700) Train Loss: 5.9911, Train Steps/Sec: 1.12 + 30%|██████████████████████████████████████▍ | 33724/110696 [10:15:49<18:58:24, 1.13it/s][2025-04-27 07:19:15] (step=0033725) Train Loss: 5.9863, Train Steps/Sec: 1.12 + 30%|██████████████████████████████████████▍ | 33749/110696 [10:16:12<19:00:25, 1.12it/s][2025-04-27 07:19:37] (step=0033750) Train Loss: 5.9202, Train Steps/Sec: 1.12 + 31%|██████████████████████████████████████▍ | 33774/110696 [10:16:34<18:53:23, 1.13it/s][2025-04-27 07:19:59] (step=0033775) Train Loss: 5.9247, Train Steps/Sec: 1.12 + 31%|██████████████████████████████████████▍ | 33799/110696 [10:16:57<19:12:20, 1.11it/s][2025-04-27 07:20:22] (step=0033800) Train Loss: 5.8545, Train Steps/Sec: 1.11 + 31%|██████████████████████████████████████▌ | 33824/110696 [10:17:19<19:28:37, 1.10it/s][2025-04-27 07:20:44] (step=0033825) Train Loss: 5.9205, Train Steps/Sec: 1.12 + 31%|██████████████████████████████████████▌ | 33849/110696 [10:17:41<19:15:22, 1.11it/s][2025-04-27 07:21:07] (step=0033850) Train Loss: 5.9053, Train Steps/Sec: 1.12 + 31%|██████████████████████████████████████▌ | 33874/110696 [10:18:04<19:07:08, 1.12it/s][2025-04-27 07:21:29] (step=0033875) Train Loss: 5.8777, Train Steps/Sec: 1.12 + 31%|██████████████████████████████████████▌ | 33899/110696 [10:18:26<18:58:14, 1.12it/s][2025-04-27 07:21:51] (step=0033900) Train Loss: 5.8924, Train Steps/Sec: 1.12 + 31%|██████████████████████████████████████▌ | 33924/110696 [10:18:48<18:57:40, 1.12it/s][2025-04-27 07:22:14] (step=0033925) Train Loss: 5.9701, Train Steps/Sec: 1.12 + 31%|██████████████████████████████████████▋ | 33949/110696 [10:19:11<18:50:50, 1.13it/s][2025-04-27 07:22:36] (step=0033950) Train Loss: 5.9150, Train Steps/Sec: 1.12 + 31%|██████████████████████████████████████▋ | 33974/110696 [10:19:33<18:52:25, 1.13it/s][2025-04-27 07:22:58] (step=0033975) Train Loss: 5.9348, Train Steps/Sec: 1.12 + 31%|██████████████████████████████████████▋ | 33999/110696 [10:19:55<18:51:31, 1.13it/s][2025-04-27 07:23:21] (step=0034000) Train Loss: 5.9628, Train Steps/Sec: 1.12 +You are using a model of type blip-2 to instantiate a model of type instructblip. This is not supported for all configurations of models and can yield errors. +[2025-04-27 07:23:21] text_config is None. Initializing the text config with default values (`OPTConfig`). +model_name_or_path:/tmp/haozhezhao/model/blip2-flan-t5-xl +image_place_holder: +tokenizer length after expend 32101 +tokenizer length before expend 32100 +100%|████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████| 5/5 [04:57<00:00, 59.51s/it] +[2025-04-27 07:29:29] Finish Eval in 34000 steps...█████████████████████████████████████████████████████████████████████████████████████████████| 5/5 [04:56<00:00, 59.20s/it] +processing left-padding... +processing left-padding... +processing left-padding... +processing left-padding... +processing left-padding... +[2025-04-27 07:29:48] Saved checkpoint to checkpoint/X2I_UltraEdiit_fourmask_LLava_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_filtered_objects_4mask_1280_context/003-GPT-XL/checkpoints/0034000.pt +[2025-04-27 07:29:50] Removed old checkpoint: checkpoint/X2I_UltraEdiit_fourmask_LLava_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_filtered_objects_4mask_1280_context/003-GPT-XL/checkpoints/0032000.pt + 31%|██████████████████████████████████████▋ | 34024/110696 [10:26:47<19:53:26, 1.07it/s][2025-04-27 07:30:13] (step=0034025) Train Loss: 5.8748, Train Steps/Sec: 0.06 + 31%|██████████████████████████████████████▊ | 34049/110696 [10:27:09<19:04:44, 1.12it/s][2025-04-27 07:30:35] (step=0034050) Train Loss: 5.9991, Train Steps/Sec: 1.12 + 31%|██████████████████████████████████████▊ | 34074/110696 [10:27:32<19:01:25, 1.12it/s][2025-04-27 07:30:57] (step=0034075) Train Loss: 5.8893, Train Steps/Sec: 1.12 + 31%|██████████████████████████████████████▊ | 34099/110696 [10:27:54<18:56:41, 1.12it/s][2025-04-27 07:31:20] (step=0034100) Train Loss: 5.9605, Train Steps/Sec: 1.12 + 31%|██████████████████████████████████████▊ | 34124/110696 [10:28:16<18:55:30, 1.12it/s][2025-04-27 07:31:42] (step=0034125) Train Loss: 5.9311, Train Steps/Sec: 1.12 + 31%|██████████████████████████████████████▊ | 34149/110696 [10:28:39<18:50:10, 1.13it/s][2025-04-27 07:32:04] (step=0034150) Train Loss: 5.9065, Train Steps/Sec: 1.12 + 31%|██████████████████████████████████████▉ | 34174/110696 [10:29:01<18:49:11, 1.13it/s][2025-04-27 07:32:26] (step=0034175) Train Loss: 5.9619, Train Steps/Sec: 1.12 + 31%|██████████████████████████████████████▉ | 34199/110696 [10:29:23<18:45:44, 1.13it/s][2025-04-27 07:32:49] (step=0034200) Train Loss: 5.9195, Train Steps/Sec: 1.12 + 31%|██████████████████████████████████████▉ | 34224/110696 [10:29:46<19:23:13, 1.10it/s][2025-04-27 07:33:11] (step=0034225) Train Loss: 5.9370, Train Steps/Sec: 1.12 + 31%|██████████████████████████████████████▉ | 34249/110696 [10:30:08<19:15:35, 1.10it/s][2025-04-27 07:33:34] (step=0034250) Train Loss: 5.8952, Train Steps/Sec: 1.12 + 31%|███████████████████████████████████████ | 34274/110696 [10:30:30<19:03:02, 1.11it/s][2025-04-27 07:33:56] (step=0034275) Train Loss: 5.8829, Train Steps/Sec: 1.12 + 31%|███████████████████████████████████████ | 34299/110696 [10:30:53<18:53:29, 1.12it/s][2025-04-27 07:34:18] (step=0034300) Train Loss: 5.9048, Train Steps/Sec: 1.12 + 31%|███████████████████████████████████████ | 34324/110696 [10:31:15<18:49:30, 1.13it/s][2025-04-27 07:34:40] (step=0034325) Train Loss: 5.9566, Train Steps/Sec: 1.12 + 31%|███████████████████████████████████████ | 34349/110696 [10:31:37<18:48:05, 1.13it/s][2025-04-27 07:35:03] (step=0034350) Train Loss: 5.9103, Train Steps/Sec: 1.12 + 31%|███████████████████████████████████████▏ | 34374/110696 [10:32:00<18:42:53, 1.13it/s][2025-04-27 07:35:25] (step=0034375) Train Loss: 5.8570, Train Steps/Sec: 1.12 + 31%|███████████████████████████████████████▏ | 34399/110696 [10:32:22<18:49:26, 1.13it/s][2025-04-27 07:35:47] (step=0034400) Train Loss: 5.9379, Train Steps/Sec: 1.11 + 31%|███████████████████████████████████████▏ | 34424/110696 [10:32:44<19:16:18, 1.10it/s][2025-04-27 07:36:10] (step=0034425) Train Loss: 5.8920, Train Steps/Sec: 1.12 + 31%|███████████████████████████████████████▏ | 34449/110696 [10:33:07<19:01:56, 1.11it/s][2025-04-27 07:36:32] (step=0034450) Train Loss: 5.9443, Train Steps/Sec: 1.11 + 31%|███████████████████████████████████████▏ | 34474/110696 [10:33:29<18:54:55, 1.12it/s][2025-04-27 07:36:55] (step=0034475) Train Loss: 5.9701, Train Steps/Sec: 1.12 + 31%|███████████████████████████████████████▎ | 34499/110696 [10:33:51<18:51:42, 1.12it/s][2025-04-27 07:37:17] (step=0034500) Train Loss: 5.8841, Train Steps/Sec: 1.12 + 31%|███████████████████████████████████████▎ | 34524/110696 [10:34:14<18:48:15, 1.13it/s][2025-04-27 07:37:39] (step=0034525) Train Loss: 5.8905, Train Steps/Sec: 1.12 + 31%|███████████████████████████████████████▎ | 34549/110696 [10:34:36<18:46:01, 1.13it/s][2025-04-27 07:38:01] (step=0034550) Train Loss: 5.8829, Train Steps/Sec: 1.12 + 31%|███████████████████████████████████████▎ | 34574/110696 [10:34:58<18:44:08, 1.13it/s][2025-04-27 07:38:24] (step=0034575) Train Loss: 5.9348, Train Steps/Sec: 1.12 + 31%|███████████████████████████████████████▍ | 34599/110696 [10:35:21<18:41:58, 1.13it/s][2025-04-27 07:38:46] (step=0034600) Train Loss: 5.8771, Train Steps/Sec: 1.11 + 31%|███████████████████████████████████████▍ | 34624/110696 [10:35:43<19:17:14, 1.10it/s][2025-04-27 07:39:09] (step=0034625) Train Loss: 5.8376, Train Steps/Sec: 1.12 + 31%|███████████████████████████████████████▍ | 34649/110696 [10:36:06<19:00:32, 1.11it/s][2025-04-27 07:39:31] (step=0034650) Train Loss: 5.9063, Train Steps/Sec: 1.12 + 31%|███████████████████████████████████████▍ | 34674/110696 [10:36:28<18:54:39, 1.12it/s][2025-04-27 07:39:53] (step=0034675) Train Loss: 5.8400, Train Steps/Sec: 1.12 + 31%|███████████████████████████████████████▍ | 34699/110696 [10:36:50<18:50:54, 1.12it/s][2025-04-27 07:40:16] (step=0034700) Train Loss: 5.9418, Train Steps/Sec: 1.12 + 31%|███████████████████████████████████████▌ | 34724/110696 [10:37:12<18:46:31, 1.12it/s][2025-04-27 07:40:38] (step=0034725) Train Loss: 5.8576, Train Steps/Sec: 1.12 + 31%|███████████████████████████████████████▌ | 34749/110696 [10:37:35<18:43:06, 1.13it/s][2025-04-27 07:41:00] (step=0034750) Train Loss: 5.9099, Train Steps/Sec: 1.12 + 31%|███████████████████████████████████████▌ | 34774/110696 [10:37:57<18:41:01, 1.13it/s][2025-04-27 07:41:23] (step=0034775) Train Loss: 5.8636, Train Steps/Sec: 1.12 + 31%|███████████████████████████████████████▌ | 34799/110696 [10:38:19<18:40:00, 1.13it/s][2025-04-27 07:41:45] (step=0034800) Train Loss: 5.9211, Train Steps/Sec: 1.12 + 31%|███████████████████████████████████████▋ | 34824/110696 [10:38:42<19:11:31, 1.10it/s][2025-04-27 07:42:07] (step=0034825) Train Loss: 5.8795, Train Steps/Sec: 1.12 + 31%|███████████████████████████████████████▋ | 34849/110696 [10:39:04<18:57:50, 1.11it/s][2025-04-27 07:42:30] (step=0034850) Train Loss: 5.9088, Train Steps/Sec: 1.12 + 32%|███████████████████████████████████████▋ | 34874/110696 [10:39:27<18:55:02, 1.11it/s][2025-04-27 07:42:52] (step=0034875) Train Loss: 5.9331, Train Steps/Sec: 1.12 + 32%|███████████████████████████████████████▋ | 34899/110696 [10:39:49<18:46:31, 1.12it/s][2025-04-27 07:43:14] (step=0034900) Train Loss: 5.9097, Train Steps/Sec: 1.12 + 32%|███████████████████████████████████████▊ | 34924/110696 [10:40:11<18:41:42, 1.13it/s][2025-04-27 07:43:37] (step=0034925) Train Loss: 5.9184, Train Steps/Sec: 1.12 + 32%|███████████████████████████████████████▊ | 34949/110696 [10:40:34<18:41:16, 1.13it/s][2025-04-27 07:43:59] (step=0034950) Train Loss: 6.0011, Train Steps/Sec: 1.12 + 32%|███████████████████████████████████████▊ | 34974/110696 [10:40:56<18:37:53, 1.13it/s][2025-04-27 07:44:21] (step=0034975) Train Loss: 5.9369, Train Steps/Sec: 1.12 + 32%|███████████████████████████████████████▊ | 34999/110696 [10:41:18<18:35:00, 1.13it/s][2025-04-27 07:44:44] (step=0035000) Train Loss: 5.9623, Train Steps/Sec: 1.12 + 32%|███████████████████████████████████████▊ | 35024/110696 [10:41:41<19:07:57, 1.10it/s][2025-04-27 07:45:06] (step=0035025) Train Loss: 5.9646, Train Steps/Sec: 1.12 + 32%|███████████████████████████████████████▉ | 35049/110696 [10:42:03<18:53:21, 1.11it/s][2025-04-27 07:45:28] (step=0035050) Train Loss: 5.8805, Train Steps/Sec: 1.12 + 32%|███████████████████████████████████████▉ | 35074/110696 [10:42:25<18:49:22, 1.12it/s][2025-04-27 07:45:51] (step=0035075) Train Loss: 5.9409, Train Steps/Sec: 1.11 + 32%|███████████████████████████████████████▉ | 35099/110696 [10:42:48<18:46:52, 1.12it/s][2025-04-27 07:46:13] (step=0035100) Train Loss: 5.9395, Train Steps/Sec: 1.12 + 32%|███████████████████████████████████████▉ | 35124/110696 [10:43:10<18:40:27, 1.12it/s][2025-04-27 07:46:35] (step=0035125) Train Loss: 5.9030, Train Steps/Sec: 1.12 + 32%|████████████████████████████████████████ | 35149/110696 [10:43:32<18:36:05, 1.13it/s][2025-04-27 07:46:58] (step=0035150) Train Loss: 5.9827, Train Steps/Sec: 1.12 + 32%|████████████████████████████████████████ | 35174/110696 [10:43:55<18:36:01, 1.13it/s][2025-04-27 07:47:20] (step=0035175) Train Loss: 5.9801, Train Steps/Sec: 1.12 + 32%|████████████████████████████████████████ | 35199/110696 [10:44:17<18:30:10, 1.13it/s][2025-04-27 07:47:42] (step=0035200) Train Loss: 5.9522, Train Steps/Sec: 1.12 + 32%|████████████████████████████████████████ | 35224/110696 [10:44:39<19:13:30, 1.09it/s][2025-04-27 07:48:05] (step=0035225) Train Loss: 5.8972, Train Steps/Sec: 1.12 + 32%|████████████████████████████████████████ | 35249/110696 [10:45:02<18:55:12, 1.11it/s][2025-04-27 07:48:27] (step=0035250) Train Loss: 5.8915, Train Steps/Sec: 1.12 + 32%|████████████████████████████████████████▏ | 35274/110696 [10:45:24<18:48:22, 1.11it/s][2025-04-27 07:48:49] (step=0035275) Train Loss: 5.8562, Train Steps/Sec: 1.12 + 32%|████████████████████████████████████████▏ | 35299/110696 [10:45:46<18:41:21, 1.12it/s][2025-04-27 07:49:12] (step=0035300) Train Loss: 5.9557, Train Steps/Sec: 1.12 + 32%|████████████████████████████████████████▏ | 35324/110696 [10:46:09<18:40:02, 1.12it/s][2025-04-27 07:49:34] (step=0035325) Train Loss: 5.9047, Train Steps/Sec: 1.12 + 32%|████████████████████████████████████████▏ | 35349/110696 [10:46:31<18:35:42, 1.13it/s][2025-04-27 07:49:57] (step=0035350) Train Loss: 5.9244, Train Steps/Sec: 1.12 + 32%|████████████████████████████████████████▎ | 35374/110696 [10:46:54<18:32:43, 1.13it/s][2025-04-27 07:50:19] (step=0035375) Train Loss: 5.8832, Train Steps/Sec: 1.12 + 32%|████████████████████████████████████████▎ | 35399/110696 [10:47:16<18:30:48, 1.13it/s][2025-04-27 07:50:41] (step=0035400) Train Loss: 5.9131, Train Steps/Sec: 1.12 + 32%|████████████████████████████████████████▎ | 35424/110696 [10:47:38<19:04:44, 1.10it/s][2025-04-27 07:51:04] (step=0035425) Train Loss: 5.9146, Train Steps/Sec: 1.12 + 32%|████████████████████████████████████████▎ | 35449/110696 [10:48:01<18:52:16, 1.11it/s][2025-04-27 07:51:26] (step=0035450) Train Loss: 5.9586, Train Steps/Sec: 1.12 + 32%|████████████████████████████████████████▍ | 35474/110696 [10:48:23<18:43:17, 1.12it/s][2025-04-27 07:51:48] (step=0035475) Train Loss: 5.9430, Train Steps/Sec: 1.12 + 32%|████████████████████████████████████████▍ | 35499/110696 [10:48:45<18:37:25, 1.12it/s][2025-04-27 07:52:11] (step=0035500) Train Loss: 5.9872, Train Steps/Sec: 1.12 + 32%|████████████████████████████████████████▍ | 35524/110696 [10:49:08<18:32:18, 1.13it/s][2025-04-27 07:52:33] (step=0035525) Train Loss: 5.8978, Train Steps/Sec: 1.12 + 32%|████████████████████████████████████████▍ | 35549/110696 [10:49:30<18:29:56, 1.13it/s][2025-04-27 07:52:55] (step=0035550) Train Loss: 5.9399, Train Steps/Sec: 1.12 + 32%|████████████████████████████████████████▍ | 35574/110696 [10:49:52<18:29:47, 1.13it/s][2025-04-27 07:53:18] (step=0035575) Train Loss: 5.8550, Train Steps/Sec: 1.12 + 32%|████████████████████████████████████████▌ | 35599/110696 [10:50:15<18:29:19, 1.13it/s][2025-04-27 07:53:40] (step=0035600) Train Loss: 5.9245, Train Steps/Sec: 1.12 + 32%|████████████████████████████████████████▌ | 35624/110696 [10:50:37<19:02:49, 1.09it/s][2025-04-27 07:54:02] (step=0035625) Train Loss: 5.9235, Train Steps/Sec: 1.12 + 32%|████████████████████████████████████████▌ | 35649/110696 [10:50:59<18:47:31, 1.11it/s][2025-04-27 07:54:25] (step=0035650) Train Loss: 5.9068, Train Steps/Sec: 1.12 + 32%|████████████████████████████████████████▌ | 35674/110696 [10:51:22<18:39:58, 1.12it/s][2025-04-27 07:54:47] (step=0035675) Train Loss: 5.8909, Train Steps/Sec: 1.12 + 32%|████████████████████████████████████████▋ | 35699/110696 [10:51:44<18:34:53, 1.12it/s][2025-04-27 07:55:09] (step=0035700) Train Loss: 5.9012, Train Steps/Sec: 1.12 + 32%|████████████████████████████████████████▋ | 35724/110696 [10:52:06<18:29:52, 1.13it/s][2025-04-27 07:55:32] (step=0035725) Train Loss: 5.9413, Train Steps/Sec: 1.12 + 32%|████████████████████████████████████████▋ | 35749/110696 [10:52:29<18:28:44, 1.13it/s][2025-04-27 07:55:54] (step=0035750) Train Loss: 5.8568, Train Steps/Sec: 1.12 + 32%|████████████████████████████████████████▋ | 35774/110696 [10:52:51<18:26:42, 1.13it/s][2025-04-27 07:56:16] (step=0035775) Train Loss: 5.9464, Train Steps/Sec: 1.12 + 32%|████████████████████████████████████████▋ | 35799/110696 [10:53:13<18:26:00, 1.13it/s][2025-04-27 07:56:39] (step=0035800) Train Loss: 5.9195, Train Steps/Sec: 1.12 + 32%|████████████████████████████████████████▊ | 35824/110696 [10:53:36<18:58:16, 1.10it/s][2025-04-27 07:57:01] (step=0035825) Train Loss: 5.8914, Train Steps/Sec: 1.12 + 32%|████████████████████████████████████████▊ | 35849/110696 [10:53:58<18:43:51, 1.11it/s][2025-04-27 07:57:24] (step=0035850) Train Loss: 5.9127, Train Steps/Sec: 1.12 + 32%|████████████████████████████████████████▊ | 35874/110696 [10:54:20<18:34:59, 1.12it/s][2025-04-27 07:57:46] (step=0035875) Train Loss: 5.8864, Train Steps/Sec: 1.12 + 32%|████████████████████████████████████████▊ | 35899/110696 [10:54:43<18:32:03, 1.12it/s][2025-04-27 07:58:08] (step=0035900) Train Loss: 5.9341, Train Steps/Sec: 1.12 + 32%|████████████████████████████████████████▉ | 35924/110696 [10:55:05<18:25:50, 1.13it/s][2025-04-27 07:58:30] (step=0035925) Train Loss: 5.8822, Train Steps/Sec: 1.12 + 32%|████████████████████████████████████████▉ | 35949/110696 [10:55:27<18:23:49, 1.13it/s][2025-04-27 07:58:53] (step=0035950) Train Loss: 5.9004, Train Steps/Sec: 1.12 + 32%|████████████████████████████████████████▉ | 35974/110696 [10:55:50<18:24:38, 1.13it/s][2025-04-27 07:59:15] (step=0035975) Train Loss: 5.9062, Train Steps/Sec: 1.12 + 33%|████████████████████████████████████████▉ | 35999/110696 [10:56:12<18:19:39, 1.13it/s][2025-04-27 07:59:38] (step=0036000) Train Loss: 5.8767, Train Steps/Sec: 1.12 +You are using a model of type blip-2 to instantiate a model of type instructblip. This is not supported for all configurations of models and can yield errors. +[2025-04-27 07:59:38] text_config is None. Initializing the text config with default values (`OPTConfig`). +model_name_or_path:/tmp/haozhezhao/model/blip2-flan-t5-xl +image_place_holder: +tokenizer length after expend 32101 +tokenizer length before expend 32100 +100%|████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████| 5/5 [04:54<00:00, 58.95s/it] +[2025-04-27 08:05:44] Finish Eval in 36000 steps...█████████████████████████████████████████████████████████████████████████████████████████████| 5/5 [04:54<00:00, 58.79s/it] +processing left-padding... +processing left-padding... +processing left-padding... +processing left-padding... +processing left-padding... +[2025-04-27 08:06:03] Saved checkpoint to checkpoint/X2I_UltraEdiit_fourmask_LLava_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_filtered_objects_4mask_1280_context/003-GPT-XL/checkpoints/0036000.pt +[2025-04-27 08:06:05] Removed old checkpoint: checkpoint/X2I_UltraEdiit_fourmask_LLava_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_filtered_objects_4mask_1280_context/003-GPT-XL/checkpoints/0034000.pt + 33%|█████████████████████████████████████████ | 36024/110696 [11:03:02<19:25:27, 1.07it/s][2025-04-27 08:06:28] (step=0036025) Train Loss: 5.9054, Train Steps/Sec: 0.06 + 33%|█████████████████████████████████████████ | 36049/110696 [11:03:25<18:39:15, 1.11it/s][2025-04-27 08:06:50] (step=0036050) Train Loss: 5.9168, Train Steps/Sec: 1.12 + 33%|█████████████████████████████████████████ | 36074/110696 [11:03:47<18:33:37, 1.12it/s][2025-04-27 08:07:12] (step=0036075) Train Loss: 5.8957, Train Steps/Sec: 1.12 + 33%|█████████████████████████████████████████ | 36099/110696 [11:04:09<18:32:44, 1.12it/s][2025-04-27 08:07:35] (step=0036100) Train Loss: 5.8885, Train Steps/Sec: 1.12 + 33%|█████████████████████████████████████████ | 36124/110696 [11:04:32<18:24:14, 1.13it/s][2025-04-27 08:07:57] (step=0036125) Train Loss: 5.9333, Train Steps/Sec: 1.12 + 33%|█████████████████████████████████████████▏ | 36149/110696 [11:04:54<18:21:28, 1.13it/s][2025-04-27 08:08:19] (step=0036150) Train Loss: 5.9329, Train Steps/Sec: 1.12 + 33%|█████████████████████████████████████████▏ | 36174/110696 [11:05:16<18:20:23, 1.13it/s][2025-04-27 08:08:42] (step=0036175) Train Loss: 5.8893, Train Steps/Sec: 1.12 + 33%|█████████████████████████████████████████▏ | 36199/110696 [11:05:39<18:16:59, 1.13it/s][2025-04-27 08:09:04] (step=0036200) Train Loss: 5.9318, Train Steps/Sec: 1.12 + 33%|█████████████████████████████████████████▏ | 36224/110696 [11:06:01<18:51:27, 1.10it/s][2025-04-27 08:09:26] (step=0036225) Train Loss: 5.8900, Train Steps/Sec: 1.12 + 33%|█████████████████████████████████████████▎ | 36249/110696 [11:06:23<18:35:42, 1.11it/s][2025-04-27 08:09:49] (step=0036250) Train Loss: 5.9515, Train Steps/Sec: 1.12 + 33%|█████████████████████████████████████████▎ | 36274/110696 [11:06:46<18:30:29, 1.12it/s][2025-04-27 08:10:11] (step=0036275) Train Loss: 5.8844, Train Steps/Sec: 1.12 + 33%|█████████████████████████████████████████▎ | 36299/110696 [11:07:08<18:26:35, 1.12it/s][2025-04-27 08:10:33] (step=0036300) Train Loss: 5.8723, Train Steps/Sec: 1.12 + 33%|█████████████████████████████████████████▎ | 36324/110696 [11:07:30<18:20:04, 1.13it/s][2025-04-27 08:10:56] (step=0036325) Train Loss: 5.8977, Train Steps/Sec: 1.12 + 33%|█████████████████████████████████████████▎ | 36349/110696 [11:07:53<18:21:10, 1.13it/s][2025-04-27 08:11:18] (step=0036350) Train Loss: 5.9080, Train Steps/Sec: 1.11 + 33%|█████████████████████████████████████████▍ | 36374/110696 [11:08:15<18:16:47, 1.13it/s][2025-04-27 08:11:41] (step=0036375) Train Loss: 5.8309, Train Steps/Sec: 1.12 + 33%|█████████████████████████████████████████▍ | 36399/110696 [11:08:37<18:15:08, 1.13it/s][2025-04-27 08:12:03] (step=0036400) Train Loss: 5.8908, Train Steps/Sec: 1.12 + 33%|█████████████████████████████████████████▍ | 36424/110696 [11:09:00<18:46:15, 1.10it/s][2025-04-27 08:12:25] (step=0036425) Train Loss: 5.9047, Train Steps/Sec: 1.12 + 33%|█████████████████████████████████████████▍ | 36449/110696 [11:09:22<18:33:06, 1.11it/s][2025-04-27 08:12:48] (step=0036450) Train Loss: 5.9678, Train Steps/Sec: 1.12 + 33%|█████████████████████████████████████████▌ | 36474/110696 [11:09:44<18:28:19, 1.12it/s][2025-04-27 08:13:10] (step=0036475) Train Loss: 5.8826, Train Steps/Sec: 1.12 + 33%|█████████████████████████████████████████▌ | 36499/110696 [11:10:07<18:20:29, 1.12it/s][2025-04-27 08:13:32] (step=0036500) Train Loss: 5.9219, Train Steps/Sec: 1.12 + 33%|█████████████████████████████████████████▌ | 36524/110696 [11:10:29<18:18:46, 1.13it/s][2025-04-27 08:13:54] (step=0036525) Train Loss: 5.8258, Train Steps/Sec: 1.12 + 33%|█████████████████████████████████████████▌ | 36549/110696 [11:10:51<18:14:55, 1.13it/s][2025-04-27 08:14:17] (step=0036550) Train Loss: 5.9249, Train Steps/Sec: 1.12 + 33%|█████████████████████████████████████████▋ | 36574/110696 [11:11:14<18:12:43, 1.13it/s][2025-04-27 08:14:39] (step=0036575) Train Loss: 5.8534, Train Steps/Sec: 1.12 + 33%|█████████████████████████████████████████▋ | 36599/110696 [11:11:36<18:11:10, 1.13it/s][2025-04-27 08:15:01] (step=0036600) Train Loss: 5.8691, Train Steps/Sec: 1.12 + 33%|█████████████████████████████████████████▋ | 36624/110696 [11:12:04<20:36:46, 1.00s/it][2025-04-27 08:15:29] (step=0036625) Train Loss: 5.8788, Train Steps/Sec: 0.91 + 33%|█████████████████████████████████████████▋ | 36649/110696 [11:12:26<18:29:59, 1.11it/s][2025-04-27 08:15:51] (step=0036650) Train Loss: 5.8979, Train Steps/Sec: 1.12 + 33%|█████████████████████████████████████████▋ | 36674/110696 [11:12:48<18:25:50, 1.12it/s][2025-04-27 08:16:14] (step=0036675) Train Loss: 5.9174, Train Steps/Sec: 1.12 + 33%|█████████████████████████████████████████▊ | 36699/110696 [11:13:10<18:18:39, 1.12it/s][2025-04-27 08:16:36] (step=0036700) Train Loss: 5.8538, Train Steps/Sec: 1.12 + 33%|█████████████████████████████████████████▊ | 36724/110696 [11:13:33<18:17:49, 1.12it/s][2025-04-27 08:16:58] (step=0036725) Train Loss: 5.8681, Train Steps/Sec: 1.12 + 33%|█████████████████████████████████████████▊ | 36749/110696 [11:13:55<18:13:09, 1.13it/s][2025-04-27 08:17:20] (step=0036750) Train Loss: 5.9342, Train Steps/Sec: 1.12 + 33%|█████████████████████████████████████████▊ | 36774/110696 [11:14:17<18:11:39, 1.13it/s][2025-04-27 08:17:43] (step=0036775) Train Loss: 5.8258, Train Steps/Sec: 1.12 + 33%|█████████████████████████████████████████▉ | 36799/110696 [11:14:44<18:11:54, 1.13it/s][2025-04-27 08:18:10] (step=0036800) Train Loss: 5.8871, Train Steps/Sec: 0.92 + 33%|█████████████████████████████████████████▉ | 36824/110696 [11:15:07<18:40:44, 1.10it/s][2025-04-27 08:18:32] (step=0036825) Train Loss: 5.9094, Train Steps/Sec: 1.12 + 33%|█████████████████████████████████████████▉ | 36849/110696 [11:15:34<18:43:05, 1.10it/s][2025-04-27 08:18:59] (step=0036850) Train Loss: 5.8226, Train Steps/Sec: 0.93 + 33%|█████████████████████████████████████████▉ | 36874/110696 [11:15:56<18:23:04, 1.12it/s][2025-04-27 08:19:22] (step=0036875) Train Loss: 5.9130, Train Steps/Sec: 1.12 + 33%|██████████████████████████████████████████ | 36899/110696 [11:16:18<18:13:37, 1.12it/s][2025-04-27 08:19:44] (step=0036900) Train Loss: 5.8419, Train Steps/Sec: 1.12 + 33%|██████████████████████████████████████████ | 36924/110696 [11:16:41<18:13:29, 1.12it/s][2025-04-27 08:20:06] (step=0036925) Train Loss: 5.8587, Train Steps/Sec: 1.12 + 33%|██████████████████████████████████████████ | 36949/110696 [11:17:03<18:09:29, 1.13it/s][2025-04-27 08:20:29] (step=0036950) Train Loss: 5.8645, Train Steps/Sec: 1.12 + 33%|██████████████████████████████████████████ | 36974/110696 [11:17:25<18:10:19, 1.13it/s][2025-04-27 08:20:51] (step=0036975) Train Loss: 5.8939, Train Steps/Sec: 1.12 + 33%|██████████████████████████████████████████ | 36999/110696 [11:17:48<18:05:59, 1.13it/s][2025-04-27 08:21:13] (step=0037000) Train Loss: 5.9451, Train Steps/Sec: 1.12 + 33%|██████████████████████████████████████████▏ | 37024/110696 [11:18:25<33:57:03, 1.66s/it][2025-04-27 08:21:50] (step=0037025) Train Loss: 5.9055, Train Steps/Sec: 0.68 + 33%|██████████████████████████████████████████▏ | 37049/110696 [11:18:53<18:25:10, 1.11it/s][2025-04-27 08:22:18] (step=0037050) Train Loss: 5.8643, Train Steps/Sec: 0.90 + 33%|██████████████████████████████████████████▏ | 37074/110696 [11:19:15<18:20:50, 1.11it/s][2025-04-27 08:22:40] (step=0037075) Train Loss: 5.8616, Train Steps/Sec: 1.12 + 34%|██████████████████████████████████████████▏ | 37099/110696 [11:19:37<18:11:22, 1.12it/s][2025-04-27 08:23:03] (step=0037100) Train Loss: 5.9064, Train Steps/Sec: 1.12 + 34%|██████████████████████████████████████████▎ | 37124/110696 [11:20:00<18:08:07, 1.13it/s][2025-04-27 08:23:25] (step=0037125) Train Loss: 5.8713, Train Steps/Sec: 1.12 + 34%|██████████████████████████████████████████▎ | 37149/110696 [11:20:22<18:06:39, 1.13it/s][2025-04-27 08:23:47] (step=0037150) Train Loss: 5.9057, Train Steps/Sec: 1.12 + 34%|██████████████████████████████████████████▎ | 37174/110696 [11:20:44<18:06:18, 1.13it/s][2025-04-27 08:24:10] (step=0037175) Train Loss: 5.9435, Train Steps/Sec: 1.12 + 34%|██████████████████████████████████████████▎ | 37199/110696 [11:21:07<18:00:49, 1.13it/s][2025-04-27 08:24:32] (step=0037200) Train Loss: 5.8732, Train Steps/Sec: 1.12 + 34%|██████████████████████████████████████████▎ | 37224/110696 [11:21:29<18:36:15, 1.10it/s][2025-04-27 08:24:54] (step=0037225) Train Loss: 5.8673, Train Steps/Sec: 1.12 + 34%|██████████████████████████████████████████▍ | 37249/110696 [11:21:51<18:20:28, 1.11it/s][2025-04-27 08:25:17] (step=0037250) Train Loss: 5.9176, Train Steps/Sec: 1.12 + 34%|██████████████████████████████████████████▍ | 37274/110696 [11:22:14<18:20:14, 1.11it/s][2025-04-27 08:25:39] (step=0037275) Train Loss: 5.9064, Train Steps/Sec: 1.12 + 34%|██████████████████████████████████████████▍ | 37299/110696 [11:22:36<18:10:32, 1.12it/s][2025-04-27 08:26:01] (step=0037300) Train Loss: 5.8174, Train Steps/Sec: 1.12 + 34%|██████████████████████████████████████████▍ | 37324/110696 [11:22:58<18:07:18, 1.12it/s][2025-04-27 08:26:24] (step=0037325) Train Loss: 5.8546, Train Steps/Sec: 1.12 + 34%|██████████████████████████████████████████▌ | 37349/110696 [11:23:21<18:04:59, 1.13it/s][2025-04-27 08:26:46] (step=0037350) Train Loss: 5.8864, Train Steps/Sec: 1.12 + 34%|██████████████████████████████████████████▌ | 37374/110696 [11:23:43<17:59:00, 1.13it/s][2025-04-27 08:27:08] (step=0037375) Train Loss: 5.9636, Train Steps/Sec: 1.12 + 34%|██████████████████████████████████████████▌ | 37399/110696 [11:24:05<18:01:02, 1.13it/s][2025-04-27 08:27:31] (step=0037400) Train Loss: 5.8853, Train Steps/Sec: 1.12 + 34%|██████████████████████████████████████████▌ | 37424/110696 [11:24:28<18:36:31, 1.09it/s][2025-04-27 08:27:53] (step=0037425) Train Loss: 5.8840, Train Steps/Sec: 1.12 + 34%|██████████████████████████████████████████▋ | 37449/110696 [11:24:50<18:17:03, 1.11it/s][2025-04-27 08:28:15] (step=0037450) Train Loss: 5.8710, Train Steps/Sec: 1.12 + 34%|██████████████████████████████████████████▋ | 37474/110696 [11:25:12<18:16:40, 1.11it/s][2025-04-27 08:28:38] (step=0037475) Train Loss: 5.9179, Train Steps/Sec: 1.12 + 34%|██████████████████████████████████████████▋ | 37499/110696 [11:25:35<18:06:45, 1.12it/s][2025-04-27 08:29:00] (step=0037500) Train Loss: 5.8596, Train Steps/Sec: 1.12 + 34%|██████████████████████████████████████████▋ | 37524/110696 [11:25:57<18:03:49, 1.13it/s][2025-04-27 08:29:22] (step=0037525) Train Loss: 5.8703, Train Steps/Sec: 1.12 + 34%|██████████████████████████████████████████▋ | 37549/110696 [11:26:19<17:58:23, 1.13it/s][2025-04-27 08:29:44] (step=0037550) Train Loss: 5.8431, Train Steps/Sec: 1.12 + 34%|██████████████████████████████████████████▊ | 37574/110696 [11:26:41<18:04:27, 1.12it/s][2025-04-27 08:30:07] (step=0037575) Train Loss: 5.9178, Train Steps/Sec: 1.12 + 34%|██████████████████████████████████████████▊ | 37599/110696 [11:27:04<18:02:15, 1.13it/s][2025-04-27 08:30:29] (step=0037600) Train Loss: 5.8849, Train Steps/Sec: 1.11 + 34%|██████████████████████████████████████████▊ | 37624/110696 [11:27:26<18:32:15, 1.09it/s][2025-04-27 08:30:52] (step=0037625) Train Loss: 5.8734, Train Steps/Sec: 1.12 + 34%|██████████████████████████████████████████▊ | 37649/110696 [11:27:49<18:18:36, 1.11it/s][2025-04-27 08:31:14] (step=0037650) Train Loss: 5.8870, Train Steps/Sec: 1.12 + 34%|██████████████████████████████████████████▉ | 37674/110696 [11:28:11<18:10:59, 1.12it/s][2025-04-27 08:31:36] (step=0037675) Train Loss: 5.9073, Train Steps/Sec: 1.12 + 34%|██████████████████████████████████████████▉ | 37699/110696 [11:28:33<18:03:15, 1.12it/s][2025-04-27 08:31:59] (step=0037700) Train Loss: 5.9240, Train Steps/Sec: 1.12 + 34%|██████████████████████████████████████████▉ | 37724/110696 [11:28:56<18:01:39, 1.12it/s][2025-04-27 08:32:21] (step=0037725) Train Loss: 5.8360, Train Steps/Sec: 1.12 + 34%|██████████████████████████████████████████▉ | 37749/110696 [11:29:18<17:58:12, 1.13it/s][2025-04-27 08:32:43] (step=0037750) Train Loss: 5.8196, Train Steps/Sec: 1.12 + 34%|██████████████████████████████████████████▉ | 37774/110696 [11:29:40<17:53:16, 1.13it/s][2025-04-27 08:33:06] (step=0037775) Train Loss: 5.9395, Train Steps/Sec: 1.12 + 34%|███████████████████████████████████████████ | 37799/110696 [11:30:03<17:55:47, 1.13it/s][2025-04-27 08:33:28] (step=0037800) Train Loss: 5.8615, Train Steps/Sec: 1.12 + 34%|███████████████████████████████████████████ | 37824/110696 [11:30:25<18:26:47, 1.10it/s][2025-04-27 08:33:50] (step=0037825) Train Loss: 5.8549, Train Steps/Sec: 1.12 + 34%|███████████████████████████████████████████ | 37849/110696 [11:30:47<18:14:53, 1.11it/s][2025-04-27 08:34:13] (step=0037850) Train Loss: 5.8736, Train Steps/Sec: 1.12 + 34%|███████████████████████████████████████████ | 37874/110696 [11:31:10<18:07:28, 1.12it/s][2025-04-27 08:34:35] (step=0037875) Train Loss: 5.8524, Train Steps/Sec: 1.12 + 34%|███████████████████████████████████████████▏ | 37899/110696 [11:31:32<18:05:02, 1.12it/s][2025-04-27 08:34:57] (step=0037900) Train Loss: 5.9079, Train Steps/Sec: 1.12 + 34%|███████████████████████████████████████████▏ | 37924/110696 [11:31:54<17:58:12, 1.12it/s][2025-04-27 08:35:20] (step=0037925) Train Loss: 5.8772, Train Steps/Sec: 1.12 + 34%|███████████████████████████████████████████▏ | 37949/110696 [11:32:17<17:54:40, 1.13it/s][2025-04-27 08:35:42] (step=0037950) Train Loss: 5.8751, Train Steps/Sec: 1.12 + 34%|███████████████████████████████████████████▏ | 37974/110696 [11:32:39<17:53:42, 1.13it/s][2025-04-27 08:36:04] (step=0037975) Train Loss: 5.8979, Train Steps/Sec: 1.12 + 34%|███████████████████████████████████████████▎ | 37999/110696 [11:33:01<17:52:29, 1.13it/s][2025-04-27 08:36:27] (step=0038000) Train Loss: 5.8520, Train Steps/Sec: 1.11 +You are using a model of type blip-2 to instantiate a model of type instructblip. This is not supported for all configurations of models and can yield errors. +[2025-04-27 08:36:27] text_config is None. Initializing the text config with default values (`OPTConfig`). +model_name_or_path:/tmp/haozhezhao/model/blip2-flan-t5-xl +image_place_holder: +tokenizer length after expend 32101 +tokenizer length before expend 32100 +100%|████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████| 5/5 [04:57<00:00, 59.57s/it] +[2025-04-27 08:42:35] Finish Eval in 38000 steps...█████████████████████████████████████████████████████████████████████████████████████████████| 5/5 [04:57<00:00, 59.36s/it] +processing left-padding... +processing left-padding... +processing left-padding... +processing left-padding... +processing left-padding... +[2025-04-27 08:42:55] Saved checkpoint to checkpoint/X2I_UltraEdiit_fourmask_LLava_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_filtered_objects_4mask_1280_context/003-GPT-XL/checkpoints/0038000.pt +[2025-04-27 08:42:57] Removed old checkpoint: checkpoint/X2I_UltraEdiit_fourmask_LLava_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_filtered_objects_4mask_1280_context/003-GPT-XL/checkpoints/0036000.pt + 34%|███████████████████████████████████████████▎ | 38024/110696 [11:39:54<18:56:53, 1.07it/s][2025-04-27 08:43:19] (step=0038025) Train Loss: 5.8474, Train Steps/Sec: 0.06 + 34%|███████████████████████████████████████████▎ | 38049/110696 [11:40:16<18:09:37, 1.11it/s][2025-04-27 08:43:41] (step=0038050) Train Loss: 5.8810, Train Steps/Sec: 1.12 + 34%|███████████████████████████████████████████▎ | 38074/110696 [11:40:38<18:06:29, 1.11it/s][2025-04-27 08:44:04] (step=0038075) Train Loss: 5.8478, Train Steps/Sec: 1.12 + 34%|███████████████████████████████████████████▎ | 38099/110696 [11:41:01<18:02:39, 1.12it/s][2025-04-27 08:44:26] (step=0038100) Train Loss: 5.8821, Train Steps/Sec: 1.12 + 34%|███████████████████████████████████████████▍ | 38124/110696 [11:41:23<17:52:52, 1.13it/s][2025-04-27 08:44:48] (step=0038125) Train Loss: 5.8513, Train Steps/Sec: 1.12 + 34%|███████████████████████████████████████████▍ | 38149/110696 [11:41:45<17:50:11, 1.13it/s][2025-04-27 08:45:11] (step=0038150) Train Loss: 5.8453, Train Steps/Sec: 1.12 + 34%|███████████████████████████████████████████▍ | 38174/110696 [11:42:08<17:51:20, 1.13it/s][2025-04-27 08:45:33] (step=0038175) Train Loss: 5.9101, Train Steps/Sec: 1.12 + 35%|███████████████████████████████████████████▍ | 38199/110696 [11:42:30<17:50:32, 1.13it/s][2025-04-27 08:45:55] (step=0038200) Train Loss: 5.8740, Train Steps/Sec: 1.12 + 35%|███████████████████████████████████████████▌ | 38224/110696 [11:42:52<18:21:38, 1.10it/s][2025-04-27 08:46:18] (step=0038225) Train Loss: 5.9316, Train Steps/Sec: 1.11 + 35%|███████████████████████████████████████████▌ | 38249/110696 [11:43:15<18:09:57, 1.11it/s][2025-04-27 08:46:40] (step=0038250) Train Loss: 5.9175, Train Steps/Sec: 1.11 + 35%|███████████████████████████████████████████▌ | 38274/110696 [11:43:37<17:56:19, 1.12it/s][2025-04-27 08:47:03] (step=0038275) Train Loss: 5.8189, Train Steps/Sec: 1.12 + 35%|███████████████████████████████████████████▌ | 38299/110696 [11:44:00<17:57:56, 1.12it/s][2025-04-27 08:47:25] (step=0038300) Train Loss: 5.8846, Train Steps/Sec: 1.12 + 35%|███████████████████████████████████████████▌ | 38324/110696 [11:44:22<17:51:15, 1.13it/s][2025-04-27 08:47:47] (step=0038325) Train Loss: 5.8771, Train Steps/Sec: 1.12 + 35%|███████████████████████████████████████████▋ | 38349/110696 [11:44:44<17:53:37, 1.12it/s][2025-04-27 08:48:10] (step=0038350) Train Loss: 5.9141, Train Steps/Sec: 1.12 + 35%|███████████████████████████████████████████▋ | 38374/110696 [11:45:07<17:50:25, 1.13it/s][2025-04-27 08:48:32] (step=0038375) Train Loss: 5.8968, Train Steps/Sec: 1.12 + 35%|███████████████████████████████████████████▋ | 38399/110696 [11:45:29<17:48:49, 1.13it/s][2025-04-27 08:48:54] (step=0038400) Train Loss: 5.8371, Train Steps/Sec: 1.11 + 35%|███████████████████████████████████████████▋ | 38424/110696 [11:45:51<18:21:10, 1.09it/s][2025-04-27 08:49:17] (step=0038425) Train Loss: 5.8292, Train Steps/Sec: 1.12 + 35%|███████████████████████████████████████████▊ | 38449/110696 [11:46:14<18:04:07, 1.11it/s][2025-04-27 08:49:39] (step=0038450) Train Loss: 5.8554, Train Steps/Sec: 1.12 + 35%|███████████████████████████████████████████▊ | 38474/110696 [11:46:36<17:56:57, 1.12it/s][2025-04-27 08:50:01] (step=0038475) Train Loss: 5.8755, Train Steps/Sec: 1.12 + 35%|███████████████████████████████████████████▊ | 38499/110696 [11:46:58<17:54:45, 1.12it/s][2025-04-27 08:50:24] (step=0038500) Train Loss: 5.8692, Train Steps/Sec: 1.12 + 35%|███████████████████████████████████████████▊ | 38524/110696 [11:47:21<17:51:08, 1.12it/s][2025-04-27 08:50:46] (step=0038525) Train Loss: 5.8169, Train Steps/Sec: 1.12 + 35%|███████████████████████████████████████████▉ | 38549/110696 [11:47:43<17:49:57, 1.12it/s][2025-04-27 08:51:09] (step=0038550) Train Loss: 5.8419, Train Steps/Sec: 1.12 + 35%|███████████████████████████████████████████▉ | 38574/110696 [11:48:06<17:43:26, 1.13it/s][2025-04-27 08:51:31] (step=0038575) Train Loss: 5.8307, Train Steps/Sec: 1.12 + 35%|███████████████████████████████████████████▉ | 38599/110696 [11:48:28<17:45:43, 1.13it/s][2025-04-27 08:51:53] (step=0038600) Train Loss: 5.9335, Train Steps/Sec: 1.11 + 35%|███████████████████████████████████████████▉ | 38624/110696 [11:48:50<18:17:44, 1.09it/s][2025-04-27 08:52:16] (step=0038625) Train Loss: 5.8898, Train Steps/Sec: 1.12 + 35%|███████████████████████████████████████████▉ | 38649/110696 [11:49:13<18:06:08, 1.11it/s][2025-04-27 08:52:38] (step=0038650) Train Loss: 5.8799, Train Steps/Sec: 1.12 + 35%|████████████████████████████████████████████ | 38674/110696 [11:49:35<17:57:48, 1.11it/s][2025-04-27 08:53:00] (step=0038675) Train Loss: 5.8949, Train Steps/Sec: 1.12 + 35%|████████████████████████████████████████████ | 38699/110696 [11:49:57<17:50:56, 1.12it/s][2025-04-27 08:53:23] (step=0038700) Train Loss: 5.8902, Train Steps/Sec: 1.12 + 35%|████████████████████████████████████████████ | 38724/110696 [11:50:20<17:45:35, 1.13it/s][2025-04-27 08:53:45] (step=0038725) Train Loss: 5.9144, Train Steps/Sec: 1.12 + 35%|████████████████████████████████████████████ | 38749/110696 [11:50:42<17:44:41, 1.13it/s][2025-04-27 08:54:07] (step=0038750) Train Loss: 5.9047, Train Steps/Sec: 1.12 + 35%|████████████████████████████████████████████▏ | 38774/110696 [11:51:04<17:40:02, 1.13it/s][2025-04-27 08:54:30] (step=0038775) Train Loss: 5.8257, Train Steps/Sec: 1.12 + 35%|████████████████████████████████████████████▏ | 38799/110696 [11:51:27<17:41:01, 1.13it/s][2025-04-27 08:54:52] (step=0038800) Train Loss: 5.9062, Train Steps/Sec: 1.11 + 35%|████████████████████████████████████████████▏ | 38824/110696 [11:51:49<18:15:35, 1.09it/s][2025-04-27 08:55:14] (step=0038825) Train Loss: 5.8924, Train Steps/Sec: 1.12 + 35%|████████████████████████████████████████████▏ | 38849/110696 [11:52:11<18:03:24, 1.11it/s][2025-04-27 08:55:37] (step=0038850) Train Loss: 5.8536, Train Steps/Sec: 1.12 + 35%|████████████████████████████████████████████▏ | 38874/110696 [11:52:34<17:49:58, 1.12it/s][2025-04-27 08:55:59] (step=0038875) Train Loss: 5.8023, Train Steps/Sec: 1.12 + 35%|████████████████████████████████████████████▎ | 38899/110696 [11:52:56<17:47:42, 1.12it/s][2025-04-27 08:56:22] (step=0038900) Train Loss: 5.8616, Train Steps/Sec: 1.12 + 35%|████████████████████████████████████████████▎ | 38924/110696 [11:53:19<17:43:30, 1.12it/s][2025-04-27 08:56:44] (step=0038925) Train Loss: 5.9045, Train Steps/Sec: 1.12 + 35%|████████████████████████████████████████████▎ | 38949/110696 [11:53:41<17:41:09, 1.13it/s][2025-04-27 08:57:06] (step=0038950) Train Loss: 5.8987, Train Steps/Sec: 1.12 + 35%|████████████████████████████████████████████▎ | 38974/110696 [11:54:03<17:40:32, 1.13it/s][2025-04-27 08:57:29] (step=0038975) Train Loss: 5.9251, Train Steps/Sec: 1.12 + 35%|████████████████████████████████████████████▍ | 38999/110696 [11:54:26<17:42:51, 1.12it/s][2025-04-27 08:57:51] (step=0039000) Train Loss: 5.9266, Train Steps/Sec: 1.11 + 35%|████████████████████████████████████████████▍ | 39024/110696 [11:54:48<18:05:34, 1.10it/s][2025-04-27 08:58:13] (step=0039025) Train Loss: 5.8465, Train Steps/Sec: 1.12 + 35%|████████████████████████████████████████████▍ | 39049/110696 [11:55:10<17:54:29, 1.11it/s][2025-04-27 08:58:36] (step=0039050) Train Loss: 5.8760, Train Steps/Sec: 1.12 + 35%|████████████████████████████████████████████▍ | 39074/110696 [11:55:33<17:41:54, 1.12it/s][2025-04-27 08:58:58] (step=0039075) Train Loss: 5.8560, Train Steps/Sec: 1.12 + 35%|████████████████████████████████████████████▌ | 39099/110696 [11:55:55<17:46:28, 1.12it/s][2025-04-27 08:59:20] (step=0039100) Train Loss: 5.9058, Train Steps/Sec: 1.12 + 35%|████████████████████████████████████████████▌ | 39124/110696 [11:56:17<17:42:21, 1.12it/s][2025-04-27 08:59:43] (step=0039125) Train Loss: 5.8840, Train Steps/Sec: 1.12 + 35%|████████████████████████████████████████████▌ | 39149/110696 [11:56:40<17:35:52, 1.13it/s][2025-04-27 09:00:05] (step=0039150) Train Loss: 5.8594, Train Steps/Sec: 1.12 + 35%|████████████████████████████████████████████▌ | 39174/110696 [11:57:02<17:34:30, 1.13it/s][2025-04-27 09:00:27] (step=0039175) Train Loss: 5.8564, Train Steps/Sec: 1.12 + 35%|████████████████████████████████████████████▌ | 39199/110696 [11:57:24<17:36:55, 1.13it/s][2025-04-27 09:00:50] (step=0039200) Train Loss: 5.8613, Train Steps/Sec: 1.11 + 35%|████████████████████████████████████████████▋ | 39224/110696 [11:57:47<18:07:36, 1.10it/s][2025-04-27 09:01:12] (step=0039225) Train Loss: 5.8879, Train Steps/Sec: 1.12 + 35%|████████████████████████████████████████████▋ | 39249/110696 [11:58:09<17:47:55, 1.12it/s][2025-04-27 09:01:34] (step=0039250) Train Loss: 5.8766, Train Steps/Sec: 1.12 + 35%|████████████████████████████████████████████▋ | 39274/110696 [11:58:31<17:46:38, 1.12it/s][2025-04-27 09:01:57] (step=0039275) Train Loss: 5.8577, Train Steps/Sec: 1.12 + 36%|████████████████████████████████████████████▋ | 39299/110696 [11:58:54<17:42:28, 1.12it/s][2025-04-27 09:02:19] (step=0039300) Train Loss: 5.9746, Train Steps/Sec: 1.12 + 36%|████████████████████████████████████████████▊ | 39324/110696 [11:59:16<17:41:27, 1.12it/s][2025-04-27 09:02:41] (step=0039325) Train Loss: 5.9384, Train Steps/Sec: 1.12 + 36%|████████████████████████████████████████████▊ | 39349/110696 [11:59:38<17:36:18, 1.13it/s][2025-04-27 09:03:04] (step=0039350) Train Loss: 5.8779, Train Steps/Sec: 1.12 + 36%|████████████████████████████████████████████▊ | 39374/110696 [12:00:01<17:31:33, 1.13it/s][2025-04-27 09:03:26] (step=0039375) Train Loss: 5.8664, Train Steps/Sec: 1.12 + 36%|████████████████████████████████████████████▊ | 39399/110696 [12:00:23<17:33:18, 1.13it/s][2025-04-27 09:03:48] (step=0039400) Train Loss: 5.7833, Train Steps/Sec: 1.11 + 36%|████████████████████████████████████████████▊ | 39424/110696 [12:00:45<18:01:41, 1.10it/s][2025-04-27 09:04:11] (step=0039425) Train Loss: 5.8260, Train Steps/Sec: 1.12 + 36%|████████████████████████████████████████████▉ | 39449/110696 [12:01:08<17:46:39, 1.11it/s][2025-04-27 09:04:33] (step=0039450) Train Loss: 5.8953, Train Steps/Sec: 1.12 + 36%|████████████████████████████████████████████▉ | 39474/110696 [12:01:30<17:39:52, 1.12it/s][2025-04-27 09:04:55] (step=0039475) Train Loss: 5.9052, Train Steps/Sec: 1.12 + 36%|████████████████████████████████████████████▉ | 39499/110696 [12:01:52<17:37:22, 1.12it/s][2025-04-27 09:05:18] (step=0039500) Train Loss: 5.9044, Train Steps/Sec: 1.12 + 36%|████████████████████████████████████████████▉ | 39524/110696 [12:02:15<17:35:18, 1.12it/s][2025-04-27 09:05:40] (step=0039525) Train Loss: 5.8151, Train Steps/Sec: 1.12 + 36%|█████████████████████████████████████████████ | 39549/110696 [12:02:37<17:32:51, 1.13it/s][2025-04-27 09:06:02] (step=0039550) Train Loss: 5.8866, Train Steps/Sec: 1.12 + 36%|█████████████████████████████████████████████ | 39574/110696 [12:02:59<17:31:21, 1.13it/s][2025-04-27 09:06:25] (step=0039575) Train Loss: 5.9259, Train Steps/Sec: 1.12 + 36%|█████████████████████████████████████████████ | 39599/110696 [12:03:22<17:28:03, 1.13it/s][2025-04-27 09:06:47] (step=0039600) Train Loss: 5.8711, Train Steps/Sec: 1.12 + 36%|█████████████████████████████████████████████ | 39624/110696 [12:03:44<17:58:27, 1.10it/s][2025-04-27 09:07:10] (step=0039625) Train Loss: 5.8156, Train Steps/Sec: 1.12 + 36%|█████████████████████████████████████████████▏ | 39649/110696 [12:04:06<17:47:10, 1.11it/s][2025-04-27 09:07:32] (step=0039650) Train Loss: 5.8775, Train Steps/Sec: 1.12 + 36%|█████████████████████████████████████████████▏ | 39674/110696 [12:04:29<17:39:38, 1.12it/s][2025-04-27 09:07:54] (step=0039675) Train Loss: 5.8296, Train Steps/Sec: 1.12 + 36%|█████████████████████████████████████████████▏ | 39699/110696 [12:04:51<17:38:10, 1.12it/s][2025-04-27 09:08:16] (step=0039700) Train Loss: 5.9460, Train Steps/Sec: 1.12 + 36%|█████████████████████████████████████████████▏ | 39724/110696 [12:05:13<17:31:08, 1.13it/s][2025-04-27 09:08:39] (step=0039725) Train Loss: 5.8665, Train Steps/Sec: 1.12 + 36%|█████████████████████████████████████████████▏ | 39749/110696 [12:05:36<17:28:13, 1.13it/s][2025-04-27 09:09:01] (step=0039750) Train Loss: 5.7933, Train Steps/Sec: 1.12 + 36%|█████████████████████████████████████████████▎ | 39774/110696 [12:05:58<17:33:46, 1.12it/s][2025-04-27 09:09:24] (step=0039775) Train Loss: 5.8691, Train Steps/Sec: 1.12 + 36%|█████████████████████████████████████████████▎ | 39799/110696 [12:06:21<17:27:53, 1.13it/s][2025-04-27 09:09:46] (step=0039800) Train Loss: 5.8602, Train Steps/Sec: 1.12 + 36%|█████████████████████████████████████████████▎ | 39824/110696 [12:06:43<18:02:11, 1.09it/s][2025-04-27 09:10:08] (step=0039825) Train Loss: 5.9218, Train Steps/Sec: 1.12 + 36%|█████████████████████████████████████████████▎ | 39849/110696 [12:07:05<17:45:08, 1.11it/s][2025-04-27 09:10:31] (step=0039850) Train Loss: 5.9743, Train Steps/Sec: 1.12 + 36%|█████████████████████████████████████████████▍ | 39874/110696 [12:07:28<17:39:49, 1.11it/s][2025-04-27 09:10:53] (step=0039875) Train Loss: 5.8544, Train Steps/Sec: 1.12 + 36%|█████████████████████████████████████████████▍ | 39899/110696 [12:07:50<17:33:47, 1.12it/s][2025-04-27 09:11:15] (step=0039900) Train Loss: 5.8429, Train Steps/Sec: 1.12 + 36%|█████████████████████████████████████████████▍ | 39924/110696 [12:08:12<17:30:16, 1.12it/s][2025-04-27 09:11:38] (step=0039925) Train Loss: 5.9170, Train Steps/Sec: 1.12 + 36%|█████████████████████████████████████████████▍ | 39949/110696 [12:08:35<17:29:20, 1.12it/s][2025-04-27 09:12:00] (step=0039950) Train Loss: 5.8360, Train Steps/Sec: 1.12 + 36%|█████████████████████████████████████████████▌ | 39974/110696 [12:08:57<17:26:34, 1.13it/s][2025-04-27 09:12:22] (step=0039975) Train Loss: 5.8082, Train Steps/Sec: 1.12 + 36%|█████████████████████████████████████████████▌ | 39999/110696 [12:09:19<17:24:45, 1.13it/s][2025-04-27 09:12:45] (step=0040000) Train Loss: 5.8571, Train Steps/Sec: 1.11 +You are using a model of type blip-2 to instantiate a model of type instructblip. This is not supported for all configurations of models and can yield errors. +[2025-04-27 09:12:45] text_config is None. Initializing the text config with default values (`OPTConfig`). +model_name_or_path:/tmp/haozhezhao/model/blip2-flan-t5-xl +image_place_holder: +tokenizer length after expend 32101 +tokenizer length before expend 32100 +100%|████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████| 5/5 [04:56<00:00, 59.39s/it] +[2025-04-27 09:18:53] Finish Eval in 40000 steps...█████████████████████████████████████████████████████████████████████████████████████████████| 5/5 [04:56<00:00, 59.21s/it] +processing left-padding... +processing left-padding... +processing left-padding... +processing left-padding... +processing left-padding... +[2025-04-27 09:19:12] Saved checkpoint to checkpoint/X2I_UltraEdiit_fourmask_LLava_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_filtered_objects_4mask_1280_context/003-GPT-XL/checkpoints/0040000.pt +[2025-04-27 09:19:14] Removed old checkpoint: checkpoint/X2I_UltraEdiit_fourmask_LLava_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_filtered_objects_4mask_1280_context/003-GPT-XL/checkpoints/0038000.pt + 36%|█████████████████████████████████████████████▌ | 40024/110696 [12:16:11<18:19:14, 1.07it/s][2025-04-27 09:19:37] (step=0040025) Train Loss: 5.8694, Train Steps/Sec: 0.06 + 36%|█████████████████████████████████████████████▌ | 40049/110696 [12:16:34<17:39:53, 1.11it/s][2025-04-27 09:19:59] (step=0040050) Train Loss: 5.9151, Train Steps/Sec: 1.12 + 36%|█████████████████████████████████████████████▌ | 40074/110696 [12:16:56<17:32:08, 1.12it/s][2025-04-27 09:20:21] (step=0040075) Train Loss: 5.8016, Train Steps/Sec: 1.12 + 36%|█████████████████████████████████████████████▋ | 40099/110696 [12:17:18<17:24:09, 1.13it/s][2025-04-27 09:20:44] (step=0040100) Train Loss: 5.9272, Train Steps/Sec: 1.12 + 36%|█████████████████████████████████████████████▋ | 40124/110696 [12:17:41<17:25:35, 1.12it/s][2025-04-27 09:21:06] (step=0040125) Train Loss: 5.8374, Train Steps/Sec: 1.12 + 36%|█████████████████████████████████████████████▋ | 40149/110696 [12:18:03<17:47:26, 1.10it/s][2025-04-27 09:21:28] (step=0040150) Train Loss: 5.9271, Train Steps/Sec: 1.11 + 36%|█████████████████████████████████████████████▋ | 40174/110696 [12:18:25<17:20:04, 1.13it/s][2025-04-27 09:21:51] (step=0040175) Train Loss: 5.8495, Train Steps/Sec: 1.12 + 36%|█████████████████████████████████████████████▊ | 40199/110696 [12:18:48<17:21:21, 1.13it/s][2025-04-27 09:22:13] (step=0040200) Train Loss: 5.9265, Train Steps/Sec: 1.11 + 36%|█████████████████████████████████████████████▊ | 40224/110696 [12:19:10<17:47:28, 1.10it/s][2025-04-27 09:22:35] (step=0040225) Train Loss: 5.9238, Train Steps/Sec: 1.12 + 36%|█████████████████████████████████████████████▊ | 40249/110696 [12:19:32<17:38:24, 1.11it/s][2025-04-27 09:22:58] (step=0040250) Train Loss: 5.8567, Train Steps/Sec: 1.12 + 36%|█████████████████████████████████████████████▊ | 40274/110696 [12:19:55<17:28:38, 1.12it/s][2025-04-27 09:23:20] (step=0040275) Train Loss: 5.8310, Train Steps/Sec: 1.12 + 36%|█████████████████████████████████████████████▊ | 40299/110696 [12:20:17<17:26:50, 1.12it/s][2025-04-27 09:23:42] (step=0040300) Train Loss: 5.8119, Train Steps/Sec: 1.12 + 36%|█████████████████████████████████████████████▉ | 40324/110696 [12:20:39<17:21:34, 1.13it/s][2025-04-27 09:24:05] (step=0040325) Train Loss: 5.8620, Train Steps/Sec: 1.12 + 36%|█████████████████████████████████████████████▉ | 40349/110696 [12:21:02<17:20:04, 1.13it/s][2025-04-27 09:24:27] (step=0040350) Train Loss: 5.8932, Train Steps/Sec: 1.12 + 36%|█████████████████████████████████████████████▉ | 40374/110696 [12:21:24<17:18:57, 1.13it/s][2025-04-27 09:24:49] (step=0040375) Train Loss: 5.9260, Train Steps/Sec: 1.12 + 36%|█████████████████████████████████████████████▉ | 40399/110696 [12:21:46<17:15:37, 1.13it/s][2025-04-27 09:25:12] (step=0040400) Train Loss: 5.8629, Train Steps/Sec: 1.12 + 37%|██████████████████████████████████████████████ | 40424/110696 [12:22:09<17:47:56, 1.10it/s][2025-04-27 09:25:34] (step=0040425) Train Loss: 5.8828, Train Steps/Sec: 1.12 + 37%|██████████████████████████████████████████████ | 40449/110696 [12:22:31<17:34:05, 1.11it/s][2025-04-27 09:25:56] (step=0040450) Train Loss: 5.9289, Train Steps/Sec: 1.12 + 37%|██████████████████████████████████████████████ | 40474/110696 [12:22:53<17:25:54, 1.12it/s][2025-04-27 09:26:19] (step=0040475) Train Loss: 5.8689, Train Steps/Sec: 1.12 + 37%|██████████████████████████████████████████████ | 40499/110696 [12:23:16<17:25:44, 1.12it/s][2025-04-27 09:26:41] (step=0040500) Train Loss: 5.8557, Train Steps/Sec: 1.12 + 37%|██████████████████████████████████████████████▏ | 40524/110696 [12:23:38<17:20:37, 1.12it/s][2025-04-27 09:27:03] (step=0040525) Train Loss: 5.8864, Train Steps/Sec: 1.12 + 37%|██████████████████████████████████████████████▏ | 40549/110696 [12:24:00<17:10:18, 1.13it/s][2025-04-27 09:27:26] (step=0040550) Train Loss: 5.8648, Train Steps/Sec: 1.12 + 37%|██████████████████████████████████████████████▏ | 40574/110696 [12:24:22<17:15:34, 1.13it/s][2025-04-27 09:27:48] (step=0040575) Train Loss: 5.8974, Train Steps/Sec: 1.12 + 37%|██████████████████████████████████████████████▏ | 40599/110696 [12:24:45<17:12:15, 1.13it/s][2025-04-27 09:28:10] (step=0040600) Train Loss: 5.8579, Train Steps/Sec: 1.11 + 37%|██████████████████████████████████████████████▏ | 40624/110696 [12:25:07<17:45:47, 1.10it/s][2025-04-27 09:28:33] (step=0040625) Train Loss: 5.8884, Train Steps/Sec: 1.12 + 37%|██████████████████████████████████████████████▎ | 40649/110696 [12:25:30<17:35:52, 1.11it/s][2025-04-27 09:28:55] (step=0040650) Train Loss: 5.9110, Train Steps/Sec: 1.12 + 37%|██████████████████████████████████████████████▎ | 40674/110696 [12:25:52<17:22:42, 1.12it/s][2025-04-27 09:29:17] (step=0040675) Train Loss: 5.8622, Train Steps/Sec: 1.12 + 37%|██████████████████████████████████████████████▎ | 40699/110696 [12:26:14<17:20:37, 1.12it/s][2025-04-27 09:29:40] (step=0040700) Train Loss: 5.8962, Train Steps/Sec: 1.12 + 37%|██████████████████████████████████████████████▎ | 40724/110696 [12:26:37<17:15:44, 1.13it/s][2025-04-27 09:30:02] (step=0040725) Train Loss: 5.8917, Train Steps/Sec: 1.12 + 37%|██████████████████████████████████████████████▍ | 40749/110696 [12:26:59<17:18:10, 1.12it/s][2025-04-27 09:30:24] (step=0040750) Train Loss: 5.8723, Train Steps/Sec: 1.12 + 37%|██████████████████████████████████████████████▍ | 40774/110696 [12:27:21<17:10:58, 1.13it/s][2025-04-27 09:30:47] (step=0040775) Train Loss: 5.7867, Train Steps/Sec: 1.12 + 37%|██████████████████████████████████████████████▍ | 40799/110696 [12:27:43<17:07:47, 1.13it/s][2025-04-27 09:31:09] (step=0040800) Train Loss: 5.8418, Train Steps/Sec: 1.12 + 37%|██████████████████████████████████████████████▍ | 40824/110696 [12:28:06<17:43:26, 1.10it/s][2025-04-27 09:31:31] (step=0040825) Train Loss: 5.8711, Train Steps/Sec: 1.12 + 37%|██████████████████████████████████████████████▍ | 40849/110696 [12:28:28<17:28:04, 1.11it/s][2025-04-27 09:31:54] (step=0040850) Train Loss: 5.8872, Train Steps/Sec: 1.12 + 37%|██████████████████████████████████████████████▌ | 40874/110696 [12:28:51<17:19:14, 1.12it/s][2025-04-27 09:32:16] (step=0040875) Train Loss: 5.8744, Train Steps/Sec: 1.12 + 37%|██████████████████████████████████████████████▌ | 40899/110696 [12:29:13<17:13:49, 1.13it/s][2025-04-27 09:32:38] (step=0040900) Train Loss: 5.8982, Train Steps/Sec: 1.12 + 37%|██████████████████████████████████████████████▌ | 40924/110696 [12:29:35<17:14:48, 1.12it/s][2025-04-27 09:33:01] (step=0040925) Train Loss: 5.9591, Train Steps/Sec: 1.12 + 37%|██████████████████████████████████████████████▌ | 40949/110696 [12:29:57<17:11:56, 1.13it/s][2025-04-27 09:33:23] (step=0040950) Train Loss: 5.7693, Train Steps/Sec: 1.12 + 37%|██████████████████████████████████████████████▋ | 40974/110696 [12:30:20<17:07:26, 1.13it/s][2025-04-27 09:33:45] (step=0040975) Train Loss: 5.8233, Train Steps/Sec: 1.12 + 37%|██████████████████████████████████████████████▋ | 40999/110696 [12:30:42<17:06:28, 1.13it/s][2025-04-27 09:34:08] (step=0041000) Train Loss: 5.8705, Train Steps/Sec: 1.12 + 37%|██████████████████████████████████████████████▋ | 41024/110696 [12:31:10<19:20:04, 1.00it/s][2025-04-27 09:34:35] (step=0041025) Train Loss: 5.8776, Train Steps/Sec: 0.91 + 37%|██████████████████████████████████████████████▋ | 41049/110696 [12:31:37<17:27:30, 1.11it/s][2025-04-27 09:35:02] (step=0041050) Train Loss: 5.8488, Train Steps/Sec: 0.93 + 37%|██████████████████████████████████████████████▊ | 41074/110696 [12:31:59<17:20:04, 1.12it/s][2025-04-27 09:35:24] (step=0041075) Train Loss: 5.8486, Train Steps/Sec: 1.12 + 37%|██████████████████████████████████████████████▊ | 41099/110696 [12:32:21<17:15:45, 1.12it/s][2025-04-27 09:35:47] (step=0041100) Train Loss: 5.8776, Train Steps/Sec: 1.12 + 37%|██████████████████████████████████████████████▊ | 41124/110696 [12:32:48<17:14:25, 1.12it/s][2025-04-27 09:36:14] (step=0041125) Train Loss: 5.8817, Train Steps/Sec: 0.93 + 37%|██████████████████████████████████████████████▊ | 41149/110696 [12:33:11<17:06:36, 1.13it/s][2025-04-27 09:36:36] (step=0041150) Train Loss: 5.8038, Train Steps/Sec: 1.12 + 37%|██████████████████████████████████████████████▊ | 41174/110696 [12:33:33<17:07:58, 1.13it/s][2025-04-27 09:36:58] (step=0041175) Train Loss: 5.8812, Train Steps/Sec: 1.12 + 37%|██████████████████████████████████████████████▉ | 41199/110696 [12:33:55<17:02:57, 1.13it/s][2025-04-27 09:37:21] (step=0041200) Train Loss: 5.8525, Train Steps/Sec: 1.12 + 37%|██████████████████████████████████████████████▉ | 41224/110696 [12:34:18<17:35:39, 1.10it/s][2025-04-27 09:37:43] (step=0041225) Train Loss: 5.9068, Train Steps/Sec: 1.12 + 37%|██████████████████████████████████████████████▉ | 41249/110696 [12:34:40<17:24:49, 1.11it/s][2025-04-27 09:38:05] (step=0041250) Train Loss: 5.8711, Train Steps/Sec: 1.12 + 37%|██████████████████████████████████████████████▉ | 41274/110696 [12:35:02<17:14:17, 1.12it/s][2025-04-27 09:38:28] (step=0041275) Train Loss: 5.8500, Train Steps/Sec: 1.12 + 37%|███████████████████████████████████████████████ | 41299/110696 [12:35:24<17:09:51, 1.12it/s][2025-04-27 09:38:50] (step=0041300) Train Loss: 5.8938, Train Steps/Sec: 1.12 + 37%|███████████████████████████████████████████████ | 41324/110696 [12:35:47<17:06:44, 1.13it/s][2025-04-27 09:39:12] (step=0041325) Train Loss: 5.8382, Train Steps/Sec: 1.12 + 37%|███████████████████████████████████████████████ | 41349/110696 [12:36:19<17:57:30, 1.07it/s][2025-04-27 09:39:44] (step=0041350) Train Loss: 5.9090, Train Steps/Sec: 0.78 + 37%|███████████████████████████████████████████████ | 41374/110696 [12:36:51<17:04:14, 1.13it/s][2025-04-27 09:40:17] (step=0041375) Train Loss: 5.9219, Train Steps/Sec: 0.77 + 37%|███████████████████████████████████████████████ | 41399/110696 [12:37:13<17:02:28, 1.13it/s][2025-04-27 09:40:39] (step=0041400) Train Loss: 5.8639, Train Steps/Sec: 1.11 + 37%|███████████████████████████████████████████████▏ | 41424/110696 [12:37:36<17:32:11, 1.10it/s][2025-04-27 09:41:01] (step=0041425) Train Loss: 5.8749, Train Steps/Sec: 1.12 + 37%|███████████████████████████████████████████████▏ | 41449/110696 [12:37:58<17:23:28, 1.11it/s][2025-04-27 09:41:24] (step=0041450) Train Loss: 5.8374, Train Steps/Sec: 1.12 + 37%|███████████████████████████████████████████████▏ | 41474/110696 [12:38:21<17:12:06, 1.12it/s][2025-04-27 09:41:46] (step=0041475) Train Loss: 5.8837, Train Steps/Sec: 1.12 + 37%|███████████████████████████████████████████████▏ | 41499/110696 [12:38:43<17:06:27, 1.12it/s][2025-04-27 09:42:08] (step=0041500) Train Loss: 5.8397, Train Steps/Sec: 1.12 + 38%|███████████████████████████████████████████████▎ | 41524/110696 [12:39:05<17:02:26, 1.13it/s][2025-04-27 09:42:31] (step=0041525) Train Loss: 5.8521, Train Steps/Sec: 1.12 + 38%|███████████████████████████████████████████████▎ | 41549/110696 [12:39:28<17:00:54, 1.13it/s][2025-04-27 09:42:53] (step=0041550) Train Loss: 5.9077, Train Steps/Sec: 1.12 + 38%|███████████████████████████████████████████████▎ | 41574/110696 [12:39:50<17:01:44, 1.13it/s][2025-04-27 09:43:15] (step=0041575) Train Loss: 5.8805, Train Steps/Sec: 1.12 + 38%|███████████████████████████████████████████████▎ | 41599/110696 [12:40:12<16:58:03, 1.13it/s][2025-04-27 09:43:38] (step=0041600) Train Loss: 5.8455, Train Steps/Sec: 1.12 + 38%|███████████████████████████████████████████████▍ | 41624/110696 [12:40:35<17:28:21, 1.10it/s][2025-04-27 09:44:00] (step=0041625) Train Loss: 5.7876, Train Steps/Sec: 1.12 + 38%|███████████████████████████████████████████████▍ | 41649/110696 [12:40:57<17:20:18, 1.11it/s][2025-04-27 09:44:22] (step=0041650) Train Loss: 5.8443, Train Steps/Sec: 1.12 + 38%|███████████████████████████████████████████████▍ | 41674/110696 [12:41:19<17:11:39, 1.12it/s][2025-04-27 09:44:45] (step=0041675) Train Loss: 5.8460, Train Steps/Sec: 1.12 + 38%|███████████████████████████████████████████████▍ | 41699/110696 [12:41:42<17:06:22, 1.12it/s][2025-04-27 09:45:07] (step=0041700) Train Loss: 5.9755, Train Steps/Sec: 1.12 + 38%|███████████████████████████████████████████████▍ | 41724/110696 [12:42:04<16:59:44, 1.13it/s][2025-04-27 09:45:29] (step=0041725) Train Loss: 5.8437, Train Steps/Sec: 1.12 + 38%|███████████████████████████████████████████████▌ | 41749/110696 [12:42:26<17:02:52, 1.12it/s][2025-04-27 09:45:52] (step=0041750) Train Loss: 5.9234, Train Steps/Sec: 1.12 + 38%|███████████████████████████████████████████████▌ | 41774/110696 [12:42:49<16:58:47, 1.13it/s][2025-04-27 09:46:14] (step=0041775) Train Loss: 5.8555, Train Steps/Sec: 1.12 + 38%|███████████████████████████████████████████████▌ | 41799/110696 [12:43:11<16:52:34, 1.13it/s][2025-04-27 09:46:36] (step=0041800) Train Loss: 5.9066, Train Steps/Sec: 1.11 + 38%|███████████████████████████████████████████████▌ | 41824/110696 [12:43:33<17:24:14, 1.10it/s][2025-04-27 09:46:59] (step=0041825) Train Loss: 5.9010, Train Steps/Sec: 1.12 + 38%|███████████████████████████████████████████████▋ | 41849/110696 [12:43:56<17:14:56, 1.11it/s][2025-04-27 09:47:21] (step=0041850) Train Loss: 5.9204, Train Steps/Sec: 1.12 + 38%|███████████████████████████████████████████████▋ | 41874/110696 [12:44:18<17:10:41, 1.11it/s][2025-04-27 09:47:43] (step=0041875) Train Loss: 5.8899, Train Steps/Sec: 1.12 + 38%|███████████████████████████████████████████████▋ | 41899/110696 [12:44:40<17:03:59, 1.12it/s][2025-04-27 09:48:06] (step=0041900) Train Loss: 5.8648, Train Steps/Sec: 1.12 + 38%|███████████████████████████████████████████████▋ | 41924/110696 [12:45:03<16:57:04, 1.13it/s][2025-04-27 09:48:28] (step=0041925) Train Loss: 5.8011, Train Steps/Sec: 1.12 + 38%|███████████████████████████████████████████████▋ | 41949/110696 [12:45:25<16:56:50, 1.13it/s][2025-04-27 09:48:50] (step=0041950) Train Loss: 5.8391, Train Steps/Sec: 1.12 + 38%|███████████████████████████████████████████████▊ | 41974/110696 [12:45:47<16:52:45, 1.13it/s][2025-04-27 09:49:13] (step=0041975) Train Loss: 5.8392, Train Steps/Sec: 1.12 + 38%|███████████████████████████████████████████████▊ | 41999/110696 [12:46:10<16:50:48, 1.13it/s][2025-04-27 09:49:35] (step=0042000) Train Loss: 5.8959, Train Steps/Sec: 1.12 +You are using a model of type blip-2 to instantiate a model of type instructblip. This is not supported for all configurations of models and can yield errors. +[2025-04-27 09:49:35] text_config is None. Initializing the text config with default values (`OPTConfig`). +model_name_or_path:/tmp/haozhezhao/model/blip2-flan-t5-xl +image_place_holder: +tokenizer length after expend 32101 +tokenizer length before expend 32100 +100%|████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████| 5/5 [05:06<00:00, 61.31s/it] +[2025-04-27 09:56:02] Finish Eval in 42000 steps...█████████████████████████████████████████████████████████████████████████████████████████████| 5/5 [05:05<00:00, 60.85s/it] +processing left-padding... +processing left-padding... +processing left-padding... +processing left-padding... +processing left-padding... +[2025-04-27 09:56:24] Saved checkpoint to checkpoint/X2I_UltraEdiit_fourmask_LLava_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_filtered_objects_4mask_1280_context/003-GPT-XL/checkpoints/0042000.pt +[2025-04-27 09:56:27] Removed old checkpoint: checkpoint/X2I_UltraEdiit_fourmask_LLava_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_filtered_objects_4mask_1280_context/003-GPT-XL/checkpoints/0040000.pt + 38%|███████████████████████████████████████████████▊ | 42024/110696 [12:53:24<17:54:19, 1.07it/s][2025-04-27 09:56:49] (step=0042025) Train Loss: 5.8824, Train Steps/Sec: 0.06 + 38%|███████████████████████████████████████████████▊ | 42049/110696 [12:53:47<17:50:19, 1.07it/s][2025-04-27 09:57:12] (step=0042050) Train Loss: 5.8154, Train Steps/Sec: 1.09 + 38%|███████████████████████████████████████████████▉ | 42074/110696 [12:54:09<17:05:29, 1.12it/s][2025-04-27 09:57:35] (step=0042075) Train Loss: 5.9001, Train Steps/Sec: 1.12 + 38%|███████████████████████████████████████████████▉ | 42099/110696 [12:54:32<16:58:34, 1.12it/s][2025-04-27 09:57:57] (step=0042100) Train Loss: 5.9237, Train Steps/Sec: 1.12 + 38%|███████████████████████████████████████████████▉ | 42124/110696 [12:54:54<16:54:06, 1.13it/s][2025-04-27 09:58:19] (step=0042125) Train Loss: 5.8363, Train Steps/Sec: 1.12 + 38%|███████████████████████████████████████████████▉ | 42149/110696 [12:55:16<16:54:44, 1.13it/s][2025-04-27 09:58:42] (step=0042150) Train Loss: 5.7919, Train Steps/Sec: 1.12 + 38%|████████████████████████████████████████████████ | 42174/110696 [12:55:39<16:52:42, 1.13it/s][2025-04-27 09:59:04] (step=0042175) Train Loss: 5.9020, Train Steps/Sec: 1.12 + 38%|████████████████████████████████████████████████ | 42199/110696 [12:56:01<16:57:44, 1.12it/s][2025-04-27 09:59:27] (step=0042200) Train Loss: 5.8871, Train Steps/Sec: 1.11 + 38%|████████████████████████████████████████████████ | 42224/110696 [12:56:24<17:20:17, 1.10it/s][2025-04-27 09:59:49] (step=0042225) Train Loss: 5.9024, Train Steps/Sec: 1.11 + 38%|████████████████████████████████████████████████ | 42249/110696 [12:56:46<17:06:34, 1.11it/s][2025-04-27 10:00:11] (step=0042250) Train Loss: 5.9021, Train Steps/Sec: 1.12 + 38%|████████████████████████████████████████████████ | 42274/110696 [12:57:09<17:10:19, 1.11it/s][2025-04-27 10:00:34] (step=0042275) Train Loss: 5.8520, Train Steps/Sec: 1.11 + 38%|████████████████████████████████████████████████▏ | 42299/110696 [12:57:31<16:58:10, 1.12it/s][2025-04-27 10:00:56] (step=0042300) Train Loss: 5.8108, Train Steps/Sec: 1.12 + 38%|████████████████████████████████████████████████▏ | 42324/110696 [12:57:53<16:52:01, 1.13it/s][2025-04-27 10:01:19] (step=0042325) Train Loss: 5.8923, Train Steps/Sec: 1.12 + 38%|████████████████████████████████████████████████▏ | 42349/110696 [12:58:16<16:54:15, 1.12it/s][2025-04-27 10:01:41] (step=0042350) Train Loss: 5.9194, Train Steps/Sec: 1.11 + 38%|████████████████████████████████████████████████▏ | 42374/110696 [12:58:38<16:58:48, 1.12it/s][2025-04-27 10:02:04] (step=0042375) Train Loss: 5.8624, Train Steps/Sec: 1.11 + 38%|████████████████████████████████████████████████▎ | 42399/110696 [12:59:01<16:57:31, 1.12it/s][2025-04-27 10:02:26] (step=0042400) Train Loss: 5.8829, Train Steps/Sec: 1.10 + 38%|████████████████████████████████████████████████▎ | 42424/110696 [12:59:23<17:29:04, 1.08it/s][2025-04-27 10:02:49] (step=0042425) Train Loss: 5.8410, Train Steps/Sec: 1.11 + 38%|████████████████████████████████████████████████▎ | 42449/110696 [12:59:46<17:08:31, 1.11it/s][2025-04-27 10:03:11] (step=0042450) Train Loss: 5.8517, Train Steps/Sec: 1.11 + 38%|████████████████████████████████████████████████▎ | 42474/110696 [13:00:08<17:03:07, 1.11it/s][2025-04-27 10:03:34] (step=0042475) Train Loss: 5.8804, Train Steps/Sec: 1.11 + 38%|████████████████████████████████████████████████▎ | 42499/110696 [13:00:31<16:54:15, 1.12it/s][2025-04-27 10:03:56] (step=0042500) Train Loss: 5.8574, Train Steps/Sec: 1.12 + 38%|████████████████████████████████████████████████▍ | 42524/110696 [13:00:53<16:51:06, 1.12it/s][2025-04-27 10:04:18] (step=0042525) Train Loss: 5.8626, Train Steps/Sec: 1.12 + 38%|████████████████████████████████████████████████▍ | 42549/110696 [13:01:15<16:46:27, 1.13it/s][2025-04-27 10:04:41] (step=0042550) Train Loss: 5.8692, Train Steps/Sec: 1.12 + 38%|████████████████████████████████████████████████▍ | 42574/110696 [13:01:38<16:49:07, 1.13it/s][2025-04-27 10:05:03] (step=0042575) Train Loss: 5.9332, Train Steps/Sec: 1.12 + 38%|████████████████████████████████████████████████▍ | 42599/110696 [13:02:00<16:49:06, 1.12it/s][2025-04-27 10:05:26] (step=0042600) Train Loss: 5.8781, Train Steps/Sec: 1.11 + 39%|████████████████████████████████████████████████▌ | 42624/110696 [13:02:23<17:20:27, 1.09it/s][2025-04-27 10:05:48] (step=0042625) Train Loss: 5.8606, Train Steps/Sec: 1.11 + 39%|████████████████████████████████████████████████▌ | 42649/110696 [13:02:45<17:01:33, 1.11it/s][2025-04-27 10:06:10] (step=0042650) Train Loss: 5.7922, Train Steps/Sec: 1.12 + 39%|████████████████████████████████████████████████▌ | 42674/110696 [13:03:07<16:59:40, 1.11it/s][2025-04-27 10:06:33] (step=0042675) Train Loss: 5.8096, Train Steps/Sec: 1.12 + 39%|████████████████████████████████████████████████▌ | 42699/110696 [13:03:30<16:48:42, 1.12it/s][2025-04-27 10:06:55] (step=0042700) Train Loss: 5.7826, Train Steps/Sec: 1.12 + 39%|████████████████████████████████████████████████▋ | 42724/110696 [13:03:52<16:47:36, 1.12it/s][2025-04-27 10:07:17] (step=0042725) Train Loss: 5.8463, Train Steps/Sec: 1.12 + 39%|████████████████████████████████████████████████▋ | 42749/110696 [13:04:14<16:42:57, 1.13it/s][2025-04-27 10:07:40] (step=0042750) Train Loss: 5.8804, Train Steps/Sec: 1.12 + 39%|████████████████████████████████████████████████▋ | 42774/110696 [13:04:37<16:41:47, 1.13it/s][2025-04-27 10:08:02] (step=0042775) Train Loss: 5.8643, Train Steps/Sec: 1.12 + 39%|████████████████████████████████████████████████▋ | 42799/110696 [13:04:59<16:44:11, 1.13it/s][2025-04-27 10:08:25] (step=0042800) Train Loss: 5.7893, Train Steps/Sec: 1.11 + 39%|████████████████████████████████████████████████▋ | 42824/110696 [13:05:22<17:14:26, 1.09it/s][2025-04-27 10:08:47] (step=0042825) Train Loss: 5.8519, Train Steps/Sec: 1.12 + 39%|████████████████████████████████████████████████▊ | 42849/110696 [13:05:44<17:00:36, 1.11it/s][2025-04-27 10:09:09] (step=0042850) Train Loss: 5.8428, Train Steps/Sec: 1.12 + 39%|████████████████████████████████████████████████▊ | 42874/110696 [13:06:06<16:49:30, 1.12it/s][2025-04-27 10:09:32] (step=0042875) Train Loss: 5.8890, Train Steps/Sec: 1.12 + 39%|████████████████████████████████████████████████▊ | 42899/110696 [13:06:29<16:49:58, 1.12it/s][2025-04-27 10:09:54] (step=0042900) Train Loss: 5.8034, Train Steps/Sec: 1.12 + 39%|████████████████████████████████████████████████▊ | 42924/110696 [13:06:51<16:42:42, 1.13it/s][2025-04-27 10:10:16] (step=0042925) Train Loss: 5.8932, Train Steps/Sec: 1.12 + 39%|████████████████████████████████████████████████▉ | 42949/110696 [13:07:13<16:40:46, 1.13it/s][2025-04-27 10:10:38] (step=0042950) Train Loss: 5.8606, Train Steps/Sec: 1.12 + 39%|████████████████████████████████████████████████▉ | 42974/110696 [13:07:35<16:43:48, 1.12it/s][2025-04-27 10:11:01] (step=0042975) Train Loss: 5.8512, Train Steps/Sec: 1.12 + 39%|████████████████████████████████████████████████▉ | 42999/110696 [13:07:58<16:38:30, 1.13it/s][2025-04-27 10:11:23] (step=0043000) Train Loss: 5.7795, Train Steps/Sec: 1.12 + 39%|████████████████████████████████████████████████▉ | 43024/110696 [13:08:20<17:09:42, 1.10it/s][2025-04-27 10:11:46] (step=0043025) Train Loss: 5.8279, Train Steps/Sec: 1.12 + 39%|█████████████████████████████████████████████████ | 43049/110696 [13:08:42<16:54:44, 1.11it/s][2025-04-27 10:12:08] (step=0043050) Train Loss: 5.8498, Train Steps/Sec: 1.12 + 39%|█████████████████████████████████████████████████ | 43074/110696 [13:09:05<16:53:58, 1.11it/s][2025-04-27 10:12:30] (step=0043075) Train Loss: 5.8675, Train Steps/Sec: 1.12 + 39%|█████████████████████████████████████████████████ | 43099/110696 [13:09:27<16:42:48, 1.12it/s][2025-04-27 10:12:52] (step=0043100) Train Loss: 5.8528, Train Steps/Sec: 1.12 + 39%|█████████████████████████████████████████████████ | 43124/110696 [13:09:49<16:40:01, 1.13it/s][2025-04-27 10:13:15] (step=0043125) Train Loss: 5.8991, Train Steps/Sec: 1.12 + 39%|█████████████████████████████████████████████████ | 43149/110696 [13:10:12<16:38:49, 1.13it/s][2025-04-27 10:13:37] (step=0043150) Train Loss: 5.8579, Train Steps/Sec: 1.12 + 39%|█████████████████████████████████████████████████▏ | 43174/110696 [13:10:34<16:37:46, 1.13it/s][2025-04-27 10:13:59] (step=0043175) Train Loss: 5.8585, Train Steps/Sec: 1.12 + 39%|█████████████████████████████████████████████████▏ | 43199/110696 [13:10:56<16:36:58, 1.13it/s][2025-04-27 10:14:22] (step=0043200) Train Loss: 5.7799, Train Steps/Sec: 1.11 + 39%|█████████████████████████████████████████████████▏ | 43224/110696 [13:11:19<17:04:07, 1.10it/s][2025-04-27 10:14:44] (step=0043225) Train Loss: 5.8778, Train Steps/Sec: 1.12 + 39%|█████████████████████████████████████████████████▏ | 43249/110696 [13:11:41<16:51:56, 1.11it/s][2025-04-27 10:15:06] (step=0043250) Train Loss: 5.8280, Train Steps/Sec: 1.12 + 39%|█████████████████████████████████████████████████▎ | 43274/110696 [13:12:03<16:42:32, 1.12it/s][2025-04-27 10:15:29] (step=0043275) Train Loss: 5.8248, Train Steps/Sec: 1.12 + 39%|█████████████████████████████████████████████████▎ | 43299/110696 [13:12:26<16:42:02, 1.12it/s][2025-04-27 10:15:51] (step=0043300) Train Loss: 5.8637, Train Steps/Sec: 1.12 + 39%|█████████████████████████████████████████████████▎ | 43324/110696 [13:12:48<16:38:01, 1.13it/s][2025-04-27 10:16:13] (step=0043325) Train Loss: 5.8435, Train Steps/Sec: 1.12 + 39%|█████████████████████████████████████████████████▎ | 43349/110696 [13:13:10<16:37:22, 1.13it/s][2025-04-27 10:16:36] (step=0043350) Train Loss: 5.8471, Train Steps/Sec: 1.12 + 39%|█████████████████████████████████████████████████▎ | 43374/110696 [13:13:33<16:32:28, 1.13it/s][2025-04-27 10:16:58] (step=0043375) Train Loss: 5.8254, Train Steps/Sec: 1.12 + 39%|█████████████████████████████████████████████████▍ | 43399/110696 [13:13:55<16:30:46, 1.13it/s][2025-04-27 10:17:20] (step=0043400) Train Loss: 5.8449, Train Steps/Sec: 1.12 + 39%|█████████████████████████████████████████████████▍ | 43424/110696 [13:14:17<17:00:00, 1.10it/s][2025-04-27 10:17:43] (step=0043425) Train Loss: 5.8331, Train Steps/Sec: 1.12 + 39%|█████████████████████████████████████████████████▍ | 43449/110696 [13:14:40<16:47:26, 1.11it/s][2025-04-27 10:18:05] (step=0043450) Train Loss: 5.8370, Train Steps/Sec: 1.12 + 39%|█████████████████████████████████████████████████▍ | 43474/110696 [13:15:02<16:43:37, 1.12it/s][2025-04-27 10:18:27] (step=0043475) Train Loss: 5.8046, Train Steps/Sec: 1.12 + 39%|█████████████████████████████████████████████████▌ | 43499/110696 [13:15:24<16:39:59, 1.12it/s][2025-04-27 10:18:50] (step=0043500) Train Loss: 5.8592, Train Steps/Sec: 1.12 + 39%|█████████████████████████████████████████████████▌ | 43524/110696 [13:15:47<16:37:01, 1.12it/s][2025-04-27 10:19:12] (step=0043525) Train Loss: 5.8800, Train Steps/Sec: 1.12 + 39%|█████████████████████████████████████████████████▌ | 43549/110696 [13:16:09<16:36:22, 1.12it/s][2025-04-27 10:19:35] (step=0043550) Train Loss: 5.9135, Train Steps/Sec: 1.12 + 39%|█████████████████████████████████████████████████▌ | 43574/110696 [13:16:31<16:32:01, 1.13it/s][2025-04-27 10:19:57] (step=0043575) Train Loss: 5.8626, Train Steps/Sec: 1.12 + 39%|█████████████████████████████████████████████████▋ | 43599/110696 [13:16:54<16:28:59, 1.13it/s][2025-04-27 10:20:19] (step=0043600) Train Loss: 5.9251, Train Steps/Sec: 1.11 + 39%|█████████████████████████████████████████████████▋ | 43624/110696 [13:17:16<16:58:31, 1.10it/s][2025-04-27 10:20:42] (step=0043625) Train Loss: 5.7884, Train Steps/Sec: 1.12 + 39%|█████████████████████████████████████████████████▋ | 43649/110696 [13:17:39<16:48:16, 1.11it/s][2025-04-27 10:21:04] (step=0043650) Train Loss: 5.8518, Train Steps/Sec: 1.12 + 39%|█████████████████████████████████████████████████▋ | 43674/110696 [13:18:01<16:42:03, 1.11it/s][2025-04-27 10:21:26] (step=0043675) Train Loss: 5.8425, Train Steps/Sec: 1.12 + 39%|█████████████████████████████████████████████████▋ | 43699/110696 [13:18:23<16:35:30, 1.12it/s][2025-04-27 10:21:49] (step=0043700) Train Loss: 5.8809, Train Steps/Sec: 1.12 + 39%|█████████████████████████████████████████████████▊ | 43724/110696 [13:18:46<16:33:27, 1.12it/s][2025-04-27 10:22:11] (step=0043725) Train Loss: 5.8595, Train Steps/Sec: 1.12 + 40%|█████████████████████████████████████████████████▊ | 43749/110696 [13:19:08<16:33:57, 1.12it/s][2025-04-27 10:22:33] (step=0043750) Train Loss: 5.8669, Train Steps/Sec: 1.12 + 40%|█████████████████████████████████████████████████▊ | 43774/110696 [13:19:30<16:27:44, 1.13it/s][2025-04-27 10:22:56] (step=0043775) Train Loss: 5.8286, Train Steps/Sec: 1.12 + 40%|█████████████████████████████████████████████████▊ | 43799/110696 [13:19:53<16:26:51, 1.13it/s][2025-04-27 10:23:18] (step=0043800) Train Loss: 5.8904, Train Steps/Sec: 1.11 + 40%|█████████████████████████████████████████████████▉ | 43824/110696 [13:20:15<16:59:59, 1.09it/s][2025-04-27 10:23:40] (step=0043825) Train Loss: 5.7997, Train Steps/Sec: 1.12 + 40%|█████████████████████████████████████████████████▉ | 43849/110696 [13:20:37<16:40:57, 1.11it/s][2025-04-27 10:24:03] (step=0043850) Train Loss: 5.8477, Train Steps/Sec: 1.12 + 40%|█████████████████████████████████████████████████▉ | 43874/110696 [13:21:00<16:35:44, 1.12it/s][2025-04-27 10:24:25] (step=0043875) Train Loss: 5.7913, Train Steps/Sec: 1.12 + 40%|█████████████████████████████████████████████████▉ | 43899/110696 [13:21:22<16:33:45, 1.12it/s][2025-04-27 10:24:47] (step=0043900) Train Loss: 5.8447, Train Steps/Sec: 1.12 + 40%|█████████████████████████████████████████████████▉ | 43924/110696 [13:21:44<16:35:17, 1.12it/s][2025-04-27 10:25:10] (step=0043925) Train Loss: 5.7923, Train Steps/Sec: 1.12 + 40%|██████████████████████████████████████████████████ | 43949/110696 [13:22:07<16:26:13, 1.13it/s][2025-04-27 10:25:32] (step=0043950) Train Loss: 5.9140, Train Steps/Sec: 1.12 + 40%|██████████████████████████████████████████████████ | 43974/110696 [13:22:29<16:26:31, 1.13it/s][2025-04-27 10:25:54] (step=0043975) Train Loss: 5.8561, Train Steps/Sec: 1.12 + 40%|██████████████████████████████████████████████████ | 43999/110696 [13:22:51<16:29:30, 1.12it/s][2025-04-27 10:26:17] (step=0044000) Train Loss: 5.8351, Train Steps/Sec: 1.11 +You are using a model of type blip-2 to instantiate a model of type instructblip. This is not supported for all configurations of models and can yield errors. +[2025-04-27 10:26:17] text_config is None. Initializing the text config with default values (`OPTConfig`). +model_name_or_path:/tmp/haozhezhao/model/blip2-flan-t5-xl +image_place_holder: +tokenizer length after expend 32101 +tokenizer length before expend 32100 +100%|████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████| 5/5 [04:58<00:00, 59.76s/it] +[2025-04-27 10:32:27] Finish Eval in 44000 steps...█████████████████████████████████████████████████████████████████████████████████████████████| 5/5 [04:57<00:00, 59.45s/it] +processing left-padding... +processing left-padding... +processing left-padding... +processing left-padding... +processing left-padding... +[2025-04-27 10:32:47] Saved checkpoint to checkpoint/X2I_UltraEdiit_fourmask_LLava_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_filtered_objects_4mask_1280_context/003-GPT-XL/checkpoints/0044000.pt +[2025-04-27 10:32:50] Removed old checkpoint: checkpoint/X2I_UltraEdiit_fourmask_LLava_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_filtered_objects_4mask_1280_context/003-GPT-XL/checkpoints/0042000.pt + 40%|██████████████████████████████████████████████████ | 44024/110696 [13:29:47<17:17:01, 1.07it/s][2025-04-27 10:33:13] (step=0044025) Train Loss: 5.8251, Train Steps/Sec: 0.06 + 40%|██████████████████████████████████████████████████▏ | 44049/110696 [13:30:10<16:40:04, 1.11it/s][2025-04-27 10:33:35] (step=0044050) Train Loss: 5.7978, Train Steps/Sec: 1.12 + 40%|██████████████████████████████████████████████████▏ | 44074/110696 [13:30:32<16:34:55, 1.12it/s][2025-04-27 10:33:57] (step=0044075) Train Loss: 5.8632, Train Steps/Sec: 1.12 + 40%|██████████████████████████████████████████████████▏ | 44099/110696 [13:30:54<16:28:53, 1.12it/s][2025-04-27 10:34:20] (step=0044100) Train Loss: 5.9039, Train Steps/Sec: 1.12 + 40%|██████████████████████████████████████████████████▏ | 44124/110696 [13:31:17<16:24:17, 1.13it/s][2025-04-27 10:34:42] (step=0044125) Train Loss: 5.8423, Train Steps/Sec: 1.12 + 40%|██████████████████████████████████████████████████▎ | 44149/110696 [13:31:39<16:23:44, 1.13it/s][2025-04-27 10:35:04] (step=0044150) Train Loss: 5.8249, Train Steps/Sec: 1.12 + 40%|██████████████████████████████████████████████████▎ | 44174/110696 [13:32:01<16:20:22, 1.13it/s][2025-04-27 10:35:27] (step=0044175) Train Loss: 5.7924, Train Steps/Sec: 1.12 + 40%|██████████████████████████████████████████████████▎ | 44199/110696 [13:32:23<16:21:21, 1.13it/s][2025-04-27 10:35:49] (step=0044200) Train Loss: 5.8078, Train Steps/Sec: 1.12 + 40%|██████████████████████████████████████████████████▎ | 44224/110696 [13:32:46<16:49:02, 1.10it/s][2025-04-27 10:36:11] (step=0044225) Train Loss: 5.9150, Train Steps/Sec: 1.12 + 40%|██████████████████████████████████████████████████▎ | 44249/110696 [13:33:08<16:40:03, 1.11it/s][2025-04-27 10:36:34] (step=0044250) Train Loss: 5.7911, Train Steps/Sec: 1.12 + 40%|██████████████████████████████████████████████████▍ | 44274/110696 [13:33:31<16:34:58, 1.11it/s][2025-04-27 10:36:56] (step=0044275) Train Loss: 5.7089, Train Steps/Sec: 1.12 + 40%|██████████████████████████████████████████████████▍ | 44299/110696 [13:33:53<16:26:09, 1.12it/s][2025-04-27 10:37:18] (step=0044300) Train Loss: 5.8582, Train Steps/Sec: 1.12 + 40%|██████████████████████████████████████████████████▍ | 44324/110696 [13:34:15<16:19:46, 1.13it/s][2025-04-27 10:37:41] (step=0044325) Train Loss: 5.8230, Train Steps/Sec: 1.12 + 40%|██████████████████████████████████████████████████▍ | 44349/110696 [13:34:37<16:16:32, 1.13it/s][2025-04-27 10:38:03] (step=0044350) Train Loss: 5.8367, Train Steps/Sec: 1.12 + 40%|██████████████████████████████████████████████████▌ | 44374/110696 [13:35:00<16:18:51, 1.13it/s][2025-04-27 10:38:25] (step=0044375) Train Loss: 5.8179, Train Steps/Sec: 1.12 + 40%|██████████████████████████████████████████████████▌ | 44399/110696 [13:35:22<16:15:38, 1.13it/s][2025-04-27 10:38:48] (step=0044400) Train Loss: 5.8105, Train Steps/Sec: 1.12 + 40%|██████████████████████████████████████████████████▌ | 44424/110696 [13:35:44<16:45:31, 1.10it/s][2025-04-27 10:39:10] (step=0044425) Train Loss: 5.9106, Train Steps/Sec: 1.12 + 40%|██████████████████████████████████████████████████▌ | 44449/110696 [13:36:07<16:34:41, 1.11it/s][2025-04-27 10:39:32] (step=0044450) Train Loss: 5.8752, Train Steps/Sec: 1.12 + 40%|██████████████████████████████████████████████████▌ | 44474/110696 [13:36:29<16:26:28, 1.12it/s][2025-04-27 10:39:54] (step=0044475) Train Loss: 5.8485, Train Steps/Sec: 1.12 + 40%|██████████████████████████████████████████████████▋ | 44499/110696 [13:36:51<16:21:34, 1.12it/s][2025-04-27 10:40:17] (step=0044500) Train Loss: 5.7759, Train Steps/Sec: 1.12 + 40%|██████████████████████████████████████████████████▋ | 44524/110696 [13:37:14<16:22:16, 1.12it/s][2025-04-27 10:40:39] (step=0044525) Train Loss: 5.8913, Train Steps/Sec: 1.12 + 40%|██████████████████████████████████████████████████▋ | 44549/110696 [13:37:36<16:19:05, 1.13it/s][2025-04-27 10:41:01] (step=0044550) Train Loss: 5.9238, Train Steps/Sec: 1.12 + 40%|██████████████████████████████████████████████████▋ | 44574/110696 [13:37:58<16:15:52, 1.13it/s][2025-04-27 10:41:24] (step=0044575) Train Loss: 5.8381, Train Steps/Sec: 1.12 + 40%|██████████████████████████████████████████████████▊ | 44599/110696 [13:38:21<16:13:05, 1.13it/s][2025-04-27 10:41:46] (step=0044600) Train Loss: 5.7485, Train Steps/Sec: 1.12 + 40%|██████████████████████████████████████████████████▊ | 44624/110696 [13:38:43<16:43:00, 1.10it/s][2025-04-27 10:42:08] (step=0044625) Train Loss: 5.8872, Train Steps/Sec: 1.12 + 40%|██████████████████████████████████████████████████▊ | 44649/110696 [13:39:05<16:32:27, 1.11it/s][2025-04-27 10:42:31] (step=0044650) Train Loss: 5.8409, Train Steps/Sec: 1.12 + 40%|██████████████████████████████████████████████████▊ | 44674/110696 [13:39:28<16:22:22, 1.12it/s][2025-04-27 10:42:53] (step=0044675) Train Loss: 5.7854, Train Steps/Sec: 1.12 + 40%|██████████████████████████████████████████████████▉ | 44699/110696 [13:39:50<16:19:20, 1.12it/s][2025-04-27 10:43:15] (step=0044700) Train Loss: 5.8466, Train Steps/Sec: 1.12 + 40%|██████████████████████████████████████████████████▉ | 44724/110696 [13:40:12<16:11:50, 1.13it/s][2025-04-27 10:43:38] (step=0044725) Train Loss: 5.9249, Train Steps/Sec: 1.12 + 40%|██████████████████████████████████████████████████▉ | 44749/110696 [13:40:35<16:17:43, 1.12it/s][2025-04-27 10:44:00] (step=0044750) Train Loss: 5.7852, Train Steps/Sec: 1.12 + 40%|██████████████████████████████████████████████████▉ | 44774/110696 [13:40:57<16:13:34, 1.13it/s][2025-04-27 10:44:22] (step=0044775) Train Loss: 5.8545, Train Steps/Sec: 1.12 + 40%|██████████████████████████████████████████████████▉ | 44799/110696 [13:41:19<16:09:38, 1.13it/s][2025-04-27 10:44:45] (step=0044800) Train Loss: 5.7478, Train Steps/Sec: 1.11 + 40%|███████████████████████████████████████████████████ | 44824/110696 [13:41:42<16:43:13, 1.09it/s][2025-04-27 10:45:07] (step=0044825) Train Loss: 5.7863, Train Steps/Sec: 1.12 + 41%|███████████████████████████████████████████████████ | 44849/110696 [13:42:04<16:28:17, 1.11it/s][2025-04-27 10:45:29] (step=0044850) Train Loss: 5.8420, Train Steps/Sec: 1.12 + 41%|███████████████████████████████████████████████████ | 44874/110696 [13:42:26<16:22:44, 1.12it/s][2025-04-27 10:45:52] (step=0044875) Train Loss: 5.8765, Train Steps/Sec: 1.12 + 41%|███████████████████████████████████████████████████ | 44899/110696 [13:42:49<16:14:17, 1.13it/s][2025-04-27 10:46:14] (step=0044900) Train Loss: 5.8184, Train Steps/Sec: 1.12 + 41%|███████████████████████████████████████████████████▏ | 44924/110696 [13:43:11<16:12:56, 1.13it/s][2025-04-27 10:46:36] (step=0044925) Train Loss: 5.8313, Train Steps/Sec: 1.12 + 41%|███████████████████████████████████████████████████▏ | 44949/110696 [13:43:33<16:09:13, 1.13it/s][2025-04-27 10:46:59] (step=0044950) Train Loss: 5.8611, Train Steps/Sec: 1.12 + 41%|███████████████████████████████████████████████████▏ | 44974/110696 [13:43:56<16:08:28, 1.13it/s][2025-04-27 10:47:21] (step=0044975) Train Loss: 5.8886, Train Steps/Sec: 1.12 + 41%|███████████████████████████████████████████████████▏ | 44999/110696 [13:44:18<16:07:02, 1.13it/s][2025-04-27 10:47:43] (step=0045000) Train Loss: 5.8739, Train Steps/Sec: 1.12 + 41%|███████████████████████████████████████████████████▏ | 45024/110696 [13:44:40<16:36:16, 1.10it/s][2025-04-27 10:48:06] (step=0045025) Train Loss: 5.8961, Train Steps/Sec: 1.12 + 41%|███████████████████████████████████████████████████▎ | 45049/110696 [13:45:03<16:24:40, 1.11it/s][2025-04-27 10:48:28] (step=0045050) Train Loss: 5.8775, Train Steps/Sec: 1.12 + 41%|███████████████████████████████████████████████████▎ | 45074/110696 [13:45:25<16:16:50, 1.12it/s][2025-04-27 10:48:50] (step=0045075) Train Loss: 5.8421, Train Steps/Sec: 1.12 + 41%|███████████████████████████████████████████████████▎ | 45099/110696 [13:45:47<16:12:14, 1.12it/s][2025-04-27 10:49:12] (step=0045100) Train Loss: 5.9003, Train Steps/Sec: 1.12 + 41%|███████████████████████████████████████████████████▎ | 45124/110696 [13:46:09<16:10:11, 1.13it/s][2025-04-27 10:49:35] (step=0045125) Train Loss: 5.8494, Train Steps/Sec: 1.12 + 41%|███████████████████████████████████████████████████▍ | 45149/110696 [13:46:32<16:08:36, 1.13it/s][2025-04-27 10:49:57] (step=0045150) Train Loss: 5.7969, Train Steps/Sec: 1.12 + 41%|███████████████████████████████████████████████████▍ | 45174/110696 [13:46:54<16:03:20, 1.13it/s][2025-04-27 10:50:19] (step=0045175) Train Loss: 5.8462, Train Steps/Sec: 1.12 + 41%|███████████████████████████████████████████████████▍ | 45199/110696 [13:47:16<16:05:59, 1.13it/s][2025-04-27 10:50:42] (step=0045200) Train Loss: 5.8106, Train Steps/Sec: 1.12 + 41%|███████████████████████████████████████████████████▍ | 45224/110696 [13:47:39<16:38:23, 1.09it/s][2025-04-27 10:51:04] (step=0045225) Train Loss: 5.8579, Train Steps/Sec: 1.12 + 41%|███████████████████████████████████████████████████▌ | 45249/110696 [13:48:01<16:24:38, 1.11it/s][2025-04-27 10:51:26] (step=0045250) Train Loss: 5.8235, Train Steps/Sec: 1.12 + 41%|███████████████████████████████████████████████████▌ | 45274/110696 [13:48:23<16:19:36, 1.11it/s][2025-04-27 10:51:49] (step=0045275) Train Loss: 5.8773, Train Steps/Sec: 1.12 + 41%|███████████████████████████████████████████████████▌ | 45299/110696 [13:48:50<19:16:00, 1.06s/it][2025-04-27 10:52:16] (step=0045300) Train Loss: 5.8133, Train Steps/Sec: 0.92 + 41%|███████████████████████████████████████████████████▌ | 45324/110696 [13:49:13<16:09:00, 1.12it/s][2025-04-27 10:52:38] (step=0045325) Train Loss: 5.8128, Train Steps/Sec: 1.12 + 41%|███████████████████████████████████████████████████▌ | 45349/110696 [13:49:35<16:03:43, 1.13it/s][2025-04-27 10:53:00] (step=0045350) Train Loss: 5.8213, Train Steps/Sec: 1.12 + 41%|███████████████████████████████████████████████████▋ | 45374/110696 [13:49:57<15:59:03, 1.14it/s][2025-04-27 10:53:23] (step=0045375) Train Loss: 5.8343, Train Steps/Sec: 1.12 + 41%|███████████████████████████████████████████████████▋ | 45399/110696 [13:50:24<16:05:53, 1.13it/s][2025-04-27 10:53:50] (step=0045400) Train Loss: 5.8803, Train Steps/Sec: 0.92 + 41%|███████████████████████████████████████████████████▋ | 45424/110696 [13:50:52<17:39:06, 1.03it/s][2025-04-27 10:54:17] (step=0045425) Train Loss: 5.7267, Train Steps/Sec: 0.91 + 41%|███████████████████████████████████████████████████▋ | 45449/110696 [13:51:14<16:20:23, 1.11it/s][2025-04-27 10:54:40] (step=0045450) Train Loss: 5.9024, Train Steps/Sec: 1.12 + 41%|███████████████████████████████████████████████████▊ | 45474/110696 [13:51:37<16:14:04, 1.12it/s][2025-04-27 10:55:02] (step=0045475) Train Loss: 5.8717, Train Steps/Sec: 1.12 + 41%|███████████████████████████████████████████████████▊ | 45499/110696 [13:51:59<16:08:41, 1.12it/s][2025-04-27 10:55:24] (step=0045500) Train Loss: 5.8034, Train Steps/Sec: 1.12 + 41%|███████████████████████████████████████████████████▊ | 45524/110696 [13:52:21<16:05:41, 1.12it/s][2025-04-27 10:55:47] (step=0045525) Train Loss: 5.8120, Train Steps/Sec: 1.12 + 41%|███████████████████████████████████████████████████▊ | 45549/110696 [13:52:44<16:03:07, 1.13it/s][2025-04-27 10:56:09] (step=0045550) Train Loss: 5.8851, Train Steps/Sec: 1.12 + 41%|███████████████████████████████████████████████████▊ | 45574/110696 [13:53:06<15:59:41, 1.13it/s][2025-04-27 10:56:31] (step=0045575) Train Loss: 5.8565, Train Steps/Sec: 1.12 + 41%|███████████████████████████████████████████████████▉ | 45599/110696 [13:53:28<15:59:09, 1.13it/s][2025-04-27 10:56:54] (step=0045600) Train Loss: 5.7946, Train Steps/Sec: 1.11 + 41%|███████████████████████████████████████████████████▉ | 45624/110696 [13:53:51<16:30:28, 1.09it/s][2025-04-27 10:57:16] (step=0045625) Train Loss: 5.8258, Train Steps/Sec: 1.12 + 41%|███████████████████████████████████████████████████▉ | 45649/110696 [13:54:13<16:15:44, 1.11it/s][2025-04-27 10:57:38] (step=0045650) Train Loss: 5.8553, Train Steps/Sec: 1.12 + 41%|███████████████████████████████████████████████████▉ | 45674/110696 [13:54:44<17:18:48, 1.04it/s][2025-04-27 10:58:10] (step=0045675) Train Loss: 5.8630, Train Steps/Sec: 0.80 + 41%|████████████████████████████████████████████████████ | 45699/110696 [13:55:16<17:48:18, 1.01it/s][2025-04-27 10:58:42] (step=0045700) Train Loss: 5.8784, Train Steps/Sec: 0.77 + 41%|████████████████████████████████████████████████████ | 45724/110696 [13:55:39<16:00:25, 1.13it/s][2025-04-27 10:59:04] (step=0045725) Train Loss: 5.8680, Train Steps/Sec: 1.12 + 41%|████████████████████████████████████████████████████ | 45749/110696 [13:56:01<16:00:04, 1.13it/s][2025-04-27 10:59:26] (step=0045750) Train Loss: 5.8417, Train Steps/Sec: 1.12 + 41%|████████████████████████████████████████████████████ | 45774/110696 [13:56:23<16:01:28, 1.13it/s][2025-04-27 10:59:49] (step=0045775) Train Loss: 5.8875, Train Steps/Sec: 1.12 + 41%|████████████████████████████████████████████████████▏ | 45799/110696 [13:56:46<15:59:22, 1.13it/s][2025-04-27 11:00:11] (step=0045800) Train Loss: 5.8468, Train Steps/Sec: 1.11 + 41%|████████████████████████████████████████████████████▏ | 45824/110696 [13:57:08<16:22:54, 1.10it/s][2025-04-27 11:00:33] (step=0045825) Train Loss: 5.7998, Train Steps/Sec: 1.12 + 41%|████████████████████████████████████████████████████▏ | 45849/110696 [13:57:30<16:15:53, 1.11it/s][2025-04-27 11:00:56] (step=0045850) Train Loss: 5.8189, Train Steps/Sec: 1.12 + 41%|████████████████████████████████████████████████████▏ | 45874/110696 [13:57:53<16:10:50, 1.11it/s][2025-04-27 11:01:18] (step=0045875) Train Loss: 5.8535, Train Steps/Sec: 1.12 + 41%|████████████████████████████████████████████████████▏ | 45899/110696 [13:58:15<16:05:08, 1.12it/s][2025-04-27 11:01:40] (step=0045900) Train Loss: 5.8831, Train Steps/Sec: 1.12 + 41%|████████████████████████████████████████████████████▎ | 45924/110696 [13:58:37<15:55:59, 1.13it/s][2025-04-27 11:02:03] (step=0045925) Train Loss: 5.8442, Train Steps/Sec: 1.12 + 42%|████████████████████████████████████████████████████▎ | 45949/110696 [13:59:00<15:59:02, 1.13it/s][2025-04-27 11:02:25] (step=0045950) Train Loss: 5.8563, Train Steps/Sec: 1.12 + 42%|████████████████████████████████████████████████████▎ | 45974/110696 [13:59:22<15:57:10, 1.13it/s][2025-04-27 11:02:47] (step=0045975) Train Loss: 5.8177, Train Steps/Sec: 1.12 + 42%|████████████████████████████████████████████████████▎ | 45999/110696 [13:59:44<15:53:13, 1.13it/s][2025-04-27 11:03:10] (step=0046000) Train Loss: 5.8056, Train Steps/Sec: 1.12 +You are using a model of type blip-2 to instantiate a model of type instructblip. This is not supported for all configurations of models and can yield errors. +[2025-04-27 11:03:10] text_config is None. Initializing the text config with default values (`OPTConfig`). +model_name_or_path:/tmp/haozhezhao/model/blip2-flan-t5-xl +image_place_holder: +tokenizer length after expend 32101 +tokenizer length before expend 32100 +100%|████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████| 5/5 [04:56<00:00, 59.30s/it] +[2025-04-27 11:09:18] Finish Eval in 46000 steps...█████████████████████████████████████████████████████████████████████████████████████████████| 5/5 [04:55<00:00, 59.05s/it] +processing left-padding... +processing left-padding... +processing left-padding... +processing left-padding... +processing left-padding... +[2025-04-27 11:09:37] Saved checkpoint to checkpoint/X2I_UltraEdiit_fourmask_LLava_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_filtered_objects_4mask_1280_context/003-GPT-XL/checkpoints/0046000.pt +[2025-04-27 11:09:39] Removed old checkpoint: checkpoint/X2I_UltraEdiit_fourmask_LLava_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_filtered_objects_4mask_1280_context/003-GPT-XL/checkpoints/0044000.pt + 42%|████████████████████████████████████████████████████▍ | 46024/110696 [14:06:36<16:42:35, 1.08it/s][2025-04-27 11:10:01] (step=0046025) Train Loss: 5.8416, Train Steps/Sec: 0.06 + 42%|████████████████████████████████████████████████████▍ | 46049/110696 [14:06:58<16:11:12, 1.11it/s][2025-04-27 11:10:24] (step=0046050) Train Loss: 5.8559, Train Steps/Sec: 1.12 + 42%|████████████████████████████████████████████████████▍ | 46074/110696 [14:07:21<16:01:42, 1.12it/s][2025-04-27 11:10:46] (step=0046075) Train Loss: 5.8362, Train Steps/Sec: 1.12 + 42%|████████████████████████████████████████████████████▍ | 46099/110696 [14:07:43<15:58:56, 1.12it/s][2025-04-27 11:11:08] (step=0046100) Train Loss: 5.8156, Train Steps/Sec: 1.12 + 42%|████████████████████████████████████████████████████▌ | 46124/110696 [14:08:05<15:56:33, 1.13it/s][2025-04-27 11:11:31] (step=0046125) Train Loss: 5.8111, Train Steps/Sec: 1.12 + 42%|████████████████████████████████████████████████████▌ | 46149/110696 [14:08:28<15:50:34, 1.13it/s][2025-04-27 11:11:53] (step=0046150) Train Loss: 5.8051, Train Steps/Sec: 1.12 + 42%|████████████████████████████████████████████████████▌ | 46174/110696 [14:08:50<15:52:49, 1.13it/s][2025-04-27 11:12:15] (step=0046175) Train Loss: 5.7943, Train Steps/Sec: 1.12 + 42%|████████████████████████████████████████████████████▌ | 46199/110696 [14:09:12<15:52:08, 1.13it/s][2025-04-27 11:12:38] (step=0046200) Train Loss: 5.8174, Train Steps/Sec: 1.11 + 42%|████████████████████████████████████████████████████▌ | 46224/110696 [14:09:35<16:22:07, 1.09it/s][2025-04-27 11:13:00] (step=0046225) Train Loss: 5.8287, Train Steps/Sec: 1.12 + 42%|████████████████████████████████████████████████████▋ | 46249/110696 [14:09:57<16:10:00, 1.11it/s][2025-04-27 11:13:22] (step=0046250) Train Loss: 5.8522, Train Steps/Sec: 1.12 + 42%|████████████████████████████████████████████████████▋ | 46274/110696 [14:10:19<15:59:22, 1.12it/s][2025-04-27 11:13:45] (step=0046275) Train Loss: 5.8166, Train Steps/Sec: 1.12 + 42%|████████████████████████████████████████████████████▋ | 46299/110696 [14:10:42<15:56:06, 1.12it/s][2025-04-27 11:14:07] (step=0046300) Train Loss: 5.8138, Train Steps/Sec: 1.12 + 42%|████████████████████████████████████████████████████▋ | 46324/110696 [14:11:04<15:51:48, 1.13it/s][2025-04-27 11:14:29] (step=0046325) Train Loss: 5.8671, Train Steps/Sec: 1.12 + 42%|████████████████████████████████████████████████████▊ | 46349/110696 [14:11:26<15:51:45, 1.13it/s][2025-04-27 11:14:52] (step=0046350) Train Loss: 5.8603, Train Steps/Sec: 1.12 + 42%|████████████████████████████████████████████████████▊ | 46374/110696 [14:11:49<15:45:25, 1.13it/s][2025-04-27 11:15:14] (step=0046375) Train Loss: 5.8490, Train Steps/Sec: 1.12 + 42%|████████████████████████████████████████████████████▊ | 46399/110696 [14:12:11<15:46:08, 1.13it/s][2025-04-27 11:15:36] (step=0046400) Train Loss: 5.8233, Train Steps/Sec: 1.11 + 42%|████████████████████████████████████████████████████▊ | 46424/110696 [14:12:33<16:16:55, 1.10it/s][2025-04-27 11:15:59] (step=0046425) Train Loss: 5.7743, Train Steps/Sec: 1.12 + 42%|████████████████████████████████████████████████████▊ | 46449/110696 [14:12:56<16:08:29, 1.11it/s][2025-04-27 11:16:21] (step=0046450) Train Loss: 5.8252, Train Steps/Sec: 1.12 + 42%|████████████████████████████████████████████████████▉ | 46474/110696 [14:13:18<15:58:21, 1.12it/s][2025-04-27 11:16:43] (step=0046475) Train Loss: 5.8781, Train Steps/Sec: 1.12 + 42%|████████████████████████████████████████████████████▉ | 46499/110696 [14:13:40<16:01:38, 1.11it/s][2025-04-27 11:17:06] (step=0046500) Train Loss: 5.8656, Train Steps/Sec: 1.12 + 42%|████████████████████████████████████████████████████▉ | 46524/110696 [14:14:03<15:49:21, 1.13it/s][2025-04-27 11:17:28] (step=0046525) Train Loss: 5.7939, Train Steps/Sec: 1.11 + 42%|████████████████████████████████████████████████████▉ | 46549/110696 [14:14:25<15:47:56, 1.13it/s][2025-04-27 11:17:51] (step=0046550) Train Loss: 5.8665, Train Steps/Sec: 1.12 + 42%|█████████████████████████████████████████████████████ | 46574/110696 [14:14:48<15:45:34, 1.13it/s][2025-04-27 11:18:13] (step=0046575) Train Loss: 5.8348, Train Steps/Sec: 1.12 + 42%|█████████████████████████████████████████████████████ | 46599/110696 [14:15:10<15:48:39, 1.13it/s][2025-04-27 11:18:35] (step=0046600) Train Loss: 5.8790, Train Steps/Sec: 1.12 + 42%|█████████████████████████████████████████████████████ | 46624/110696 [14:15:32<16:14:20, 1.10it/s][2025-04-27 11:18:58] (step=0046625) Train Loss: 5.8414, Train Steps/Sec: 1.12 + 42%|█████████████████████████████████████████████████████ | 46649/110696 [14:15:55<16:01:24, 1.11it/s][2025-04-27 11:19:20] (step=0046650) Train Loss: 5.8508, Train Steps/Sec: 1.12 + 42%|█████████████████████████████████████████████████████▏ | 46674/110696 [14:16:17<15:55:14, 1.12it/s][2025-04-27 11:19:42] (step=0046675) Train Loss: 5.8673, Train Steps/Sec: 1.12 + 42%|█████████████████████████████████████████████████████▏ | 46699/110696 [14:16:39<15:54:57, 1.12it/s][2025-04-27 11:20:05] (step=0046700) Train Loss: 5.8546, Train Steps/Sec: 1.12 + 42%|█████████████████████████████████████████████████████▏ | 46724/110696 [14:17:02<15:47:52, 1.12it/s][2025-04-27 11:20:27] (step=0046725) Train Loss: 5.9066, Train Steps/Sec: 1.12 + 42%|█████████████████████████████████████████████████████▏ | 46749/110696 [14:17:24<15:43:04, 1.13it/s][2025-04-27 11:20:49] (step=0046750) Train Loss: 5.8451, Train Steps/Sec: 1.12 + 42%|█████████████████████████████████████████████████████▏ | 46774/110696 [14:17:46<15:41:27, 1.13it/s][2025-04-27 11:21:12] (step=0046775) Train Loss: 5.8236, Train Steps/Sec: 1.12 + 42%|█████████████████████████████████████████████████████▎ | 46799/110696 [14:18:08<15:42:33, 1.13it/s][2025-04-27 11:21:34] (step=0046800) Train Loss: 5.7969, Train Steps/Sec: 1.12 + 42%|█████████████████████████████████████████████████████▎ | 46824/110696 [14:18:31<16:09:24, 1.10it/s][2025-04-27 11:21:56] (step=0046825) Train Loss: 5.8352, Train Steps/Sec: 1.12 + 42%|█████████████████████████████████████████████████████▎ | 46849/110696 [14:18:53<15:58:34, 1.11it/s][2025-04-27 11:22:19] (step=0046850) Train Loss: 5.8318, Train Steps/Sec: 1.12 + 42%|█████████████████████████████████████████████████████▎ | 46874/110696 [14:19:16<15:53:09, 1.12it/s][2025-04-27 11:22:41] (step=0046875) Train Loss: 5.8679, Train Steps/Sec: 1.12 + 42%|█████████████████████████████████████████████████████▍ | 46899/110696 [14:19:38<15:46:30, 1.12it/s][2025-04-27 11:23:03] (step=0046900) Train Loss: 5.8561, Train Steps/Sec: 1.12 + 42%|█████████████████████████████████████████████████████▍ | 46924/110696 [14:20:00<15:45:26, 1.12it/s][2025-04-27 11:23:26] (step=0046925) Train Loss: 5.8392, Train Steps/Sec: 1.12 + 42%|█████████████████████████████████████████████████████▍ | 46949/110696 [14:20:23<15:45:20, 1.12it/s][2025-04-27 11:23:48] (step=0046950) Train Loss: 5.8197, Train Steps/Sec: 1.12 + 42%|█████████████████████████████████████████████████████▍ | 46974/110696 [14:20:45<15:39:43, 1.13it/s][2025-04-27 11:24:10] (step=0046975) Train Loss: 5.8643, Train Steps/Sec: 1.12 + 42%|█████████████████████████████████████████████████████▍ | 46999/110696 [14:21:07<15:38:47, 1.13it/s][2025-04-27 11:24:33] (step=0047000) Train Loss: 5.8273, Train Steps/Sec: 1.12 + 42%|█████████████████████████████████████████████████████▌ | 47024/110696 [14:21:30<16:08:02, 1.10it/s][2025-04-27 11:24:55] (step=0047025) Train Loss: 5.8548, Train Steps/Sec: 1.12 + 43%|█████████████████████████████████████████████████████▌ | 47049/110696 [14:21:52<15:56:18, 1.11it/s][2025-04-27 11:25:17] (step=0047050) Train Loss: 5.8400, Train Steps/Sec: 1.12 + 43%|█████████████████████████████████████████████████████▌ | 47074/110696 [14:22:14<15:46:39, 1.12it/s][2025-04-27 11:25:40] (step=0047075) Train Loss: 5.8553, Train Steps/Sec: 1.12 + 43%|█████████████████████████████████████████████████████▌ | 47099/110696 [14:22:37<15:46:46, 1.12it/s][2025-04-27 11:26:02] (step=0047100) Train Loss: 5.9012, Train Steps/Sec: 1.12 + 43%|█████████████████████████████████████████████████████▋ | 47124/110696 [14:22:59<15:42:07, 1.12it/s][2025-04-27 11:26:24] (step=0047125) Train Loss: 5.7677, Train Steps/Sec: 1.12 + 43%|█████████████████████████████████████████████████████▋ | 47149/110696 [14:23:21<15:38:06, 1.13it/s][2025-04-27 11:26:47] (step=0047150) Train Loss: 5.7956, Train Steps/Sec: 1.12 + 43%|█████████████████████████████████████████████████████▋ | 47174/110696 [14:23:44<15:38:57, 1.13it/s][2025-04-27 11:27:09] (step=0047175) Train Loss: 5.8051, Train Steps/Sec: 1.12 + 43%|█████████████████████████████████████████████████████▋ | 47199/110696 [14:24:06<15:36:16, 1.13it/s][2025-04-27 11:27:32] (step=0047200) Train Loss: 5.8697, Train Steps/Sec: 1.11 + 43%|█████████████████████████████████████████████████████▊ | 47224/110696 [14:24:29<16:03:30, 1.10it/s][2025-04-27 11:27:54] (step=0047225) Train Loss: 5.7726, Train Steps/Sec: 1.12 + 43%|█████████████████████████████████████████████████████▊ | 47249/110696 [14:24:51<15:54:01, 1.11it/s][2025-04-27 11:28:16] (step=0047250) Train Loss: 5.8691, Train Steps/Sec: 1.12 + 43%|█████████████████████████████████████████████████████▊ | 47274/110696 [14:25:13<15:47:48, 1.12it/s][2025-04-27 11:28:39] (step=0047275) Train Loss: 5.7965, Train Steps/Sec: 1.12 + 43%|█████████████████████████████████████████████████████▊ | 47299/110696 [14:25:36<15:41:50, 1.12it/s][2025-04-27 11:29:01] (step=0047300) Train Loss: 5.8873, Train Steps/Sec: 1.12 + 43%|█████████████████████████████████████████████████████▊ | 47324/110696 [14:25:58<15:41:29, 1.12it/s][2025-04-27 11:29:23] (step=0047325) Train Loss: 5.8407, Train Steps/Sec: 1.12 + 43%|█████████████████████████████████████████████████████▉ | 47349/110696 [14:26:20<15:38:39, 1.12it/s][2025-04-27 11:29:46] (step=0047350) Train Loss: 5.8695, Train Steps/Sec: 1.12 + 43%|█████████████████████████████████████████████████████▉ | 47374/110696 [14:26:43<15:34:27, 1.13it/s][2025-04-27 11:30:08] (step=0047375) Train Loss: 5.7754, Train Steps/Sec: 1.12 + 43%|█████████████████████████████████████████████████████▉ | 47399/110696 [14:27:05<15:34:21, 1.13it/s][2025-04-27 11:30:30] (step=0047400) Train Loss: 5.8162, Train Steps/Sec: 1.11 + 43%|█████████████████████████████████████████████████████▉ | 47424/110696 [14:27:27<16:02:12, 1.10it/s][2025-04-27 11:30:53] (step=0047425) Train Loss: 5.8494, Train Steps/Sec: 1.12 + 43%|██████████████████████████████████████████████████████ | 47449/110696 [14:27:50<15:50:36, 1.11it/s][2025-04-27 11:31:15] (step=0047450) Train Loss: 5.7741, Train Steps/Sec: 1.12 + 43%|██████████████████████████████████████████████████████ | 47474/110696 [14:28:12<15:41:52, 1.12it/s][2025-04-27 11:31:38] (step=0047475) Train Loss: 5.8112, Train Steps/Sec: 1.12 + 43%|██████████████████████████████████████████████████████ | 47499/110696 [14:28:35<15:40:10, 1.12it/s][2025-04-27 11:32:00] (step=0047500) Train Loss: 5.7527, Train Steps/Sec: 1.12 + 43%|██████████████████████████████████████████████████████ | 47524/110696 [14:28:57<15:36:54, 1.12it/s][2025-04-27 11:32:22] (step=0047525) Train Loss: 5.8203, Train Steps/Sec: 1.12 + 43%|██████████████████████████████████████████████████████ | 47549/110696 [14:29:19<15:36:37, 1.12it/s][2025-04-27 11:32:45] (step=0047550) Train Loss: 5.7804, Train Steps/Sec: 1.12 + 43%|██████████████████████████████████████████████████████▏ | 47574/110696 [14:29:42<15:30:39, 1.13it/s][2025-04-27 11:33:07] (step=0047575) Train Loss: 5.8647, Train Steps/Sec: 1.12 + 43%|██████████████████████████████████████████████████████▏ | 47599/110696 [14:30:04<15:30:08, 1.13it/s][2025-04-27 11:33:29] (step=0047600) Train Loss: 5.7774, Train Steps/Sec: 1.11 + 43%|██████████████████████████████████████████████████████▏ | 47624/110696 [14:30:26<16:04:25, 1.09it/s][2025-04-27 11:33:52] (step=0047625) Train Loss: 5.8650, Train Steps/Sec: 1.12 + 43%|██████████████████████████████████████████████████████▏ | 47649/110696 [14:30:49<15:48:56, 1.11it/s][2025-04-27 11:34:14] (step=0047650) Train Loss: 5.9065, Train Steps/Sec: 1.12 + 43%|██████████████████████████████████████████████████████▎ | 47674/110696 [14:31:11<15:42:11, 1.11it/s][2025-04-27 11:34:36] (step=0047675) Train Loss: 5.7967, Train Steps/Sec: 1.12 + 43%|██████████████████████████████████████████████████████▎ | 47699/110696 [14:31:33<15:35:27, 1.12it/s][2025-04-27 11:34:59] (step=0047700) Train Loss: 5.8055, Train Steps/Sec: 1.12 + 43%|██████████████████████████████████████████████████████▎ | 47724/110696 [14:31:56<15:31:06, 1.13it/s][2025-04-27 11:35:21] (step=0047725) Train Loss: 5.7982, Train Steps/Sec: 1.12 + 43%|██████████████████████████████████████████████████████▎ | 47749/110696 [14:32:18<15:32:11, 1.13it/s][2025-04-27 11:35:43] (step=0047750) Train Loss: 5.8353, Train Steps/Sec: 1.12 + 43%|██████████████████████████████████████████████████████▍ | 47774/110696 [14:32:40<15:28:35, 1.13it/s][2025-04-27 11:36:06] (step=0047775) Train Loss: 5.7870, Train Steps/Sec: 1.12 + 43%|██████████████████████████████████████████████████████▍ | 47799/110696 [14:33:03<15:26:41, 1.13it/s][2025-04-27 11:36:28] (step=0047800) Train Loss: 5.8349, Train Steps/Sec: 1.12 + 43%|██████████████████████████████████████████████████████▍ | 47824/110696 [14:33:25<15:57:36, 1.09it/s][2025-04-27 11:36:50] (step=0047825) Train Loss: 5.8156, Train Steps/Sec: 1.12 + 43%|██████████████████████████████████████████████████████▍ | 47849/110696 [14:33:47<15:43:19, 1.11it/s][2025-04-27 11:37:13] (step=0047850) Train Loss: 5.8131, Train Steps/Sec: 1.12 + 43%|██████████████████████████████████████████████████████▍ | 47874/110696 [14:34:10<15:37:39, 1.12it/s][2025-04-27 11:37:35] (step=0047875) Train Loss: 5.8180, Train Steps/Sec: 1.12 + 43%|██████████████████████████████████████████████████████▌ | 47899/110696 [14:34:32<15:34:16, 1.12it/s][2025-04-27 11:37:57] (step=0047900) Train Loss: 5.7896, Train Steps/Sec: 1.12 + 43%|██████████████████████████████████████████████████████▌ | 47924/110696 [14:34:54<15:30:05, 1.12it/s][2025-04-27 11:38:20] (step=0047925) Train Loss: 5.7484, Train Steps/Sec: 1.12 + 43%|██████████████████████████████████████████████████████▌ | 47949/110696 [14:35:17<15:29:03, 1.13it/s][2025-04-27 11:38:42] (step=0047950) Train Loss: 5.8704, Train Steps/Sec: 1.12 + 43%|██████████████████████████████████████████████████████▌ | 47974/110696 [14:35:39<15:26:23, 1.13it/s][2025-04-27 11:39:04] (step=0047975) Train Loss: 5.8445, Train Steps/Sec: 1.12 + 43%|██████████████████████████████████████████████████████▋ | 47999/110696 [14:36:01<15:23:12, 1.13it/s][2025-04-27 11:39:27] (step=0048000) Train Loss: 5.7777, Train Steps/Sec: 1.12 +You are using a model of type blip-2 to instantiate a model of type instructblip. This is not supported for all configurations of models and can yield errors. +[2025-04-27 11:39:27] text_config is None. Initializing the text config with default values (`OPTConfig`). +model_name_or_path:/tmp/haozhezhao/model/blip2-flan-t5-xl +image_place_holder: +tokenizer length after expend 32101 +tokenizer length before expend 32100 +100%|████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████| 5/5 [04:56<00:00, 59.26s/it] +[2025-04-27 11:45:36] Finish Eval in 48000 steps...█████████████████████████████████████████████████████████████████████████████████████████████| 5/5 [04:55<00:00, 59.05s/it] +processing left-padding... +processing left-padding... +processing left-padding... +processing left-padding... +processing left-padding... +[2025-04-27 11:45:55] Saved checkpoint to checkpoint/X2I_UltraEdiit_fourmask_LLava_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_filtered_objects_4mask_1280_context/003-GPT-XL/checkpoints/0048000.pt +[2025-04-27 11:45:57] Removed old checkpoint: checkpoint/X2I_UltraEdiit_fourmask_LLava_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_filtered_objects_4mask_1280_context/003-GPT-XL/checkpoints/0046000.pt + 43%|██████████████████████████████████████████████████████▋ | 48024/110696 [14:42:55<16:16:00, 1.07it/s][2025-04-27 11:46:20] (step=0048025) Train Loss: 5.8620, Train Steps/Sec: 0.06 + 43%|██████████████████████████████████████████████████████▋ | 48049/110696 [14:43:17<15:35:23, 1.12it/s][2025-04-27 11:46:42] (step=0048050) Train Loss: 5.8776, Train Steps/Sec: 1.12 + 43%|██████████████████████████████████████████████████████▋ | 48074/110696 [14:43:39<15:34:48, 1.12it/s][2025-04-27 11:47:04] (step=0048075) Train Loss: 5.8011, Train Steps/Sec: 1.12 + 43%|██████████████████████████████████████████████████████▋ | 48099/110696 [14:44:01<15:25:30, 1.13it/s][2025-04-27 11:47:27] (step=0048100) Train Loss: 5.8558, Train Steps/Sec: 1.12 + 43%|██████████████████████████████████████████████████████▊ | 48124/110696 [14:44:24<15:25:07, 1.13it/s][2025-04-27 11:47:49] (step=0048125) Train Loss: 5.9142, Train Steps/Sec: 1.12 + 43%|██████████████████████████████████████████████████████▊ | 48149/110696 [14:44:46<15:20:54, 1.13it/s][2025-04-27 11:48:11] (step=0048150) Train Loss: 5.8290, Train Steps/Sec: 1.12 + 44%|██████████████████████████████████████████████████████▊ | 48174/110696 [14:45:08<15:22:03, 1.13it/s][2025-04-27 11:48:34] (step=0048175) Train Loss: 5.8427, Train Steps/Sec: 1.12 + 44%|██████████████████████████████████████████████████████▊ | 48199/110696 [14:45:31<15:19:21, 1.13it/s][2025-04-27 11:48:56] (step=0048200) Train Loss: 5.8734, Train Steps/Sec: 1.12 + 44%|██████████████████████████████████████████████████████▉ | 48224/110696 [14:45:53<15:50:23, 1.10it/s][2025-04-27 11:49:18] (step=0048225) Train Loss: 5.8898, Train Steps/Sec: 1.12 + 44%|██████████████████████████████████████████████████████▉ | 48249/110696 [14:46:15<15:38:08, 1.11it/s][2025-04-27 11:49:41] (step=0048250) Train Loss: 5.7715, Train Steps/Sec: 1.12 + 44%|██████████████████████████████████████████████████████▉ | 48274/110696 [14:46:38<15:34:07, 1.11it/s][2025-04-27 11:50:03] (step=0048275) Train Loss: 5.8195, Train Steps/Sec: 1.12 + 44%|██████████████████████████████████████████████████████▉ | 48299/110696 [14:47:00<15:24:29, 1.12it/s][2025-04-27 11:50:25] (step=0048300) Train Loss: 5.8449, Train Steps/Sec: 1.12 + 44%|███████████████████████████████████████████████████████ | 48324/110696 [14:47:22<15:21:30, 1.13it/s][2025-04-27 11:50:47] (step=0048325) Train Loss: 5.8006, Train Steps/Sec: 1.12 + 44%|███████████████████████████████████████████████████████ | 48349/110696 [14:47:44<15:23:04, 1.13it/s][2025-04-27 11:51:10] (step=0048350) Train Loss: 5.8292, Train Steps/Sec: 1.12 + 44%|███████████████████████████████████████████████████████ | 48374/110696 [14:48:07<15:19:11, 1.13it/s][2025-04-27 11:51:32] (step=0048375) Train Loss: 5.8179, Train Steps/Sec: 1.12 + 44%|███████████████████████████████████████████████████████ | 48399/110696 [14:48:29<15:18:10, 1.13it/s][2025-04-27 11:51:54] (step=0048400) Train Loss: 5.7734, Train Steps/Sec: 1.12 + 44%|███████████████████████████████████████████████████████ | 48424/110696 [14:48:52<15:57:46, 1.08it/s][2025-04-27 11:52:17] (step=0048425) Train Loss: 5.8546, Train Steps/Sec: 1.10 + 44%|███████████████████████████████████████████████████████▏ | 48449/110696 [14:49:14<15:31:51, 1.11it/s][2025-04-27 11:52:39] (step=0048450) Train Loss: 5.8485, Train Steps/Sec: 1.12 + 44%|███████████████████████████████████████████████████████▏ | 48474/110696 [14:49:36<15:26:49, 1.12it/s][2025-04-27 11:53:02] (step=0048475) Train Loss: 5.8306, Train Steps/Sec: 1.12 + 44%|███████████████████████████████████████████████████████▏ | 48499/110696 [14:49:59<15:22:56, 1.12it/s][2025-04-27 11:53:24] (step=0048500) Train Loss: 5.7978, Train Steps/Sec: 1.12 + 44%|███████████████████████████████████████████████████████▏ | 48524/110696 [14:50:21<15:20:59, 1.13it/s][2025-04-27 11:53:46] (step=0048525) Train Loss: 5.8071, Train Steps/Sec: 1.12 + 44%|███████████████████████████████████████████████████████▎ | 48549/110696 [14:50:43<15:13:26, 1.13it/s][2025-04-27 11:54:09] (step=0048550) Train Loss: 5.8294, Train Steps/Sec: 1.12 + 44%|███████████████████████████████████████████████████████▎ | 48574/110696 [14:51:06<15:17:54, 1.13it/s][2025-04-27 11:54:31] (step=0048575) Train Loss: 5.8188, Train Steps/Sec: 1.12 + 44%|███████████████████████████████████████████████████████▎ | 48599/110696 [14:51:28<15:13:21, 1.13it/s][2025-04-27 11:54:53] (step=0048600) Train Loss: 5.8220, Train Steps/Sec: 1.12 + 44%|███████████████████████████████████████████████████████▎ | 48624/110696 [14:51:50<15:41:13, 1.10it/s][2025-04-27 11:55:16] (step=0048625) Train Loss: 5.8622, Train Steps/Sec: 1.12 + 44%|███████████████████████████████████████████████████████▎ | 48649/110696 [14:52:13<15:33:16, 1.11it/s][2025-04-27 11:55:38] (step=0048650) Train Loss: 5.8044, Train Steps/Sec: 1.12 + 44%|███████████████████████████████████████████████████████▍ | 48674/110696 [14:52:35<15:21:15, 1.12it/s][2025-04-27 11:56:00] (step=0048675) Train Loss: 5.8428, Train Steps/Sec: 1.12 + 44%|███████████████████████████████████████████████████████▍ | 48699/110696 [14:52:57<15:20:10, 1.12it/s][2025-04-27 11:56:22] (step=0048700) Train Loss: 5.8284, Train Steps/Sec: 1.12 + 44%|███████████████████████████████████████████████████████▍ | 48724/110696 [14:53:19<15:17:19, 1.13it/s][2025-04-27 11:56:45] (step=0048725) Train Loss: 5.8628, Train Steps/Sec: 1.12 + 44%|███████████████████████████████████████████████████████▍ | 48749/110696 [14:53:42<15:14:48, 1.13it/s][2025-04-27 11:57:07] (step=0048750) Train Loss: 5.7929, Train Steps/Sec: 1.12 + 44%|███████████████████████████████████████████████████████▌ | 48774/110696 [14:54:04<15:10:38, 1.13it/s][2025-04-27 11:57:29] (step=0048775) Train Loss: 5.8308, Train Steps/Sec: 1.12 + 44%|███████████████████████████████████████████████████████▌ | 48799/110696 [14:54:26<15:10:23, 1.13it/s][2025-04-27 11:57:52] (step=0048800) Train Loss: 5.7855, Train Steps/Sec: 1.12 + 44%|███████████████████████████████████████████████████████▌ | 48824/110696 [14:54:49<15:38:16, 1.10it/s][2025-04-27 11:58:14] (step=0048825) Train Loss: 5.7973, Train Steps/Sec: 1.12 + 44%|███████████████████████████████████████████████████████▌ | 48849/110696 [14:55:11<15:26:38, 1.11it/s][2025-04-27 11:58:36] (step=0048850) Train Loss: 5.7700, Train Steps/Sec: 1.12 + 44%|███████████████████████████████████████████████████████▋ | 48874/110696 [14:55:33<15:21:04, 1.12it/s][2025-04-27 11:58:59] (step=0048875) Train Loss: 5.8744, Train Steps/Sec: 1.12 + 44%|███████████████████████████████████████████████████████▋ | 48899/110696 [14:55:55<15:18:09, 1.12it/s][2025-04-27 11:59:21] (step=0048900) Train Loss: 5.8351, Train Steps/Sec: 1.12 + 44%|███████████████████████████████████████████████████████▋ | 48924/110696 [14:56:18<15:17:28, 1.12it/s][2025-04-27 11:59:43] (step=0048925) Train Loss: 5.7933, Train Steps/Sec: 1.12 + 44%|███████████████████████████████████████████████████████▋ | 48949/110696 [14:56:40<15:11:04, 1.13it/s][2025-04-27 12:00:05] (step=0048950) Train Loss: 5.7890, Train Steps/Sec: 1.12 + 44%|███████████████████████████████████████████████████████▋ | 48974/110696 [14:57:02<15:10:34, 1.13it/s][2025-04-27 12:00:28] (step=0048975) Train Loss: 5.8110, Train Steps/Sec: 1.12 + 44%|███████████████████████████████████████████████████████▊ | 48999/110696 [14:57:25<15:08:19, 1.13it/s][2025-04-27 12:00:50] (step=0049000) Train Loss: 5.7610, Train Steps/Sec: 1.12 + 44%|███████████████████████████████████████████████████████▊ | 49024/110696 [14:57:47<15:37:44, 1.10it/s][2025-04-27 12:01:13] (step=0049025) Train Loss: 5.8518, Train Steps/Sec: 1.12 + 44%|███████████████████████████████████████████████████████▊ | 49049/110696 [14:58:09<15:27:08, 1.11it/s][2025-04-27 12:01:35] (step=0049050) Train Loss: 5.8827, Train Steps/Sec: 1.12 + 44%|███████████████████████████████████████████████████████▊ | 49074/110696 [14:58:32<15:20:29, 1.12it/s][2025-04-27 12:01:57] (step=0049075) Train Loss: 5.8588, Train Steps/Sec: 1.12 + 44%|███████████████████████████████████████████████████████▉ | 49099/110696 [14:58:54<15:16:56, 1.12it/s][2025-04-27 12:02:19] (step=0049100) Train Loss: 5.8275, Train Steps/Sec: 1.12 + 44%|███████████████████████████████████████████████████████▉ | 49124/110696 [14:59:16<15:12:03, 1.13it/s][2025-04-27 12:02:42] (step=0049125) Train Loss: 5.8426, Train Steps/Sec: 1.12 + 44%|███████████████████████████████████████████████████████▉ | 49149/110696 [14:59:39<15:06:50, 1.13it/s][2025-04-27 12:03:04] (step=0049150) Train Loss: 5.8909, Train Steps/Sec: 1.12 + 44%|███████████████████████████████████████████████████████▉ | 49174/110696 [15:00:01<15:05:31, 1.13it/s][2025-04-27 12:03:26] (step=0049175) Train Loss: 5.8546, Train Steps/Sec: 1.12 + 44%|████████████████████████████████████████████████████████ | 49199/110696 [15:00:23<15:09:13, 1.13it/s][2025-04-27 12:03:49] (step=0049200) Train Loss: 5.8607, Train Steps/Sec: 1.12 + 44%|████████████████████████████████████████████████████████ | 49224/110696 [15:00:46<15:33:03, 1.10it/s][2025-04-27 12:04:11] (step=0049225) Train Loss: 5.8349, Train Steps/Sec: 1.12 + 44%|████████████████████████████████████████████████████████ | 49249/110696 [15:01:08<15:20:33, 1.11it/s][2025-04-27 12:04:33] (step=0049250) Train Loss: 5.8225, Train Steps/Sec: 1.12 + 45%|████████████████████████████████████████████████████████ | 49274/110696 [15:01:30<15:15:11, 1.12it/s][2025-04-27 12:04:56] (step=0049275) Train Loss: 5.8545, Train Steps/Sec: 1.12 + 45%|████████████████████████████████████████████████████████ | 49299/110696 [15:01:53<15:12:40, 1.12it/s][2025-04-27 12:05:18] (step=0049300) Train Loss: 5.7511, Train Steps/Sec: 1.12 + 45%|████████████████████████████████████████████████████████▏ | 49324/110696 [15:02:15<15:08:00, 1.13it/s][2025-04-27 12:05:40] (step=0049325) Train Loss: 5.8370, Train Steps/Sec: 1.12 + 45%|████████████████████████████████████████████████████████▏ | 49349/110696 [15:02:37<15:07:57, 1.13it/s][2025-04-27 12:06:03] (step=0049350) Train Loss: 5.8323, Train Steps/Sec: 1.12 + 45%|████████████████████████████████████████████████████████▏ | 49374/110696 [15:02:59<15:07:34, 1.13it/s][2025-04-27 12:06:25] (step=0049375) Train Loss: 5.8022, Train Steps/Sec: 1.12 + 45%|████████████████████████████████████████████████████████▏ | 49399/110696 [15:03:22<15:04:39, 1.13it/s][2025-04-27 12:06:47] (step=0049400) Train Loss: 5.8785, Train Steps/Sec: 1.12 + 45%|████████████████████████████████████████████████████████▎ | 49424/110696 [15:03:44<15:33:16, 1.09it/s][2025-04-27 12:07:10] (step=0049425) Train Loss: 5.7835, Train Steps/Sec: 1.12 + 45%|████████████████████████████████████████████████████████▎ | 49449/110696 [15:04:07<15:20:05, 1.11it/s][2025-04-27 12:07:32] (step=0049450) Train Loss: 5.8241, Train Steps/Sec: 1.12 + 45%|████████████████████████████████████████████████████████▎ | 49474/110696 [15:04:29<15:13:22, 1.12it/s][2025-04-27 12:07:54] (step=0049475) Train Loss: 5.8468, Train Steps/Sec: 1.12 + 45%|████████████████████████████████████████████████████████▎ | 49499/110696 [15:04:51<15:07:23, 1.12it/s][2025-04-27 12:08:17] (step=0049500) Train Loss: 5.8565, Train Steps/Sec: 1.12 + 45%|████████████████████████████████████████████████████████▎ | 49524/110696 [15:05:13<15:03:11, 1.13it/s][2025-04-27 12:08:39] (step=0049525) Train Loss: 5.7854, Train Steps/Sec: 1.12 + 45%|████████████████████████████████████████████████████████▍ | 49549/110696 [15:05:40<20:41:17, 1.22s/it][2025-04-27 12:09:06] (step=0049550) Train Loss: 5.8192, Train Steps/Sec: 0.93 + 45%|████████████████████████████████████████████████████████▍ | 49574/110696 [15:06:03<15:01:51, 1.13it/s][2025-04-27 12:09:28] (step=0049575) Train Loss: 5.8180, Train Steps/Sec: 1.12 + 45%|████████████████████████████████████████████████████████▍ | 49599/110696 [15:06:25<14:59:24, 1.13it/s][2025-04-27 12:09:50] (step=0049600) Train Loss: 5.8607, Train Steps/Sec: 1.12 + 45%|████████████████████████████████████████████████████████▍ | 49624/110696 [15:06:47<15:23:50, 1.10it/s][2025-04-27 12:10:13] (step=0049625) Train Loss: 5.8605, Train Steps/Sec: 1.12 + 45%|████████████████████████████████████████████████████████▌ | 49649/110696 [15:07:14<26:59:49, 1.59s/it][2025-04-27 12:10:40] (step=0049650) Train Loss: 5.8208, Train Steps/Sec: 0.93 + 45%|████████████████████████████████████████████████████████▌ | 49674/110696 [15:07:37<15:11:01, 1.12it/s][2025-04-27 12:11:02] (step=0049675) Train Loss: 5.8851, Train Steps/Sec: 1.12 + 45%|████████████████████████████████████████████████████████▌ | 49699/110696 [15:07:59<15:04:00, 1.12it/s][2025-04-27 12:11:24] (step=0049700) Train Loss: 5.8484, Train Steps/Sec: 1.12 + 45%|████████████████████████████████████████████████████████▌ | 49724/110696 [15:08:21<15:06:36, 1.12it/s][2025-04-27 12:11:47] (step=0049725) Train Loss: 5.7450, Train Steps/Sec: 1.12 + 45%|████████████████████████████████████████████████████████▋ | 49749/110696 [15:08:44<14:59:27, 1.13it/s][2025-04-27 12:12:09] (step=0049750) Train Loss: 5.8500, Train Steps/Sec: 1.12 + 45%|████████████████████████████████████████████████████████▋ | 49774/110696 [15:09:06<14:57:36, 1.13it/s][2025-04-27 12:12:31] (step=0049775) Train Loss: 5.8935, Train Steps/Sec: 1.12 + 45%|████████████████████████████████████████████████████████▋ | 49799/110696 [15:09:28<14:56:39, 1.13it/s][2025-04-27 12:12:54] (step=0049800) Train Loss: 5.8149, Train Steps/Sec: 1.12 + 45%|████████████████████████████████████████████████████████▋ | 49824/110696 [15:09:56<19:44:45, 1.17s/it][2025-04-27 12:13:21] (step=0049825) Train Loss: 5.8320, Train Steps/Sec: 0.91 + 45%|████████████████████████████████████████████████████████▋ | 49849/110696 [15:10:18<15:16:24, 1.11it/s][2025-04-27 12:13:43] (step=0049850) Train Loss: 5.8376, Train Steps/Sec: 1.12 + 45%|████████████████████████████████████████████████████████▊ | 49874/110696 [15:10:40<15:09:48, 1.11it/s][2025-04-27 12:14:06] (step=0049875) Train Loss: 5.8137, Train Steps/Sec: 1.12 + 45%|████████████████████████████████████████████████████████▊ | 49899/110696 [15:11:03<15:00:05, 1.13it/s][2025-04-27 12:14:28] (step=0049900) Train Loss: 5.7979, Train Steps/Sec: 1.12 + 45%|████████████████████████████████████████████████████████▊ | 49924/110696 [15:11:25<15:00:10, 1.13it/s][2025-04-27 12:14:50] (step=0049925) Train Loss: 5.8670, Train Steps/Sec: 1.12 + 45%|████████████████████████████████████████████████████████▊ | 49949/110696 [15:11:47<15:00:14, 1.12it/s][2025-04-27 12:15:13] (step=0049950) Train Loss: 5.7946, Train Steps/Sec: 1.12 + 45%|████████████████████████████████████████████████████████▉ | 49974/110696 [15:12:10<14:58:42, 1.13it/s][2025-04-27 12:15:35] (step=0049975) Train Loss: 5.8140, Train Steps/Sec: 1.12 + 45%|████████████████████████████████████████████████████████▉ | 49999/110696 [15:12:40<15:52:37, 1.06it/s][2025-04-27 12:16:06] (step=0050000) Train Loss: 5.8324, Train Steps/Sec: 0.81 +You are using a model of type blip-2 to instantiate a model of type instructblip. This is not supported for all configurations of models and can yield errors. +[2025-04-27 12:16:06] text_config is None. Initializing the text config with default values (`OPTConfig`). +model_name_or_path:/tmp/haozhezhao/model/blip2-flan-t5-xl +image_place_holder: +tokenizer length after expend 32101 +tokenizer length before expend 32100 +100%|████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████| 5/5 [04:56<00:00, 59.36s/it] +[2025-04-27 12:22:15] Finish Eval in 50000 steps...█████████████████████████████████████████████████████████████████████████████████████████████| 5/5 [04:56<00:00, 59.02s/it] +processing left-padding... +processing left-padding... +processing left-padding... +processing left-padding... +processing left-padding... +[2025-04-27 12:22:34] Saved checkpoint to checkpoint/X2I_UltraEdiit_fourmask_LLava_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_filtered_objects_4mask_1280_context/003-GPT-XL/checkpoints/0050000.pt +[2025-04-27 12:22:36] Removed old checkpoint: checkpoint/X2I_UltraEdiit_fourmask_LLava_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_filtered_objects_4mask_1280_context/003-GPT-XL/checkpoints/0048000.pt + 45%|████████████████████████████████████████████████████████▉ | 50024/110696 [15:19:43<48:17:37, 2.87s/it][2025-04-27 12:23:08] (step=0050025) Train Loss: 5.8653, Train Steps/Sec: 0.06 + 45%|████████████████████████████████████████████████████████▉ | 50049/110696 [15:20:05<15:10:32, 1.11it/s][2025-04-27 12:23:31] (step=0050050) Train Loss: 5.7756, Train Steps/Sec: 1.12 + 45%|████████████████████████████████████████████████████████▉ | 50074/110696 [15:20:28<15:06:58, 1.11it/s][2025-04-27 12:23:53] (step=0050075) Train Loss: 5.8337, Train Steps/Sec: 1.12 + 45%|█████████████████████████████████████████████████████████ | 50099/110696 [15:20:50<15:02:22, 1.12it/s][2025-04-27 12:24:15] (step=0050100) Train Loss: 5.7787, Train Steps/Sec: 1.12 + 45%|█████████████████████████████████████████████████████████ | 50124/110696 [15:21:12<14:54:59, 1.13it/s][2025-04-27 12:24:38] (step=0050125) Train Loss: 5.9065, Train Steps/Sec: 1.12 + 45%|█████████████████████████████████████████████████████████ | 50149/110696 [15:21:34<14:54:23, 1.13it/s][2025-04-27 12:25:00] (step=0050150) Train Loss: 5.7701, Train Steps/Sec: 1.12 + 45%|█████████████████████████████████████████████████████████ | 50174/110696 [15:21:57<14:54:46, 1.13it/s][2025-04-27 12:25:22] (step=0050175) Train Loss: 5.7830, Train Steps/Sec: 1.12 + 45%|█████████████████████████████████████████████████████████▏ | 50199/110696 [15:22:19<14:51:25, 1.13it/s][2025-04-27 12:25:45] (step=0050200) Train Loss: 5.7792, Train Steps/Sec: 1.11 + 45%|█████████████████████████████████████████████████████████▏ | 50224/110696 [15:22:42<15:19:34, 1.10it/s][2025-04-27 12:26:07] (step=0050225) Train Loss: 5.8056, Train Steps/Sec: 1.12 + 45%|█████████████████████████████████████████████████████████▏ | 50249/110696 [15:23:04<15:07:14, 1.11it/s][2025-04-27 12:26:29] (step=0050250) Train Loss: 5.8495, Train Steps/Sec: 1.12 + 45%|█████████████████████████████████████████████████████████▏ | 50274/110696 [15:23:26<15:02:56, 1.12it/s][2025-04-27 12:26:52] (step=0050275) Train Loss: 5.7627, Train Steps/Sec: 1.12 + 45%|█████████████████████████████████████████████████████████▎ | 50299/110696 [15:23:49<14:55:10, 1.12it/s][2025-04-27 12:27:14] (step=0050300) Train Loss: 5.8159, Train Steps/Sec: 1.12 + 45%|█████████████████████████████████████████████████████████▎ | 50324/110696 [15:24:11<15:00:07, 1.12it/s][2025-04-27 12:27:37] (step=0050325) Train Loss: 5.8549, Train Steps/Sec: 1.11 + 45%|█████████████████████████████████████████████████████████▎ | 50349/110696 [15:24:33<14:51:41, 1.13it/s][2025-04-27 12:27:59] (step=0050350) Train Loss: 5.8331, Train Steps/Sec: 1.12 + 46%|█████████████████████████████████████████████████████████▎ | 50374/110696 [15:24:56<14:49:39, 1.13it/s][2025-04-27 12:28:21] (step=0050375) Train Loss: 5.8433, Train Steps/Sec: 1.12 + 46%|█████████████████████████████████████████████████████████▎ | 50399/110696 [15:25:18<14:47:10, 1.13it/s][2025-04-27 12:28:44] (step=0050400) Train Loss: 5.8115, Train Steps/Sec: 1.12 + 46%|█████████████████████████████████████████████████████████▍ | 50424/110696 [15:25:40<15:16:17, 1.10it/s][2025-04-27 12:29:06] (step=0050425) Train Loss: 5.7438, Train Steps/Sec: 1.12 + 46%|█████████████████████████████████████████████████████████▍ | 50449/110696 [15:26:03<15:05:22, 1.11it/s][2025-04-27 12:29:28] (step=0050450) Train Loss: 5.8081, Train Steps/Sec: 1.12 + 46%|█████████████████████████████████████████████████████████▍ | 50474/110696 [15:26:25<15:00:28, 1.11it/s][2025-04-27 12:29:50] (step=0050475) Train Loss: 5.8680, Train Steps/Sec: 1.12 + 46%|█████████████████████████████████████████████████████████▍ | 50499/110696 [15:26:47<14:55:20, 1.12it/s][2025-04-27 12:30:13] (step=0050500) Train Loss: 5.8609, Train Steps/Sec: 1.12 + 46%|█████████████████████████████████████████████████████████▌ | 50524/110696 [15:27:10<14:53:23, 1.12it/s][2025-04-27 12:30:35] (step=0050525) Train Loss: 5.8412, Train Steps/Sec: 1.12 + 46%|█████████████████████████████████████████████████████████▌ | 50549/110696 [15:27:32<14:50:09, 1.13it/s][2025-04-27 12:30:57] (step=0050550) Train Loss: 5.8251, Train Steps/Sec: 1.12 + 46%|█████████████████████████████████████████████████████████▌ | 50574/110696 [15:27:54<14:48:32, 1.13it/s][2025-04-27 12:31:20] (step=0050575) Train Loss: 5.7646, Train Steps/Sec: 1.12 + 46%|█████████████████████████████████████████████████████████▌ | 50599/110696 [15:28:17<14:47:19, 1.13it/s][2025-04-27 12:31:42] (step=0050600) Train Loss: 5.8506, Train Steps/Sec: 1.11 + 46%|█████████████████████████████████████████████████████████▌ | 50624/110696 [15:28:39<15:12:11, 1.10it/s][2025-04-27 12:32:05] (step=0050625) Train Loss: 5.8510, Train Steps/Sec: 1.12 + 46%|█████████████████████████████████████████████████████████▋ | 50649/110696 [15:29:02<15:01:23, 1.11it/s][2025-04-27 12:32:27] (step=0050650) Train Loss: 5.8196, Train Steps/Sec: 1.12 + 46%|█████████████████████████████████████████████████████████▋ | 50674/110696 [15:29:24<14:57:57, 1.11it/s][2025-04-27 12:32:49] (step=0050675) Train Loss: 5.7585, Train Steps/Sec: 1.12 + 46%|█████████████████████████████████████████████████████████▋ | 50699/110696 [15:29:46<14:50:33, 1.12it/s][2025-04-27 12:33:12] (step=0050700) Train Loss: 5.8042, Train Steps/Sec: 1.12 + 46%|█████████████████████████████████████████████████████████▋ | 50724/110696 [15:30:08<14:49:57, 1.12it/s][2025-04-27 12:33:34] (step=0050725) Train Loss: 5.8326, Train Steps/Sec: 1.12 + 46%|█████████████████████████████████████████████████████████▊ | 50749/110696 [15:30:31<14:47:11, 1.13it/s][2025-04-27 12:33:56] (step=0050750) Train Loss: 5.8422, Train Steps/Sec: 1.12 + 46%|█████████████████████████████████████████████████████████▊ | 50774/110696 [15:30:53<14:43:27, 1.13it/s][2025-04-27 12:34:18] (step=0050775) Train Loss: 5.7564, Train Steps/Sec: 1.12 + 46%|█████████████████████████████████████████████████████████▊ | 50799/110696 [15:31:15<14:41:52, 1.13it/s][2025-04-27 12:34:41] (step=0050800) Train Loss: 5.8148, Train Steps/Sec: 1.12 + 46%|█████████████████████████████████████████████████████████▊ | 50824/110696 [15:31:38<15:06:49, 1.10it/s][2025-04-27 12:35:03] (step=0050825) Train Loss: 5.7933, Train Steps/Sec: 1.12 + 46%|█████████████████████████████████████████████████████████▉ | 50849/110696 [15:32:00<15:01:26, 1.11it/s][2025-04-27 12:35:25] (step=0050850) Train Loss: 5.8680, Train Steps/Sec: 1.12 + 46%|█████████████████████████████████████████████████████████▉ | 50874/110696 [15:32:22<14:50:38, 1.12it/s][2025-04-27 12:35:48] (step=0050875) Train Loss: 5.7729, Train Steps/Sec: 1.12 + 46%|█████████████████████████████████████████████████████████▉ | 50899/110696 [15:32:45<14:46:41, 1.12it/s][2025-04-27 12:36:10] (step=0050900) Train Loss: 5.8294, Train Steps/Sec: 1.12 + 46%|█████████████████████████████████████████████████████████▉ | 50924/110696 [15:33:07<14:44:30, 1.13it/s][2025-04-27 12:36:32] (step=0050925) Train Loss: 5.8134, Train Steps/Sec: 1.12 + 46%|█████████████████████████████████████████████████████████▉ | 50949/110696 [15:33:29<14:41:03, 1.13it/s][2025-04-27 12:36:55] (step=0050950) Train Loss: 5.8275, Train Steps/Sec: 1.12 + 46%|██████████████████████████████████████████████████████████ | 50974/110696 [15:33:52<14:38:10, 1.13it/s][2025-04-27 12:37:17] (step=0050975) Train Loss: 5.8262, Train Steps/Sec: 1.12 + 46%|██████████████████████████████████████████████████████████ | 50999/110696 [15:34:14<14:43:04, 1.13it/s][2025-04-27 12:37:39] (step=0051000) Train Loss: 5.8605, Train Steps/Sec: 1.11 + 46%|██████████████████████████████████████████████████████████ | 51024/110696 [15:34:36<15:03:01, 1.10it/s][2025-04-27 12:38:02] (step=0051025) Train Loss: 5.7406, Train Steps/Sec: 1.12 + 46%|██████████████████████████████████████████████████████████ | 51049/110696 [15:34:59<14:55:09, 1.11it/s][2025-04-27 12:38:24] (step=0051050) Train Loss: 5.8594, Train Steps/Sec: 1.12 + 46%|██████████████████████████████████████████████████████████▏ | 51074/110696 [15:35:21<14:48:09, 1.12it/s][2025-04-27 12:38:46] (step=0051075) Train Loss: 5.8230, Train Steps/Sec: 1.12 + 46%|██████████████████████████████████████████████████████████▏ | 51099/110696 [15:35:43<14:43:17, 1.12it/s][2025-04-27 12:39:09] (step=0051100) Train Loss: 5.8033, Train Steps/Sec: 1.12 + 46%|██████████████████████████████████████████████████████████▏ | 51124/110696 [15:36:05<14:41:39, 1.13it/s][2025-04-27 12:39:31] (step=0051125) Train Loss: 5.7866, Train Steps/Sec: 1.12 + 46%|██████████████████████████████████████████████████████████▏ | 51149/110696 [15:36:28<14:42:22, 1.12it/s][2025-04-27 12:39:53] (step=0051150) Train Loss: 5.7969, Train Steps/Sec: 1.12 + 46%|██████████████████████████████████████████████████████████▏ | 51174/110696 [15:36:50<14:37:50, 1.13it/s][2025-04-27 12:40:16] (step=0051175) Train Loss: 5.8134, Train Steps/Sec: 1.12 + 46%|██████████████████████████████████████████████████████████▎ | 51199/110696 [15:37:13<14:38:52, 1.13it/s][2025-04-27 12:40:38] (step=0051200) Train Loss: 5.7648, Train Steps/Sec: 1.11 + 46%|██████████████████████████████████████████████████████████▎ | 51224/110696 [15:37:35<15:03:48, 1.10it/s][2025-04-27 12:41:00] (step=0051225) Train Loss: 5.7778, Train Steps/Sec: 1.12 + 46%|██████████████████████████████████████████████████████████▎ | 51249/110696 [15:37:57<14:53:26, 1.11it/s][2025-04-27 12:41:23] (step=0051250) Train Loss: 5.8544, Train Steps/Sec: 1.12 + 46%|██████████████████████████████████████████████████████████▎ | 51274/110696 [15:38:20<14:47:00, 1.12it/s][2025-04-27 12:41:45] (step=0051275) Train Loss: 5.7872, Train Steps/Sec: 1.12 + 46%|██████████████████████████████████████████████████████████▍ | 51299/110696 [15:38:42<14:41:48, 1.12it/s][2025-04-27 12:42:07] (step=0051300) Train Loss: 5.8653, Train Steps/Sec: 1.12 + 46%|██████████████████████████████████████████████████████████▍ | 51324/110696 [15:39:04<14:39:36, 1.12it/s][2025-04-27 12:42:30] (step=0051325) Train Loss: 5.7729, Train Steps/Sec: 1.12 + 46%|██████████████████████████████████████████████████████████▍ | 51349/110696 [15:39:27<14:38:31, 1.13it/s][2025-04-27 12:42:52] (step=0051350) Train Loss: 5.7932, Train Steps/Sec: 1.12 + 46%|██████████████████████████████████████████████████████████▍ | 51374/110696 [15:39:49<14:36:17, 1.13it/s][2025-04-27 12:43:14] (step=0051375) Train Loss: 5.7564, Train Steps/Sec: 1.12 + 46%|██████████████████████████████████████████████████████████▌ | 51399/110696 [15:40:11<14:35:58, 1.13it/s][2025-04-27 12:43:37] (step=0051400) Train Loss: 5.8480, Train Steps/Sec: 1.11 + 46%|██████████████████████████████████████████████████████████▌ | 51424/110696 [15:40:34<14:58:33, 1.10it/s][2025-04-27 12:43:59] (step=0051425) Train Loss: 5.7932, Train Steps/Sec: 1.12 + 46%|██████████████████████████████████████████████████████████▌ | 51449/110696 [15:40:56<14:45:50, 1.11it/s][2025-04-27 12:44:21] (step=0051450) Train Loss: 5.8157, Train Steps/Sec: 1.12 + 47%|██████████████████████████████████████████████████████████▌ | 51474/110696 [15:41:18<14:47:00, 1.11it/s][2025-04-27 12:44:44] (step=0051475) Train Loss: 5.8744, Train Steps/Sec: 1.12 + 47%|██████████████████████████████████████████████████████████▌ | 51499/110696 [15:41:41<14:35:47, 1.13it/s][2025-04-27 12:45:06] (step=0051500) Train Loss: 5.8522, Train Steps/Sec: 1.12 + 47%|██████████████████████████████████████████████████████████▋ | 51524/110696 [15:42:03<14:38:34, 1.12it/s][2025-04-27 12:45:28] (step=0051525) Train Loss: 5.7804, Train Steps/Sec: 1.12 + 47%|██████████████████████████████████████████████████████████▋ | 51549/110696 [15:42:25<14:36:28, 1.12it/s][2025-04-27 12:45:51] (step=0051550) Train Loss: 5.8312, Train Steps/Sec: 1.12 + 47%|██████████████████████████████████████████████████████████▋ | 51574/110696 [15:42:47<14:33:46, 1.13it/s][2025-04-27 12:46:13] (step=0051575) Train Loss: 5.7996, Train Steps/Sec: 1.12 + 47%|██████████████████████████████████████████████████████████▋ | 51599/110696 [15:43:10<14:31:19, 1.13it/s][2025-04-27 12:46:35] (step=0051600) Train Loss: 5.8068, Train Steps/Sec: 1.12 + 47%|██████████████████████████████████████████████████████████▊ | 51624/110696 [15:43:32<15:01:23, 1.09it/s][2025-04-27 12:46:58] (step=0051625) Train Loss: 5.7449, Train Steps/Sec: 1.12 + 47%|██████████████████████████████████████████████████████████▊ | 51649/110696 [15:43:55<14:48:46, 1.11it/s][2025-04-27 12:47:20] (step=0051650) Train Loss: 5.8036, Train Steps/Sec: 1.12 + 47%|██████████████████████████████████████████████████████████▊ | 51674/110696 [15:44:17<14:38:13, 1.12it/s][2025-04-27 12:47:42] (step=0051675) Train Loss: 5.8697, Train Steps/Sec: 1.12 + 47%|██████████████████████████████████████████████████████████▊ | 51699/110696 [15:44:39<14:35:13, 1.12it/s][2025-04-27 12:48:05] (step=0051700) Train Loss: 5.7883, Train Steps/Sec: 1.12 + 47%|██████████████████████████████████████████████████████████▊ | 51724/110696 [15:45:02<14:34:07, 1.12it/s][2025-04-27 12:48:27] (step=0051725) Train Loss: 5.7937, Train Steps/Sec: 1.12 + 47%|██████████████████████████████████████████████████████████▉ | 51749/110696 [15:45:24<14:32:37, 1.13it/s][2025-04-27 12:48:49] (step=0051750) Train Loss: 5.8208, Train Steps/Sec: 1.12 + 47%|██████████████████████████████████████████████████████████▉ | 51774/110696 [15:45:46<14:28:10, 1.13it/s][2025-04-27 12:49:12] (step=0051775) Train Loss: 5.7605, Train Steps/Sec: 1.12 + 47%|██████████████████████████████████████████████████████████▉ | 51799/110696 [15:46:08<14:27:35, 1.13it/s][2025-04-27 12:49:34] (step=0051800) Train Loss: 5.7830, Train Steps/Sec: 1.12 + 47%|██████████████████████████████████████████████████████████▉ | 51824/110696 [15:46:31<14:59:11, 1.09it/s][2025-04-27 12:49:56] (step=0051825) Train Loss: 5.7771, Train Steps/Sec: 1.12 + 47%|███████████████████████████████████████████████████████████ | 51849/110696 [15:46:53<14:44:39, 1.11it/s][2025-04-27 12:50:19] (step=0051850) Train Loss: 5.7913, Train Steps/Sec: 1.12 + 47%|███████████████████████████████████████████████████████████ | 51874/110696 [15:47:16<14:36:56, 1.12it/s][2025-04-27 12:50:41] (step=0051875) Train Loss: 5.8828, Train Steps/Sec: 1.12 + 47%|███████████████████████████████████████████████████████████ | 51899/110696 [15:47:38<14:34:30, 1.12it/s][2025-04-27 12:51:03] (step=0051900) Train Loss: 5.7846, Train Steps/Sec: 1.12 + 47%|███████████████████████████████████████████████████████████ | 51924/110696 [15:48:00<14:30:52, 1.12it/s][2025-04-27 12:51:26] (step=0051925) Train Loss: 5.8163, Train Steps/Sec: 1.12 + 47%|███████████████████████████████████████████████████████████▏ | 51949/110696 [15:48:23<14:26:28, 1.13it/s][2025-04-27 12:51:48] (step=0051950) Train Loss: 5.8479, Train Steps/Sec: 1.12 + 47%|███████████████████████████████████████████████████████████▏ | 51974/110696 [15:48:45<14:27:31, 1.13it/s][2025-04-27 12:52:10] (step=0051975) Train Loss: 5.8151, Train Steps/Sec: 1.12 + 47%|███████████████████████████████████████████████████████████▏ | 51999/110696 [15:49:07<14:23:42, 1.13it/s][2025-04-27 12:52:33] (step=0052000) Train Loss: 5.8805, Train Steps/Sec: 1.12 +You are using a model of type blip-2 to instantiate a model of type instructblip. This is not supported for all configurations of models and can yield errors. +[2025-04-27 12:52:33] text_config is None. Initializing the text config with default values (`OPTConfig`). +model_name_or_path:/tmp/haozhezhao/model/blip2-flan-t5-xl +image_place_holder: +tokenizer length after expend 32101 +tokenizer length before expend 32100 +100%|████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████| 5/5 [04:56<00:00, 59.38s/it] +[2025-04-27 12:58:42] Finish Eval in 52000 steps...█████████████████████████████████████████████████████████████████████████████████████████████| 5/5 [04:56<00:00, 59.21s/it] +processing left-padding... +processing left-padding... +processing left-padding... +processing left-padding... +processing left-padding... +[2025-04-27 12:59:02] Saved checkpoint to checkpoint/X2I_UltraEdiit_fourmask_LLava_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_filtered_objects_4mask_1280_context/003-GPT-XL/checkpoints/0052000.pt +[2025-04-27 12:59:04] Removed old checkpoint: checkpoint/X2I_UltraEdiit_fourmask_LLava_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_filtered_objects_4mask_1280_context/003-GPT-XL/checkpoints/0050000.pt + 47%|███████████████████████████████████████████████████████████▏ | 52024/110696 [15:56:01<15:13:35, 1.07it/s][2025-04-27 12:59:26] (step=0052025) Train Loss: 5.7631, Train Steps/Sec: 0.06 + 47%|███████████████████████████████████████████████████████████▏ | 52049/110696 [15:56:23<14:36:56, 1.11it/s][2025-04-27 12:59:48] (step=0052050) Train Loss: 5.8168, Train Steps/Sec: 1.12 + 47%|███████████████████████████████████████████████████████████▎ | 52074/110696 [15:56:45<14:35:58, 1.12it/s][2025-04-27 13:00:11] (step=0052075) Train Loss: 5.8613, Train Steps/Sec: 1.12 + 47%|███████████████████████████████████████████████████████████▎ | 52099/110696 [15:57:08<14:30:45, 1.12it/s][2025-04-27 13:00:33] (step=0052100) Train Loss: 5.8383, Train Steps/Sec: 1.12 + 47%|███████████████████████████████████████████████████████████▎ | 52124/110696 [15:57:30<14:24:01, 1.13it/s][2025-04-27 13:00:55] (step=0052125) Train Loss: 5.8050, Train Steps/Sec: 1.12 + 47%|███████████████████████████████████████████████████████████▎ | 52149/110696 [15:57:52<14:20:51, 1.13it/s][2025-04-27 13:01:17] (step=0052150) Train Loss: 5.8656, Train Steps/Sec: 1.12 + 47%|███████████████████████████████████████████████████████████▍ | 52174/110696 [15:58:14<14:20:02, 1.13it/s][2025-04-27 13:01:40] (step=0052175) Train Loss: 5.7717, Train Steps/Sec: 1.13 + 47%|███████████████████████████████████████████████████████████▍ | 52199/110696 [15:58:37<14:21:14, 1.13it/s][2025-04-27 13:02:02] (step=0052200) Train Loss: 5.8569, Train Steps/Sec: 1.12 + 47%|███████████████████████████████████████████████████████████▍ | 52224/110696 [15:58:59<15:30:41, 1.05it/s][2025-04-27 13:02:25] (step=0052225) Train Loss: 5.8101, Train Steps/Sec: 1.11 + 47%|███████████████████████████████████████████████████████████▍ | 52249/110696 [15:59:21<14:38:25, 1.11it/s][2025-04-27 13:02:47] (step=0052250) Train Loss: 5.8467, Train Steps/Sec: 1.12 + 47%|███████████████████████████████████████████████████████████▌ | 52274/110696 [15:59:44<14:29:51, 1.12it/s][2025-04-27 13:03:09] (step=0052275) Train Loss: 5.8442, Train Steps/Sec: 1.12 + 47%|███████████████████████████████████████████████████████████▌ | 52299/110696 [16:00:06<14:24:13, 1.13it/s][2025-04-27 13:03:31] (step=0052300) Train Loss: 5.7810, Train Steps/Sec: 1.12 + 47%|███████████████████████████████████████████████████████████▌ | 52324/110696 [16:00:28<14:20:50, 1.13it/s][2025-04-27 13:03:54] (step=0052325) Train Loss: 5.8323, Train Steps/Sec: 1.12 + 47%|███████████████████████████████████████████████████████████▌ | 52349/110696 [16:00:51<14:19:51, 1.13it/s][2025-04-27 13:04:16] (step=0052350) Train Loss: 5.7931, Train Steps/Sec: 1.12 + 47%|███████████████████████████████████████████████████████████▌ | 52374/110696 [16:01:13<14:19:57, 1.13it/s][2025-04-27 13:04:38] (step=0052375) Train Loss: 5.7956, Train Steps/Sec: 1.12 + 47%|███████████████████████████████████████████████████████████▋ | 52399/110696 [16:01:35<14:19:14, 1.13it/s][2025-04-27 13:05:01] (step=0052400) Train Loss: 5.8602, Train Steps/Sec: 1.12 + 47%|███████████████████████████████████████████████████████████▋ | 52424/110696 [16:01:57<14:43:02, 1.10it/s][2025-04-27 13:05:23] (step=0052425) Train Loss: 5.8046, Train Steps/Sec: 1.12 + 47%|███████████████████████████████████████████████████████████▋ | 52449/110696 [16:02:20<14:34:51, 1.11it/s][2025-04-27 13:05:45] (step=0052450) Train Loss: 5.8219, Train Steps/Sec: 1.12 + 47%|███████████████████████████████████████████████████████████▋ | 52474/110696 [16:02:42<14:27:01, 1.12it/s][2025-04-27 13:06:07] (step=0052475) Train Loss: 5.8417, Train Steps/Sec: 1.12 + 47%|███████████████████████████████████████████████████████████▊ | 52499/110696 [16:03:04<14:22:22, 1.12it/s][2025-04-27 13:06:30] (step=0052500) Train Loss: 5.7979, Train Steps/Sec: 1.12 + 47%|███████████████████████████████████████████████████████████▊ | 52524/110696 [16:03:27<14:20:11, 1.13it/s][2025-04-27 13:06:52] (step=0052525) Train Loss: 5.7926, Train Steps/Sec: 1.12 + 47%|███████████████████████████████████████████████████████████▊ | 52549/110696 [16:03:49<14:18:29, 1.13it/s][2025-04-27 13:07:14] (step=0052550) Train Loss: 5.7560, Train Steps/Sec: 1.12 + 47%|███████████████████████████████████████████████████████████▊ | 52574/110696 [16:04:11<14:14:04, 1.13it/s][2025-04-27 13:07:36] (step=0052575) Train Loss: 5.7791, Train Steps/Sec: 1.12 + 48%|███████████████████████████████████████████████████████████▊ | 52599/110696 [16:04:33<14:18:14, 1.13it/s][2025-04-27 13:07:59] (step=0052600) Train Loss: 5.8128, Train Steps/Sec: 1.12 + 48%|███████████████████████████████████████████████████████████▉ | 52624/110696 [16:04:56<14:41:29, 1.10it/s][2025-04-27 13:08:21] (step=0052625) Train Loss: 5.7915, Train Steps/Sec: 1.12 + 48%|███████████████████████████████████████████████████████████▉ | 52649/110696 [16:05:18<14:30:31, 1.11it/s][2025-04-27 13:08:43] (step=0052650) Train Loss: 5.8365, Train Steps/Sec: 1.12 + 48%|███████████████████████████████████████████████████████████▉ | 52674/110696 [16:05:40<14:21:08, 1.12it/s][2025-04-27 13:09:06] (step=0052675) Train Loss: 5.7933, Train Steps/Sec: 1.12 + 48%|███████████████████████████████████████████████████████████▉ | 52699/110696 [16:06:03<14:22:48, 1.12it/s][2025-04-27 13:09:28] (step=0052700) Train Loss: 5.7884, Train Steps/Sec: 1.12 + 48%|████████████████████████████████████████████████████████████ | 52724/110696 [16:06:25<14:14:27, 1.13it/s][2025-04-27 13:09:50] (step=0052725) Train Loss: 5.7967, Train Steps/Sec: 1.12 + 48%|████████████████████████████████████████████████████████████ | 52749/110696 [16:06:47<14:17:52, 1.13it/s][2025-04-27 13:10:12] (step=0052750) Train Loss: 5.8262, Train Steps/Sec: 1.12 + 48%|████████████████████████████████████████████████████████████ | 52774/110696 [16:07:09<14:14:16, 1.13it/s][2025-04-27 13:10:35] (step=0052775) Train Loss: 5.8482, Train Steps/Sec: 1.12 + 48%|████████████████████████████████████████████████████████████ | 52799/110696 [16:07:32<14:12:48, 1.13it/s][2025-04-27 13:10:57] (step=0052800) Train Loss: 5.8174, Train Steps/Sec: 1.12 + 48%|████████████████████████████████████████████████████████████▏ | 52824/110696 [16:07:54<14:39:29, 1.10it/s][2025-04-27 13:11:19] (step=0052825) Train Loss: 5.7195, Train Steps/Sec: 1.12 + 48%|████████████████████████████████████████████████████████████▏ | 52849/110696 [16:08:16<14:30:32, 1.11it/s][2025-04-27 13:11:42] (step=0052850) Train Loss: 5.8238, Train Steps/Sec: 1.12 + 48%|████████████████████████████████████████████████████████████▏ | 52874/110696 [16:08:39<14:23:19, 1.12it/s][2025-04-27 13:12:04] (step=0052875) Train Loss: 5.8091, Train Steps/Sec: 1.12 + 48%|████████████████████████████████████████████████████████████▏ | 52899/110696 [16:09:01<14:16:52, 1.12it/s][2025-04-27 13:12:26] (step=0052900) Train Loss: 5.8630, Train Steps/Sec: 1.12 + 48%|████████████████████████████████████████████████████████████▏ | 52924/110696 [16:09:23<14:13:10, 1.13it/s][2025-04-27 13:12:49] (step=0052925) Train Loss: 5.8625, Train Steps/Sec: 1.12 + 48%|████████████████████████████████████████████████████████████▎ | 52949/110696 [16:09:46<14:11:59, 1.13it/s][2025-04-27 13:13:11] (step=0052950) Train Loss: 5.8413, Train Steps/Sec: 1.12 + 48%|████████████████████████████████████████████████████████████▎ | 52974/110696 [16:10:08<14:11:50, 1.13it/s][2025-04-27 13:13:33] (step=0052975) Train Loss: 5.8414, Train Steps/Sec: 1.12 + 48%|████████████████████████████████████████████████████████████▎ | 52999/110696 [16:10:30<14:07:49, 1.13it/s][2025-04-27 13:13:56] (step=0053000) Train Loss: 5.8397, Train Steps/Sec: 1.12 + 48%|████████████████████████████████████████████████████████████▎ | 53024/110696 [16:10:52<14:34:52, 1.10it/s][2025-04-27 13:14:18] (step=0053025) Train Loss: 5.7212, Train Steps/Sec: 1.12 + 48%|████████████████████████████████████████████████████████████▍ | 53049/110696 [16:11:15<14:21:21, 1.12it/s][2025-04-27 13:14:40] (step=0053050) Train Loss: 5.7422, Train Steps/Sec: 1.12 + 48%|████████████████████████████████████████████████████████████▍ | 53074/110696 [16:11:37<14:20:49, 1.12it/s][2025-04-27 13:15:02] (step=0053075) Train Loss: 5.8462, Train Steps/Sec: 1.12 + 48%|████████████████████████████████████████████████████████████▍ | 53099/110696 [16:11:59<14:10:26, 1.13it/s][2025-04-27 13:15:25] (step=0053100) Train Loss: 5.8384, Train Steps/Sec: 1.13 + 48%|████████████████████████████████████████████████████████████▍ | 53124/110696 [16:12:22<14:11:40, 1.13it/s][2025-04-27 13:15:47] (step=0053125) Train Loss: 5.8886, Train Steps/Sec: 1.12 + 48%|████████████████████████████████████████████████████████████▍ | 53149/110696 [16:12:44<14:09:57, 1.13it/s][2025-04-27 13:16:09] (step=0053150) Train Loss: 5.7617, Train Steps/Sec: 1.12 + 48%|████████████████████████████████████████████████████████████▌ | 53174/110696 [16:13:06<14:05:27, 1.13it/s][2025-04-27 13:16:31] (step=0053175) Train Loss: 5.7533, Train Steps/Sec: 1.12 + 48%|████████████████████████████████████████████████████████████▌ | 53199/110696 [16:13:28<14:05:29, 1.13it/s][2025-04-27 13:16:54] (step=0053200) Train Loss: 5.7798, Train Steps/Sec: 1.12 + 48%|████████████████████████████████████████████████████████████▌ | 53224/110696 [16:13:51<14:33:13, 1.10it/s][2025-04-27 13:17:16] (step=0053225) Train Loss: 5.7703, Train Steps/Sec: 1.12 + 48%|████████████████████████████████████████████████████████████▌ | 53249/110696 [16:14:13<14:18:42, 1.11it/s][2025-04-27 13:17:38] (step=0053250) Train Loss: 5.7948, Train Steps/Sec: 1.12 + 48%|████████████████████████████████████████████████████████████▋ | 53274/110696 [16:14:35<14:18:23, 1.11it/s][2025-04-27 13:18:01] (step=0053275) Train Loss: 5.8027, Train Steps/Sec: 1.12 + 48%|████████████████████████████████████████████████████████████▋ | 53299/110696 [16:14:57<14:08:55, 1.13it/s][2025-04-27 13:18:23] (step=0053300) Train Loss: 5.8050, Train Steps/Sec: 1.12 + 48%|████████████████████████████████████████████████████████████▋ | 53324/110696 [16:15:20<14:14:56, 1.12it/s][2025-04-27 13:18:45] (step=0053325) Train Loss: 5.7738, Train Steps/Sec: 1.12 + 48%|████████████████████████████████████████████████████████████▋ | 53349/110696 [16:15:42<14:05:32, 1.13it/s][2025-04-27 13:19:07] (step=0053350) Train Loss: 5.8381, Train Steps/Sec: 1.12 + 48%|████████████████████████████████████████████████████████████▊ | 53374/110696 [16:16:04<14:02:03, 1.13it/s][2025-04-27 13:19:30] (step=0053375) Train Loss: 5.7908, Train Steps/Sec: 1.12 + 48%|████████████████████████████████████████████████████████████▊ | 53399/110696 [16:16:27<14:02:57, 1.13it/s][2025-04-27 13:19:52] (step=0053400) Train Loss: 5.7841, Train Steps/Sec: 1.12 + 48%|████████████████████████████████████████████████████████████▊ | 53424/110696 [16:16:49<14:29:40, 1.10it/s][2025-04-27 13:20:14] (step=0053425) Train Loss: 5.8420, Train Steps/Sec: 1.12 + 48%|████████████████████████████████████████████████████████████▊ | 53449/110696 [16:17:11<14:17:01, 1.11it/s][2025-04-27 13:20:37] (step=0053450) Train Loss: 5.8162, Train Steps/Sec: 1.12 + 48%|████████████████████████████████████████████████████████████▊ | 53474/110696 [16:17:33<14:10:15, 1.12it/s][2025-04-27 13:20:59] (step=0053475) Train Loss: 5.7983, Train Steps/Sec: 1.12 + 48%|████████████████████████████████████████████████████████████▉ | 53499/110696 [16:17:56<14:11:35, 1.12it/s][2025-04-27 13:21:21] (step=0053500) Train Loss: 5.8713, Train Steps/Sec: 1.12 + 48%|████████████████████████████████████████████████████████████▉ | 53524/110696 [16:18:18<14:06:47, 1.13it/s][2025-04-27 13:21:43] (step=0053525) Train Loss: 5.7537, Train Steps/Sec: 1.12 + 48%|████████████████████████████████████████████████████████████▉ | 53549/110696 [16:18:40<14:02:55, 1.13it/s][2025-04-27 13:22:06] (step=0053550) Train Loss: 5.7240, Train Steps/Sec: 1.12 + 48%|████████████████████████████████████████████████████████████▉ | 53574/110696 [16:19:03<14:01:13, 1.13it/s][2025-04-27 13:22:28] (step=0053575) Train Loss: 5.8256, Train Steps/Sec: 1.12 + 48%|█████████████████████████████████████████████████████████████ | 53599/110696 [16:19:25<13:57:41, 1.14it/s][2025-04-27 13:22:50] (step=0053600) Train Loss: 5.8174, Train Steps/Sec: 1.12 + 48%|█████████████████████████████████████████████████████████████ | 53624/110696 [16:19:47<14:26:11, 1.10it/s][2025-04-27 13:23:13] (step=0053625) Train Loss: 5.8753, Train Steps/Sec: 1.12 + 48%|█████████████████████████████████████████████████████████████ | 53649/110696 [16:20:09<14:15:31, 1.11it/s][2025-04-27 13:23:35] (step=0053650) Train Loss: 5.7424, Train Steps/Sec: 1.12 + 48%|█████████████████████████████████████████████████████████████ | 53674/110696 [16:20:32<14:08:52, 1.12it/s][2025-04-27 13:23:57] (step=0053675) Train Loss: 5.8367, Train Steps/Sec: 1.12 + 49%|█████████████████████████████████████████████████████████████ | 53699/110696 [16:20:54<14:02:48, 1.13it/s][2025-04-27 13:24:19] (step=0053700) Train Loss: 5.7372, Train Steps/Sec: 1.13 + 49%|█████████████████████████████████████████████████████████████▏ | 53724/110696 [16:21:16<14:05:58, 1.12it/s][2025-04-27 13:24:42] (step=0053725) Train Loss: 5.7795, Train Steps/Sec: 1.12 + 49%|█████████████████████████████████████████████████████████████▏ | 53749/110696 [16:21:39<13:59:09, 1.13it/s][2025-04-27 13:25:04] (step=0053750) Train Loss: 5.8005, Train Steps/Sec: 1.12 + 49%|█████████████████████████████████████████████████████████████▏ | 53774/110696 [16:22:01<13:56:43, 1.13it/s][2025-04-27 13:25:26] (step=0053775) Train Loss: 5.7903, Train Steps/Sec: 1.12 + 49%|█████████████████████████████████████████████████████████████▏ | 53799/110696 [16:22:28<24:26:58, 1.55s/it][2025-04-27 13:25:53] (step=0053800) Train Loss: 5.7296, Train Steps/Sec: 0.93 + 49%|█████████████████████████████████████████████████████████████▎ | 53824/110696 [16:22:50<14:23:41, 1.10it/s][2025-04-27 13:26:15] (step=0053825) Train Loss: 5.8310, Train Steps/Sec: 1.12 + 49%|█████████████████████████████████████████████████████████████▎ | 53849/110696 [16:23:12<14:09:03, 1.12it/s][2025-04-27 13:26:37] (step=0053850) Train Loss: 5.8460, Train Steps/Sec: 1.13 + 49%|█████████████████████████████████████████████████████████████▎ | 53874/110696 [16:23:34<14:05:24, 1.12it/s][2025-04-27 13:27:00] (step=0053875) Train Loss: 5.8339, Train Steps/Sec: 1.12 + 49%|█████████████████████████████████████████████████████████████▎ | 53899/110696 [16:23:57<14:01:34, 1.12it/s][2025-04-27 13:27:22] (step=0053900) Train Loss: 5.7667, Train Steps/Sec: 1.13 + 49%|█████████████████████████████████████████████████████████████▍ | 53924/110696 [16:24:24<14:14:52, 1.11it/s][2025-04-27 13:27:49] (step=0053925) Train Loss: 5.8106, Train Steps/Sec: 0.93 + 49%|█████████████████████████████████████████████████████████████▍ | 53949/110696 [16:24:46<13:52:57, 1.14it/s][2025-04-27 13:28:11] (step=0053950) Train Loss: 5.7747, Train Steps/Sec: 1.12 + 49%|█████████████████████████████████████████████████████████████▍ | 53974/110696 [16:25:08<13:54:59, 1.13it/s][2025-04-27 13:28:33] (step=0053975) Train Loss: 5.7797, Train Steps/Sec: 1.12 + 49%|█████████████████████████████████████████████████████████████▍ | 53999/110696 [16:25:30<13:54:01, 1.13it/s][2025-04-27 13:28:56] (step=0054000) Train Loss: 5.9028, Train Steps/Sec: 1.12 +You are using a model of type blip-2 to instantiate a model of type instructblip. This is not supported for all configurations of models and can yield errors. +[2025-04-27 13:28:56] text_config is None. Initializing the text config with default values (`OPTConfig`). +model_name_or_path:/tmp/haozhezhao/model/blip2-flan-t5-xl +image_place_holder: +tokenizer length after expend 32101 +tokenizer length before expend 32100 +100%|████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████| 5/5 [04:56<00:00, 59.32s/it] +[2025-04-27 13:35:05] Finish Eval in 54000 steps...█████████████████████████████████████████████████████████████████████████████████████████████| 5/5 [04:55<00:00, 59.08s/it] +processing left-padding... +processing left-padding... +processing left-padding... +processing left-padding... +processing left-padding... +[2025-04-27 13:35:24] Saved checkpoint to checkpoint/X2I_UltraEdiit_fourmask_LLava_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_filtered_objects_4mask_1280_context/003-GPT-XL/checkpoints/0054000.pt +[2025-04-27 13:35:25] Removed old checkpoint: checkpoint/X2I_UltraEdiit_fourmask_LLava_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_filtered_objects_4mask_1280_context/003-GPT-XL/checkpoints/0052000.pt + 49%|█████████████████████████████████████████████████████████████▍ | 54024/110696 [16:32:23<14:39:18, 1.07it/s][2025-04-27 13:35:48] (step=0054025) Train Loss: 5.7993, Train Steps/Sec: 0.06 + 49%|█████████████████████████████████████████████████████████████▌ | 54049/110696 [16:32:45<14:04:30, 1.12it/s][2025-04-27 13:36:10] (step=0054050) Train Loss: 5.8045, Train Steps/Sec: 1.13 + 49%|█████████████████████████████████████████████████████████████▌ | 54074/110696 [16:33:07<13:59:46, 1.12it/s][2025-04-27 13:36:32] (step=0054075) Train Loss: 5.7795, Train Steps/Sec: 1.13 + 49%|█████████████████████████████████████████████████████████████▌ | 54099/110696 [16:33:29<14:02:10, 1.12it/s][2025-04-27 13:36:55] (step=0054100) Train Loss: 5.7540, Train Steps/Sec: 1.12 + 49%|█████████████████████████████████████████████████████████████▌ | 54124/110696 [16:33:52<13:56:16, 1.13it/s][2025-04-27 13:37:17] (step=0054125) Train Loss: 5.7212, Train Steps/Sec: 1.12 + 49%|█████████████████████████████████████████████████████████████▋ | 54149/110696 [16:34:14<13:55:38, 1.13it/s][2025-04-27 13:37:40] (step=0054150) Train Loss: 5.7862, Train Steps/Sec: 1.12 + 49%|█████████████████████████████████████████████████████████████▋ | 54174/110696 [16:34:37<13:53:45, 1.13it/s][2025-04-27 13:38:02] (step=0054175) Train Loss: 5.7633, Train Steps/Sec: 1.12 + 49%|█████████████████████████████████████████████████████████████▋ | 54199/110696 [16:34:59<13:51:39, 1.13it/s][2025-04-27 13:38:24] (step=0054200) Train Loss: 5.7641, Train Steps/Sec: 1.12 + 49%|█████████████████████████████████████████████████████████████▋ | 54224/110696 [16:35:26<26:09:20, 1.67s/it][2025-04-27 13:38:52] (step=0054225) Train Loss: 5.8516, Train Steps/Sec: 0.91 + 49%|█████████████████████████████████████████████████████████████▋ | 54249/110696 [16:35:49<14:06:40, 1.11it/s][2025-04-27 13:39:14] (step=0054250) Train Loss: 5.7948, Train Steps/Sec: 1.12 + 49%|█████████████████████████████████████████████████████████████▊ | 54274/110696 [16:36:11<14:00:21, 1.12it/s][2025-04-27 13:39:36] (step=0054275) Train Loss: 5.8668, Train Steps/Sec: 1.12 + 49%|█████████████████████████████████████████████████████████████▊ | 54299/110696 [16:36:33<13:57:34, 1.12it/s][2025-04-27 13:40:02] (step=0054300) Train Loss: 5.7804, Train Steps/Sec: 0.96 + 49%|█████████████████████████████████████████████████████████████▊ | 54324/110696 [16:37:04<14:13:36, 1.10it/s][2025-04-27 13:40:29] (step=0054325) Train Loss: 5.8224, Train Steps/Sec: 0.93 + 49%|█████████████████████████████████████████████████████████████▊ | 54349/110696 [16:37:31<19:03:38, 1.22s/it][2025-04-27 13:40:56] (step=0054350) Train Loss: 5.8056, Train Steps/Sec: 0.93 + 49%|█████████████████████████████████████████████████████████████▉ | 54374/110696 [16:37:58<13:50:45, 1.13it/s][2025-04-27 13:41:24] (step=0054375) Train Loss: 5.8844, Train Steps/Sec: 0.91 + 49%|█████████████████████████████████████████████████████████████▉ | 54399/110696 [16:38:21<13:47:19, 1.13it/s][2025-04-27 13:41:46] (step=0054400) Train Loss: 5.8464, Train Steps/Sec: 1.12 + 49%|█████████████████████████████████████████████████████████████▉ | 54424/110696 [16:38:43<14:14:03, 1.10it/s][2025-04-27 13:42:09] (step=0054425) Train Loss: 5.7792, Train Steps/Sec: 1.12 + 49%|█████████████████████████████████████████████████████████████▉ | 54449/110696 [16:39:05<14:02:15, 1.11it/s][2025-04-27 13:42:31] (step=0054450) Train Loss: 5.7713, Train Steps/Sec: 1.12 + 49%|██████████████████████████████████████████████████████████████ | 54474/110696 [16:39:28<13:57:34, 1.12it/s][2025-04-27 13:42:53] (step=0054475) Train Loss: 5.7977, Train Steps/Sec: 1.12 + 49%|██████████████████████████████████████████████████████████████ | 54499/110696 [16:39:50<13:57:11, 1.12it/s][2025-04-27 13:43:15] (step=0054500) Train Loss: 5.7966, Train Steps/Sec: 1.12 + 49%|██████████████████████████████████████████████████████████████ | 54524/110696 [16:40:12<13:49:32, 1.13it/s][2025-04-27 13:43:38] (step=0054525) Train Loss: 5.7542, Train Steps/Sec: 1.12 + 49%|██████████████████████████████████████████████████████████████ | 54549/110696 [16:40:35<13:48:41, 1.13it/s][2025-04-27 13:44:00] (step=0054550) Train Loss: 5.8237, Train Steps/Sec: 1.12 + 49%|██████████████████████████████████████████████████████████████ | 54574/110696 [16:40:57<13:50:32, 1.13it/s][2025-04-27 13:44:22] (step=0054575) Train Loss: 5.8258, Train Steps/Sec: 1.12 + 49%|██████████████████████████████████████████████████████████████▏ | 54599/110696 [16:41:19<13:49:20, 1.13it/s][2025-04-27 13:44:45] (step=0054600) Train Loss: 5.8108, Train Steps/Sec: 1.11 + 49%|██████████████████████████████████████████████████████████████▏ | 54624/110696 [16:41:42<14:12:37, 1.10it/s][2025-04-27 13:45:07] (step=0054625) Train Loss: 5.7913, Train Steps/Sec: 1.12 + 49%|██████████████████████████████████████████████████████████████▏ | 54649/110696 [16:42:04<14:01:09, 1.11it/s][2025-04-27 13:45:29] (step=0054650) Train Loss: 5.8631, Train Steps/Sec: 1.12 + 49%|██████████████████████████████████████████████████████████████▏ | 54674/110696 [16:42:26<13:54:33, 1.12it/s][2025-04-27 13:45:52] (step=0054675) Train Loss: 5.8213, Train Steps/Sec: 1.12 + 49%|██████████████████████████████████████████████████████████████▎ | 54699/110696 [16:42:49<13:53:40, 1.12it/s][2025-04-27 13:46:14] (step=0054700) Train Loss: 5.8105, Train Steps/Sec: 1.12 + 49%|██████████████████████████████████████████████████████████████▎ | 54724/110696 [16:43:11<13:50:25, 1.12it/s][2025-04-27 13:46:36] (step=0054725) Train Loss: 5.7459, Train Steps/Sec: 1.12 + 49%|██████████████████████████████████████████████████████████████▎ | 54749/110696 [16:43:33<13:49:22, 1.12it/s][2025-04-27 13:46:59] (step=0054750) Train Loss: 5.7718, Train Steps/Sec: 1.12 + 49%|██████████████████████████████████████████████████████████████▎ | 54774/110696 [16:43:56<13:45:47, 1.13it/s][2025-04-27 13:47:21] (step=0054775) Train Loss: 5.8489, Train Steps/Sec: 1.12 + 50%|██████████████████████████████████████████████████████████████▍ | 54799/110696 [16:44:18<13:45:57, 1.13it/s][2025-04-27 13:47:43] (step=0054800) Train Loss: 5.8394, Train Steps/Sec: 1.11 + 50%|██████████████████████████████████████████████████████████████▍ | 54824/110696 [16:44:40<14:07:14, 1.10it/s][2025-04-27 13:48:06] (step=0054825) Train Loss: 5.7946, Train Steps/Sec: 1.12 + 50%|██████████████████████████████████████████████████████████████▍ | 54849/110696 [16:45:03<13:59:56, 1.11it/s][2025-04-27 13:48:28] (step=0054850) Train Loss: 5.8388, Train Steps/Sec: 1.12 + 50%|██████████████████████████████████████████████████████████████▍ | 54874/110696 [16:45:25<13:53:59, 1.12it/s][2025-04-27 13:48:50] (step=0054875) Train Loss: 5.8450, Train Steps/Sec: 1.12 + 50%|██████████████████████████████████████████████████████████████▍ | 54899/110696 [16:45:47<13:49:12, 1.12it/s][2025-04-27 13:49:13] (step=0054900) Train Loss: 5.8255, Train Steps/Sec: 1.12 + 50%|██████████████████████████████████████████████████████████████▌ | 54924/110696 [16:46:10<13:48:40, 1.12it/s][2025-04-27 13:49:35] (step=0054925) Train Loss: 5.8241, Train Steps/Sec: 1.12 + 50%|██████████████████████████████████████████████████████████████▌ | 54949/110696 [16:46:32<13:45:54, 1.12it/s][2025-04-27 13:49:57] (step=0054950) Train Loss: 5.8129, Train Steps/Sec: 1.12 + 50%|██████████████████████████████████████████████████████████████▌ | 54974/110696 [16:46:54<13:40:37, 1.13it/s][2025-04-27 13:50:20] (step=0054975) Train Loss: 5.7316, Train Steps/Sec: 1.12 + 50%|██████████████████████████████████████████████████████████████▌ | 54999/110696 [16:47:17<13:47:45, 1.12it/s][2025-04-27 13:50:42] (step=0055000) Train Loss: 5.8142, Train Steps/Sec: 1.11 + 50%|██████████████████████████████████████████████████████████████▋ | 55024/110696 [16:47:39<14:07:55, 1.09it/s][2025-04-27 13:51:04] (step=0055025) Train Loss: 5.8332, Train Steps/Sec: 1.12 + 50%|██████████████████████████████████████████████████████████████▋ | 55049/110696 [16:48:01<13:57:41, 1.11it/s][2025-04-27 13:51:27] (step=0055050) Train Loss: 5.7974, Train Steps/Sec: 1.12 + 50%|██████████████████████████████████████████████████████████████▋ | 55074/110696 [16:48:24<13:50:47, 1.12it/s][2025-04-27 13:51:49] (step=0055075) Train Loss: 5.7856, Train Steps/Sec: 1.12 + 50%|██████████████████████████████████████████████████████████████▋ | 55099/110696 [16:48:46<13:44:31, 1.12it/s][2025-04-27 13:52:11] (step=0055100) Train Loss: 5.7918, Train Steps/Sec: 1.12 + 50%|██████████████████████████████████████████████████████████████▋ | 55124/110696 [16:49:08<13:43:32, 1.12it/s][2025-04-27 13:52:34] (step=0055125) Train Loss: 5.8146, Train Steps/Sec: 1.12 + 50%|██████████████████████████████████████████████████████████████▊ | 55149/110696 [16:49:31<13:41:23, 1.13it/s][2025-04-27 13:52:56] (step=0055150) Train Loss: 5.6463, Train Steps/Sec: 1.12 + 50%|██████████████████████████████████████████████████████████████▊ | 55174/110696 [16:49:53<13:37:54, 1.13it/s][2025-04-27 13:53:18] (step=0055175) Train Loss: 5.8319, Train Steps/Sec: 1.12 + 50%|██████████████████████████████████████████████████████████████▊ | 55199/110696 [16:50:15<13:38:58, 1.13it/s][2025-04-27 13:53:41] (step=0055200) Train Loss: 5.7919, Train Steps/Sec: 1.12 + 50%|██████████████████████████████████████████████████████████████▊ | 55224/110696 [16:50:38<14:05:39, 1.09it/s][2025-04-27 13:54:03] (step=0055225) Train Loss: 5.7622, Train Steps/Sec: 1.12 + 50%|██████████████████████████████████████████████████████████████▉ | 55249/110696 [16:51:00<13:51:22, 1.11it/s][2025-04-27 13:54:25] (step=0055250) Train Loss: 5.7679, Train Steps/Sec: 1.12 + 50%|██████████████████████████████████████████████████████████████▉ | 55274/110696 [16:51:22<13:49:12, 1.11it/s][2025-04-27 13:54:48] (step=0055275) Train Loss: 5.7535, Train Steps/Sec: 1.12 + 50%|██████████████████████████████████████████████████████████████▉ | 55299/110696 [16:51:45<13:44:44, 1.12it/s][2025-04-27 13:55:10] (step=0055300) Train Loss: 5.7608, Train Steps/Sec: 1.12 + 50%|██████████████████████████████████████████████████████████████▉ | 55324/110696 [16:52:07<13:37:51, 1.13it/s][2025-04-27 13:55:32] (step=0055325) Train Loss: 5.8030, Train Steps/Sec: 1.12 + 50%|███████████████████████████████████████████████████████████████ | 55349/110696 [16:52:29<13:38:28, 1.13it/s][2025-04-27 13:55:55] (step=0055350) Train Loss: 5.8261, Train Steps/Sec: 1.12 + 50%|███████████████████████████████████████████████████████████████ | 55374/110696 [16:52:51<13:34:32, 1.13it/s][2025-04-27 13:56:17] (step=0055375) Train Loss: 5.8021, Train Steps/Sec: 1.12 + 50%|███████████████████████████████████████████████████████████████ | 55399/110696 [16:53:14<13:33:31, 1.13it/s][2025-04-27 13:56:39] (step=0055400) Train Loss: 5.7660, Train Steps/Sec: 1.12 + 50%|███████████████████████████████████████████████████████████████ | 55424/110696 [16:53:36<14:01:35, 1.09it/s][2025-04-27 13:57:02] (step=0055425) Train Loss: 5.7797, Train Steps/Sec: 1.12 + 50%|███████████████████████████████████████████████████████████████ | 55449/110696 [16:53:59<13:52:12, 1.11it/s][2025-04-27 13:57:24] (step=0055450) Train Loss: 5.7545, Train Steps/Sec: 1.12 + 50%|███████████████████████████████████████████████████████████████▏ | 55474/110696 [16:54:21<13:42:46, 1.12it/s][2025-04-27 13:57:46] (step=0055475) Train Loss: 5.8011, Train Steps/Sec: 1.12 + 50%|███████████████████████████████████████████████████████████████▏ | 55499/110696 [16:54:43<13:44:31, 1.12it/s][2025-04-27 13:58:09] (step=0055500) Train Loss: 5.7928, Train Steps/Sec: 1.12 + 50%|███████████████████████████████████████████████████████████████▏ | 55524/110696 [16:55:05<13:35:18, 1.13it/s][2025-04-27 13:58:31] (step=0055525) Train Loss: 5.7790, Train Steps/Sec: 1.12 + 50%|███████████████████████████████████████████████████████████████▏ | 55549/110696 [16:55:28<13:34:38, 1.13it/s][2025-04-27 13:58:53] (step=0055550) Train Loss: 5.7472, Train Steps/Sec: 1.12 + 50%|███████████████████████████████████████████████████████████████▎ | 55574/110696 [16:55:50<13:34:20, 1.13it/s][2025-04-27 13:59:15] (step=0055575) Train Loss: 5.7803, Train Steps/Sec: 1.12 + 50%|███████████████████████████████████████████████████████████████▎ | 55599/110696 [16:56:12<13:33:22, 1.13it/s][2025-04-27 13:59:38] (step=0055600) Train Loss: 5.8024, Train Steps/Sec: 1.11 + 50%|███████████████████████████████████████████████████████████████▎ | 55624/110696 [16:56:35<13:53:32, 1.10it/s][2025-04-27 14:00:00] (step=0055625) Train Loss: 5.7645, Train Steps/Sec: 1.12 + 50%|███████████████████████████████████████████████████████████████▎ | 55649/110696 [16:56:57<13:51:36, 1.10it/s][2025-04-27 14:00:22] (step=0055650) Train Loss: 5.8414, Train Steps/Sec: 1.12 + 50%|███████████████████████████████████████████████████████████████▎ | 55674/110696 [16:57:19<13:40:36, 1.12it/s][2025-04-27 14:00:45] (step=0055675) Train Loss: 5.7996, Train Steps/Sec: 1.12 + 50%|███████████████████████████████████████████████████████████████▍ | 55699/110696 [16:57:42<13:38:48, 1.12it/s][2025-04-27 14:01:07] (step=0055700) Train Loss: 5.7602, Train Steps/Sec: 1.12 + 50%|███████████████████████████████████████████████████████████████▍ | 55724/110696 [16:58:04<13:30:25, 1.13it/s][2025-04-27 14:01:29] (step=0055725) Train Loss: 5.7983, Train Steps/Sec: 1.12 + 50%|███████████████████████████████████████████████████████████████▍ | 55749/110696 [16:58:26<13:32:17, 1.13it/s][2025-04-27 14:01:52] (step=0055750) Train Loss: 5.8281, Train Steps/Sec: 1.12 + 50%|███████████████████████████████████████████████████████████████▍ | 55774/110696 [16:58:49<13:29:00, 1.13it/s][2025-04-27 14:02:14] (step=0055775) Train Loss: 5.8336, Train Steps/Sec: 1.12 + 50%|███████████████████████████████████████████████████████████████▌ | 55799/110696 [16:59:11<13:28:04, 1.13it/s][2025-04-27 14:02:36] (step=0055800) Train Loss: 5.7761, Train Steps/Sec: 1.12 + 50%|███████████████████████████████████████████████████████████████▌ | 55824/110696 [16:59:33<13:51:34, 1.10it/s][2025-04-27 14:02:59] (step=0055825) Train Loss: 5.7860, Train Steps/Sec: 1.12 + 50%|███████████████████████████████████████████████████████████████▌ | 55849/110696 [16:59:56<13:43:45, 1.11it/s][2025-04-27 14:03:21] (step=0055850) Train Loss: 5.8751, Train Steps/Sec: 1.12 + 50%|███████████████████████████████████████████████████████████████▌ | 55874/110696 [17:00:18<13:35:45, 1.12it/s][2025-04-27 14:03:43] (step=0055875) Train Loss: 5.8026, Train Steps/Sec: 1.12 + 50%|███████████████████████████████████████████████████████████████▋ | 55899/110696 [17:00:40<13:32:32, 1.12it/s][2025-04-27 14:04:06] (step=0055900) Train Loss: 5.7985, Train Steps/Sec: 1.12 + 51%|███████████████████████████████████████████████████████████████▋ | 55924/110696 [17:01:03<13:33:54, 1.12it/s][2025-04-27 14:04:28] (step=0055925) Train Loss: 5.7922, Train Steps/Sec: 1.12 + 51%|███████████████████████████████████████████████████████████████▋ | 55949/110696 [17:01:25<13:28:04, 1.13it/s][2025-04-27 14:04:50] (step=0055950) Train Loss: 5.8307, Train Steps/Sec: 1.12 + 51%|███████████████████████████████████████████████████████████████▋ | 55974/110696 [17:01:47<13:24:20, 1.13it/s][2025-04-27 14:05:13] (step=0055975) Train Loss: 5.8380, Train Steps/Sec: 1.12 + 51%|███████████████████████████████████████████████████████████████▋ | 55999/110696 [17:02:09<13:26:04, 1.13it/s][2025-04-27 14:05:35] (step=0056000) Train Loss: 5.7695, Train Steps/Sec: 1.12 +You are using a model of type blip-2 to instantiate a model of type instructblip. This is not supported for all configurations of models and can yield errors. +[2025-04-27 14:05:35] text_config is None. Initializing the text config with default values (`OPTConfig`). +model_name_or_path:/tmp/haozhezhao/model/blip2-flan-t5-xl +image_place_holder: +tokenizer length after expend 32101 +tokenizer length before expend 32100 +100%|████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████| 5/5 [04:55<00:00, 59.03s/it] +[2025-04-27 14:11:43] Finish Eval in 56000 steps...█████████████████████████████████████████████████████████████████████████████████████████████| 5/5 [04:54<00:00, 58.80s/it] +processing left-padding... +processing left-padding... +processing left-padding... +processing left-padding... +processing left-padding... +[2025-04-27 14:12:02] Saved checkpoint to checkpoint/X2I_UltraEdiit_fourmask_LLava_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_filtered_objects_4mask_1280_context/003-GPT-XL/checkpoints/0056000.pt +[2025-04-27 14:12:04] Removed old checkpoint: checkpoint/X2I_UltraEdiit_fourmask_LLava_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_filtered_objects_4mask_1280_context/003-GPT-XL/checkpoints/0054000.pt + 51%|███████████████████████████████████████████████████████████████▊ | 56024/110696 [17:09:01<14:07:45, 1.07it/s][2025-04-27 14:12:26] (step=0056025) Train Loss: 5.8289, Train Steps/Sec: 0.06 + 51%|███████████████████████████████████████████████████████████████▊ | 56049/110696 [17:09:24<13:37:40, 1.11it/s][2025-04-27 14:12:49] (step=0056050) Train Loss: 5.8056, Train Steps/Sec: 1.11 + 51%|███████████████████████████████████████████████████████████████▊ | 56074/110696 [17:09:46<13:34:53, 1.12it/s][2025-04-27 14:13:11] (step=0056075) Train Loss: 5.8062, Train Steps/Sec: 1.12 + 51%|███████████████████████████████████████████████████████████████▊ | 56099/110696 [17:10:08<13:30:14, 1.12it/s][2025-04-27 14:13:34] (step=0056100) Train Loss: 5.8100, Train Steps/Sec: 1.12 + 51%|███████████████████████████████████████████████████████████████▉ | 56124/110696 [17:10:30<13:28:01, 1.13it/s][2025-04-27 14:13:56] (step=0056125) Train Loss: 5.7554, Train Steps/Sec: 1.12 + 51%|███████████████████████████████████████████████████████████████▉ | 56149/110696 [17:10:53<13:24:07, 1.13it/s][2025-04-27 14:14:18] (step=0056150) Train Loss: 5.6926, Train Steps/Sec: 1.12 + 51%|███████████████████████████████████████████████████████████████▉ | 56174/110696 [17:11:15<13:26:07, 1.13it/s][2025-04-27 14:14:40] (step=0056175) Train Loss: 5.7688, Train Steps/Sec: 1.12 + 51%|███████████████████████████████████████████████████████████████▉ | 56199/110696 [17:11:37<13:20:40, 1.13it/s][2025-04-27 14:15:03] (step=0056200) Train Loss: 5.7003, Train Steps/Sec: 1.12 + 51%|███████████████████████████████████████████████████████████████▉ | 56224/110696 [17:12:00<13:44:59, 1.10it/s][2025-04-27 14:15:25] (step=0056225) Train Loss: 5.7772, Train Steps/Sec: 1.12 + 51%|████████████████████████████████████████████████████████████████ | 56249/110696 [17:12:22<13:35:24, 1.11it/s][2025-04-27 14:15:47] (step=0056250) Train Loss: 5.8121, Train Steps/Sec: 1.12 + 51%|████████████████████████████████████████████████████████████████ | 56274/110696 [17:12:44<13:29:03, 1.12it/s][2025-04-27 14:16:10] (step=0056275) Train Loss: 5.7703, Train Steps/Sec: 1.12 + 51%|████████████████████████████████████████████████████████████████ | 56299/110696 [17:13:06<13:28:19, 1.12it/s][2025-04-27 14:16:32] (step=0056300) Train Loss: 5.7627, Train Steps/Sec: 1.12 + 51%|████████████████████████████████████████████████████████████████ | 56324/110696 [17:13:29<13:25:05, 1.13it/s][2025-04-27 14:16:54] (step=0056325) Train Loss: 5.7750, Train Steps/Sec: 1.12 + 51%|████████████████████████████████████████████████████████████████▏ | 56349/110696 [17:13:51<13:20:26, 1.13it/s][2025-04-27 14:17:16] (step=0056350) Train Loss: 5.7984, Train Steps/Sec: 1.12 + 51%|████████████████████████████████████████████████████████████████▏ | 56374/110696 [17:14:13<13:18:51, 1.13it/s][2025-04-27 14:17:39] (step=0056375) Train Loss: 5.8157, Train Steps/Sec: 1.12 + 51%|████████████████████████████████████████████████████████████████▏ | 56399/110696 [17:14:36<13:22:48, 1.13it/s][2025-04-27 14:18:01] (step=0056400) Train Loss: 5.7628, Train Steps/Sec: 1.12 + 51%|████████████████████████████████████████████████████████████████▏ | 56424/110696 [17:14:58<13:44:35, 1.10it/s][2025-04-27 14:18:23] (step=0056425) Train Loss: 5.7460, Train Steps/Sec: 1.12 + 51%|████████████████████████████████████████████████████████████████▎ | 56449/110696 [17:15:20<13:36:18, 1.11it/s][2025-04-27 14:18:46] (step=0056450) Train Loss: 5.8653, Train Steps/Sec: 1.12 + 51%|████████████████████████████████████████████████████████████████▎ | 56474/110696 [17:15:43<13:27:22, 1.12it/s][2025-04-27 14:19:08] (step=0056475) Train Loss: 5.7945, Train Steps/Sec: 1.12 + 51%|████████████████████████████████████████████████████████████████▎ | 56499/110696 [17:16:05<13:23:06, 1.12it/s][2025-04-27 14:19:30] (step=0056500) Train Loss: 5.7887, Train Steps/Sec: 1.12 + 51%|████████████████████████████████████████████████████████████████▎ | 56524/110696 [17:16:27<13:19:57, 1.13it/s][2025-04-27 14:19:52] (step=0056525) Train Loss: 5.7188, Train Steps/Sec: 1.12 + 51%|████████████████████████████████████████████████████████████████▎ | 56549/110696 [17:16:49<13:22:24, 1.12it/s][2025-04-27 14:20:15] (step=0056550) Train Loss: 5.8649, Train Steps/Sec: 1.12 + 51%|████████████████████████████████████████████████████████████████▍ | 56574/110696 [17:17:12<13:17:07, 1.13it/s][2025-04-27 14:20:37] (step=0056575) Train Loss: 5.7891, Train Steps/Sec: 1.12 + 51%|████████████████████████████████████████████████████████████████▍ | 56599/110696 [17:17:34<13:14:33, 1.13it/s][2025-04-27 14:20:59] (step=0056600) Train Loss: 5.8471, Train Steps/Sec: 1.12 + 51%|████████████████████████████████████████████████████████████████▍ | 56624/110696 [17:17:56<13:40:04, 1.10it/s][2025-04-27 14:21:22] (step=0056625) Train Loss: 5.7715, Train Steps/Sec: 1.12 + 51%|████████████████████████████████████████████████████████████████▍ | 56649/110696 [17:18:19<13:30:24, 1.11it/s][2025-04-27 14:21:44] (step=0056650) Train Loss: 5.8381, Train Steps/Sec: 1.12 + 51%|████████████████████████████████████████████████████████████████▌ | 56674/110696 [17:18:41<13:23:47, 1.12it/s][2025-04-27 14:22:06] (step=0056675) Train Loss: 5.7744, Train Steps/Sec: 1.12 + 51%|████████████████████████████████████████████████████████████████▌ | 56699/110696 [17:19:03<13:22:01, 1.12it/s][2025-04-27 14:22:29] (step=0056700) Train Loss: 5.7159, Train Steps/Sec: 1.12 + 51%|████████████████████████████████████████████████████████████████▌ | 56724/110696 [17:19:25<13:17:44, 1.13it/s][2025-04-27 14:22:51] (step=0056725) Train Loss: 5.7230, Train Steps/Sec: 1.12 + 51%|████████████████████████████████████████████████████████████████▌ | 56749/110696 [17:19:48<13:21:16, 1.12it/s][2025-04-27 14:23:13] (step=0056750) Train Loss: 5.8204, Train Steps/Sec: 1.12 + 51%|████████████████████████████████████████████████████████████████▌ | 56774/110696 [17:20:10<13:14:40, 1.13it/s][2025-04-27 14:23:36] (step=0056775) Train Loss: 5.7692, Train Steps/Sec: 1.12 + 51%|████████████████████████████████████████████████████████████████▋ | 56799/110696 [17:20:32<13:17:45, 1.13it/s][2025-04-27 14:23:58] (step=0056800) Train Loss: 5.8437, Train Steps/Sec: 1.12 + 51%|████████████████████████████████████████████████████████████████▋ | 56824/110696 [17:20:55<13:41:20, 1.09it/s][2025-04-27 14:24:20] (step=0056825) Train Loss: 5.8385, Train Steps/Sec: 1.12 + 51%|████████████████████████████████████████████████████████████████▋ | 56849/110696 [17:21:17<13:24:55, 1.11it/s][2025-04-27 14:24:43] (step=0056850) Train Loss: 5.7802, Train Steps/Sec: 1.12 + 51%|████████████████████████████████████████████████████████████████▋ | 56874/110696 [17:21:39<13:21:42, 1.12it/s][2025-04-27 14:25:05] (step=0056875) Train Loss: 5.8371, Train Steps/Sec: 1.12 + 51%|████████████████████████████████████████████████████████████████▊ | 56899/110696 [17:22:02<13:20:05, 1.12it/s][2025-04-27 14:25:27] (step=0056900) Train Loss: 5.8682, Train Steps/Sec: 1.12 + 51%|████████████████████████████████████████████████████████████████▊ | 56924/110696 [17:22:24<13:16:09, 1.13it/s][2025-04-27 14:25:49] (step=0056925) Train Loss: 5.7540, Train Steps/Sec: 1.12 + 51%|████████████████████████████████████████████████████████████████▊ | 56949/110696 [17:22:46<13:12:26, 1.13it/s][2025-04-27 14:26:12] (step=0056950) Train Loss: 5.7955, Train Steps/Sec: 1.12 + 51%|████████████████████████████████████████████████████████████████▊ | 56974/110696 [17:23:09<13:09:59, 1.13it/s][2025-04-27 14:26:34] (step=0056975) Train Loss: 5.7868, Train Steps/Sec: 1.12 + 51%|████████████████████████████████████████████████████████████████▉ | 56999/110696 [17:23:31<13:10:20, 1.13it/s][2025-04-27 14:26:56] (step=0057000) Train Loss: 5.8225, Train Steps/Sec: 1.12 + 52%|████████████████████████████████████████████████████████████████▉ | 57024/110696 [17:23:53<13:35:45, 1.10it/s][2025-04-27 14:27:19] (step=0057025) Train Loss: 5.7458, Train Steps/Sec: 1.12 + 52%|████████████████████████████████████████████████████████████████▉ | 57049/110696 [17:24:16<13:23:53, 1.11it/s][2025-04-27 14:27:41] (step=0057050) Train Loss: 5.7793, Train Steps/Sec: 1.12 + 52%|████████████████████████████████████████████████████████████████▉ | 57074/110696 [17:24:38<13:19:31, 1.12it/s][2025-04-27 14:28:03] (step=0057075) Train Loss: 5.8635, Train Steps/Sec: 1.12 + 52%|████████████████████████████████████████████████████████████████▉ | 57099/110696 [17:25:00<13:13:39, 1.13it/s][2025-04-27 14:28:26] (step=0057100) Train Loss: 5.7832, Train Steps/Sec: 1.12 + 52%|█████████████████████████████████████████████████████████████████ | 57124/110696 [17:25:22<13:14:07, 1.12it/s][2025-04-27 14:28:48] (step=0057125) Train Loss: 5.8243, Train Steps/Sec: 1.12 + 52%|█████████████████████████████████████████████████████████████████ | 57149/110696 [17:25:45<13:08:57, 1.13it/s][2025-04-27 14:29:10] (step=0057150) Train Loss: 5.7258, Train Steps/Sec: 1.12 + 52%|█████████████████████████████████████████████████████████████████ | 57174/110696 [17:26:07<13:08:21, 1.13it/s][2025-04-27 14:29:32] (step=0057175) Train Loss: 5.7780, Train Steps/Sec: 1.12 + 52%|█████████████████████████████████████████████████████████████████ | 57199/110696 [17:26:29<13:09:43, 1.13it/s][2025-04-27 14:29:55] (step=0057200) Train Loss: 5.7611, Train Steps/Sec: 1.12 + 52%|█████████████████████████████████████████████████████████████████▏ | 57224/110696 [17:26:52<13:31:57, 1.10it/s][2025-04-27 14:30:17] (step=0057225) Train Loss: 5.8122, Train Steps/Sec: 1.12 + 52%|█████████████████████████████████████████████████████████████████▏ | 57249/110696 [17:27:14<13:22:08, 1.11it/s][2025-04-27 14:30:39] (step=0057250) Train Loss: 5.8352, Train Steps/Sec: 1.12 + 52%|█████████████████████████████████████████████████████████████████▏ | 57274/110696 [17:27:36<13:18:58, 1.11it/s][2025-04-27 14:31:02] (step=0057275) Train Loss: 5.8399, Train Steps/Sec: 1.12 + 52%|█████████████████████████████████████████████████████████████████▏ | 57299/110696 [17:27:59<13:16:10, 1.12it/s][2025-04-27 14:31:24] (step=0057300) Train Loss: 5.8726, Train Steps/Sec: 1.12 + 52%|█████████████████████████████████████████████████████████████████▏ | 57324/110696 [17:28:21<13:10:41, 1.13it/s][2025-04-27 14:31:46] (step=0057325) Train Loss: 5.8442, Train Steps/Sec: 1.12 + 52%|█████████████████████████████████████████████████████████████████▎ | 57349/110696 [17:28:43<13:07:35, 1.13it/s][2025-04-27 14:32:09] (step=0057350) Train Loss: 5.8296, Train Steps/Sec: 1.12 + 52%|█████████████████████████████████████████████████████████████████▎ | 57374/110696 [17:29:06<13:03:24, 1.13it/s][2025-04-27 14:32:31] (step=0057375) Train Loss: 5.8085, Train Steps/Sec: 1.12 + 52%|█████████████████████████████████████████████████████████████████▎ | 57399/110696 [17:29:28<13:03:31, 1.13it/s][2025-04-27 14:32:53] (step=0057400) Train Loss: 5.8122, Train Steps/Sec: 1.12 + 52%|█████████████████████████████████████████████████████████████████▎ | 57424/110696 [17:29:50<13:23:34, 1.10it/s][2025-04-27 14:33:15] (step=0057425) Train Loss: 5.7464, Train Steps/Sec: 1.13 + 52%|█████████████████████████████████████████████████████████████████▍ | 57449/110696 [17:30:12<13:16:26, 1.11it/s][2025-04-27 14:33:38] (step=0057450) Train Loss: 5.7569, Train Steps/Sec: 1.12 + 52%|█████████████████████████████████████████████████████████████████▍ | 57474/110696 [17:30:35<13:12:21, 1.12it/s][2025-04-27 14:34:00] (step=0057475) Train Loss: 5.7616, Train Steps/Sec: 1.12 + 52%|█████████████████████████████████████████████████████████████████▍ | 57499/110696 [17:30:57<13:09:20, 1.12it/s][2025-04-27 14:34:22] (step=0057500) Train Loss: 5.7575, Train Steps/Sec: 1.12 + 52%|█████████████████████████████████████████████████████████████████▍ | 57524/110696 [17:31:19<13:05:17, 1.13it/s][2025-04-27 14:34:45] (step=0057525) Train Loss: 5.7522, Train Steps/Sec: 1.12 + 52%|█████████████████████████████████████████████████████████████████▌ | 57549/110696 [17:31:41<13:03:35, 1.13it/s][2025-04-27 14:35:07] (step=0057550) Train Loss: 5.8239, Train Steps/Sec: 1.13 + 52%|█████████████████████████████████████████████████████████████████▌ | 57574/110696 [17:32:04<13:05:56, 1.13it/s][2025-04-27 14:35:29] (step=0057575) Train Loss: 5.8705, Train Steps/Sec: 1.12 + 52%|█████████████████████████████████████████████████████████████████▌ | 57599/110696 [17:32:26<13:00:04, 1.13it/s][2025-04-27 14:35:51] (step=0057600) Train Loss: 5.7653, Train Steps/Sec: 1.12 + 52%|█████████████████████████████████████████████████████████████████▌ | 57624/110696 [17:32:48<13:25:42, 1.10it/s][2025-04-27 14:36:14] (step=0057625) Train Loss: 5.7647, Train Steps/Sec: 1.12 + 52%|█████████████████████████████████████████████████████████████████▌ | 57649/110696 [17:33:11<13:15:12, 1.11it/s][2025-04-27 14:36:36] (step=0057650) Train Loss: 5.7852, Train Steps/Sec: 1.12 + 52%|█████████████████████████████████████████████████████████████████▋ | 57674/110696 [17:33:33<13:12:01, 1.12it/s][2025-04-27 14:36:58] (step=0057675) Train Loss: 5.7867, Train Steps/Sec: 1.12 + 52%|█████████████████████████████████████████████████████████████████▋ | 57699/110696 [17:33:55<13:09:13, 1.12it/s][2025-04-27 14:37:21] (step=0057700) Train Loss: 5.7638, Train Steps/Sec: 1.12 + 52%|█████████████████████████████████████████████████████████████████▋ | 57724/110696 [17:34:18<13:03:34, 1.13it/s][2025-04-27 14:37:43] (step=0057725) Train Loss: 5.7539, Train Steps/Sec: 1.12 + 52%|█████████████████████████████████████████████████████████████████▋ | 57749/110696 [17:34:40<13:01:10, 1.13it/s][2025-04-27 14:38:05] (step=0057750) Train Loss: 5.8046, Train Steps/Sec: 1.12 + 52%|█████████████████████████████████████████████████████████████████▊ | 57774/110696 [17:35:02<12:57:35, 1.13it/s][2025-04-27 14:38:28] (step=0057775) Train Loss: 5.8314, Train Steps/Sec: 1.12 + 52%|█████████████████████████████████████████████████████████████████▊ | 57799/110696 [17:35:24<12:57:44, 1.13it/s][2025-04-27 14:38:50] (step=0057800) Train Loss: 5.7602, Train Steps/Sec: 1.12 + 52%|█████████████████████████████████████████████████████████████████▊ | 57824/110696 [17:35:47<13:23:54, 1.10it/s][2025-04-27 14:39:12] (step=0057825) Train Loss: 5.7690, Train Steps/Sec: 1.12 + 52%|█████████████████████████████████████████████████████████████████▊ | 57849/110696 [17:36:09<13:09:08, 1.12it/s][2025-04-27 14:39:34] (step=0057850) Train Loss: 5.8320, Train Steps/Sec: 1.12 + 52%|█████████████████████████████████████████████████████████████████▉ | 57874/110696 [17:36:31<13:06:15, 1.12it/s][2025-04-27 14:39:57] (step=0057875) Train Loss: 5.8325, Train Steps/Sec: 1.12 + 52%|█████████████████████████████████████████████████████████████████▉ | 57899/110696 [17:36:54<13:04:43, 1.12it/s][2025-04-27 14:40:19] (step=0057900) Train Loss: 5.8511, Train Steps/Sec: 1.12 + 52%|█████████████████████████████████████████████████████████████████▉ | 57924/110696 [17:37:16<12:58:55, 1.13it/s][2025-04-27 14:40:41] (step=0057925) Train Loss: 5.7939, Train Steps/Sec: 1.12 + 52%|█████████████████████████████████████████████████████████████████▉ | 57949/110696 [17:37:38<12:58:35, 1.13it/s][2025-04-27 14:41:04] (step=0057950) Train Loss: 5.7412, Train Steps/Sec: 1.12 + 52%|█████████████████████████████████████████████████████████████████▉ | 57974/110696 [17:38:01<12:55:23, 1.13it/s][2025-04-27 14:41:26] (step=0057975) Train Loss: 5.8578, Train Steps/Sec: 1.12 + 52%|██████████████████████████████████████████████████████████████████ | 57999/110696 [17:38:23<12:53:20, 1.14it/s][2025-04-27 14:41:48] (step=0058000) Train Loss: 5.7512, Train Steps/Sec: 1.12 +You are using a model of type blip-2 to instantiate a model of type instructblip. This is not supported for all configurations of models and can yield errors. +[2025-04-27 14:41:48] text_config is None. Initializing the text config with default values (`OPTConfig`). +model_name_or_path:/tmp/haozhezhao/model/blip2-flan-t5-xl +image_place_holder: +tokenizer length after expend 32101 +tokenizer length before expend 32100 +100%|████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████| 5/5 [04:55<00:00, 59.16s/it] +[2025-04-27 14:47:56] Finish Eval in 58000 steps...█████████████████████████████████████████████████████████████████████████████████████████████| 5/5 [04:55<00:00, 58.91s/it] +processing left-padding... +processing left-padding... +processing left-padding... +processing left-padding... +processing left-padding... +[2025-04-27 14:48:16] Saved checkpoint to checkpoint/X2I_UltraEdiit_fourmask_LLava_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_filtered_objects_4mask_1280_context/003-GPT-XL/checkpoints/0058000.pt +[2025-04-27 14:48:18] Removed old checkpoint: checkpoint/X2I_UltraEdiit_fourmask_LLava_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_filtered_objects_4mask_1280_context/003-GPT-XL/checkpoints/0056000.pt + 52%|██████████████████████████████████████████████████████████████████ | 58024/110696 [17:45:15<13:39:06, 1.07it/s][2025-04-27 14:48:40] (step=0058025) Train Loss: 5.8519, Train Steps/Sec: 0.06 + 52%|██████████████████████████████████████████████████████████████████ | 58049/110696 [17:45:37<13:11:22, 1.11it/s][2025-04-27 14:49:03] (step=0058050) Train Loss: 5.8059, Train Steps/Sec: 1.12 + 52%|██████████████████████████████████████████████████████████████████ | 58074/110696 [17:46:04<13:03:50, 1.12it/s][2025-04-27 14:49:29] (step=0058075) Train Loss: 5.7056, Train Steps/Sec: 0.93 + 52%|██████████████████████████████████████████████████████████████████▏ | 58099/110696 [17:46:26<13:00:43, 1.12it/s][2025-04-27 14:49:52] (step=0058100) Train Loss: 5.7506, Train Steps/Sec: 1.12 + 53%|██████████████████████████████████████████████████████████████████▏ | 58124/110696 [17:46:49<12:54:51, 1.13it/s][2025-04-27 14:50:14] (step=0058125) Train Loss: 5.8088, Train Steps/Sec: 1.12 + 53%|██████████████████████████████████████████████████████████████████▏ | 58149/110696 [17:47:11<12:55:58, 1.13it/s][2025-04-27 14:50:36] (step=0058150) Train Loss: 5.7521, Train Steps/Sec: 1.12 + 53%|██████████████████████████████████████████████████████████████████▏ | 58174/110696 [17:47:33<12:56:08, 1.13it/s][2025-04-27 14:50:59] (step=0058175) Train Loss: 5.7651, Train Steps/Sec: 1.12 + 53%|██████████████████████████████████████████████████████████████████▏ | 58199/110696 [17:48:00<16:19:27, 1.12s/it][2025-04-27 14:51:26] (step=0058200) Train Loss: 5.8518, Train Steps/Sec: 0.93 + 53%|██████████████████████████████████████████████████████████████████▎ | 58224/110696 [17:48:22<13:16:28, 1.10it/s][2025-04-27 14:51:48] (step=0058225) Train Loss: 5.7997, Train Steps/Sec: 1.12 + 53%|██████████████████████████████████████████████████████████████████▎ | 58249/110696 [17:48:45<13:13:00, 1.10it/s][2025-04-27 14:52:10] (step=0058250) Train Loss: 5.8301, Train Steps/Sec: 1.12 + 53%|██████████████████████████████████████████████████████████████████▎ | 58274/110696 [17:49:07<13:00:21, 1.12it/s][2025-04-27 14:52:32] (step=0058275) Train Loss: 5.7840, Train Steps/Sec: 1.12 + 53%|██████████████████████████████████████████████████████████████████▎ | 58299/110696 [17:49:29<12:55:19, 1.13it/s][2025-04-27 14:52:55] (step=0058300) Train Loss: 5.7047, Train Steps/Sec: 1.12 + 53%|██████████████████████████████████████████████████████████████████▍ | 58324/110696 [17:49:52<12:54:50, 1.13it/s][2025-04-27 14:53:17] (step=0058325) Train Loss: 5.7322, Train Steps/Sec: 1.12 + 53%|██████████████████████████████████████████████████████████████████▍ | 58349/110696 [17:50:14<12:51:44, 1.13it/s][2025-04-27 14:53:39] (step=0058350) Train Loss: 5.7718, Train Steps/Sec: 1.12 + 53%|██████████████████████████████████████████████████████████████████▍ | 58374/110696 [17:50:36<12:51:57, 1.13it/s][2025-04-27 14:54:02] (step=0058375) Train Loss: 5.7703, Train Steps/Sec: 1.12 + 53%|██████████████████████████████████████████████████████████████████▍ | 58399/110696 [17:50:59<12:47:29, 1.14it/s][2025-04-27 14:54:24] (step=0058400) Train Loss: 5.7163, Train Steps/Sec: 1.12 + 53%|██████████████████████████████████████████████████████████████████▌ | 58424/110696 [17:51:21<13:19:55, 1.09it/s][2025-04-27 14:54:46] (step=0058425) Train Loss: 5.8399, Train Steps/Sec: 1.12 + 53%|██████████████████████████████████████████████████████████████████▌ | 58449/110696 [17:51:43<13:07:53, 1.11it/s][2025-04-27 14:55:09] (step=0058450) Train Loss: 5.8523, Train Steps/Sec: 1.12 + 53%|██████████████████████████████████████████████████████████████████▌ | 58474/110696 [17:52:06<12:58:00, 1.12it/s][2025-04-27 14:55:31] (step=0058475) Train Loss: 5.8346, Train Steps/Sec: 1.12 + 53%|██████████████████████████████████████████████████████████████████▌ | 58499/110696 [17:52:28<12:55:40, 1.12it/s][2025-04-27 14:55:53] (step=0058500) Train Loss: 5.7876, Train Steps/Sec: 1.12 + 53%|██████████████████████████████████████████████████████████████████▌ | 58524/110696 [17:52:50<12:52:31, 1.13it/s][2025-04-27 14:56:16] (step=0058525) Train Loss: 5.7028, Train Steps/Sec: 1.12 + 53%|██████████████████████████████████████████████████████████████████▋ | 58549/110696 [17:53:13<12:52:04, 1.13it/s][2025-04-27 14:56:38] (step=0058550) Train Loss: 5.7430, Train Steps/Sec: 1.12 + 53%|██████████████████████████████████████████████████████████████████▋ | 58574/110696 [17:53:35<12:47:53, 1.13it/s][2025-04-27 14:57:00] (step=0058575) Train Loss: 5.7867, Train Steps/Sec: 1.12 + 53%|██████████████████████████████████████████████████████████████████▋ | 58599/110696 [17:53:58<13:14:05, 1.09it/s][2025-04-27 14:57:23] (step=0058600) Train Loss: 5.8207, Train Steps/Sec: 1.10 + 53%|██████████████████████████████████████████████████████████████████▋ | 58624/110696 [17:54:20<13:12:57, 1.09it/s][2025-04-27 14:57:51] (step=0058625) Train Loss: 5.7623, Train Steps/Sec: 0.91 + 53%|██████████████████████████████████████████████████████████████████▊ | 58649/110696 [17:54:56<14:28:30, 1.00s/it][2025-04-27 14:58:21] (step=0058650) Train Loss: 5.7390, Train Steps/Sec: 0.82 + 53%|██████████████████████████████████████████████████████████████████▊ | 58674/110696 [17:55:23<15:12:32, 1.05s/it][2025-04-27 14:58:48] (step=0058675) Train Loss: 5.8145, Train Steps/Sec: 0.93 + 53%|██████████████████████████████████████████████████████████████████▊ | 58699/110696 [17:55:50<12:58:37, 1.11it/s][2025-04-27 14:59:16] (step=0058700) Train Loss: 5.8313, Train Steps/Sec: 0.90 + 53%|██████████████████████████████████████████████████████████████████▊ | 58724/110696 [17:56:13<12:46:17, 1.13it/s][2025-04-27 14:59:38] (step=0058725) Train Loss: 5.7542, Train Steps/Sec: 1.12 + 53%|██████████████████████████████████████████████████████████████████▊ | 58749/110696 [17:56:35<12:45:38, 1.13it/s][2025-04-27 15:00:00] (step=0058750) Train Loss: 5.8093, Train Steps/Sec: 1.12 + 53%|██████████████████████████████████████████████████████████████████▉ | 58774/110696 [17:56:57<12:43:15, 1.13it/s][2025-04-27 15:00:23] (step=0058775) Train Loss: 5.7157, Train Steps/Sec: 1.12 + 53%|██████████████████████████████████████████████████████████████████▉ | 58799/110696 [17:57:20<12:44:35, 1.13it/s][2025-04-27 15:00:45] (step=0058800) Train Loss: 5.8387, Train Steps/Sec: 1.12 + 53%|██████████████████████████████████████████████████████████████████▉ | 58824/110696 [17:57:42<13:06:31, 1.10it/s][2025-04-27 15:01:07] (step=0058825) Train Loss: 5.7493, Train Steps/Sec: 1.12 + 53%|██████████████████████████████████████████████████████████████████▉ | 58849/110696 [17:58:04<12:59:50, 1.11it/s][2025-04-27 15:01:30] (step=0058850) Train Loss: 5.7528, Train Steps/Sec: 1.12 + 53%|███████████████████████████████████████████████████████████████████ | 58874/110696 [17:58:27<12:51:11, 1.12it/s][2025-04-27 15:01:52] (step=0058875) Train Loss: 5.7829, Train Steps/Sec: 1.12 + 53%|███████████████████████████████████████████████████████████████████ | 58899/110696 [17:58:49<12:49:03, 1.12it/s][2025-04-27 15:02:14] (step=0058900) Train Loss: 5.8513, Train Steps/Sec: 1.12 + 53%|███████████████████████████████████████████████████████████████████ | 58924/110696 [17:59:11<12:48:58, 1.12it/s][2025-04-27 15:02:37] (step=0058925) Train Loss: 5.7874, Train Steps/Sec: 1.12 + 53%|███████████████████████████████████████████████████████████████████ | 58949/110696 [17:59:34<12:44:48, 1.13it/s][2025-04-27 15:02:59] (step=0058950) Train Loss: 5.7713, Train Steps/Sec: 1.12 + 53%|███████████████████████████████████████████████████████████████████▏ | 58974/110696 [17:59:56<12:46:13, 1.13it/s][2025-04-27 15:03:21] (step=0058975) Train Loss: 5.7406, Train Steps/Sec: 1.12 + 53%|███████████████████████████████████████████████████████████████████▏ | 58999/110696 [18:00:18<12:42:24, 1.13it/s][2025-04-27 15:03:44] (step=0059000) Train Loss: 5.8598, Train Steps/Sec: 1.12 + 53%|███████████████████████████████████████████████████████████████████▏ | 59024/110696 [18:00:41<13:05:28, 1.10it/s][2025-04-27 15:04:06] (step=0059025) Train Loss: 5.7772, Train Steps/Sec: 1.12 + 53%|███████████████████████████████████████████████████████████████████▏ | 59049/110696 [18:01:03<12:56:08, 1.11it/s][2025-04-27 15:04:28] (step=0059050) Train Loss: 5.7474, Train Steps/Sec: 1.12 + 53%|███████████████████████████████████████████████████████████████████▏ | 59074/110696 [18:01:25<12:48:01, 1.12it/s][2025-04-27 15:04:51] (step=0059075) Train Loss: 5.7897, Train Steps/Sec: 1.12 + 53%|███████████████████████████████████████████████████████████████████▎ | 59099/110696 [18:01:48<12:46:29, 1.12it/s][2025-04-27 15:05:13] (step=0059100) Train Loss: 5.7803, Train Steps/Sec: 1.12 + 53%|███████████████████████████████████████████████████████████████████▎ | 59124/110696 [18:02:10<12:41:02, 1.13it/s][2025-04-27 15:05:35] (step=0059125) Train Loss: 5.7661, Train Steps/Sec: 1.12 + 53%|███████████████████████████████████████████████████████████████████▎ | 59149/110696 [18:02:32<12:41:39, 1.13it/s][2025-04-27 15:05:58] (step=0059150) Train Loss: 5.7059, Train Steps/Sec: 1.12 + 53%|███████████████████████████████████████████████████████████████████▎ | 59174/110696 [18:02:55<12:41:04, 1.13it/s][2025-04-27 15:06:20] (step=0059175) Train Loss: 5.6871, Train Steps/Sec: 1.12 + 53%|███████████████████████████████████████████████████████████████████▍ | 59199/110696 [18:03:17<12:42:34, 1.13it/s][2025-04-27 15:06:42] (step=0059200) Train Loss: 5.8231, Train Steps/Sec: 1.12 + 54%|███████████████████████████████████████████████████████████████████▍ | 59224/110696 [18:03:39<12:59:58, 1.10it/s][2025-04-27 15:07:05] (step=0059225) Train Loss: 5.7870, Train Steps/Sec: 1.12 + 54%|███████████████████████████████████████████████████████████████████▍ | 59249/110696 [18:04:02<12:54:07, 1.11it/s][2025-04-27 15:07:27] (step=0059250) Train Loss: 5.8231, Train Steps/Sec: 1.12 + 54%|███████████████████████████████████████████████████████████████████▍ | 59274/110696 [18:04:24<12:45:55, 1.12it/s][2025-04-27 15:07:49] (step=0059275) Train Loss: 5.7853, Train Steps/Sec: 1.12 + 54%|███████████████████████████████████████████████████████████████████▍ | 59299/110696 [18:04:46<12:47:11, 1.12it/s][2025-04-27 15:08:12] (step=0059300) Train Loss: 5.7373, Train Steps/Sec: 1.12 + 54%|███████████████████████████████████████████████████████████████████▌ | 59324/110696 [18:05:09<12:40:28, 1.13it/s][2025-04-27 15:08:34] (step=0059325) Train Loss: 5.8618, Train Steps/Sec: 1.12 + 54%|███████████████████████████████████████████████████████████████████▌ | 59349/110696 [18:05:31<12:37:07, 1.13it/s][2025-04-27 15:08:56] (step=0059350) Train Loss: 5.7357, Train Steps/Sec: 1.12 + 54%|███████████████████████████████████████████████████████████████████▌ | 59374/110696 [18:05:53<12:37:39, 1.13it/s][2025-04-27 15:09:19] (step=0059375) Train Loss: 5.7579, Train Steps/Sec: 1.12 + 54%|███████████████████████████████████████████████████████████████████▌ | 59399/110696 [18:06:16<12:36:02, 1.13it/s][2025-04-27 15:09:41] (step=0059400) Train Loss: 5.7611, Train Steps/Sec: 1.12 + 54%|███████████████████████████████████████████████████████████████████▋ | 59424/110696 [18:06:38<12:58:21, 1.10it/s][2025-04-27 15:10:03] (step=0059425) Train Loss: 5.7475, Train Steps/Sec: 1.12 + 54%|███████████████████████████████████████████████████████████████████▋ | 59449/110696 [18:07:00<12:50:37, 1.11it/s][2025-04-27 15:10:26] (step=0059450) Train Loss: 5.7525, Train Steps/Sec: 1.12 + 54%|███████████████████████████████████████████████████████████████████▋ | 59474/110696 [18:07:22<12:43:11, 1.12it/s][2025-04-27 15:10:48] (step=0059475) Train Loss: 5.7253, Train Steps/Sec: 1.12 + 54%|███████████████████████████████████████████████████████████████████▋ | 59499/110696 [18:07:45<12:42:45, 1.12it/s][2025-04-27 15:11:10] (step=0059500) Train Loss: 5.7642, Train Steps/Sec: 1.12 + 54%|███████████████████████████████████████████████████████████████████▊ | 59524/110696 [18:08:07<12:36:09, 1.13it/s][2025-04-27 15:11:33] (step=0059525) Train Loss: 5.7888, Train Steps/Sec: 1.12 + 54%|███████████████████████████████████████████████████████████████████▊ | 59549/110696 [18:08:29<12:34:41, 1.13it/s][2025-04-27 15:11:55] (step=0059550) Train Loss: 5.7348, Train Steps/Sec: 1.12 + 54%|███████████████████████████████████████████████████████████████████▊ | 59574/110696 [18:08:52<12:32:12, 1.13it/s][2025-04-27 15:12:17] (step=0059575) Train Loss: 5.7846, Train Steps/Sec: 1.12 + 54%|███████████████████████████████████████████████████████████████████▊ | 59599/110696 [18:09:14<12:31:05, 1.13it/s][2025-04-27 15:12:40] (step=0059600) Train Loss: 5.7790, Train Steps/Sec: 1.12 + 54%|███████████████████████████████████████████████████████████████████▊ | 59624/110696 [18:09:36<12:53:57, 1.10it/s][2025-04-27 15:13:02] (step=0059625) Train Loss: 5.8215, Train Steps/Sec: 1.12 + 54%|███████████████████████████████████████████████████████████████████▉ | 59649/110696 [18:09:59<12:43:19, 1.11it/s][2025-04-27 15:13:24] (step=0059650) Train Loss: 5.7836, Train Steps/Sec: 1.12 + 54%|███████████████████████████████████████████████████████████████████▉ | 59674/110696 [18:10:21<12:39:54, 1.12it/s][2025-04-27 15:13:46] (step=0059675) Train Loss: 5.7723, Train Steps/Sec: 1.12 + 54%|███████████████████████████████████████████████████████████████████▉ | 59699/110696 [18:10:43<12:36:26, 1.12it/s][2025-04-27 15:14:09] (step=0059700) Train Loss: 5.7902, Train Steps/Sec: 1.12 + 54%|███████████████████████████████████████████████████████████████████▉ | 59724/110696 [18:11:06<12:36:38, 1.12it/s][2025-04-27 15:14:31] (step=0059725) Train Loss: 5.7162, Train Steps/Sec: 1.12 + 54%|████████████████████████████████████████████████████████████████████ | 59749/110696 [18:11:28<12:32:56, 1.13it/s][2025-04-27 15:14:53] (step=0059750) Train Loss: 5.7036, Train Steps/Sec: 1.12 + 54%|████████████████████████████████████████████████████████████████████ | 59774/110696 [18:11:50<12:30:55, 1.13it/s][2025-04-27 15:15:16] (step=0059775) Train Loss: 5.7370, Train Steps/Sec: 1.12 + 54%|████████████████████████████████████████████████████████████████████ | 59799/110696 [18:12:13<12:28:20, 1.13it/s][2025-04-27 15:15:38] (step=0059800) Train Loss: 5.7596, Train Steps/Sec: 1.12 + 54%|████████████████████████████████████████████████████████████████████ | 59824/110696 [18:12:35<12:53:54, 1.10it/s][2025-04-27 15:16:00] (step=0059825) Train Loss: 5.7548, Train Steps/Sec: 1.12 + 54%|████████████████████████████████████████████████████████████████████ | 59849/110696 [18:12:57<12:43:16, 1.11it/s][2025-04-27 15:16:23] (step=0059850) Train Loss: 5.7396, Train Steps/Sec: 1.12 + 54%|████████████████████████████████████████████████████████████████████▏ | 59874/110696 [18:13:20<12:41:44, 1.11it/s][2025-04-27 15:16:45] (step=0059875) Train Loss: 5.7476, Train Steps/Sec: 1.12 + 54%|████████████████████████████████████████████████████████████████████▏ | 59899/110696 [18:13:42<12:33:40, 1.12it/s][2025-04-27 15:17:07] (step=0059900) Train Loss: 5.7858, Train Steps/Sec: 1.12 + 54%|████████████████████████████████████████████████████████████████████▏ | 59924/110696 [18:14:04<12:31:15, 1.13it/s][2025-04-27 15:17:30] (step=0059925) Train Loss: 5.7376, Train Steps/Sec: 1.12 + 54%|████████████████████████████████████████████████████████████████████▏ | 59949/110696 [18:14:27<12:27:27, 1.13it/s][2025-04-27 15:17:52] (step=0059950) Train Loss: 5.7353, Train Steps/Sec: 1.12 + 54%|████████████████████████████████████████████████████████████████████▎ | 59974/110696 [18:14:49<12:24:55, 1.13it/s][2025-04-27 15:18:14] (step=0059975) Train Loss: 5.7653, Train Steps/Sec: 1.12 + 54%|████████████████████████████████████████████████████████████████████▎ | 59999/110696 [18:15:11<12:27:06, 1.13it/s][2025-04-27 15:18:36] (step=0060000) Train Loss: 5.7907, Train Steps/Sec: 1.12 +You are using a model of type blip-2 to instantiate a model of type instructblip. This is not supported for all configurations of models and can yield errors. +[2025-04-27 15:18:37] text_config is None. Initializing the text config with default values (`OPTConfig`). +model_name_or_path:/tmp/haozhezhao/model/blip2-flan-t5-xl +image_place_holder: +tokenizer length after expend 32101 +tokenizer length before expend 32100 +100%|████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████| 5/5 [04:58<00:00, 59.60s/it] +[2025-04-27 15:24:52] Finish Eval in 60000 steps...█████████████████████████████████████████████████████████████████████████████████████████████| 5/5 [04:57<00:00, 59.35s/it] +processing left-padding... +processing left-padding... +processing left-padding... +processing left-padding... +processing left-padding... +[2025-04-27 15:25:12] Saved checkpoint to checkpoint/X2I_UltraEdiit_fourmask_LLava_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_filtered_objects_4mask_1280_context/003-GPT-XL/checkpoints/0060000.pt +[2025-04-27 15:25:15] Removed old checkpoint: checkpoint/X2I_UltraEdiit_fourmask_LLava_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_filtered_objects_4mask_1280_context/003-GPT-XL/checkpoints/0058000.pt + 54%|████████████████████████████████████████████████████████████████████▎ | 60024/110696 [18:22:12<13:14:14, 1.06it/s][2025-04-27 15:25:37] (step=0060025) Train Loss: 5.7344, Train Steps/Sec: 0.06 + 54%|████████████████████████████████████████████████████████████████████▎ | 60049/110696 [18:22:35<12:47:39, 1.10it/s][2025-04-27 15:26:00] (step=0060050) Train Loss: 5.7604, Train Steps/Sec: 1.11 + 54%|████████████████████████████████████████████████████████████████████▍ | 60074/110696 [18:22:57<12:33:16, 1.12it/s][2025-04-27 15:26:22] (step=0060075) Train Loss: 5.7831, Train Steps/Sec: 1.12 + 54%|████████████████████████████████████████████████████████████████████▍ | 60099/110696 [18:23:19<12:31:46, 1.12it/s][2025-04-27 15:26:45] (step=0060100) Train Loss: 5.7995, Train Steps/Sec: 1.12 + 54%|████████████████████████████████████████████████████████████████████▍ | 60124/110696 [18:23:42<12:36:54, 1.11it/s][2025-04-27 15:27:07] (step=0060125) Train Loss: 5.7997, Train Steps/Sec: 1.11 + 54%|████████████████████████████████████████████████████████████████████▍ | 60149/110696 [18:24:04<12:23:38, 1.13it/s][2025-04-27 15:27:29] (step=0060150) Train Loss: 5.7676, Train Steps/Sec: 1.12 + 54%|████████████████████████████████████████████████████████████████████▍ | 60174/110696 [18:24:26<12:24:41, 1.13it/s][2025-04-27 15:27:52] (step=0060175) Train Loss: 5.7983, Train Steps/Sec: 1.12 + 54%|████████████████████████████████████████████████████████████████████▌ | 60199/110696 [18:24:49<12:30:04, 1.12it/s][2025-04-27 15:28:14] (step=0060200) Train Loss: 5.7304, Train Steps/Sec: 1.11 + 54%|████████████████████████████████████████████████████████████████████▌ | 60224/110696 [18:25:11<12:50:48, 1.09it/s][2025-04-27 15:28:37] (step=0060225) Train Loss: 5.7218, Train Steps/Sec: 1.11 + 54%|████████████████████████████████████████████████████████████████████▌ | 60249/110696 [18:25:34<12:39:20, 1.11it/s][2025-04-27 15:28:59] (step=0060250) Train Loss: 5.7814, Train Steps/Sec: 1.11 + 54%|████████████████████████████████████████████████████████████████████▌ | 60274/110696 [18:25:56<12:33:12, 1.12it/s][2025-04-27 15:29:21] (step=0060275) Train Loss: 5.7788, Train Steps/Sec: 1.12 + 54%|████████████████████████████████████████████████████████████████████▋ | 60299/110696 [18:26:19<12:29:29, 1.12it/s][2025-04-27 15:29:44] (step=0060300) Train Loss: 5.7936, Train Steps/Sec: 1.11 + 54%|████████████████████████████████████████████████████████████████████▋ | 60324/110696 [18:26:41<12:27:11, 1.12it/s][2025-04-27 15:30:06] (step=0060325) Train Loss: 5.8051, Train Steps/Sec: 1.11 + 55%|████████████████████████████████████████████████████████████████████▋ | 60349/110696 [18:27:03<12:29:02, 1.12it/s][2025-04-27 15:30:29] (step=0060350) Train Loss: 5.7688, Train Steps/Sec: 1.12 + 55%|████████████████████████████████████████████████████████████████████▋ | 60374/110696 [18:27:26<12:20:28, 1.13it/s][2025-04-27 15:30:51] (step=0060375) Train Loss: 5.7273, Train Steps/Sec: 1.12 + 55%|████████████████████████████████████████████████████████████████████▋ | 60399/110696 [18:27:48<12:24:08, 1.13it/s][2025-04-27 15:31:14] (step=0060400) Train Loss: 5.7312, Train Steps/Sec: 1.11 + 55%|████████████████████████████████████████████████████████████████████▊ | 60424/110696 [18:28:11<12:44:33, 1.10it/s][2025-04-27 15:31:36] (step=0060425) Train Loss: 5.7730, Train Steps/Sec: 1.12 + 55%|████████████████████████████████████████████████████████████████████▊ | 60449/110696 [18:28:33<12:35:51, 1.11it/s][2025-04-27 15:31:58] (step=0060450) Train Loss: 5.7267, Train Steps/Sec: 1.12 + 55%|████████████████████████████████████████████████████████████████████▊ | 60474/110696 [18:28:55<12:29:16, 1.12it/s][2025-04-27 15:32:21] (step=0060475) Train Loss: 5.7414, Train Steps/Sec: 1.12 + 55%|████████████████████████████████████████████████████████████████████▊ | 60499/110696 [18:29:18<12:54:55, 1.08it/s][2025-04-27 15:32:43] (step=0060500) Train Loss: 5.7235, Train Steps/Sec: 1.10 + 55%|████████████████████████████████████████████████████████████████████▉ | 60524/110696 [18:29:40<12:28:24, 1.12it/s][2025-04-27 15:33:06] (step=0060525) Train Loss: 5.7769, Train Steps/Sec: 1.12 + 55%|████████████████████████████████████████████████████████████████████▉ | 60549/110696 [18:30:03<12:26:11, 1.12it/s][2025-04-27 15:33:28] (step=0060550) Train Loss: 5.7516, Train Steps/Sec: 1.11 + 55%|████████████████████████████████████████████████████████████████████▉ | 60574/110696 [18:30:25<12:22:42, 1.12it/s][2025-04-27 15:33:51] (step=0060575) Train Loss: 5.7268, Train Steps/Sec: 1.12 + 55%|████████████████████████████████████████████████████████████████████▉ | 60599/110696 [18:30:48<12:17:09, 1.13it/s][2025-04-27 15:34:13] (step=0060600) Train Loss: 5.8066, Train Steps/Sec: 1.11 + 55%|█████████████████████████████████████████████████████████████████████ | 60624/110696 [18:31:10<12:40:52, 1.10it/s][2025-04-27 15:34:35] (step=0060625) Train Loss: 5.7575, Train Steps/Sec: 1.12 + 55%|█████████████████████████████████████████████████████████████████████ | 60649/110696 [18:31:32<12:35:19, 1.10it/s][2025-04-27 15:34:58] (step=0060650) Train Loss: 5.8001, Train Steps/Sec: 1.12 + 55%|█████████████████████████████████████████████████████████████████████ | 60674/110696 [18:31:55<12:24:54, 1.12it/s][2025-04-27 15:35:20] (step=0060675) Train Loss: 5.7328, Train Steps/Sec: 1.12 + 55%|█████████████████████████████████████████████████████████████████████ | 60699/110696 [18:32:17<12:20:23, 1.13it/s][2025-04-27 15:35:42] (step=0060700) Train Loss: 5.7971, Train Steps/Sec: 1.12 + 55%|█████████████████████████████████████████████████████████████████████ | 60724/110696 [18:32:39<12:19:36, 1.13it/s][2025-04-27 15:36:05] (step=0060725) Train Loss: 5.8110, Train Steps/Sec: 1.12 + 55%|█████████████████████████████████████████████████████████████████████▏ | 60749/110696 [18:33:02<12:18:19, 1.13it/s][2025-04-27 15:36:27] (step=0060750) Train Loss: 5.7600, Train Steps/Sec: 1.12 + 55%|█████████████████████████████████████████████████████████████████████▏ | 60774/110696 [18:33:24<12:19:22, 1.13it/s][2025-04-27 15:36:50] (step=0060775) Train Loss: 5.7736, Train Steps/Sec: 1.12 + 55%|█████████████████████████████████████████████████████████████████████▏ | 60799/110696 [18:33:47<12:23:01, 1.12it/s][2025-04-27 15:37:12] (step=0060800) Train Loss: 5.7659, Train Steps/Sec: 1.11 + 55%|█████████████████████████████████████████████████████████████████████▏ | 60824/110696 [18:34:09<12:37:08, 1.10it/s][2025-04-27 15:37:34] (step=0060825) Train Loss: 5.7854, Train Steps/Sec: 1.12 + 55%|█████████████████████████████████████████████████████████████████████▎ | 60849/110696 [18:34:31<12:33:02, 1.10it/s][2025-04-27 15:37:57] (step=0060850) Train Loss: 5.8069, Train Steps/Sec: 1.12 + 55%|█████████████████████████████████████████████████████████████████████▎ | 60874/110696 [18:34:54<12:25:49, 1.11it/s][2025-04-27 15:38:19] (step=0060875) Train Loss: 5.7845, Train Steps/Sec: 1.12 + 55%|█████████████████████████████████████████████████████████████████████▎ | 60899/110696 [18:35:16<12:23:37, 1.12it/s][2025-04-27 15:38:42] (step=0060900) Train Loss: 5.7972, Train Steps/Sec: 1.12 + 55%|█████████████████████████████████████████████████████████████████████▎ | 60924/110696 [18:35:39<12:19:57, 1.12it/s][2025-04-27 15:39:04] (step=0060925) Train Loss: 5.7704, Train Steps/Sec: 1.12 + 55%|█████████████████████████████████████████████████████████████████████▍ | 60949/110696 [18:36:01<12:16:59, 1.12it/s][2025-04-27 15:39:26] (step=0060950) Train Loss: 5.7591, Train Steps/Sec: 1.12 + 55%|█████████████████████████████████████████████████████████████████████▍ | 60974/110696 [18:36:23<12:17:18, 1.12it/s][2025-04-27 15:39:49] (step=0060975) Train Loss: 5.7906, Train Steps/Sec: 1.12 + 55%|█████████████████████████████████████████████████████████████████████▍ | 60999/110696 [18:36:46<12:14:47, 1.13it/s][2025-04-27 15:40:11] (step=0061000) Train Loss: 5.7385, Train Steps/Sec: 1.11 + 55%|█████████████████████████████████████████████████████████████████████▍ | 61024/110696 [18:37:08<12:39:46, 1.09it/s][2025-04-27 15:40:34] (step=0061025) Train Loss: 5.7861, Train Steps/Sec: 1.12 + 55%|█████████████████████████████████████████████████████████████████████▍ | 61049/110696 [18:37:31<12:30:08, 1.10it/s][2025-04-27 15:40:56] (step=0061050) Train Loss: 5.7856, Train Steps/Sec: 1.12 + 55%|█████████████████████████████████████████████████████████████████████▌ | 61074/110696 [18:37:53<12:23:45, 1.11it/s][2025-04-27 15:41:18] (step=0061075) Train Loss: 5.7643, Train Steps/Sec: 1.12 + 55%|█████████████████████████████████████████████████████████████████████▌ | 61099/110696 [18:38:16<12:25:20, 1.11it/s][2025-04-27 15:41:41] (step=0061100) Train Loss: 5.9311, Train Steps/Sec: 1.11 + 55%|█████████████████████████████████████████████████████████████████████▌ | 61124/110696 [18:38:38<12:21:27, 1.11it/s][2025-04-27 15:42:03] (step=0061125) Train Loss: 5.7453, Train Steps/Sec: 1.11 + 55%|█████████████████████████████████████████████████████████████████████▌ | 61149/110696 [18:39:00<12:13:49, 1.13it/s][2025-04-27 15:42:26] (step=0061150) Train Loss: 5.7855, Train Steps/Sec: 1.12 + 55%|█████████████████████████████████████████████████████████████████████▋ | 61174/110696 [18:39:23<12:10:32, 1.13it/s][2025-04-27 15:42:48] (step=0061175) Train Loss: 5.7420, Train Steps/Sec: 1.12 + 55%|█████████████████████████████████████████████████████████████████████▋ | 61199/110696 [18:39:45<12:11:18, 1.13it/s][2025-04-27 15:43:10] (step=0061200) Train Loss: 5.7269, Train Steps/Sec: 1.12 + 55%|█████████████████████████████████████████████████████████████████████▋ | 61224/110696 [18:40:07<12:30:19, 1.10it/s][2025-04-27 15:43:33] (step=0061225) Train Loss: 5.7836, Train Steps/Sec: 1.12 + 55%|█████████████████████████████████████████████████████████████████████▋ | 61249/110696 [18:40:30<12:22:31, 1.11it/s][2025-04-27 15:43:55] (step=0061250) Train Loss: 5.7578, Train Steps/Sec: 1.12 + 55%|█████████████████████████████████████████████████████████████████████▋ | 61274/110696 [18:40:52<12:16:09, 1.12it/s][2025-04-27 15:44:17] (step=0061275) Train Loss: 5.8153, Train Steps/Sec: 1.12 + 55%|█████████████████████████████████████████████████████████████████████▊ | 61299/110696 [18:41:14<12:15:00, 1.12it/s][2025-04-27 15:44:40] (step=0061300) Train Loss: 5.8085, Train Steps/Sec: 1.12 + 55%|█████████████████████████████████████████████████████████████████████▊ | 61324/110696 [18:41:37<12:09:29, 1.13it/s][2025-04-27 15:45:02] (step=0061325) Train Loss: 5.7838, Train Steps/Sec: 1.12 + 55%|█████████████████████████████████████████████████████████████████████▊ | 61349/110696 [18:41:59<12:07:13, 1.13it/s][2025-04-27 15:45:24] (step=0061350) Train Loss: 5.7496, Train Steps/Sec: 1.12 + 55%|█████████████████████████████████████████████████████████████████████▊ | 61374/110696 [18:42:21<12:08:24, 1.13it/s][2025-04-27 15:45:47] (step=0061375) Train Loss: 5.8375, Train Steps/Sec: 1.12 + 55%|█████████████████████████████████████████████████████████████████████▉ | 61399/110696 [18:42:44<12:05:36, 1.13it/s][2025-04-27 15:46:09] (step=0061400) Train Loss: 5.8632, Train Steps/Sec: 1.12 + 55%|█████████████████████████████████████████████████████████████████████▉ | 61424/110696 [18:43:06<12:29:44, 1.10it/s][2025-04-27 15:46:32] (step=0061425) Train Loss: 5.7898, Train Steps/Sec: 1.12 + 56%|█████████████████████████████████████████████████████████████████████▉ | 61449/110696 [18:43:28<12:19:37, 1.11it/s][2025-04-27 15:46:54] (step=0061450) Train Loss: 5.7313, Train Steps/Sec: 1.12 + 56%|█████████████████████████████████████████████████████████████████████▉ | 61474/110696 [18:43:51<12:11:46, 1.12it/s][2025-04-27 15:47:16] (step=0061475) Train Loss: 5.7839, Train Steps/Sec: 1.12 + 56%|██████████████████████████████████████████████████████████████████████ | 61499/110696 [18:44:13<12:12:07, 1.12it/s][2025-04-27 15:47:38] (step=0061500) Train Loss: 5.8047, Train Steps/Sec: 1.12 + 56%|██████████████████████████████████████████████████████████████████████ | 61524/110696 [18:44:35<12:08:36, 1.12it/s][2025-04-27 15:48:01] (step=0061525) Train Loss: 5.7391, Train Steps/Sec: 1.12 + 56%|██████████████████████████████████████████████████████████████████████ | 61549/110696 [18:44:58<12:05:34, 1.13it/s][2025-04-27 15:48:23] (step=0061550) Train Loss: 5.7552, Train Steps/Sec: 1.12 + 56%|██████████████████████████████████████████████████████████████████████ | 61574/110696 [18:45:20<12:07:57, 1.12it/s][2025-04-27 15:48:45] (step=0061575) Train Loss: 5.7652, Train Steps/Sec: 1.12 + 56%|██████████████████████████████████████████████████████████████████████ | 61599/110696 [18:45:42<12:02:43, 1.13it/s][2025-04-27 15:49:08] (step=0061600) Train Loss: 5.7994, Train Steps/Sec: 1.12 + 56%|██████████████████████████████████████████████████████████████████████▏ | 61624/110696 [18:46:05<12:25:22, 1.10it/s][2025-04-27 15:49:30] (step=0061625) Train Loss: 5.8085, Train Steps/Sec: 1.12 + 56%|██████████████████████████████████████████████████████████████████████▏ | 61649/110696 [18:46:27<12:16:05, 1.11it/s][2025-04-27 15:49:53] (step=0061650) Train Loss: 5.7458, Train Steps/Sec: 1.12 + 56%|██████████████████████████████████████████████████████████████████████▏ | 61674/110696 [18:46:50<12:12:02, 1.12it/s][2025-04-27 15:50:15] (step=0061675) Train Loss: 5.8260, Train Steps/Sec: 1.12 + 56%|██████████████████████████████████████████████████████████████████████▏ | 61699/110696 [18:47:12<12:07:23, 1.12it/s][2025-04-27 15:50:37] (step=0061700) Train Loss: 5.7984, Train Steps/Sec: 1.12 + 56%|██████████████████████████████████████████████████████████████████████▎ | 61724/110696 [18:47:34<12:13:44, 1.11it/s][2025-04-27 15:51:00] (step=0061725) Train Loss: 5.7607, Train Steps/Sec: 1.11 + 56%|██████████████████████████████████████████████████████████████████████▎ | 61749/110696 [18:47:57<12:07:57, 1.12it/s][2025-04-27 15:51:22] (step=0061750) Train Loss: 5.8463, Train Steps/Sec: 1.12 + 56%|██████████████████████████████████████████████████████████████████████▎ | 61774/110696 [18:48:19<12:06:21, 1.12it/s][2025-04-27 15:51:45] (step=0061775) Train Loss: 5.7481, Train Steps/Sec: 1.12 + 56%|██████████████████████████████████████████████████████████████████████▎ | 61799/110696 [18:48:42<12:04:38, 1.12it/s][2025-04-27 15:52:07] (step=0061800) Train Loss: 5.7433, Train Steps/Sec: 1.11 + 56%|██████████████████████████████████████████████████████████████████████▎ | 61824/110696 [18:49:04<12:31:14, 1.08it/s][2025-04-27 15:52:29] (step=0061825) Train Loss: 5.7468, Train Steps/Sec: 1.11 + 56%|██████████████████████████████████████████████████████████████████████▍ | 61849/110696 [18:49:26<12:13:25, 1.11it/s][2025-04-27 15:52:52] (step=0061850) Train Loss: 5.7823, Train Steps/Sec: 1.11 + 56%|██████████████████████████████████████████████████████████████████████▍ | 61874/110696 [18:49:49<12:08:16, 1.12it/s][2025-04-27 15:53:14] (step=0061875) Train Loss: 5.7694, Train Steps/Sec: 1.12 + 56%|██████████████████████████████████████████████████████████████████████▍ | 61899/110696 [18:50:11<12:06:54, 1.12it/s][2025-04-27 15:53:37] (step=0061900) Train Loss: 5.7816, Train Steps/Sec: 1.12 + 56%|██████████████████████████████████████████████████████████████████████▍ | 61924/110696 [18:50:34<12:01:55, 1.13it/s][2025-04-27 15:53:59] (step=0061925) Train Loss: 5.8340, Train Steps/Sec: 1.12 + 56%|██████████████████████████████████████████████████████████████████████▌ | 61949/110696 [18:50:56<11:58:29, 1.13it/s][2025-04-27 15:54:21] (step=0061950) Train Loss: 5.7952, Train Steps/Sec: 1.12 + 56%|██████████████████████████████████████████████████████████████████████▌ | 61974/110696 [18:51:18<11:58:22, 1.13it/s][2025-04-27 15:54:43] (step=0061975) Train Loss: 5.7756, Train Steps/Sec: 1.12 + 56%|██████████████████████████████████████████████████████████████████████▌ | 61999/110696 [18:51:40<11:58:01, 1.13it/s][2025-04-27 15:55:06] (step=0062000) Train Loss: 5.7679, Train Steps/Sec: 1.12 +You are using a model of type blip-2 to instantiate a model of type instructblip. This is not supported for all configurations of models and can yield errors. +[2025-04-27 15:55:06] text_config is None. Initializing the text config with default values (`OPTConfig`). +model_name_or_path:/tmp/haozhezhao/model/blip2-flan-t5-xl +image_place_holder: +tokenizer length after expend 32101 +tokenizer length before expend 32100 +100%|████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████| 5/5 [05:01<00:00, 60.34s/it] +[2025-04-27 16:01:23] Finish Eval in 62000 steps...█████████████████████████████████████████████████████████████████████████████████████████████| 5/5 [05:01<00:00, 60.28s/it] +processing left-padding... +processing left-padding... +processing left-padding... +processing left-padding... +processing left-padding... +[2025-04-27 16:01:44] Saved checkpoint to checkpoint/X2I_UltraEdiit_fourmask_LLava_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_filtered_objects_4mask_1280_context/003-GPT-XL/checkpoints/0062000.pt +[2025-04-27 16:01:46] Removed old checkpoint: checkpoint/X2I_UltraEdiit_fourmask_LLava_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_filtered_objects_4mask_1280_context/003-GPT-XL/checkpoints/0060000.pt + 56%|██████████████████████████████████████████████████████████████████████▌ | 62024/110696 [18:58:43<12:41:33, 1.07it/s][2025-04-27 16:02:08] (step=0062025) Train Loss: 5.7849, Train Steps/Sec: 0.06 + 56%|██████████████████████████████████████████████████████████████████████▋ | 62049/110696 [18:59:05<12:13:56, 1.10it/s][2025-04-27 16:02:31] (step=0062050) Train Loss: 5.7468, Train Steps/Sec: 1.11 + 56%|██████████████████████████████████████████████████████████████████████▋ | 62074/110696 [18:59:28<12:06:48, 1.11it/s][2025-04-27 16:02:53] (step=0062075) Train Loss: 5.8140, Train Steps/Sec: 1.12 + 56%|██████████████████████████████████████████████████████████████████████▋ | 62099/110696 [18:59:50<12:02:29, 1.12it/s][2025-04-27 16:03:16] (step=0062100) Train Loss: 5.7962, Train Steps/Sec: 1.12 + 56%|██████████████████████████████████████████████████████████████████████▋ | 62124/110696 [19:00:13<12:04:05, 1.12it/s][2025-04-27 16:03:38] (step=0062125) Train Loss: 5.7314, Train Steps/Sec: 1.12 + 56%|██████████████████████████████████████████████████████████████████████▋ | 62149/110696 [19:00:35<12:14:35, 1.10it/s][2025-04-27 16:04:01] (step=0062150) Train Loss: 5.7185, Train Steps/Sec: 1.11 + 56%|██████████████████████████████████████████████████████████████████████▊ | 62174/110696 [19:00:58<12:00:16, 1.12it/s][2025-04-27 16:04:23] (step=0062175) Train Loss: 5.7561, Train Steps/Sec: 1.11 + 56%|██████████████████████████████████████████████████████████████████████▊ | 62199/110696 [19:01:20<11:54:44, 1.13it/s][2025-04-27 16:04:46] (step=0062200) Train Loss: 5.7795, Train Steps/Sec: 1.11 + 56%|██████████████████████████████████████████████████████████████████████▊ | 62224/110696 [19:01:43<12:22:56, 1.09it/s][2025-04-27 16:05:08] (step=0062225) Train Loss: 5.7751, Train Steps/Sec: 1.11 + 56%|██████████████████████████████████████████████████████████████████████▊ | 62249/110696 [19:02:05<12:13:54, 1.10it/s][2025-04-27 16:05:31] (step=0062250) Train Loss: 5.7562, Train Steps/Sec: 1.11 + 56%|██████████████████████████████████████████████████████████████████████▉ | 62274/110696 [19:02:28<12:05:41, 1.11it/s][2025-04-27 16:05:53] (step=0062275) Train Loss: 5.7677, Train Steps/Sec: 1.11 + 56%|██████████████████████████████████████████████████████████████████████▉ | 62299/110696 [19:02:50<12:02:06, 1.12it/s][2025-04-27 16:06:15] (step=0062300) Train Loss: 5.7618, Train Steps/Sec: 1.12 + 56%|██████████████████████████████████████████████████████████████████████▉ | 62324/110696 [19:03:17<12:06:20, 1.11it/s][2025-04-27 16:06:42] (step=0062325) Train Loss: 5.7755, Train Steps/Sec: 0.93 + 56%|██████████████████████████████████████████████████████████████████████▉ | 62349/110696 [19:03:39<11:55:50, 1.13it/s][2025-04-27 16:07:05] (step=0062350) Train Loss: 5.8036, Train Steps/Sec: 1.12 + 56%|██████████████████████████████████████████████████████████████████████▉ | 62374/110696 [19:04:02<11:53:26, 1.13it/s][2025-04-27 16:07:27] (step=0062375) Train Loss: 5.7485, Train Steps/Sec: 1.12 + 56%|███████████████████████████████████████████████████████████████████████ | 62399/110696 [19:04:24<12:57:42, 1.04it/s][2025-04-27 16:07:50] (step=0062400) Train Loss: 5.7851, Train Steps/Sec: 1.10 + 56%|███████████████████████████████████████████████████████████████████████ | 62424/110696 [19:04:47<12:12:37, 1.10it/s][2025-04-27 16:08:12] (step=0062425) Train Loss: 5.7490, Train Steps/Sec: 1.12 + 56%|███████████████████████████████████████████████████████████████████████ | 62449/110696 [19:05:09<12:03:09, 1.11it/s][2025-04-27 16:08:34] (step=0062450) Train Loss: 5.7490, Train Steps/Sec: 1.12 + 56%|███████████████████████████████████████████████████████████████████████ | 62474/110696 [19:05:36<16:37:37, 1.24s/it][2025-04-27 16:09:01] (step=0062475) Train Loss: 5.7376, Train Steps/Sec: 0.92 + 56%|███████████████████████████████████████████████████████████████████████▏ | 62499/110696 [19:05:58<11:55:41, 1.12it/s][2025-04-27 16:09:24] (step=0062500) Train Loss: 5.8654, Train Steps/Sec: 1.12 + 56%|███████████████████████████████████████████████████████████████████████▏ | 62524/110696 [19:06:21<11:55:11, 1.12it/s][2025-04-27 16:09:46] (step=0062525) Train Loss: 5.7737, Train Steps/Sec: 1.12 + 57%|███████████████████████████████████████████████████████████████████████▏ | 62549/110696 [19:06:43<11:50:19, 1.13it/s][2025-04-27 16:10:08] (step=0062550) Train Loss: 5.7323, Train Steps/Sec: 1.12 + 57%|███████████████████████████████████████████████████████████████████████▏ | 62574/110696 [19:07:05<11:53:12, 1.12it/s][2025-04-27 16:10:31] (step=0062575) Train Loss: 5.7977, Train Steps/Sec: 1.12 + 57%|███████████████████████████████████████████████████████████████████████▎ | 62599/110696 [19:07:28<11:49:49, 1.13it/s][2025-04-27 16:10:53] (step=0062600) Train Loss: 5.7715, Train Steps/Sec: 1.12 + 57%|███████████████████████████████████████████████████████████████████████▎ | 62624/110696 [19:07:50<12:12:35, 1.09it/s][2025-04-27 16:11:16] (step=0062625) Train Loss: 5.7743, Train Steps/Sec: 1.12 + 57%|███████████████████████████████████████████████████████████████████████▎ | 62649/110696 [19:08:13<12:03:27, 1.11it/s][2025-04-27 16:11:38] (step=0062650) Train Loss: 5.7606, Train Steps/Sec: 1.12 + 57%|███████████████████████████████████████████████████████████████████████▎ | 62674/110696 [19:08:35<11:56:52, 1.12it/s][2025-04-27 16:12:00] (step=0062675) Train Loss: 5.7651, Train Steps/Sec: 1.12 + 57%|███████████████████████████████████████████████████████████████████████▎ | 62699/110696 [19:08:57<11:54:49, 1.12it/s][2025-04-27 16:12:23] (step=0062700) Train Loss: 5.7807, Train Steps/Sec: 1.12 + 57%|███████████████████████████████████████████████████████████████████████▍ | 62724/110696 [19:09:20<11:51:14, 1.12it/s][2025-04-27 16:12:45] (step=0062725) Train Loss: 5.8207, Train Steps/Sec: 1.12 + 57%|███████████████████████████████████████████████████████████████████████▍ | 62749/110696 [19:09:42<11:46:37, 1.13it/s][2025-04-27 16:13:07] (step=0062750) Train Loss: 5.7255, Train Steps/Sec: 1.12 + 57%|███████████████████████████████████████████████████████████████████████▍ | 62774/110696 [19:10:04<11:46:09, 1.13it/s][2025-04-27 16:13:30] (step=0062775) Train Loss: 5.7111, Train Steps/Sec: 1.12 + 57%|███████████████████████████████████████████████████████████████████████▍ | 62799/110696 [19:10:27<11:48:28, 1.13it/s][2025-04-27 16:13:52] (step=0062800) Train Loss: 5.7597, Train Steps/Sec: 1.11 + 57%|███████████████████████████████████████████████████████████████████████▌ | 62824/110696 [19:10:49<12:05:27, 1.10it/s][2025-04-27 16:14:14] (step=0062825) Train Loss: 5.7796, Train Steps/Sec: 1.12 + 57%|███████████████████████████████████████████████████████████████████████▌ | 62849/110696 [19:11:11<11:58:35, 1.11it/s][2025-04-27 16:14:37] (step=0062850) Train Loss: 5.8206, Train Steps/Sec: 1.12 + 57%|███████████████████████████████████████████████████████████████████████▌ | 62874/110696 [19:11:34<11:55:49, 1.11it/s][2025-04-27 16:14:59] (step=0062875) Train Loss: 5.6717, Train Steps/Sec: 1.12 + 57%|███████████████████████████████████████████████████████████████████████▌ | 62899/110696 [19:11:56<11:51:02, 1.12it/s][2025-04-27 16:15:21] (step=0062900) Train Loss: 5.6923, Train Steps/Sec: 1.12 + 57%|███████████████████████████████████████████████████████████████████████▌ | 62924/110696 [19:12:18<11:47:06, 1.13it/s][2025-04-27 16:15:44] (step=0062925) Train Loss: 5.7892, Train Steps/Sec: 1.12 + 57%|███████████████████████████████████████████████████████████████████████▋ | 62949/110696 [19:12:41<11:44:55, 1.13it/s][2025-04-27 16:16:06] (step=0062950) Train Loss: 5.8051, Train Steps/Sec: 1.12 + 57%|███████████████████████████████████████████████████████████████████████▋ | 62974/110696 [19:13:11<17:12:31, 1.30s/it][2025-04-27 16:16:37] (step=0062975) Train Loss: 5.7864, Train Steps/Sec: 0.82 + 57%|███████████████████████████████████████████████████████████████████████▋ | 62999/110696 [19:13:38<11:59:26, 1.10it/s][2025-04-27 16:17:03] (step=0063000) Train Loss: 5.8109, Train Steps/Sec: 0.94 + 57%|███████████████████████████████████████████████████████████████████████▋ | 63024/110696 [19:14:11<33:00:37, 2.49s/it][2025-04-27 16:17:36] (step=0063025) Train Loss: 5.8343, Train Steps/Sec: 0.76 + 57%|███████████████████████████████████████████████████████████████████████▊ | 63049/110696 [19:14:33<11:57:57, 1.11it/s][2025-04-27 16:17:59] (step=0063050) Train Loss: 5.7449, Train Steps/Sec: 1.12 + 57%|███████████████████████████████████████████████████████████████████████▊ | 63074/110696 [19:14:56<11:54:00, 1.11it/s][2025-04-27 16:18:21] (step=0063075) Train Loss: 5.7464, Train Steps/Sec: 1.11 + 57%|███████████████████████████████████████████████████████████████████████▊ | 63099/110696 [19:15:18<11:49:49, 1.12it/s][2025-04-27 16:18:43] (step=0063100) Train Loss: 5.7515, Train Steps/Sec: 1.12 + 57%|███████████████████████████████████████████████████████████████████████▊ | 63124/110696 [19:15:40<11:47:04, 1.12it/s][2025-04-27 16:19:06] (step=0063125) Train Loss: 5.7185, Train Steps/Sec: 1.12 + 57%|███████████████████████████████████████████████████████████████████████▉ | 63149/110696 [19:16:03<11:45:09, 1.12it/s][2025-04-27 16:19:28] (step=0063150) Train Loss: 5.7386, Train Steps/Sec: 1.12 + 57%|███████████████████████████████████████████████████████████████████████▉ | 63174/110696 [19:16:25<11:39:45, 1.13it/s][2025-04-27 16:19:51] (step=0063175) Train Loss: 5.7571, Train Steps/Sec: 1.12 + 57%|███████████████████████████████████████████████████████████████████████▉ | 63199/110696 [19:16:48<11:42:23, 1.13it/s][2025-04-27 16:20:13] (step=0063200) Train Loss: 5.7604, Train Steps/Sec: 1.11 + 57%|███████████████████████████████████████████████████████████████████████▉ | 63224/110696 [19:17:10<12:04:14, 1.09it/s][2025-04-27 16:20:35] (step=0063225) Train Loss: 5.7015, Train Steps/Sec: 1.12 + 57%|███████████████████████████████████████████████████████████████████████▉ | 63249/110696 [19:17:32<11:53:13, 1.11it/s][2025-04-27 16:20:58] (step=0063250) Train Loss: 5.7440, Train Steps/Sec: 1.12 + 57%|████████████████████████████████████████████████████████████████████████ | 63274/110696 [19:17:55<11:49:23, 1.11it/s][2025-04-27 16:21:20] (step=0063275) Train Loss: 5.7387, Train Steps/Sec: 1.12 + 57%|████████████████████████████████████████████████████████████████████████ | 63299/110696 [19:18:17<11:43:36, 1.12it/s][2025-04-27 16:21:42] (step=0063300) Train Loss: 5.7462, Train Steps/Sec: 1.12 + 57%|████████████████████████████████████████████████████████████████████████ | 63324/110696 [19:18:39<11:40:12, 1.13it/s][2025-04-27 16:22:05] (step=0063325) Train Loss: 5.7127, Train Steps/Sec: 1.12 + 57%|████████████████████████████████████████████████████████████████████████ | 63349/110696 [19:19:02<11:38:39, 1.13it/s][2025-04-27 16:22:27] (step=0063350) Train Loss: 5.7658, Train Steps/Sec: 1.12 + 57%|████████████████████████████████████████████████████████████████████████▏ | 63374/110696 [19:19:24<11:38:58, 1.13it/s][2025-04-27 16:22:49] (step=0063375) Train Loss: 5.7850, Train Steps/Sec: 1.12 + 57%|████████████████████████████████████████████████████████████████████████▏ | 63399/110696 [19:19:46<11:34:16, 1.14it/s][2025-04-27 16:23:12] (step=0063400) Train Loss: 5.7415, Train Steps/Sec: 1.12 + 57%|████████████████████████████████████████████████████████████████████████▏ | 63424/110696 [19:20:09<11:59:56, 1.09it/s][2025-04-27 16:23:34] (step=0063425) Train Loss: 5.7504, Train Steps/Sec: 1.12 + 57%|████████████████████████████████████████████████████████████████████████▏ | 63449/110696 [19:20:31<11:49:23, 1.11it/s][2025-04-27 16:23:56] (step=0063450) Train Loss: 5.7515, Train Steps/Sec: 1.12 + 57%|████████████████████████████████████████████████████████████████████████▏ | 63474/110696 [19:20:53<11:44:21, 1.12it/s][2025-04-27 16:24:19] (step=0063475) Train Loss: 5.8003, Train Steps/Sec: 1.12 + 57%|████████████████████████████████████████████████████████████████████████▎ | 63499/110696 [19:21:16<11:41:39, 1.12it/s][2025-04-27 16:24:41] (step=0063500) Train Loss: 5.7063, Train Steps/Sec: 1.12 + 57%|████████████████████████████████████████████████████████████████████████▎ | 63524/110696 [19:21:38<11:39:29, 1.12it/s][2025-04-27 16:25:03] (step=0063525) Train Loss: 5.8486, Train Steps/Sec: 1.12 + 57%|████████████████████████████████████████████████████████████████████████▎ | 63549/110696 [19:22:00<11:36:22, 1.13it/s][2025-04-27 16:25:26] (step=0063550) Train Loss: 5.7835, Train Steps/Sec: 1.12 + 57%|████████████████████████████████████████████████████████████████████████▎ | 63574/110696 [19:22:23<11:38:09, 1.12it/s][2025-04-27 16:25:48] (step=0063575) Train Loss: 5.7700, Train Steps/Sec: 1.12 + 57%|████████████████████████████████████████████████████████████████████████▍ | 63599/110696 [19:22:45<11:34:25, 1.13it/s][2025-04-27 16:26:11] (step=0063600) Train Loss: 5.7493, Train Steps/Sec: 1.12 + 57%|████████████████████████████████████████████████████████████████████████▍ | 63624/110696 [19:23:07<11:57:17, 1.09it/s][2025-04-27 16:26:33] (step=0063625) Train Loss: 5.7623, Train Steps/Sec: 1.12 + 57%|████████████████████████████████████████████████████████████████████████▍ | 63649/110696 [19:23:30<11:46:59, 1.11it/s][2025-04-27 16:26:55] (step=0063650) Train Loss: 5.7592, Train Steps/Sec: 1.12 + 58%|████████████████████████████████████████████████████████████████████████▍ | 63674/110696 [19:23:52<11:39:15, 1.12it/s][2025-04-27 16:27:17] (step=0063675) Train Loss: 5.8028, Train Steps/Sec: 1.12 + 58%|████████████████████████████████████████████████████████████████████████▌ | 63699/110696 [19:24:14<11:38:44, 1.12it/s][2025-04-27 16:27:40] (step=0063700) Train Loss: 5.8158, Train Steps/Sec: 1.12 + 58%|████████████████████████████████████████████████████████████████████████▌ | 63724/110696 [19:24:37<11:41:16, 1.12it/s][2025-04-27 16:28:02] (step=0063725) Train Loss: 5.7741, Train Steps/Sec: 1.12 + 58%|████████████████████████████████████████████████████████████████████████▌ | 63749/110696 [19:24:59<11:33:57, 1.13it/s][2025-04-27 16:28:24] (step=0063750) Train Loss: 5.7778, Train Steps/Sec: 1.12 + 58%|████████████████████████████████████████████████████████████████████████▌ | 63774/110696 [19:25:21<11:31:13, 1.13it/s][2025-04-27 16:28:47] (step=0063775) Train Loss: 5.8168, Train Steps/Sec: 1.12 + 58%|████████████████████████████████████████████████████████████████████████▌ | 63799/110696 [19:25:44<11:33:13, 1.13it/s][2025-04-27 16:29:09] (step=0063800) Train Loss: 5.7099, Train Steps/Sec: 1.11 + 58%|████████████████████████████████████████████████████████████████████████▋ | 63824/110696 [19:26:06<11:54:18, 1.09it/s][2025-04-27 16:29:32] (step=0063825) Train Loss: 5.7562, Train Steps/Sec: 1.12 + 58%|████████████████████████████████████████████████████████████████████████▋ | 63849/110696 [19:26:29<11:44:15, 1.11it/s][2025-04-27 16:29:54] (step=0063850) Train Loss: 5.7583, Train Steps/Sec: 1.12 + 58%|████████████████████████████████████████████████████████████████████████▋ | 63874/110696 [19:26:51<11:41:38, 1.11it/s][2025-04-27 16:30:16] (step=0063875) Train Loss: 5.7657, Train Steps/Sec: 1.12 + 58%|████████████████████████████████████████████████████████████████████████▋ | 63899/110696 [19:27:13<11:36:01, 1.12it/s][2025-04-27 16:30:39] (step=0063900) Train Loss: 5.6851, Train Steps/Sec: 1.12 + 58%|████████████████████████████████████████████████████████████████████████▊ | 63924/110696 [19:27:36<11:32:20, 1.13it/s][2025-04-27 16:31:01] (step=0063925) Train Loss: 5.7762, Train Steps/Sec: 1.12 + 58%|████████████████████████████████████████████████████████████████████████▊ | 63949/110696 [19:27:58<11:31:59, 1.13it/s][2025-04-27 16:31:23] (step=0063950) Train Loss: 5.7864, Train Steps/Sec: 1.12 + 58%|████████████████████████████████████████████████████████████████████████▊ | 63974/110696 [19:28:20<11:29:25, 1.13it/s][2025-04-27 16:31:46] (step=0063975) Train Loss: 5.7642, Train Steps/Sec: 1.12 + 58%|████████████████████████████████████████████████████████████████████████▊ | 63999/110696 [19:28:43<11:26:23, 1.13it/s][2025-04-27 16:32:08] (step=0064000) Train Loss: 5.8009, Train Steps/Sec: 1.11 +You are using a model of type blip-2 to instantiate a model of type instructblip. This is not supported for all configurations of models and can yield errors. +[2025-04-27 16:32:08] text_config is None. Initializing the text config with default values (`OPTConfig`). +model_name_or_path:/tmp/haozhezhao/model/blip2-flan-t5-xl +image_place_holder: +tokenizer length after expend 32101 +tokenizer length before expend 32100 +100%|████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████| 5/5 [05:04<00:00, 60.99s/it] +[2025-04-27 16:38:31] Finish Eval in 64000 steps...█████████████████████████████████████████████████████████████████████████████████████████████| 5/5 [05:04<00:00, 60.62s/it] +processing left-padding... +processing left-padding... +processing left-padding... +processing left-padding... +processing left-padding... +[2025-04-27 16:38:51] Saved checkpoint to checkpoint/X2I_UltraEdiit_fourmask_LLava_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_filtered_objects_4mask_1280_context/003-GPT-XL/checkpoints/0064000.pt +[2025-04-27 16:38:53] Removed old checkpoint: checkpoint/X2I_UltraEdiit_fourmask_LLava_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_filtered_objects_4mask_1280_context/003-GPT-XL/checkpoints/0062000.pt + 58%|████████████████████████████████████████████████████████████████████████▉ | 64024/110696 [19:35:50<12:08:53, 1.07it/s][2025-04-27 16:39:16] (step=0064025) Train Loss: 5.8236, Train Steps/Sec: 0.06 + 58%|████████████████████████████████████████████████████████████████████████▉ | 64049/110696 [19:36:13<11:41:26, 1.11it/s][2025-04-27 16:39:38] (step=0064050) Train Loss: 5.7731, Train Steps/Sec: 1.12 + 58%|████████████████████████████████████████████████████████████████████████▉ | 64074/110696 [19:36:35<11:33:37, 1.12it/s][2025-04-27 16:40:00] (step=0064075) Train Loss: 5.7783, Train Steps/Sec: 1.12 + 58%|████████████████████████████████████████████████████████████████████████▉ | 64099/110696 [19:36:57<11:31:15, 1.12it/s][2025-04-27 16:40:22] (step=0064100) Train Loss: 5.8387, Train Steps/Sec: 1.12 + 58%|████████████████████████████████████████████████████████████████████████▉ | 64124/110696 [19:37:19<11:28:55, 1.13it/s][2025-04-27 16:40:45] (step=0064125) Train Loss: 5.7503, Train Steps/Sec: 1.12 + 58%|█████████████████████████████████████████████████████████████████████████ | 64149/110696 [19:37:42<11:26:24, 1.13it/s][2025-04-27 16:41:07] (step=0064150) Train Loss: 5.7619, Train Steps/Sec: 1.12 + 58%|█████████████████████████████████████████████████████████████████████████ | 64174/110696 [19:38:04<11:25:47, 1.13it/s][2025-04-27 16:41:30] (step=0064175) Train Loss: 5.7831, Train Steps/Sec: 1.12 + 58%|█████████████████████████████████████████████████████████████████████████ | 64199/110696 [19:38:26<11:25:26, 1.13it/s][2025-04-27 16:41:52] (step=0064200) Train Loss: 5.8300, Train Steps/Sec: 1.12 + 58%|█████████████████████████████████████████████████████████████████████████ | 64224/110696 [19:38:49<11:44:13, 1.10it/s][2025-04-27 16:42:14] (step=0064225) Train Loss: 5.7570, Train Steps/Sec: 1.12 + 58%|█████████████████████████████████████████████████████████████████████████▏ | 64249/110696 [19:39:11<11:36:28, 1.11it/s][2025-04-27 16:42:36] (step=0064250) Train Loss: 5.8014, Train Steps/Sec: 1.12 + 58%|█████████████████████████████████████████████████████████████████████████▏ | 64274/110696 [19:39:33<11:31:05, 1.12it/s][2025-04-27 16:42:59] (step=0064275) Train Loss: 5.7853, Train Steps/Sec: 1.12 + 58%|█████████████████████████████████████████████████████████████████████████▏ | 64299/110696 [19:39:56<13:19:12, 1.03s/it][2025-04-27 16:43:22] (step=0064300) Train Loss: 5.7902, Train Steps/Sec: 1.09 + 58%|█████████████████████████████████████████████████████████████████████████▏ | 64324/110696 [19:40:19<11:28:17, 1.12it/s][2025-04-27 16:43:44] (step=0064325) Train Loss: 5.8110, Train Steps/Sec: 1.12 + 58%|█████████████████████████████████████████████████████████████████████████▏ | 64349/110696 [19:40:41<11:21:59, 1.13it/s][2025-04-27 16:44:06] (step=0064350) Train Loss: 5.7959, Train Steps/Sec: 1.12 + 58%|█████████████████████████████████████████████████████████████████████████▎ | 64374/110696 [19:41:03<11:23:06, 1.13it/s][2025-04-27 16:44:29] (step=0064375) Train Loss: 5.7330, Train Steps/Sec: 1.12 + 58%|█████████████████████████████████████████████████████████████████████████▎ | 64399/110696 [19:41:25<11:20:09, 1.13it/s][2025-04-27 16:44:51] (step=0064400) Train Loss: 5.7844, Train Steps/Sec: 1.12 + 58%|█████████████████████████████████████████████████████████████████████████▎ | 64424/110696 [19:41:48<11:41:53, 1.10it/s][2025-04-27 16:45:13] (step=0064425) Train Loss: 5.7352, Train Steps/Sec: 1.12 + 58%|█████████████████████████████████████████████████████████████████████████▎ | 64449/110696 [19:42:10<11:34:43, 1.11it/s][2025-04-27 16:45:36] (step=0064450) Train Loss: 5.7401, Train Steps/Sec: 1.12 + 58%|█████████████████████████████████████████████████████████████████████████▍ | 64474/110696 [19:42:32<11:28:16, 1.12it/s][2025-04-27 16:45:58] (step=0064475) Train Loss: 5.8244, Train Steps/Sec: 1.12 + 58%|█████████████████████████████████████████████████████████████████████████▍ | 64499/110696 [19:42:55<11:27:07, 1.12it/s][2025-04-27 16:46:20] (step=0064500) Train Loss: 5.8563, Train Steps/Sec: 1.12 + 58%|█████████████████████████████████████████████████████████████████████████▍ | 64524/110696 [19:43:17<11:20:50, 1.13it/s][2025-04-27 16:46:42] (step=0064525) Train Loss: 5.7654, Train Steps/Sec: 1.12 + 58%|█████████████████████████████████████████████████████████████████████████▍ | 64549/110696 [19:43:39<11:22:37, 1.13it/s][2025-04-27 16:47:05] (step=0064550) Train Loss: 5.7478, Train Steps/Sec: 1.12 + 58%|█████████████████████████████████████████████████████████████████████████▌ | 64574/110696 [19:44:02<11:18:30, 1.13it/s][2025-04-27 16:47:27] (step=0064575) Train Loss: 5.7938, Train Steps/Sec: 1.12 + 58%|█████████████████████████████████████████████████████████████████████████▌ | 64599/110696 [19:44:24<11:15:31, 1.14it/s][2025-04-27 16:47:49] (step=0064600) Train Loss: 5.7252, Train Steps/Sec: 1.12 + 58%|█████████████████████████████████████████████████████████████████████████▌ | 64624/110696 [19:44:46<11:42:47, 1.09it/s][2025-04-27 16:48:12] (step=0064625) Train Loss: 5.6685, Train Steps/Sec: 1.12 + 58%|█████████████████████████████████████████████████████████████████████████▌ | 64649/110696 [19:45:09<11:30:45, 1.11it/s][2025-04-27 16:48:34] (step=0064650) Train Loss: 5.7458, Train Steps/Sec: 1.12 + 58%|█████████████████████████████████████████████████████████████████████████▌ | 64674/110696 [19:45:31<11:23:16, 1.12it/s][2025-04-27 16:48:56] (step=0064675) Train Loss: 5.7520, Train Steps/Sec: 1.12 + 58%|█████████████████████████████████████████████████████████████████████████▋ | 64699/110696 [19:45:53<11:23:04, 1.12it/s][2025-04-27 16:49:19] (step=0064700) Train Loss: 5.7685, Train Steps/Sec: 1.12 + 58%|█████████████████████████████████████████████████████████████████████████▋ | 64724/110696 [19:46:15<11:22:20, 1.12it/s][2025-04-27 16:49:41] (step=0064725) Train Loss: 5.7923, Train Steps/Sec: 1.12 + 58%|█████████████████████████████████████████████████████████████████████████▋ | 64749/110696 [19:46:38<11:15:27, 1.13it/s][2025-04-27 16:50:03] (step=0064750) Train Loss: 5.8430, Train Steps/Sec: 1.12 + 59%|█████████████████████████████████████████████████████████████████████████▋ | 64774/110696 [19:47:00<11:15:22, 1.13it/s][2025-04-27 16:50:25] (step=0064775) Train Loss: 5.7164, Train Steps/Sec: 1.13 + 59%|█████████████████████████████████████████████████████████████████████████▊ | 64799/110696 [19:47:22<11:14:46, 1.13it/s][2025-04-27 16:50:48] (step=0064800) Train Loss: 5.7685, Train Steps/Sec: 1.12 + 59%|█████████████████████████████████████████████████████████████████████████▊ | 64824/110696 [19:47:45<11:37:52, 1.10it/s][2025-04-27 16:51:10] (step=0064825) Train Loss: 5.7588, Train Steps/Sec: 1.12 + 59%|█████████████████████████████████████████████████████████████████████████▊ | 64849/110696 [19:48:07<11:27:00, 1.11it/s][2025-04-27 16:51:32] (step=0064850) Train Loss: 5.8585, Train Steps/Sec: 1.12 + 59%|█████████████████████████████████████████████████████████████████████████▊ | 64874/110696 [19:48:29<11:22:36, 1.12it/s][2025-04-27 16:51:55] (step=0064875) Train Loss: 5.7571, Train Steps/Sec: 1.12 + 59%|█████████████████████████████████████████████████████████████████████████▊ | 64899/110696 [19:48:51<11:20:35, 1.12it/s][2025-04-27 16:52:17] (step=0064900) Train Loss: 5.7059, Train Steps/Sec: 1.12 + 59%|█████████████████████████████████████████████████████████████████████████▉ | 64924/110696 [19:49:14<11:14:18, 1.13it/s][2025-04-27 16:52:39] (step=0064925) Train Loss: 5.7625, Train Steps/Sec: 1.12 + 59%|█████████████████████████████████████████████████████████████████████████▉ | 64949/110696 [19:49:36<11:13:03, 1.13it/s][2025-04-27 16:53:01] (step=0064950) Train Loss: 5.7666, Train Steps/Sec: 1.12 + 59%|█████████████████████████████████████████████████████████████████████████▉ | 64974/110696 [19:49:58<11:16:21, 1.13it/s][2025-04-27 16:53:24] (step=0064975) Train Loss: 5.6663, Train Steps/Sec: 1.12 + 59%|█████████████████████████████████████████████████████████████████████████▉ | 64999/110696 [19:50:21<11:10:59, 1.14it/s][2025-04-27 16:53:46] (step=0065000) Train Loss: 5.7398, Train Steps/Sec: 1.12 + 59%|██████████████████████████████████████████████████████████████████████████ | 65024/110696 [19:50:43<11:35:45, 1.09it/s][2025-04-27 16:54:08] (step=0065025) Train Loss: 5.7139, Train Steps/Sec: 1.12 + 59%|██████████████████████████████████████████████████████████████████████████ | 65049/110696 [19:51:05<11:23:20, 1.11it/s][2025-04-27 16:54:31] (step=0065050) Train Loss: 5.7711, Train Steps/Sec: 1.12 + 59%|██████████████████████████████████████████████████████████████████████████ | 65074/110696 [19:51:28<11:19:15, 1.12it/s][2025-04-27 16:54:53] (step=0065075) Train Loss: 5.7995, Train Steps/Sec: 1.12 + 59%|██████████████████████████████████████████████████████████████████████████ | 65099/110696 [19:51:50<11:17:05, 1.12it/s][2025-04-27 16:55:15] (step=0065100) Train Loss: 5.7179, Train Steps/Sec: 1.12 + 59%|██████████████████████████████████████████████████████████████████████████▏ | 65124/110696 [19:52:12<11:14:22, 1.13it/s][2025-04-27 16:55:38] (step=0065125) Train Loss: 5.7962, Train Steps/Sec: 1.12 + 59%|██████████████████████████████████████████████████████████████████████████▏ | 65149/110696 [19:52:35<11:12:29, 1.13it/s][2025-04-27 16:56:00] (step=0065150) Train Loss: 5.7755, Train Steps/Sec: 1.12 + 59%|██████████████████████████████████████████████████████████████████████████▏ | 65174/110696 [19:52:57<11:14:01, 1.13it/s][2025-04-27 16:56:22] (step=0065175) Train Loss: 5.7072, Train Steps/Sec: 1.12 + 59%|██████████████████████████████████████████████████████████████████████████▏ | 65199/110696 [19:53:19<11:08:42, 1.13it/s][2025-04-27 16:56:45] (step=0065200) Train Loss: 5.7031, Train Steps/Sec: 1.12 + 59%|██████████████████████████████████████████████████████████████████████████▏ | 65224/110696 [19:53:41<11:29:01, 1.10it/s][2025-04-27 16:57:07] (step=0065225) Train Loss: 5.7400, Train Steps/Sec: 1.12 + 59%|██████████████████████████████████████████████████████████████████████████▎ | 65249/110696 [19:54:04<11:19:27, 1.11it/s][2025-04-27 16:57:29] (step=0065250) Train Loss: 5.7622, Train Steps/Sec: 1.12 + 59%|██████████████████████████████████████████████████████████████████████████▎ | 65274/110696 [19:54:26<11:16:29, 1.12it/s][2025-04-27 16:57:51] (step=0065275) Train Loss: 5.7448, Train Steps/Sec: 1.12 + 59%|██████████████████████████████████████████████████████████████████████████▎ | 65299/110696 [19:54:48<11:11:58, 1.13it/s][2025-04-27 16:58:14] (step=0065300) Train Loss: 5.7653, Train Steps/Sec: 1.12 + 59%|██████████████████████████████████████████████████████████████████████████▎ | 65324/110696 [19:55:11<11:11:25, 1.13it/s][2025-04-27 16:58:36] (step=0065325) Train Loss: 5.8096, Train Steps/Sec: 1.12 + 59%|██████████████████████████████████████████████████████████████████████████▍ | 65349/110696 [19:55:33<11:08:34, 1.13it/s][2025-04-27 16:58:58] (step=0065350) Train Loss: 5.7688, Train Steps/Sec: 1.12 + 59%|██████████████████████████████████████████████████████████████████████████▍ | 65374/110696 [19:55:55<11:07:17, 1.13it/s][2025-04-27 16:59:21] (step=0065375) Train Loss: 5.7737, Train Steps/Sec: 1.12 + 59%|██████████████████████████████████████████████████████████████████████████▍ | 65399/110696 [19:56:18<11:09:29, 1.13it/s][2025-04-27 16:59:43] (step=0065400) Train Loss: 5.7429, Train Steps/Sec: 1.12 + 59%|██████████████████████████████████████████████████████████████████████████▍ | 65424/110696 [19:56:40<11:28:46, 1.10it/s][2025-04-27 17:00:05] (step=0065425) Train Loss: 5.8250, Train Steps/Sec: 1.12 + 59%|██████████████████████████████████████████████████████████████████████████▍ | 65449/110696 [19:57:02<11:18:20, 1.11it/s][2025-04-27 17:00:28] (step=0065450) Train Loss: 5.7000, Train Steps/Sec: 1.12 + 59%|██████████████████████████████████████████████████████████████████████████▌ | 65474/110696 [19:57:25<11:14:16, 1.12it/s][2025-04-27 17:00:50] (step=0065475) Train Loss: 5.8099, Train Steps/Sec: 1.12 + 59%|██████████████████████████████████████████████████████████████████████████▌ | 65499/110696 [19:57:47<11:10:51, 1.12it/s][2025-04-27 17:01:12] (step=0065500) Train Loss: 5.8023, Train Steps/Sec: 1.12 + 59%|██████████████████████████████████████████████████████████████████████████▌ | 65524/110696 [19:58:09<11:06:13, 1.13it/s][2025-04-27 17:01:35] (step=0065525) Train Loss: 5.8064, Train Steps/Sec: 1.12 + 59%|██████████████████████████████████████████████████████████████████████████▌ | 65549/110696 [19:58:31<11:05:18, 1.13it/s][2025-04-27 17:01:57] (step=0065550) Train Loss: 5.7611, Train Steps/Sec: 1.12 + 59%|██████████████████████████████████████████████████████████████████████████▋ | 65574/110696 [19:58:54<11:02:57, 1.13it/s][2025-04-27 17:02:19] (step=0065575) Train Loss: 5.7641, Train Steps/Sec: 1.12 + 59%|██████████████████████████████████████████████████████████████████████████▋ | 65599/110696 [19:59:16<11:04:38, 1.13it/s][2025-04-27 17:02:41] (step=0065600) Train Loss: 5.7722, Train Steps/Sec: 1.12 + 59%|██████████████████████████████████████████████████████████████████████████▋ | 65624/110696 [19:59:38<11:22:27, 1.10it/s][2025-04-27 17:03:04] (step=0065625) Train Loss: 5.7229, Train Steps/Sec: 1.12 + 59%|██████████████████████████████████████████████████████████████████████████▋ | 65649/110696 [20:00:01<11:15:22, 1.11it/s][2025-04-27 17:03:26] (step=0065650) Train Loss: 5.7628, Train Steps/Sec: 1.12 + 59%|██████████████████████████████████████████████████████████████████████████▊ | 65674/110696 [20:00:23<11:08:52, 1.12it/s][2025-04-27 17:03:48] (step=0065675) Train Loss: 5.7431, Train Steps/Sec: 1.12 + 59%|██████████████████████████████████████████████████████████████████████████▊ | 65699/110696 [20:00:45<11:08:28, 1.12it/s][2025-04-27 17:04:11] (step=0065700) Train Loss: 5.6945, Train Steps/Sec: 1.12 + 59%|██████████████████████████████████████████████████████████████████████████▊ | 65724/110696 [20:01:08<11:05:11, 1.13it/s][2025-04-27 17:04:33] (step=0065725) Train Loss: 5.8078, Train Steps/Sec: 1.12 + 59%|██████████████████████████████████████████████████████████████████████████▊ | 65749/110696 [20:01:30<11:03:32, 1.13it/s][2025-04-27 17:04:55] (step=0065750) Train Loss: 5.7606, Train Steps/Sec: 1.12 + 59%|██████████████████████████████████████████████████████████████████████████▊ | 65774/110696 [20:01:52<11:00:48, 1.13it/s][2025-04-27 17:05:17] (step=0065775) Train Loss: 5.7167, Train Steps/Sec: 1.12 + 59%|██████████████████████████████████████████████████████████████████████████▉ | 65799/110696 [20:02:14<11:04:11, 1.13it/s][2025-04-27 17:05:40] (step=0065800) Train Loss: 5.7902, Train Steps/Sec: 1.12 + 59%|██████████████████████████████████████████████████████████████████████████▉ | 65824/110696 [20:02:37<11:22:48, 1.10it/s][2025-04-27 17:06:02] (step=0065825) Train Loss: 5.7732, Train Steps/Sec: 1.11 + 59%|██████████████████████████████████████████████████████████████████████████▉ | 65849/110696 [20:02:59<11:15:15, 1.11it/s][2025-04-27 17:06:25] (step=0065850) Train Loss: 5.7592, Train Steps/Sec: 1.12 + 60%|██████████████████████████████████████████████████████████████████████████▉ | 65874/110696 [20:03:21<11:05:38, 1.12it/s][2025-04-27 17:06:47] (step=0065875) Train Loss: 5.6994, Train Steps/Sec: 1.12 + 60%|███████████████████████████████████████████████████████████████████████████ | 65899/110696 [20:03:44<11:04:34, 1.12it/s][2025-04-27 17:07:09] (step=0065900) Train Loss: 5.7150, Train Steps/Sec: 1.12 + 60%|███████████████████████████████████████████████████████████████████████████ | 65924/110696 [20:04:06<11:03:02, 1.13it/s][2025-04-27 17:07:31] (step=0065925) Train Loss: 5.7667, Train Steps/Sec: 1.12 + 60%|███████████████████████████████████████████████████████████████████████████ | 65949/110696 [20:04:28<11:00:18, 1.13it/s][2025-04-27 17:07:54] (step=0065950) Train Loss: 5.7466, Train Steps/Sec: 1.12 + 60%|███████████████████████████████████████████████████████████████████████████ | 65974/110696 [20:04:51<11:01:43, 1.13it/s][2025-04-27 17:08:16] (step=0065975) Train Loss: 5.7648, Train Steps/Sec: 1.12 + 60%|███████████████████████████████████████████████████████████████████████████ | 65999/110696 [20:05:13<10:58:55, 1.13it/s][2025-04-27 17:08:38] (step=0066000) Train Loss: 5.7544, Train Steps/Sec: 1.12 +You are using a model of type blip-2 to instantiate a model of type instructblip. This is not supported for all configurations of models and can yield errors. +[2025-04-27 17:08:38] text_config is None. Initializing the text config with default values (`OPTConfig`). +model_name_or_path:/tmp/haozhezhao/model/blip2-flan-t5-xl +image_place_holder: +tokenizer length after expend 32101 +tokenizer length before expend 32100 +100%|████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████| 5/5 [05:05<00:00, 61.11s/it] +[2025-04-27 17:14:58] Finish Eval in 66000 steps...█████████████████████████████████████████████████████████████████████████████████████████████| 5/5 [05:04<00:00, 60.74s/it] +processing left-padding... +processing left-padding... +processing left-padding... +processing left-padding... +processing left-padding... +[2025-04-27 17:15:18] Saved checkpoint to checkpoint/X2I_UltraEdiit_fourmask_LLava_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_filtered_objects_4mask_1280_context/003-GPT-XL/checkpoints/0066000.pt +[2025-04-27 17:15:20] Removed old checkpoint: checkpoint/X2I_UltraEdiit_fourmask_LLava_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_filtered_objects_4mask_1280_context/003-GPT-XL/checkpoints/0064000.pt + 60%|███████████████████████████████████████████████████████████████████████████▏ | 66024/110696 [20:12:18<11:37:00, 1.07it/s][2025-04-27 17:15:43] (step=0066025) Train Loss: 5.7493, Train Steps/Sec: 0.06 + 60%|███████████████████████████████████████████████████████████████████████████▏ | 66049/110696 [20:12:40<11:10:34, 1.11it/s][2025-04-27 17:16:05] (step=0066050) Train Loss: 5.7910, Train Steps/Sec: 1.12 + 60%|███████████████████████████████████████████████████████████████████████████▏ | 66074/110696 [20:13:02<11:04:44, 1.12it/s][2025-04-27 17:16:28] (step=0066075) Train Loss: 5.7336, Train Steps/Sec: 1.12 + 60%|███████████████████████████████████████████████████████████████████████████▏ | 66099/110696 [20:13:25<11:00:39, 1.13it/s][2025-04-27 17:16:50] (step=0066100) Train Loss: 5.7509, Train Steps/Sec: 1.12 + 60%|███████████████████████████████████████████████████████████████████████████▎ | 66124/110696 [20:13:47<10:58:17, 1.13it/s][2025-04-27 17:17:12] (step=0066125) Train Loss: 5.7552, Train Steps/Sec: 1.12 + 60%|███████████████████████████████████████████████████████████████████████████▎ | 66149/110696 [20:14:09<10:58:18, 1.13it/s][2025-04-27 17:17:34] (step=0066150) Train Loss: 5.6799, Train Steps/Sec: 1.12 + 60%|███████████████████████████████████████████████████████████████████████████▎ | 66174/110696 [20:14:31<10:55:32, 1.13it/s][2025-04-27 17:17:57] (step=0066175) Train Loss: 5.7296, Train Steps/Sec: 1.12 + 60%|███████████████████████████████████████████████████████████████████████████▎ | 66199/110696 [20:14:54<10:54:42, 1.13it/s][2025-04-27 17:18:19] (step=0066200) Train Loss: 5.7656, Train Steps/Sec: 1.12 + 60%|███████████████████████████████████████████████████████████████████████████▍ | 66224/110696 [20:15:17<11:15:17, 1.10it/s][2025-04-27 17:18:42] (step=0066225) Train Loss: 5.7875, Train Steps/Sec: 1.12 + 60%|███████████████████████████████████████████████████████████████████████████▍ | 66249/110696 [20:15:39<11:06:41, 1.11it/s][2025-04-27 17:19:04] (step=0066250) Train Loss: 5.7311, Train Steps/Sec: 1.12 + 60%|███████████████████████████████████████████████████████████████████████████▍ | 66274/110696 [20:16:01<11:02:31, 1.12it/s][2025-04-27 17:19:26] (step=0066275) Train Loss: 5.7400, Train Steps/Sec: 1.12 + 60%|███████████████████████████████████████████████████████████████████████████▍ | 66299/110696 [20:16:23<11:02:10, 1.12it/s][2025-04-27 17:19:49] (step=0066300) Train Loss: 5.7605, Train Steps/Sec: 1.12 + 60%|███████████████████████████████████████████████████████████████████████████▍ | 66324/110696 [20:16:46<10:56:56, 1.13it/s][2025-04-27 17:20:11] (step=0066325) Train Loss: 5.8007, Train Steps/Sec: 1.12 + 60%|███████████████████████████████████████████████████████████████████████████▌ | 66349/110696 [20:17:08<10:52:43, 1.13it/s][2025-04-27 17:20:33] (step=0066350) Train Loss: 5.7957, Train Steps/Sec: 1.12 + 60%|███████████████████████████████████████████████████████████████████████████▌ | 66374/110696 [20:17:30<10:53:14, 1.13it/s][2025-04-27 17:20:56] (step=0066375) Train Loss: 5.7241, Train Steps/Sec: 1.12 + 60%|███████████████████████████████████████████████████████████████████████████▌ | 66399/110696 [20:17:53<10:51:22, 1.13it/s][2025-04-27 17:21:18] (step=0066400) Train Loss: 5.7466, Train Steps/Sec: 1.12 + 60%|███████████████████████████████████████████████████████████████████████████▌ | 66424/110696 [20:18:15<11:10:21, 1.10it/s][2025-04-27 17:21:40] (step=0066425) Train Loss: 5.8021, Train Steps/Sec: 1.12 + 60%|███████████████████████████████████████████████████████████████████████████▋ | 66449/110696 [20:18:37<11:06:04, 1.11it/s][2025-04-27 17:22:03] (step=0066450) Train Loss: 5.7209, Train Steps/Sec: 1.12 + 60%|███████████████████████████████████████████████████████████████████████████▋ | 66474/110696 [20:19:00<10:58:25, 1.12it/s][2025-04-27 17:22:25] (step=0066475) Train Loss: 5.7719, Train Steps/Sec: 1.12 + 60%|███████████████████████████████████████████████████████████████████████████▋ | 66499/110696 [20:19:22<10:56:57, 1.12it/s][2025-04-27 17:22:47] (step=0066500) Train Loss: 5.7879, Train Steps/Sec: 1.12 + 60%|███████████████████████████████████████████████████████████████████████████▋ | 66524/110696 [20:19:44<10:53:09, 1.13it/s][2025-04-27 17:23:10] (step=0066525) Train Loss: 5.7381, Train Steps/Sec: 1.12 + 60%|███████████████████████████████████████████████████████████████████████████▋ | 66549/110696 [20:20:07<10:50:45, 1.13it/s][2025-04-27 17:23:32] (step=0066550) Train Loss: 5.7753, Train Steps/Sec: 1.12 + 60%|███████████████████████████████████████████████████████████████████████████▊ | 66574/110696 [20:20:33<11:31:49, 1.06it/s][2025-04-27 17:23:59] (step=0066575) Train Loss: 5.7429, Train Steps/Sec: 0.93 + 60%|███████████████████████████████████████████████████████████████████████████▊ | 66599/110696 [20:20:56<10:48:55, 1.13it/s][2025-04-27 17:24:21] (step=0066600) Train Loss: 5.7274, Train Steps/Sec: 1.12 + 60%|███████████████████████████████████████████████████████████████████████████▊ | 66624/110696 [20:21:18<11:12:29, 1.09it/s][2025-04-27 17:24:43] (step=0066625) Train Loss: 5.7660, Train Steps/Sec: 1.12 + 60%|███████████████████████████████████████████████████████████████████████████▊ | 66649/110696 [20:21:40<11:00:03, 1.11it/s][2025-04-27 17:25:06] (step=0066650) Train Loss: 5.7113, Train Steps/Sec: 1.12 + 60%|███████████████████████████████████████████████████████████████████████████▉ | 66674/110696 [20:22:03<10:57:13, 1.12it/s][2025-04-27 17:25:28] (step=0066675) Train Loss: 5.7964, Train Steps/Sec: 1.12 + 60%|███████████████████████████████████████████████████████████████████████████▉ | 66699/110696 [20:22:25<10:50:57, 1.13it/s][2025-04-27 17:25:50] (step=0066700) Train Loss: 5.7528, Train Steps/Sec: 1.12 + 60%|███████████████████████████████████████████████████████████████████████████▉ | 66724/110696 [20:22:47<10:52:40, 1.12it/s][2025-04-27 17:26:12] (step=0066725) Train Loss: 5.7495, Train Steps/Sec: 1.12 + 60%|███████████████████████████████████████████████████████████████████████████▉ | 66749/110696 [20:23:14<11:09:23, 1.09it/s][2025-04-27 17:26:40] (step=0066750) Train Loss: 5.7939, Train Steps/Sec: 0.93 + 60%|████████████████████████████████████████████████████████████████████████████ | 66774/110696 [20:23:36<10:45:36, 1.13it/s][2025-04-27 17:27:02] (step=0066775) Train Loss: 5.7285, Train Steps/Sec: 1.12 + 60%|████████████████████████████████████████████████████████████████████████████ | 66799/110696 [20:23:59<10:46:03, 1.13it/s][2025-04-27 17:27:24] (step=0066800) Train Loss: 5.7746, Train Steps/Sec: 1.12 + 60%|████████████████████████████████████████████████████████████████████████████ | 66824/110696 [20:24:21<11:06:02, 1.10it/s][2025-04-27 17:27:47] (step=0066825) Train Loss: 5.7017, Train Steps/Sec: 1.12 + 60%|████████████████████████████████████████████████████████████████████████████ | 66849/110696 [20:24:44<10:58:59, 1.11it/s][2025-04-27 17:28:09] (step=0066850) Train Loss: 5.7074, Train Steps/Sec: 1.12 + 60%|████████████████████████████████████████████████████████████████████████████ | 66874/110696 [20:25:06<10:54:58, 1.12it/s][2025-04-27 17:28:31] (step=0066875) Train Loss: 5.8048, Train Steps/Sec: 1.12 + 60%|████████████████████████████████████████████████████████████████████████████▏ | 66899/110696 [20:25:28<10:49:11, 1.12it/s][2025-04-27 17:28:54] (step=0066900) Train Loss: 5.7790, Train Steps/Sec: 1.12 + 60%|████████████████████████████████████████████████████████████████████████████▏ | 66924/110696 [20:25:50<10:48:10, 1.13it/s][2025-04-27 17:29:16] (step=0066925) Train Loss: 5.8089, Train Steps/Sec: 1.12 + 60%|████████████████████████████████████████████████████████████████████████████▏ | 66949/110696 [20:26:13<10:46:29, 1.13it/s][2025-04-27 17:29:38] (step=0066950) Train Loss: 5.7355, Train Steps/Sec: 1.12 + 61%|████████████████████████████████████████████████████████████████████████████▏ | 66974/110696 [20:26:35<10:45:00, 1.13it/s][2025-04-27 17:30:00] (step=0066975) Train Loss: 5.7345, Train Steps/Sec: 1.12 + 61%|████████████████████████████████████████████████████████████████████████████▎ | 66999/110696 [20:26:57<10:44:18, 1.13it/s][2025-04-27 17:30:23] (step=0067000) Train Loss: 5.7640, Train Steps/Sec: 1.12 + 61%|████████████████████████████████████████████████████████████████████████████▎ | 67024/110696 [20:27:20<11:02:51, 1.10it/s][2025-04-27 17:30:45] (step=0067025) Train Loss: 5.7251, Train Steps/Sec: 1.12 + 61%|████████████████████████████████████████████████████████████████████████████▎ | 67049/110696 [20:27:42<10:55:48, 1.11it/s][2025-04-27 17:31:07] (step=0067050) Train Loss: 5.7479, Train Steps/Sec: 1.12 + 61%|████████████████████████████████████████████████████████████████████████████▎ | 67074/110696 [20:28:04<10:49:18, 1.12it/s][2025-04-27 17:31:30] (step=0067075) Train Loss: 5.7227, Train Steps/Sec: 1.12 + 61%|████████████████████████████████████████████████████████████████████████████▍ | 67099/110696 [20:28:27<10:45:21, 1.13it/s][2025-04-27 17:31:52] (step=0067100) Train Loss: 5.7378, Train Steps/Sec: 1.12 + 61%|████████████████████████████████████████████████████████████████████████████▍ | 67124/110696 [20:28:49<10:46:18, 1.12it/s][2025-04-27 17:32:14] (step=0067125) Train Loss: 5.7598, Train Steps/Sec: 1.12 + 61%|████████████████████████████████████████████████████████████████████████████▍ | 67149/110696 [20:29:11<10:43:43, 1.13it/s][2025-04-27 17:32:37] (step=0067150) Train Loss: 5.6923, Train Steps/Sec: 1.12 + 61%|████████████████████████████████████████████████████████████████████████████▍ | 67174/110696 [20:29:34<10:43:58, 1.13it/s][2025-04-27 17:32:59] (step=0067175) Train Loss: 5.7084, Train Steps/Sec: 1.12 + 61%|████████████████████████████████████████████████████████████████████████████▍ | 67199/110696 [20:29:56<10:41:15, 1.13it/s][2025-04-27 17:33:21] (step=0067200) Train Loss: 5.7806, Train Steps/Sec: 1.12 + 61%|████████████████████████████████████████████████████████████████████████████▌ | 67224/110696 [20:30:18<11:01:56, 1.09it/s][2025-04-27 17:33:44] (step=0067225) Train Loss: 5.7828, Train Steps/Sec: 1.12 + 61%|████████████████████████████████████████████████████████████████████████████▌ | 67249/110696 [20:30:41<10:51:06, 1.11it/s][2025-04-27 17:34:06] (step=0067250) Train Loss: 5.7608, Train Steps/Sec: 1.12 + 61%|████████████████████████████████████████████████████████████████████████████▌ | 67274/110696 [20:31:03<10:48:31, 1.12it/s][2025-04-27 17:34:28] (step=0067275) Train Loss: 5.7505, Train Steps/Sec: 1.12 + 61%|████████████████████████████████████████████████████████████████████████████▌ | 67299/110696 [20:31:29<10:51:06, 1.11it/s][2025-04-27 17:34:55] (step=0067300) Train Loss: 5.7716, Train Steps/Sec: 0.96 + 61%|████████████████████████████████████████████████████████████████████████████▋ | 67324/110696 [20:32:00<11:47:25, 1.02it/s][2025-04-27 17:35:26] (step=0067325) Train Loss: 5.7723, Train Steps/Sec: 0.81 + 61%|████████████████████████████████████████████████████████████████████████████▋ | 67349/110696 [20:32:28<23:46:12, 1.97s/it][2025-04-27 17:35:53] (step=0067350) Train Loss: 5.7964, Train Steps/Sec: 0.91 + 61%|████████████████████████████████████████████████████████████████████████████▋ | 67374/110696 [20:32:50<10:37:59, 1.13it/s][2025-04-27 17:36:15] (step=0067375) Train Loss: 5.8184, Train Steps/Sec: 1.12 + 61%|████████████████████████████████████████████████████████████████████████████▋ | 67399/110696 [20:33:12<10:37:29, 1.13it/s][2025-04-27 17:36:38] (step=0067400) Train Loss: 5.7927, Train Steps/Sec: 1.12 + 61%|████████████████████████████████████████████████████████████████████████████▋ | 67424/110696 [20:33:35<10:57:19, 1.10it/s][2025-04-27 17:37:00] (step=0067425) Train Loss: 5.7378, Train Steps/Sec: 1.12 + 61%|████████████████████████████████████████████████████████████████████████████▊ | 67449/110696 [20:34:02<10:48:59, 1.11it/s][2025-04-27 17:37:27] (step=0067450) Train Loss: 5.7308, Train Steps/Sec: 0.91 + 61%|████████████████████████████████████████████████████████████████████████████▊ | 67474/110696 [20:34:24<10:43:48, 1.12it/s][2025-04-27 17:37:50] (step=0067475) Train Loss: 5.7467, Train Steps/Sec: 1.12 + 61%|████████████████████████████████████████████████████████████████████████████▊ | 67499/110696 [20:34:47<10:39:42, 1.13it/s][2025-04-27 17:38:12] (step=0067500) Train Loss: 5.7464, Train Steps/Sec: 1.12 + 61%|████████████████████████████████████████████████████████████████████████████▊ | 67524/110696 [20:35:09<10:45:15, 1.12it/s][2025-04-27 17:38:34] (step=0067525) Train Loss: 5.7944, Train Steps/Sec: 1.12 + 61%|████████████████████████████████████████████████████████████████████████████▉ | 67549/110696 [20:35:31<10:35:56, 1.13it/s][2025-04-27 17:38:57] (step=0067550) Train Loss: 5.7935, Train Steps/Sec: 1.12 + 61%|████████████████████████████████████████████████████████████████████████████▉ | 67574/110696 [20:35:53<10:34:27, 1.13it/s][2025-04-27 17:39:19] (step=0067575) Train Loss: 5.7861, Train Steps/Sec: 1.12 + 61%|████████████████████████████████████████████████████████████████████████████▉ | 67599/110696 [20:36:16<10:34:58, 1.13it/s][2025-04-27 17:39:41] (step=0067600) Train Loss: 5.6957, Train Steps/Sec: 1.12 + 61%|████████████████████████████████████████████████████████████████████████████▉ | 67624/110696 [20:36:38<10:55:57, 1.09it/s][2025-04-27 17:40:04] (step=0067625) Train Loss: 5.7342, Train Steps/Sec: 1.12 + 61%|█████████████████████████████████████████████████████████████████████████████ | 67649/110696 [20:37:00<10:45:12, 1.11it/s][2025-04-27 17:40:26] (step=0067650) Train Loss: 5.7345, Train Steps/Sec: 1.12 + 61%|█████████████████████████████████████████████████████████████████████████████ | 67674/110696 [20:37:23<10:39:09, 1.12it/s][2025-04-27 17:40:48] (step=0067675) Train Loss: 5.8059, Train Steps/Sec: 1.12 + 61%|█████████████████████████████████████████████████████████████████████████████ | 67699/110696 [20:37:45<10:38:53, 1.12it/s][2025-04-27 17:41:10] (step=0067700) Train Loss: 5.6982, Train Steps/Sec: 1.12 + 61%|█████████████████████████████████████████████████████████████████████████████ | 67724/110696 [20:38:07<10:36:08, 1.13it/s][2025-04-27 17:41:33] (step=0067725) Train Loss: 5.7323, Train Steps/Sec: 1.12 + 61%|█████████████████████████████████████████████████████████████████████████████ | 67749/110696 [20:38:30<10:32:18, 1.13it/s][2025-04-27 17:41:55] (step=0067750) Train Loss: 5.7328, Train Steps/Sec: 1.12 + 61%|█████████████████████████████████████████████████████████████████████████████▏ | 67774/110696 [20:38:52<10:31:19, 1.13it/s][2025-04-27 17:42:17] (step=0067775) Train Loss: 5.7475, Train Steps/Sec: 1.12 + 61%|█████████████████████████████████████████████████████████████████████████████▏ | 67799/110696 [20:39:14<10:31:20, 1.13it/s][2025-04-27 17:42:40] (step=0067800) Train Loss: 5.7930, Train Steps/Sec: 1.12 + 61%|█████████████████████████████████████████████████████████████████████████████▏ | 67824/110696 [20:39:37<10:52:02, 1.10it/s][2025-04-27 17:43:02] (step=0067825) Train Loss: 5.7570, Train Steps/Sec: 1.12 + 61%|█████████████████████████████████████████████████████████████████████████████▏ | 67849/110696 [20:39:59<10:40:45, 1.11it/s][2025-04-27 17:43:24] (step=0067850) Train Loss: 5.7218, Train Steps/Sec: 1.12 + 61%|█████████████████████████████████████████████████████████████████████████████▎ | 67874/110696 [20:40:21<10:39:14, 1.12it/s][2025-04-27 17:43:46] (step=0067875) Train Loss: 5.7603, Train Steps/Sec: 1.12 + 61%|█████████████████████████████████████████████████████████████████████████████▎ | 67899/110696 [20:40:43<10:34:50, 1.12it/s][2025-04-27 17:44:09] (step=0067900) Train Loss: 5.7319, Train Steps/Sec: 1.12 + 61%|█████████████████████████████████████████████████████████████████████████████▎ | 67924/110696 [20:41:06<10:32:38, 1.13it/s][2025-04-27 17:44:31] (step=0067925) Train Loss: 5.7260, Train Steps/Sec: 1.12 + 61%|█████████████████████████████████████████████████████████████████████████████▎ | 67949/110696 [20:41:28<10:29:54, 1.13it/s][2025-04-27 17:44:53] (step=0067950) Train Loss: 5.7465, Train Steps/Sec: 1.12 + 61%|█████████████████████████████████████████████████████████████████████████████▎ | 67974/110696 [20:41:50<10:29:03, 1.13it/s][2025-04-27 17:45:16] (step=0067975) Train Loss: 5.7541, Train Steps/Sec: 1.12 + 61%|█████████████████████████████████████████████████████████████████████████████▍ | 67999/110696 [20:42:12<10:28:50, 1.13it/s][2025-04-27 17:45:38] (step=0068000) Train Loss: 5.8193, Train Steps/Sec: 1.12 +You are using a model of type blip-2 to instantiate a model of type instructblip. This is not supported for all configurations of models and can yield errors. +[2025-04-27 17:45:38] text_config is None. Initializing the text config with default values (`OPTConfig`). +model_name_or_path:/tmp/haozhezhao/model/blip2-flan-t5-xl +image_place_holder: +tokenizer length after expend 32101 +tokenizer length before expend 32100 +100%|████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████| 5/5 [05:05<00:00, 61.20s/it] +[2025-04-27 17:51:58] Finish Eval in 68000 steps...█████████████████████████████████████████████████████████████████████████████████████████████| 5/5 [05:05<00:00, 60.83s/it] +processing left-padding... +processing left-padding... +processing left-padding... +processing left-padding... +processing left-padding... +[2025-04-27 17:52:18] Saved checkpoint to checkpoint/X2I_UltraEdiit_fourmask_LLava_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_filtered_objects_4mask_1280_context/003-GPT-XL/checkpoints/0068000.pt +[2025-04-27 17:52:20] Removed old checkpoint: checkpoint/X2I_UltraEdiit_fourmask_LLava_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_filtered_objects_4mask_1280_context/003-GPT-XL/checkpoints/0066000.pt + 61%|█████████████████████████████████████████████████████████████████████████████▍ | 68024/110696 [20:49:17<11:02:51, 1.07it/s][2025-04-27 17:52:43] (step=0068025) Train Loss: 5.7897, Train Steps/Sec: 0.06 + 61%|█████████████████████████████████████████████████████████████████████████████▍ | 68049/110696 [20:49:40<10:36:56, 1.12it/s][2025-04-27 17:53:05] (step=0068050) Train Loss: 5.7264, Train Steps/Sec: 1.12 + 61%|█████████████████████████████████████████████████████████████████████████████▍ | 68074/110696 [20:50:02<10:34:26, 1.12it/s][2025-04-27 17:53:27] (step=0068075) Train Loss: 5.7102, Train Steps/Sec: 1.12 + 62%|█████████████████████████████████████████████████████████████████████████████▌ | 68099/110696 [20:50:24<10:31:09, 1.12it/s][2025-04-27 17:53:50] (step=0068100) Train Loss: 5.8192, Train Steps/Sec: 1.12 + 62%|█████████████████████████████████████████████████████████████████████████████▌ | 68124/110696 [20:50:47<10:31:35, 1.12it/s][2025-04-27 17:54:12] (step=0068125) Train Loss: 5.7023, Train Steps/Sec: 1.12 + 62%|█████████████████████████████████████████████████████████████████████████████▌ | 68149/110696 [20:51:09<10:24:53, 1.13it/s][2025-04-27 17:54:35] (step=0068150) Train Loss: 5.7768, Train Steps/Sec: 1.12 + 62%|█████████████████████████████████████████████████████████████████████████████▌ | 68174/110696 [20:51:32<10:25:42, 1.13it/s][2025-04-27 17:54:57] (step=0068175) Train Loss: 5.7650, Train Steps/Sec: 1.12 + 62%|█████████████████████████████████████████████████████████████████████████████▋ | 68199/110696 [20:51:54<10:23:51, 1.14it/s][2025-04-27 17:55:19] (step=0068200) Train Loss: 5.7337, Train Steps/Sec: 1.12 + 62%|█████████████████████████████████████████████████████████████████████████████▋ | 68224/110696 [20:52:16<10:43:02, 1.10it/s][2025-04-27 17:55:42] (step=0068225) Train Loss: 5.7147, Train Steps/Sec: 1.12 + 62%|█████████████████████████████████████████████████████████████████████████████▋ | 68249/110696 [20:52:38<10:35:14, 1.11it/s][2025-04-27 17:56:04] (step=0068250) Train Loss: 5.6666, Train Steps/Sec: 1.12 + 62%|█████████████████████████████████████████████████████████████████████████████▋ | 68274/110696 [20:53:01<10:29:29, 1.12it/s][2025-04-27 17:56:26] (step=0068275) Train Loss: 5.7900, Train Steps/Sec: 1.12 + 62%|█████████████████████████████████████████████████████████████████████████████▋ | 68299/110696 [20:53:23<10:28:31, 1.12it/s][2025-04-27 17:56:48] (step=0068300) Train Loss: 5.7424, Train Steps/Sec: 1.13 + 62%|█████████████████████████████████████████████████████████████████████████████▊ | 68324/110696 [20:53:45<10:27:15, 1.13it/s][2025-04-27 17:57:11] (step=0068325) Train Loss: 5.7320, Train Steps/Sec: 1.12 + 62%|█████████████████████████████████████████████████████████████████████████████▊ | 68349/110696 [20:54:07<10:23:16, 1.13it/s][2025-04-27 17:57:33] (step=0068350) Train Loss: 5.7284, Train Steps/Sec: 1.12 + 62%|█████████████████████████████████████████████████████████████████████████████▊ | 68374/110696 [20:54:30<10:21:42, 1.13it/s][2025-04-27 17:57:55] (step=0068375) Train Loss: 5.7486, Train Steps/Sec: 1.12 + 62%|█████████████████████████████████████████████████████████████████████████████▊ | 68399/110696 [20:54:52<10:19:40, 1.14it/s][2025-04-27 17:58:17] (step=0068400) Train Loss: 5.6980, Train Steps/Sec: 1.12 + 62%|█████████████████████████████████████████████████████████████████████████████▉ | 68424/110696 [20:55:14<10:43:37, 1.09it/s][2025-04-27 17:58:40] (step=0068425) Train Loss: 5.7812, Train Steps/Sec: 1.12 + 62%|█████████████████████████████████████████████████████████████████████████████▉ | 68449/110696 [20:55:37<10:32:44, 1.11it/s][2025-04-27 17:59:02] (step=0068450) Train Loss: 5.7079, Train Steps/Sec: 1.12 + 62%|█████████████████████████████████████████████████████████████████████████████▉ | 68474/110696 [20:55:59<10:28:40, 1.12it/s][2025-04-27 17:59:24] (step=0068475) Train Loss: 5.7727, Train Steps/Sec: 1.12 + 62%|█████████████████████████████████████████████████████████████████████████████▉ | 68499/110696 [20:56:21<10:26:09, 1.12it/s][2025-04-27 17:59:46] (step=0068500) Train Loss: 5.7567, Train Steps/Sec: 1.12 + 62%|█████████████████████████████████████████████████████████████████████████████▉ | 68524/110696 [20:56:43<10:23:58, 1.13it/s][2025-04-27 18:00:09] (step=0068525) Train Loss: 5.7080, Train Steps/Sec: 1.12 + 62%|██████████████████████████████████████████████████████████████████████████████ | 68549/110696 [20:57:06<10:20:14, 1.13it/s][2025-04-27 18:00:31] (step=0068550) Train Loss: 5.7637, Train Steps/Sec: 1.12 + 62%|██████████████████████████████████████████████████████████████████████████████ | 68574/110696 [20:57:28<10:19:58, 1.13it/s][2025-04-27 18:00:53] (step=0068575) Train Loss: 5.7026, Train Steps/Sec: 1.12 + 62%|██████████████████████████████████████████████████████████████████████████████ | 68599/110696 [20:57:50<10:17:59, 1.14it/s][2025-04-27 18:01:16] (step=0068600) Train Loss: 5.8155, Train Steps/Sec: 1.12 + 62%|██████████████████████████████████████████████████████████████████████████████ | 68624/110696 [20:58:12<10:38:18, 1.10it/s][2025-04-27 18:01:38] (step=0068625) Train Loss: 5.7846, Train Steps/Sec: 1.12 + 62%|██████████████████████████████████████████████████████████████████████████████▏ | 68649/110696 [20:58:35<10:28:13, 1.12it/s][2025-04-27 18:02:00] (step=0068650) Train Loss: 5.7280, Train Steps/Sec: 1.13 + 62%|██████████████████████████████████████████████████████████████████████████████▏ | 68674/110696 [20:58:57<10:27:29, 1.12it/s][2025-04-27 18:02:22] (step=0068675) Train Loss: 5.7887, Train Steps/Sec: 1.12 + 62%|██████████████████████████████████████████████████████████████████████████████▏ | 68699/110696 [20:59:19<10:22:48, 1.12it/s][2025-04-27 18:02:45] (step=0068700) Train Loss: 5.7409, Train Steps/Sec: 1.12 + 62%|██████████████████████████████████████████████████████████████████████████████▏ | 68724/110696 [20:59:42<10:21:43, 1.13it/s][2025-04-27 18:03:07] (step=0068725) Train Loss: 5.6965, Train Steps/Sec: 1.12 + 62%|██████████████████████████████████████████████████████████████████████████████▎ | 68749/110696 [21:00:04<10:17:25, 1.13it/s][2025-04-27 18:03:29] (step=0068750) Train Loss: 5.7537, Train Steps/Sec: 1.12 + 62%|██████████████████████████████████████████████████████████████████████████████▎ | 68774/110696 [21:00:26<10:18:31, 1.13it/s][2025-04-27 18:03:52] (step=0068775) Train Loss: 5.6965, Train Steps/Sec: 1.12 + 62%|██████████████████████████████████████████████████████████████████████████████▎ | 68799/110696 [21:00:48<10:14:41, 1.14it/s][2025-04-27 18:04:14] (step=0068800) Train Loss: 5.7998, Train Steps/Sec: 1.12 + 62%|██████████████████████████████████████████████████████████████████████████████▎ | 68824/110696 [21:01:11<10:37:09, 1.10it/s][2025-04-27 18:04:36] (step=0068825) Train Loss: 5.6756, Train Steps/Sec: 1.12 + 62%|██████████████████████████████████████████████████████████████████████████████▎ | 68849/110696 [21:01:33<10:25:22, 1.12it/s][2025-04-27 18:04:58] (step=0068850) Train Loss: 5.7876, Train Steps/Sec: 1.12 + 62%|██████████████████████████████████████████████████████████████████████████████▍ | 68874/110696 [21:01:55<10:21:28, 1.12it/s][2025-04-27 18:05:21] (step=0068875) Train Loss: 5.8342, Train Steps/Sec: 1.12 + 62%|██████████████████████████████████████████████████████████████████████████████▍ | 68899/110696 [21:02:17<10:16:58, 1.13it/s][2025-04-27 18:05:43] (step=0068900) Train Loss: 5.7272, Train Steps/Sec: 1.13 + 62%|██████████████████████████████████████████████████████████████████████████████▍ | 68924/110696 [21:02:40<10:14:34, 1.13it/s][2025-04-27 18:06:05] (step=0068925) Train Loss: 5.7128, Train Steps/Sec: 1.13 + 62%|██████████████████████████████████████████████████████████████████████████████▍ | 68949/110696 [21:03:02<10:14:45, 1.13it/s][2025-04-27 18:06:27] (step=0068950) Train Loss: 5.7007, Train Steps/Sec: 1.12 + 62%|██████████████████████████████████████████████████████████████████████████████▌ | 68974/110696 [21:03:24<10:15:02, 1.13it/s][2025-04-27 18:06:50] (step=0068975) Train Loss: 5.8050, Train Steps/Sec: 1.13 + 62%|██████████████████████████████████████████████████████████████████████████████▌ | 68999/110696 [21:03:46<10:12:29, 1.13it/s][2025-04-27 18:07:12] (step=0069000) Train Loss: 5.7134, Train Steps/Sec: 1.12 + 62%|██████████████████████████████████████████████████████████████████████████████▌ | 69024/110696 [21:04:09<10:30:22, 1.10it/s][2025-04-27 18:07:34] (step=0069025) Train Loss: 5.8004, Train Steps/Sec: 1.13 + 62%|██████████████████████████████████████████████████████████████████████████████▌ | 69049/110696 [21:04:31<10:23:37, 1.11it/s][2025-04-27 18:07:56] (step=0069050) Train Loss: 5.7390, Train Steps/Sec: 1.12 + 62%|██████████████████████████████████████████████████████████████████████████████▌ | 69074/110696 [21:04:53<10:17:42, 1.12it/s][2025-04-27 18:08:19] (step=0069075) Train Loss: 5.7418, Train Steps/Sec: 1.12 + 62%|██████████████████████████████████████████████████████████████████████████████▋ | 69099/110696 [21:05:16<10:14:40, 1.13it/s][2025-04-27 18:08:41] (step=0069100) Train Loss: 5.8079, Train Steps/Sec: 1.12 + 62%|██████████████████████████████████████████████████████████████████████████████▋ | 69124/110696 [21:05:38<10:14:35, 1.13it/s][2025-04-27 18:09:03] (step=0069125) Train Loss: 5.7736, Train Steps/Sec: 1.13 + 62%|██████████████████████████████████████████████████████████████████████████████▋ | 69149/110696 [21:06:00<10:13:24, 1.13it/s][2025-04-27 18:09:25] (step=0069150) Train Loss: 5.7290, Train Steps/Sec: 1.12 + 62%|██████████████████████████████████████████████████████████████████████████████▋ | 69174/110696 [21:06:22<10:09:42, 1.14it/s][2025-04-27 18:09:48] (step=0069175) Train Loss: 5.7348, Train Steps/Sec: 1.12 + 63%|██████████████████████████████████████████████████████████████████████████████▊ | 69199/110696 [21:06:45<10:08:27, 1.14it/s][2025-04-27 18:10:10] (step=0069200) Train Loss: 5.7389, Train Steps/Sec: 1.12 + 63%|██████████████████████████████████████████████████████████████████████████████▊ | 69224/110696 [21:07:07<10:28:05, 1.10it/s][2025-04-27 18:10:32] (step=0069225) Train Loss: 5.6673, Train Steps/Sec: 1.13 + 63%|██████████████████████████████████████████████████████████████████████████████▊ | 69249/110696 [21:07:29<10:21:04, 1.11it/s][2025-04-27 18:10:55] (step=0069250) Train Loss: 5.7965, Train Steps/Sec: 1.12 + 63%|██████████████████████████████████████████████████████████████████████████████▊ | 69274/110696 [21:07:51<10:17:07, 1.12it/s][2025-04-27 18:11:17] (step=0069275) Train Loss: 5.7417, Train Steps/Sec: 1.12 + 63%|██████████████████████████████████████████████████████████████████████████████▉ | 69299/110696 [21:08:14<10:14:58, 1.12it/s][2025-04-27 18:11:39] (step=0069300) Train Loss: 5.7141, Train Steps/Sec: 1.12 + 63%|██████████████████████████████████████████████████████████████████████████████▉ | 69324/110696 [21:08:36<10:11:51, 1.13it/s][2025-04-27 18:12:01] (step=0069325) Train Loss: 5.7526, Train Steps/Sec: 1.12 + 63%|██████████████████████████████████████████████████████████████████████████████▉ | 69349/110696 [21:08:58<10:09:07, 1.13it/s][2025-04-27 18:12:24] (step=0069350) Train Loss: 5.8276, Train Steps/Sec: 1.13 + 63%|██████████████████████████████████████████████████████████████████████████████▉ | 69374/110696 [21:09:21<10:08:12, 1.13it/s][2025-04-27 18:12:46] (step=0069375) Train Loss: 5.7238, Train Steps/Sec: 1.12 + 63%|██████████████████████████████████████████████████████████████████████████████▉ | 69399/110696 [21:09:43<10:06:22, 1.14it/s][2025-04-27 18:13:08] (step=0069400) Train Loss: 5.7763, Train Steps/Sec: 1.12 + 63%|███████████████████████████████████████████████████████████████████████████████ | 69424/110696 [21:10:05<10:23:28, 1.10it/s][2025-04-27 18:13:31] (step=0069425) Train Loss: 5.7821, Train Steps/Sec: 1.12 + 63%|███████████████████████████████████████████████████████████████████████████████ | 69449/110696 [21:10:28<10:15:42, 1.12it/s][2025-04-27 18:13:53] (step=0069450) Train Loss: 5.7973, Train Steps/Sec: 1.12 + 63%|███████████████████████████████████████████████████████████████████████████████ | 69474/110696 [21:10:50<10:13:33, 1.12it/s][2025-04-27 18:14:15] (step=0069475) Train Loss: 5.7290, Train Steps/Sec: 1.12 + 63%|███████████████████████████████████████████████████████████████████████████████ | 69499/110696 [21:11:12<10:10:30, 1.12it/s][2025-04-27 18:14:37] (step=0069500) Train Loss: 5.7711, Train Steps/Sec: 1.12 + 63%|███████████████████████████████████████████████████████████████████████████████▏ | 69524/110696 [21:11:34<10:07:57, 1.13it/s][2025-04-27 18:15:00] (step=0069525) Train Loss: 5.7610, Train Steps/Sec: 1.13 + 63%|███████████████████████████████████████████████████████████████████████████████▏ | 69549/110696 [21:11:57<10:06:13, 1.13it/s][2025-04-27 18:15:22] (step=0069550) Train Loss: 5.7397, Train Steps/Sec: 1.12 + 63%|███████████████████████████████████████████████████████████████████████████████▏ | 69574/110696 [21:12:19<10:06:47, 1.13it/s][2025-04-27 18:15:44] (step=0069575) Train Loss: 5.7663, Train Steps/Sec: 1.12 + 63%|███████████████████████████████████████████████████████████████████████████████▏ | 69599/110696 [21:12:41<10:05:30, 1.13it/s][2025-04-27 18:16:07] (step=0069600) Train Loss: 5.7835, Train Steps/Sec: 1.12 + 63%|███████████████████████████████████████████████████████████████████████████████▏ | 69624/110696 [21:13:04<10:25:32, 1.09it/s][2025-04-27 18:16:29] (step=0069625) Train Loss: 5.6902, Train Steps/Sec: 1.12 + 63%|███████████████████████████████████████████████████████████████████████████████▎ | 69649/110696 [21:13:26<10:16:01, 1.11it/s][2025-04-27 18:16:51] (step=0069650) Train Loss: 5.7882, Train Steps/Sec: 1.12 + 63%|███████████████████████████████████████████████████████████████████████████████▎ | 69674/110696 [21:13:48<10:10:03, 1.12it/s][2025-04-27 18:17:13] (step=0069675) Train Loss: 5.6815, Train Steps/Sec: 1.12 + 63%|███████████████████████████████████████████████████████████████████████████████▎ | 69699/110696 [21:14:10<10:08:37, 1.12it/s][2025-04-27 18:17:36] (step=0069700) Train Loss: 5.7482, Train Steps/Sec: 1.12 + 63%|███████████████████████████████████████████████████████████████████████████████▎ | 69724/110696 [21:14:33<10:05:12, 1.13it/s][2025-04-27 18:17:58] (step=0069725) Train Loss: 5.7828, Train Steps/Sec: 1.12 + 63%|███████████████████████████████████████████████████████████████████████████████▍ | 69749/110696 [21:14:55<10:05:47, 1.13it/s][2025-04-27 18:18:20] (step=0069750) Train Loss: 5.7409, Train Steps/Sec: 1.12 + 63%|███████████████████████████████████████████████████████████████████████████████▍ | 69774/110696 [21:15:17<10:05:01, 1.13it/s][2025-04-27 18:18:43] (step=0069775) Train Loss: 5.6648, Train Steps/Sec: 1.12 + 63%|███████████████████████████████████████████████████████████████████████████████▍ | 69799/110696 [21:15:40<10:03:09, 1.13it/s][2025-04-27 18:19:05] (step=0069800) Train Loss: 5.7995, Train Steps/Sec: 1.11 + 63%|███████████████████████████████████████████████████████████████████████████████▍ | 69824/110696 [21:16:02<10:21:34, 1.10it/s][2025-04-27 18:19:27] (step=0069825) Train Loss: 5.7686, Train Steps/Sec: 1.12 + 63%|███████████████████████████████████████████████████████████████████████████████▌ | 69849/110696 [21:16:24<10:09:45, 1.12it/s][2025-04-27 18:19:50] (step=0069850) Train Loss: 5.7451, Train Steps/Sec: 1.12 + 63%|███████████████████████████████████████████████████████████████████████████████▌ | 69874/110696 [21:16:46<10:07:20, 1.12it/s][2025-04-27 18:20:12] (step=0069875) Train Loss: 5.7720, Train Steps/Sec: 1.12 + 63%|███████████████████████████████████████████████████████████████████████████████▌ | 69899/110696 [21:17:09<10:05:32, 1.12it/s][2025-04-27 18:20:34] (step=0069900) Train Loss: 5.7113, Train Steps/Sec: 1.12 + 63%|███████████████████████████████████████████████████████████████████████████████▌ | 69924/110696 [21:17:31<10:02:46, 1.13it/s][2025-04-27 18:20:56] (step=0069925) Train Loss: 5.6982, Train Steps/Sec: 1.12 + 63%|███████████████████████████████████████████████████████████████████████████████▌ | 69949/110696 [21:17:53<10:00:55, 1.13it/s][2025-04-27 18:21:19] (step=0069950) Train Loss: 5.7595, Train Steps/Sec: 1.12 + 63%|███████████████████████████████████████████████████████████████████████████████▋ | 69974/110696 [21:18:16<10:02:20, 1.13it/s][2025-04-27 18:21:41] (step=0069975) Train Loss: 5.7736, Train Steps/Sec: 1.12 + 63%|███████████████████████████████████████████████████████████████████████████████▋ | 69999/110696 [21:18:38<10:01:05, 1.13it/s][2025-04-27 18:22:03] (step=0070000) Train Loss: 5.7053, Train Steps/Sec: 1.12 +You are using a model of type blip-2 to instantiate a model of type instructblip. This is not supported for all configurations of models and can yield errors. +[2025-04-27 18:22:03] text_config is None. Initializing the text config with default values (`OPTConfig`). +model_name_or_path:/tmp/haozhezhao/model/blip2-flan-t5-xl +image_place_holder: +tokenizer length after expend 32101 +tokenizer length before expend 32100 +100%|████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████| 5/5 [05:04<00:00, 61.00s/it] +[2025-04-27 18:28:22] Finish Eval in 70000 steps...█████████████████████████████████████████████████████████████████████████████████████████████| 5/5 [05:04<00:00, 60.67s/it] +processing left-padding... +processing left-padding... +processing left-padding... +processing left-padding... +processing left-padding... +[2025-04-27 18:28:43] Saved checkpoint to checkpoint/X2I_UltraEdiit_fourmask_LLava_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_filtered_objects_4mask_1280_context/003-GPT-XL/checkpoints/0070000.pt +[2025-04-27 18:28:45] Removed old checkpoint: checkpoint/X2I_UltraEdiit_fourmask_LLava_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_filtered_objects_4mask_1280_context/003-GPT-XL/checkpoints/0068000.pt + 63%|███████████████████████████████████████████████████████████████████████████████▋ | 70024/110696 [21:25:42<10:34:44, 1.07it/s][2025-04-27 18:29:08] (step=0070025) Train Loss: 5.7028, Train Steps/Sec: 0.06 + 63%|███████████████████████████████████████████████████████████████████████████████▋ | 70049/110696 [21:26:04<10:10:13, 1.11it/s][2025-04-27 18:29:30] (step=0070050) Train Loss: 5.8009, Train Steps/Sec: 1.12 + 63%|███████████████████████████████████████████████████████████████████████████████▊ | 70074/110696 [21:26:27<10:03:21, 1.12it/s][2025-04-27 18:29:52] (step=0070075) Train Loss: 5.7069, Train Steps/Sec: 1.12 + 63%|███████████████████████████████████████████████████████████████████████████████▊ | 70099/110696 [21:26:49<10:00:58, 1.13it/s][2025-04-27 18:30:14] (step=0070100) Train Loss: 5.7105, Train Steps/Sec: 1.12 + 63%|███████████████████████████████████████████████████████████████████████████████▊ | 70124/110696 [21:27:11<10:00:26, 1.13it/s][2025-04-27 18:30:37] (step=0070125) Train Loss: 5.8008, Train Steps/Sec: 1.12 + 63%|████████████████████████████████████████████████████████████████████████████████▍ | 70149/110696 [21:27:34<9:56:07, 1.13it/s][2025-04-27 18:30:59] (step=0070150) Train Loss: 5.7598, Train Steps/Sec: 1.12 + 63%|████████████████████████████████████████████████████████████████████████████████▌ | 70174/110696 [21:27:56<9:55:11, 1.13it/s][2025-04-27 18:31:21] (step=0070175) Train Loss: 5.6955, Train Steps/Sec: 1.12 + 63%|████████████████████████████████████████████████████████████████████████████████▌ | 70199/110696 [21:28:18<9:59:00, 1.13it/s][2025-04-27 18:31:44] (step=0070200) Train Loss: 5.7582, Train Steps/Sec: 1.12 + 63%|███████████████████████████████████████████████████████████████████████████████▉ | 70224/110696 [21:28:40<10:16:38, 1.09it/s][2025-04-27 18:32:06] (step=0070225) Train Loss: 5.7420, Train Steps/Sec: 1.12 + 63%|███████████████████████████████████████████████████████████████████████████████▉ | 70249/110696 [21:29:03<10:06:22, 1.11it/s][2025-04-27 18:32:28] (step=0070250) Train Loss: 5.7596, Train Steps/Sec: 1.12 + 63%|███████████████████████████████████████████████████████████████████████████████▉ | 70274/110696 [21:29:25<10:05:29, 1.11it/s][2025-04-27 18:32:50] (step=0070275) Train Loss: 5.7119, Train Steps/Sec: 1.12 + 64%|████████████████████████████████████████████████████████████████████████████████▋ | 70299/110696 [21:29:47<9:57:55, 1.13it/s][2025-04-27 18:33:13] (step=0070300) Train Loss: 5.7568, Train Steps/Sec: 1.12 + 64%|████████████████████████████████████████████████████████████████████████████████▋ | 70324/110696 [21:30:10<9:56:22, 1.13it/s][2025-04-27 18:33:35] (step=0070325) Train Loss: 5.7608, Train Steps/Sec: 1.12 + 64%|████████████████████████████████████████████████████████████████████████████████▋ | 70349/110696 [21:30:32<9:58:05, 1.12it/s][2025-04-27 18:33:57] (step=0070350) Train Loss: 5.7527, Train Steps/Sec: 1.12 + 64%|████████████████████████████████████████████████████████████████████████████████▋ | 70374/110696 [21:30:54<9:52:13, 1.13it/s][2025-04-27 18:34:20] (step=0070375) Train Loss: 5.7280, Train Steps/Sec: 1.12 + 64%|████████████████████████████████████████████████████████████████████████████████▊ | 70399/110696 [21:31:17<9:55:54, 1.13it/s][2025-04-27 18:34:42] (step=0070400) Train Loss: 5.7472, Train Steps/Sec: 1.12 + 64%|████████████████████████████████████████████████████████████████████████████████▏ | 70424/110696 [21:31:39<10:09:32, 1.10it/s][2025-04-27 18:35:04] (step=0070425) Train Loss: 5.8042, Train Steps/Sec: 1.12 + 64%|████████████████████████████████████████████████████████████████████████████████▏ | 70449/110696 [21:32:01<10:05:40, 1.11it/s][2025-04-27 18:35:27] (step=0070450) Train Loss: 5.7715, Train Steps/Sec: 1.12 + 64%|████████████████████████████████████████████████████████████████████████████████▏ | 70474/110696 [21:32:24<10:01:49, 1.11it/s][2025-04-27 18:35:49] (step=0070475) Train Loss: 5.7702, Train Steps/Sec: 1.12 + 64%|████████████████████████████████████████████████████████████████████████████████▉ | 70499/110696 [21:32:46<9:57:46, 1.12it/s][2025-04-27 18:36:11] (step=0070500) Train Loss: 5.8133, Train Steps/Sec: 1.12 + 64%|████████████████████████████████████████████████████████████████████████████████▉ | 70524/110696 [21:33:08<9:54:59, 1.13it/s][2025-04-27 18:36:34] (step=0070525) Train Loss: 5.7727, Train Steps/Sec: 1.12 + 64%|████████████████████████████████████████████████████████████████████████████████▉ | 70549/110696 [21:33:30<9:52:49, 1.13it/s][2025-04-27 18:36:56] (step=0070550) Train Loss: 5.6694, Train Steps/Sec: 1.12 + 64%|████████████████████████████████████████████████████████████████████████████████▉ | 70574/110696 [21:33:53<9:53:58, 1.13it/s][2025-04-27 18:37:18] (step=0070575) Train Loss: 5.7688, Train Steps/Sec: 1.12 + 64%|████████████████████████████████████████████████████████████████████████████████▉ | 70599/110696 [21:34:15<9:49:23, 1.13it/s][2025-04-27 18:37:41] (step=0070600) Train Loss: 5.7314, Train Steps/Sec: 1.12 + 64%|████████████████████████████████████████████████████████████████████████████████▍ | 70624/110696 [21:34:38<10:07:09, 1.10it/s][2025-04-27 18:38:03] (step=0070625) Train Loss: 5.8011, Train Steps/Sec: 1.12 + 64%|█████████████████████████████████████████████████████████████████████████████████ | 70649/110696 [21:35:00<9:59:05, 1.11it/s][2025-04-27 18:38:25] (step=0070650) Train Loss: 5.8194, Train Steps/Sec: 1.12 + 64%|█████████████████████████████████████████████████████████████████████████████████ | 70674/110696 [21:35:22<9:58:55, 1.11it/s][2025-04-27 18:38:48] (step=0070675) Train Loss: 5.7389, Train Steps/Sec: 1.12 + 64%|█████████████████████████████████████████████████████████████████████████████████ | 70699/110696 [21:35:44<9:50:53, 1.13it/s][2025-04-27 18:39:10] (step=0070700) Train Loss: 5.7040, Train Steps/Sec: 1.12 + 64%|█████████████████████████████████████████████████████████████████████████████████▏ | 70724/110696 [21:36:07<9:51:47, 1.13it/s][2025-04-27 18:39:32] (step=0070725) Train Loss: 5.7279, Train Steps/Sec: 1.12 + 64%|█████████████████████████████████████████████████████████████████████████████████▏ | 70749/110696 [21:36:29<9:48:46, 1.13it/s][2025-04-27 18:39:54] (step=0070750) Train Loss: 5.7617, Train Steps/Sec: 1.12 + 64%|█████████████████████████████████████████████████████████████████████████████████▏ | 70774/110696 [21:36:51<9:51:03, 1.13it/s][2025-04-27 18:40:17] (step=0070775) Train Loss: 5.7676, Train Steps/Sec: 1.12 + 64%|█████████████████████████████████████████████████████████████████████████████████▏ | 70799/110696 [21:37:14<9:48:24, 1.13it/s][2025-04-27 18:40:39] (step=0070800) Train Loss: 5.7442, Train Steps/Sec: 1.12 + 64%|████████████████████████████████████████████████████████████████████████████████▌ | 70824/110696 [21:37:40<11:46:50, 1.06s/it][2025-04-27 18:41:06] (step=0070825) Train Loss: 5.7646, Train Steps/Sec: 0.94 + 64%|█████████████████████████████████████████████████████████████████████████████████▎ | 70849/110696 [21:38:03<9:55:58, 1.11it/s][2025-04-27 18:41:28] (step=0070850) Train Loss: 5.7242, Train Steps/Sec: 1.12 + 64%|█████████████████████████████████████████████████████████████████████████████████▎ | 70874/110696 [21:38:25<9:53:55, 1.12it/s][2025-04-27 18:41:50] (step=0070875) Train Loss: 5.7253, Train Steps/Sec: 1.12 + 64%|█████████████████████████████████████████████████████████████████████████████████▎ | 70899/110696 [21:38:47<9:50:28, 1.12it/s][2025-04-27 18:42:13] (step=0070900) Train Loss: 5.7293, Train Steps/Sec: 1.12 + 64%|█████████████████████████████████████████████████████████████████████████████████▎ | 70924/110696 [21:39:10<9:50:49, 1.12it/s][2025-04-27 18:42:35] (step=0070925) Train Loss: 5.7248, Train Steps/Sec: 1.12 + 64%|█████████████████████████████████████████████████████████████████████████████████▍ | 70949/110696 [21:39:32<9:48:26, 1.13it/s][2025-04-27 18:42:57] (step=0070950) Train Loss: 5.7474, Train Steps/Sec: 1.12 + 64%|█████████████████████████████████████████████████████████████████████████████████▍ | 70974/110696 [21:39:54<9:45:24, 1.13it/s][2025-04-27 18:43:20] (step=0070975) Train Loss: 5.7027, Train Steps/Sec: 1.12 + 64%|█████████████████████████████████████████████████████████████████████████████████▍ | 70999/110696 [21:40:17<9:47:07, 1.13it/s][2025-04-27 18:43:42] (step=0071000) Train Loss: 5.6776, Train Steps/Sec: 1.12 + 64%|████████████████████████████████████████████████████████████████████████████████▊ | 71024/110696 [21:40:44<11:21:57, 1.03s/it][2025-04-27 18:44:09] (step=0071025) Train Loss: 5.7592, Train Steps/Sec: 0.92 + 64%|█████████████████████████████████████████████████████████████████████████████████▌ | 71049/110696 [21:41:06<9:52:26, 1.12it/s][2025-04-27 18:44:32] (step=0071050) Train Loss: 5.6859, Train Steps/Sec: 1.12 + 64%|█████████████████████████████████████████████████████████████████████████████████▌ | 71074/110696 [21:41:28<9:50:06, 1.12it/s][2025-04-27 18:44:54] (step=0071075) Train Loss: 5.7517, Train Steps/Sec: 1.12 + 64%|█████████████████████████████████████████████████████████████████████████████████▌ | 71099/110696 [21:41:51<9:47:22, 1.12it/s][2025-04-27 18:45:16] (step=0071100) Train Loss: 5.7736, Train Steps/Sec: 1.12 + 64%|█████████████████████████████████████████████████████████████████████████████████▌ | 71124/110696 [21:42:13<9:47:13, 1.12it/s][2025-04-27 18:45:38] (step=0071125) Train Loss: 5.7502, Train Steps/Sec: 1.12 + 64%|█████████████████████████████████████████████████████████████████████████████████▋ | 71149/110696 [21:42:35<9:43:04, 1.13it/s][2025-04-27 18:46:01] (step=0071150) Train Loss: 5.7146, Train Steps/Sec: 1.12 + 64%|█████████████████████████████████████████████████████████████████████████████████▋ | 71174/110696 [21:42:58<9:42:21, 1.13it/s][2025-04-27 18:46:23] (step=0071175) Train Loss: 5.6955, Train Steps/Sec: 1.12 + 64%|█████████████████████████████████████████████████████████████████████████████████▋ | 71199/110696 [21:43:20<9:42:59, 1.13it/s][2025-04-27 18:46:45] (step=0071200) Train Loss: 5.6876, Train Steps/Sec: 1.11 + 64%|█████████████████████████████████████████████████████████████████████████████████▋ | 71224/110696 [21:43:42<9:56:49, 1.10it/s][2025-04-27 18:47:08] (step=0071225) Train Loss: 5.7820, Train Steps/Sec: 1.12 + 64%|█████████████████████████████████████████████████████████████████████████████████▋ | 71249/110696 [21:44:05<9:50:19, 1.11it/s][2025-04-27 18:47:30] (step=0071250) Train Loss: 5.7298, Train Steps/Sec: 1.12 + 64%|█████████████████████████████████████████████████████████████████████████████████▊ | 71274/110696 [21:44:27<9:48:49, 1.12it/s][2025-04-27 18:47:52] (step=0071275) Train Loss: 5.7708, Train Steps/Sec: 1.12 + 64%|█████████████████████████████████████████████████████████████████████████████████▊ | 71299/110696 [21:44:49<9:44:52, 1.12it/s][2025-04-27 18:48:15] (step=0071300) Train Loss: 5.7604, Train Steps/Sec: 1.12 + 64%|█████████████████████████████████████████████████████████████████████████████████▊ | 71324/110696 [21:45:11<9:41:44, 1.13it/s][2025-04-27 18:48:37] (step=0071325) Train Loss: 5.7145, Train Steps/Sec: 1.12 + 64%|█████████████████████████████████████████████████████████████████████████████████▊ | 71349/110696 [21:45:34<9:41:14, 1.13it/s][2025-04-27 18:48:59] (step=0071350) Train Loss: 5.7147, Train Steps/Sec: 1.12 + 64%|█████████████████████████████████████████████████████████████████████████████████▉ | 71374/110696 [21:45:56<9:41:39, 1.13it/s][2025-04-27 18:49:21] (step=0071375) Train Loss: 5.7400, Train Steps/Sec: 1.12 + 65%|█████████████████████████████████████████████████████████████████████████████████▉ | 71399/110696 [21:46:18<9:36:19, 1.14it/s][2025-04-27 18:49:44] (step=0071400) Train Loss: 5.6741, Train Steps/Sec: 1.11 + 65%|█████████████████████████████████████████████████████████████████████████████████▉ | 71424/110696 [21:46:41<9:53:54, 1.10it/s][2025-04-27 18:50:06] (step=0071425) Train Loss: 5.7642, Train Steps/Sec: 1.12 + 65%|█████████████████████████████████████████████████████████████████████████████████▉ | 71449/110696 [21:47:03<9:49:16, 1.11it/s][2025-04-27 18:50:28] (step=0071450) Train Loss: 5.7298, Train Steps/Sec: 1.12 + 65%|██████████████████████████████████████████████████████████████████████████████████ | 71474/110696 [21:47:25<9:45:07, 1.12it/s][2025-04-27 18:50:51] (step=0071475) Train Loss: 5.7392, Train Steps/Sec: 1.12 + 65%|██████████████████████████████████████████████████████████████████████████████████ | 71499/110696 [21:47:48<9:40:15, 1.13it/s][2025-04-27 18:51:13] (step=0071500) Train Loss: 5.7823, Train Steps/Sec: 1.12 + 65%|██████████████████████████████████████████████████████████████████████████████████ | 71524/110696 [21:48:10<9:39:20, 1.13it/s][2025-04-27 18:51:35] (step=0071525) Train Loss: 5.6913, Train Steps/Sec: 1.12 + 65%|██████████████████████████████████████████████████████████████████████████████████ | 71549/110696 [21:48:32<9:35:12, 1.13it/s][2025-04-27 18:51:58] (step=0071550) Train Loss: 5.8002, Train Steps/Sec: 1.12 + 65%|██████████████████████████████████████████████████████████████████████████████████ | 71574/110696 [21:48:55<9:33:35, 1.14it/s][2025-04-27 18:52:20] (step=0071575) Train Loss: 5.7080, Train Steps/Sec: 1.12 + 65%|██████████████████████████████████████████████████████████████████████████████████▏ | 71599/110696 [21:49:17<9:34:22, 1.13it/s][2025-04-27 18:52:42] (step=0071600) Train Loss: 5.7649, Train Steps/Sec: 1.11 + 65%|█████████████████████████████████████████████████████████████████████████████████▌ | 71624/110696 [21:49:43<10:05:38, 1.08it/s][2025-04-27 18:53:08] (step=0071625) Train Loss: 5.7800, Train Steps/Sec: 0.96 + 65%|█████████████████████████████████████████████████████████████████████████████████▌ | 71649/110696 [21:50:14<15:06:48, 1.39s/it][2025-04-27 18:53:40] (step=0071650) Train Loss: 5.7631, Train Steps/Sec: 0.80 + 65%|██████████████████████████████████████████████████████████████████████████████████▏ | 71674/110696 [21:50:37<9:43:30, 1.11it/s][2025-04-27 18:54:02] (step=0071675) Train Loss: 5.6343, Train Steps/Sec: 1.12 + 65%|██████████████████████████████████████████████████████████████████████████████████▎ | 71699/110696 [21:51:04<9:45:30, 1.11it/s][2025-04-27 18:54:30] (step=0071700) Train Loss: 5.7202, Train Steps/Sec: 0.90 + 65%|██████████████████████████████████████████████████████████████████████████████████▎ | 71724/110696 [21:51:27<9:33:43, 1.13it/s][2025-04-27 18:54:52] (step=0071725) Train Loss: 5.7995, Train Steps/Sec: 1.12 + 65%|██████████████████████████████████████████████████████████████████████████████████▎ | 71749/110696 [21:51:49<9:36:47, 1.13it/s][2025-04-27 18:55:14] (step=0071750) Train Loss: 5.7438, Train Steps/Sec: 1.12 + 65%|██████████████████████████████████████████████████████████████████████████████████▎ | 71774/110696 [21:52:11<9:31:59, 1.13it/s][2025-04-27 18:55:36] (step=0071775) Train Loss: 5.7041, Train Steps/Sec: 1.12 + 65%|██████████████████████████████████████████████████████████████████████████████████▎ | 71799/110696 [21:52:33<9:33:21, 1.13it/s][2025-04-27 18:55:59] (step=0071800) Train Loss: 5.7477, Train Steps/Sec: 1.12 + 65%|██████████████████████████████████████████████████████████████████████████████████▍ | 71824/110696 [21:52:56<9:53:31, 1.09it/s][2025-04-27 18:56:21] (step=0071825) Train Loss: 5.7196, Train Steps/Sec: 1.12 + 65%|██████████████████████████████████████████████████████████████████████████████████▍ | 71849/110696 [21:53:23<9:49:14, 1.10it/s][2025-04-27 18:56:48] (step=0071850) Train Loss: 5.8030, Train Steps/Sec: 0.92 + 65%|██████████████████████████████████████████████████████████████████████████████████▍ | 71874/110696 [21:53:45<9:37:10, 1.12it/s][2025-04-27 18:57:11] (step=0071875) Train Loss: 5.7486, Train Steps/Sec: 1.12 + 65%|██████████████████████████████████████████████████████████████████████████████████▍ | 71899/110696 [21:54:08<9:37:30, 1.12it/s][2025-04-27 18:57:33] (step=0071900) Train Loss: 5.7364, Train Steps/Sec: 1.12 + 65%|██████████████████████████████████████████████████████████████████████████████████▌ | 71924/110696 [21:54:30<9:33:56, 1.13it/s][2025-04-27 18:57:55] (step=0071925) Train Loss: 5.7185, Train Steps/Sec: 1.12 + 65%|██████████████████████████████████████████████████████████████████████████████████▌ | 71949/110696 [21:54:52<9:31:32, 1.13it/s][2025-04-27 18:58:18] (step=0071950) Train Loss: 5.7453, Train Steps/Sec: 1.12 + 65%|██████████████████████████████████████████████████████████████████████████████████▌ | 71974/110696 [21:55:15<9:31:12, 1.13it/s][2025-04-27 18:58:40] (step=0071975) Train Loss: 5.7713, Train Steps/Sec: 1.12 + 65%|██████████████████████████████████████████████████████████████████████████████████▌ | 71999/110696 [21:55:37<9:32:03, 1.13it/s][2025-04-27 18:59:02] (step=0072000) Train Loss: 5.7667, Train Steps/Sec: 1.11 +You are using a model of type blip-2 to instantiate a model of type instructblip. This is not supported for all configurations of models and can yield errors. +[2025-04-27 18:59:02] text_config is None. Initializing the text config with default values (`OPTConfig`). +model_name_or_path:/tmp/haozhezhao/model/blip2-flan-t5-xl +image_place_holder: +tokenizer length after expend 32101 +tokenizer length before expend 32100 +100%|████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████| 5/5 [05:05<00:00, 61.12s/it] +[2025-04-27 19:05:22] Finish Eval in 72000 steps...█████████████████████████████████████████████████████████████████████████████████████████████| 5/5 [05:04<00:00, 60.80s/it] +processing left-padding... +processing left-padding... +processing left-padding... +processing left-padding... +processing left-padding... +[2025-04-27 19:05:41] Saved checkpoint to checkpoint/X2I_UltraEdiit_fourmask_LLava_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_filtered_objects_4mask_1280_context/003-GPT-XL/checkpoints/0072000.pt +[2025-04-27 19:05:43] Removed old checkpoint: checkpoint/X2I_UltraEdiit_fourmask_LLava_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_filtered_objects_4mask_1280_context/003-GPT-XL/checkpoints/0070000.pt + 65%|█████████████████████████████████████████████████████████████████████████████████▉ | 72024/110696 [22:02:41<10:05:23, 1.06it/s][2025-04-27 19:06:06] (step=0072025) Train Loss: 5.7760, Train Steps/Sec: 0.06 + 65%|██████████████████████████████████████████████████████████████████████████████████▋ | 72049/110696 [22:03:03<9:38:22, 1.11it/s][2025-04-27 19:06:28] (step=0072050) Train Loss: 5.7930, Train Steps/Sec: 1.12 + 65%|██████████████████████████████████████████████████████████████████████████████████▋ | 72074/110696 [22:03:25<9:36:08, 1.12it/s][2025-04-27 19:06:51] (step=0072075) Train Loss: 5.7698, Train Steps/Sec: 1.12 + 65%|██████████████████████████████████████████████████████████████████████████████████▋ | 72099/110696 [22:03:47<9:32:46, 1.12it/s][2025-04-27 19:07:13] (step=0072100) Train Loss: 5.7274, Train Steps/Sec: 1.12 + 65%|██████████████████████████████████████████████████████████████████████████████████▋ | 72124/110696 [22:04:10<9:29:55, 1.13it/s][2025-04-27 19:07:35] (step=0072125) Train Loss: 5.7864, Train Steps/Sec: 1.12 + 65%|██████████████████████████████████████████████████████████████████████████████████▊ | 72149/110696 [22:04:32<9:29:01, 1.13it/s][2025-04-27 19:07:57] (step=0072150) Train Loss: 5.7677, Train Steps/Sec: 1.12 + 65%|██████████████████████████████████████████████████████████████████████████████████▊ | 72174/110696 [22:04:54<9:29:30, 1.13it/s][2025-04-27 19:08:20] (step=0072175) Train Loss: 5.7257, Train Steps/Sec: 1.12 + 65%|██████████████████████████████████████████████████████████████████████████████████▊ | 72199/110696 [22:05:17<9:25:22, 1.13it/s][2025-04-27 19:08:42] (step=0072200) Train Loss: 5.6819, Train Steps/Sec: 1.12 + 65%|██████████████████████████████████████████████████████████████████████████████████▊ | 72224/110696 [22:05:39<9:44:24, 1.10it/s][2025-04-27 19:09:04] (step=0072225) Train Loss: 5.7252, Train Steps/Sec: 1.12 + 65%|██████████████████████████████████████████████████████████████████████████████████▉ | 72249/110696 [22:06:01<9:35:39, 1.11it/s][2025-04-27 19:09:27] (step=0072250) Train Loss: 5.6863, Train Steps/Sec: 1.12 + 65%|██████████████████████████████████████████████████████████████████████████████████▉ | 72274/110696 [22:06:23<9:30:05, 1.12it/s][2025-04-27 19:09:49] (step=0072275) Train Loss: 5.7034, Train Steps/Sec: 1.12 + 65%|██████████████████████████████████████████████████████████████████████████████████▉ | 72299/110696 [22:06:46<9:29:54, 1.12it/s][2025-04-27 19:10:11] (step=0072300) Train Loss: 5.7038, Train Steps/Sec: 1.12 + 65%|██████████████████████████████████████████████████████████████████████████████████▉ | 72324/110696 [22:07:08<9:26:37, 1.13it/s][2025-04-27 19:10:33] (step=0072325) Train Loss: 5.8282, Train Steps/Sec: 1.12 + 65%|███████████████████████████████████████████████████████████████████████████████████ | 72349/110696 [22:07:30<9:25:15, 1.13it/s][2025-04-27 19:10:56] (step=0072350) Train Loss: 5.7618, Train Steps/Sec: 1.12 + 65%|███████████████████████████████████████████████████████████████████████████████████ | 72374/110696 [22:07:52<9:22:53, 1.13it/s][2025-04-27 19:11:18] (step=0072375) Train Loss: 5.7333, Train Steps/Sec: 1.12 + 65%|███████████████████████████████████████████████████████████████████████████████████ | 72399/110696 [22:08:15<9:20:51, 1.14it/s][2025-04-27 19:11:40] (step=0072400) Train Loss: 5.7145, Train Steps/Sec: 1.12 + 65%|███████████████████████████████████████████████████████████████████████████████████ | 72424/110696 [22:08:37<9:41:28, 1.10it/s][2025-04-27 19:12:02] (step=0072425) Train Loss: 5.7108, Train Steps/Sec: 1.12 + 65%|███████████████████████████████████████████████████████████████████████████████████ | 72449/110696 [22:08:59<9:32:22, 1.11it/s][2025-04-27 19:12:25] (step=0072450) Train Loss: 5.7554, Train Steps/Sec: 1.12 + 65%|███████████████████████████████████████████████████████████████████████████████████▏ | 72474/110696 [22:09:22<9:28:22, 1.12it/s][2025-04-27 19:12:47] (step=0072475) Train Loss: 5.7535, Train Steps/Sec: 1.12 + 65%|███████████████████████████████████████████████████████████████████████████████████▏ | 72499/110696 [22:09:44<9:26:46, 1.12it/s][2025-04-27 19:13:09] (step=0072500) Train Loss: 5.7891, Train Steps/Sec: 1.12 + 66%|███████████████████████████████████████████████████████████████████████████████████▏ | 72524/110696 [22:10:06<9:24:12, 1.13it/s][2025-04-27 19:13:32] (step=0072525) Train Loss: 5.7557, Train Steps/Sec: 1.12 + 66%|███████████████████████████████████████████████████████████████████████████████████▏ | 72549/110696 [22:10:28<9:22:06, 1.13it/s][2025-04-27 19:13:54] (step=0072550) Train Loss: 5.6986, Train Steps/Sec: 1.12 + 66%|███████████████████████████████████████████████████████████████████████████████████▎ | 72574/110696 [22:10:51<9:31:04, 1.11it/s][2025-04-27 19:14:17] (step=0072575) Train Loss: 5.6911, Train Steps/Sec: 1.10 + 66%|███████████████████████████████████████████████████████████████████████████████████▎ | 72599/110696 [22:11:13<9:21:37, 1.13it/s][2025-04-27 19:14:39] (step=0072600) Train Loss: 5.7664, Train Steps/Sec: 1.12 + 66%|███████████████████████████████████████████████████████████████████████████████████▎ | 72624/110696 [22:11:36<9:38:46, 1.10it/s][2025-04-27 19:15:01] (step=0072625) Train Loss: 5.7254, Train Steps/Sec: 1.12 + 66%|███████████████████████████████████████████████████████████████████████████████████▎ | 72649/110696 [22:11:58<9:30:52, 1.11it/s][2025-04-27 19:15:23] (step=0072650) Train Loss: 5.7423, Train Steps/Sec: 1.12 + 66%|███████████████████████████████████████████████████████████████████████████████████▍ | 72674/110696 [22:12:20<9:25:49, 1.12it/s][2025-04-27 19:15:46] (step=0072675) Train Loss: 5.7783, Train Steps/Sec: 1.12 + 66%|███████████████████████████████████████████████████████████████████████████████████▍ | 72699/110696 [22:12:43<9:23:06, 1.12it/s][2025-04-27 19:16:08] (step=0072700) Train Loss: 5.7569, Train Steps/Sec: 1.12 + 66%|███████████████████████████████████████████████████████████████████████████████████▍ | 72724/110696 [22:13:05<9:21:32, 1.13it/s][2025-04-27 19:16:30] (step=0072725) Train Loss: 5.7509, Train Steps/Sec: 1.12 + 66%|███████████████████████████████████████████████████████████████████████████████████▍ | 72749/110696 [22:13:27<9:21:01, 1.13it/s][2025-04-27 19:16:53] (step=0072750) Train Loss: 5.6785, Train Steps/Sec: 1.12 + 66%|███████████████████████████████████████████████████████████████████████████████████▍ | 72774/110696 [22:13:50<9:18:04, 1.13it/s][2025-04-27 19:17:15] (step=0072775) Train Loss: 5.7755, Train Steps/Sec: 1.12 + 66%|███████████████████████████████████████████████████████████████████████████████████▌ | 72799/110696 [22:14:12<9:17:41, 1.13it/s][2025-04-27 19:17:37] (step=0072800) Train Loss: 5.6921, Train Steps/Sec: 1.12 + 66%|███████████████████████████████████████████████████████████████████████████████████▌ | 72824/110696 [22:14:34<9:31:47, 1.10it/s][2025-04-27 19:17:59] (step=0072825) Train Loss: 5.7098, Train Steps/Sec: 1.12 + 66%|███████████████████████████████████████████████████████████████████████████████████▌ | 72849/110696 [22:14:56<9:28:42, 1.11it/s][2025-04-27 19:18:22] (step=0072850) Train Loss: 5.7552, Train Steps/Sec: 1.12 + 66%|███████████████████████████████████████████████████████████████████████████████████▌ | 72874/110696 [22:15:19<9:23:47, 1.12it/s][2025-04-27 19:18:44] (step=0072875) Train Loss: 5.7057, Train Steps/Sec: 1.12 + 66%|███████████████████████████████████████████████████████████████████████████████████▋ | 72899/110696 [22:15:41<9:20:20, 1.12it/s][2025-04-27 19:19:06] (step=0072900) Train Loss: 5.7526, Train Steps/Sec: 1.12 + 66%|███████████████████████████████████████████████████████████████████████████████████▋ | 72924/110696 [22:16:03<9:17:39, 1.13it/s][2025-04-27 19:19:29] (step=0072925) Train Loss: 5.7120, Train Steps/Sec: 1.12 + 66%|███████████████████████████████████████████████████████████████████████████████████▋ | 72949/110696 [22:16:26<9:17:09, 1.13it/s][2025-04-27 19:19:51] (step=0072950) Train Loss: 5.7929, Train Steps/Sec: 1.12 + 66%|███████████████████████████████████████████████████████████████████████████████████▋ | 72974/110696 [22:16:48<9:18:31, 1.13it/s][2025-04-27 19:20:13] (step=0072975) Train Loss: 5.7757, Train Steps/Sec: 1.12 + 66%|███████████████████████████████████████████████████████████████████████████████████▊ | 72999/110696 [22:17:10<9:15:14, 1.13it/s][2025-04-27 19:20:36] (step=0073000) Train Loss: 5.6968, Train Steps/Sec: 1.12 + 66%|███████████████████████████████████████████████████████████████████████████████████▊ | 73024/110696 [22:17:33<9:29:38, 1.10it/s][2025-04-27 19:20:58] (step=0073025) Train Loss: 5.7167, Train Steps/Sec: 1.13 + 66%|███████████████████████████████████████████████████████████████████████████████████▊ | 73049/110696 [22:17:55<9:24:57, 1.11it/s][2025-04-27 19:21:20] (step=0073050) Train Loss: 5.7088, Train Steps/Sec: 1.12 + 66%|███████████████████████████████████████████████████████████████████████████████████▊ | 73074/110696 [22:18:17<9:18:56, 1.12it/s][2025-04-27 19:21:42] (step=0073075) Train Loss: 5.7270, Train Steps/Sec: 1.12 + 66%|███████████████████████████████████████████████████████████████████████████████████▊ | 73099/110696 [22:18:39<9:17:52, 1.12it/s][2025-04-27 19:22:05] (step=0073100) Train Loss: 5.6994, Train Steps/Sec: 1.11 + 66%|███████████████████████████████████████████████████████████████████████████████████▉ | 73124/110696 [22:19:02<9:14:29, 1.13it/s][2025-04-27 19:22:27] (step=0073125) Train Loss: 5.7565, Train Steps/Sec: 1.12 + 66%|███████████████████████████████████████████████████████████████████████████████████▉ | 73149/110696 [22:19:24<9:14:10, 1.13it/s][2025-04-27 19:22:49] (step=0073150) Train Loss: 5.7112, Train Steps/Sec: 1.12 + 66%|███████████████████████████████████████████████████████████████████████████████████▉ | 73174/110696 [22:19:46<9:13:50, 1.13it/s][2025-04-27 19:23:12] (step=0073175) Train Loss: 5.7605, Train Steps/Sec: 1.12 + 66%|███████████████████████████████████████████████████████████████████████████████████▉ | 73199/110696 [22:20:09<9:11:19, 1.13it/s][2025-04-27 19:23:34] (step=0073200) Train Loss: 5.7109, Train Steps/Sec: 1.11 + 66%|████████████████████████████████████████████████████████████████████████████████████ | 73224/110696 [22:20:31<9:26:23, 1.10it/s][2025-04-27 19:23:56] (step=0073225) Train Loss: 5.7892, Train Steps/Sec: 1.13 + 66%|████████████████████████████████████████████████████████████████████████████████████ | 73249/110696 [22:20:53<9:21:59, 1.11it/s][2025-04-27 19:24:19] (step=0073250) Train Loss: 5.7307, Train Steps/Sec: 1.12 + 66%|████████████████████████████████████████████████████████████████████████████████████ | 73274/110696 [22:21:16<9:19:06, 1.12it/s][2025-04-27 19:24:41] (step=0073275) Train Loss: 5.7243, Train Steps/Sec: 1.12 + 66%|████████████████████████████████████████████████████████████████████████████████████ | 73299/110696 [22:21:38<9:15:12, 1.12it/s][2025-04-27 19:25:03] (step=0073300) Train Loss: 5.7862, Train Steps/Sec: 1.12 + 66%|████████████████████████████████████████████████████████████████████████████████████ | 73324/110696 [22:22:00<9:13:11, 1.13it/s][2025-04-27 19:25:26] (step=0073325) Train Loss: 5.7194, Train Steps/Sec: 1.12 + 66%|████████████████████████████████████████████████████████████████████████████████████▏ | 73349/110696 [22:22:23<9:10:24, 1.13it/s][2025-04-27 19:25:48] (step=0073350) Train Loss: 5.8119, Train Steps/Sec: 1.12 + 66%|████████████████████████████████████████████████████████████████████████████████████▏ | 73374/110696 [22:22:45<9:10:11, 1.13it/s][2025-04-27 19:26:10] (step=0073375) Train Loss: 5.7047, Train Steps/Sec: 1.12 + 66%|████████████████████████████████████████████████████████████████████████████████████▏ | 73399/110696 [22:23:07<9:10:26, 1.13it/s][2025-04-27 19:26:33] (step=0073400) Train Loss: 5.7521, Train Steps/Sec: 1.12 + 66%|████████████████████████████████████████████████████████████████████████████████████▏ | 73424/110696 [22:23:30<9:25:54, 1.10it/s][2025-04-27 19:26:55] (step=0073425) Train Loss: 5.7159, Train Steps/Sec: 1.12 + 66%|████████████████████████████████████████████████████████████████████████████████████▎ | 73449/110696 [22:23:52<9:20:04, 1.11it/s][2025-04-27 19:27:17] (step=0073450) Train Loss: 5.6808, Train Steps/Sec: 1.12 + 66%|████████████████████████████████████████████████████████████████████████████████████▎ | 73474/110696 [22:24:14<9:11:52, 1.12it/s][2025-04-27 19:27:40] (step=0073475) Train Loss: 5.6852, Train Steps/Sec: 1.12 + 66%|████████████████████████████████████████████████████████████████████████████████████▎ | 73499/110696 [22:24:37<9:12:48, 1.12it/s][2025-04-27 19:28:02] (step=0073500) Train Loss: 5.7072, Train Steps/Sec: 1.12 + 66%|████████████████████████████████████████████████████████████████████████████████████▎ | 73524/110696 [22:24:59<9:09:53, 1.13it/s][2025-04-27 19:28:24] (step=0073525) Train Loss: 5.6907, Train Steps/Sec: 1.12 + 66%|████████████████████████████████████████████████████████████████████████████████████▍ | 73549/110696 [22:25:21<9:08:21, 1.13it/s][2025-04-27 19:28:47] (step=0073550) Train Loss: 5.7365, Train Steps/Sec: 1.12 + 66%|████████████████████████████████████████████████████████████████████████████████████▍ | 73574/110696 [22:25:44<9:05:18, 1.13it/s][2025-04-27 19:29:09] (step=0073575) Train Loss: 5.7058, Train Steps/Sec: 1.12 + 66%|████████████████████████████████████████████████████████████████████████████████████▍ | 73599/110696 [22:26:06<9:09:35, 1.12it/s][2025-04-27 19:29:31] (step=0073600) Train Loss: 5.7637, Train Steps/Sec: 1.11 + 67%|████████████████████████████████████████████████████████████████████████████████████▍ | 73624/110696 [22:26:28<9:25:14, 1.09it/s][2025-04-27 19:29:54] (step=0073625) Train Loss: 5.7006, Train Steps/Sec: 1.12 + 67%|████████████████████████████████████████████████████████████████████████████████████▍ | 73649/110696 [22:26:51<9:15:07, 1.11it/s][2025-04-27 19:30:16] (step=0073650) Train Loss: 5.7122, Train Steps/Sec: 1.12 + 67%|████████████████████████████████████████████████████████████████████████████████████▌ | 73674/110696 [22:27:13<9:11:04, 1.12it/s][2025-04-27 19:30:38] (step=0073675) Train Loss: 5.7070, Train Steps/Sec: 1.12 + 67%|████████████████████████████████████████████████████████████████████████████████████▌ | 73699/110696 [22:27:35<9:08:45, 1.12it/s][2025-04-27 19:31:01] (step=0073700) Train Loss: 5.7880, Train Steps/Sec: 1.12 + 67%|████████████████████████████████████████████████████████████████████████████████████▌ | 73724/110696 [22:27:58<9:08:18, 1.12it/s][2025-04-27 19:31:23] (step=0073725) Train Loss: 5.7524, Train Steps/Sec: 1.12 + 67%|████████████████████████████████████████████████████████████████████████████████████▌ | 73749/110696 [22:28:20<9:05:35, 1.13it/s][2025-04-27 19:31:45] (step=0073750) Train Loss: 5.7361, Train Steps/Sec: 1.12 + 67%|████████████████████████████████████████████████████████████████████████████████████▋ | 73774/110696 [22:28:42<9:05:54, 1.13it/s][2025-04-27 19:32:07] (step=0073775) Train Loss: 5.7128, Train Steps/Sec: 1.12 + 67%|████████████████████████████████████████████████████████████████████████████████████▋ | 73799/110696 [22:29:04<9:02:06, 1.13it/s][2025-04-27 19:32:30] (step=0073800) Train Loss: 5.7960, Train Steps/Sec: 1.12 + 67%|████████████████████████████████████████████████████████████████████████████████████▋ | 73824/110696 [22:29:27<9:18:34, 1.10it/s][2025-04-27 19:32:52] (step=0073825) Train Loss: 5.7617, Train Steps/Sec: 1.12 + 67%|████████████████████████████████████████████████████████████████████████████████████▋ | 73849/110696 [22:29:49<9:13:22, 1.11it/s][2025-04-27 19:33:14] (step=0073850) Train Loss: 5.7813, Train Steps/Sec: 1.12 + 67%|████████████████████████████████████████████████████████████████████████████████████▊ | 73874/110696 [22:30:11<9:08:54, 1.12it/s][2025-04-27 19:33:37] (step=0073875) Train Loss: 5.7240, Train Steps/Sec: 1.12 + 67%|████████████████████████████████████████████████████████████████████████████████████▊ | 73899/110696 [22:30:34<9:07:31, 1.12it/s][2025-04-27 19:33:59] (step=0073900) Train Loss: 5.7395, Train Steps/Sec: 1.12 + 67%|████████████████████████████████████████████████████████████████████████████████████▊ | 73924/110696 [22:30:56<9:03:54, 1.13it/s][2025-04-27 19:34:21] (step=0073925) Train Loss: 5.7057, Train Steps/Sec: 1.12 + 67%|████████████████████████████████████████████████████████████████████████████████████▊ | 73949/110696 [22:31:18<9:01:03, 1.13it/s][2025-04-27 19:34:44] (step=0073950) Train Loss: 5.7378, Train Steps/Sec: 1.12 + 67%|████████████████████████████████████████████████████████████████████████████████████▊ | 73974/110696 [22:31:41<9:02:44, 1.13it/s][2025-04-27 19:35:06] (step=0073975) Train Loss: 5.7402, Train Steps/Sec: 1.12 + 67%|████████████████████████████████████████████████████████████████████████████████████▉ | 73999/110696 [22:32:03<9:00:35, 1.13it/s][2025-04-27 19:35:28] (step=0074000) Train Loss: 5.6978, Train Steps/Sec: 1.11 +You are using a model of type blip-2 to instantiate a model of type instructblip. This is not supported for all configurations of models and can yield errors. +[2025-04-27 19:35:28] text_config is None. Initializing the text config with default values (`OPTConfig`). +model_name_or_path:/tmp/haozhezhao/model/blip2-flan-t5-xl +image_place_holder: +tokenizer length after expend 32101 +tokenizer length before expend 32100 +100%|████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████| 5/5 [05:05<00:00, 61.08s/it] +[2025-04-27 19:41:47] Finish Eval in 74000 steps...█████████████████████████████████████████████████████████████████████████████████████████████| 5/5 [05:04<00:00, 60.67s/it] +processing left-padding... +processing left-padding... +processing left-padding... +processing left-padding... +processing left-padding... +[2025-04-27 19:42:08] Saved checkpoint to checkpoint/X2I_UltraEdiit_fourmask_LLava_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_filtered_objects_4mask_1280_context/003-GPT-XL/checkpoints/0074000.pt +[2025-04-27 19:42:10] Removed old checkpoint: checkpoint/X2I_UltraEdiit_fourmask_LLava_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_filtered_objects_4mask_1280_context/003-GPT-XL/checkpoints/0072000.pt + 67%|████████████████████████████████████████████████████████████████████████████████████▉ | 74024/110696 [22:39:07<9:31:01, 1.07it/s][2025-04-27 19:42:33] (step=0074025) Train Loss: 5.7721, Train Steps/Sec: 0.06 + 67%|████████████████████████████████████████████████████████████████████████████████████▉ | 74049/110696 [22:39:29<9:07:43, 1.12it/s][2025-04-27 19:42:55] (step=0074050) Train Loss: 5.6573, Train Steps/Sec: 1.12 + 67%|████████████████████████████████████████████████████████████████████████████████████▉ | 74074/110696 [22:39:52<9:03:26, 1.12it/s][2025-04-27 19:43:17] (step=0074075) Train Loss: 5.7614, Train Steps/Sec: 1.12 + 67%|█████████████████████████████████████████████████████████████████████████████████████ | 74099/110696 [22:40:14<9:03:11, 1.12it/s][2025-04-27 19:43:39] (step=0074100) Train Loss: 5.7687, Train Steps/Sec: 1.12 + 67%|█████████████████████████████████████████████████████████████████████████████████████ | 74124/110696 [22:40:36<8:59:33, 1.13it/s][2025-04-27 19:44:02] (step=0074125) Train Loss: 5.7464, Train Steps/Sec: 1.12 + 67%|█████████████████████████████████████████████████████████████████████████████████████ | 74149/110696 [22:40:58<8:59:08, 1.13it/s][2025-04-27 19:44:24] (step=0074150) Train Loss: 5.7295, Train Steps/Sec: 1.12 + 67%|█████████████████████████████████████████████████████████████████████████████████████ | 74174/110696 [22:41:21<8:57:31, 1.13it/s][2025-04-27 19:44:46] (step=0074175) Train Loss: 5.7339, Train Steps/Sec: 1.12 + 67%|█████████████████████████████████████████████████████████████████████████████████████▏ | 74199/110696 [22:41:43<9:01:09, 1.12it/s][2025-04-27 19:45:09] (step=0074200) Train Loss: 5.6535, Train Steps/Sec: 1.12 + 67%|█████████████████████████████████████████████████████████████████████████████████████▏ | 74224/110696 [22:42:06<9:15:03, 1.10it/s][2025-04-27 19:45:31] (step=0074225) Train Loss: 5.6877, Train Steps/Sec: 1.12 + 67%|█████████████████████████████████████████████████████████████████████████████████████▏ | 74249/110696 [22:42:28<9:07:42, 1.11it/s][2025-04-27 19:45:53] (step=0074250) Train Loss: 5.7601, Train Steps/Sec: 1.12 + 67%|█████████████████████████████████████████████████████████████████████████████████████▏ | 74274/110696 [22:42:50<9:03:16, 1.12it/s][2025-04-27 19:46:15] (step=0074275) Train Loss: 5.7443, Train Steps/Sec: 1.12 + 67%|█████████████████████████████████████████████████████████████████████████████████████▏ | 74299/110696 [22:43:12<8:58:34, 1.13it/s][2025-04-27 19:46:38] (step=0074300) Train Loss: 5.7373, Train Steps/Sec: 1.12 + 67%|█████████████████████████████████████████████████████████████████████████████████████▎ | 74324/110696 [22:43:35<8:56:18, 1.13it/s][2025-04-27 19:47:00] (step=0074325) Train Loss: 5.7036, Train Steps/Sec: 1.13 + 67%|█████████████████████████████████████████████████████████████████████████████████████▎ | 74349/110696 [22:43:57<8:56:13, 1.13it/s][2025-04-27 19:47:22] (step=0074350) Train Loss: 5.7590, Train Steps/Sec: 1.12 + 67%|█████████████████████████████████████████████████████████████████████████████████████▎ | 74374/110696 [22:44:19<8:56:59, 1.13it/s][2025-04-27 19:47:45] (step=0074375) Train Loss: 5.7624, Train Steps/Sec: 1.12 + 67%|█████████████████████████████████████████████████████████████████████████████████████▎ | 74399/110696 [22:44:42<8:55:42, 1.13it/s][2025-04-27 19:48:07] (step=0074400) Train Loss: 5.7275, Train Steps/Sec: 1.11 + 67%|█████████████████████████████████████████████████████████████████████████████████████▍ | 74424/110696 [22:45:04<9:11:40, 1.10it/s][2025-04-27 19:48:29] (step=0074425) Train Loss: 5.7070, Train Steps/Sec: 1.12 + 67%|█████████████████████████████████████████████████████████████████████████████████████▍ | 74449/110696 [22:45:26<9:02:37, 1.11it/s][2025-04-27 19:48:52] (step=0074450) Train Loss: 5.6965, Train Steps/Sec: 1.12 + 67%|█████████████████████████████████████████████████████████████████████████████████████▍ | 74474/110696 [22:45:49<9:19:41, 1.08it/s][2025-04-27 19:49:14] (step=0074475) Train Loss: 5.7359, Train Steps/Sec: 1.10 + 67%|█████████████████████████████████████████████████████████████████████████████████████▍ | 74499/110696 [22:46:11<8:58:22, 1.12it/s][2025-04-27 19:49:37] (step=0074500) Train Loss: 5.7461, Train Steps/Sec: 1.12 + 67%|█████████████████████████████████████████████████████████████████████████████████████▌ | 74524/110696 [22:46:34<8:55:37, 1.13it/s][2025-04-27 19:49:59] (step=0074525) Train Loss: 5.7630, Train Steps/Sec: 1.12 + 67%|█████████████████████████████████████████████████████████████████████████████████████▌ | 74549/110696 [22:46:56<8:52:17, 1.13it/s][2025-04-27 19:50:21] (step=0074550) Train Loss: 5.6933, Train Steps/Sec: 1.12 + 67%|█████████████████████████████████████████████████████████████████████████████████████▌ | 74574/110696 [22:47:18<8:52:21, 1.13it/s][2025-04-27 19:50:44] (step=0074575) Train Loss: 5.7472, Train Steps/Sec: 1.12 + 67%|█████████████████████████████████████████████████████████████████████████████████████▌ | 74599/110696 [22:47:41<8:51:47, 1.13it/s][2025-04-27 19:51:06] (step=0074600) Train Loss: 5.6954, Train Steps/Sec: 1.12 + 67%|█████████████████████████████████████████████████████████████████████████████████████▌ | 74624/110696 [22:48:03<9:08:32, 1.10it/s][2025-04-27 19:51:28] (step=0074625) Train Loss: 5.6965, Train Steps/Sec: 1.12 + 67%|█████████████████████████████████████████████████████████████████████████████████████▋ | 74649/110696 [22:48:25<9:02:23, 1.11it/s][2025-04-27 19:51:51] (step=0074650) Train Loss: 5.7242, Train Steps/Sec: 1.12 + 67%|█████████████████████████████████████████████████████████████████████████████████████▋ | 74674/110696 [22:48:48<8:59:32, 1.11it/s][2025-04-27 19:52:13] (step=0074675) Train Loss: 5.7934, Train Steps/Sec: 1.12 + 67%|█████████████████████████████████████████████████████████████████████████████████████▋ | 74699/110696 [22:49:10<8:54:14, 1.12it/s][2025-04-27 19:52:35] (step=0074700) Train Loss: 5.6675, Train Steps/Sec: 1.12 + 68%|█████████████████████████████████████████████████████████████████████████████████████▋ | 74724/110696 [22:49:32<8:52:26, 1.13it/s][2025-04-27 19:52:58] (step=0074725) Train Loss: 5.7213, Train Steps/Sec: 1.12 + 68%|█████████████████████████████████████████████████████████████████████████████████████▊ | 74749/110696 [22:49:55<8:49:24, 1.13it/s][2025-04-27 19:53:20] (step=0074750) Train Loss: 5.7610, Train Steps/Sec: 1.12 + 68%|█████████████████████████████████████████████████████████████████████████████████████▊ | 74774/110696 [22:50:17<8:52:25, 1.12it/s][2025-04-27 19:53:42] (step=0074775) Train Loss: 5.7227, Train Steps/Sec: 1.12 + 68%|█████████████████████████████████████████████████████████████████████████████████████▊ | 74799/110696 [22:50:39<8:46:36, 1.14it/s][2025-04-27 19:54:05] (step=0074800) Train Loss: 5.7354, Train Steps/Sec: 1.12 + 68%|█████████████████████████████████████████████████████████████████████████████████████▊ | 74824/110696 [22:51:02<9:08:47, 1.09it/s][2025-04-27 19:54:27] (step=0074825) Train Loss: 5.7442, Train Steps/Sec: 1.12 + 68%|█████████████████████████████████████████████████████████████████████████████████████▊ | 74849/110696 [22:51:24<8:57:02, 1.11it/s][2025-04-27 19:54:49] (step=0074850) Train Loss: 5.7098, Train Steps/Sec: 1.12 + 68%|█████████████████████████████████████████████████████████████████████████████████████▉ | 74874/110696 [22:51:46<8:54:09, 1.12it/s][2025-04-27 19:55:12] (step=0074875) Train Loss: 5.7777, Train Steps/Sec: 1.12 + 68%|█████████████████████████████████████████████████████████████████████████████████████▉ | 74899/110696 [22:52:09<8:51:47, 1.12it/s][2025-04-27 19:55:34] (step=0074900) Train Loss: 5.7104, Train Steps/Sec: 1.12 + 68%|█████████████████████████████████████████████████████████████████████████████████████▉ | 74924/110696 [22:52:31<8:50:12, 1.12it/s][2025-04-27 19:55:56] (step=0074925) Train Loss: 5.7372, Train Steps/Sec: 1.12 + 68%|█████████████████████████████████████████████████████████████████████████████████████▉ | 74949/110696 [22:52:53<8:48:09, 1.13it/s][2025-04-27 19:56:18] (step=0074950) Train Loss: 5.7109, Train Steps/Sec: 1.12 + 68%|██████████████████████████████████████████████████████████████████████████████████████ | 74974/110696 [22:53:15<8:48:46, 1.13it/s][2025-04-27 19:56:41] (step=0074975) Train Loss: 5.7632, Train Steps/Sec: 1.12 + 68%|██████████████████████████████████████████████████████████████████████████████████████ | 74999/110696 [22:53:38<8:46:30, 1.13it/s][2025-04-27 19:57:03] (step=0075000) Train Loss: 5.7678, Train Steps/Sec: 1.11 + 68%|██████████████████████████████████████████████████████████████████████████████████████ | 75024/110696 [22:54:00<9:02:22, 1.10it/s][2025-04-27 19:57:26] (step=0075025) Train Loss: 5.7229, Train Steps/Sec: 1.12 + 68%|██████████████████████████████████████████████████████████████████████████████████████ | 75049/110696 [22:54:22<8:54:23, 1.11it/s][2025-04-27 19:57:48] (step=0075050) Train Loss: 5.6992, Train Steps/Sec: 1.12 + 68%|█████████████████████████████████████████████████████████████████████████████████████▍ | 75074/110696 [22:54:49<13:24:42, 1.36s/it][2025-04-27 19:58:15] (step=0075075) Train Loss: 5.7934, Train Steps/Sec: 0.93 + 68%|██████████████████████████████████████████████████████████████████████████████████████▏ | 75099/110696 [22:55:12<8:48:03, 1.12it/s][2025-04-27 19:58:37] (step=0075100) Train Loss: 5.6244, Train Steps/Sec: 1.12 + 68%|██████████████████████████████████████████████████████████████████████████████████████▏ | 75124/110696 [22:55:34<8:46:44, 1.13it/s][2025-04-27 19:58:59] (step=0075125) Train Loss: 5.8105, Train Steps/Sec: 1.12 + 68%|██████████████████████████████████████████████████████████████████████████████████████▏ | 75149/110696 [22:55:56<8:46:34, 1.13it/s][2025-04-27 19:59:22] (step=0075150) Train Loss: 5.7188, Train Steps/Sec: 1.12 + 68%|██████████████████████████████████████████████████████████████████████████████████████▏ | 75174/110696 [22:56:19<8:44:32, 1.13it/s][2025-04-27 19:59:44] (step=0075175) Train Loss: 5.7058, Train Steps/Sec: 1.12 + 68%|██████████████████████████████████████████████████████████████████████████████████████▎ | 75199/110696 [22:56:41<8:41:34, 1.13it/s][2025-04-27 20:00:06] (step=0075200) Train Loss: 5.7529, Train Steps/Sec: 1.12 + 68%|██████████████████████████████████████████████████████████████████████████████████████▎ | 75224/110696 [22:57:03<8:59:04, 1.10it/s][2025-04-27 20:00:29] (step=0075225) Train Loss: 5.7394, Train Steps/Sec: 1.12 + 68%|██████████████████████████████████████████████████████████████████████████████████████▎ | 75249/110696 [22:57:26<8:51:29, 1.11it/s][2025-04-27 20:00:51] (step=0075250) Train Loss: 5.7660, Train Steps/Sec: 1.12 + 68%|██████████████████████████████████████████████████████████████████████████████████████▎ | 75274/110696 [22:57:48<8:48:37, 1.12it/s][2025-04-27 20:01:13] (step=0075275) Train Loss: 5.7507, Train Steps/Sec: 1.12 + 68%|█████████████████████████████████████████████████████████████████████████████████████▋ | 75299/110696 [22:58:15<13:28:10, 1.37s/it][2025-04-27 20:01:40] (step=0075300) Train Loss: 5.7227, Train Steps/Sec: 0.93 + 68%|██████████████████████████████████████████████████████████████████████████████████████▍ | 75324/110696 [22:58:37<8:44:36, 1.12it/s][2025-04-27 20:02:03] (step=0075325) Train Loss: 5.6992, Train Steps/Sec: 1.12 + 68%|██████████████████████████████████████████████████████████████████████████████████████▍ | 75349/110696 [22:59:00<8:41:31, 1.13it/s][2025-04-27 20:02:25] (step=0075350) Train Loss: 5.7038, Train Steps/Sec: 1.12 + 68%|██████████████████████████████████████████████████████████████████████████████████████▍ | 75374/110696 [22:59:22<8:40:16, 1.13it/s][2025-04-27 20:02:47] (step=0075375) Train Loss: 5.7150, Train Steps/Sec: 1.12 + 68%|██████████████████████████████████████████████████████████████████████████████████████▌ | 75399/110696 [22:59:44<8:38:03, 1.14it/s][2025-04-27 20:03:10] (step=0075400) Train Loss: 5.7224, Train Steps/Sec: 1.12 + 68%|██████████████████████████████████████████████████████████████████████████████████████▌ | 75424/110696 [23:00:06<8:54:30, 1.10it/s][2025-04-27 20:03:32] (step=0075425) Train Loss: 5.6826, Train Steps/Sec: 1.12 + 68%|██████████████████████████████████████████████████████████████████████████████████████▌ | 75449/110696 [23:00:29<8:47:58, 1.11it/s][2025-04-27 20:03:54] (step=0075450) Train Loss: 5.7195, Train Steps/Sec: 1.12 + 68%|██████████████████████████████████████████████████████████████████████████████████████▌ | 75474/110696 [23:00:51<8:44:43, 1.12it/s][2025-04-27 20:04:16] (step=0075475) Train Loss: 5.7410, Train Steps/Sec: 1.12 + 68%|██████████████████████████████████████████████████████████████████████████████████████▌ | 75499/110696 [23:01:13<8:42:42, 1.12it/s][2025-04-27 20:04:39] (step=0075500) Train Loss: 5.7243, Train Steps/Sec: 1.12 + 68%|██████████████████████████████████████████████████████████████████████████████████████▋ | 75524/110696 [23:01:36<8:40:28, 1.13it/s][2025-04-27 20:05:01] (step=0075525) Train Loss: 5.7159, Train Steps/Sec: 1.12 + 68%|██████████████████████████████████████████████████████████████████████████████████████▋ | 75549/110696 [23:01:58<8:38:08, 1.13it/s][2025-04-27 20:05:23] (step=0075550) Train Loss: 5.7429, Train Steps/Sec: 1.12 + 68%|██████████████████████████████████████████████████████████████████████████████████████▋ | 75574/110696 [23:02:20<8:37:36, 1.13it/s][2025-04-27 20:05:45] (step=0075575) Train Loss: 5.6991, Train Steps/Sec: 1.12 + 68%|██████████████████████████████████████████████████████████████████████████████████████▋ | 75599/110696 [23:02:42<8:36:36, 1.13it/s][2025-04-27 20:06:08] (step=0075600) Train Loss: 5.6775, Train Steps/Sec: 1.11 + 68%|██████████████████████████████████████████████████████████████████████████████████████▊ | 75624/110696 [23:03:05<8:53:38, 1.10it/s][2025-04-27 20:06:30] (step=0075625) Train Loss: 5.7091, Train Steps/Sec: 1.12 + 68%|██████████████████████████████████████████████████████████████████████████████████████▊ | 75649/110696 [23:03:27<8:43:21, 1.12it/s][2025-04-27 20:06:52] (step=0075650) Train Loss: 5.7268, Train Steps/Sec: 1.13 + 68%|██████████████████████████████████████████████████████████████████████████████████████▊ | 75674/110696 [23:03:49<8:40:51, 1.12it/s][2025-04-27 20:07:15] (step=0075675) Train Loss: 5.6942, Train Steps/Sec: 1.12 + 68%|██████████████████████████████████████████████████████████████████████████████████████▊ | 75699/110696 [23:04:12<8:39:22, 1.12it/s][2025-04-27 20:07:37] (step=0075700) Train Loss: 5.6816, Train Steps/Sec: 1.12 + 68%|██████████████████████████████████████████████████████████████████████████████████████▉ | 75724/110696 [23:04:34<8:38:49, 1.12it/s][2025-04-27 20:07:59] (step=0075725) Train Loss: 5.7684, Train Steps/Sec: 1.12 + 68%|██████████████████████████████████████████████████████████████████████████████████████▉ | 75749/110696 [23:04:56<8:34:27, 1.13it/s][2025-04-27 20:08:22] (step=0075750) Train Loss: 5.7469, Train Steps/Sec: 1.12 + 68%|██████████████████████████████████████████████████████████████████████████████████████▉ | 75774/110696 [23:05:19<8:36:00, 1.13it/s][2025-04-27 20:08:44] (step=0075775) Train Loss: 5.7208, Train Steps/Sec: 1.12 + 68%|██████████████████████████████████████████████████████████████████████████████████████▉ | 75799/110696 [23:05:41<8:37:59, 1.12it/s][2025-04-27 20:09:06] (step=0075800) Train Loss: 5.6633, Train Steps/Sec: 1.11 + 68%|██████████████████████████████████████████████████████████████████████████████████████▉ | 75824/110696 [23:06:03<8:48:35, 1.10it/s][2025-04-27 20:09:29] (step=0075825) Train Loss: 5.7462, Train Steps/Sec: 1.12 + 69%|███████████████████████████████████████████████████████████████████████████████████████ | 75849/110696 [23:06:26<8:43:13, 1.11it/s][2025-04-27 20:09:51] (step=0075850) Train Loss: 5.7390, Train Steps/Sec: 1.12 + 69%|███████████████████████████████████████████████████████████████████████████████████████ | 75874/110696 [23:06:48<8:40:06, 1.12it/s][2025-04-27 20:10:13] (step=0075875) Train Loss: 5.6789, Train Steps/Sec: 1.12 + 69%|███████████████████████████████████████████████████████████████████████████████████████ | 75899/110696 [23:07:10<8:37:31, 1.12it/s][2025-04-27 20:10:36] (step=0075900) Train Loss: 5.7237, Train Steps/Sec: 1.12 + 69%|███████████████████████████████████████████████████████████████████████████████████████ | 75924/110696 [23:07:33<8:34:44, 1.13it/s][2025-04-27 20:10:58] (step=0075925) Train Loss: 5.7162, Train Steps/Sec: 1.12 + 69%|███████████████████████████████████████████████████████████████████████████████████████▏ | 75949/110696 [23:07:55<8:32:13, 1.13it/s][2025-04-27 20:11:20] (step=0075950) Train Loss: 5.7140, Train Steps/Sec: 1.12 + 69%|██████████████████████████████████████████████████████████████████████████████████████▍ | 75974/110696 [23:08:26<10:43:39, 1.11s/it][2025-04-27 20:11:51] (step=0075975) Train Loss: 5.6973, Train Steps/Sec: 0.81 + 69%|███████████████████████████████████████████████████████████████████████████████████████▏ | 75999/110696 [23:08:52<8:36:28, 1.12it/s][2025-04-27 20:12:18] (step=0076000) Train Loss: 5.7591, Train Steps/Sec: 0.94 +You are using a model of type blip-2 to instantiate a model of type instructblip. This is not supported for all configurations of models and can yield errors. +[2025-04-27 20:12:18] text_config is None. Initializing the text config with default values (`OPTConfig`). +model_name_or_path:/tmp/haozhezhao/model/blip2-flan-t5-xl +image_place_holder: +tokenizer length after expend 32101 +tokenizer length before expend 32100 +100%|████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████| 5/5 [05:04<00:00, 60.92s/it] +[2025-04-27 20:18:36] Finish Eval in 76000 steps...█████████████████████████████████████████████████████████████████████████████████████████████| 5/5 [05:03<00:00, 60.54s/it] +processing left-padding... +processing left-padding... +processing left-padding... +processing left-padding... +processing left-padding... +[2025-04-27 20:18:56] Saved checkpoint to checkpoint/X2I_UltraEdiit_fourmask_LLava_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_filtered_objects_4mask_1280_context/003-GPT-XL/checkpoints/0076000.pt +[2025-04-27 20:18:58] Removed old checkpoint: checkpoint/X2I_UltraEdiit_fourmask_LLava_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_filtered_objects_4mask_1280_context/003-GPT-XL/checkpoints/0074000.pt + 69%|███████████████████████████████████████████████████████████████████████████████████████▏ | 76024/110696 [23:16:00<9:25:15, 1.02it/s][2025-04-27 20:19:25] (step=0076025) Train Loss: 5.6696, Train Steps/Sec: 0.06 + 69%|███████████████████████████████████████████████████████████████████████████████████████▏ | 76049/110696 [23:16:22<8:38:40, 1.11it/s][2025-04-27 20:19:48] (step=0076050) Train Loss: 5.6943, Train Steps/Sec: 1.12 + 69%|███████████████████████████████████████████████████████████████████████████████████████▎ | 76074/110696 [23:16:45<8:34:47, 1.12it/s][2025-04-27 20:20:10] (step=0076075) Train Loss: 5.6934, Train Steps/Sec: 1.12 + 69%|███████████████████████████████████████████████████████████████████████████████████████▎ | 76099/110696 [23:17:07<8:35:11, 1.12it/s][2025-04-27 20:20:32] (step=0076100) Train Loss: 5.7573, Train Steps/Sec: 1.12 + 69%|███████████████████████████████████████████████████████████████████████████████████████▎ | 76124/110696 [23:17:29<8:31:13, 1.13it/s][2025-04-27 20:20:55] (step=0076125) Train Loss: 5.7233, Train Steps/Sec: 1.12 + 69%|███████████████████████████████████████████████████████████████████████████████████████▎ | 76149/110696 [23:17:51<8:28:36, 1.13it/s][2025-04-27 20:21:17] (step=0076150) Train Loss: 5.7097, Train Steps/Sec: 1.12 + 69%|███████████████████████████████████████████████████████████████████████████████████████▍ | 76174/110696 [23:18:14<8:29:57, 1.13it/s][2025-04-27 20:21:39] (step=0076175) Train Loss: 5.7543, Train Steps/Sec: 1.12 + 69%|███████████████████████████████████████████████████████████████████████████████████████▍ | 76199/110696 [23:18:36<8:29:42, 1.13it/s][2025-04-27 20:22:02] (step=0076200) Train Loss: 5.7857, Train Steps/Sec: 1.12 + 69%|███████████████████████████████████████████████████████████████████████████████████████▍ | 76224/110696 [23:18:58<8:43:04, 1.10it/s][2025-04-27 20:22:24] (step=0076225) Train Loss: 5.7199, Train Steps/Sec: 1.12 + 69%|███████████████████████████████████████████████████████████████████████████████████████▍ | 76249/110696 [23:19:26<8:37:13, 1.11it/s][2025-04-27 20:22:51] (step=0076250) Train Loss: 5.7736, Train Steps/Sec: 0.92 + 69%|███████████████████████████████████████████████████████████████████████████████████████▌ | 76274/110696 [23:19:48<8:32:33, 1.12it/s][2025-04-27 20:23:13] (step=0076275) Train Loss: 5.6506, Train Steps/Sec: 1.12 + 69%|███████████████████████████████████████████████████████████████████████████████████████▌ | 76299/110696 [23:20:10<8:29:57, 1.12it/s][2025-04-27 20:23:35] (step=0076300) Train Loss: 5.7243, Train Steps/Sec: 1.12 + 69%|███████████████████████████████████████████████████████████████████████████████████████▌ | 76324/110696 [23:20:32<8:28:29, 1.13it/s][2025-04-27 20:23:58] (step=0076325) Train Loss: 5.7546, Train Steps/Sec: 1.12 + 69%|███████████████████████████████████████████████████████████████████████████████████████▌ | 76349/110696 [23:20:55<8:26:20, 1.13it/s][2025-04-27 20:24:20] (step=0076350) Train Loss: 5.6752, Train Steps/Sec: 1.12 + 69%|███████████████████████████████████████████████████████████████████████████████████████▌ | 76374/110696 [23:21:17<8:44:49, 1.09it/s][2025-04-27 20:24:43] (step=0076375) Train Loss: 5.7136, Train Steps/Sec: 1.10 + 69%|███████████████████████████████████████████████████████████████████████████████████████▋ | 76399/110696 [23:21:40<8:25:09, 1.13it/s][2025-04-27 20:25:05] (step=0076400) Train Loss: 5.6707, Train Steps/Sec: 1.12 + 69%|███████████████████████████████████████████████████████████████████████████████████████▋ | 76424/110696 [23:22:02<8:40:16, 1.10it/s][2025-04-27 20:25:27] (step=0076425) Train Loss: 5.7918, Train Steps/Sec: 1.12 + 69%|███████████████████████████████████████████████████████████████████████████████████████▋ | 76449/110696 [23:22:24<8:35:05, 1.11it/s][2025-04-27 20:25:50] (step=0076450) Train Loss: 5.7957, Train Steps/Sec: 1.12 + 69%|███████████████████████████████████████████████████████████████████████████████████████▋ | 76474/110696 [23:22:47<8:28:44, 1.12it/s][2025-04-27 20:26:12] (step=0076475) Train Loss: 5.6796, Train Steps/Sec: 1.12 + 69%|███████████████████████████████████████████████████████████████████████████████████████▊ | 76499/110696 [23:23:09<8:27:23, 1.12it/s][2025-04-27 20:26:34] (step=0076500) Train Loss: 5.6525, Train Steps/Sec: 1.12 + 69%|███████████████████████████████████████████████████████████████████████████████████████▊ | 76524/110696 [23:23:31<8:24:59, 1.13it/s][2025-04-27 20:26:56] (step=0076525) Train Loss: 5.6823, Train Steps/Sec: 1.12 + 69%|███████████████████████████████████████████████████████████████████████████████████████▊ | 76549/110696 [23:23:54<8:25:48, 1.13it/s][2025-04-27 20:27:19] (step=0076550) Train Loss: 5.7147, Train Steps/Sec: 1.12 + 69%|███████████████████████████████████████████████████████████████████████████████████████▊ | 76574/110696 [23:24:16<8:24:32, 1.13it/s][2025-04-27 20:27:41] (step=0076575) Train Loss: 5.7535, Train Steps/Sec: 1.12 + 69%|███████████████████████████████████████████████████████████████████████████████████████▉ | 76599/110696 [23:24:38<8:19:47, 1.14it/s][2025-04-27 20:28:04] (step=0076600) Train Loss: 5.7049, Train Steps/Sec: 1.12 + 69%|███████████████████████████████████████████████████████████████████████████████████████▉ | 76624/110696 [23:25:00<8:38:57, 1.09it/s][2025-04-27 20:28:26] (step=0076625) Train Loss: 5.7371, Train Steps/Sec: 1.12 + 69%|███████████████████████████████████████████████████████████████████████████████████████▉ | 76649/110696 [23:25:23<8:29:47, 1.11it/s][2025-04-27 20:28:48] (step=0076650) Train Loss: 5.7015, Train Steps/Sec: 1.12 + 69%|███████████████████████████████████████████████████████████████████████████████████████▉ | 76674/110696 [23:25:45<8:26:37, 1.12it/s][2025-04-27 20:29:10] (step=0076675) Train Loss: 5.7297, Train Steps/Sec: 1.12 + 69%|███████████████████████████████████████████████████████████████████████████████████████▉ | 76699/110696 [23:26:07<8:23:44, 1.12it/s][2025-04-27 20:29:33] (step=0076700) Train Loss: 5.8067, Train Steps/Sec: 1.12 + 69%|████████████████████████████████████████████████████████████████████████████████████████ | 76724/110696 [23:26:30<8:23:32, 1.12it/s][2025-04-27 20:29:55] (step=0076725) Train Loss: 5.7075, Train Steps/Sec: 1.12 + 69%|████████████████████████████████████████████████████████████████████████████████████████ | 76749/110696 [23:26:52<8:20:42, 1.13it/s][2025-04-27 20:30:17] (step=0076750) Train Loss: 5.7099, Train Steps/Sec: 1.12 + 69%|████████████████████████████████████████████████████████████████████████████████████████ | 76774/110696 [23:27:14<8:20:57, 1.13it/s][2025-04-27 20:30:40] (step=0076775) Train Loss: 5.7005, Train Steps/Sec: 1.12 + 69%|████████████████████████████████████████████████████████████████████████████████████████ | 76799/110696 [23:27:37<8:21:42, 1.13it/s][2025-04-27 20:31:02] (step=0076800) Train Loss: 5.7753, Train Steps/Sec: 1.12 + 69%|████████████████████████████████████████████████████████████████████████████████████████▏ | 76824/110696 [23:27:59<8:35:16, 1.10it/s][2025-04-27 20:31:24] (step=0076825) Train Loss: 5.7228, Train Steps/Sec: 1.12 + 69%|████████████████████████████████████████████████████████████████████████████████████████▏ | 76849/110696 [23:28:21<8:26:16, 1.11it/s][2025-04-27 20:31:47] (step=0076850) Train Loss: 5.7663, Train Steps/Sec: 1.12 + 69%|████████████████████████████████████████████████████████████████████████████████████████▏ | 76874/110696 [23:28:44<8:22:37, 1.12it/s][2025-04-27 20:32:09] (step=0076875) Train Loss: 5.7125, Train Steps/Sec: 1.12 + 69%|████████████████████████████████████████████████████████████████████████████████████████▏ | 76899/110696 [23:29:06<8:22:47, 1.12it/s][2025-04-27 20:32:31] (step=0076900) Train Loss: 5.7393, Train Steps/Sec: 1.12 + 69%|████████████████████████████████████████████████████████████████████████████████████████▎ | 76924/110696 [23:29:28<8:19:42, 1.13it/s][2025-04-27 20:32:54] (step=0076925) Train Loss: 5.7744, Train Steps/Sec: 1.12 + 70%|████████████████████████████████████████████████████████████████████████████████████████▎ | 76949/110696 [23:29:51<8:18:47, 1.13it/s][2025-04-27 20:33:16] (step=0076950) Train Loss: 5.6764, Train Steps/Sec: 1.12 + 70%|████████████████████████████████████████████████████████████████████████████████████████▎ | 76974/110696 [23:30:13<8:17:25, 1.13it/s][2025-04-27 20:33:38] (step=0076975) Train Loss: 5.7356, Train Steps/Sec: 1.12 + 70%|████████████████████████████████████████████████████████████████████████████████████████▎ | 76999/110696 [23:30:35<8:15:03, 1.13it/s][2025-04-27 20:34:01] (step=0077000) Train Loss: 5.7037, Train Steps/Sec: 1.12 + 70%|████████████████████████████████████████████████████████████████████████████████████████▎ | 77024/110696 [23:30:58<8:31:44, 1.10it/s][2025-04-27 20:34:23] (step=0077025) Train Loss: 5.7430, Train Steps/Sec: 1.12 + 70%|████████████████████████████████████████████████████████████████████████████████████████▍ | 77049/110696 [23:31:20<8:25:47, 1.11it/s][2025-04-27 20:34:45] (step=0077050) Train Loss: 5.6990, Train Steps/Sec: 1.12 + 70%|████████████████████████████████████████████████████████████████████████████████████████▍ | 77074/110696 [23:31:42<8:19:42, 1.12it/s][2025-04-27 20:35:08] (step=0077075) Train Loss: 5.6896, Train Steps/Sec: 1.12 + 70%|████████████████████████████████████████████████████████████████████████████████████████▍ | 77099/110696 [23:32:05<8:18:24, 1.12it/s][2025-04-27 20:35:30] (step=0077100) Train Loss: 5.7429, Train Steps/Sec: 1.12 + 70%|████████████████████████████████████████████████████████████████████████████████████████▍ | 77124/110696 [23:32:27<8:15:28, 1.13it/s][2025-04-27 20:35:52] (step=0077125) Train Loss: 5.7996, Train Steps/Sec: 1.12 + 70%|████████████████████████████████████████████████████████████████████████████████████████▌ | 77149/110696 [23:32:49<8:14:00, 1.13it/s][2025-04-27 20:36:14] (step=0077150) Train Loss: 5.7497, Train Steps/Sec: 1.12 + 70%|████████████████████████████████████████████████████████████████████████████████████████▌ | 77174/110696 [23:33:11<8:15:22, 1.13it/s][2025-04-27 20:36:37] (step=0077175) Train Loss: 5.7268, Train Steps/Sec: 1.12 + 70%|████████████████████████████████████████████████████████████████████████████████████████▌ | 77199/110696 [23:33:34<8:14:17, 1.13it/s][2025-04-27 20:36:59] (step=0077200) Train Loss: 5.7813, Train Steps/Sec: 1.12 + 70%|████████████████████████████████████████████████████████████████████████████████████████▌ | 77224/110696 [23:33:56<8:26:10, 1.10it/s][2025-04-27 20:37:21] (step=0077225) Train Loss: 5.7120, Train Steps/Sec: 1.12 + 70%|████████████████████████████████████████████████████████████████████████████████████████▋ | 77249/110696 [23:34:18<8:21:36, 1.11it/s][2025-04-27 20:37:44] (step=0077250) Train Loss: 5.7619, Train Steps/Sec: 1.12 + 70%|████████████████████████████████████████████████████████████████████████████████████████▋ | 77274/110696 [23:34:41<8:18:02, 1.12it/s][2025-04-27 20:38:06] (step=0077275) Train Loss: 5.7927, Train Steps/Sec: 1.12 + 70%|████████████████████████████████████████████████████████████████████████████████████████▋ | 77299/110696 [23:35:03<8:16:21, 1.12it/s][2025-04-27 20:38:28] (step=0077300) Train Loss: 5.7105, Train Steps/Sec: 1.12 + 70%|████████████████████████████████████████████████████████████████████████████████████████▋ | 77324/110696 [23:35:25<8:14:14, 1.13it/s][2025-04-27 20:38:51] (step=0077325) Train Loss: 5.6451, Train Steps/Sec: 1.12 + 70%|████████████████████████████████████████████████████████████████████████████████████████▋ | 77349/110696 [23:35:48<8:13:51, 1.13it/s][2025-04-27 20:39:13] (step=0077350) Train Loss: 5.7461, Train Steps/Sec: 1.12 + 70%|████████████████████████████████████████████████████████████████████████████████████████▊ | 77374/110696 [23:36:10<8:11:58, 1.13it/s][2025-04-27 20:39:35] (step=0077375) Train Loss: 5.7718, Train Steps/Sec: 1.12 + 70%|████████████████████████████████████████████████████████████████████████████████████████▊ | 77399/110696 [23:36:32<8:11:26, 1.13it/s][2025-04-27 20:39:58] (step=0077400) Train Loss: 5.7640, Train Steps/Sec: 1.11 + 70%|████████████████████████████████████████████████████████████████████████████████████████▊ | 77424/110696 [23:36:55<8:26:05, 1.10it/s][2025-04-27 20:40:20] (step=0077425) Train Loss: 5.6955, Train Steps/Sec: 1.12 + 70%|████████████████████████████████████████████████████████████████████████████████████████▊ | 77449/110696 [23:37:17<8:19:03, 1.11it/s][2025-04-27 20:40:42] (step=0077450) Train Loss: 5.7781, Train Steps/Sec: 1.12 + 70%|████████████████████████████████████████████████████████████████████████████████████████▉ | 77474/110696 [23:37:39<8:17:08, 1.11it/s][2025-04-27 20:41:05] (step=0077475) Train Loss: 5.7168, Train Steps/Sec: 1.12 + 70%|████████████████████████████████████████████████████████████████████████████████████████▉ | 77499/110696 [23:38:02<8:12:05, 1.12it/s][2025-04-27 20:41:27] (step=0077500) Train Loss: 5.6614, Train Steps/Sec: 1.12 + 70%|████████████████████████████████████████████████████████████████████████████████████████▉ | 77524/110696 [23:38:24<8:08:32, 1.13it/s][2025-04-27 20:41:49] (step=0077525) Train Loss: 5.7237, Train Steps/Sec: 1.12 + 70%|████████████████████████████████████████████████████████████████████████████████████████▉ | 77549/110696 [23:38:46<8:10:38, 1.13it/s][2025-04-27 20:42:12] (step=0077550) Train Loss: 5.6897, Train Steps/Sec: 1.12 + 70%|████████████████████████████████████████████████████████████████████████████████████████▉ | 77574/110696 [23:39:09<8:09:40, 1.13it/s][2025-04-27 20:42:34] (step=0077575) Train Loss: 5.7029, Train Steps/Sec: 1.12 + 70%|█████████████████████████████████████████████████████████████████████████████████████████ | 77599/110696 [23:39:31<8:06:43, 1.13it/s][2025-04-27 20:42:56] (step=0077600) Train Loss: 5.7288, Train Steps/Sec: 1.12 + 70%|█████████████████████████████████████████████████████████████████████████████████████████ | 77624/110696 [23:39:53<8:21:28, 1.10it/s][2025-04-27 20:43:19] (step=0077625) Train Loss: 5.6730, Train Steps/Sec: 1.12 + 70%|█████████████████████████████████████████████████████████████████████████████████████████ | 77649/110696 [23:40:15<8:14:53, 1.11it/s][2025-04-27 20:43:41] (step=0077650) Train Loss: 5.7057, Train Steps/Sec: 1.12 + 70%|█████████████████████████████████████████████████████████████████████████████████████████ | 77674/110696 [23:40:38<8:12:16, 1.12it/s][2025-04-27 20:44:03] (step=0077675) Train Loss: 5.7060, Train Steps/Sec: 1.12 + 70%|█████████████████████████████████████████████████████████████████████████████████████████▏ | 77699/110696 [23:41:00<8:10:47, 1.12it/s][2025-04-27 20:44:25] (step=0077700) Train Loss: 5.6839, Train Steps/Sec: 1.12 + 70%|█████████████████████████████████████████████████████████████████████████████████████████▏ | 77724/110696 [23:41:22<8:09:52, 1.12it/s][2025-04-27 20:44:48] (step=0077725) Train Loss: 5.7517, Train Steps/Sec: 1.12 + 70%|█████████████████████████████████████████████████████████████████████████████████████████▏ | 77749/110696 [23:41:45<8:06:37, 1.13it/s][2025-04-27 20:45:10] (step=0077750) Train Loss: 5.6762, Train Steps/Sec: 1.12 + 70%|█████████████████████████████████████████████████████████████████████████████████████████▏ | 77774/110696 [23:42:07<8:05:24, 1.13it/s][2025-04-27 20:45:32] (step=0077775) Train Loss: 5.7888, Train Steps/Sec: 1.12 + 70%|█████████████████████████████████████████████████████████████████████████████████████████▎ | 77799/110696 [23:42:29<8:05:42, 1.13it/s][2025-04-27 20:45:55] (step=0077800) Train Loss: 5.7860, Train Steps/Sec: 1.12 + 70%|█████████████████████████████████████████████████████████████████████████████████████████▎ | 77824/110696 [23:42:52<8:17:06, 1.10it/s][2025-04-27 20:46:17] (step=0077825) Train Loss: 5.7246, Train Steps/Sec: 1.12 + 70%|█████████████████████████████████████████████████████████████████████████████████████████▎ | 77849/110696 [23:43:14<8:13:54, 1.11it/s][2025-04-27 20:46:39] (step=0077850) Train Loss: 5.7439, Train Steps/Sec: 1.12 + 70%|█████████████████████████████████████████████████████████████████████████████████████████▎ | 77874/110696 [23:43:36<8:10:55, 1.11it/s][2025-04-27 20:47:02] (step=0077875) Train Loss: 5.7032, Train Steps/Sec: 1.12 + 70%|█████████████████████████████████████████████████████████████████████████████████████████▎ | 77899/110696 [23:43:59<8:07:50, 1.12it/s][2025-04-27 20:47:24] (step=0077900) Train Loss: 5.7598, Train Steps/Sec: 1.12 + 70%|█████████████████████████████████████████████████████████████████████████████████████████▍ | 77924/110696 [23:44:21<8:04:50, 1.13it/s][2025-04-27 20:47:46] (step=0077925) Train Loss: 5.7616, Train Steps/Sec: 1.12 + 70%|█████████████████████████████████████████████████████████████████████████████████████████▍ | 77949/110696 [23:44:43<8:05:51, 1.12it/s][2025-04-27 20:48:09] (step=0077950) Train Loss: 5.7475, Train Steps/Sec: 1.12 + 70%|█████████████████████████████████████████████████████████████████████████████████████████▍ | 77974/110696 [23:45:06<8:04:15, 1.13it/s][2025-04-27 20:48:31] (step=0077975) Train Loss: 5.6824, Train Steps/Sec: 1.12 + 70%|█████████████████████████████████████████████████████████████████████████████████████████▍ | 77999/110696 [23:45:28<8:03:51, 1.13it/s][2025-04-27 20:48:53] (step=0078000) Train Loss: 5.7386, Train Steps/Sec: 1.11 +You are using a model of type blip-2 to instantiate a model of type instructblip. This is not supported for all configurations of models and can yield errors. +[2025-04-27 20:48:53] text_config is None. Initializing the text config with default values (`OPTConfig`). +model_name_or_path:/tmp/haozhezhao/model/blip2-flan-t5-xl +image_place_holder: +tokenizer length after expend 32101 +tokenizer length before expend 32100 +100%|████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████| 5/5 [05:03<00:00, 60.72s/it] +[2025-04-27 20:55:10] Finish Eval in 78000 steps...█████████████████████████████████████████████████████████████████████████████████████████████| 5/5 [05:03<00:00, 60.41s/it] +processing left-padding... +processing left-padding... +processing left-padding... +processing left-padding... +processing left-padding... +[2025-04-27 20:55:30] Saved checkpoint to checkpoint/X2I_UltraEdiit_fourmask_LLava_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_filtered_objects_4mask_1280_context/003-GPT-XL/checkpoints/0078000.pt +[2025-04-27 20:55:32] Removed old checkpoint: checkpoint/X2I_UltraEdiit_fourmask_LLava_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_filtered_objects_4mask_1280_context/003-GPT-XL/checkpoints/0076000.pt + 70%|█████████████████████████████████████████████████████████████████████████████████████████▌ | 78024/110696 [23:52:29<8:28:06, 1.07it/s][2025-04-27 20:55:55] (step=0078025) Train Loss: 5.7780, Train Steps/Sec: 0.06 + 71%|█████████████████████████████████████████████████████████████████████████████████████████▌ | 78049/110696 [23:52:52<8:11:27, 1.11it/s][2025-04-27 20:56:17] (step=0078050) Train Loss: 5.6752, Train Steps/Sec: 1.12 + 71%|█████████████████████████████████████████████████████████████████████████████████████████▌ | 78074/110696 [23:53:14<8:05:24, 1.12it/s][2025-04-27 20:56:39] (step=0078075) Train Loss: 5.6859, Train Steps/Sec: 1.12 + 71%|█████████████████████████████████████████████████████████████████████████████████████████▌ | 78099/110696 [23:53:36<8:02:44, 1.13it/s][2025-04-27 20:57:01] (step=0078100) Train Loss: 5.7633, Train Steps/Sec: 1.12 + 71%|█████████████████████████████████████████████████████████████████████████████████████████▋ | 78124/110696 [23:53:58<7:59:36, 1.13it/s][2025-04-27 20:57:24] (step=0078125) Train Loss: 5.7159, Train Steps/Sec: 1.12 + 71%|█████████████████████████████████████████████████████████████████████████████████████████▋ | 78149/110696 [23:54:21<8:00:47, 1.13it/s][2025-04-27 20:57:46] (step=0078150) Train Loss: 5.7515, Train Steps/Sec: 1.12 + 71%|█████████████████████████████████████████████████████████████████████████████████████████▋ | 78174/110696 [23:54:43<7:57:29, 1.14it/s][2025-04-27 20:58:08] (step=0078175) Train Loss: 5.7067, Train Steps/Sec: 1.13 + 71%|█████████████████████████████████████████████████████████████████████████████████████████▋ | 78199/110696 [23:55:05<7:58:20, 1.13it/s][2025-04-27 20:58:30] (step=0078200) Train Loss: 5.7095, Train Steps/Sec: 1.12 + 71%|█████████████████████████████████████████████████████████████████████████████████████████▋ | 78224/110696 [23:55:27<8:14:01, 1.10it/s][2025-04-27 20:58:53] (step=0078225) Train Loss: 5.7550, Train Steps/Sec: 1.12 + 71%|█████████████████████████████████████████████████████████████████████████████████████████▊ | 78249/110696 [23:55:50<8:04:55, 1.12it/s][2025-04-27 20:59:15] (step=0078250) Train Loss: 5.6855, Train Steps/Sec: 1.13 + 71%|█████████████████████████████████████████████████████████████████████████████████████████▊ | 78274/110696 [23:56:12<8:01:46, 1.12it/s][2025-04-27 20:59:38] (step=0078275) Train Loss: 5.6456, Train Steps/Sec: 1.11 + 71%|█████████████████████████████████████████████████████████████████████████████████████████▊ | 78299/110696 [23:56:35<8:00:07, 1.12it/s][2025-04-27 21:00:00] (step=0078300) Train Loss: 5.7888, Train Steps/Sec: 1.12 + 71%|█████████████████████████████████████████████████████████████████████████████████████████▊ | 78324/110696 [23:56:57<7:59:01, 1.13it/s][2025-04-27 21:00:22] (step=0078325) Train Loss: 5.7384, Train Steps/Sec: 1.12 + 71%|█████████████████████████████████████████████████████████████████████████████████████████▉ | 78349/110696 [23:57:19<7:57:00, 1.13it/s][2025-04-27 21:00:44] (step=0078350) Train Loss: 5.6986, Train Steps/Sec: 1.12 + 71%|█████████████████████████████████████████████████████████████████████████████████████████▉ | 78374/110696 [23:57:41<7:54:53, 1.13it/s][2025-04-27 21:01:07] (step=0078375) Train Loss: 5.7102, Train Steps/Sec: 1.12 + 71%|█████████████████████████████████████████████████████████████████████████████████████████▉ | 78399/110696 [23:58:04<7:54:21, 1.13it/s][2025-04-27 21:01:29] (step=0078400) Train Loss: 5.7824, Train Steps/Sec: 1.12 + 71%|█████████████████████████████████████████████████████████████████████████████████████████▉ | 78424/110696 [23:58:26<8:08:37, 1.10it/s][2025-04-27 21:01:51] (step=0078425) Train Loss: 5.7604, Train Steps/Sec: 1.12 + 71%|██████████████████████████████████████████████████████████████████████████████████████████ | 78449/110696 [23:58:48<8:05:15, 1.11it/s][2025-04-27 21:02:14] (step=0078450) Train Loss: 5.6498, Train Steps/Sec: 1.12 + 71%|██████████████████████████████████████████████████████████████████████████████████████████ | 78474/110696 [23:59:11<8:02:01, 1.11it/s][2025-04-27 21:02:36] (step=0078475) Train Loss: 5.7302, Train Steps/Sec: 1.12 + 71%|██████████████████████████████████████████████████████████████████████████████████████████ | 78499/110696 [23:59:33<7:57:21, 1.12it/s][2025-04-27 21:02:58] (step=0078500) Train Loss: 5.6521, Train Steps/Sec: 1.12 + 71%|██████████████████████████████████████████████████████████████████████████████████████████ | 78524/110696 [23:59:55<7:56:04, 1.13it/s][2025-04-27 21:03:21] (step=0078525) Train Loss: 5.7017, Train Steps/Sec: 1.12 + 71%|██████████████████████████████████████████████████████████████████████████████████████████ | 78549/110696 [24:00:18<7:53:31, 1.13it/s][2025-04-27 21:03:43] (step=0078550) Train Loss: 5.7102, Train Steps/Sec: 1.12 + 71%|██████████████████████████████████████████████████████████████████████████████████████████▏ | 78574/110696 [24:00:40<7:54:43, 1.13it/s][2025-04-27 21:04:05] (step=0078575) Train Loss: 5.7726, Train Steps/Sec: 1.12 + 71%|██████████████████████████████████████████████████████████████████████████████████████████▏ | 78599/110696 [24:01:02<7:52:19, 1.13it/s][2025-04-27 21:04:28] (step=0078600) Train Loss: 5.7319, Train Steps/Sec: 1.12 + 71%|██████████████████████████████████████████████████████████████████████████████████████████▏ | 78624/110696 [24:01:25<8:06:15, 1.10it/s][2025-04-27 21:04:50] (step=0078625) Train Loss: 5.7254, Train Steps/Sec: 1.12 + 71%|██████████████████████████████████████████████████████████████████████████████████████████▏ | 78649/110696 [24:01:47<7:59:10, 1.11it/s][2025-04-27 21:05:12] (step=0078650) Train Loss: 5.7103, Train Steps/Sec: 1.12 + 71%|██████████████████████████████████████████████████████████████████████████████████████████▎ | 78674/110696 [24:02:09<7:55:59, 1.12it/s][2025-04-27 21:05:34] (step=0078675) Train Loss: 5.7113, Train Steps/Sec: 1.12 + 71%|██████████████████████████████████████████████████████████████████████████████████████████▎ | 78699/110696 [24:02:31<7:54:34, 1.12it/s][2025-04-27 21:05:57] (step=0078700) Train Loss: 5.7402, Train Steps/Sec: 1.12 + 71%|██████████████████████████████████████████████████████████████████████████████████████████▎ | 78724/110696 [24:02:54<7:53:02, 1.13it/s][2025-04-27 21:06:19] (step=0078725) Train Loss: 5.7180, Train Steps/Sec: 1.12 + 71%|██████████████████████████████████████████████████████████████████████████████████████████▎ | 78749/110696 [24:03:16<7:51:38, 1.13it/s][2025-04-27 21:06:41] (step=0078750) Train Loss: 5.6692, Train Steps/Sec: 1.12 + 71%|██████████████████████████████████████████████████████████████████████████████████████████▍ | 78774/110696 [24:03:38<7:50:01, 1.13it/s][2025-04-27 21:07:03] (step=0078775) Train Loss: 5.7363, Train Steps/Sec: 1.12 + 71%|██████████████████████████████████████████████████████████████████████████████████████████▍ | 78799/110696 [24:04:00<7:47:26, 1.14it/s][2025-04-27 21:07:26] (step=0078800) Train Loss: 5.7168, Train Steps/Sec: 1.12 + 71%|██████████████████████████████████████████████████████████████████████████████████████████▍ | 78824/110696 [24:04:23<8:02:48, 1.10it/s][2025-04-27 21:07:48] (step=0078825) Train Loss: 5.7787, Train Steps/Sec: 1.12 + 71%|██████████████████████████████████████████████████████████████████████████████████████████▍ | 78849/110696 [24:04:45<7:58:41, 1.11it/s][2025-04-27 21:08:10] (step=0078850) Train Loss: 5.7033, Train Steps/Sec: 1.12 + 71%|██████████████████████████████████████████████████████████████████████████████████████████▍ | 78874/110696 [24:05:07<7:52:32, 1.12it/s][2025-04-27 21:08:33] (step=0078875) Train Loss: 5.7221, Train Steps/Sec: 1.12 + 71%|██████████████████████████████████████████████████████████████████████████████████████████▌ | 78899/110696 [24:05:30<7:51:09, 1.12it/s][2025-04-27 21:08:55] (step=0078900) Train Loss: 5.7370, Train Steps/Sec: 1.12 + 71%|██████████████████████████████████████████████████████████████████████████████████████████▌ | 78924/110696 [24:05:52<7:47:31, 1.13it/s][2025-04-27 21:09:17] (step=0078925) Train Loss: 5.6703, Train Steps/Sec: 1.12 + 71%|██████████████████████████████████████████████████████████████████████████████████████████▌ | 78949/110696 [24:06:14<7:48:11, 1.13it/s][2025-04-27 21:09:40] (step=0078950) Train Loss: 5.7563, Train Steps/Sec: 1.12 + 71%|██████████████████████████████████████████████████████████████████████████████████████████▌ | 78974/110696 [24:06:37<7:46:59, 1.13it/s][2025-04-27 21:10:02] (step=0078975) Train Loss: 5.7335, Train Steps/Sec: 1.12 + 71%|██████████████████████████████████████████████████████████████████████████████████████████▋ | 78999/110696 [24:06:59<7:47:39, 1.13it/s][2025-04-27 21:10:24] (step=0079000) Train Loss: 5.7567, Train Steps/Sec: 1.12 + 71%|██████████████████████████████████████████████████████████████████████████████████████████▋ | 79024/110696 [24:07:21<7:58:12, 1.10it/s][2025-04-27 21:10:47] (step=0079025) Train Loss: 5.7492, Train Steps/Sec: 1.13 + 71%|██████████████████████████████████████████████████████████████████████████████████████████▋ | 79049/110696 [24:07:43<7:54:39, 1.11it/s][2025-04-27 21:11:09] (step=0079050) Train Loss: 5.7394, Train Steps/Sec: 1.12 + 71%|██████████████████████████████████████████████████████████████████████████████████████████▋ | 79074/110696 [24:08:06<7:53:32, 1.11it/s][2025-04-27 21:11:31] (step=0079075) Train Loss: 5.7095, Train Steps/Sec: 1.12 + 71%|██████████████████████████████████████████████████████████████████████████████████████████▋ | 79099/110696 [24:08:28<7:49:41, 1.12it/s][2025-04-27 21:11:53] (step=0079100) Train Loss: 5.7487, Train Steps/Sec: 1.12 + 71%|██████████████████████████████████████████████████████████████████████████████████████████▊ | 79124/110696 [24:08:50<7:46:31, 1.13it/s][2025-04-27 21:12:16] (step=0079125) Train Loss: 5.7204, Train Steps/Sec: 1.12 + 72%|██████████████████████████████████████████████████████████████████████████████████████████▊ | 79149/110696 [24:09:13<7:46:35, 1.13it/s][2025-04-27 21:12:38] (step=0079150) Train Loss: 5.7512, Train Steps/Sec: 1.12 + 72%|██████████████████████████████████████████████████████████████████████████████████████████▊ | 79174/110696 [24:09:35<7:46:01, 1.13it/s][2025-04-27 21:13:00] (step=0079175) Train Loss: 5.6861, Train Steps/Sec: 1.12 + 72%|██████████████████████████████████████████████████████████████████████████████████████████▊ | 79199/110696 [24:09:57<7:44:18, 1.13it/s][2025-04-27 21:13:23] (step=0079200) Train Loss: 5.6972, Train Steps/Sec: 1.11 + 72%|██████████████████████████████████████████████████████████████████████████████████████████▉ | 79224/110696 [24:10:20<7:58:56, 1.10it/s][2025-04-27 21:13:45] (step=0079225) Train Loss: 5.7041, Train Steps/Sec: 1.12 + 72%|██████████████████████████████████████████████████████████████████████████████████████████▉ | 79249/110696 [24:10:42<7:53:56, 1.11it/s][2025-04-27 21:14:07] (step=0079250) Train Loss: 5.6990, Train Steps/Sec: 1.12 + 72%|██████████████████████████████████████████████████████████████████████████████████████████▉ | 79274/110696 [24:11:04<7:49:46, 1.11it/s][2025-04-27 21:14:30] (step=0079275) Train Loss: 5.7544, Train Steps/Sec: 1.12 + 72%|██████████████████████████████████████████████████████████████████████████████████████████▉ | 79299/110696 [24:11:26<7:44:54, 1.13it/s][2025-04-27 21:14:52] (step=0079300) Train Loss: 5.7192, Train Steps/Sec: 1.12 + 72%|██████████████████████████████████████████████████████████████████████████████████████████▎ | 79324/110696 [24:11:53<19:19:09, 2.22s/it][2025-04-27 21:15:19] (step=0079325) Train Loss: 5.6975, Train Steps/Sec: 0.94 + 72%|███████████████████████████████████████████████████████████████████████████████████████████ | 79349/110696 [24:12:15<7:42:03, 1.13it/s][2025-04-27 21:15:41] (step=0079350) Train Loss: 5.7374, Train Steps/Sec: 1.12 + 72%|███████████████████████████████████████████████████████████████████████████████████████████ | 79374/110696 [24:12:38<7:44:48, 1.12it/s][2025-04-27 21:16:03] (step=0079375) Train Loss: 5.7054, Train Steps/Sec: 1.12 + 72%|███████████████████████████████████████████████████████████████████████████████████████████ | 79399/110696 [24:13:00<7:39:45, 1.13it/s][2025-04-27 21:16:26] (step=0079400) Train Loss: 5.8172, Train Steps/Sec: 1.12 + 72%|███████████████████████████████████████████████████████████████████████████████████████████ | 79424/110696 [24:13:22<7:54:10, 1.10it/s][2025-04-27 21:16:48] (step=0079425) Train Loss: 5.7287, Train Steps/Sec: 1.13 + 72%|███████████████████████████████████████████████████████████████████████████████████████████▏ | 79449/110696 [24:13:45<7:48:24, 1.11it/s][2025-04-27 21:17:10] (step=0079450) Train Loss: 5.6929, Train Steps/Sec: 1.12 + 72%|███████████████████████████████████████████████████████████████████████████████████████████▏ | 79474/110696 [24:14:07<7:45:37, 1.12it/s][2025-04-27 21:17:32] (step=0079475) Train Loss: 5.6777, Train Steps/Sec: 1.12 + 72%|███████████████████████████████████████████████████████████████████████████████████████████▏ | 79499/110696 [24:14:29<7:41:04, 1.13it/s][2025-04-27 21:17:55] (step=0079500) Train Loss: 5.7285, Train Steps/Sec: 1.12 + 72%|███████████████████████████████████████████████████████████████████████████████████████████▏ | 79524/110696 [24:14:51<7:39:03, 1.13it/s][2025-04-27 21:18:17] (step=0079525) Train Loss: 5.6441, Train Steps/Sec: 1.13 + 72%|███████████████████████████████████████████████████████████████████████████████████████████▎ | 79549/110696 [24:15:14<7:43:00, 1.12it/s][2025-04-27 21:18:39] (step=0079550) Train Loss: 5.7539, Train Steps/Sec: 1.12 + 72%|███████████████████████████████████████████████████████████████████████████████████████████▎ | 79574/110696 [24:15:41<8:08:56, 1.06it/s][2025-04-27 21:19:06] (step=0079575) Train Loss: 5.7067, Train Steps/Sec: 0.93 + 72%|███████████████████████████████████████████████████████████████████████████████████████████▎ | 79599/110696 [24:16:03<7:37:11, 1.13it/s][2025-04-27 21:19:29] (step=0079600) Train Loss: 5.7267, Train Steps/Sec: 1.12 + 72%|███████████████████████████████████████████████████████████████████████████████████████████▎ | 79624/110696 [24:16:25<7:51:18, 1.10it/s][2025-04-27 21:19:51] (step=0079625) Train Loss: 5.7300, Train Steps/Sec: 1.12 + 72%|███████████████████████████████████████████████████████████████████████████████████████████▍ | 79649/110696 [24:16:48<7:46:01, 1.11it/s][2025-04-27 21:20:13] (step=0079650) Train Loss: 5.7583, Train Steps/Sec: 1.12 + 72%|███████████████████████████████████████████████████████████████████████████████████████████▍ | 79674/110696 [24:17:10<7:41:47, 1.12it/s][2025-04-27 21:20:35] (step=0079675) Train Loss: 5.6433, Train Steps/Sec: 1.12 + 72%|███████████████████████████████████████████████████████████████████████████████████████████▍ | 79699/110696 [24:17:32<7:39:35, 1.12it/s][2025-04-27 21:20:58] (step=0079700) Train Loss: 5.6668, Train Steps/Sec: 1.12 + 72%|███████████████████████████████████████████████████████████████████████████████████████████▍ | 79724/110696 [24:17:54<7:35:25, 1.13it/s][2025-04-27 21:21:20] (step=0079725) Train Loss: 5.6938, Train Steps/Sec: 1.13 + 72%|███████████████████████████████████████████████████████████████████████████████████████████▍ | 79749/110696 [24:18:17<7:36:02, 1.13it/s][2025-04-27 21:21:42] (step=0079750) Train Loss: 5.7003, Train Steps/Sec: 1.13 + 72%|███████████████████████████████████████████████████████████████████████████████████████████▌ | 79774/110696 [24:18:39<7:35:46, 1.13it/s][2025-04-27 21:22:04] (step=0079775) Train Loss: 5.6471, Train Steps/Sec: 1.12 + 72%|███████████████████████████████████████████████████████████████████████████████████████████▌ | 79799/110696 [24:19:01<7:34:01, 1.13it/s][2025-04-27 21:22:27] (step=0079800) Train Loss: 5.7392, Train Steps/Sec: 1.12 + 72%|███████████████████████████████████████████████████████████████████████████████████████████▌ | 79824/110696 [24:19:24<7:50:49, 1.09it/s][2025-04-27 21:22:49] (step=0079825) Train Loss: 5.7002, Train Steps/Sec: 1.12 + 72%|███████████████████████████████████████████████████████████████████████████████████████████▌ | 79849/110696 [24:19:46<7:43:24, 1.11it/s][2025-04-27 21:23:11] (step=0079850) Train Loss: 5.6756, Train Steps/Sec: 1.12 + 72%|███████████████████████████████████████████████████████████████████████████████████████████▋ | 79874/110696 [24:20:08<7:40:11, 1.12it/s][2025-04-27 21:23:34] (step=0079875) Train Loss: 5.7165, Train Steps/Sec: 1.12 + 72%|███████████████████████████████████████████████████████████████████████████████████████████▋ | 79899/110696 [24:20:31<7:35:49, 1.13it/s][2025-04-27 21:23:56] (step=0079900) Train Loss: 5.7446, Train Steps/Sec: 1.12 + 72%|███████████████████████████████████████████████████████████████████████████████████████████▋ | 79924/110696 [24:20:53<7:33:13, 1.13it/s][2025-04-27 21:24:18] (step=0079925) Train Loss: 5.6686, Train Steps/Sec: 1.12 + 72%|███████████████████████████████████████████████████████████████████████████████████████████▋ | 79949/110696 [24:21:15<7:32:54, 1.13it/s][2025-04-27 21:24:40] (step=0079950) Train Loss: 5.7316, Train Steps/Sec: 1.12 + 72%|███████████████████████████████████████████████████████████████████████████████████████████▊ | 79974/110696 [24:21:37<7:33:11, 1.13it/s][2025-04-27 21:25:03] (step=0079975) Train Loss: 5.7983, Train Steps/Sec: 1.12 + 72%|███████████████████████████████████████████████████████████████████████████████████████████▊ | 79999/110696 [24:22:00<7:30:26, 1.14it/s][2025-04-27 21:25:25] (step=0080000) Train Loss: 5.7237, Train Steps/Sec: 1.12 +You are using a model of type blip-2 to instantiate a model of type instructblip. This is not supported for all configurations of models and can yield errors. +[2025-04-27 21:25:25] text_config is None. Initializing the text config with default values (`OPTConfig`). +model_name_or_path:/tmp/haozhezhao/model/blip2-flan-t5-xl +image_place_holder: +tokenizer length after expend 32101 +tokenizer length before expend 32100 +100%|████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████| 5/5 [05:05<00:00, 61.09s/it] +[2025-04-27 21:31:45] Finish Eval in 80000 steps...█████████████████████████████████████████████████████████████████████████████████████████████| 5/5 [05:04<00:00, 60.75s/it] +processing left-padding... +processing left-padding... +processing left-padding... +processing left-padding... +processing left-padding... +[2025-04-27 21:32:05] Saved checkpoint to checkpoint/X2I_UltraEdiit_fourmask_LLava_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_filtered_objects_4mask_1280_context/003-GPT-XL/checkpoints/0080000.pt +[2025-04-27 21:32:07] Removed old checkpoint: checkpoint/X2I_UltraEdiit_fourmask_LLava_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_filtered_objects_4mask_1280_context/003-GPT-XL/checkpoints/0078000.pt + 72%|███████████████████████████████████████████████████████████████████████████████████████████▊ | 80024/110696 [24:29:04<7:57:32, 1.07it/s][2025-04-27 21:32:30] (step=0080025) Train Loss: 5.7411, Train Steps/Sec: 0.06 + 72%|███████████████████████████████████████████████████████████████████████████████████████████▊ | 80049/110696 [24:29:26<7:38:46, 1.11it/s][2025-04-27 21:32:52] (step=0080050) Train Loss: 5.7015, Train Steps/Sec: 1.12 + 72%|███████████████████████████████████████████████████████████████████████████████████████████▊ | 80074/110696 [24:29:49<7:37:17, 1.12it/s][2025-04-27 21:33:14] (step=0080075) Train Loss: 5.7247, Train Steps/Sec: 1.12 + 72%|███████████████████████████████████████████████████████████████████████████████████████████▉ | 80099/110696 [24:30:11<7:33:19, 1.12it/s][2025-04-27 21:33:36] (step=0080100) Train Loss: 5.7823, Train Steps/Sec: 1.12 + 72%|███████████████████████████████████████████████████████████████████████████████████████████▉ | 80124/110696 [24:30:33<7:31:40, 1.13it/s][2025-04-27 21:33:59] (step=0080125) Train Loss: 5.7032, Train Steps/Sec: 1.12 + 72%|███████████████████████████████████████████████████████████████████████████████████████████▉ | 80149/110696 [24:30:56<7:29:32, 1.13it/s][2025-04-27 21:34:21] (step=0080150) Train Loss: 5.7284, Train Steps/Sec: 1.12 + 72%|███████████████████████████████████████████████████████████████████████████████████████████▉ | 80174/110696 [24:31:18<7:29:27, 1.13it/s][2025-04-27 21:34:43] (step=0080175) Train Loss: 5.6436, Train Steps/Sec: 1.12 + 72%|████████████████████████████████████████████████████████████████████████████████████████████ | 80199/110696 [24:31:41<7:28:56, 1.13it/s][2025-04-27 21:35:06] (step=0080200) Train Loss: 5.7453, Train Steps/Sec: 1.09 + 72%|████████████████████████████████████████████████████████████████████████████████████████████ | 80224/110696 [24:32:03<7:44:43, 1.09it/s][2025-04-27 21:35:28] (step=0080225) Train Loss: 5.7208, Train Steps/Sec: 1.12 + 72%|████████████████████████████████████████████████████████████████████████████████████████████ | 80249/110696 [24:32:25<7:36:02, 1.11it/s][2025-04-27 21:35:51] (step=0080250) Train Loss: 5.7474, Train Steps/Sec: 1.12 + 73%|████████████████████████████████████████████████████████████████████████████████████████████ | 80274/110696 [24:32:48<7:33:30, 1.12it/s][2025-04-27 21:36:13] (step=0080275) Train Loss: 5.6310, Train Steps/Sec: 1.12 + 73%|████████████████████████████████████████████████████████████████████████████████████████████▏ | 80299/110696 [24:33:18<7:36:59, 1.11it/s][2025-04-27 21:36:43] (step=0080300) Train Loss: 5.6718, Train Steps/Sec: 0.82 + 73%|████████████████████████████████████████████████████████████████████████████████████████████▏ | 80324/110696 [24:33:45<7:35:03, 1.11it/s][2025-04-27 21:37:10] (step=0080325) Train Loss: 5.7636, Train Steps/Sec: 0.94 + 73%|████████████████████████████████████████████████████████████████████████████████████████████▏ | 80349/110696 [24:34:12<9:39:44, 1.15s/it][2025-04-27 21:37:37] (step=0080350) Train Loss: 5.7279, Train Steps/Sec: 0.91 + 73%|████████████████████████████████████████████████████████████████████████████████████████████▏ | 80374/110696 [24:34:34<7:26:46, 1.13it/s][2025-04-27 21:38:00] (step=0080375) Train Loss: 5.6776, Train Steps/Sec: 1.12 + 73%|████████████████████████████████████████████████████████████████████████████████████████████▏ | 80399/110696 [24:34:57<7:25:30, 1.13it/s][2025-04-27 21:38:22] (step=0080400) Train Loss: 5.7098, Train Steps/Sec: 1.12 + 73%|████████████████████████████████████████████████████████████████████████████████████████████▎ | 80424/110696 [24:35:19<7:39:48, 1.10it/s][2025-04-27 21:38:44] (step=0080425) Train Loss: 5.7267, Train Steps/Sec: 1.12 + 73%|████████████████████████████████████████████████████████████████████████████████████████████▎ | 80449/110696 [24:35:41<7:32:53, 1.11it/s][2025-04-27 21:39:07] (step=0080450) Train Loss: 5.7492, Train Steps/Sec: 1.12 + 73%|████████████████████████████████████████████████████████████████████████████████████████████▎ | 80474/110696 [24:36:04<7:30:38, 1.12it/s][2025-04-27 21:39:29] (step=0080475) Train Loss: 5.7493, Train Steps/Sec: 1.12 + 73%|████████████████████████████████████████████████████████████████████████████████████████████▎ | 80499/110696 [24:36:26<7:27:40, 1.12it/s][2025-04-27 21:39:51] (step=0080500) Train Loss: 5.7365, Train Steps/Sec: 1.12 + 73%|████████████████████████████████████████████████████████████████████████████████████████████▍ | 80524/110696 [24:36:48<7:25:58, 1.13it/s][2025-04-27 21:40:14] (step=0080525) Train Loss: 5.7479, Train Steps/Sec: 1.12 + 73%|████████████████████████████████████████████████████████████████████████████████████████████▍ | 80549/110696 [24:37:11<7:23:36, 1.13it/s][2025-04-27 21:40:36] (step=0080550) Train Loss: 5.7392, Train Steps/Sec: 1.12 + 73%|████████████████████████████████████████████████████████████████████████████████████████████▍ | 80574/110696 [24:37:33<7:24:33, 1.13it/s][2025-04-27 21:40:58] (step=0080575) Train Loss: 5.6386, Train Steps/Sec: 1.12 + 73%|████████████████████████████████████████████████████████████████████████████████████████████▍ | 80599/110696 [24:37:55<7:22:55, 1.13it/s][2025-04-27 21:41:21] (step=0080600) Train Loss: 5.7757, Train Steps/Sec: 1.12 + 73%|████████████████████████████████████████████████████████████████████████████████████████████▍ | 80624/110696 [24:38:18<7:39:41, 1.09it/s][2025-04-27 21:41:43] (step=0080625) Train Loss: 5.7486, Train Steps/Sec: 1.12 + 73%|████████████████████████████████████████████████████████████████████████████████████████████▌ | 80649/110696 [24:38:45<7:39:20, 1.09it/s][2025-04-27 21:42:10] (step=0080650) Train Loss: 5.6948, Train Steps/Sec: 0.92 + 73%|████████████████████████████████████████████████████████████████████████████████████████████▌ | 80674/110696 [24:39:07<7:28:52, 1.11it/s][2025-04-27 21:42:32] (step=0080675) Train Loss: 5.7412, Train Steps/Sec: 1.12 + 73%|████████████████████████████████████████████████████████████████████████████████████████████▌ | 80699/110696 [24:39:29<7:23:27, 1.13it/s][2025-04-27 21:42:55] (step=0080700) Train Loss: 5.7273, Train Steps/Sec: 1.12 + 73%|████████████████████████████████████████████████████████████████████████████████████████████▌ | 80724/110696 [24:39:51<7:22:33, 1.13it/s][2025-04-27 21:43:17] (step=0080725) Train Loss: 5.7458, Train Steps/Sec: 1.12 + 73%|████████████████████████████████████████████████████████████████████████████████████████████▋ | 80749/110696 [24:40:14<7:22:05, 1.13it/s][2025-04-27 21:43:39] (step=0080750) Train Loss: 5.7475, Train Steps/Sec: 1.12 + 73%|████████████████████████████████████████████████████████████████████████████████████████████▋ | 80774/110696 [24:40:36<7:20:53, 1.13it/s][2025-04-27 21:44:01] (step=0080775) Train Loss: 5.7737, Train Steps/Sec: 1.12 + 73%|████████████████████████████████████████████████████████████████████████████████████████████▋ | 80799/110696 [24:40:58<7:19:34, 1.13it/s][2025-04-27 21:44:24] (step=0080800) Train Loss: 5.7249, Train Steps/Sec: 1.12 + 73%|████████████████████████████████████████████████████████████████████████████████████████████▋ | 80824/110696 [24:41:21<7:34:24, 1.10it/s][2025-04-27 21:44:46] (step=0080825) Train Loss: 5.7064, Train Steps/Sec: 1.12 + 73%|████████████████████████████████████████████████████████████████████████████████████████████▊ | 80849/110696 [24:41:43<7:26:29, 1.11it/s][2025-04-27 21:45:08] (step=0080850) Train Loss: 5.7084, Train Steps/Sec: 1.12 + 73%|████████████████████████████████████████████████████████████████████████████████████████████▊ | 80874/110696 [24:42:05<7:24:57, 1.12it/s][2025-04-27 21:45:31] (step=0080875) Train Loss: 5.6962, Train Steps/Sec: 1.12 + 73%|████████████████████████████████████████████████████████████████████████████████████████████▊ | 80899/110696 [24:42:28<7:22:17, 1.12it/s][2025-04-27 21:45:53] (step=0080900) Train Loss: 5.7035, Train Steps/Sec: 1.12 + 73%|████████████████████████████████████████████████████████████████████████████████████████████▊ | 80924/110696 [24:42:50<7:21:18, 1.12it/s][2025-04-27 21:46:15] (step=0080925) Train Loss: 5.6863, Train Steps/Sec: 1.12 + 73%|████████████████████████████████████████████████████████████████████████████████████████████▊ | 80949/110696 [24:43:12<7:19:35, 1.13it/s][2025-04-27 21:46:38] (step=0080950) Train Loss: 5.7624, Train Steps/Sec: 1.12 + 73%|████████████████████████████████████████████████████████████████████████████████████████████▉ | 80974/110696 [24:43:35<7:15:10, 1.14it/s][2025-04-27 21:47:00] (step=0080975) Train Loss: 5.7487, Train Steps/Sec: 1.12 + 73%|████████████████████████████████████████████████████████████████████████████████████████████▉ | 80999/110696 [24:43:57<7:16:35, 1.13it/s][2025-04-27 21:47:22] (step=0081000) Train Loss: 5.7172, Train Steps/Sec: 1.12 + 73%|████████████████████████████████████████████████████████████████████████████████████████████▉ | 81024/110696 [24:44:19<7:28:54, 1.10it/s][2025-04-27 21:47:44] (step=0081025) Train Loss: 5.7207, Train Steps/Sec: 1.12 + 73%|████████████████████████████████████████████████████████████████████████████████████████████▉ | 81049/110696 [24:44:41<7:26:01, 1.11it/s][2025-04-27 21:48:07] (step=0081050) Train Loss: 5.7074, Train Steps/Sec: 1.12 + 73%|█████████████████████████████████████████████████████████████████████████████████████████████ | 81074/110696 [24:45:04<7:20:48, 1.12it/s][2025-04-27 21:48:29] (step=0081075) Train Loss: 5.7157, Train Steps/Sec: 1.12 + 73%|█████████████████████████████████████████████████████████████████████████████████████████████ | 81099/110696 [24:45:26<7:19:16, 1.12it/s][2025-04-27 21:48:51] (step=0081100) Train Loss: 5.7350, Train Steps/Sec: 1.12 + 73%|█████████████████████████████████████████████████████████████████████████████████████████████ | 81124/110696 [24:45:48<7:16:19, 1.13it/s][2025-04-27 21:49:14] (step=0081125) Train Loss: 5.7097, Train Steps/Sec: 1.12 + 73%|█████████████████████████████████████████████████████████████████████████████████████████████ | 81149/110696 [24:46:11<7:17:47, 1.12it/s][2025-04-27 21:49:36] (step=0081150) Train Loss: 5.7220, Train Steps/Sec: 1.12 + 73%|█████████████████████████████████████████████████████████████████████████████████████████████▏ | 81174/110696 [24:46:33<7:16:00, 1.13it/s][2025-04-27 21:49:58] (step=0081175) Train Loss: 5.7145, Train Steps/Sec: 1.12 + 73%|█████████████████████████████████████████████████████████████████████████████████████████████▏ | 81199/110696 [24:46:55<7:12:36, 1.14it/s][2025-04-27 21:50:21] (step=0081200) Train Loss: 5.7599, Train Steps/Sec: 1.12 + 73%|█████████████████████████████████████████████████████████████████████████████████████████████▏ | 81224/110696 [24:47:18<7:27:06, 1.10it/s][2025-04-27 21:50:43] (step=0081225) Train Loss: 5.6874, Train Steps/Sec: 1.12 + 73%|█████████████████████████████████████████████████████████████████████████████████████████████▏ | 81249/110696 [24:47:40<7:20:08, 1.12it/s][2025-04-27 21:51:05] (step=0081250) Train Loss: 5.6677, Train Steps/Sec: 1.12 + 73%|█████████████████████████████████████████████████████████████████████████████████████████████▏ | 81274/110696 [24:48:02<7:17:56, 1.12it/s][2025-04-27 21:51:27] (step=0081275) Train Loss: 5.7177, Train Steps/Sec: 1.12 + 73%|█████████████████████████████████████████████████████████████████████████████████████████████▎ | 81299/110696 [24:48:24<7:16:24, 1.12it/s][2025-04-27 21:51:50] (step=0081300) Train Loss: 5.6862, Train Steps/Sec: 1.12 + 73%|█████████████████████████████████████████████████████████████████████████████████████████████▎ | 81324/110696 [24:48:47<7:14:10, 1.13it/s][2025-04-27 21:52:12] (step=0081325) Train Loss: 5.7284, Train Steps/Sec: 1.12 + 73%|█████████████████████████████████████████████████████████████████████████████████████████████▎ | 81349/110696 [24:49:09<7:13:06, 1.13it/s][2025-04-27 21:52:34] (step=0081350) Train Loss: 5.6993, Train Steps/Sec: 1.12 + 74%|█████████████████████████████████████████████████████████████████████████████████████████████▎ | 81374/110696 [24:49:31<7:14:33, 1.12it/s][2025-04-27 21:52:57] (step=0081375) Train Loss: 5.7459, Train Steps/Sec: 1.12 + 74%|█████████████████████████████████████████████████████████████████████████████████████████████▍ | 81399/110696 [24:49:54<7:11:04, 1.13it/s][2025-04-27 21:53:19] (step=0081400) Train Loss: 5.7236, Train Steps/Sec: 1.12 + 74%|█████████████████████████████████████████████████████████████████████████████████████████████▍ | 81424/110696 [24:50:16<7:22:54, 1.10it/s][2025-04-27 21:53:41] (step=0081425) Train Loss: 5.6397, Train Steps/Sec: 1.12 + 74%|█████████████████████████████████████████████████████████████████████████████████████████████▍ | 81449/110696 [24:50:38<7:18:37, 1.11it/s][2025-04-27 21:54:04] (step=0081450) Train Loss: 5.7623, Train Steps/Sec: 1.12 + 74%|█████████████████████████████████████████████████████████████████████████████████████████████▍ | 81474/110696 [24:51:01<7:15:20, 1.12it/s][2025-04-27 21:54:26] (step=0081475) Train Loss: 5.7378, Train Steps/Sec: 1.12 + 74%|█████████████████████████████████████████████████████████████████████████████████████████████▌ | 81499/110696 [24:51:23<7:12:14, 1.13it/s][2025-04-27 21:54:48] (step=0081500) Train Loss: 5.7191, Train Steps/Sec: 1.12 + 74%|█████████████████████████████████████████████████████████████████████████████████████████████▌ | 81524/110696 [24:51:45<7:11:38, 1.13it/s][2025-04-27 21:55:11] (step=0081525) Train Loss: 5.7210, Train Steps/Sec: 1.12 + 74%|█████████████████████████████████████████████████████████████████████████████████████████████▌ | 81549/110696 [24:52:07<7:10:57, 1.13it/s][2025-04-27 21:55:33] (step=0081550) Train Loss: 5.6772, Train Steps/Sec: 1.12 + 74%|█████████████████████████████████████████████████████████████████████████████████████████████▌ | 81574/110696 [24:52:30<7:08:04, 1.13it/s][2025-04-27 21:55:55] (step=0081575) Train Loss: 5.6470, Train Steps/Sec: 1.12 + 74%|█████████████████████████████████████████████████████████████████████████████████████████████▌ | 81599/110696 [24:52:52<7:09:48, 1.13it/s][2025-04-27 21:56:18] (step=0081600) Train Loss: 5.6866, Train Steps/Sec: 1.12 + 74%|█████████████████████████████████████████████████████████████████████████████████████████████▋ | 81624/110696 [24:53:14<7:21:43, 1.10it/s][2025-04-27 21:56:40] (step=0081625) Train Loss: 5.7508, Train Steps/Sec: 1.12 + 74%|█████████████████████████████████████████████████████████████████████████████████████████████▋ | 81649/110696 [24:53:37<7:15:33, 1.11it/s][2025-04-27 21:57:02] (step=0081650) Train Loss: 5.7055, Train Steps/Sec: 1.12 + 74%|█████████████████████████████████████████████████████████████████████████████████████████████▋ | 81674/110696 [24:53:59<7:12:16, 1.12it/s][2025-04-27 21:57:24] (step=0081675) Train Loss: 5.7767, Train Steps/Sec: 1.12 + 74%|█████████████████████████████████████████████████████████████████████████████████████████████▋ | 81699/110696 [24:54:21<7:10:30, 1.12it/s][2025-04-27 21:57:47] (step=0081700) Train Loss: 5.6001, Train Steps/Sec: 1.12 + 74%|█████████████████████████████████████████████████████████████████████████████████████████████▊ | 81724/110696 [24:54:44<7:07:47, 1.13it/s][2025-04-27 21:58:09] (step=0081725) Train Loss: 5.7170, Train Steps/Sec: 1.12 + 74%|█████████████████████████████████████████████████████████████████████████████████████████████▊ | 81749/110696 [24:55:06<7:08:37, 1.13it/s][2025-04-27 21:58:31] (step=0081750) Train Loss: 5.7292, Train Steps/Sec: 1.12 + 74%|█████████████████████████████████████████████████████████████████████████████████████████████▊ | 81774/110696 [24:55:28<7:06:43, 1.13it/s][2025-04-27 21:58:54] (step=0081775) Train Loss: 5.7452, Train Steps/Sec: 1.12 + 74%|█████████████████████████████████████████████████████████████████████████████████████████████▊ | 81799/110696 [24:55:51<7:05:23, 1.13it/s][2025-04-27 21:59:16] (step=0081800) Train Loss: 5.6708, Train Steps/Sec: 1.12 + 74%|█████████████████████████████████████████████████████████████████████████████████████████████▉ | 81824/110696 [24:56:13<7:18:34, 1.10it/s][2025-04-27 21:59:38] (step=0081825) Train Loss: 5.7133, Train Steps/Sec: 1.12 + 74%|█████████████████████████████████████████████████████████████████████████████████████████████▉ | 81849/110696 [24:56:35<7:12:56, 1.11it/s][2025-04-27 22:00:01] (step=0081850) Train Loss: 5.7478, Train Steps/Sec: 1.12 + 74%|█████████████████████████████████████████████████████████████████████████████████████████████▉ | 81874/110696 [24:56:58<7:07:50, 1.12it/s][2025-04-27 22:00:23] (step=0081875) Train Loss: 5.7593, Train Steps/Sec: 1.12 + 74%|█████████████████████████████████████████████████████████████████████████████████████████████▉ | 81899/110696 [24:57:20<7:06:14, 1.13it/s][2025-04-27 22:00:45] (step=0081900) Train Loss: 5.7281, Train Steps/Sec: 1.12 + 74%|█████████████████████████████████████████████████████████████████████████████████████████████▉ | 81924/110696 [24:57:42<7:05:45, 1.13it/s][2025-04-27 22:01:07] (step=0081925) Train Loss: 5.7103, Train Steps/Sec: 1.12 + 74%|██████████████████████████████████████████████████████████████████████████████████████████████ | 81949/110696 [24:58:04<7:04:11, 1.13it/s][2025-04-27 22:01:30] (step=0081950) Train Loss: 5.7950, Train Steps/Sec: 1.12 + 74%|██████████████████████████████████████████████████████████████████████████████████████████████ | 81974/110696 [24:58:27<7:04:10, 1.13it/s][2025-04-27 22:01:52] (step=0081975) Train Loss: 5.7054, Train Steps/Sec: 1.12 + 74%|██████████████████████████████████████████████████████████████████████████████████████████████ | 81999/110696 [24:58:49<7:02:07, 1.13it/s][2025-04-27 22:02:14] (step=0082000) Train Loss: 5.7378, Train Steps/Sec: 1.12 +You are using a model of type blip-2 to instantiate a model of type instructblip. This is not supported for all configurations of models and can yield errors. +[2025-04-27 22:02:14] text_config is None. Initializing the text config with default values (`OPTConfig`). +model_name_or_path:/tmp/haozhezhao/model/blip2-flan-t5-xl +image_place_holder: +tokenizer length after expend 32101 +tokenizer length before expend 32100 +100%|████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████| 5/5 [05:04<00:00, 60.90s/it] +[2025-04-27 22:08:33] Finish Eval in 82000 steps...█████████████████████████████████████████████████████████████████████████████████████████████| 5/5 [05:03<00:00, 60.59s/it] +processing left-padding... +processing left-padding... +processing left-padding... +processing left-padding... +processing left-padding... +[2025-04-27 22:08:53] Saved checkpoint to checkpoint/X2I_UltraEdiit_fourmask_LLava_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_filtered_objects_4mask_1280_context/003-GPT-XL/checkpoints/0082000.pt +[2025-04-27 22:08:55] Removed old checkpoint: checkpoint/X2I_UltraEdiit_fourmask_LLava_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_filtered_objects_4mask_1280_context/003-GPT-XL/checkpoints/0080000.pt + 74%|██████████████████████████████████████████████████████████████████████████████████████████████ | 82024/110696 [25:05:52<7:26:43, 1.07it/s][2025-04-27 22:09:18] (step=0082025) Train Loss: 5.6959, Train Steps/Sec: 0.06 + 74%|██████████████████████████████████████████████████████████████████████████████████████████████▏ | 82049/110696 [25:06:15<7:08:59, 1.11it/s][2025-04-27 22:09:40] (step=0082050) Train Loss: 5.7004, Train Steps/Sec: 1.12 + 74%|██████████████████████████████████████████████████████████████████████████████████████████████▏ | 82074/110696 [25:06:37<7:05:29, 1.12it/s][2025-04-27 22:10:02] (step=0082075) Train Loss: 5.7105, Train Steps/Sec: 1.12 + 74%|██████████████████████████████████████████████████████████████████████████████████████████████▏ | 82099/110696 [25:06:59<7:04:55, 1.12it/s][2025-04-27 22:10:25] (step=0082100) Train Loss: 5.7161, Train Steps/Sec: 1.10 + 74%|██████████████████████████████████████████████████████████████████████████████████████████████▏ | 82124/110696 [25:07:22<7:03:00, 1.13it/s][2025-04-27 22:10:47] (step=0082125) Train Loss: 5.7298, Train Steps/Sec: 1.12 + 74%|██████████████████████████████████████████████████████████████████████████████████████████████▏ | 82149/110696 [25:07:44<7:01:29, 1.13it/s][2025-04-27 22:11:09] (step=0082150) Train Loss: 5.7614, Train Steps/Sec: 1.12 + 74%|██████████████████████████████████████████████████████████████████████████████████████████████▎ | 82174/110696 [25:08:06<6:59:17, 1.13it/s][2025-04-27 22:11:32] (step=0082175) Train Loss: 5.6564, Train Steps/Sec: 1.12 + 74%|██████████████████████████████████████████████████████████████████████████████████████████████▎ | 82199/110696 [25:08:29<7:00:02, 1.13it/s][2025-04-27 22:11:54] (step=0082200) Train Loss: 5.7258, Train Steps/Sec: 1.12 + 74%|██████████████████████████████████████████████████████████████████████████████████████████████▎ | 82224/110696 [25:08:51<7:10:49, 1.10it/s][2025-04-27 22:12:16] (step=0082225) Train Loss: 5.7487, Train Steps/Sec: 1.12 + 74%|██████████████████████████████████████████████████████████████████████████████████████████████▎ | 82249/110696 [25:09:13<7:05:39, 1.11it/s][2025-04-27 22:12:38] (step=0082250) Train Loss: 5.7056, Train Steps/Sec: 1.12 + 74%|██████████████████████████████████████████████████████████████████████████████████████████████▍ | 82274/110696 [25:09:35<7:04:08, 1.12it/s][2025-04-27 22:13:01] (step=0082275) Train Loss: 5.7602, Train Steps/Sec: 1.12 + 74%|██████████████████████████████████████████████████████████████████████████████████████████████▍ | 82299/110696 [25:09:58<7:00:59, 1.12it/s][2025-04-27 22:13:23] (step=0082300) Train Loss: 5.7621, Train Steps/Sec: 1.12 + 74%|██████████████████████████████████████████████████████████████████████████████████████████████▍ | 82324/110696 [25:10:20<6:59:40, 1.13it/s][2025-04-27 22:13:45] (step=0082325) Train Loss: 5.6342, Train Steps/Sec: 1.12 + 74%|██████████████████████████████████████████████████████████████████████████████████████████████▍ | 82349/110696 [25:10:42<6:58:59, 1.13it/s][2025-04-27 22:14:08] (step=0082350) Train Loss: 5.7166, Train Steps/Sec: 1.12 + 74%|██████████████████████████████████████████████████████████████████████████████████████████████▌ | 82374/110696 [25:11:05<6:56:45, 1.13it/s][2025-04-27 22:14:30] (step=0082375) Train Loss: 5.7221, Train Steps/Sec: 1.12 + 74%|██████████████████████████████████████████████████████████████████████████████████████████████▌ | 82399/110696 [25:11:27<6:55:48, 1.13it/s][2025-04-27 22:14:52] (step=0082400) Train Loss: 5.7253, Train Steps/Sec: 1.12 + 74%|██████████████████████████████████████████████████████████████████████████████████████████████▌ | 82424/110696 [25:11:49<7:08:06, 1.10it/s][2025-04-27 22:15:14] (step=0082425) Train Loss: 5.7038, Train Steps/Sec: 1.12 + 74%|██████████████████████████████████████████████████████████████████████████████████████████████▌ | 82449/110696 [25:12:11<7:02:10, 1.12it/s][2025-04-27 22:15:37] (step=0082450) Train Loss: 5.7428, Train Steps/Sec: 1.12 + 75%|██████████████████████████████████████████████████████████████████████████████████████████████▌ | 82474/110696 [25:12:34<7:00:30, 1.12it/s][2025-04-27 22:15:59] (step=0082475) Train Loss: 5.7849, Train Steps/Sec: 1.12 + 75%|██████████████████████████████████████████████████████████████████████████████████████████████▋ | 82499/110696 [25:12:56<6:56:47, 1.13it/s][2025-04-27 22:16:21] (step=0082500) Train Loss: 5.6996, Train Steps/Sec: 1.12 + 75%|██████████████████████████████████████████████████████████████████████████████████████████████▋ | 82524/110696 [25:13:18<6:56:54, 1.13it/s][2025-04-27 22:16:43] (step=0082525) Train Loss: 5.6924, Train Steps/Sec: 1.12 + 75%|██████████████████████████████████████████████████████████████████████████████████████████████▋ | 82549/110696 [25:13:40<6:53:40, 1.13it/s][2025-04-27 22:17:06] (step=0082550) Train Loss: 5.7301, Train Steps/Sec: 1.12 + 75%|██████████████████████████████████████████████████████████████████████████████████████████████▋ | 82574/110696 [25:14:03<6:55:49, 1.13it/s][2025-04-27 22:17:28] (step=0082575) Train Loss: 5.7365, Train Steps/Sec: 1.12 + 75%|██████████████████████████████████████████████████████████████████████████████████████████████▊ | 82599/110696 [25:14:25<6:52:30, 1.14it/s][2025-04-27 22:17:50] (step=0082600) Train Loss: 5.7343, Train Steps/Sec: 1.12 + 75%|██████████████████████████████████████████████████████████████████████████████████████████████▊ | 82624/110696 [25:14:47<7:03:55, 1.10it/s][2025-04-27 22:18:13] (step=0082625) Train Loss: 5.6706, Train Steps/Sec: 1.12 + 75%|██████████████████████████████████████████████████████████████████████████████████████████████▊ | 82649/110696 [25:15:10<7:00:05, 1.11it/s][2025-04-27 22:18:35] (step=0082650) Train Loss: 5.7887, Train Steps/Sec: 1.12 + 75%|██████████████████████████████████████████████████████████████████████████████████████████████▊ | 82674/110696 [25:15:32<6:55:33, 1.12it/s][2025-04-27 22:18:57] (step=0082675) Train Loss: 5.7197, Train Steps/Sec: 1.12 + 75%|██████████████████████████████████████████████████████████████████████████████████████████████▉ | 82699/110696 [25:15:54<6:53:19, 1.13it/s][2025-04-27 22:19:19] (step=0082700) Train Loss: 5.6927, Train Steps/Sec: 1.13 + 75%|██████████████████████████████████████████████████████████████████████████████████████████████▉ | 82724/110696 [25:16:16<6:52:47, 1.13it/s][2025-04-27 22:19:42] (step=0082725) Train Loss: 5.6701, Train Steps/Sec: 1.12 + 75%|██████████████████████████████████████████████████████████████████████████████████████████████▉ | 82749/110696 [25:16:39<6:54:23, 1.12it/s][2025-04-27 22:20:04] (step=0082750) Train Loss: 5.7175, Train Steps/Sec: 1.12 + 75%|██████████████████████████████████████████████████████████████████████████████████████████████▉ | 82774/110696 [25:17:01<6:52:13, 1.13it/s][2025-04-27 22:20:26] (step=0082775) Train Loss: 5.6631, Train Steps/Sec: 1.12 + 75%|██████████████████████████████████████████████████████████████████████████████████████████████▉ | 82799/110696 [25:17:23<6:49:46, 1.13it/s][2025-04-27 22:20:49] (step=0082800) Train Loss: 5.7600, Train Steps/Sec: 1.12 + 75%|███████████████████████████████████████████████████████████████████████████████████████████████ | 82824/110696 [25:17:46<7:04:35, 1.09it/s][2025-04-27 22:21:11] (step=0082825) Train Loss: 5.6903, Train Steps/Sec: 1.12 + 75%|███████████████████████████████████████████████████████████████████████████████████████████████ | 82849/110696 [25:18:08<6:55:23, 1.12it/s][2025-04-27 22:21:33] (step=0082850) Train Loss: 5.7082, Train Steps/Sec: 1.13 + 75%|███████████████████████████████████████████████████████████████████████████████████████████████ | 82874/110696 [25:18:30<6:52:57, 1.12it/s][2025-04-27 22:21:55] (step=0082875) Train Loss: 5.7143, Train Steps/Sec: 1.12 + 75%|███████████████████████████████████████████████████████████████████████████████████████████████ | 82899/110696 [25:18:52<6:51:51, 1.12it/s][2025-04-27 22:22:18] (step=0082900) Train Loss: 5.6657, Train Steps/Sec: 1.12 + 75%|███████████████████████████████████████████████████████████████████████████████████████████████▏ | 82924/110696 [25:19:15<6:52:04, 1.12it/s][2025-04-27 22:22:40] (step=0082925) Train Loss: 5.7258, Train Steps/Sec: 1.12 + 75%|███████████████████████████████████████████████████████████████████████████████████████████████▏ | 82949/110696 [25:19:37<6:47:57, 1.13it/s][2025-04-27 22:23:02] (step=0082950) Train Loss: 5.7621, Train Steps/Sec: 1.12 + 75%|███████████████████████████████████████████████████████████████████████████████████████████████▏ | 82974/110696 [25:19:59<6:46:43, 1.14it/s][2025-04-27 22:23:24] (step=0082975) Train Loss: 5.7599, Train Steps/Sec: 1.13 + 75%|███████████████████████████████████████████████████████████████████████████████████████████████▏ | 82999/110696 [25:20:21<6:46:40, 1.14it/s][2025-04-27 22:23:47] (step=0083000) Train Loss: 5.7075, Train Steps/Sec: 1.11 + 75%|███████████████████████████████████████████████████████████████████████████████████████████████▎ | 83024/110696 [25:20:44<7:00:22, 1.10it/s][2025-04-27 22:24:09] (step=0083025) Train Loss: 5.6711, Train Steps/Sec: 1.12 + 75%|███████████████████████████████████████████████████████████████████████████████████████████████▎ | 83049/110696 [25:21:06<6:52:13, 1.12it/s][2025-04-27 22:24:32] (step=0083050) Train Loss: 5.7285, Train Steps/Sec: 1.12 + 75%|███████████████████████████████████████████████████████████████████████████████████████████████▎ | 83074/110696 [25:21:28<6:51:32, 1.12it/s][2025-04-27 22:24:54] (step=0083075) Train Loss: 5.7017, Train Steps/Sec: 1.12 + 75%|███████████████████████████████████████████████████████████████████████████████████████████████▎ | 83099/110696 [25:21:51<6:47:52, 1.13it/s][2025-04-27 22:25:16] (step=0083100) Train Loss: 5.7485, Train Steps/Sec: 1.13 + 75%|███████████████████████████████████████████████████████████████████████████████████████████████▎ | 83124/110696 [25:22:13<6:47:37, 1.13it/s][2025-04-27 22:25:38] (step=0083125) Train Loss: 5.7712, Train Steps/Sec: 1.12 + 75%|███████████████████████████████████████████████████████████████████████████████████████████████▍ | 83149/110696 [25:22:35<6:48:49, 1.12it/s][2025-04-27 22:26:01] (step=0083150) Train Loss: 5.6508, Train Steps/Sec: 1.12 + 75%|███████████████████████████████████████████████████████████████████████████████████████████████▍ | 83174/110696 [25:22:57<6:45:26, 1.13it/s][2025-04-27 22:26:23] (step=0083175) Train Loss: 5.7643, Train Steps/Sec: 1.13 + 75%|███████████████████████████████████████████████████████████████████████████████████████████████▍ | 83199/110696 [25:23:20<6:43:39, 1.14it/s][2025-04-27 22:26:45] (step=0083200) Train Loss: 5.7061, Train Steps/Sec: 1.12 + 75%|███████████████████████████████████████████████████████████████████████████████████████████████▍ | 83224/110696 [25:23:42<6:57:33, 1.10it/s][2025-04-27 22:27:07] (step=0083225) Train Loss: 5.6824, Train Steps/Sec: 1.12 + 75%|███████████████████████████████████████████████████████████████████████████████████████████████▌ | 83249/110696 [25:24:04<6:51:40, 1.11it/s][2025-04-27 22:27:30] (step=0083250) Train Loss: 5.7328, Train Steps/Sec: 1.12 + 75%|███████████████████████████████████████████████████████████████████████████████████████████████▌ | 83274/110696 [25:24:27<6:48:55, 1.12it/s][2025-04-27 22:27:52] (step=0083275) Train Loss: 5.7143, Train Steps/Sec: 1.12 + 75%|███████████████████████████████████████████████████████████████████████████████████████████████▌ | 83299/110696 [25:24:49<6:47:16, 1.12it/s][2025-04-27 22:28:14] (step=0083300) Train Loss: 5.6227, Train Steps/Sec: 1.12 + 75%|███████████████████████████████████████████████████████████████████████████████████████████████▌ | 83324/110696 [25:25:11<6:46:24, 1.12it/s][2025-04-27 22:28:37] (step=0083325) Train Loss: 5.7180, Train Steps/Sec: 1.12 + 75%|███████████████████████████████████████████████████████████████████████████████████████████████▋ | 83349/110696 [25:25:34<6:43:42, 1.13it/s][2025-04-27 22:28:59] (step=0083350) Train Loss: 5.7718, Train Steps/Sec: 1.12 + 75%|███████████████████████████████████████████████████████████████████████████████████████████████▋ | 83374/110696 [25:25:56<6:43:30, 1.13it/s][2025-04-27 22:29:21] (step=0083375) Train Loss: 5.7194, Train Steps/Sec: 1.12 + 75%|███████████████████████████████████████████████████████████████████████████████████████████████▋ | 83399/110696 [25:26:18<6:41:27, 1.13it/s][2025-04-27 22:29:44] (step=0083400) Train Loss: 5.7223, Train Steps/Sec: 1.12 + 75%|███████████████████████████████████████████████████████████████████████████████████████████████▋ | 83424/110696 [25:26:41<6:53:09, 1.10it/s][2025-04-27 22:30:06] (step=0083425) Train Loss: 5.7415, Train Steps/Sec: 1.12 + 75%|███████████████████████████████████████████████████████████████████████████████████████████████▋ | 83449/110696 [25:27:03<6:48:55, 1.11it/s][2025-04-27 22:30:28] (step=0083450) Train Loss: 5.6701, Train Steps/Sec: 1.12 + 75%|███████████████████████████████████████████████████████████████████████████████████████████████▊ | 83474/110696 [25:27:25<6:44:56, 1.12it/s][2025-04-27 22:30:50] (step=0083475) Train Loss: 5.7291, Train Steps/Sec: 1.12 + 75%|███████████████████████████████████████████████████████████████████████████████████████████████▊ | 83499/110696 [25:27:47<6:41:50, 1.13it/s][2025-04-27 22:31:13] (step=0083500) Train Loss: 5.7073, Train Steps/Sec: 1.13 + 75%|███████████████████████████████████████████████████████████████████████████████████████████████▊ | 83524/110696 [25:28:10<6:41:22, 1.13it/s][2025-04-27 22:31:35] (step=0083525) Train Loss: 5.7555, Train Steps/Sec: 1.12 + 75%|███████████████████████████████████████████████████████████████████████████████████████████████▊ | 83549/110696 [25:28:32<6:40:41, 1.13it/s][2025-04-27 22:31:57] (step=0083550) Train Loss: 5.7056, Train Steps/Sec: 1.12 + 75%|███████████████████████████████████████████████████████████████████████████████████████████████▉ | 83574/110696 [25:28:58<9:03:25, 1.20s/it][2025-04-27 22:32:24] (step=0083575) Train Loss: 5.7000, Train Steps/Sec: 0.94 + 76%|███████████████████████████████████████████████████████████████████████████████████████████████▉ | 83599/110696 [25:29:21<6:38:25, 1.13it/s][2025-04-27 22:32:46] (step=0083600) Train Loss: 5.7301, Train Steps/Sec: 1.12 + 76%|███████████████████████████████████████████████████████████████████████████████████████████████▉ | 83624/110696 [25:29:43<6:50:26, 1.10it/s][2025-04-27 22:33:08] (step=0083625) Train Loss: 5.7505, Train Steps/Sec: 1.12 + 76%|███████████████████████████████████████████████████████████████████████████████████████████████▉ | 83649/110696 [25:30:05<6:44:53, 1.11it/s][2025-04-27 22:33:31] (step=0083650) Train Loss: 5.6828, Train Steps/Sec: 1.12 + 76%|███████████████████████████████████████████████████████████████████████████████████████████████▉ | 83674/110696 [25:30:28<6:42:06, 1.12it/s][2025-04-27 22:33:53] (step=0083675) Train Loss: 5.7185, Train Steps/Sec: 1.12 + 76%|████████████████████████████████████████████████████████████████████████████████████████████████ | 83699/110696 [25:30:50<6:42:12, 1.12it/s][2025-04-27 22:34:15] (step=0083700) Train Loss: 5.6956, Train Steps/Sec: 1.12 + 76%|████████████████████████████████████████████████████████████████████████████████████████████████ | 83724/110696 [25:31:12<6:39:58, 1.12it/s][2025-04-27 22:34:38] (step=0083725) Train Loss: 5.6945, Train Steps/Sec: 1.12 + 76%|████████████████████████████████████████████████████████████████████████████████████████████████ | 83749/110696 [25:31:35<6:38:48, 1.13it/s][2025-04-27 22:35:00] (step=0083750) Train Loss: 5.7020, Train Steps/Sec: 1.12 + 76%|████████████████████████████████████████████████████████████████████████████████████████████████ | 83774/110696 [25:31:57<6:37:20, 1.13it/s][2025-04-27 22:35:22] (step=0083775) Train Loss: 5.6849, Train Steps/Sec: 1.12 + 76%|████████████████████████████████████████████████████████████████████████████████████████████████▏ | 83799/110696 [25:32:19<6:34:17, 1.14it/s][2025-04-27 22:35:45] (step=0083800) Train Loss: 5.7239, Train Steps/Sec: 1.12 + 76%|████████████████████████████████████████████████████████████████████████████████████████████████▏ | 83824/110696 [25:32:42<6:46:13, 1.10it/s][2025-04-27 22:36:07] (step=0083825) Train Loss: 5.7326, Train Steps/Sec: 1.12 + 76%|████████████████████████████████████████████████████████████████████████████████████████████████▏ | 83849/110696 [25:33:09<7:08:27, 1.04it/s][2025-04-27 22:36:34] (step=0083850) Train Loss: 5.7449, Train Steps/Sec: 0.92 + 76%|████████████████████████████████████████████████████████████████████████████████████████████████▏ | 83874/110696 [25:33:31<6:37:55, 1.12it/s][2025-04-27 22:36:56] (step=0083875) Train Loss: 5.7893, Train Steps/Sec: 1.12 + 76%|████████████████████████████████████████████████████████████████████████████████████████████████▎ | 83899/110696 [25:33:53<6:36:29, 1.13it/s][2025-04-27 22:37:18] (step=0083900) Train Loss: 5.7186, Train Steps/Sec: 1.12 + 76%|████████████████████████████████████████████████████████████████████████████████████████████████▎ | 83924/110696 [25:34:15<6:34:52, 1.13it/s][2025-04-27 22:37:41] (step=0083925) Train Loss: 5.7331, Train Steps/Sec: 1.13 + 76%|████████████████████████████████████████████████████████████████████████████████████████████████▎ | 83949/110696 [25:34:38<6:35:25, 1.13it/s][2025-04-27 22:38:03] (step=0083950) Train Loss: 5.7431, Train Steps/Sec: 1.12 + 76%|████████████████████████████████████████████████████████████████████████████████████████████████▎ | 83974/110696 [25:35:00<6:34:36, 1.13it/s][2025-04-27 22:38:25] (step=0083975) Train Loss: 5.6727, Train Steps/Sec: 1.12 + 76%|████████████████████████████████████████████████████████████████████████████████████████████████▎ | 83999/110696 [25:35:22<6:31:57, 1.14it/s][2025-04-27 22:38:48] (step=0084000) Train Loss: 5.7709, Train Steps/Sec: 1.12 +You are using a model of type blip-2 to instantiate a model of type instructblip. This is not supported for all configurations of models and can yield errors. +[2025-04-27 22:38:48] text_config is None. Initializing the text config with default values (`OPTConfig`). +model_name_or_path:/tmp/haozhezhao/model/blip2-flan-t5-xl +image_place_holder: +tokenizer length after expend 32101 +tokenizer length before expend 32100 +100%|████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████| 5/5 [05:04<00:00, 60.86s/it] +[2025-04-27 22:45:06] Finish Eval in 84000 steps...█████████████████████████████████████████████████████████████████████████████████████████████| 5/5 [05:03<00:00, 60.52s/it] +processing left-padding... +processing left-padding... +processing left-padding... +processing left-padding... +processing left-padding... +[2025-04-27 22:45:26] Saved checkpoint to checkpoint/X2I_UltraEdiit_fourmask_LLava_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_filtered_objects_4mask_1280_context/003-GPT-XL/checkpoints/0084000.pt +[2025-04-27 22:45:29] Removed old checkpoint: checkpoint/X2I_UltraEdiit_fourmask_LLava_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_filtered_objects_4mask_1280_context/003-GPT-XL/checkpoints/0082000.pt + 76%|████████████████████████████████████████████████████████████████████████████████████████████████▍ | 84024/110696 [25:42:26<6:54:59, 1.07it/s][2025-04-27 22:45:51] (step=0084025) Train Loss: 5.6930, Train Steps/Sec: 0.06 + 76%|████████████████████████████████████████████████████████████████████████████████████████████████▍ | 84049/110696 [25:42:48<6:37:44, 1.12it/s][2025-04-27 22:46:13] (step=0084050) Train Loss: 5.7144, Train Steps/Sec: 1.13 + 76%|████████████████████████████████████████████████████████████████████████████████████████████████▍ | 84074/110696 [25:43:10<6:36:13, 1.12it/s][2025-04-27 22:46:35] (step=0084075) Train Loss: 5.7187, Train Steps/Sec: 1.13 + 76%|████████████████████████████████████████████████████████████████████████████████████████████████▍ | 84099/110696 [25:43:32<6:34:40, 1.12it/s][2025-04-27 22:46:58] (step=0084100) Train Loss: 5.6925, Train Steps/Sec: 1.12 + 76%|████████████████████████████████████████████████████████████████████████████████████████████████▌ | 84124/110696 [25:43:55<6:33:06, 1.13it/s][2025-04-27 22:47:20] (step=0084125) Train Loss: 5.7306, Train Steps/Sec: 1.12 + 76%|████████████████████████████████████████████████████████████████████████████████████████████████▌ | 84149/110696 [25:44:17<6:31:48, 1.13it/s][2025-04-27 22:47:42] (step=0084150) Train Loss: 5.7394, Train Steps/Sec: 1.12 + 76%|████████████████████████████████████████████████████████████████████████████████████████████████▌ | 84174/110696 [25:44:39<6:29:34, 1.13it/s][2025-04-27 22:48:05] (step=0084175) Train Loss: 5.7025, Train Steps/Sec: 1.12 + 76%|████████████████████████████████████████████████████████████████████████████████████████████████▌ | 84199/110696 [25:45:01<6:29:46, 1.13it/s][2025-04-27 22:48:27] (step=0084200) Train Loss: 5.7623, Train Steps/Sec: 1.12 + 76%|████████████████████████████████████████████████████████████████████████████████████████████████▋ | 84224/110696 [25:45:24<6:42:06, 1.10it/s][2025-04-27 22:48:49] (step=0084225) Train Loss: 5.6858, Train Steps/Sec: 1.12 + 76%|████████████████████████████████████████████████████████████████████████████████████████████████▋ | 84249/110696 [25:45:46<6:36:53, 1.11it/s][2025-04-27 22:49:11] (step=0084250) Train Loss: 5.7613, Train Steps/Sec: 1.12 + 76%|████████████████████████████████████████████████████████████████████████████████████████████████▋ | 84274/110696 [25:46:08<6:33:20, 1.12it/s][2025-04-27 22:49:34] (step=0084275) Train Loss: 5.7183, Train Steps/Sec: 1.12 + 76%|████████████████████████████████████████████████████████████████████████████████████████████████▋ | 84299/110696 [25:46:31<6:31:00, 1.13it/s][2025-04-27 22:49:56] (step=0084300) Train Loss: 5.7136, Train Steps/Sec: 1.12 + 76%|████████████████████████████████████████████████████████████████████████████████████████████████▋ | 84324/110696 [25:46:53<6:28:39, 1.13it/s][2025-04-27 22:50:18] (step=0084325) Train Loss: 5.6908, Train Steps/Sec: 1.13 + 76%|████████████████████████████████████████████████████████████████████████████████████████████████▊ | 84349/110696 [25:47:15<6:28:20, 1.13it/s][2025-04-27 22:50:40] (step=0084350) Train Loss: 5.7028, Train Steps/Sec: 1.12 + 76%|████████████████████████████████████████████████████████████████████████████████████████████████▊ | 84374/110696 [25:47:37<6:28:23, 1.13it/s][2025-04-27 22:51:03] (step=0084375) Train Loss: 5.7411, Train Steps/Sec: 1.12 + 76%|████████████████████████████████████████████████████████████████████████████████████████████████▊ | 84399/110696 [25:48:00<6:26:36, 1.13it/s][2025-04-27 22:51:25] (step=0084400) Train Loss: 5.6990, Train Steps/Sec: 1.12 + 76%|████████████████████████████████████████████████████████████████████████████████████████████████▊ | 84424/110696 [25:48:22<6:39:08, 1.10it/s][2025-04-27 22:51:47] (step=0084425) Train Loss: 5.6942, Train Steps/Sec: 1.12 + 76%|████████████████████████████████████████████████████████████████████████████████████████████████▉ | 84449/110696 [25:48:44<6:31:30, 1.12it/s][2025-04-27 22:52:09] (step=0084450) Train Loss: 5.7551, Train Steps/Sec: 1.13 + 76%|████████████████████████████████████████████████████████████████████████████████████████████████▉ | 84474/110696 [25:49:06<6:30:29, 1.12it/s][2025-04-27 22:52:32] (step=0084475) Train Loss: 5.6724, Train Steps/Sec: 1.12 + 76%|████████████████████████████████████████████████████████████████████████████████████████████████▉ | 84499/110696 [25:49:29<6:26:59, 1.13it/s][2025-04-27 22:52:54] (step=0084500) Train Loss: 5.7127, Train Steps/Sec: 1.12 + 76%|████████████████████████████████████████████████████████████████████████████████████████████████▉ | 84524/110696 [25:49:51<6:25:58, 1.13it/s][2025-04-27 22:53:16] (step=0084525) Train Loss: 5.7217, Train Steps/Sec: 1.12 + 76%|█████████████████████████████████████████████████████████████████████████████████████████████████ | 84549/110696 [25:50:13<6:25:52, 1.13it/s][2025-04-27 22:53:38] (step=0084550) Train Loss: 5.7166, Train Steps/Sec: 1.12 + 76%|█████████████████████████████████████████████████████████████████████████████████████████████████ | 84574/110696 [25:50:35<6:24:20, 1.13it/s][2025-04-27 22:54:01] (step=0084575) Train Loss: 5.6808, Train Steps/Sec: 1.12 + 76%|█████████████████████████████████████████████████████████████████████████████████████████████████ | 84599/110696 [25:50:58<6:22:52, 1.14it/s][2025-04-27 22:54:23] (step=0084600) Train Loss: 5.7758, Train Steps/Sec: 1.12 + 76%|█████████████████████████████████████████████████████████████████████████████████████████████████ | 84624/110696 [25:51:28<9:01:01, 1.25s/it][2025-04-27 22:54:54] (step=0084625) Train Loss: 5.7479, Train Steps/Sec: 0.81 + 76%|█████████████████████████████████████████████████████████████████████████████████████████████████ | 84649/110696 [25:51:55<6:34:23, 1.10it/s][2025-04-27 22:55:21] (step=0084650) Train Loss: 5.6793, Train Steps/Sec: 0.94 + 76%|████████████████████████████████████████████████████████████████████████████████████████████████▍ | 84674/110696 [25:52:23<14:34:23, 2.02s/it][2025-04-27 22:55:48] (step=0084675) Train Loss: 5.7269, Train Steps/Sec: 0.91 + 77%|█████████████████████████████████████████████████████████████████████████████████████████████████▏ | 84699/110696 [25:52:45<6:26:09, 1.12it/s][2025-04-27 22:56:10] (step=0084700) Train Loss: 5.7340, Train Steps/Sec: 1.12 + 77%|█████████████████████████████████████████████████████████████████████████████████████████████████▏ | 84724/110696 [25:53:07<6:25:25, 1.12it/s][2025-04-27 22:56:33] (step=0084725) Train Loss: 5.7289, Train Steps/Sec: 1.12 + 77%|█████████████████████████████████████████████████████████████████████████████████████████████████▏ | 84749/110696 [25:53:30<6:23:28, 1.13it/s][2025-04-27 22:56:55] (step=0084750) Train Loss: 5.7107, Train Steps/Sec: 1.12 + 77%|█████████████████████████████████████████████████████████████████████████████████████████████████▎ | 84774/110696 [25:53:52<6:21:57, 1.13it/s][2025-04-27 22:57:17] (step=0084775) Train Loss: 5.6654, Train Steps/Sec: 1.12 + 77%|█████████████████████████████████████████████████████████████████████████████████████████████████▎ | 84799/110696 [25:54:14<6:20:49, 1.13it/s][2025-04-27 22:57:40] (step=0084800) Train Loss: 5.7324, Train Steps/Sec: 1.12 + 77%|█████████████████████████████████████████████████████████████████████████████████████████████████▎ | 84824/110696 [25:54:37<6:33:26, 1.10it/s][2025-04-27 22:58:02] (step=0084825) Train Loss: 5.6362, Train Steps/Sec: 1.12 + 77%|█████████████████████████████████████████████████████████████████████████████████████████████████▎ | 84849/110696 [25:54:59<6:26:58, 1.11it/s][2025-04-27 22:58:24] (step=0084850) Train Loss: 5.6456, Train Steps/Sec: 1.12 + 77%|█████████████████████████████████████████████████████████████████████████████████████████████████▎ | 84874/110696 [25:55:21<6:24:17, 1.12it/s][2025-04-27 22:58:47] (step=0084875) Train Loss: 5.7357, Train Steps/Sec: 1.12 + 77%|█████████████████████████████████████████████████████████████████████████████████████████████████▍ | 84899/110696 [25:55:43<6:22:40, 1.12it/s][2025-04-27 22:59:09] (step=0084900) Train Loss: 5.6527, Train Steps/Sec: 1.12 + 77%|█████████████████████████████████████████████████████████████████████████████████████████████████▍ | 84924/110696 [25:56:06<6:20:32, 1.13it/s][2025-04-27 22:59:31] (step=0084925) Train Loss: 5.7034, Train Steps/Sec: 1.12 + 77%|█████████████████████████████████████████████████████████████████████████████████████████████████▍ | 84949/110696 [25:56:28<6:20:01, 1.13it/s][2025-04-27 22:59:53] (step=0084950) Train Loss: 5.7067, Train Steps/Sec: 1.12 + 77%|█████████████████████████████████████████████████████████████████████████████████████████████████▍ | 84974/110696 [25:56:50<6:19:41, 1.13it/s][2025-04-27 23:00:16] (step=0084975) Train Loss: 5.6913, Train Steps/Sec: 1.12 + 77%|█████████████████████████████████████████████████████████████████████████████████████████████████▌ | 84999/110696 [25:57:13<6:17:31, 1.13it/s][2025-04-27 23:00:38] (step=0085000) Train Loss: 5.6940, Train Steps/Sec: 1.12 + 77%|█████████████████████████████████████████████████████████████████████████████████████████████████▌ | 85024/110696 [25:57:35<6:29:28, 1.10it/s][2025-04-27 23:01:00] (step=0085025) Train Loss: 5.7046, Train Steps/Sec: 1.12 + 77%|█████████████████████████████████████████████████████████████████████████████████████████████████▌ | 85049/110696 [25:58:02<8:09:39, 1.15s/it][2025-04-27 23:01:27] (step=0085050) Train Loss: 5.6951, Train Steps/Sec: 0.92 + 77%|█████████████████████████████████████████████████████████████████████████████████████████████████▌ | 85074/110696 [25:58:24<6:20:41, 1.12it/s][2025-04-27 23:01:50] (step=0085075) Train Loss: 5.6815, Train Steps/Sec: 1.12 + 77%|█████████████████████████████████████████████████████████████████████████████████████████████████▋ | 85099/110696 [25:58:47<6:19:52, 1.12it/s][2025-04-27 23:02:12] (step=0085100) Train Loss: 5.7035, Train Steps/Sec: 1.12 + 77%|█████████████████████████████████████████████████████████████████████████████████████████████████▋ | 85124/110696 [25:59:09<6:18:45, 1.13it/s][2025-04-27 23:02:34] (step=0085125) Train Loss: 5.7365, Train Steps/Sec: 1.12 + 77%|█████████████████████████████████████████████████████████████████████████████████████████████████▋ | 85149/110696 [25:59:31<6:17:34, 1.13it/s][2025-04-27 23:02:57] (step=0085150) Train Loss: 5.7258, Train Steps/Sec: 1.12 + 77%|█████████████████████████████████████████████████████████████████████████████████████████████████▋ | 85174/110696 [25:59:53<6:15:40, 1.13it/s][2025-04-27 23:03:19] (step=0085175) Train Loss: 5.6694, Train Steps/Sec: 1.12 + 77%|█████████████████████████████████████████████████████████████████████████████████████████████████▋ | 85199/110696 [26:00:16<6:13:57, 1.14it/s][2025-04-27 23:03:41] (step=0085200) Train Loss: 5.7110, Train Steps/Sec: 1.12 + 77%|█████████████████████████████████████████████████████████████████████████████████████████████████▊ | 85224/110696 [26:00:38<6:27:52, 1.09it/s][2025-04-27 23:04:04] (step=0085225) Train Loss: 5.6965, Train Steps/Sec: 1.12 + 77%|█████████████████████████████████████████████████████████████████████████████████████████████████▊ | 85249/110696 [26:01:00<6:21:11, 1.11it/s][2025-04-27 23:04:26] (step=0085250) Train Loss: 5.6834, Train Steps/Sec: 1.12 + 77%|█████████████████████████████████████████████████████████████████████████████████████████████████▊ | 85274/110696 [26:01:23<6:18:59, 1.12it/s][2025-04-27 23:04:48] (step=0085275) Train Loss: 5.7014, Train Steps/Sec: 1.12 + 77%|█████████████████████████████████████████████████████████████████████████████████████████████████▊ | 85299/110696 [26:01:45<6:16:44, 1.12it/s][2025-04-27 23:05:10] (step=0085300) Train Loss: 5.6944, Train Steps/Sec: 1.12 + 77%|█████████████████████████████████████████████████████████████████████████████████████████████████▉ | 85324/110696 [26:02:07<6:16:06, 1.12it/s][2025-04-27 23:05:33] (step=0085325) Train Loss: 5.7178, Train Steps/Sec: 1.12 + 77%|█████████████████████████████████████████████████████████████████████████████████████████████████▉ | 85349/110696 [26:02:30<6:15:28, 1.13it/s][2025-04-27 23:05:55] (step=0085350) Train Loss: 5.6632, Train Steps/Sec: 1.12 + 77%|█████████████████████████████████████████████████████████████████████████████████████████████████▉ | 85374/110696 [26:02:52<6:11:59, 1.13it/s][2025-04-27 23:06:17] (step=0085375) Train Loss: 5.7104, Train Steps/Sec: 1.12 + 77%|█████████████████████████████████████████████████████████████████████████████████████████████████▉ | 85399/110696 [26:03:14<6:11:31, 1.13it/s][2025-04-27 23:06:40] (step=0085400) Train Loss: 5.6982, Train Steps/Sec: 1.12 + 77%|██████████████████████████████████████████████████████████████████████████████████████████████████ | 85424/110696 [26:03:37<6:22:42, 1.10it/s][2025-04-27 23:07:02] (step=0085425) Train Loss: 5.6735, Train Steps/Sec: 1.12 + 77%|██████████████████████████████████████████████████████████████████████████████████████████████████ | 85449/110696 [26:03:59<6:17:20, 1.12it/s][2025-04-27 23:07:24] (step=0085450) Train Loss: 5.6979, Train Steps/Sec: 1.12 + 77%|██████████████████████████████████████████████████████████████████████████████████████████████████ | 85474/110696 [26:04:21<6:15:42, 1.12it/s][2025-04-27 23:07:46] (step=0085475) Train Loss: 5.6812, Train Steps/Sec: 1.12 + 77%|██████████████████████████████████████████████████████████████████████████████████████████████████ | 85499/110696 [26:04:43<6:13:52, 1.12it/s][2025-04-27 23:08:09] (step=0085500) Train Loss: 5.7029, Train Steps/Sec: 1.12 + 77%|██████████████████████████████████████████████████████████████████████████████████████████████████ | 85524/110696 [26:05:06<6:13:18, 1.12it/s][2025-04-27 23:08:31] (step=0085525) Train Loss: 5.6764, Train Steps/Sec: 1.12 + 77%|██████████████████████████████████████████████████████████████████████████████████████████████████▏ | 85549/110696 [26:05:28<6:10:38, 1.13it/s][2025-04-27 23:08:53] (step=0085550) Train Loss: 5.7288, Train Steps/Sec: 1.12 + 77%|██████████████████████████████████████████████████████████████████████████████████████████████████▏ | 85574/110696 [26:05:50<6:09:21, 1.13it/s][2025-04-27 23:09:16] (step=0085575) Train Loss: 5.7521, Train Steps/Sec: 1.12 + 77%|██████████████████████████████████████████████████████████████████████████████████████████████████▏ | 85599/110696 [26:06:12<6:10:16, 1.13it/s][2025-04-27 23:09:38] (step=0085600) Train Loss: 5.6973, Train Steps/Sec: 1.12 + 77%|██████████████████████████████████████████████████████████████████████████████████████████████████▏ | 85624/110696 [26:06:35<6:20:49, 1.10it/s][2025-04-27 23:10:00] (step=0085625) Train Loss: 5.6637, Train Steps/Sec: 1.12 + 77%|██████████████████████████████████████████████████████████████████████████████████████████████████▎ | 85649/110696 [26:06:57<6:14:17, 1.12it/s][2025-04-27 23:10:22] (step=0085650) Train Loss: 5.7091, Train Steps/Sec: 1.12 + 77%|██████████████████████████████████████████████████████████████████████████████████████████████████▎ | 85674/110696 [26:07:19<6:11:52, 1.12it/s][2025-04-27 23:10:45] (step=0085675) Train Loss: 5.7254, Train Steps/Sec: 1.12 + 77%|██████████████████████████████████████████████████████████████████████████████████████████████████▎ | 85699/110696 [26:07:42<6:09:42, 1.13it/s][2025-04-27 23:11:07] (step=0085700) Train Loss: 5.7016, Train Steps/Sec: 1.12 + 77%|██████████████████████████████████████████████████████████████████████████████████████████████████▎ | 85724/110696 [26:08:04<6:08:42, 1.13it/s][2025-04-27 23:11:29] (step=0085725) Train Loss: 5.6822, Train Steps/Sec: 1.12 + 77%|██████████████████████████████████████████████████████████████████████████████████████████████████▍ | 85749/110696 [26:08:26<6:08:29, 1.13it/s][2025-04-27 23:11:52] (step=0085750) Train Loss: 5.7104, Train Steps/Sec: 1.12 + 77%|██████████████████████████████████████████████████████████████████████████████████████████████████▍ | 85774/110696 [26:08:49<6:07:33, 1.13it/s][2025-04-27 23:12:14] (step=0085775) Train Loss: 5.6568, Train Steps/Sec: 1.12 + 78%|██████████████████████████████████████████████████████████████████████████████████████████████████▍ | 85799/110696 [26:09:11<6:05:09, 1.14it/s][2025-04-27 23:12:36] (step=0085800) Train Loss: 5.6148, Train Steps/Sec: 1.12 + 78%|██████████████████████████████████████████████████████████████████████████████████████████████████▍ | 85824/110696 [26:09:33<6:18:55, 1.09it/s][2025-04-27 23:12:58] (step=0085825) Train Loss: 5.7155, Train Steps/Sec: 1.12 + 78%|██████████████████████████████████████████████████████████████████████████████████████████████████▍ | 85849/110696 [26:09:55<6:12:37, 1.11it/s][2025-04-27 23:13:21] (step=0085850) Train Loss: 5.7031, Train Steps/Sec: 1.12 + 78%|██████████████████████████████████████████████████████████████████████████████████████████████████▌ | 85874/110696 [26:10:18<6:09:09, 1.12it/s][2025-04-27 23:13:43] (step=0085875) Train Loss: 5.6866, Train Steps/Sec: 1.12 + 78%|██████████████████████████████████████████████████████████████████████████████████████████████████▌ | 85899/110696 [26:10:40<6:06:37, 1.13it/s][2025-04-27 23:14:05] (step=0085900) Train Loss: 5.7289, Train Steps/Sec: 1.12 + 78%|██████████████████████████████████████████████████████████████████████████████████████████████████▌ | 85924/110696 [26:11:02<6:07:48, 1.12it/s][2025-04-27 23:14:28] (step=0085925) Train Loss: 5.7308, Train Steps/Sec: 1.12 + 78%|██████████████████████████████████████████████████████████████████████████████████████████████████▌ | 85949/110696 [26:11:24<6:05:52, 1.13it/s][2025-04-27 23:14:50] (step=0085950) Train Loss: 5.6454, Train Steps/Sec: 1.12 + 78%|██████████████████████████████████████████████████████████████████████████████████████████████████▋ | 85974/110696 [26:11:47<6:06:43, 1.12it/s][2025-04-27 23:15:12] (step=0085975) Train Loss: 5.6625, Train Steps/Sec: 1.12 + 78%|██████████████████████████████████████████████████████████████████████████████████████████████████▋ | 85999/110696 [26:12:09<6:03:26, 1.13it/s][2025-04-27 23:15:35] (step=0086000) Train Loss: 5.6339, Train Steps/Sec: 1.12 +You are using a model of type blip-2 to instantiate a model of type instructblip. This is not supported for all configurations of models and can yield errors. +[2025-04-27 23:15:35] text_config is None. Initializing the text config with default values (`OPTConfig`). +model_name_or_path:/tmp/haozhezhao/model/blip2-flan-t5-xl +image_place_holder: +tokenizer length after expend 32101 +tokenizer length before expend 32100 +100%|████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████| 5/5 [05:04<00:00, 60.98s/it] +[2025-04-27 23:21:53] Finish Eval in 86000 steps...█████████████████████████████████████████████████████████████████████████████████████████████| 5/5 [05:04<00:00, 60.71s/it] +processing left-padding... +processing left-padding... +processing left-padding... +processing left-padding... +processing left-padding... +[2025-04-27 23:22:12] Saved checkpoint to checkpoint/X2I_UltraEdiit_fourmask_LLava_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_filtered_objects_4mask_1280_context/003-GPT-XL/checkpoints/0086000.pt +[2025-04-27 23:22:14] Removed old checkpoint: checkpoint/X2I_UltraEdiit_fourmask_LLava_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_filtered_objects_4mask_1280_context/003-GPT-XL/checkpoints/0084000.pt + 78%|██████████████████████████████████████████████████████████████████████████████████████████████████▋ | 86024/110696 [26:19:12<6:24:06, 1.07it/s][2025-04-27 23:22:37] (step=0086025) Train Loss: 5.6778, Train Steps/Sec: 0.06 + 78%|██████████████████████████████████████████████████████████████████████████████████████████████████▋ | 86049/110696 [26:19:34<6:08:41, 1.11it/s][2025-04-27 23:22:59] (step=0086050) Train Loss: 5.7297, Train Steps/Sec: 1.12 + 78%|██████████████████████████████████████████████████████████████████████████████████████████████████▊ | 86074/110696 [26:19:56<6:05:56, 1.12it/s][2025-04-27 23:23:21] (step=0086075) Train Loss: 5.6614, Train Steps/Sec: 1.13 + 78%|██████████████████████████████████████████████████████████████████████████████████████████████████▊ | 86099/110696 [26:20:18<6:04:23, 1.13it/s][2025-04-27 23:23:44] (step=0086100) Train Loss: 5.6788, Train Steps/Sec: 1.12 + 78%|██████████████████████████████████████████████████████████████████████████████████████████████████▊ | 86124/110696 [26:20:40<6:04:00, 1.13it/s][2025-04-27 23:24:06] (step=0086125) Train Loss: 5.7083, Train Steps/Sec: 1.12 + 78%|██████████████████████████████████████████████████████████████████████████████████████████████████▊ | 86149/110696 [26:21:03<6:02:03, 1.13it/s][2025-04-27 23:24:28] (step=0086150) Train Loss: 5.7718, Train Steps/Sec: 1.12 + 78%|██████████████████████████████████████████████████████████████████████████████████████████████████▊ | 86174/110696 [26:21:25<6:01:06, 1.13it/s][2025-04-27 23:24:50] (step=0086175) Train Loss: 5.6889, Train Steps/Sec: 1.12 + 78%|██████████████████████████████████████████████████████████████████████████████████████████████████▉ | 86199/110696 [26:21:47<6:00:51, 1.13it/s][2025-04-27 23:25:13] (step=0086200) Train Loss: 5.6899, Train Steps/Sec: 1.12 + 78%|██████████████████████████████████████████████████████████████████████████████████████████████████▉ | 86224/110696 [26:22:10<6:12:17, 1.10it/s][2025-04-27 23:25:35] (step=0086225) Train Loss: 5.7252, Train Steps/Sec: 1.12 + 78%|██████████████████████████████████████████████████████████████████████████████████████████████████▉ | 86249/110696 [26:22:32<6:05:19, 1.12it/s][2025-04-27 23:25:57] (step=0086250) Train Loss: 5.7197, Train Steps/Sec: 1.12 + 78%|██████████████████████████████████████████████████████████████████████████████████████████████████▉ | 86274/110696 [26:22:54<6:04:26, 1.12it/s][2025-04-27 23:26:20] (step=0086275) Train Loss: 5.7082, Train Steps/Sec: 1.12 + 78%|███████████████████████████████████████████████████████████████████████████████████████████████████ | 86299/110696 [26:23:16<6:03:47, 1.12it/s][2025-04-27 23:26:42] (step=0086300) Train Loss: 5.6963, Train Steps/Sec: 1.12 + 78%|███████████████████████████████████████████████████████████████████████████████████████████████████ | 86324/110696 [26:23:39<5:58:44, 1.13it/s][2025-04-27 23:27:04] (step=0086325) Train Loss: 5.6967, Train Steps/Sec: 1.12 + 78%|███████████████████████████████████████████████████████████████████████████████████████████████████ | 86349/110696 [26:24:01<6:01:10, 1.12it/s][2025-04-27 23:27:26] (step=0086350) Train Loss: 5.6815, Train Steps/Sec: 1.12 + 78%|███████████████████████████████████████████████████████████████████████████████████████████████████ | 86374/110696 [26:24:23<5:57:00, 1.14it/s][2025-04-27 23:27:49] (step=0086375) Train Loss: 5.7694, Train Steps/Sec: 1.12 + 78%|███████████████████████████████████████████████████████████████████████████████████████████████████ | 86399/110696 [26:24:45<5:57:11, 1.13it/s][2025-04-27 23:28:11] (step=0086400) Train Loss: 5.7587, Train Steps/Sec: 1.12 + 78%|███████████████████████████████████████████████████████████████████████████████████████████████████▏ | 86424/110696 [26:25:08<6:07:15, 1.10it/s][2025-04-27 23:28:33] (step=0086425) Train Loss: 5.6572, Train Steps/Sec: 1.12 + 78%|███████████████████████████████████████████████████████████████████████████████████████████████████▏ | 86449/110696 [26:25:30<6:01:55, 1.12it/s][2025-04-27 23:28:55] (step=0086450) Train Loss: 5.7088, Train Steps/Sec: 1.12 + 78%|███████████████████████████████████████████████████████████████████████████████████████████████████▏ | 86474/110696 [26:25:52<6:00:51, 1.12it/s][2025-04-27 23:29:18] (step=0086475) Train Loss: 5.6921, Train Steps/Sec: 1.12 + 78%|███████████████████████████████████████████████████████████████████████████████████████████████████▏ | 86499/110696 [26:26:15<5:58:15, 1.13it/s][2025-04-27 23:29:40] (step=0086500) Train Loss: 5.7787, Train Steps/Sec: 1.12 + 78%|███████████████████████████████████████████████████████████████████████████████████████████████████▎ | 86524/110696 [26:26:37<5:55:58, 1.13it/s][2025-04-27 23:30:02] (step=0086525) Train Loss: 5.7279, Train Steps/Sec: 1.12 + 78%|███████████████████████████████████████████████████████████████████████████████████████████████████▎ | 86549/110696 [26:27:00<6:07:35, 1.09it/s][2025-04-27 23:30:25] (step=0086550) Train Loss: 5.7070, Train Steps/Sec: 1.10 + 78%|███████████████████████████████████████████████████████████████████████████████████████████████████▎ | 86574/110696 [26:27:22<5:54:35, 1.13it/s][2025-04-27 23:30:47] (step=0086575) Train Loss: 5.7498, Train Steps/Sec: 1.12 + 78%|███████████████████████████████████████████████████████████████████████████████████████████████████▎ | 86599/110696 [26:27:44<5:53:37, 1.14it/s][2025-04-27 23:31:09] (step=0086600) Train Loss: 5.6886, Train Steps/Sec: 1.12 + 78%|███████████████████████████████████████████████████████████████████████████████████████████████████▍ | 86624/110696 [26:28:06<6:05:47, 1.10it/s][2025-04-27 23:31:32] (step=0086625) Train Loss: 5.7292, Train Steps/Sec: 1.12 + 78%|███████████████████████████████████████████████████████████████████████████████████████████████████▍ | 86649/110696 [26:28:29<6:00:24, 1.11it/s][2025-04-27 23:31:54] (step=0086650) Train Loss: 5.6942, Train Steps/Sec: 1.12 + 78%|███████████████████████████████████████████████████████████████████████████████████████████████████▍ | 86674/110696 [26:28:51<5:56:20, 1.12it/s][2025-04-27 23:32:16] (step=0086675) Train Loss: 5.6816, Train Steps/Sec: 1.12 + 78%|███████████████████████████████████████████████████████████████████████████████████████████████████▍ | 86699/110696 [26:29:13<5:55:32, 1.12it/s][2025-04-27 23:32:39] (step=0086700) Train Loss: 5.7893, Train Steps/Sec: 1.12 + 78%|███████████████████████████████████████████████████████████████████████████████████████████████████▍ | 86724/110696 [26:29:36<5:57:57, 1.12it/s][2025-04-27 23:33:01] (step=0086725) Train Loss: 5.6266, Train Steps/Sec: 1.12 + 78%|███████████████████████████████████████████████████████████████████████████████████████████████████▌ | 86749/110696 [26:29:58<5:53:34, 1.13it/s][2025-04-27 23:33:23] (step=0086750) Train Loss: 5.7318, Train Steps/Sec: 1.12 + 78%|███████████████████████████████████████████████████████████████████████████████████████████████████▌ | 86774/110696 [26:30:20<5:52:19, 1.13it/s][2025-04-27 23:33:45] (step=0086775) Train Loss: 5.6662, Train Steps/Sec: 1.12 + 78%|███████████████████████████████████████████████████████████████████████████████████████████████████▌ | 86799/110696 [26:30:42<5:52:07, 1.13it/s][2025-04-27 23:34:08] (step=0086800) Train Loss: 5.7433, Train Steps/Sec: 1.12 + 78%|███████████████████████████████████████████████████████████████████████████████████████████████████▌ | 86824/110696 [26:31:05<6:02:24, 1.10it/s][2025-04-27 23:34:30] (step=0086825) Train Loss: 5.7196, Train Steps/Sec: 1.12 + 78%|███████████████████████████████████████████████████████████████████████████████████████████████████▋ | 86849/110696 [26:31:27<5:57:06, 1.11it/s][2025-04-27 23:34:52] (step=0086850) Train Loss: 5.6787, Train Steps/Sec: 1.12 + 78%|███████████████████████████████████████████████████████████████████████████████████████████████████▋ | 86874/110696 [26:31:49<5:54:39, 1.12it/s][2025-04-27 23:35:15] (step=0086875) Train Loss: 5.6985, Train Steps/Sec: 1.12 + 79%|███████████████████████████████████████████████████████████████████████████████████████████████████▋ | 86899/110696 [26:32:12<5:52:32, 1.13it/s][2025-04-27 23:35:37] (step=0086900) Train Loss: 5.7095, Train Steps/Sec: 1.12 + 79%|███████████████████████████████████████████████████████████████████████████████████████████████████▋ | 86924/110696 [26:32:34<5:52:16, 1.12it/s][2025-04-27 23:35:59] (step=0086925) Train Loss: 5.6857, Train Steps/Sec: 1.12 + 79%|███████████████████████████████████████████████████████████████████████████████████████████████████▊ | 86949/110696 [26:32:56<5:48:57, 1.13it/s][2025-04-27 23:36:21] (step=0086950) Train Loss: 5.7469, Train Steps/Sec: 1.12 + 79%|███████████████████████████████████████████████████████████████████████████████████████████████████▊ | 86974/110696 [26:33:18<5:50:06, 1.13it/s][2025-04-27 23:36:44] (step=0086975) Train Loss: 5.6917, Train Steps/Sec: 1.12 + 79%|███████████████████████████████████████████████████████████████████████████████████████████████████▊ | 86999/110696 [26:33:41<5:48:17, 1.13it/s][2025-04-27 23:37:06] (step=0087000) Train Loss: 5.7376, Train Steps/Sec: 1.12 + 79%|███████████████████████████████████████████████████████████████████████████████████████████████████▊ | 87024/110696 [26:34:03<5:58:37, 1.10it/s][2025-04-27 23:37:28] (step=0087025) Train Loss: 5.6663, Train Steps/Sec: 1.12 + 79%|███████████████████████████████████████████████████████████████████████████████████████████████████▊ | 87049/110696 [26:34:25<5:53:14, 1.12it/s][2025-04-27 23:37:51] (step=0087050) Train Loss: 5.6541, Train Steps/Sec: 1.12 + 79%|███████████████████████████████████████████████████████████████████████████████████████████████████▉ | 87074/110696 [26:34:48<5:51:41, 1.12it/s][2025-04-27 23:38:13] (step=0087075) Train Loss: 5.6086, Train Steps/Sec: 1.12 + 79%|███████████████████████████████████████████████████████████████████████████████████████████████████▉ | 87099/110696 [26:35:10<5:50:33, 1.12it/s][2025-04-27 23:38:35] (step=0087100) Train Loss: 5.6115, Train Steps/Sec: 1.12 + 79%|███████████████████████████████████████████████████████████████████████████████████████████████████▉ | 87124/110696 [26:35:32<5:49:24, 1.12it/s][2025-04-27 23:38:57] (step=0087125) Train Loss: 5.7014, Train Steps/Sec: 1.12 + 79%|███████████████████████████████████████████████████████████████████████████████████████████████████▉ | 87149/110696 [26:35:54<5:46:23, 1.13it/s][2025-04-27 23:39:20] (step=0087150) Train Loss: 5.6933, Train Steps/Sec: 1.12 + 79%|████████████████████████████████████████████████████████████████████████████████████████████████████ | 87174/110696 [26:36:17<5:46:23, 1.13it/s][2025-04-27 23:39:42] (step=0087175) Train Loss: 5.7770, Train Steps/Sec: 1.12 + 79%|████████████████████████████████████████████████████████████████████████████████████████████████████ | 87199/110696 [26:36:39<5:46:15, 1.13it/s][2025-04-27 23:40:04] (step=0087200) Train Loss: 5.6982, Train Steps/Sec: 1.12 + 79%|████████████████████████████████████████████████████████████████████████████████████████████████████ | 87224/110696 [26:37:01<5:55:47, 1.10it/s][2025-04-27 23:40:27] (step=0087225) Train Loss: 5.7127, Train Steps/Sec: 1.12 + 79%|████████████████████████████████████████████████████████████████████████████████████████████████████ | 87249/110696 [26:37:24<5:51:40, 1.11it/s][2025-04-27 23:40:49] (step=0087250) Train Loss: 5.6595, Train Steps/Sec: 1.12 + 79%|████████████████████████████████████████████████████████████████████████████████████████████████████▏ | 87274/110696 [26:37:46<5:49:18, 1.12it/s][2025-04-27 23:41:11] (step=0087275) Train Loss: 5.6859, Train Steps/Sec: 1.12 + 79%|████████████████████████████████████████████████████████████████████████████████████████████████████▏ | 87299/110696 [26:38:08<5:47:21, 1.12it/s][2025-04-27 23:41:34] (step=0087300) Train Loss: 5.6539, Train Steps/Sec: 1.12 + 79%|████████████████████████████████████████████████████████████████████████████████████████████████████▏ | 87324/110696 [26:38:31<5:47:10, 1.12it/s][2025-04-27 23:41:56] (step=0087325) Train Loss: 5.7166, Train Steps/Sec: 1.12 + 79%|████████████████████████████████████████████████████████████████████████████████████████████████████▏ | 87349/110696 [26:38:53<5:45:05, 1.13it/s][2025-04-27 23:42:18] (step=0087350) Train Loss: 5.7141, Train Steps/Sec: 1.12 + 79%|████████████████████████████████████████████████████████████████████████████████████████████████████▏ | 87374/110696 [26:39:15<5:44:03, 1.13it/s][2025-04-27 23:42:41] (step=0087375) Train Loss: 5.6517, Train Steps/Sec: 1.12 + 79%|████████████████████████████████████████████████████████████████████████████████████████████████████▎ | 87399/110696 [26:39:37<5:42:25, 1.13it/s][2025-04-27 23:43:03] (step=0087400) Train Loss: 5.7341, Train Steps/Sec: 1.12 + 79%|████████████████████████████████████████████████████████████████████████████████████████████████████▎ | 87424/110696 [26:40:00<5:53:57, 1.10it/s][2025-04-27 23:43:25] (step=0087425) Train Loss: 5.7334, Train Steps/Sec: 1.12 + 79%|████████████████████████████████████████████████████████████████████████████████████████████████████▎ | 87449/110696 [26:40:22<5:48:47, 1.11it/s][2025-04-27 23:43:47] (step=0087450) Train Loss: 5.5675, Train Steps/Sec: 1.12 + 79%|████████████████████████████████████████████████████████████████████████████████████████████████████▎ | 87474/110696 [26:40:44<5:47:13, 1.11it/s][2025-04-27 23:44:10] (step=0087475) Train Loss: 5.7574, Train Steps/Sec: 1.12 + 79%|████████████████████████████████████████████████████████████████████████████████████████████████████▍ | 87499/110696 [26:41:07<5:44:08, 1.12it/s][2025-04-27 23:44:32] (step=0087500) Train Loss: 5.6816, Train Steps/Sec: 1.12 + 79%|████████████████████████████████████████████████████████████████████████████████████████████████████▍ | 87524/110696 [26:41:29<5:42:21, 1.13it/s][2025-04-27 23:44:54] (step=0087525) Train Loss: 5.7332, Train Steps/Sec: 1.12 + 79%|████████████████████████████████████████████████████████████████████████████████████████████████████▍ | 87549/110696 [26:41:51<5:41:24, 1.13it/s][2025-04-27 23:45:17] (step=0087550) Train Loss: 5.6187, Train Steps/Sec: 1.12 + 79%|████████████████████████████████████████████████████████████████████████████████████████████████████▍ | 87574/110696 [26:42:14<5:43:38, 1.12it/s][2025-04-27 23:45:39] (step=0087575) Train Loss: 5.7431, Train Steps/Sec: 1.12 + 79%|████████████████████████████████████████████████████████████████████████████████████████████████████▌ | 87599/110696 [26:42:36<5:40:32, 1.13it/s][2025-04-27 23:46:01] (step=0087600) Train Loss: 5.6970, Train Steps/Sec: 1.12 + 79%|████████████████████████████████████████████████████████████████████████████████████████████████████▌ | 87624/110696 [26:42:58<5:49:39, 1.10it/s][2025-04-27 23:46:24] (step=0087625) Train Loss: 5.7150, Train Steps/Sec: 1.12 + 79%|████████████████████████████████████████████████████████████████████████████████████████████████████▌ | 87649/110696 [26:43:21<5:44:15, 1.12it/s][2025-04-27 23:46:46] (step=0087650) Train Loss: 5.7127, Train Steps/Sec: 1.13 + 79%|████████████████████████████████████████████████████████████████████████████████████████████████████▌ | 87674/110696 [26:43:43<5:43:34, 1.12it/s][2025-04-27 23:47:08] (step=0087675) Train Loss: 5.7622, Train Steps/Sec: 1.12 + 79%|████████████████████████████████████████████████████████████████████████████████████████████████████▌ | 87699/110696 [26:44:05<5:41:19, 1.12it/s][2025-04-27 23:47:30] (step=0087700) Train Loss: 5.7315, Train Steps/Sec: 1.12 + 79%|████████████████████████████████████████████████████████████████████████████████████████████████████▋ | 87724/110696 [26:44:27<5:38:18, 1.13it/s][2025-04-27 23:47:53] (step=0087725) Train Loss: 5.7446, Train Steps/Sec: 1.12 + 79%|████████████████████████████████████████████████████████████████████████████████████████████████████▋ | 87749/110696 [26:44:50<5:37:54, 1.13it/s][2025-04-27 23:48:15] (step=0087750) Train Loss: 5.7517, Train Steps/Sec: 1.12 + 79%|████████████████████████████████████████████████████████████████████████████████████████████████████▋ | 87774/110696 [26:45:12<5:37:35, 1.13it/s][2025-04-27 23:48:37] (step=0087775) Train Loss: 5.6449, Train Steps/Sec: 1.12 + 79%|████████████████████████████████████████████████████████████████████████████████████████████████████▋ | 87799/110696 [26:45:34<5:36:27, 1.13it/s][2025-04-27 23:49:00] (step=0087800) Train Loss: 5.6657, Train Steps/Sec: 1.12 + 79%|████████████████████████████████████████████████████████████████████████████████████████████████████▊ | 87824/110696 [26:45:57<5:46:57, 1.10it/s][2025-04-27 23:49:22] (step=0087825) Train Loss: 5.7055, Train Steps/Sec: 1.12 + 79%|████████████████████████████████████████████████████████████████████████████████████████████████████▊ | 87849/110696 [26:46:23<5:42:08, 1.11it/s][2025-04-27 23:49:49] (step=0087850) Train Loss: 5.6516, Train Steps/Sec: 0.93 + 79%|████████████████████████████████████████████████████████████████████████████████████████████████████▊ | 87874/110696 [26:46:46<5:40:33, 1.12it/s][2025-04-27 23:50:11] (step=0087875) Train Loss: 5.6452, Train Steps/Sec: 1.12 + 79%|████████████████████████████████████████████████████████████████████████████████████████████████████▊ | 87899/110696 [26:47:08<5:38:24, 1.12it/s][2025-04-27 23:50:33] (step=0087900) Train Loss: 5.6504, Train Steps/Sec: 1.12 + 79%|████████████████████████████████████████████████████████████████████████████████████████████████████▊ | 87924/110696 [26:47:30<5:36:42, 1.13it/s][2025-04-27 23:50:56] (step=0087925) Train Loss: 5.6880, Train Steps/Sec: 1.12 + 79%|████████████████████████████████████████████████████████████████████████████████████████████████████▉ | 87949/110696 [26:47:53<5:36:28, 1.13it/s][2025-04-27 23:51:18] (step=0087950) Train Loss: 5.7783, Train Steps/Sec: 1.12 + 79%|████████████████████████████████████████████████████████████████████████████████████████████████████▉ | 87974/110696 [26:48:15<5:34:57, 1.13it/s][2025-04-27 23:51:40] (step=0087975) Train Loss: 5.7572, Train Steps/Sec: 1.12 + 79%|████████████████████████████████████████████████████████████████████████████████████████████████████▉ | 87999/110696 [26:48:37<5:36:20, 1.12it/s][2025-04-27 23:52:03] (step=0088000) Train Loss: 5.6709, Train Steps/Sec: 1.12 +You are using a model of type blip-2 to instantiate a model of type instructblip. This is not supported for all configurations of models and can yield errors. +[2025-04-27 23:52:03] text_config is None. Initializing the text config with default values (`OPTConfig`). +model_name_or_path:/tmp/haozhezhao/model/blip2-flan-t5-xl +image_place_holder: +tokenizer length after expend 32101 +tokenizer length before expend 32100 +100%|████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████| 5/5 [05:05<00:00, 61.12s/it] +[2025-04-27 23:58:22] Finish Eval in 88000 steps...█████████████████████████████████████████████████████████████████████████████████████████████| 5/5 [05:04<00:00, 60.82s/it] +processing left-padding... +processing left-padding... +processing left-padding... +processing left-padding... +processing left-padding... +[2025-04-27 23:58:42] Saved checkpoint to checkpoint/X2I_UltraEdiit_fourmask_LLava_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_filtered_objects_4mask_1280_context/003-GPT-XL/checkpoints/0088000.pt +[2025-04-27 23:58:44] Removed old checkpoint: checkpoint/X2I_UltraEdiit_fourmask_LLava_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_filtered_objects_4mask_1280_context/003-GPT-XL/checkpoints/0086000.pt + 80%|████████████████████████████████████████████████████████████████████████████████████████████████████▉ | 88024/110696 [26:55:41<5:53:04, 1.07it/s][2025-04-27 23:59:07] (step=0088025) Train Loss: 5.7294, Train Steps/Sec: 0.06 + 80%|█████████████████████████████████████████████████████████████████████████████████████████████████████ | 88049/110696 [26:56:03<5:38:26, 1.12it/s][2025-04-27 23:59:29] (step=0088050) Train Loss: 5.7266, Train Steps/Sec: 1.12 + 80%|█████████████████████████████████████████████████████████████████████████████████████████████████████ | 88074/110696 [26:56:26<5:36:13, 1.12it/s][2025-04-27 23:59:51] (step=0088075) Train Loss: 5.7064, Train Steps/Sec: 1.12 + 80%|█████████████████████████████████████████████████████████████████████████████████████████████████████ | 88099/110696 [26:56:48<5:35:11, 1.12it/s][2025-04-28 00:00:13] (step=0088100) Train Loss: 5.6797, Train Steps/Sec: 1.12 + 80%|█████████████████████████████████████████████████████████████████████████████████████████████████████ | 88124/110696 [26:57:15<8:40:57, 1.38s/it][2025-04-28 00:00:41] (step=0088125) Train Loss: 5.6539, Train Steps/Sec: 0.92 + 80%|█████████████████████████████████████████████████████████████████████████████████████████████████████▏ | 88149/110696 [26:57:37<5:33:08, 1.13it/s][2025-04-28 00:01:03] (step=0088150) Train Loss: 5.6855, Train Steps/Sec: 1.12 + 80%|█████████████████████████████████████████████████████████████████████████████████████████████████████▏ | 88174/110696 [26:58:00<5:31:59, 1.13it/s][2025-04-28 00:01:25] (step=0088175) Train Loss: 5.7131, Train Steps/Sec: 1.12 + 80%|█████████████████████████████████████████████████████████████████████████████████████████████████████▏ | 88199/110696 [26:58:22<5:31:00, 1.13it/s][2025-04-28 00:01:47] (step=0088200) Train Loss: 5.6872, Train Steps/Sec: 1.12 + 80%|█████████████████████████████████████████████████████████████████████████████████████████████████████▏ | 88224/110696 [26:58:44<5:39:55, 1.10it/s][2025-04-28 00:02:10] (step=0088225) Train Loss: 5.7532, Train Steps/Sec: 1.12 + 80%|█████████████████████████████████████████████████████████████████████████████████████████████████████▏ | 88249/110696 [26:59:07<5:35:45, 1.11it/s][2025-04-28 00:02:32] (step=0088250) Train Loss: 5.6946, Train Steps/Sec: 1.12 + 80%|█████████████████████████████████████████████████████████████████████████████████████████████████████▎ | 88274/110696 [26:59:29<5:33:35, 1.12it/s][2025-04-28 00:02:54] (step=0088275) Train Loss: 5.7084, Train Steps/Sec: 1.12 + 80%|█████████████████████████████████████████████████████████████████████████████████████████████████████▎ | 88299/110696 [26:59:51<5:31:56, 1.12it/s][2025-04-28 00:03:16] (step=0088300) Train Loss: 5.7004, Train Steps/Sec: 1.12 + 80%|█████████████████████████████████████████████████████████████████████████████████████████████████████▎ | 88324/110696 [27:00:13<5:30:49, 1.13it/s][2025-04-28 00:03:39] (step=0088325) Train Loss: 5.7864, Train Steps/Sec: 1.12 + 80%|█████████████████████████████████████████████████████████████████████████████████████████████████████▎ | 88349/110696 [27:00:36<5:29:50, 1.13it/s][2025-04-28 00:04:01] (step=0088350) Train Loss: 5.6601, Train Steps/Sec: 1.12 + 80%|█████████████████████████████████████████████████████████████████████████████████████████████████████▍ | 88374/110696 [27:00:58<5:27:54, 1.13it/s][2025-04-28 00:04:23] (step=0088375) Train Loss: 5.6689, Train Steps/Sec: 1.12 + 80%|█████████████████████████████████████████████████████████████████████████████████████████████████████▍ | 88399/110696 [27:01:20<5:27:58, 1.13it/s][2025-04-28 00:04:46] (step=0088400) Train Loss: 5.7123, Train Steps/Sec: 1.12 + 80%|█████████████████████████████████████████████████████████████████████████████████████████████████████▍ | 88424/110696 [27:01:43<5:38:05, 1.10it/s][2025-04-28 00:05:08] (step=0088425) Train Loss: 5.7291, Train Steps/Sec: 1.12 + 80%|█████████████████████████████████████████████████████████████████████████████████████████████████████▍ | 88449/110696 [27:02:05<5:41:13, 1.09it/s][2025-04-28 00:05:30] (step=0088450) Train Loss: 5.6779, Train Steps/Sec: 1.11 + 80%|█████████████████████████████████████████████████████████████████████████████████████████████████████▌ | 88474/110696 [27:02:27<5:29:29, 1.12it/s][2025-04-28 00:05:53] (step=0088475) Train Loss: 5.7649, Train Steps/Sec: 1.12 + 80%|█████████████████████████████████████████████████████████████████████████████████████████████████████▌ | 88499/110696 [27:02:50<5:28:34, 1.13it/s][2025-04-28 00:06:15] (step=0088500) Train Loss: 5.7453, Train Steps/Sec: 1.12 + 80%|█████████████████████████████████████████████████████████████████████████████████████████████████████▌ | 88524/110696 [27:03:12<5:27:15, 1.13it/s][2025-04-28 00:06:37] (step=0088525) Train Loss: 5.6938, Train Steps/Sec: 1.12 + 80%|█████████████████████████████████████████████████████████████████████████████████████████████████████▌ | 88549/110696 [27:03:34<5:26:21, 1.13it/s][2025-04-28 00:06:59] (step=0088550) Train Loss: 5.7757, Train Steps/Sec: 1.12 + 80%|█████████████████████████████████████████████████████████████████████████████████████████████████████▌ | 88574/110696 [27:03:56<5:25:04, 1.13it/s][2025-04-28 00:07:22] (step=0088575) Train Loss: 5.7305, Train Steps/Sec: 1.12 + 80%|█████████████████████████████████████████████████████████████████████████████████████████████████████▋ | 88599/110696 [27:04:19<5:26:24, 1.13it/s][2025-04-28 00:07:44] (step=0088600) Train Loss: 5.7315, Train Steps/Sec: 1.11 + 80%|█████████████████████████████████████████████████████████████████████████████████████████████████████▋ | 88624/110696 [27:04:41<5:33:58, 1.10it/s][2025-04-28 00:08:06] (step=0088625) Train Loss: 5.6790, Train Steps/Sec: 1.12 + 80%|█████████████████████████████████████████████████████████████████████████████████████████████████████▋ | 88649/110696 [27:05:03<5:29:02, 1.12it/s][2025-04-28 00:08:29] (step=0088650) Train Loss: 5.6662, Train Steps/Sec: 1.12 + 80%|█████████████████████████████████████████████████████████████████████████████████████████████████████▋ | 88674/110696 [27:05:25<5:28:09, 1.12it/s][2025-04-28 00:08:51] (step=0088675) Train Loss: 5.6666, Train Steps/Sec: 1.12 + 80%|█████████████████████████████████████████████████████████████████████████████████████████████████████▊ | 88699/110696 [27:05:48<5:25:41, 1.13it/s][2025-04-28 00:09:13] (step=0088700) Train Loss: 5.6861, Train Steps/Sec: 1.12 + 80%|█████████████████████████████████████████████████████████████████████████████████████████████████████▊ | 88724/110696 [27:06:10<5:24:08, 1.13it/s][2025-04-28 00:09:35] (step=0088725) Train Loss: 5.6688, Train Steps/Sec: 1.12 + 80%|█████████████████████████████████████████████████████████████████████████████████████████████████████▊ | 88749/110696 [27:06:32<5:24:07, 1.13it/s][2025-04-28 00:09:58] (step=0088750) Train Loss: 5.7686, Train Steps/Sec: 1.12 + 80%|█████████████████████████████████████████████████████████████████████████████████████████████████████▊ | 88774/110696 [27:06:55<5:22:56, 1.13it/s][2025-04-28 00:10:20] (step=0088775) Train Loss: 5.7248, Train Steps/Sec: 1.12 + 80%|█████████████████████████████████████████████████████████████████████████████████████████████████████▉ | 88799/110696 [27:07:17<5:21:33, 1.13it/s][2025-04-28 00:10:42] (step=0088800) Train Loss: 5.7078, Train Steps/Sec: 1.12 + 80%|█████████████████████████████████████████████████████████████████████████████████████████████████████▉ | 88824/110696 [27:07:39<5:31:14, 1.10it/s][2025-04-28 00:11:05] (step=0088825) Train Loss: 5.7366, Train Steps/Sec: 1.12 + 80%|█████████████████████████████████████████████████████████████████████████████████████████████████████▉ | 88849/110696 [27:08:01<5:27:51, 1.11it/s][2025-04-28 00:11:27] (step=0088850) Train Loss: 5.6987, Train Steps/Sec: 1.12 + 80%|█████████████████████████████████████████████████████████████████████████████████████████████████████▉ | 88874/110696 [27:08:24<5:24:53, 1.12it/s][2025-04-28 00:11:49] (step=0088875) Train Loss: 5.7016, Train Steps/Sec: 1.12 + 80%|█████████████████████████████████████████████████████████████████████████████████████████████████████▉ | 88899/110696 [27:08:46<5:22:44, 1.13it/s][2025-04-28 00:12:11] (step=0088900) Train Loss: 5.6910, Train Steps/Sec: 1.12 + 80%|██████████████████████████████████████████████████████████████████████████████████████████████████████ | 88924/110696 [27:09:08<5:23:55, 1.12it/s][2025-04-28 00:12:34] (step=0088925) Train Loss: 5.7167, Train Steps/Sec: 1.12 + 80%|██████████████████████████████████████████████████████████████████████████████████████████████████████ | 88949/110696 [27:09:39<7:33:23, 1.25s/it][2025-04-28 00:13:04] (step=0088950) Train Loss: 5.6573, Train Steps/Sec: 0.81 + 80%|██████████████████████████████████████████████████████████████████████████████████████████████████████ | 88974/110696 [27:10:06<6:37:09, 1.10s/it][2025-04-28 00:13:31] (step=0088975) Train Loss: 5.6901, Train Steps/Sec: 0.94 + 80%|██████████████████████████████████████████████████████████████████████████████████████████████████████ | 88999/110696 [27:10:33<8:33:16, 1.42s/it][2025-04-28 00:13:59] (step=0089000) Train Loss: 5.6843, Train Steps/Sec: 0.91 + 80%|██████████████████████████████████████████████████████████████████████████████████████████████████████▏ | 89024/110696 [27:10:56<5:30:27, 1.09it/s][2025-04-28 00:14:21] (step=0089025) Train Loss: 5.7196, Train Steps/Sec: 1.12 + 80%|██████████████████████████████████████████████████████████████████████████████████████████████████████▏ | 89049/110696 [27:11:18<5:23:42, 1.11it/s][2025-04-28 00:14:43] (step=0089050) Train Loss: 5.7204, Train Steps/Sec: 1.12 + 80%|██████████████████████████████████████████████████████████████████████████████████████████████████████▏ | 89074/110696 [27:11:40<5:20:24, 1.12it/s][2025-04-28 00:15:05] (step=0089075) Train Loss: 5.7048, Train Steps/Sec: 1.13 + 80%|██████████████████████████████████████████████████████████████████████████████████████████████████████▏ | 89099/110696 [27:12:02<5:19:51, 1.13it/s][2025-04-28 00:15:28] (step=0089100) Train Loss: 5.6859, Train Steps/Sec: 1.12 + 81%|██████████████████████████████████████████████████████████████████████████████████████████████████████▎ | 89124/110696 [27:12:25<5:18:37, 1.13it/s][2025-04-28 00:15:50] (step=0089125) Train Loss: 5.6174, Train Steps/Sec: 1.12 + 81%|██████████████████████████████████████████████████████████████████████████████████████████████████████▎ | 89149/110696 [27:12:47<5:17:51, 1.13it/s][2025-04-28 00:16:12] (step=0089150) Train Loss: 5.6527, Train Steps/Sec: 1.12 + 81%|██████████████████████████████████████████████████████████████████████████████████████████████████████▎ | 89174/110696 [27:13:09<5:16:44, 1.13it/s][2025-04-28 00:16:34] (step=0089175) Train Loss: 5.6826, Train Steps/Sec: 1.12 + 81%|██████████████████████████████████████████████████████████████████████████████████████████████████████▎ | 89199/110696 [27:13:31<5:17:34, 1.13it/s][2025-04-28 00:16:57] (step=0089200) Train Loss: 5.7478, Train Steps/Sec: 1.12 + 81%|██████████████████████████████████████████████████████████████████████████████████████████████████████▎ | 89224/110696 [27:13:54<5:25:34, 1.10it/s][2025-04-28 00:17:19] (step=0089225) Train Loss: 5.6360, Train Steps/Sec: 1.12 + 81%|██████████████████████████████████████████████████████████████████████████████████████████████████████▍ | 89249/110696 [27:14:16<5:20:23, 1.12it/s][2025-04-28 00:17:41] (step=0089250) Train Loss: 5.7251, Train Steps/Sec: 1.12 + 81%|██████████████████████████████████████████████████████████████████████████████████████████████████████▍ | 89274/110696 [27:14:38<5:18:18, 1.12it/s][2025-04-28 00:18:04] (step=0089275) Train Loss: 5.6973, Train Steps/Sec: 1.12 + 81%|██████████████████████████████████████████████████████████████████████████████████████████████████████▍ | 89299/110696 [27:15:01<5:16:25, 1.13it/s][2025-04-28 00:18:26] (step=0089300) Train Loss: 5.6856, Train Steps/Sec: 1.12 + 81%|██████████████████████████████████████████████████████████████████████████████████████████████████████▍ | 89324/110696 [27:15:23<5:14:41, 1.13it/s][2025-04-28 00:18:48] (step=0089325) Train Loss: 5.6942, Train Steps/Sec: 1.12 + 81%|██████████████████████████████████████████████████████████████████████████████████████████████████████▌ | 89349/110696 [27:15:45<5:16:05, 1.13it/s][2025-04-28 00:19:10] (step=0089350) Train Loss: 5.7047, Train Steps/Sec: 1.12 + 81%|██████████████████████████████████████████████████████████████████████████████████████████████████████▌ | 89374/110696 [27:16:07<5:14:15, 1.13it/s][2025-04-28 00:19:33] (step=0089375) Train Loss: 5.6975, Train Steps/Sec: 1.12 + 81%|██████████████████████████████████████████████████████████████████████████████████████████████████████▌ | 89399/110696 [27:16:30<5:12:54, 1.13it/s][2025-04-28 00:19:55] (step=0089400) Train Loss: 5.6230, Train Steps/Sec: 1.12 + 81%|██████████████████████████████████████████████████████████████████████████████████████████████████████▌ | 89424/110696 [27:16:52<5:22:07, 1.10it/s][2025-04-28 00:20:17] (step=0089425) Train Loss: 5.7248, Train Steps/Sec: 1.12 + 81%|██████████████████████████████████████████████████████████████████████████████████████████████████████▌ | 89449/110696 [27:17:19<9:34:24, 1.62s/it][2025-04-28 00:20:45] (step=0089450) Train Loss: 5.7345, Train Steps/Sec: 0.92 + 81%|██████████████████████████████████████████████████████████████████████████████████████████████████████▋ | 89474/110696 [27:17:41<5:15:34, 1.12it/s][2025-04-28 00:21:07] (step=0089475) Train Loss: 5.7596, Train Steps/Sec: 1.13 + 81%|██████████████████████████████████████████████████████████████████████████████████████████████████████▋ | 89499/110696 [27:18:04<5:13:49, 1.13it/s][2025-04-28 00:21:29] (step=0089500) Train Loss: 5.7198, Train Steps/Sec: 1.12 + 81%|██████████████████████████████████████████████████████████████████████████████████████████████████████▋ | 89524/110696 [27:18:26<5:13:02, 1.13it/s][2025-04-28 00:21:51] (step=0089525) Train Loss: 5.7153, Train Steps/Sec: 1.12 + 81%|██████████████████████████████████████████████████████████████████████████████████████████████████████▋ | 89549/110696 [27:18:48<5:11:18, 1.13it/s][2025-04-28 00:22:14] (step=0089550) Train Loss: 5.6707, Train Steps/Sec: 1.12 + 81%|██████████████████████████████████████████████████████████████████████████████████████████████████████▊ | 89574/110696 [27:19:10<5:11:48, 1.13it/s][2025-04-28 00:22:36] (step=0089575) Train Loss: 5.7229, Train Steps/Sec: 1.12 + 81%|██████████████████████████████████████████████████████████████████████████████████████████████████████▊ | 89599/110696 [27:19:33<5:10:15, 1.13it/s][2025-04-28 00:22:58] (step=0089600) Train Loss: 5.6959, Train Steps/Sec: 1.12 + 81%|██████████████████████████████████████████████████████████████████████████████████████████████████████▊ | 89624/110696 [27:19:55<5:20:26, 1.10it/s][2025-04-28 00:23:20] (step=0089625) Train Loss: 5.6657, Train Steps/Sec: 1.12 + 81%|██████████████████████████████████████████████████████████████████████████████████████████████████████▊ | 89649/110696 [27:20:17<5:15:08, 1.11it/s][2025-04-28 00:23:43] (step=0089650) Train Loss: 5.6923, Train Steps/Sec: 1.12 + 81%|██████████████████████████████████████████████████████████████████████████████████████████████████████▉ | 89674/110696 [27:20:40<5:13:11, 1.12it/s][2025-04-28 00:24:05] (step=0089675) Train Loss: 5.7061, Train Steps/Sec: 1.12 + 81%|██████████████████████████████████████████████████████████████████████████████████████████████████████▉ | 89699/110696 [27:21:02<5:10:22, 1.13it/s][2025-04-28 00:24:27] (step=0089700) Train Loss: 5.7874, Train Steps/Sec: 1.12 + 81%|██████████████████████████████████████████████████████████████████████████████████████████████████████▉ | 89724/110696 [27:21:24<5:10:04, 1.13it/s][2025-04-28 00:24:49] (step=0089725) Train Loss: 5.7079, Train Steps/Sec: 1.12 + 81%|██████████████████████████████████████████████████████████████████████████████████████████████████████▉ | 89749/110696 [27:21:46<5:08:56, 1.13it/s][2025-04-28 00:25:12] (step=0089750) Train Loss: 5.7393, Train Steps/Sec: 1.12 + 81%|██████████████████████████████████████████████████████████████████████████████████████████████████████▉ | 89774/110696 [27:22:09<5:08:52, 1.13it/s][2025-04-28 00:25:34] (step=0089775) Train Loss: 5.7192, Train Steps/Sec: 1.12 + 81%|███████████████████████████████████████████████████████████████████████████████████████████████████████ | 89799/110696 [27:22:31<5:07:20, 1.13it/s][2025-04-28 00:25:56] (step=0089800) Train Loss: 5.6587, Train Steps/Sec: 1.12 + 81%|███████████████████████████████████████████████████████████████████████████████████████████████████████ | 89824/110696 [27:22:53<5:16:57, 1.10it/s][2025-04-28 00:26:19] (step=0089825) Train Loss: 5.6806, Train Steps/Sec: 1.12 + 81%|███████████████████████████████████████████████████████████████████████████████████████████████████████ | 89849/110696 [27:23:16<5:11:32, 1.12it/s][2025-04-28 00:26:41] (step=0089850) Train Loss: 5.6442, Train Steps/Sec: 1.12 + 81%|███████████████████████████████████████████████████████████████████████████████████████████████████████ | 89874/110696 [27:23:38<5:11:07, 1.12it/s][2025-04-28 00:27:03] (step=0089875) Train Loss: 5.6562, Train Steps/Sec: 1.12 + 81%|███████████████████████████████████████████████████████████████████████████████████████████████████████▏ | 89899/110696 [27:24:00<5:07:58, 1.13it/s][2025-04-28 00:27:26] (step=0089900) Train Loss: 5.7224, Train Steps/Sec: 1.12 + 81%|███████████████████████████████████████████████████████████████████████████████████████████████████████▏ | 89924/110696 [27:24:22<5:06:34, 1.13it/s][2025-04-28 00:27:48] (step=0089925) Train Loss: 5.7055, Train Steps/Sec: 1.12 + 81%|███████████████████████████████████████████████████████████████████████████████████████████████████████▏ | 89949/110696 [27:24:45<5:07:31, 1.12it/s][2025-04-28 00:28:10] (step=0089950) Train Loss: 5.6883, Train Steps/Sec: 1.12 + 81%|███████████████████████████████████████████████████████████████████████████████████████████████████████▏ | 89974/110696 [27:25:07<5:04:51, 1.13it/s][2025-04-28 00:28:32] (step=0089975) Train Loss: 5.6880, Train Steps/Sec: 1.12 + 81%|███████████████████████████████████████████████████████████████████████████████████████████████████████▎ | 89999/110696 [27:25:29<5:04:10, 1.13it/s][2025-04-28 00:28:55] (step=0090000) Train Loss: 5.7202, Train Steps/Sec: 1.12 +You are using a model of type blip-2 to instantiate a model of type instructblip. This is not supported for all configurations of models and can yield errors. +[2025-04-28 00:28:55] text_config is None. Initializing the text config with default values (`OPTConfig`). +model_name_or_path:/tmp/haozhezhao/model/blip2-flan-t5-xl +image_place_holder: +tokenizer length after expend 32101 +tokenizer length before expend 32100 +100%|████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████| 5/5 [05:03<00:00, 60.80s/it] +[2025-04-28 00:35:14] Finish Eval in 90000 steps...█████████████████████████████████████████████████████████████████████████████████████████████| 5/5 [05:03<00:00, 60.47s/it] +processing left-padding... +processing left-padding... +processing left-padding... +processing left-padding... +processing left-padding... +[2025-04-28 00:35:34] Saved checkpoint to checkpoint/X2I_UltraEdiit_fourmask_LLava_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_filtered_objects_4mask_1280_context/003-GPT-XL/checkpoints/0090000.pt +[2025-04-28 00:35:36] Removed old checkpoint: checkpoint/X2I_UltraEdiit_fourmask_LLava_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_filtered_objects_4mask_1280_context/003-GPT-XL/checkpoints/0088000.pt + 81%|███████████████████████████████████████████████████████████████████████████████████████████████████████▎ | 90024/110696 [27:32:33<5:22:42, 1.07it/s][2025-04-28 00:35:58] (step=0090025) Train Loss: 5.6885, Train Steps/Sec: 0.06 + 81%|███████████████████████████████████████████████████████████████████████████████████████████████████████▎ | 90049/110696 [27:32:55<5:09:01, 1.11it/s][2025-04-28 00:36:21] (step=0090050) Train Loss: 5.7443, Train Steps/Sec: 1.12 + 81%|███████████████████████████████████████████████████████████████████████████████████████████████████████▎ | 90074/110696 [27:33:18<5:06:58, 1.12it/s][2025-04-28 00:36:43] (step=0090075) Train Loss: 5.6504, Train Steps/Sec: 1.12 + 81%|███████████████████████████████████████████████████████████████████████████████████████████████████████▎ | 90099/110696 [27:33:40<5:06:42, 1.12it/s][2025-04-28 00:37:05] (step=0090100) Train Loss: 5.7389, Train Steps/Sec: 1.12 + 81%|███████████████████████████████████████████████████████████████████████████████████████████████████████▍ | 90124/110696 [27:34:02<5:04:28, 1.13it/s][2025-04-28 00:37:28] (step=0090125) Train Loss: 5.6784, Train Steps/Sec: 1.12 + 81%|███████████████████████████████████████████████████████████████████████████████████████████████████████▍ | 90149/110696 [27:34:25<5:03:26, 1.13it/s][2025-04-28 00:37:50] (step=0090150) Train Loss: 5.7046, Train Steps/Sec: 1.12 + 81%|███████████████████████████████████████████████████████████████████████████████████████████████████████▍ | 90174/110696 [27:34:47<5:02:56, 1.13it/s][2025-04-28 00:38:12] (step=0090175) Train Loss: 5.6435, Train Steps/Sec: 1.12 + 81%|███████████████████████████████████████████████████████████████████████████████████████████████████████▍ | 90199/110696 [27:35:09<5:01:32, 1.13it/s][2025-04-28 00:38:35] (step=0090200) Train Loss: 5.6938, Train Steps/Sec: 1.12 + 82%|███████████████████████████████████████████████████████████████████████████████████████████████████████▌ | 90224/110696 [27:35:32<5:11:23, 1.10it/s][2025-04-28 00:38:57] (step=0090225) Train Loss: 5.6632, Train Steps/Sec: 1.12 + 82%|███████████████████████████████████████████████████████████████████████████████████████████████████████▌ | 90249/110696 [27:35:54<5:06:40, 1.11it/s][2025-04-28 00:39:19] (step=0090250) Train Loss: 5.6851, Train Steps/Sec: 1.12 + 82%|███████████████████████████████████████████████████████████████████████████████████████████████████████▌ | 90274/110696 [27:36:16<5:03:51, 1.12it/s][2025-04-28 00:39:42] (step=0090275) Train Loss: 5.6276, Train Steps/Sec: 1.12 + 82%|███████████████████████████████████████████████████████████████████████████████████████████████████████▌ | 90299/110696 [27:36:39<5:03:41, 1.12it/s][2025-04-28 00:40:04] (step=0090300) Train Loss: 5.6471, Train Steps/Sec: 1.12 + 82%|███████████████████████████████████████████████████████████████████████████████████████████████████████▋ | 90324/110696 [27:37:01<5:01:41, 1.13it/s][2025-04-28 00:40:26] (step=0090325) Train Loss: 5.6571, Train Steps/Sec: 1.12 + 82%|███████████████████████████████████████████████████████████████████████████████████████████████████████▋ | 90349/110696 [27:37:24<5:27:55, 1.03it/s][2025-04-28 00:40:49] (step=0090350) Train Loss: 5.7344, Train Steps/Sec: 1.10 + 82%|███████████████████████████████████████████████████████████████████████████████████████████████████████▋ | 90374/110696 [27:37:46<5:00:38, 1.13it/s][2025-04-28 00:41:11] (step=0090375) Train Loss: 5.7017, Train Steps/Sec: 1.12 + 82%|███████████████████████████████████████████████████████████████████████████████████████████████████████▋ | 90399/110696 [27:38:08<4:59:11, 1.13it/s][2025-04-28 00:41:34] (step=0090400) Train Loss: 5.6685, Train Steps/Sec: 1.12 + 82%|███████████████████████████████████████████████████████████████████████████████████████████████████████▋ | 90424/110696 [27:38:31<5:07:33, 1.10it/s][2025-04-28 00:41:56] (step=0090425) Train Loss: 5.7942, Train Steps/Sec: 1.12 + 82%|███████████████████████████████████████████████████████████████████████████████████████████████████████▊ | 90449/110696 [27:38:53<5:02:39, 1.11it/s][2025-04-28 00:42:18] (step=0090450) Train Loss: 5.7237, Train Steps/Sec: 1.12 + 82%|███████████████████████████████████████████████████████████████████████████████████████████████████████▊ | 90474/110696 [27:39:15<5:02:34, 1.11it/s][2025-04-28 00:42:41] (step=0090475) Train Loss: 5.6767, Train Steps/Sec: 1.12 + 82%|███████████████████████████████████████████████████████████████████████████████████████████████████████▊ | 90499/110696 [27:39:38<4:59:46, 1.12it/s][2025-04-28 00:43:03] (step=0090500) Train Loss: 5.7121, Train Steps/Sec: 1.12 + 82%|███████████████████████████████████████████████████████████████████████████████████████████████████████▊ | 90524/110696 [27:40:00<4:58:30, 1.13it/s][2025-04-28 00:43:25] (step=0090525) Train Loss: 5.7396, Train Steps/Sec: 1.12 + 82%|███████████████████████████████████████████████████████████████████████████████████████████████████████▉ | 90549/110696 [27:40:22<4:56:57, 1.13it/s][2025-04-28 00:43:48] (step=0090550) Train Loss: 5.7267, Train Steps/Sec: 1.12 + 82%|███████████████████████████████████████████████████████████████████████████████████████████████████████▉ | 90574/110696 [27:40:45<4:57:08, 1.13it/s][2025-04-28 00:44:10] (step=0090575) Train Loss: 5.6392, Train Steps/Sec: 1.12 + 82%|███████████████████████████████████████████████████████████████████████████████████████████████████████▉ | 90599/110696 [27:41:07<4:56:59, 1.13it/s][2025-04-28 00:44:32] (step=0090600) Train Loss: 5.7545, Train Steps/Sec: 1.12 + 82%|███████████████████████████████████████████████████████████████████████████████████████████████████████▉ | 90624/110696 [27:41:29<5:06:08, 1.09it/s][2025-04-28 00:44:55] (step=0090625) Train Loss: 5.6585, Train Steps/Sec: 1.12 + 82%|████████████████████████████████████████████████████████████████████████████████████████████████████████ | 90649/110696 [27:41:52<5:01:06, 1.11it/s][2025-04-28 00:45:17] (step=0090650) Train Loss: 5.6555, Train Steps/Sec: 1.12 + 82%|████████████████████████████████████████████████████████████████████████████████████████████████████████ | 90674/110696 [27:42:14<4:58:44, 1.12it/s][2025-04-28 00:45:39] (step=0090675) Train Loss: 5.7466, Train Steps/Sec: 1.12 + 82%|████████████████████████████████████████████████████████████████████████████████████████████████████████ | 90699/110696 [27:42:36<4:56:47, 1.12it/s][2025-04-28 00:46:01] (step=0090700) Train Loss: 5.6684, Train Steps/Sec: 1.12 + 82%|████████████████████████████████████████████████████████████████████████████████████████████████████████ | 90724/110696 [27:42:58<4:55:21, 1.13it/s][2025-04-28 00:46:24] (step=0090725) Train Loss: 5.6406, Train Steps/Sec: 1.12 + 82%|████████████████████████████████████████████████████████████████████████████████████████████████████████ | 90749/110696 [27:43:21<4:55:12, 1.13it/s][2025-04-28 00:46:46] (step=0090750) Train Loss: 5.6526, Train Steps/Sec: 1.12 + 82%|████████████████████████████████████████████████████████████████████████████████████████████████████████▏ | 90774/110696 [27:43:43<4:54:00, 1.13it/s][2025-04-28 00:47:08] (step=0090775) Train Loss: 5.7470, Train Steps/Sec: 1.12 + 82%|████████████████████████████████████████████████████████████████████████████████████████████████████████▏ | 90799/110696 [27:44:05<4:53:25, 1.13it/s][2025-04-28 00:47:31] (step=0090800) Train Loss: 5.6691, Train Steps/Sec: 1.12 + 82%|████████████████████████████████████████████████████████████████████████████████████████████████████████▏ | 90824/110696 [27:44:28<5:00:57, 1.10it/s][2025-04-28 00:47:53] (step=0090825) Train Loss: 5.6375, Train Steps/Sec: 1.12 + 82%|████████████████████████████████████████████████████████████████████████████████████████████████████████▏ | 90849/110696 [27:44:50<4:58:19, 1.11it/s][2025-04-28 00:48:15] (step=0090850) Train Loss: 5.7053, Train Steps/Sec: 1.12 + 82%|████████████████████████████████████████████████████████████████████████████████████████████████████████▎ | 90874/110696 [27:45:12<4:55:15, 1.12it/s][2025-04-28 00:48:38] (step=0090875) Train Loss: 5.6599, Train Steps/Sec: 1.12 + 82%|████████████████████████████████████████████████████████████████████████████████████████████████████████▎ | 90899/110696 [27:45:35<4:54:50, 1.12it/s][2025-04-28 00:49:00] (step=0090900) Train Loss: 5.7138, Train Steps/Sec: 1.12 + 82%|████████████████████████████████████████████████████████████████████████████████████████████████████████▎ | 90924/110696 [27:45:57<4:52:34, 1.13it/s][2025-04-28 00:49:22] (step=0090925) Train Loss: 5.6693, Train Steps/Sec: 1.12 + 82%|████████████████████████████████████████████████████████████████████████████████████████████████████████▎ | 90949/110696 [27:46:19<4:51:59, 1.13it/s][2025-04-28 00:49:45] (step=0090950) Train Loss: 5.7932, Train Steps/Sec: 1.12 + 82%|████████████████████████████████████████████████████████████████████████████████████████████████████████▎ | 90974/110696 [27:46:41<4:50:04, 1.13it/s][2025-04-28 00:50:07] (step=0090975) Train Loss: 5.6900, Train Steps/Sec: 1.12 + 82%|████████████████████████████████████████████████████████████████████████████████████████████████████████▍ | 90999/110696 [27:47:04<4:50:23, 1.13it/s][2025-04-28 00:50:29] (step=0091000) Train Loss: 5.7629, Train Steps/Sec: 1.12 + 82%|████████████████████████████████████████████████████████████████████████████████████████████████████████▍ | 91024/110696 [27:47:26<4:58:37, 1.10it/s][2025-04-28 00:50:52] (step=0091025) Train Loss: 5.7294, Train Steps/Sec: 1.12 + 82%|████████████████████████████████████████████████████████████████████████████████████████████████████████▍ | 91049/110696 [27:47:49<4:55:44, 1.11it/s][2025-04-28 00:51:14] (step=0091050) Train Loss: 5.6976, Train Steps/Sec: 1.12 + 82%|████████████████████████████████████████████████████████████████████████████████████████████████████████▍ | 91074/110696 [27:48:11<4:53:07, 1.12it/s][2025-04-28 00:51:36] (step=0091075) Train Loss: 5.6914, Train Steps/Sec: 1.12 + 82%|████████████████████████████████████████████████████████████████████████████████████████████████████████▌ | 91099/110696 [27:48:33<4:50:11, 1.13it/s][2025-04-28 00:51:58] (step=0091100) Train Loss: 5.6975, Train Steps/Sec: 1.12 + 82%|████████████████████████████████████████████████████████████████████████████████████████████████████████▌ | 91124/110696 [27:48:55<4:49:36, 1.13it/s][2025-04-28 00:52:21] (step=0091125) Train Loss: 5.7786, Train Steps/Sec: 1.12 + 82%|████████████████████████████████████████████████████████████████████████████████████████████████████████▌ | 91149/110696 [27:49:18<4:48:53, 1.13it/s][2025-04-28 00:52:43] (step=0091150) Train Loss: 5.6120, Train Steps/Sec: 1.12 + 82%|████████████████████████████████████████████████████████████████████████████████████████████████████████▌ | 91174/110696 [27:49:40<4:47:38, 1.13it/s][2025-04-28 00:53:05] (step=0091175) Train Loss: 5.7638, Train Steps/Sec: 1.12 + 82%|████████████████████████████████████████████████████████████████████████████████████████████████████████▋ | 91199/110696 [27:50:02<4:46:07, 1.14it/s][2025-04-28 00:53:28] (step=0091200) Train Loss: 5.7098, Train Steps/Sec: 1.12 + 82%|████████████████████████████████████████████████████████████████████████████████████████████████████████▋ | 91224/110696 [27:50:25<4:55:49, 1.10it/s][2025-04-28 00:53:50] (step=0091225) Train Loss: 5.6783, Train Steps/Sec: 1.12 + 82%|████████████████████████████████████████████████████████████████████████████████████████████████████████▋ | 91249/110696 [27:50:47<4:51:11, 1.11it/s][2025-04-28 00:54:12] (step=0091250) Train Loss: 5.7445, Train Steps/Sec: 1.12 + 82%|████████████████████████████████████████████████████████████████████████████████████████████████████████▋ | 91274/110696 [27:51:09<4:49:17, 1.12it/s][2025-04-28 00:54:35] (step=0091275) Train Loss: 5.6933, Train Steps/Sec: 1.12 + 82%|████████████████████████████████████████████████████████████████████████████████████████████████████████▋ | 91299/110696 [27:51:31<4:47:46, 1.12it/s][2025-04-28 00:54:57] (step=0091300) Train Loss: 5.7063, Train Steps/Sec: 1.12 + 82%|████████████████████████████████████████████████████████████████████████████████████████████████████████▊ | 91324/110696 [27:51:54<4:47:49, 1.12it/s][2025-04-28 00:55:19] (step=0091325) Train Loss: 5.7635, Train Steps/Sec: 1.12 + 83%|████████████████████████████████████████████████████████████████████████████████████████████████████████▊ | 91349/110696 [27:52:16<4:45:35, 1.13it/s][2025-04-28 00:55:41] (step=0091350) Train Loss: 5.7082, Train Steps/Sec: 1.12 + 83%|████████████████████████████████████████████████████████████████████████████████████████████████████████▊ | 91374/110696 [27:52:38<4:45:30, 1.13it/s][2025-04-28 00:56:04] (step=0091375) Train Loss: 5.6910, Train Steps/Sec: 1.12 + 83%|████████████████████████████████████████████████████████████████████████████████████████████████████████▊ | 91399/110696 [27:53:01<4:44:52, 1.13it/s][2025-04-28 00:56:26] (step=0091400) Train Loss: 5.8075, Train Steps/Sec: 1.12 + 83%|████████████████████████████████████████████████████████████████████████████████████████████████████████▉ | 91424/110696 [27:53:23<4:54:02, 1.09it/s][2025-04-28 00:56:49] (step=0091425) Train Loss: 5.7414, Train Steps/Sec: 1.12 + 83%|████████████████████████████████████████████████████████████████████████████████████████████████████████▉ | 91449/110696 [27:53:46<4:48:39, 1.11it/s][2025-04-28 00:57:11] (step=0091450) Train Loss: 5.6526, Train Steps/Sec: 1.12 + 83%|████████████████████████████████████████████████████████████████████████████████████████████████████████▉ | 91474/110696 [27:54:08<4:45:11, 1.12it/s][2025-04-28 00:57:33] (step=0091475) Train Loss: 5.6707, Train Steps/Sec: 1.12 + 83%|████████████████████████████████████████████████████████████████████████████████████████████████████████▉ | 91499/110696 [27:54:30<4:44:00, 1.13it/s][2025-04-28 00:57:55] (step=0091500) Train Loss: 5.6779, Train Steps/Sec: 1.12 + 83%|█████████████████████████████████████████████████████████████████████████████████████████████████████████ | 91524/110696 [27:54:52<4:43:52, 1.13it/s][2025-04-28 00:58:18] (step=0091525) Train Loss: 5.6514, Train Steps/Sec: 1.12 + 83%|█████████████████████████████████████████████████████████████████████████████████████████████████████████ | 91549/110696 [27:55:15<4:42:56, 1.13it/s][2025-04-28 00:58:40] (step=0091550) Train Loss: 5.6789, Train Steps/Sec: 1.12 + 83%|█████████████████████████████████████████████████████████████████████████████████████████████████████████ | 91574/110696 [27:55:37<4:42:02, 1.13it/s][2025-04-28 00:59:02] (step=0091575) Train Loss: 5.6942, Train Steps/Sec: 1.12 + 83%|█████████████████████████████████████████████████████████████████████████████████████████████████████████ | 91599/110696 [27:55:59<4:41:06, 1.13it/s][2025-04-28 00:59:25] (step=0091600) Train Loss: 5.7347, Train Steps/Sec: 1.12 + 83%|█████████████████████████████████████████████████████████████████████████████████████████████████████████ | 91624/110696 [27:56:22<4:49:40, 1.10it/s][2025-04-28 00:59:47] (step=0091625) Train Loss: 5.7033, Train Steps/Sec: 1.12 + 83%|█████████████████████████████████████████████████████████████████████████████████████████████████████████▏ | 91649/110696 [27:56:44<4:46:46, 1.11it/s][2025-04-28 01:00:09] (step=0091650) Train Loss: 5.6451, Train Steps/Sec: 1.12 + 83%|█████████████████████████████████████████████████████████████████████████████████████████████████████████▏ | 91674/110696 [27:57:06<4:47:38, 1.10it/s][2025-04-28 01:00:32] (step=0091675) Train Loss: 5.6340, Train Steps/Sec: 1.12 + 83%|█████████████████████████████████████████████████████████████████████████████████████████████████████████▏ | 91699/110696 [27:57:29<4:42:22, 1.12it/s][2025-04-28 01:00:54] (step=0091700) Train Loss: 5.6715, Train Steps/Sec: 1.12 + 83%|█████████████████████████████████████████████████████████████████████████████████████████████████████████▏ | 91724/110696 [27:57:51<4:40:34, 1.13it/s][2025-04-28 01:01:16] (step=0091725) Train Loss: 5.6150, Train Steps/Sec: 1.12 + 83%|█████████████████████████████████████████████████████████████████████████████████████████████████████████▎ | 91749/110696 [27:58:13<4:40:10, 1.13it/s][2025-04-28 01:01:39] (step=0091750) Train Loss: 5.8065, Train Steps/Sec: 1.12 + 83%|█████████████████████████████████████████████████████████████████████████████████████████████████████████▎ | 91774/110696 [27:58:36<4:39:40, 1.13it/s][2025-04-28 01:02:01] (step=0091775) Train Loss: 5.7151, Train Steps/Sec: 1.12 + 83%|█████████████████████████████████████████████████████████████████████████████████████████████████████████▎ | 91799/110696 [27:58:58<4:37:44, 1.13it/s][2025-04-28 01:02:23] (step=0091800) Train Loss: 5.6624, Train Steps/Sec: 1.12 + 83%|█████████████████████████████████████████████████████████████████████████████████████████████████████████▎ | 91824/110696 [27:59:20<4:48:05, 1.09it/s][2025-04-28 01:02:46] (step=0091825) Train Loss: 5.7143, Train Steps/Sec: 1.12 + 83%|█████████████████████████████████████████████████████████████████████████████████████████████████████████▍ | 91849/110696 [27:59:43<4:42:23, 1.11it/s][2025-04-28 01:03:08] (step=0091850) Train Loss: 5.6553, Train Steps/Sec: 1.12 + 83%|█████████████████████████████████████████████████████████████████████████████████████████████████████████▍ | 91874/110696 [28:00:05<4:41:11, 1.12it/s][2025-04-28 01:03:30] (step=0091875) Train Loss: 5.6677, Train Steps/Sec: 1.12 + 83%|█████████████████████████████████████████████████████████████████████████████████████████████████████████▍ | 91899/110696 [28:00:27<4:39:09, 1.12it/s][2025-04-28 01:03:53] (step=0091900) Train Loss: 5.7056, Train Steps/Sec: 1.12 + 83%|█████████████████████████████████████████████████████████████████████████████████████████████████████████▍ | 91924/110696 [28:00:50<4:37:46, 1.13it/s][2025-04-28 01:04:15] (step=0091925) Train Loss: 5.7193, Train Steps/Sec: 1.12 + 83%|█████████████████████████████████████████████████████████████████████████████████████████████████████████▍ | 91949/110696 [28:01:12<4:39:39, 1.12it/s][2025-04-28 01:04:37] (step=0091950) Train Loss: 5.6999, Train Steps/Sec: 1.12 + 83%|█████████████████████████████████████████████████████████████████████████████████████████████████████████▌ | 91974/110696 [28:01:34<4:35:17, 1.13it/s][2025-04-28 01:05:00] (step=0091975) Train Loss: 5.6834, Train Steps/Sec: 1.12 + 83%|█████████████████████████████████████████████████████████████████████████████████████████████████████████▌ | 91999/110696 [28:01:57<4:35:04, 1.13it/s][2025-04-28 01:05:22] (step=0092000) Train Loss: 5.7076, Train Steps/Sec: 1.12 +You are using a model of type blip-2 to instantiate a model of type instructblip. This is not supported for all configurations of models and can yield errors. +[2025-04-28 01:05:22] text_config is None. Initializing the text config with default values (`OPTConfig`). +model_name_or_path:/tmp/haozhezhao/model/blip2-flan-t5-xl +image_place_holder: +tokenizer length after expend 32101 +tokenizer length before expend 32100 +100%|████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████| 5/5 [05:04<00:00, 60.93s/it] +[2025-04-28 01:11:40] Finish Eval in 92000 steps...█████████████████████████████████████████████████████████████████████████████████████████████| 5/5 [05:04<00:00, 60.67s/it] +processing left-padding... +processing left-padding... +processing left-padding... +processing left-padding... +processing left-padding... +[2025-04-28 01:11:59] Saved checkpoint to checkpoint/X2I_UltraEdiit_fourmask_LLava_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_filtered_objects_4mask_1280_context/003-GPT-XL/checkpoints/0092000.pt +[2025-04-28 01:12:01] Removed old checkpoint: checkpoint/X2I_UltraEdiit_fourmask_LLava_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_filtered_objects_4mask_1280_context/003-GPT-XL/checkpoints/0090000.pt + 83%|█████████████████████████████████████████████████████████████████████████████████████████████████████████▌ | 92024/110696 [28:08:59<4:49:41, 1.07it/s][2025-04-28 01:12:24] (step=0092025) Train Loss: 5.6565, Train Steps/Sec: 0.06 + 83%|█████████████████████████████████████████████████████████████████████████████████████████████████████████▌ | 92049/110696 [28:09:21<4:38:20, 1.12it/s][2025-04-28 01:12:46] (step=0092050) Train Loss: 5.6869, Train Steps/Sec: 1.12 + 83%|█████████████████████████████████████████████████████████████████████████████████████████████████████████▋ | 92074/110696 [28:09:43<4:37:51, 1.12it/s][2025-04-28 01:13:08] (step=0092075) Train Loss: 5.7229, Train Steps/Sec: 1.12 + 83%|█████████████████████████████████████████████████████████████████████████████████████████████████████████▋ | 92099/110696 [28:10:10<4:36:04, 1.12it/s][2025-04-28 01:13:35] (step=0092100) Train Loss: 5.6278, Train Steps/Sec: 0.93 + 83%|█████████████████████████████████████████████████████████████████████████████████████████████████████████▋ | 92124/110696 [28:10:32<4:34:11, 1.13it/s][2025-04-28 01:13:57] (step=0092125) Train Loss: 5.8197, Train Steps/Sec: 1.12 + 83%|█████████████████████████████████████████████████████████████████████████████████████████████████████████▋ | 92149/110696 [28:10:54<4:33:26, 1.13it/s][2025-04-28 01:14:20] (step=0092150) Train Loss: 5.7098, Train Steps/Sec: 1.12 + 83%|█████████████████████████████████████████████████████████████████████████████████████████████████████████▋ | 92174/110696 [28:11:17<4:35:09, 1.12it/s][2025-04-28 01:14:42] (step=0092175) Train Loss: 5.6568, Train Steps/Sec: 1.12 + 83%|█████████████████████████████████████████████████████████████████████████████████████████████████████████▊ | 92199/110696 [28:11:39<4:30:32, 1.14it/s][2025-04-28 01:15:04] (step=0092200) Train Loss: 5.6617, Train Steps/Sec: 1.12 + 83%|█████████████████████████████████████████████████████████████████████████████████████████████████████████▊ | 92224/110696 [28:12:01<4:40:30, 1.10it/s][2025-04-28 01:15:27] (step=0092225) Train Loss: 5.7796, Train Steps/Sec: 1.12 + 83%|█████████████████████████████████████████████████████████████████████████████████████████████████████████▊ | 92249/110696 [28:12:24<4:36:00, 1.11it/s][2025-04-28 01:15:49] (step=0092250) Train Loss: 5.6753, Train Steps/Sec: 1.12 + 83%|█████████████████████████████████████████████████████████████████████████████████████████████████████████▊ | 92274/110696 [28:12:46<4:35:17, 1.12it/s][2025-04-28 01:16:12] (step=0092275) Train Loss: 5.7019, Train Steps/Sec: 1.12 + 83%|█████████████████████████████████████████████████████████████████████████████████████████████████████████▉ | 92299/110696 [28:13:08<4:32:31, 1.13it/s][2025-04-28 01:16:34] (step=0092300) Train Loss: 5.6507, Train Steps/Sec: 1.12 + 83%|█████████████████████████████████████████████████████████████████████████████████████████████████████████▉ | 92324/110696 [28:13:31<4:31:56, 1.13it/s][2025-04-28 01:16:56] (step=0092325) Train Loss: 5.6815, Train Steps/Sec: 1.12 + 83%|█████████████████████████████████████████████████████████████████████████████████████████████████████████▉ | 92349/110696 [28:13:53<4:30:58, 1.13it/s][2025-04-28 01:17:18] (step=0092350) Train Loss: 5.6913, Train Steps/Sec: 1.12 + 83%|█████████████████████████████████████████████████████████████████████████████████████████████████████████▉ | 92374/110696 [28:14:15<4:29:12, 1.13it/s][2025-04-28 01:17:41] (step=0092375) Train Loss: 5.7307, Train Steps/Sec: 1.13 + 83%|██████████████████████████████████████████████████████████████████████████████████████████████████████████ | 92399/110696 [28:14:42<4:31:49, 1.12it/s][2025-04-28 01:18:08] (step=0092400) Train Loss: 5.6658, Train Steps/Sec: 0.92 + 83%|██████████████████████████████████████████████████████████████████████████████████████████████████████████ | 92424/110696 [28:15:05<4:37:36, 1.10it/s][2025-04-28 01:18:30] (step=0092425) Train Loss: 5.6755, Train Steps/Sec: 1.12 + 84%|██████████████████████████████████████████████████████████████████████████████████████████████████████████ | 92449/110696 [28:15:27<4:33:35, 1.11it/s][2025-04-28 01:18:52] (step=0092450) Train Loss: 5.6396, Train Steps/Sec: 1.12 + 84%|██████████████████████████████████████████████████████████████████████████████████████████████████████████ | 92474/110696 [28:15:49<4:30:27, 1.12it/s][2025-04-28 01:19:15] (step=0092475) Train Loss: 5.6025, Train Steps/Sec: 1.12 + 84%|██████████████████████████████████████████████████████████████████████████████████████████████████████████ | 92499/110696 [28:16:12<4:29:15, 1.13it/s][2025-04-28 01:19:37] (step=0092500) Train Loss: 5.6244, Train Steps/Sec: 1.12 + 84%|██████████████████████████████████████████████████████████████████████████████████████████████████████████▏ | 92524/110696 [28:16:34<4:29:14, 1.12it/s][2025-04-28 01:19:59] (step=0092525) Train Loss: 5.7422, Train Steps/Sec: 1.12 + 84%|██████████████████████████████████████████████████████████████████████████████████████████████████████████▏ | 92549/110696 [28:16:56<4:28:54, 1.12it/s][2025-04-28 01:20:22] (step=0092550) Train Loss: 5.6400, Train Steps/Sec: 1.12 + 84%|██████████████████████████████████████████████████████████████████████████████████████████████████████████▏ | 92574/110696 [28:17:18<4:27:30, 1.13it/s][2025-04-28 01:20:44] (step=0092575) Train Loss: 5.6868, Train Steps/Sec: 1.12 + 84%|██████████████████████████████████████████████████████████████████████████████████████████████████████████▏ | 92599/110696 [28:17:41<4:26:25, 1.13it/s][2025-04-28 01:21:06] (step=0092600) Train Loss: 5.7238, Train Steps/Sec: 1.12 + 84%|██████████████████████████████████████████████████████████████████████████████████████████████████████████▎ | 92624/110696 [28:18:03<4:34:09, 1.10it/s][2025-04-28 01:21:28] (step=0092625) Train Loss: 5.6674, Train Steps/Sec: 1.12 + 84%|██████████████████████████████████████████████████████████████████████████████████████████████████████████▎ | 92649/110696 [28:18:25<4:30:23, 1.11it/s][2025-04-28 01:21:51] (step=0092650) Train Loss: 5.7250, Train Steps/Sec: 1.12 + 84%|██████████████████████████████████████████████████████████████████████████████████████████████████████████▎ | 92674/110696 [28:18:48<4:28:28, 1.12it/s][2025-04-28 01:22:13] (step=0092675) Train Loss: 5.6559, Train Steps/Sec: 1.12 + 84%|██████████████████████████████████████████████████████████████████████████████████████████████████████████▎ | 92699/110696 [28:19:10<4:27:42, 1.12it/s][2025-04-28 01:22:35] (step=0092700) Train Loss: 5.6414, Train Steps/Sec: 1.12 + 84%|██████████████████████████████████████████████████████████████████████████████████████████████████████████▍ | 92724/110696 [28:19:32<4:24:55, 1.13it/s][2025-04-28 01:22:58] (step=0092725) Train Loss: 5.6994, Train Steps/Sec: 1.12 + 84%|██████████████████████████████████████████████████████████████████████████████████████████████████████████▍ | 92749/110696 [28:19:54<4:24:32, 1.13it/s][2025-04-28 01:23:20] (step=0092750) Train Loss: 5.6047, Train Steps/Sec: 1.12 + 84%|██████████████████████████████████████████████████████████████████████████████████████████████████████████▍ | 92774/110696 [28:20:17<4:23:29, 1.13it/s][2025-04-28 01:23:42] (step=0092775) Train Loss: 5.7008, Train Steps/Sec: 1.12 + 84%|██████████████████████████████████████████████████████████████████████████████████████████████████████████▍ | 92799/110696 [28:20:39<4:24:04, 1.13it/s][2025-04-28 01:24:04] (step=0092800) Train Loss: 5.6411, Train Steps/Sec: 1.12 + 84%|██████████████████████████████████████████████████████████████████████████████████████████████████████████▍ | 92824/110696 [28:21:01<4:32:00, 1.10it/s][2025-04-28 01:24:27] (step=0092825) Train Loss: 5.6756, Train Steps/Sec: 1.12 + 84%|██████████████████████████████████████████████████████████████████████████████████████████████████████████▌ | 92849/110696 [28:21:24<4:27:48, 1.11it/s][2025-04-28 01:24:49] (step=0092850) Train Loss: 5.6824, Train Steps/Sec: 1.12 + 84%|██████████████████████████████████████████████████████████████████████████████████████████████████████████▌ | 92874/110696 [28:21:46<4:24:46, 1.12it/s][2025-04-28 01:25:11] (step=0092875) Train Loss: 5.7048, Train Steps/Sec: 1.12 + 84%|██████████████████████████████████████████████████████████████████████████████████████████████████████████▌ | 92899/110696 [28:22:08<4:23:45, 1.12it/s][2025-04-28 01:25:34] (step=0092900) Train Loss: 5.7817, Train Steps/Sec: 1.12 + 84%|██████████████████████████████████████████████████████████████████████████████████████████████████████████▌ | 92924/110696 [28:22:31<4:22:41, 1.13it/s][2025-04-28 01:25:56] (step=0092925) Train Loss: 5.7415, Train Steps/Sec: 1.12 + 84%|██████████████████████████████████████████████████████████████████████████████████████████████████████████▋ | 92949/110696 [28:22:53<4:21:38, 1.13it/s][2025-04-28 01:26:18] (step=0092950) Train Loss: 5.6841, Train Steps/Sec: 1.12 + 84%|██████████████████████████████████████████████████████████████████████████████████████████████████████████▋ | 92974/110696 [28:23:15<4:22:09, 1.13it/s][2025-04-28 01:26:40] (step=0092975) Train Loss: 5.6600, Train Steps/Sec: 1.12 + 84%|██████████████████████████████████████████████████████████████████████████████████████████████████████████▋ | 92999/110696 [28:23:37<4:19:19, 1.14it/s][2025-04-28 01:27:03] (step=0093000) Train Loss: 5.6771, Train Steps/Sec: 1.12 + 84%|██████████████████████████████████████████████████████████████████████████████████████████████████████████▋ | 93024/110696 [28:24:00<4:27:27, 1.10it/s][2025-04-28 01:27:25] (step=0093025) Train Loss: 5.6780, Train Steps/Sec: 1.12 + 84%|██████████████████████████████████████████████████████████████████████████████████████████████████████████▊ | 93049/110696 [28:24:22<4:24:42, 1.11it/s][2025-04-28 01:27:47] (step=0093050) Train Loss: 5.6731, Train Steps/Sec: 1.12 + 84%|██████████████████████████████████████████████████████████████████████████████████████████████████████████▊ | 93074/110696 [28:24:44<4:22:16, 1.12it/s][2025-04-28 01:28:10] (step=0093075) Train Loss: 5.7408, Train Steps/Sec: 1.12 + 84%|██████████████████████████████████████████████████████████████████████████████████████████████████████████▊ | 93099/110696 [28:25:06<4:21:10, 1.12it/s][2025-04-28 01:28:32] (step=0093100) Train Loss: 5.7415, Train Steps/Sec: 1.12 + 84%|██████████████████████████████████████████████████████████████████████████████████████████████████████████▊ | 93124/110696 [28:25:29<4:21:19, 1.12it/s][2025-04-28 01:28:54] (step=0093125) Train Loss: 5.6853, Train Steps/Sec: 1.12 + 84%|██████████████████████████████████████████████████████████████████████████████████████████████████████████▊ | 93149/110696 [28:25:51<4:19:32, 1.13it/s][2025-04-28 01:29:16] (step=0093150) Train Loss: 5.7227, Train Steps/Sec: 1.12 + 84%|██████████████████████████████████████████████████████████████████████████████████████████████████████████▉ | 93174/110696 [28:26:13<4:18:26, 1.13it/s][2025-04-28 01:29:39] (step=0093175) Train Loss: 5.6878, Train Steps/Sec: 1.12 + 84%|██████████████████████████████████████████████████████████████████████████████████████████████████████████▉ | 93199/110696 [28:26:36<4:17:19, 1.13it/s][2025-04-28 01:30:01] (step=0093200) Train Loss: 5.6709, Train Steps/Sec: 1.12 + 84%|██████████████████████████████████████████████████████████████████████████████████████████████████████████▉ | 93224/110696 [28:26:58<4:25:52, 1.10it/s][2025-04-28 01:30:23] (step=0093225) Train Loss: 5.6878, Train Steps/Sec: 1.12 + 84%|██████████████████████████████████████████████████████████████████████████████████████████████████████████▉ | 93249/110696 [28:27:20<4:21:39, 1.11it/s][2025-04-28 01:30:46] (step=0093250) Train Loss: 5.6325, Train Steps/Sec: 1.12 + 84%|███████████████████████████████████████████████████████████████████████████████████████████████████████████ | 93274/110696 [28:27:46<8:11:14, 1.69s/it][2025-04-28 01:31:16] (step=0093275) Train Loss: 5.6809, Train Steps/Sec: 0.82 + 84%|███████████████████████████████████████████████████████████████████████████████████████████████████████████ | 93299/110696 [28:28:13<4:17:31, 1.13it/s][2025-04-28 01:31:39] (step=0093300) Train Loss: 5.6762, Train Steps/Sec: 1.12 + 84%|███████████████████████████████████████████████████████████████████████████████████████████████████████████ | 93324/110696 [28:28:40<4:16:49, 1.13it/s][2025-04-28 01:32:05] (step=0093325) Train Loss: 5.6710, Train Steps/Sec: 0.94 + 84%|███████████████████████████████████████████████████████████████████████████████████████████████████████████ | 93349/110696 [28:29:07<4:16:24, 1.13it/s][2025-04-28 01:32:33] (step=0093350) Train Loss: 5.6285, Train Steps/Sec: 0.91 + 84%|███████████████████████████████████████████████████████████████████████████████████████████████████████████▏ | 93374/110696 [28:29:30<4:14:44, 1.13it/s][2025-04-28 01:32:55] (step=0093375) Train Loss: 5.7136, Train Steps/Sec: 1.12 + 84%|███████████████████████████████████████████████████████████████████████████████████████████████████████████▏ | 93399/110696 [28:29:52<4:14:22, 1.13it/s][2025-04-28 01:33:17] (step=0093400) Train Loss: 5.6827, Train Steps/Sec: 1.12 + 84%|███████████████████████████████████████████████████████████████████████████████████████████████████████████▏ | 93424/110696 [28:30:14<4:21:47, 1.10it/s][2025-04-28 01:33:40] (step=0093425) Train Loss: 5.7446, Train Steps/Sec: 1.12 + 84%|███████████████████████████████████████████████████████████████████████████████████████████████████████████▏ | 93449/110696 [28:30:36<4:18:58, 1.11it/s][2025-04-28 01:34:02] (step=0093450) Train Loss: 5.7183, Train Steps/Sec: 1.12 + 84%|███████████████████████████████████████████████████████████████████████████████████████████████████████████▏ | 93474/110696 [28:30:59<4:16:17, 1.12it/s][2025-04-28 01:34:24] (step=0093475) Train Loss: 5.6875, Train Steps/Sec: 1.12 + 84%|███████████████████████████████████████████████████████████████████████████████████████████████████████████▎ | 93499/110696 [28:31:21<4:15:11, 1.12it/s][2025-04-28 01:34:46] (step=0093500) Train Loss: 5.5967, Train Steps/Sec: 1.12 + 84%|███████████████████████████████████████████████████████████████████████████████████████████████████████████▎ | 93524/110696 [28:31:43<4:14:35, 1.12it/s][2025-04-28 01:35:09] (step=0093525) Train Loss: 5.7827, Train Steps/Sec: 1.12 + 85%|███████████████████████████████████████████████████████████████████████████████████████████████████████████▎ | 93549/110696 [28:32:06<4:12:56, 1.13it/s][2025-04-28 01:35:31] (step=0093550) Train Loss: 5.6466, Train Steps/Sec: 1.12 + 85%|███████████████████████████████████████████████████████████████████████████████████████████████████████████▎ | 93574/110696 [28:32:28<4:13:01, 1.13it/s][2025-04-28 01:35:53] (step=0093575) Train Loss: 5.6218, Train Steps/Sec: 1.12 + 85%|███████████████████████████████████████████████████████████████████████████████████████████████████████████▍ | 93599/110696 [28:32:50<4:12:13, 1.13it/s][2025-04-28 01:36:16] (step=0093600) Train Loss: 5.6942, Train Steps/Sec: 1.12 + 85%|███████████████████████████████████████████████████████████████████████████████████████████████████████████▍ | 93624/110696 [28:33:13<4:18:57, 1.10it/s][2025-04-28 01:36:38] (step=0093625) Train Loss: 5.6305, Train Steps/Sec: 1.12 + 85%|███████████████████████████████████████████████████████████████████████████████████████████████████████████▍ | 93649/110696 [28:33:35<4:15:53, 1.11it/s][2025-04-28 01:37:00] (step=0093650) Train Loss: 5.7153, Train Steps/Sec: 1.12 + 85%|███████████████████████████████████████████████████████████████████████████████████████████████████████████▍ | 93674/110696 [28:33:57<4:13:05, 1.12it/s][2025-04-28 01:37:23] (step=0093675) Train Loss: 5.6634, Train Steps/Sec: 1.12 + 85%|███████████████████████████████████████████████████████████████████████████████████████████████████████████▍ | 93699/110696 [28:34:20<4:12:00, 1.12it/s][2025-04-28 01:37:45] (step=0093700) Train Loss: 5.6470, Train Steps/Sec: 1.12 + 85%|███████████████████████████████████████████████████████████████████████████████████████████████████████████▌ | 93724/110696 [28:34:42<4:11:32, 1.12it/s][2025-04-28 01:38:07] (step=0093725) Train Loss: 5.6665, Train Steps/Sec: 1.12 + 85%|███████████████████████████████████████████████████████████████████████████████████████████████████████████▌ | 93749/110696 [28:35:04<4:09:57, 1.13it/s][2025-04-28 01:38:29] (step=0093750) Train Loss: 5.6987, Train Steps/Sec: 1.12 + 85%|███████████████████████████████████████████████████████████████████████████████████████████████████████████▌ | 93774/110696 [28:35:26<4:09:41, 1.13it/s][2025-04-28 01:38:52] (step=0093775) Train Loss: 5.6488, Train Steps/Sec: 1.12 + 85%|███████████████████████████████████████████████████████████████████████████████████████████████████████████▌ | 93799/110696 [28:35:49<4:08:09, 1.13it/s][2025-04-28 01:39:14] (step=0093800) Train Loss: 5.6664, Train Steps/Sec: 1.12 + 85%|███████████████████████████████████████████████████████████████████████████████████████████████████████████▋ | 93824/110696 [28:36:11<4:16:35, 1.10it/s][2025-04-28 01:39:36] (step=0093825) Train Loss: 5.6825, Train Steps/Sec: 1.12 + 85%|███████████████████████████████████████████████████████████████████████████████████████████████████████████▋ | 93849/110696 [28:36:38<7:32:00, 1.61s/it][2025-04-28 01:40:04] (step=0093850) Train Loss: 5.6860, Train Steps/Sec: 0.92 + 85%|███████████████████████████████████████████████████████████████████████████████████████████████████████████▋ | 93874/110696 [28:37:00<4:09:53, 1.12it/s][2025-04-28 01:40:26] (step=0093875) Train Loss: 5.7626, Train Steps/Sec: 1.12 + 85%|███████████████████████████████████████████████████████████████████████████████████████████████████████████▋ | 93899/110696 [28:37:23<4:09:13, 1.12it/s][2025-04-28 01:40:48] (step=0093900) Train Loss: 5.7313, Train Steps/Sec: 1.12 + 85%|███████████████████████████████████████████████████████████████████████████████████████████████████████████▊ | 93924/110696 [28:37:45<4:08:00, 1.13it/s][2025-04-28 01:41:10] (step=0093925) Train Loss: 5.6901, Train Steps/Sec: 1.12 + 85%|███████████████████████████████████████████████████████████████████████████████████████████████████████████▊ | 93949/110696 [28:38:07<4:07:02, 1.13it/s][2025-04-28 01:41:33] (step=0093950) Train Loss: 5.6110, Train Steps/Sec: 1.12 + 85%|███████████████████████████████████████████████████████████████████████████████████████████████████████████▊ | 93974/110696 [28:38:30<4:06:30, 1.13it/s][2025-04-28 01:41:55] (step=0093975) Train Loss: 5.6854, Train Steps/Sec: 1.12 + 85%|███████████████████████████████████████████████████████████████████████████████████████████████████████████▊ | 93999/110696 [28:38:52<4:05:14, 1.13it/s][2025-04-28 01:42:17] (step=0094000) Train Loss: 5.6774, Train Steps/Sec: 1.12 +You are using a model of type blip-2 to instantiate a model of type instructblip. This is not supported for all configurations of models and can yield errors. +[2025-04-28 01:42:17] text_config is None. Initializing the text config with default values (`OPTConfig`). +model_name_or_path:/tmp/haozhezhao/model/blip2-flan-t5-xl +image_place_holder: +tokenizer length after expend 32101 +tokenizer length before expend 32100 +100%|████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████| 5/5 [05:03<00:00, 60.73s/it] +[2025-04-28 01:48:35] Finish Eval in 94000 steps...█████████████████████████████████████████████████████████████████████████████████████████████| 5/5 [05:02<00:00, 60.41s/it] +processing left-padding... +processing left-padding... +processing left-padding... +processing left-padding... +processing left-padding... +[2025-04-28 01:48:55] Saved checkpoint to checkpoint/X2I_UltraEdiit_fourmask_LLava_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_filtered_objects_4mask_1280_context/003-GPT-XL/checkpoints/0094000.pt +[2025-04-28 01:48:57] Removed old checkpoint: checkpoint/X2I_UltraEdiit_fourmask_LLava_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_filtered_objects_4mask_1280_context/003-GPT-XL/checkpoints/0092000.pt + 85%|███████████████████████████████████████████████████████████████████████████████████████████████████████████▊ | 94024/110696 [28:45:54<4:18:54, 1.07it/s][2025-04-28 01:49:19] (step=0094025) Train Loss: 5.6328, Train Steps/Sec: 0.06 + 85%|███████████████████████████████████████████████████████████████████████████████████████████████████████████▉ | 94049/110696 [28:46:16<4:09:01, 1.11it/s][2025-04-28 01:49:42] (step=0094050) Train Loss: 5.7090, Train Steps/Sec: 1.12 + 85%|███████████████████████████████████████████████████████████████████████████████████████████████████████████▉ | 94074/110696 [28:46:39<4:08:01, 1.12it/s][2025-04-28 01:50:04] (step=0094075) Train Loss: 5.6303, Train Steps/Sec: 1.12 + 85%|███████████████████████████████████████████████████████████████████████████████████████████████████████████▉ | 94099/110696 [28:47:01<4:05:30, 1.13it/s][2025-04-28 01:50:26] (step=0094100) Train Loss: 5.7241, Train Steps/Sec: 1.12 + 85%|███████████████████████████████████████████████████████████████████████████████████████████████████████████▉ | 94124/110696 [28:47:23<4:04:50, 1.13it/s][2025-04-28 01:50:49] (step=0094125) Train Loss: 5.6866, Train Steps/Sec: 1.12 + 85%|████████████████████████████████████████████████████████████████████████████████████████████████████████████ | 94149/110696 [28:47:45<4:04:06, 1.13it/s][2025-04-28 01:51:11] (step=0094150) Train Loss: 5.6186, Train Steps/Sec: 1.12 + 85%|████████████████████████████████████████████████████████████████████████████████████████████████████████████ | 94174/110696 [28:48:08<4:03:13, 1.13it/s][2025-04-28 01:51:33] (step=0094175) Train Loss: 5.7162, Train Steps/Sec: 1.12 + 85%|████████████████████████████████████████████████████████████████████████████████████████████████████████████ | 94199/110696 [28:48:30<4:03:07, 1.13it/s][2025-04-28 01:51:56] (step=0094200) Train Loss: 5.6589, Train Steps/Sec: 1.11 + 85%|████████████████████████████████████████████████████████████████████████████████████████████████████████████ | 94224/110696 [28:48:53<4:10:31, 1.10it/s][2025-04-28 01:52:18] (step=0094225) Train Loss: 5.6063, Train Steps/Sec: 1.12 + 85%|████████████████████████████████████████████████████████████████████████████████████████████████████████████▏ | 94249/110696 [28:49:15<4:06:12, 1.11it/s][2025-04-28 01:52:41] (step=0094250) Train Loss: 5.6669, Train Steps/Sec: 1.12 + 85%|████████████████████████████████████████████████████████████████████████████████████████████████████████████▏ | 94274/110696 [28:49:37<4:04:13, 1.12it/s][2025-04-28 01:53:03] (step=0094275) Train Loss: 5.6841, Train Steps/Sec: 1.12 + 85%|████████████████████████████████████████████████████████████████████████████████████████████████████████████▏ | 94299/110696 [28:50:00<4:03:19, 1.12it/s][2025-04-28 01:53:25] (step=0094300) Train Loss: 5.6363, Train Steps/Sec: 1.12 + 85%|████████████████████████████████████████████████████████████████████████████████████████████████████████████▏ | 94324/110696 [28:50:22<4:02:07, 1.13it/s][2025-04-28 01:53:47] (step=0094325) Train Loss: 5.7129, Train Steps/Sec: 1.12 + 85%|████████████████████████████████████████████████████████████████████████████████████████████████████████████▏ | 94349/110696 [28:50:44<4:01:20, 1.13it/s][2025-04-28 01:54:10] (step=0094350) Train Loss: 5.6910, Train Steps/Sec: 1.12 + 85%|████████████████████████████████████████████████████████████████████████████████████████████████████████████▎ | 94374/110696 [28:51:07<4:00:55, 1.13it/s][2025-04-28 01:54:32] (step=0094375) Train Loss: 5.6583, Train Steps/Sec: 1.12 + 85%|████████████████████████████████████████████████████████████████████████████████████████████████████████████▎ | 94399/110696 [28:51:29<3:59:43, 1.13it/s][2025-04-28 01:54:54] (step=0094400) Train Loss: 5.7083, Train Steps/Sec: 1.12 + 85%|████████████████████████████████████████████████████████████████████████████████████████████████████████████▎ | 94424/110696 [28:51:51<4:07:53, 1.09it/s][2025-04-28 01:55:17] (step=0094425) Train Loss: 5.6972, Train Steps/Sec: 1.12 + 85%|████████████████████████████████████████████████████████████████████████████████████████████████████████████▎ | 94449/110696 [28:52:14<4:03:28, 1.11it/s][2025-04-28 01:55:39] (step=0094450) Train Loss: 5.6410, Train Steps/Sec: 1.12 + 85%|████████████████████████████████████████████████████████████████████████████████████████████████████████████▍ | 94474/110696 [28:52:36<4:01:58, 1.12it/s][2025-04-28 01:56:01] (step=0094475) Train Loss: 5.7020, Train Steps/Sec: 1.12 + 85%|████████████████████████████████████████████████████████████████████████████████████████████████████████████▍ | 94499/110696 [28:52:58<3:59:42, 1.13it/s][2025-04-28 01:56:24] (step=0094500) Train Loss: 5.7281, Train Steps/Sec: 1.12 + 85%|████████████████████████████████████████████████████████████████████████████████████████████████████████████▍ | 94524/110696 [28:53:20<3:59:35, 1.12it/s][2025-04-28 01:56:46] (step=0094525) Train Loss: 5.6374, Train Steps/Sec: 1.12 + 85%|████████████████████████████████████████████████████████████████████████████████████████████████████████████▍ | 94549/110696 [28:53:43<3:58:41, 1.13it/s][2025-04-28 01:57:08] (step=0094550) Train Loss: 5.7088, Train Steps/Sec: 1.12 + 85%|████████████████████████████████████████████████████████████████████████████████████████████████████████████▌ | 94574/110696 [28:54:05<3:57:20, 1.13it/s][2025-04-28 01:57:30] (step=0094575) Train Loss: 5.6711, Train Steps/Sec: 1.12 + 85%|████████████████████████████████████████████████████████████████████████████████████████████████████████████▌ | 94599/110696 [28:54:27<3:55:46, 1.14it/s][2025-04-28 01:57:53] (step=0094600) Train Loss: 5.6573, Train Steps/Sec: 1.12 + 85%|████████████████████████████████████████████████████████████████████████████████████████████████████████████▌ | 94624/110696 [28:54:50<4:03:09, 1.10it/s][2025-04-28 01:58:15] (step=0094625) Train Loss: 5.7184, Train Steps/Sec: 1.12 + 86%|████████████████████████████████████████████████████████████████████████████████████████████████████████████▌ | 94649/110696 [28:55:12<3:59:34, 1.12it/s][2025-04-28 01:58:37] (step=0094650) Train Loss: 5.7352, Train Steps/Sec: 1.12 + 86%|████████████████████████████████████████████████████████████████████████████████████████████████████████████▌ | 94674/110696 [28:55:34<3:58:15, 1.12it/s][2025-04-28 01:59:00] (step=0094675) Train Loss: 5.7308, Train Steps/Sec: 1.12 + 86%|████████████████████████████████████████████████████████████████████████████████████████████████████████████▋ | 94699/110696 [28:55:56<3:57:31, 1.12it/s][2025-04-28 01:59:22] (step=0094700) Train Loss: 5.6472, Train Steps/Sec: 1.12 + 86%|████████████████████████████████████████████████████████████████████████████████████████████████████████████▋ | 94724/110696 [28:56:19<3:55:56, 1.13it/s][2025-04-28 01:59:44] (step=0094725) Train Loss: 5.7084, Train Steps/Sec: 1.12 + 86%|████████████████████████████████████████████████████████████████████████████████████████████████████████████▋ | 94749/110696 [28:56:41<3:55:26, 1.13it/s][2025-04-28 02:00:06] (step=0094750) Train Loss: 5.6631, Train Steps/Sec: 1.12 + 86%|████████████████████████████████████████████████████████████████████████████████████████████████████████████▋ | 94774/110696 [28:57:03<3:54:30, 1.13it/s][2025-04-28 02:00:29] (step=0094775) Train Loss: 5.6913, Train Steps/Sec: 1.12 + 86%|████████████████████████████████████████████████████████████████████████████████████████████████████████████▊ | 94799/110696 [28:57:26<3:54:35, 1.13it/s][2025-04-28 02:00:51] (step=0094800) Train Loss: 5.7390, Train Steps/Sec: 1.12 + 86%|████████████████████████████████████████████████████████████████████████████████████████████████████████████▊ | 94824/110696 [28:57:48<4:01:52, 1.09it/s][2025-04-28 02:01:13] (step=0094825) Train Loss: 5.6667, Train Steps/Sec: 1.12 + 86%|████████████████████████████████████████████████████████████████████████████████████████████████████████████▊ | 94849/110696 [28:58:10<3:58:15, 1.11it/s][2025-04-28 02:01:36] (step=0094850) Train Loss: 5.6912, Train Steps/Sec: 1.12 + 86%|████████████████████████████████████████████████████████████████████████████████████████████████████████████▊ | 94874/110696 [28:58:33<3:56:07, 1.12it/s][2025-04-28 02:01:58] (step=0094875) Train Loss: 5.6806, Train Steps/Sec: 1.12 + 86%|████████████████████████████████████████████████████████████████████████████████████████████████████████████▉ | 94899/110696 [28:58:55<3:55:02, 1.12it/s][2025-04-28 02:02:20] (step=0094900) Train Loss: 5.6721, Train Steps/Sec: 1.12 + 86%|████████████████████████████████████████████████████████████████████████████████████████████████████████████▉ | 94924/110696 [28:59:17<3:53:55, 1.12it/s][2025-04-28 02:02:43] (step=0094925) Train Loss: 5.6349, Train Steps/Sec: 1.12 + 86%|████████████████████████████████████████████████████████████████████████████████████████████████████████████▉ | 94949/110696 [28:59:40<3:53:21, 1.12it/s][2025-04-28 02:03:05] (step=0094950) Train Loss: 5.6830, Train Steps/Sec: 1.12 + 86%|████████████████████████████████████████████████████████████████████████████████████████████████████████████▉ | 94974/110696 [29:00:02<3:51:51, 1.13it/s][2025-04-28 02:03:27] (step=0094975) Train Loss: 5.7297, Train Steps/Sec: 1.12 + 86%|████████████████████████████████████████████████████████████████████████████████████████████████████████████▉ | 94999/110696 [29:00:24<3:51:17, 1.13it/s][2025-04-28 02:03:50] (step=0095000) Train Loss: 5.6576, Train Steps/Sec: 1.11 + 86%|█████████████████████████████████████████████████████████████████████████████████████████████████████████████ | 95024/110696 [29:00:47<3:57:56, 1.10it/s][2025-04-28 02:04:12] (step=0095025) Train Loss: 5.6612, Train Steps/Sec: 1.12 + 86%|█████████████████████████████████████████████████████████████████████████████████████████████████████████████ | 95049/110696 [29:01:09<3:54:49, 1.11it/s][2025-04-28 02:04:35] (step=0095050) Train Loss: 5.7610, Train Steps/Sec: 1.12 + 86%|█████████████████████████████████████████████████████████████████████████████████████████████████████████████ | 95074/110696 [29:01:31<3:52:59, 1.12it/s][2025-04-28 02:04:57] (step=0095075) Train Loss: 5.6623, Train Steps/Sec: 1.12 + 86%|█████████████████████████████████████████████████████████████████████████████████████████████████████████████ | 95099/110696 [29:01:54<3:51:31, 1.12it/s][2025-04-28 02:05:19] (step=0095100) Train Loss: 5.7058, Train Steps/Sec: 1.12 + 86%|█████████████████████████████████████████████████████████████████████████████████████████████████████████████▏ | 95124/110696 [29:02:16<3:49:43, 1.13it/s][2025-04-28 02:05:41] (step=0095125) Train Loss: 5.7150, Train Steps/Sec: 1.12 + 86%|█████████████████████████████████████████████████████████████████████████████████████████████████████████████▏ | 95149/110696 [29:02:38<3:49:14, 1.13it/s][2025-04-28 02:06:04] (step=0095150) Train Loss: 5.6434, Train Steps/Sec: 1.12 + 86%|█████████████████████████████████████████████████████████████████████████████████████████████████████████████▏ | 95174/110696 [29:03:01<3:48:38, 1.13it/s][2025-04-28 02:06:26] (step=0095175) Train Loss: 5.7724, Train Steps/Sec: 1.12 + 86%|█████████████████████████████████████████████████████████████████████████████████████████████████████████████▏ | 95199/110696 [29:03:23<3:47:55, 1.13it/s][2025-04-28 02:06:48] (step=0095200) Train Loss: 5.6577, Train Steps/Sec: 1.12 + 86%|█████████████████████████████████████████████████████████████████████████████████████████████████████████████▏ | 95224/110696 [29:03:45<3:55:22, 1.10it/s][2025-04-28 02:07:11] (step=0095225) Train Loss: 5.6778, Train Steps/Sec: 1.12 + 86%|█████████████████████████████████████████████████████████████████████████████████████████████████████████████▎ | 95249/110696 [29:04:08<3:52:32, 1.11it/s][2025-04-28 02:07:33] (step=0095250) Train Loss: 5.7470, Train Steps/Sec: 1.12 + 86%|█████████████████████████████████████████████████████████████████████████████████████████████████████████████▎ | 95274/110696 [29:04:30<3:49:54, 1.12it/s][2025-04-28 02:07:55] (step=0095275) Train Loss: 5.6746, Train Steps/Sec: 1.12 + 86%|█████████████████████████████████████████████████████████████████████████████████████████████████████████████▎ | 95299/110696 [29:04:52<3:48:29, 1.12it/s][2025-04-28 02:08:18] (step=0095300) Train Loss: 5.7368, Train Steps/Sec: 1.12 + 86%|█████████████████████████████████████████████████████████████████████████████████████████████████████████████▎ | 95324/110696 [29:05:15<3:47:40, 1.13it/s][2025-04-28 02:08:40] (step=0095325) Train Loss: 5.6722, Train Steps/Sec: 1.12 + 86%|█████████████████████████████████████████████████████████████████████████████████████████████████████████████▍ | 95349/110696 [29:05:37<3:46:38, 1.13it/s][2025-04-28 02:09:02] (step=0095350) Train Loss: 5.7102, Train Steps/Sec: 1.12 + 86%|█████████████████████████████████████████████████████████████████████████████████████████████████████████████▍ | 95374/110696 [29:05:59<3:45:30, 1.13it/s][2025-04-28 02:09:25] (step=0095375) Train Loss: 5.7302, Train Steps/Sec: 1.12 + 86%|█████████████████████████████████████████████████████████████████████████████████████████████████████████████▍ | 95399/110696 [29:06:22<3:45:47, 1.13it/s][2025-04-28 02:09:47] (step=0095400) Train Loss: 5.6148, Train Steps/Sec: 1.12 + 86%|█████████████████████████████████████████████████████████████████████████████████████████████████████████████▍ | 95424/110696 [29:06:44<3:52:02, 1.10it/s][2025-04-28 02:10:09] (step=0095425) Train Loss: 5.7232, Train Steps/Sec: 1.12 + 86%|█████████████████████████████████████████████████████████████████████████████████████████████████████████████▌ | 95449/110696 [29:07:06<3:48:59, 1.11it/s][2025-04-28 02:10:32] (step=0095450) Train Loss: 5.6650, Train Steps/Sec: 1.12 + 86%|█████████████████████████████████████████████████████████████████████████████████████████████████████████████▌ | 95474/110696 [29:07:29<3:47:06, 1.12it/s][2025-04-28 02:10:54] (step=0095475) Train Loss: 5.6448, Train Steps/Sec: 1.12 + 86%|█████████████████████████████████████████████████████████████████████████████████████████████████████████████▌ | 95499/110696 [29:07:51<3:45:59, 1.12it/s][2025-04-28 02:11:16] (step=0095500) Train Loss: 5.6736, Train Steps/Sec: 1.12 + 86%|█████████████████████████████████████████████████████████████████████████████████████████████████████████████▌ | 95524/110696 [29:08:13<3:44:46, 1.12it/s][2025-04-28 02:11:39] (step=0095525) Train Loss: 5.6602, Train Steps/Sec: 1.12 + 86%|█████████████████████████████████████████████████████████████████████████████████████████████████████████████▌ | 95549/110696 [29:08:36<3:44:46, 1.12it/s][2025-04-28 02:12:01] (step=0095550) Train Loss: 5.7284, Train Steps/Sec: 1.12 + 86%|█████████████████████████████████████████████████████████████████████████████████████████████████████████████▋ | 95574/110696 [29:08:58<3:43:16, 1.13it/s][2025-04-28 02:12:23] (step=0095575) Train Loss: 5.7370, Train Steps/Sec: 1.12 + 86%|█████████████████████████████████████████████████████████████████████████████████████████████████████████████▋ | 95599/110696 [29:09:20<3:43:01, 1.13it/s][2025-04-28 02:12:46] (step=0095600) Train Loss: 5.7008, Train Steps/Sec: 1.12 + 86%|█████████████████████████████████████████████████████████████████████████████████████████████████████████████▋ | 95624/110696 [29:09:43<3:49:35, 1.09it/s][2025-04-28 02:13:08] (step=0095625) Train Loss: 5.6994, Train Steps/Sec: 1.12 + 86%|█████████████████████████████████████████████████████████████████████████████████████████████████████████████▋ | 95649/110696 [29:10:05<3:46:23, 1.11it/s][2025-04-28 02:13:30] (step=0095650) Train Loss: 5.5852, Train Steps/Sec: 1.12 + 86%|█████████████████████████████████████████████████████████████████████████████████████████████████████████████▊ | 95674/110696 [29:10:27<3:43:53, 1.12it/s][2025-04-28 02:13:53] (step=0095675) Train Loss: 5.6343, Train Steps/Sec: 1.12 + 86%|█████████████████████████████████████████████████████████████████████████████████████████████████████████████▊ | 95699/110696 [29:10:49<3:42:36, 1.12it/s][2025-04-28 02:14:15] (step=0095700) Train Loss: 5.6603, Train Steps/Sec: 1.12 + 86%|█████████████████████████████████████████████████████████████████████████████████████████████████████████████▊ | 95724/110696 [29:11:12<3:42:01, 1.12it/s][2025-04-28 02:14:37] (step=0095725) Train Loss: 5.7015, Train Steps/Sec: 1.12 + 86%|█████████████████████████████████████████████████████████████████████████████████████████████████████████████▊ | 95749/110696 [29:11:34<3:40:54, 1.13it/s][2025-04-28 02:15:00] (step=0095750) Train Loss: 5.6721, Train Steps/Sec: 1.12 + 87%|█████████████████████████████████████████████████████████████████████████████████████████████████████████████▉ | 95774/110696 [29:11:56<3:39:54, 1.13it/s][2025-04-28 02:15:22] (step=0095775) Train Loss: 5.6584, Train Steps/Sec: 1.12 + 87%|█████████████████████████████████████████████████████████████████████████████████████████████████████████████▉ | 95799/110696 [29:12:19<3:39:12, 1.13it/s][2025-04-28 02:15:44] (step=0095800) Train Loss: 5.7048, Train Steps/Sec: 1.12 + 87%|█████████████████████████████████████████████████████████████████████████████████████████████████████████████▉ | 95824/110696 [29:12:41<3:46:28, 1.09it/s][2025-04-28 02:16:07] (step=0095825) Train Loss: 5.6500, Train Steps/Sec: 1.12 + 87%|█████████████████████████████████████████████████████████████████████████████████████████████████████████████▉ | 95849/110696 [29:13:04<3:41:56, 1.11it/s][2025-04-28 02:16:29] (step=0095850) Train Loss: 5.5986, Train Steps/Sec: 1.12 + 87%|█████████████████████████████████████████████████████████████████████████████████████████████████████████████▉ | 95874/110696 [29:13:26<3:41:18, 1.12it/s][2025-04-28 02:16:51] (step=0095875) Train Loss: 5.7032, Train Steps/Sec: 1.12 + 87%|██████████████████████████████████████████████████████████████████████████████████████████████████████████████ | 95899/110696 [29:13:48<3:39:47, 1.12it/s][2025-04-28 02:17:14] (step=0095900) Train Loss: 5.6644, Train Steps/Sec: 1.12 + 87%|██████████████████████████████████████████████████████████████████████████████████████████████████████████████ | 95924/110696 [29:14:11<3:39:06, 1.12it/s][2025-04-28 02:17:36] (step=0095925) Train Loss: 5.6752, Train Steps/Sec: 1.12 + 87%|██████████████████████████████████████████████████████████████████████████████████████████████████████████████ | 95949/110696 [29:14:33<3:38:13, 1.13it/s][2025-04-28 02:17:58] (step=0095950) Train Loss: 5.6920, Train Steps/Sec: 1.12 + 87%|██████████████████████████████████████████████████████████████████████████████████████████████████████████████ | 95974/110696 [29:14:55<3:36:23, 1.13it/s][2025-04-28 02:18:21] (step=0095975) Train Loss: 5.7754, Train Steps/Sec: 1.12 + 87%|██████████████████████████████████████████████████████████████████████████████████████████████████████████████▏ | 95999/110696 [29:15:17<3:36:16, 1.13it/s][2025-04-28 02:18:43] (step=0096000) Train Loss: 5.6544, Train Steps/Sec: 1.12 +You are using a model of type blip-2 to instantiate a model of type instructblip. This is not supported for all configurations of models and can yield errors. +[2025-04-28 02:18:43] text_config is None. Initializing the text config with default values (`OPTConfig`). +model_name_or_path:/tmp/haozhezhao/model/blip2-flan-t5-xl +image_place_holder: +tokenizer length after expend 32101 +tokenizer length before expend 32100 +100%|████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████| 5/5 [05:03<00:00, 60.78s/it] +[2025-04-28 02:25:00] Finish Eval in 96000 steps...█████████████████████████████████████████████████████████████████████████████████████████████| 5/5 [05:03<00:00, 60.49s/it] +processing left-padding... +processing left-padding... +processing left-padding... +processing left-padding... +processing left-padding... +[2025-04-28 02:25:20] Saved checkpoint to checkpoint/X2I_UltraEdiit_fourmask_LLava_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_filtered_objects_4mask_1280_context/003-GPT-XL/checkpoints/0096000.pt +[2025-04-28 02:25:22] Removed old checkpoint: checkpoint/X2I_UltraEdiit_fourmask_LLava_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_filtered_objects_4mask_1280_context/003-GPT-XL/checkpoints/0094000.pt + 87%|██████████████████████████████████████████████████████████████████████████████████████████████████████████████▏ | 96024/110696 [29:22:19<3:48:46, 1.07it/s][2025-04-28 02:25:44] (step=0096025) Train Loss: 5.6806, Train Steps/Sec: 0.06 + 87%|██████████████████████████████████████████████████████████████████████████████████████████████████████████████▏ | 96049/110696 [29:22:41<3:40:33, 1.11it/s][2025-04-28 02:26:07] (step=0096050) Train Loss: 5.7286, Train Steps/Sec: 1.12 + 87%|██████████████████████████████████████████████████████████████████████████████████████████████████████████████▏ | 96074/110696 [29:23:04<3:37:41, 1.12it/s][2025-04-28 02:26:29] (step=0096075) Train Loss: 5.6890, Train Steps/Sec: 1.10 + 87%|██████████████████████████████████████████████████████████████████████████████████████████████████████████████▎ | 96099/110696 [29:23:26<3:36:36, 1.12it/s][2025-04-28 02:26:52] (step=0096100) Train Loss: 5.7228, Train Steps/Sec: 1.12 + 87%|██████████████████████████████████████████████████████████████████████████████████████████████████████████████▎ | 96124/110696 [29:23:48<3:35:10, 1.13it/s][2025-04-28 02:27:14] (step=0096125) Train Loss: 5.7554, Train Steps/Sec: 1.12 + 87%|██████████████████████████████████████████████████████████████████████████████████████████████████████████████▎ | 96149/110696 [29:24:11<3:35:25, 1.13it/s][2025-04-28 02:27:36] (step=0096150) Train Loss: 5.6759, Train Steps/Sec: 1.12 + 87%|██████████████████████████████████████████████████████████████████████████████████████████████████████████████▎ | 96174/110696 [29:24:33<3:34:56, 1.13it/s][2025-04-28 02:27:58] (step=0096175) Train Loss: 5.6807, Train Steps/Sec: 1.12 + 87%|██████████████████████████████████████████████████████████████████████████████████████████████████████████████▎ | 96199/110696 [29:24:55<3:33:21, 1.13it/s][2025-04-28 02:28:21] (step=0096200) Train Loss: 5.6753, Train Steps/Sec: 1.12 + 87%|██████████████████████████████████████████████████████████████████████████████████████████████████████████████▍ | 96224/110696 [29:25:18<3:38:53, 1.10it/s][2025-04-28 02:28:43] (step=0096225) Train Loss: 5.6639, Train Steps/Sec: 1.12 + 87%|██████████████████████████████████████████████████████████████████████████████████████████████████████████████▍ | 96249/110696 [29:25:40<3:36:29, 1.11it/s][2025-04-28 02:29:05] (step=0096250) Train Loss: 5.6562, Train Steps/Sec: 1.12 + 87%|██████████████████████████████████████████████████████████████████████████████████████████████████████████████▍ | 96274/110696 [29:26:02<3:35:12, 1.12it/s][2025-04-28 02:29:28] (step=0096275) Train Loss: 5.7223, Train Steps/Sec: 1.12 + 87%|██████████████████████████████████████████████████████████████████████████████████████████████████████████████▍ | 96299/110696 [29:26:25<3:32:46, 1.13it/s][2025-04-28 02:29:50] (step=0096300) Train Loss: 5.6409, Train Steps/Sec: 1.12 + 87%|██████████████████████████████████████████████████████████████████████████████████████████████████████████████▌ | 96324/110696 [29:26:47<3:32:12, 1.13it/s][2025-04-28 02:30:12] (step=0096325) Train Loss: 5.6944, Train Steps/Sec: 1.12 + 87%|██████████████████████████████████████████████████████████████████████████████████████████████████████████████▌ | 96349/110696 [29:27:13<3:33:59, 1.12it/s][2025-04-28 02:30:39] (step=0096350) Train Loss: 5.7074, Train Steps/Sec: 0.94 + 87%|██████████████████████████████████████████████████████████████████████████████████████████████████████████████▌ | 96374/110696 [29:27:36<3:31:00, 1.13it/s][2025-04-28 02:31:01] (step=0096375) Train Loss: 5.6625, Train Steps/Sec: 1.12 + 87%|██████████████████████████████████████████████████████████████████████████████████████████████████████████████▌ | 96399/110696 [29:27:58<3:30:56, 1.13it/s][2025-04-28 02:31:24] (step=0096400) Train Loss: 5.6956, Train Steps/Sec: 1.12 + 87%|██████████████████████████████████████████████████████████████████████████████████████████████████████████████▋ | 96424/110696 [29:28:20<3:36:37, 1.10it/s][2025-04-28 02:31:46] (step=0096425) Train Loss: 5.6629, Train Steps/Sec: 1.12 + 87%|██████████████████████████████████████████████████████████████████████████████████████████████████████████████▋ | 96449/110696 [29:28:43<3:33:53, 1.11it/s][2025-04-28 02:32:08] (step=0096450) Train Loss: 5.6805, Train Steps/Sec: 1.12 + 87%|██████████████████████████████████████████████████████████████████████████████████████████████████████████████▋ | 96474/110696 [29:29:05<3:31:33, 1.12it/s][2025-04-28 02:32:30] (step=0096475) Train Loss: 5.6630, Train Steps/Sec: 1.12 + 87%|██████████████████████████████████████████████████████████████████████████████████████████████████████████████▋ | 96499/110696 [29:29:27<3:30:44, 1.12it/s][2025-04-28 02:32:53] (step=0096500) Train Loss: 5.6758, Train Steps/Sec: 1.12 + 87%|██████████████████████████████████████████████████████████████████████████████████████████████████████████████▋ | 96524/110696 [29:29:50<3:29:45, 1.13it/s][2025-04-28 02:33:15] (step=0096525) Train Loss: 5.6460, Train Steps/Sec: 1.12 + 87%|██████████████████████████████████████████████████████████████████████████████████████████████████████████████▊ | 96549/110696 [29:30:12<3:28:11, 1.13it/s][2025-04-28 02:33:37] (step=0096550) Train Loss: 5.6458, Train Steps/Sec: 1.12 + 87%|██████████████████████████████████████████████████████████████████████████████████████████████████████████████▊ | 96574/110696 [29:30:34<3:28:19, 1.13it/s][2025-04-28 02:34:00] (step=0096575) Train Loss: 5.7129, Train Steps/Sec: 1.12 + 87%|██████████████████████████████████████████████████████████████████████████████████████████████████████████████▊ | 96599/110696 [29:30:56<3:26:55, 1.14it/s][2025-04-28 02:34:22] (step=0096600) Train Loss: 5.6223, Train Steps/Sec: 1.12 + 87%|██████████████████████████████████████████████████████████████████████████████████████████████████████████████▊ | 96624/110696 [29:31:19<3:33:58, 1.10it/s][2025-04-28 02:34:44] (step=0096625) Train Loss: 5.7280, Train Steps/Sec: 1.12 + 87%|██████████████████████████████████████████████████████████████████████████████████████████████████████████████▉ | 96649/110696 [29:31:46<7:22:04, 1.89s/it][2025-04-28 02:35:11] (step=0096650) Train Loss: 5.6753, Train Steps/Sec: 0.93 + 87%|██████████████████████████████████████████████████████████████████████████████████████████████████████████████▉ | 96674/110696 [29:32:08<3:28:24, 1.12it/s][2025-04-28 02:35:33] (step=0096675) Train Loss: 5.7137, Train Steps/Sec: 1.12 + 87%|██████████████████████████████████████████████████████████████████████████████████████████████████████████████▉ | 96699/110696 [29:32:30<3:27:49, 1.12it/s][2025-04-28 02:35:56] (step=0096700) Train Loss: 5.6738, Train Steps/Sec: 1.12 + 87%|██████████████████████████████████████████████████████████████████████████████████████████████████████████████▉ | 96724/110696 [29:32:53<3:27:33, 1.12it/s][2025-04-28 02:36:18] (step=0096725) Train Loss: 5.6542, Train Steps/Sec: 1.12 + 87%|██████████████████████████████████████████████████████████████████████████████████████████████████████████████▉ | 96749/110696 [29:33:15<3:25:29, 1.13it/s][2025-04-28 02:36:40] (step=0096750) Train Loss: 5.6881, Train Steps/Sec: 1.12 + 87%|███████████████████████████████████████████████████████████████████████████████████████████████████████████████ | 96774/110696 [29:33:37<3:25:05, 1.13it/s][2025-04-28 02:37:03] (step=0096775) Train Loss: 5.7009, Train Steps/Sec: 1.12 + 87%|███████████████████████████████████████████████████████████████████████████████████████████████████████████████ | 96799/110696 [29:33:59<3:23:34, 1.14it/s][2025-04-28 02:37:25] (step=0096800) Train Loss: 5.7230, Train Steps/Sec: 1.12 + 87%|███████████████████████████████████████████████████████████████████████████████████████████████████████████████ | 96824/110696 [29:34:22<3:31:17, 1.09it/s][2025-04-28 02:37:47] (step=0096825) Train Loss: 5.6088, Train Steps/Sec: 1.12 + 87%|███████████████████████████████████████████████████████████████████████████████████████████████████████████████ | 96849/110696 [29:34:44<3:27:27, 1.11it/s][2025-04-28 02:38:09] (step=0096850) Train Loss: 5.6684, Train Steps/Sec: 1.12 + 88%|███████████████████████████████████████████████████████████████████████████████████████████████████████████████▏ | 96874/110696 [29:35:06<3:26:00, 1.12it/s][2025-04-28 02:38:32] (step=0096875) Train Loss: 5.6491, Train Steps/Sec: 1.12 + 88%|███████████████████████████████████████████████████████████████████████████████████████████████████████████████▏ | 96899/110696 [29:35:29<3:24:18, 1.13it/s][2025-04-28 02:38:54] (step=0096900) Train Loss: 5.7184, Train Steps/Sec: 1.12 + 88%|███████████████████████████████████████████████████████████████████████████████████████████████████████████████▏ | 96924/110696 [29:35:51<3:23:11, 1.13it/s][2025-04-28 02:39:16] (step=0096925) Train Loss: 5.7054, Train Steps/Sec: 1.12 + 88%|███████████████████████████████████████████████████████████████████████████████████████████████████████████████▏ | 96949/110696 [29:36:13<3:22:52, 1.13it/s][2025-04-28 02:39:39] (step=0096950) Train Loss: 5.6911, Train Steps/Sec: 1.12 + 88%|███████████████████████████████████████████████████████████████████████████████████████████████████████████████▎ | 96974/110696 [29:36:35<3:21:08, 1.14it/s][2025-04-28 02:40:01] (step=0096975) Train Loss: 5.6724, Train Steps/Sec: 1.12 + 88%|███████████████████████████████████████████████████████████████████████████████████████████████████████████████▎ | 96999/110696 [29:36:58<3:21:05, 1.14it/s][2025-04-28 02:40:23] (step=0097000) Train Loss: 5.6680, Train Steps/Sec: 1.12 + 88%|███████████████████████████████████████████████████████████████████████████████████████████████████████████████▎ | 97024/110696 [29:37:20<3:27:52, 1.10it/s][2025-04-28 02:40:45] (step=0097025) Train Loss: 5.7212, Train Steps/Sec: 1.12 + 88%|███████████████████████████████████████████████████████████████████████████████████████████████████████████████▎ | 97049/110696 [29:37:42<3:24:36, 1.11it/s][2025-04-28 02:41:08] (step=0097050) Train Loss: 5.7542, Train Steps/Sec: 1.12 + 88%|███████████████████████████████████████████████████████████████████████████████████████████████████████████████▎ | 97074/110696 [29:38:04<3:22:49, 1.12it/s][2025-04-28 02:41:30] (step=0097075) Train Loss: 5.6960, Train Steps/Sec: 1.12 + 88%|███████████████████████████████████████████████████████████████████████████████████████████████████████████████▍ | 97099/110696 [29:38:27<3:21:48, 1.12it/s][2025-04-28 02:41:52] (step=0097100) Train Loss: 5.6460, Train Steps/Sec: 1.12 + 88%|███████████████████████████████████████████████████████████████████████████████████████████████████████████████▍ | 97124/110696 [29:38:49<3:20:10, 1.13it/s][2025-04-28 02:42:14] (step=0097125) Train Loss: 5.6137, Train Steps/Sec: 1.12 + 88%|███████████████████████████████████████████████████████████████████████████████████████████████████████████████▍ | 97149/110696 [29:39:11<3:19:40, 1.13it/s][2025-04-28 02:42:37] (step=0097150) Train Loss: 5.7263, Train Steps/Sec: 1.12 + 88%|███████████████████████████████████████████████████████████████████████████████████████████████████████████████▍ | 97174/110696 [29:39:34<3:19:16, 1.13it/s][2025-04-28 02:42:59] (step=0097175) Train Loss: 5.6948, Train Steps/Sec: 1.12 + 88%|███████████████████████████████████████████████████████████████████████████████████████████████████████████████▌ | 97199/110696 [29:39:56<3:18:34, 1.13it/s][2025-04-28 02:43:21] (step=0097200) Train Loss: 5.6863, Train Steps/Sec: 1.12 + 88%|███████████████████████████████████████████████████████████████████████████████████████████████████████████████▌ | 97224/110696 [29:40:18<3:24:07, 1.10it/s][2025-04-28 02:43:44] (step=0097225) Train Loss: 5.6661, Train Steps/Sec: 1.12 + 88%|███████████████████████████████████████████████████████████████████████████████████████████████████████████████▌ | 97249/110696 [29:40:40<3:20:28, 1.12it/s][2025-04-28 02:44:06] (step=0097250) Train Loss: 5.6434, Train Steps/Sec: 1.13 + 88%|███████████████████████████████████████████████████████████████████████████████████████████████████████████████▌ | 97274/110696 [29:41:03<3:19:34, 1.12it/s][2025-04-28 02:44:28] (step=0097275) Train Loss: 5.6233, Train Steps/Sec: 1.12 + 88%|███████████████████████████████████████████████████████████████████████████████████████████████████████████████▋ | 97299/110696 [29:41:25<3:17:52, 1.13it/s][2025-04-28 02:44:50] (step=0097300) Train Loss: 5.7897, Train Steps/Sec: 1.12 + 88%|███████████████████████████████████████████████████████████████████████████████████████████████████████████████▋ | 97324/110696 [29:41:47<3:16:51, 1.13it/s][2025-04-28 02:45:13] (step=0097325) Train Loss: 5.6049, Train Steps/Sec: 1.13 + 88%|███████████████████████████████████████████████████████████████████████████████████████████████████████████████▋ | 97349/110696 [29:42:09<3:17:02, 1.13it/s][2025-04-28 02:45:35] (step=0097350) Train Loss: 5.6211, Train Steps/Sec: 1.12 + 88%|███████████████████████████████████████████████████████████████████████████████████████████████████████████████▋ | 97374/110696 [29:42:32<3:16:33, 1.13it/s][2025-04-28 02:45:57] (step=0097375) Train Loss: 5.6811, Train Steps/Sec: 1.12 + 88%|███████████████████████████████████████████████████████████████████████████████████████████████████████████████▋ | 97399/110696 [29:42:54<3:15:38, 1.13it/s][2025-04-28 02:46:19] (step=0097400) Train Loss: 5.6838, Train Steps/Sec: 1.12 + 88%|███████████████████████████████████████████████████████████████████████████████████████████████████████████████▊ | 97424/110696 [29:43:16<3:20:40, 1.10it/s][2025-04-28 02:46:42] (step=0097425) Train Loss: 5.6669, Train Steps/Sec: 1.12 + 88%|███████████████████████████████████████████████████████████████████████████████████████████████████████████████▊ | 97449/110696 [29:43:39<3:18:46, 1.11it/s][2025-04-28 02:47:04] (step=0097450) Train Loss: 5.6717, Train Steps/Sec: 1.12 + 88%|███████████████████████████████████████████████████████████████████████████████████████████████████████████████▊ | 97474/110696 [29:44:01<3:17:38, 1.12it/s][2025-04-28 02:47:26] (step=0097475) Train Loss: 5.6957, Train Steps/Sec: 1.12 + 88%|███████████████████████████████████████████████████████████████████████████████████████████████████████████████▊ | 97499/110696 [29:44:23<3:15:46, 1.12it/s][2025-04-28 02:47:49] (step=0097500) Train Loss: 5.7033, Train Steps/Sec: 1.12 + 88%|███████████████████████████████████████████████████████████████████████████████████████████████████████████████▉ | 97524/110696 [29:44:45<3:14:38, 1.13it/s][2025-04-28 02:48:11] (step=0097525) Train Loss: 5.5787, Train Steps/Sec: 1.12 + 88%|███████████████████████████████████████████████████████████████████████████████████████████████████████████████▉ | 97549/110696 [29:45:08<3:13:55, 1.13it/s][2025-04-28 02:48:33] (step=0097550) Train Loss: 5.7249, Train Steps/Sec: 1.12 + 88%|███████████████████████████████████████████████████████████████████████████████████████████████████████████████▉ | 97574/110696 [29:45:30<3:13:17, 1.13it/s][2025-04-28 02:48:55] (step=0097575) Train Loss: 5.7181, Train Steps/Sec: 1.12 + 88%|███████████████████████████████████████████████████████████████████████████████████████████████████████████████▉ | 97599/110696 [29:45:52<3:12:13, 1.14it/s][2025-04-28 02:49:22] (step=0097600) Train Loss: 5.6781, Train Steps/Sec: 0.96 + 88%|████████████████████████████████████████████████████████████████████████████████████████████████████████████████ | 97624/110696 [29:46:23<3:22:37, 1.08it/s][2025-04-28 02:49:48] (step=0097625) Train Loss: 5.6591, Train Steps/Sec: 0.93 + 88%|████████████████████████████████████████████████████████████████████████████████████████████████████████████████ | 97649/110696 [29:46:49<3:15:07, 1.11it/s][2025-04-28 02:50:15] (step=0097650) Train Loss: 5.6378, Train Steps/Sec: 0.94 + 88%|████████████████████████████████████████████████████████████████████████████████████████████████████████████████ | 97674/110696 [29:47:17<3:14:10, 1.12it/s][2025-04-28 02:50:42] (step=0097675) Train Loss: 5.6691, Train Steps/Sec: 0.91 + 88%|████████████████████████████████████████████████████████████████████████████████████████████████████████████████ | 97699/110696 [29:47:39<3:13:02, 1.12it/s][2025-04-28 02:51:05] (step=0097700) Train Loss: 5.6803, Train Steps/Sec: 1.12 + 88%|████████████████████████████████████████████████████████████████████████████████████████████████████████████████ | 97724/110696 [29:48:01<3:11:50, 1.13it/s][2025-04-28 02:51:27] (step=0097725) Train Loss: 5.6957, Train Steps/Sec: 1.12 + 88%|████████████████████████████████████████████████████████████████████████████████████████████████████████████████▏ | 97749/110696 [29:48:24<3:11:40, 1.13it/s][2025-04-28 02:51:49] (step=0097750) Train Loss: 5.6755, Train Steps/Sec: 1.12 + 88%|████████████████████████████████████████████████████████████████████████████████████████████████████████████████▏ | 97774/110696 [29:48:46<3:10:23, 1.13it/s][2025-04-28 02:52:11] (step=0097775) Train Loss: 5.6517, Train Steps/Sec: 1.12 + 88%|████████████████████████████████████████████████████████████████████████████████████████████████████████████████▏ | 97799/110696 [29:49:08<3:10:17, 1.13it/s][2025-04-28 02:52:34] (step=0097800) Train Loss: 5.7008, Train Steps/Sec: 1.12 + 88%|████████████████████████████████████████████████████████████████████████████████████████████████████████████████▏ | 97824/110696 [29:49:31<3:15:56, 1.09it/s][2025-04-28 02:52:56] (step=0097825) Train Loss: 5.7344, Train Steps/Sec: 1.12 + 88%|████████████████████████████████████████████████████████████████████████████████████████████████████████████████▎ | 97849/110696 [29:49:53<3:12:42, 1.11it/s][2025-04-28 02:53:18] (step=0097850) Train Loss: 5.7254, Train Steps/Sec: 1.12 + 88%|████████████████████████████████████████████████████████████████████████████████████████████████████████████████▎ | 97874/110696 [29:50:15<3:10:33, 1.12it/s][2025-04-28 02:53:41] (step=0097875) Train Loss: 5.6669, Train Steps/Sec: 1.12 + 88%|████████████████████████████████████████████████████████████████████████████████████████████████████████████████▎ | 97899/110696 [29:50:38<3:09:32, 1.13it/s][2025-04-28 02:54:03] (step=0097900) Train Loss: 5.6258, Train Steps/Sec: 1.12 + 88%|████████████████████████████████████████████████████████████████████████████████████████████████████████████████▎ | 97924/110696 [29:51:00<3:09:40, 1.12it/s][2025-04-28 02:54:25] (step=0097925) Train Loss: 5.7080, Train Steps/Sec: 1.12 + 88%|████████████████████████████████████████████████████████████████████████████████████████████████████████████████▍ | 97949/110696 [29:51:22<3:08:21, 1.13it/s][2025-04-28 02:54:48] (step=0097950) Train Loss: 5.6701, Train Steps/Sec: 1.12 + 89%|████████████████████████████████████████████████████████████████████████████████████████████████████████████████▍ | 97974/110696 [29:51:45<3:07:44, 1.13it/s][2025-04-28 02:55:10] (step=0097975) Train Loss: 5.6450, Train Steps/Sec: 1.12 + 89%|████████████████████████████████████████████████████████████████████████████████████████████████████████████████▍ | 97999/110696 [29:52:07<3:07:04, 1.13it/s][2025-04-28 02:55:32] (step=0098000) Train Loss: 5.6770, Train Steps/Sec: 1.12 +You are using a model of type blip-2 to instantiate a model of type instructblip. This is not supported for all configurations of models and can yield errors. +[2025-04-28 02:55:32] text_config is None. Initializing the text config with default values (`OPTConfig`). +model_name_or_path:/tmp/haozhezhao/model/blip2-flan-t5-xl +image_place_holder: +tokenizer length after expend 32101 +tokenizer length before expend 32100 +100%|████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████| 5/5 [05:03<00:00, 60.80s/it] +[2025-04-28 03:01:55] Finish Eval in 98000 steps...█████████████████████████████████████████████████████████████████████████████████████████████| 5/5 [05:03<00:00, 60.45s/it] +processing left-padding... +processing left-padding... +processing left-padding... +processing left-padding... +processing left-padding... +[2025-04-28 03:02:15] Saved checkpoint to checkpoint/X2I_UltraEdiit_fourmask_LLava_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_filtered_objects_4mask_1280_context/003-GPT-XL/checkpoints/0098000.pt +[2025-04-28 03:02:17] Removed old checkpoint: checkpoint/X2I_UltraEdiit_fourmask_LLava_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_filtered_objects_4mask_1280_context/003-GPT-XL/checkpoints/0096000.pt + 89%|████████████████████████████████████████████████████████████████████████████████████████████████████████████████▍ | 98024/110696 [29:59:14<3:17:36, 1.07it/s][2025-04-28 03:02:39] (step=0098025) Train Loss: 5.6641, Train Steps/Sec: 0.06 + 89%|████████████████████████████████████████████████████████████████████████████████████████████████████████████████▍ | 98049/110696 [29:59:36<3:09:17, 1.11it/s][2025-04-28 03:03:01] (step=0098050) Train Loss: 5.6866, Train Steps/Sec: 1.12 + 89%|████████████████████████████████████████████████████████████████████████████████████████████████████████████████▌ | 98074/110696 [29:59:58<3:08:06, 1.12it/s][2025-04-28 03:03:24] (step=0098075) Train Loss: 5.6637, Train Steps/Sec: 1.12 + 89%|████████████████████████████████████████████████████████████████████████████████████████████████████████████████▌ | 98099/110696 [30:00:21<3:06:22, 1.13it/s][2025-04-28 03:03:46] (step=0098100) Train Loss: 5.6378, Train Steps/Sec: 1.12 + 89%|████████████████████████████████████████████████████████████████████████████████████████████████████████████████▌ | 98124/110696 [30:00:43<3:05:41, 1.13it/s][2025-04-28 03:04:08] (step=0098125) Train Loss: 5.7048, Train Steps/Sec: 1.12 + 89%|████████████████████████████████████████████████████████████████████████████████████████████████████████████████▌ | 98149/110696 [30:01:05<3:04:13, 1.14it/s][2025-04-28 03:04:30] (step=0098150) Train Loss: 5.7012, Train Steps/Sec: 1.13 + 89%|████████████████████████████████████████████████████████████████████████████████████████████████████████████████▋ | 98174/110696 [30:01:27<3:04:24, 1.13it/s][2025-04-28 03:04:53] (step=0098175) Train Loss: 5.6602, Train Steps/Sec: 1.12 + 89%|████████████████████████████████████████████████████████████████████████████████████████████████████████████████▋ | 98199/110696 [30:01:50<3:03:55, 1.13it/s][2025-04-28 03:05:15] (step=0098200) Train Loss: 5.6779, Train Steps/Sec: 1.12 + 89%|████████████████████████████████████████████████████████████████████████████████████████████████████████████████▋ | 98224/110696 [30:02:12<3:08:56, 1.10it/s][2025-04-28 03:05:37] (step=0098225) Train Loss: 5.6544, Train Steps/Sec: 1.12 + 89%|████████████████████████████████████████████████████████████████████████████████████████████████████████████████▋ | 98249/110696 [30:02:39<3:23:30, 1.02it/s][2025-04-28 03:06:05] (step=0098250) Train Loss: 5.6997, Train Steps/Sec: 0.92 + 89%|████████████████████████████████████████████████████████████████████████████████████████████████████████████████▋ | 98274/110696 [30:03:02<3:05:18, 1.12it/s][2025-04-28 03:06:27] (step=0098275) Train Loss: 5.6872, Train Steps/Sec: 1.12 + 89%|████████████████████████████████████████████████████████████████████████████████████████████████████████████████▊ | 98299/110696 [30:03:24<3:03:44, 1.12it/s][2025-04-28 03:06:49] (step=0098300) Train Loss: 5.6479, Train Steps/Sec: 1.12 + 89%|████████████████████████████████████████████████████████████████████████████████████████████████████████████████▊ | 98324/110696 [30:03:46<3:02:30, 1.13it/s][2025-04-28 03:07:11] (step=0098325) Train Loss: 5.7382, Train Steps/Sec: 1.12 + 89%|████████████████████████████████████████████████████████████████████████████████████████████████████████████████▊ | 98349/110696 [30:04:08<3:01:50, 1.13it/s][2025-04-28 03:07:34] (step=0098350) Train Loss: 5.6529, Train Steps/Sec: 1.12 + 89%|████████████████████████████████████████████████████████████████████████████████████████████████████████████████▊ | 98374/110696 [30:04:31<3:01:14, 1.13it/s][2025-04-28 03:07:56] (step=0098375) Train Loss: 5.6830, Train Steps/Sec: 1.12 + 89%|████████████████████████████████████████████████████████████████████████████████████████████████████████████████▉ | 98399/110696 [30:04:53<3:00:41, 1.13it/s][2025-04-28 03:08:18] (step=0098400) Train Loss: 5.7248, Train Steps/Sec: 1.12 + 89%|████████████████████████████████████████████████████████████████████████████████████████████████████████████████▉ | 98424/110696 [30:05:15<3:06:23, 1.10it/s][2025-04-28 03:08:41] (step=0098425) Train Loss: 5.6550, Train Steps/Sec: 1.12 + 89%|████████████████████████████████████████████████████████████████████████████████████████████████████████████████▉ | 98449/110696 [30:05:37<3:03:38, 1.11it/s][2025-04-28 03:09:03] (step=0098450) Train Loss: 5.7588, Train Steps/Sec: 1.12 + 89%|████████████████████████████████████████████████████████████████████████████████████████████████████████████████▉ | 98474/110696 [30:06:00<3:01:38, 1.12it/s][2025-04-28 03:09:25] (step=0098475) Train Loss: 5.6737, Train Steps/Sec: 1.12 + 89%|█████████████████████████████████████████████████████████████████████████████████████████████████████████████████ | 98499/110696 [30:06:22<3:00:54, 1.12it/s][2025-04-28 03:09:47] (step=0098500) Train Loss: 5.6557, Train Steps/Sec: 1.12 + 89%|█████████████████████████████████████████████████████████████████████████████████████████████████████████████████ | 98524/110696 [30:06:44<2:59:23, 1.13it/s][2025-04-28 03:10:10] (step=0098525) Train Loss: 5.6318, Train Steps/Sec: 1.12 + 89%|█████████████████████████████████████████████████████████████████████████████████████████████████████████████████ | 98549/110696 [30:07:07<2:59:04, 1.13it/s][2025-04-28 03:10:32] (step=0098550) Train Loss: 5.6720, Train Steps/Sec: 1.12 + 89%|█████████████████████████████████████████████████████████████████████████████████████████████████████████████████ | 98574/110696 [30:07:29<2:59:13, 1.13it/s][2025-04-28 03:10:54] (step=0098575) Train Loss: 5.6744, Train Steps/Sec: 1.12 + 89%|█████████████████████████████████████████████████████████████████████████████████████████████████████████████████ | 98599/110696 [30:07:51<2:57:32, 1.14it/s][2025-04-28 03:11:17] (step=0098600) Train Loss: 5.6161, Train Steps/Sec: 1.12 + 89%|█████████████████████████████████████████████████████████████████████████████████████████████████████████████████▏ | 98624/110696 [30:08:14<3:03:29, 1.10it/s][2025-04-28 03:11:39] (step=0098625) Train Loss: 5.7121, Train Steps/Sec: 1.12 + 89%|█████████████████████████████████████████████████████████████████████████████████████████████████████████████████▏ | 98649/110696 [30:08:36<2:59:49, 1.12it/s][2025-04-28 03:12:01] (step=0098650) Train Loss: 5.6572, Train Steps/Sec: 1.12 + 89%|█████████████████████████████████████████████████████████████████████████████████████████████████████████████████▏ | 98674/110696 [30:08:58<2:58:58, 1.12it/s][2025-04-28 03:12:24] (step=0098675) Train Loss: 5.7682, Train Steps/Sec: 1.12 + 89%|█████████████████████████████████████████████████████████████████████████████████████████████████████████████████▏ | 98699/110696 [30:09:20<2:57:58, 1.12it/s][2025-04-28 03:12:46] (step=0098700) Train Loss: 5.7138, Train Steps/Sec: 1.12 + 89%|█████████████████████████████████████████████████████████████████████████████████████████████████████████████████▎ | 98724/110696 [30:09:43<2:56:29, 1.13it/s][2025-04-28 03:13:08] (step=0098725) Train Loss: 5.6636, Train Steps/Sec: 1.12 + 89%|█████████████████████████████████████████████████████████████████████████████████████████████████████████████████▎ | 98749/110696 [30:10:05<2:55:24, 1.14it/s][2025-04-28 03:13:30] (step=0098750) Train Loss: 5.6824, Train Steps/Sec: 1.12 + 89%|█████████████████████████████████████████████████████████████████████████████████████████████████████████████████▎ | 98774/110696 [30:10:27<2:55:07, 1.13it/s][2025-04-28 03:13:53] (step=0098775) Train Loss: 5.6300, Train Steps/Sec: 1.12 + 89%|█████████████████████████████████████████████████████████████████████████████████████████████████████████████████▎ | 98799/110696 [30:10:50<2:55:48, 1.13it/s][2025-04-28 03:14:15] (step=0098800) Train Loss: 5.6832, Train Steps/Sec: 1.12 + 89%|█████████████████████████████████████████████████████████████████████████████████████████████████████████████████▍ | 98824/110696 [30:11:12<3:01:11, 1.09it/s][2025-04-28 03:14:37] (step=0098825) Train Loss: 5.6926, Train Steps/Sec: 1.12 + 89%|█████████████████████████████████████████████████████████████████████████████████████████████████████████████████▍ | 98849/110696 [30:11:34<2:57:32, 1.11it/s][2025-04-28 03:15:00] (step=0098850) Train Loss: 5.6563, Train Steps/Sec: 1.13 + 89%|█████████████████████████████████████████████████████████████████████████████████████████████████████████████████▍ | 98874/110696 [30:11:56<2:56:01, 1.12it/s][2025-04-28 03:15:22] (step=0098875) Train Loss: 5.6867, Train Steps/Sec: 1.12 + 89%|█████████████████████████████████████████████████████████████████████████████████████████████████████████████████▍ | 98899/110696 [30:12:19<2:54:57, 1.12it/s][2025-04-28 03:15:44] (step=0098900) Train Loss: 5.7025, Train Steps/Sec: 1.12 + 89%|█████████████████████████████████████████████████████████████████████████████████████████████████████████████████▍ | 98924/110696 [30:12:41<2:53:57, 1.13it/s][2025-04-28 03:16:06] (step=0098925) Train Loss: 5.6525, Train Steps/Sec: 1.12 + 89%|█████████████████████████████████████████████████████████████████████████████████████████████████████████████████▌ | 98949/110696 [30:13:03<2:53:56, 1.13it/s][2025-04-28 03:16:29] (step=0098950) Train Loss: 5.7354, Train Steps/Sec: 1.12 + 89%|█████████████████████████████████████████████████████████████████████████████████████████████████████████████████▌ | 98974/110696 [30:13:26<2:52:45, 1.13it/s][2025-04-28 03:16:51] (step=0098975) Train Loss: 5.7048, Train Steps/Sec: 1.12 + 89%|█████████████████████████████████████████████████████████████████████████████████████████████████████████████████▌ | 98999/110696 [30:13:48<2:51:33, 1.14it/s][2025-04-28 03:17:13] (step=0099000) Train Loss: 5.6546, Train Steps/Sec: 1.12 + 89%|█████████████████████████████████████████████████████████████████████████████████████████████████████████████████▌ | 99024/110696 [30:14:10<2:57:30, 1.10it/s][2025-04-28 03:17:36] (step=0099025) Train Loss: 5.6579, Train Steps/Sec: 1.12 + 89%|█████████████████████████████████████████████████████████████████████████████████████████████████████████████████▋ | 99049/110696 [30:14:33<2:54:47, 1.11it/s][2025-04-28 03:17:58] (step=0099050) Train Loss: 5.7574, Train Steps/Sec: 1.12 + 90%|█████████████████████████████████████████████████████████████████████████████████████████████████████████████████▋ | 99074/110696 [30:14:55<2:52:34, 1.12it/s][2025-04-28 03:18:20] (step=0099075) Train Loss: 5.6845, Train Steps/Sec: 1.12 + 90%|█████████████████████████████████████████████████████████████████████████████████████████████████████████████████▋ | 99099/110696 [30:15:17<2:51:29, 1.13it/s][2025-04-28 03:18:42] (step=0099100) Train Loss: 5.6532, Train Steps/Sec: 1.12 + 90%|█████████████████████████████████████████████████████████████████████████████████████████████████████████████████▋ | 99124/110696 [30:15:39<2:51:11, 1.13it/s][2025-04-28 03:19:05] (step=0099125) Train Loss: 5.6318, Train Steps/Sec: 1.12 + 90%|█████████████████████████████████████████████████████████████████████████████████████████████████████████████████▊ | 99149/110696 [30:16:02<2:50:09, 1.13it/s][2025-04-28 03:19:27] (step=0099150) Train Loss: 5.7035, Train Steps/Sec: 1.12 + 90%|█████████████████████████████████████████████████████████████████████████████████████████████████████████████████▊ | 99174/110696 [30:16:24<2:49:59, 1.13it/s][2025-04-28 03:19:49] (step=0099175) Train Loss: 5.6758, Train Steps/Sec: 1.12 + 90%|█████████████████████████████████████████████████████████████████████████████████████████████████████████████████▊ | 99199/110696 [30:16:46<2:48:31, 1.14it/s][2025-04-28 03:20:12] (step=0099200) Train Loss: 5.6373, Train Steps/Sec: 1.12 + 90%|█████████████████████████████████████████████████████████████████████████████████████████████████████████████████▊ | 99224/110696 [30:17:09<2:53:40, 1.10it/s][2025-04-28 03:20:34] (step=0099225) Train Loss: 5.7275, Train Steps/Sec: 1.12 + 90%|█████████████████████████████████████████████████████████████████████████████████████████████████████████████████▊ | 99249/110696 [30:17:31<2:51:11, 1.11it/s][2025-04-28 03:20:56] (step=0099250) Train Loss: 5.6223, Train Steps/Sec: 1.12 + 90%|█████████████████████████████████████████████████████████████████████████████████████████████████████████████████▉ | 99274/110696 [30:17:53<2:49:46, 1.12it/s][2025-04-28 03:21:18] (step=0099275) Train Loss: 5.6952, Train Steps/Sec: 1.12 + 90%|█████████████████████████████████████████████████████████████████████████████████████████████████████████████████▉ | 99299/110696 [30:18:15<2:49:20, 1.12it/s][2025-04-28 03:21:41] (step=0099300) Train Loss: 5.7340, Train Steps/Sec: 1.12 + 90%|█████████████████████████████████████████████████████████████████████████████████████████████████████████████████▉ | 99324/110696 [30:18:38<2:48:22, 1.13it/s][2025-04-28 03:22:03] (step=0099325) Train Loss: 5.6813, Train Steps/Sec: 1.12 + 90%|█████████████████████████████████████████████████████████████████████████████████████████████████████████████████▉ | 99349/110696 [30:19:00<2:47:21, 1.13it/s][2025-04-28 03:22:25] (step=0099350) Train Loss: 5.6191, Train Steps/Sec: 1.12 + 90%|██████████████████████████████████████████████████████████████████████████████████████████████████████████████████ | 99374/110696 [30:19:22<2:46:31, 1.13it/s][2025-04-28 03:22:48] (step=0099375) Train Loss: 5.6968, Train Steps/Sec: 1.12 + 90%|██████████████████████████████████████████████████████████████████████████████████████████████████████████████████ | 99399/110696 [30:19:45<2:45:43, 1.14it/s][2025-04-28 03:23:10] (step=0099400) Train Loss: 5.6797, Train Steps/Sec: 1.12 + 90%|██████████████████████████████████████████████████████████████████████████████████████████████████████████████████ | 99424/110696 [30:20:07<2:50:57, 1.10it/s][2025-04-28 03:23:32] (step=0099425) Train Loss: 5.6799, Train Steps/Sec: 1.12 + 90%|██████████████████████████████████████████████████████████████████████████████████████████████████████████████████ | 99449/110696 [30:20:29<2:48:06, 1.12it/s][2025-04-28 03:23:54] (step=0099450) Train Loss: 5.6238, Train Steps/Sec: 1.12 + 90%|██████████████████████████████████████████████████████████████████████████████████████████████████████████████████▏ | 99474/110696 [30:20:51<2:46:41, 1.12it/s][2025-04-28 03:24:17] (step=0099475) Train Loss: 5.6483, Train Steps/Sec: 1.12 + 90%|██████████████████████████████████████████████████████████████████████████████████████████████████████████████████▏ | 99499/110696 [30:21:14<2:45:37, 1.13it/s][2025-04-28 03:24:39] (step=0099500) Train Loss: 5.6906, Train Steps/Sec: 1.12 + 90%|██████████████████████████████████████████████████████████████████████████████████████████████████████████████████▏ | 99524/110696 [30:21:36<2:45:16, 1.13it/s][2025-04-28 03:25:01] (step=0099525) Train Loss: 5.6607, Train Steps/Sec: 1.12 + 90%|██████████████████████████████████████████████████████████████████████████████████████████████████████████████████▏ | 99549/110696 [30:21:58<2:44:52, 1.13it/s][2025-04-28 03:25:24] (step=0099550) Train Loss: 5.6534, Train Steps/Sec: 1.12 + 90%|██████████████████████████████████████████████████████████████████████████████████████████████████████████████████▏ | 99574/110696 [30:22:21<2:44:22, 1.13it/s][2025-04-28 03:25:46] (step=0099575) Train Loss: 5.6461, Train Steps/Sec: 1.12 + 90%|██████████████████████████████████████████████████████████████████████████████████████████████████████████████████▎ | 99599/110696 [30:22:43<2:43:20, 1.13it/s][2025-04-28 03:26:08] (step=0099600) Train Loss: 5.6843, Train Steps/Sec: 1.12 + 90%|██████████████████████████████████████████████████████████████████████████████████████████████████████████████████▎ | 99624/110696 [30:23:05<2:47:48, 1.10it/s][2025-04-28 03:26:31] (step=0099625) Train Loss: 5.6255, Train Steps/Sec: 1.13 + 90%|██████████████████████████████████████████████████████████████████████████████████████████████████████████████████▎ | 99649/110696 [30:23:27<2:45:39, 1.11it/s][2025-04-28 03:26:53] (step=0099650) Train Loss: 5.6169, Train Steps/Sec: 1.12 + 90%|██████████████████████████████████████████████████████████████████████████████████████████████████████████████████▎ | 99674/110696 [30:23:50<2:43:59, 1.12it/s][2025-04-28 03:27:15] (step=0099675) Train Loss: 5.6341, Train Steps/Sec: 1.12 + 90%|██████████████████████████████████████████████████████████████████████████████████████████████████████████████████▍ | 99699/110696 [30:24:12<2:42:31, 1.13it/s][2025-04-28 03:27:37] (step=0099700) Train Loss: 5.6921, Train Steps/Sec: 1.12 + 90%|██████████████████████████████████████████████████████████████████████████████████████████████████████████████████▍ | 99724/110696 [30:24:34<2:42:18, 1.13it/s][2025-04-28 03:28:00] (step=0099725) Train Loss: 5.7276, Train Steps/Sec: 1.12 + 90%|██████████████████████████████████████████████████████████████████████████████████████████████████████████████████▍ | 99749/110696 [30:24:57<2:41:41, 1.13it/s][2025-04-28 03:28:22] (step=0099750) Train Loss: 5.6612, Train Steps/Sec: 1.12 + 90%|██████████████████████████████████████████████████████████████████████████████████████████████████████████████████▍ | 99774/110696 [30:25:19<2:40:40, 1.13it/s][2025-04-28 03:28:44] (step=0099775) Train Loss: 5.6342, Train Steps/Sec: 1.12 + 90%|██████████████████████████████████████████████████████████████████████████████████████████████████████████████████▍ | 99799/110696 [30:25:41<2:40:21, 1.13it/s][2025-04-28 03:29:07] (step=0099800) Train Loss: 5.6309, Train Steps/Sec: 1.12 + 90%|██████████████████████████████████████████████████████████████████████████████████████████████████████████████████▌ | 99824/110696 [30:26:04<2:45:06, 1.10it/s][2025-04-28 03:29:29] (step=0099825) Train Loss: 5.7062, Train Steps/Sec: 1.12 + 90%|██████████████████████████████████████████████████████████████████████████████████████████████████████████████████▌ | 99849/110696 [30:26:26<2:42:56, 1.11it/s][2025-04-28 03:29:51] (step=0099850) Train Loss: 5.6889, Train Steps/Sec: 1.12 + 90%|██████████████████████████████████████████████████████████████████████████████████████████████████████████████████▌ | 99874/110696 [30:26:48<2:41:42, 1.12it/s][2025-04-28 03:30:14] (step=0099875) Train Loss: 5.6950, Train Steps/Sec: 1.12 + 90%|██████████████████████████████████████████████████████████████████████████████████████████████████████████████████▌ | 99899/110696 [30:27:10<2:39:58, 1.12it/s][2025-04-28 03:30:36] (step=0099900) Train Loss: 5.6346, Train Steps/Sec: 1.12 + 90%|██████████████████████████████████████████████████████████████████████████████████████████████████████████████████▋ | 99924/110696 [30:27:33<2:39:27, 1.13it/s][2025-04-28 03:30:58] (step=0099925) Train Loss: 5.6356, Train Steps/Sec: 1.12 + 90%|██████████████████████████████████████████████████████████████████████████████████████████████████████████████████▋ | 99949/110696 [30:27:55<2:39:29, 1.12it/s][2025-04-28 03:31:20] (step=0099950) Train Loss: 5.6926, Train Steps/Sec: 1.12 + 90%|██████████████████████████████████████████████████████████████████████████████████████████████████████████████████▋ | 99974/110696 [30:28:17<2:38:04, 1.13it/s][2025-04-28 03:31:43] (step=0099975) Train Loss: 5.6845, Train Steps/Sec: 1.12 + 90%|██████████████████████████████████████████████████████████████████████████████████████████████████████████████████▋ | 99999/110696 [30:28:40<2:37:02, 1.14it/s][2025-04-28 03:32:05] (step=0100000) Train Loss: 5.7195, Train Steps/Sec: 1.12 +You are using a model of type blip-2 to instantiate a model of type instructblip. This is not supported for all configurations of models and can yield errors. +[2025-04-28 03:32:05] text_config is None. Initializing the text config with default values (`OPTConfig`). +model_name_or_path:/tmp/haozhezhao/model/blip2-flan-t5-xl +image_place_holder: +tokenizer length after expend 32101 +tokenizer length before expend 32100 +100%|████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████| 5/5 [05:03<00:00, 60.69s/it] +[2025-04-28 03:38:22] Finish Eval in 100000 steps...████████████████████████████████████████████████████████████████████████████████████████████| 5/5 [05:02<00:00, 60.42s/it] +processing left-padding... +processing left-padding... +processing left-padding... +processing left-padding... +processing left-padding... +[2025-04-28 03:38:41] Saved checkpoint to checkpoint/X2I_UltraEdiit_fourmask_LLava_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_filtered_objects_4mask_1280_context/003-GPT-XL/checkpoints/0100000.pt +[2025-04-28 03:38:43] Removed old checkpoint: checkpoint/X2I_UltraEdiit_fourmask_LLava_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_filtered_objects_4mask_1280_context/003-GPT-XL/checkpoints/0098000.pt + 90%|█████████████████████████████████████████████████████████████████████████████████████████████████████████████████▊ | 100024/110696 [30:35:40<2:45:59, 1.07it/s][2025-04-28 03:39:06] (step=0100025) Train Loss: 5.6511, Train Steps/Sec: 0.06 + 90%|█████████████████████████████████████████████████████████████████████████████████████████████████████████████████▉ | 100049/110696 [30:36:03<2:39:02, 1.12it/s][2025-04-28 03:39:28] (step=0100050) Train Loss: 5.6591, Train Steps/Sec: 1.12 + 90%|█████████████████████████████████████████████████████████████████████████████████████████████████████████████████▉ | 100074/110696 [30:36:25<2:38:02, 1.12it/s][2025-04-28 03:39:50] (step=0100075) Train Loss: 5.6294, Train Steps/Sec: 1.12 + 90%|█████████████████████████████████████████████████████████████████████████████████████████████████████████████████▉ | 100099/110696 [30:36:47<2:37:00, 1.12it/s][2025-04-28 03:40:12] (step=0100100) Train Loss: 5.7044, Train Steps/Sec: 1.12 + 90%|█████████████████████████████████████████████████████████████████████████████████████████████████████████████████▉ | 100124/110696 [30:37:09<2:36:26, 1.13it/s][2025-04-28 03:40:35] (step=0100125) Train Loss: 5.7240, Train Steps/Sec: 1.12 + 90%|█████████████████████████████████████████████████████████████████████████████████████████████████████████████████▉ | 100149/110696 [30:37:32<2:35:19, 1.13it/s][2025-04-28 03:40:57] (step=0100150) Train Loss: 5.6056, Train Steps/Sec: 1.12 + 90%|██████████████████████████████████████████████████████████████████████████████████████████████████████████████████ | 100174/110696 [30:37:54<2:35:12, 1.13it/s][2025-04-28 03:41:19] (step=0100175) Train Loss: 5.6583, Train Steps/Sec: 1.12 + 91%|██████████████████████████████████████████████████████████████████████████████████████████████████████████████████ | 100199/110696 [30:38:16<2:33:50, 1.14it/s][2025-04-28 03:41:42] (step=0100200) Train Loss: 5.7007, Train Steps/Sec: 1.12 + 91%|██████████████████████████████████████████████████████████████████████████████████████████████████████████████████ | 100224/110696 [30:38:38<2:39:05, 1.10it/s][2025-04-28 03:42:04] (step=0100225) Train Loss: 5.6820, Train Steps/Sec: 1.12 + 91%|██████████████████████████████████████████████████████████████████████████████████████████████████████████████████ | 100249/110696 [30:39:01<2:36:08, 1.12it/s][2025-04-28 03:42:26] (step=0100250) Train Loss: 5.6584, Train Steps/Sec: 1.12 + 91%|██████████████████████████████████████████████████████████████████████████████████████████████████████████████████▏ | 100274/110696 [30:39:23<2:35:09, 1.12it/s][2025-04-28 03:42:48] (step=0100275) Train Loss: 5.7000, Train Steps/Sec: 1.13 + 91%|██████████████████████████████████████████████████████████████████████████████████████████████████████████████████▏ | 100299/110696 [30:39:45<2:34:05, 1.12it/s][2025-04-28 03:43:11] (step=0100300) Train Loss: 5.7081, Train Steps/Sec: 1.12 + 91%|██████████████████████████████████████████████████████████████████████████████████████████████████████████████████▏ | 100324/110696 [30:40:07<2:33:16, 1.13it/s][2025-04-28 03:43:33] (step=0100325) Train Loss: 5.6880, Train Steps/Sec: 1.12 + 91%|██████████████████████████████████████████████████████████████████████████████████████████████████████████████████▏ | 100349/110696 [30:40:30<2:32:45, 1.13it/s][2025-04-28 03:43:55] (step=0100350) Train Loss: 5.6872, Train Steps/Sec: 1.12 + 91%|██████████████████████████████████████████████████████████████████████████████████████████████████████████████████▎ | 100374/110696 [30:40:52<2:32:17, 1.13it/s][2025-04-28 03:44:17] (step=0100375) Train Loss: 5.6555, Train Steps/Sec: 1.12 + 91%|██████████████████████████████████████████████████████████████████████████████████████████████████████████████████▎ | 100399/110696 [30:41:14<2:31:02, 1.14it/s][2025-04-28 03:44:40] (step=0100400) Train Loss: 5.6029, Train Steps/Sec: 1.12 + 91%|██████████████████████████████████████████████████████████████████████████████████████████████████████████████████▎ | 100424/110696 [30:41:37<2:35:37, 1.10it/s][2025-04-28 03:45:02] (step=0100425) Train Loss: 5.6558, Train Steps/Sec: 1.13 + 91%|██████████████████████████████████████████████████████████████████████████████████████████████████████████████████▎ | 100449/110696 [30:41:59<2:33:13, 1.11it/s][2025-04-28 03:45:24] (step=0100450) Train Loss: 5.7090, Train Steps/Sec: 1.12 + 91%|██████████████████████████████████████████████████████████████████████████████████████████████████████████████████▎ | 100474/110696 [30:42:21<2:32:14, 1.12it/s][2025-04-28 03:45:46] (step=0100475) Train Loss: 5.6122, Train Steps/Sec: 1.12 + 91%|██████████████████████████████████████████████████████████████████████████████████████████████████████████████████▍ | 100499/110696 [30:42:43<2:30:53, 1.13it/s][2025-04-28 03:46:09] (step=0100500) Train Loss: 5.5975, Train Steps/Sec: 1.12 + 91%|██████████████████████████████████████████████████████████████████████████████████████████████████████████████████▍ | 100524/110696 [30:43:06<2:42:40, 1.04it/s][2025-04-28 03:46:31] (step=0100525) Train Loss: 5.7055, Train Steps/Sec: 1.11 + 91%|██████████████████████████████████████████████████████████████████████████████████████████████████████████████████▍ | 100549/110696 [30:43:28<2:29:43, 1.13it/s][2025-04-28 03:46:54] (step=0100550) Train Loss: 5.6064, Train Steps/Sec: 1.12 + 91%|██████████████████████████████████████████████████████████████████████████████████████████████████████████████████▍ | 100574/110696 [30:43:51<2:29:24, 1.13it/s][2025-04-28 03:47:16] (step=0100575) Train Loss: 5.7554, Train Steps/Sec: 1.12 + 91%|██████████████████████████████████████████████████████████████████████████████████████████████████████████████████▌ | 100599/110696 [30:44:17<2:41:23, 1.04it/s][2025-04-28 03:47:43] (step=0100600) Train Loss: 5.6328, Train Steps/Sec: 0.93 + 91%|██████████████████████████████████████████████████████████████████████████████████████████████████████████████████▌ | 100624/110696 [30:44:40<2:33:02, 1.10it/s][2025-04-28 03:48:05] (step=0100625) Train Loss: 5.6381, Train Steps/Sec: 1.12 + 91%|██████████████████████████████████████████████████████████████████████████████████████████████████████████████████▌ | 100649/110696 [30:45:02<2:30:39, 1.11it/s][2025-04-28 03:48:27] (step=0100650) Train Loss: 5.6511, Train Steps/Sec: 1.12 + 91%|██████████████████████████████████████████████████████████████████████████████████████████████████████████████████▌ | 100674/110696 [30:45:24<2:29:10, 1.12it/s][2025-04-28 03:48:50] (step=0100675) Train Loss: 5.6411, Train Steps/Sec: 1.12 + 91%|██████████████████████████████████████████████████████████████████████████████████████████████████████████████████▌ | 100699/110696 [30:45:47<2:28:27, 1.12it/s][2025-04-28 03:49:12] (step=0100700) Train Loss: 5.6792, Train Steps/Sec: 1.12 + 91%|██████████████████████████████████████████████████████████████████████████████████████████████████████████████████▋ | 100724/110696 [30:46:09<2:27:22, 1.13it/s][2025-04-28 03:49:34] (step=0100725) Train Loss: 5.6769, Train Steps/Sec: 1.12 + 91%|██████████████████████████████████████████████████████████████████████████████████████████████████████████████████▋ | 100749/110696 [30:46:31<2:27:21, 1.13it/s][2025-04-28 03:49:57] (step=0100750) Train Loss: 5.6729, Train Steps/Sec: 1.12 + 91%|██████████████████████████████████████████████████████████████████████████████████████████████████████████████████▋ | 100774/110696 [30:46:54<2:25:53, 1.13it/s][2025-04-28 03:50:19] (step=0100775) Train Loss: 5.6243, Train Steps/Sec: 1.12 + 91%|██████████████████████████████████████████████████████████████████████████████████████████████████████████████████▋ | 100799/110696 [30:47:16<2:25:46, 1.13it/s][2025-04-28 03:50:41] (step=0100800) Train Loss: 5.6476, Train Steps/Sec: 1.12 + 91%|██████████████████████████████████████████████████████████████████████████████████████████████████████████████████▊ | 100824/110696 [30:47:38<2:29:11, 1.10it/s][2025-04-28 03:51:03] (step=0100825) Train Loss: 5.6813, Train Steps/Sec: 1.12 + 91%|██████████████████████████████████████████████████████████████████████████████████████████████████████████████████▊ | 100849/110696 [30:48:00<2:27:31, 1.11it/s][2025-04-28 03:51:26] (step=0100850) Train Loss: 5.7674, Train Steps/Sec: 1.12 + 91%|██████████████████████████████████████████████████████████████████████████████████████████████████████████████████▊ | 100874/110696 [30:48:23<2:26:33, 1.12it/s][2025-04-28 03:51:48] (step=0100875) Train Loss: 5.6877, Train Steps/Sec: 1.12 + 91%|██████████████████████████████████████████████████████████████████████████████████████████████████████████████████▊ | 100899/110696 [30:48:45<2:25:13, 1.12it/s][2025-04-28 03:52:10] (step=0100900) Train Loss: 5.7132, Train Steps/Sec: 1.12 + 91%|██████████████████████████████████████████████████████████████████████████████████████████████████████████████████▉ | 100924/110696 [30:49:12<2:33:52, 1.06it/s][2025-04-28 03:52:37] (step=0100925) Train Loss: 5.6587, Train Steps/Sec: 0.92 + 91%|██████████████████████████████████████████████████████████████████████████████████████████████████████████████████▉ | 100949/110696 [30:49:34<2:23:30, 1.13it/s][2025-04-28 03:53:00] (step=0100950) Train Loss: 5.6957, Train Steps/Sec: 1.12 + 91%|██████████████████████████████████████████████████████████████████████████████████████████████████████████████████▉ | 100974/110696 [30:49:56<2:22:37, 1.14it/s][2025-04-28 03:53:22] (step=0100975) Train Loss: 5.6311, Train Steps/Sec: 1.13 + 91%|██████████████████████████████████████████████████████████████████████████████████████████████████████████████████▉ | 100999/110696 [30:50:19<2:22:41, 1.13it/s][2025-04-28 03:53:44] (step=0101000) Train Loss: 5.7214, Train Steps/Sec: 1.12 + 91%|██████████████████████████████████████████████████████████████████████████████████████████████████████████████████▉ | 101024/110696 [30:50:41<2:27:15, 1.09it/s][2025-04-28 03:54:07] (step=0101025) Train Loss: 5.7446, Train Steps/Sec: 1.12 + 91%|███████████████████████████████████████████████████████████████████████████████████████████████████████████████████ | 101049/110696 [30:51:03<2:24:48, 1.11it/s][2025-04-28 03:54:29] (step=0101050) Train Loss: 5.6871, Train Steps/Sec: 1.12 + 91%|███████████████████████████████████████████████████████████████████████████████████████████████████████████████████ | 101074/110696 [30:51:26<2:23:20, 1.12it/s][2025-04-28 03:54:51] (step=0101075) Train Loss: 5.7434, Train Steps/Sec: 1.12 + 91%|███████████████████████████████████████████████████████████████████████████████████████████████████████████████████ | 101099/110696 [30:51:48<2:21:50, 1.13it/s][2025-04-28 03:55:13] (step=0101100) Train Loss: 5.6773, Train Steps/Sec: 1.12 + 91%|███████████████████████████████████████████████████████████████████████████████████████████████████████████████████ | 101124/110696 [30:52:10<2:21:47, 1.13it/s][2025-04-28 03:55:36] (step=0101125) Train Loss: 5.6628, Train Steps/Sec: 1.12 + 91%|███████████████████████████████████████████████████████████████████████████████████████████████████████████████████▏ | 101149/110696 [30:52:33<2:20:24, 1.13it/s][2025-04-28 03:55:58] (step=0101150) Train Loss: 5.7078, Train Steps/Sec: 1.12 + 91%|███████████████████████████████████████████████████████████████████████████████████████████████████████████████████▏ | 101174/110696 [30:52:55<2:20:33, 1.13it/s][2025-04-28 03:56:20] (step=0101175) Train Loss: 5.6605, Train Steps/Sec: 1.12 + 91%|███████████████████████████████████████████████████████████████████████████████████████████████████████████████████▏ | 101199/110696 [30:53:17<2:19:30, 1.13it/s][2025-04-28 03:56:43] (step=0101200) Train Loss: 5.5937, Train Steps/Sec: 1.12 + 91%|███████████████████████████████████████████████████████████████████████████████████████████████████████████████████▏ | 101224/110696 [30:53:40<2:24:34, 1.09it/s][2025-04-28 03:57:05] (step=0101225) Train Loss: 5.7106, Train Steps/Sec: 1.12 + 91%|███████████████████████████████████████████████████████████████████████████████████████████████████████████████████▏ | 101249/110696 [30:54:02<2:21:19, 1.11it/s][2025-04-28 03:57:27] (step=0101250) Train Loss: 5.6526, Train Steps/Sec: 1.12 + 91%|███████████████████████████████████████████████████████████████████████████████████████████████████████████████████▎ | 101274/110696 [30:54:24<2:20:12, 1.12it/s][2025-04-28 03:57:49] (step=0101275) Train Loss: 5.6474, Train Steps/Sec: 1.12 + 92%|███████████████████████████████████████████████████████████████████████████████████████████████████████████████████▎ | 101299/110696 [30:54:46<2:18:51, 1.13it/s][2025-04-28 03:58:12] (step=0101300) Train Loss: 5.6524, Train Steps/Sec: 1.12 + 92%|███████████████████████████████████████████████████████████████████████████████████████████████████████████████████▎ | 101324/110696 [30:55:09<2:18:44, 1.13it/s][2025-04-28 03:58:34] (step=0101325) Train Loss: 5.6013, Train Steps/Sec: 1.12 + 92%|███████████████████████████████████████████████████████████████████████████████████████████████████████████████████▎ | 101349/110696 [30:55:31<2:17:53, 1.13it/s][2025-04-28 03:58:56] (step=0101350) Train Loss: 5.6242, Train Steps/Sec: 1.12 + 92%|███████████████████████████████████████████████████████████████████████████████████████████████████████████████████▍ | 101374/110696 [30:55:53<2:17:22, 1.13it/s][2025-04-28 03:59:19] (step=0101375) Train Loss: 5.6229, Train Steps/Sec: 1.12 + 92%|███████████████████████████████████████████████████████████████████████████████████████████████████████████████████▍ | 101399/110696 [30:56:16<2:17:24, 1.13it/s][2025-04-28 03:59:41] (step=0101400) Train Loss: 5.6965, Train Steps/Sec: 1.11 + 92%|███████████████████████████████████████████████████████████████████████████████████████████████████████████████████▍ | 101424/110696 [30:56:38<2:20:16, 1.10it/s][2025-04-28 04:00:03] (step=0101425) Train Loss: 5.6164, Train Steps/Sec: 1.12 + 92%|███████████████████████████████████████████████████████████████████████████████████████████████████████████████████▍ | 101449/110696 [30:57:00<2:18:33, 1.11it/s][2025-04-28 04:00:26] (step=0101450) Train Loss: 5.7047, Train Steps/Sec: 1.12 + 92%|███████████████████████████████████████████████████████████████████████████████████████████████████████████████████▌ | 101474/110696 [30:57:22<2:17:06, 1.12it/s][2025-04-28 04:00:48] (step=0101475) Train Loss: 5.6603, Train Steps/Sec: 1.12 + 92%|███████████████████████████████████████████████████████████████████████████████████████████████████████████████████▌ | 101499/110696 [30:57:45<2:15:35, 1.13it/s][2025-04-28 04:01:10] (step=0101500) Train Loss: 5.6765, Train Steps/Sec: 1.13 + 92%|███████████████████████████████████████████████████████████████████████████████████████████████████████████████████▌ | 101524/110696 [30:58:07<2:15:31, 1.13it/s][2025-04-28 04:01:32] (step=0101525) Train Loss: 5.6951, Train Steps/Sec: 1.12 + 92%|███████████████████████████████████████████████████████████████████████████████████████████████████████████████████▌ | 101549/110696 [30:58:29<2:15:27, 1.13it/s][2025-04-28 04:01:55] (step=0101550) Train Loss: 5.6594, Train Steps/Sec: 1.12 + 92%|███████████████████████████████████████████████████████████████████████████████████████████████████████████████████▌ | 101574/110696 [30:58:52<2:14:30, 1.13it/s][2025-04-28 04:02:17] (step=0101575) Train Loss: 5.6170, Train Steps/Sec: 1.12 + 92%|███████████████████████████████████████████████████████████████████████████████████████████████████████████████████▋ | 101599/110696 [30:59:14<2:13:25, 1.14it/s][2025-04-28 04:02:39] (step=0101600) Train Loss: 5.6542, Train Steps/Sec: 1.12 + 92%|███████████████████████████████████████████████████████████████████████████████████████████████████████████████████▋ | 101624/110696 [30:59:36<2:17:49, 1.10it/s][2025-04-28 04:03:02] (step=0101625) Train Loss: 5.6737, Train Steps/Sec: 1.12 + 92%|███████████████████████████████████████████████████████████████████████████████████████████████████████████████████▋ | 101649/110696 [30:59:59<2:15:37, 1.11it/s][2025-04-28 04:03:24] (step=0101650) Train Loss: 5.6582, Train Steps/Sec: 1.12 + 92%|███████████████████████████████████████████████████████████████████████████████████████████████████████████████████▋ | 101674/110696 [31:00:21<2:14:10, 1.12it/s][2025-04-28 04:03:46] (step=0101675) Train Loss: 5.6690, Train Steps/Sec: 1.12 + 92%|███████████████████████████████████████████████████████████████████████████████████████████████████████████████████▊ | 101699/110696 [31:00:43<2:13:25, 1.12it/s][2025-04-28 04:04:08] (step=0101700) Train Loss: 5.6396, Train Steps/Sec: 1.12 + 92%|███████████████████████████████████████████████████████████████████████████████████████████████████████████████████▊ | 101724/110696 [31:01:05<2:12:43, 1.13it/s][2025-04-28 04:04:31] (step=0101725) Train Loss: 5.6636, Train Steps/Sec: 1.12 + 92%|███████████████████████████████████████████████████████████████████████████████████████████████████████████████████▊ | 101749/110696 [31:01:28<2:11:44, 1.13it/s][2025-04-28 04:04:53] (step=0101750) Train Loss: 5.6638, Train Steps/Sec: 1.12 + 92%|███████████████████████████████████████████████████████████████████████████████████████████████████████████████████▊ | 101774/110696 [31:01:50<2:12:05, 1.13it/s][2025-04-28 04:05:15] (step=0101775) Train Loss: 5.6327, Train Steps/Sec: 1.12 + 92%|███████████████████████████████████████████████████████████████████████████████████████████████████████████████████▊ | 101799/110696 [31:02:12<2:11:00, 1.13it/s][2025-04-28 04:05:38] (step=0101800) Train Loss: 5.6865, Train Steps/Sec: 1.12 + 92%|███████████████████████████████████████████████████████████████████████████████████████████████████████████████████▉ | 101824/110696 [31:02:35<2:14:35, 1.10it/s][2025-04-28 04:06:00] (step=0101825) Train Loss: 5.5983, Train Steps/Sec: 1.12 + 92%|███████████████████████████████████████████████████████████████████████████████████████████████████████████████████▉ | 101849/110696 [31:02:57<2:14:14, 1.10it/s][2025-04-28 04:06:22] (step=0101850) Train Loss: 5.6999, Train Steps/Sec: 1.12 + 92%|███████████████████████████████████████████████████████████████████████████████████████████████████████████████████▉ | 101874/110696 [31:03:19<2:11:28, 1.12it/s][2025-04-28 04:06:45] (step=0101875) Train Loss: 5.6517, Train Steps/Sec: 1.12 + 92%|███████████████████████████████████████████████████████████████████████████████████████████████████████████████████▉ | 101899/110696 [31:03:42<2:10:27, 1.12it/s][2025-04-28 04:07:07] (step=0101900) Train Loss: 5.6498, Train Steps/Sec: 1.12 + 92%|████████████████████████████████████████████████████████████████████████████████████████████████████████████████████ | 101924/110696 [31:04:04<2:09:56, 1.13it/s][2025-04-28 04:07:29] (step=0101925) Train Loss: 5.7594, Train Steps/Sec: 1.12 + 92%|████████████████████████████████████████████████████████████████████████████████████████████████████████████████████ | 101949/110696 [31:04:34<2:10:00, 1.12it/s][2025-04-28 04:08:00] (step=0101950) Train Loss: 5.6108, Train Steps/Sec: 0.82 + 92%|████████████████████████████████████████████████████████████████████████████████████████████████████████████████████ | 101974/110696 [31:05:06<4:47:59, 1.98s/it][2025-04-28 04:08:32] (step=0101975) Train Loss: 5.6536, Train Steps/Sec: 0.79 + 92%|████████████████████████████████████████████████████████████████████████████████████████████████████████████████████ | 101999/110696 [31:05:28<2:07:39, 1.14it/s][2025-04-28 04:08:54] (step=0102000) Train Loss: 5.6247, Train Steps/Sec: 1.12 +You are using a model of type blip-2 to instantiate a model of type instructblip. This is not supported for all configurations of models and can yield errors. +[2025-04-28 04:08:54] text_config is None. Initializing the text config with default values (`OPTConfig`). +model_name_or_path:/tmp/haozhezhao/model/blip2-flan-t5-xl +image_place_holder: +tokenizer length after expend 32101 +tokenizer length before expend 32100 +100%|████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████| 5/5 [05:03<00:00, 60.76s/it] +[2025-04-28 04:15:12] Finish Eval in 102000 steps...████████████████████████████████████████████████████████████████████████████████████████████| 5/5 [05:03<00:00, 60.46s/it] +processing left-padding... +processing left-padding... +processing left-padding... +processing left-padding... +processing left-padding... +[2025-04-28 04:15:31] Saved checkpoint to checkpoint/X2I_UltraEdiit_fourmask_LLava_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_filtered_objects_4mask_1280_context/003-GPT-XL/checkpoints/0102000.pt +[2025-04-28 04:15:33] Removed old checkpoint: checkpoint/X2I_UltraEdiit_fourmask_LLava_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_filtered_objects_4mask_1280_context/003-GPT-XL/checkpoints/0100000.pt + 92%|████████████████████████████████████████████████████████████████████████████████████████████████████████████████████▏ | 102024/110696 [31:12:30<2:14:54, 1.07it/s][2025-04-28 04:15:56] (step=0102025) Train Loss: 5.6695, Train Steps/Sec: 0.06 + 92%|████████████████████████████████████████████████████████████████████████████████████████████████████████████████████▏ | 102049/110696 [31:12:53<2:09:32, 1.11it/s][2025-04-28 04:16:18] (step=0102050) Train Loss: 5.7094, Train Steps/Sec: 1.12 + 92%|████████████████████████████████████████████████████████████████████████████████████████████████████████████████████▏ | 102074/110696 [31:13:15<2:08:03, 1.12it/s][2025-04-28 04:16:40] (step=0102075) Train Loss: 5.6780, Train Steps/Sec: 1.12 + 92%|████████████████████████████████████████████████████████████████████████████████████████████████████████████████████▏ | 102099/110696 [31:13:37<2:07:08, 1.13it/s][2025-04-28 04:17:02] (step=0102100) Train Loss: 5.6988, Train Steps/Sec: 1.12 + 92%|████████████████████████████████████████████████████████████████████████████████████████████████████████████████████▏ | 102124/110696 [31:13:59<2:06:19, 1.13it/s][2025-04-28 04:17:25] (step=0102125) Train Loss: 5.6840, Train Steps/Sec: 1.13 + 92%|████████████████████████████████████████████████████████████████████████████████████████████████████████████████████▎ | 102149/110696 [31:14:21<2:05:52, 1.13it/s][2025-04-28 04:17:47] (step=0102150) Train Loss: 5.6552, Train Steps/Sec: 1.12 + 92%|████████████████████████████████████████████████████████████████████████████████████████████████████████████████████▎ | 102174/110696 [31:14:44<2:04:57, 1.14it/s][2025-04-28 04:18:09] (step=0102175) Train Loss: 5.6527, Train Steps/Sec: 1.12 + 92%|████████████████████████████████████████████████████████████████████████████████████████████████████████████████████▎ | 102199/110696 [31:15:06<2:04:31, 1.14it/s][2025-04-28 04:18:31] (step=0102200) Train Loss: 5.7136, Train Steps/Sec: 1.12 + 92%|████████████████████████████████████████████████████████████████████████████████████████████████████████████████████▎ | 102224/110696 [31:15:28<2:08:28, 1.10it/s][2025-04-28 04:18:54] (step=0102225) Train Loss: 5.7066, Train Steps/Sec: 1.12 + 92%|████████████████████████████████████████████████████████████████████████████████████████████████████████████████████▍ | 102249/110696 [31:15:51<2:06:38, 1.11it/s][2025-04-28 04:19:16] (step=0102250) Train Loss: 5.6224, Train Steps/Sec: 1.12 + 92%|████████████████████████████████████████████████████████████████████████████████████████████████████████████████████▍ | 102274/110696 [31:16:13<2:05:09, 1.12it/s][2025-04-28 04:19:38] (step=0102275) Train Loss: 5.6394, Train Steps/Sec: 1.12 + 92%|████████████████████████████████████████████████████████████████████████████████████████████████████████████████████▍ | 102299/110696 [31:16:35<2:04:08, 1.13it/s][2025-04-28 04:20:00] (step=0102300) Train Loss: 5.6300, Train Steps/Sec: 1.12 + 92%|████████████████████████████████████████████████████████████████████████████████████████████████████████████████████▍ | 102324/110696 [31:16:57<2:03:32, 1.13it/s][2025-04-28 04:20:23] (step=0102325) Train Loss: 5.6929, Train Steps/Sec: 1.13 + 92%|████████████████████████████████████████████████████████████████████████████████████████████████████████████████████▍ | 102349/110696 [31:17:20<2:03:14, 1.13it/s][2025-04-28 04:20:45] (step=0102350) Train Loss: 5.5800, Train Steps/Sec: 1.12 + 92%|████████████████████████████████████████████████████████████████████████████████████████████████████████████████████▌ | 102374/110696 [31:17:42<2:02:29, 1.13it/s][2025-04-28 04:21:07] (step=0102375) Train Loss: 5.6901, Train Steps/Sec: 1.12 + 93%|████████████████████████████████████████████████████████████████████████████████████████████████████████████████████▌ | 102399/110696 [31:18:04<2:01:52, 1.13it/s][2025-04-28 04:21:29] (step=0102400) Train Loss: 5.7106, Train Steps/Sec: 1.12 + 93%|████████████████████████████████████████████████████████████████████████████████████████████████████████████████████▌ | 102424/110696 [31:18:27<2:16:52, 1.01it/s][2025-04-28 04:21:52] (step=0102425) Train Loss: 5.6833, Train Steps/Sec: 1.10 + 93%|████████████████████████████████████████████████████████████████████████████████████████████████████████████████████▌ | 102449/110696 [31:18:49<2:04:06, 1.11it/s][2025-04-28 04:22:15] (step=0102450) Train Loss: 5.6636, Train Steps/Sec: 1.12 + 93%|████████████████████████████████████████████████████████████████████████████████████████████████████████████████████▋ | 102474/110696 [31:19:11<2:02:09, 1.12it/s][2025-04-28 04:22:37] (step=0102475) Train Loss: 5.6992, Train Steps/Sec: 1.12 + 93%|████████████████████████████████████████████████████████████████████████████████████████████████████████████████████▋ | 102499/110696 [31:19:34<2:01:44, 1.12it/s][2025-04-28 04:22:59] (step=0102500) Train Loss: 5.6804, Train Steps/Sec: 1.12 + 93%|████████████████████████████████████████████████████████████████████████████████████████████████████████████████████▋ | 102524/110696 [31:19:56<2:00:46, 1.13it/s][2025-04-28 04:23:21] (step=0102525) Train Loss: 5.5894, Train Steps/Sec: 1.12 + 93%|████████████████████████████████████████████████████████████████████████████████████████████████████████████████████▋ | 102549/110696 [31:20:18<2:00:22, 1.13it/s][2025-04-28 04:23:44] (step=0102550) Train Loss: 5.7648, Train Steps/Sec: 1.12 + 93%|████████████████████████████████████████████████████████████████████████████████████████████████████████████████████▊ | 102574/110696 [31:20:41<1:59:41, 1.13it/s][2025-04-28 04:24:06] (step=0102575) Train Loss: 5.6593, Train Steps/Sec: 1.12 + 93%|████████████████████████████████████████████████████████████████████████████████████████████████████████████████████▊ | 102599/110696 [31:21:03<1:59:28, 1.13it/s][2025-04-28 04:24:28] (step=0102600) Train Loss: 5.6976, Train Steps/Sec: 1.12 + 93%|████████████████████████████████████████████████████████████████████████████████████████████████████████████████████▊ | 102624/110696 [31:21:25<2:02:21, 1.10it/s][2025-04-28 04:24:51] (step=0102625) Train Loss: 5.6759, Train Steps/Sec: 1.12 + 93%|████████████████████████████████████████████████████████████████████████████████████████████████████████████████████▊ | 102649/110696 [31:21:52<3:08:35, 1.41s/it][2025-04-28 04:25:18] (step=0102650) Train Loss: 5.7008, Train Steps/Sec: 0.92 + 93%|████████████████████████████████████████████████████████████████████████████████████████████████████████████████████▊ | 102674/110696 [31:22:15<1:59:10, 1.12it/s][2025-04-28 04:25:40] (step=0102675) Train Loss: 5.6179, Train Steps/Sec: 1.12 + 93%|████████████████████████████████████████████████████████████████████████████████████████████████████████████████████▉ | 102699/110696 [31:22:37<1:58:27, 1.13it/s][2025-04-28 04:26:02] (step=0102700) Train Loss: 5.6266, Train Steps/Sec: 1.13 + 93%|████████████████████████████████████████████████████████████████████████████████████████████████████████████████████▉ | 102724/110696 [31:22:59<1:57:49, 1.13it/s][2025-04-28 04:26:24] (step=0102725) Train Loss: 5.6885, Train Steps/Sec: 1.12 + 93%|████████████████████████████████████████████████████████████████████████████████████████████████████████████████████▉ | 102749/110696 [31:23:21<1:57:06, 1.13it/s][2025-04-28 04:26:47] (step=0102750) Train Loss: 5.7082, Train Steps/Sec: 1.13 + 93%|████████████████████████████████████████████████████████████████████████████████████████████████████████████████████▉ | 102774/110696 [31:23:44<1:56:36, 1.13it/s][2025-04-28 04:27:09] (step=0102775) Train Loss: 5.6584, Train Steps/Sec: 1.12 + 93%|█████████████████████████████████████████████████████████████████████████████████████████████████████████████████████ | 102799/110696 [31:24:06<1:55:57, 1.14it/s][2025-04-28 04:27:31] (step=0102800) Train Loss: 5.6590, Train Steps/Sec: 1.12 + 93%|█████████████████████████████████████████████████████████████████████████████████████████████████████████████████████ | 102824/110696 [31:24:28<1:59:44, 1.10it/s][2025-04-28 04:27:54] (step=0102825) Train Loss: 5.6984, Train Steps/Sec: 1.12 + 93%|█████████████████████████████████████████████████████████████████████████████████████████████████████████████████████ | 102849/110696 [31:24:50<1:57:27, 1.11it/s][2025-04-28 04:28:16] (step=0102850) Train Loss: 5.6681, Train Steps/Sec: 1.12 + 93%|█████████████████████████████████████████████████████████████████████████████████████████████████████████████████████ | 102874/110696 [31:25:13<1:56:42, 1.12it/s][2025-04-28 04:28:38] (step=0102875) Train Loss: 5.7023, Train Steps/Sec: 1.12 + 93%|█████████████████████████████████████████████████████████████████████████████████████████████████████████████████████▏ | 102899/110696 [31:25:35<1:55:53, 1.12it/s][2025-04-28 04:29:01] (step=0102900) Train Loss: 5.6146, Train Steps/Sec: 1.12 + 93%|█████████████████████████████████████████████████████████████████████████████████████████████████████████████████████▏ | 102924/110696 [31:25:57<1:55:17, 1.12it/s][2025-04-28 04:29:23] (step=0102925) Train Loss: 5.6445, Train Steps/Sec: 1.12 + 93%|█████████████████████████████████████████████████████████████████████████████████████████████████████████████████████▏ | 102949/110696 [31:26:20<1:54:15, 1.13it/s][2025-04-28 04:29:45] (step=0102950) Train Loss: 5.6958, Train Steps/Sec: 1.12 + 93%|█████████████████████████████████████████████████████████████████████████████████████████████████████████████████████▏ | 102974/110696 [31:26:42<1:53:44, 1.13it/s][2025-04-28 04:30:07] (step=0102975) Train Loss: 5.6946, Train Steps/Sec: 1.12 + 93%|█████████████████████████████████████████████████████████████████████████████████████████████████████████████████████▏ | 102999/110696 [31:27:04<1:53:10, 1.13it/s][2025-04-28 04:30:30] (step=0103000) Train Loss: 5.6230, Train Steps/Sec: 1.12 + 93%|█████████████████████████████████████████████████████████████████████████████████████████████████████████████████████▎ | 103024/110696 [31:27:27<1:56:15, 1.10it/s][2025-04-28 04:30:52] (step=0103025) Train Loss: 5.6614, Train Steps/Sec: 1.12 + 93%|█████████████████████████████████████████████████████████████████████████████████████████████████████████████████████▎ | 103049/110696 [31:27:49<1:54:56, 1.11it/s][2025-04-28 04:31:14] (step=0103050) Train Loss: 5.7380, Train Steps/Sec: 1.12 + 93%|█████████████████████████████████████████████████████████████████████████████████████████████████████████████████████▎ | 103074/110696 [31:28:11<1:53:19, 1.12it/s][2025-04-28 04:31:37] (step=0103075) Train Loss: 5.7132, Train Steps/Sec: 1.12 + 93%|█████████████████████████████████████████████████████████████████████████████████████████████████████████████████████▎ | 103099/110696 [31:28:34<1:52:44, 1.12it/s][2025-04-28 04:31:59] (step=0103100) Train Loss: 5.6151, Train Steps/Sec: 1.12 + 93%|█████████████████████████████████████████████████████████████████████████████████████████████████████████████████████▍ | 103124/110696 [31:28:56<1:52:26, 1.12it/s][2025-04-28 04:32:21] (step=0103125) Train Loss: 5.6593, Train Steps/Sec: 1.12 + 93%|█████████████████████████████████████████████████████████████████████████████████████████████████████████████████████▍ | 103149/110696 [31:29:18<1:51:42, 1.13it/s][2025-04-28 04:32:44] (step=0103150) Train Loss: 5.6724, Train Steps/Sec: 1.12 + 93%|█████████████████████████████████████████████████████████████████████████████████████████████████████████████████████▍ | 103174/110696 [31:29:40<1:51:08, 1.13it/s][2025-04-28 04:33:06] (step=0103175) Train Loss: 5.7030, Train Steps/Sec: 1.12 + 93%|█████████████████████████████████████████████████████████████████████████████████████████████████████████████████████▍ | 103199/110696 [31:30:03<1:49:44, 1.14it/s][2025-04-28 04:33:28] (step=0103200) Train Loss: 5.7043, Train Steps/Sec: 1.12 + 93%|█████████████████████████████████████████████████████████████████████████████████████████████████████████████████████▍ | 103224/110696 [31:30:25<1:53:04, 1.10it/s][2025-04-28 04:33:50] (step=0103225) Train Loss: 5.6568, Train Steps/Sec: 1.13 + 93%|█████████████████████████████████████████████████████████████████████████████████████████████████████████████████████▌ | 103249/110696 [31:30:47<1:51:56, 1.11it/s][2025-04-28 04:34:13] (step=0103250) Train Loss: 5.6681, Train Steps/Sec: 1.12 + 93%|█████████████████████████████████████████████████████████████████████████████████████████████████████████████████████▌ | 103274/110696 [31:31:10<1:50:19, 1.12it/s][2025-04-28 04:34:35] (step=0103275) Train Loss: 5.6422, Train Steps/Sec: 1.12 + 93%|█████████████████████████████████████████████████████████████████████████████████████████████████████████████████████▌ | 103299/110696 [31:31:32<1:49:45, 1.12it/s][2025-04-28 04:34:57] (step=0103300) Train Loss: 5.6954, Train Steps/Sec: 1.12 + 93%|█████████████████████████████████████████████████████████████████████████████████████████████████████████████████████▌ | 103324/110696 [31:31:54<1:48:59, 1.13it/s][2025-04-28 04:35:20] (step=0103325) Train Loss: 5.6809, Train Steps/Sec: 1.12 + 93%|█████████████████████████████████████████████████████████████████████████████████████████████████████████████████████▋ | 103349/110696 [31:32:16<1:48:46, 1.13it/s][2025-04-28 04:35:42] (step=0103350) Train Loss: 5.6553, Train Steps/Sec: 1.12 + 93%|█████████████████████████████████████████████████████████████████████████████████████████████████████████████████████▋ | 103374/110696 [31:32:39<1:47:50, 1.13it/s][2025-04-28 04:36:04] (step=0103375) Train Loss: 5.6475, Train Steps/Sec: 1.12 + 93%|█████████████████████████████████████████████████████████████████████████████████████████████████████████████████████▋ | 103399/110696 [31:33:01<1:47:19, 1.13it/s][2025-04-28 04:36:26] (step=0103400) Train Loss: 5.5749, Train Steps/Sec: 1.12 + 93%|█████████████████████████████████████████████████████████████████████████████████████████████████████████████████████▋ | 103424/110696 [31:33:23<1:50:23, 1.10it/s][2025-04-28 04:36:49] (step=0103425) Train Loss: 5.6366, Train Steps/Sec: 1.12 + 93%|█████████████████████████████████████████████████████████████████████████████████████████████████████████████████████▊ | 103449/110696 [31:33:46<1:48:33, 1.11it/s][2025-04-28 04:37:11] (step=0103450) Train Loss: 5.6298, Train Steps/Sec: 1.12 + 93%|█████████████████████████████████████████████████████████████████████████████████████████████████████████████████████▊ | 103474/110696 [31:34:08<1:47:44, 1.12it/s][2025-04-28 04:37:33] (step=0103475) Train Loss: 5.7032, Train Steps/Sec: 1.12 + 93%|█████████████████████████████████████████████████████████████████████████████████████████████████████████████████████▊ | 103499/110696 [31:34:30<1:47:18, 1.12it/s][2025-04-28 04:37:56] (step=0103500) Train Loss: 5.6550, Train Steps/Sec: 1.12 + 94%|█████████████████████████████████████████████████████████████████████████████████████████████████████████████████████▊ | 103524/110696 [31:34:53<1:46:10, 1.13it/s][2025-04-28 04:38:18] (step=0103525) Train Loss: 5.6697, Train Steps/Sec: 1.12 + 94%|█████████████████████████████████████████████████████████████████████████████████████████████████████████████████████▊ | 103549/110696 [31:35:15<1:45:16, 1.13it/s][2025-04-28 04:38:40] (step=0103550) Train Loss: 5.6298, Train Steps/Sec: 1.12 + 94%|█████████████████████████████████████████████████████████████████████████████████████████████████████████████████████▉ | 103574/110696 [31:35:37<1:44:45, 1.13it/s][2025-04-28 04:39:02] (step=0103575) Train Loss: 5.6551, Train Steps/Sec: 1.12 + 94%|█████████████████████████████████████████████████████████████████████████████████████████████████████████████████████▉ | 103599/110696 [31:35:59<1:44:16, 1.13it/s][2025-04-28 04:39:25] (step=0103600) Train Loss: 5.7166, Train Steps/Sec: 1.12 + 94%|█████████████████████████████████████████████████████████████████████████████████████████████████████████████████████▉ | 103624/110696 [31:36:22<1:46:57, 1.10it/s][2025-04-28 04:39:47] (step=0103625) Train Loss: 5.7148, Train Steps/Sec: 1.12 + 94%|█████████████████████████████████████████████████████████████████████████████████████████████████████████████████████▉ | 103649/110696 [31:36:44<1:45:37, 1.11it/s][2025-04-28 04:40:09] (step=0103650) Train Loss: 5.6383, Train Steps/Sec: 1.12 + 94%|██████████████████████████████████████████████████████████████████████████████████████████████████████████████████████ | 103674/110696 [31:37:06<1:44:13, 1.12it/s][2025-04-28 04:40:32] (step=0103675) Train Loss: 5.6079, Train Steps/Sec: 1.13 + 94%|██████████████████████████████████████████████████████████████████████████████████████████████████████████████████████ | 103699/110696 [31:37:28<1:43:21, 1.13it/s][2025-04-28 04:40:54] (step=0103700) Train Loss: 5.6269, Train Steps/Sec: 1.12 + 94%|██████████████████████████████████████████████████████████████████████████████████████████████████████████████████████ | 103724/110696 [31:37:51<1:43:18, 1.12it/s][2025-04-28 04:41:16] (step=0103725) Train Loss: 5.6319, Train Steps/Sec: 1.12 + 94%|██████████████████████████████████████████████████████████████████████████████████████████████████████████████████████ | 103749/110696 [31:38:13<1:43:27, 1.12it/s][2025-04-28 04:41:38] (step=0103750) Train Loss: 5.6479, Train Steps/Sec: 1.12 + 94%|██████████████████████████████████████████████████████████████████████████████████████████████████████████████████████ | 103774/110696 [31:38:35<1:41:45, 1.13it/s][2025-04-28 04:42:01] (step=0103775) Train Loss: 5.6134, Train Steps/Sec: 1.12 + 94%|██████████████████████████████████████████████████████████████████████████████████████████████████████████████████████▏ | 103799/110696 [31:38:58<1:41:25, 1.13it/s][2025-04-28 04:42:23] (step=0103800) Train Loss: 5.6975, Train Steps/Sec: 1.12 + 94%|██████████████████████████████████████████████████████████████████████████████████████████████████████████████████████▏ | 103824/110696 [31:39:20<1:44:59, 1.09it/s][2025-04-28 04:42:45] (step=0103825) Train Loss: 5.6719, Train Steps/Sec: 1.12 + 94%|██████████████████████████████████████████████████████████████████████████████████████████████████████████████████████▏ | 103849/110696 [31:39:42<1:42:35, 1.11it/s][2025-04-28 04:43:08] (step=0103850) Train Loss: 5.6731, Train Steps/Sec: 1.12 + 94%|██████████████████████████████████████████████████████████████████████████████████████████████████████████████████████▏ | 103874/110696 [31:40:05<1:41:35, 1.12it/s][2025-04-28 04:43:30] (step=0103875) Train Loss: 5.7238, Train Steps/Sec: 1.12 + 94%|██████████████████████████████████████████████████████████████████████████████████████████████████████████████████████▎ | 103899/110696 [31:40:27<1:40:33, 1.13it/s][2025-04-28 04:43:52] (step=0103900) Train Loss: 5.6306, Train Steps/Sec: 1.12 + 94%|██████████████████████████████████████████████████████████████████████████████████████████████████████████████████████▎ | 103924/110696 [31:40:49<1:39:53, 1.13it/s][2025-04-28 04:44:14] (step=0103925) Train Loss: 5.6715, Train Steps/Sec: 1.12 + 94%|██████████████████████████████████████████████████████████████████████████████████████████████████████████████████████▎ | 103949/110696 [31:41:11<1:39:40, 1.13it/s][2025-04-28 04:44:37] (step=0103950) Train Loss: 5.6783, Train Steps/Sec: 1.12 + 94%|██████████████████████████████████████████████████████████████████████████████████████████████████████████████████████▎ | 103974/110696 [31:41:34<1:38:51, 1.13it/s][2025-04-28 04:44:59] (step=0103975) Train Loss: 5.6428, Train Steps/Sec: 1.12 + 94%|██████████████████████████████████████████████████████████████████████████████████████████████████████████████████████▍ | 103999/110696 [31:41:56<1:38:17, 1.14it/s][2025-04-28 04:45:21] (step=0104000) Train Loss: 5.7248, Train Steps/Sec: 1.12 +You are using a model of type blip-2 to instantiate a model of type instructblip. This is not supported for all configurations of models and can yield errors. +[2025-04-28 04:45:21] text_config is None. Initializing the text config with default values (`OPTConfig`). +model_name_or_path:/tmp/haozhezhao/model/blip2-flan-t5-xl +image_place_holder: +tokenizer length after expend 32101 +tokenizer length before expend 32100 +100%|████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████| 5/5 [05:04<00:00, 60.97s/it] +[2025-04-28 04:51:40] Finish Eval in 104000 steps...████████████████████████████████████████████████████████████████████████████████████████████| 5/5 [05:04<00:00, 60.59s/it] +processing left-padding... +processing left-padding... +processing left-padding... +processing left-padding... +processing left-padding... +[2025-04-28 04:52:00] Saved checkpoint to checkpoint/X2I_UltraEdiit_fourmask_LLava_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_filtered_objects_4mask_1280_context/003-GPT-XL/checkpoints/0104000.pt +[2025-04-28 04:52:02] Removed old checkpoint: checkpoint/X2I_UltraEdiit_fourmask_LLava_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_filtered_objects_4mask_1280_context/003-GPT-XL/checkpoints/0102000.pt + 94%|██████████████████████████████████████████████████████████████████████████████████████████████████████████████████████▍ | 104024/110696 [31:48:59<1:43:57, 1.07it/s][2025-04-28 04:52:24] (step=0104025) Train Loss: 5.6286, Train Steps/Sec: 0.06 + 94%|██████████████████████████████████████████████████████████████████████████████████████████████████████████████████████▍ | 104049/110696 [31:49:21<1:39:24, 1.11it/s][2025-04-28 04:52:46] (step=0104050) Train Loss: 5.6815, Train Steps/Sec: 1.12 + 94%|██████████████████████████████████████████████████████████████████████████████████████████████████████████████████████▍ | 104074/110696 [31:49:43<1:38:30, 1.12it/s][2025-04-28 04:53:09] (step=0104075) Train Loss: 5.6616, Train Steps/Sec: 1.12 + 94%|██████████████████████████████████████████████████████████████████████████████████████████████████████████████████████▍ | 104099/110696 [31:50:06<1:37:41, 1.13it/s][2025-04-28 04:53:31] (step=0104100) Train Loss: 5.6505, Train Steps/Sec: 1.12 + 94%|██████████████████████████████████████████████████████████████████████████████████████████████████████████████████████▌ | 104124/110696 [31:50:28<1:36:50, 1.13it/s][2025-04-28 04:53:53] (step=0104125) Train Loss: 5.6988, Train Steps/Sec: 1.12 + 94%|██████████████████████████████████████████████████████████████████████████████████████████████████████████████████████▌ | 104149/110696 [31:50:50<1:36:32, 1.13it/s][2025-04-28 04:54:15] (step=0104150) Train Loss: 5.6185, Train Steps/Sec: 1.12 + 94%|██████████████████████████████████████████████████████████████████████████████████████████████████████████████████████▌ | 104174/110696 [31:51:12<1:35:51, 1.13it/s][2025-04-28 04:54:38] (step=0104175) Train Loss: 5.6522, Train Steps/Sec: 1.12 + 94%|██████████████████████████████████████████████████████████████████████████████████████████████████████████████████████▌ | 104199/110696 [31:51:35<1:35:56, 1.13it/s][2025-04-28 04:55:00] (step=0104200) Train Loss: 5.6767, Train Steps/Sec: 1.12 + 94%|██████████████████████████████████████████████████████████████████████████████████████████████████████████████████████▋ | 104224/110696 [31:51:57<1:37:45, 1.10it/s][2025-04-28 04:55:22] (step=0104225) Train Loss: 5.7029, Train Steps/Sec: 1.13 + 94%|██████████████████████████████████████████████████████████████████████████████████████████████████████████████████████▋ | 104249/110696 [31:52:19<1:36:01, 1.12it/s][2025-04-28 04:55:45] (step=0104250) Train Loss: 5.5823, Train Steps/Sec: 1.12 + 94%|██████████████████████████████████████████████████████████████████████████████████████████████████████████████████████▋ | 104274/110696 [31:52:41<1:35:12, 1.12it/s][2025-04-28 04:56:07] (step=0104275) Train Loss: 5.6710, Train Steps/Sec: 1.12 + 94%|██████████████████████████████████████████████████████████████████████████████████████████████████████████████████████▋ | 104299/110696 [31:53:04<1:34:31, 1.13it/s][2025-04-28 04:56:29] (step=0104300) Train Loss: 5.6820, Train Steps/Sec: 1.12 + 94%|██████████████████████████████████████████████████████████████████████████████████████████████████████████████████████▋ | 104324/110696 [31:53:26<1:34:13, 1.13it/s][2025-04-28 04:56:51] (step=0104325) Train Loss: 5.6094, Train Steps/Sec: 1.12 + 94%|██████████████████████████████████████████████████████████████████████████████████████████████████████████████████████▊ | 104349/110696 [31:53:49<1:33:33, 1.13it/s][2025-04-28 04:57:14] (step=0104350) Train Loss: 5.7068, Train Steps/Sec: 1.12 + 94%|██████████████████████████████████████████████████████████████████████████████████████████████████████████████████████▊ | 104374/110696 [31:54:11<1:33:57, 1.12it/s][2025-04-28 04:57:36] (step=0104375) Train Loss: 5.7071, Train Steps/Sec: 1.12 + 94%|██████████████████████████████████████████████████████████████████████████████████████████████████████████████████████▊ | 104399/110696 [31:54:33<1:32:33, 1.13it/s][2025-04-28 04:57:59] (step=0104400) Train Loss: 5.6844, Train Steps/Sec: 1.12 + 94%|██████████████████████████████████████████████████████████████████████████████████████████████████████████████████████▊ | 104424/110696 [31:54:56<1:35:29, 1.09it/s][2025-04-28 04:58:21] (step=0104425) Train Loss: 5.6802, Train Steps/Sec: 1.12 + 94%|██████████████████████████████████████████████████████████████████████████████████████████████████████████████████████▉ | 104449/110696 [31:55:18<1:33:34, 1.11it/s][2025-04-28 04:58:43] (step=0104450) Train Loss: 5.7649, Train Steps/Sec: 1.12 + 94%|██████████████████████████████████████████████████████████████████████████████████████████████████████████████████████▉ | 104474/110696 [31:55:40<1:32:37, 1.12it/s][2025-04-28 04:59:05] (step=0104475) Train Loss: 5.6328, Train Steps/Sec: 1.12 + 94%|██████████████████████████████████████████████████████████████████████████████████████████████████████████████████████▉ | 104499/110696 [31:56:02<1:32:00, 1.12it/s][2025-04-28 04:59:28] (step=0104500) Train Loss: 5.6158, Train Steps/Sec: 1.12 + 94%|██████████████████████████████████████████████████████████████████████████████████████████████████████████████████████▉ | 104524/110696 [31:56:25<1:31:24, 1.13it/s][2025-04-28 04:59:50] (step=0104525) Train Loss: 5.6232, Train Steps/Sec: 1.12 + 94%|███████████████████████████████████████████████████████████████████████████████████████████████████████████████████████ | 104549/110696 [31:56:47<1:30:40, 1.13it/s][2025-04-28 05:00:12] (step=0104550) Train Loss: 5.6202, Train Steps/Sec: 1.12 + 94%|███████████████████████████████████████████████████████████████████████████████████████████████████████████████████████ | 104574/110696 [31:57:09<1:30:14, 1.13it/s][2025-04-28 05:00:35] (step=0104575) Train Loss: 5.6900, Train Steps/Sec: 1.12 + 94%|███████████████████████████████████████████████████████████████████████████████████████████████████████████████████████ | 104599/110696 [31:57:32<1:29:46, 1.13it/s][2025-04-28 05:00:57] (step=0104600) Train Loss: 5.6840, Train Steps/Sec: 1.12 + 95%|███████████████████████████████████████████████████████████████████████████████████████████████████████████████████████ | 104624/110696 [31:57:54<1:32:01, 1.10it/s][2025-04-28 05:01:19] (step=0104625) Train Loss: 5.6924, Train Steps/Sec: 1.12 + 95%|███████████████████████████████████████████████████████████████████████████████████████████████████████████████████████ | 104649/110696 [31:58:16<1:30:49, 1.11it/s][2025-04-28 05:01:42] (step=0104650) Train Loss: 5.7199, Train Steps/Sec: 1.12 + 95%|███████████████████████████████████████████████████████████████████████████████████████████████████████████████████████▏ | 104674/110696 [31:58:38<1:29:28, 1.12it/s][2025-04-28 05:02:04] (step=0104675) Train Loss: 5.6982, Train Steps/Sec: 1.12 + 95%|███████████████████████████████████████████████████████████████████████████████████████████████████████████████████████▏ | 104699/110696 [31:59:01<1:28:43, 1.13it/s][2025-04-28 05:02:26] (step=0104700) Train Loss: 5.6402, Train Steps/Sec: 1.12 + 95%|███████████████████████████████████████████████████████████████████████████████████████████████████████████████████████▏ | 104724/110696 [31:59:23<1:28:18, 1.13it/s][2025-04-28 05:02:48] (step=0104725) Train Loss: 5.6147, Train Steps/Sec: 1.12 + 95%|███████████████████████████████████████████████████████████████████████████████████████████████████████████████████████▏ | 104749/110696 [31:59:45<1:28:01, 1.13it/s][2025-04-28 05:03:11] (step=0104750) Train Loss: 5.6616, Train Steps/Sec: 1.12 + 95%|███████████████████████████████████████████████████████████████████████████████████████████████████████████████████████▎ | 104774/110696 [32:00:08<1:27:22, 1.13it/s][2025-04-28 05:03:33] (step=0104775) Train Loss: 5.7037, Train Steps/Sec: 1.12 + 95%|███████████████████████████████████████████████████████████████████████████████████████████████████████████████████████▎ | 104799/110696 [32:00:30<1:26:44, 1.13it/s][2025-04-28 05:03:55] (step=0104800) Train Loss: 5.7020, Train Steps/Sec: 1.12 + 95%|███████████████████████████████████████████████████████████████████████████████████████████████████████████████████████▎ | 104824/110696 [32:00:52<1:29:24, 1.09it/s][2025-04-28 05:04:17] (step=0104825) Train Loss: 5.6442, Train Steps/Sec: 1.12 + 95%|███████████████████████████████████████████████████████████████████████████████████████████████████████████████████████▎ | 104849/110696 [32:01:14<1:27:27, 1.11it/s][2025-04-28 05:04:40] (step=0104850) Train Loss: 5.6295, Train Steps/Sec: 1.12 + 95%|███████████████████████████████████████████████████████████████████████████████████████████████████████████████████████▎ | 104874/110696 [32:01:41<1:26:39, 1.12it/s][2025-04-28 05:05:06] (step=0104875) Train Loss: 5.6307, Train Steps/Sec: 0.94 + 95%|███████████████████████████████████████████████████████████████████████████████████████████████████████████████████████▍ | 104899/110696 [32:02:03<1:25:56, 1.12it/s][2025-04-28 05:05:29] (step=0104900) Train Loss: 5.6720, Train Steps/Sec: 1.12 + 95%|███████████████████████████████████████████████████████████████████████████████████████████████████████████████████████▍ | 104924/110696 [32:02:26<1:25:02, 1.13it/s][2025-04-28 05:05:51] (step=0104925) Train Loss: 5.6253, Train Steps/Sec: 1.12 + 95%|███████████████████████████████████████████████████████████████████████████████████████████████████████████████████████▍ | 104949/110696 [32:02:48<1:24:48, 1.13it/s][2025-04-28 05:06:13] (step=0104950) Train Loss: 5.6822, Train Steps/Sec: 1.12 + 95%|███████████████████████████████████████████████████████████████████████████████████████████████████████████████████████▍ | 104974/110696 [32:03:10<1:24:01, 1.14it/s][2025-04-28 05:06:36] (step=0104975) Train Loss: 5.6185, Train Steps/Sec: 1.12 + 95%|███████████████████████████████████████████████████████████████████████████████████████████████████████████████████████▌ | 104999/110696 [32:03:33<1:23:53, 1.13it/s][2025-04-28 05:06:58] (step=0105000) Train Loss: 5.7470, Train Steps/Sec: 1.12 + 95%|███████████████████████████████████████████████████████████████████████████████████████████████████████████████████████▌ | 105024/110696 [32:03:55<1:26:09, 1.10it/s][2025-04-28 05:07:20] (step=0105025) Train Loss: 5.6415, Train Steps/Sec: 1.12 + 95%|███████████████████████████████████████████████████████████████████████████████████████████████████████████████████████▌ | 105049/110696 [32:04:17<1:24:46, 1.11it/s][2025-04-28 05:07:42] (step=0105050) Train Loss: 5.6485, Train Steps/Sec: 1.12 + 95%|███████████████████████████████████████████████████████████████████████████████████████████████████████████████████████▌ | 105074/110696 [32:04:39<1:23:42, 1.12it/s][2025-04-28 05:08:05] (step=0105075) Train Loss: 5.7041, Train Steps/Sec: 1.12 + 95%|███████████████████████████████████████████████████████████████████████████████████████████████████████████████████████▋ | 105099/110696 [32:05:02<1:22:49, 1.13it/s][2025-04-28 05:08:27] (step=0105100) Train Loss: 5.6191, Train Steps/Sec: 1.12 + 95%|███████████████████████████████████████████████████████████████████████████████████████████████████████████████████████▋ | 105124/110696 [32:05:24<1:22:28, 1.13it/s][2025-04-28 05:08:49] (step=0105125) Train Loss: 5.7136, Train Steps/Sec: 1.12 + 95%|███████████████████████████████████████████████████████████████████████████████████████████████████████████████████████▋ | 105149/110696 [32:05:46<1:21:50, 1.13it/s][2025-04-28 05:09:12] (step=0105150) Train Loss: 5.6208, Train Steps/Sec: 1.12 + 95%|███████████████████████████████████████████████████████████████████████████████████████████████████████████████████████▋ | 105174/110696 [32:06:09<1:21:28, 1.13it/s][2025-04-28 05:09:34] (step=0105175) Train Loss: 5.6519, Train Steps/Sec: 1.12 + 95%|███████████████████████████████████████████████████████████████████████████████████████████████████████████████████████▋ | 105199/110696 [32:06:36<1:31:21, 1.00it/s][2025-04-28 05:10:01] (step=0105200) Train Loss: 5.7018, Train Steps/Sec: 0.92 + 95%|███████████████████████████████████████████████████████████████████████████████████████████████████████████████████████▊ | 105224/110696 [32:06:58<1:23:10, 1.10it/s][2025-04-28 05:10:23] (step=0105225) Train Loss: 5.7527, Train Steps/Sec: 1.12 + 95%|███████████████████████████████████████████████████████████████████████████████████████████████████████████████████████▊ | 105249/110696 [32:07:20<1:21:31, 1.11it/s][2025-04-28 05:10:46] (step=0105250) Train Loss: 5.5952, Train Steps/Sec: 1.12 + 95%|███████████████████████████████████████████████████████████████████████████████████████████████████████████████████████▊ | 105274/110696 [32:07:42<1:20:36, 1.12it/s][2025-04-28 05:11:08] (step=0105275) Train Loss: 5.6289, Train Steps/Sec: 1.12 + 95%|███████████████████████████████████████████████████████████████████████████████████████████████████████████████████████▊ | 105299/110696 [32:08:05<1:20:13, 1.12it/s][2025-04-28 05:11:30] (step=0105300) Train Loss: 5.6615, Train Steps/Sec: 1.12 + 95%|███████████████████████████████████████████████████████████████████████████████████████████████████████████████████████▉ | 105324/110696 [32:08:27<1:19:12, 1.13it/s][2025-04-28 05:11:52] (step=0105325) Train Loss: 5.6667, Train Steps/Sec: 1.12 + 95%|███████████████████████████████████████████████████████████████████████████████████████████████████████████████████████▉ | 105349/110696 [32:08:49<1:18:52, 1.13it/s][2025-04-28 05:12:15] (step=0105350) Train Loss: 5.7322, Train Steps/Sec: 1.12 + 95%|███████████████████████████████████████████████████████████████████████████████████████████████████████████████████████▉ | 105374/110696 [32:09:12<1:18:37, 1.13it/s][2025-04-28 05:12:37] (step=0105375) Train Loss: 5.6177, Train Steps/Sec: 1.12 + 95%|███████████████████████████████████████████████████████████████████████████████████████████████████████████████████████▉ | 105399/110696 [32:09:34<1:17:59, 1.13it/s][2025-04-28 05:12:59] (step=0105400) Train Loss: 5.6619, Train Steps/Sec: 1.12 + 95%|███████████████████████████████████████████████████████████████████████████████████████████████████████████████████████▉ | 105424/110696 [32:09:56<1:19:53, 1.10it/s][2025-04-28 05:13:22] (step=0105425) Train Loss: 5.6395, Train Steps/Sec: 1.13 + 95%|████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████ | 105449/110696 [32:10:19<1:18:38, 1.11it/s][2025-04-28 05:13:44] (step=0105450) Train Loss: 5.6864, Train Steps/Sec: 1.12 + 95%|████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████ | 105474/110696 [32:10:41<1:18:01, 1.12it/s][2025-04-28 05:14:06] (step=0105475) Train Loss: 5.6774, Train Steps/Sec: 1.12 + 95%|████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████ | 105499/110696 [32:11:03<1:16:53, 1.13it/s][2025-04-28 05:14:28] (step=0105500) Train Loss: 5.6698, Train Steps/Sec: 1.12 + 95%|████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████ | 105524/110696 [32:11:25<1:16:20, 1.13it/s][2025-04-28 05:14:51] (step=0105525) Train Loss: 5.6273, Train Steps/Sec: 1.12 + 95%|████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████▏ | 105549/110696 [32:11:48<1:16:00, 1.13it/s][2025-04-28 05:15:13] (step=0105550) Train Loss: 5.6333, Train Steps/Sec: 1.12 + 95%|████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████▏ | 105574/110696 [32:12:10<1:15:44, 1.13it/s][2025-04-28 05:15:35] (step=0105575) Train Loss: 5.5996, Train Steps/Sec: 1.12 + 95%|████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████▏ | 105599/110696 [32:12:32<1:15:02, 1.13it/s][2025-04-28 05:15:58] (step=0105600) Train Loss: 5.6513, Train Steps/Sec: 1.12 + 95%|████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████▏ | 105624/110696 [32:12:55<1:17:00, 1.10it/s][2025-04-28 05:16:20] (step=0105625) Train Loss: 5.6297, Train Steps/Sec: 1.12 + 95%|████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████▎ | 105649/110696 [32:13:17<1:15:28, 1.11it/s][2025-04-28 05:16:42] (step=0105650) Train Loss: 5.6239, Train Steps/Sec: 1.12 + 95%|████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████▎ | 105674/110696 [32:13:39<1:14:44, 1.12it/s][2025-04-28 05:17:05] (step=0105675) Train Loss: 5.6696, Train Steps/Sec: 1.12 + 95%|████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████▎ | 105699/110696 [32:14:02<1:13:49, 1.13it/s][2025-04-28 05:17:27] (step=0105700) Train Loss: 5.7294, Train Steps/Sec: 1.12 + 96%|████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████▎ | 105724/110696 [32:14:24<1:13:37, 1.13it/s][2025-04-28 05:17:49] (step=0105725) Train Loss: 5.5749, Train Steps/Sec: 1.12 + 96%|████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████▎ | 105749/110696 [32:14:46<1:13:01, 1.13it/s][2025-04-28 05:18:12] (step=0105750) Train Loss: 5.7510, Train Steps/Sec: 1.12 + 96%|████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████▍ | 105774/110696 [32:15:09<1:12:34, 1.13it/s][2025-04-28 05:18:34] (step=0105775) Train Loss: 5.6422, Train Steps/Sec: 1.12 + 96%|████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████▍ | 105799/110696 [32:15:31<1:12:13, 1.13it/s][2025-04-28 05:18:56] (step=0105800) Train Loss: 5.7241, Train Steps/Sec: 1.12 + 96%|████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████▍ | 105824/110696 [32:15:53<1:13:57, 1.10it/s][2025-04-28 05:19:19] (step=0105825) Train Loss: 5.7239, Train Steps/Sec: 1.12 + 96%|████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████▍ | 105849/110696 [32:16:15<1:12:24, 1.12it/s][2025-04-28 05:19:41] (step=0105850) Train Loss: 5.6135, Train Steps/Sec: 1.12 + 96%|████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████▌ | 105874/110696 [32:16:38<1:11:58, 1.12it/s][2025-04-28 05:20:03] (step=0105875) Train Loss: 5.6584, Train Steps/Sec: 1.12 + 96%|████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████▌ | 105899/110696 [32:17:00<1:11:08, 1.12it/s][2025-04-28 05:20:25] (step=0105900) Train Loss: 5.7054, Train Steps/Sec: 1.12 + 96%|████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████▌ | 105924/110696 [32:17:22<1:10:37, 1.13it/s][2025-04-28 05:20:48] (step=0105925) Train Loss: 5.6962, Train Steps/Sec: 1.12 + 96%|████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████▌ | 105949/110696 [32:17:45<1:09:56, 1.13it/s][2025-04-28 05:21:10] (step=0105950) Train Loss: 5.7337, Train Steps/Sec: 1.12 + 96%|████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████▋ | 105974/110696 [32:18:07<1:09:34, 1.13it/s][2025-04-28 05:21:32] (step=0105975) Train Loss: 5.6620, Train Steps/Sec: 1.12 + 96%|████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████▋ | 105999/110696 [32:18:29<1:09:09, 1.13it/s][2025-04-28 05:21:55] (step=0106000) Train Loss: 5.6149, Train Steps/Sec: 1.12 +You are using a model of type blip-2 to instantiate a model of type instructblip. This is not supported for all configurations of models and can yield errors. +[2025-04-28 05:21:55] text_config is None. Initializing the text config with default values (`OPTConfig`). +model_name_or_path:/tmp/haozhezhao/model/blip2-flan-t5-xl +image_place_holder: +tokenizer length after expend 32101 +tokenizer length before expend 32100 +100%|████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████| 5/5 [05:04<00:00, 60.86s/it] +[2025-04-28 05:28:13] Finish Eval in 106000 steps...████████████████████████████████████████████████████████████████████████████████████████████| 5/5 [05:03<00:00, 60.59s/it] +processing left-padding... +processing left-padding... +processing left-padding... +processing left-padding... +processing left-padding... +[2025-04-28 05:28:33] Saved checkpoint to checkpoint/X2I_UltraEdiit_fourmask_LLava_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_filtered_objects_4mask_1280_context/003-GPT-XL/checkpoints/0106000.pt +[2025-04-28 05:28:35] Removed old checkpoint: checkpoint/X2I_UltraEdiit_fourmask_LLava_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_filtered_objects_4mask_1280_context/003-GPT-XL/checkpoints/0104000.pt + 96%|████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████▋ | 106024/110696 [32:25:32<1:12:40, 1.07it/s][2025-04-28 05:28:57] (step=0106025) Train Loss: 5.6616, Train Steps/Sec: 0.06 + 96%|████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████▋ | 106049/110696 [32:25:54<1:09:32, 1.11it/s][2025-04-28 05:29:20] (step=0106050) Train Loss: 5.6860, Train Steps/Sec: 1.12 + 96%|████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████▋ | 106074/110696 [32:26:17<1:09:10, 1.11it/s][2025-04-28 05:29:42] (step=0106075) Train Loss: 5.7142, Train Steps/Sec: 1.12 + 96%|████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████▊ | 106099/110696 [32:26:39<1:07:58, 1.13it/s][2025-04-28 05:30:04] (step=0106100) Train Loss: 5.6275, Train Steps/Sec: 1.12 + 96%|████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████▊ | 106124/110696 [32:27:01<1:07:41, 1.13it/s][2025-04-28 05:30:27] (step=0106125) Train Loss: 5.6484, Train Steps/Sec: 1.12 + 96%|████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████▊ | 106149/110696 [32:27:24<1:06:58, 1.13it/s][2025-04-28 05:30:49] (step=0106150) Train Loss: 5.6182, Train Steps/Sec: 1.12 + 96%|████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████▊ | 106174/110696 [32:27:46<1:06:30, 1.13it/s][2025-04-28 05:31:11] (step=0106175) Train Loss: 5.6594, Train Steps/Sec: 1.12 + 96%|████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████▉ | 106199/110696 [32:28:08<1:05:59, 1.14it/s][2025-04-28 05:31:33] (step=0106200) Train Loss: 5.6233, Train Steps/Sec: 1.12 + 96%|████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████▉ | 106224/110696 [32:28:30<1:07:49, 1.10it/s][2025-04-28 05:31:56] (step=0106225) Train Loss: 5.6929, Train Steps/Sec: 1.12 + 96%|████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████▉ | 106249/110696 [32:28:53<1:06:24, 1.12it/s][2025-04-28 05:32:18] (step=0106250) Train Loss: 5.6785, Train Steps/Sec: 1.11 + 96%|████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████▉ | 106274/110696 [32:29:24<1:06:27, 1.11it/s][2025-04-28 05:32:49] (step=0106275) Train Loss: 5.6600, Train Steps/Sec: 0.81 + 96%|████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████▉ | 106299/110696 [32:29:50<1:05:14, 1.12it/s][2025-04-28 05:33:16] (step=0106300) Train Loss: 5.6614, Train Steps/Sec: 0.94 + 96%|█████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████ | 106324/110696 [32:30:18<1:04:45, 1.13it/s][2025-04-28 05:33:43] (step=0106325) Train Loss: 5.6518, Train Steps/Sec: 0.91 + 96%|█████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████ | 106349/110696 [32:30:40<1:03:58, 1.13it/s][2025-04-28 05:34:05] (step=0106350) Train Loss: 5.6222, Train Steps/Sec: 1.12 + 96%|█████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████ | 106374/110696 [32:31:02<1:03:48, 1.13it/s][2025-04-28 05:34:28] (step=0106375) Train Loss: 5.5722, Train Steps/Sec: 1.12 + 96%|█████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████ | 106399/110696 [32:31:24<1:03:07, 1.13it/s][2025-04-28 05:34:50] (step=0106400) Train Loss: 5.6212, Train Steps/Sec: 1.12 + 96%|█████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████▏ | 106424/110696 [32:31:47<1:04:43, 1.10it/s][2025-04-28 05:35:12] (step=0106425) Train Loss: 5.6859, Train Steps/Sec: 1.12 + 96%|█████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████▏ | 106449/110696 [32:32:09<1:03:34, 1.11it/s][2025-04-28 05:35:34] (step=0106450) Train Loss: 5.6593, Train Steps/Sec: 1.13 + 96%|█████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████▏ | 106474/110696 [32:32:31<1:02:44, 1.12it/s][2025-04-28 05:35:57] (step=0106475) Train Loss: 5.6533, Train Steps/Sec: 1.12 + 96%|█████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████▏ | 106499/110696 [32:32:54<1:02:24, 1.12it/s][2025-04-28 05:36:19] (step=0106500) Train Loss: 5.6748, Train Steps/Sec: 1.12 + 96%|█████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████▎ | 106524/110696 [32:33:16<1:01:33, 1.13it/s][2025-04-28 05:36:41] (step=0106525) Train Loss: 5.5965, Train Steps/Sec: 1.12 + 96%|█████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████▎ | 106549/110696 [32:33:38<1:01:07, 1.13it/s][2025-04-28 05:37:03] (step=0106550) Train Loss: 5.6909, Train Steps/Sec: 1.12 + 96%|█████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████▎ | 106574/110696 [32:34:00<1:00:41, 1.13it/s][2025-04-28 05:37:26] (step=0106575) Train Loss: 5.6592, Train Steps/Sec: 1.12 + 96%|█████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████▎ | 106599/110696 [32:34:23<1:00:22, 1.13it/s][2025-04-28 05:37:48] (step=0106600) Train Loss: 5.6932, Train Steps/Sec: 1.12 + 96%|█████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████▎ | 106624/110696 [32:34:45<1:01:37, 1.10it/s][2025-04-28 05:38:10] (step=0106625) Train Loss: 5.6847, Train Steps/Sec: 1.13 + 96%|█████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████▍ | 106649/110696 [32:35:07<1:00:22, 1.12it/s][2025-04-28 05:38:33] (step=0106650) Train Loss: 5.7112, Train Steps/Sec: 1.12 + 96%|███████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████▎ | 106674/110696 [32:35:30<59:57, 1.12it/s][2025-04-28 05:38:55] (step=0106675) Train Loss: 5.6570, Train Steps/Sec: 1.12 + 96%|███████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████▍ | 106699/110696 [32:35:52<59:08, 1.13it/s][2025-04-28 05:39:17] (step=0106700) Train Loss: 5.6682, Train Steps/Sec: 1.12 + 96%|███████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████▍ | 106724/110696 [32:36:14<58:52, 1.12it/s][2025-04-28 05:39:39] (step=0106725) Train Loss: 5.6847, Train Steps/Sec: 1.12 + 96%|███████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████▍ | 106749/110696 [32:36:36<57:54, 1.14it/s][2025-04-28 05:40:02] (step=0106750) Train Loss: 5.7051, Train Steps/Sec: 1.13 + 96%|███████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████▍ | 106774/110696 [32:36:59<57:43, 1.13it/s][2025-04-28 05:40:24] (step=0106775) Train Loss: 5.7418, Train Steps/Sec: 1.12 + 96%|███████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████▍ | 106799/110696 [32:37:21<57:17, 1.13it/s][2025-04-28 05:40:46] (step=0106800) Train Loss: 5.6276, Train Steps/Sec: 1.12 + 97%|███████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████▌ | 106824/110696 [32:37:43<58:52, 1.10it/s][2025-04-28 05:41:09] (step=0106825) Train Loss: 5.6928, Train Steps/Sec: 1.12 + 97%|███████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████▌ | 106849/110696 [32:38:06<57:30, 1.11it/s][2025-04-28 05:41:31] (step=0106850) Train Loss: 5.7028, Train Steps/Sec: 1.12 + 97%|███████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████▌ | 106874/110696 [32:38:28<56:56, 1.12it/s][2025-04-28 05:41:53] (step=0106875) Train Loss: 5.6421, Train Steps/Sec: 1.12 + 97%|███████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████▌ | 106899/110696 [32:38:50<56:25, 1.12it/s][2025-04-28 05:42:15] (step=0106900) Train Loss: 5.7371, Train Steps/Sec: 1.12 + 97%|███████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████▋ | 106924/110696 [32:39:12<55:54, 1.12it/s][2025-04-28 05:42:38] (step=0106925) Train Loss: 5.5983, Train Steps/Sec: 1.12 + 97%|███████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████▋ | 106949/110696 [32:39:35<55:15, 1.13it/s][2025-04-28 05:43:00] (step=0106950) Train Loss: 5.6492, Train Steps/Sec: 1.13 + 97%|███████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████▋ | 106974/110696 [32:39:57<54:43, 1.13it/s][2025-04-28 05:43:22] (step=0106975) Train Loss: 5.6312, Train Steps/Sec: 1.12 + 97%|███████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████▋ | 106999/110696 [32:40:19<54:12, 1.14it/s][2025-04-28 05:43:45] (step=0107000) Train Loss: 5.6815, Train Steps/Sec: 1.12 + 97%|███████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████▊ | 107024/110696 [32:40:42<55:51, 1.10it/s][2025-04-28 05:44:07] (step=0107025) Train Loss: 5.6344, Train Steps/Sec: 1.12 + 97%|███████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████▊ | 107049/110696 [32:41:04<54:28, 1.12it/s][2025-04-28 05:44:29] (step=0107050) Train Loss: 5.7710, Train Steps/Sec: 1.12 + 97%|███████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████▊ | 107074/110696 [32:41:31<54:20, 1.11it/s][2025-04-28 05:44:56] (step=0107075) Train Loss: 5.5905, Train Steps/Sec: 0.92 + 97%|███████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████▊ | 107099/110696 [32:41:53<53:10, 1.13it/s][2025-04-28 05:45:19] (step=0107100) Train Loss: 5.6663, Train Steps/Sec: 1.12 + 97%|███████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████▊ | 107124/110696 [32:42:15<52:41, 1.13it/s][2025-04-28 05:45:41] (step=0107125) Train Loss: 5.6756, Train Steps/Sec: 1.12 + 97%|███████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████▉ | 107149/110696 [32:42:38<52:05, 1.13it/s][2025-04-28 05:46:03] (step=0107150) Train Loss: 5.6508, Train Steps/Sec: 1.12 + 97%|███████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████▉ | 107174/110696 [32:43:00<52:08, 1.13it/s][2025-04-28 05:46:25] (step=0107175) Train Loss: 5.6479, Train Steps/Sec: 1.12 + 97%|███████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████▉ | 107199/110696 [32:43:22<51:33, 1.13it/s][2025-04-28 05:46:48] (step=0107200) Train Loss: 5.7219, Train Steps/Sec: 1.12 + 97%|███████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████▉ | 107224/110696 [32:43:45<52:39, 1.10it/s][2025-04-28 05:47:10] (step=0107225) Train Loss: 5.6614, Train Steps/Sec: 1.12 + 97%|████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████ | 107249/110696 [32:44:07<51:41, 1.11it/s][2025-04-28 05:47:32] (step=0107250) Train Loss: 5.6385, Train Steps/Sec: 1.12 + 97%|████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████ | 107274/110696 [32:44:29<50:54, 1.12it/s][2025-04-28 05:47:55] (step=0107275) Train Loss: 5.6041, Train Steps/Sec: 1.12 + 97%|████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████ | 107299/110696 [32:44:51<50:23, 1.12it/s][2025-04-28 05:48:17] (step=0107300) Train Loss: 5.7071, Train Steps/Sec: 1.12 + 97%|████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████ | 107324/110696 [32:45:14<49:30, 1.14it/s][2025-04-28 05:48:39] (step=0107325) Train Loss: 5.6523, Train Steps/Sec: 1.12 + 97%|████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████▏ | 107349/110696 [32:45:36<49:25, 1.13it/s][2025-04-28 05:49:01] (step=0107350) Train Loss: 5.7322, Train Steps/Sec: 1.12 + 97%|████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████▏ | 107374/110696 [32:45:58<48:52, 1.13it/s][2025-04-28 05:49:24] (step=0107375) Train Loss: 5.6573, Train Steps/Sec: 1.12 + 97%|████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████▏ | 107399/110696 [32:46:21<48:28, 1.13it/s][2025-04-28 05:49:46] (step=0107400) Train Loss: 5.6853, Train Steps/Sec: 1.12 + 97%|████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████▏ | 107424/110696 [32:46:43<49:47, 1.10it/s][2025-04-28 05:50:08] (step=0107425) Train Loss: 5.7120, Train Steps/Sec: 1.12 + 97%|████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████▏ | 107449/110696 [32:47:05<48:39, 1.11it/s][2025-04-28 05:50:31] (step=0107450) Train Loss: 5.6531, Train Steps/Sec: 1.12 + 97%|████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████▎ | 107474/110696 [32:47:28<47:58, 1.12it/s][2025-04-28 05:50:53] (step=0107475) Train Loss: 5.6696, Train Steps/Sec: 1.12 + 97%|████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████▎ | 107499/110696 [32:47:50<47:17, 1.13it/s][2025-04-28 05:51:15] (step=0107500) Train Loss: 5.6240, Train Steps/Sec: 1.12 + 97%|████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████▎ | 107524/110696 [32:48:12<46:49, 1.13it/s][2025-04-28 05:51:37] (step=0107525) Train Loss: 5.6913, Train Steps/Sec: 1.12 + 97%|████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████▎ | 107549/110696 [32:48:34<46:21, 1.13it/s][2025-04-28 05:52:00] (step=0107550) Train Loss: 5.6039, Train Steps/Sec: 1.12 + 97%|████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████▍ | 107574/110696 [32:48:57<45:53, 1.13it/s][2025-04-28 05:52:22] (step=0107575) Train Loss: 5.6692, Train Steps/Sec: 1.12 + 97%|████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████▍ | 107599/110696 [32:49:19<45:35, 1.13it/s][2025-04-28 05:52:44] (step=0107600) Train Loss: 5.6389, Train Steps/Sec: 1.12 + 97%|████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████▍ | 107624/110696 [32:49:41<46:37, 1.10it/s][2025-04-28 05:53:07] (step=0107625) Train Loss: 5.6716, Train Steps/Sec: 1.12 + 97%|████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████▍ | 107649/110696 [32:50:04<45:47, 1.11it/s][2025-04-28 05:53:29] (step=0107650) Train Loss: 5.6596, Train Steps/Sec: 1.12 + 97%|████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████▌ | 107674/110696 [32:50:26<45:12, 1.11it/s][2025-04-28 05:53:51] (step=0107675) Train Loss: 5.6890, Train Steps/Sec: 1.12 + 97%|████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████▌ | 107699/110696 [32:50:48<44:20, 1.13it/s][2025-04-28 05:54:14] (step=0107700) Train Loss: 5.6765, Train Steps/Sec: 1.12 + 97%|████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████▌ | 107724/110696 [32:51:10<43:57, 1.13it/s][2025-04-28 05:54:36] (step=0107725) Train Loss: 5.6471, Train Steps/Sec: 1.12 + 97%|████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████▌ | 107749/110696 [32:51:33<43:34, 1.13it/s][2025-04-28 05:54:58] (step=0107750) Train Loss: 5.6349, Train Steps/Sec: 1.12 + 97%|████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████▌ | 107774/110696 [32:51:55<42:58, 1.13it/s][2025-04-28 05:55:20] (step=0107775) Train Loss: 5.7140, Train Steps/Sec: 1.12 + 97%|████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████▋ | 107799/110696 [32:52:17<42:34, 1.13it/s][2025-04-28 05:55:43] (step=0107800) Train Loss: 5.6696, Train Steps/Sec: 1.12 + 97%|████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████▋ | 107824/110696 [32:52:40<43:32, 1.10it/s][2025-04-28 05:56:05] (step=0107825) Train Loss: 5.6253, Train Steps/Sec: 1.12 + 97%|████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████▋ | 107849/110696 [32:53:02<42:36, 1.11it/s][2025-04-28 05:56:27] (step=0107850) Train Loss: 5.6703, Train Steps/Sec: 1.12 + 97%|████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████▋ | 107874/110696 [32:53:24<41:55, 1.12it/s][2025-04-28 05:56:50] (step=0107875) Train Loss: 5.6673, Train Steps/Sec: 1.12 + 97%|████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████▊ | 107899/110696 [32:53:46<41:24, 1.13it/s][2025-04-28 05:57:12] (step=0107900) Train Loss: 5.7142, Train Steps/Sec: 1.12 + 97%|████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████▊ | 107924/110696 [32:54:09<41:02, 1.13it/s][2025-04-28 05:57:34] (step=0107925) Train Loss: 5.7046, Train Steps/Sec: 1.12 + 98%|████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████▊ | 107949/110696 [32:54:31<40:18, 1.14it/s][2025-04-28 05:57:56] (step=0107950) Train Loss: 5.6355, Train Steps/Sec: 1.12 + 98%|████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████▊ | 107974/110696 [32:54:53<40:10, 1.13it/s][2025-04-28 05:58:19] (step=0107975) Train Loss: 5.6673, Train Steps/Sec: 1.12 + 98%|████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████▉ | 107999/110696 [32:55:16<39:45, 1.13it/s][2025-04-28 05:58:41] (step=0108000) Train Loss: 5.7060, Train Steps/Sec: 1.12 +You are using a model of type blip-2 to instantiate a model of type instructblip. This is not supported for all configurations of models and can yield errors. +[2025-04-28 05:58:41] text_config is None. Initializing the text config with default values (`OPTConfig`). +model_name_or_path:/tmp/haozhezhao/model/blip2-flan-t5-xl +image_place_holder: +tokenizer length after expend 32101 +tokenizer length before expend 32100 +100%|████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████| 5/5 [05:04<00:00, 60.83s/it] +[2025-04-28 06:05:00] Finish Eval in 108000 steps...████████████████████████████████████████████████████████████████████████████████████████████| 5/5 [05:03<00:00, 60.52s/it] +processing left-padding... +processing left-padding... +processing left-padding... +processing left-padding... +processing left-padding... +[2025-04-28 06:05:20] Saved checkpoint to checkpoint/X2I_UltraEdiit_fourmask_LLava_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_filtered_objects_4mask_1280_context/003-GPT-XL/checkpoints/0108000.pt +[2025-04-28 06:05:22] Removed old checkpoint: checkpoint/X2I_UltraEdiit_fourmask_LLava_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_filtered_objects_4mask_1280_context/003-GPT-XL/checkpoints/0106000.pt + 98%|████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████▉ | 108024/110696 [33:02:19<41:25, 1.08it/s][2025-04-28 06:05:44] (step=0108025) Train Loss: 5.7117, Train Steps/Sec: 0.06 + 98%|████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████▉ | 108049/110696 [33:02:41<39:35, 1.11it/s][2025-04-28 06:06:06] (step=0108050) Train Loss: 5.7686, Train Steps/Sec: 1.12 + 98%|████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████▉ | 108074/110696 [33:03:03<39:07, 1.12it/s][2025-04-28 06:06:29] (step=0108075) Train Loss: 5.6358, Train Steps/Sec: 1.12 + 98%|████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████▉ | 108099/110696 [33:03:26<38:42, 1.12it/s][2025-04-28 06:06:51] (step=0108100) Train Loss: 5.6790, Train Steps/Sec: 1.12 + 98%|█████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████ | 108124/110696 [33:03:48<37:58, 1.13it/s][2025-04-28 06:07:13] (step=0108125) Train Loss: 5.6760, Train Steps/Sec: 1.12 + 98%|█████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████ | 108149/110696 [33:04:11<37:41, 1.13it/s][2025-04-28 06:07:36] (step=0108150) Train Loss: 5.6635, Train Steps/Sec: 1.10 + 98%|█████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████ | 108174/110696 [33:04:33<37:15, 1.13it/s][2025-04-28 06:07:58] (step=0108175) Train Loss: 5.6564, Train Steps/Sec: 1.12 + 98%|█████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████ | 108199/110696 [33:04:55<36:46, 1.13it/s][2025-04-28 06:08:21] (step=0108200) Train Loss: 5.6676, Train Steps/Sec: 1.11 + 98%|█████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████▏ | 108224/110696 [33:05:18<37:34, 1.10it/s][2025-04-28 06:08:43] (step=0108225) Train Loss: 5.6975, Train Steps/Sec: 1.12 + 98%|█████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████▏ | 108249/110696 [33:05:40<36:41, 1.11it/s][2025-04-28 06:09:05] (step=0108250) Train Loss: 5.7219, Train Steps/Sec: 1.12 + 98%|█████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████▏ | 108274/110696 [33:06:02<36:06, 1.12it/s][2025-04-28 06:09:28] (step=0108275) Train Loss: 5.6147, Train Steps/Sec: 1.12 + 98%|█████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████▏ | 108299/110696 [33:06:25<35:35, 1.12it/s][2025-04-28 06:09:50] (step=0108300) Train Loss: 5.7275, Train Steps/Sec: 1.12 + 98%|█████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████▎ | 108324/110696 [33:06:47<35:07, 1.13it/s][2025-04-28 06:10:12] (step=0108325) Train Loss: 5.6888, Train Steps/Sec: 1.12 + 98%|█████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████▎ | 108349/110696 [33:07:09<34:33, 1.13it/s][2025-04-28 06:10:35] (step=0108350) Train Loss: 5.6635, Train Steps/Sec: 1.12 + 98%|█████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████▎ | 108374/110696 [33:07:31<34:08, 1.13it/s][2025-04-28 06:10:57] (step=0108375) Train Loss: 5.6532, Train Steps/Sec: 1.12 + 98%|█████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████▎ | 108399/110696 [33:07:54<33:54, 1.13it/s][2025-04-28 06:11:19] (step=0108400) Train Loss: 5.7016, Train Steps/Sec: 1.12 + 98%|█████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████▎ | 108424/110696 [33:08:16<34:25, 1.10it/s][2025-04-28 06:11:41] (step=0108425) Train Loss: 5.6836, Train Steps/Sec: 1.12 + 98%|█████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████▍ | 108449/110696 [33:08:38<33:33, 1.12it/s][2025-04-28 06:12:04] (step=0108450) Train Loss: 5.6696, Train Steps/Sec: 1.13 + 98%|█████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████▍ | 108474/110696 [33:09:01<33:09, 1.12it/s][2025-04-28 06:12:26] (step=0108475) Train Loss: 5.6215, Train Steps/Sec: 1.12 + 98%|█████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████▍ | 108499/110696 [33:09:23<32:35, 1.12it/s][2025-04-28 06:12:48] (step=0108500) Train Loss: 5.6226, Train Steps/Sec: 1.12 + 98%|█████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████▍ | 108524/110696 [33:09:45<32:08, 1.13it/s][2025-04-28 06:13:11] (step=0108525) Train Loss: 5.6640, Train Steps/Sec: 1.12 + 98%|█████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████▌ | 108549/110696 [33:10:08<31:45, 1.13it/s][2025-04-28 06:13:33] (step=0108550) Train Loss: 5.6623, Train Steps/Sec: 1.12 + 98%|█████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████▌ | 108574/110696 [33:10:30<31:08, 1.14it/s][2025-04-28 06:13:55] (step=0108575) Train Loss: 5.7157, Train Steps/Sec: 1.12 + 98%|█████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████▌ | 108599/110696 [33:10:52<30:56, 1.13it/s][2025-04-28 06:14:18] (step=0108600) Train Loss: 5.6495, Train Steps/Sec: 1.11 + 98%|█████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████▌ | 108624/110696 [33:11:15<31:34, 1.09it/s][2025-04-28 06:14:40] (step=0108625) Train Loss: 5.7214, Train Steps/Sec: 1.12 + 98%|█████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████▋ | 108649/110696 [33:11:37<30:40, 1.11it/s][2025-04-28 06:15:02] (step=0108650) Train Loss: 5.6718, Train Steps/Sec: 1.12 + 98%|█████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████▋ | 108674/110696 [33:11:59<30:10, 1.12it/s][2025-04-28 06:15:24] (step=0108675) Train Loss: 5.5886, Train Steps/Sec: 1.12 + 98%|█████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████▋ | 108699/110696 [33:12:21<29:41, 1.12it/s][2025-04-28 06:15:47] (step=0108700) Train Loss: 5.5783, Train Steps/Sec: 1.12 + 98%|█████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████▋ | 108724/110696 [33:12:44<29:11, 1.13it/s][2025-04-28 06:16:09] (step=0108725) Train Loss: 5.5424, Train Steps/Sec: 1.12 + 98%|█████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████▋ | 108749/110696 [33:13:06<28:35, 1.13it/s][2025-04-28 06:16:31] (step=0108750) Train Loss: 5.6489, Train Steps/Sec: 1.12 + 98%|█████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████▊ | 108774/110696 [33:13:28<28:24, 1.13it/s][2025-04-28 06:16:54] (step=0108775) Train Loss: 5.6288, Train Steps/Sec: 1.12 + 98%|█████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████▊ | 108799/110696 [33:13:51<28:07, 1.12it/s][2025-04-28 06:17:16] (step=0108800) Train Loss: 5.6587, Train Steps/Sec: 1.11 + 98%|█████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████▊ | 108824/110696 [33:14:13<28:27, 1.10it/s][2025-04-28 06:17:38] (step=0108825) Train Loss: 5.6620, Train Steps/Sec: 1.12 + 98%|█████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████▊ | 108849/110696 [33:14:35<27:36, 1.11it/s][2025-04-28 06:18:01] (step=0108850) Train Loss: 5.6392, Train Steps/Sec: 1.12 + 98%|█████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████▉ | 108874/110696 [33:14:58<27:06, 1.12it/s][2025-04-28 06:18:23] (step=0108875) Train Loss: 5.5899, Train Steps/Sec: 1.12 + 98%|█████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████▉ | 108899/110696 [33:15:20<26:43, 1.12it/s][2025-04-28 06:18:45] (step=0108900) Train Loss: 5.6960, Train Steps/Sec: 1.12 + 98%|█████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████▉ | 108924/110696 [33:15:42<26:12, 1.13it/s][2025-04-28 06:19:08] (step=0108925) Train Loss: 5.6765, Train Steps/Sec: 1.12 + 98%|█████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████▉ | 108949/110696 [33:16:05<25:45, 1.13it/s][2025-04-28 06:19:30] (step=0108950) Train Loss: 5.7514, Train Steps/Sec: 1.12 + 98%|██████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████ | 108974/110696 [33:16:27<25:24, 1.13it/s][2025-04-28 06:19:52] (step=0108975) Train Loss: 5.7057, Train Steps/Sec: 1.12 + 98%|██████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████ | 108999/110696 [33:16:49<24:57, 1.13it/s][2025-04-28 06:20:15] (step=0109000) Train Loss: 5.6216, Train Steps/Sec: 1.12 + 98%|██████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████ | 109024/110696 [33:17:12<25:29, 1.09it/s][2025-04-28 06:20:37] (step=0109025) Train Loss: 5.6664, Train Steps/Sec: 1.12 + 99%|██████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████ | 109049/110696 [33:17:34<24:40, 1.11it/s][2025-04-28 06:20:59] (step=0109050) Train Loss: 5.6447, Train Steps/Sec: 1.12 + 99%|██████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████ | 109074/110696 [33:17:56<24:07, 1.12it/s][2025-04-28 06:21:21] (step=0109075) Train Loss: 5.6742, Train Steps/Sec: 1.12 + 99%|██████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████▏ | 109099/110696 [33:18:18<23:43, 1.12it/s][2025-04-28 06:21:44] (step=0109100) Train Loss: 5.6565, Train Steps/Sec: 1.12 + 99%|██████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████▏ | 109124/110696 [33:18:45<24:02, 1.09it/s][2025-04-28 06:22:11] (step=0109125) Train Loss: 5.6200, Train Steps/Sec: 0.93 + 99%|██████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████▏ | 109149/110696 [33:19:08<22:45, 1.13it/s][2025-04-28 06:22:33] (step=0109150) Train Loss: 5.6786, Train Steps/Sec: 1.12 + 99%|██████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████▏ | 109174/110696 [33:19:30<22:24, 1.13it/s][2025-04-28 06:22:55] (step=0109175) Train Loss: 5.7064, Train Steps/Sec: 1.12 + 99%|██████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████▎ | 109199/110696 [33:19:52<22:05, 1.13it/s][2025-04-28 06:23:18] (step=0109200) Train Loss: 5.6454, Train Steps/Sec: 1.12 + 99%|██████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████▎ | 109224/110696 [33:20:15<22:23, 1.10it/s][2025-04-28 06:23:40] (step=0109225) Train Loss: 5.6551, Train Steps/Sec: 1.12 + 99%|██████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████▎ | 109249/110696 [33:20:37<21:41, 1.11it/s][2025-04-28 06:24:02] (step=0109250) Train Loss: 5.7165, Train Steps/Sec: 1.12 + 99%|██████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████▎ | 109274/110696 [33:20:59<21:08, 1.12it/s][2025-04-28 06:24:24] (step=0109275) Train Loss: 5.7141, Train Steps/Sec: 1.12 + 99%|██████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████▍ | 109299/110696 [33:21:21<20:45, 1.12it/s][2025-04-28 06:24:47] (step=0109300) Train Loss: 5.5816, Train Steps/Sec: 1.12 + 99%|██████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████▍ | 109324/110696 [33:21:44<20:16, 1.13it/s][2025-04-28 06:25:09] (step=0109325) Train Loss: 5.6663, Train Steps/Sec: 1.12 + 99%|██████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████▍ | 109349/110696 [33:22:06<19:52, 1.13it/s][2025-04-28 06:25:31] (step=0109350) Train Loss: 5.7142, Train Steps/Sec: 1.12 + 99%|██████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████▍ | 109374/110696 [33:22:28<19:26, 1.13it/s][2025-04-28 06:25:54] (step=0109375) Train Loss: 5.7033, Train Steps/Sec: 1.12 + 99%|██████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████▌ | 109399/110696 [33:22:50<19:02, 1.13it/s][2025-04-28 06:26:16] (step=0109400) Train Loss: 5.5975, Train Steps/Sec: 1.12 + 99%|██████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████▌ | 109424/110696 [33:23:13<19:21, 1.10it/s][2025-04-28 06:26:38] (step=0109425) Train Loss: 5.6214, Train Steps/Sec: 1.12 + 99%|██████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████▌ | 109449/110696 [33:23:35<18:40, 1.11it/s][2025-04-28 06:27:00] (step=0109450) Train Loss: 5.6980, Train Steps/Sec: 1.12 + 99%|██████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████▌ | 109474/110696 [33:24:02<21:36, 1.06s/it][2025-04-28 06:27:27] (step=0109475) Train Loss: 5.6878, Train Steps/Sec: 0.92 + 99%|██████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████▌ | 109499/110696 [33:24:24<17:43, 1.13it/s][2025-04-28 06:27:50] (step=0109500) Train Loss: 5.5591, Train Steps/Sec: 1.12 + 99%|██████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████▋ | 109524/110696 [33:24:47<17:19, 1.13it/s][2025-04-28 06:28:12] (step=0109525) Train Loss: 5.6242, Train Steps/Sec: 1.12 + 99%|██████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████▋ | 109549/110696 [33:25:09<17:00, 1.12it/s][2025-04-28 06:28:34] (step=0109550) Train Loss: 5.6647, Train Steps/Sec: 1.12 + 99%|██████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████▋ | 109574/110696 [33:25:31<16:33, 1.13it/s][2025-04-28 06:28:57] (step=0109575) Train Loss: 5.6834, Train Steps/Sec: 1.12 + 99%|██████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████▋ | 109599/110696 [33:25:54<16:06, 1.14it/s][2025-04-28 06:29:19] (step=0109600) Train Loss: 5.6525, Train Steps/Sec: 1.12 + 99%|██████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████▊ | 109624/110696 [33:26:16<16:14, 1.10it/s][2025-04-28 06:29:41] (step=0109625) Train Loss: 5.6323, Train Steps/Sec: 1.12 + 99%|██████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████▊ | 109649/110696 [33:26:38<15:41, 1.11it/s][2025-04-28 06:30:04] (step=0109650) Train Loss: 5.6584, Train Steps/Sec: 1.12 + 99%|██████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████▊ | 109674/110696 [33:27:01<15:12, 1.12it/s][2025-04-28 06:30:26] (step=0109675) Train Loss: 5.6350, Train Steps/Sec: 1.12 + 99%|██████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████▊ | 109699/110696 [33:27:23<14:47, 1.12it/s][2025-04-28 06:30:48] (step=0109700) Train Loss: 5.6594, Train Steps/Sec: 1.12 + 99%|██████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████▉ | 109724/110696 [33:27:45<14:21, 1.13it/s][2025-04-28 06:31:10] (step=0109725) Train Loss: 5.7304, Train Steps/Sec: 1.12 + 99%|██████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████▉ | 109749/110696 [33:28:07<14:00, 1.13it/s][2025-04-28 06:31:33] (step=0109750) Train Loss: 5.6889, Train Steps/Sec: 1.12 + 99%|██████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████▉ | 109774/110696 [33:28:30<13:35, 1.13it/s][2025-04-28 06:31:55] (step=0109775) Train Loss: 5.6489, Train Steps/Sec: 1.12 + 99%|██████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████▉ | 109799/110696 [33:28:52<13:10, 1.13it/s][2025-04-28 06:32:17] (step=0109800) Train Loss: 5.6170, Train Steps/Sec: 1.12 + 99%|██████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████▉ | 109824/110696 [33:29:14<13:15, 1.10it/s][2025-04-28 06:32:40] (step=0109825) Train Loss: 5.6850, Train Steps/Sec: 1.12 + 99%|███████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████ | 109849/110696 [33:29:37<12:43, 1.11it/s][2025-04-28 06:33:02] (step=0109850) Train Loss: 5.6664, Train Steps/Sec: 1.12 + 99%|███████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████ | 109874/110696 [33:29:59<12:14, 1.12it/s][2025-04-28 06:33:24] (step=0109875) Train Loss: 5.6091, Train Steps/Sec: 1.12 + 99%|███████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████ | 109899/110696 [33:30:21<11:48, 1.13it/s][2025-04-28 06:33:46] (step=0109900) Train Loss: 5.6328, Train Steps/Sec: 1.12 + 99%|███████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████ | 109924/110696 [33:30:43<11:23, 1.13it/s][2025-04-28 06:34:09] (step=0109925) Train Loss: 5.6571, Train Steps/Sec: 1.12 + 99%|███████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████▏| 109949/110696 [33:31:06<11:00, 1.13it/s][2025-04-28 06:34:31] (step=0109950) Train Loss: 5.6161, Train Steps/Sec: 1.12 + 99%|███████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████▏| 109974/110696 [33:31:28<10:35, 1.14it/s][2025-04-28 06:34:53] (step=0109975) Train Loss: 5.6925, Train Steps/Sec: 1.12 + 99%|███████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████▏| 109999/110696 [33:31:50<10:15, 1.13it/s][2025-04-28 06:35:16] (step=0110000) Train Loss: 5.6601, Train Steps/Sec: 1.12 +You are using a model of type blip-2 to instantiate a model of type instructblip. This is not supported for all configurations of models and can yield errors. +[2025-04-28 06:35:16] text_config is None. Initializing the text config with default values (`OPTConfig`). +model_name_or_path:/tmp/haozhezhao/model/blip2-flan-t5-xl +image_place_holder: +tokenizer length after expend 32101 +tokenizer length before expend 32100 +100%|████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████| 5/5 [05:04<00:00, 60.96s/it] +[2025-04-28 06:41:34] Finish Eval in 110000 steps...████████████████████████████████████████████████████████████████████████████████████████████| 5/5 [05:04<00:00, 60.71s/it] +processing left-padding... +processing left-padding... +processing left-padding... +processing left-padding... +processing left-padding... +[2025-04-28 06:41:53] Saved checkpoint to checkpoint/X2I_UltraEdiit_fourmask_LLava_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_filtered_objects_4mask_1280_context/003-GPT-XL/checkpoints/0110000.pt +[2025-04-28 06:41:55] Removed old checkpoint: checkpoint/X2I_UltraEdiit_fourmask_LLava_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_filtered_objects_4mask_1280_context/003-GPT-XL/checkpoints/0108000.pt + 99%|███████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████▏| 110024/110696 [33:38:52<10:26, 1.07it/s][2025-04-28 06:42:17] (step=0110025) Train Loss: 5.7012, Train Steps/Sec: 0.06 + 99%|███████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████▎| 110049/110696 [33:39:15<09:42, 1.11it/s][2025-04-28 06:42:40] (step=0110050) Train Loss: 5.6505, Train Steps/Sec: 1.10 + 99%|███████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████▎| 110074/110696 [33:39:37<09:15, 1.12it/s][2025-04-28 06:43:02] (step=0110075) Train Loss: 5.6027, Train Steps/Sec: 1.12 + 99%|███████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████▎| 110099/110696 [33:39:59<08:51, 1.12it/s][2025-04-28 06:43:25] (step=0110100) Train Loss: 5.6725, Train Steps/Sec: 1.12 + 99%|███████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████▎| 110124/110696 [33:40:22<08:28, 1.12it/s][2025-04-28 06:43:47] (step=0110125) Train Loss: 5.7205, Train Steps/Sec: 1.12 +100%|███████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████▎| 110149/110696 [33:40:44<08:04, 1.13it/s][2025-04-28 06:44:09] (step=0110150) Train Loss: 5.6553, Train Steps/Sec: 1.12 +100%|███████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████▍| 110174/110696 [33:41:06<07:40, 1.13it/s][2025-04-28 06:44:31] (step=0110175) Train Loss: 5.6895, Train Steps/Sec: 1.12 +100%|███████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████▍| 110199/110696 [33:41:28<07:18, 1.13it/s][2025-04-28 06:44:54] (step=0110200) Train Loss: 5.7215, Train Steps/Sec: 1.12 +100%|███████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████▍| 110224/110696 [33:41:51<07:10, 1.10it/s][2025-04-28 06:45:16] (step=0110225) Train Loss: 5.6276, Train Steps/Sec: 1.12 +100%|███████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████▍| 110249/110696 [33:42:13<06:41, 1.11it/s][2025-04-28 06:45:38] (step=0110250) Train Loss: 5.6122, Train Steps/Sec: 1.12 +100%|███████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████▌| 110274/110696 [33:42:35<06:17, 1.12it/s][2025-04-28 06:46:01] (step=0110275) Train Loss: 5.7012, Train Steps/Sec: 1.12 +100%|███████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████▌| 110299/110696 [33:42:57<05:54, 1.12it/s][2025-04-28 06:46:23] (step=0110300) Train Loss: 5.6694, Train Steps/Sec: 1.12 +100%|███████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████▌| 110324/110696 [33:43:20<05:29, 1.13it/s][2025-04-28 06:46:45] (step=0110325) Train Loss: 5.6489, Train Steps/Sec: 1.12 +100%|███████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████▌| 110349/110696 [33:43:42<05:06, 1.13it/s][2025-04-28 06:47:07] (step=0110350) Train Loss: 5.6281, Train Steps/Sec: 1.12 +100%|███████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████▋| 110374/110696 [33:44:04<04:43, 1.13it/s][2025-04-28 06:47:30] (step=0110375) Train Loss: 5.6736, Train Steps/Sec: 1.12 +100%|███████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████▋| 110399/110696 [33:44:27<04:22, 1.13it/s][2025-04-28 06:47:52] (step=0110400) Train Loss: 5.6578, Train Steps/Sec: 1.12 +100%|███████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████▋| 110424/110696 [33:44:49<04:08, 1.10it/s][2025-04-28 06:48:14] (step=0110425) Train Loss: 5.6692, Train Steps/Sec: 1.12 +100%|███████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████▋| 110449/110696 [33:45:11<03:40, 1.12it/s][2025-04-28 06:48:37] (step=0110450) Train Loss: 5.6487, Train Steps/Sec: 1.12 +100%|███████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████▋| 110474/110696 [33:45:33<03:17, 1.12it/s][2025-04-28 06:48:59] (step=0110475) Train Loss: 5.6544, Train Steps/Sec: 1.12 +100%|███████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████▊| 110499/110696 [33:45:56<02:54, 1.13it/s][2025-04-28 06:49:21] (step=0110500) Train Loss: 5.6840, Train Steps/Sec: 1.12 +100%|███████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████▊| 110524/110696 [33:46:18<02:32, 1.13it/s][2025-04-28 06:49:43] (step=0110525) Train Loss: 5.5940, Train Steps/Sec: 1.12 +100%|███████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████▊| 110549/110696 [33:46:40<02:09, 1.13it/s][2025-04-28 06:50:06] (step=0110550) Train Loss: 5.6951, Train Steps/Sec: 1.12 +100%|███████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████▊| 110574/110696 [33:47:03<01:47, 1.13it/s][2025-04-28 06:50:28] (step=0110575) Train Loss: 5.6126, Train Steps/Sec: 1.12 +100%|███████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████▉| 110599/110696 [33:47:37<05:01, 3.11s/it][2025-04-28 06:51:03] (step=0110600) Train Loss: 5.6076, Train Steps/Sec: 0.71 +100%|███████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████▉| 110624/110696 [33:48:00<01:05, 1.10it/s][2025-04-28 06:51:25] (step=0110625) Train Loss: 5.6658, Train Steps/Sec: 1.12 +100%|███████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████▉| 110649/110696 [33:48:27<00:42, 1.11it/s][2025-04-28 06:51:53] (step=0110650) Train Loss: 5.6951, Train Steps/Sec: 0.91 +100%|███████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████▉| 110674/110696 [33:48:50<00:19, 1.12it/s][2025-04-28 06:52:15] (step=0110675) Train Loss: 5.6242, Train Steps/Sec: 1.12 +100%|████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████| 110696/110696 [33:49:12<00:00, 1.10s/it] +[2025-04-28 06:52:36] Done! diff --git a/CKPTS/X2I_UltraEdiit_fourmask_LLava_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_filtered_objects_4mask_1280_context/wandb/run-20250426_210323-vtrxdt92/files/requirements.txt b/CKPTS/X2I_UltraEdiit_fourmask_LLava_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_filtered_objects_4mask_1280_context/wandb/run-20250426_210323-vtrxdt92/files/requirements.txt new file mode 100644 index 0000000000000000000000000000000000000000..06dc78369ffff807b210006a0e79d705ffe2a7d7 --- /dev/null +++ b/CKPTS/X2I_UltraEdiit_fourmask_LLava_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_filtered_objects_4mask_1280_context/wandb/run-20250426_210323-vtrxdt92/files/requirements.txt @@ -0,0 +1,131 @@ +typing_extensions==4.12.2 +pyzmq==26.3.0 +nvidia-cufft-cu12==11.0.2.54 +triton==3.1.0 +nvidia-cublas-cu12==12.1.3.1 +psutil==7.0.0 +nvidia-cuda-cupti-cu12==12.1.105 +smmap==5.0.2 +nvidia-cuda-runtime-cu12==12.1.105 +aiohappyeyeballs==2.6.1 +asttokens==3.0.0 +huggingface-hub==0.29.3 +pyarrow==19.0.1 +fonttools==4.56.0 +python-dateutil==2.9.0.post0 +GitPython==3.1.44 +aiohttp==3.11.14 +wandb==0.19.8 +setproctitle==1.3.5 +PyYAML==6.0.2 +pydantic_core==2.27.2 +safetensors==0.5.3 +nvidia-nvjitlink-cu12==12.1.105 +aiosignal==1.3.2 +dill==0.3.8 +nvidia-cuda-nvrtc-cu12==12.1.105 +multiprocess==0.70.16 +pure_eval==0.2.3 +stack_data==0.6.3 +pydantic==2.10.6 +MarkupSafe==2.1.5 +tornado==6.4.2 +executing==2.1.0 +executing==2.2.0 +opencv-python==4.11.0.86 +nvitop==1.4.2 +multidict==6.2.0 +Jinja2==3.1.4 +torch==2.5.1+cu121 +nvidia-curand-cu12==10.3.2.106 +platformdirs==4.3.6 +six==1.17.0 +mpmath==1.3.0 +zipp==3.21.0 +packaging==24.2 +requests==2.32.3 +certifi==2025.1.31 +docker-pycreds==0.4.0 +torchvision==0.20.1+cu121 +pandas==2.2.3 +networkx==3.3 +exceptiongroup==1.2.2 +pickleshare==0.7.5 +tokenizers==0.21.1 +charset-normalizer==3.4.1 +jupyter_core==5.7.2 +wcwidth==0.2.13 +nvidia-nvtx-cu12==12.1.105 +prompt_toolkit==3.0.50 +fsspec==2024.12.0 +pillow==11.1.0 +propcache==0.3.0 +regex==2024.11.6 +ptyprocess==0.7.0 +contourpy==1.3.1 +importlib_metadata==8.6.1 +idna==3.10 +comm==0.2.2 +protobuf==5.29.3 +yarl==1.18.3 +ipython_pygments_lexers==1.1.1 +pip==25.0 +parso==0.8.4 +joblib==1.4.2 +nvidia-nccl-cu12==2.21.5 +hf_transfer==0.1.9 +Pygments==2.19.1 +decorator==5.2.1 +filelock==3.18.0 +nvidia-cusparse-cu12==12.1.0.106 +debugpy==1.8.13 +urllib3==2.3.0 +traitlets==5.14.3 +tzdata==2025.1 +matplotlib-inline==0.1.7 +matplotlib==3.10.1 +kiwisolver==1.4.8 +nest_asyncio==1.6.0 +frozenlist==1.5.0 +nvidia-ml-py==12.570.86 +transformers==4.49.0 +nltk==3.9.1 +ipykernel==6.29.5 +click==8.1.8 +gitdb==4.0.12 +pyparsing==3.2.1 +attrs==25.3.0 +jedi==0.19.2 +ipython==9.0.2 +nvidia-cudnn-cu12==9.1.0.70 +pexpect==4.9.0 +nvidia-cusolver-cu12==11.4.5.107 +numpy==2.2.4 +tqdm==4.67.1 +pytz==2025.1 +wheel==0.45.1 +sentry-sdk==2.23.1 +torchaudio==2.5.1+cu121 +jupyter_client==8.6.3 +cycler==0.12.1 +annotated-types==0.7.0 +sympy==1.13.1 +xxhash==3.5.0 +datasets==3.4.1 +setuptools==75.8.0 +typing_extensions==4.12.2 +wheel==0.43.0 +importlib_metadata==8.0.0 +backports.tarfile==1.2.0 +autocommand==2.2.2 +packaging==24.2 +tomli==2.0.1 +typeguard==4.3.0 +zipp==3.19.2 +jaraco.context==5.3.0 +jaraco.functools==4.0.1 +more-itertools==10.3.0 +platformdirs==4.2.2 +jaraco.text==3.12.1 +jaraco.collections==5.1.0 +inflect==7.3.1 diff --git a/CKPTS/X2I_UltraEdiit_fourmask_LLava_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_filtered_objects_4mask_1280_context/wandb/run-20250426_210323-vtrxdt92/files/wandb-metadata.json b/CKPTS/X2I_UltraEdiit_fourmask_LLava_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_filtered_objects_4mask_1280_context/wandb/run-20250426_210323-vtrxdt92/files/wandb-metadata.json new file mode 100644 index 0000000000000000000000000000000000000000..26594398fd77be0f921adeb441beeba61f1fbcc7 --- /dev/null +++ b/CKPTS/X2I_UltraEdiit_fourmask_LLava_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_filtered_objects_4mask_1280_context/wandb/run-20250426_210323-vtrxdt92/files/wandb-metadata.json @@ -0,0 +1,149 @@ +{ + "os": "Linux-5.15.0-1064-azure-x86_64-with-glibc2.31", + "python": "CPython 3.11.11", + "startedAt": "2025-04-26T21:03:23.737060Z", + "args": [ + "--vq-ckpt", + "/tmp/haozhezhao/model/llamagen_t2i/vq_ds16_t2i.pt", + "--data-path", + "/tmp/haozhezhao/MLLMG/jsonl_data/multiimage_training_for_llava_X2I_UltraEdiit_fourmask.jsonl", + "--dataset", + "ti2i", + "--image-size", + "512", + "--results-dir", + "checkpoint/X2I_UltraEdiit_fourmask_LLava_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_filtered_objects_4mask_1280_context", + "--cloud-save-path", + "/tmp/haozhezhao/MLLMG/checkpoint", + "--lr", + "1e-4", + "--val_data_path", + "/tmp/haozhezhao/MLLMG/jsonl_data/multiimage_val_for_llava_X2I_UltraEdiit_fourmask_dreambenplus.jsonl", + "--use_vision_tower", + "--model_name_or_path", + "/tmp/haozhezhao/model/blip2-flan-t5-xl", + "--image_place_holder", + "", + "--do_eval", + "--eval_steps", + "2000", + "--max_eval_samples", + "128", + "--cfg-scale", + "7.5", + "--top-k", + "16384", + "--load_from_checkpoint", + "/tmp/haozhezhao/MLLMG/checkpoint/FIXed_3M_1epoch_step32000.pt", + "--global-batch-size", + "24", + "--num-workers", + "4", + "--warmup", + "0.05", + "--gradient-accumulation-steps", + "8", + "--train_text_encoder", + "--ckpt-every", + "2000", + "--epochs", + "1", + "--subject_driven", + "--reference_data_path", + "/tmp/haozhezhao/MLLMG/cc12m_reference_tunnel.jsonl", + "--multimodal_encoder", + "llava", + "--do_recovery", + "--find_unused_parameters", + "--cls-token-num", + "1280", + "--train_all", + "--load_fixed_llamagen", + "--save_total_limit", + "1", + "--fix", + "gpt-empty-fix", + "--load_language_projection", + "/tmp/haozhezhao/MLLMG/llava-v1.5-flant5_fixed-pretrain/mm_projector.bin", + "--mm_vision_tower", + "openai/clip-vit-large-patch14", + "--gpt-ckpt", + "/tmp/haozhezhao/MLLMG/checkpoint/Afteri2i_100kckpt_LLAVAT5_empty_Fixed_just_segment_pretrain_stage3_dreambench__recap_400k_t2i_400k_flux_200k_midjourney_150k_recovery_150k_grounding_100fluxseg_50samseg/005-GPT-XL/checkpoints/0078000.pt", + "--dreambench_eval" + ], + "program": "/tmp/haozhezhao/MLLMG/autoregressive/train/train_t2i.py", + "codePath": "autoregressive/train/train_t2i.py", + "email": "mimazhe55360@gmail.com", + "root": "checkpoint/X2I_UltraEdiit_fourmask_LLava_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_filtered_objects_4mask_1280_context", + "host": "447cc403a8794092814259713c51c1df00001X", + "executable": "/tmp/haozhezhao/anaconda3/envs/nlp/bin/python", + "codePathLocal": "autoregressive/train/train_t2i.py", + "cpu_count": 96, + "cpu_count_logical": 96, + "gpu": "NVIDIA A100-SXM4-80GB", + "gpu_count": 8, + "disk": { + "/": { + "total": "133003395072", + "used": "65759305728" + } + }, + "memory": { + "total": "1902387884032" + }, + "cpu": { + "count": 96, + "countLogical": 96 + }, + "gpu_nvidia": [ + { + "name": "NVIDIA A100-SXM4-80GB", + "memoryTotal": "85899345920", + "cudaCores": 6912, + "architecture": "Ampere" + }, + { + "name": "NVIDIA A100-SXM4-80GB", + "memoryTotal": "85899345920", + "cudaCores": 6912, + "architecture": "Ampere" + }, + { + "name": "NVIDIA A100-SXM4-80GB", + "memoryTotal": "85899345920", + "cudaCores": 6912, + "architecture": "Ampere" + }, + { + "name": "NVIDIA A100-SXM4-80GB", + "memoryTotal": "85899345920", + "cudaCores": 6912, + "architecture": "Ampere" + }, + { + "name": "NVIDIA A100-SXM4-80GB", + "memoryTotal": "85899345920", + "cudaCores": 6912, + "architecture": "Ampere" + }, + { + "name": "NVIDIA A100-SXM4-80GB", + "memoryTotal": "85899345920", + "cudaCores": 6912, + "architecture": "Ampere" + }, + { + "name": "NVIDIA A100-SXM4-80GB", + "memoryTotal": "85899345920", + "cudaCores": 6912, + "architecture": "Ampere" + }, + { + "name": "NVIDIA A100-SXM4-80GB", + "memoryTotal": "85899345920", + "cudaCores": 6912, + "architecture": "Ampere" + } + ], + "cudaVersion": "12.2" +} \ No newline at end of file diff --git a/CKPTS/X2I_UltraEdiit_fourmask_LLava_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_filtered_objects_4mask_1280_context/wandb/run-20250426_210323-vtrxdt92/files/wandb-summary.json b/CKPTS/X2I_UltraEdiit_fourmask_LLava_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_filtered_objects_4mask_1280_context/wandb/run-20250426_210323-vtrxdt92/files/wandb-summary.json new file mode 100644 index 0000000000000000000000000000000000000000..4fa4886349293d155e98c50dac202d934a7225b1 --- /dev/null +++ b/CKPTS/X2I_UltraEdiit_fourmask_LLava_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_filtered_objects_4mask_1280_context/wandb/run-20250426_210323-vtrxdt92/files/wandb-summary.json @@ -0,0 +1 @@ +{"train loss":5.624237060546875,"Train Steps/Sec":1.1206378432714594,"_timestamp":1.7458231354667306e+09,"_runtime":121759.010573283,"_step":110675,"_wandb":{"runtime":121759},"train lr":9.875754066995826e-05} \ No newline at end of file diff --git a/CKPTS/X2I_UltraEdiit_fourmask_LLava_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_filtered_objects_4mask_1280_context/wandb/run-20250426_210323-vtrxdt92/logs/debug-core.log b/CKPTS/X2I_UltraEdiit_fourmask_LLava_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_filtered_objects_4mask_1280_context/wandb/run-20250426_210323-vtrxdt92/logs/debug-core.log new file mode 100644 index 0000000000000000000000000000000000000000..e45e27f5be397d70554713e1b6c32d6c5750fbfe --- /dev/null +++ b/CKPTS/X2I_UltraEdiit_fourmask_LLava_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_filtered_objects_4mask_1280_context/wandb/run-20250426_210323-vtrxdt92/logs/debug-core.log @@ -0,0 +1,13 @@ +{"time":"2025-04-26T21:03:23.254775521Z","level":"INFO","msg":"main: starting server","port-filename":"/tmp/tmp3z65w4le/port-3723769.txt","pid":3723769,"log-level":0,"disable-analytics":false,"shutdown-on-parent-exit":false} +{"time":"2025-04-26T21:03:23.256072329Z","level":"INFO","msg":"Will exit if parent process dies.","ppid":3723769} +{"time":"2025-04-26T21:03:23.256063988Z","level":"INFO","msg":"server is running","addr":{"IP":"127.0.0.1","Port":41685,"Zone":""}} +{"time":"2025-04-26T21:03:23.44265469Z","level":"INFO","msg":"connection: ManageConnectionData: new connection created","id":"127.0.0.1:36960"} +{"time":"2025-04-26T21:03:23.738338588Z","level":"INFO","msg":"handleInformInit: received","streamId":"vtrxdt92","id":"127.0.0.1:36960"} +{"time":"2025-04-26T21:03:23.945289824Z","level":"INFO","msg":"handleInformInit: stream started","streamId":"vtrxdt92","id":"127.0.0.1:36960"} +{"time":"2025-04-28T06:52:42.74746049Z","level":"INFO","msg":"handleInformTeardown: server teardown initiated","id":"127.0.0.1:36960"} +{"time":"2025-04-28T06:52:42.747606187Z","level":"INFO","msg":"server is shutting down"} +{"time":"2025-04-28T06:52:42.74759435Z","level":"INFO","msg":"connection: closing","id":"127.0.0.1:36960"} +{"time":"2025-04-28T06:52:42.747753638Z","level":"INFO","msg":"connection: closed successfully","id":"127.0.0.1:36960"} +{"time":"2025-04-28T06:52:44.290799223Z","level":"INFO","msg":"handleInformTeardown: server shutdown complete","id":"127.0.0.1:36960"} +{"time":"2025-04-28T06:52:44.290825041Z","level":"INFO","msg":"connection: ManageConnectionData: connection closed","id":"127.0.0.1:36960"} +{"time":"2025-04-28T06:52:44.290838536Z","level":"INFO","msg":"server is closed"} diff --git a/CKPTS/X2I_UltraEdiit_fourmask_LLava_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_filtered_objects_4mask_1280_context/wandb/run-20250426_210323-vtrxdt92/logs/debug-internal.log b/CKPTS/X2I_UltraEdiit_fourmask_LLava_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_filtered_objects_4mask_1280_context/wandb/run-20250426_210323-vtrxdt92/logs/debug-internal.log new file mode 100644 index 0000000000000000000000000000000000000000..b84e7a539e16b615eccb2774280b7e984207c8bd --- /dev/null +++ b/CKPTS/X2I_UltraEdiit_fourmask_LLava_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_filtered_objects_4mask_1280_context/wandb/run-20250426_210323-vtrxdt92/logs/debug-internal.log @@ -0,0 +1,15 @@ +{"time":"2025-04-26T21:03:23.738486816Z","level":"INFO","msg":"stream: starting","core version":"0.19.8","symlink path":"checkpoint/X2I_UltraEdiit_fourmask_LLava_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_filtered_objects_4mask_1280_context/wandb/run-20250426_210323-vtrxdt92/logs/debug-core.log"} +{"time":"2025-04-26T21:03:23.945252394Z","level":"INFO","msg":"created new stream","id":"vtrxdt92"} +{"time":"2025-04-26T21:03:23.945284534Z","level":"INFO","msg":"stream: started","id":"vtrxdt92"} +{"time":"2025-04-26T21:03:23.945335685Z","level":"INFO","msg":"writer: Do: started","stream_id":"vtrxdt92"} +{"time":"2025-04-26T21:03:23.945496626Z","level":"INFO","msg":"handler: started","stream_id":"vtrxdt92"} +{"time":"2025-04-26T21:03:23.945514981Z","level":"INFO","msg":"sender: started","stream_id":"vtrxdt92"} +{"time":"2025-04-26T21:03:24.215962364Z","level":"INFO","msg":"Starting system monitor"} +{"time":"2025-04-28T06:52:42.747611793Z","level":"INFO","msg":"stream: closing","id":"vtrxdt92"} +{"time":"2025-04-28T06:52:42.747654202Z","level":"INFO","msg":"Stopping system monitor"} +{"time":"2025-04-28T06:52:42.748621161Z","level":"INFO","msg":"Stopped system monitor"} +{"time":"2025-04-28T06:52:44.085866147Z","level":"INFO","msg":"fileTransfer: Close: file transfer manager closed"} +{"time":"2025-04-28T06:52:44.290568215Z","level":"INFO","msg":"handler: closed","stream_id":"vtrxdt92"} +{"time":"2025-04-28T06:52:44.290611926Z","level":"INFO","msg":"sender: closed","stream_id":"vtrxdt92"} +{"time":"2025-04-28T06:52:44.290607003Z","level":"INFO","msg":"writer: Close: closed","stream_id":"vtrxdt92"} +{"time":"2025-04-28T06:52:44.290714634Z","level":"INFO","msg":"stream: closed","id":"vtrxdt92"} diff --git a/CKPTS/X2I_UltraEdiit_fourmask_LLava_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_filtered_objects_4mask_1280_context/wandb/run-20250426_210323-vtrxdt92/logs/debug.log b/CKPTS/X2I_UltraEdiit_fourmask_LLava_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_filtered_objects_4mask_1280_context/wandb/run-20250426_210323-vtrxdt92/logs/debug.log new file mode 100644 index 0000000000000000000000000000000000000000..d9a06495e04d3aebe3d21430ae11c2af92cdf189 --- /dev/null +++ b/CKPTS/X2I_UltraEdiit_fourmask_LLava_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_filtered_objects_4mask_1280_context/wandb/run-20250426_210323-vtrxdt92/logs/debug.log @@ -0,0 +1,23 @@ +2025-04-26 21:03:23,731 INFO MainThread:3723769 [wandb_setup.py:_flush():67] Current SDK version is 0.19.8 +2025-04-26 21:03:23,731 INFO MainThread:3723769 [wandb_setup.py:_flush():67] Configure stats pid to 3723769 +2025-04-26 21:03:23,731 INFO MainThread:3723769 [wandb_setup.py:_flush():67] Loading settings from /tmp/haozhezhao/.config/wandb/settings +2025-04-26 21:03:23,731 INFO MainThread:3723769 [wandb_setup.py:_flush():67] Loading settings from /tmp/haozhezhao/MLLMG/wandb/settings +2025-04-26 21:03:23,731 INFO MainThread:3723769 [wandb_setup.py:_flush():67] Loading settings from environment variables +2025-04-26 21:03:23,731 INFO MainThread:3723769 [wandb_init.py:setup_run_log_directory():647] Logging user logs to checkpoint/X2I_UltraEdiit_fourmask_LLava_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_filtered_objects_4mask_1280_context/wandb/run-20250426_210323-vtrxdt92/logs/debug.log +2025-04-26 21:03:23,731 INFO MainThread:3723769 [wandb_init.py:setup_run_log_directory():648] Logging internal logs to checkpoint/X2I_UltraEdiit_fourmask_LLava_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_filtered_objects_4mask_1280_context/wandb/run-20250426_210323-vtrxdt92/logs/debug-internal.log +2025-04-26 21:03:23,731 INFO MainThread:3723769 [wandb_init.py:init():761] calling init triggers +2025-04-26 21:03:23,731 INFO MainThread:3723769 [wandb_init.py:init():766] wandb.init called with sweep_config: {} +config: {'data_path': '/tmp/haozhezhao/MLLMG/jsonl_data/multiimage_training_for_llava_X2I_UltraEdiit_fourmask.jsonl', 'cloud_save_path': '/tmp/haozhezhao/MLLMG/checkpoint', 'no_local_save': False, 'vq_model': 'VQ-16', 'vq_ckpt': '/tmp/haozhezhao/model/llamagen_t2i/vq_ds16_t2i.pt', 'codebook_size': 16384, 'codebook_embed_dim': 8, 'gpt_model': 'GPT-XL', 'gpt_ckpt': '/tmp/haozhezhao/MLLMG/checkpoint/Afteri2i_100kckpt_LLAVAT5_empty_Fixed_just_segment_pretrain_stage3_dreambench__recap_400k_t2i_400k_flux_200k_midjourney_150k_recovery_150k_grounding_100fluxseg_50samseg/005-GPT-XL/checkpoints/0078000.pt', 'gpt_type': 't2i', 'vocab_size': 16384, 'cls_token_num': 1280, 'dropout_p': 0.1, 'token_dropout_p': 0.1, 'drop_path': 0.0, 'no_compile': False, 'results_dir': 'checkpoint/X2I_UltraEdiit_fourmask_LLava_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_filtered_objects_4mask_1280_context', 'dataset': 'ti2i', 'image_size': 512, 'downsample_size': 16, 'num_classes': 1000, 'epochs': 1, 'lr': 0.0001, 'weight_decay': 0.05, 'beta1': 0.9, 'beta2': 0.95, 'max_grad_norm': 1.0, 'global_batch_size': 24, 'global_seed': 0, 'num_workers': 4, 'log_every': 25, 'ckpt_every': 2000, 'gradient_accumulation_steps': 8, 'mixed_precision': 'bf16', 'val_data_path': '/tmp/haozhezhao/MLLMG/jsonl_data/multiimage_val_for_llava_X2I_UltraEdiit_fourmask_dreambenplus.jsonl', 'use_vision_tower': True, 'model_name_or_path': '/tmp/haozhezhao/model/blip2-flan-t5-xl', 'image_place_holder': '', 'processor_path': None, 'do_eval': True, 'max_eval_samples': 128, 'train_text_encoder': True, 'no_left_padding': False, 'cfg_scale': 7.5, 'top_k': 16384, 'temperature': 0.9, 'top_p': 1.0, 'eval_steps': 2000, 'project_name': 'llamagen_ti2i', 'load_from_checkpoint': '/tmp/haozhezhao/MLLMG/checkpoint/FIXed_3M_1epoch_step32000.pt', 'warmup': 0.05, 'lr_decay_style': 'cosine', 'lr_decay_ratio': 0.1, 'train_iters': 500000, 'class_dropout_prob': 0.1, 'with_image_only': False, 'image_only_rate': 0.1, 'stage2': False, 'subject_driven': True, 'load_subject_embedding': None, 'reference_data_path': '/tmp/haozhezhao/MLLMG/cc12m_reference_tunnel.jsonl', 'multimodal_encoder': 'llava', 'do_recovery': True, 'no_replace': False, 'resume': False, 'dreambench_eval': True, 'find_unused_parameters': True, 'load_visual_encoder': False, 'continue_stage1': False, 'replace_subject': False, 'train_all': True, 'save_total_limit': 1, 'load_language_projection': '/tmp/haozhezhao/MLLMG/llava-v1.5-flant5_fixed-pretrain/mm_projector.bin', 'mm_vision_tower': 'openai/clip-vit-large-patch14', 'load_fixed_llamagen': True, 'unfreeze_output': False, 'fix': 'gpt-empty-fix', 'rank': 0, 'world_size': 8, 'gpu': 0, 'dist_url': 'env://', 'distributed': True, 'dist_backend': 'nccl', '_wandb': {}} +2025-04-26 21:03:23,732 INFO MainThread:3723769 [wandb_init.py:init():784] starting backend +2025-04-26 21:03:23,732 INFO MainThread:3723769 [wandb_init.py:init():788] sending inform_init request +2025-04-26 21:03:23,736 INFO MainThread:3723769 [backend.py:_multiprocessing_setup():101] multiprocessing start_methods=fork,spawn,forkserver, using: spawn +2025-04-26 21:03:23,736 INFO MainThread:3723769 [wandb_init.py:init():798] backend started and connected +2025-04-26 21:03:23,738 INFO MainThread:3723769 [wandb_init.py:init():891] updated telemetry +2025-04-26 21:03:23,739 INFO MainThread:3723769 [wandb_init.py:init():915] communicating run to backend with 90.0 second timeout +2025-04-26 21:03:24,213 INFO MainThread:3723769 [wandb_init.py:init():990] starting run threads in backend +2025-04-26 21:03:24,296 INFO MainThread:3723769 [wandb_run.py:_console_start():2375] atexit reg +2025-04-26 21:03:24,296 INFO MainThread:3723769 [wandb_run.py:_redirect():2227] redirect: wrap_raw +2025-04-26 21:03:24,297 INFO MainThread:3723769 [wandb_run.py:_redirect():2292] Wrapping output streams. +2025-04-26 21:03:24,297 INFO MainThread:3723769 [wandb_run.py:_redirect():2315] Redirects installed. +2025-04-26 21:03:24,298 INFO MainThread:3723769 [wandb_init.py:init():1032] run started, returning control to user process +2025-04-28 06:52:42,746 INFO MsgRouterThr:3723769 [mailbox.py:close():129] Closing mailbox, abandoning 1 handles. diff --git a/CKPTS/X2I_UltraEdiit_fourmask_LLava_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_filtered_objects_4mask_1280_context/wandb/run-20250426_210323-vtrxdt92/run-vtrxdt92.wandb b/CKPTS/X2I_UltraEdiit_fourmask_LLava_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_filtered_objects_4mask_1280_context/wandb/run-20250426_210323-vtrxdt92/run-vtrxdt92.wandb new file mode 100644 index 0000000000000000000000000000000000000000..ab7fe8efbdc6f6f62339e02b815dfbd860083abb --- /dev/null +++ b/CKPTS/X2I_UltraEdiit_fourmask_LLava_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_filtered_objects_4mask_1280_context/wandb/run-20250426_210323-vtrxdt92/run-vtrxdt92.wandb @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:17c6f47ec4d5c59d91f5148ba2e2a67821fe942438f5906e71a9628d0e45573d +size 71948553 diff --git a/CKPTS/X2I_UltraEdiit_fourmask_LLava_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_filtered_objects_4mask_1280_context_after_stage1/000-GPT-XL/checkpoints/eval_step_2000/batch_0_cfg_7.5_topk_16384.jpg b/CKPTS/X2I_UltraEdiit_fourmask_LLava_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_filtered_objects_4mask_1280_context_after_stage1/000-GPT-XL/checkpoints/eval_step_2000/batch_0_cfg_7.5_topk_16384.jpg new file mode 100644 index 0000000000000000000000000000000000000000..f6458282052e82007f025ed6662bedf29cf0eb1c --- /dev/null +++ b/CKPTS/X2I_UltraEdiit_fourmask_LLava_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_filtered_objects_4mask_1280_context_after_stage1/000-GPT-XL/checkpoints/eval_step_2000/batch_0_cfg_7.5_topk_16384.jpg @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:dc2ea2f4c4174cf06d801449ce760decd82eddc498b91f330c1b62c338cc2658 +size 351167 diff --git a/CKPTS/X2I_UltraEdiit_fourmask_LLava_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_filtered_objects_4mask_1280_context_after_stage1/000-GPT-XL/checkpoints/eval_step_2000/batch_1_cfg_7.5_topk_16384.jpg b/CKPTS/X2I_UltraEdiit_fourmask_LLava_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_filtered_objects_4mask_1280_context_after_stage1/000-GPT-XL/checkpoints/eval_step_2000/batch_1_cfg_7.5_topk_16384.jpg new file mode 100644 index 0000000000000000000000000000000000000000..1ffa6619792941eb70c56d14be03ff034c52e248 --- /dev/null +++ b/CKPTS/X2I_UltraEdiit_fourmask_LLava_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_filtered_objects_4mask_1280_context_after_stage1/000-GPT-XL/checkpoints/eval_step_2000/batch_1_cfg_7.5_topk_16384.jpg @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:bcd085d3ec93731b50c083292bf313f353644c1753f541d4c2b0f0ed5af652ad +size 294566 diff --git a/CKPTS/X2I_UltraEdiit_fourmask_LLava_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_filtered_objects_4mask_1280_context_after_stage1/000-GPT-XL/checkpoints/eval_step_2000/batch_2_cfg_7.5_topk_16384.jpg b/CKPTS/X2I_UltraEdiit_fourmask_LLava_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_filtered_objects_4mask_1280_context_after_stage1/000-GPT-XL/checkpoints/eval_step_2000/batch_2_cfg_7.5_topk_16384.jpg new file mode 100644 index 0000000000000000000000000000000000000000..87da81aabfe9365077ffa6b3a79b6fce4756ec7b --- /dev/null +++ b/CKPTS/X2I_UltraEdiit_fourmask_LLava_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_filtered_objects_4mask_1280_context_after_stage1/000-GPT-XL/checkpoints/eval_step_2000/batch_2_cfg_7.5_topk_16384.jpg @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:56007acbb67f5476355fde205f1da09121b74e05e0baead4528fe9e94c4032c3 +size 312571 diff --git a/CKPTS/X2I_UltraEdiit_fourmask_LLava_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_filtered_objects_4mask_1280_context_after_stage1/000-GPT-XL/checkpoints/eval_step_2000/batch_3_cfg_7.5_topk_16384.jpg b/CKPTS/X2I_UltraEdiit_fourmask_LLava_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_filtered_objects_4mask_1280_context_after_stage1/000-GPT-XL/checkpoints/eval_step_2000/batch_3_cfg_7.5_topk_16384.jpg new file mode 100644 index 0000000000000000000000000000000000000000..8f525304cd9bfc32e0c69e3c7a963d4d23b9393f --- /dev/null +++ b/CKPTS/X2I_UltraEdiit_fourmask_LLava_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_filtered_objects_4mask_1280_context_after_stage1/000-GPT-XL/checkpoints/eval_step_2000/batch_3_cfg_7.5_topk_16384.jpg @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:43f9ce0363b3503508ee1eeb0a7e86f6cec8a388db1d0cc782a9173b8083deed +size 356787 diff --git a/CKPTS/X2I_UltraEdiit_fourmask_LLava_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_filtered_objects_4mask_1280_context_after_stage1/000-GPT-XL/checkpoints/eval_step_2000/batch_4_cfg_7.5_topk_16384.jpg b/CKPTS/X2I_UltraEdiit_fourmask_LLava_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_filtered_objects_4mask_1280_context_after_stage1/000-GPT-XL/checkpoints/eval_step_2000/batch_4_cfg_7.5_topk_16384.jpg new file mode 100644 index 0000000000000000000000000000000000000000..ef0c8d6da34ab8e86d893c3523e6321499f6a7d4 --- /dev/null +++ b/CKPTS/X2I_UltraEdiit_fourmask_LLava_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_filtered_objects_4mask_1280_context_after_stage1/000-GPT-XL/checkpoints/eval_step_2000/batch_4_cfg_7.5_topk_16384.jpg @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:ef6b5340d22860dee0adffba8f9cacdc0ff4c17d755c58fa779238d9946132e9 +size 391554 diff --git a/CKPTS/X2I_UltraEdiit_fourmask_LLava_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_filtered_objects_4mask_1280_context_after_stage1/000-GPT-XL/checkpoints/eval_step_2000/batch_5_cfg_7.5_topk_16384.jpg b/CKPTS/X2I_UltraEdiit_fourmask_LLava_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_filtered_objects_4mask_1280_context_after_stage1/000-GPT-XL/checkpoints/eval_step_2000/batch_5_cfg_7.5_topk_16384.jpg new file mode 100644 index 0000000000000000000000000000000000000000..2dc5552719c78703104582f21f3ab1a14333f564 --- /dev/null +++ b/CKPTS/X2I_UltraEdiit_fourmask_LLava_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_filtered_objects_4mask_1280_context_after_stage1/000-GPT-XL/checkpoints/eval_step_2000/batch_5_cfg_7.5_topk_16384.jpg @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:f73655406617b7ba830fefcff88449ae4b3ad18c594b4551c203bc024aad1338 +size 334767 diff --git a/CKPTS/X2I_UltraEdiit_fourmask_LLava_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_filtered_objects_4mask_1280_context_after_stage1/000-GPT-XL/checkpoints/eval_step_4000/batch_0_cfg_7.5_topk_16384.jpg b/CKPTS/X2I_UltraEdiit_fourmask_LLava_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_filtered_objects_4mask_1280_context_after_stage1/000-GPT-XL/checkpoints/eval_step_4000/batch_0_cfg_7.5_topk_16384.jpg new file mode 100644 index 0000000000000000000000000000000000000000..8e3a1031ba9ffd3c7d92a7d2f9e40f9931e5953f --- /dev/null +++ b/CKPTS/X2I_UltraEdiit_fourmask_LLava_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_filtered_objects_4mask_1280_context_after_stage1/000-GPT-XL/checkpoints/eval_step_4000/batch_0_cfg_7.5_topk_16384.jpg @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:e1a41a70813607088fc3302fdf74e034bd757d1980be91301b72f97f6a408c47 +size 389695 diff --git a/CKPTS/X2I_UltraEdiit_fourmask_LLava_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_filtered_objects_4mask_1280_context_after_stage1/000-GPT-XL/checkpoints/eval_step_4000/batch_1_cfg_7.5_topk_16384.jpg b/CKPTS/X2I_UltraEdiit_fourmask_LLava_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_filtered_objects_4mask_1280_context_after_stage1/000-GPT-XL/checkpoints/eval_step_4000/batch_1_cfg_7.5_topk_16384.jpg new file mode 100644 index 0000000000000000000000000000000000000000..f9a11c3a15f5a1259d84d2d3c75c8a60e3d29d0c --- /dev/null +++ b/CKPTS/X2I_UltraEdiit_fourmask_LLava_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_filtered_objects_4mask_1280_context_after_stage1/000-GPT-XL/checkpoints/eval_step_4000/batch_1_cfg_7.5_topk_16384.jpg @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:a9e36998400fe416fb8f2ef52a76db10893078c15f02f5e7c729b7894e715af8 +size 298773 diff --git a/CKPTS/X2I_UltraEdiit_fourmask_LLava_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_filtered_objects_4mask_1280_context_after_stage1/000-GPT-XL/checkpoints/eval_step_4000/batch_2_cfg_7.5_topk_16384.jpg b/CKPTS/X2I_UltraEdiit_fourmask_LLava_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_filtered_objects_4mask_1280_context_after_stage1/000-GPT-XL/checkpoints/eval_step_4000/batch_2_cfg_7.5_topk_16384.jpg new file mode 100644 index 0000000000000000000000000000000000000000..1b6280905ee80755ba4f3cfe59bff0774bfd199b --- /dev/null +++ b/CKPTS/X2I_UltraEdiit_fourmask_LLava_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_filtered_objects_4mask_1280_context_after_stage1/000-GPT-XL/checkpoints/eval_step_4000/batch_2_cfg_7.5_topk_16384.jpg @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:0b41fba67f02622504d4f089449e24acb41d24cdd9b4ba055a0c39b84c58ea48 +size 376108 diff --git a/CKPTS/X2I_UltraEdiit_fourmask_LLava_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_filtered_objects_4mask_1280_context_after_stage1/000-GPT-XL/checkpoints/eval_step_4000/batch_3_cfg_7.5_topk_16384.jpg b/CKPTS/X2I_UltraEdiit_fourmask_LLava_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_filtered_objects_4mask_1280_context_after_stage1/000-GPT-XL/checkpoints/eval_step_4000/batch_3_cfg_7.5_topk_16384.jpg new file mode 100644 index 0000000000000000000000000000000000000000..c09e916a3ec4d494e26336b7ef827187ffd4b7d0 --- /dev/null +++ b/CKPTS/X2I_UltraEdiit_fourmask_LLava_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_filtered_objects_4mask_1280_context_after_stage1/000-GPT-XL/checkpoints/eval_step_4000/batch_3_cfg_7.5_topk_16384.jpg @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:813c62d6f6081b9ba04b47e6fc69c71468ba69a3a11d69cb33fd69a1c29df64d +size 373538 diff --git a/CKPTS/X2I_UltraEdiit_fourmask_LLava_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_filtered_objects_4mask_1280_context_after_stage1/000-GPT-XL/checkpoints/eval_step_4000/batch_4_cfg_7.5_topk_16384.jpg b/CKPTS/X2I_UltraEdiit_fourmask_LLava_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_filtered_objects_4mask_1280_context_after_stage1/000-GPT-XL/checkpoints/eval_step_4000/batch_4_cfg_7.5_topk_16384.jpg new file mode 100644 index 0000000000000000000000000000000000000000..05df3061bbaaca500ad855775b8c2b8a8970f8d8 --- /dev/null +++ b/CKPTS/X2I_UltraEdiit_fourmask_LLava_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_filtered_objects_4mask_1280_context_after_stage1/000-GPT-XL/checkpoints/eval_step_4000/batch_4_cfg_7.5_topk_16384.jpg @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:67f8e0026dd63934cd9aa33ed2e76ee8d23eea011bf2584df5aecfda00af1800 +size 399940 diff --git a/CKPTS/X2I_UltraEdiit_fourmask_LLava_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_filtered_objects_4mask_1280_context_after_stage1/000-GPT-XL/checkpoints/eval_step_4000/batch_5_cfg_7.5_topk_16384.jpg b/CKPTS/X2I_UltraEdiit_fourmask_LLava_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_filtered_objects_4mask_1280_context_after_stage1/000-GPT-XL/checkpoints/eval_step_4000/batch_5_cfg_7.5_topk_16384.jpg new file mode 100644 index 0000000000000000000000000000000000000000..6e90b1acc40ff9435f9131e9b50b0093bc04f2eb --- /dev/null +++ b/CKPTS/X2I_UltraEdiit_fourmask_LLava_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_filtered_objects_4mask_1280_context_after_stage1/000-GPT-XL/checkpoints/eval_step_4000/batch_5_cfg_7.5_topk_16384.jpg @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:1016aa4556678e8aa402f0fdf8486dfc3f88b5e5d10d241e390ea0cd852db474 +size 394092 diff --git a/CKPTS/X2I_UltraEdiit_fourmask_LLava_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_filtered_objects_4mask_1280_context_after_stage1/000-GPT-XL/log.txt b/CKPTS/X2I_UltraEdiit_fourmask_LLava_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_filtered_objects_4mask_1280_context_after_stage1/000-GPT-XL/log.txt new file mode 100644 index 0000000000000000000000000000000000000000..1db854f75e61c4831397736ed29dde7f4990118d --- /dev/null +++ b/CKPTS/X2I_UltraEdiit_fourmask_LLava_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_filtered_objects_4mask_1280_context_after_stage1/000-GPT-XL/log.txt @@ -0,0 +1,210 @@ +[2025-04-28 08:49:15] Experiment directory created at checkpoint/X2I_UltraEdiit_fourmask_LLava_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_filtered_objects_4mask_1280_context_after_stage1/000-GPT-XL +[2025-04-28 08:49:15] Namespace(data_path='/tmp/haozhezhao/MLLMG/jsonl_data/multiimage_training_for_llava_X2I_fourmask.jsonl', cloud_save_path='/tmp/haozhezhao/MLLMG/checkpoint', no_local_save=False, vq_model='VQ-16', vq_ckpt='/tmp/haozhezhao/model/llamagen_t2i/vq_ds16_t2i.pt', codebook_size=16384, codebook_embed_dim=8, gpt_model='GPT-XL', gpt_ckpt='/tmp/haozhezhao/MLLMG/MLLMG_ckpts/checkpoints/AfterI2I_llavat5_subject_train_all_T5_subject_t2i_ti2i_120_w_flux_segment_noimageonly_empty_fix_train_all/0100000.pt', gpt_type='t2i', vocab_size=16384, cls_token_num=1280, dropout_p=0.1, token_dropout_p=0.1, drop_path=0.0, no_compile=False, results_dir='checkpoint/X2I_UltraEdiit_fourmask_LLava_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_filtered_objects_4mask_1280_context_after_stage1', dataset='ti2i', image_size=512, downsample_size=16, num_classes=1000, epochs=1, lr=0.0001, weight_decay=0.05, beta1=0.9, beta2=0.95, max_grad_norm=1.0, global_batch_size=24, global_seed=0, num_workers=4, log_every=25, ckpt_every=2000, gradient_accumulation_steps=8, mixed_precision='bf16', val_data_path='/tmp/haozhezhao/MLLMG/jsonl_data/multiimage_val_for_llava_X2I_fourmask_dreambenplus.jsonl', use_vision_tower=True, model_name_or_path='/tmp/haozhezhao/model/blip2-flan-t5-xl', image_place_holder='', processor_path=None, do_eval=True, max_eval_samples=128, train_text_encoder=True, no_left_padding=False, cfg_scale=7.5, top_k=16384, temperature=0.9, top_p=1.0, eval_steps=2000, project_name='llamagen_ti2i', load_from_checkpoint='/tmp/haozhezhao/MLLMG/checkpoint/FIXed_3M_1epoch_step32000.pt', warmup=0.05, lr_decay_style='cosine', lr_decay_ratio=0.1, train_iters=500000, class_dropout_prob=0.1, with_image_only=False, image_only_rate=0.1, stage2=False, subject_driven=True, load_subject_embedding=None, reference_data_path='/tmp/haozhezhao/MLLMG/cc12m_reference_tunnel.jsonl', multimodal_encoder='llava', do_recovery=True, no_replace=False, resume=False, dreambench_eval=True, find_unused_parameters=True, load_visual_encoder=False, continue_stage1=False, replace_subject=False, train_all=True, save_total_limit=1, load_language_projection='/tmp/haozhezhao/MLLMG/llava-v1.5-flant5_fixed-pretrain/mm_projector.bin', mm_vision_tower='openai/clip-vit-large-patch14', load_fixed_llamagen=True, unfreeze_output=False, fix='gpt-empty-fix', rank=0, world_size=8, gpu=0, dist_url='env://', distributed=True, dist_backend='nccl') +[2025-04-28 08:49:15] Starting rank=0, seed=0, world_size=8. +[2025-04-28 08:49:15] text_config is None. Initializing the text config with default values (`OPTConfig`). +[2025-04-28 08:50:21] GPT Parameters: 2,310,680,832 +[2025-04-28 08:50:21] num decayed parameter tensors: 356, with 2,007,303,168 parameters +[2025-04-28 08:50:21] num non-decayed parameter tensors: 124, with 197,888 parameters +[2025-04-28 08:50:21] using fused AdamW: True +[2025-04-28 08:50:51] Dataset contains 2,157,728 images +[2025-04-28 08:50:51] Train iters 89905 , warmup 4495.25, len of loader 89905 +[2025-04-28 08:51:22] ### LOAD pretraining weights from checkpoint: /tmp/haozhezhao/MLLMG/MLLMG_ckpts/checkpoints/AfterI2I_llavat5_subject_train_all_T5_subject_t2i_ti2i_120_w_flux_segment_noimageonly_empty_fix_train_all/0100000.pt +[2025-04-28 08:51:22] Initial state: steps=0, epochs=0 +[2025-04-28 08:51:22] compiling the model... (may take several minutes) +[2025-04-28 08:51:23] freeze the vit +[2025-04-28 08:51:23] ***** total param is 2310680832 ***** +[2025-04-28 08:51:23] ***** total trained param is 2007501056 ***** +[2025-04-28 08:51:28] Training for 1 epochs... +[2025-04-28 08:51:28] Beginning epoch 0... +[2025-04-28 08:56:49] (step=0000025) Train Loss: 8.1098, Train Steps/Sec: 0.08 +[2025-04-28 08:57:12] (step=0000050) Train Loss: 8.1527, Train Steps/Sec: 1.11 +[2025-04-28 08:57:34] (step=0000075) Train Loss: 8.0368, Train Steps/Sec: 1.10 +[2025-04-28 08:57:57] (step=0000100) Train Loss: 8.0892, Train Steps/Sec: 1.12 +[2025-04-28 08:58:19] (step=0000125) Train Loss: 8.0331, Train Steps/Sec: 1.12 +[2025-04-28 08:58:41] (step=0000150) Train Loss: 8.1135, Train Steps/Sec: 1.12 +[2025-04-28 08:59:04] (step=0000175) Train Loss: 8.0643, Train Steps/Sec: 1.12 +[2025-04-28 08:59:26] (step=0000200) Train Loss: 8.0926, Train Steps/Sec: 1.11 +[2025-04-28 08:59:48] (step=0000225) Train Loss: 7.9835, Train Steps/Sec: 1.12 +[2025-04-28 09:00:11] (step=0000250) Train Loss: 7.9569, Train Steps/Sec: 1.12 +[2025-04-28 09:00:33] (step=0000275) Train Loss: 7.9222, Train Steps/Sec: 1.12 +[2025-04-28 09:00:56] (step=0000300) Train Loss: 7.9842, Train Steps/Sec: 1.11 +[2025-04-28 09:01:18] (step=0000325) Train Loss: 7.8966, Train Steps/Sec: 1.11 +[2025-04-28 09:01:41] (step=0000350) Train Loss: 7.8805, Train Steps/Sec: 1.12 +[2025-04-28 09:02:03] (step=0000375) Train Loss: 7.9262, Train Steps/Sec: 1.12 +[2025-04-28 09:02:25] (step=0000400) Train Loss: 7.7872, Train Steps/Sec: 1.11 +[2025-04-28 09:02:48] (step=0000425) Train Loss: 7.7399, Train Steps/Sec: 1.12 +[2025-04-28 09:03:10] (step=0000450) Train Loss: 7.7485, Train Steps/Sec: 1.12 +[2025-04-28 09:03:32] (step=0000475) Train Loss: 7.6619, Train Steps/Sec: 1.12 +[2025-04-28 09:03:55] (step=0000500) Train Loss: 7.5847, Train Steps/Sec: 1.12 +[2025-04-28 09:04:17] (step=0000525) Train Loss: 7.5771, Train Steps/Sec: 1.12 +[2025-04-28 09:04:39] (step=0000550) Train Loss: 7.5592, Train Steps/Sec: 1.12 +[2025-04-28 09:05:02] (step=0000575) Train Loss: 7.5491, Train Steps/Sec: 1.12 +[2025-04-28 09:05:24] (step=0000600) Train Loss: 7.3793, Train Steps/Sec: 1.11 +[2025-04-28 09:05:47] (step=0000625) Train Loss: 7.4401, Train Steps/Sec: 1.12 +[2025-04-28 09:06:09] (step=0000650) Train Loss: 7.4419, Train Steps/Sec: 1.12 +[2025-04-28 09:06:31] (step=0000675) Train Loss: 7.4914, Train Steps/Sec: 1.12 +[2025-04-28 09:06:54] (step=0000700) Train Loss: 7.3870, Train Steps/Sec: 1.12 +[2025-04-28 09:07:16] (step=0000725) Train Loss: 7.2848, Train Steps/Sec: 1.12 +[2025-04-28 09:07:38] (step=0000750) Train Loss: 7.3533, Train Steps/Sec: 1.12 +[2025-04-28 09:08:01] (step=0000775) Train Loss: 7.2940, Train Steps/Sec: 1.12 +[2025-04-28 09:08:23] (step=0000800) Train Loss: 7.1500, Train Steps/Sec: 1.11 +[2025-04-28 09:08:46] (step=0000825) Train Loss: 7.1671, Train Steps/Sec: 1.12 +[2025-04-28 09:09:08] (step=0000850) Train Loss: 7.1889, Train Steps/Sec: 1.12 +[2025-04-28 09:09:30] (step=0000875) Train Loss: 7.1471, Train Steps/Sec: 1.12 +[2025-04-28 09:09:52] (step=0000900) Train Loss: 7.0972, Train Steps/Sec: 1.12 +[2025-04-28 09:10:15] (step=0000925) Train Loss: 7.0956, Train Steps/Sec: 1.12 +[2025-04-28 09:10:37] (step=0000950) Train Loss: 6.9854, Train Steps/Sec: 1.12 +[2025-04-28 09:11:00] (step=0000975) Train Loss: 7.0482, Train Steps/Sec: 1.12 +[2025-04-28 09:11:22] (step=0001000) Train Loss: 7.0426, Train Steps/Sec: 1.11 +[2025-04-28 09:11:55] (step=0001025) Train Loss: 6.9699, Train Steps/Sec: 0.76 +[2025-04-28 09:12:44] (step=0001050) Train Loss: 6.9854, Train Steps/Sec: 0.51 +[2025-04-28 09:13:17] (step=0001075) Train Loss: 6.9727, Train Steps/Sec: 0.77 +[2025-04-28 09:13:39] (step=0001100) Train Loss: 6.9168, Train Steps/Sec: 1.12 +[2025-04-28 09:14:01] (step=0001125) Train Loss: 6.9377, Train Steps/Sec: 1.12 +[2025-04-28 09:14:24] (step=0001150) Train Loss: 6.9271, Train Steps/Sec: 1.12 +[2025-04-28 09:14:46] (step=0001175) Train Loss: 6.9455, Train Steps/Sec: 1.12 +[2025-04-28 09:15:08] (step=0001200) Train Loss: 6.8419, Train Steps/Sec: 1.11 +[2025-04-28 09:15:31] (step=0001225) Train Loss: 6.8882, Train Steps/Sec: 1.12 +[2025-04-28 09:15:53] (step=0001250) Train Loss: 6.8683, Train Steps/Sec: 1.12 +[2025-04-28 09:16:15] (step=0001275) Train Loss: 6.8570, Train Steps/Sec: 1.12 +[2025-04-28 09:16:48] (step=0001300) Train Loss: 6.7967, Train Steps/Sec: 0.77 +[2025-04-28 09:17:20] (step=0001325) Train Loss: 6.8396, Train Steps/Sec: 0.78 +[2025-04-28 09:17:52] (step=0001350) Train Loss: 6.7753, Train Steps/Sec: 0.78 +[2025-04-28 09:18:24] (step=0001375) Train Loss: 6.8184, Train Steps/Sec: 0.78 +[2025-04-28 09:18:46] (step=0001400) Train Loss: 6.7812, Train Steps/Sec: 1.12 +[2025-04-28 09:19:09] (step=0001425) Train Loss: 6.7392, Train Steps/Sec: 1.12 +[2025-04-28 09:19:31] (step=0001450) Train Loss: 6.8378, Train Steps/Sec: 1.12 +[2025-04-28 09:19:53] (step=0001475) Train Loss: 6.7880, Train Steps/Sec: 1.12 +[2025-04-28 09:20:16] (step=0001500) Train Loss: 6.7570, Train Steps/Sec: 1.12 +[2025-04-28 09:20:38] (step=0001525) Train Loss: 6.7325, Train Steps/Sec: 1.12 +[2025-04-28 09:21:00] (step=0001550) Train Loss: 6.7641, Train Steps/Sec: 1.12 +[2025-04-28 09:21:22] (step=0001575) Train Loss: 6.8103, Train Steps/Sec: 1.12 +[2025-04-28 09:21:45] (step=0001600) Train Loss: 6.7041, Train Steps/Sec: 1.12 +[2025-04-28 09:22:07] (step=0001625) Train Loss: 6.7932, Train Steps/Sec: 1.12 +[2025-04-28 09:22:29] (step=0001650) Train Loss: 6.7049, Train Steps/Sec: 1.12 +[2025-04-28 09:22:52] (step=0001675) Train Loss: 6.7478, Train Steps/Sec: 1.12 +[2025-04-28 09:23:14] (step=0001700) Train Loss: 6.6683, Train Steps/Sec: 1.12 +[2025-04-28 09:23:36] (step=0001725) Train Loss: 6.7006, Train Steps/Sec: 1.12 +[2025-04-28 09:23:59] (step=0001750) Train Loss: 6.7351, Train Steps/Sec: 1.12 +[2025-04-28 09:24:21] (step=0001775) Train Loss: 6.7044, Train Steps/Sec: 1.12 +[2025-04-28 09:24:44] (step=0001800) Train Loss: 6.7295, Train Steps/Sec: 1.11 +[2025-04-28 09:25:06] (step=0001825) Train Loss: 6.6875, Train Steps/Sec: 1.12 +[2025-04-28 09:25:28] (step=0001850) Train Loss: 6.6428, Train Steps/Sec: 1.12 +[2025-04-28 09:25:50] (step=0001875) Train Loss: 6.6406, Train Steps/Sec: 1.12 +[2025-04-28 09:26:13] (step=0001900) Train Loss: 6.6806, Train Steps/Sec: 1.12 +[2025-04-28 09:26:35] (step=0001925) Train Loss: 6.6908, Train Steps/Sec: 1.12 +[2025-04-28 09:26:57] (step=0001950) Train Loss: 6.6597, Train Steps/Sec: 1.12 +[2025-04-28 09:27:20] (step=0001975) Train Loss: 6.6953, Train Steps/Sec: 1.12 +[2025-04-28 09:27:42] (step=0002000) Train Loss: 6.6593, Train Steps/Sec: 1.12 +[2025-04-28 09:27:42] text_config is None. Initializing the text config with default values (`OPTConfig`). +[2025-04-28 09:34:50] Finish Eval in 2000 steps... +[2025-04-28 09:35:09] Saved checkpoint to checkpoint/X2I_UltraEdiit_fourmask_LLava_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_filtered_objects_4mask_1280_context_after_stage1/000-GPT-XL/checkpoints/0002000.pt +[2025-04-28 09:35:32] (step=0002025) Train Loss: 6.6644, Train Steps/Sec: 0.05 +[2025-04-28 09:35:55] (step=0002050) Train Loss: 6.6933, Train Steps/Sec: 1.12 +[2025-04-28 09:36:17] (step=0002075) Train Loss: 6.6850, Train Steps/Sec: 1.12 +[2025-04-28 09:36:39] (step=0002100) Train Loss: 6.7265, Train Steps/Sec: 1.12 +[2025-04-28 09:37:01] (step=0002125) Train Loss: 6.6219, Train Steps/Sec: 1.12 +[2025-04-28 09:37:24] (step=0002150) Train Loss: 6.6068, Train Steps/Sec: 1.12 +[2025-04-28 09:37:46] (step=0002175) Train Loss: 6.6236, Train Steps/Sec: 1.11 +[2025-04-28 09:38:09] (step=0002200) Train Loss: 6.6728, Train Steps/Sec: 1.11 +[2025-04-28 09:38:31] (step=0002225) Train Loss: 6.6412, Train Steps/Sec: 1.12 +[2025-04-28 09:38:54] (step=0002250) Train Loss: 6.5835, Train Steps/Sec: 1.12 +[2025-04-28 09:39:16] (step=0002275) Train Loss: 6.6870, Train Steps/Sec: 1.12 +[2025-04-28 09:39:38] (step=0002300) Train Loss: 6.7122, Train Steps/Sec: 1.12 +[2025-04-28 09:40:01] (step=0002325) Train Loss: 6.5762, Train Steps/Sec: 1.12 +[2025-04-28 09:40:23] (step=0002350) Train Loss: 6.6358, Train Steps/Sec: 1.10 +[2025-04-28 09:40:46] (step=0002375) Train Loss: 6.6176, Train Steps/Sec: 1.12 +[2025-04-28 09:41:08] (step=0002400) Train Loss: 6.6492, Train Steps/Sec: 1.11 +[2025-04-28 09:41:30] (step=0002425) Train Loss: 6.5980, Train Steps/Sec: 1.12 +[2025-04-28 09:41:53] (step=0002450) Train Loss: 6.6289, Train Steps/Sec: 1.12 +[2025-04-28 09:42:15] (step=0002475) Train Loss: 6.6270, Train Steps/Sec: 1.11 +[2025-04-28 09:42:37] (step=0002500) Train Loss: 6.6066, Train Steps/Sec: 1.12 +[2025-04-28 09:43:00] (step=0002525) Train Loss: 6.5997, Train Steps/Sec: 1.12 +[2025-04-28 09:43:22] (step=0002550) Train Loss: 6.6163, Train Steps/Sec: 1.12 +[2025-04-28 09:43:45] (step=0002575) Train Loss: 6.5980, Train Steps/Sec: 1.12 +[2025-04-28 09:44:07] (step=0002600) Train Loss: 6.5537, Train Steps/Sec: 1.11 +[2025-04-28 09:44:29] (step=0002625) Train Loss: 6.6536, Train Steps/Sec: 1.12 +[2025-04-28 09:44:52] (step=0002650) Train Loss: 6.6638, Train Steps/Sec: 1.12 +[2025-04-28 09:45:14] (step=0002675) Train Loss: 6.6210, Train Steps/Sec: 1.12 +[2025-04-28 09:45:36] (step=0002700) Train Loss: 6.6485, Train Steps/Sec: 1.12 +[2025-04-28 09:45:59] (step=0002725) Train Loss: 6.6062, Train Steps/Sec: 1.12 +[2025-04-28 09:46:21] (step=0002750) Train Loss: 6.6374, Train Steps/Sec: 1.12 +[2025-04-28 09:46:43] (step=0002775) Train Loss: 6.7047, Train Steps/Sec: 1.12 +[2025-04-28 09:47:06] (step=0002800) Train Loss: 6.6175, Train Steps/Sec: 1.11 +[2025-04-28 09:47:28] (step=0002825) Train Loss: 6.6502, Train Steps/Sec: 1.11 +[2025-04-28 09:47:51] (step=0002850) Train Loss: 6.6497, Train Steps/Sec: 1.12 +[2025-04-28 09:48:13] (step=0002875) Train Loss: 6.6005, Train Steps/Sec: 1.12 +[2025-04-28 09:48:36] (step=0002900) Train Loss: 6.6339, Train Steps/Sec: 1.12 +[2025-04-28 09:48:58] (step=0002925) Train Loss: 6.6220, Train Steps/Sec: 1.11 +[2025-04-28 09:49:20] (step=0002950) Train Loss: 6.5686, Train Steps/Sec: 1.12 +[2025-04-28 09:49:43] (step=0002975) Train Loss: 6.6179, Train Steps/Sec: 1.12 +[2025-04-28 09:50:05] (step=0003000) Train Loss: 6.6589, Train Steps/Sec: 1.11 +[2025-04-28 09:50:28] (step=0003025) Train Loss: 6.6127, Train Steps/Sec: 1.12 +[2025-04-28 09:50:50] (step=0003050) Train Loss: 6.5536, Train Steps/Sec: 1.12 +[2025-04-28 09:51:12] (step=0003075) Train Loss: 6.5970, Train Steps/Sec: 1.12 +[2025-04-28 09:51:35] (step=0003100) Train Loss: 6.6391, Train Steps/Sec: 1.12 +[2025-04-28 09:51:57] (step=0003125) Train Loss: 6.6305, Train Steps/Sec: 1.12 +[2025-04-28 09:52:19] (step=0003150) Train Loss: 6.6107, Train Steps/Sec: 1.12 +[2025-04-28 09:52:42] (step=0003175) Train Loss: 6.5604, Train Steps/Sec: 1.12 +[2025-04-28 09:53:04] (step=0003200) Train Loss: 6.5156, Train Steps/Sec: 1.12 +[2025-04-28 09:53:26] (step=0003225) Train Loss: 6.5188, Train Steps/Sec: 1.12 +[2025-04-28 09:53:49] (step=0003250) Train Loss: 6.6348, Train Steps/Sec: 1.12 +[2025-04-28 09:54:11] (step=0003275) Train Loss: 6.5770, Train Steps/Sec: 1.12 +[2025-04-28 09:54:33] (step=0003300) Train Loss: 6.6317, Train Steps/Sec: 1.12 +[2025-04-28 09:54:56] (step=0003325) Train Loss: 6.6281, Train Steps/Sec: 1.12 +[2025-04-28 09:55:18] (step=0003350) Train Loss: 6.6367, Train Steps/Sec: 1.12 +[2025-04-28 09:55:40] (step=0003375) Train Loss: 6.6152, Train Steps/Sec: 1.12 +[2025-04-28 09:56:03] (step=0003400) Train Loss: 6.5761, Train Steps/Sec: 1.11 +[2025-04-28 09:56:25] (step=0003425) Train Loss: 6.5764, Train Steps/Sec: 1.12 +[2025-04-28 09:56:48] (step=0003450) Train Loss: 6.5810, Train Steps/Sec: 1.12 +[2025-04-28 09:57:10] (step=0003475) Train Loss: 6.5939, Train Steps/Sec: 1.12 +[2025-04-28 09:57:32] (step=0003500) Train Loss: 6.5922, Train Steps/Sec: 1.12 +[2025-04-28 09:57:55] (step=0003525) Train Loss: 6.5983, Train Steps/Sec: 1.12 +[2025-04-28 09:58:17] (step=0003550) Train Loss: 6.5894, Train Steps/Sec: 1.12 +[2025-04-28 09:58:39] (step=0003575) Train Loss: 6.5798, Train Steps/Sec: 1.12 +[2025-04-28 09:59:02] (step=0003600) Train Loss: 6.5903, Train Steps/Sec: 1.12 +[2025-04-28 09:59:24] (step=0003625) Train Loss: 6.6086, Train Steps/Sec: 1.12 +[2025-04-28 09:59:46] (step=0003650) Train Loss: 6.6304, Train Steps/Sec: 1.12 +[2025-04-28 10:00:09] (step=0003675) Train Loss: 6.5407, Train Steps/Sec: 1.12 +[2025-04-28 10:00:31] (step=0003700) Train Loss: 6.5788, Train Steps/Sec: 1.12 +[2025-04-28 10:00:53] (step=0003725) Train Loss: 6.5568, Train Steps/Sec: 1.12 +[2025-04-28 10:01:16] (step=0003750) Train Loss: 6.6197, Train Steps/Sec: 1.12 +[2025-04-28 10:01:38] (step=0003775) Train Loss: 6.5456, Train Steps/Sec: 1.12 +[2025-04-28 10:02:01] (step=0003800) Train Loss: 6.6183, Train Steps/Sec: 1.11 +[2025-04-28 10:02:23] (step=0003825) Train Loss: 6.5983, Train Steps/Sec: 1.12 +[2025-04-28 10:02:45] (step=0003850) Train Loss: 6.5378, Train Steps/Sec: 1.12 +[2025-04-28 10:03:08] (step=0003875) Train Loss: 6.5376, Train Steps/Sec: 1.12 +[2025-04-28 10:03:30] (step=0003900) Train Loss: 6.5656, Train Steps/Sec: 1.12 +[2025-04-28 10:03:52] (step=0003925) Train Loss: 6.5361, Train Steps/Sec: 1.12 +[2025-04-28 10:04:15] (step=0003950) Train Loss: 6.6249, Train Steps/Sec: 1.12 +[2025-04-28 10:04:37] (step=0003975) Train Loss: 6.5434, Train Steps/Sec: 1.12 +[2025-04-28 10:04:59] (step=0004000) Train Loss: 6.6078, Train Steps/Sec: 1.12 +[2025-04-28 10:04:59] text_config is None. Initializing the text config with default values (`OPTConfig`). +[2025-04-28 10:12:05] Finish Eval in 4000 steps... +[2025-04-28 10:12:25] Saved checkpoint to checkpoint/X2I_UltraEdiit_fourmask_LLava_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_filtered_objects_4mask_1280_context_after_stage1/000-GPT-XL/checkpoints/0004000.pt +[2025-04-28 10:12:27] Removed old checkpoint: checkpoint/X2I_UltraEdiit_fourmask_LLava_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_filtered_objects_4mask_1280_context_after_stage1/000-GPT-XL/checkpoints/0002000.pt +[2025-04-28 10:12:49] (step=0004025) Train Loss: 6.5141, Train Steps/Sec: 0.05 +[2025-04-28 10:13:12] (step=0004050) Train Loss: 6.6416, Train Steps/Sec: 1.12 +[2025-04-28 10:13:34] (step=0004075) Train Loss: 6.5725, Train Steps/Sec: 1.12 +[2025-04-28 10:13:56] (step=0004100) Train Loss: 6.5724, Train Steps/Sec: 1.12 +[2025-04-28 10:14:19] (step=0004125) Train Loss: 6.5356, Train Steps/Sec: 1.12 +[2025-04-28 10:14:41] (step=0004150) Train Loss: 6.5415, Train Steps/Sec: 1.12 +[2025-04-28 10:15:03] (step=0004175) Train Loss: 6.5581, Train Steps/Sec: 1.12 +[2025-04-28 10:15:26] (step=0004200) Train Loss: 6.5575, Train Steps/Sec: 1.12 +[2025-04-28 10:15:48] (step=0004225) Train Loss: 6.5850, Train Steps/Sec: 1.12 +[2025-04-28 10:16:10] (step=0004250) Train Loss: 6.6256, Train Steps/Sec: 1.12 +[2025-04-28 10:16:33] (step=0004275) Train Loss: 6.6425, Train Steps/Sec: 1.12 +[2025-04-28 10:16:55] (step=0004300) Train Loss: 6.5594, Train Steps/Sec: 1.12 +[2025-04-28 10:17:17] (step=0004325) Train Loss: 6.5325, Train Steps/Sec: 1.12 +[2025-04-28 10:17:40] (step=0004350) Train Loss: 6.5210, Train Steps/Sec: 1.12 +[2025-04-28 10:18:02] (step=0004375) Train Loss: 6.5835, Train Steps/Sec: 1.11 +[2025-04-28 10:18:25] (step=0004400) Train Loss: 6.5274, Train Steps/Sec: 1.11 +[2025-04-28 10:18:47] (step=0004425) Train Loss: 6.5315, Train Steps/Sec: 1.12 +[2025-04-28 10:19:09] (step=0004450) Train Loss: 6.6087, Train Steps/Sec: 1.12 +[2025-04-28 10:19:32] (step=0004475) Train Loss: 6.5947, Train Steps/Sec: 1.12 +[2025-04-28 10:19:54] (step=0004500) Train Loss: 6.5741, Train Steps/Sec: 1.12 +[2025-04-28 10:20:16] (step=0004525) Train Loss: 6.5401, Train Steps/Sec: 1.12 +[2025-04-28 10:20:39] (step=0004550) Train Loss: 6.5409, Train Steps/Sec: 1.12 +[2025-04-28 10:21:01] (step=0004575) Train Loss: 6.4968, Train Steps/Sec: 1.12 +[2025-04-28 10:21:23] (step=0004600) Train Loss: 6.4970, Train Steps/Sec: 1.12 +[2025-04-28 10:21:46] (step=0004625) Train Loss: 6.5954, Train Steps/Sec: 1.12 diff --git a/CKPTS/X2I_UltraEdiit_fourmask_LLava_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_filtered_objects_4mask_1280_context_after_stage1/wandb/debug-internal.log b/CKPTS/X2I_UltraEdiit_fourmask_LLava_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_filtered_objects_4mask_1280_context_after_stage1/wandb/debug-internal.log new file mode 100644 index 0000000000000000000000000000000000000000..8f64c359a6ffa0d89249cb24aa67c3fd6b5cfb5f --- /dev/null +++ b/CKPTS/X2I_UltraEdiit_fourmask_LLava_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_filtered_objects_4mask_1280_context_after_stage1/wandb/debug-internal.log @@ -0,0 +1,7 @@ +{"time":"2025-04-28T08:51:27.690319766Z","level":"INFO","msg":"stream: starting","core version":"0.19.8","symlink path":"checkpoint/X2I_UltraEdiit_fourmask_LLava_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_filtered_objects_4mask_1280_context_after_stage1/wandb/run-20250428_085127-jzkufte5/logs/debug-core.log"} +{"time":"2025-04-28T08:51:27.79684421Z","level":"INFO","msg":"created new stream","id":"jzkufte5"} +{"time":"2025-04-28T08:51:27.796891148Z","level":"INFO","msg":"stream: started","id":"jzkufte5"} +{"time":"2025-04-28T08:51:27.796929319Z","level":"INFO","msg":"writer: Do: started","stream_id":"jzkufte5"} +{"time":"2025-04-28T08:51:27.797220158Z","level":"INFO","msg":"sender: started","stream_id":"jzkufte5"} +{"time":"2025-04-28T08:51:27.797463809Z","level":"INFO","msg":"handler: started","stream_id":"jzkufte5"} +{"time":"2025-04-28T08:51:28.283077266Z","level":"INFO","msg":"Starting system monitor"} diff --git a/CKPTS/X2I_UltraEdiit_fourmask_LLava_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_filtered_objects_4mask_1280_context_after_stage1/wandb/debug.log b/CKPTS/X2I_UltraEdiit_fourmask_LLava_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_filtered_objects_4mask_1280_context_after_stage1/wandb/debug.log new file mode 100644 index 0000000000000000000000000000000000000000..09308f18950371e14488501facbb4ac7f04090b4 --- /dev/null +++ b/CKPTS/X2I_UltraEdiit_fourmask_LLava_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_filtered_objects_4mask_1280_context_after_stage1/wandb/debug.log @@ -0,0 +1,22 @@ +2025-04-28 08:51:27,683 INFO MainThread:3812340 [wandb_setup.py:_flush():67] Current SDK version is 0.19.8 +2025-04-28 08:51:27,683 INFO MainThread:3812340 [wandb_setup.py:_flush():67] Configure stats pid to 3812340 +2025-04-28 08:51:27,683 INFO MainThread:3812340 [wandb_setup.py:_flush():67] Loading settings from /tmp/haozhezhao/.config/wandb/settings +2025-04-28 08:51:27,683 INFO MainThread:3812340 [wandb_setup.py:_flush():67] Loading settings from /tmp/haozhezhao/MLLMG/wandb/settings +2025-04-28 08:51:27,683 INFO MainThread:3812340 [wandb_setup.py:_flush():67] Loading settings from environment variables +2025-04-28 08:51:27,683 INFO MainThread:3812340 [wandb_init.py:setup_run_log_directory():647] Logging user logs to checkpoint/X2I_UltraEdiit_fourmask_LLava_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_filtered_objects_4mask_1280_context_after_stage1/wandb/run-20250428_085127-jzkufte5/logs/debug.log +2025-04-28 08:51:27,683 INFO MainThread:3812340 [wandb_init.py:setup_run_log_directory():648] Logging internal logs to checkpoint/X2I_UltraEdiit_fourmask_LLava_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_filtered_objects_4mask_1280_context_after_stage1/wandb/run-20250428_085127-jzkufte5/logs/debug-internal.log +2025-04-28 08:51:27,683 INFO MainThread:3812340 [wandb_init.py:init():761] calling init triggers +2025-04-28 08:51:27,683 INFO MainThread:3812340 [wandb_init.py:init():766] wandb.init called with sweep_config: {} +config: {'data_path': '/tmp/haozhezhao/MLLMG/jsonl_data/multiimage_training_for_llava_X2I_fourmask.jsonl', 'cloud_save_path': '/tmp/haozhezhao/MLLMG/checkpoint', 'no_local_save': False, 'vq_model': 'VQ-16', 'vq_ckpt': '/tmp/haozhezhao/model/llamagen_t2i/vq_ds16_t2i.pt', 'codebook_size': 16384, 'codebook_embed_dim': 8, 'gpt_model': 'GPT-XL', 'gpt_ckpt': '/tmp/haozhezhao/MLLMG/MLLMG_ckpts/checkpoints/AfterI2I_llavat5_subject_train_all_T5_subject_t2i_ti2i_120_w_flux_segment_noimageonly_empty_fix_train_all/0100000.pt', 'gpt_type': 't2i', 'vocab_size': 16384, 'cls_token_num': 1280, 'dropout_p': 0.1, 'token_dropout_p': 0.1, 'drop_path': 0.0, 'no_compile': False, 'results_dir': 'checkpoint/X2I_UltraEdiit_fourmask_LLava_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_filtered_objects_4mask_1280_context_after_stage1', 'dataset': 'ti2i', 'image_size': 512, 'downsample_size': 16, 'num_classes': 1000, 'epochs': 1, 'lr': 0.0001, 'weight_decay': 0.05, 'beta1': 0.9, 'beta2': 0.95, 'max_grad_norm': 1.0, 'global_batch_size': 24, 'global_seed': 0, 'num_workers': 4, 'log_every': 25, 'ckpt_every': 2000, 'gradient_accumulation_steps': 8, 'mixed_precision': 'bf16', 'val_data_path': '/tmp/haozhezhao/MLLMG/jsonl_data/multiimage_val_for_llava_X2I_fourmask_dreambenplus.jsonl', 'use_vision_tower': True, 'model_name_or_path': '/tmp/haozhezhao/model/blip2-flan-t5-xl', 'image_place_holder': '', 'processor_path': None, 'do_eval': True, 'max_eval_samples': 128, 'train_text_encoder': True, 'no_left_padding': False, 'cfg_scale': 7.5, 'top_k': 16384, 'temperature': 0.9, 'top_p': 1.0, 'eval_steps': 2000, 'project_name': 'llamagen_ti2i', 'load_from_checkpoint': '/tmp/haozhezhao/MLLMG/checkpoint/FIXed_3M_1epoch_step32000.pt', 'warmup': 0.05, 'lr_decay_style': 'cosine', 'lr_decay_ratio': 0.1, 'train_iters': 500000, 'class_dropout_prob': 0.1, 'with_image_only': False, 'image_only_rate': 0.1, 'stage2': False, 'subject_driven': True, 'load_subject_embedding': None, 'reference_data_path': '/tmp/haozhezhao/MLLMG/cc12m_reference_tunnel.jsonl', 'multimodal_encoder': 'llava', 'do_recovery': True, 'no_replace': False, 'resume': False, 'dreambench_eval': True, 'find_unused_parameters': True, 'load_visual_encoder': False, 'continue_stage1': False, 'replace_subject': False, 'train_all': True, 'save_total_limit': 1, 'load_language_projection': '/tmp/haozhezhao/MLLMG/llava-v1.5-flant5_fixed-pretrain/mm_projector.bin', 'mm_vision_tower': 'openai/clip-vit-large-patch14', 'load_fixed_llamagen': True, 'unfreeze_output': False, 'fix': 'gpt-empty-fix', 'rank': 0, 'world_size': 8, 'gpu': 0, 'dist_url': 'env://', 'distributed': True, 'dist_backend': 'nccl', '_wandb': {}} +2025-04-28 08:51:27,683 INFO MainThread:3812340 [wandb_init.py:init():784] starting backend +2025-04-28 08:51:27,683 INFO MainThread:3812340 [wandb_init.py:init():788] sending inform_init request +2025-04-28 08:51:27,688 INFO MainThread:3812340 [backend.py:_multiprocessing_setup():101] multiprocessing start_methods=fork,spawn,forkserver, using: spawn +2025-04-28 08:51:27,688 INFO MainThread:3812340 [wandb_init.py:init():798] backend started and connected +2025-04-28 08:51:27,690 INFO MainThread:3812340 [wandb_init.py:init():891] updated telemetry +2025-04-28 08:51:27,690 INFO MainThread:3812340 [wandb_init.py:init():915] communicating run to backend with 90.0 second timeout +2025-04-28 08:51:28,280 INFO MainThread:3812340 [wandb_init.py:init():990] starting run threads in backend +2025-04-28 08:51:28,388 INFO MainThread:3812340 [wandb_run.py:_console_start():2375] atexit reg +2025-04-28 08:51:28,388 INFO MainThread:3812340 [wandb_run.py:_redirect():2227] redirect: wrap_raw +2025-04-28 08:51:28,388 INFO MainThread:3812340 [wandb_run.py:_redirect():2292] Wrapping output streams. +2025-04-28 08:51:28,388 INFO MainThread:3812340 [wandb_run.py:_redirect():2315] Redirects installed. +2025-04-28 08:51:28,390 INFO MainThread:3812340 [wandb_init.py:init():1032] run started, returning control to user process diff --git a/CKPTS/X2I_UltraEdiit_fourmask_LLava_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_filtered_objects_4mask_1280_context_after_stage1/wandb/run-20250428_085127-jzkufte5/files/output.log b/CKPTS/X2I_UltraEdiit_fourmask_LLava_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_filtered_objects_4mask_1280_context_after_stage1/wandb/run-20250428_085127-jzkufte5/files/output.log new file mode 100644 index 0000000000000000000000000000000000000000..2dda75f766fbe9998cb814a4d7dfa89854433ce2 --- /dev/null +++ b/CKPTS/X2I_UltraEdiit_fourmask_LLava_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_filtered_objects_4mask_1280_context_after_stage1/wandb/run-20250428_085127-jzkufte5/files/output.log @@ -0,0 +1,244 @@ +[2025-04-28 08:51:28] Training for 1 epochs... +[2025-04-28 08:51:28] Beginning epoch 0... + 0%| | 0/89905 [00:00 +tokenizer length after expend 32101 +tokenizer length before expend 32100 + /tmp/haozhezhao/anaconda3/envs/nlp/lib/python3.11/contextlib.py:105: FutureWarning: `torch.backends.cuda.sdp_kernel()` is deprecated. In the future, this context manager will be removed. Please see `torch.nn.attention.sdpa_kernel()` for the new context manager, with updated signature. + self.gen = func(*args, **kwds) | 0/6 [00:00 +tokenizer length after expend 32101 +tokenizer length before expend 32100 +100%|████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████| 6/6 [05:50<00:00, 58.36s/it] +[2025-04-28 10:12:05] Finish Eval in 4000 steps...██████████████████████████████████████████████████████████████████████████████████████████████| 6/6 [05:49<00:00, 58.16s/it] +processing left-padding... +processing left-padding... +processing left-padding... +processing left-padding... +processing left-padding... +processing left-padding... +[2025-04-28 10:12:25] Saved checkpoint to checkpoint/X2I_UltraEdiit_fourmask_LLava_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_filtered_objects_4mask_1280_context_after_stage1/000-GPT-XL/checkpoints/0004000.pt +[2025-04-28 10:12:27] Removed old checkpoint: checkpoint/X2I_UltraEdiit_fourmask_LLava_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_filtered_objects_4mask_1280_context_after_stage1/000-GPT-XL/checkpoints/0002000.pt + 4%|██████▌ | 4024/89905 [1:21:20<22:16:41, 1.07it/s][2025-04-28 10:12:49] (step=0004025) Train Loss: 6.5141, Train Steps/Sec: 0.05 + 5%|██████▋ | 4049/89905 [1:21:42<21:31:16, 1.11it/s][2025-04-28 10:13:12] (step=0004050) Train Loss: 6.6416, Train Steps/Sec: 1.12 + 5%|██████▋ | 4074/89905 [1:22:04<21:21:47, 1.12it/s][2025-04-28 10:13:34] (step=0004075) Train Loss: 6.5725, Train Steps/Sec: 1.12 + 5%|██████▋ | 4099/89905 [1:22:27<21:09:37, 1.13it/s][2025-04-28 10:13:56] (step=0004100) Train Loss: 6.5724, Train Steps/Sec: 1.12 + 5%|██████▊ | 4124/89905 [1:22:49<21:13:33, 1.12it/s][2025-04-28 10:14:19] (step=0004125) Train Loss: 6.5356, Train Steps/Sec: 1.12 + 5%|██████▊ | 4149/89905 [1:23:11<21:09:36, 1.13it/s][2025-04-28 10:14:41] (step=0004150) Train Loss: 6.5415, Train Steps/Sec: 1.12 + 5%|██████▊ | 4174/89905 [1:23:34<21:04:08, 1.13it/s][2025-04-28 10:15:03] (step=0004175) Train Loss: 6.5581, Train Steps/Sec: 1.12 + 5%|██████▉ | 4199/89905 [1:23:56<21:00:08, 1.13it/s][2025-04-28 10:15:26] (step=0004200) Train Loss: 6.5575, Train Steps/Sec: 1.12 + 5%|██████▉ | 4224/89905 [1:24:18<21:43:04, 1.10it/s][2025-04-28 10:15:48] (step=0004225) Train Loss: 6.5850, Train Steps/Sec: 1.12 + 5%|██████▉ | 4249/89905 [1:24:41<21:31:11, 1.11it/s][2025-04-28 10:16:10] (step=0004250) Train Loss: 6.6256, Train Steps/Sec: 1.12 + 5%|███████ | 4274/89905 [1:25:03<21:21:12, 1.11it/s][2025-04-28 10:16:33] (step=0004275) Train Loss: 6.6425, Train Steps/Sec: 1.12 + 5%|███████ | 4299/89905 [1:25:25<21:11:48, 1.12it/s][2025-04-28 10:16:55] (step=0004300) Train Loss: 6.5594, Train Steps/Sec: 1.12 + 5%|███████ | 4324/89905 [1:25:48<21:12:15, 1.12it/s][2025-04-28 10:17:17] (step=0004325) Train Loss: 6.5325, Train Steps/Sec: 1.12 + 5%|███████▏ | 4349/89905 [1:26:10<21:05:36, 1.13it/s][2025-04-28 10:17:40] (step=0004350) Train Loss: 6.5210, Train Steps/Sec: 1.12 + 5%|███████▏ | 4374/89905 [1:26:33<21:03:28, 1.13it/s][2025-04-28 10:18:02] (step=0004375) Train Loss: 6.5835, Train Steps/Sec: 1.11 + 5%|███████▏ | 4399/89905 [1:26:55<21:04:21, 1.13it/s][2025-04-28 10:18:25] (step=0004400) Train Loss: 6.5274, Train Steps/Sec: 1.11 + 5%|███████▎ | 4424/89905 [1:27:17<21:34:53, 1.10it/s][2025-04-28 10:18:47] (step=0004425) Train Loss: 6.5315, Train Steps/Sec: 1.12 + 5%|███████▎ | 4449/89905 [1:27:40<21:27:22, 1.11it/s][2025-04-28 10:19:09] (step=0004450) Train Loss: 6.6087, Train Steps/Sec: 1.12 + 5%|███████▎ | 4474/89905 [1:28:02<21:11:20, 1.12it/s][2025-04-28 10:19:32] (step=0004475) Train Loss: 6.5947, Train Steps/Sec: 1.12 + 5%|███████▍ | 4499/89905 [1:28:24<21:12:57, 1.12it/s][2025-04-28 10:19:54] (step=0004500) Train Loss: 6.5741, Train Steps/Sec: 1.12 + 5%|███████▍ | 4524/89905 [1:28:47<21:03:56, 1.13it/s][2025-04-28 10:20:16] (step=0004525) Train Loss: 6.5401, Train Steps/Sec: 1.12 + 5%|███████▍ | 4549/89905 [1:29:09<21:00:09, 1.13it/s][2025-04-28 10:20:39] (step=0004550) Train Loss: 6.5409, Train Steps/Sec: 1.12 + 5%|███████▌ | 4574/89905 [1:29:31<20:57:31, 1.13it/s][2025-04-28 10:21:01] (step=0004575) Train Loss: 6.4968, Train Steps/Sec: 1.12 + 5%|███████▌ | 4599/89905 [1:29:54<20:57:33, 1.13it/s][2025-04-28 10:21:23] (step=0004600) Train Loss: 6.4970, Train Steps/Sec: 1.12 + 5%|███████▌ | 4624/89905 [1:30:16<21:35:55, 1.10it/s][2025-04-28 10:21:46] (step=0004625) Train Loss: 6.5954, Train Steps/Sec: 1.12 + 5%|███████▌ | 4627/89905 [1:30:19<21:05:32, 1.12it/s] diff --git a/CKPTS/X2I_UltraEdiit_fourmask_LLava_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_filtered_objects_4mask_1280_context_after_stage1/wandb/run-20250428_085127-jzkufte5/files/requirements.txt b/CKPTS/X2I_UltraEdiit_fourmask_LLava_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_filtered_objects_4mask_1280_context_after_stage1/wandb/run-20250428_085127-jzkufte5/files/requirements.txt new file mode 100644 index 0000000000000000000000000000000000000000..06dc78369ffff807b210006a0e79d705ffe2a7d7 --- /dev/null +++ b/CKPTS/X2I_UltraEdiit_fourmask_LLava_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_filtered_objects_4mask_1280_context_after_stage1/wandb/run-20250428_085127-jzkufte5/files/requirements.txt @@ -0,0 +1,131 @@ +typing_extensions==4.12.2 +pyzmq==26.3.0 +nvidia-cufft-cu12==11.0.2.54 +triton==3.1.0 +nvidia-cublas-cu12==12.1.3.1 +psutil==7.0.0 +nvidia-cuda-cupti-cu12==12.1.105 +smmap==5.0.2 +nvidia-cuda-runtime-cu12==12.1.105 +aiohappyeyeballs==2.6.1 +asttokens==3.0.0 +huggingface-hub==0.29.3 +pyarrow==19.0.1 +fonttools==4.56.0 +python-dateutil==2.9.0.post0 +GitPython==3.1.44 +aiohttp==3.11.14 +wandb==0.19.8 +setproctitle==1.3.5 +PyYAML==6.0.2 +pydantic_core==2.27.2 +safetensors==0.5.3 +nvidia-nvjitlink-cu12==12.1.105 +aiosignal==1.3.2 +dill==0.3.8 +nvidia-cuda-nvrtc-cu12==12.1.105 +multiprocess==0.70.16 +pure_eval==0.2.3 +stack_data==0.6.3 +pydantic==2.10.6 +MarkupSafe==2.1.5 +tornado==6.4.2 +executing==2.1.0 +executing==2.2.0 +opencv-python==4.11.0.86 +nvitop==1.4.2 +multidict==6.2.0 +Jinja2==3.1.4 +torch==2.5.1+cu121 +nvidia-curand-cu12==10.3.2.106 +platformdirs==4.3.6 +six==1.17.0 +mpmath==1.3.0 +zipp==3.21.0 +packaging==24.2 +requests==2.32.3 +certifi==2025.1.31 +docker-pycreds==0.4.0 +torchvision==0.20.1+cu121 +pandas==2.2.3 +networkx==3.3 +exceptiongroup==1.2.2 +pickleshare==0.7.5 +tokenizers==0.21.1 +charset-normalizer==3.4.1 +jupyter_core==5.7.2 +wcwidth==0.2.13 +nvidia-nvtx-cu12==12.1.105 +prompt_toolkit==3.0.50 +fsspec==2024.12.0 +pillow==11.1.0 +propcache==0.3.0 +regex==2024.11.6 +ptyprocess==0.7.0 +contourpy==1.3.1 +importlib_metadata==8.6.1 +idna==3.10 +comm==0.2.2 +protobuf==5.29.3 +yarl==1.18.3 +ipython_pygments_lexers==1.1.1 +pip==25.0 +parso==0.8.4 +joblib==1.4.2 +nvidia-nccl-cu12==2.21.5 +hf_transfer==0.1.9 +Pygments==2.19.1 +decorator==5.2.1 +filelock==3.18.0 +nvidia-cusparse-cu12==12.1.0.106 +debugpy==1.8.13 +urllib3==2.3.0 +traitlets==5.14.3 +tzdata==2025.1 +matplotlib-inline==0.1.7 +matplotlib==3.10.1 +kiwisolver==1.4.8 +nest_asyncio==1.6.0 +frozenlist==1.5.0 +nvidia-ml-py==12.570.86 +transformers==4.49.0 +nltk==3.9.1 +ipykernel==6.29.5 +click==8.1.8 +gitdb==4.0.12 +pyparsing==3.2.1 +attrs==25.3.0 +jedi==0.19.2 +ipython==9.0.2 +nvidia-cudnn-cu12==9.1.0.70 +pexpect==4.9.0 +nvidia-cusolver-cu12==11.4.5.107 +numpy==2.2.4 +tqdm==4.67.1 +pytz==2025.1 +wheel==0.45.1 +sentry-sdk==2.23.1 +torchaudio==2.5.1+cu121 +jupyter_client==8.6.3 +cycler==0.12.1 +annotated-types==0.7.0 +sympy==1.13.1 +xxhash==3.5.0 +datasets==3.4.1 +setuptools==75.8.0 +typing_extensions==4.12.2 +wheel==0.43.0 +importlib_metadata==8.0.0 +backports.tarfile==1.2.0 +autocommand==2.2.2 +packaging==24.2 +tomli==2.0.1 +typeguard==4.3.0 +zipp==3.19.2 +jaraco.context==5.3.0 +jaraco.functools==4.0.1 +more-itertools==10.3.0 +platformdirs==4.2.2 +jaraco.text==3.12.1 +jaraco.collections==5.1.0 +inflect==7.3.1 diff --git a/CKPTS/X2I_UltraEdiit_fourmask_LLava_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_filtered_objects_4mask_1280_context_after_stage1/wandb/run-20250428_085127-jzkufte5/files/wandb-metadata.json b/CKPTS/X2I_UltraEdiit_fourmask_LLava_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_filtered_objects_4mask_1280_context_after_stage1/wandb/run-20250428_085127-jzkufte5/files/wandb-metadata.json new file mode 100644 index 0000000000000000000000000000000000000000..853d410b301465e013d5a6c3b0c304ef24d3ec7b --- /dev/null +++ b/CKPTS/X2I_UltraEdiit_fourmask_LLava_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_filtered_objects_4mask_1280_context_after_stage1/wandb/run-20250428_085127-jzkufte5/files/wandb-metadata.json @@ -0,0 +1,149 @@ +{ + "os": "Linux-5.15.0-1064-azure-x86_64-with-glibc2.31", + "python": "CPython 3.11.11", + "startedAt": "2025-04-28T08:51:27.688522Z", + "args": [ + "--vq-ckpt", + "/tmp/haozhezhao/model/llamagen_t2i/vq_ds16_t2i.pt", + "--data-path", + "/tmp/haozhezhao/MLLMG/jsonl_data/multiimage_training_for_llava_X2I_fourmask.jsonl", + "--dataset", + "ti2i", + "--image-size", + "512", + "--results-dir", + "checkpoint/X2I_UltraEdiit_fourmask_LLava_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_filtered_objects_4mask_1280_context_after_stage1", + "--cloud-save-path", + "/tmp/haozhezhao/MLLMG/checkpoint", + "--lr", + "1e-4", + "--val_data_path", + "/tmp/haozhezhao/MLLMG/jsonl_data/multiimage_val_for_llava_X2I_fourmask_dreambenplus.jsonl", + "--use_vision_tower", + "--model_name_or_path", + "/tmp/haozhezhao/model/blip2-flan-t5-xl", + "--image_place_holder", + "", + "--do_eval", + "--eval_steps", + "2000", + "--max_eval_samples", + "128", + "--cfg-scale", + "7.5", + "--top-k", + "16384", + "--load_from_checkpoint", + "/tmp/haozhezhao/MLLMG/checkpoint/FIXed_3M_1epoch_step32000.pt", + "--global-batch-size", + "24", + "--num-workers", + "4", + "--warmup", + "0.05", + "--gradient-accumulation-steps", + "8", + "--train_text_encoder", + "--ckpt-every", + "2000", + "--epochs", + "1", + "--subject_driven", + "--reference_data_path", + "/tmp/haozhezhao/MLLMG/cc12m_reference_tunnel.jsonl", + "--multimodal_encoder", + "llava", + "--do_recovery", + "--find_unused_parameters", + "--cls-token-num", + "1280", + "--train_all", + "--load_fixed_llamagen", + "--save_total_limit", + "1", + "--fix", + "gpt-empty-fix", + "--load_language_projection", + "/tmp/haozhezhao/MLLMG/llava-v1.5-flant5_fixed-pretrain/mm_projector.bin", + "--mm_vision_tower", + "openai/clip-vit-large-patch14", + "--gpt-ckpt", + "/tmp/haozhezhao/MLLMG/MLLMG_ckpts/checkpoints/AfterI2I_llavat5_subject_train_all_T5_subject_t2i_ti2i_120_w_flux_segment_noimageonly_empty_fix_train_all/0100000.pt", + "--dreambench_eval" + ], + "program": "/tmp/haozhezhao/MLLMG/autoregressive/train/train_t2i.py", + "codePath": "autoregressive/train/train_t2i.py", + "email": "mimazhe55360@gmail.com", + "root": "checkpoint/X2I_UltraEdiit_fourmask_LLava_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_filtered_objects_4mask_1280_context_after_stage1", + "host": "447cc403a8794092814259713c51c1df00001X", + "executable": "/tmp/haozhezhao/anaconda3/envs/nlp/bin/python", + "codePathLocal": "autoregressive/train/train_t2i.py", + "cpu_count": 96, + "cpu_count_logical": 96, + "gpu": "NVIDIA A100-SXM4-80GB", + "gpu_count": 8, + "disk": { + "/": { + "total": "133003395072", + "used": "67469406208" + } + }, + "memory": { + "total": "1902387884032" + }, + "cpu": { + "count": 96, + "countLogical": 96 + }, + "gpu_nvidia": [ + { + "name": "NVIDIA A100-SXM4-80GB", + "memoryTotal": "85899345920", + "cudaCores": 6912, + "architecture": "Ampere" + }, + { + "name": "NVIDIA A100-SXM4-80GB", + "memoryTotal": "85899345920", + "cudaCores": 6912, + "architecture": "Ampere" + }, + { + "name": "NVIDIA A100-SXM4-80GB", + "memoryTotal": "85899345920", + "cudaCores": 6912, + "architecture": "Ampere" + }, + { + "name": "NVIDIA A100-SXM4-80GB", + "memoryTotal": "85899345920", + "cudaCores": 6912, + "architecture": "Ampere" + }, + { + "name": "NVIDIA A100-SXM4-80GB", + "memoryTotal": "85899345920", + "cudaCores": 6912, + "architecture": "Ampere" + }, + { + "name": "NVIDIA A100-SXM4-80GB", + "memoryTotal": "85899345920", + "cudaCores": 6912, + "architecture": "Ampere" + }, + { + "name": "NVIDIA A100-SXM4-80GB", + "memoryTotal": "85899345920", + "cudaCores": 6912, + "architecture": "Ampere" + }, + { + "name": "NVIDIA A100-SXM4-80GB", + "memoryTotal": "85899345920", + "cudaCores": 6912, + "architecture": "Ampere" + } + ], + "cudaVersion": "12.2" +} \ No newline at end of file diff --git a/CKPTS/X2I_UltraEdiit_fourmask_LLava_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_filtered_objects_4mask_1280_context_after_stage1/wandb/run-20250428_085127-jzkufte5/logs/debug-core.log b/CKPTS/X2I_UltraEdiit_fourmask_LLava_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_filtered_objects_4mask_1280_context_after_stage1/wandb/run-20250428_085127-jzkufte5/logs/debug-core.log new file mode 100644 index 0000000000000000000000000000000000000000..d7b8707926a3e378bf62c134eea198088f301482 --- /dev/null +++ b/CKPTS/X2I_UltraEdiit_fourmask_LLava_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_filtered_objects_4mask_1280_context_after_stage1/wandb/run-20250428_085127-jzkufte5/logs/debug-core.log @@ -0,0 +1,7 @@ +{"time":"2025-04-28T08:51:27.058536024Z","level":"INFO","msg":"main: starting server","port-filename":"/tmp/tmpwmncgk_c/port-3812340.txt","pid":3812340,"log-level":0,"disable-analytics":false,"shutdown-on-parent-exit":false} +{"time":"2025-04-28T08:51:27.060296022Z","level":"INFO","msg":"Will exit if parent process dies.","ppid":3812340} +{"time":"2025-04-28T08:51:27.06026544Z","level":"INFO","msg":"server is running","addr":{"IP":"127.0.0.1","Port":46513,"Zone":""}} +{"time":"2025-04-28T08:51:27.244803551Z","level":"INFO","msg":"connection: ManageConnectionData: new connection created","id":"127.0.0.1:38786"} +{"time":"2025-04-28T08:51:27.690046795Z","level":"INFO","msg":"handleInformInit: received","streamId":"jzkufte5","id":"127.0.0.1:38786"} +{"time":"2025-04-28T08:51:27.796896898Z","level":"INFO","msg":"handleInformInit: stream started","streamId":"jzkufte5","id":"127.0.0.1:38786"} +{"time":"2025-04-28T10:21:51.155372678Z","level":"INFO","msg":"Parent process exited, terminating service process."} diff --git a/CKPTS/X2I_UltraEdiit_fourmask_LLava_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_filtered_objects_4mask_1280_context_after_stage1/wandb/run-20250428_085127-jzkufte5/logs/debug-internal.log b/CKPTS/X2I_UltraEdiit_fourmask_LLava_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_filtered_objects_4mask_1280_context_after_stage1/wandb/run-20250428_085127-jzkufte5/logs/debug-internal.log new file mode 100644 index 0000000000000000000000000000000000000000..8f64c359a6ffa0d89249cb24aa67c3fd6b5cfb5f --- /dev/null +++ b/CKPTS/X2I_UltraEdiit_fourmask_LLava_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_filtered_objects_4mask_1280_context_after_stage1/wandb/run-20250428_085127-jzkufte5/logs/debug-internal.log @@ -0,0 +1,7 @@ +{"time":"2025-04-28T08:51:27.690319766Z","level":"INFO","msg":"stream: starting","core version":"0.19.8","symlink path":"checkpoint/X2I_UltraEdiit_fourmask_LLava_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_filtered_objects_4mask_1280_context_after_stage1/wandb/run-20250428_085127-jzkufte5/logs/debug-core.log"} +{"time":"2025-04-28T08:51:27.79684421Z","level":"INFO","msg":"created new stream","id":"jzkufte5"} +{"time":"2025-04-28T08:51:27.796891148Z","level":"INFO","msg":"stream: started","id":"jzkufte5"} +{"time":"2025-04-28T08:51:27.796929319Z","level":"INFO","msg":"writer: Do: started","stream_id":"jzkufte5"} +{"time":"2025-04-28T08:51:27.797220158Z","level":"INFO","msg":"sender: started","stream_id":"jzkufte5"} +{"time":"2025-04-28T08:51:27.797463809Z","level":"INFO","msg":"handler: started","stream_id":"jzkufte5"} +{"time":"2025-04-28T08:51:28.283077266Z","level":"INFO","msg":"Starting system monitor"} diff --git a/CKPTS/X2I_UltraEdiit_fourmask_LLava_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_filtered_objects_4mask_1280_context_after_stage1/wandb/run-20250428_085127-jzkufte5/logs/debug.log b/CKPTS/X2I_UltraEdiit_fourmask_LLava_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_filtered_objects_4mask_1280_context_after_stage1/wandb/run-20250428_085127-jzkufte5/logs/debug.log new file mode 100644 index 0000000000000000000000000000000000000000..09308f18950371e14488501facbb4ac7f04090b4 --- /dev/null +++ b/CKPTS/X2I_UltraEdiit_fourmask_LLava_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_filtered_objects_4mask_1280_context_after_stage1/wandb/run-20250428_085127-jzkufte5/logs/debug.log @@ -0,0 +1,22 @@ +2025-04-28 08:51:27,683 INFO MainThread:3812340 [wandb_setup.py:_flush():67] Current SDK version is 0.19.8 +2025-04-28 08:51:27,683 INFO MainThread:3812340 [wandb_setup.py:_flush():67] Configure stats pid to 3812340 +2025-04-28 08:51:27,683 INFO MainThread:3812340 [wandb_setup.py:_flush():67] Loading settings from /tmp/haozhezhao/.config/wandb/settings +2025-04-28 08:51:27,683 INFO MainThread:3812340 [wandb_setup.py:_flush():67] Loading settings from /tmp/haozhezhao/MLLMG/wandb/settings +2025-04-28 08:51:27,683 INFO MainThread:3812340 [wandb_setup.py:_flush():67] Loading settings from environment variables +2025-04-28 08:51:27,683 INFO MainThread:3812340 [wandb_init.py:setup_run_log_directory():647] Logging user logs to checkpoint/X2I_UltraEdiit_fourmask_LLava_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_filtered_objects_4mask_1280_context_after_stage1/wandb/run-20250428_085127-jzkufte5/logs/debug.log +2025-04-28 08:51:27,683 INFO MainThread:3812340 [wandb_init.py:setup_run_log_directory():648] Logging internal logs to checkpoint/X2I_UltraEdiit_fourmask_LLava_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_filtered_objects_4mask_1280_context_after_stage1/wandb/run-20250428_085127-jzkufte5/logs/debug-internal.log +2025-04-28 08:51:27,683 INFO MainThread:3812340 [wandb_init.py:init():761] calling init triggers +2025-04-28 08:51:27,683 INFO MainThread:3812340 [wandb_init.py:init():766] wandb.init called with sweep_config: {} +config: {'data_path': '/tmp/haozhezhao/MLLMG/jsonl_data/multiimage_training_for_llava_X2I_fourmask.jsonl', 'cloud_save_path': '/tmp/haozhezhao/MLLMG/checkpoint', 'no_local_save': False, 'vq_model': 'VQ-16', 'vq_ckpt': '/tmp/haozhezhao/model/llamagen_t2i/vq_ds16_t2i.pt', 'codebook_size': 16384, 'codebook_embed_dim': 8, 'gpt_model': 'GPT-XL', 'gpt_ckpt': '/tmp/haozhezhao/MLLMG/MLLMG_ckpts/checkpoints/AfterI2I_llavat5_subject_train_all_T5_subject_t2i_ti2i_120_w_flux_segment_noimageonly_empty_fix_train_all/0100000.pt', 'gpt_type': 't2i', 'vocab_size': 16384, 'cls_token_num': 1280, 'dropout_p': 0.1, 'token_dropout_p': 0.1, 'drop_path': 0.0, 'no_compile': False, 'results_dir': 'checkpoint/X2I_UltraEdiit_fourmask_LLava_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_filtered_objects_4mask_1280_context_after_stage1', 'dataset': 'ti2i', 'image_size': 512, 'downsample_size': 16, 'num_classes': 1000, 'epochs': 1, 'lr': 0.0001, 'weight_decay': 0.05, 'beta1': 0.9, 'beta2': 0.95, 'max_grad_norm': 1.0, 'global_batch_size': 24, 'global_seed': 0, 'num_workers': 4, 'log_every': 25, 'ckpt_every': 2000, 'gradient_accumulation_steps': 8, 'mixed_precision': 'bf16', 'val_data_path': '/tmp/haozhezhao/MLLMG/jsonl_data/multiimage_val_for_llava_X2I_fourmask_dreambenplus.jsonl', 'use_vision_tower': True, 'model_name_or_path': '/tmp/haozhezhao/model/blip2-flan-t5-xl', 'image_place_holder': '', 'processor_path': None, 'do_eval': True, 'max_eval_samples': 128, 'train_text_encoder': True, 'no_left_padding': False, 'cfg_scale': 7.5, 'top_k': 16384, 'temperature': 0.9, 'top_p': 1.0, 'eval_steps': 2000, 'project_name': 'llamagen_ti2i', 'load_from_checkpoint': '/tmp/haozhezhao/MLLMG/checkpoint/FIXed_3M_1epoch_step32000.pt', 'warmup': 0.05, 'lr_decay_style': 'cosine', 'lr_decay_ratio': 0.1, 'train_iters': 500000, 'class_dropout_prob': 0.1, 'with_image_only': False, 'image_only_rate': 0.1, 'stage2': False, 'subject_driven': True, 'load_subject_embedding': None, 'reference_data_path': '/tmp/haozhezhao/MLLMG/cc12m_reference_tunnel.jsonl', 'multimodal_encoder': 'llava', 'do_recovery': True, 'no_replace': False, 'resume': False, 'dreambench_eval': True, 'find_unused_parameters': True, 'load_visual_encoder': False, 'continue_stage1': False, 'replace_subject': False, 'train_all': True, 'save_total_limit': 1, 'load_language_projection': '/tmp/haozhezhao/MLLMG/llava-v1.5-flant5_fixed-pretrain/mm_projector.bin', 'mm_vision_tower': 'openai/clip-vit-large-patch14', 'load_fixed_llamagen': True, 'unfreeze_output': False, 'fix': 'gpt-empty-fix', 'rank': 0, 'world_size': 8, 'gpu': 0, 'dist_url': 'env://', 'distributed': True, 'dist_backend': 'nccl', '_wandb': {}} +2025-04-28 08:51:27,683 INFO MainThread:3812340 [wandb_init.py:init():784] starting backend +2025-04-28 08:51:27,683 INFO MainThread:3812340 [wandb_init.py:init():788] sending inform_init request +2025-04-28 08:51:27,688 INFO MainThread:3812340 [backend.py:_multiprocessing_setup():101] multiprocessing start_methods=fork,spawn,forkserver, using: spawn +2025-04-28 08:51:27,688 INFO MainThread:3812340 [wandb_init.py:init():798] backend started and connected +2025-04-28 08:51:27,690 INFO MainThread:3812340 [wandb_init.py:init():891] updated telemetry +2025-04-28 08:51:27,690 INFO MainThread:3812340 [wandb_init.py:init():915] communicating run to backend with 90.0 second timeout +2025-04-28 08:51:28,280 INFO MainThread:3812340 [wandb_init.py:init():990] starting run threads in backend +2025-04-28 08:51:28,388 INFO MainThread:3812340 [wandb_run.py:_console_start():2375] atexit reg +2025-04-28 08:51:28,388 INFO MainThread:3812340 [wandb_run.py:_redirect():2227] redirect: wrap_raw +2025-04-28 08:51:28,388 INFO MainThread:3812340 [wandb_run.py:_redirect():2292] Wrapping output streams. +2025-04-28 08:51:28,388 INFO MainThread:3812340 [wandb_run.py:_redirect():2315] Redirects installed. +2025-04-28 08:51:28,390 INFO MainThread:3812340 [wandb_init.py:init():1032] run started, returning control to user process diff --git a/CKPTS/X2I_UltraEdiit_fourmask_LLava_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_filtered_objects_4mask_1280_context_after_stage1/wandb/run-20250428_085127-jzkufte5/run-jzkufte5.wandb b/CKPTS/X2I_UltraEdiit_fourmask_LLava_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_filtered_objects_4mask_1280_context_after_stage1/wandb/run-20250428_085127-jzkufte5/run-jzkufte5.wandb new file mode 100644 index 0000000000000000000000000000000000000000..036a3764cd3df7397213dc18a4be2381d227fb8a --- /dev/null +++ b/CKPTS/X2I_UltraEdiit_fourmask_LLava_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_filtered_objects_4mask_1280_context_after_stage1/wandb/run-20250428_085127-jzkufte5/run-jzkufte5.wandb @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:44c1ffaf5fa568fc90d953099a75e21f4e20062ffad262d3e1b9f6012910245a +size 2588672 diff --git a/CKPTS/X2I_UltraEdiit_fourmask_LLava_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_filtered_objects_4mask_1280_context_after_stage1_5e4/000-GPT-XL/checkpoints/0050000.pt b/CKPTS/X2I_UltraEdiit_fourmask_LLava_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_filtered_objects_4mask_1280_context_after_stage1_5e4/000-GPT-XL/checkpoints/0050000.pt new file mode 100644 index 0000000000000000000000000000000000000000..39a0e6bb3aeb47a2329564fb615b3fb58dffe57f --- /dev/null +++ b/CKPTS/X2I_UltraEdiit_fourmask_LLava_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_filtered_objects_4mask_1280_context_after_stage1_5e4/000-GPT-XL/checkpoints/0050000.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:6e7fa75748574f32c05728ee457f3f7340728e4fa9cc014cdfcc9471861f8f79 +size 17328771066 diff --git a/CKPTS/X2I_UltraEdiit_fourmask_LLava_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_filtered_objects_4mask_1280_context_after_stage1_5e4/000-GPT-XL/checkpoints/eval_step_10000/batch_0_cfg_7.5_topk_16384.jpg b/CKPTS/X2I_UltraEdiit_fourmask_LLava_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_filtered_objects_4mask_1280_context_after_stage1_5e4/000-GPT-XL/checkpoints/eval_step_10000/batch_0_cfg_7.5_topk_16384.jpg new file mode 100644 index 0000000000000000000000000000000000000000..052ce85664e6ed2423f038b5e64fb8b380e218c2 --- /dev/null +++ b/CKPTS/X2I_UltraEdiit_fourmask_LLava_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_filtered_objects_4mask_1280_context_after_stage1_5e4/000-GPT-XL/checkpoints/eval_step_10000/batch_0_cfg_7.5_topk_16384.jpg @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:ad9e0da5bf4ded28f3081b3d3b06397d55ddea0145ba1666803f8b3952157888 +size 385641 diff --git a/CKPTS/X2I_UltraEdiit_fourmask_LLava_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_filtered_objects_4mask_1280_context_after_stage1_5e4/000-GPT-XL/checkpoints/eval_step_10000/batch_1_cfg_7.5_topk_16384.jpg b/CKPTS/X2I_UltraEdiit_fourmask_LLava_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_filtered_objects_4mask_1280_context_after_stage1_5e4/000-GPT-XL/checkpoints/eval_step_10000/batch_1_cfg_7.5_topk_16384.jpg new file mode 100644 index 0000000000000000000000000000000000000000..51b92eb6963a15ed36f1cea0759cf25fc45f7ce4 --- /dev/null +++ b/CKPTS/X2I_UltraEdiit_fourmask_LLava_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_filtered_objects_4mask_1280_context_after_stage1_5e4/000-GPT-XL/checkpoints/eval_step_10000/batch_1_cfg_7.5_topk_16384.jpg @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:bcb1188fa7e40114c7bdbc794db4ccb16fde2e64fc99243ad53751b3bc07b365 +size 340453 diff --git a/CKPTS/X2I_UltraEdiit_fourmask_LLava_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_filtered_objects_4mask_1280_context_after_stage1_5e4/000-GPT-XL/checkpoints/eval_step_10000/batch_2_cfg_7.5_topk_16384.jpg b/CKPTS/X2I_UltraEdiit_fourmask_LLava_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_filtered_objects_4mask_1280_context_after_stage1_5e4/000-GPT-XL/checkpoints/eval_step_10000/batch_2_cfg_7.5_topk_16384.jpg new file mode 100644 index 0000000000000000000000000000000000000000..925954f5415a4b64c5b75716e07be9cd62a7a37c --- /dev/null +++ b/CKPTS/X2I_UltraEdiit_fourmask_LLava_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_filtered_objects_4mask_1280_context_after_stage1_5e4/000-GPT-XL/checkpoints/eval_step_10000/batch_2_cfg_7.5_topk_16384.jpg @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:b811d45033df1176e208bacfb01628578a2e5d3d12f061fc5c6be7f947c500e4 +size 355816 diff --git a/CKPTS/X2I_UltraEdiit_fourmask_LLava_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_filtered_objects_4mask_1280_context_after_stage1_5e4/000-GPT-XL/checkpoints/eval_step_10000/batch_3_cfg_7.5_topk_16384.jpg b/CKPTS/X2I_UltraEdiit_fourmask_LLava_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_filtered_objects_4mask_1280_context_after_stage1_5e4/000-GPT-XL/checkpoints/eval_step_10000/batch_3_cfg_7.5_topk_16384.jpg new file mode 100644 index 0000000000000000000000000000000000000000..cbe842a9115c02d0368154be0caa3ef5cc265db1 --- /dev/null +++ b/CKPTS/X2I_UltraEdiit_fourmask_LLava_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_filtered_objects_4mask_1280_context_after_stage1_5e4/000-GPT-XL/checkpoints/eval_step_10000/batch_3_cfg_7.5_topk_16384.jpg @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:0dc7c4a4d177d810b61d6495191e072a7a03a0fbb0ade8fcac2673ed357207ff +size 387102 diff --git a/CKPTS/X2I_UltraEdiit_fourmask_LLava_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_filtered_objects_4mask_1280_context_after_stage1_5e4/000-GPT-XL/checkpoints/eval_step_10000/batch_4_cfg_7.5_topk_16384.jpg b/CKPTS/X2I_UltraEdiit_fourmask_LLava_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_filtered_objects_4mask_1280_context_after_stage1_5e4/000-GPT-XL/checkpoints/eval_step_10000/batch_4_cfg_7.5_topk_16384.jpg new file mode 100644 index 0000000000000000000000000000000000000000..e77a707f330f559e141d7d5d2c180a78dc32905a --- /dev/null +++ b/CKPTS/X2I_UltraEdiit_fourmask_LLava_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_filtered_objects_4mask_1280_context_after_stage1_5e4/000-GPT-XL/checkpoints/eval_step_10000/batch_4_cfg_7.5_topk_16384.jpg @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:1bc82bfbab48994ade101a417a39a3824d9737556803ba7de521aea7762dd5dd +size 385855 diff --git a/CKPTS/X2I_UltraEdiit_fourmask_LLava_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_filtered_objects_4mask_1280_context_after_stage1_5e4/000-GPT-XL/checkpoints/eval_step_10000/batch_5_cfg_7.5_topk_16384.jpg b/CKPTS/X2I_UltraEdiit_fourmask_LLava_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_filtered_objects_4mask_1280_context_after_stage1_5e4/000-GPT-XL/checkpoints/eval_step_10000/batch_5_cfg_7.5_topk_16384.jpg new file mode 100644 index 0000000000000000000000000000000000000000..2672039cb94b160dc54496c2ff4f687b812e1ccc --- /dev/null +++ b/CKPTS/X2I_UltraEdiit_fourmask_LLava_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_filtered_objects_4mask_1280_context_after_stage1_5e4/000-GPT-XL/checkpoints/eval_step_10000/batch_5_cfg_7.5_topk_16384.jpg @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:de8568fa28c02ff7a06d57363a6ad09a4fc222ab11dbd086a5bfb7cbdd07c735 +size 381841 diff --git a/CKPTS/X2I_UltraEdiit_fourmask_LLava_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_filtered_objects_4mask_1280_context_after_stage1_5e4/000-GPT-XL/checkpoints/eval_step_12000/batch_0_cfg_7.5_topk_16384.jpg b/CKPTS/X2I_UltraEdiit_fourmask_LLava_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_filtered_objects_4mask_1280_context_after_stage1_5e4/000-GPT-XL/checkpoints/eval_step_12000/batch_0_cfg_7.5_topk_16384.jpg new file mode 100644 index 0000000000000000000000000000000000000000..0b7dde9adf2bb884a1ff5ec87ee22f9f03faaa3a --- /dev/null +++ b/CKPTS/X2I_UltraEdiit_fourmask_LLava_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_filtered_objects_4mask_1280_context_after_stage1_5e4/000-GPT-XL/checkpoints/eval_step_12000/batch_0_cfg_7.5_topk_16384.jpg @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:21df9fea183900250a5e7669c717e727c03ef91635f5cb35def1a68b07ba24db +size 384493 diff --git a/CKPTS/X2I_UltraEdiit_fourmask_LLava_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_filtered_objects_4mask_1280_context_after_stage1_5e4/000-GPT-XL/checkpoints/eval_step_12000/batch_1_cfg_7.5_topk_16384.jpg b/CKPTS/X2I_UltraEdiit_fourmask_LLava_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_filtered_objects_4mask_1280_context_after_stage1_5e4/000-GPT-XL/checkpoints/eval_step_12000/batch_1_cfg_7.5_topk_16384.jpg new file mode 100644 index 0000000000000000000000000000000000000000..552e32f8477ad13d20cfebce73ef250096be0693 --- /dev/null +++ b/CKPTS/X2I_UltraEdiit_fourmask_LLava_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_filtered_objects_4mask_1280_context_after_stage1_5e4/000-GPT-XL/checkpoints/eval_step_12000/batch_1_cfg_7.5_topk_16384.jpg @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:207fb96ddd2cadac44a91c729199150bf63ef148e416883bfe50c21ea44bb450 +size 344918 diff --git a/CKPTS/X2I_UltraEdiit_fourmask_LLava_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_filtered_objects_4mask_1280_context_after_stage1_5e4/000-GPT-XL/checkpoints/eval_step_12000/batch_2_cfg_7.5_topk_16384.jpg b/CKPTS/X2I_UltraEdiit_fourmask_LLava_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_filtered_objects_4mask_1280_context_after_stage1_5e4/000-GPT-XL/checkpoints/eval_step_12000/batch_2_cfg_7.5_topk_16384.jpg new file mode 100644 index 0000000000000000000000000000000000000000..3059b99812019f7fcab9a59892fc41c8c14f3637 --- /dev/null +++ b/CKPTS/X2I_UltraEdiit_fourmask_LLava_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_filtered_objects_4mask_1280_context_after_stage1_5e4/000-GPT-XL/checkpoints/eval_step_12000/batch_2_cfg_7.5_topk_16384.jpg @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:dce00f4b4e9598c10e60ac1f4e1402eab19c91861c2d28356b9e7a6f6a855023 +size 357502 diff --git a/CKPTS/X2I_UltraEdiit_fourmask_LLava_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_filtered_objects_4mask_1280_context_after_stage1_5e4/000-GPT-XL/checkpoints/eval_step_12000/batch_3_cfg_7.5_topk_16384.jpg b/CKPTS/X2I_UltraEdiit_fourmask_LLava_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_filtered_objects_4mask_1280_context_after_stage1_5e4/000-GPT-XL/checkpoints/eval_step_12000/batch_3_cfg_7.5_topk_16384.jpg new file mode 100644 index 0000000000000000000000000000000000000000..b501da548ff96e6f79627a7899b06f16ad1ff9b1 --- /dev/null +++ b/CKPTS/X2I_UltraEdiit_fourmask_LLava_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_filtered_objects_4mask_1280_context_after_stage1_5e4/000-GPT-XL/checkpoints/eval_step_12000/batch_3_cfg_7.5_topk_16384.jpg @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:d9c0b82b2e3c6c2bf6aa133eb4e277342e843e14c1e1f160db88979a34d8fbf5 +size 394432 diff --git a/CKPTS/X2I_UltraEdiit_fourmask_LLava_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_filtered_objects_4mask_1280_context_after_stage1_5e4/000-GPT-XL/checkpoints/eval_step_12000/batch_4_cfg_7.5_topk_16384.jpg b/CKPTS/X2I_UltraEdiit_fourmask_LLava_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_filtered_objects_4mask_1280_context_after_stage1_5e4/000-GPT-XL/checkpoints/eval_step_12000/batch_4_cfg_7.5_topk_16384.jpg new file mode 100644 index 0000000000000000000000000000000000000000..4ada074b3ea0643f07c5c8c47486ecc3b4309963 --- /dev/null +++ b/CKPTS/X2I_UltraEdiit_fourmask_LLava_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_filtered_objects_4mask_1280_context_after_stage1_5e4/000-GPT-XL/checkpoints/eval_step_12000/batch_4_cfg_7.5_topk_16384.jpg @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:c5b1bcaf8ab651a8e8b9d601ebe93899a7f6b7635eaa66b4cd46f3aa098bbef2 +size 419003 diff --git a/CKPTS/X2I_UltraEdiit_fourmask_LLava_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_filtered_objects_4mask_1280_context_after_stage1_5e4/000-GPT-XL/checkpoints/eval_step_12000/batch_5_cfg_7.5_topk_16384.jpg b/CKPTS/X2I_UltraEdiit_fourmask_LLava_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_filtered_objects_4mask_1280_context_after_stage1_5e4/000-GPT-XL/checkpoints/eval_step_12000/batch_5_cfg_7.5_topk_16384.jpg new file mode 100644 index 0000000000000000000000000000000000000000..25459babb8a91cd6b055f0e1347d4189ab90ff4f --- /dev/null +++ b/CKPTS/X2I_UltraEdiit_fourmask_LLava_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_filtered_objects_4mask_1280_context_after_stage1_5e4/000-GPT-XL/checkpoints/eval_step_12000/batch_5_cfg_7.5_topk_16384.jpg @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:3c74654a54e5f6c34ab3ce1dfc0e73519cf5621e79fd1a4b6960ceecc44ce610 +size 380026 diff --git a/CKPTS/X2I_UltraEdiit_fourmask_LLava_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_filtered_objects_4mask_1280_context_after_stage1_5e4/000-GPT-XL/checkpoints/eval_step_14000/batch_0_cfg_7.5_topk_16384.jpg b/CKPTS/X2I_UltraEdiit_fourmask_LLava_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_filtered_objects_4mask_1280_context_after_stage1_5e4/000-GPT-XL/checkpoints/eval_step_14000/batch_0_cfg_7.5_topk_16384.jpg new file mode 100644 index 0000000000000000000000000000000000000000..796728dacf51c02cd89a3f5cffcdb58acb8bfeae --- /dev/null +++ b/CKPTS/X2I_UltraEdiit_fourmask_LLava_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_filtered_objects_4mask_1280_context_after_stage1_5e4/000-GPT-XL/checkpoints/eval_step_14000/batch_0_cfg_7.5_topk_16384.jpg @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:2adfe31e0bbe64d995abeb24c4d385522150ff69fe4f08fe517d895e9cdf5f14 +size 381941 diff --git a/CKPTS/X2I_UltraEdiit_fourmask_LLava_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_filtered_objects_4mask_1280_context_after_stage1_5e4/000-GPT-XL/checkpoints/eval_step_14000/batch_1_cfg_7.5_topk_16384.jpg b/CKPTS/X2I_UltraEdiit_fourmask_LLava_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_filtered_objects_4mask_1280_context_after_stage1_5e4/000-GPT-XL/checkpoints/eval_step_14000/batch_1_cfg_7.5_topk_16384.jpg new file mode 100644 index 0000000000000000000000000000000000000000..eb5c8f0754a4a34d9bc77e95f85722cdf58fd7a0 --- /dev/null +++ b/CKPTS/X2I_UltraEdiit_fourmask_LLava_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_filtered_objects_4mask_1280_context_after_stage1_5e4/000-GPT-XL/checkpoints/eval_step_14000/batch_1_cfg_7.5_topk_16384.jpg @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:584439598d009c1b35fdbf8be3f3b73bb41fc7571750a84aaf2a4dc8ba3b4446 +size 348726 diff --git a/CKPTS/X2I_UltraEdiit_fourmask_LLava_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_filtered_objects_4mask_1280_context_after_stage1_5e4/000-GPT-XL/checkpoints/eval_step_14000/batch_2_cfg_7.5_topk_16384.jpg b/CKPTS/X2I_UltraEdiit_fourmask_LLava_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_filtered_objects_4mask_1280_context_after_stage1_5e4/000-GPT-XL/checkpoints/eval_step_14000/batch_2_cfg_7.5_topk_16384.jpg new file mode 100644 index 0000000000000000000000000000000000000000..b88f70b4c7807902e5d7d076276e936fe1356e09 --- /dev/null +++ b/CKPTS/X2I_UltraEdiit_fourmask_LLava_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_filtered_objects_4mask_1280_context_after_stage1_5e4/000-GPT-XL/checkpoints/eval_step_14000/batch_2_cfg_7.5_topk_16384.jpg @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:0c7ec1e8f1608c30d2d2a70bd5d5192a3b5d42473e0e003b9c8b9e062eeb2872 +size 353980 diff --git a/CKPTS/X2I_UltraEdiit_fourmask_LLava_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_filtered_objects_4mask_1280_context_after_stage1_5e4/000-GPT-XL/checkpoints/eval_step_14000/batch_3_cfg_7.5_topk_16384.jpg b/CKPTS/X2I_UltraEdiit_fourmask_LLava_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_filtered_objects_4mask_1280_context_after_stage1_5e4/000-GPT-XL/checkpoints/eval_step_14000/batch_3_cfg_7.5_topk_16384.jpg new file mode 100644 index 0000000000000000000000000000000000000000..fb8ec5afa2cb8dbae4453edbd7de8971350b00f3 --- /dev/null +++ b/CKPTS/X2I_UltraEdiit_fourmask_LLava_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_filtered_objects_4mask_1280_context_after_stage1_5e4/000-GPT-XL/checkpoints/eval_step_14000/batch_3_cfg_7.5_topk_16384.jpg @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:71e5424ae1127c1e8a2db5d90a56ddfecc29b152d0f5d1c979a3593ac072445f +size 390800 diff --git a/CKPTS/X2I_UltraEdiit_fourmask_LLava_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_filtered_objects_4mask_1280_context_after_stage1_5e4/000-GPT-XL/checkpoints/eval_step_14000/batch_4_cfg_7.5_topk_16384.jpg b/CKPTS/X2I_UltraEdiit_fourmask_LLava_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_filtered_objects_4mask_1280_context_after_stage1_5e4/000-GPT-XL/checkpoints/eval_step_14000/batch_4_cfg_7.5_topk_16384.jpg new file mode 100644 index 0000000000000000000000000000000000000000..0d44739f17da5333a7eeed5a6642c4b1aab80591 --- /dev/null +++ b/CKPTS/X2I_UltraEdiit_fourmask_LLava_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_filtered_objects_4mask_1280_context_after_stage1_5e4/000-GPT-XL/checkpoints/eval_step_14000/batch_4_cfg_7.5_topk_16384.jpg @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:457eb0890c6c466dda5d8b5bff50136a0c6d97e99d9ba18eddd666477dc603a8 +size 400433 diff --git a/CKPTS/X2I_UltraEdiit_fourmask_LLava_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_filtered_objects_4mask_1280_context_after_stage1_5e4/000-GPT-XL/checkpoints/eval_step_14000/batch_5_cfg_7.5_topk_16384.jpg b/CKPTS/X2I_UltraEdiit_fourmask_LLava_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_filtered_objects_4mask_1280_context_after_stage1_5e4/000-GPT-XL/checkpoints/eval_step_14000/batch_5_cfg_7.5_topk_16384.jpg new file mode 100644 index 0000000000000000000000000000000000000000..5bfe2ac29ca7984d8db6d40d5131b40cc5794d41 --- /dev/null +++ b/CKPTS/X2I_UltraEdiit_fourmask_LLava_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_filtered_objects_4mask_1280_context_after_stage1_5e4/000-GPT-XL/checkpoints/eval_step_14000/batch_5_cfg_7.5_topk_16384.jpg @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:74f0a4584547ee1dfee5d67158bd20a2dfbc30bdefa96366819417ff7dd3aea0 +size 383100 diff --git a/CKPTS/X2I_UltraEdiit_fourmask_LLava_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_filtered_objects_4mask_1280_context_after_stage1_5e4/000-GPT-XL/checkpoints/eval_step_16000/batch_0_cfg_7.5_topk_16384.jpg b/CKPTS/X2I_UltraEdiit_fourmask_LLava_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_filtered_objects_4mask_1280_context_after_stage1_5e4/000-GPT-XL/checkpoints/eval_step_16000/batch_0_cfg_7.5_topk_16384.jpg new file mode 100644 index 0000000000000000000000000000000000000000..703542624b5aa73103943f401f46c12db29ef370 --- /dev/null +++ b/CKPTS/X2I_UltraEdiit_fourmask_LLava_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_filtered_objects_4mask_1280_context_after_stage1_5e4/000-GPT-XL/checkpoints/eval_step_16000/batch_0_cfg_7.5_topk_16384.jpg @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:fb35da58c3f162616c9e38edc3224d9572f1cf401dfb74afe19d14ed8165c720 +size 377847 diff --git a/CKPTS/X2I_UltraEdiit_fourmask_LLava_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_filtered_objects_4mask_1280_context_after_stage1_5e4/000-GPT-XL/checkpoints/eval_step_16000/batch_1_cfg_7.5_topk_16384.jpg b/CKPTS/X2I_UltraEdiit_fourmask_LLava_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_filtered_objects_4mask_1280_context_after_stage1_5e4/000-GPT-XL/checkpoints/eval_step_16000/batch_1_cfg_7.5_topk_16384.jpg new file mode 100644 index 0000000000000000000000000000000000000000..e507e6e1a7742506513c23cf1aab67b3c94da156 --- /dev/null +++ b/CKPTS/X2I_UltraEdiit_fourmask_LLava_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_filtered_objects_4mask_1280_context_after_stage1_5e4/000-GPT-XL/checkpoints/eval_step_16000/batch_1_cfg_7.5_topk_16384.jpg @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:3430d902ad62bfa4180cd6b67d116988fe10d660998dae13ee922d9943f38e24 +size 343786 diff --git a/CKPTS/X2I_UltraEdiit_fourmask_LLava_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_filtered_objects_4mask_1280_context_after_stage1_5e4/000-GPT-XL/checkpoints/eval_step_16000/batch_2_cfg_7.5_topk_16384.jpg b/CKPTS/X2I_UltraEdiit_fourmask_LLava_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_filtered_objects_4mask_1280_context_after_stage1_5e4/000-GPT-XL/checkpoints/eval_step_16000/batch_2_cfg_7.5_topk_16384.jpg new file mode 100644 index 0000000000000000000000000000000000000000..c41c94ad0e40f0c53247f4fbd888f30758df6dc6 --- /dev/null +++ b/CKPTS/X2I_UltraEdiit_fourmask_LLava_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_filtered_objects_4mask_1280_context_after_stage1_5e4/000-GPT-XL/checkpoints/eval_step_16000/batch_2_cfg_7.5_topk_16384.jpg @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:cbd5e9eb4151929cbe17b011bdffd8dd0dac7b0f3f05b0c440de09103237ef08 +size 355896 diff --git a/CKPTS/X2I_UltraEdiit_fourmask_LLava_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_filtered_objects_4mask_1280_context_after_stage1_5e4/000-GPT-XL/checkpoints/eval_step_16000/batch_3_cfg_7.5_topk_16384.jpg b/CKPTS/X2I_UltraEdiit_fourmask_LLava_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_filtered_objects_4mask_1280_context_after_stage1_5e4/000-GPT-XL/checkpoints/eval_step_16000/batch_3_cfg_7.5_topk_16384.jpg new file mode 100644 index 0000000000000000000000000000000000000000..ea5c14310493372449e48b5c4fc783715de8ddbb --- /dev/null +++ b/CKPTS/X2I_UltraEdiit_fourmask_LLava_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_filtered_objects_4mask_1280_context_after_stage1_5e4/000-GPT-XL/checkpoints/eval_step_16000/batch_3_cfg_7.5_topk_16384.jpg @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:b9655480dfa43d40516d65f4b44aa7dd8d5990bd8c51b01495aae1f8ff7df27b +size 388084 diff --git a/CKPTS/X2I_UltraEdiit_fourmask_LLava_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_filtered_objects_4mask_1280_context_after_stage1_5e4/000-GPT-XL/checkpoints/eval_step_16000/batch_4_cfg_7.5_topk_16384.jpg b/CKPTS/X2I_UltraEdiit_fourmask_LLava_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_filtered_objects_4mask_1280_context_after_stage1_5e4/000-GPT-XL/checkpoints/eval_step_16000/batch_4_cfg_7.5_topk_16384.jpg new file mode 100644 index 0000000000000000000000000000000000000000..9c0ae5de2df7595a81c00bbcbb6300a382d41411 --- /dev/null +++ b/CKPTS/X2I_UltraEdiit_fourmask_LLava_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_filtered_objects_4mask_1280_context_after_stage1_5e4/000-GPT-XL/checkpoints/eval_step_16000/batch_4_cfg_7.5_topk_16384.jpg @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:721c2c3fc7eb0099b5d4f5a79cf142d97ebc8d6a2c1b3525846fa1a3357985cb +size 414791 diff --git a/CKPTS/X2I_UltraEdiit_fourmask_LLava_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_filtered_objects_4mask_1280_context_after_stage1_5e4/000-GPT-XL/checkpoints/eval_step_16000/batch_5_cfg_7.5_topk_16384.jpg b/CKPTS/X2I_UltraEdiit_fourmask_LLava_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_filtered_objects_4mask_1280_context_after_stage1_5e4/000-GPT-XL/checkpoints/eval_step_16000/batch_5_cfg_7.5_topk_16384.jpg new file mode 100644 index 0000000000000000000000000000000000000000..8e03a09d2a79e906cda14041cdcf29f866848978 --- /dev/null +++ b/CKPTS/X2I_UltraEdiit_fourmask_LLava_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_filtered_objects_4mask_1280_context_after_stage1_5e4/000-GPT-XL/checkpoints/eval_step_16000/batch_5_cfg_7.5_topk_16384.jpg @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:5ea0577f47eb33f63fe3e9e9d0e93ef52dc40ab8beb240975cdeb6cfdc830d97 +size 377451 diff --git a/CKPTS/X2I_UltraEdiit_fourmask_LLava_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_filtered_objects_4mask_1280_context_after_stage1_5e4/000-GPT-XL/checkpoints/eval_step_18000/batch_0_cfg_7.5_topk_16384.jpg b/CKPTS/X2I_UltraEdiit_fourmask_LLava_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_filtered_objects_4mask_1280_context_after_stage1_5e4/000-GPT-XL/checkpoints/eval_step_18000/batch_0_cfg_7.5_topk_16384.jpg new file mode 100644 index 0000000000000000000000000000000000000000..37bc654d7a845315584cd497c85febe6291eb65b --- /dev/null +++ b/CKPTS/X2I_UltraEdiit_fourmask_LLava_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_filtered_objects_4mask_1280_context_after_stage1_5e4/000-GPT-XL/checkpoints/eval_step_18000/batch_0_cfg_7.5_topk_16384.jpg @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:137778fb8ecb5522799c409bfeb3e6cc39945c0fa5492d96e21d401d441dac95 +size 388551 diff --git a/CKPTS/X2I_UltraEdiit_fourmask_LLava_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_filtered_objects_4mask_1280_context_after_stage1_5e4/000-GPT-XL/checkpoints/eval_step_18000/batch_1_cfg_7.5_topk_16384.jpg b/CKPTS/X2I_UltraEdiit_fourmask_LLava_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_filtered_objects_4mask_1280_context_after_stage1_5e4/000-GPT-XL/checkpoints/eval_step_18000/batch_1_cfg_7.5_topk_16384.jpg new file mode 100644 index 0000000000000000000000000000000000000000..e1f283cce85f689c10afc15c31e40886e6ba525c --- /dev/null +++ b/CKPTS/X2I_UltraEdiit_fourmask_LLava_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_filtered_objects_4mask_1280_context_after_stage1_5e4/000-GPT-XL/checkpoints/eval_step_18000/batch_1_cfg_7.5_topk_16384.jpg @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:26fd6ba86d07acc19d7bf4314ef87d4af75be39f4a6c32b8c229cbcfff497ff7 +size 350073 diff --git a/CKPTS/X2I_UltraEdiit_fourmask_LLava_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_filtered_objects_4mask_1280_context_after_stage1_5e4/000-GPT-XL/checkpoints/eval_step_18000/batch_2_cfg_7.5_topk_16384.jpg b/CKPTS/X2I_UltraEdiit_fourmask_LLava_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_filtered_objects_4mask_1280_context_after_stage1_5e4/000-GPT-XL/checkpoints/eval_step_18000/batch_2_cfg_7.5_topk_16384.jpg new file mode 100644 index 0000000000000000000000000000000000000000..ddfbbd77fcee89f91e34f9fa29d70a9b790bb60d --- /dev/null +++ b/CKPTS/X2I_UltraEdiit_fourmask_LLava_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_filtered_objects_4mask_1280_context_after_stage1_5e4/000-GPT-XL/checkpoints/eval_step_18000/batch_2_cfg_7.5_topk_16384.jpg @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:0cd4b11c247d855fc00d322e7b68158da0f3eada38ac5cf971b3e99e2f290a16 +size 362608 diff --git a/CKPTS/X2I_UltraEdiit_fourmask_LLava_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_filtered_objects_4mask_1280_context_after_stage1_5e4/000-GPT-XL/checkpoints/eval_step_18000/batch_3_cfg_7.5_topk_16384.jpg b/CKPTS/X2I_UltraEdiit_fourmask_LLava_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_filtered_objects_4mask_1280_context_after_stage1_5e4/000-GPT-XL/checkpoints/eval_step_18000/batch_3_cfg_7.5_topk_16384.jpg new file mode 100644 index 0000000000000000000000000000000000000000..a975f88b1b186edd12bdf9e316aaa8c574bde1ab --- /dev/null +++ b/CKPTS/X2I_UltraEdiit_fourmask_LLava_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_filtered_objects_4mask_1280_context_after_stage1_5e4/000-GPT-XL/checkpoints/eval_step_18000/batch_3_cfg_7.5_topk_16384.jpg @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:4fe14401e6a67090bbdffa71dc19278f936ec47254af29642cb26df2781fa2f7 +size 393597 diff --git a/CKPTS/X2I_UltraEdiit_fourmask_LLava_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_filtered_objects_4mask_1280_context_after_stage1_5e4/000-GPT-XL/checkpoints/eval_step_18000/batch_4_cfg_7.5_topk_16384.jpg b/CKPTS/X2I_UltraEdiit_fourmask_LLava_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_filtered_objects_4mask_1280_context_after_stage1_5e4/000-GPT-XL/checkpoints/eval_step_18000/batch_4_cfg_7.5_topk_16384.jpg new file mode 100644 index 0000000000000000000000000000000000000000..d0692b5b237c29ea709c8ad3970e41abf7868d8e --- /dev/null +++ b/CKPTS/X2I_UltraEdiit_fourmask_LLava_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_filtered_objects_4mask_1280_context_after_stage1_5e4/000-GPT-XL/checkpoints/eval_step_18000/batch_4_cfg_7.5_topk_16384.jpg @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:9c08b2249497a0a615079658611cc38c0bf86d51d53a1bd2c0b8de2c52520615 +size 414981 diff --git a/CKPTS/X2I_UltraEdiit_fourmask_LLava_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_filtered_objects_4mask_1280_context_after_stage1_5e4/000-GPT-XL/checkpoints/eval_step_18000/batch_5_cfg_7.5_topk_16384.jpg b/CKPTS/X2I_UltraEdiit_fourmask_LLava_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_filtered_objects_4mask_1280_context_after_stage1_5e4/000-GPT-XL/checkpoints/eval_step_18000/batch_5_cfg_7.5_topk_16384.jpg new file mode 100644 index 0000000000000000000000000000000000000000..32711a1dca4066ab48333b208058c8d0c207f3ed --- /dev/null +++ b/CKPTS/X2I_UltraEdiit_fourmask_LLava_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_filtered_objects_4mask_1280_context_after_stage1_5e4/000-GPT-XL/checkpoints/eval_step_18000/batch_5_cfg_7.5_topk_16384.jpg @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:340510cd4decb20cbed8da44db95f13cb32ca567c6e786838aab7069b8c6b4ce +size 370546 diff --git a/CKPTS/X2I_UltraEdiit_fourmask_LLava_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_filtered_objects_4mask_1280_context_after_stage1_5e4/000-GPT-XL/checkpoints/eval_step_2000/batch_0_cfg_7.5_topk_16384.jpg b/CKPTS/X2I_UltraEdiit_fourmask_LLava_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_filtered_objects_4mask_1280_context_after_stage1_5e4/000-GPT-XL/checkpoints/eval_step_2000/batch_0_cfg_7.5_topk_16384.jpg new file mode 100644 index 0000000000000000000000000000000000000000..8b0ca2cfebb963f70cc4ec5d793107397f7dc72a --- /dev/null +++ b/CKPTS/X2I_UltraEdiit_fourmask_LLava_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_filtered_objects_4mask_1280_context_after_stage1_5e4/000-GPT-XL/checkpoints/eval_step_2000/batch_0_cfg_7.5_topk_16384.jpg @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:8ef84e6b796449e7e752a61764d1200c0389f4db071f7b5284d9c7138fd96a95 +size 389118 diff --git a/CKPTS/X2I_UltraEdiit_fourmask_LLava_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_filtered_objects_4mask_1280_context_after_stage1_5e4/000-GPT-XL/checkpoints/eval_step_2000/batch_1_cfg_7.5_topk_16384.jpg b/CKPTS/X2I_UltraEdiit_fourmask_LLava_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_filtered_objects_4mask_1280_context_after_stage1_5e4/000-GPT-XL/checkpoints/eval_step_2000/batch_1_cfg_7.5_topk_16384.jpg new file mode 100644 index 0000000000000000000000000000000000000000..1e99a8b952eae0463e7b3b31f9f444dd7e6f4f0e --- /dev/null +++ b/CKPTS/X2I_UltraEdiit_fourmask_LLava_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_filtered_objects_4mask_1280_context_after_stage1_5e4/000-GPT-XL/checkpoints/eval_step_2000/batch_1_cfg_7.5_topk_16384.jpg @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:17718cc029293b365babcb28f1748590070077a44902bb7e2c76b7ccb63a2952 +size 364681 diff --git a/CKPTS/X2I_UltraEdiit_fourmask_LLava_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_filtered_objects_4mask_1280_context_after_stage1_5e4/000-GPT-XL/checkpoints/eval_step_2000/batch_2_cfg_7.5_topk_16384.jpg b/CKPTS/X2I_UltraEdiit_fourmask_LLava_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_filtered_objects_4mask_1280_context_after_stage1_5e4/000-GPT-XL/checkpoints/eval_step_2000/batch_2_cfg_7.5_topk_16384.jpg new file mode 100644 index 0000000000000000000000000000000000000000..685dc8b942eab6be37f09acd2d6136d1f869b7b9 --- /dev/null +++ b/CKPTS/X2I_UltraEdiit_fourmask_LLava_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_filtered_objects_4mask_1280_context_after_stage1_5e4/000-GPT-XL/checkpoints/eval_step_2000/batch_2_cfg_7.5_topk_16384.jpg @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:a6fc531e1b4c4157804393ceb856109a61ce1f6084790145f82bea015215a243 +size 361525 diff --git a/CKPTS/X2I_UltraEdiit_fourmask_LLava_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_filtered_objects_4mask_1280_context_after_stage1_5e4/000-GPT-XL/checkpoints/eval_step_2000/batch_3_cfg_7.5_topk_16384.jpg b/CKPTS/X2I_UltraEdiit_fourmask_LLava_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_filtered_objects_4mask_1280_context_after_stage1_5e4/000-GPT-XL/checkpoints/eval_step_2000/batch_3_cfg_7.5_topk_16384.jpg new file mode 100644 index 0000000000000000000000000000000000000000..9cfc8154f6e5c19bf5a2ba1bf9d81fc68c51d1a3 --- /dev/null +++ b/CKPTS/X2I_UltraEdiit_fourmask_LLava_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_filtered_objects_4mask_1280_context_after_stage1_5e4/000-GPT-XL/checkpoints/eval_step_2000/batch_3_cfg_7.5_topk_16384.jpg @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:db62c81b2e6984574595d78ef23f9c4518264dd320d94d5ba1c6bdca2de7beef +size 393575 diff --git a/CKPTS/X2I_UltraEdiit_fourmask_LLava_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_filtered_objects_4mask_1280_context_after_stage1_5e4/000-GPT-XL/checkpoints/eval_step_2000/batch_4_cfg_7.5_topk_16384.jpg b/CKPTS/X2I_UltraEdiit_fourmask_LLava_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_filtered_objects_4mask_1280_context_after_stage1_5e4/000-GPT-XL/checkpoints/eval_step_2000/batch_4_cfg_7.5_topk_16384.jpg new file mode 100644 index 0000000000000000000000000000000000000000..3fdfc47c679eda67a464f798921b327c8c8ae20b --- /dev/null +++ b/CKPTS/X2I_UltraEdiit_fourmask_LLava_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_filtered_objects_4mask_1280_context_after_stage1_5e4/000-GPT-XL/checkpoints/eval_step_2000/batch_4_cfg_7.5_topk_16384.jpg @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:c35dd61eb9bdc3dac4b487c2824ab83d31eb14362dc4c8d4bcb0115106a5074d +size 410243 diff --git a/CKPTS/X2I_UltraEdiit_fourmask_LLava_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_filtered_objects_4mask_1280_context_after_stage1_5e4/000-GPT-XL/checkpoints/eval_step_2000/batch_5_cfg_7.5_topk_16384.jpg b/CKPTS/X2I_UltraEdiit_fourmask_LLava_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_filtered_objects_4mask_1280_context_after_stage1_5e4/000-GPT-XL/checkpoints/eval_step_2000/batch_5_cfg_7.5_topk_16384.jpg new file mode 100644 index 0000000000000000000000000000000000000000..3d2c5ccb96c94803451f3587c6f06959343525cc --- /dev/null +++ b/CKPTS/X2I_UltraEdiit_fourmask_LLava_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_filtered_objects_4mask_1280_context_after_stage1_5e4/000-GPT-XL/checkpoints/eval_step_2000/batch_5_cfg_7.5_topk_16384.jpg @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:45063a3830772b45ac55b55b2544db9fab9ff24d58af416c69f23be1c7b8d50f +size 380708 diff --git a/CKPTS/X2I_UltraEdiit_fourmask_LLava_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_filtered_objects_4mask_1280_context_after_stage1_5e4/000-GPT-XL/checkpoints/eval_step_20000/batch_0_cfg_7.5_topk_16384.jpg b/CKPTS/X2I_UltraEdiit_fourmask_LLava_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_filtered_objects_4mask_1280_context_after_stage1_5e4/000-GPT-XL/checkpoints/eval_step_20000/batch_0_cfg_7.5_topk_16384.jpg new file mode 100644 index 0000000000000000000000000000000000000000..0c6e264781152a6cd04f05056b0120b86101e093 --- /dev/null +++ b/CKPTS/X2I_UltraEdiit_fourmask_LLava_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_filtered_objects_4mask_1280_context_after_stage1_5e4/000-GPT-XL/checkpoints/eval_step_20000/batch_0_cfg_7.5_topk_16384.jpg @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:f0d23390fe8a145c6bb39bb2739f8fdbc68bc0e57850bd78944834aa430d3003 +size 385137 diff --git a/CKPTS/X2I_UltraEdiit_fourmask_LLava_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_filtered_objects_4mask_1280_context_after_stage1_5e4/000-GPT-XL/checkpoints/eval_step_20000/batch_1_cfg_7.5_topk_16384.jpg b/CKPTS/X2I_UltraEdiit_fourmask_LLava_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_filtered_objects_4mask_1280_context_after_stage1_5e4/000-GPT-XL/checkpoints/eval_step_20000/batch_1_cfg_7.5_topk_16384.jpg new file mode 100644 index 0000000000000000000000000000000000000000..1e29e469abbed8317f92a22a939ced0a12f1ff08 --- /dev/null +++ b/CKPTS/X2I_UltraEdiit_fourmask_LLava_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_filtered_objects_4mask_1280_context_after_stage1_5e4/000-GPT-XL/checkpoints/eval_step_20000/batch_1_cfg_7.5_topk_16384.jpg @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:a6bc81a2d10a5fb1f6b4fffac4975df03ff5bdb6f67a8c47b6813b210a1c681f +size 337366 diff --git a/CKPTS/X2I_UltraEdiit_fourmask_LLava_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_filtered_objects_4mask_1280_context_after_stage1_5e4/000-GPT-XL/checkpoints/eval_step_20000/batch_2_cfg_7.5_topk_16384.jpg b/CKPTS/X2I_UltraEdiit_fourmask_LLava_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_filtered_objects_4mask_1280_context_after_stage1_5e4/000-GPT-XL/checkpoints/eval_step_20000/batch_2_cfg_7.5_topk_16384.jpg new file mode 100644 index 0000000000000000000000000000000000000000..210e990af984d790053a0c30d5518049492ff5f2 --- /dev/null +++ b/CKPTS/X2I_UltraEdiit_fourmask_LLava_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_filtered_objects_4mask_1280_context_after_stage1_5e4/000-GPT-XL/checkpoints/eval_step_20000/batch_2_cfg_7.5_topk_16384.jpg @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:61b85ca30aff43c8b66ec7900e0566cf43ced6a5889b374a691bc0973d16dedf +size 358060 diff --git a/CKPTS/X2I_UltraEdiit_fourmask_LLava_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_filtered_objects_4mask_1280_context_after_stage1_5e4/000-GPT-XL/checkpoints/eval_step_20000/batch_3_cfg_7.5_topk_16384.jpg b/CKPTS/X2I_UltraEdiit_fourmask_LLava_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_filtered_objects_4mask_1280_context_after_stage1_5e4/000-GPT-XL/checkpoints/eval_step_20000/batch_3_cfg_7.5_topk_16384.jpg new file mode 100644 index 0000000000000000000000000000000000000000..92b84615c6642ef3555e08700debf16f1cd8f044 --- /dev/null +++ b/CKPTS/X2I_UltraEdiit_fourmask_LLava_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_filtered_objects_4mask_1280_context_after_stage1_5e4/000-GPT-XL/checkpoints/eval_step_20000/batch_3_cfg_7.5_topk_16384.jpg @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:9f49a8335cf6bbdfb870739131ca2febec127eb75f9981044a927c8b5f56e8a1 +size 388916 diff --git a/CKPTS/X2I_UltraEdiit_fourmask_LLava_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_filtered_objects_4mask_1280_context_after_stage1_5e4/000-GPT-XL/checkpoints/eval_step_20000/batch_4_cfg_7.5_topk_16384.jpg b/CKPTS/X2I_UltraEdiit_fourmask_LLava_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_filtered_objects_4mask_1280_context_after_stage1_5e4/000-GPT-XL/checkpoints/eval_step_20000/batch_4_cfg_7.5_topk_16384.jpg new file mode 100644 index 0000000000000000000000000000000000000000..b95cfcb3566c74c75b0f66f053a3761e9d8ab992 --- /dev/null +++ b/CKPTS/X2I_UltraEdiit_fourmask_LLava_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_filtered_objects_4mask_1280_context_after_stage1_5e4/000-GPT-XL/checkpoints/eval_step_20000/batch_4_cfg_7.5_topk_16384.jpg @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:f714204192faf1c8a7c4c136c43eaeae0174e7f6574c18704b1f84bfd1d425be +size 393853 diff --git a/CKPTS/X2I_UltraEdiit_fourmask_LLava_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_filtered_objects_4mask_1280_context_after_stage1_5e4/000-GPT-XL/checkpoints/eval_step_20000/batch_5_cfg_7.5_topk_16384.jpg b/CKPTS/X2I_UltraEdiit_fourmask_LLava_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_filtered_objects_4mask_1280_context_after_stage1_5e4/000-GPT-XL/checkpoints/eval_step_20000/batch_5_cfg_7.5_topk_16384.jpg new file mode 100644 index 0000000000000000000000000000000000000000..daacd91cca792e5ab9cee92cd66393aa972a06e7 --- /dev/null +++ b/CKPTS/X2I_UltraEdiit_fourmask_LLava_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_filtered_objects_4mask_1280_context_after_stage1_5e4/000-GPT-XL/checkpoints/eval_step_20000/batch_5_cfg_7.5_topk_16384.jpg @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:800e9fd631b03bd641374b91cee7ee403e7fdc32c3b946ca1befae7c0f5aea05 +size 377044 diff --git a/CKPTS/X2I_UltraEdiit_fourmask_LLava_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_filtered_objects_4mask_1280_context_after_stage1_5e4/000-GPT-XL/checkpoints/eval_step_22000/batch_0_cfg_7.5_topk_16384.jpg b/CKPTS/X2I_UltraEdiit_fourmask_LLava_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_filtered_objects_4mask_1280_context_after_stage1_5e4/000-GPT-XL/checkpoints/eval_step_22000/batch_0_cfg_7.5_topk_16384.jpg new file mode 100644 index 0000000000000000000000000000000000000000..dfe22c94620e34ee59849775084b1cd111a05af7 --- /dev/null +++ b/CKPTS/X2I_UltraEdiit_fourmask_LLava_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_filtered_objects_4mask_1280_context_after_stage1_5e4/000-GPT-XL/checkpoints/eval_step_22000/batch_0_cfg_7.5_topk_16384.jpg @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:deaf2aea7c29969eac6264b0b84fe0b0c9636a20feee5c475129d9d3c7f5e46b +size 382401 diff --git a/CKPTS/X2I_UltraEdiit_fourmask_LLava_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_filtered_objects_4mask_1280_context_after_stage1_5e4/000-GPT-XL/checkpoints/eval_step_22000/batch_1_cfg_7.5_topk_16384.jpg b/CKPTS/X2I_UltraEdiit_fourmask_LLava_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_filtered_objects_4mask_1280_context_after_stage1_5e4/000-GPT-XL/checkpoints/eval_step_22000/batch_1_cfg_7.5_topk_16384.jpg new file mode 100644 index 0000000000000000000000000000000000000000..580bcc73c747baa907f8adf504b82cc0cdd77e50 --- /dev/null +++ b/CKPTS/X2I_UltraEdiit_fourmask_LLava_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_filtered_objects_4mask_1280_context_after_stage1_5e4/000-GPT-XL/checkpoints/eval_step_22000/batch_1_cfg_7.5_topk_16384.jpg @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:8173dfb8de6938efb75236d63af864e39995f4b2d8b7c6f6299b14a09e0c96e2 +size 349149 diff --git a/CKPTS/X2I_UltraEdiit_fourmask_LLava_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_filtered_objects_4mask_1280_context_after_stage1_5e4/000-GPT-XL/checkpoints/eval_step_22000/batch_2_cfg_7.5_topk_16384.jpg b/CKPTS/X2I_UltraEdiit_fourmask_LLava_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_filtered_objects_4mask_1280_context_after_stage1_5e4/000-GPT-XL/checkpoints/eval_step_22000/batch_2_cfg_7.5_topk_16384.jpg new file mode 100644 index 0000000000000000000000000000000000000000..b87de89eedc1786237f794a67239d21a2d4ffca9 --- /dev/null +++ b/CKPTS/X2I_UltraEdiit_fourmask_LLava_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_filtered_objects_4mask_1280_context_after_stage1_5e4/000-GPT-XL/checkpoints/eval_step_22000/batch_2_cfg_7.5_topk_16384.jpg @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:c8610539be4258a6236566d858061bd36de5d460b8a5f78819a547d26a3f4a68 +size 359536 diff --git a/CKPTS/X2I_UltraEdiit_fourmask_LLava_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_filtered_objects_4mask_1280_context_after_stage1_5e4/000-GPT-XL/checkpoints/eval_step_22000/batch_3_cfg_7.5_topk_16384.jpg b/CKPTS/X2I_UltraEdiit_fourmask_LLava_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_filtered_objects_4mask_1280_context_after_stage1_5e4/000-GPT-XL/checkpoints/eval_step_22000/batch_3_cfg_7.5_topk_16384.jpg new file mode 100644 index 0000000000000000000000000000000000000000..a1fbc5097490949f84bc33b00d89a69330eabe55 --- /dev/null +++ b/CKPTS/X2I_UltraEdiit_fourmask_LLava_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_filtered_objects_4mask_1280_context_after_stage1_5e4/000-GPT-XL/checkpoints/eval_step_22000/batch_3_cfg_7.5_topk_16384.jpg @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:9356ef0d2df4334c3344b3a0b3fc72c59a692750e434ac36db3333d238d597bc +size 386427 diff --git a/CKPTS/X2I_UltraEdiit_fourmask_LLava_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_filtered_objects_4mask_1280_context_after_stage1_5e4/000-GPT-XL/checkpoints/eval_step_22000/batch_4_cfg_7.5_topk_16384.jpg b/CKPTS/X2I_UltraEdiit_fourmask_LLava_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_filtered_objects_4mask_1280_context_after_stage1_5e4/000-GPT-XL/checkpoints/eval_step_22000/batch_4_cfg_7.5_topk_16384.jpg new file mode 100644 index 0000000000000000000000000000000000000000..0e413b4dc2ff1d0ec87dd81649d42893be086ac4 --- /dev/null +++ b/CKPTS/X2I_UltraEdiit_fourmask_LLava_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_filtered_objects_4mask_1280_context_after_stage1_5e4/000-GPT-XL/checkpoints/eval_step_22000/batch_4_cfg_7.5_topk_16384.jpg @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:27cc11ba77fd4b1523d91b3ca71c5376c9e986fa2c7724693d5289d43b39857e +size 395768 diff --git a/CKPTS/X2I_UltraEdiit_fourmask_LLava_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_filtered_objects_4mask_1280_context_after_stage1_5e4/000-GPT-XL/checkpoints/eval_step_22000/batch_5_cfg_7.5_topk_16384.jpg b/CKPTS/X2I_UltraEdiit_fourmask_LLava_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_filtered_objects_4mask_1280_context_after_stage1_5e4/000-GPT-XL/checkpoints/eval_step_22000/batch_5_cfg_7.5_topk_16384.jpg new file mode 100644 index 0000000000000000000000000000000000000000..ab82144897e0300f5b46634ff99247fda9d75b5d --- /dev/null +++ b/CKPTS/X2I_UltraEdiit_fourmask_LLava_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_filtered_objects_4mask_1280_context_after_stage1_5e4/000-GPT-XL/checkpoints/eval_step_22000/batch_5_cfg_7.5_topk_16384.jpg @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:5f3917985626e8bd7adb309d2a83ffbefd6b86d0375ed6505ccad3e7653c6cb9 +size 383528 diff --git a/CKPTS/X2I_UltraEdiit_fourmask_LLava_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_filtered_objects_4mask_1280_context_after_stage1_5e4/000-GPT-XL/checkpoints/eval_step_24000/batch_0_cfg_7.5_topk_16384.jpg b/CKPTS/X2I_UltraEdiit_fourmask_LLava_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_filtered_objects_4mask_1280_context_after_stage1_5e4/000-GPT-XL/checkpoints/eval_step_24000/batch_0_cfg_7.5_topk_16384.jpg new file mode 100644 index 0000000000000000000000000000000000000000..615126b06844f78b2032d5cc11910205505ba6e9 --- /dev/null +++ b/CKPTS/X2I_UltraEdiit_fourmask_LLava_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_filtered_objects_4mask_1280_context_after_stage1_5e4/000-GPT-XL/checkpoints/eval_step_24000/batch_0_cfg_7.5_topk_16384.jpg @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:2f310c93635dcf96a18e30c312ed3f559354ea0612d83882b07fe1bf201b833a +size 387254 diff --git a/CKPTS/X2I_UltraEdiit_fourmask_LLava_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_filtered_objects_4mask_1280_context_after_stage1_5e4/000-GPT-XL/checkpoints/eval_step_24000/batch_1_cfg_7.5_topk_16384.jpg b/CKPTS/X2I_UltraEdiit_fourmask_LLava_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_filtered_objects_4mask_1280_context_after_stage1_5e4/000-GPT-XL/checkpoints/eval_step_24000/batch_1_cfg_7.5_topk_16384.jpg new file mode 100644 index 0000000000000000000000000000000000000000..56f97e4c40209a16ca9d4d01c78c4032951e1aa5 --- /dev/null +++ b/CKPTS/X2I_UltraEdiit_fourmask_LLava_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_filtered_objects_4mask_1280_context_after_stage1_5e4/000-GPT-XL/checkpoints/eval_step_24000/batch_1_cfg_7.5_topk_16384.jpg @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:a62c580a39194cccbac3d39b8ad57b6967ff4bfd6c4b1766a3fc9c42e31c81ca +size 366867 diff --git a/CKPTS/X2I_UltraEdiit_fourmask_LLava_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_filtered_objects_4mask_1280_context_after_stage1_5e4/000-GPT-XL/checkpoints/eval_step_24000/batch_2_cfg_7.5_topk_16384.jpg b/CKPTS/X2I_UltraEdiit_fourmask_LLava_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_filtered_objects_4mask_1280_context_after_stage1_5e4/000-GPT-XL/checkpoints/eval_step_24000/batch_2_cfg_7.5_topk_16384.jpg new file mode 100644 index 0000000000000000000000000000000000000000..5e14369e1a3aa29038b4528e7d263e2e504b0a1e --- /dev/null +++ b/CKPTS/X2I_UltraEdiit_fourmask_LLava_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_filtered_objects_4mask_1280_context_after_stage1_5e4/000-GPT-XL/checkpoints/eval_step_24000/batch_2_cfg_7.5_topk_16384.jpg @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:72a01ed4c8ae39b70e39c8cb90857f7bcd221cb6468d91caafc04c1d8eef3470 +size 348578 diff --git a/CKPTS/X2I_UltraEdiit_fourmask_LLava_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_filtered_objects_4mask_1280_context_after_stage1_5e4/000-GPT-XL/checkpoints/eval_step_24000/batch_3_cfg_7.5_topk_16384.jpg b/CKPTS/X2I_UltraEdiit_fourmask_LLava_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_filtered_objects_4mask_1280_context_after_stage1_5e4/000-GPT-XL/checkpoints/eval_step_24000/batch_3_cfg_7.5_topk_16384.jpg new file mode 100644 index 0000000000000000000000000000000000000000..c1f4d6e5cb0f954677f7245d9aff25ebb1e7864a --- /dev/null +++ b/CKPTS/X2I_UltraEdiit_fourmask_LLava_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_filtered_objects_4mask_1280_context_after_stage1_5e4/000-GPT-XL/checkpoints/eval_step_24000/batch_3_cfg_7.5_topk_16384.jpg @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:3514bf82fcfe50a8d1f7da7c2a6d3d1aa092e7ced0c51a2a6b8365e2d2a8e650 +size 381464 diff --git a/CKPTS/X2I_UltraEdiit_fourmask_LLava_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_filtered_objects_4mask_1280_context_after_stage1_5e4/000-GPT-XL/checkpoints/eval_step_24000/batch_4_cfg_7.5_topk_16384.jpg b/CKPTS/X2I_UltraEdiit_fourmask_LLava_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_filtered_objects_4mask_1280_context_after_stage1_5e4/000-GPT-XL/checkpoints/eval_step_24000/batch_4_cfg_7.5_topk_16384.jpg new file mode 100644 index 0000000000000000000000000000000000000000..d1d9ea84365b373bfdc2ca8b36cdc76ee383dacc --- /dev/null +++ b/CKPTS/X2I_UltraEdiit_fourmask_LLava_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_filtered_objects_4mask_1280_context_after_stage1_5e4/000-GPT-XL/checkpoints/eval_step_24000/batch_4_cfg_7.5_topk_16384.jpg @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:fb3590130ddb0fcc43fbda9f49599971300a5ee4c65eaa60d0ff105652283b4a +size 398346 diff --git a/CKPTS/X2I_UltraEdiit_fourmask_LLava_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_filtered_objects_4mask_1280_context_after_stage1_5e4/000-GPT-XL/checkpoints/eval_step_24000/batch_5_cfg_7.5_topk_16384.jpg b/CKPTS/X2I_UltraEdiit_fourmask_LLava_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_filtered_objects_4mask_1280_context_after_stage1_5e4/000-GPT-XL/checkpoints/eval_step_24000/batch_5_cfg_7.5_topk_16384.jpg new file mode 100644 index 0000000000000000000000000000000000000000..f548a569f0f560105ce3b7629ae312cc15061b04 --- /dev/null +++ b/CKPTS/X2I_UltraEdiit_fourmask_LLava_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_filtered_objects_4mask_1280_context_after_stage1_5e4/000-GPT-XL/checkpoints/eval_step_24000/batch_5_cfg_7.5_topk_16384.jpg @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:d0a164ca6c76297a917056f47b362d16e6f38ca382571c08d188809302d0f248 +size 368739 diff --git a/CKPTS/X2I_UltraEdiit_fourmask_LLava_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_filtered_objects_4mask_1280_context_after_stage1_5e4/000-GPT-XL/checkpoints/eval_step_26000/batch_0_cfg_7.5_topk_16384.jpg b/CKPTS/X2I_UltraEdiit_fourmask_LLava_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_filtered_objects_4mask_1280_context_after_stage1_5e4/000-GPT-XL/checkpoints/eval_step_26000/batch_0_cfg_7.5_topk_16384.jpg new file mode 100644 index 0000000000000000000000000000000000000000..4f34d7457ecaf363e38ac5447c565725818c3688 --- /dev/null +++ b/CKPTS/X2I_UltraEdiit_fourmask_LLava_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_filtered_objects_4mask_1280_context_after_stage1_5e4/000-GPT-XL/checkpoints/eval_step_26000/batch_0_cfg_7.5_topk_16384.jpg @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:e304fb5d4b6d58ae9f7e403825476a41606bcbd3ced1da744975b1704f4dab1c +size 385628 diff --git a/CKPTS/X2I_UltraEdiit_fourmask_LLava_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_filtered_objects_4mask_1280_context_after_stage1_5e4/000-GPT-XL/checkpoints/eval_step_26000/batch_1_cfg_7.5_topk_16384.jpg b/CKPTS/X2I_UltraEdiit_fourmask_LLava_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_filtered_objects_4mask_1280_context_after_stage1_5e4/000-GPT-XL/checkpoints/eval_step_26000/batch_1_cfg_7.5_topk_16384.jpg new file mode 100644 index 0000000000000000000000000000000000000000..9dda2a69471cf8d3cf11a79a4ec8f7c34e23dd42 --- /dev/null +++ b/CKPTS/X2I_UltraEdiit_fourmask_LLava_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_filtered_objects_4mask_1280_context_after_stage1_5e4/000-GPT-XL/checkpoints/eval_step_26000/batch_1_cfg_7.5_topk_16384.jpg @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:e069de58e8a965702fffa8c45617bf13ffac4555a72444c4e4d02a94d196628a +size 340392 diff --git a/CKPTS/X2I_UltraEdiit_fourmask_LLava_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_filtered_objects_4mask_1280_context_after_stage1_5e4/000-GPT-XL/checkpoints/eval_step_26000/batch_2_cfg_7.5_topk_16384.jpg b/CKPTS/X2I_UltraEdiit_fourmask_LLava_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_filtered_objects_4mask_1280_context_after_stage1_5e4/000-GPT-XL/checkpoints/eval_step_26000/batch_2_cfg_7.5_topk_16384.jpg new file mode 100644 index 0000000000000000000000000000000000000000..5dd6ba65c783e98a79227bb7318133d96a85f5d1 --- /dev/null +++ b/CKPTS/X2I_UltraEdiit_fourmask_LLava_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_filtered_objects_4mask_1280_context_after_stage1_5e4/000-GPT-XL/checkpoints/eval_step_26000/batch_2_cfg_7.5_topk_16384.jpg @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:826dc413e1d542ab0413de6b7093e9f8c4e0174b05f3b77b759ddf864fa03a62 +size 357268 diff --git a/CKPTS/X2I_UltraEdiit_fourmask_LLava_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_filtered_objects_4mask_1280_context_after_stage1_5e4/000-GPT-XL/checkpoints/eval_step_26000/batch_3_cfg_7.5_topk_16384.jpg b/CKPTS/X2I_UltraEdiit_fourmask_LLava_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_filtered_objects_4mask_1280_context_after_stage1_5e4/000-GPT-XL/checkpoints/eval_step_26000/batch_3_cfg_7.5_topk_16384.jpg new file mode 100644 index 0000000000000000000000000000000000000000..7a9050b2e2cc6197102ac985c19ede1aaa2ff300 --- /dev/null +++ b/CKPTS/X2I_UltraEdiit_fourmask_LLava_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_filtered_objects_4mask_1280_context_after_stage1_5e4/000-GPT-XL/checkpoints/eval_step_26000/batch_3_cfg_7.5_topk_16384.jpg @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:250b6454e81c0277d5e703672e68a820624ab9fdb921e7b537d81f49469ff3f7 +size 389830 diff --git a/CKPTS/X2I_UltraEdiit_fourmask_LLava_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_filtered_objects_4mask_1280_context_after_stage1_5e4/000-GPT-XL/checkpoints/eval_step_26000/batch_4_cfg_7.5_topk_16384.jpg b/CKPTS/X2I_UltraEdiit_fourmask_LLava_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_filtered_objects_4mask_1280_context_after_stage1_5e4/000-GPT-XL/checkpoints/eval_step_26000/batch_4_cfg_7.5_topk_16384.jpg new file mode 100644 index 0000000000000000000000000000000000000000..275bf0a7275c4de31dd65284d525162a540840ec --- /dev/null +++ b/CKPTS/X2I_UltraEdiit_fourmask_LLava_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_filtered_objects_4mask_1280_context_after_stage1_5e4/000-GPT-XL/checkpoints/eval_step_26000/batch_4_cfg_7.5_topk_16384.jpg @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:9648a5b752fff5bb6487138d269576c5305978b73c9cfa8f5c2544e57b779b75 +size 400461 diff --git a/CKPTS/X2I_UltraEdiit_fourmask_LLava_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_filtered_objects_4mask_1280_context_after_stage1_5e4/000-GPT-XL/checkpoints/eval_step_26000/batch_5_cfg_7.5_topk_16384.jpg b/CKPTS/X2I_UltraEdiit_fourmask_LLava_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_filtered_objects_4mask_1280_context_after_stage1_5e4/000-GPT-XL/checkpoints/eval_step_26000/batch_5_cfg_7.5_topk_16384.jpg new file mode 100644 index 0000000000000000000000000000000000000000..4749ada9c029b5897b76e34dff34e34b4d3f3e51 --- /dev/null +++ b/CKPTS/X2I_UltraEdiit_fourmask_LLava_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_filtered_objects_4mask_1280_context_after_stage1_5e4/000-GPT-XL/checkpoints/eval_step_26000/batch_5_cfg_7.5_topk_16384.jpg @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:963bb6f9e4f4a95b8ec65a566fc964c2a9aa42c854f22c56446771496f3ecc98 +size 374849 diff --git a/CKPTS/X2I_UltraEdiit_fourmask_LLava_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_filtered_objects_4mask_1280_context_after_stage1_5e4/000-GPT-XL/checkpoints/eval_step_28000/batch_0_cfg_7.5_topk_16384.jpg b/CKPTS/X2I_UltraEdiit_fourmask_LLava_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_filtered_objects_4mask_1280_context_after_stage1_5e4/000-GPT-XL/checkpoints/eval_step_28000/batch_0_cfg_7.5_topk_16384.jpg new file mode 100644 index 0000000000000000000000000000000000000000..7d19e4a680973185ab2b5258f8bb99f807fbbcfd --- /dev/null +++ b/CKPTS/X2I_UltraEdiit_fourmask_LLava_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_filtered_objects_4mask_1280_context_after_stage1_5e4/000-GPT-XL/checkpoints/eval_step_28000/batch_0_cfg_7.5_topk_16384.jpg @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:791170add9d261f51b7e421427bb710a649ae37777874d730cf66fa28960ef0a +size 392011 diff --git a/CKPTS/X2I_UltraEdiit_fourmask_LLava_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_filtered_objects_4mask_1280_context_after_stage1_5e4/000-GPT-XL/checkpoints/eval_step_28000/batch_1_cfg_7.5_topk_16384.jpg b/CKPTS/X2I_UltraEdiit_fourmask_LLava_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_filtered_objects_4mask_1280_context_after_stage1_5e4/000-GPT-XL/checkpoints/eval_step_28000/batch_1_cfg_7.5_topk_16384.jpg new file mode 100644 index 0000000000000000000000000000000000000000..3213f6b4d33a7d85aec7223acf1f9587362be5bc --- /dev/null +++ b/CKPTS/X2I_UltraEdiit_fourmask_LLava_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_filtered_objects_4mask_1280_context_after_stage1_5e4/000-GPT-XL/checkpoints/eval_step_28000/batch_1_cfg_7.5_topk_16384.jpg @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:3a34a11f9aa88d0f6f6798591ef6938f85ac5ae5d7f19963549c08aaee99b92a +size 329172 diff --git a/CKPTS/X2I_UltraEdiit_fourmask_LLava_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_filtered_objects_4mask_1280_context_after_stage1_5e4/000-GPT-XL/checkpoints/eval_step_28000/batch_2_cfg_7.5_topk_16384.jpg b/CKPTS/X2I_UltraEdiit_fourmask_LLava_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_filtered_objects_4mask_1280_context_after_stage1_5e4/000-GPT-XL/checkpoints/eval_step_28000/batch_2_cfg_7.5_topk_16384.jpg new file mode 100644 index 0000000000000000000000000000000000000000..5e248816edb8eec227b422abf7a3712a91c9794b --- /dev/null +++ b/CKPTS/X2I_UltraEdiit_fourmask_LLava_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_filtered_objects_4mask_1280_context_after_stage1_5e4/000-GPT-XL/checkpoints/eval_step_28000/batch_2_cfg_7.5_topk_16384.jpg @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:c9ed512542709efaabbbb16b49d6cb660186609dd3a28e19d55b63c18ea52076 +size 363073 diff --git a/CKPTS/X2I_UltraEdiit_fourmask_LLava_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_filtered_objects_4mask_1280_context_after_stage1_5e4/000-GPT-XL/checkpoints/eval_step_28000/batch_3_cfg_7.5_topk_16384.jpg b/CKPTS/X2I_UltraEdiit_fourmask_LLava_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_filtered_objects_4mask_1280_context_after_stage1_5e4/000-GPT-XL/checkpoints/eval_step_28000/batch_3_cfg_7.5_topk_16384.jpg new file mode 100644 index 0000000000000000000000000000000000000000..e283c357c94d55eec6726257e51ee3219aa621c9 --- /dev/null +++ b/CKPTS/X2I_UltraEdiit_fourmask_LLava_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_filtered_objects_4mask_1280_context_after_stage1_5e4/000-GPT-XL/checkpoints/eval_step_28000/batch_3_cfg_7.5_topk_16384.jpg @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:b9fa7e3675bd1516a9d2c4a502b8557ba10fbd91fd0ba9aa5252267bbf8d52ab +size 393399 diff --git a/CKPTS/X2I_UltraEdiit_fourmask_LLava_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_filtered_objects_4mask_1280_context_after_stage1_5e4/000-GPT-XL/checkpoints/eval_step_28000/batch_4_cfg_7.5_topk_16384.jpg b/CKPTS/X2I_UltraEdiit_fourmask_LLava_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_filtered_objects_4mask_1280_context_after_stage1_5e4/000-GPT-XL/checkpoints/eval_step_28000/batch_4_cfg_7.5_topk_16384.jpg new file mode 100644 index 0000000000000000000000000000000000000000..3ff6959ff4d7b00bb4dfaeb479c791a38cd27463 --- /dev/null +++ b/CKPTS/X2I_UltraEdiit_fourmask_LLava_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_filtered_objects_4mask_1280_context_after_stage1_5e4/000-GPT-XL/checkpoints/eval_step_28000/batch_4_cfg_7.5_topk_16384.jpg @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:93ffb3d8770f43e6f9915810c944bc3cbb216c28237b51721a3c90e34f9b32d7 +size 404720 diff --git a/CKPTS/X2I_UltraEdiit_fourmask_LLava_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_filtered_objects_4mask_1280_context_after_stage1_5e4/000-GPT-XL/checkpoints/eval_step_28000/batch_5_cfg_7.5_topk_16384.jpg b/CKPTS/X2I_UltraEdiit_fourmask_LLava_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_filtered_objects_4mask_1280_context_after_stage1_5e4/000-GPT-XL/checkpoints/eval_step_28000/batch_5_cfg_7.5_topk_16384.jpg new file mode 100644 index 0000000000000000000000000000000000000000..21750a9dd76bea30192dc9dd1e2a18a31ac1dddc --- /dev/null +++ b/CKPTS/X2I_UltraEdiit_fourmask_LLava_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_filtered_objects_4mask_1280_context_after_stage1_5e4/000-GPT-XL/checkpoints/eval_step_28000/batch_5_cfg_7.5_topk_16384.jpg @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:9ee30ffa84ed129019cfe7b69dd95c4e84e2326f313ee345f177c4ee9f25cad1 +size 370597 diff --git a/CKPTS/X2I_UltraEdiit_fourmask_LLava_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_filtered_objects_4mask_1280_context_after_stage1_5e4/000-GPT-XL/checkpoints/eval_step_30000/batch_0_cfg_7.5_topk_16384.jpg b/CKPTS/X2I_UltraEdiit_fourmask_LLava_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_filtered_objects_4mask_1280_context_after_stage1_5e4/000-GPT-XL/checkpoints/eval_step_30000/batch_0_cfg_7.5_topk_16384.jpg new file mode 100644 index 0000000000000000000000000000000000000000..19165f1b5b2f5c811c6aaee5d7b58f726f4b55d1 --- /dev/null +++ b/CKPTS/X2I_UltraEdiit_fourmask_LLava_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_filtered_objects_4mask_1280_context_after_stage1_5e4/000-GPT-XL/checkpoints/eval_step_30000/batch_0_cfg_7.5_topk_16384.jpg @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:9d8b6bea61db152c0010076ab81279e45276b2d417eb3cce3bf162ccc2a19d10 +size 395554 diff --git a/CKPTS/X2I_UltraEdiit_fourmask_LLava_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_filtered_objects_4mask_1280_context_after_stage1_5e4/000-GPT-XL/checkpoints/eval_step_30000/batch_1_cfg_7.5_topk_16384.jpg b/CKPTS/X2I_UltraEdiit_fourmask_LLava_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_filtered_objects_4mask_1280_context_after_stage1_5e4/000-GPT-XL/checkpoints/eval_step_30000/batch_1_cfg_7.5_topk_16384.jpg new file mode 100644 index 0000000000000000000000000000000000000000..ceb41ce122462a5181de27ea87692342d86b4660 --- /dev/null +++ b/CKPTS/X2I_UltraEdiit_fourmask_LLava_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_filtered_objects_4mask_1280_context_after_stage1_5e4/000-GPT-XL/checkpoints/eval_step_30000/batch_1_cfg_7.5_topk_16384.jpg @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:cc1a8ae883fe15d943cacf956481297d60fc08c39f9f85a46db8f5119b0fdc04 +size 326810 diff --git a/CKPTS/X2I_UltraEdiit_fourmask_LLava_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_filtered_objects_4mask_1280_context_after_stage1_5e4/000-GPT-XL/checkpoints/eval_step_30000/batch_2_cfg_7.5_topk_16384.jpg b/CKPTS/X2I_UltraEdiit_fourmask_LLava_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_filtered_objects_4mask_1280_context_after_stage1_5e4/000-GPT-XL/checkpoints/eval_step_30000/batch_2_cfg_7.5_topk_16384.jpg new file mode 100644 index 0000000000000000000000000000000000000000..4c0cb6c538b37d88f60b2a14f8424dd5c56120ea --- /dev/null +++ b/CKPTS/X2I_UltraEdiit_fourmask_LLava_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_filtered_objects_4mask_1280_context_after_stage1_5e4/000-GPT-XL/checkpoints/eval_step_30000/batch_2_cfg_7.5_topk_16384.jpg @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:f8f16eb430b080144f4d3fe2a51c5d11558f510b77e15e627e58852c6ea803f7 +size 358209 diff --git a/CKPTS/X2I_UltraEdiit_fourmask_LLava_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_filtered_objects_4mask_1280_context_after_stage1_5e4/000-GPT-XL/checkpoints/eval_step_30000/batch_3_cfg_7.5_topk_16384.jpg b/CKPTS/X2I_UltraEdiit_fourmask_LLava_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_filtered_objects_4mask_1280_context_after_stage1_5e4/000-GPT-XL/checkpoints/eval_step_30000/batch_3_cfg_7.5_topk_16384.jpg new file mode 100644 index 0000000000000000000000000000000000000000..6370f9cbb946df758e18e50047a906acc4280934 --- /dev/null +++ b/CKPTS/X2I_UltraEdiit_fourmask_LLava_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_filtered_objects_4mask_1280_context_after_stage1_5e4/000-GPT-XL/checkpoints/eval_step_30000/batch_3_cfg_7.5_topk_16384.jpg @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:5d90e5f46e7565e07b0393e47d30bd8ad143dba672084b78ba2947a8d9d189ea +size 386204 diff --git a/CKPTS/X2I_UltraEdiit_fourmask_LLava_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_filtered_objects_4mask_1280_context_after_stage1_5e4/000-GPT-XL/checkpoints/eval_step_30000/batch_4_cfg_7.5_topk_16384.jpg b/CKPTS/X2I_UltraEdiit_fourmask_LLava_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_filtered_objects_4mask_1280_context_after_stage1_5e4/000-GPT-XL/checkpoints/eval_step_30000/batch_4_cfg_7.5_topk_16384.jpg new file mode 100644 index 0000000000000000000000000000000000000000..2c8ab6e2471106dfc72de16d0ba73ec082c03095 --- /dev/null +++ b/CKPTS/X2I_UltraEdiit_fourmask_LLava_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_filtered_objects_4mask_1280_context_after_stage1_5e4/000-GPT-XL/checkpoints/eval_step_30000/batch_4_cfg_7.5_topk_16384.jpg @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:b9afb8a3c88d031934a2a6802fe9b4a1a3d08e0aa57c00e1c8dde407acd6505e +size 405871 diff --git a/CKPTS/X2I_UltraEdiit_fourmask_LLava_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_filtered_objects_4mask_1280_context_after_stage1_5e4/000-GPT-XL/checkpoints/eval_step_30000/batch_5_cfg_7.5_topk_16384.jpg b/CKPTS/X2I_UltraEdiit_fourmask_LLava_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_filtered_objects_4mask_1280_context_after_stage1_5e4/000-GPT-XL/checkpoints/eval_step_30000/batch_5_cfg_7.5_topk_16384.jpg new file mode 100644 index 0000000000000000000000000000000000000000..975194536746c922014a4922cd8dc02fc5bab5a2 --- /dev/null +++ b/CKPTS/X2I_UltraEdiit_fourmask_LLava_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_filtered_objects_4mask_1280_context_after_stage1_5e4/000-GPT-XL/checkpoints/eval_step_30000/batch_5_cfg_7.5_topk_16384.jpg @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:dbb3c7a45306f79626bf0eda2a31fceb7632d2f48a073cc45d1e8b482b54d6a4 +size 374385 diff --git a/CKPTS/X2I_UltraEdiit_fourmask_LLava_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_filtered_objects_4mask_1280_context_after_stage1_5e4/000-GPT-XL/checkpoints/eval_step_32000/batch_0_cfg_7.5_topk_16384.jpg b/CKPTS/X2I_UltraEdiit_fourmask_LLava_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_filtered_objects_4mask_1280_context_after_stage1_5e4/000-GPT-XL/checkpoints/eval_step_32000/batch_0_cfg_7.5_topk_16384.jpg new file mode 100644 index 0000000000000000000000000000000000000000..d450797e868526a6578383f6c8794c067d3ce09d --- /dev/null +++ b/CKPTS/X2I_UltraEdiit_fourmask_LLava_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_filtered_objects_4mask_1280_context_after_stage1_5e4/000-GPT-XL/checkpoints/eval_step_32000/batch_0_cfg_7.5_topk_16384.jpg @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:c8630e6f10ed4b1dae05440f9132f79195b1464e80a612c0629abe4d5b1cddc3 +size 390567 diff --git a/CKPTS/X2I_UltraEdiit_fourmask_LLava_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_filtered_objects_4mask_1280_context_after_stage1_5e4/000-GPT-XL/checkpoints/eval_step_32000/batch_1_cfg_7.5_topk_16384.jpg b/CKPTS/X2I_UltraEdiit_fourmask_LLava_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_filtered_objects_4mask_1280_context_after_stage1_5e4/000-GPT-XL/checkpoints/eval_step_32000/batch_1_cfg_7.5_topk_16384.jpg new file mode 100644 index 0000000000000000000000000000000000000000..117e9e4e149b56409effc906618f5c931e2775f4 --- /dev/null +++ b/CKPTS/X2I_UltraEdiit_fourmask_LLava_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_filtered_objects_4mask_1280_context_after_stage1_5e4/000-GPT-XL/checkpoints/eval_step_32000/batch_1_cfg_7.5_topk_16384.jpg @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:187052ec0e2eb505b48228f96359a3df9139c5fd3779aedf6aa0171d37bdd20d +size 337882 diff --git a/CKPTS/X2I_UltraEdiit_fourmask_LLava_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_filtered_objects_4mask_1280_context_after_stage1_5e4/000-GPT-XL/checkpoints/eval_step_32000/batch_2_cfg_7.5_topk_16384.jpg b/CKPTS/X2I_UltraEdiit_fourmask_LLava_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_filtered_objects_4mask_1280_context_after_stage1_5e4/000-GPT-XL/checkpoints/eval_step_32000/batch_2_cfg_7.5_topk_16384.jpg new file mode 100644 index 0000000000000000000000000000000000000000..951b50d03bf03d5800fcdbe9f8ed9c39b1e6969b --- /dev/null +++ b/CKPTS/X2I_UltraEdiit_fourmask_LLava_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_filtered_objects_4mask_1280_context_after_stage1_5e4/000-GPT-XL/checkpoints/eval_step_32000/batch_2_cfg_7.5_topk_16384.jpg @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:7f9c3330f0b83f9df345dcce2195b76ed21e38fb20b1a53c8090f54962add305 +size 353625 diff --git a/CKPTS/X2I_UltraEdiit_fourmask_LLava_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_filtered_objects_4mask_1280_context_after_stage1_5e4/000-GPT-XL/checkpoints/eval_step_32000/batch_3_cfg_7.5_topk_16384.jpg b/CKPTS/X2I_UltraEdiit_fourmask_LLava_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_filtered_objects_4mask_1280_context_after_stage1_5e4/000-GPT-XL/checkpoints/eval_step_32000/batch_3_cfg_7.5_topk_16384.jpg new file mode 100644 index 0000000000000000000000000000000000000000..61e791df293dd0eb1daf37dd0fafb65f11c4323f --- /dev/null +++ b/CKPTS/X2I_UltraEdiit_fourmask_LLava_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_filtered_objects_4mask_1280_context_after_stage1_5e4/000-GPT-XL/checkpoints/eval_step_32000/batch_3_cfg_7.5_topk_16384.jpg @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:87b866c71b88665ce3734923e5098483dd90e3bfb1212f535a382414b4916076 +size 385406 diff --git a/CKPTS/X2I_UltraEdiit_fourmask_LLava_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_filtered_objects_4mask_1280_context_after_stage1_5e4/000-GPT-XL/checkpoints/eval_step_32000/batch_4_cfg_7.5_topk_16384.jpg b/CKPTS/X2I_UltraEdiit_fourmask_LLava_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_filtered_objects_4mask_1280_context_after_stage1_5e4/000-GPT-XL/checkpoints/eval_step_32000/batch_4_cfg_7.5_topk_16384.jpg new file mode 100644 index 0000000000000000000000000000000000000000..6b24815831cb3abcf3674c31f4a67f82dc819254 --- /dev/null +++ b/CKPTS/X2I_UltraEdiit_fourmask_LLava_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_filtered_objects_4mask_1280_context_after_stage1_5e4/000-GPT-XL/checkpoints/eval_step_32000/batch_4_cfg_7.5_topk_16384.jpg @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:e3f5140401938c1ca54c3ba30403b033f6544297d94c6075aae83fd828ff7b30 +size 401525 diff --git a/CKPTS/X2I_UltraEdiit_fourmask_LLava_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_filtered_objects_4mask_1280_context_after_stage1_5e4/000-GPT-XL/checkpoints/eval_step_32000/batch_5_cfg_7.5_topk_16384.jpg b/CKPTS/X2I_UltraEdiit_fourmask_LLava_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_filtered_objects_4mask_1280_context_after_stage1_5e4/000-GPT-XL/checkpoints/eval_step_32000/batch_5_cfg_7.5_topk_16384.jpg new file mode 100644 index 0000000000000000000000000000000000000000..59b0b0a0793d981727c6dc13ad89c05f4122dcb6 --- /dev/null +++ b/CKPTS/X2I_UltraEdiit_fourmask_LLava_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_filtered_objects_4mask_1280_context_after_stage1_5e4/000-GPT-XL/checkpoints/eval_step_32000/batch_5_cfg_7.5_topk_16384.jpg @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:a0017b8392185f50b6627723eb92a4bafffce3aafd5b6a8fe52b79f59cdfe774 +size 376309 diff --git a/CKPTS/X2I_UltraEdiit_fourmask_LLava_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_filtered_objects_4mask_1280_context_after_stage1_5e4/000-GPT-XL/checkpoints/eval_step_34000/batch_0_cfg_7.5_topk_16384.jpg b/CKPTS/X2I_UltraEdiit_fourmask_LLava_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_filtered_objects_4mask_1280_context_after_stage1_5e4/000-GPT-XL/checkpoints/eval_step_34000/batch_0_cfg_7.5_topk_16384.jpg new file mode 100644 index 0000000000000000000000000000000000000000..0f740f8efb80773cf59bfa5e531c67973f6ea784 --- /dev/null +++ b/CKPTS/X2I_UltraEdiit_fourmask_LLava_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_filtered_objects_4mask_1280_context_after_stage1_5e4/000-GPT-XL/checkpoints/eval_step_34000/batch_0_cfg_7.5_topk_16384.jpg @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:3156f72ddb1d81936bcd7b398ae9ea2f331736ef46c7b8f58af345224178e12b +size 391695 diff --git a/CKPTS/X2I_UltraEdiit_fourmask_LLava_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_filtered_objects_4mask_1280_context_after_stage1_5e4/000-GPT-XL/checkpoints/eval_step_34000/batch_1_cfg_7.5_topk_16384.jpg b/CKPTS/X2I_UltraEdiit_fourmask_LLava_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_filtered_objects_4mask_1280_context_after_stage1_5e4/000-GPT-XL/checkpoints/eval_step_34000/batch_1_cfg_7.5_topk_16384.jpg new file mode 100644 index 0000000000000000000000000000000000000000..23ec0eabe50fa1d8c47cc566fe858eeea074a5ec --- /dev/null +++ b/CKPTS/X2I_UltraEdiit_fourmask_LLava_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_filtered_objects_4mask_1280_context_after_stage1_5e4/000-GPT-XL/checkpoints/eval_step_34000/batch_1_cfg_7.5_topk_16384.jpg @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:cca9b76b03533af08875ffc81e5f429a19a3f4356fcaeed674bfee290379d4d4 +size 338885 diff --git a/CKPTS/X2I_UltraEdiit_fourmask_LLava_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_filtered_objects_4mask_1280_context_after_stage1_5e4/000-GPT-XL/checkpoints/eval_step_34000/batch_2_cfg_7.5_topk_16384.jpg b/CKPTS/X2I_UltraEdiit_fourmask_LLava_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_filtered_objects_4mask_1280_context_after_stage1_5e4/000-GPT-XL/checkpoints/eval_step_34000/batch_2_cfg_7.5_topk_16384.jpg new file mode 100644 index 0000000000000000000000000000000000000000..9ebc29dbb8f08851d9e90f941d17bcf545cef5ae --- /dev/null +++ b/CKPTS/X2I_UltraEdiit_fourmask_LLava_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_filtered_objects_4mask_1280_context_after_stage1_5e4/000-GPT-XL/checkpoints/eval_step_34000/batch_2_cfg_7.5_topk_16384.jpg @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:c9b413e831966922e2495ff91a32830eafb5b7de1f2b302cf89fe9a723a194f1 +size 353794 diff --git a/CKPTS/X2I_UltraEdiit_fourmask_LLava_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_filtered_objects_4mask_1280_context_after_stage1_5e4/000-GPT-XL/checkpoints/eval_step_34000/batch_3_cfg_7.5_topk_16384.jpg b/CKPTS/X2I_UltraEdiit_fourmask_LLava_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_filtered_objects_4mask_1280_context_after_stage1_5e4/000-GPT-XL/checkpoints/eval_step_34000/batch_3_cfg_7.5_topk_16384.jpg new file mode 100644 index 0000000000000000000000000000000000000000..530122fed997a6f03f9a7b63395412e9cc25520f --- /dev/null +++ b/CKPTS/X2I_UltraEdiit_fourmask_LLava_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_filtered_objects_4mask_1280_context_after_stage1_5e4/000-GPT-XL/checkpoints/eval_step_34000/batch_3_cfg_7.5_topk_16384.jpg @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:8246e267dd5dd4c5301f3e34340b369d3f6b7e23e53ce47f74123e3ff90cee7c +size 389429 diff --git a/CKPTS/X2I_UltraEdiit_fourmask_LLava_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_filtered_objects_4mask_1280_context_after_stage1_5e4/000-GPT-XL/checkpoints/eval_step_34000/batch_4_cfg_7.5_topk_16384.jpg b/CKPTS/X2I_UltraEdiit_fourmask_LLava_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_filtered_objects_4mask_1280_context_after_stage1_5e4/000-GPT-XL/checkpoints/eval_step_34000/batch_4_cfg_7.5_topk_16384.jpg new file mode 100644 index 0000000000000000000000000000000000000000..d1bc551fbca6f170c9deafd94d2e09908e7c8887 --- /dev/null +++ b/CKPTS/X2I_UltraEdiit_fourmask_LLava_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_filtered_objects_4mask_1280_context_after_stage1_5e4/000-GPT-XL/checkpoints/eval_step_34000/batch_4_cfg_7.5_topk_16384.jpg @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:a24e282b1ddcd097a62e3dc17ad627103c0e4d91952835bd5e159e62f988caef +size 386887 diff --git a/CKPTS/X2I_UltraEdiit_fourmask_LLava_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_filtered_objects_4mask_1280_context_after_stage1_5e4/000-GPT-XL/checkpoints/eval_step_34000/batch_5_cfg_7.5_topk_16384.jpg b/CKPTS/X2I_UltraEdiit_fourmask_LLava_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_filtered_objects_4mask_1280_context_after_stage1_5e4/000-GPT-XL/checkpoints/eval_step_34000/batch_5_cfg_7.5_topk_16384.jpg new file mode 100644 index 0000000000000000000000000000000000000000..3f0c2404ef4698fbf1c40c26e0d4c998ba33eae2 --- /dev/null +++ b/CKPTS/X2I_UltraEdiit_fourmask_LLava_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_filtered_objects_4mask_1280_context_after_stage1_5e4/000-GPT-XL/checkpoints/eval_step_34000/batch_5_cfg_7.5_topk_16384.jpg @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:ba73649f7c3e450d371a1736a54cd4495e4f07cf97f3141adc5f7f3c593d720c +size 384546 diff --git a/CKPTS/X2I_UltraEdiit_fourmask_LLava_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_filtered_objects_4mask_1280_context_after_stage1_5e4/000-GPT-XL/checkpoints/eval_step_36000/batch_0_cfg_7.5_topk_16384.jpg b/CKPTS/X2I_UltraEdiit_fourmask_LLava_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_filtered_objects_4mask_1280_context_after_stage1_5e4/000-GPT-XL/checkpoints/eval_step_36000/batch_0_cfg_7.5_topk_16384.jpg new file mode 100644 index 0000000000000000000000000000000000000000..3e2f9cc0efc48472e5ade234aa424fd7feab203e --- /dev/null +++ b/CKPTS/X2I_UltraEdiit_fourmask_LLava_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_filtered_objects_4mask_1280_context_after_stage1_5e4/000-GPT-XL/checkpoints/eval_step_36000/batch_0_cfg_7.5_topk_16384.jpg @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:5ec31d6310d68e51754d7ffd62703ddb37068e8ce2d347a928938857a5c1a567 +size 392674 diff --git a/CKPTS/X2I_UltraEdiit_fourmask_LLava_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_filtered_objects_4mask_1280_context_after_stage1_5e4/000-GPT-XL/checkpoints/eval_step_36000/batch_1_cfg_7.5_topk_16384.jpg b/CKPTS/X2I_UltraEdiit_fourmask_LLava_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_filtered_objects_4mask_1280_context_after_stage1_5e4/000-GPT-XL/checkpoints/eval_step_36000/batch_1_cfg_7.5_topk_16384.jpg new file mode 100644 index 0000000000000000000000000000000000000000..d0f371e75b93cd5a4492be73a915a3b55b0c205a --- /dev/null +++ b/CKPTS/X2I_UltraEdiit_fourmask_LLava_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_filtered_objects_4mask_1280_context_after_stage1_5e4/000-GPT-XL/checkpoints/eval_step_36000/batch_1_cfg_7.5_topk_16384.jpg @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:13f0f42eae1b0a3f038f0382b2c96e7b1266885009d5f3447fc643552d741bd9 +size 332108 diff --git a/CKPTS/X2I_UltraEdiit_fourmask_LLava_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_filtered_objects_4mask_1280_context_after_stage1_5e4/000-GPT-XL/checkpoints/eval_step_36000/batch_2_cfg_7.5_topk_16384.jpg b/CKPTS/X2I_UltraEdiit_fourmask_LLava_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_filtered_objects_4mask_1280_context_after_stage1_5e4/000-GPT-XL/checkpoints/eval_step_36000/batch_2_cfg_7.5_topk_16384.jpg new file mode 100644 index 0000000000000000000000000000000000000000..79050a267d0d1f9695d59b0292dc6d9fb7e868bd --- /dev/null +++ b/CKPTS/X2I_UltraEdiit_fourmask_LLava_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_filtered_objects_4mask_1280_context_after_stage1_5e4/000-GPT-XL/checkpoints/eval_step_36000/batch_2_cfg_7.5_topk_16384.jpg @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:601c6a0490d860a2531675167b6a0a782c4dffd545356e963903a6356852e278 +size 355323 diff --git a/CKPTS/X2I_UltraEdiit_fourmask_LLava_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_filtered_objects_4mask_1280_context_after_stage1_5e4/000-GPT-XL/checkpoints/eval_step_36000/batch_3_cfg_7.5_topk_16384.jpg b/CKPTS/X2I_UltraEdiit_fourmask_LLava_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_filtered_objects_4mask_1280_context_after_stage1_5e4/000-GPT-XL/checkpoints/eval_step_36000/batch_3_cfg_7.5_topk_16384.jpg new file mode 100644 index 0000000000000000000000000000000000000000..e5a7d1cf0e73a547be6c0a54d86272f9bcd8d273 --- /dev/null +++ b/CKPTS/X2I_UltraEdiit_fourmask_LLava_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_filtered_objects_4mask_1280_context_after_stage1_5e4/000-GPT-XL/checkpoints/eval_step_36000/batch_3_cfg_7.5_topk_16384.jpg @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:1c69d8fa0721cbcb776e678c2ac4541447735171ba0e3a7df14a8ecf02247c13 +size 390257 diff --git a/CKPTS/X2I_UltraEdiit_fourmask_LLava_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_filtered_objects_4mask_1280_context_after_stage1_5e4/000-GPT-XL/checkpoints/eval_step_36000/batch_4_cfg_7.5_topk_16384.jpg b/CKPTS/X2I_UltraEdiit_fourmask_LLava_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_filtered_objects_4mask_1280_context_after_stage1_5e4/000-GPT-XL/checkpoints/eval_step_36000/batch_4_cfg_7.5_topk_16384.jpg new file mode 100644 index 0000000000000000000000000000000000000000..e126bfe706cd988e6e34937cc836f5ac8efc10df --- /dev/null +++ b/CKPTS/X2I_UltraEdiit_fourmask_LLava_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_filtered_objects_4mask_1280_context_after_stage1_5e4/000-GPT-XL/checkpoints/eval_step_36000/batch_4_cfg_7.5_topk_16384.jpg @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:945c36203d61f6ff5f6ff118f65e691d07287dcaabc7b6b1e70195ae12d9df68 +size 412622 diff --git a/CKPTS/X2I_UltraEdiit_fourmask_LLava_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_filtered_objects_4mask_1280_context_after_stage1_5e4/000-GPT-XL/checkpoints/eval_step_36000/batch_5_cfg_7.5_topk_16384.jpg b/CKPTS/X2I_UltraEdiit_fourmask_LLava_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_filtered_objects_4mask_1280_context_after_stage1_5e4/000-GPT-XL/checkpoints/eval_step_36000/batch_5_cfg_7.5_topk_16384.jpg new file mode 100644 index 0000000000000000000000000000000000000000..573b896f52a1b3fcfdd9024ef3b25096813b238e --- /dev/null +++ b/CKPTS/X2I_UltraEdiit_fourmask_LLava_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_filtered_objects_4mask_1280_context_after_stage1_5e4/000-GPT-XL/checkpoints/eval_step_36000/batch_5_cfg_7.5_topk_16384.jpg @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:a0b0d2414d0deb15cf6e92a99c00c70b6f9e9ea7aa8e6ff3a7b9f345f2d9444d +size 376024 diff --git a/CKPTS/X2I_UltraEdiit_fourmask_LLava_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_filtered_objects_4mask_1280_context_after_stage1_5e4/000-GPT-XL/checkpoints/eval_step_38000/batch_0_cfg_7.5_topk_16384.jpg b/CKPTS/X2I_UltraEdiit_fourmask_LLava_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_filtered_objects_4mask_1280_context_after_stage1_5e4/000-GPT-XL/checkpoints/eval_step_38000/batch_0_cfg_7.5_topk_16384.jpg new file mode 100644 index 0000000000000000000000000000000000000000..901d3a6d5abf00e2a11942f211b5318208b98e5d --- /dev/null +++ b/CKPTS/X2I_UltraEdiit_fourmask_LLava_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_filtered_objects_4mask_1280_context_after_stage1_5e4/000-GPT-XL/checkpoints/eval_step_38000/batch_0_cfg_7.5_topk_16384.jpg @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:e957da6b0b64a51ec4ae73db0428cd8ccfb8a1b276aad33566794008602547e7 +size 391828 diff --git a/CKPTS/X2I_UltraEdiit_fourmask_LLava_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_filtered_objects_4mask_1280_context_after_stage1_5e4/000-GPT-XL/checkpoints/eval_step_38000/batch_1_cfg_7.5_topk_16384.jpg b/CKPTS/X2I_UltraEdiit_fourmask_LLava_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_filtered_objects_4mask_1280_context_after_stage1_5e4/000-GPT-XL/checkpoints/eval_step_38000/batch_1_cfg_7.5_topk_16384.jpg new file mode 100644 index 0000000000000000000000000000000000000000..957e0a95b1f36baf7edc0293937e924d23fe400c --- /dev/null +++ b/CKPTS/X2I_UltraEdiit_fourmask_LLava_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_filtered_objects_4mask_1280_context_after_stage1_5e4/000-GPT-XL/checkpoints/eval_step_38000/batch_1_cfg_7.5_topk_16384.jpg @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:ef93419bf788c2e9407aa9ef492c41141d42bb2585ef1a6e0d99ef9d7c8f4abe +size 346779 diff --git a/CKPTS/X2I_UltraEdiit_fourmask_LLava_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_filtered_objects_4mask_1280_context_after_stage1_5e4/000-GPT-XL/checkpoints/eval_step_38000/batch_2_cfg_7.5_topk_16384.jpg b/CKPTS/X2I_UltraEdiit_fourmask_LLava_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_filtered_objects_4mask_1280_context_after_stage1_5e4/000-GPT-XL/checkpoints/eval_step_38000/batch_2_cfg_7.5_topk_16384.jpg new file mode 100644 index 0000000000000000000000000000000000000000..7e3e0df7297620e86e22cb8ede06e9c007c4dc7a --- /dev/null +++ b/CKPTS/X2I_UltraEdiit_fourmask_LLava_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_filtered_objects_4mask_1280_context_after_stage1_5e4/000-GPT-XL/checkpoints/eval_step_38000/batch_2_cfg_7.5_topk_16384.jpg @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:4019f158a3bcdb7b9637ba88f236ffd9216fc607af240db9eee7fe04bb478f27 +size 357938 diff --git a/CKPTS/X2I_UltraEdiit_fourmask_LLava_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_filtered_objects_4mask_1280_context_after_stage1_5e4/000-GPT-XL/checkpoints/eval_step_38000/batch_3_cfg_7.5_topk_16384.jpg b/CKPTS/X2I_UltraEdiit_fourmask_LLava_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_filtered_objects_4mask_1280_context_after_stage1_5e4/000-GPT-XL/checkpoints/eval_step_38000/batch_3_cfg_7.5_topk_16384.jpg new file mode 100644 index 0000000000000000000000000000000000000000..68cdd89c9c4d7eacdb095b34c70e288f164388e3 --- /dev/null +++ b/CKPTS/X2I_UltraEdiit_fourmask_LLava_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_filtered_objects_4mask_1280_context_after_stage1_5e4/000-GPT-XL/checkpoints/eval_step_38000/batch_3_cfg_7.5_topk_16384.jpg @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:71a006263d227e0bf9958c99fcb955e5e5c528dcd03d0f0cc3f786f9928588f4 +size 391241 diff --git a/CKPTS/X2I_UltraEdiit_fourmask_LLava_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_filtered_objects_4mask_1280_context_after_stage1_5e4/000-GPT-XL/checkpoints/eval_step_38000/batch_4_cfg_7.5_topk_16384.jpg b/CKPTS/X2I_UltraEdiit_fourmask_LLava_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_filtered_objects_4mask_1280_context_after_stage1_5e4/000-GPT-XL/checkpoints/eval_step_38000/batch_4_cfg_7.5_topk_16384.jpg new file mode 100644 index 0000000000000000000000000000000000000000..2bb5576bed458654155af2a059eb0349eb136050 --- /dev/null +++ b/CKPTS/X2I_UltraEdiit_fourmask_LLava_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_filtered_objects_4mask_1280_context_after_stage1_5e4/000-GPT-XL/checkpoints/eval_step_38000/batch_4_cfg_7.5_topk_16384.jpg @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:8bfbb1322b8e63824044169a6adb9218d9323c0bd04e79422f9fff1af4532368 +size 392100 diff --git a/CKPTS/X2I_UltraEdiit_fourmask_LLava_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_filtered_objects_4mask_1280_context_after_stage1_5e4/000-GPT-XL/checkpoints/eval_step_38000/batch_5_cfg_7.5_topk_16384.jpg b/CKPTS/X2I_UltraEdiit_fourmask_LLava_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_filtered_objects_4mask_1280_context_after_stage1_5e4/000-GPT-XL/checkpoints/eval_step_38000/batch_5_cfg_7.5_topk_16384.jpg new file mode 100644 index 0000000000000000000000000000000000000000..74032ff4af6bdb155ed9756138a0b18b06ab9ec9 --- /dev/null +++ b/CKPTS/X2I_UltraEdiit_fourmask_LLava_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_filtered_objects_4mask_1280_context_after_stage1_5e4/000-GPT-XL/checkpoints/eval_step_38000/batch_5_cfg_7.5_topk_16384.jpg @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:cac9c448c5579501c8e5bb98a2b8393e9a04b0f97e883903567d682d7a16fe89 +size 369273 diff --git a/CKPTS/X2I_UltraEdiit_fourmask_LLava_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_filtered_objects_4mask_1280_context_after_stage1_5e4/000-GPT-XL/checkpoints/eval_step_4000/batch_0_cfg_7.5_topk_16384.jpg b/CKPTS/X2I_UltraEdiit_fourmask_LLava_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_filtered_objects_4mask_1280_context_after_stage1_5e4/000-GPT-XL/checkpoints/eval_step_4000/batch_0_cfg_7.5_topk_16384.jpg new file mode 100644 index 0000000000000000000000000000000000000000..9b9ba2efc5bb30017aa95c695cd5e13982bdfd1b --- /dev/null +++ b/CKPTS/X2I_UltraEdiit_fourmask_LLava_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_filtered_objects_4mask_1280_context_after_stage1_5e4/000-GPT-XL/checkpoints/eval_step_4000/batch_0_cfg_7.5_topk_16384.jpg @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:78c7df73931c8ba2e2ebe1276584dae7589a42cc1a868f484bd1275665730b57 +size 385990 diff --git a/CKPTS/X2I_UltraEdiit_fourmask_LLava_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_filtered_objects_4mask_1280_context_after_stage1_5e4/000-GPT-XL/checkpoints/eval_step_4000/batch_1_cfg_7.5_topk_16384.jpg b/CKPTS/X2I_UltraEdiit_fourmask_LLava_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_filtered_objects_4mask_1280_context_after_stage1_5e4/000-GPT-XL/checkpoints/eval_step_4000/batch_1_cfg_7.5_topk_16384.jpg new file mode 100644 index 0000000000000000000000000000000000000000..fc7cf85b5b90f327fab5e984867fae87058ac68f --- /dev/null +++ b/CKPTS/X2I_UltraEdiit_fourmask_LLava_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_filtered_objects_4mask_1280_context_after_stage1_5e4/000-GPT-XL/checkpoints/eval_step_4000/batch_1_cfg_7.5_topk_16384.jpg @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:3d0879b49e953483034afa754e679278cbb366c7ed596526b6388084136ca576 +size 331807 diff --git a/CKPTS/X2I_UltraEdiit_fourmask_LLava_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_filtered_objects_4mask_1280_context_after_stage1_5e4/000-GPT-XL/checkpoints/eval_step_4000/batch_2_cfg_7.5_topk_16384.jpg b/CKPTS/X2I_UltraEdiit_fourmask_LLava_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_filtered_objects_4mask_1280_context_after_stage1_5e4/000-GPT-XL/checkpoints/eval_step_4000/batch_2_cfg_7.5_topk_16384.jpg new file mode 100644 index 0000000000000000000000000000000000000000..8f638305da6ed24186668ee93c66e23e8aad1a10 --- /dev/null +++ b/CKPTS/X2I_UltraEdiit_fourmask_LLava_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_filtered_objects_4mask_1280_context_after_stage1_5e4/000-GPT-XL/checkpoints/eval_step_4000/batch_2_cfg_7.5_topk_16384.jpg @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:0af0332f893dbbcd80e562817376189c1363fd0f7cb0e5308065ecdb228b723d +size 348327 diff --git a/CKPTS/X2I_UltraEdiit_fourmask_LLava_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_filtered_objects_4mask_1280_context_after_stage1_5e4/000-GPT-XL/checkpoints/eval_step_4000/batch_3_cfg_7.5_topk_16384.jpg b/CKPTS/X2I_UltraEdiit_fourmask_LLava_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_filtered_objects_4mask_1280_context_after_stage1_5e4/000-GPT-XL/checkpoints/eval_step_4000/batch_3_cfg_7.5_topk_16384.jpg new file mode 100644 index 0000000000000000000000000000000000000000..b2908703fdb034cc810427eef162d45e1ec836ed --- /dev/null +++ b/CKPTS/X2I_UltraEdiit_fourmask_LLava_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_filtered_objects_4mask_1280_context_after_stage1_5e4/000-GPT-XL/checkpoints/eval_step_4000/batch_3_cfg_7.5_topk_16384.jpg @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:6a22ed91af0b3080689467eb2b43759c6645353ef7f2852ebe79f20b15734df6 +size 382308 diff --git a/CKPTS/X2I_UltraEdiit_fourmask_LLava_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_filtered_objects_4mask_1280_context_after_stage1_5e4/000-GPT-XL/checkpoints/eval_step_4000/batch_4_cfg_7.5_topk_16384.jpg b/CKPTS/X2I_UltraEdiit_fourmask_LLava_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_filtered_objects_4mask_1280_context_after_stage1_5e4/000-GPT-XL/checkpoints/eval_step_4000/batch_4_cfg_7.5_topk_16384.jpg new file mode 100644 index 0000000000000000000000000000000000000000..193a1ec114efbfcc41646ac148335bd655607c67 --- /dev/null +++ b/CKPTS/X2I_UltraEdiit_fourmask_LLava_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_filtered_objects_4mask_1280_context_after_stage1_5e4/000-GPT-XL/checkpoints/eval_step_4000/batch_4_cfg_7.5_topk_16384.jpg @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:256e7ef96e801c6bc2a7e634b0fec88ee014236b28dde74e998a401adb49e91b +size 410561 diff --git a/CKPTS/X2I_UltraEdiit_fourmask_LLava_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_filtered_objects_4mask_1280_context_after_stage1_5e4/000-GPT-XL/checkpoints/eval_step_4000/batch_5_cfg_7.5_topk_16384.jpg b/CKPTS/X2I_UltraEdiit_fourmask_LLava_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_filtered_objects_4mask_1280_context_after_stage1_5e4/000-GPT-XL/checkpoints/eval_step_4000/batch_5_cfg_7.5_topk_16384.jpg new file mode 100644 index 0000000000000000000000000000000000000000..cf532f4ae46dd4d932dbd5d2588591addda43dcd --- /dev/null +++ b/CKPTS/X2I_UltraEdiit_fourmask_LLava_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_filtered_objects_4mask_1280_context_after_stage1_5e4/000-GPT-XL/checkpoints/eval_step_4000/batch_5_cfg_7.5_topk_16384.jpg @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:3a9f844425a5720195fba2c5525d5dcc8e4d6d4eb525a70ccfe41eff86802c47 +size 377349 diff --git a/CKPTS/X2I_UltraEdiit_fourmask_LLava_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_filtered_objects_4mask_1280_context_after_stage1_5e4/000-GPT-XL/checkpoints/eval_step_40000/batch_0_cfg_7.5_topk_16384.jpg b/CKPTS/X2I_UltraEdiit_fourmask_LLava_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_filtered_objects_4mask_1280_context_after_stage1_5e4/000-GPT-XL/checkpoints/eval_step_40000/batch_0_cfg_7.5_topk_16384.jpg new file mode 100644 index 0000000000000000000000000000000000000000..f01ba7fd8e5e2c8384eff6ff8b9717a2b0124196 --- /dev/null +++ b/CKPTS/X2I_UltraEdiit_fourmask_LLava_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_filtered_objects_4mask_1280_context_after_stage1_5e4/000-GPT-XL/checkpoints/eval_step_40000/batch_0_cfg_7.5_topk_16384.jpg @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:713173c074c1c874b78f0bc2a29215839d9df06f3c4feea4d01cf93a17bf378c +size 391896 diff --git a/CKPTS/X2I_UltraEdiit_fourmask_LLava_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_filtered_objects_4mask_1280_context_after_stage1_5e4/000-GPT-XL/checkpoints/eval_step_40000/batch_1_cfg_7.5_topk_16384.jpg b/CKPTS/X2I_UltraEdiit_fourmask_LLava_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_filtered_objects_4mask_1280_context_after_stage1_5e4/000-GPT-XL/checkpoints/eval_step_40000/batch_1_cfg_7.5_topk_16384.jpg new file mode 100644 index 0000000000000000000000000000000000000000..a63590906344d41bf87cf19c28561b25f2d7ff06 --- /dev/null +++ b/CKPTS/X2I_UltraEdiit_fourmask_LLava_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_filtered_objects_4mask_1280_context_after_stage1_5e4/000-GPT-XL/checkpoints/eval_step_40000/batch_1_cfg_7.5_topk_16384.jpg @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:81cda7437b8e3cbb07800053b8117635ffd6e0312e317a475e8030f2f14a5a7b +size 333127 diff --git a/CKPTS/X2I_UltraEdiit_fourmask_LLava_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_filtered_objects_4mask_1280_context_after_stage1_5e4/000-GPT-XL/checkpoints/eval_step_40000/batch_2_cfg_7.5_topk_16384.jpg b/CKPTS/X2I_UltraEdiit_fourmask_LLava_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_filtered_objects_4mask_1280_context_after_stage1_5e4/000-GPT-XL/checkpoints/eval_step_40000/batch_2_cfg_7.5_topk_16384.jpg new file mode 100644 index 0000000000000000000000000000000000000000..4f95a29e2e1df4a7e51d995b41f0ad01b38e7151 --- /dev/null +++ b/CKPTS/X2I_UltraEdiit_fourmask_LLava_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_filtered_objects_4mask_1280_context_after_stage1_5e4/000-GPT-XL/checkpoints/eval_step_40000/batch_2_cfg_7.5_topk_16384.jpg @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:e9b721acf190af399f28cc36cd5c4de9a06d6695f73f77388aa72dcce0605c34 +size 355525 diff --git a/CKPTS/X2I_UltraEdiit_fourmask_LLava_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_filtered_objects_4mask_1280_context_after_stage1_5e4/000-GPT-XL/checkpoints/eval_step_40000/batch_3_cfg_7.5_topk_16384.jpg b/CKPTS/X2I_UltraEdiit_fourmask_LLava_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_filtered_objects_4mask_1280_context_after_stage1_5e4/000-GPT-XL/checkpoints/eval_step_40000/batch_3_cfg_7.5_topk_16384.jpg new file mode 100644 index 0000000000000000000000000000000000000000..290a78bf13f14bec669cb67695e306f883586190 --- /dev/null +++ b/CKPTS/X2I_UltraEdiit_fourmask_LLava_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_filtered_objects_4mask_1280_context_after_stage1_5e4/000-GPT-XL/checkpoints/eval_step_40000/batch_3_cfg_7.5_topk_16384.jpg @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:a00bec499d4a17c4e9c6d539e821e423a6ea7f2989a35570d6108a423d17aff9 +size 385096 diff --git a/CKPTS/X2I_UltraEdiit_fourmask_LLava_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_filtered_objects_4mask_1280_context_after_stage1_5e4/000-GPT-XL/checkpoints/eval_step_40000/batch_4_cfg_7.5_topk_16384.jpg b/CKPTS/X2I_UltraEdiit_fourmask_LLava_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_filtered_objects_4mask_1280_context_after_stage1_5e4/000-GPT-XL/checkpoints/eval_step_40000/batch_4_cfg_7.5_topk_16384.jpg new file mode 100644 index 0000000000000000000000000000000000000000..ead84b1542239122c31ef76a8830b906f4e8b2fd --- /dev/null +++ b/CKPTS/X2I_UltraEdiit_fourmask_LLava_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_filtered_objects_4mask_1280_context_after_stage1_5e4/000-GPT-XL/checkpoints/eval_step_40000/batch_4_cfg_7.5_topk_16384.jpg @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:012e923315368c24dc855fc5a2ba7928a6e8d5ae269aedc170fc32e3451424c8 +size 392449 diff --git a/CKPTS/X2I_UltraEdiit_fourmask_LLava_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_filtered_objects_4mask_1280_context_after_stage1_5e4/000-GPT-XL/checkpoints/eval_step_40000/batch_5_cfg_7.5_topk_16384.jpg b/CKPTS/X2I_UltraEdiit_fourmask_LLava_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_filtered_objects_4mask_1280_context_after_stage1_5e4/000-GPT-XL/checkpoints/eval_step_40000/batch_5_cfg_7.5_topk_16384.jpg new file mode 100644 index 0000000000000000000000000000000000000000..5864b26e12267d9dc406b7964d2a28e7bf6029e9 --- /dev/null +++ b/CKPTS/X2I_UltraEdiit_fourmask_LLava_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_filtered_objects_4mask_1280_context_after_stage1_5e4/000-GPT-XL/checkpoints/eval_step_40000/batch_5_cfg_7.5_topk_16384.jpg @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:4ec300549be53e4ebda1e800b9a53623aad437f0c68dbc1fcb9700b58dfcc509 +size 369635 diff --git a/CKPTS/X2I_UltraEdiit_fourmask_LLava_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_filtered_objects_4mask_1280_context_after_stage1_5e4/000-GPT-XL/checkpoints/eval_step_42000/batch_0_cfg_7.5_topk_16384.jpg b/CKPTS/X2I_UltraEdiit_fourmask_LLava_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_filtered_objects_4mask_1280_context_after_stage1_5e4/000-GPT-XL/checkpoints/eval_step_42000/batch_0_cfg_7.5_topk_16384.jpg new file mode 100644 index 0000000000000000000000000000000000000000..fc627afc6615ae18a884d8661a389ae2ff89d788 --- /dev/null +++ b/CKPTS/X2I_UltraEdiit_fourmask_LLava_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_filtered_objects_4mask_1280_context_after_stage1_5e4/000-GPT-XL/checkpoints/eval_step_42000/batch_0_cfg_7.5_topk_16384.jpg @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:b4a83ebd8547042610608036ced95bf2a40b5827e99503cc072576dc74693745 +size 398337 diff --git a/CKPTS/X2I_UltraEdiit_fourmask_LLava_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_filtered_objects_4mask_1280_context_after_stage1_5e4/000-GPT-XL/checkpoints/eval_step_42000/batch_1_cfg_7.5_topk_16384.jpg b/CKPTS/X2I_UltraEdiit_fourmask_LLava_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_filtered_objects_4mask_1280_context_after_stage1_5e4/000-GPT-XL/checkpoints/eval_step_42000/batch_1_cfg_7.5_topk_16384.jpg new file mode 100644 index 0000000000000000000000000000000000000000..e4b31ecf5a91f27a76d5ce601a9197a66c39e33d --- /dev/null +++ b/CKPTS/X2I_UltraEdiit_fourmask_LLava_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_filtered_objects_4mask_1280_context_after_stage1_5e4/000-GPT-XL/checkpoints/eval_step_42000/batch_1_cfg_7.5_topk_16384.jpg @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:a4fde822455136047fd2f94ca02d0ba4f11aa55e3c0bb37de4cccb8bd32c6f92 +size 323164 diff --git a/CKPTS/X2I_UltraEdiit_fourmask_LLava_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_filtered_objects_4mask_1280_context_after_stage1_5e4/000-GPT-XL/checkpoints/eval_step_42000/batch_2_cfg_7.5_topk_16384.jpg b/CKPTS/X2I_UltraEdiit_fourmask_LLava_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_filtered_objects_4mask_1280_context_after_stage1_5e4/000-GPT-XL/checkpoints/eval_step_42000/batch_2_cfg_7.5_topk_16384.jpg new file mode 100644 index 0000000000000000000000000000000000000000..76c6946b108b26527620a9c9fe7650bc75abe007 --- /dev/null +++ b/CKPTS/X2I_UltraEdiit_fourmask_LLava_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_filtered_objects_4mask_1280_context_after_stage1_5e4/000-GPT-XL/checkpoints/eval_step_42000/batch_2_cfg_7.5_topk_16384.jpg @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:ffe33d3593eb90f6055e9d3d6be21400a71a4cbec09b8dd517073ca551daa405 +size 361081 diff --git a/CKPTS/X2I_UltraEdiit_fourmask_LLava_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_filtered_objects_4mask_1280_context_after_stage1_5e4/000-GPT-XL/checkpoints/eval_step_42000/batch_3_cfg_7.5_topk_16384.jpg b/CKPTS/X2I_UltraEdiit_fourmask_LLava_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_filtered_objects_4mask_1280_context_after_stage1_5e4/000-GPT-XL/checkpoints/eval_step_42000/batch_3_cfg_7.5_topk_16384.jpg new file mode 100644 index 0000000000000000000000000000000000000000..fb839e1bbdcd9505f18b206eb7db9af00df9515c --- /dev/null +++ b/CKPTS/X2I_UltraEdiit_fourmask_LLava_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_filtered_objects_4mask_1280_context_after_stage1_5e4/000-GPT-XL/checkpoints/eval_step_42000/batch_3_cfg_7.5_topk_16384.jpg @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:9d78b6272247caa1582e16392758562a75ca48ec5f692adf046165533da317b0 +size 386504 diff --git a/CKPTS/X2I_UltraEdiit_fourmask_LLava_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_filtered_objects_4mask_1280_context_after_stage1_5e4/000-GPT-XL/checkpoints/eval_step_42000/batch_4_cfg_7.5_topk_16384.jpg b/CKPTS/X2I_UltraEdiit_fourmask_LLava_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_filtered_objects_4mask_1280_context_after_stage1_5e4/000-GPT-XL/checkpoints/eval_step_42000/batch_4_cfg_7.5_topk_16384.jpg new file mode 100644 index 0000000000000000000000000000000000000000..02a1883ce7367967da5281b9e817d4e5e85b6484 --- /dev/null +++ b/CKPTS/X2I_UltraEdiit_fourmask_LLava_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_filtered_objects_4mask_1280_context_after_stage1_5e4/000-GPT-XL/checkpoints/eval_step_42000/batch_4_cfg_7.5_topk_16384.jpg @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:d06a014570b5fcc05c15912fd3621e477150be3bc038f4b272879521c54d6215 +size 396986 diff --git a/CKPTS/X2I_UltraEdiit_fourmask_LLava_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_filtered_objects_4mask_1280_context_after_stage1_5e4/000-GPT-XL/checkpoints/eval_step_42000/batch_5_cfg_7.5_topk_16384.jpg b/CKPTS/X2I_UltraEdiit_fourmask_LLava_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_filtered_objects_4mask_1280_context_after_stage1_5e4/000-GPT-XL/checkpoints/eval_step_42000/batch_5_cfg_7.5_topk_16384.jpg new file mode 100644 index 0000000000000000000000000000000000000000..6151c0298363f1ef2a277c84c5a2cd6470c3772c --- /dev/null +++ b/CKPTS/X2I_UltraEdiit_fourmask_LLava_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_filtered_objects_4mask_1280_context_after_stage1_5e4/000-GPT-XL/checkpoints/eval_step_42000/batch_5_cfg_7.5_topk_16384.jpg @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:f23baf374b4f6fd7e935e390148b14d637535f4288823e0714978611c644bc50 +size 380256 diff --git a/CKPTS/X2I_UltraEdiit_fourmask_LLava_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_filtered_objects_4mask_1280_context_after_stage1_5e4/000-GPT-XL/checkpoints/eval_step_44000/batch_0_cfg_7.5_topk_16384.jpg b/CKPTS/X2I_UltraEdiit_fourmask_LLava_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_filtered_objects_4mask_1280_context_after_stage1_5e4/000-GPT-XL/checkpoints/eval_step_44000/batch_0_cfg_7.5_topk_16384.jpg new file mode 100644 index 0000000000000000000000000000000000000000..b40dd39bde13c9bac8a773898828c02fe95f7fce --- /dev/null +++ b/CKPTS/X2I_UltraEdiit_fourmask_LLava_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_filtered_objects_4mask_1280_context_after_stage1_5e4/000-GPT-XL/checkpoints/eval_step_44000/batch_0_cfg_7.5_topk_16384.jpg @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:ed7b202c8e8360128de7a7017c45f95ffdd9324c6aeac1d86b778a5fa97c1467 +size 385752 diff --git a/CKPTS/X2I_UltraEdiit_fourmask_LLava_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_filtered_objects_4mask_1280_context_after_stage1_5e4/000-GPT-XL/checkpoints/eval_step_44000/batch_1_cfg_7.5_topk_16384.jpg b/CKPTS/X2I_UltraEdiit_fourmask_LLava_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_filtered_objects_4mask_1280_context_after_stage1_5e4/000-GPT-XL/checkpoints/eval_step_44000/batch_1_cfg_7.5_topk_16384.jpg new file mode 100644 index 0000000000000000000000000000000000000000..34a97a88ecf49d4946b4dd5e0728c78ca857492c --- /dev/null +++ b/CKPTS/X2I_UltraEdiit_fourmask_LLava_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_filtered_objects_4mask_1280_context_after_stage1_5e4/000-GPT-XL/checkpoints/eval_step_44000/batch_1_cfg_7.5_topk_16384.jpg @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:6d38a533efcaba5c6989f48b8dfa057f5a6001cbc4e619331bca907dfcd506de +size 348138 diff --git a/CKPTS/X2I_UltraEdiit_fourmask_LLava_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_filtered_objects_4mask_1280_context_after_stage1_5e4/000-GPT-XL/checkpoints/eval_step_44000/batch_2_cfg_7.5_topk_16384.jpg b/CKPTS/X2I_UltraEdiit_fourmask_LLava_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_filtered_objects_4mask_1280_context_after_stage1_5e4/000-GPT-XL/checkpoints/eval_step_44000/batch_2_cfg_7.5_topk_16384.jpg new file mode 100644 index 0000000000000000000000000000000000000000..204d3a5960b5f37dd524a156d8a25daed2ba71c5 --- /dev/null +++ b/CKPTS/X2I_UltraEdiit_fourmask_LLava_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_filtered_objects_4mask_1280_context_after_stage1_5e4/000-GPT-XL/checkpoints/eval_step_44000/batch_2_cfg_7.5_topk_16384.jpg @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:92df7ae7f516cae2ee5fc011437c6800b7171c5bfe7f9c5b2fd27ed7b8d3028b +size 353133 diff --git a/CKPTS/X2I_UltraEdiit_fourmask_LLava_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_filtered_objects_4mask_1280_context_after_stage1_5e4/000-GPT-XL/checkpoints/eval_step_44000/batch_3_cfg_7.5_topk_16384.jpg b/CKPTS/X2I_UltraEdiit_fourmask_LLava_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_filtered_objects_4mask_1280_context_after_stage1_5e4/000-GPT-XL/checkpoints/eval_step_44000/batch_3_cfg_7.5_topk_16384.jpg new file mode 100644 index 0000000000000000000000000000000000000000..c11a08f3db945be13f0fc5e6b19cfe08ec0b66b3 --- /dev/null +++ b/CKPTS/X2I_UltraEdiit_fourmask_LLava_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_filtered_objects_4mask_1280_context_after_stage1_5e4/000-GPT-XL/checkpoints/eval_step_44000/batch_3_cfg_7.5_topk_16384.jpg @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:3524315b0b90687048363792aea3b72e8dd285206b17c264d06f4f311b7bf67f +size 393817 diff --git a/CKPTS/X2I_UltraEdiit_fourmask_LLava_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_filtered_objects_4mask_1280_context_after_stage1_5e4/000-GPT-XL/checkpoints/eval_step_44000/batch_4_cfg_7.5_topk_16384.jpg b/CKPTS/X2I_UltraEdiit_fourmask_LLava_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_filtered_objects_4mask_1280_context_after_stage1_5e4/000-GPT-XL/checkpoints/eval_step_44000/batch_4_cfg_7.5_topk_16384.jpg new file mode 100644 index 0000000000000000000000000000000000000000..c787bdddf9a30fd9d227e49ee7aa5c70e07b5d7a --- /dev/null +++ b/CKPTS/X2I_UltraEdiit_fourmask_LLava_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_filtered_objects_4mask_1280_context_after_stage1_5e4/000-GPT-XL/checkpoints/eval_step_44000/batch_4_cfg_7.5_topk_16384.jpg @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:dd1e9a815015196ce06ae80e256debb22b5adb596ccf8ace653bb8cb67d1eeeb +size 405368 diff --git a/CKPTS/X2I_UltraEdiit_fourmask_LLava_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_filtered_objects_4mask_1280_context_after_stage1_5e4/000-GPT-XL/checkpoints/eval_step_44000/batch_5_cfg_7.5_topk_16384.jpg b/CKPTS/X2I_UltraEdiit_fourmask_LLava_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_filtered_objects_4mask_1280_context_after_stage1_5e4/000-GPT-XL/checkpoints/eval_step_44000/batch_5_cfg_7.5_topk_16384.jpg new file mode 100644 index 0000000000000000000000000000000000000000..64e579bfe6ec537400f26be6f44897db6cced914 --- /dev/null +++ b/CKPTS/X2I_UltraEdiit_fourmask_LLava_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_filtered_objects_4mask_1280_context_after_stage1_5e4/000-GPT-XL/checkpoints/eval_step_44000/batch_5_cfg_7.5_topk_16384.jpg @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:1edacd1c32ffa77e6332105c0bb5160548634dd76f74e158e648a425c1a7537a +size 365030 diff --git a/CKPTS/X2I_UltraEdiit_fourmask_LLava_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_filtered_objects_4mask_1280_context_after_stage1_5e4/000-GPT-XL/checkpoints/eval_step_46000/batch_0_cfg_7.5_topk_16384.jpg b/CKPTS/X2I_UltraEdiit_fourmask_LLava_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_filtered_objects_4mask_1280_context_after_stage1_5e4/000-GPT-XL/checkpoints/eval_step_46000/batch_0_cfg_7.5_topk_16384.jpg new file mode 100644 index 0000000000000000000000000000000000000000..401b3bb918d5af69f40e91d87cb88eebc3c2f84c --- /dev/null +++ b/CKPTS/X2I_UltraEdiit_fourmask_LLava_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_filtered_objects_4mask_1280_context_after_stage1_5e4/000-GPT-XL/checkpoints/eval_step_46000/batch_0_cfg_7.5_topk_16384.jpg @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:4407c7343beb4ab1cf13efdc665e3d8dd445b0ba3f3daf6f148e33ffc2615434 +size 398074 diff --git a/CKPTS/X2I_UltraEdiit_fourmask_LLava_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_filtered_objects_4mask_1280_context_after_stage1_5e4/000-GPT-XL/checkpoints/eval_step_46000/batch_1_cfg_7.5_topk_16384.jpg b/CKPTS/X2I_UltraEdiit_fourmask_LLava_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_filtered_objects_4mask_1280_context_after_stage1_5e4/000-GPT-XL/checkpoints/eval_step_46000/batch_1_cfg_7.5_topk_16384.jpg new file mode 100644 index 0000000000000000000000000000000000000000..6a15c995e4163aee98f732f9ffafac624b7c4c03 --- /dev/null +++ b/CKPTS/X2I_UltraEdiit_fourmask_LLava_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_filtered_objects_4mask_1280_context_after_stage1_5e4/000-GPT-XL/checkpoints/eval_step_46000/batch_1_cfg_7.5_topk_16384.jpg @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:935bc43479c801de6d62cfe034c8bae1d085ccc9f52a9f48316e071550738619 +size 344520 diff --git a/CKPTS/X2I_UltraEdiit_fourmask_LLava_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_filtered_objects_4mask_1280_context_after_stage1_5e4/000-GPT-XL/checkpoints/eval_step_46000/batch_2_cfg_7.5_topk_16384.jpg b/CKPTS/X2I_UltraEdiit_fourmask_LLava_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_filtered_objects_4mask_1280_context_after_stage1_5e4/000-GPT-XL/checkpoints/eval_step_46000/batch_2_cfg_7.5_topk_16384.jpg new file mode 100644 index 0000000000000000000000000000000000000000..52d811aa7b1d241405a82a714b36d47850a6d28c --- /dev/null +++ b/CKPTS/X2I_UltraEdiit_fourmask_LLava_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_filtered_objects_4mask_1280_context_after_stage1_5e4/000-GPT-XL/checkpoints/eval_step_46000/batch_2_cfg_7.5_topk_16384.jpg @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:8b190d3a7aeae809664859e02851080b7a7281a85c188d45d55181842c9de510 +size 354279 diff --git a/CKPTS/X2I_UltraEdiit_fourmask_LLava_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_filtered_objects_4mask_1280_context_after_stage1_5e4/000-GPT-XL/checkpoints/eval_step_46000/batch_3_cfg_7.5_topk_16384.jpg b/CKPTS/X2I_UltraEdiit_fourmask_LLava_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_filtered_objects_4mask_1280_context_after_stage1_5e4/000-GPT-XL/checkpoints/eval_step_46000/batch_3_cfg_7.5_topk_16384.jpg new file mode 100644 index 0000000000000000000000000000000000000000..a950c3e7b70e1ca79da9876270a76b256cb2fab9 --- /dev/null +++ b/CKPTS/X2I_UltraEdiit_fourmask_LLava_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_filtered_objects_4mask_1280_context_after_stage1_5e4/000-GPT-XL/checkpoints/eval_step_46000/batch_3_cfg_7.5_topk_16384.jpg @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:4aa97a3b851b166bf8954070d9695c3388a48b580d754f2522e759bc45553fc0 +size 379866 diff --git a/CKPTS/X2I_UltraEdiit_fourmask_LLava_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_filtered_objects_4mask_1280_context_after_stage1_5e4/000-GPT-XL/checkpoints/eval_step_46000/batch_4_cfg_7.5_topk_16384.jpg b/CKPTS/X2I_UltraEdiit_fourmask_LLava_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_filtered_objects_4mask_1280_context_after_stage1_5e4/000-GPT-XL/checkpoints/eval_step_46000/batch_4_cfg_7.5_topk_16384.jpg new file mode 100644 index 0000000000000000000000000000000000000000..1d5375eb67d0d76104e39159519b73c9c6bcdb61 --- /dev/null +++ b/CKPTS/X2I_UltraEdiit_fourmask_LLava_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_filtered_objects_4mask_1280_context_after_stage1_5e4/000-GPT-XL/checkpoints/eval_step_46000/batch_4_cfg_7.5_topk_16384.jpg @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:70e1a8d473e3d24ef58036d03d57a33fece9013ed347924073e55e5385cd6dd7 +size 408804 diff --git a/CKPTS/X2I_UltraEdiit_fourmask_LLava_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_filtered_objects_4mask_1280_context_after_stage1_5e4/000-GPT-XL/checkpoints/eval_step_46000/batch_5_cfg_7.5_topk_16384.jpg b/CKPTS/X2I_UltraEdiit_fourmask_LLava_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_filtered_objects_4mask_1280_context_after_stage1_5e4/000-GPT-XL/checkpoints/eval_step_46000/batch_5_cfg_7.5_topk_16384.jpg new file mode 100644 index 0000000000000000000000000000000000000000..dd5652886ff41799fe3161a8bd0d72b284505fd3 --- /dev/null +++ b/CKPTS/X2I_UltraEdiit_fourmask_LLava_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_filtered_objects_4mask_1280_context_after_stage1_5e4/000-GPT-XL/checkpoints/eval_step_46000/batch_5_cfg_7.5_topk_16384.jpg @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:088dd43f67f76a40b1500ffa348a099741bbf2c041f48630fbbe2751be09ab91 +size 374050 diff --git a/CKPTS/X2I_UltraEdiit_fourmask_LLava_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_filtered_objects_4mask_1280_context_after_stage1_5e4/000-GPT-XL/checkpoints/eval_step_48000/batch_0_cfg_7.5_topk_16384.jpg b/CKPTS/X2I_UltraEdiit_fourmask_LLava_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_filtered_objects_4mask_1280_context_after_stage1_5e4/000-GPT-XL/checkpoints/eval_step_48000/batch_0_cfg_7.5_topk_16384.jpg new file mode 100644 index 0000000000000000000000000000000000000000..822c229843144dc74a7f83141096ae0f2595eaf6 --- /dev/null +++ b/CKPTS/X2I_UltraEdiit_fourmask_LLava_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_filtered_objects_4mask_1280_context_after_stage1_5e4/000-GPT-XL/checkpoints/eval_step_48000/batch_0_cfg_7.5_topk_16384.jpg @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:f8a6f11a5d61b97df5f2c007adfaf1bffbdd336f65ef8867f1dbfbec70304bcf +size 407043 diff --git a/CKPTS/X2I_UltraEdiit_fourmask_LLava_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_filtered_objects_4mask_1280_context_after_stage1_5e4/000-GPT-XL/checkpoints/eval_step_48000/batch_1_cfg_7.5_topk_16384.jpg b/CKPTS/X2I_UltraEdiit_fourmask_LLava_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_filtered_objects_4mask_1280_context_after_stage1_5e4/000-GPT-XL/checkpoints/eval_step_48000/batch_1_cfg_7.5_topk_16384.jpg new file mode 100644 index 0000000000000000000000000000000000000000..c7bf6c709611763ecf2baa60bb51b6f1dfa54068 --- /dev/null +++ b/CKPTS/X2I_UltraEdiit_fourmask_LLava_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_filtered_objects_4mask_1280_context_after_stage1_5e4/000-GPT-XL/checkpoints/eval_step_48000/batch_1_cfg_7.5_topk_16384.jpg @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:0f9b95ff60fbb29d077300298e93a67192faa8252febbbb00ab274787f2726b6 +size 343939 diff --git a/CKPTS/X2I_UltraEdiit_fourmask_LLava_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_filtered_objects_4mask_1280_context_after_stage1_5e4/000-GPT-XL/checkpoints/eval_step_48000/batch_2_cfg_7.5_topk_16384.jpg b/CKPTS/X2I_UltraEdiit_fourmask_LLava_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_filtered_objects_4mask_1280_context_after_stage1_5e4/000-GPT-XL/checkpoints/eval_step_48000/batch_2_cfg_7.5_topk_16384.jpg new file mode 100644 index 0000000000000000000000000000000000000000..351ea2832f640083ce391e3b33a1e40855ff99a1 --- /dev/null +++ b/CKPTS/X2I_UltraEdiit_fourmask_LLava_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_filtered_objects_4mask_1280_context_after_stage1_5e4/000-GPT-XL/checkpoints/eval_step_48000/batch_2_cfg_7.5_topk_16384.jpg @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:7d4d3e7271b7a8bca17b51b3e8c25043250a87f9cf775b53d6194964dfe186dd +size 361973 diff --git a/CKPTS/X2I_UltraEdiit_fourmask_LLava_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_filtered_objects_4mask_1280_context_after_stage1_5e4/000-GPT-XL/checkpoints/eval_step_48000/batch_3_cfg_7.5_topk_16384.jpg b/CKPTS/X2I_UltraEdiit_fourmask_LLava_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_filtered_objects_4mask_1280_context_after_stage1_5e4/000-GPT-XL/checkpoints/eval_step_48000/batch_3_cfg_7.5_topk_16384.jpg new file mode 100644 index 0000000000000000000000000000000000000000..eab08a5c3af607610b92ddadd8dea36b0433b04a --- /dev/null +++ b/CKPTS/X2I_UltraEdiit_fourmask_LLava_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_filtered_objects_4mask_1280_context_after_stage1_5e4/000-GPT-XL/checkpoints/eval_step_48000/batch_3_cfg_7.5_topk_16384.jpg @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:332e2c2be5e88ca00886811591133a50a5115e3fe9e80740d316bb21fef6c753 +size 389741 diff --git a/CKPTS/X2I_UltraEdiit_fourmask_LLava_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_filtered_objects_4mask_1280_context_after_stage1_5e4/000-GPT-XL/checkpoints/eval_step_48000/batch_4_cfg_7.5_topk_16384.jpg b/CKPTS/X2I_UltraEdiit_fourmask_LLava_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_filtered_objects_4mask_1280_context_after_stage1_5e4/000-GPT-XL/checkpoints/eval_step_48000/batch_4_cfg_7.5_topk_16384.jpg new file mode 100644 index 0000000000000000000000000000000000000000..c0fe47c116636c9e3f0e2240c4847cdfaaa7ce95 --- /dev/null +++ b/CKPTS/X2I_UltraEdiit_fourmask_LLava_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_filtered_objects_4mask_1280_context_after_stage1_5e4/000-GPT-XL/checkpoints/eval_step_48000/batch_4_cfg_7.5_topk_16384.jpg @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:1d308c388cc10172ede667bcd555450f93397acdd8e80577cbcbf5f56d471b05 +size 394771 diff --git a/CKPTS/X2I_UltraEdiit_fourmask_LLava_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_filtered_objects_4mask_1280_context_after_stage1_5e4/000-GPT-XL/checkpoints/eval_step_48000/batch_5_cfg_7.5_topk_16384.jpg b/CKPTS/X2I_UltraEdiit_fourmask_LLava_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_filtered_objects_4mask_1280_context_after_stage1_5e4/000-GPT-XL/checkpoints/eval_step_48000/batch_5_cfg_7.5_topk_16384.jpg new file mode 100644 index 0000000000000000000000000000000000000000..9a42cddb9cf18cc71145008cb98802b9c553e00a --- /dev/null +++ b/CKPTS/X2I_UltraEdiit_fourmask_LLava_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_filtered_objects_4mask_1280_context_after_stage1_5e4/000-GPT-XL/checkpoints/eval_step_48000/batch_5_cfg_7.5_topk_16384.jpg @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:6a2ee58871e513108d55c97666f9531c9f6a7c971254b8eb8ba82948fa8411af +size 383184 diff --git a/CKPTS/X2I_UltraEdiit_fourmask_LLava_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_filtered_objects_4mask_1280_context_after_stage1_5e4/000-GPT-XL/checkpoints/eval_step_50000/batch_0_cfg_7.5_topk_16384.jpg b/CKPTS/X2I_UltraEdiit_fourmask_LLava_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_filtered_objects_4mask_1280_context_after_stage1_5e4/000-GPT-XL/checkpoints/eval_step_50000/batch_0_cfg_7.5_topk_16384.jpg new file mode 100644 index 0000000000000000000000000000000000000000..8dc01a44702cdea8d7b91633eb5372e85f66fcc4 --- /dev/null +++ b/CKPTS/X2I_UltraEdiit_fourmask_LLava_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_filtered_objects_4mask_1280_context_after_stage1_5e4/000-GPT-XL/checkpoints/eval_step_50000/batch_0_cfg_7.5_topk_16384.jpg @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:619d40595b0ba8451d6d03c00d72076125116a7b1bc84e4c452f5b5af9b23abf +size 400148 diff --git a/CKPTS/X2I_UltraEdiit_fourmask_LLava_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_filtered_objects_4mask_1280_context_after_stage1_5e4/000-GPT-XL/checkpoints/eval_step_50000/batch_1_cfg_7.5_topk_16384.jpg b/CKPTS/X2I_UltraEdiit_fourmask_LLava_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_filtered_objects_4mask_1280_context_after_stage1_5e4/000-GPT-XL/checkpoints/eval_step_50000/batch_1_cfg_7.5_topk_16384.jpg new file mode 100644 index 0000000000000000000000000000000000000000..f55d56c2011ca222b143402349f26bdfe0bdb0cb --- /dev/null +++ b/CKPTS/X2I_UltraEdiit_fourmask_LLava_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_filtered_objects_4mask_1280_context_after_stage1_5e4/000-GPT-XL/checkpoints/eval_step_50000/batch_1_cfg_7.5_topk_16384.jpg @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:ca763f129d6d4d10e12820d644515d589adc4b8799f1d1c05041a9b7c092cda1 +size 338023 diff --git a/CKPTS/X2I_UltraEdiit_fourmask_LLava_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_filtered_objects_4mask_1280_context_after_stage1_5e4/000-GPT-XL/checkpoints/eval_step_50000/batch_2_cfg_7.5_topk_16384.jpg b/CKPTS/X2I_UltraEdiit_fourmask_LLava_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_filtered_objects_4mask_1280_context_after_stage1_5e4/000-GPT-XL/checkpoints/eval_step_50000/batch_2_cfg_7.5_topk_16384.jpg new file mode 100644 index 0000000000000000000000000000000000000000..f843c7306a895791683b3c47a08687240fc617ba --- /dev/null +++ b/CKPTS/X2I_UltraEdiit_fourmask_LLava_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_filtered_objects_4mask_1280_context_after_stage1_5e4/000-GPT-XL/checkpoints/eval_step_50000/batch_2_cfg_7.5_topk_16384.jpg @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:000a41a307752a148e2c08b5b9a6281640ee0d23e193bb7d55f1aa86f3ed48d2 +size 357870 diff --git a/CKPTS/X2I_UltraEdiit_fourmask_LLava_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_filtered_objects_4mask_1280_context_after_stage1_5e4/000-GPT-XL/checkpoints/eval_step_50000/batch_3_cfg_7.5_topk_16384.jpg b/CKPTS/X2I_UltraEdiit_fourmask_LLava_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_filtered_objects_4mask_1280_context_after_stage1_5e4/000-GPT-XL/checkpoints/eval_step_50000/batch_3_cfg_7.5_topk_16384.jpg new file mode 100644 index 0000000000000000000000000000000000000000..d558472023ea81494305c113340b3fbde42f0723 --- /dev/null +++ b/CKPTS/X2I_UltraEdiit_fourmask_LLava_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_filtered_objects_4mask_1280_context_after_stage1_5e4/000-GPT-XL/checkpoints/eval_step_50000/batch_3_cfg_7.5_topk_16384.jpg @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:e93b099f2e09cd64954cf227165af76d6e14857567904528061a85ac12c15f67 +size 387373 diff --git a/CKPTS/X2I_UltraEdiit_fourmask_LLava_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_filtered_objects_4mask_1280_context_after_stage1_5e4/000-GPT-XL/checkpoints/eval_step_50000/batch_4_cfg_7.5_topk_16384.jpg b/CKPTS/X2I_UltraEdiit_fourmask_LLava_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_filtered_objects_4mask_1280_context_after_stage1_5e4/000-GPT-XL/checkpoints/eval_step_50000/batch_4_cfg_7.5_topk_16384.jpg new file mode 100644 index 0000000000000000000000000000000000000000..4655fb39607380f345bb534257e5e01bb08d59cd --- /dev/null +++ b/CKPTS/X2I_UltraEdiit_fourmask_LLava_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_filtered_objects_4mask_1280_context_after_stage1_5e4/000-GPT-XL/checkpoints/eval_step_50000/batch_4_cfg_7.5_topk_16384.jpg @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:9a4fcff523049001eb3492e663e04e785c96f7eaf7c55d358d43b7d36aa6d615 +size 396853 diff --git a/CKPTS/X2I_UltraEdiit_fourmask_LLava_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_filtered_objects_4mask_1280_context_after_stage1_5e4/000-GPT-XL/checkpoints/eval_step_50000/batch_5_cfg_7.5_topk_16384.jpg b/CKPTS/X2I_UltraEdiit_fourmask_LLava_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_filtered_objects_4mask_1280_context_after_stage1_5e4/000-GPT-XL/checkpoints/eval_step_50000/batch_5_cfg_7.5_topk_16384.jpg new file mode 100644 index 0000000000000000000000000000000000000000..0ccc8aaea4bf6332d45dce14b4d016ec6659ffb2 --- /dev/null +++ b/CKPTS/X2I_UltraEdiit_fourmask_LLava_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_filtered_objects_4mask_1280_context_after_stage1_5e4/000-GPT-XL/checkpoints/eval_step_50000/batch_5_cfg_7.5_topk_16384.jpg @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:a566e22e8295e056a528634939ec85cd1201fa85e2375b62411538ff7b229b37 +size 357290 diff --git a/CKPTS/X2I_UltraEdiit_fourmask_LLava_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_filtered_objects_4mask_1280_context_after_stage1_5e4/000-GPT-XL/checkpoints/eval_step_6000/batch_0_cfg_7.5_topk_16384.jpg b/CKPTS/X2I_UltraEdiit_fourmask_LLava_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_filtered_objects_4mask_1280_context_after_stage1_5e4/000-GPT-XL/checkpoints/eval_step_6000/batch_0_cfg_7.5_topk_16384.jpg new file mode 100644 index 0000000000000000000000000000000000000000..9e34280b3591db6a261d7c00b9c3613890402526 --- /dev/null +++ b/CKPTS/X2I_UltraEdiit_fourmask_LLava_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_filtered_objects_4mask_1280_context_after_stage1_5e4/000-GPT-XL/checkpoints/eval_step_6000/batch_0_cfg_7.5_topk_16384.jpg @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:82990ad4004df9e3319885c6fa8db4ad2da67497f4f8e6e71f39f93c07613c19 +size 392400 diff --git a/CKPTS/X2I_UltraEdiit_fourmask_LLava_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_filtered_objects_4mask_1280_context_after_stage1_5e4/000-GPT-XL/checkpoints/eval_step_6000/batch_1_cfg_7.5_topk_16384.jpg b/CKPTS/X2I_UltraEdiit_fourmask_LLava_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_filtered_objects_4mask_1280_context_after_stage1_5e4/000-GPT-XL/checkpoints/eval_step_6000/batch_1_cfg_7.5_topk_16384.jpg new file mode 100644 index 0000000000000000000000000000000000000000..222fce381b5c0f287f84954828a941f2d8017b42 --- /dev/null +++ b/CKPTS/X2I_UltraEdiit_fourmask_LLava_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_filtered_objects_4mask_1280_context_after_stage1_5e4/000-GPT-XL/checkpoints/eval_step_6000/batch_1_cfg_7.5_topk_16384.jpg @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:e2e9b3076d7a0a13edc17657e84c05588cb9620bd20b5b639c076bad55600b20 +size 331581 diff --git a/CKPTS/X2I_UltraEdiit_fourmask_LLava_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_filtered_objects_4mask_1280_context_after_stage1_5e4/000-GPT-XL/checkpoints/eval_step_6000/batch_2_cfg_7.5_topk_16384.jpg b/CKPTS/X2I_UltraEdiit_fourmask_LLava_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_filtered_objects_4mask_1280_context_after_stage1_5e4/000-GPT-XL/checkpoints/eval_step_6000/batch_2_cfg_7.5_topk_16384.jpg new file mode 100644 index 0000000000000000000000000000000000000000..5291ce4c5879d2780117c0d1360ebc598f816170 --- /dev/null +++ b/CKPTS/X2I_UltraEdiit_fourmask_LLava_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_filtered_objects_4mask_1280_context_after_stage1_5e4/000-GPT-XL/checkpoints/eval_step_6000/batch_2_cfg_7.5_topk_16384.jpg @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:c36873035bfc56e1789d2ee22947b1fde73ab43a068f0833a640182c066bf45f +size 341959 diff --git a/CKPTS/X2I_UltraEdiit_fourmask_LLava_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_filtered_objects_4mask_1280_context_after_stage1_5e4/000-GPT-XL/checkpoints/eval_step_6000/batch_3_cfg_7.5_topk_16384.jpg b/CKPTS/X2I_UltraEdiit_fourmask_LLava_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_filtered_objects_4mask_1280_context_after_stage1_5e4/000-GPT-XL/checkpoints/eval_step_6000/batch_3_cfg_7.5_topk_16384.jpg new file mode 100644 index 0000000000000000000000000000000000000000..119e18af5b7053b23522e096e7f41dc3abb37e7c --- /dev/null +++ b/CKPTS/X2I_UltraEdiit_fourmask_LLava_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_filtered_objects_4mask_1280_context_after_stage1_5e4/000-GPT-XL/checkpoints/eval_step_6000/batch_3_cfg_7.5_topk_16384.jpg @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:5fbaa8b9101aa3ca46c38ee608ddf95d41a3dfd97ccb9340922a2a9796dc1aa3 +size 384614 diff --git a/CKPTS/X2I_UltraEdiit_fourmask_LLava_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_filtered_objects_4mask_1280_context_after_stage1_5e4/000-GPT-XL/checkpoints/eval_step_6000/batch_4_cfg_7.5_topk_16384.jpg b/CKPTS/X2I_UltraEdiit_fourmask_LLava_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_filtered_objects_4mask_1280_context_after_stage1_5e4/000-GPT-XL/checkpoints/eval_step_6000/batch_4_cfg_7.5_topk_16384.jpg new file mode 100644 index 0000000000000000000000000000000000000000..e7944cf3e0f8ded36e64e07661cd13168a7f8e81 --- /dev/null +++ b/CKPTS/X2I_UltraEdiit_fourmask_LLava_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_filtered_objects_4mask_1280_context_after_stage1_5e4/000-GPT-XL/checkpoints/eval_step_6000/batch_4_cfg_7.5_topk_16384.jpg @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:a24c5401bf31f16ab8ea5af989ebda9bee6e599cd338d2bac2faf1b5c845c2fe +size 424036 diff --git a/CKPTS/X2I_UltraEdiit_fourmask_LLava_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_filtered_objects_4mask_1280_context_after_stage1_5e4/000-GPT-XL/checkpoints/eval_step_6000/batch_5_cfg_7.5_topk_16384.jpg b/CKPTS/X2I_UltraEdiit_fourmask_LLava_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_filtered_objects_4mask_1280_context_after_stage1_5e4/000-GPT-XL/checkpoints/eval_step_6000/batch_5_cfg_7.5_topk_16384.jpg new file mode 100644 index 0000000000000000000000000000000000000000..0d8151d86950961197d475833694fa4091c69f3e --- /dev/null +++ b/CKPTS/X2I_UltraEdiit_fourmask_LLava_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_filtered_objects_4mask_1280_context_after_stage1_5e4/000-GPT-XL/checkpoints/eval_step_6000/batch_5_cfg_7.5_topk_16384.jpg @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:a92dc258f86e661f339e9f479abecc47df487dda1e291f1d5ab35f092e96bcc3 +size 390735 diff --git a/CKPTS/X2I_UltraEdiit_fourmask_LLava_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_filtered_objects_4mask_1280_context_after_stage1_5e4/000-GPT-XL/checkpoints/eval_step_8000/batch_0_cfg_7.5_topk_16384.jpg b/CKPTS/X2I_UltraEdiit_fourmask_LLava_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_filtered_objects_4mask_1280_context_after_stage1_5e4/000-GPT-XL/checkpoints/eval_step_8000/batch_0_cfg_7.5_topk_16384.jpg new file mode 100644 index 0000000000000000000000000000000000000000..c431a04bfb9285c0313b53dec7cd2c12524ab51a --- /dev/null +++ b/CKPTS/X2I_UltraEdiit_fourmask_LLava_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_filtered_objects_4mask_1280_context_after_stage1_5e4/000-GPT-XL/checkpoints/eval_step_8000/batch_0_cfg_7.5_topk_16384.jpg @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:908eefbeef93073f50b40b7e1abf45ad5896f3a13cb878c43e9a88bc91461d92 +size 387124 diff --git a/CKPTS/X2I_UltraEdiit_fourmask_LLava_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_filtered_objects_4mask_1280_context_after_stage1_5e4/000-GPT-XL/checkpoints/eval_step_8000/batch_1_cfg_7.5_topk_16384.jpg b/CKPTS/X2I_UltraEdiit_fourmask_LLava_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_filtered_objects_4mask_1280_context_after_stage1_5e4/000-GPT-XL/checkpoints/eval_step_8000/batch_1_cfg_7.5_topk_16384.jpg new file mode 100644 index 0000000000000000000000000000000000000000..960e952859176f2ac35d21a7a6476eb382261336 --- /dev/null +++ b/CKPTS/X2I_UltraEdiit_fourmask_LLava_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_filtered_objects_4mask_1280_context_after_stage1_5e4/000-GPT-XL/checkpoints/eval_step_8000/batch_1_cfg_7.5_topk_16384.jpg @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:0883afafc53059eb4ac73fc196b5db6f532ea7ee47b3bee717f94229334ab293 +size 368386 diff --git a/CKPTS/X2I_UltraEdiit_fourmask_LLava_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_filtered_objects_4mask_1280_context_after_stage1_5e4/000-GPT-XL/checkpoints/eval_step_8000/batch_2_cfg_7.5_topk_16384.jpg b/CKPTS/X2I_UltraEdiit_fourmask_LLava_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_filtered_objects_4mask_1280_context_after_stage1_5e4/000-GPT-XL/checkpoints/eval_step_8000/batch_2_cfg_7.5_topk_16384.jpg new file mode 100644 index 0000000000000000000000000000000000000000..7fef12936902386fe6f63390a3ee502fe6f36f47 --- /dev/null +++ b/CKPTS/X2I_UltraEdiit_fourmask_LLava_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_filtered_objects_4mask_1280_context_after_stage1_5e4/000-GPT-XL/checkpoints/eval_step_8000/batch_2_cfg_7.5_topk_16384.jpg @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:e1542a37ad5eb47699b315f70f38872fe348dd77d094f22708d71051028f3a2d +size 352017 diff --git a/CKPTS/X2I_UltraEdiit_fourmask_LLava_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_filtered_objects_4mask_1280_context_after_stage1_5e4/000-GPT-XL/checkpoints/eval_step_8000/batch_3_cfg_7.5_topk_16384.jpg b/CKPTS/X2I_UltraEdiit_fourmask_LLava_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_filtered_objects_4mask_1280_context_after_stage1_5e4/000-GPT-XL/checkpoints/eval_step_8000/batch_3_cfg_7.5_topk_16384.jpg new file mode 100644 index 0000000000000000000000000000000000000000..17344292a0b7fc4694688303f9cf3af93770845a --- /dev/null +++ b/CKPTS/X2I_UltraEdiit_fourmask_LLava_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_filtered_objects_4mask_1280_context_after_stage1_5e4/000-GPT-XL/checkpoints/eval_step_8000/batch_3_cfg_7.5_topk_16384.jpg @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:cd44587131da73e5106e488643a01627b9e1512b6e170fb6a776f861cf408734 +size 391700 diff --git a/CKPTS/X2I_UltraEdiit_fourmask_LLava_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_filtered_objects_4mask_1280_context_after_stage1_5e4/000-GPT-XL/checkpoints/eval_step_8000/batch_4_cfg_7.5_topk_16384.jpg b/CKPTS/X2I_UltraEdiit_fourmask_LLava_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_filtered_objects_4mask_1280_context_after_stage1_5e4/000-GPT-XL/checkpoints/eval_step_8000/batch_4_cfg_7.5_topk_16384.jpg new file mode 100644 index 0000000000000000000000000000000000000000..386016d69a60f807504da58d32e5b04a151530f9 --- /dev/null +++ b/CKPTS/X2I_UltraEdiit_fourmask_LLava_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_filtered_objects_4mask_1280_context_after_stage1_5e4/000-GPT-XL/checkpoints/eval_step_8000/batch_4_cfg_7.5_topk_16384.jpg @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:a7f2858ca98991f0e1c2c91a1700121810dc2130d6e5e71af5bb77a0f2b0cdc6 +size 423569 diff --git a/CKPTS/X2I_UltraEdiit_fourmask_LLava_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_filtered_objects_4mask_1280_context_after_stage1_5e4/000-GPT-XL/checkpoints/eval_step_8000/batch_5_cfg_7.5_topk_16384.jpg b/CKPTS/X2I_UltraEdiit_fourmask_LLava_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_filtered_objects_4mask_1280_context_after_stage1_5e4/000-GPT-XL/checkpoints/eval_step_8000/batch_5_cfg_7.5_topk_16384.jpg new file mode 100644 index 0000000000000000000000000000000000000000..bcc6e317b8f28864cda42e4e440a101aacc0018e --- /dev/null +++ b/CKPTS/X2I_UltraEdiit_fourmask_LLava_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_filtered_objects_4mask_1280_context_after_stage1_5e4/000-GPT-XL/checkpoints/eval_step_8000/batch_5_cfg_7.5_topk_16384.jpg @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:ae80cb26cab06dbe8c3a4455d9e74d36ef01ca7bfca435037ce8f7d4a5b1b535 +size 388130 diff --git a/CKPTS/X2I_UltraEdiit_fourmask_LLava_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_filtered_objects_4mask_1280_context_after_stage1_5e4/000-GPT-XL/log.txt b/CKPTS/X2I_UltraEdiit_fourmask_LLava_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_filtered_objects_4mask_1280_context_after_stage1_5e4/000-GPT-XL/log.txt new file mode 100644 index 0000000000000000000000000000000000000000..8b8721208ea3f3fbd32dff861e6e69bd1850b373 --- /dev/null +++ b/CKPTS/X2I_UltraEdiit_fourmask_LLava_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_filtered_objects_4mask_1280_context_after_stage1_5e4/000-GPT-XL/log.txt @@ -0,0 +1,2177 @@ +[2025-04-28 10:22:34] Experiment directory created at checkpoint/X2I_UltraEdiit_fourmask_LLava_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_filtered_objects_4mask_1280_context_after_stage1_5e4/000-GPT-XL +[2025-04-28 10:22:34] Namespace(data_path='/tmp/haozhezhao/MLLMG/jsonl_data/multiimage_training_for_llava_X2I_fourmask.jsonl', cloud_save_path='/tmp/haozhezhao/MLLMG/checkpoint', no_local_save=False, vq_model='VQ-16', vq_ckpt='/tmp/haozhezhao/model/llamagen_t2i/vq_ds16_t2i.pt', codebook_size=16384, codebook_embed_dim=8, gpt_model='GPT-XL', gpt_ckpt='/tmp/haozhezhao/MLLMG/MLLMG_ckpts/checkpoints/AfterI2I_llavat5_subject_train_all_T5_subject_t2i_ti2i_120_w_flux_segment_noimageonly_empty_fix_train_all/0100000.pt', gpt_type='t2i', vocab_size=16384, cls_token_num=1280, dropout_p=0.1, token_dropout_p=0.1, drop_path=0.0, no_compile=False, results_dir='checkpoint/X2I_UltraEdiit_fourmask_LLava_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_filtered_objects_4mask_1280_context_after_stage1_5e4', dataset='ti2i', image_size=512, downsample_size=16, num_classes=1000, epochs=1, lr=0.0005, weight_decay=0.05, beta1=0.9, beta2=0.95, max_grad_norm=1.0, global_batch_size=24, global_seed=0, num_workers=4, log_every=25, ckpt_every=2000, gradient_accumulation_steps=8, mixed_precision='bf16', val_data_path='/tmp/haozhezhao/MLLMG/jsonl_data/multiimage_val_for_llava_X2I_fourmask_dreambenplus.jsonl', use_vision_tower=True, model_name_or_path='/tmp/haozhezhao/model/blip2-flan-t5-xl', image_place_holder='', processor_path=None, do_eval=True, max_eval_samples=128, train_text_encoder=True, no_left_padding=False, cfg_scale=7.5, top_k=16384, temperature=0.9, top_p=1.0, eval_steps=2000, project_name='llamagen_ti2i', load_from_checkpoint='/tmp/haozhezhao/MLLMG/checkpoint/FIXed_3M_1epoch_step32000.pt', warmup=0.05, lr_decay_style='cosine', lr_decay_ratio=0.1, train_iters=500000, class_dropout_prob=0.1, with_image_only=False, image_only_rate=0.1, stage2=False, subject_driven=True, load_subject_embedding=None, reference_data_path='/tmp/haozhezhao/MLLMG/cc12m_reference_tunnel.jsonl', multimodal_encoder='llava', do_recovery=True, no_replace=False, resume=False, dreambench_eval=True, find_unused_parameters=True, load_visual_encoder=False, continue_stage1=False, replace_subject=False, train_all=True, save_total_limit=1, load_language_projection='/tmp/haozhezhao/MLLMG/llava-v1.5-flant5_fixed-pretrain/mm_projector.bin', mm_vision_tower='openai/clip-vit-large-patch14', load_fixed_llamagen=True, unfreeze_output=False, fix='gpt-empty-fix', rank=0, world_size=8, gpu=0, dist_url='env://', distributed=True, dist_backend='nccl') +[2025-04-28 10:22:34] Starting rank=0, seed=0, world_size=8. +[2025-04-28 10:22:34] text_config is None. Initializing the text config with default values (`OPTConfig`). +[2025-04-28 10:23:39] GPT Parameters: 2,310,680,832 +[2025-04-28 10:23:39] num decayed parameter tensors: 356, with 2,007,303,168 parameters +[2025-04-28 10:23:39] num non-decayed parameter tensors: 124, with 197,888 parameters +[2025-04-28 10:23:39] using fused AdamW: True +[2025-04-28 10:24:10] Dataset contains 2,157,728 images +[2025-04-28 10:24:10] Train iters 89905 , warmup 4495.25, len of loader 89905 +[2025-04-28 10:25:00] ### LOAD pretraining weights from checkpoint: /tmp/haozhezhao/MLLMG/MLLMG_ckpts/checkpoints/AfterI2I_llavat5_subject_train_all_T5_subject_t2i_ti2i_120_w_flux_segment_noimageonly_empty_fix_train_all/0100000.pt +[2025-04-28 10:25:00] Initial state: steps=0, epochs=0 +[2025-04-28 10:25:00] compiling the model... (may take several minutes) +[2025-04-28 10:25:00] freeze the vit +[2025-04-28 10:25:01] ***** total param is 2310680832 ***** +[2025-04-28 10:25:01] ***** total trained param is 2007501056 ***** +[2025-04-28 10:25:02] Training for 1 epochs... +[2025-04-28 10:25:02] Beginning epoch 0... +[2025-04-28 10:30:18] (step=0000025) Train Loss: 8.1092, Train Steps/Sec: 0.08 +[2025-04-28 10:30:40] (step=0000050) Train Loss: 8.1421, Train Steps/Sec: 1.10 +[2025-04-28 10:31:03] (step=0000075) Train Loss: 8.0019, Train Steps/Sec: 1.10 +[2025-04-28 10:31:26] (step=0000100) Train Loss: 8.0168, Train Steps/Sec: 1.12 +[2025-04-28 10:31:48] (step=0000125) Train Loss: 7.9105, Train Steps/Sec: 1.12 +[2025-04-28 10:32:10] (step=0000150) Train Loss: 7.9324, Train Steps/Sec: 1.12 +[2025-04-28 10:32:33] (step=0000175) Train Loss: 7.8198, Train Steps/Sec: 1.12 +[2025-04-28 10:32:55] (step=0000200) Train Loss: 7.7789, Train Steps/Sec: 1.11 +[2025-04-28 10:33:17] (step=0000225) Train Loss: 7.6225, Train Steps/Sec: 1.12 +[2025-04-28 10:33:40] (step=0000250) Train Loss: 7.5357, Train Steps/Sec: 1.12 +[2025-04-28 10:34:02] (step=0000275) Train Loss: 7.4374, Train Steps/Sec: 1.12 +[2025-04-28 10:34:24] (step=0000300) Train Loss: 7.4600, Train Steps/Sec: 1.12 +[2025-04-28 10:34:47] (step=0000325) Train Loss: 7.3324, Train Steps/Sec: 1.12 +[2025-04-28 10:35:09] (step=0000350) Train Loss: 7.2950, Train Steps/Sec: 1.12 +[2025-04-28 10:35:32] (step=0000375) Train Loss: 7.2966, Train Steps/Sec: 1.12 +[2025-04-28 10:35:54] (step=0000400) Train Loss: 7.1496, Train Steps/Sec: 1.11 +[2025-04-28 10:36:16] (step=0000425) Train Loss: 7.0687, Train Steps/Sec: 1.12 +[2025-04-28 10:36:39] (step=0000450) Train Loss: 7.0731, Train Steps/Sec: 1.12 +[2025-04-28 10:37:01] (step=0000475) Train Loss: 6.9842, Train Steps/Sec: 1.12 +[2025-04-28 10:37:23] (step=0000500) Train Loss: 6.9064, Train Steps/Sec: 1.12 +[2025-04-28 10:37:46] (step=0000525) Train Loss: 6.8972, Train Steps/Sec: 1.12 +[2025-04-28 10:38:08] (step=0000550) Train Loss: 6.8811, Train Steps/Sec: 1.12 +[2025-04-28 10:38:30] (step=0000575) Train Loss: 6.8756, Train Steps/Sec: 1.12 +[2025-04-28 10:38:53] (step=0000600) Train Loss: 6.7432, Train Steps/Sec: 1.11 +[2025-04-28 10:39:15] (step=0000625) Train Loss: 6.7971, Train Steps/Sec: 1.12 +[2025-04-28 10:39:38] (step=0000650) Train Loss: 6.8317, Train Steps/Sec: 1.12 +[2025-04-28 10:40:00] (step=0000675) Train Loss: 6.8772, Train Steps/Sec: 1.12 +[2025-04-28 10:40:22] (step=0000700) Train Loss: 6.7981, Train Steps/Sec: 1.12 +[2025-04-28 10:40:45] (step=0000725) Train Loss: 6.7361, Train Steps/Sec: 1.12 +[2025-04-28 10:41:07] (step=0000750) Train Loss: 6.8179, Train Steps/Sec: 1.12 +[2025-04-28 10:41:29] (step=0000775) Train Loss: 6.7837, Train Steps/Sec: 1.12 +[2025-04-28 10:41:52] (step=0000800) Train Loss: 6.6719, Train Steps/Sec: 1.11 +[2025-04-28 10:42:14] (step=0000825) Train Loss: 6.7018, Train Steps/Sec: 1.12 +[2025-04-28 10:42:36] (step=0000850) Train Loss: 6.7435, Train Steps/Sec: 1.12 +[2025-04-28 10:42:59] (step=0000875) Train Loss: 6.6905, Train Steps/Sec: 1.12 +[2025-04-28 10:43:21] (step=0000900) Train Loss: 6.7018, Train Steps/Sec: 1.12 +[2025-04-28 10:43:43] (step=0000925) Train Loss: 6.6906, Train Steps/Sec: 1.12 +[2025-04-28 10:44:06] (step=0000950) Train Loss: 6.6143, Train Steps/Sec: 1.12 +[2025-04-28 10:44:28] (step=0000975) Train Loss: 6.6678, Train Steps/Sec: 1.12 +[2025-04-28 10:44:50] (step=0001000) Train Loss: 6.6981, Train Steps/Sec: 1.12 +[2025-04-28 10:45:13] (step=0001025) Train Loss: 6.6264, Train Steps/Sec: 1.12 +[2025-04-28 10:45:35] (step=0001050) Train Loss: 6.6604, Train Steps/Sec: 1.12 +[2025-04-28 10:45:58] (step=0001075) Train Loss: 6.6570, Train Steps/Sec: 1.12 +[2025-04-28 10:46:20] (step=0001100) Train Loss: 6.6121, Train Steps/Sec: 1.12 +[2025-04-28 10:46:42] (step=0001125) Train Loss: 6.6586, Train Steps/Sec: 1.12 +[2025-04-28 10:47:05] (step=0001150) Train Loss: 6.6565, Train Steps/Sec: 1.12 +[2025-04-28 10:47:27] (step=0001175) Train Loss: 6.6763, Train Steps/Sec: 1.12 +[2025-04-28 10:47:49] (step=0001200) Train Loss: 6.5940, Train Steps/Sec: 1.12 +[2025-04-28 10:48:12] (step=0001225) Train Loss: 6.6367, Train Steps/Sec: 1.12 +[2025-04-28 10:48:34] (step=0001250) Train Loss: 6.6159, Train Steps/Sec: 1.12 +[2025-04-28 10:48:56] (step=0001275) Train Loss: 6.6290, Train Steps/Sec: 1.12 +[2025-04-28 10:49:19] (step=0001300) Train Loss: 6.5799, Train Steps/Sec: 1.12 +[2025-04-28 10:49:53] (step=0001325) Train Loss: 6.6413, Train Steps/Sec: 0.72 +[2025-04-28 10:50:26] (step=0001350) Train Loss: 6.5724, Train Steps/Sec: 0.78 +[2025-04-28 10:51:28] (step=0001375) Train Loss: 6.6245, Train Steps/Sec: 0.40 +[2025-04-28 10:52:10] (step=0001400) Train Loss: 6.5878, Train Steps/Sec: 0.59 +[2025-04-28 10:52:33] (step=0001425) Train Loss: 6.5532, Train Steps/Sec: 1.12 +[2025-04-28 10:52:55] (step=0001450) Train Loss: 6.6474, Train Steps/Sec: 1.12 +[2025-04-28 10:53:18] (step=0001475) Train Loss: 6.6164, Train Steps/Sec: 1.12 +[2025-04-28 10:53:40] (step=0001500) Train Loss: 6.5906, Train Steps/Sec: 1.12 +[2025-04-28 10:54:02] (step=0001525) Train Loss: 6.5675, Train Steps/Sec: 1.12 +[2025-04-28 10:54:25] (step=0001550) Train Loss: 6.6065, Train Steps/Sec: 1.12 +[2025-04-28 10:54:47] (step=0001575) Train Loss: 6.6480, Train Steps/Sec: 1.12 +[2025-04-28 10:55:09] (step=0001600) Train Loss: 6.5494, Train Steps/Sec: 1.11 +[2025-04-28 10:55:32] (step=0001625) Train Loss: 6.6368, Train Steps/Sec: 1.12 +[2025-04-28 10:55:54] (step=0001650) Train Loss: 6.5613, Train Steps/Sec: 1.12 +[2025-04-28 10:56:16] (step=0001675) Train Loss: 6.6066, Train Steps/Sec: 1.12 +[2025-04-28 10:56:39] (step=0001700) Train Loss: 6.5331, Train Steps/Sec: 1.12 +[2025-04-28 10:57:01] (step=0001725) Train Loss: 6.5667, Train Steps/Sec: 1.12 +[2025-04-28 10:57:23] (step=0001750) Train Loss: 6.5901, Train Steps/Sec: 1.12 +[2025-04-28 10:57:46] (step=0001775) Train Loss: 6.5751, Train Steps/Sec: 1.12 +[2025-04-28 10:58:08] (step=0001800) Train Loss: 6.6029, Train Steps/Sec: 1.11 +[2025-04-28 10:58:31] (step=0001825) Train Loss: 6.5654, Train Steps/Sec: 1.12 +[2025-04-28 10:58:53] (step=0001850) Train Loss: 6.5113, Train Steps/Sec: 1.12 +[2025-04-28 10:59:15] (step=0001875) Train Loss: 6.5072, Train Steps/Sec: 1.12 +[2025-04-28 10:59:38] (step=0001900) Train Loss: 6.5506, Train Steps/Sec: 1.12 +[2025-04-28 11:00:00] (step=0001925) Train Loss: 6.5729, Train Steps/Sec: 1.12 +[2025-04-28 11:00:22] (step=0001950) Train Loss: 6.5451, Train Steps/Sec: 1.12 +[2025-04-28 11:00:45] (step=0001975) Train Loss: 6.5847, Train Steps/Sec: 1.12 +[2025-04-28 11:01:07] (step=0002000) Train Loss: 6.5440, Train Steps/Sec: 1.12 +[2025-04-28 11:01:07] text_config is None. Initializing the text config with default values (`OPTConfig`). +[2025-04-28 11:08:27] Finish Eval in 2000 steps... +[2025-04-28 11:08:47] Saved checkpoint to checkpoint/X2I_UltraEdiit_fourmask_LLava_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_filtered_objects_4mask_1280_context_after_stage1_5e4/000-GPT-XL/checkpoints/0002000.pt +[2025-04-28 11:09:10] (step=0002025) Train Loss: 6.5534, Train Steps/Sec: 0.05 +[2025-04-28 11:09:32] (step=0002050) Train Loss: 6.5801, Train Steps/Sec: 1.12 +[2025-04-28 11:09:54] (step=0002075) Train Loss: 6.5736, Train Steps/Sec: 1.12 +[2025-04-28 11:10:17] (step=0002100) Train Loss: 6.6217, Train Steps/Sec: 1.12 +[2025-04-28 11:10:39] (step=0002125) Train Loss: 6.5207, Train Steps/Sec: 1.12 +[2025-04-28 11:11:01] (step=0002150) Train Loss: 6.4993, Train Steps/Sec: 1.12 +[2025-04-28 11:11:24] (step=0002175) Train Loss: 6.5182, Train Steps/Sec: 1.12 +[2025-04-28 11:11:46] (step=0002200) Train Loss: 6.5620, Train Steps/Sec: 1.12 +[2025-04-28 11:12:08] (step=0002225) Train Loss: 6.5376, Train Steps/Sec: 1.12 +[2025-04-28 11:12:31] (step=0002250) Train Loss: 6.4775, Train Steps/Sec: 1.12 +[2025-04-28 11:12:53] (step=0002275) Train Loss: 6.5781, Train Steps/Sec: 1.12 +[2025-04-28 11:13:15] (step=0002300) Train Loss: 6.6102, Train Steps/Sec: 1.12 +[2025-04-28 11:13:38] (step=0002325) Train Loss: 6.4687, Train Steps/Sec: 1.12 +[2025-04-28 11:14:00] (step=0002350) Train Loss: 6.5302, Train Steps/Sec: 1.12 +[2025-04-28 11:14:22] (step=0002375) Train Loss: 6.5095, Train Steps/Sec: 1.12 +[2025-04-28 11:14:45] (step=0002400) Train Loss: 6.5516, Train Steps/Sec: 1.11 +[2025-04-28 11:15:07] (step=0002425) Train Loss: 6.4959, Train Steps/Sec: 1.12 +[2025-04-28 11:15:29] (step=0002450) Train Loss: 6.5264, Train Steps/Sec: 1.12 +[2025-04-28 11:15:52] (step=0002475) Train Loss: 6.5302, Train Steps/Sec: 1.12 +[2025-04-28 11:16:14] (step=0002500) Train Loss: 6.5048, Train Steps/Sec: 1.12 +[2025-04-28 11:16:37] (step=0002525) Train Loss: 6.4928, Train Steps/Sec: 1.12 +[2025-04-28 11:16:59] (step=0002550) Train Loss: 6.5111, Train Steps/Sec: 1.12 +[2025-04-28 11:17:21] (step=0002575) Train Loss: 6.4855, Train Steps/Sec: 1.12 +[2025-04-28 11:17:44] (step=0002600) Train Loss: 6.4489, Train Steps/Sec: 1.11 +[2025-04-28 11:18:06] (step=0002625) Train Loss: 6.5543, Train Steps/Sec: 1.12 +[2025-04-28 11:18:28] (step=0002650) Train Loss: 6.5589, Train Steps/Sec: 1.12 +[2025-04-28 11:18:51] (step=0002675) Train Loss: 6.5175, Train Steps/Sec: 1.12 +[2025-04-28 11:19:13] (step=0002700) Train Loss: 6.5470, Train Steps/Sec: 1.12 +[2025-04-28 11:19:35] (step=0002725) Train Loss: 6.4999, Train Steps/Sec: 1.12 +[2025-04-28 11:19:58] (step=0002750) Train Loss: 6.5397, Train Steps/Sec: 1.12 +[2025-04-28 11:20:20] (step=0002775) Train Loss: 6.6104, Train Steps/Sec: 1.12 +[2025-04-28 11:20:42] (step=0002800) Train Loss: 6.5031, Train Steps/Sec: 1.12 +[2025-04-28 11:21:05] (step=0002825) Train Loss: 6.5526, Train Steps/Sec: 1.12 +[2025-04-28 11:21:27] (step=0002850) Train Loss: 6.5375, Train Steps/Sec: 1.12 +[2025-04-28 11:21:50] (step=0002875) Train Loss: 6.4974, Train Steps/Sec: 1.12 +[2025-04-28 11:22:12] (step=0002900) Train Loss: 6.5393, Train Steps/Sec: 1.12 +[2025-04-28 11:22:34] (step=0002925) Train Loss: 6.5221, Train Steps/Sec: 1.12 +[2025-04-28 11:22:57] (step=0002950) Train Loss: 6.4657, Train Steps/Sec: 1.12 +[2025-04-28 11:23:19] (step=0002975) Train Loss: 6.5025, Train Steps/Sec: 1.12 +[2025-04-28 11:23:41] (step=0003000) Train Loss: 6.5557, Train Steps/Sec: 1.12 +[2025-04-28 11:24:04] (step=0003025) Train Loss: 6.4961, Train Steps/Sec: 1.12 +[2025-04-28 11:24:26] (step=0003050) Train Loss: 6.4553, Train Steps/Sec: 1.12 +[2025-04-28 11:24:48] (step=0003075) Train Loss: 6.4916, Train Steps/Sec: 1.12 +[2025-04-28 11:25:11] (step=0003100) Train Loss: 6.5366, Train Steps/Sec: 1.12 +[2025-04-28 11:25:33] (step=0003125) Train Loss: 6.5343, Train Steps/Sec: 1.12 +[2025-04-28 11:25:55] (step=0003150) Train Loss: 6.5077, Train Steps/Sec: 1.12 +[2025-04-28 11:26:18] (step=0003175) Train Loss: 6.4589, Train Steps/Sec: 1.11 +[2025-04-28 11:26:40] (step=0003200) Train Loss: 6.3991, Train Steps/Sec: 1.12 +[2025-04-28 11:27:03] (step=0003225) Train Loss: 6.4099, Train Steps/Sec: 1.12 +[2025-04-28 11:27:25] (step=0003250) Train Loss: 6.5213, Train Steps/Sec: 1.12 +[2025-04-28 11:27:47] (step=0003275) Train Loss: 6.4586, Train Steps/Sec: 1.12 +[2025-04-28 11:28:10] (step=0003300) Train Loss: 6.5156, Train Steps/Sec: 1.12 +[2025-04-28 11:28:32] (step=0003325) Train Loss: 6.5228, Train Steps/Sec: 1.12 +[2025-04-28 11:28:54] (step=0003350) Train Loss: 6.5242, Train Steps/Sec: 1.12 +[2025-04-28 11:29:17] (step=0003375) Train Loss: 6.5061, Train Steps/Sec: 1.12 +[2025-04-28 11:29:39] (step=0003400) Train Loss: 6.4710, Train Steps/Sec: 1.12 +[2025-04-28 11:30:01] (step=0003425) Train Loss: 6.4517, Train Steps/Sec: 1.12 +[2025-04-28 11:30:24] (step=0003450) Train Loss: 6.4769, Train Steps/Sec: 1.12 +[2025-04-28 11:30:46] (step=0003475) Train Loss: 6.4765, Train Steps/Sec: 1.12 +[2025-04-28 11:31:08] (step=0003500) Train Loss: 6.4750, Train Steps/Sec: 1.12 +[2025-04-28 11:31:31] (step=0003525) Train Loss: 6.4758, Train Steps/Sec: 1.12 +[2025-04-28 11:31:53] (step=0003550) Train Loss: 6.4850, Train Steps/Sec: 1.12 +[2025-04-28 11:32:15] (step=0003575) Train Loss: 6.4601, Train Steps/Sec: 1.12 +[2025-04-28 11:32:38] (step=0003600) Train Loss: 6.4813, Train Steps/Sec: 1.12 +[2025-04-28 11:33:00] (step=0003625) Train Loss: 6.4902, Train Steps/Sec: 1.12 +[2025-04-28 11:33:22] (step=0003650) Train Loss: 6.5125, Train Steps/Sec: 1.12 +[2025-04-28 11:33:45] (step=0003675) Train Loss: 6.4308, Train Steps/Sec: 1.12 +[2025-04-28 11:34:07] (step=0003700) Train Loss: 6.4558, Train Steps/Sec: 1.12 +[2025-04-28 11:34:30] (step=0003725) Train Loss: 6.4281, Train Steps/Sec: 1.12 +[2025-04-28 11:34:52] (step=0003750) Train Loss: 6.5049, Train Steps/Sec: 1.12 +[2025-04-28 11:35:14] (step=0003775) Train Loss: 6.4229, Train Steps/Sec: 1.12 +[2025-04-28 11:35:37] (step=0003800) Train Loss: 6.5067, Train Steps/Sec: 1.11 +[2025-04-28 11:35:59] (step=0003825) Train Loss: 6.4834, Train Steps/Sec: 1.12 +[2025-04-28 11:36:21] (step=0003850) Train Loss: 6.4106, Train Steps/Sec: 1.12 +[2025-04-28 11:36:44] (step=0003875) Train Loss: 6.4231, Train Steps/Sec: 1.12 +[2025-04-28 11:37:06] (step=0003900) Train Loss: 6.4437, Train Steps/Sec: 1.12 +[2025-04-28 11:37:28] (step=0003925) Train Loss: 6.4155, Train Steps/Sec: 1.12 +[2025-04-28 11:37:51] (step=0003950) Train Loss: 6.5193, Train Steps/Sec: 1.12 +[2025-04-28 11:38:13] (step=0003975) Train Loss: 6.4155, Train Steps/Sec: 1.12 +[2025-04-28 11:38:35] (step=0004000) Train Loss: 6.4832, Train Steps/Sec: 1.12 +[2025-04-28 11:38:36] text_config is None. Initializing the text config with default values (`OPTConfig`). +[2025-04-28 11:45:53] Finish Eval in 4000 steps... +[2025-04-28 11:46:13] Saved checkpoint to checkpoint/X2I_UltraEdiit_fourmask_LLava_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_filtered_objects_4mask_1280_context_after_stage1_5e4/000-GPT-XL/checkpoints/0004000.pt +[2025-04-28 11:46:15] Removed old checkpoint: checkpoint/X2I_UltraEdiit_fourmask_LLava_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_filtered_objects_4mask_1280_context_after_stage1_5e4/000-GPT-XL/checkpoints/0002000.pt +[2025-04-28 11:46:37] (step=0004025) Train Loss: 6.4099, Train Steps/Sec: 0.05 +[2025-04-28 11:46:59] (step=0004050) Train Loss: 6.5192, Train Steps/Sec: 1.12 +[2025-04-28 11:47:22] (step=0004075) Train Loss: 6.4452, Train Steps/Sec: 1.12 +[2025-04-28 11:47:44] (step=0004100) Train Loss: 6.4579, Train Steps/Sec: 1.12 +[2025-04-28 11:48:06] (step=0004125) Train Loss: 6.4193, Train Steps/Sec: 1.12 +[2025-04-28 11:48:29] (step=0004150) Train Loss: 6.4320, Train Steps/Sec: 1.12 +[2025-04-28 11:48:51] (step=0004175) Train Loss: 6.4474, Train Steps/Sec: 1.12 +[2025-04-28 11:49:13] (step=0004200) Train Loss: 6.4446, Train Steps/Sec: 1.11 +[2025-04-28 11:49:36] (step=0004225) Train Loss: 6.4663, Train Steps/Sec: 1.12 +[2025-04-28 11:49:58] (step=0004250) Train Loss: 6.5018, Train Steps/Sec: 1.12 +[2025-04-28 11:50:20] (step=0004275) Train Loss: 6.5116, Train Steps/Sec: 1.12 +[2025-04-28 11:50:43] (step=0004300) Train Loss: 6.4493, Train Steps/Sec: 1.12 +[2025-04-28 11:51:05] (step=0004325) Train Loss: 6.4124, Train Steps/Sec: 1.12 +[2025-04-28 11:51:27] (step=0004350) Train Loss: 6.4117, Train Steps/Sec: 1.12 +[2025-04-28 11:51:50] (step=0004375) Train Loss: 6.4720, Train Steps/Sec: 1.11 +[2025-04-28 11:52:12] (step=0004400) Train Loss: 6.4215, Train Steps/Sec: 1.11 +[2025-04-28 11:52:35] (step=0004425) Train Loss: 6.4024, Train Steps/Sec: 1.12 +[2025-04-28 11:52:57] (step=0004450) Train Loss: 6.4817, Train Steps/Sec: 1.12 +[2025-04-28 11:53:19] (step=0004475) Train Loss: 6.4606, Train Steps/Sec: 1.12 +[2025-04-28 11:53:42] (step=0004500) Train Loss: 6.4599, Train Steps/Sec: 1.12 +[2025-04-28 11:54:04] (step=0004525) Train Loss: 6.4290, Train Steps/Sec: 1.12 +[2025-04-28 11:54:26] (step=0004550) Train Loss: 6.4144, Train Steps/Sec: 1.12 +[2025-04-28 11:54:49] (step=0004575) Train Loss: 6.3784, Train Steps/Sec: 1.12 +[2025-04-28 11:55:11] (step=0004600) Train Loss: 6.3721, Train Steps/Sec: 1.12 +[2025-04-28 11:55:33] (step=0004625) Train Loss: 6.4663, Train Steps/Sec: 1.12 +[2025-04-28 11:55:56] (step=0004650) Train Loss: 6.4239, Train Steps/Sec: 1.12 +[2025-04-28 11:56:18] (step=0004675) Train Loss: 6.4599, Train Steps/Sec: 1.12 +[2025-04-28 11:56:40] (step=0004700) Train Loss: 6.3608, Train Steps/Sec: 1.12 +[2025-04-28 11:57:03] (step=0004725) Train Loss: 6.4572, Train Steps/Sec: 1.12 +[2025-04-28 11:57:25] (step=0004750) Train Loss: 6.4561, Train Steps/Sec: 1.12 +[2025-04-28 11:57:47] (step=0004775) Train Loss: 6.4806, Train Steps/Sec: 1.12 +[2025-04-28 11:58:10] (step=0004800) Train Loss: 6.4253, Train Steps/Sec: 1.11 +[2025-04-28 11:58:32] (step=0004825) Train Loss: 6.4200, Train Steps/Sec: 1.12 +[2025-04-28 11:58:54] (step=0004850) Train Loss: 6.4268, Train Steps/Sec: 1.12 +[2025-04-28 11:59:17] (step=0004875) Train Loss: 6.3626, Train Steps/Sec: 1.12 +[2025-04-28 11:59:39] (step=0004900) Train Loss: 6.4606, Train Steps/Sec: 1.12 +[2025-04-28 12:00:30] (step=0004925) Train Loss: 6.3901, Train Steps/Sec: 0.49 +[2025-04-28 12:01:02] (step=0004950) Train Loss: 6.4584, Train Steps/Sec: 0.79 +[2025-04-28 12:01:33] (step=0004975) Train Loss: 6.4209, Train Steps/Sec: 0.79 +[2025-04-28 12:01:56] (step=0005000) Train Loss: 6.3990, Train Steps/Sec: 1.12 +[2025-04-28 12:02:18] (step=0005025) Train Loss: 6.4649, Train Steps/Sec: 1.12 +[2025-04-28 12:02:40] (step=0005050) Train Loss: 6.3706, Train Steps/Sec: 1.12 +[2025-04-28 12:03:03] (step=0005075) Train Loss: 6.4158, Train Steps/Sec: 1.12 +[2025-04-28 12:03:25] (step=0005100) Train Loss: 6.3972, Train Steps/Sec: 1.12 +[2025-04-28 12:03:47] (step=0005125) Train Loss: 6.3681, Train Steps/Sec: 1.12 +[2025-04-28 12:04:10] (step=0005150) Train Loss: 6.3981, Train Steps/Sec: 1.12 +[2025-04-28 12:04:32] (step=0005175) Train Loss: 6.4441, Train Steps/Sec: 1.12 +[2025-04-28 12:04:54] (step=0005200) Train Loss: 6.4513, Train Steps/Sec: 1.11 +[2025-04-28 12:05:17] (step=0005225) Train Loss: 6.4187, Train Steps/Sec: 1.12 +[2025-04-28 12:05:39] (step=0005250) Train Loss: 6.3721, Train Steps/Sec: 1.12 +[2025-04-28 12:06:01] (step=0005275) Train Loss: 6.4778, Train Steps/Sec: 1.12 +[2025-04-28 12:06:24] (step=0005300) Train Loss: 6.4105, Train Steps/Sec: 1.12 +[2025-04-28 12:06:46] (step=0005325) Train Loss: 6.4380, Train Steps/Sec: 1.12 +[2025-04-28 12:07:08] (step=0005350) Train Loss: 6.4355, Train Steps/Sec: 1.12 +[2025-04-28 12:07:31] (step=0005375) Train Loss: 6.3652, Train Steps/Sec: 1.12 +[2025-04-28 12:07:53] (step=0005400) Train Loss: 6.3738, Train Steps/Sec: 1.12 +[2025-04-28 12:08:15] (step=0005425) Train Loss: 6.4244, Train Steps/Sec: 1.12 +[2025-04-28 12:08:38] (step=0005450) Train Loss: 6.4083, Train Steps/Sec: 1.12 +[2025-04-28 12:09:00] (step=0005475) Train Loss: 6.4378, Train Steps/Sec: 1.12 +[2025-04-28 12:09:22] (step=0005500) Train Loss: 6.4167, Train Steps/Sec: 1.12 +[2025-04-28 12:09:45] (step=0005525) Train Loss: 6.3858, Train Steps/Sec: 1.12 +[2025-04-28 12:10:07] (step=0005550) Train Loss: 6.4631, Train Steps/Sec: 1.12 +[2025-04-28 12:10:29] (step=0005575) Train Loss: 6.3906, Train Steps/Sec: 1.12 +[2025-04-28 12:10:52] (step=0005600) Train Loss: 6.3885, Train Steps/Sec: 1.12 +[2025-04-28 12:11:14] (step=0005625) Train Loss: 6.3565, Train Steps/Sec: 1.12 +[2025-04-28 12:11:37] (step=0005650) Train Loss: 6.3953, Train Steps/Sec: 1.12 +[2025-04-28 12:11:59] (step=0005675) Train Loss: 6.3692, Train Steps/Sec: 1.12 +[2025-04-28 12:12:21] (step=0005700) Train Loss: 6.3866, Train Steps/Sec: 1.12 +[2025-04-28 12:12:44] (step=0005725) Train Loss: 6.5024, Train Steps/Sec: 1.12 +[2025-04-28 12:13:06] (step=0005750) Train Loss: 6.4182, Train Steps/Sec: 1.12 +[2025-04-28 12:13:28] (step=0005775) Train Loss: 6.4550, Train Steps/Sec: 1.12 +[2025-04-28 12:13:51] (step=0005800) Train Loss: 6.3405, Train Steps/Sec: 1.12 +[2025-04-28 12:14:13] (step=0005825) Train Loss: 6.4379, Train Steps/Sec: 1.12 +[2025-04-28 12:14:35] (step=0005850) Train Loss: 6.3994, Train Steps/Sec: 1.12 +[2025-04-28 12:14:58] (step=0005875) Train Loss: 6.3801, Train Steps/Sec: 1.12 +[2025-04-28 12:15:20] (step=0005900) Train Loss: 6.4163, Train Steps/Sec: 1.12 +[2025-04-28 12:15:42] (step=0005925) Train Loss: 6.3760, Train Steps/Sec: 1.12 +[2025-04-28 12:16:05] (step=0005950) Train Loss: 6.3828, Train Steps/Sec: 1.12 +[2025-04-28 12:16:27] (step=0005975) Train Loss: 6.3960, Train Steps/Sec: 1.12 +[2025-04-28 12:16:50] (step=0006000) Train Loss: 6.3470, Train Steps/Sec: 1.10 +[2025-04-28 12:16:50] text_config is None. Initializing the text config with default values (`OPTConfig`). +[2025-04-28 12:24:09] Finish Eval in 6000 steps... +[2025-04-28 12:24:28] Saved checkpoint to checkpoint/X2I_UltraEdiit_fourmask_LLava_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_filtered_objects_4mask_1280_context_after_stage1_5e4/000-GPT-XL/checkpoints/0006000.pt +[2025-04-28 12:24:30] Removed old checkpoint: checkpoint/X2I_UltraEdiit_fourmask_LLava_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_filtered_objects_4mask_1280_context_after_stage1_5e4/000-GPT-XL/checkpoints/0004000.pt +[2025-04-28 12:24:53] (step=0006025) Train Loss: 6.3688, Train Steps/Sec: 0.05 +[2025-04-28 12:25:16] (step=0006050) Train Loss: 6.3762, Train Steps/Sec: 1.11 +[2025-04-28 12:25:38] (step=0006075) Train Loss: 6.3538, Train Steps/Sec: 1.12 +[2025-04-28 12:26:01] (step=0006100) Train Loss: 6.3220, Train Steps/Sec: 1.12 +[2025-04-28 12:26:23] (step=0006125) Train Loss: 6.3212, Train Steps/Sec: 1.12 +[2025-04-28 12:26:45] (step=0006150) Train Loss: 6.3841, Train Steps/Sec: 1.12 +[2025-04-28 12:27:08] (step=0006175) Train Loss: 6.4498, Train Steps/Sec: 1.12 +[2025-04-28 12:27:30] (step=0006200) Train Loss: 6.4097, Train Steps/Sec: 1.11 +[2025-04-28 12:27:52] (step=0006225) Train Loss: 6.4438, Train Steps/Sec: 1.12 +[2025-04-28 12:28:15] (step=0006250) Train Loss: 6.3409, Train Steps/Sec: 1.11 +[2025-04-28 12:28:37] (step=0006275) Train Loss: 6.3623, Train Steps/Sec: 1.11 +[2025-04-28 12:29:00] (step=0006300) Train Loss: 6.3150, Train Steps/Sec: 1.10 +[2025-04-28 12:29:22] (step=0006325) Train Loss: 6.3871, Train Steps/Sec: 1.11 +[2025-04-28 12:29:45] (step=0006350) Train Loss: 6.3271, Train Steps/Sec: 1.12 +[2025-04-28 12:30:07] (step=0006375) Train Loss: 6.3487, Train Steps/Sec: 1.11 +[2025-04-28 12:30:30] (step=0006400) Train Loss: 6.3749, Train Steps/Sec: 1.11 +[2025-04-28 12:30:52] (step=0006425) Train Loss: 6.3576, Train Steps/Sec: 1.11 +[2025-04-28 12:31:15] (step=0006450) Train Loss: 6.3883, Train Steps/Sec: 1.12 +[2025-04-28 12:31:37] (step=0006475) Train Loss: 6.3179, Train Steps/Sec: 1.12 +[2025-04-28 12:32:00] (step=0006500) Train Loss: 6.3753, Train Steps/Sec: 1.12 +[2025-04-28 12:32:22] (step=0006525) Train Loss: 6.3083, Train Steps/Sec: 1.12 +[2025-04-28 12:32:44] (step=0006550) Train Loss: 6.3374, Train Steps/Sec: 1.11 +[2025-04-28 12:33:07] (step=0006575) Train Loss: 6.3211, Train Steps/Sec: 1.11 +[2025-04-28 12:33:30] (step=0006600) Train Loss: 6.2962, Train Steps/Sec: 1.10 +[2025-04-28 12:33:52] (step=0006625) Train Loss: 6.3448, Train Steps/Sec: 1.11 +[2025-04-28 12:34:15] (step=0006650) Train Loss: 6.3616, Train Steps/Sec: 1.11 +[2025-04-28 12:34:37] (step=0006675) Train Loss: 6.2940, Train Steps/Sec: 1.11 +[2025-04-28 12:34:59] (step=0006700) Train Loss: 6.3012, Train Steps/Sec: 1.12 +[2025-04-28 12:35:22] (step=0006725) Train Loss: 6.3618, Train Steps/Sec: 1.12 +[2025-04-28 12:35:44] (step=0006750) Train Loss: 6.3523, Train Steps/Sec: 1.11 +[2025-04-28 12:36:07] (step=0006775) Train Loss: 6.2996, Train Steps/Sec: 1.11 +[2025-04-28 12:36:29] (step=0006800) Train Loss: 6.3512, Train Steps/Sec: 1.11 +[2025-04-28 12:36:52] (step=0006825) Train Loss: 6.3491, Train Steps/Sec: 1.12 +[2025-04-28 12:37:14] (step=0006850) Train Loss: 6.3301, Train Steps/Sec: 1.12 +[2025-04-28 12:37:37] (step=0006875) Train Loss: 6.3186, Train Steps/Sec: 1.11 +[2025-04-28 12:37:59] (step=0006900) Train Loss: 6.3850, Train Steps/Sec: 1.11 +[2025-04-28 12:38:22] (step=0006925) Train Loss: 6.3080, Train Steps/Sec: 1.11 +[2025-04-28 12:38:44] (step=0006950) Train Loss: 6.2326, Train Steps/Sec: 1.11 +[2025-04-28 12:39:07] (step=0006975) Train Loss: 6.3361, Train Steps/Sec: 1.11 +[2025-04-28 12:39:29] (step=0007000) Train Loss: 6.3091, Train Steps/Sec: 1.11 +[2025-04-28 12:39:52] (step=0007025) Train Loss: 6.3572, Train Steps/Sec: 1.11 +[2025-04-28 12:40:14] (step=0007050) Train Loss: 6.3504, Train Steps/Sec: 1.12 +[2025-04-28 12:40:37] (step=0007075) Train Loss: 6.3142, Train Steps/Sec: 1.11 +[2025-04-28 12:40:59] (step=0007100) Train Loss: 6.2948, Train Steps/Sec: 1.11 +[2025-04-28 12:41:22] (step=0007125) Train Loss: 6.2980, Train Steps/Sec: 1.11 +[2025-04-28 12:41:44] (step=0007150) Train Loss: 6.2722, Train Steps/Sec: 1.12 +[2025-04-28 12:42:06] (step=0007175) Train Loss: 6.2736, Train Steps/Sec: 1.12 +[2025-04-28 12:42:29] (step=0007200) Train Loss: 6.3163, Train Steps/Sec: 1.11 +[2025-04-28 12:42:51] (step=0007225) Train Loss: 6.3273, Train Steps/Sec: 1.12 +[2025-04-28 12:43:14] (step=0007250) Train Loss: 6.2165, Train Steps/Sec: 1.12 +[2025-04-28 12:43:36] (step=0007275) Train Loss: 6.3048, Train Steps/Sec: 1.12 +[2025-04-28 12:43:58] (step=0007300) Train Loss: 6.3032, Train Steps/Sec: 1.12 +[2025-04-28 12:44:21] (step=0007325) Train Loss: 6.2981, Train Steps/Sec: 1.12 +[2025-04-28 12:44:43] (step=0007350) Train Loss: 6.3170, Train Steps/Sec: 1.12 +[2025-04-28 12:45:05] (step=0007375) Train Loss: 6.2532, Train Steps/Sec: 1.12 +[2025-04-28 12:45:28] (step=0007400) Train Loss: 6.3362, Train Steps/Sec: 1.11 +[2025-04-28 12:45:50] (step=0007425) Train Loss: 6.2670, Train Steps/Sec: 1.12 +[2025-04-28 12:46:13] (step=0007450) Train Loss: 6.2893, Train Steps/Sec: 1.12 +[2025-04-28 12:46:35] (step=0007475) Train Loss: 6.2692, Train Steps/Sec: 1.12 +[2025-04-28 12:46:57] (step=0007500) Train Loss: 6.2634, Train Steps/Sec: 1.12 +[2025-04-28 12:47:20] (step=0007525) Train Loss: 6.2611, Train Steps/Sec: 1.12 +[2025-04-28 12:47:42] (step=0007550) Train Loss: 6.2244, Train Steps/Sec: 1.12 +[2025-04-28 12:48:04] (step=0007575) Train Loss: 6.2689, Train Steps/Sec: 1.12 +[2025-04-28 12:48:27] (step=0007600) Train Loss: 6.3749, Train Steps/Sec: 1.11 +[2025-04-28 12:48:49] (step=0007625) Train Loss: 6.3153, Train Steps/Sec: 1.12 +[2025-04-28 12:49:11] (step=0007650) Train Loss: 6.2967, Train Steps/Sec: 1.12 +[2025-04-28 12:49:34] (step=0007675) Train Loss: 6.2646, Train Steps/Sec: 1.12 +[2025-04-28 12:49:56] (step=0007700) Train Loss: 6.2918, Train Steps/Sec: 1.12 +[2025-04-28 12:50:18] (step=0007725) Train Loss: 6.2810, Train Steps/Sec: 1.12 +[2025-04-28 12:50:41] (step=0007750) Train Loss: 6.2429, Train Steps/Sec: 1.12 +[2025-04-28 12:51:03] (step=0007775) Train Loss: 6.2446, Train Steps/Sec: 1.12 +[2025-04-28 12:51:26] (step=0007800) Train Loss: 6.2847, Train Steps/Sec: 1.11 +[2025-04-28 12:51:48] (step=0007825) Train Loss: 6.2613, Train Steps/Sec: 1.12 +[2025-04-28 12:52:10] (step=0007850) Train Loss: 6.3335, Train Steps/Sec: 1.12 +[2025-04-28 12:52:33] (step=0007875) Train Loss: 6.2170, Train Steps/Sec: 1.12 +[2025-04-28 12:52:55] (step=0007900) Train Loss: 6.2997, Train Steps/Sec: 1.12 +[2025-04-28 12:53:17] (step=0007925) Train Loss: 6.2252, Train Steps/Sec: 1.12 +[2025-04-28 12:53:40] (step=0007950) Train Loss: 6.3446, Train Steps/Sec: 1.12 +[2025-04-28 12:54:02] (step=0007975) Train Loss: 6.2191, Train Steps/Sec: 1.12 +[2025-04-28 12:54:25] (step=0008000) Train Loss: 6.2446, Train Steps/Sec: 1.11 +[2025-04-28 12:54:25] text_config is None. Initializing the text config with default values (`OPTConfig`). +[2025-04-28 13:01:45] Finish Eval in 8000 steps... +[2025-04-28 13:02:03] Saved checkpoint to checkpoint/X2I_UltraEdiit_fourmask_LLava_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_filtered_objects_4mask_1280_context_after_stage1_5e4/000-GPT-XL/checkpoints/0008000.pt +[2025-04-28 13:02:05] Removed old checkpoint: checkpoint/X2I_UltraEdiit_fourmask_LLava_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_filtered_objects_4mask_1280_context_after_stage1_5e4/000-GPT-XL/checkpoints/0006000.pt +[2025-04-28 13:02:29] (step=0008025) Train Loss: 6.3299, Train Steps/Sec: 0.05 +[2025-04-28 13:02:52] (step=0008050) Train Loss: 6.3708, Train Steps/Sec: 1.12 +[2025-04-28 13:03:14] (step=0008075) Train Loss: 6.2980, Train Steps/Sec: 1.12 +[2025-04-28 13:03:36] (step=0008100) Train Loss: 6.2168, Train Steps/Sec: 1.12 +[2025-04-28 13:03:59] (step=0008125) Train Loss: 6.2687, Train Steps/Sec: 1.12 +[2025-04-28 13:04:21] (step=0008150) Train Loss: 6.2848, Train Steps/Sec: 1.11 +[2025-04-28 13:04:43] (step=0008175) Train Loss: 6.3156, Train Steps/Sec: 1.12 +[2025-04-28 13:05:06] (step=0008200) Train Loss: 6.2778, Train Steps/Sec: 1.12 +[2025-04-28 13:05:28] (step=0008225) Train Loss: 6.2969, Train Steps/Sec: 1.12 +[2025-04-28 13:05:51] (step=0008250) Train Loss: 6.2563, Train Steps/Sec: 1.12 +[2025-04-28 13:06:13] (step=0008275) Train Loss: 6.2420, Train Steps/Sec: 1.12 +[2025-04-28 13:06:35] (step=0008300) Train Loss: 6.2647, Train Steps/Sec: 1.12 +[2025-04-28 13:06:58] (step=0008325) Train Loss: 6.2270, Train Steps/Sec: 1.12 +[2025-04-28 13:07:20] (step=0008350) Train Loss: 6.2998, Train Steps/Sec: 1.12 +[2025-04-28 13:07:42] (step=0008375) Train Loss: 6.2033, Train Steps/Sec: 1.12 +[2025-04-28 13:08:05] (step=0008400) Train Loss: 6.2364, Train Steps/Sec: 1.11 +[2025-04-28 13:08:27] (step=0008425) Train Loss: 6.2595, Train Steps/Sec: 1.12 +[2025-04-28 13:08:50] (step=0008450) Train Loss: 6.2640, Train Steps/Sec: 1.12 +[2025-04-28 13:09:12] (step=0008475) Train Loss: 6.2303, Train Steps/Sec: 1.12 +[2025-04-28 13:10:15] (step=0008500) Train Loss: 6.2357, Train Steps/Sec: 0.40 +[2025-04-28 13:10:56] (step=0008525) Train Loss: 6.2744, Train Steps/Sec: 0.61 +[2025-04-28 13:11:27] (step=0008550) Train Loss: 6.3118, Train Steps/Sec: 0.79 +[2025-04-28 13:11:50] (step=0008575) Train Loss: 6.2234, Train Steps/Sec: 1.11 +[2025-04-28 13:12:12] (step=0008600) Train Loss: 6.2050, Train Steps/Sec: 1.11 +[2025-04-28 13:12:35] (step=0008625) Train Loss: 6.3078, Train Steps/Sec: 1.12 +[2025-04-28 13:12:57] (step=0008650) Train Loss: 6.2587, Train Steps/Sec: 1.12 +[2025-04-28 13:13:19] (step=0008675) Train Loss: 6.2271, Train Steps/Sec: 1.12 +[2025-04-28 13:13:42] (step=0008700) Train Loss: 6.2190, Train Steps/Sec: 1.12 +[2025-04-28 13:14:04] (step=0008725) Train Loss: 6.2314, Train Steps/Sec: 1.12 +[2025-04-28 13:14:26] (step=0008750) Train Loss: 6.2384, Train Steps/Sec: 1.12 +[2025-04-28 13:14:49] (step=0008775) Train Loss: 6.2243, Train Steps/Sec: 1.12 +[2025-04-28 13:15:11] (step=0008800) Train Loss: 6.1961, Train Steps/Sec: 1.11 +[2025-04-28 13:15:34] (step=0008825) Train Loss: 6.2154, Train Steps/Sec: 1.12 +[2025-04-28 13:15:56] (step=0008850) Train Loss: 6.1745, Train Steps/Sec: 1.12 +[2025-04-28 13:16:18] (step=0008875) Train Loss: 6.1864, Train Steps/Sec: 1.12 +[2025-04-28 13:16:41] (step=0008900) Train Loss: 6.1780, Train Steps/Sec: 1.12 +[2025-04-28 13:17:03] (step=0008925) Train Loss: 6.1923, Train Steps/Sec: 1.12 +[2025-04-28 13:17:25] (step=0008950) Train Loss: 6.2190, Train Steps/Sec: 1.12 +[2025-04-28 13:17:48] (step=0008975) Train Loss: 6.2236, Train Steps/Sec: 1.12 +[2025-04-28 13:18:10] (step=0009000) Train Loss: 6.1870, Train Steps/Sec: 1.11 +[2025-04-28 13:18:33] (step=0009025) Train Loss: 6.2279, Train Steps/Sec: 1.12 +[2025-04-28 13:18:55] (step=0009050) Train Loss: 6.1987, Train Steps/Sec: 1.12 +[2025-04-28 13:19:17] (step=0009075) Train Loss: 6.2127, Train Steps/Sec: 1.12 +[2025-04-28 13:19:40] (step=0009100) Train Loss: 6.2156, Train Steps/Sec: 1.12 +[2025-04-28 13:20:02] (step=0009125) Train Loss: 6.2214, Train Steps/Sec: 1.12 +[2025-04-28 13:20:24] (step=0009150) Train Loss: 6.1848, Train Steps/Sec: 1.12 +[2025-04-28 13:20:47] (step=0009175) Train Loss: 6.2138, Train Steps/Sec: 1.12 +[2025-04-28 13:21:09] (step=0009200) Train Loss: 6.2082, Train Steps/Sec: 1.11 +[2025-04-28 13:21:31] (step=0009225) Train Loss: 6.1742, Train Steps/Sec: 1.12 +[2025-04-28 13:21:54] (step=0009250) Train Loss: 6.1494, Train Steps/Sec: 1.12 +[2025-04-28 13:22:16] (step=0009275) Train Loss: 6.1587, Train Steps/Sec: 1.12 +[2025-04-28 13:22:38] (step=0009300) Train Loss: 6.2256, Train Steps/Sec: 1.12 +[2025-04-28 13:23:01] (step=0009325) Train Loss: 6.2427, Train Steps/Sec: 1.12 +[2025-04-28 13:23:23] (step=0009350) Train Loss: 6.1845, Train Steps/Sec: 1.12 +[2025-04-28 13:23:45] (step=0009375) Train Loss: 6.2201, Train Steps/Sec: 1.12 +[2025-04-28 13:24:08] (step=0009400) Train Loss: 6.2459, Train Steps/Sec: 1.11 +[2025-04-28 13:24:30] (step=0009425) Train Loss: 6.2020, Train Steps/Sec: 1.12 +[2025-04-28 13:24:53] (step=0009450) Train Loss: 6.1587, Train Steps/Sec: 1.12 +[2025-04-28 13:25:15] (step=0009475) Train Loss: 6.1994, Train Steps/Sec: 1.11 +[2025-04-28 13:25:37] (step=0009500) Train Loss: 6.1808, Train Steps/Sec: 1.12 +[2025-04-28 13:26:00] (step=0009525) Train Loss: 6.2306, Train Steps/Sec: 1.12 +[2025-04-28 13:26:22] (step=0009550) Train Loss: 6.2154, Train Steps/Sec: 1.12 +[2025-04-28 13:26:44] (step=0009575) Train Loss: 6.1647, Train Steps/Sec: 1.12 +[2025-04-28 13:27:07] (step=0009600) Train Loss: 6.1963, Train Steps/Sec: 1.11 +[2025-04-28 13:27:29] (step=0009625) Train Loss: 6.1774, Train Steps/Sec: 1.12 +[2025-04-28 13:27:52] (step=0009650) Train Loss: 6.2172, Train Steps/Sec: 1.12 +[2025-04-28 13:28:14] (step=0009675) Train Loss: 6.2165, Train Steps/Sec: 1.12 +[2025-04-28 13:28:36] (step=0009700) Train Loss: 6.1574, Train Steps/Sec: 1.12 +[2025-04-28 13:28:59] (step=0009725) Train Loss: 6.1364, Train Steps/Sec: 1.12 +[2025-04-28 13:29:21] (step=0009750) Train Loss: 6.1662, Train Steps/Sec: 1.12 +[2025-04-28 13:29:43] (step=0009775) Train Loss: 6.0988, Train Steps/Sec: 1.12 +[2025-04-28 13:30:06] (step=0009800) Train Loss: 6.1786, Train Steps/Sec: 1.11 +[2025-04-28 13:30:28] (step=0009825) Train Loss: 6.2315, Train Steps/Sec: 1.12 +[2025-04-28 13:30:50] (step=0009850) Train Loss: 6.1760, Train Steps/Sec: 1.12 +[2025-04-28 13:31:13] (step=0009875) Train Loss: 6.2055, Train Steps/Sec: 1.12 +[2025-04-28 13:31:35] (step=0009900) Train Loss: 6.1653, Train Steps/Sec: 1.12 +[2025-04-28 13:31:57] (step=0009925) Train Loss: 6.1622, Train Steps/Sec: 1.12 +[2025-04-28 13:32:20] (step=0009950) Train Loss: 6.2154, Train Steps/Sec: 1.12 +[2025-04-28 13:32:42] (step=0009975) Train Loss: 6.1415, Train Steps/Sec: 1.12 +[2025-04-28 13:33:05] (step=0010000) Train Loss: 6.1778, Train Steps/Sec: 1.11 +[2025-04-28 13:33:05] text_config is None. Initializing the text config with default values (`OPTConfig`). +[2025-04-28 13:40:23] Finish Eval in 10000 steps... +[2025-04-28 13:40:43] Saved checkpoint to checkpoint/X2I_UltraEdiit_fourmask_LLava_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_filtered_objects_4mask_1280_context_after_stage1_5e4/000-GPT-XL/checkpoints/0010000.pt +[2025-04-28 13:40:45] Removed old checkpoint: checkpoint/X2I_UltraEdiit_fourmask_LLava_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_filtered_objects_4mask_1280_context_after_stage1_5e4/000-GPT-XL/checkpoints/0008000.pt +[2025-04-28 13:41:08] (step=0010025) Train Loss: 6.2247, Train Steps/Sec: 0.05 +[2025-04-28 13:41:30] (step=0010050) Train Loss: 6.1984, Train Steps/Sec: 1.11 +[2025-04-28 13:41:53] (step=0010075) Train Loss: 6.1763, Train Steps/Sec: 1.12 +[2025-04-28 13:42:15] (step=0010100) Train Loss: 6.1886, Train Steps/Sec: 1.12 +[2025-04-28 13:42:37] (step=0010125) Train Loss: 6.1658, Train Steps/Sec: 1.12 +[2025-04-28 13:43:00] (step=0010150) Train Loss: 6.1557, Train Steps/Sec: 1.12 +[2025-04-28 13:43:22] (step=0010175) Train Loss: 6.1529, Train Steps/Sec: 1.12 +[2025-04-28 13:43:44] (step=0010200) Train Loss: 6.1729, Train Steps/Sec: 1.11 +[2025-04-28 13:44:07] (step=0010225) Train Loss: 6.1500, Train Steps/Sec: 1.12 +[2025-04-28 13:44:29] (step=0010250) Train Loss: 6.1658, Train Steps/Sec: 1.12 +[2025-04-28 13:44:51] (step=0010275) Train Loss: 6.1547, Train Steps/Sec: 1.12 +[2025-04-28 13:45:14] (step=0010300) Train Loss: 6.1202, Train Steps/Sec: 1.12 +[2025-04-28 13:45:36] (step=0010325) Train Loss: 6.1266, Train Steps/Sec: 1.12 +[2025-04-28 13:45:58] (step=0010350) Train Loss: 6.1741, Train Steps/Sec: 1.12 +[2025-04-28 13:46:21] (step=0010375) Train Loss: 6.2101, Train Steps/Sec: 1.12 +[2025-04-28 13:46:43] (step=0010400) Train Loss: 6.2194, Train Steps/Sec: 1.11 +[2025-04-28 13:47:06] (step=0010425) Train Loss: 6.2016, Train Steps/Sec: 1.12 +[2025-04-28 13:47:28] (step=0010450) Train Loss: 6.1883, Train Steps/Sec: 1.12 +[2025-04-28 13:47:50] (step=0010475) Train Loss: 6.1930, Train Steps/Sec: 1.12 +[2025-04-28 13:48:13] (step=0010500) Train Loss: 6.1420, Train Steps/Sec: 1.12 +[2025-04-28 13:48:35] (step=0010525) Train Loss: 6.1227, Train Steps/Sec: 1.12 +[2025-04-28 13:48:57] (step=0010550) Train Loss: 6.1811, Train Steps/Sec: 1.12 +[2025-04-28 13:49:20] (step=0010575) Train Loss: 6.1873, Train Steps/Sec: 1.12 +[2025-04-28 13:49:42] (step=0010600) Train Loss: 6.1637, Train Steps/Sec: 1.11 +[2025-04-28 13:50:05] (step=0010625) Train Loss: 6.1720, Train Steps/Sec: 1.12 +[2025-04-28 13:50:27] (step=0010650) Train Loss: 6.1231, Train Steps/Sec: 1.12 +[2025-04-28 13:50:49] (step=0010675) Train Loss: 6.1595, Train Steps/Sec: 1.12 +[2025-04-28 13:51:12] (step=0010700) Train Loss: 6.1113, Train Steps/Sec: 1.12 +[2025-04-28 13:51:34] (step=0010725) Train Loss: 6.1167, Train Steps/Sec: 1.12 +[2025-04-28 13:51:56] (step=0010750) Train Loss: 6.1147, Train Steps/Sec: 1.12 +[2025-04-28 13:52:19] (step=0010775) Train Loss: 6.1617, Train Steps/Sec: 1.12 +[2025-04-28 13:52:41] (step=0010800) Train Loss: 6.1974, Train Steps/Sec: 1.12 +[2025-04-28 13:53:03] (step=0010825) Train Loss: 6.1710, Train Steps/Sec: 1.12 +[2025-04-28 13:53:26] (step=0010850) Train Loss: 6.1261, Train Steps/Sec: 1.12 +[2025-04-28 13:53:48] (step=0010875) Train Loss: 6.1458, Train Steps/Sec: 1.12 +[2025-04-28 13:54:11] (step=0010900) Train Loss: 6.1283, Train Steps/Sec: 1.12 +[2025-04-28 13:54:33] (step=0010925) Train Loss: 6.0329, Train Steps/Sec: 1.12 +[2025-04-28 13:54:55] (step=0010950) Train Loss: 6.1676, Train Steps/Sec: 1.12 +[2025-04-28 13:55:18] (step=0010975) Train Loss: 6.1697, Train Steps/Sec: 1.12 +[2025-04-28 13:55:40] (step=0011000) Train Loss: 6.0695, Train Steps/Sec: 1.12 +[2025-04-28 13:56:02] (step=0011025) Train Loss: 6.1667, Train Steps/Sec: 1.12 +[2025-04-28 13:56:25] (step=0011050) Train Loss: 6.2007, Train Steps/Sec: 1.12 +[2025-04-28 13:56:47] (step=0011075) Train Loss: 6.1430, Train Steps/Sec: 1.12 +[2025-04-28 13:57:09] (step=0011100) Train Loss: 6.1191, Train Steps/Sec: 1.12 +[2025-04-28 13:57:32] (step=0011125) Train Loss: 6.1769, Train Steps/Sec: 1.12 +[2025-04-28 13:57:54] (step=0011150) Train Loss: 6.1164, Train Steps/Sec: 1.12 +[2025-04-28 13:58:16] (step=0011175) Train Loss: 6.1840, Train Steps/Sec: 1.12 +[2025-04-28 13:58:39] (step=0011200) Train Loss: 6.0763, Train Steps/Sec: 1.11 +[2025-04-28 13:59:01] (step=0011225) Train Loss: 6.1249, Train Steps/Sec: 1.12 +[2025-04-28 13:59:23] (step=0011250) Train Loss: 6.1757, Train Steps/Sec: 1.12 +[2025-04-28 13:59:46] (step=0011275) Train Loss: 6.0938, Train Steps/Sec: 1.12 +[2025-04-28 14:00:08] (step=0011300) Train Loss: 6.1085, Train Steps/Sec: 1.12 +[2025-04-28 14:00:30] (step=0011325) Train Loss: 6.1990, Train Steps/Sec: 1.12 +[2025-04-28 14:00:53] (step=0011350) Train Loss: 6.1306, Train Steps/Sec: 1.12 +[2025-04-28 14:01:15] (step=0011375) Train Loss: 6.1198, Train Steps/Sec: 1.12 +[2025-04-28 14:01:38] (step=0011400) Train Loss: 6.0427, Train Steps/Sec: 1.12 +[2025-04-28 14:02:00] (step=0011425) Train Loss: 6.0730, Train Steps/Sec: 1.12 +[2025-04-28 14:02:22] (step=0011450) Train Loss: 6.0705, Train Steps/Sec: 1.12 +[2025-04-28 14:02:45] (step=0011475) Train Loss: 6.1082, Train Steps/Sec: 1.12 +[2025-04-28 14:03:07] (step=0011500) Train Loss: 6.0998, Train Steps/Sec: 1.12 +[2025-04-28 14:03:29] (step=0011525) Train Loss: 6.0717, Train Steps/Sec: 1.12 +[2025-04-28 14:03:52] (step=0011550) Train Loss: 6.0723, Train Steps/Sec: 1.12 +[2025-04-28 14:04:14] (step=0011575) Train Loss: 6.0871, Train Steps/Sec: 1.12 +[2025-04-28 14:04:36] (step=0011600) Train Loss: 6.0766, Train Steps/Sec: 1.12 +[2025-04-28 14:04:59] (step=0011625) Train Loss: 6.0883, Train Steps/Sec: 1.12 +[2025-04-28 14:05:21] (step=0011650) Train Loss: 6.0654, Train Steps/Sec: 1.12 +[2025-04-28 14:05:44] (step=0011675) Train Loss: 6.0538, Train Steps/Sec: 1.12 +[2025-04-28 14:06:06] (step=0011700) Train Loss: 6.1222, Train Steps/Sec: 1.12 +[2025-04-28 14:06:28] (step=0011725) Train Loss: 6.0667, Train Steps/Sec: 1.12 +[2025-04-28 14:06:51] (step=0011750) Train Loss: 6.1275, Train Steps/Sec: 1.12 +[2025-04-28 14:07:13] (step=0011775) Train Loss: 6.1009, Train Steps/Sec: 1.12 +[2025-04-28 14:07:35] (step=0011800) Train Loss: 6.1053, Train Steps/Sec: 1.11 +[2025-04-28 14:07:58] (step=0011825) Train Loss: 6.1412, Train Steps/Sec: 1.12 +[2025-04-28 14:08:20] (step=0011850) Train Loss: 6.0658, Train Steps/Sec: 1.12 +[2025-04-28 14:08:42] (step=0011875) Train Loss: 6.0757, Train Steps/Sec: 1.12 +[2025-04-28 14:09:05] (step=0011900) Train Loss: 6.1077, Train Steps/Sec: 1.12 +[2025-04-28 14:09:27] (step=0011925) Train Loss: 6.0672, Train Steps/Sec: 1.12 +[2025-04-28 14:09:49] (step=0011950) Train Loss: 6.1049, Train Steps/Sec: 1.12 +[2025-04-28 14:10:12] (step=0011975) Train Loss: 6.0695, Train Steps/Sec: 1.12 +[2025-04-28 14:10:34] (step=0012000) Train Loss: 6.0708, Train Steps/Sec: 1.11 +[2025-04-28 14:10:34] text_config is None. Initializing the text config with default values (`OPTConfig`). +[2025-04-28 14:17:53] Finish Eval in 12000 steps... +[2025-04-28 14:18:13] Saved checkpoint to checkpoint/X2I_UltraEdiit_fourmask_LLava_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_filtered_objects_4mask_1280_context_after_stage1_5e4/000-GPT-XL/checkpoints/0012000.pt +[2025-04-28 14:18:14] Removed old checkpoint: checkpoint/X2I_UltraEdiit_fourmask_LLava_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_filtered_objects_4mask_1280_context_after_stage1_5e4/000-GPT-XL/checkpoints/0010000.pt +[2025-04-28 14:18:37] (step=0012025) Train Loss: 6.1536, Train Steps/Sec: 0.05 +[2025-04-28 14:19:27] (step=0012050) Train Loss: 6.0633, Train Steps/Sec: 0.50 +[2025-04-28 14:20:09] (step=0012075) Train Loss: 6.1331, Train Steps/Sec: 0.60 +[2025-04-28 14:20:41] (step=0012100) Train Loss: 6.0836, Train Steps/Sec: 0.79 +[2025-04-28 14:21:12] (step=0012125) Train Loss: 6.1123, Train Steps/Sec: 0.79 +[2025-04-28 14:21:35] (step=0012150) Train Loss: 6.0963, Train Steps/Sec: 1.12 +[2025-04-28 14:21:57] (step=0012175) Train Loss: 6.0990, Train Steps/Sec: 1.12 +[2025-04-28 14:22:19] (step=0012200) Train Loss: 6.0688, Train Steps/Sec: 1.12 +[2025-04-28 14:22:42] (step=0012225) Train Loss: 6.0953, Train Steps/Sec: 1.12 +[2025-04-28 14:23:04] (step=0012250) Train Loss: 6.0852, Train Steps/Sec: 1.12 +[2025-04-28 14:23:26] (step=0012275) Train Loss: 6.1081, Train Steps/Sec: 1.12 +[2025-04-28 14:23:49] (step=0012300) Train Loss: 6.0392, Train Steps/Sec: 1.12 +[2025-04-28 14:24:11] (step=0012325) Train Loss: 6.1210, Train Steps/Sec: 1.12 +[2025-04-28 14:24:33] (step=0012350) Train Loss: 6.0219, Train Steps/Sec: 1.12 +[2025-04-28 14:24:56] (step=0012375) Train Loss: 6.0390, Train Steps/Sec: 1.12 +[2025-04-28 14:25:18] (step=0012400) Train Loss: 6.0404, Train Steps/Sec: 1.11 +[2025-04-28 14:25:41] (step=0012425) Train Loss: 6.0278, Train Steps/Sec: 1.12 +[2025-04-28 14:26:03] (step=0012450) Train Loss: 6.1133, Train Steps/Sec: 1.12 +[2025-04-28 14:26:25] (step=0012475) Train Loss: 6.1029, Train Steps/Sec: 1.12 +[2025-04-28 14:26:48] (step=0012500) Train Loss: 6.1089, Train Steps/Sec: 1.12 +[2025-04-28 14:27:10] (step=0012525) Train Loss: 6.1052, Train Steps/Sec: 1.12 +[2025-04-28 14:27:32] (step=0012550) Train Loss: 6.1075, Train Steps/Sec: 1.12 +[2025-04-28 14:27:55] (step=0012575) Train Loss: 6.0938, Train Steps/Sec: 1.12 +[2025-04-28 14:28:17] (step=0012600) Train Loss: 6.0822, Train Steps/Sec: 1.11 +[2025-04-28 14:28:40] (step=0012625) Train Loss: 6.1434, Train Steps/Sec: 1.12 +[2025-04-28 14:29:02] (step=0012650) Train Loss: 6.0677, Train Steps/Sec: 1.12 +[2025-04-28 14:29:24] (step=0012675) Train Loss: 6.1159, Train Steps/Sec: 1.12 +[2025-04-28 14:29:47] (step=0012700) Train Loss: 5.9995, Train Steps/Sec: 1.12 +[2025-04-28 14:30:09] (step=0012725) Train Loss: 6.0708, Train Steps/Sec: 1.12 +[2025-04-28 14:30:31] (step=0012750) Train Loss: 6.0237, Train Steps/Sec: 1.12 +[2025-04-28 14:30:54] (step=0012775) Train Loss: 6.0792, Train Steps/Sec: 1.12 +[2025-04-28 14:31:16] (step=0012800) Train Loss: 6.0647, Train Steps/Sec: 1.11 +[2025-04-28 14:31:38] (step=0012825) Train Loss: 6.0566, Train Steps/Sec: 1.12 +[2025-04-28 14:32:01] (step=0012850) Train Loss: 6.0930, Train Steps/Sec: 1.12 +[2025-04-28 14:32:23] (step=0012875) Train Loss: 6.0327, Train Steps/Sec: 1.12 +[2025-04-28 14:32:45] (step=0012900) Train Loss: 6.0725, Train Steps/Sec: 1.12 +[2025-04-28 14:33:08] (step=0012925) Train Loss: 6.0380, Train Steps/Sec: 1.12 +[2025-04-28 14:33:30] (step=0012950) Train Loss: 6.0675, Train Steps/Sec: 1.12 +[2025-04-28 14:33:52] (step=0012975) Train Loss: 6.0337, Train Steps/Sec: 1.12 +[2025-04-28 14:34:15] (step=0013000) Train Loss: 6.0589, Train Steps/Sec: 1.11 +[2025-04-28 14:34:37] (step=0013025) Train Loss: 6.0369, Train Steps/Sec: 1.12 +[2025-04-28 14:35:00] (step=0013050) Train Loss: 5.9883, Train Steps/Sec: 1.12 +[2025-04-28 14:35:22] (step=0013075) Train Loss: 6.1005, Train Steps/Sec: 1.12 +[2025-04-28 14:35:44] (step=0013100) Train Loss: 5.9906, Train Steps/Sec: 1.12 +[2025-04-28 14:36:07] (step=0013125) Train Loss: 6.0224, Train Steps/Sec: 1.12 +[2025-04-28 14:36:29] (step=0013150) Train Loss: 6.0482, Train Steps/Sec: 1.12 +[2025-04-28 14:36:51] (step=0013175) Train Loss: 6.0040, Train Steps/Sec: 1.12 +[2025-04-28 14:37:14] (step=0013200) Train Loss: 6.0621, Train Steps/Sec: 1.12 +[2025-04-28 14:37:36] (step=0013225) Train Loss: 6.0146, Train Steps/Sec: 1.12 +[2025-04-28 14:37:58] (step=0013250) Train Loss: 6.0281, Train Steps/Sec: 1.12 +[2025-04-28 14:38:21] (step=0013275) Train Loss: 6.0044, Train Steps/Sec: 1.12 +[2025-04-28 14:38:43] (step=0013300) Train Loss: 6.0963, Train Steps/Sec: 1.12 +[2025-04-28 14:39:05] (step=0013325) Train Loss: 6.0286, Train Steps/Sec: 1.12 +[2025-04-28 14:39:28] (step=0013350) Train Loss: 6.0859, Train Steps/Sec: 1.12 +[2025-04-28 14:39:50] (step=0013375) Train Loss: 6.0255, Train Steps/Sec: 1.12 +[2025-04-28 14:40:12] (step=0013400) Train Loss: 6.0236, Train Steps/Sec: 1.12 +[2025-04-28 14:40:35] (step=0013425) Train Loss: 6.0473, Train Steps/Sec: 1.12 +[2025-04-28 14:40:57] (step=0013450) Train Loss: 6.0152, Train Steps/Sec: 1.12 +[2025-04-28 14:41:19] (step=0013475) Train Loss: 6.0877, Train Steps/Sec: 1.12 +[2025-04-28 14:41:42] (step=0013500) Train Loss: 6.0035, Train Steps/Sec: 1.12 +[2025-04-28 14:42:04] (step=0013525) Train Loss: 6.0206, Train Steps/Sec: 1.12 +[2025-04-28 14:42:26] (step=0013550) Train Loss: 6.0226, Train Steps/Sec: 1.12 +[2025-04-28 14:42:49] (step=0013575) Train Loss: 5.9984, Train Steps/Sec: 1.11 +[2025-04-28 14:43:12] (step=0013600) Train Loss: 5.9311, Train Steps/Sec: 1.11 +[2025-04-28 14:43:34] (step=0013625) Train Loss: 6.0040, Train Steps/Sec: 1.12 +[2025-04-28 14:43:56] (step=0013650) Train Loss: 6.0289, Train Steps/Sec: 1.12 +[2025-04-28 14:44:19] (step=0013675) Train Loss: 6.0451, Train Steps/Sec: 1.12 +[2025-04-28 14:44:41] (step=0013700) Train Loss: 6.0177, Train Steps/Sec: 1.12 +[2025-04-28 14:45:03] (step=0013725) Train Loss: 6.0121, Train Steps/Sec: 1.12 +[2025-04-28 14:45:26] (step=0013750) Train Loss: 6.0214, Train Steps/Sec: 1.12 +[2025-04-28 14:45:48] (step=0013775) Train Loss: 6.0418, Train Steps/Sec: 1.12 +[2025-04-28 14:46:10] (step=0013800) Train Loss: 6.0957, Train Steps/Sec: 1.11 +[2025-04-28 14:46:33] (step=0013825) Train Loss: 6.0199, Train Steps/Sec: 1.12 +[2025-04-28 14:46:55] (step=0013850) Train Loss: 6.0093, Train Steps/Sec: 1.12 +[2025-04-28 14:47:17] (step=0013875) Train Loss: 6.0680, Train Steps/Sec: 1.12 +[2025-04-28 14:47:40] (step=0013900) Train Loss: 5.9879, Train Steps/Sec: 1.12 +[2025-04-28 14:48:02] (step=0013925) Train Loss: 6.0420, Train Steps/Sec: 1.12 +[2025-04-28 14:48:24] (step=0013950) Train Loss: 6.0141, Train Steps/Sec: 1.12 +[2025-04-28 14:48:47] (step=0013975) Train Loss: 6.0472, Train Steps/Sec: 1.12 +[2025-04-28 14:49:09] (step=0014000) Train Loss: 6.0177, Train Steps/Sec: 1.11 +[2025-04-28 14:49:09] text_config is None. Initializing the text config with default values (`OPTConfig`). +[2025-04-28 14:56:27] Finish Eval in 14000 steps... +[2025-04-28 14:56:48] Saved checkpoint to checkpoint/X2I_UltraEdiit_fourmask_LLava_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_filtered_objects_4mask_1280_context_after_stage1_5e4/000-GPT-XL/checkpoints/0014000.pt +[2025-04-28 14:56:50] Removed old checkpoint: checkpoint/X2I_UltraEdiit_fourmask_LLava_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_filtered_objects_4mask_1280_context_after_stage1_5e4/000-GPT-XL/checkpoints/0012000.pt +[2025-04-28 14:57:12] (step=0014025) Train Loss: 6.0807, Train Steps/Sec: 0.05 +[2025-04-28 14:57:35] (step=0014050) Train Loss: 6.0478, Train Steps/Sec: 1.12 +[2025-04-28 14:57:57] (step=0014075) Train Loss: 5.9515, Train Steps/Sec: 1.12 +[2025-04-28 14:58:19] (step=0014100) Train Loss: 5.9942, Train Steps/Sec: 1.12 +[2025-04-28 14:58:42] (step=0014125) Train Loss: 6.0392, Train Steps/Sec: 1.12 +[2025-04-28 14:59:04] (step=0014150) Train Loss: 5.9694, Train Steps/Sec: 1.12 +[2025-04-28 14:59:26] (step=0014175) Train Loss: 6.0606, Train Steps/Sec: 1.12 +[2025-04-28 14:59:49] (step=0014200) Train Loss: 6.0793, Train Steps/Sec: 1.12 +[2025-04-28 15:00:11] (step=0014225) Train Loss: 6.0607, Train Steps/Sec: 1.12 +[2025-04-28 15:00:33] (step=0014250) Train Loss: 5.9762, Train Steps/Sec: 1.12 +[2025-04-28 15:00:55] (step=0014275) Train Loss: 6.0151, Train Steps/Sec: 1.12 +[2025-04-28 15:01:18] (step=0014300) Train Loss: 5.9556, Train Steps/Sec: 1.12 +[2025-04-28 15:01:40] (step=0014325) Train Loss: 6.0588, Train Steps/Sec: 1.12 +[2025-04-28 15:02:02] (step=0014350) Train Loss: 6.0310, Train Steps/Sec: 1.12 +[2025-04-28 15:02:25] (step=0014375) Train Loss: 6.0249, Train Steps/Sec: 1.12 +[2025-04-28 15:02:47] (step=0014400) Train Loss: 5.9545, Train Steps/Sec: 1.12 +[2025-04-28 15:03:09] (step=0014425) Train Loss: 5.9751, Train Steps/Sec: 1.12 +[2025-04-28 15:03:32] (step=0014450) Train Loss: 6.0320, Train Steps/Sec: 1.12 +[2025-04-28 15:03:54] (step=0014475) Train Loss: 6.0591, Train Steps/Sec: 1.11 +[2025-04-28 15:04:17] (step=0014500) Train Loss: 5.9171, Train Steps/Sec: 1.12 +[2025-04-28 15:04:39] (step=0014525) Train Loss: 6.0492, Train Steps/Sec: 1.12 +[2025-04-28 15:05:01] (step=0014550) Train Loss: 6.0193, Train Steps/Sec: 1.12 +[2025-04-28 15:05:23] (step=0014575) Train Loss: 5.9699, Train Steps/Sec: 1.12 +[2025-04-28 15:05:46] (step=0014600) Train Loss: 5.9690, Train Steps/Sec: 1.12 +[2025-04-28 15:06:08] (step=0014625) Train Loss: 6.0200, Train Steps/Sec: 1.12 +[2025-04-28 15:06:31] (step=0014650) Train Loss: 6.0558, Train Steps/Sec: 1.12 +[2025-04-28 15:06:53] (step=0014675) Train Loss: 6.0372, Train Steps/Sec: 1.12 +[2025-04-28 15:07:15] (step=0014700) Train Loss: 5.9751, Train Steps/Sec: 1.12 +[2025-04-28 15:07:37] (step=0014725) Train Loss: 5.9759, Train Steps/Sec: 1.12 +[2025-04-28 15:08:00] (step=0014750) Train Loss: 5.9466, Train Steps/Sec: 1.12 +[2025-04-28 15:08:22] (step=0014775) Train Loss: 6.0743, Train Steps/Sec: 1.12 +[2025-04-28 15:08:45] (step=0014800) Train Loss: 5.9694, Train Steps/Sec: 1.11 +[2025-04-28 15:09:07] (step=0014825) Train Loss: 5.9509, Train Steps/Sec: 1.12 +[2025-04-28 15:09:29] (step=0014850) Train Loss: 5.9954, Train Steps/Sec: 1.12 +[2025-04-28 15:09:52] (step=0014875) Train Loss: 6.0090, Train Steps/Sec: 1.12 +[2025-04-28 15:10:14] (step=0014900) Train Loss: 6.0149, Train Steps/Sec: 1.12 +[2025-04-28 15:10:36] (step=0014925) Train Loss: 5.9759, Train Steps/Sec: 1.12 +[2025-04-28 15:10:58] (step=0014950) Train Loss: 6.0020, Train Steps/Sec: 1.12 +[2025-04-28 15:11:21] (step=0014975) Train Loss: 5.9754, Train Steps/Sec: 1.12 +[2025-04-28 15:11:43] (step=0015000) Train Loss: 5.9494, Train Steps/Sec: 1.12 +[2025-04-28 15:12:06] (step=0015025) Train Loss: 5.9044, Train Steps/Sec: 1.12 +[2025-04-28 15:12:28] (step=0015050) Train Loss: 5.9769, Train Steps/Sec: 1.12 +[2025-04-28 15:12:50] (step=0015075) Train Loss: 6.0237, Train Steps/Sec: 1.12 +[2025-04-28 15:13:12] (step=0015100) Train Loss: 6.0295, Train Steps/Sec: 1.12 +[2025-04-28 15:13:35] (step=0015125) Train Loss: 6.0049, Train Steps/Sec: 1.12 +[2025-04-28 15:13:57] (step=0015150) Train Loss: 6.0243, Train Steps/Sec: 1.12 +[2025-04-28 15:14:19] (step=0015175) Train Loss: 5.9977, Train Steps/Sec: 1.12 +[2025-04-28 15:14:42] (step=0015200) Train Loss: 5.9976, Train Steps/Sec: 1.12 +[2025-04-28 15:15:04] (step=0015225) Train Loss: 5.9914, Train Steps/Sec: 1.12 +[2025-04-28 15:15:26] (step=0015250) Train Loss: 6.0081, Train Steps/Sec: 1.12 +[2025-04-28 15:15:49] (step=0015275) Train Loss: 5.9754, Train Steps/Sec: 1.12 +[2025-04-28 15:16:11] (step=0015300) Train Loss: 5.9914, Train Steps/Sec: 1.12 +[2025-04-28 15:16:33] (step=0015325) Train Loss: 6.0026, Train Steps/Sec: 1.12 +[2025-04-28 15:16:56] (step=0015350) Train Loss: 5.9776, Train Steps/Sec: 1.12 +[2025-04-28 15:17:18] (step=0015375) Train Loss: 6.0162, Train Steps/Sec: 1.12 +[2025-04-28 15:17:41] (step=0015400) Train Loss: 6.0034, Train Steps/Sec: 1.11 +[2025-04-28 15:18:03] (step=0015425) Train Loss: 5.9350, Train Steps/Sec: 1.12 +[2025-04-28 15:18:25] (step=0015450) Train Loss: 5.9396, Train Steps/Sec: 1.12 +[2025-04-28 15:18:47] (step=0015475) Train Loss: 5.9610, Train Steps/Sec: 1.12 +[2025-04-28 15:19:10] (step=0015500) Train Loss: 6.0061, Train Steps/Sec: 1.12 +[2025-04-28 15:19:32] (step=0015525) Train Loss: 5.9912, Train Steps/Sec: 1.12 +[2025-04-28 15:19:54] (step=0015550) Train Loss: 5.9695, Train Steps/Sec: 1.12 +[2025-04-28 15:20:17] (step=0015575) Train Loss: 5.9883, Train Steps/Sec: 1.12 +[2025-04-28 15:20:39] (step=0015600) Train Loss: 5.9789, Train Steps/Sec: 1.12 +[2025-04-28 15:21:49] (step=0015625) Train Loss: 5.9784, Train Steps/Sec: 0.36 +[2025-04-28 15:22:20] (step=0015650) Train Loss: 6.0302, Train Steps/Sec: 0.80 +[2025-04-28 15:22:51] (step=0015675) Train Loss: 5.9327, Train Steps/Sec: 0.79 +[2025-04-28 15:23:14] (step=0015700) Train Loss: 5.9652, Train Steps/Sec: 1.12 +[2025-04-28 15:23:36] (step=0015725) Train Loss: 5.9430, Train Steps/Sec: 1.12 +[2025-04-28 15:23:58] (step=0015750) Train Loss: 5.9395, Train Steps/Sec: 1.12 +[2025-04-28 15:24:21] (step=0015775) Train Loss: 5.9757, Train Steps/Sec: 1.12 +[2025-04-28 15:24:43] (step=0015800) Train Loss: 5.9704, Train Steps/Sec: 1.11 +[2025-04-28 15:25:05] (step=0015825) Train Loss: 5.8617, Train Steps/Sec: 1.12 +[2025-04-28 15:25:28] (step=0015850) Train Loss: 6.0001, Train Steps/Sec: 1.12 +[2025-04-28 15:25:50] (step=0015875) Train Loss: 5.9245, Train Steps/Sec: 1.12 +[2025-04-28 15:26:12] (step=0015900) Train Loss: 5.9441, Train Steps/Sec: 1.12 +[2025-04-28 15:26:35] (step=0015925) Train Loss: 5.9410, Train Steps/Sec: 1.12 +[2025-04-28 15:26:57] (step=0015950) Train Loss: 5.9531, Train Steps/Sec: 1.12 +[2025-04-28 15:27:19] (step=0015975) Train Loss: 5.9442, Train Steps/Sec: 1.12 +[2025-04-28 15:27:42] (step=0016000) Train Loss: 5.9726, Train Steps/Sec: 1.11 +[2025-04-28 15:27:42] text_config is None. Initializing the text config with default values (`OPTConfig`). +[2025-04-28 15:35:00] Finish Eval in 16000 steps... +[2025-04-28 15:35:20] Saved checkpoint to checkpoint/X2I_UltraEdiit_fourmask_LLava_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_filtered_objects_4mask_1280_context_after_stage1_5e4/000-GPT-XL/checkpoints/0016000.pt +[2025-04-28 15:35:22] Removed old checkpoint: checkpoint/X2I_UltraEdiit_fourmask_LLava_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_filtered_objects_4mask_1280_context_after_stage1_5e4/000-GPT-XL/checkpoints/0014000.pt +[2025-04-28 15:35:45] (step=0016025) Train Loss: 5.8962, Train Steps/Sec: 0.05 +[2025-04-28 15:36:07] (step=0016050) Train Loss: 5.9402, Train Steps/Sec: 1.12 +[2025-04-28 15:36:29] (step=0016075) Train Loss: 5.9906, Train Steps/Sec: 1.12 +[2025-04-28 15:36:51] (step=0016100) Train Loss: 6.0432, Train Steps/Sec: 1.12 +[2025-04-28 15:37:14] (step=0016125) Train Loss: 5.9399, Train Steps/Sec: 1.12 +[2025-04-28 15:37:36] (step=0016150) Train Loss: 5.9174, Train Steps/Sec: 1.12 +[2025-04-28 15:37:58] (step=0016175) Train Loss: 5.9304, Train Steps/Sec: 1.12 +[2025-04-28 15:38:21] (step=0016200) Train Loss: 5.9640, Train Steps/Sec: 1.12 +[2025-04-28 15:38:43] (step=0016225) Train Loss: 5.9457, Train Steps/Sec: 1.12 +[2025-04-28 15:39:06] (step=0016250) Train Loss: 5.9469, Train Steps/Sec: 1.12 +[2025-04-28 15:39:28] (step=0016275) Train Loss: 5.8930, Train Steps/Sec: 1.12 +[2025-04-28 15:39:50] (step=0016300) Train Loss: 5.9471, Train Steps/Sec: 1.12 +[2025-04-28 15:40:12] (step=0016325) Train Loss: 5.9980, Train Steps/Sec: 1.12 +[2025-04-28 15:40:35] (step=0016350) Train Loss: 5.9456, Train Steps/Sec: 1.12 +[2025-04-28 15:40:57] (step=0016375) Train Loss: 5.8609, Train Steps/Sec: 1.12 +[2025-04-28 15:41:20] (step=0016400) Train Loss: 5.9608, Train Steps/Sec: 1.12 +[2025-04-28 15:41:42] (step=0016425) Train Loss: 5.9370, Train Steps/Sec: 1.12 +[2025-04-28 15:42:04] (step=0016450) Train Loss: 5.8545, Train Steps/Sec: 1.12 +[2025-04-28 15:42:27] (step=0016475) Train Loss: 5.8663, Train Steps/Sec: 1.12 +[2025-04-28 15:42:49] (step=0016500) Train Loss: 5.9601, Train Steps/Sec: 1.12 +[2025-04-28 15:43:11] (step=0016525) Train Loss: 5.9354, Train Steps/Sec: 1.12 +[2025-04-28 15:43:34] (step=0016550) Train Loss: 5.9224, Train Steps/Sec: 1.12 +[2025-04-28 15:43:56] (step=0016575) Train Loss: 5.9486, Train Steps/Sec: 1.12 +[2025-04-28 15:44:18] (step=0016600) Train Loss: 5.9610, Train Steps/Sec: 1.12 +[2025-04-28 15:44:41] (step=0016625) Train Loss: 5.9015, Train Steps/Sec: 1.12 +[2025-04-28 15:45:03] (step=0016650) Train Loss: 5.9494, Train Steps/Sec: 1.12 +[2025-04-28 15:45:25] (step=0016675) Train Loss: 5.9957, Train Steps/Sec: 1.12 +[2025-04-28 15:45:48] (step=0016700) Train Loss: 5.9662, Train Steps/Sec: 1.12 +[2025-04-28 15:46:10] (step=0016725) Train Loss: 5.9848, Train Steps/Sec: 1.12 +[2025-04-28 15:46:32] (step=0016750) Train Loss: 5.9237, Train Steps/Sec: 1.12 +[2025-04-28 15:46:55] (step=0016775) Train Loss: 5.8634, Train Steps/Sec: 1.12 +[2025-04-28 15:47:17] (step=0016800) Train Loss: 5.9497, Train Steps/Sec: 1.11 +[2025-04-28 15:47:39] (step=0016825) Train Loss: 5.9778, Train Steps/Sec: 1.12 +[2025-04-28 15:48:02] (step=0016850) Train Loss: 5.9171, Train Steps/Sec: 1.12 +[2025-04-28 15:48:24] (step=0016875) Train Loss: 5.9030, Train Steps/Sec: 1.12 +[2025-04-28 15:48:46] (step=0016900) Train Loss: 5.9116, Train Steps/Sec: 1.12 +[2025-04-28 15:49:09] (step=0016925) Train Loss: 5.9239, Train Steps/Sec: 1.12 +[2025-04-28 15:49:31] (step=0016950) Train Loss: 5.9444, Train Steps/Sec: 1.12 +[2025-04-28 15:49:53] (step=0016975) Train Loss: 5.9355, Train Steps/Sec: 1.12 +[2025-04-28 15:50:16] (step=0017000) Train Loss: 5.9297, Train Steps/Sec: 1.11 +[2025-04-28 15:50:38] (step=0017025) Train Loss: 5.9212, Train Steps/Sec: 1.12 +[2025-04-28 15:51:01] (step=0017050) Train Loss: 5.9798, Train Steps/Sec: 1.12 +[2025-04-28 15:51:23] (step=0017075) Train Loss: 5.9200, Train Steps/Sec: 1.12 +[2025-04-28 15:51:45] (step=0017100) Train Loss: 5.9390, Train Steps/Sec: 1.12 +[2025-04-28 15:52:08] (step=0017125) Train Loss: 5.9526, Train Steps/Sec: 1.12 +[2025-04-28 15:52:30] (step=0017150) Train Loss: 5.9380, Train Steps/Sec: 1.12 +[2025-04-28 15:52:52] (step=0017175) Train Loss: 5.9884, Train Steps/Sec: 1.12 +[2025-04-28 15:53:15] (step=0017200) Train Loss: 5.9680, Train Steps/Sec: 1.12 +[2025-04-28 15:53:37] (step=0017225) Train Loss: 5.8722, Train Steps/Sec: 1.12 +[2025-04-28 15:53:59] (step=0017250) Train Loss: 5.9967, Train Steps/Sec: 1.12 +[2025-04-28 15:54:22] (step=0017275) Train Loss: 5.8936, Train Steps/Sec: 1.12 +[2025-04-28 15:54:44] (step=0017300) Train Loss: 5.9390, Train Steps/Sec: 1.12 +[2025-04-28 15:55:06] (step=0017325) Train Loss: 5.8953, Train Steps/Sec: 1.12 +[2025-04-28 15:55:29] (step=0017350) Train Loss: 5.9512, Train Steps/Sec: 1.12 +[2025-04-28 15:55:51] (step=0017375) Train Loss: 5.9250, Train Steps/Sec: 1.12 +[2025-04-28 15:56:13] (step=0017400) Train Loss: 5.9069, Train Steps/Sec: 1.11 +[2025-04-28 15:56:36] (step=0017425) Train Loss: 5.8674, Train Steps/Sec: 1.12 +[2025-04-28 15:56:58] (step=0017450) Train Loss: 5.9503, Train Steps/Sec: 1.12 +[2025-04-28 15:57:20] (step=0017475) Train Loss: 5.9811, Train Steps/Sec: 1.12 +[2025-04-28 15:57:43] (step=0017500) Train Loss: 5.9118, Train Steps/Sec: 1.12 +[2025-04-28 15:58:05] (step=0017525) Train Loss: 5.9279, Train Steps/Sec: 1.12 +[2025-04-28 15:58:27] (step=0017550) Train Loss: 5.9098, Train Steps/Sec: 1.12 +[2025-04-28 15:58:50] (step=0017575) Train Loss: 5.9085, Train Steps/Sec: 1.12 +[2025-04-28 15:59:12] (step=0017600) Train Loss: 5.9378, Train Steps/Sec: 1.11 +[2025-04-28 15:59:34] (step=0017625) Train Loss: 5.8908, Train Steps/Sec: 1.12 +[2025-04-28 15:59:57] (step=0017650) Train Loss: 5.9165, Train Steps/Sec: 1.12 +[2025-04-28 16:00:19] (step=0017675) Train Loss: 5.9625, Train Steps/Sec: 1.12 +[2025-04-28 16:00:41] (step=0017700) Train Loss: 5.9401, Train Steps/Sec: 1.11 +[2025-04-28 16:01:04] (step=0017725) Train Loss: 5.9363, Train Steps/Sec: 1.12 +[2025-04-28 16:01:26] (step=0017750) Train Loss: 5.9035, Train Steps/Sec: 1.12 +[2025-04-28 16:01:48] (step=0017775) Train Loss: 5.9369, Train Steps/Sec: 1.12 +[2025-04-28 16:02:11] (step=0017800) Train Loss: 5.9033, Train Steps/Sec: 1.11 +[2025-04-28 16:02:33] (step=0017825) Train Loss: 5.9144, Train Steps/Sec: 1.12 +[2025-04-28 16:02:56] (step=0017850) Train Loss: 5.9025, Train Steps/Sec: 1.12 +[2025-04-28 16:03:18] (step=0017875) Train Loss: 5.9290, Train Steps/Sec: 1.12 +[2025-04-28 16:03:40] (step=0017900) Train Loss: 5.9009, Train Steps/Sec: 1.12 +[2025-04-28 16:04:03] (step=0017925) Train Loss: 5.9162, Train Steps/Sec: 1.12 +[2025-04-28 16:04:25] (step=0017950) Train Loss: 5.8941, Train Steps/Sec: 1.12 +[2025-04-28 16:04:47] (step=0017975) Train Loss: 5.9289, Train Steps/Sec: 1.12 +[2025-04-28 16:05:10] (step=0018000) Train Loss: 5.8731, Train Steps/Sec: 1.12 +[2025-04-28 16:05:10] text_config is None. Initializing the text config with default values (`OPTConfig`). +[2025-04-28 16:12:28] Finish Eval in 18000 steps... +[2025-04-28 16:12:48] Saved checkpoint to checkpoint/X2I_UltraEdiit_fourmask_LLava_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_filtered_objects_4mask_1280_context_after_stage1_5e4/000-GPT-XL/checkpoints/0018000.pt +[2025-04-28 16:12:50] Removed old checkpoint: checkpoint/X2I_UltraEdiit_fourmask_LLava_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_filtered_objects_4mask_1280_context_after_stage1_5e4/000-GPT-XL/checkpoints/0016000.pt +[2025-04-28 16:13:12] (step=0018025) Train Loss: 5.9053, Train Steps/Sec: 0.05 +[2025-04-28 16:13:34] (step=0018050) Train Loss: 5.8165, Train Steps/Sec: 1.12 +[2025-04-28 16:13:57] (step=0018075) Train Loss: 5.9039, Train Steps/Sec: 1.12 +[2025-04-28 16:14:19] (step=0018100) Train Loss: 5.8945, Train Steps/Sec: 1.12 +[2025-04-28 16:14:41] (step=0018125) Train Loss: 5.9189, Train Steps/Sec: 1.12 +[2025-04-28 16:15:04] (step=0018150) Train Loss: 5.9610, Train Steps/Sec: 1.12 +[2025-04-28 16:15:26] (step=0018175) Train Loss: 5.8812, Train Steps/Sec: 1.12 +[2025-04-28 16:15:49] (step=0018200) Train Loss: 5.8938, Train Steps/Sec: 1.11 +[2025-04-28 16:16:11] (step=0018225) Train Loss: 5.9171, Train Steps/Sec: 1.12 +[2025-04-28 16:16:33] (step=0018250) Train Loss: 5.9023, Train Steps/Sec: 1.12 +[2025-04-28 16:16:56] (step=0018275) Train Loss: 5.9244, Train Steps/Sec: 1.12 +[2025-04-28 16:17:18] (step=0018300) Train Loss: 5.8111, Train Steps/Sec: 1.12 +[2025-04-28 16:17:40] (step=0018325) Train Loss: 5.9025, Train Steps/Sec: 1.12 +[2025-04-28 16:18:03] (step=0018350) Train Loss: 5.9115, Train Steps/Sec: 1.12 +[2025-04-28 16:18:25] (step=0018375) Train Loss: 5.8695, Train Steps/Sec: 1.12 +[2025-04-28 16:18:47] (step=0018400) Train Loss: 5.9346, Train Steps/Sec: 1.11 +[2025-04-28 16:19:10] (step=0018425) Train Loss: 5.8625, Train Steps/Sec: 1.12 +[2025-04-28 16:19:32] (step=0018450) Train Loss: 5.8998, Train Steps/Sec: 1.12 +[2025-04-28 16:19:54] (step=0018475) Train Loss: 5.9787, Train Steps/Sec: 1.12 +[2025-04-28 16:20:17] (step=0018500) Train Loss: 5.8726, Train Steps/Sec: 1.12 +[2025-04-28 16:20:39] (step=0018525) Train Loss: 5.8758, Train Steps/Sec: 1.12 +[2025-04-28 16:21:01] (step=0018550) Train Loss: 5.9138, Train Steps/Sec: 1.12 +[2025-04-28 16:21:24] (step=0018575) Train Loss: 5.9201, Train Steps/Sec: 1.12 +[2025-04-28 16:21:46] (step=0018600) Train Loss: 5.9273, Train Steps/Sec: 1.12 +[2025-04-28 16:22:08] (step=0018625) Train Loss: 5.8874, Train Steps/Sec: 1.12 +[2025-04-28 16:22:31] (step=0018650) Train Loss: 5.8914, Train Steps/Sec: 1.12 +[2025-04-28 16:22:53] (step=0018675) Train Loss: 5.8707, Train Steps/Sec: 1.12 +[2025-04-28 16:23:15] (step=0018700) Train Loss: 5.9067, Train Steps/Sec: 1.12 +[2025-04-28 16:23:38] (step=0018725) Train Loss: 5.8876, Train Steps/Sec: 1.12 +[2025-04-28 16:24:00] (step=0018750) Train Loss: 5.9384, Train Steps/Sec: 1.12 +[2025-04-28 16:24:22] (step=0018775) Train Loss: 5.9170, Train Steps/Sec: 1.12 +[2025-04-28 16:24:45] (step=0018800) Train Loss: 5.9049, Train Steps/Sec: 1.12 +[2025-04-28 16:25:07] (step=0018825) Train Loss: 5.9064, Train Steps/Sec: 1.12 +[2025-04-28 16:25:29] (step=0018850) Train Loss: 5.9130, Train Steps/Sec: 1.12 +[2025-04-28 16:25:51] (step=0018875) Train Loss: 5.8333, Train Steps/Sec: 1.12 +[2025-04-28 16:26:14] (step=0018900) Train Loss: 5.9168, Train Steps/Sec: 1.12 +[2025-04-28 16:26:36] (step=0018925) Train Loss: 5.9400, Train Steps/Sec: 1.12 +[2025-04-28 16:26:58] (step=0018950) Train Loss: 5.9034, Train Steps/Sec: 1.12 +[2025-04-28 16:27:21] (step=0018975) Train Loss: 5.8981, Train Steps/Sec: 1.12 +[2025-04-28 16:27:43] (step=0019000) Train Loss: 5.8830, Train Steps/Sec: 1.11 +[2025-04-28 16:28:05] (step=0019025) Train Loss: 5.8705, Train Steps/Sec: 1.12 +[2025-04-28 16:28:28] (step=0019050) Train Loss: 5.9037, Train Steps/Sec: 1.12 +[2025-04-28 16:28:50] (step=0019075) Train Loss: 5.8579, Train Steps/Sec: 1.12 +[2025-04-28 16:29:12] (step=0019100) Train Loss: 5.9198, Train Steps/Sec: 1.12 +[2025-04-28 16:29:35] (step=0019125) Train Loss: 5.8713, Train Steps/Sec: 1.12 +[2025-04-28 16:29:57] (step=0019150) Train Loss: 5.8841, Train Steps/Sec: 1.12 +[2025-04-28 16:30:47] (step=0019175) Train Loss: 5.9350, Train Steps/Sec: 0.51 +[2025-04-28 16:31:38] (step=0019200) Train Loss: 5.8646, Train Steps/Sec: 0.49 +[2025-04-28 16:32:09] (step=0019225) Train Loss: 5.8698, Train Steps/Sec: 0.80 +[2025-04-28 16:32:32] (step=0019250) Train Loss: 5.9725, Train Steps/Sec: 1.12 +[2025-04-28 16:32:54] (step=0019275) Train Loss: 5.8840, Train Steps/Sec: 1.12 +[2025-04-28 16:33:16] (step=0019300) Train Loss: 5.8247, Train Steps/Sec: 1.12 +[2025-04-28 16:33:39] (step=0019325) Train Loss: 5.8060, Train Steps/Sec: 1.12 +[2025-04-28 16:34:01] (step=0019350) Train Loss: 5.9428, Train Steps/Sec: 1.12 +[2025-04-28 16:34:23] (step=0019375) Train Loss: 5.8445, Train Steps/Sec: 1.12 +[2025-04-28 16:34:46] (step=0019400) Train Loss: 5.8688, Train Steps/Sec: 1.12 +[2025-04-28 16:35:08] (step=0019425) Train Loss: 5.8530, Train Steps/Sec: 1.12 +[2025-04-28 16:35:30] (step=0019450) Train Loss: 5.9125, Train Steps/Sec: 1.12 +[2025-04-28 16:35:53] (step=0019475) Train Loss: 5.8960, Train Steps/Sec: 1.12 +[2025-04-28 16:36:15] (step=0019500) Train Loss: 5.8947, Train Steps/Sec: 1.12 +[2025-04-28 16:36:37] (step=0019525) Train Loss: 5.8772, Train Steps/Sec: 1.12 +[2025-04-28 16:36:59] (step=0019550) Train Loss: 5.9241, Train Steps/Sec: 1.12 +[2025-04-28 16:37:22] (step=0019575) Train Loss: 5.9176, Train Steps/Sec: 1.12 +[2025-04-28 16:37:44] (step=0019600) Train Loss: 5.9039, Train Steps/Sec: 1.12 +[2025-04-28 16:38:06] (step=0019625) Train Loss: 5.8247, Train Steps/Sec: 1.12 +[2025-04-28 16:38:29] (step=0019650) Train Loss: 5.8421, Train Steps/Sec: 1.12 +[2025-04-28 16:38:51] (step=0019675) Train Loss: 5.9632, Train Steps/Sec: 1.12 +[2025-04-28 16:39:13] (step=0019700) Train Loss: 5.8522, Train Steps/Sec: 1.12 +[2025-04-28 16:39:35] (step=0019725) Train Loss: 5.8771, Train Steps/Sec: 1.12 +[2025-04-28 16:39:58] (step=0019750) Train Loss: 5.8460, Train Steps/Sec: 1.12 +[2025-04-28 16:40:20] (step=0019775) Train Loss: 5.8805, Train Steps/Sec: 1.12 +[2025-04-28 16:40:42] (step=0019800) Train Loss: 5.9148, Train Steps/Sec: 1.12 +[2025-04-28 16:41:05] (step=0019825) Train Loss: 5.8743, Train Steps/Sec: 1.12 +[2025-04-28 16:41:27] (step=0019850) Train Loss: 5.7941, Train Steps/Sec: 1.12 +[2025-04-28 16:41:49] (step=0019875) Train Loss: 5.9963, Train Steps/Sec: 1.12 +[2025-04-28 16:42:12] (step=0019900) Train Loss: 5.8847, Train Steps/Sec: 1.12 +[2025-04-28 16:42:34] (step=0019925) Train Loss: 5.8934, Train Steps/Sec: 1.12 +[2025-04-28 16:42:56] (step=0019950) Train Loss: 5.8758, Train Steps/Sec: 1.12 +[2025-04-28 16:43:18] (step=0019975) Train Loss: 5.8940, Train Steps/Sec: 1.12 +[2025-04-28 16:43:41] (step=0020000) Train Loss: 5.9034, Train Steps/Sec: 1.12 +[2025-04-28 16:43:41] text_config is None. Initializing the text config with default values (`OPTConfig`). +[2025-04-28 16:50:59] Finish Eval in 20000 steps... +[2025-04-28 16:51:19] Saved checkpoint to checkpoint/X2I_UltraEdiit_fourmask_LLava_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_filtered_objects_4mask_1280_context_after_stage1_5e4/000-GPT-XL/checkpoints/0020000.pt +[2025-04-28 16:51:21] Removed old checkpoint: checkpoint/X2I_UltraEdiit_fourmask_LLava_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_filtered_objects_4mask_1280_context_after_stage1_5e4/000-GPT-XL/checkpoints/0018000.pt +[2025-04-28 16:51:43] (step=0020025) Train Loss: 5.8259, Train Steps/Sec: 0.05 +[2025-04-28 16:52:06] (step=0020050) Train Loss: 5.8313, Train Steps/Sec: 1.12 +[2025-04-28 16:52:28] (step=0020075) Train Loss: 5.8557, Train Steps/Sec: 1.12 +[2025-04-28 16:52:50] (step=0020100) Train Loss: 5.8188, Train Steps/Sec: 1.12 +[2025-04-28 16:53:13] (step=0020125) Train Loss: 5.9005, Train Steps/Sec: 1.12 +[2025-04-28 16:53:35] (step=0020150) Train Loss: 5.8278, Train Steps/Sec: 1.11 +[2025-04-28 16:53:58] (step=0020175) Train Loss: 5.8725, Train Steps/Sec: 1.12 +[2025-04-28 16:54:20] (step=0020200) Train Loss: 5.9271, Train Steps/Sec: 1.11 +[2025-04-28 16:54:43] (step=0020225) Train Loss: 5.8309, Train Steps/Sec: 1.11 +[2025-04-28 16:55:05] (step=0020250) Train Loss: 5.8869, Train Steps/Sec: 1.12 +[2025-04-28 16:55:27] (step=0020275) Train Loss: 5.9066, Train Steps/Sec: 1.12 +[2025-04-28 16:55:50] (step=0020300) Train Loss: 5.9052, Train Steps/Sec: 1.12 +[2025-04-28 16:56:12] (step=0020325) Train Loss: 5.8845, Train Steps/Sec: 1.12 +[2025-04-28 16:56:34] (step=0020350) Train Loss: 5.8953, Train Steps/Sec: 1.12 +[2025-04-28 16:56:57] (step=0020375) Train Loss: 5.8841, Train Steps/Sec: 1.12 +[2025-04-28 16:57:19] (step=0020400) Train Loss: 5.8900, Train Steps/Sec: 1.11 +[2025-04-28 16:57:42] (step=0020425) Train Loss: 5.8234, Train Steps/Sec: 1.12 +[2025-04-28 16:58:04] (step=0020450) Train Loss: 5.8128, Train Steps/Sec: 1.12 +[2025-04-28 16:58:26] (step=0020475) Train Loss: 5.9233, Train Steps/Sec: 1.12 +[2025-04-28 16:58:49] (step=0020500) Train Loss: 5.8407, Train Steps/Sec: 1.12 +[2025-04-28 16:59:11] (step=0020525) Train Loss: 5.8937, Train Steps/Sec: 1.11 +[2025-04-28 16:59:33] (step=0020550) Train Loss: 5.8181, Train Steps/Sec: 1.12 +[2025-04-28 16:59:56] (step=0020575) Train Loss: 5.9061, Train Steps/Sec: 1.12 +[2025-04-28 17:00:18] (step=0020600) Train Loss: 5.9097, Train Steps/Sec: 1.11 +[2025-04-28 17:00:40] (step=0020625) Train Loss: 5.8740, Train Steps/Sec: 1.12 +[2025-04-28 17:01:03] (step=0020650) Train Loss: 5.7996, Train Steps/Sec: 1.12 +[2025-04-28 17:01:25] (step=0020675) Train Loss: 5.8285, Train Steps/Sec: 1.12 +[2025-04-28 17:01:47] (step=0020700) Train Loss: 5.9078, Train Steps/Sec: 1.12 +[2025-04-28 17:02:10] (step=0020725) Train Loss: 5.8164, Train Steps/Sec: 1.12 +[2025-04-28 17:02:32] (step=0020750) Train Loss: 5.8457, Train Steps/Sec: 1.12 +[2025-04-28 17:02:54] (step=0020775) Train Loss: 5.8140, Train Steps/Sec: 1.12 +[2025-04-28 17:03:17] (step=0020800) Train Loss: 5.9246, Train Steps/Sec: 1.11 +[2025-04-28 17:03:39] (step=0020825) Train Loss: 5.8289, Train Steps/Sec: 1.12 +[2025-04-28 17:04:01] (step=0020850) Train Loss: 5.8003, Train Steps/Sec: 1.12 +[2025-04-28 17:04:24] (step=0020875) Train Loss: 5.8321, Train Steps/Sec: 1.12 +[2025-04-28 17:04:46] (step=0020900) Train Loss: 5.9103, Train Steps/Sec: 1.12 +[2025-04-28 17:05:08] (step=0020925) Train Loss: 5.8175, Train Steps/Sec: 1.12 +[2025-04-28 17:05:31] (step=0020950) Train Loss: 5.8762, Train Steps/Sec: 1.12 +[2025-04-28 17:05:53] (step=0020975) Train Loss: 5.8033, Train Steps/Sec: 1.12 +[2025-04-28 17:06:15] (step=0021000) Train Loss: 5.8838, Train Steps/Sec: 1.12 +[2025-04-28 17:06:38] (step=0021025) Train Loss: 5.9170, Train Steps/Sec: 1.12 +[2025-04-28 17:07:00] (step=0021050) Train Loss: 5.7975, Train Steps/Sec: 1.12 +[2025-04-28 17:07:22] (step=0021075) Train Loss: 5.8656, Train Steps/Sec: 1.12 +[2025-04-28 17:07:44] (step=0021100) Train Loss: 5.8788, Train Steps/Sec: 1.12 +[2025-04-28 17:08:07] (step=0021125) Train Loss: 5.9253, Train Steps/Sec: 1.12 +[2025-04-28 17:08:29] (step=0021150) Train Loss: 5.8185, Train Steps/Sec: 1.12 +[2025-04-28 17:08:51] (step=0021175) Train Loss: 5.8546, Train Steps/Sec: 1.12 +[2025-04-28 17:09:14] (step=0021200) Train Loss: 5.8222, Train Steps/Sec: 1.11 +[2025-04-28 17:09:36] (step=0021225) Train Loss: 5.8791, Train Steps/Sec: 1.12 +[2025-04-28 17:09:58] (step=0021250) Train Loss: 5.8633, Train Steps/Sec: 1.12 +[2025-04-28 17:10:21] (step=0021275) Train Loss: 5.8676, Train Steps/Sec: 1.12 +[2025-04-28 17:10:43] (step=0021300) Train Loss: 5.8639, Train Steps/Sec: 1.12 +[2025-04-28 17:11:05] (step=0021325) Train Loss: 5.7685, Train Steps/Sec: 1.12 +[2025-04-28 17:11:28] (step=0021350) Train Loss: 5.8246, Train Steps/Sec: 1.12 +[2025-04-28 17:11:50] (step=0021375) Train Loss: 5.7889, Train Steps/Sec: 1.12 +[2025-04-28 17:12:12] (step=0021400) Train Loss: 5.8123, Train Steps/Sec: 1.11 +[2025-04-28 17:12:35] (step=0021425) Train Loss: 5.7993, Train Steps/Sec: 1.12 +[2025-04-28 17:12:57] (step=0021450) Train Loss: 5.8750, Train Steps/Sec: 1.12 +[2025-04-28 17:13:19] (step=0021475) Train Loss: 5.8134, Train Steps/Sec: 1.12 +[2025-04-28 17:13:42] (step=0021500) Train Loss: 5.8814, Train Steps/Sec: 1.12 +[2025-04-28 17:14:04] (step=0021525) Train Loss: 5.8299, Train Steps/Sec: 1.12 +[2025-04-28 17:14:26] (step=0021550) Train Loss: 5.7884, Train Steps/Sec: 1.12 +[2025-04-28 17:14:49] (step=0021575) Train Loss: 5.8921, Train Steps/Sec: 1.12 +[2025-04-28 17:15:11] (step=0021600) Train Loss: 5.9266, Train Steps/Sec: 1.12 +[2025-04-28 17:15:33] (step=0021625) Train Loss: 5.8653, Train Steps/Sec: 1.12 +[2025-04-28 17:15:56] (step=0021650) Train Loss: 5.8211, Train Steps/Sec: 1.12 +[2025-04-28 17:16:18] (step=0021675) Train Loss: 5.8338, Train Steps/Sec: 1.12 +[2025-04-28 17:16:40] (step=0021700) Train Loss: 5.7888, Train Steps/Sec: 1.12 +[2025-04-28 17:17:02] (step=0021725) Train Loss: 5.8767, Train Steps/Sec: 1.12 +[2025-04-28 17:17:25] (step=0021750) Train Loss: 5.9026, Train Steps/Sec: 1.12 +[2025-04-28 17:17:47] (step=0021775) Train Loss: 5.8991, Train Steps/Sec: 1.12 +[2025-04-28 17:18:09] (step=0021800) Train Loss: 5.8847, Train Steps/Sec: 1.12 +[2025-04-28 17:18:32] (step=0021825) Train Loss: 5.7940, Train Steps/Sec: 1.12 +[2025-04-28 17:18:54] (step=0021850) Train Loss: 5.8380, Train Steps/Sec: 1.12 +[2025-04-28 17:19:16] (step=0021875) Train Loss: 5.7468, Train Steps/Sec: 1.12 +[2025-04-28 17:19:39] (step=0021900) Train Loss: 5.7818, Train Steps/Sec: 1.12 +[2025-04-28 17:20:01] (step=0021925) Train Loss: 5.8480, Train Steps/Sec: 1.12 +[2025-04-28 17:20:23] (step=0021950) Train Loss: 5.7643, Train Steps/Sec: 1.12 +[2025-04-28 17:20:45] (step=0021975) Train Loss: 5.8577, Train Steps/Sec: 1.12 +[2025-04-28 17:21:08] (step=0022000) Train Loss: 5.8168, Train Steps/Sec: 1.12 +[2025-04-28 17:21:08] text_config is None. Initializing the text config with default values (`OPTConfig`). +[2025-04-28 17:28:26] Finish Eval in 22000 steps... +[2025-04-28 17:28:46] Saved checkpoint to checkpoint/X2I_UltraEdiit_fourmask_LLava_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_filtered_objects_4mask_1280_context_after_stage1_5e4/000-GPT-XL/checkpoints/0022000.pt +[2025-04-28 17:28:48] Removed old checkpoint: checkpoint/X2I_UltraEdiit_fourmask_LLava_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_filtered_objects_4mask_1280_context_after_stage1_5e4/000-GPT-XL/checkpoints/0020000.pt +[2025-04-28 17:29:10] (step=0022025) Train Loss: 5.9128, Train Steps/Sec: 0.05 +[2025-04-28 17:29:33] (step=0022050) Train Loss: 5.7904, Train Steps/Sec: 1.12 +[2025-04-28 17:29:55] (step=0022075) Train Loss: 5.8722, Train Steps/Sec: 1.12 +[2025-04-28 17:30:17] (step=0022100) Train Loss: 5.8021, Train Steps/Sec: 1.12 +[2025-04-28 17:30:40] (step=0022125) Train Loss: 5.8196, Train Steps/Sec: 1.12 +[2025-04-28 17:31:02] (step=0022150) Train Loss: 5.8287, Train Steps/Sec: 1.12 +[2025-04-28 17:31:24] (step=0022175) Train Loss: 5.8001, Train Steps/Sec: 1.12 +[2025-04-28 17:31:47] (step=0022200) Train Loss: 5.8713, Train Steps/Sec: 1.12 +[2025-04-28 17:32:09] (step=0022225) Train Loss: 5.8454, Train Steps/Sec: 1.12 +[2025-04-28 17:32:31] (step=0022250) Train Loss: 5.8816, Train Steps/Sec: 1.12 +[2025-04-28 17:32:54] (step=0022275) Train Loss: 5.8939, Train Steps/Sec: 1.12 +[2025-04-28 17:33:16] (step=0022300) Train Loss: 5.8071, Train Steps/Sec: 1.12 +[2025-04-28 17:33:38] (step=0022325) Train Loss: 5.7634, Train Steps/Sec: 1.12 +[2025-04-28 17:34:00] (step=0022350) Train Loss: 5.8143, Train Steps/Sec: 1.12 +[2025-04-28 17:34:23] (step=0022375) Train Loss: 5.8216, Train Steps/Sec: 1.12 +[2025-04-28 17:34:45] (step=0022400) Train Loss: 5.8641, Train Steps/Sec: 1.11 +[2025-04-28 17:35:08] (step=0022425) Train Loss: 5.7800, Train Steps/Sec: 1.12 +[2025-04-28 17:35:30] (step=0022450) Train Loss: 5.8380, Train Steps/Sec: 1.12 +[2025-04-28 17:35:52] (step=0022475) Train Loss: 5.8105, Train Steps/Sec: 1.12 +[2025-04-28 17:36:14] (step=0022500) Train Loss: 5.8361, Train Steps/Sec: 1.12 +[2025-04-28 17:36:37] (step=0022525) Train Loss: 5.8642, Train Steps/Sec: 1.12 +[2025-04-28 17:36:59] (step=0022550) Train Loss: 5.8234, Train Steps/Sec: 1.12 +[2025-04-28 17:37:21] (step=0022575) Train Loss: 5.8226, Train Steps/Sec: 1.12 +[2025-04-28 17:37:44] (step=0022600) Train Loss: 5.8856, Train Steps/Sec: 1.12 +[2025-04-28 17:38:06] (step=0022625) Train Loss: 5.7675, Train Steps/Sec: 1.12 +[2025-04-28 17:38:28] (step=0022650) Train Loss: 5.7974, Train Steps/Sec: 1.12 +[2025-04-28 17:38:51] (step=0022675) Train Loss: 5.8495, Train Steps/Sec: 1.12 +[2025-04-28 17:39:13] (step=0022700) Train Loss: 5.8789, Train Steps/Sec: 1.12 +[2025-04-28 17:39:35] (step=0022725) Train Loss: 5.8057, Train Steps/Sec: 1.12 +[2025-04-28 17:40:36] (step=0022750) Train Loss: 5.8202, Train Steps/Sec: 0.41 +[2025-04-28 17:40:58] (step=0022775) Train Loss: 5.8007, Train Steps/Sec: 1.12 +[2025-04-28 17:41:29] (step=0022800) Train Loss: 5.8185, Train Steps/Sec: 0.79 +[2025-04-28 17:41:52] (step=0022825) Train Loss: 5.8626, Train Steps/Sec: 1.12 +[2025-04-28 17:42:14] (step=0022850) Train Loss: 5.7999, Train Steps/Sec: 1.12 +[2025-04-28 17:42:36] (step=0022875) Train Loss: 5.8938, Train Steps/Sec: 1.12 +[2025-04-28 17:42:59] (step=0022900) Train Loss: 5.9044, Train Steps/Sec: 1.12 +[2025-04-28 17:43:21] (step=0022925) Train Loss: 5.8231, Train Steps/Sec: 1.12 +[2025-04-28 17:43:43] (step=0022950) Train Loss: 5.8349, Train Steps/Sec: 1.12 +[2025-04-28 17:44:06] (step=0022975) Train Loss: 5.7947, Train Steps/Sec: 1.12 +[2025-04-28 17:44:28] (step=0023000) Train Loss: 5.7859, Train Steps/Sec: 1.12 +[2025-04-28 17:44:50] (step=0023025) Train Loss: 5.8042, Train Steps/Sec: 1.12 +[2025-04-28 17:45:13] (step=0023050) Train Loss: 5.8531, Train Steps/Sec: 1.12 +[2025-04-28 17:45:35] (step=0023075) Train Loss: 5.8250, Train Steps/Sec: 1.12 +[2025-04-28 17:45:57] (step=0023100) Train Loss: 5.8150, Train Steps/Sec: 1.12 +[2025-04-28 17:46:19] (step=0023125) Train Loss: 5.8239, Train Steps/Sec: 1.12 +[2025-04-28 17:46:42] (step=0023150) Train Loss: 5.8663, Train Steps/Sec: 1.12 +[2025-04-28 17:47:04] (step=0023175) Train Loss: 5.8841, Train Steps/Sec: 1.12 +[2025-04-28 17:47:26] (step=0023200) Train Loss: 5.8761, Train Steps/Sec: 1.12 +[2025-04-28 17:47:49] (step=0023225) Train Loss: 5.8085, Train Steps/Sec: 1.12 +[2025-04-28 17:48:11] (step=0023250) Train Loss: 5.8137, Train Steps/Sec: 1.12 +[2025-04-28 17:48:33] (step=0023275) Train Loss: 5.7950, Train Steps/Sec: 1.12 +[2025-04-28 17:48:56] (step=0023300) Train Loss: 5.7640, Train Steps/Sec: 1.12 +[2025-04-28 17:49:18] (step=0023325) Train Loss: 5.7652, Train Steps/Sec: 1.12 +[2025-04-28 17:49:40] (step=0023350) Train Loss: 5.8488, Train Steps/Sec: 1.12 +[2025-04-28 17:50:02] (step=0023375) Train Loss: 5.7958, Train Steps/Sec: 1.12 +[2025-04-28 17:50:25] (step=0023400) Train Loss: 5.7392, Train Steps/Sec: 1.12 +[2025-04-28 17:50:47] (step=0023425) Train Loss: 5.7877, Train Steps/Sec: 1.12 +[2025-04-28 17:51:09] (step=0023450) Train Loss: 5.8280, Train Steps/Sec: 1.12 +[2025-04-28 17:51:32] (step=0023475) Train Loss: 5.8364, Train Steps/Sec: 1.12 +[2025-04-28 17:51:54] (step=0023500) Train Loss: 5.7993, Train Steps/Sec: 1.12 +[2025-04-28 17:52:16] (step=0023525) Train Loss: 5.8390, Train Steps/Sec: 1.12 +[2025-04-28 17:52:39] (step=0023550) Train Loss: 5.9050, Train Steps/Sec: 1.12 +[2025-04-28 17:53:01] (step=0023575) Train Loss: 5.8076, Train Steps/Sec: 1.12 +[2025-04-28 17:53:23] (step=0023600) Train Loss: 5.7562, Train Steps/Sec: 1.12 +[2025-04-28 17:53:46] (step=0023625) Train Loss: 5.8553, Train Steps/Sec: 1.12 +[2025-04-28 17:54:08] (step=0023650) Train Loss: 5.7517, Train Steps/Sec: 1.12 +[2025-04-28 17:54:30] (step=0023675) Train Loss: 5.8674, Train Steps/Sec: 1.12 +[2025-04-28 17:54:52] (step=0023700) Train Loss: 5.8613, Train Steps/Sec: 1.12 +[2025-04-28 17:55:15] (step=0023725) Train Loss: 5.8403, Train Steps/Sec: 1.12 +[2025-04-28 17:55:37] (step=0023750) Train Loss: 5.8107, Train Steps/Sec: 1.12 +[2025-04-28 17:55:59] (step=0023775) Train Loss: 5.8560, Train Steps/Sec: 1.12 +[2025-04-28 17:56:22] (step=0023800) Train Loss: 5.7976, Train Steps/Sec: 1.12 +[2025-04-28 17:56:44] (step=0023825) Train Loss: 5.7902, Train Steps/Sec: 1.12 +[2025-04-28 17:57:06] (step=0023850) Train Loss: 5.8058, Train Steps/Sec: 1.12 +[2025-04-28 17:57:29] (step=0023875) Train Loss: 5.8382, Train Steps/Sec: 1.12 +[2025-04-28 17:57:51] (step=0023900) Train Loss: 5.7586, Train Steps/Sec: 1.12 +[2025-04-28 17:58:13] (step=0023925) Train Loss: 5.7929, Train Steps/Sec: 1.12 +[2025-04-28 17:58:35] (step=0023950) Train Loss: 5.7765, Train Steps/Sec: 1.12 +[2025-04-28 17:58:58] (step=0023975) Train Loss: 5.8548, Train Steps/Sec: 1.12 +[2025-04-28 17:59:20] (step=0024000) Train Loss: 5.8276, Train Steps/Sec: 1.12 +[2025-04-28 17:59:20] text_config is None. Initializing the text config with default values (`OPTConfig`). +[2025-04-28 18:06:38] Finish Eval in 24000 steps... +[2025-04-28 18:06:58] Saved checkpoint to checkpoint/X2I_UltraEdiit_fourmask_LLava_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_filtered_objects_4mask_1280_context_after_stage1_5e4/000-GPT-XL/checkpoints/0024000.pt +[2025-04-28 18:06:59] Removed old checkpoint: checkpoint/X2I_UltraEdiit_fourmask_LLava_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_filtered_objects_4mask_1280_context_after_stage1_5e4/000-GPT-XL/checkpoints/0022000.pt +[2025-04-28 18:07:22] (step=0024025) Train Loss: 5.7891, Train Steps/Sec: 0.05 +[2025-04-28 18:07:44] (step=0024050) Train Loss: 5.8163, Train Steps/Sec: 1.12 +[2025-04-28 18:08:07] (step=0024075) Train Loss: 5.8346, Train Steps/Sec: 1.12 +[2025-04-28 18:08:29] (step=0024100) Train Loss: 5.8374, Train Steps/Sec: 1.12 +[2025-04-28 18:08:51] (step=0024125) Train Loss: 5.7830, Train Steps/Sec: 1.12 +[2025-04-28 18:09:13] (step=0024150) Train Loss: 5.8201, Train Steps/Sec: 1.12 +[2025-04-28 18:09:36] (step=0024175) Train Loss: 5.8447, Train Steps/Sec: 1.12 +[2025-04-28 18:09:58] (step=0024200) Train Loss: 5.7398, Train Steps/Sec: 1.12 +[2025-04-28 18:10:20] (step=0024225) Train Loss: 5.7928, Train Steps/Sec: 1.12 +[2025-04-28 18:10:43] (step=0024250) Train Loss: 5.7996, Train Steps/Sec: 1.12 +[2025-04-28 18:11:05] (step=0024275) Train Loss: 5.7839, Train Steps/Sec: 1.12 +[2025-04-28 18:11:27] (step=0024300) Train Loss: 5.7365, Train Steps/Sec: 1.12 +[2025-04-28 18:11:50] (step=0024325) Train Loss: 5.8292, Train Steps/Sec: 1.12 +[2025-04-28 18:12:12] (step=0024350) Train Loss: 5.7675, Train Steps/Sec: 1.12 +[2025-04-28 18:12:34] (step=0024375) Train Loss: 5.9459, Train Steps/Sec: 1.12 +[2025-04-28 18:12:57] (step=0024400) Train Loss: 5.7731, Train Steps/Sec: 1.12 +[2025-04-28 18:13:19] (step=0024425) Train Loss: 5.7576, Train Steps/Sec: 1.12 +[2025-04-28 18:13:41] (step=0024450) Train Loss: 5.8063, Train Steps/Sec: 1.12 +[2025-04-28 18:14:03] (step=0024475) Train Loss: 5.8363, Train Steps/Sec: 1.12 +[2025-04-28 18:14:26] (step=0024500) Train Loss: 5.7661, Train Steps/Sec: 1.12 +[2025-04-28 18:14:48] (step=0024525) Train Loss: 5.7798, Train Steps/Sec: 1.12 +[2025-04-28 18:15:10] (step=0024550) Train Loss: 5.7840, Train Steps/Sec: 1.12 +[2025-04-28 18:15:33] (step=0024575) Train Loss: 5.7691, Train Steps/Sec: 1.12 +[2025-04-28 18:15:55] (step=0024600) Train Loss: 5.7841, Train Steps/Sec: 1.12 +[2025-04-28 18:16:17] (step=0024625) Train Loss: 5.8028, Train Steps/Sec: 1.12 +[2025-04-28 18:16:40] (step=0024650) Train Loss: 5.8290, Train Steps/Sec: 1.12 +[2025-04-28 18:17:02] (step=0024675) Train Loss: 5.8274, Train Steps/Sec: 1.12 +[2025-04-28 18:17:24] (step=0024700) Train Loss: 5.8626, Train Steps/Sec: 1.12 +[2025-04-28 18:17:47] (step=0024725) Train Loss: 5.8492, Train Steps/Sec: 1.12 +[2025-04-28 18:18:09] (step=0024750) Train Loss: 5.8436, Train Steps/Sec: 1.12 +[2025-04-28 18:18:31] (step=0024775) Train Loss: 5.8311, Train Steps/Sec: 1.12 +[2025-04-28 18:18:54] (step=0024800) Train Loss: 5.8001, Train Steps/Sec: 1.12 +[2025-04-28 18:19:16] (step=0024825) Train Loss: 5.8418, Train Steps/Sec: 1.12 +[2025-04-28 18:19:38] (step=0024850) Train Loss: 5.8153, Train Steps/Sec: 1.12 +[2025-04-28 18:20:01] (step=0024875) Train Loss: 5.8516, Train Steps/Sec: 1.12 +[2025-04-28 18:20:23] (step=0024900) Train Loss: 5.8115, Train Steps/Sec: 1.12 +[2025-04-28 18:20:45] (step=0024925) Train Loss: 5.7978, Train Steps/Sec: 1.12 +[2025-04-28 18:21:07] (step=0024950) Train Loss: 5.7316, Train Steps/Sec: 1.12 +[2025-04-28 18:21:30] (step=0024975) Train Loss: 5.8020, Train Steps/Sec: 1.12 +[2025-04-28 18:21:52] (step=0025000) Train Loss: 5.7831, Train Steps/Sec: 1.12 +[2025-04-28 18:22:15] (step=0025025) Train Loss: 5.7303, Train Steps/Sec: 1.12 +[2025-04-28 18:22:37] (step=0025050) Train Loss: 5.7515, Train Steps/Sec: 1.12 +[2025-04-28 18:22:59] (step=0025075) Train Loss: 5.7632, Train Steps/Sec: 1.12 +[2025-04-28 18:23:21] (step=0025100) Train Loss: 5.7978, Train Steps/Sec: 1.12 +[2025-04-28 18:23:44] (step=0025125) Train Loss: 5.7717, Train Steps/Sec: 1.12 +[2025-04-28 18:24:06] (step=0025150) Train Loss: 5.8025, Train Steps/Sec: 1.12 +[2025-04-28 18:24:28] (step=0025175) Train Loss: 5.8365, Train Steps/Sec: 1.12 +[2025-04-28 18:24:51] (step=0025200) Train Loss: 5.7700, Train Steps/Sec: 1.12 +[2025-04-28 18:25:13] (step=0025225) Train Loss: 5.8032, Train Steps/Sec: 1.12 +[2025-04-28 18:25:35] (step=0025250) Train Loss: 5.7752, Train Steps/Sec: 1.12 +[2025-04-28 18:25:58] (step=0025275) Train Loss: 5.7929, Train Steps/Sec: 1.12 +[2025-04-28 18:26:20] (step=0025300) Train Loss: 5.7339, Train Steps/Sec: 1.12 +[2025-04-28 18:26:42] (step=0025325) Train Loss: 5.7483, Train Steps/Sec: 1.12 +[2025-04-28 18:27:04] (step=0025350) Train Loss: 5.7573, Train Steps/Sec: 1.12 +[2025-04-28 18:27:27] (step=0025375) Train Loss: 5.8258, Train Steps/Sec: 1.12 +[2025-04-28 18:27:49] (step=0025400) Train Loss: 5.7418, Train Steps/Sec: 1.12 +[2025-04-28 18:28:11] (step=0025425) Train Loss: 5.7913, Train Steps/Sec: 1.12 +[2025-04-28 18:28:34] (step=0025450) Train Loss: 5.7338, Train Steps/Sec: 1.12 +[2025-04-28 18:28:56] (step=0025475) Train Loss: 5.8132, Train Steps/Sec: 1.12 +[2025-04-28 18:29:18] (step=0025500) Train Loss: 5.8019, Train Steps/Sec: 1.12 +[2025-04-28 18:29:41] (step=0025525) Train Loss: 5.7642, Train Steps/Sec: 1.12 +[2025-04-28 18:30:03] (step=0025550) Train Loss: 5.7844, Train Steps/Sec: 1.12 +[2025-04-28 18:30:25] (step=0025575) Train Loss: 5.7659, Train Steps/Sec: 1.12 +[2025-04-28 18:30:48] (step=0025600) Train Loss: 5.7834, Train Steps/Sec: 1.12 +[2025-04-28 18:31:10] (step=0025625) Train Loss: 5.7418, Train Steps/Sec: 1.12 +[2025-04-28 18:31:32] (step=0025650) Train Loss: 5.7490, Train Steps/Sec: 1.12 +[2025-04-28 18:31:55] (step=0025675) Train Loss: 5.7936, Train Steps/Sec: 1.12 +[2025-04-28 18:32:17] (step=0025700) Train Loss: 5.8064, Train Steps/Sec: 1.12 +[2025-04-28 18:32:39] (step=0025725) Train Loss: 5.8796, Train Steps/Sec: 1.12 +[2025-04-28 18:33:01] (step=0025750) Train Loss: 5.8135, Train Steps/Sec: 1.12 +[2025-04-28 18:33:24] (step=0025775) Train Loss: 5.7404, Train Steps/Sec: 1.12 +[2025-04-28 18:33:46] (step=0025800) Train Loss: 5.7782, Train Steps/Sec: 1.12 +[2025-04-28 18:34:08] (step=0025825) Train Loss: 5.7650, Train Steps/Sec: 1.12 +[2025-04-28 18:34:31] (step=0025850) Train Loss: 5.7790, Train Steps/Sec: 1.12 +[2025-04-28 18:34:53] (step=0025875) Train Loss: 5.8129, Train Steps/Sec: 1.12 +[2025-04-28 18:35:15] (step=0025900) Train Loss: 5.7789, Train Steps/Sec: 1.12 +[2025-04-28 18:35:37] (step=0025925) Train Loss: 5.7799, Train Steps/Sec: 1.12 +[2025-04-28 18:36:00] (step=0025950) Train Loss: 5.8439, Train Steps/Sec: 1.12 +[2025-04-28 18:36:22] (step=0025975) Train Loss: 5.7907, Train Steps/Sec: 1.12 +[2025-04-28 18:36:45] (step=0026000) Train Loss: 5.7854, Train Steps/Sec: 1.11 +[2025-04-28 18:36:45] text_config is None. Initializing the text config with default values (`OPTConfig`). +[2025-04-28 18:44:02] Finish Eval in 26000 steps... +[2025-04-28 18:44:22] Saved checkpoint to checkpoint/X2I_UltraEdiit_fourmask_LLava_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_filtered_objects_4mask_1280_context_after_stage1_5e4/000-GPT-XL/checkpoints/0026000.pt +[2025-04-28 18:44:24] Removed old checkpoint: checkpoint/X2I_UltraEdiit_fourmask_LLava_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_filtered_objects_4mask_1280_context_after_stage1_5e4/000-GPT-XL/checkpoints/0024000.pt +[2025-04-28 18:44:46] (step=0026025) Train Loss: 5.8157, Train Steps/Sec: 0.05 +[2025-04-28 18:45:09] (step=0026050) Train Loss: 5.7660, Train Steps/Sec: 1.12 +[2025-04-28 18:45:31] (step=0026075) Train Loss: 5.7568, Train Steps/Sec: 1.12 +[2025-04-28 18:45:53] (step=0026100) Train Loss: 5.7773, Train Steps/Sec: 1.12 +[2025-04-28 18:46:16] (step=0026125) Train Loss: 5.7023, Train Steps/Sec: 1.12 +[2025-04-28 18:46:38] (step=0026150) Train Loss: 5.8370, Train Steps/Sec: 1.12 +[2025-04-28 18:47:00] (step=0026175) Train Loss: 5.8062, Train Steps/Sec: 1.12 +[2025-04-28 18:47:23] (step=0026200) Train Loss: 5.7596, Train Steps/Sec: 1.12 +[2025-04-28 18:47:45] (step=0026225) Train Loss: 5.7724, Train Steps/Sec: 1.12 +[2025-04-28 18:48:07] (step=0026250) Train Loss: 5.7880, Train Steps/Sec: 1.12 +[2025-04-28 18:48:30] (step=0026275) Train Loss: 5.7499, Train Steps/Sec: 1.12 +[2025-04-28 18:49:19] (step=0026300) Train Loss: 5.7684, Train Steps/Sec: 0.51 +[2025-04-28 18:50:10] (step=0026325) Train Loss: 5.7743, Train Steps/Sec: 0.49 +[2025-04-28 18:50:32] (step=0026350) Train Loss: 5.7724, Train Steps/Sec: 1.12 +[2025-04-28 18:51:03] (step=0026375) Train Loss: 5.7189, Train Steps/Sec: 0.80 +[2025-04-28 18:51:26] (step=0026400) Train Loss: 5.7312, Train Steps/Sec: 1.11 +[2025-04-28 18:51:48] (step=0026425) Train Loss: 5.7747, Train Steps/Sec: 1.12 +[2025-04-28 18:52:11] (step=0026450) Train Loss: 5.7679, Train Steps/Sec: 1.12 +[2025-04-28 18:52:33] (step=0026475) Train Loss: 5.8458, Train Steps/Sec: 1.12 +[2025-04-28 18:52:55] (step=0026500) Train Loss: 5.8095, Train Steps/Sec: 1.12 +[2025-04-28 18:53:18] (step=0026525) Train Loss: 5.8064, Train Steps/Sec: 1.12 +[2025-04-28 18:53:40] (step=0026550) Train Loss: 5.8509, Train Steps/Sec: 1.12 +[2025-04-28 18:54:02] (step=0026575) Train Loss: 5.8336, Train Steps/Sec: 1.12 +[2025-04-28 18:54:25] (step=0026600) Train Loss: 5.7799, Train Steps/Sec: 1.12 +[2025-04-28 18:54:47] (step=0026625) Train Loss: 5.8367, Train Steps/Sec: 1.12 +[2025-04-28 18:55:09] (step=0026650) Train Loss: 5.7712, Train Steps/Sec: 1.12 +[2025-04-28 18:55:32] (step=0026675) Train Loss: 5.7993, Train Steps/Sec: 1.12 +[2025-04-28 18:55:54] (step=0026700) Train Loss: 5.7314, Train Steps/Sec: 1.12 +[2025-04-28 18:56:16] (step=0026725) Train Loss: 5.7131, Train Steps/Sec: 1.12 +[2025-04-28 18:56:39] (step=0026750) Train Loss: 5.7987, Train Steps/Sec: 1.12 +[2025-04-28 18:57:01] (step=0026775) Train Loss: 5.8135, Train Steps/Sec: 1.12 +[2025-04-28 18:57:23] (step=0026800) Train Loss: 5.7490, Train Steps/Sec: 1.12 +[2025-04-28 18:57:46] (step=0026825) Train Loss: 5.7403, Train Steps/Sec: 1.12 +[2025-04-28 18:58:08] (step=0026850) Train Loss: 5.7657, Train Steps/Sec: 1.12 +[2025-04-28 18:58:30] (step=0026875) Train Loss: 5.7431, Train Steps/Sec: 1.12 +[2025-04-28 18:58:52] (step=0026900) Train Loss: 5.6861, Train Steps/Sec: 1.12 +[2025-04-28 18:59:15] (step=0026925) Train Loss: 5.8510, Train Steps/Sec: 1.12 +[2025-04-28 18:59:37] (step=0026950) Train Loss: 5.7466, Train Steps/Sec: 1.12 +[2025-04-28 18:59:59] (step=0026975) Train Loss: 5.8489, Train Steps/Sec: 1.12 +[2025-04-28 19:00:22] (step=0027000) Train Loss: 5.7876, Train Steps/Sec: 1.12 +[2025-04-28 19:00:44] (step=0027025) Train Loss: 5.7910, Train Steps/Sec: 1.12 +[2025-04-28 19:01:06] (step=0027050) Train Loss: 5.7989, Train Steps/Sec: 1.12 +[2025-04-28 19:01:29] (step=0027075) Train Loss: 5.7440, Train Steps/Sec: 1.12 +[2025-04-28 19:01:51] (step=0027100) Train Loss: 5.7788, Train Steps/Sec: 1.12 +[2025-04-28 19:02:13] (step=0027125) Train Loss: 5.7670, Train Steps/Sec: 1.12 +[2025-04-28 19:02:36] (step=0027150) Train Loss: 5.8336, Train Steps/Sec: 1.12 +[2025-04-28 19:02:58] (step=0027175) Train Loss: 5.6953, Train Steps/Sec: 1.12 +[2025-04-28 19:03:20] (step=0027200) Train Loss: 5.7796, Train Steps/Sec: 1.12 +[2025-04-28 19:03:43] (step=0027225) Train Loss: 5.7483, Train Steps/Sec: 1.12 +[2025-04-28 19:04:05] (step=0027250) Train Loss: 5.7692, Train Steps/Sec: 1.12 +[2025-04-28 19:04:27] (step=0027275) Train Loss: 5.7820, Train Steps/Sec: 1.12 +[2025-04-28 19:04:50] (step=0027300) Train Loss: 5.8047, Train Steps/Sec: 1.12 +[2025-04-28 19:05:12] (step=0027325) Train Loss: 5.8264, Train Steps/Sec: 1.12 +[2025-04-28 19:05:34] (step=0027350) Train Loss: 5.7881, Train Steps/Sec: 1.12 +[2025-04-28 19:05:57] (step=0027375) Train Loss: 5.7213, Train Steps/Sec: 1.12 +[2025-04-28 19:06:19] (step=0027400) Train Loss: 5.7100, Train Steps/Sec: 1.12 +[2025-04-28 19:06:41] (step=0027425) Train Loss: 5.7808, Train Steps/Sec: 1.12 +[2025-04-28 19:07:04] (step=0027450) Train Loss: 5.7842, Train Steps/Sec: 1.12 +[2025-04-28 19:07:26] (step=0027475) Train Loss: 5.7789, Train Steps/Sec: 1.12 +[2025-04-28 19:07:48] (step=0027500) Train Loss: 5.7548, Train Steps/Sec: 1.12 +[2025-04-28 19:08:11] (step=0027525) Train Loss: 5.8317, Train Steps/Sec: 1.12 +[2025-04-28 19:08:33] (step=0027550) Train Loss: 5.7657, Train Steps/Sec: 1.12 +[2025-04-28 19:08:55] (step=0027575) Train Loss: 5.7479, Train Steps/Sec: 1.12 +[2025-04-28 19:09:18] (step=0027600) Train Loss: 5.8683, Train Steps/Sec: 1.12 +[2025-04-28 19:09:40] (step=0027625) Train Loss: 5.7908, Train Steps/Sec: 1.12 +[2025-04-28 19:10:02] (step=0027650) Train Loss: 5.7558, Train Steps/Sec: 1.12 +[2025-04-28 19:10:25] (step=0027675) Train Loss: 5.7499, Train Steps/Sec: 1.12 +[2025-04-28 19:10:47] (step=0027700) Train Loss: 5.7164, Train Steps/Sec: 1.12 +[2025-04-28 19:11:09] (step=0027725) Train Loss: 5.7763, Train Steps/Sec: 1.12 +[2025-04-28 19:11:32] (step=0027750) Train Loss: 5.7482, Train Steps/Sec: 1.12 +[2025-04-28 19:11:54] (step=0027775) Train Loss: 5.6903, Train Steps/Sec: 1.12 +[2025-04-28 19:12:16] (step=0027800) Train Loss: 5.7504, Train Steps/Sec: 1.11 +[2025-04-28 19:12:39] (step=0027825) Train Loss: 5.7791, Train Steps/Sec: 1.12 +[2025-04-28 19:13:01] (step=0027850) Train Loss: 5.6717, Train Steps/Sec: 1.12 +[2025-04-28 19:13:23] (step=0027875) Train Loss: 5.8017, Train Steps/Sec: 1.12 +[2025-04-28 19:13:45] (step=0027900) Train Loss: 5.7312, Train Steps/Sec: 1.12 +[2025-04-28 19:14:08] (step=0027925) Train Loss: 5.8101, Train Steps/Sec: 1.12 +[2025-04-28 19:14:30] (step=0027950) Train Loss: 5.8122, Train Steps/Sec: 1.12 +[2025-04-28 19:14:52] (step=0027975) Train Loss: 5.7779, Train Steps/Sec: 1.12 +[2025-04-28 19:15:15] (step=0028000) Train Loss: 5.7598, Train Steps/Sec: 1.12 +[2025-04-28 19:15:15] text_config is None. Initializing the text config with default values (`OPTConfig`). +[2025-04-28 19:22:34] Finish Eval in 28000 steps... +[2025-04-28 19:22:54] Saved checkpoint to checkpoint/X2I_UltraEdiit_fourmask_LLava_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_filtered_objects_4mask_1280_context_after_stage1_5e4/000-GPT-XL/checkpoints/0028000.pt +[2025-04-28 19:22:56] Removed old checkpoint: checkpoint/X2I_UltraEdiit_fourmask_LLava_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_filtered_objects_4mask_1280_context_after_stage1_5e4/000-GPT-XL/checkpoints/0026000.pt +[2025-04-28 19:23:18] (step=0028025) Train Loss: 5.7491, Train Steps/Sec: 0.05 +[2025-04-28 19:23:41] (step=0028050) Train Loss: 5.7993, Train Steps/Sec: 1.12 +[2025-04-28 19:24:03] (step=0028075) Train Loss: 5.7361, Train Steps/Sec: 1.12 +[2025-04-28 19:24:25] (step=0028100) Train Loss: 5.7470, Train Steps/Sec: 1.12 +[2025-04-28 19:24:48] (step=0028125) Train Loss: 5.7701, Train Steps/Sec: 1.12 +[2025-04-28 19:25:10] (step=0028150) Train Loss: 5.7818, Train Steps/Sec: 1.12 +[2025-04-28 19:25:32] (step=0028175) Train Loss: 5.7242, Train Steps/Sec: 1.12 +[2025-04-28 19:25:55] (step=0028200) Train Loss: 5.7440, Train Steps/Sec: 1.12 +[2025-04-28 19:26:17] (step=0028225) Train Loss: 5.7221, Train Steps/Sec: 1.12 +[2025-04-28 19:26:39] (step=0028250) Train Loss: 5.7925, Train Steps/Sec: 1.12 +[2025-04-28 19:27:01] (step=0028275) Train Loss: 5.8085, Train Steps/Sec: 1.12 +[2025-04-28 19:27:24] (step=0028300) Train Loss: 5.7734, Train Steps/Sec: 1.12 +[2025-04-28 19:27:46] (step=0028325) Train Loss: 5.8093, Train Steps/Sec: 1.12 +[2025-04-28 19:28:09] (step=0028350) Train Loss: 5.7456, Train Steps/Sec: 1.11 +[2025-04-28 19:28:31] (step=0028375) Train Loss: 5.6855, Train Steps/Sec: 1.12 +[2025-04-28 19:28:53] (step=0028400) Train Loss: 5.7576, Train Steps/Sec: 1.12 +[2025-04-28 19:29:16] (step=0028425) Train Loss: 5.7682, Train Steps/Sec: 1.12 +[2025-04-28 19:29:38] (step=0028450) Train Loss: 5.7377, Train Steps/Sec: 1.12 +[2025-04-28 19:30:00] (step=0028475) Train Loss: 5.7951, Train Steps/Sec: 1.12 +[2025-04-28 19:30:23] (step=0028500) Train Loss: 5.7360, Train Steps/Sec: 1.12 +[2025-04-28 19:30:45] (step=0028525) Train Loss: 5.7532, Train Steps/Sec: 1.12 +[2025-04-28 19:31:07] (step=0028550) Train Loss: 5.8186, Train Steps/Sec: 1.12 +[2025-04-28 19:31:29] (step=0028575) Train Loss: 5.7906, Train Steps/Sec: 1.12 +[2025-04-28 19:31:52] (step=0028600) Train Loss: 5.7042, Train Steps/Sec: 1.12 +[2025-04-28 19:32:14] (step=0028625) Train Loss: 5.7526, Train Steps/Sec: 1.12 +[2025-04-28 19:32:36] (step=0028650) Train Loss: 5.7657, Train Steps/Sec: 1.12 +[2025-04-28 19:32:59] (step=0028675) Train Loss: 5.7706, Train Steps/Sec: 1.12 +[2025-04-28 19:33:21] (step=0028700) Train Loss: 5.7701, Train Steps/Sec: 1.12 +[2025-04-28 19:33:43] (step=0028725) Train Loss: 5.6989, Train Steps/Sec: 1.12 +[2025-04-28 19:34:06] (step=0028750) Train Loss: 5.7890, Train Steps/Sec: 1.12 +[2025-04-28 19:34:28] (step=0028775) Train Loss: 5.7170, Train Steps/Sec: 1.12 +[2025-04-28 19:34:50] (step=0028800) Train Loss: 5.7648, Train Steps/Sec: 1.12 +[2025-04-28 19:35:13] (step=0028825) Train Loss: 5.7906, Train Steps/Sec: 1.12 +[2025-04-28 19:35:35] (step=0028850) Train Loss: 5.7798, Train Steps/Sec: 1.12 +[2025-04-28 19:35:57] (step=0028875) Train Loss: 5.7935, Train Steps/Sec: 1.12 +[2025-04-28 19:36:20] (step=0028900) Train Loss: 5.7783, Train Steps/Sec: 1.12 +[2025-04-28 19:36:42] (step=0028925) Train Loss: 5.8133, Train Steps/Sec: 1.12 +[2025-04-28 19:37:04] (step=0028950) Train Loss: 5.7790, Train Steps/Sec: 1.12 +[2025-04-28 19:37:26] (step=0028975) Train Loss: 5.7062, Train Steps/Sec: 1.12 +[2025-04-28 19:37:49] (step=0029000) Train Loss: 5.7164, Train Steps/Sec: 1.12 +[2025-04-28 19:38:11] (step=0029025) Train Loss: 5.7563, Train Steps/Sec: 1.12 +[2025-04-28 19:38:33] (step=0029050) Train Loss: 5.7928, Train Steps/Sec: 1.12 +[2025-04-28 19:38:56] (step=0029075) Train Loss: 5.7335, Train Steps/Sec: 1.12 +[2025-04-28 19:39:18] (step=0029100) Train Loss: 5.7275, Train Steps/Sec: 1.12 +[2025-04-28 19:39:40] (step=0029125) Train Loss: 5.7662, Train Steps/Sec: 1.12 +[2025-04-28 19:40:03] (step=0029150) Train Loss: 5.6929, Train Steps/Sec: 1.12 +[2025-04-28 19:40:25] (step=0029175) Train Loss: 5.7550, Train Steps/Sec: 1.12 +[2025-04-28 19:40:47] (step=0029200) Train Loss: 5.7757, Train Steps/Sec: 1.12 +[2025-04-28 19:41:10] (step=0029225) Train Loss: 5.7420, Train Steps/Sec: 1.12 +[2025-04-28 19:41:32] (step=0029250) Train Loss: 5.8289, Train Steps/Sec: 1.12 +[2025-04-28 19:41:54] (step=0029275) Train Loss: 5.7030, Train Steps/Sec: 1.12 +[2025-04-28 19:42:17] (step=0029300) Train Loss: 5.7142, Train Steps/Sec: 1.12 +[2025-04-28 19:42:39] (step=0029325) Train Loss: 5.7175, Train Steps/Sec: 1.12 +[2025-04-28 19:43:01] (step=0029350) Train Loss: 5.7881, Train Steps/Sec: 1.12 +[2025-04-28 19:43:23] (step=0029375) Train Loss: 5.7740, Train Steps/Sec: 1.12 +[2025-04-28 19:43:46] (step=0029400) Train Loss: 5.8085, Train Steps/Sec: 1.12 +[2025-04-28 19:44:08] (step=0029425) Train Loss: 5.7718, Train Steps/Sec: 1.12 +[2025-04-28 19:44:30] (step=0029450) Train Loss: 5.7364, Train Steps/Sec: 1.12 +[2025-04-28 19:44:53] (step=0029475) Train Loss: 5.7540, Train Steps/Sec: 1.12 +[2025-04-28 19:45:15] (step=0029500) Train Loss: 5.7341, Train Steps/Sec: 1.12 +[2025-04-28 19:45:37] (step=0029525) Train Loss: 5.8173, Train Steps/Sec: 1.12 +[2025-04-28 19:46:00] (step=0029550) Train Loss: 5.7843, Train Steps/Sec: 1.12 +[2025-04-28 19:46:22] (step=0029575) Train Loss: 5.6721, Train Steps/Sec: 1.12 +[2025-04-28 19:46:44] (step=0029600) Train Loss: 5.7916, Train Steps/Sec: 1.11 +[2025-04-28 19:47:07] (step=0029625) Train Loss: 5.7758, Train Steps/Sec: 1.12 +[2025-04-28 19:47:29] (step=0029650) Train Loss: 5.7513, Train Steps/Sec: 1.12 +[2025-04-28 19:47:51] (step=0029675) Train Loss: 5.8102, Train Steps/Sec: 1.12 +[2025-04-28 19:48:14] (step=0029700) Train Loss: 5.7195, Train Steps/Sec: 1.12 +[2025-04-28 19:48:36] (step=0029725) Train Loss: 5.7518, Train Steps/Sec: 1.12 +[2025-04-28 19:48:58] (step=0029750) Train Loss: 5.7434, Train Steps/Sec: 1.12 +[2025-04-28 19:49:21] (step=0029775) Train Loss: 5.7848, Train Steps/Sec: 1.12 +[2025-04-28 19:49:43] (step=0029800) Train Loss: 5.7478, Train Steps/Sec: 1.12 +[2025-04-28 19:50:05] (step=0029825) Train Loss: 5.7437, Train Steps/Sec: 1.12 +[2025-04-28 19:50:45] (step=0029850) Train Loss: 5.7620, Train Steps/Sec: 0.63 +[2025-04-28 19:51:27] (step=0029875) Train Loss: 5.7785, Train Steps/Sec: 0.60 +[2025-04-28 19:51:58] (step=0029900) Train Loss: 5.7832, Train Steps/Sec: 0.79 +[2025-04-28 19:52:30] (step=0029925) Train Loss: 5.7612, Train Steps/Sec: 0.79 +[2025-04-28 19:52:52] (step=0029950) Train Loss: 5.7649, Train Steps/Sec: 1.12 +[2025-04-28 19:53:14] (step=0029975) Train Loss: 5.7484, Train Steps/Sec: 1.12 +[2025-04-28 19:53:37] (step=0030000) Train Loss: 5.7199, Train Steps/Sec: 1.12 +[2025-04-28 19:53:37] text_config is None. Initializing the text config with default values (`OPTConfig`). +[2025-04-28 20:00:57] Finish Eval in 30000 steps... +[2025-04-28 20:01:17] Saved checkpoint to checkpoint/X2I_UltraEdiit_fourmask_LLava_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_filtered_objects_4mask_1280_context_after_stage1_5e4/000-GPT-XL/checkpoints/0030000.pt +[2025-04-28 20:01:19] Removed old checkpoint: checkpoint/X2I_UltraEdiit_fourmask_LLava_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_filtered_objects_4mask_1280_context_after_stage1_5e4/000-GPT-XL/checkpoints/0028000.pt +[2025-04-28 20:01:41] (step=0030025) Train Loss: 5.8017, Train Steps/Sec: 0.05 +[2025-04-28 20:02:04] (step=0030050) Train Loss: 5.7140, Train Steps/Sec: 1.12 +[2025-04-28 20:02:26] (step=0030075) Train Loss: 5.7819, Train Steps/Sec: 1.12 +[2025-04-28 20:02:48] (step=0030100) Train Loss: 5.7306, Train Steps/Sec: 1.12 +[2025-04-28 20:03:11] (step=0030125) Train Loss: 5.6795, Train Steps/Sec: 1.12 +[2025-04-28 20:03:33] (step=0030150) Train Loss: 5.7199, Train Steps/Sec: 1.12 +[2025-04-28 20:03:55] (step=0030175) Train Loss: 5.7943, Train Steps/Sec: 1.12 +[2025-04-28 20:04:18] (step=0030200) Train Loss: 5.7593, Train Steps/Sec: 1.11 +[2025-04-28 20:04:40] (step=0030225) Train Loss: 5.7964, Train Steps/Sec: 1.12 +[2025-04-28 20:05:03] (step=0030250) Train Loss: 5.7190, Train Steps/Sec: 1.11 +[2025-04-28 20:05:25] (step=0030275) Train Loss: 5.7197, Train Steps/Sec: 1.12 +[2025-04-28 20:05:47] (step=0030300) Train Loss: 5.8365, Train Steps/Sec: 1.12 +[2025-04-28 20:06:09] (step=0030325) Train Loss: 5.7860, Train Steps/Sec: 1.12 +[2025-04-28 20:06:32] (step=0030350) Train Loss: 5.7517, Train Steps/Sec: 1.12 +[2025-04-28 20:06:54] (step=0030375) Train Loss: 5.6922, Train Steps/Sec: 1.12 +[2025-04-28 20:07:16] (step=0030400) Train Loss: 5.7484, Train Steps/Sec: 1.11 +[2025-04-28 20:07:39] (step=0030425) Train Loss: 5.6814, Train Steps/Sec: 1.12 +[2025-04-28 20:08:01] (step=0030450) Train Loss: 5.7319, Train Steps/Sec: 1.12 +[2025-04-28 20:08:23] (step=0030475) Train Loss: 5.8237, Train Steps/Sec: 1.12 +[2025-04-28 20:08:46] (step=0030500) Train Loss: 5.7452, Train Steps/Sec: 1.12 +[2025-04-28 20:09:08] (step=0030525) Train Loss: 5.7061, Train Steps/Sec: 1.12 +[2025-04-28 20:09:30] (step=0030550) Train Loss: 5.7481, Train Steps/Sec: 1.12 +[2025-04-28 20:09:53] (step=0030575) Train Loss: 5.7504, Train Steps/Sec: 1.12 +[2025-04-28 20:10:15] (step=0030600) Train Loss: 5.7360, Train Steps/Sec: 1.12 +[2025-04-28 20:10:37] (step=0030625) Train Loss: 5.6676, Train Steps/Sec: 1.12 +[2025-04-28 20:11:00] (step=0030650) Train Loss: 5.7608, Train Steps/Sec: 1.12 +[2025-04-28 20:11:22] (step=0030675) Train Loss: 5.6693, Train Steps/Sec: 1.12 +[2025-04-28 20:11:44] (step=0030700) Train Loss: 5.7223, Train Steps/Sec: 1.12 +[2025-04-28 20:12:07] (step=0030725) Train Loss: 5.7608, Train Steps/Sec: 1.12 +[2025-04-28 20:12:29] (step=0030750) Train Loss: 5.7139, Train Steps/Sec: 1.12 +[2025-04-28 20:12:51] (step=0030775) Train Loss: 5.7359, Train Steps/Sec: 1.12 +[2025-04-28 20:13:14] (step=0030800) Train Loss: 5.7503, Train Steps/Sec: 1.12 +[2025-04-28 20:13:36] (step=0030825) Train Loss: 5.7402, Train Steps/Sec: 1.12 +[2025-04-28 20:13:58] (step=0030850) Train Loss: 5.7258, Train Steps/Sec: 1.12 +[2025-04-28 20:14:21] (step=0030875) Train Loss: 5.7663, Train Steps/Sec: 1.12 +[2025-04-28 20:14:43] (step=0030900) Train Loss: 5.6854, Train Steps/Sec: 1.12 +[2025-04-28 20:15:05] (step=0030925) Train Loss: 5.7526, Train Steps/Sec: 1.12 +[2025-04-28 20:15:28] (step=0030950) Train Loss: 5.7828, Train Steps/Sec: 1.12 +[2025-04-28 20:15:50] (step=0030975) Train Loss: 5.7292, Train Steps/Sec: 1.12 +[2025-04-28 20:16:12] (step=0031000) Train Loss: 5.6985, Train Steps/Sec: 1.12 +[2025-04-28 20:16:34] (step=0031025) Train Loss: 5.8217, Train Steps/Sec: 1.12 +[2025-04-28 20:16:57] (step=0031050) Train Loss: 5.7229, Train Steps/Sec: 1.12 +[2025-04-28 20:17:19] (step=0031075) Train Loss: 5.7492, Train Steps/Sec: 1.12 +[2025-04-28 20:17:41] (step=0031100) Train Loss: 5.7321, Train Steps/Sec: 1.12 +[2025-04-28 20:18:04] (step=0031125) Train Loss: 5.7653, Train Steps/Sec: 1.12 +[2025-04-28 20:18:26] (step=0031150) Train Loss: 5.7325, Train Steps/Sec: 1.12 +[2025-04-28 20:18:48] (step=0031175) Train Loss: 5.7317, Train Steps/Sec: 1.12 +[2025-04-28 20:19:11] (step=0031200) Train Loss: 5.7444, Train Steps/Sec: 1.12 +[2025-04-28 20:19:33] (step=0031225) Train Loss: 5.7337, Train Steps/Sec: 1.12 +[2025-04-28 20:19:55] (step=0031250) Train Loss: 5.7173, Train Steps/Sec: 1.12 +[2025-04-28 20:20:18] (step=0031275) Train Loss: 5.7721, Train Steps/Sec: 1.12 +[2025-04-28 20:20:40] (step=0031300) Train Loss: 5.7619, Train Steps/Sec: 1.12 +[2025-04-28 20:21:02] (step=0031325) Train Loss: 5.7400, Train Steps/Sec: 1.12 +[2025-04-28 20:21:25] (step=0031350) Train Loss: 5.7561, Train Steps/Sec: 1.12 +[2025-04-28 20:21:47] (step=0031375) Train Loss: 5.7455, Train Steps/Sec: 1.12 +[2025-04-28 20:22:09] (step=0031400) Train Loss: 5.7293, Train Steps/Sec: 1.12 +[2025-04-28 20:22:32] (step=0031425) Train Loss: 5.7178, Train Steps/Sec: 1.12 +[2025-04-28 20:22:54] (step=0031450) Train Loss: 5.6944, Train Steps/Sec: 1.12 +[2025-04-28 20:23:16] (step=0031475) Train Loss: 5.6916, Train Steps/Sec: 1.12 +[2025-04-28 20:23:38] (step=0031500) Train Loss: 5.6788, Train Steps/Sec: 1.12 +[2025-04-28 20:24:01] (step=0031525) Train Loss: 5.7317, Train Steps/Sec: 1.12 +[2025-04-28 20:24:23] (step=0031550) Train Loss: 5.7233, Train Steps/Sec: 1.12 +[2025-04-28 20:24:45] (step=0031575) Train Loss: 5.8137, Train Steps/Sec: 1.12 +[2025-04-28 20:25:08] (step=0031600) Train Loss: 5.7100, Train Steps/Sec: 1.12 +[2025-04-28 20:25:30] (step=0031625) Train Loss: 5.7066, Train Steps/Sec: 1.12 +[2025-04-28 20:25:52] (step=0031650) Train Loss: 5.6537, Train Steps/Sec: 1.12 +[2025-04-28 20:26:15] (step=0031675) Train Loss: 5.7534, Train Steps/Sec: 1.12 +[2025-04-28 20:26:37] (step=0031700) Train Loss: 5.7189, Train Steps/Sec: 1.12 +[2025-04-28 20:26:59] (step=0031725) Train Loss: 5.7719, Train Steps/Sec: 1.12 +[2025-04-28 20:27:22] (step=0031750) Train Loss: 5.6816, Train Steps/Sec: 1.12 +[2025-04-28 20:27:44] (step=0031775) Train Loss: 5.8111, Train Steps/Sec: 1.12 +[2025-04-28 20:28:06] (step=0031800) Train Loss: 5.7296, Train Steps/Sec: 1.12 +[2025-04-28 20:28:29] (step=0031825) Train Loss: 5.7066, Train Steps/Sec: 1.12 +[2025-04-28 20:28:51] (step=0031850) Train Loss: 5.7197, Train Steps/Sec: 1.12 +[2025-04-28 20:29:13] (step=0031875) Train Loss: 5.7455, Train Steps/Sec: 1.12 +[2025-04-28 20:29:35] (step=0031900) Train Loss: 5.7541, Train Steps/Sec: 1.12 +[2025-04-28 20:29:58] (step=0031925) Train Loss: 5.7562, Train Steps/Sec: 1.12 +[2025-04-28 20:30:20] (step=0031950) Train Loss: 5.7626, Train Steps/Sec: 1.12 +[2025-04-28 20:30:42] (step=0031975) Train Loss: 5.7301, Train Steps/Sec: 1.12 +[2025-04-28 20:31:05] (step=0032000) Train Loss: 5.7126, Train Steps/Sec: 1.12 +[2025-04-28 20:31:05] text_config is None. Initializing the text config with default values (`OPTConfig`). +[2025-04-28 20:38:22] Finish Eval in 32000 steps... +[2025-04-28 20:38:41] Saved checkpoint to checkpoint/X2I_UltraEdiit_fourmask_LLava_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_filtered_objects_4mask_1280_context_after_stage1_5e4/000-GPT-XL/checkpoints/0032000.pt +[2025-04-28 20:38:43] Removed old checkpoint: checkpoint/X2I_UltraEdiit_fourmask_LLava_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_filtered_objects_4mask_1280_context_after_stage1_5e4/000-GPT-XL/checkpoints/0030000.pt +[2025-04-28 20:39:06] (step=0032025) Train Loss: 5.6773, Train Steps/Sec: 0.05 +[2025-04-28 20:39:28] (step=0032050) Train Loss: 5.8275, Train Steps/Sec: 1.12 +[2025-04-28 20:39:50] (step=0032075) Train Loss: 5.7273, Train Steps/Sec: 1.12 +[2025-04-28 20:40:13] (step=0032100) Train Loss: 5.8341, Train Steps/Sec: 1.12 +[2025-04-28 20:40:35] (step=0032125) Train Loss: 5.7492, Train Steps/Sec: 1.11 +[2025-04-28 20:40:57] (step=0032150) Train Loss: 5.7623, Train Steps/Sec: 1.12 +[2025-04-28 20:41:20] (step=0032175) Train Loss: 5.7361, Train Steps/Sec: 1.12 +[2025-04-28 20:41:42] (step=0032200) Train Loss: 5.7357, Train Steps/Sec: 1.12 +[2025-04-28 20:42:04] (step=0032225) Train Loss: 5.6968, Train Steps/Sec: 1.12 +[2025-04-28 20:42:27] (step=0032250) Train Loss: 5.6980, Train Steps/Sec: 1.12 +[2025-04-28 20:42:49] (step=0032275) Train Loss: 5.7434, Train Steps/Sec: 1.12 +[2025-04-28 20:43:11] (step=0032300) Train Loss: 5.7372, Train Steps/Sec: 1.12 +[2025-04-28 20:43:33] (step=0032325) Train Loss: 5.7245, Train Steps/Sec: 1.12 +[2025-04-28 20:43:56] (step=0032350) Train Loss: 5.7384, Train Steps/Sec: 1.12 +[2025-04-28 20:44:18] (step=0032375) Train Loss: 5.7279, Train Steps/Sec: 1.12 +[2025-04-28 20:44:40] (step=0032400) Train Loss: 5.6804, Train Steps/Sec: 1.12 +[2025-04-28 20:45:03] (step=0032425) Train Loss: 5.7007, Train Steps/Sec: 1.12 +[2025-04-28 20:45:25] (step=0032450) Train Loss: 5.6885, Train Steps/Sec: 1.12 +[2025-04-28 20:45:47] (step=0032475) Train Loss: 5.7891, Train Steps/Sec: 1.12 +[2025-04-28 20:46:09] (step=0032500) Train Loss: 5.7389, Train Steps/Sec: 1.12 +[2025-04-28 20:46:32] (step=0032525) Train Loss: 5.7544, Train Steps/Sec: 1.12 +[2025-04-28 20:46:54] (step=0032550) Train Loss: 5.7177, Train Steps/Sec: 1.12 +[2025-04-28 20:47:16] (step=0032575) Train Loss: 5.6903, Train Steps/Sec: 1.12 +[2025-04-28 20:47:39] (step=0032600) Train Loss: 5.7532, Train Steps/Sec: 1.12 +[2025-04-28 20:48:01] (step=0032625) Train Loss: 5.7229, Train Steps/Sec: 1.12 +[2025-04-28 20:48:23] (step=0032650) Train Loss: 5.7315, Train Steps/Sec: 1.12 +[2025-04-28 20:48:45] (step=0032675) Train Loss: 5.6599, Train Steps/Sec: 1.12 +[2025-04-28 20:49:08] (step=0032700) Train Loss: 5.7817, Train Steps/Sec: 1.12 +[2025-04-28 20:49:30] (step=0032725) Train Loss: 5.6754, Train Steps/Sec: 1.12 +[2025-04-28 20:49:52] (step=0032750) Train Loss: 5.7331, Train Steps/Sec: 1.12 +[2025-04-28 20:50:15] (step=0032775) Train Loss: 5.7383, Train Steps/Sec: 1.12 +[2025-04-28 20:50:37] (step=0032800) Train Loss: 5.6960, Train Steps/Sec: 1.12 +[2025-04-28 20:50:59] (step=0032825) Train Loss: 5.7684, Train Steps/Sec: 1.12 +[2025-04-28 20:51:22] (step=0032850) Train Loss: 5.6890, Train Steps/Sec: 1.12 +[2025-04-28 20:51:44] (step=0032875) Train Loss: 5.7248, Train Steps/Sec: 1.12 +[2025-04-28 20:52:06] (step=0032900) Train Loss: 5.6911, Train Steps/Sec: 1.12 +[2025-04-28 20:52:28] (step=0032925) Train Loss: 5.7395, Train Steps/Sec: 1.12 +[2025-04-28 20:52:51] (step=0032950) Train Loss: 5.7640, Train Steps/Sec: 1.12 +[2025-04-28 20:53:13] (step=0032975) Train Loss: 5.7362, Train Steps/Sec: 1.12 +[2025-04-28 20:53:35] (step=0033000) Train Loss: 5.8173, Train Steps/Sec: 1.12 +[2025-04-28 20:53:58] (step=0033025) Train Loss: 5.7319, Train Steps/Sec: 1.12 +[2025-04-28 20:54:20] (step=0033050) Train Loss: 5.7222, Train Steps/Sec: 1.12 +[2025-04-28 20:54:42] (step=0033075) Train Loss: 5.7329, Train Steps/Sec: 1.12 +[2025-04-28 20:55:05] (step=0033100) Train Loss: 5.7163, Train Steps/Sec: 1.12 +[2025-04-28 20:55:27] (step=0033125) Train Loss: 5.7226, Train Steps/Sec: 1.12 +[2025-04-28 20:55:49] (step=0033150) Train Loss: 5.7942, Train Steps/Sec: 1.12 +[2025-04-28 20:56:11] (step=0033175) Train Loss: 5.7732, Train Steps/Sec: 1.12 +[2025-04-28 20:56:34] (step=0033200) Train Loss: 5.7491, Train Steps/Sec: 1.12 +[2025-04-28 20:56:56] (step=0033225) Train Loss: 5.6619, Train Steps/Sec: 1.12 +[2025-04-28 20:57:18] (step=0033250) Train Loss: 5.6865, Train Steps/Sec: 1.12 +[2025-04-28 20:57:41] (step=0033275) Train Loss: 5.7316, Train Steps/Sec: 1.12 +[2025-04-28 20:58:03] (step=0033300) Train Loss: 5.7431, Train Steps/Sec: 1.12 +[2025-04-28 20:58:25] (step=0033325) Train Loss: 5.7587, Train Steps/Sec: 1.12 +[2025-04-28 20:58:47] (step=0033350) Train Loss: 5.7638, Train Steps/Sec: 1.12 +[2025-04-28 20:59:10] (step=0033375) Train Loss: 5.7484, Train Steps/Sec: 1.12 +[2025-04-28 20:59:32] (step=0033400) Train Loss: 5.7428, Train Steps/Sec: 1.12 +[2025-04-28 21:00:12] (step=0033425) Train Loss: 5.7329, Train Steps/Sec: 0.62 +[2025-04-28 21:00:54] (step=0033450) Train Loss: 5.7259, Train Steps/Sec: 0.60 +[2025-04-28 21:01:25] (step=0033475) Train Loss: 5.6997, Train Steps/Sec: 0.80 +[2025-04-28 21:01:57] (step=0033500) Train Loss: 5.8014, Train Steps/Sec: 0.80 +[2025-04-28 21:02:19] (step=0033525) Train Loss: 5.7161, Train Steps/Sec: 1.12 +[2025-04-28 21:02:41] (step=0033550) Train Loss: 5.7739, Train Steps/Sec: 1.12 +[2025-04-28 21:03:03] (step=0033575) Train Loss: 5.7089, Train Steps/Sec: 1.12 +[2025-04-28 21:03:26] (step=0033600) Train Loss: 5.7108, Train Steps/Sec: 1.12 +[2025-04-28 21:03:48] (step=0033625) Train Loss: 5.7595, Train Steps/Sec: 1.13 +[2025-04-28 21:04:10] (step=0033650) Train Loss: 5.7198, Train Steps/Sec: 1.12 +[2025-04-28 21:04:33] (step=0033675) Train Loss: 5.6692, Train Steps/Sec: 1.12 +[2025-04-28 21:04:55] (step=0033700) Train Loss: 5.7307, Train Steps/Sec: 1.12 +[2025-04-28 21:05:17] (step=0033725) Train Loss: 5.7001, Train Steps/Sec: 1.12 +[2025-04-28 21:05:40] (step=0033750) Train Loss: 5.7337, Train Steps/Sec: 1.12 +[2025-04-28 21:06:02] (step=0033775) Train Loss: 5.7112, Train Steps/Sec: 1.12 +[2025-04-28 21:06:24] (step=0033800) Train Loss: 5.7075, Train Steps/Sec: 1.11 +[2025-04-28 21:06:47] (step=0033825) Train Loss: 5.7779, Train Steps/Sec: 1.12 +[2025-04-28 21:07:09] (step=0033850) Train Loss: 5.7571, Train Steps/Sec: 1.12 +[2025-04-28 21:07:31] (step=0033875) Train Loss: 5.7889, Train Steps/Sec: 1.12 +[2025-04-28 21:07:53] (step=0033900) Train Loss: 5.7674, Train Steps/Sec: 1.12 +[2025-04-28 21:08:16] (step=0033925) Train Loss: 5.7743, Train Steps/Sec: 1.12 +[2025-04-28 21:08:38] (step=0033950) Train Loss: 5.7238, Train Steps/Sec: 1.12 +[2025-04-28 21:09:00] (step=0033975) Train Loss: 5.7147, Train Steps/Sec: 1.12 +[2025-04-28 21:09:23] (step=0034000) Train Loss: 5.7533, Train Steps/Sec: 1.12 +[2025-04-28 21:09:23] text_config is None. Initializing the text config with default values (`OPTConfig`). +[2025-04-28 21:16:42] Finish Eval in 34000 steps... +[2025-04-28 21:17:02] Saved checkpoint to checkpoint/X2I_UltraEdiit_fourmask_LLava_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_filtered_objects_4mask_1280_context_after_stage1_5e4/000-GPT-XL/checkpoints/0034000.pt +[2025-04-28 21:17:04] Removed old checkpoint: checkpoint/X2I_UltraEdiit_fourmask_LLava_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_filtered_objects_4mask_1280_context_after_stage1_5e4/000-GPT-XL/checkpoints/0032000.pt +[2025-04-28 21:17:27] (step=0034025) Train Loss: 5.7266, Train Steps/Sec: 0.05 +[2025-04-28 21:17:49] (step=0034050) Train Loss: 5.7174, Train Steps/Sec: 1.12 +[2025-04-28 21:18:11] (step=0034075) Train Loss: 5.7087, Train Steps/Sec: 1.12 +[2025-04-28 21:18:33] (step=0034100) Train Loss: 5.7833, Train Steps/Sec: 1.12 +[2025-04-28 21:18:56] (step=0034125) Train Loss: 5.7219, Train Steps/Sec: 1.12 +[2025-04-28 21:19:18] (step=0034150) Train Loss: 5.7024, Train Steps/Sec: 1.12 +[2025-04-28 21:19:40] (step=0034175) Train Loss: 5.6929, Train Steps/Sec: 1.12 +[2025-04-28 21:20:03] (step=0034200) Train Loss: 5.7879, Train Steps/Sec: 1.12 +[2025-04-28 21:20:25] (step=0034225) Train Loss: 5.7549, Train Steps/Sec: 1.12 +[2025-04-28 21:20:47] (step=0034250) Train Loss: 5.6834, Train Steps/Sec: 1.12 +[2025-04-28 21:21:10] (step=0034275) Train Loss: 5.7592, Train Steps/Sec: 1.12 +[2025-04-28 21:21:32] (step=0034300) Train Loss: 5.7056, Train Steps/Sec: 1.12 +[2025-04-28 21:21:54] (step=0034325) Train Loss: 5.7661, Train Steps/Sec: 1.12 +[2025-04-28 21:22:17] (step=0034350) Train Loss: 5.7982, Train Steps/Sec: 1.12 +[2025-04-28 21:22:39] (step=0034375) Train Loss: 5.7112, Train Steps/Sec: 1.12 +[2025-04-28 21:23:01] (step=0034400) Train Loss: 5.7866, Train Steps/Sec: 1.12 +[2025-04-28 21:23:24] (step=0034425) Train Loss: 5.7331, Train Steps/Sec: 1.12 +[2025-04-28 21:23:46] (step=0034450) Train Loss: 5.7497, Train Steps/Sec: 1.12 +[2025-04-28 21:24:08] (step=0034475) Train Loss: 5.7143, Train Steps/Sec: 1.12 +[2025-04-28 21:24:30] (step=0034500) Train Loss: 5.6959, Train Steps/Sec: 1.12 +[2025-04-28 21:24:53] (step=0034525) Train Loss: 5.6838, Train Steps/Sec: 1.12 +[2025-04-28 21:25:15] (step=0034550) Train Loss: 5.6606, Train Steps/Sec: 1.12 +[2025-04-28 21:25:37] (step=0034575) Train Loss: 5.7517, Train Steps/Sec: 1.12 +[2025-04-28 21:26:00] (step=0034600) Train Loss: 5.7099, Train Steps/Sec: 1.12 +[2025-04-28 21:26:22] (step=0034625) Train Loss: 5.6893, Train Steps/Sec: 1.12 +[2025-04-28 21:26:44] (step=0034650) Train Loss: 5.6958, Train Steps/Sec: 1.12 +[2025-04-28 21:27:07] (step=0034675) Train Loss: 5.7545, Train Steps/Sec: 1.12 +[2025-04-28 21:27:29] (step=0034700) Train Loss: 5.6539, Train Steps/Sec: 1.12 +[2025-04-28 21:27:51] (step=0034725) Train Loss: 5.7361, Train Steps/Sec: 1.12 +[2025-04-28 21:28:14] (step=0034750) Train Loss: 5.6570, Train Steps/Sec: 1.12 +[2025-04-28 21:28:36] (step=0034775) Train Loss: 5.6457, Train Steps/Sec: 1.12 +[2025-04-28 21:28:58] (step=0034800) Train Loss: 5.7418, Train Steps/Sec: 1.11 +[2025-04-28 21:29:21] (step=0034825) Train Loss: 5.6786, Train Steps/Sec: 1.12 +[2025-04-28 21:29:43] (step=0034850) Train Loss: 5.6909, Train Steps/Sec: 1.12 +[2025-04-28 21:30:05] (step=0034875) Train Loss: 5.7391, Train Steps/Sec: 1.12 +[2025-04-28 21:30:27] (step=0034900) Train Loss: 5.7712, Train Steps/Sec: 1.12 +[2025-04-28 21:30:50] (step=0034925) Train Loss: 5.6927, Train Steps/Sec: 1.12 +[2025-04-28 21:31:12] (step=0034950) Train Loss: 5.7717, Train Steps/Sec: 1.12 +[2025-04-28 21:31:34] (step=0034975) Train Loss: 5.8055, Train Steps/Sec: 1.12 +[2025-04-28 21:31:57] (step=0035000) Train Loss: 5.7163, Train Steps/Sec: 1.12 +[2025-04-28 21:32:19] (step=0035025) Train Loss: 5.7175, Train Steps/Sec: 1.12 +[2025-04-28 21:32:41] (step=0035050) Train Loss: 5.7261, Train Steps/Sec: 1.12 +[2025-04-28 21:33:04] (step=0035075) Train Loss: 5.7208, Train Steps/Sec: 1.12 +[2025-04-28 21:33:26] (step=0035100) Train Loss: 5.7082, Train Steps/Sec: 1.12 +[2025-04-28 21:33:48] (step=0035125) Train Loss: 5.7542, Train Steps/Sec: 1.12 +[2025-04-28 21:34:10] (step=0035150) Train Loss: 5.7635, Train Steps/Sec: 1.12 +[2025-04-28 21:34:33] (step=0035175) Train Loss: 5.7260, Train Steps/Sec: 1.12 +[2025-04-28 21:34:55] (step=0035200) Train Loss: 5.7031, Train Steps/Sec: 1.12 +[2025-04-28 21:35:17] (step=0035225) Train Loss: 5.7705, Train Steps/Sec: 1.12 +[2025-04-28 21:35:40] (step=0035250) Train Loss: 5.7131, Train Steps/Sec: 1.12 +[2025-04-28 21:36:02] (step=0035275) Train Loss: 5.6865, Train Steps/Sec: 1.12 +[2025-04-28 21:36:24] (step=0035300) Train Loss: 5.7278, Train Steps/Sec: 1.12 +[2025-04-28 21:36:47] (step=0035325) Train Loss: 5.7047, Train Steps/Sec: 1.12 +[2025-04-28 21:37:09] (step=0035350) Train Loss: 5.7503, Train Steps/Sec: 1.12 +[2025-04-28 21:37:31] (step=0035375) Train Loss: 5.8148, Train Steps/Sec: 1.12 +[2025-04-28 21:37:54] (step=0035400) Train Loss: 5.7671, Train Steps/Sec: 1.12 +[2025-04-28 21:38:16] (step=0035425) Train Loss: 5.7650, Train Steps/Sec: 1.12 +[2025-04-28 21:38:38] (step=0035450) Train Loss: 5.7829, Train Steps/Sec: 1.12 +[2025-04-28 21:39:00] (step=0035475) Train Loss: 5.7103, Train Steps/Sec: 1.12 +[2025-04-28 21:39:23] (step=0035500) Train Loss: 5.7325, Train Steps/Sec: 1.12 +[2025-04-28 21:39:45] (step=0035525) Train Loss: 5.7215, Train Steps/Sec: 1.12 +[2025-04-28 21:40:07] (step=0035550) Train Loss: 5.6786, Train Steps/Sec: 1.12 +[2025-04-28 21:40:30] (step=0035575) Train Loss: 5.7145, Train Steps/Sec: 1.12 +[2025-04-28 21:40:52] (step=0035600) Train Loss: 5.7148, Train Steps/Sec: 1.12 +[2025-04-28 21:41:14] (step=0035625) Train Loss: 5.7105, Train Steps/Sec: 1.12 +[2025-04-28 21:41:37] (step=0035650) Train Loss: 5.6635, Train Steps/Sec: 1.12 +[2025-04-28 21:41:59] (step=0035675) Train Loss: 5.6668, Train Steps/Sec: 1.12 +[2025-04-28 21:42:21] (step=0035700) Train Loss: 5.7109, Train Steps/Sec: 1.12 +[2025-04-28 21:42:44] (step=0035725) Train Loss: 5.7723, Train Steps/Sec: 1.12 +[2025-04-28 21:43:06] (step=0035750) Train Loss: 5.7345, Train Steps/Sec: 1.12 +[2025-04-28 21:43:28] (step=0035775) Train Loss: 5.6923, Train Steps/Sec: 1.12 +[2025-04-28 21:43:50] (step=0035800) Train Loss: 5.6771, Train Steps/Sec: 1.12 +[2025-04-28 21:44:13] (step=0035825) Train Loss: 5.7000, Train Steps/Sec: 1.12 +[2025-04-28 21:44:35] (step=0035850) Train Loss: 5.6787, Train Steps/Sec: 1.12 +[2025-04-28 21:44:57] (step=0035875) Train Loss: 5.7167, Train Steps/Sec: 1.12 +[2025-04-28 21:45:20] (step=0035900) Train Loss: 5.7969, Train Steps/Sec: 1.12 +[2025-04-28 21:45:42] (step=0035925) Train Loss: 5.7548, Train Steps/Sec: 1.12 +[2025-04-28 21:46:04] (step=0035950) Train Loss: 5.7564, Train Steps/Sec: 1.12 +[2025-04-28 21:46:27] (step=0035975) Train Loss: 5.6578, Train Steps/Sec: 1.12 +[2025-04-28 21:46:49] (step=0036000) Train Loss: 5.6801, Train Steps/Sec: 1.11 +[2025-04-28 21:46:49] text_config is None. Initializing the text config with default values (`OPTConfig`). +[2025-04-28 21:54:06] Finish Eval in 36000 steps... +[2025-04-28 21:54:26] Saved checkpoint to checkpoint/X2I_UltraEdiit_fourmask_LLava_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_filtered_objects_4mask_1280_context_after_stage1_5e4/000-GPT-XL/checkpoints/0036000.pt +[2025-04-28 21:54:28] Removed old checkpoint: checkpoint/X2I_UltraEdiit_fourmask_LLava_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_filtered_objects_4mask_1280_context_after_stage1_5e4/000-GPT-XL/checkpoints/0034000.pt +[2025-04-28 21:54:51] (step=0036025) Train Loss: 5.7398, Train Steps/Sec: 0.05 +[2025-04-28 21:55:13] (step=0036050) Train Loss: 5.7708, Train Steps/Sec: 1.12 +[2025-04-28 21:55:35] (step=0036075) Train Loss: 5.7037, Train Steps/Sec: 1.12 +[2025-04-28 21:55:57] (step=0036100) Train Loss: 5.6726, Train Steps/Sec: 1.12 +[2025-04-28 21:56:20] (step=0036125) Train Loss: 5.7392, Train Steps/Sec: 1.12 +[2025-04-28 21:56:42] (step=0036150) Train Loss: 5.6931, Train Steps/Sec: 1.12 +[2025-04-28 21:57:04] (step=0036175) Train Loss: 5.6925, Train Steps/Sec: 1.12 +[2025-04-28 21:57:27] (step=0036200) Train Loss: 5.6923, Train Steps/Sec: 1.12 +[2025-04-28 21:57:49] (step=0036225) Train Loss: 5.8041, Train Steps/Sec: 1.12 +[2025-04-28 21:58:11] (step=0036250) Train Loss: 5.7962, Train Steps/Sec: 1.12 +[2025-04-28 21:58:33] (step=0036275) Train Loss: 5.7530, Train Steps/Sec: 1.12 +[2025-04-28 21:58:56] (step=0036300) Train Loss: 5.6788, Train Steps/Sec: 1.12 +[2025-04-28 21:59:18] (step=0036325) Train Loss: 5.6854, Train Steps/Sec: 1.12 +[2025-04-28 21:59:40] (step=0036350) Train Loss: 5.7182, Train Steps/Sec: 1.12 +[2025-04-28 22:00:03] (step=0036375) Train Loss: 5.7081, Train Steps/Sec: 1.12 +[2025-04-28 22:00:25] (step=0036400) Train Loss: 5.7376, Train Steps/Sec: 1.12 +[2025-04-28 22:00:47] (step=0036425) Train Loss: 5.8012, Train Steps/Sec: 1.12 +[2025-04-28 22:01:09] (step=0036450) Train Loss: 5.7303, Train Steps/Sec: 1.12 +[2025-04-28 22:01:32] (step=0036475) Train Loss: 5.7495, Train Steps/Sec: 1.12 +[2025-04-28 22:01:54] (step=0036500) Train Loss: 5.7064, Train Steps/Sec: 1.12 +[2025-04-28 22:02:16] (step=0036525) Train Loss: 5.6864, Train Steps/Sec: 1.12 +[2025-04-28 22:02:39] (step=0036550) Train Loss: 5.6577, Train Steps/Sec: 1.12 +[2025-04-28 22:03:01] (step=0036575) Train Loss: 5.7237, Train Steps/Sec: 1.12 +[2025-04-28 22:03:23] (step=0036600) Train Loss: 5.6721, Train Steps/Sec: 1.12 +[2025-04-28 22:03:46] (step=0036625) Train Loss: 5.6229, Train Steps/Sec: 1.12 +[2025-04-28 22:04:08] (step=0036650) Train Loss: 5.7044, Train Steps/Sec: 1.12 +[2025-04-28 22:04:30] (step=0036675) Train Loss: 5.6728, Train Steps/Sec: 1.12 +[2025-04-28 22:04:53] (step=0036700) Train Loss: 5.7544, Train Steps/Sec: 1.12 +[2025-04-28 22:05:15] (step=0036725) Train Loss: 5.7528, Train Steps/Sec: 1.12 +[2025-04-28 22:05:37] (step=0036750) Train Loss: 5.6199, Train Steps/Sec: 1.12 +[2025-04-28 22:05:59] (step=0036775) Train Loss: 5.7201, Train Steps/Sec: 1.12 +[2025-04-28 22:06:22] (step=0036800) Train Loss: 5.7652, Train Steps/Sec: 1.12 +[2025-04-28 22:06:44] (step=0036825) Train Loss: 5.6457, Train Steps/Sec: 1.12 +[2025-04-28 22:07:06] (step=0036850) Train Loss: 5.7227, Train Steps/Sec: 1.12 +[2025-04-28 22:07:29] (step=0036875) Train Loss: 5.6403, Train Steps/Sec: 1.12 +[2025-04-28 22:07:51] (step=0036900) Train Loss: 5.6457, Train Steps/Sec: 1.12 +[2025-04-28 22:08:13] (step=0036925) Train Loss: 5.7065, Train Steps/Sec: 1.12 +[2025-04-28 22:08:36] (step=0036950) Train Loss: 5.6639, Train Steps/Sec: 1.12 +[2025-04-28 22:09:07] (step=0036975) Train Loss: 5.7081, Train Steps/Sec: 0.81 +[2025-04-28 22:09:49] (step=0037000) Train Loss: 5.7103, Train Steps/Sec: 0.59 +[2025-04-28 22:10:29] (step=0037025) Train Loss: 5.7567, Train Steps/Sec: 0.62 +[2025-04-28 22:11:00] (step=0037050) Train Loss: 5.7605, Train Steps/Sec: 0.79 +[2025-04-28 22:11:23] (step=0037075) Train Loss: 5.7059, Train Steps/Sec: 1.12 +[2025-04-28 22:11:45] (step=0037100) Train Loss: 5.7503, Train Steps/Sec: 1.12 +[2025-04-28 22:12:07] (step=0037125) Train Loss: 5.6719, Train Steps/Sec: 1.12 +[2025-04-28 22:12:30] (step=0037150) Train Loss: 5.6837, Train Steps/Sec: 1.12 +[2025-04-28 22:12:52] (step=0037175) Train Loss: 5.6936, Train Steps/Sec: 1.12 +[2025-04-28 22:13:14] (step=0037200) Train Loss: 5.6971, Train Steps/Sec: 1.12 +[2025-04-28 22:13:37] (step=0037225) Train Loss: 5.6633, Train Steps/Sec: 1.12 +[2025-04-28 22:13:59] (step=0037250) Train Loss: 5.6747, Train Steps/Sec: 1.12 +[2025-04-28 22:14:21] (step=0037275) Train Loss: 5.6409, Train Steps/Sec: 1.12 +[2025-04-28 22:14:43] (step=0037300) Train Loss: 5.6878, Train Steps/Sec: 1.12 +[2025-04-28 22:15:06] (step=0037325) Train Loss: 5.6884, Train Steps/Sec: 1.12 +[2025-04-28 22:15:28] (step=0037350) Train Loss: 5.7060, Train Steps/Sec: 1.12 +[2025-04-28 22:15:50] (step=0037375) Train Loss: 5.7784, Train Steps/Sec: 1.12 +[2025-04-28 22:16:13] (step=0037400) Train Loss: 5.6881, Train Steps/Sec: 1.12 +[2025-04-28 22:16:35] (step=0037425) Train Loss: 5.6917, Train Steps/Sec: 1.12 +[2025-04-28 22:16:57] (step=0037450) Train Loss: 5.6653, Train Steps/Sec: 1.12 +[2025-04-28 22:17:20] (step=0037475) Train Loss: 5.6357, Train Steps/Sec: 1.12 +[2025-04-28 22:17:42] (step=0037500) Train Loss: 5.7615, Train Steps/Sec: 1.12 +[2025-04-28 22:18:04] (step=0037525) Train Loss: 5.7529, Train Steps/Sec: 1.12 +[2025-04-28 22:18:26] (step=0037550) Train Loss: 5.7341, Train Steps/Sec: 1.12 +[2025-04-28 22:18:49] (step=0037575) Train Loss: 5.6826, Train Steps/Sec: 1.12 +[2025-04-28 22:19:11] (step=0037600) Train Loss: 5.6852, Train Steps/Sec: 1.11 +[2025-04-28 22:19:34] (step=0037625) Train Loss: 5.7020, Train Steps/Sec: 1.12 +[2025-04-28 22:19:56] (step=0037650) Train Loss: 5.7029, Train Steps/Sec: 1.12 +[2025-04-28 22:20:18] (step=0037675) Train Loss: 5.6959, Train Steps/Sec: 1.12 +[2025-04-28 22:20:41] (step=0037700) Train Loss: 5.6998, Train Steps/Sec: 1.12 +[2025-04-28 22:21:03] (step=0037725) Train Loss: 5.7070, Train Steps/Sec: 1.12 +[2025-04-28 22:21:25] (step=0037750) Train Loss: 5.7652, Train Steps/Sec: 1.12 +[2025-04-28 22:21:47] (step=0037775) Train Loss: 5.6868, Train Steps/Sec: 1.12 +[2025-04-28 22:22:10] (step=0037800) Train Loss: 5.6366, Train Steps/Sec: 1.12 +[2025-04-28 22:22:32] (step=0037825) Train Loss: 5.7096, Train Steps/Sec: 1.12 +[2025-04-28 22:22:54] (step=0037850) Train Loss: 5.7493, Train Steps/Sec: 1.12 +[2025-04-28 22:23:17] (step=0037875) Train Loss: 5.7320, Train Steps/Sec: 1.12 +[2025-04-28 22:23:39] (step=0037900) Train Loss: 5.6506, Train Steps/Sec: 1.11 +[2025-04-28 22:24:01] (step=0037925) Train Loss: 5.7309, Train Steps/Sec: 1.12 +[2025-04-28 22:24:24] (step=0037950) Train Loss: 5.7203, Train Steps/Sec: 1.12 +[2025-04-28 22:24:46] (step=0037975) Train Loss: 5.7073, Train Steps/Sec: 1.12 +[2025-04-28 22:25:08] (step=0038000) Train Loss: 5.7432, Train Steps/Sec: 1.12 +[2025-04-28 22:25:08] text_config is None. Initializing the text config with default values (`OPTConfig`). +[2025-04-28 22:32:28] Finish Eval in 38000 steps... +[2025-04-28 22:32:48] Saved checkpoint to checkpoint/X2I_UltraEdiit_fourmask_LLava_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_filtered_objects_4mask_1280_context_after_stage1_5e4/000-GPT-XL/checkpoints/0038000.pt +[2025-04-28 22:32:50] Removed old checkpoint: checkpoint/X2I_UltraEdiit_fourmask_LLava_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_filtered_objects_4mask_1280_context_after_stage1_5e4/000-GPT-XL/checkpoints/0036000.pt +[2025-04-28 22:33:12] (step=0038025) Train Loss: 5.7008, Train Steps/Sec: 0.05 +[2025-04-28 22:33:34] (step=0038050) Train Loss: 5.6067, Train Steps/Sec: 1.12 +[2025-04-28 22:33:57] (step=0038075) Train Loss: 5.6629, Train Steps/Sec: 1.12 +[2025-04-28 22:34:19] (step=0038100) Train Loss: 5.6873, Train Steps/Sec: 1.12 +[2025-04-28 22:34:41] (step=0038125) Train Loss: 5.7104, Train Steps/Sec: 1.12 +[2025-04-28 22:35:04] (step=0038150) Train Loss: 5.7586, Train Steps/Sec: 1.12 +[2025-04-28 22:35:26] (step=0038175) Train Loss: 5.6545, Train Steps/Sec: 1.12 +[2025-04-28 22:35:48] (step=0038200) Train Loss: 5.6861, Train Steps/Sec: 1.12 +[2025-04-28 22:36:11] (step=0038225) Train Loss: 5.6485, Train Steps/Sec: 1.12 +[2025-04-28 22:36:33] (step=0038250) Train Loss: 5.7123, Train Steps/Sec: 1.12 +[2025-04-28 22:36:55] (step=0038275) Train Loss: 5.7171, Train Steps/Sec: 1.12 +[2025-04-28 22:37:18] (step=0038300) Train Loss: 5.6631, Train Steps/Sec: 1.12 +[2025-04-28 22:37:40] (step=0038325) Train Loss: 5.6364, Train Steps/Sec: 1.12 +[2025-04-28 22:38:02] (step=0038350) Train Loss: 5.6929, Train Steps/Sec: 1.12 +[2025-04-28 22:38:24] (step=0038375) Train Loss: 5.7316, Train Steps/Sec: 1.12 +[2025-04-28 22:38:47] (step=0038400) Train Loss: 5.7773, Train Steps/Sec: 1.12 +[2025-04-28 22:39:09] (step=0038425) Train Loss: 5.6918, Train Steps/Sec: 1.12 +[2025-04-28 22:39:31] (step=0038450) Train Loss: 5.6255, Train Steps/Sec: 1.12 +[2025-04-28 22:39:54] (step=0038475) Train Loss: 5.7548, Train Steps/Sec: 1.12 +[2025-04-28 22:40:16] (step=0038500) Train Loss: 5.6324, Train Steps/Sec: 1.12 +[2025-04-28 22:40:38] (step=0038525) Train Loss: 5.7120, Train Steps/Sec: 1.12 +[2025-04-28 22:41:01] (step=0038550) Train Loss: 5.6975, Train Steps/Sec: 1.12 +[2025-04-28 22:41:23] (step=0038575) Train Loss: 5.6434, Train Steps/Sec: 1.12 +[2025-04-28 22:41:45] (step=0038600) Train Loss: 5.7460, Train Steps/Sec: 1.12 +[2025-04-28 22:42:08] (step=0038625) Train Loss: 5.6444, Train Steps/Sec: 1.12 +[2025-04-28 22:42:30] (step=0038650) Train Loss: 5.7648, Train Steps/Sec: 1.12 +[2025-04-28 22:42:52] (step=0038675) Train Loss: 5.6974, Train Steps/Sec: 1.12 +[2025-04-28 22:43:15] (step=0038700) Train Loss: 5.7563, Train Steps/Sec: 1.12 +[2025-04-28 22:43:37] (step=0038725) Train Loss: 5.6754, Train Steps/Sec: 1.12 +[2025-04-28 22:43:59] (step=0038750) Train Loss: 5.7098, Train Steps/Sec: 1.12 +[2025-04-28 22:44:22] (step=0038775) Train Loss: 5.7961, Train Steps/Sec: 1.12 +[2025-04-28 22:44:44] (step=0038800) Train Loss: 5.7561, Train Steps/Sec: 1.12 +[2025-04-28 22:45:06] (step=0038825) Train Loss: 5.7198, Train Steps/Sec: 1.12 +[2025-04-28 22:45:29] (step=0038850) Train Loss: 5.6574, Train Steps/Sec: 1.12 +[2025-04-28 22:45:51] (step=0038875) Train Loss: 5.6790, Train Steps/Sec: 1.12 +[2025-04-28 22:46:13] (step=0038900) Train Loss: 5.6921, Train Steps/Sec: 1.12 +[2025-04-28 22:46:36] (step=0038925) Train Loss: 5.6348, Train Steps/Sec: 1.12 +[2025-04-28 22:46:58] (step=0038950) Train Loss: 5.7746, Train Steps/Sec: 1.12 +[2025-04-28 22:47:20] (step=0038975) Train Loss: 5.6976, Train Steps/Sec: 1.12 +[2025-04-28 22:47:43] (step=0039000) Train Loss: 5.6305, Train Steps/Sec: 1.12 +[2025-04-28 22:48:05] (step=0039025) Train Loss: 5.7870, Train Steps/Sec: 1.12 +[2025-04-28 22:48:27] (step=0039050) Train Loss: 5.6255, Train Steps/Sec: 1.12 +[2025-04-28 22:48:50] (step=0039075) Train Loss: 5.6349, Train Steps/Sec: 1.12 +[2025-04-28 22:49:12] (step=0039100) Train Loss: 5.7117, Train Steps/Sec: 1.12 +[2025-04-28 22:49:34] (step=0039125) Train Loss: 5.6522, Train Steps/Sec: 1.12 +[2025-04-28 22:49:56] (step=0039150) Train Loss: 5.6704, Train Steps/Sec: 1.12 +[2025-04-28 22:50:19] (step=0039175) Train Loss: 5.6421, Train Steps/Sec: 1.12 +[2025-04-28 22:50:41] (step=0039200) Train Loss: 5.6788, Train Steps/Sec: 1.12 +[2025-04-28 22:51:03] (step=0039225) Train Loss: 5.7322, Train Steps/Sec: 1.12 +[2025-04-28 22:51:26] (step=0039250) Train Loss: 5.6956, Train Steps/Sec: 1.12 +[2025-04-28 22:51:48] (step=0039275) Train Loss: 5.7243, Train Steps/Sec: 1.12 +[2025-04-28 22:52:10] (step=0039300) Train Loss: 5.7246, Train Steps/Sec: 1.12 +[2025-04-28 22:52:33] (step=0039325) Train Loss: 5.7039, Train Steps/Sec: 1.12 +[2025-04-28 22:52:55] (step=0039350) Train Loss: 5.6298, Train Steps/Sec: 1.12 +[2025-04-28 22:53:17] (step=0039375) Train Loss: 5.6635, Train Steps/Sec: 1.12 +[2025-04-28 22:53:40] (step=0039400) Train Loss: 5.6847, Train Steps/Sec: 1.11 +[2025-04-28 22:54:02] (step=0039425) Train Loss: 5.6837, Train Steps/Sec: 1.12 +[2025-04-28 22:54:24] (step=0039450) Train Loss: 5.7182, Train Steps/Sec: 1.12 +[2025-04-28 22:54:47] (step=0039475) Train Loss: 5.6954, Train Steps/Sec: 1.12 +[2025-04-28 22:55:09] (step=0039500) Train Loss: 5.6937, Train Steps/Sec: 1.12 +[2025-04-28 22:55:31] (step=0039525) Train Loss: 5.7296, Train Steps/Sec: 1.12 +[2025-04-28 22:55:53] (step=0039550) Train Loss: 5.6709, Train Steps/Sec: 1.12 +[2025-04-28 22:56:16] (step=0039575) Train Loss: 5.7184, Train Steps/Sec: 1.12 +[2025-04-28 22:56:38] (step=0039600) Train Loss: 5.7036, Train Steps/Sec: 1.12 +[2025-04-28 22:57:00] (step=0039625) Train Loss: 5.7011, Train Steps/Sec: 1.12 +[2025-04-28 22:57:23] (step=0039650) Train Loss: 5.6754, Train Steps/Sec: 1.12 +[2025-04-28 22:57:45] (step=0039675) Train Loss: 5.7097, Train Steps/Sec: 1.12 +[2025-04-28 22:58:07] (step=0039700) Train Loss: 5.6544, Train Steps/Sec: 1.12 +[2025-04-28 22:58:30] (step=0039725) Train Loss: 5.6836, Train Steps/Sec: 1.12 +[2025-04-28 22:58:52] (step=0039750) Train Loss: 5.7264, Train Steps/Sec: 1.12 +[2025-04-28 22:59:14] (step=0039775) Train Loss: 5.6839, Train Steps/Sec: 1.12 +[2025-04-28 22:59:37] (step=0039800) Train Loss: 5.6228, Train Steps/Sec: 1.12 +[2025-04-28 22:59:59] (step=0039825) Train Loss: 5.7261, Train Steps/Sec: 1.12 +[2025-04-28 23:00:21] (step=0039850) Train Loss: 5.7001, Train Steps/Sec: 1.12 +[2025-04-28 23:00:43] (step=0039875) Train Loss: 5.6941, Train Steps/Sec: 1.12 +[2025-04-28 23:01:06] (step=0039900) Train Loss: 5.6182, Train Steps/Sec: 1.12 +[2025-04-28 23:01:28] (step=0039925) Train Loss: 5.6652, Train Steps/Sec: 1.12 +[2025-04-28 23:01:50] (step=0039950) Train Loss: 5.6224, Train Steps/Sec: 1.12 +[2025-04-28 23:02:13] (step=0039975) Train Loss: 5.7015, Train Steps/Sec: 1.12 +[2025-04-28 23:02:35] (step=0040000) Train Loss: 5.7176, Train Steps/Sec: 1.12 +[2025-04-28 23:02:35] text_config is None. Initializing the text config with default values (`OPTConfig`). +[2025-04-28 23:09:56] Finish Eval in 40000 steps... +[2025-04-28 23:10:16] Saved checkpoint to checkpoint/X2I_UltraEdiit_fourmask_LLava_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_filtered_objects_4mask_1280_context_after_stage1_5e4/000-GPT-XL/checkpoints/0040000.pt +[2025-04-28 23:10:18] Removed old checkpoint: checkpoint/X2I_UltraEdiit_fourmask_LLava_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_filtered_objects_4mask_1280_context_after_stage1_5e4/000-GPT-XL/checkpoints/0038000.pt +[2025-04-28 23:10:40] (step=0040025) Train Loss: 5.7022, Train Steps/Sec: 0.05 +[2025-04-28 23:11:03] (step=0040050) Train Loss: 5.6942, Train Steps/Sec: 1.12 +[2025-04-28 23:11:25] (step=0040075) Train Loss: 5.7171, Train Steps/Sec: 1.12 +[2025-04-28 23:11:47] (step=0040100) Train Loss: 5.6620, Train Steps/Sec: 1.12 +[2025-04-28 23:12:10] (step=0040125) Train Loss: 5.7108, Train Steps/Sec: 1.12 +[2025-04-28 23:12:32] (step=0040150) Train Loss: 5.7042, Train Steps/Sec: 1.12 +[2025-04-28 23:12:54] (step=0040175) Train Loss: 5.6852, Train Steps/Sec: 1.12 +[2025-04-28 23:13:17] (step=0040200) Train Loss: 5.7164, Train Steps/Sec: 1.12 +[2025-04-28 23:13:39] (step=0040225) Train Loss: 5.6808, Train Steps/Sec: 1.12 +[2025-04-28 23:14:01] (step=0040250) Train Loss: 5.7013, Train Steps/Sec: 1.12 +[2025-04-28 23:14:23] (step=0040275) Train Loss: 5.6397, Train Steps/Sec: 1.12 +[2025-04-28 23:14:46] (step=0040300) Train Loss: 5.6560, Train Steps/Sec: 1.12 +[2025-04-28 23:15:08] (step=0040325) Train Loss: 5.7312, Train Steps/Sec: 1.12 +[2025-04-28 23:15:30] (step=0040350) Train Loss: 5.7267, Train Steps/Sec: 1.12 +[2025-04-28 23:15:53] (step=0040375) Train Loss: 5.7302, Train Steps/Sec: 1.12 +[2025-04-28 23:16:15] (step=0040400) Train Loss: 5.6905, Train Steps/Sec: 1.11 +[2025-04-28 23:16:38] (step=0040425) Train Loss: 5.6178, Train Steps/Sec: 1.12 +[2025-04-28 23:17:00] (step=0040450) Train Loss: 5.6696, Train Steps/Sec: 1.12 +[2025-04-28 23:17:22] (step=0040475) Train Loss: 5.6624, Train Steps/Sec: 1.12 +[2025-04-28 23:17:45] (step=0040500) Train Loss: 5.6979, Train Steps/Sec: 1.12 +[2025-04-28 23:18:07] (step=0040525) Train Loss: 5.6687, Train Steps/Sec: 1.12 +[2025-04-28 23:18:46] (step=0040550) Train Loss: 5.6051, Train Steps/Sec: 0.63 +[2025-04-28 23:19:38] (step=0040575) Train Loss: 5.6749, Train Steps/Sec: 0.49 +[2025-04-28 23:20:09] (step=0040600) Train Loss: 5.7421, Train Steps/Sec: 0.80 +[2025-04-28 23:20:41] (step=0040625) Train Loss: 5.6511, Train Steps/Sec: 0.78 +[2025-04-28 23:21:03] (step=0040650) Train Loss: 5.6884, Train Steps/Sec: 1.12 +[2025-04-28 23:21:26] (step=0040675) Train Loss: 5.6409, Train Steps/Sec: 1.12 +[2025-04-28 23:21:48] (step=0040700) Train Loss: 5.6610, Train Steps/Sec: 1.12 +[2025-04-28 23:22:10] (step=0040725) Train Loss: 5.6914, Train Steps/Sec: 1.12 +[2025-04-28 23:22:33] (step=0040750) Train Loss: 5.7263, Train Steps/Sec: 1.12 +[2025-04-28 23:22:55] (step=0040775) Train Loss: 5.6511, Train Steps/Sec: 1.12 +[2025-04-28 23:23:17] (step=0040800) Train Loss: 5.6446, Train Steps/Sec: 1.12 +[2025-04-28 23:23:40] (step=0040825) Train Loss: 5.6876, Train Steps/Sec: 1.12 +[2025-04-28 23:24:02] (step=0040850) Train Loss: 5.6487, Train Steps/Sec: 1.12 +[2025-04-28 23:24:24] (step=0040875) Train Loss: 5.7187, Train Steps/Sec: 1.12 +[2025-04-28 23:24:47] (step=0040900) Train Loss: 5.6826, Train Steps/Sec: 1.12 +[2025-04-28 23:25:09] (step=0040925) Train Loss: 5.7394, Train Steps/Sec: 1.12 +[2025-04-28 23:25:31] (step=0040950) Train Loss: 5.7489, Train Steps/Sec: 1.12 +[2025-04-28 23:25:53] (step=0040975) Train Loss: 5.7062, Train Steps/Sec: 1.12 +[2025-04-28 23:26:16] (step=0041000) Train Loss: 5.6439, Train Steps/Sec: 1.12 +[2025-04-28 23:26:38] (step=0041025) Train Loss: 5.6978, Train Steps/Sec: 1.12 +[2025-04-28 23:27:00] (step=0041050) Train Loss: 5.5955, Train Steps/Sec: 1.12 +[2025-04-28 23:27:23] (step=0041075) Train Loss: 5.7583, Train Steps/Sec: 1.12 +[2025-04-28 23:27:45] (step=0041100) Train Loss: 5.6113, Train Steps/Sec: 1.12 +[2025-04-28 23:28:07] (step=0041125) Train Loss: 5.6859, Train Steps/Sec: 1.12 +[2025-04-28 23:28:30] (step=0041150) Train Loss: 5.6775, Train Steps/Sec: 1.12 +[2025-04-28 23:28:52] (step=0041175) Train Loss: 5.7132, Train Steps/Sec: 1.12 +[2025-04-28 23:29:14] (step=0041200) Train Loss: 5.6937, Train Steps/Sec: 1.12 +[2025-04-28 23:29:37] (step=0041225) Train Loss: 5.6944, Train Steps/Sec: 1.12 +[2025-04-28 23:29:59] (step=0041250) Train Loss: 5.6879, Train Steps/Sec: 1.12 +[2025-04-28 23:30:21] (step=0041275) Train Loss: 5.7038, Train Steps/Sec: 1.12 +[2025-04-28 23:30:44] (step=0041300) Train Loss: 5.7164, Train Steps/Sec: 1.12 +[2025-04-28 23:31:06] (step=0041325) Train Loss: 5.6224, Train Steps/Sec: 1.12 +[2025-04-28 23:31:28] (step=0041350) Train Loss: 5.6068, Train Steps/Sec: 1.12 +[2025-04-28 23:31:51] (step=0041375) Train Loss: 5.6587, Train Steps/Sec: 1.12 +[2025-04-28 23:32:13] (step=0041400) Train Loss: 5.6697, Train Steps/Sec: 1.12 +[2025-04-28 23:32:35] (step=0041425) Train Loss: 5.7030, Train Steps/Sec: 1.12 +[2025-04-28 23:32:57] (step=0041450) Train Loss: 5.6953, Train Steps/Sec: 1.12 +[2025-04-28 23:33:20] (step=0041475) Train Loss: 5.6086, Train Steps/Sec: 1.12 +[2025-04-28 23:33:42] (step=0041500) Train Loss: 5.7182, Train Steps/Sec: 1.12 +[2025-04-28 23:34:04] (step=0041525) Train Loss: 5.6865, Train Steps/Sec: 1.12 +[2025-04-28 23:34:27] (step=0041550) Train Loss: 5.6465, Train Steps/Sec: 1.12 +[2025-04-28 23:34:49] (step=0041575) Train Loss: 5.6332, Train Steps/Sec: 1.12 +[2025-04-28 23:35:11] (step=0041600) Train Loss: 5.6448, Train Steps/Sec: 1.11 +[2025-04-28 23:35:34] (step=0041625) Train Loss: 5.6693, Train Steps/Sec: 1.12 +[2025-04-28 23:35:56] (step=0041650) Train Loss: 5.6649, Train Steps/Sec: 1.12 +[2025-04-28 23:36:18] (step=0041675) Train Loss: 5.6704, Train Steps/Sec: 1.12 +[2025-04-28 23:36:41] (step=0041700) Train Loss: 5.6305, Train Steps/Sec: 1.12 +[2025-04-28 23:37:03] (step=0041725) Train Loss: 5.6705, Train Steps/Sec: 1.12 +[2025-04-28 23:37:25] (step=0041750) Train Loss: 5.7412, Train Steps/Sec: 1.12 +[2025-04-28 23:37:48] (step=0041775) Train Loss: 5.6773, Train Steps/Sec: 1.12 +[2025-04-28 23:38:10] (step=0041800) Train Loss: 5.6996, Train Steps/Sec: 1.11 +[2025-04-28 23:38:33] (step=0041825) Train Loss: 5.7208, Train Steps/Sec: 1.12 +[2025-04-28 23:38:55] (step=0041850) Train Loss: 5.6683, Train Steps/Sec: 1.12 +[2025-04-28 23:39:17] (step=0041875) Train Loss: 5.6298, Train Steps/Sec: 1.12 +[2025-04-28 23:39:39] (step=0041900) Train Loss: 5.6729, Train Steps/Sec: 1.12 +[2025-04-28 23:40:02] (step=0041925) Train Loss: 5.6558, Train Steps/Sec: 1.12 +[2025-04-28 23:40:24] (step=0041950) Train Loss: 5.7145, Train Steps/Sec: 1.12 +[2025-04-28 23:40:46] (step=0041975) Train Loss: 5.6427, Train Steps/Sec: 1.12 +[2025-04-28 23:41:09] (step=0042000) Train Loss: 5.6402, Train Steps/Sec: 1.12 +[2025-04-28 23:41:09] text_config is None. Initializing the text config with default values (`OPTConfig`). +[2025-04-28 23:48:29] Finish Eval in 42000 steps... +[2025-04-28 23:48:49] Saved checkpoint to checkpoint/X2I_UltraEdiit_fourmask_LLava_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_filtered_objects_4mask_1280_context_after_stage1_5e4/000-GPT-XL/checkpoints/0042000.pt +[2025-04-28 23:48:51] Removed old checkpoint: checkpoint/X2I_UltraEdiit_fourmask_LLava_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_filtered_objects_4mask_1280_context_after_stage1_5e4/000-GPT-XL/checkpoints/0040000.pt +[2025-04-28 23:49:13] (step=0042025) Train Loss: 5.6136, Train Steps/Sec: 0.05 +[2025-04-28 23:49:36] (step=0042050) Train Loss: 5.6493, Train Steps/Sec: 1.12 +[2025-04-28 23:49:58] (step=0042075) Train Loss: 5.6209, Train Steps/Sec: 1.12 +[2025-04-28 23:50:20] (step=0042100) Train Loss: 5.6950, Train Steps/Sec: 1.12 +[2025-04-28 23:50:42] (step=0042125) Train Loss: 5.6629, Train Steps/Sec: 1.12 +[2025-04-28 23:51:05] (step=0042150) Train Loss: 5.6806, Train Steps/Sec: 1.12 +[2025-04-28 23:51:27] (step=0042175) Train Loss: 5.5678, Train Steps/Sec: 1.12 +[2025-04-28 23:51:49] (step=0042200) Train Loss: 5.6823, Train Steps/Sec: 1.12 +[2025-04-28 23:52:12] (step=0042225) Train Loss: 5.7068, Train Steps/Sec: 1.12 +[2025-04-28 23:52:34] (step=0042250) Train Loss: 5.6857, Train Steps/Sec: 1.11 +[2025-04-28 23:52:57] (step=0042275) Train Loss: 5.6382, Train Steps/Sec: 1.12 +[2025-04-28 23:53:19] (step=0042300) Train Loss: 5.6656, Train Steps/Sec: 1.12 +[2025-04-28 23:53:41] (step=0042325) Train Loss: 5.6743, Train Steps/Sec: 1.12 +[2025-04-28 23:54:03] (step=0042350) Train Loss: 5.6660, Train Steps/Sec: 1.12 +[2025-04-28 23:54:26] (step=0042375) Train Loss: 5.6581, Train Steps/Sec: 1.12 +[2025-04-28 23:54:48] (step=0042400) Train Loss: 5.6170, Train Steps/Sec: 1.11 +[2025-04-28 23:55:10] (step=0042425) Train Loss: 5.5935, Train Steps/Sec: 1.12 +[2025-04-28 23:55:33] (step=0042450) Train Loss: 5.6772, Train Steps/Sec: 1.12 +[2025-04-28 23:55:55] (step=0042475) Train Loss: 5.6675, Train Steps/Sec: 1.12 +[2025-04-28 23:56:17] (step=0042500) Train Loss: 5.6686, Train Steps/Sec: 1.12 +[2025-04-28 23:56:40] (step=0042525) Train Loss: 5.6684, Train Steps/Sec: 1.12 +[2025-04-28 23:57:02] (step=0042550) Train Loss: 5.6257, Train Steps/Sec: 1.12 +[2025-04-28 23:57:24] (step=0042575) Train Loss: 5.6646, Train Steps/Sec: 1.12 +[2025-04-28 23:57:47] (step=0042600) Train Loss: 5.6683, Train Steps/Sec: 1.12 +[2025-04-28 23:58:09] (step=0042625) Train Loss: 5.7004, Train Steps/Sec: 1.12 +[2025-04-28 23:58:31] (step=0042650) Train Loss: 5.6882, Train Steps/Sec: 1.12 +[2025-04-28 23:58:53] (step=0042675) Train Loss: 5.6686, Train Steps/Sec: 1.12 +[2025-04-28 23:59:16] (step=0042700) Train Loss: 5.6066, Train Steps/Sec: 1.12 +[2025-04-28 23:59:38] (step=0042725) Train Loss: 5.6940, Train Steps/Sec: 1.12 +[2025-04-29 00:00:00] (step=0042750) Train Loss: 5.6306, Train Steps/Sec: 1.12 +[2025-04-29 00:00:23] (step=0042775) Train Loss: 5.7247, Train Steps/Sec: 1.12 +[2025-04-29 00:00:45] (step=0042800) Train Loss: 5.6675, Train Steps/Sec: 1.12 +[2025-04-29 00:01:07] (step=0042825) Train Loss: 5.5962, Train Steps/Sec: 1.12 +[2025-04-29 00:01:30] (step=0042850) Train Loss: 5.7044, Train Steps/Sec: 1.12 +[2025-04-29 00:01:52] (step=0042875) Train Loss: 5.6512, Train Steps/Sec: 1.12 +[2025-04-29 00:02:14] (step=0042900) Train Loss: 5.5813, Train Steps/Sec: 1.12 +[2025-04-29 00:02:37] (step=0042925) Train Loss: 5.6691, Train Steps/Sec: 1.12 +[2025-04-29 00:02:59] (step=0042950) Train Loss: 5.6384, Train Steps/Sec: 1.12 +[2025-04-29 00:03:21] (step=0042975) Train Loss: 5.7155, Train Steps/Sec: 1.12 +[2025-04-29 00:03:44] (step=0043000) Train Loss: 5.6430, Train Steps/Sec: 1.12 +[2025-04-29 00:04:06] (step=0043025) Train Loss: 5.6643, Train Steps/Sec: 1.12 +[2025-04-29 00:04:28] (step=0043050) Train Loss: 5.6765, Train Steps/Sec: 1.12 +[2025-04-29 00:04:51] (step=0043075) Train Loss: 5.6856, Train Steps/Sec: 1.12 +[2025-04-29 00:05:13] (step=0043100) Train Loss: 5.6523, Train Steps/Sec: 1.12 +[2025-04-29 00:05:35] (step=0043125) Train Loss: 5.6304, Train Steps/Sec: 1.12 +[2025-04-29 00:05:57] (step=0043150) Train Loss: 5.6913, Train Steps/Sec: 1.12 +[2025-04-29 00:06:20] (step=0043175) Train Loss: 5.6324, Train Steps/Sec: 1.12 +[2025-04-29 00:06:42] (step=0043200) Train Loss: 5.7218, Train Steps/Sec: 1.12 +[2025-04-29 00:07:04] (step=0043225) Train Loss: 5.6937, Train Steps/Sec: 1.12 +[2025-04-29 00:07:27] (step=0043250) Train Loss: 5.6725, Train Steps/Sec: 1.12 +[2025-04-29 00:07:49] (step=0043275) Train Loss: 5.5983, Train Steps/Sec: 1.12 +[2025-04-29 00:08:11] (step=0043300) Train Loss: 5.6985, Train Steps/Sec: 1.12 +[2025-04-29 00:08:34] (step=0043325) Train Loss: 5.5995, Train Steps/Sec: 1.12 +[2025-04-29 00:08:56] (step=0043350) Train Loss: 5.6473, Train Steps/Sec: 1.12 +[2025-04-29 00:09:18] (step=0043375) Train Loss: 5.6176, Train Steps/Sec: 1.12 +[2025-04-29 00:09:41] (step=0043400) Train Loss: 5.6200, Train Steps/Sec: 1.11 +[2025-04-29 00:10:03] (step=0043425) Train Loss: 5.6522, Train Steps/Sec: 1.12 +[2025-04-29 00:10:25] (step=0043450) Train Loss: 5.6896, Train Steps/Sec: 1.12 +[2025-04-29 00:10:48] (step=0043475) Train Loss: 5.6889, Train Steps/Sec: 1.12 +[2025-04-29 00:11:10] (step=0043500) Train Loss: 5.7090, Train Steps/Sec: 1.12 +[2025-04-29 00:11:32] (step=0043525) Train Loss: 5.6773, Train Steps/Sec: 1.12 +[2025-04-29 00:11:55] (step=0043550) Train Loss: 5.7125, Train Steps/Sec: 1.12 +[2025-04-29 00:12:17] (step=0043575) Train Loss: 5.6334, Train Steps/Sec: 1.12 +[2025-04-29 00:12:39] (step=0043600) Train Loss: 5.6817, Train Steps/Sec: 1.12 +[2025-04-29 00:13:02] (step=0043625) Train Loss: 5.7197, Train Steps/Sec: 1.12 +[2025-04-29 00:13:24] (step=0043650) Train Loss: 5.6980, Train Steps/Sec: 1.12 +[2025-04-29 00:13:46] (step=0043675) Train Loss: 5.6495, Train Steps/Sec: 1.12 +[2025-04-29 00:14:09] (step=0043700) Train Loss: 5.7045, Train Steps/Sec: 1.12 +[2025-04-29 00:14:31] (step=0043725) Train Loss: 5.7056, Train Steps/Sec: 1.11 +[2025-04-29 00:14:54] (step=0043750) Train Loss: 5.6773, Train Steps/Sec: 1.11 +[2025-04-29 00:15:16] (step=0043775) Train Loss: 5.6808, Train Steps/Sec: 1.12 +[2025-04-29 00:15:39] (step=0043800) Train Loss: 5.6732, Train Steps/Sec: 1.11 +[2025-04-29 00:16:01] (step=0043825) Train Loss: 5.7029, Train Steps/Sec: 1.12 +[2025-04-29 00:16:23] (step=0043850) Train Loss: 5.6863, Train Steps/Sec: 1.12 +[2025-04-29 00:16:45] (step=0043875) Train Loss: 5.6860, Train Steps/Sec: 1.12 +[2025-04-29 00:17:08] (step=0043900) Train Loss: 5.6361, Train Steps/Sec: 1.12 +[2025-04-29 00:17:30] (step=0043925) Train Loss: 5.6031, Train Steps/Sec: 1.12 +[2025-04-29 00:17:52] (step=0043950) Train Loss: 5.5885, Train Steps/Sec: 1.12 +[2025-04-29 00:18:15] (step=0043975) Train Loss: 5.6607, Train Steps/Sec: 1.12 +[2025-04-29 00:18:37] (step=0044000) Train Loss: 5.5520, Train Steps/Sec: 1.12 +[2025-04-29 00:18:37] text_config is None. Initializing the text config with default values (`OPTConfig`). +[2025-04-29 00:25:57] Finish Eval in 44000 steps... +[2025-04-29 00:26:17] Saved checkpoint to checkpoint/X2I_UltraEdiit_fourmask_LLava_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_filtered_objects_4mask_1280_context_after_stage1_5e4/000-GPT-XL/checkpoints/0044000.pt +[2025-04-29 00:26:19] Removed old checkpoint: checkpoint/X2I_UltraEdiit_fourmask_LLava_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_filtered_objects_4mask_1280_context_after_stage1_5e4/000-GPT-XL/checkpoints/0042000.pt +[2025-04-29 00:26:41] (step=0044025) Train Loss: 5.6225, Train Steps/Sec: 0.05 +[2025-04-29 00:27:03] (step=0044050) Train Loss: 5.6339, Train Steps/Sec: 1.12 +[2025-04-29 00:27:26] (step=0044075) Train Loss: 5.6957, Train Steps/Sec: 1.12 +[2025-04-29 00:27:57] (step=0044100) Train Loss: 5.6669, Train Steps/Sec: 0.81 +[2025-04-29 00:28:38] (step=0044125) Train Loss: 5.6462, Train Steps/Sec: 0.60 +[2025-04-29 00:29:19] (step=0044150) Train Loss: 5.6389, Train Steps/Sec: 0.62 +[2025-04-29 00:29:50] (step=0044175) Train Loss: 5.7078, Train Steps/Sec: 0.79 +[2025-04-29 00:30:13] (step=0044200) Train Loss: 5.6398, Train Steps/Sec: 1.12 +[2025-04-29 00:30:35] (step=0044225) Train Loss: 5.6437, Train Steps/Sec: 1.12 +[2025-04-29 00:30:57] (step=0044250) Train Loss: 5.5646, Train Steps/Sec: 1.12 +[2025-04-29 00:31:20] (step=0044275) Train Loss: 5.6698, Train Steps/Sec: 1.12 +[2025-04-29 00:31:42] (step=0044300) Train Loss: 5.6446, Train Steps/Sec: 1.12 +[2025-04-29 00:32:04] (step=0044325) Train Loss: 5.6691, Train Steps/Sec: 1.12 +[2025-04-29 00:32:27] (step=0044350) Train Loss: 5.6494, Train Steps/Sec: 1.12 +[2025-04-29 00:32:49] (step=0044375) Train Loss: 5.6703, Train Steps/Sec: 1.12 +[2025-04-29 00:33:11] (step=0044400) Train Loss: 5.6893, Train Steps/Sec: 1.12 +[2025-04-29 00:33:33] (step=0044425) Train Loss: 5.7007, Train Steps/Sec: 1.12 +[2025-04-29 00:33:56] (step=0044450) Train Loss: 5.7023, Train Steps/Sec: 1.12 +[2025-04-29 00:34:18] (step=0044475) Train Loss: 5.6614, Train Steps/Sec: 1.12 +[2025-04-29 00:34:40] (step=0044500) Train Loss: 5.6989, Train Steps/Sec: 1.12 +[2025-04-29 00:35:03] (step=0044525) Train Loss: 5.7172, Train Steps/Sec: 1.12 +[2025-04-29 00:35:25] (step=0044550) Train Loss: 5.6632, Train Steps/Sec: 1.12 +[2025-04-29 00:35:47] (step=0044575) Train Loss: 5.6868, Train Steps/Sec: 1.12 +[2025-04-29 00:36:10] (step=0044600) Train Loss: 5.7103, Train Steps/Sec: 1.12 +[2025-04-29 00:36:32] (step=0044625) Train Loss: 5.6649, Train Steps/Sec: 1.12 +[2025-04-29 00:36:54] (step=0044650) Train Loss: 5.6666, Train Steps/Sec: 1.12 +[2025-04-29 00:37:16] (step=0044675) Train Loss: 5.6089, Train Steps/Sec: 1.12 +[2025-04-29 00:37:39] (step=0044700) Train Loss: 5.6965, Train Steps/Sec: 1.12 +[2025-04-29 00:38:01] (step=0044725) Train Loss: 5.7018, Train Steps/Sec: 1.12 +[2025-04-29 00:38:23] (step=0044750) Train Loss: 5.6461, Train Steps/Sec: 1.12 +[2025-04-29 00:38:46] (step=0044775) Train Loss: 5.5549, Train Steps/Sec: 1.12 +[2025-04-29 00:39:08] (step=0044800) Train Loss: 5.6778, Train Steps/Sec: 1.12 +[2025-04-29 00:39:30] (step=0044825) Train Loss: 5.7400, Train Steps/Sec: 1.12 +[2025-04-29 00:39:53] (step=0044850) Train Loss: 5.6828, Train Steps/Sec: 1.12 +[2025-04-29 00:40:15] (step=0044875) Train Loss: 5.7268, Train Steps/Sec: 1.12 +[2025-04-29 00:40:37] (step=0044900) Train Loss: 5.7104, Train Steps/Sec: 1.12 +[2025-04-29 00:41:00] (step=0044925) Train Loss: 5.7082, Train Steps/Sec: 1.12 +[2025-04-29 00:41:22] (step=0044950) Train Loss: 5.6950, Train Steps/Sec: 1.12 +[2025-04-29 00:41:44] (step=0044975) Train Loss: 5.6528, Train Steps/Sec: 1.12 +[2025-04-29 00:42:06] (step=0045000) Train Loss: 5.6408, Train Steps/Sec: 1.12 +[2025-04-29 00:42:29] (step=0045025) Train Loss: 5.6076, Train Steps/Sec: 1.12 +[2025-04-29 00:42:51] (step=0045050) Train Loss: 5.6475, Train Steps/Sec: 1.12 +[2025-04-29 00:43:13] (step=0045075) Train Loss: 5.6710, Train Steps/Sec: 1.12 +[2025-04-29 00:43:36] (step=0045100) Train Loss: 5.6068, Train Steps/Sec: 1.12 +[2025-04-29 00:43:58] (step=0045125) Train Loss: 5.6347, Train Steps/Sec: 1.12 +[2025-04-29 00:44:20] (step=0045150) Train Loss: 5.6640, Train Steps/Sec: 1.12 +[2025-04-29 00:44:43] (step=0045175) Train Loss: 5.6386, Train Steps/Sec: 1.12 +[2025-04-29 00:45:05] (step=0045200) Train Loss: 5.6848, Train Steps/Sec: 1.12 +[2025-04-29 00:45:27] (step=0045225) Train Loss: 5.7438, Train Steps/Sec: 1.12 +[2025-04-29 00:45:50] (step=0045250) Train Loss: 5.6714, Train Steps/Sec: 1.12 +[2025-04-29 00:46:12] (step=0045275) Train Loss: 5.6269, Train Steps/Sec: 1.12 +[2025-04-29 00:46:34] (step=0045300) Train Loss: 5.6300, Train Steps/Sec: 1.12 +[2025-04-29 00:46:56] (step=0045325) Train Loss: 5.6631, Train Steps/Sec: 1.12 +[2025-04-29 00:47:19] (step=0045350) Train Loss: 5.6476, Train Steps/Sec: 1.12 +[2025-04-29 00:47:41] (step=0045375) Train Loss: 5.6634, Train Steps/Sec: 1.12 +[2025-04-29 00:48:03] (step=0045400) Train Loss: 5.6954, Train Steps/Sec: 1.12 +[2025-04-29 00:48:26] (step=0045425) Train Loss: 5.6846, Train Steps/Sec: 1.12 +[2025-04-29 00:48:48] (step=0045450) Train Loss: 5.6599, Train Steps/Sec: 1.12 +[2025-04-29 00:49:10] (step=0045475) Train Loss: 5.6389, Train Steps/Sec: 1.12 +[2025-04-29 00:49:32] (step=0045500) Train Loss: 5.6133, Train Steps/Sec: 1.12 +[2025-04-29 00:49:55] (step=0045525) Train Loss: 5.6518, Train Steps/Sec: 1.12 +[2025-04-29 00:50:17] (step=0045550) Train Loss: 5.7374, Train Steps/Sec: 1.12 +[2025-04-29 00:50:39] (step=0045575) Train Loss: 5.7107, Train Steps/Sec: 1.12 +[2025-04-29 00:51:02] (step=0045600) Train Loss: 5.6553, Train Steps/Sec: 1.12 +[2025-04-29 00:51:24] (step=0045625) Train Loss: 5.6526, Train Steps/Sec: 1.12 +[2025-04-29 00:51:46] (step=0045650) Train Loss: 5.6825, Train Steps/Sec: 1.12 +[2025-04-29 00:52:08] (step=0045675) Train Loss: 5.6364, Train Steps/Sec: 1.12 +[2025-04-29 00:52:31] (step=0045700) Train Loss: 5.5499, Train Steps/Sec: 1.12 +[2025-04-29 00:52:53] (step=0045725) Train Loss: 5.6614, Train Steps/Sec: 1.12 +[2025-04-29 00:53:15] (step=0045750) Train Loss: 5.6531, Train Steps/Sec: 1.12 +[2025-04-29 00:53:38] (step=0045775) Train Loss: 5.6939, Train Steps/Sec: 1.12 +[2025-04-29 00:54:00] (step=0045800) Train Loss: 5.6931, Train Steps/Sec: 1.12 +[2025-04-29 00:54:22] (step=0045825) Train Loss: 5.6661, Train Steps/Sec: 1.12 +[2025-04-29 00:54:45] (step=0045850) Train Loss: 5.6094, Train Steps/Sec: 1.12 +[2025-04-29 00:55:07] (step=0045875) Train Loss: 5.6916, Train Steps/Sec: 1.12 +[2025-04-29 00:55:29] (step=0045900) Train Loss: 5.5855, Train Steps/Sec: 1.12 +[2025-04-29 00:55:51] (step=0045925) Train Loss: 5.6305, Train Steps/Sec: 1.12 +[2025-04-29 00:56:14] (step=0045950) Train Loss: 5.6352, Train Steps/Sec: 1.12 +[2025-04-29 00:56:36] (step=0045975) Train Loss: 5.6476, Train Steps/Sec: 1.12 +[2025-04-29 00:56:58] (step=0046000) Train Loss: 5.6467, Train Steps/Sec: 1.12 +[2025-04-29 00:56:58] text_config is None. Initializing the text config with default values (`OPTConfig`). +[2025-04-29 01:04:19] Finish Eval in 46000 steps... +[2025-04-29 01:04:39] Saved checkpoint to checkpoint/X2I_UltraEdiit_fourmask_LLava_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_filtered_objects_4mask_1280_context_after_stage1_5e4/000-GPT-XL/checkpoints/0046000.pt +[2025-04-29 01:04:41] Removed old checkpoint: checkpoint/X2I_UltraEdiit_fourmask_LLava_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_filtered_objects_4mask_1280_context_after_stage1_5e4/000-GPT-XL/checkpoints/0044000.pt +[2025-04-29 01:05:03] (step=0046025) Train Loss: 5.6026, Train Steps/Sec: 0.05 +[2025-04-29 01:05:26] (step=0046050) Train Loss: 5.6951, Train Steps/Sec: 1.12 +[2025-04-29 01:05:48] (step=0046075) Train Loss: 5.6402, Train Steps/Sec: 1.12 +[2025-04-29 01:06:10] (step=0046100) Train Loss: 5.6464, Train Steps/Sec: 1.12 +[2025-04-29 01:06:32] (step=0046125) Train Loss: 5.5826, Train Steps/Sec: 1.12 +[2025-04-29 01:06:55] (step=0046150) Train Loss: 5.5786, Train Steps/Sec: 1.12 +[2025-04-29 01:07:17] (step=0046175) Train Loss: 5.6548, Train Steps/Sec: 1.12 +[2025-04-29 01:07:40] (step=0046200) Train Loss: 5.6580, Train Steps/Sec: 1.11 +[2025-04-29 01:08:02] (step=0046225) Train Loss: 5.6407, Train Steps/Sec: 1.12 +[2025-04-29 01:08:24] (step=0046250) Train Loss: 5.6010, Train Steps/Sec: 1.12 +[2025-04-29 01:08:46] (step=0046275) Train Loss: 5.6310, Train Steps/Sec: 1.12 +[2025-04-29 01:09:09] (step=0046300) Train Loss: 5.6470, Train Steps/Sec: 1.12 +[2025-04-29 01:09:31] (step=0046325) Train Loss: 5.6593, Train Steps/Sec: 1.12 +[2025-04-29 01:09:53] (step=0046350) Train Loss: 5.6668, Train Steps/Sec: 1.12 +[2025-04-29 01:10:16] (step=0046375) Train Loss: 5.7013, Train Steps/Sec: 1.12 +[2025-04-29 01:10:38] (step=0046400) Train Loss: 5.5764, Train Steps/Sec: 1.12 +[2025-04-29 01:11:00] (step=0046425) Train Loss: 5.6383, Train Steps/Sec: 1.12 +[2025-04-29 01:11:23] (step=0046450) Train Loss: 5.6294, Train Steps/Sec: 1.12 +[2025-04-29 01:11:45] (step=0046475) Train Loss: 5.5900, Train Steps/Sec: 1.12 +[2025-04-29 01:12:07] (step=0046500) Train Loss: 5.6494, Train Steps/Sec: 1.12 +[2025-04-29 01:12:30] (step=0046525) Train Loss: 5.6869, Train Steps/Sec: 1.12 +[2025-04-29 01:12:52] (step=0046550) Train Loss: 5.6843, Train Steps/Sec: 1.12 +[2025-04-29 01:13:14] (step=0046575) Train Loss: 5.5866, Train Steps/Sec: 1.12 +[2025-04-29 01:13:37] (step=0046600) Train Loss: 5.6570, Train Steps/Sec: 1.11 +[2025-04-29 01:13:59] (step=0046625) Train Loss: 5.7337, Train Steps/Sec: 1.12 +[2025-04-29 01:14:21] (step=0046650) Train Loss: 5.7141, Train Steps/Sec: 1.12 +[2025-04-29 01:14:44] (step=0046675) Train Loss: 5.6506, Train Steps/Sec: 1.12 +[2025-04-29 01:15:06] (step=0046700) Train Loss: 5.5793, Train Steps/Sec: 1.12 +[2025-04-29 01:15:28] (step=0046725) Train Loss: 5.6166, Train Steps/Sec: 1.12 +[2025-04-29 01:15:51] (step=0046750) Train Loss: 5.6955, Train Steps/Sec: 1.12 +[2025-04-29 01:16:13] (step=0046775) Train Loss: 5.6770, Train Steps/Sec: 1.12 +[2025-04-29 01:16:35] (step=0046800) Train Loss: 5.6471, Train Steps/Sec: 1.12 +[2025-04-29 01:16:58] (step=0046825) Train Loss: 5.6445, Train Steps/Sec: 1.12 +[2025-04-29 01:17:20] (step=0046850) Train Loss: 5.6422, Train Steps/Sec: 1.12 +[2025-04-29 01:17:42] (step=0046875) Train Loss: 5.5732, Train Steps/Sec: 1.12 +[2025-04-29 01:18:05] (step=0046900) Train Loss: 5.6204, Train Steps/Sec: 1.12 +[2025-04-29 01:18:27] (step=0046925) Train Loss: 5.6200, Train Steps/Sec: 1.12 +[2025-04-29 01:18:49] (step=0046950) Train Loss: 5.6061, Train Steps/Sec: 1.12 +[2025-04-29 01:19:12] (step=0046975) Train Loss: 5.6811, Train Steps/Sec: 1.12 +[2025-04-29 01:19:34] (step=0047000) Train Loss: 5.7023, Train Steps/Sec: 1.11 +[2025-04-29 01:19:56] (step=0047025) Train Loss: 5.6133, Train Steps/Sec: 1.12 +[2025-04-29 01:20:19] (step=0047050) Train Loss: 5.6947, Train Steps/Sec: 1.12 +[2025-04-29 01:20:41] (step=0047075) Train Loss: 5.6534, Train Steps/Sec: 1.12 +[2025-04-29 01:21:03] (step=0047100) Train Loss: 5.5772, Train Steps/Sec: 1.12 +[2025-04-29 01:21:26] (step=0047125) Train Loss: 5.6863, Train Steps/Sec: 1.12 +[2025-04-29 01:21:48] (step=0047150) Train Loss: 5.6075, Train Steps/Sec: 1.12 +[2025-04-29 01:22:10] (step=0047175) Train Loss: 5.6845, Train Steps/Sec: 1.12 +[2025-04-29 01:22:33] (step=0047200) Train Loss: 5.6236, Train Steps/Sec: 1.12 +[2025-04-29 01:22:55] (step=0047225) Train Loss: 5.6184, Train Steps/Sec: 1.12 +[2025-04-29 01:23:17] (step=0047250) Train Loss: 5.6793, Train Steps/Sec: 1.12 +[2025-04-29 01:23:40] (step=0047275) Train Loss: 5.6733, Train Steps/Sec: 1.12 +[2025-04-29 01:24:02] (step=0047300) Train Loss: 5.6135, Train Steps/Sec: 1.12 +[2025-04-29 01:24:24] (step=0047325) Train Loss: 5.6454, Train Steps/Sec: 1.12 +[2025-04-29 01:24:47] (step=0047350) Train Loss: 5.6420, Train Steps/Sec: 1.12 +[2025-04-29 01:25:09] (step=0047375) Train Loss: 5.6583, Train Steps/Sec: 1.12 +[2025-04-29 01:25:31] (step=0047400) Train Loss: 5.6078, Train Steps/Sec: 1.11 +[2025-04-29 01:25:54] (step=0047425) Train Loss: 5.6280, Train Steps/Sec: 1.12 +[2025-04-29 01:26:16] (step=0047450) Train Loss: 5.6332, Train Steps/Sec: 1.12 +[2025-04-29 01:26:38] (step=0047475) Train Loss: 5.6841, Train Steps/Sec: 1.12 +[2025-04-29 01:27:01] (step=0047500) Train Loss: 5.6310, Train Steps/Sec: 1.12 +[2025-04-29 01:27:23] (step=0047525) Train Loss: 5.6379, Train Steps/Sec: 1.12 +[2025-04-29 01:27:45] (step=0047550) Train Loss: 5.5910, Train Steps/Sec: 1.12 +[2025-04-29 01:28:07] (step=0047575) Train Loss: 5.6121, Train Steps/Sec: 1.12 +[2025-04-29 01:28:30] (step=0047600) Train Loss: 5.6309, Train Steps/Sec: 1.12 +[2025-04-29 01:28:52] (step=0047625) Train Loss: 5.6770, Train Steps/Sec: 1.12 +[2025-04-29 01:29:14] (step=0047650) Train Loss: 5.6189, Train Steps/Sec: 1.12 +[2025-04-29 01:29:56] (step=0047675) Train Loss: 5.6766, Train Steps/Sec: 0.61 +[2025-04-29 01:30:27] (step=0047700) Train Loss: 5.6503, Train Steps/Sec: 0.79 +[2025-04-29 01:31:08] (step=0047725) Train Loss: 5.6335, Train Steps/Sec: 0.62 +[2025-04-29 01:31:39] (step=0047750) Train Loss: 5.6307, Train Steps/Sec: 0.79 +[2025-04-29 01:32:02] (step=0047775) Train Loss: 5.6656, Train Steps/Sec: 1.12 +[2025-04-29 01:32:24] (step=0047800) Train Loss: 5.6320, Train Steps/Sec: 1.12 +[2025-04-29 01:32:46] (step=0047825) Train Loss: 5.6139, Train Steps/Sec: 1.12 +[2025-04-29 01:33:09] (step=0047850) Train Loss: 5.6424, Train Steps/Sec: 1.12 +[2025-04-29 01:33:31] (step=0047875) Train Loss: 5.6570, Train Steps/Sec: 1.12 +[2025-04-29 01:33:53] (step=0047900) Train Loss: 5.6031, Train Steps/Sec: 1.12 +[2025-04-29 01:34:15] (step=0047925) Train Loss: 5.6467, Train Steps/Sec: 1.12 +[2025-04-29 01:34:38] (step=0047950) Train Loss: 5.7381, Train Steps/Sec: 1.12 +[2025-04-29 01:35:00] (step=0047975) Train Loss: 5.6946, Train Steps/Sec: 1.12 +[2025-04-29 01:35:23] (step=0048000) Train Loss: 5.7035, Train Steps/Sec: 1.12 +[2025-04-29 01:35:23] text_config is None. Initializing the text config with default values (`OPTConfig`). +[2025-04-29 01:42:41] Finish Eval in 48000 steps... +[2025-04-29 01:43:01] Saved checkpoint to checkpoint/X2I_UltraEdiit_fourmask_LLava_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_filtered_objects_4mask_1280_context_after_stage1_5e4/000-GPT-XL/checkpoints/0048000.pt +[2025-04-29 01:43:03] Removed old checkpoint: checkpoint/X2I_UltraEdiit_fourmask_LLava_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_filtered_objects_4mask_1280_context_after_stage1_5e4/000-GPT-XL/checkpoints/0046000.pt +[2025-04-29 01:43:26] (step=0048025) Train Loss: 5.6746, Train Steps/Sec: 0.05 +[2025-04-29 01:43:48] (step=0048050) Train Loss: 5.6302, Train Steps/Sec: 1.12 +[2025-04-29 01:44:11] (step=0048075) Train Loss: 5.5830, Train Steps/Sec: 1.12 +[2025-04-29 01:44:33] (step=0048100) Train Loss: 5.6308, Train Steps/Sec: 1.12 +[2025-04-29 01:44:55] (step=0048125) Train Loss: 5.6776, Train Steps/Sec: 1.12 +[2025-04-29 01:45:17] (step=0048150) Train Loss: 5.6290, Train Steps/Sec: 1.12 +[2025-04-29 01:45:40] (step=0048175) Train Loss: 5.5863, Train Steps/Sec: 1.12 +[2025-04-29 01:46:02] (step=0048200) Train Loss: 5.6674, Train Steps/Sec: 1.12 +[2025-04-29 01:46:24] (step=0048225) Train Loss: 5.6372, Train Steps/Sec: 1.12 +[2025-04-29 01:46:47] (step=0048250) Train Loss: 5.6469, Train Steps/Sec: 1.12 +[2025-04-29 01:47:09] (step=0048275) Train Loss: 5.6135, Train Steps/Sec: 1.12 +[2025-04-29 01:47:31] (step=0048300) Train Loss: 5.6578, Train Steps/Sec: 1.12 +[2025-04-29 01:47:53] (step=0048325) Train Loss: 5.6603, Train Steps/Sec: 1.12 +[2025-04-29 01:48:16] (step=0048350) Train Loss: 5.6501, Train Steps/Sec: 1.12 +[2025-04-29 01:48:38] (step=0048375) Train Loss: 5.6077, Train Steps/Sec: 1.12 +[2025-04-29 01:49:00] (step=0048400) Train Loss: 5.6114, Train Steps/Sec: 1.12 +[2025-04-29 01:49:23] (step=0048425) Train Loss: 5.6512, Train Steps/Sec: 1.12 +[2025-04-29 01:49:45] (step=0048450) Train Loss: 5.5629, Train Steps/Sec: 1.12 +[2025-04-29 01:50:07] (step=0048475) Train Loss: 5.6476, Train Steps/Sec: 1.12 +[2025-04-29 01:50:29] (step=0048500) Train Loss: 5.7002, Train Steps/Sec: 1.12 +[2025-04-29 01:50:52] (step=0048525) Train Loss: 5.6317, Train Steps/Sec: 1.12 +[2025-04-29 01:51:14] (step=0048550) Train Loss: 5.6838, Train Steps/Sec: 1.12 +[2025-04-29 01:51:36] (step=0048575) Train Loss: 5.6232, Train Steps/Sec: 1.12 +[2025-04-29 01:51:59] (step=0048600) Train Loss: 5.5992, Train Steps/Sec: 1.12 +[2025-04-29 01:52:21] (step=0048625) Train Loss: 5.5585, Train Steps/Sec: 1.12 +[2025-04-29 01:52:43] (step=0048650) Train Loss: 5.5860, Train Steps/Sec: 1.12 +[2025-04-29 01:53:06] (step=0048675) Train Loss: 5.6252, Train Steps/Sec: 1.12 +[2025-04-29 01:53:28] (step=0048700) Train Loss: 5.6658, Train Steps/Sec: 1.12 +[2025-04-29 01:53:50] (step=0048725) Train Loss: 5.6747, Train Steps/Sec: 1.12 +[2025-04-29 01:54:12] (step=0048750) Train Loss: 5.6583, Train Steps/Sec: 1.12 +[2025-04-29 01:54:35] (step=0048775) Train Loss: 5.6206, Train Steps/Sec: 1.12 +[2025-04-29 01:54:57] (step=0048800) Train Loss: 5.5280, Train Steps/Sec: 1.12 +[2025-04-29 01:55:19] (step=0048825) Train Loss: 5.6271, Train Steps/Sec: 1.12 +[2025-04-29 01:55:42] (step=0048850) Train Loss: 5.5945, Train Steps/Sec: 1.12 +[2025-04-29 01:56:04] (step=0048875) Train Loss: 5.7063, Train Steps/Sec: 1.12 +[2025-04-29 01:56:26] (step=0048900) Train Loss: 5.6527, Train Steps/Sec: 1.12 +[2025-04-29 01:56:48] (step=0048925) Train Loss: 5.6405, Train Steps/Sec: 1.12 +[2025-04-29 01:57:11] (step=0048950) Train Loss: 5.6762, Train Steps/Sec: 1.12 +[2025-04-29 01:57:33] (step=0048975) Train Loss: 5.5684, Train Steps/Sec: 1.12 +[2025-04-29 01:57:55] (step=0049000) Train Loss: 5.5945, Train Steps/Sec: 1.12 +[2025-04-29 01:58:18] (step=0049025) Train Loss: 5.6057, Train Steps/Sec: 1.12 +[2025-04-29 01:58:40] (step=0049050) Train Loss: 5.6087, Train Steps/Sec: 1.12 +[2025-04-29 01:59:02] (step=0049075) Train Loss: 5.5803, Train Steps/Sec: 1.12 +[2025-04-29 01:59:25] (step=0049100) Train Loss: 5.6757, Train Steps/Sec: 1.12 +[2025-04-29 01:59:47] (step=0049125) Train Loss: 5.6529, Train Steps/Sec: 1.12 +[2025-04-29 02:00:09] (step=0049150) Train Loss: 5.6560, Train Steps/Sec: 1.12 +[2025-04-29 02:00:31] (step=0049175) Train Loss: 5.6609, Train Steps/Sec: 1.12 +[2025-04-29 02:00:54] (step=0049200) Train Loss: 5.5957, Train Steps/Sec: 1.12 +[2025-04-29 02:01:16] (step=0049225) Train Loss: 5.5572, Train Steps/Sec: 1.12 +[2025-04-29 02:01:38] (step=0049250) Train Loss: 5.6461, Train Steps/Sec: 1.12 +[2025-04-29 02:02:01] (step=0049275) Train Loss: 5.5932, Train Steps/Sec: 1.12 +[2025-04-29 02:02:23] (step=0049300) Train Loss: 5.6418, Train Steps/Sec: 1.12 +[2025-04-29 02:02:45] (step=0049325) Train Loss: 5.7004, Train Steps/Sec: 1.12 +[2025-04-29 02:03:07] (step=0049350) Train Loss: 5.6505, Train Steps/Sec: 1.12 +[2025-04-29 02:03:30] (step=0049375) Train Loss: 5.7060, Train Steps/Sec: 1.12 +[2025-04-29 02:03:52] (step=0049400) Train Loss: 5.6231, Train Steps/Sec: 1.12 +[2025-04-29 02:04:14] (step=0049425) Train Loss: 5.6134, Train Steps/Sec: 1.12 +[2025-04-29 02:04:37] (step=0049450) Train Loss: 5.6258, Train Steps/Sec: 1.12 +[2025-04-29 02:04:59] (step=0049475) Train Loss: 5.6864, Train Steps/Sec: 1.12 +[2025-04-29 02:05:21] (step=0049500) Train Loss: 5.6474, Train Steps/Sec: 1.12 +[2025-04-29 02:05:44] (step=0049525) Train Loss: 5.6010, Train Steps/Sec: 1.12 +[2025-04-29 02:06:06] (step=0049550) Train Loss: 5.6772, Train Steps/Sec: 1.12 +[2025-04-29 02:06:28] (step=0049575) Train Loss: 5.5847, Train Steps/Sec: 1.12 +[2025-04-29 02:06:51] (step=0049600) Train Loss: 5.6657, Train Steps/Sec: 1.12 +[2025-04-29 02:07:13] (step=0049625) Train Loss: 5.6329, Train Steps/Sec: 1.12 +[2025-04-29 02:07:35] (step=0049650) Train Loss: 5.6197, Train Steps/Sec: 1.12 +[2025-04-29 02:07:57] (step=0049675) Train Loss: 5.6223, Train Steps/Sec: 1.12 +[2025-04-29 02:08:20] (step=0049700) Train Loss: 5.7153, Train Steps/Sec: 1.12 +[2025-04-29 02:08:42] (step=0049725) Train Loss: 5.6035, Train Steps/Sec: 1.12 +[2025-04-29 02:09:04] (step=0049750) Train Loss: 5.5965, Train Steps/Sec: 1.12 +[2025-04-29 02:09:27] (step=0049775) Train Loss: 5.6668, Train Steps/Sec: 1.12 +[2025-04-29 02:09:49] (step=0049800) Train Loss: 5.6839, Train Steps/Sec: 1.12 +[2025-04-29 02:10:11] (step=0049825) Train Loss: 5.6108, Train Steps/Sec: 1.12 +[2025-04-29 02:10:34] (step=0049850) Train Loss: 5.5880, Train Steps/Sec: 1.12 +[2025-04-29 02:10:56] (step=0049875) Train Loss: 5.5870, Train Steps/Sec: 1.12 +[2025-04-29 02:11:18] (step=0049900) Train Loss: 5.6143, Train Steps/Sec: 1.12 +[2025-04-29 02:11:40] (step=0049925) Train Loss: 5.6079, Train Steps/Sec: 1.12 +[2025-04-29 02:12:03] (step=0049950) Train Loss: 5.5630, Train Steps/Sec: 1.12 +[2025-04-29 02:12:25] (step=0049975) Train Loss: 5.6036, Train Steps/Sec: 1.12 +[2025-04-29 02:12:47] (step=0050000) Train Loss: 5.7343, Train Steps/Sec: 1.12 +[2025-04-29 02:12:47] text_config is None. Initializing the text config with default values (`OPTConfig`). +[2025-04-29 02:20:08] Finish Eval in 50000 steps... +[2025-04-29 02:20:27] Saved checkpoint to checkpoint/X2I_UltraEdiit_fourmask_LLava_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_filtered_objects_4mask_1280_context_after_stage1_5e4/000-GPT-XL/checkpoints/0050000.pt +[2025-04-29 02:20:29] Removed old checkpoint: checkpoint/X2I_UltraEdiit_fourmask_LLava_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_filtered_objects_4mask_1280_context_after_stage1_5e4/000-GPT-XL/checkpoints/0048000.pt +[2025-04-29 02:20:51] (step=0050025) Train Loss: 5.5493, Train Steps/Sec: 0.05 +[2025-04-29 02:21:14] (step=0050050) Train Loss: 5.6900, Train Steps/Sec: 1.12 +[2025-04-29 02:21:36] (step=0050075) Train Loss: 5.6614, Train Steps/Sec: 1.12 +[2025-04-29 02:21:58] (step=0050100) Train Loss: 5.6324, Train Steps/Sec: 1.12 +[2025-04-29 02:22:21] (step=0050125) Train Loss: 5.6211, Train Steps/Sec: 1.12 +[2025-04-29 02:22:43] (step=0050150) Train Loss: 5.6057, Train Steps/Sec: 1.12 +[2025-04-29 02:23:05] (step=0050175) Train Loss: 5.6238, Train Steps/Sec: 1.12 +[2025-04-29 02:23:28] (step=0050200) Train Loss: 5.7026, Train Steps/Sec: 1.12 +[2025-04-29 02:23:50] (step=0050225) Train Loss: 5.6001, Train Steps/Sec: 1.12 +[2025-04-29 02:24:12] (step=0050250) Train Loss: 5.6326, Train Steps/Sec: 1.12 +[2025-04-29 02:24:34] (step=0050275) Train Loss: 5.6424, Train Steps/Sec: 1.12 +[2025-04-29 02:24:57] (step=0050300) Train Loss: 5.6366, Train Steps/Sec: 1.12 +[2025-04-29 02:25:19] (step=0050325) Train Loss: 5.5785, Train Steps/Sec: 1.12 +[2025-04-29 02:25:41] (step=0050350) Train Loss: 5.5692, Train Steps/Sec: 1.12 +[2025-04-29 02:26:04] (step=0050375) Train Loss: 5.6382, Train Steps/Sec: 1.12 +[2025-04-29 02:26:26] (step=0050400) Train Loss: 5.5711, Train Steps/Sec: 1.12 +[2025-04-29 02:26:48] (step=0050425) Train Loss: 5.6069, Train Steps/Sec: 1.12 +[2025-04-29 02:27:11] (step=0050450) Train Loss: 5.6342, Train Steps/Sec: 1.12 +[2025-04-29 02:27:33] (step=0050475) Train Loss: 5.6478, Train Steps/Sec: 1.12 +[2025-04-29 02:27:55] (step=0050500) Train Loss: 5.6652, Train Steps/Sec: 1.12 +[2025-04-29 02:28:18] (step=0050525) Train Loss: 5.5626, Train Steps/Sec: 1.12 +[2025-04-29 02:28:40] (step=0050550) Train Loss: 5.6514, Train Steps/Sec: 1.12 +[2025-04-29 02:29:02] (step=0050575) Train Loss: 5.6636, Train Steps/Sec: 1.12 +[2025-04-29 02:29:25] (step=0050600) Train Loss: 5.5701, Train Steps/Sec: 1.12 +[2025-04-29 02:29:47] (step=0050625) Train Loss: 5.5989, Train Steps/Sec: 1.12 +[2025-04-29 02:30:09] (step=0050650) Train Loss: 5.6398, Train Steps/Sec: 1.12 +[2025-04-29 02:30:31] (step=0050675) Train Loss: 5.5757, Train Steps/Sec: 1.12 +[2025-04-29 02:30:54] (step=0050700) Train Loss: 5.5719, Train Steps/Sec: 1.12 +[2025-04-29 02:31:16] (step=0050725) Train Loss: 5.6483, Train Steps/Sec: 1.12 +[2025-04-29 02:31:38] (step=0050750) Train Loss: 5.6335, Train Steps/Sec: 1.12 +[2025-04-29 02:32:01] (step=0050775) Train Loss: 5.5834, Train Steps/Sec: 1.12 +[2025-04-29 02:32:23] (step=0050800) Train Loss: 5.6430, Train Steps/Sec: 1.12 +[2025-04-29 02:32:45] (step=0050825) Train Loss: 5.6490, Train Steps/Sec: 1.12 +[2025-04-29 02:33:08] (step=0050850) Train Loss: 5.6493, Train Steps/Sec: 1.12 +[2025-04-29 02:33:30] (step=0050875) Train Loss: 5.6415, Train Steps/Sec: 1.12 +[2025-04-29 02:33:52] (step=0050900) Train Loss: 5.6429, Train Steps/Sec: 1.12 +[2025-04-29 02:34:15] (step=0050925) Train Loss: 5.6457, Train Steps/Sec: 1.12 +[2025-04-29 02:34:37] (step=0050950) Train Loss: 5.5692, Train Steps/Sec: 1.12 +[2025-04-29 02:34:59] (step=0050975) Train Loss: 5.5841, Train Steps/Sec: 1.12 +[2025-04-29 02:35:22] (step=0051000) Train Loss: 5.6499, Train Steps/Sec: 1.12 +[2025-04-29 02:35:44] (step=0051025) Train Loss: 5.5649, Train Steps/Sec: 1.12 +[2025-04-29 02:36:06] (step=0051050) Train Loss: 5.5786, Train Steps/Sec: 1.12 +[2025-04-29 02:36:29] (step=0051075) Train Loss: 5.6848, Train Steps/Sec: 1.12 +[2025-04-29 02:36:51] (step=0051100) Train Loss: 5.5620, Train Steps/Sec: 1.12 +[2025-04-29 02:37:13] (step=0051125) Train Loss: 5.5884, Train Steps/Sec: 1.12 +[2025-04-29 02:37:36] (step=0051150) Train Loss: 5.6708, Train Steps/Sec: 1.12 +[2025-04-29 02:37:58] (step=0051175) Train Loss: 5.6038, Train Steps/Sec: 1.12 +[2025-04-29 02:38:20] (step=0051200) Train Loss: 5.6930, Train Steps/Sec: 1.12 +[2025-04-29 02:38:51] (step=0051225) Train Loss: 5.6314, Train Steps/Sec: 0.81 +[2025-04-29 02:39:41] (step=0051250) Train Loss: 5.5874, Train Steps/Sec: 0.50 +[2025-04-29 02:40:22] (step=0051275) Train Loss: 5.5342, Train Steps/Sec: 0.61 +[2025-04-29 02:40:54] (step=0051300) Train Loss: 5.6334, Train Steps/Sec: 0.79 +[2025-04-29 02:41:16] (step=0051325) Train Loss: 5.6428, Train Steps/Sec: 1.12 +[2025-04-29 02:41:38] (step=0051350) Train Loss: 5.5513, Train Steps/Sec: 1.12 +[2025-04-29 02:42:01] (step=0051375) Train Loss: 5.5896, Train Steps/Sec: 1.12 +[2025-04-29 02:42:23] (step=0051400) Train Loss: 5.6367, Train Steps/Sec: 1.12 +[2025-04-29 02:42:45] (step=0051425) Train Loss: 5.6231, Train Steps/Sec: 1.12 +[2025-04-29 02:43:08] (step=0051450) Train Loss: 5.6133, Train Steps/Sec: 1.12 +[2025-04-29 02:43:30] (step=0051475) Train Loss: 5.5458, Train Steps/Sec: 1.12 +[2025-04-29 02:43:52] (step=0051500) Train Loss: 5.6182, Train Steps/Sec: 1.12 diff --git a/CKPTS/X2I_UltraEdiit_fourmask_LLava_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_filtered_objects_4mask_1280_context_after_stage1_5e4/wandb/debug-internal.log b/CKPTS/X2I_UltraEdiit_fourmask_LLava_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_filtered_objects_4mask_1280_context_after_stage1_5e4/wandb/debug-internal.log new file mode 100644 index 0000000000000000000000000000000000000000..abddb029fc34b637ef4cd2a655c3bc311d194bb6 --- /dev/null +++ b/CKPTS/X2I_UltraEdiit_fourmask_LLava_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_filtered_objects_4mask_1280_context_after_stage1_5e4/wandb/debug-internal.log @@ -0,0 +1,10 @@ +{"time":"2025-04-28T10:25:01.670761577Z","level":"INFO","msg":"stream: starting","core version":"0.19.8","symlink path":"checkpoint/X2I_UltraEdiit_fourmask_LLava_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_filtered_objects_4mask_1280_context_after_stage1_5e4/wandb/run-20250428_102501-vl9wvenp/logs/debug-core.log"} +{"time":"2025-04-28T10:25:01.90209546Z","level":"INFO","msg":"created new stream","id":"vl9wvenp"} +{"time":"2025-04-28T10:25:01.902133521Z","level":"INFO","msg":"stream: started","id":"vl9wvenp"} +{"time":"2025-04-28T10:25:01.9021675Z","level":"INFO","msg":"handler: started","stream_id":"vl9wvenp"} +{"time":"2025-04-28T10:25:01.902172604Z","level":"INFO","msg":"writer: Do: started","stream_id":"vl9wvenp"} +{"time":"2025-04-28T10:25:01.902460333Z","level":"INFO","msg":"sender: started","stream_id":"vl9wvenp"} +{"time":"2025-04-28T10:25:02.279006691Z","level":"INFO","msg":"Starting system monitor"} +{"time":"2025-04-28T16:35:20.050683354Z","level":"INFO","msg":"api: retrying HTTP error","status":429,"url":"https://api.wandb.ai/files/haozhezhao/llamagen_ti2i/vl9wvenp/file_stream","body":"{\"error\":\"rate limit exceeded\"}"} +{"time":"2025-04-28T16:35:22.70871011Z","level":"INFO","msg":"api: retrying HTTP error","status":429,"url":"https://api.wandb.ai/files/haozhezhao/llamagen_ti2i/vl9wvenp/file_stream","body":"{\"error\":\"rate limit exceeded\"}"} +{"time":"2025-04-28T16:35:27.831923692Z","level":"INFO","msg":"api: retrying HTTP error","status":429,"url":"https://api.wandb.ai/files/haozhezhao/llamagen_ti2i/vl9wvenp/file_stream","body":"{\"error\":\"rate limit exceeded\"}"} diff --git a/CKPTS/X2I_UltraEdiit_fourmask_LLava_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_filtered_objects_4mask_1280_context_after_stage1_5e4/wandb/debug.log b/CKPTS/X2I_UltraEdiit_fourmask_LLava_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_filtered_objects_4mask_1280_context_after_stage1_5e4/wandb/debug.log new file mode 100644 index 0000000000000000000000000000000000000000..ca6b02c3535942361246b77338437cf4c8706dd5 --- /dev/null +++ b/CKPTS/X2I_UltraEdiit_fourmask_LLava_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_filtered_objects_4mask_1280_context_after_stage1_5e4/wandb/debug.log @@ -0,0 +1,22 @@ +2025-04-28 10:25:01,638 INFO MainThread:3827947 [wandb_setup.py:_flush():67] Current SDK version is 0.19.8 +2025-04-28 10:25:01,638 INFO MainThread:3827947 [wandb_setup.py:_flush():67] Configure stats pid to 3827947 +2025-04-28 10:25:01,638 INFO MainThread:3827947 [wandb_setup.py:_flush():67] Loading settings from /tmp/haozhezhao/.config/wandb/settings +2025-04-28 10:25:01,638 INFO MainThread:3827947 [wandb_setup.py:_flush():67] Loading settings from /tmp/haozhezhao/MLLMG/wandb/settings +2025-04-28 10:25:01,638 INFO MainThread:3827947 [wandb_setup.py:_flush():67] Loading settings from environment variables +2025-04-28 10:25:01,638 INFO MainThread:3827947 [wandb_init.py:setup_run_log_directory():647] Logging user logs to checkpoint/X2I_UltraEdiit_fourmask_LLava_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_filtered_objects_4mask_1280_context_after_stage1_5e4/wandb/run-20250428_102501-vl9wvenp/logs/debug.log +2025-04-28 10:25:01,638 INFO MainThread:3827947 [wandb_init.py:setup_run_log_directory():648] Logging internal logs to checkpoint/X2I_UltraEdiit_fourmask_LLava_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_filtered_objects_4mask_1280_context_after_stage1_5e4/wandb/run-20250428_102501-vl9wvenp/logs/debug-internal.log +2025-04-28 10:25:01,638 INFO MainThread:3827947 [wandb_init.py:init():761] calling init triggers +2025-04-28 10:25:01,638 INFO MainThread:3827947 [wandb_init.py:init():766] wandb.init called with sweep_config: {} +config: {'data_path': '/tmp/haozhezhao/MLLMG/jsonl_data/multiimage_training_for_llava_X2I_fourmask.jsonl', 'cloud_save_path': '/tmp/haozhezhao/MLLMG/checkpoint', 'no_local_save': False, 'vq_model': 'VQ-16', 'vq_ckpt': '/tmp/haozhezhao/model/llamagen_t2i/vq_ds16_t2i.pt', 'codebook_size': 16384, 'codebook_embed_dim': 8, 'gpt_model': 'GPT-XL', 'gpt_ckpt': '/tmp/haozhezhao/MLLMG/MLLMG_ckpts/checkpoints/AfterI2I_llavat5_subject_train_all_T5_subject_t2i_ti2i_120_w_flux_segment_noimageonly_empty_fix_train_all/0100000.pt', 'gpt_type': 't2i', 'vocab_size': 16384, 'cls_token_num': 1280, 'dropout_p': 0.1, 'token_dropout_p': 0.1, 'drop_path': 0.0, 'no_compile': False, 'results_dir': 'checkpoint/X2I_UltraEdiit_fourmask_LLava_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_filtered_objects_4mask_1280_context_after_stage1_5e4', 'dataset': 'ti2i', 'image_size': 512, 'downsample_size': 16, 'num_classes': 1000, 'epochs': 1, 'lr': 0.0005, 'weight_decay': 0.05, 'beta1': 0.9, 'beta2': 0.95, 'max_grad_norm': 1.0, 'global_batch_size': 24, 'global_seed': 0, 'num_workers': 4, 'log_every': 25, 'ckpt_every': 2000, 'gradient_accumulation_steps': 8, 'mixed_precision': 'bf16', 'val_data_path': '/tmp/haozhezhao/MLLMG/jsonl_data/multiimage_val_for_llava_X2I_fourmask_dreambenplus.jsonl', 'use_vision_tower': True, 'model_name_or_path': '/tmp/haozhezhao/model/blip2-flan-t5-xl', 'image_place_holder': '', 'processor_path': None, 'do_eval': True, 'max_eval_samples': 128, 'train_text_encoder': True, 'no_left_padding': False, 'cfg_scale': 7.5, 'top_k': 16384, 'temperature': 0.9, 'top_p': 1.0, 'eval_steps': 2000, 'project_name': 'llamagen_ti2i', 'load_from_checkpoint': '/tmp/haozhezhao/MLLMG/checkpoint/FIXed_3M_1epoch_step32000.pt', 'warmup': 0.05, 'lr_decay_style': 'cosine', 'lr_decay_ratio': 0.1, 'train_iters': 500000, 'class_dropout_prob': 0.1, 'with_image_only': False, 'image_only_rate': 0.1, 'stage2': False, 'subject_driven': True, 'load_subject_embedding': None, 'reference_data_path': '/tmp/haozhezhao/MLLMG/cc12m_reference_tunnel.jsonl', 'multimodal_encoder': 'llava', 'do_recovery': True, 'no_replace': False, 'resume': False, 'dreambench_eval': True, 'find_unused_parameters': True, 'load_visual_encoder': False, 'continue_stage1': False, 'replace_subject': False, 'train_all': True, 'save_total_limit': 1, 'load_language_projection': '/tmp/haozhezhao/MLLMG/llava-v1.5-flant5_fixed-pretrain/mm_projector.bin', 'mm_vision_tower': 'openai/clip-vit-large-patch14', 'load_fixed_llamagen': True, 'unfreeze_output': False, 'fix': 'gpt-empty-fix', 'rank': 0, 'world_size': 8, 'gpu': 0, 'dist_url': 'env://', 'distributed': True, 'dist_backend': 'nccl', '_wandb': {}} +2025-04-28 10:25:01,638 INFO MainThread:3827947 [wandb_init.py:init():784] starting backend +2025-04-28 10:25:01,638 INFO MainThread:3827947 [wandb_init.py:init():788] sending inform_init request +2025-04-28 10:25:01,668 INFO MainThread:3827947 [backend.py:_multiprocessing_setup():101] multiprocessing start_methods=fork,spawn,forkserver, using: spawn +2025-04-28 10:25:01,669 INFO MainThread:3827947 [wandb_init.py:init():798] backend started and connected +2025-04-28 10:25:01,672 INFO MainThread:3827947 [wandb_init.py:init():891] updated telemetry +2025-04-28 10:25:01,672 INFO MainThread:3827947 [wandb_init.py:init():915] communicating run to backend with 90.0 second timeout +2025-04-28 10:25:02,276 INFO MainThread:3827947 [wandb_init.py:init():990] starting run threads in backend +2025-04-28 10:25:02,358 INFO MainThread:3827947 [wandb_run.py:_console_start():2375] atexit reg +2025-04-28 10:25:02,359 INFO MainThread:3827947 [wandb_run.py:_redirect():2227] redirect: wrap_raw +2025-04-28 10:25:02,359 INFO MainThread:3827947 [wandb_run.py:_redirect():2292] Wrapping output streams. +2025-04-28 10:25:02,359 INFO MainThread:3827947 [wandb_run.py:_redirect():2315] Redirects installed. +2025-04-28 10:25:02,361 INFO MainThread:3827947 [wandb_init.py:init():1032] run started, returning control to user process diff --git a/CKPTS/X2I_UltraEdiit_fourmask_LLava_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_filtered_objects_4mask_1280_context_after_stage1_5e4/wandb/run-20250428_102501-vl9wvenp/files/output.log b/CKPTS/X2I_UltraEdiit_fourmask_LLava_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_filtered_objects_4mask_1280_context_after_stage1_5e4/wandb/run-20250428_102501-vl9wvenp/files/output.log new file mode 100644 index 0000000000000000000000000000000000000000..21c14564d166e7cb43ddfe5d0a27bd4b14fb930c --- /dev/null +++ b/CKPTS/X2I_UltraEdiit_fourmask_LLava_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_filtered_objects_4mask_1280_context_after_stage1_5e4/wandb/run-20250428_102501-vl9wvenp/files/output.log @@ -0,0 +1,2487 @@ +[2025-04-28 10:25:02] Training for 1 epochs... +[2025-04-28 10:25:02] Beginning epoch 0... + 0%| | 0/89905 [00:00 +tokenizer length after expend 32101 +tokenizer length before expend 32100 + /tmp/haozhezhao/anaconda3/envs/nlp/lib/python3.11/contextlib.py:105: FutureWarning: `torch.backends.cuda.sdp_kernel()` is deprecated. In the future, this context manager will be removed. Please see `torch.nn.attention.sdpa_kernel()` for the new context manager, with updated signature. + self.gen = func(*args, **kwds) | 0/6 [00:00 +tokenizer length after expend 32101 +tokenizer length before expend 32100 +100%|███████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████| 6/6 [06:00<00:00, 60.10s/it] +[2025-04-28 11:45:53] Finish Eval in 4000 steps...█████████████████████████████████████████████████████████████████████| 6/6 [06:00<00:00, 59.92s/it] +processing left-padding... +processing left-padding... +processing left-padding... +processing left-padding... +processing left-padding... +processing left-padding... +[2025-04-28 11:46:13] Saved checkpoint to checkpoint/X2I_UltraEdiit_fourmask_LLava_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_filtered_objects_4mask_1280_context_after_stage1_5e4/000-GPT-XL/checkpoints/0004000.pt +[2025-04-28 11:46:15] Removed old checkpoint: checkpoint/X2I_UltraEdiit_fourmask_LLava_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_filtered_objects_4mask_1280_context_after_stage1_5e4/000-GPT-XL/checkpoints/0002000.pt + 4%|██████ | 4024/89905 [1:21:34<22:19:30, 1.07it/s][2025-04-28 11:46:37] (step=0004025) Train Loss: 6.4099, Train Steps/Sec: 0.05 + 5%|██████ | 4049/89905 [1:21:56<21:25:53, 1.11it/s][2025-04-28 11:46:59] (step=0004050) Train Loss: 6.5192, Train Steps/Sec: 1.12 + 5%|██████▏ | 4074/89905 [1:22:18<21:16:55, 1.12it/s][2025-04-28 11:47:22] (step=0004075) Train Loss: 6.4452, Train Steps/Sec: 1.12 + 5%|██████▏ | 4099/89905 [1:22:41<21:10:26, 1.13it/s][2025-04-28 11:47:44] (step=0004100) Train Loss: 6.4579, Train Steps/Sec: 1.12 + 5%|██████▏ | 4124/89905 [1:23:03<21:12:45, 1.12it/s][2025-04-28 11:48:06] (step=0004125) Train Loss: 6.4193, Train Steps/Sec: 1.12 + 5%|██████▎ | 4149/89905 [1:23:25<21:14:00, 1.12it/s][2025-04-28 11:48:29] (step=0004150) Train Loss: 6.4320, Train Steps/Sec: 1.12 + 5%|██████▎ | 4174/89905 [1:23:48<21:06:58, 1.13it/s][2025-04-28 11:48:51] (step=0004175) Train Loss: 6.4474, Train Steps/Sec: 1.12 + 5%|██████▎ | 4199/89905 [1:24:10<21:09:41, 1.13it/s][2025-04-28 11:49:13] (step=0004200) Train Loss: 6.4446, Train Steps/Sec: 1.11 + 5%|██████▍ | 4224/89905 [1:24:32<21:41:37, 1.10it/s][2025-04-28 11:49:36] (step=0004225) Train Loss: 6.4663, Train Steps/Sec: 1.12 + 5%|██████▍ | 4249/89905 [1:24:55<21:27:26, 1.11it/s][2025-04-28 11:49:58] (step=0004250) Train Loss: 6.5018, Train Steps/Sec: 1.12 + 5%|██████▍ | 4274/89905 [1:25:17<21:16:26, 1.12it/s][2025-04-28 11:50:20] (step=0004275) Train Loss: 6.5116, Train Steps/Sec: 1.12 + 5%|██████▌ | 4299/89905 [1:25:39<21:14:59, 1.12it/s][2025-04-28 11:50:43] (step=0004300) Train Loss: 6.4493, Train Steps/Sec: 1.12 + 5%|██████▌ | 4324/89905 [1:26:02<21:09:26, 1.12it/s][2025-04-28 11:51:05] (step=0004325) Train Loss: 6.4124, Train Steps/Sec: 1.12 + 5%|██████▌ | 4349/89905 [1:26:24<21:02:48, 1.13it/s][2025-04-28 11:51:27] (step=0004350) Train Loss: 6.4117, Train Steps/Sec: 1.12 + 5%|██████▌ | 4374/89905 [1:26:46<21:10:48, 1.12it/s][2025-04-28 11:51:50] (step=0004375) Train Loss: 6.4720, Train Steps/Sec: 1.11 + 5%|██████▋ | 4399/89905 [1:27:09<21:02:15, 1.13it/s][2025-04-28 11:52:12] (step=0004400) Train Loss: 6.4215, Train Steps/Sec: 1.11 + 5%|██████▋ | 4424/89905 [1:27:31<21:36:16, 1.10it/s][2025-04-28 11:52:35] (step=0004425) Train Loss: 6.4024, Train Steps/Sec: 1.12 + 5%|██████▋ | 4449/89905 [1:27:54<21:22:59, 1.11it/s][2025-04-28 11:52:57] (step=0004450) Train Loss: 6.4817, Train Steps/Sec: 1.12 + 5%|██████▊ | 4474/89905 [1:28:16<21:13:48, 1.12it/s][2025-04-28 11:53:19] (step=0004475) Train Loss: 6.4606, Train Steps/Sec: 1.12 + 5%|██████▊ | 4499/89905 [1:28:38<21:11:52, 1.12it/s][2025-04-28 11:53:42] (step=0004500) Train Loss: 6.4599, Train Steps/Sec: 1.12 + 5%|██████▊ | 4524/89905 [1:29:01<21:03:04, 1.13it/s][2025-04-28 11:54:04] (step=0004525) Train Loss: 6.4290, Train Steps/Sec: 1.12 + 5%|██████▉ | 4549/89905 [1:29:23<21:01:23, 1.13it/s][2025-04-28 11:54:26] (step=0004550) Train Loss: 6.4144, Train Steps/Sec: 1.12 + 5%|██████▉ | 4574/89905 [1:29:45<21:00:03, 1.13it/s][2025-04-28 11:54:49] (step=0004575) Train Loss: 6.3784, Train Steps/Sec: 1.12 + 5%|██████▉ | 4599/89905 [1:30:07<20:57:50, 1.13it/s][2025-04-28 11:55:11] (step=0004600) Train Loss: 6.3721, Train Steps/Sec: 1.12 + 5%|██████▉ | 4624/89905 [1:30:30<21:37:17, 1.10it/s][2025-04-28 11:55:33] (step=0004625) Train Loss: 6.4663, Train Steps/Sec: 1.12 + 5%|███████ | 4649/89905 [1:30:52<21:22:12, 1.11it/s][2025-04-28 11:55:56] (step=0004650) Train Loss: 6.4239, Train Steps/Sec: 1.12 + 5%|███████ | 4674/89905 [1:31:15<21:11:53, 1.12it/s][2025-04-28 11:56:18] (step=0004675) Train Loss: 6.4599, Train Steps/Sec: 1.12 + 5%|███████ | 4699/89905 [1:31:37<21:05:01, 1.12it/s][2025-04-28 11:56:40] (step=0004700) Train Loss: 6.3608, Train Steps/Sec: 1.12 + 5%|███████▏ | 4724/89905 [1:31:59<21:00:36, 1.13it/s][2025-04-28 11:57:03] (step=0004725) Train Loss: 6.4572, Train Steps/Sec: 1.12 + 5%|███████▏ | 4749/89905 [1:32:22<20:56:58, 1.13it/s][2025-04-28 11:57:25] (step=0004750) Train Loss: 6.4561, Train Steps/Sec: 1.12 + 5%|███████▏ | 4774/89905 [1:32:44<20:58:16, 1.13it/s][2025-04-28 11:57:47] (step=0004775) Train Loss: 6.4806, Train Steps/Sec: 1.12 + 5%|███████▎ | 4799/89905 [1:33:06<20:58:24, 1.13it/s][2025-04-28 11:58:10] (step=0004800) Train Loss: 6.4253, Train Steps/Sec: 1.11 + 5%|███████▎ | 4824/89905 [1:33:29<21:32:10, 1.10it/s][2025-04-28 11:58:32] (step=0004825) Train Loss: 6.4200, Train Steps/Sec: 1.12 + 5%|███████▎ | 4849/89905 [1:33:51<21:19:00, 1.11it/s][2025-04-28 11:58:54] (step=0004850) Train Loss: 6.4268, Train Steps/Sec: 1.12 + 5%|███████▎ | 4874/89905 [1:34:13<21:11:50, 1.11it/s][2025-04-28 11:59:17] (step=0004875) Train Loss: 6.3626, Train Steps/Sec: 1.12 + 5%|███████▍ | 4899/89905 [1:34:36<21:03:27, 1.12it/s][2025-04-28 11:59:39] (step=0004900) Train Loss: 6.4606, Train Steps/Sec: 1.12 + 5%|███████▍ | 4924/89905 [1:35:17<83:35:55, 3.54s/it][2025-04-28 12:00:30] (step=0004925) Train Loss: 6.3901, Train Steps/Sec: 0.49 + 6%|███████▍ | 4949/89905 [1:35:58<21:51:40, 1.08it/s][2025-04-28 12:01:02] (step=0004950) Train Loss: 6.4584, Train Steps/Sec: 0.79 + 6%|███████▌ | 4974/89905 [1:36:30<21:04:43, 1.12it/s][2025-04-28 12:01:33] (step=0004975) Train Loss: 6.4209, Train Steps/Sec: 0.79 + 6%|███████▌ | 4999/89905 [1:36:52<20:47:45, 1.13it/s][2025-04-28 12:01:56] (step=0005000) Train Loss: 6.3990, Train Steps/Sec: 1.12 + 6%|███████▌ | 5024/89905 [1:37:15<21:30:24, 1.10it/s][2025-04-28 12:02:18] (step=0005025) Train Loss: 6.4649, Train Steps/Sec: 1.12 + 6%|███████▋ | 5049/89905 [1:37:37<21:13:22, 1.11it/s][2025-04-28 12:02:40] (step=0005050) Train Loss: 6.3706, Train Steps/Sec: 1.12 + 6%|███████▋ | 5074/89905 [1:37:59<21:02:31, 1.12it/s][2025-04-28 12:03:03] (step=0005075) Train Loss: 6.4158, Train Steps/Sec: 1.12 + 6%|███████▋ | 5099/89905 [1:38:22<21:00:13, 1.12it/s][2025-04-28 12:03:25] (step=0005100) Train Loss: 6.3972, Train Steps/Sec: 1.12 + 6%|███████▊ | 5124/89905 [1:38:44<20:53:29, 1.13it/s][2025-04-28 12:03:47] (step=0005125) Train Loss: 6.3681, Train Steps/Sec: 1.12 + 6%|███████▊ | 5149/89905 [1:39:06<20:50:11, 1.13it/s][2025-04-28 12:04:10] (step=0005150) Train Loss: 6.3981, Train Steps/Sec: 1.12 + 6%|███████▊ | 5174/89905 [1:39:29<20:50:48, 1.13it/s][2025-04-28 12:04:32] (step=0005175) Train Loss: 6.4441, Train Steps/Sec: 1.12 + 6%|███████▊ | 5199/89905 [1:39:51<20:47:37, 1.13it/s][2025-04-28 12:04:54] (step=0005200) Train Loss: 6.4513, Train Steps/Sec: 1.11 + 6%|███████▉ | 5224/89905 [1:40:13<21:29:23, 1.09it/s][2025-04-28 12:05:17] (step=0005225) Train Loss: 6.4187, Train Steps/Sec: 1.12 + 6%|███████▉ | 5249/89905 [1:40:36<21:12:16, 1.11it/s][2025-04-28 12:05:39] (step=0005250) Train Loss: 6.3721, Train Steps/Sec: 1.12 + 6%|███████▉ | 5274/89905 [1:40:58<21:05:11, 1.11it/s][2025-04-28 12:06:01] (step=0005275) Train Loss: 6.4778, Train Steps/Sec: 1.12 + 6%|████████ | 5299/89905 [1:41:20<20:55:41, 1.12it/s][2025-04-28 12:06:24] (step=0005300) Train Loss: 6.4105, Train Steps/Sec: 1.12 + 6%|████████ | 5324/89905 [1:41:43<20:56:45, 1.12it/s][2025-04-28 12:06:46] (step=0005325) Train Loss: 6.4380, Train Steps/Sec: 1.12 + 6%|████████ | 5349/89905 [1:42:05<20:52:50, 1.12it/s][2025-04-28 12:07:08] (step=0005350) Train Loss: 6.4355, Train Steps/Sec: 1.12 + 6%|████████▏ | 5374/89905 [1:42:27<20:47:50, 1.13it/s][2025-04-28 12:07:31] (step=0005375) Train Loss: 6.3652, Train Steps/Sec: 1.12 + 6%|████████▏ | 5399/89905 [1:42:50<20:41:39, 1.13it/s][2025-04-28 12:07:53] (step=0005400) Train Loss: 6.3738, Train Steps/Sec: 1.12 + 6%|████████▏ | 5424/89905 [1:43:12<21:25:54, 1.09it/s][2025-04-28 12:08:15] (step=0005425) Train Loss: 6.4244, Train Steps/Sec: 1.12 + 6%|████████▏ | 5449/89905 [1:43:34<21:10:31, 1.11it/s][2025-04-28 12:08:38] (step=0005450) Train Loss: 6.4083, Train Steps/Sec: 1.12 + 6%|████████▎ | 5474/89905 [1:43:57<21:00:31, 1.12it/s][2025-04-28 12:09:00] (step=0005475) Train Loss: 6.4378, Train Steps/Sec: 1.12 + 6%|████████▎ | 5499/89905 [1:44:19<20:53:29, 1.12it/s][2025-04-28 12:09:22] (step=0005500) Train Loss: 6.4167, Train Steps/Sec: 1.12 + 6%|████████▎ | 5524/89905 [1:44:41<20:53:37, 1.12it/s][2025-04-28 12:09:45] (step=0005525) Train Loss: 6.3858, Train Steps/Sec: 1.12 + 6%|████████▍ | 5549/89905 [1:45:04<20:47:18, 1.13it/s][2025-04-28 12:10:07] (step=0005550) Train Loss: 6.4631, Train Steps/Sec: 1.12 + 6%|████████▍ | 5574/89905 [1:45:26<20:45:36, 1.13it/s][2025-04-28 12:10:29] (step=0005575) Train Loss: 6.3906, Train Steps/Sec: 1.12 + 6%|████████▍ | 5599/89905 [1:45:48<20:41:17, 1.13it/s][2025-04-28 12:10:52] (step=0005600) Train Loss: 6.3885, Train Steps/Sec: 1.12 + 6%|████████▌ | 5624/89905 [1:46:11<21:24:54, 1.09it/s][2025-04-28 12:11:14] (step=0005625) Train Loss: 6.3565, Train Steps/Sec: 1.12 + 6%|████████▌ | 5649/89905 [1:46:33<21:05:49, 1.11it/s][2025-04-28 12:11:37] (step=0005650) Train Loss: 6.3953, Train Steps/Sec: 1.12 + 6%|████████▌ | 5674/89905 [1:46:55<20:58:24, 1.12it/s][2025-04-28 12:11:59] (step=0005675) Train Loss: 6.3692, Train Steps/Sec: 1.12 + 6%|████████▌ | 5699/89905 [1:47:18<20:50:24, 1.12it/s][2025-04-28 12:12:21] (step=0005700) Train Loss: 6.3866, Train Steps/Sec: 1.12 + 6%|████████▋ | 5724/89905 [1:47:40<20:47:36, 1.12it/s][2025-04-28 12:12:44] (step=0005725) Train Loss: 6.5024, Train Steps/Sec: 1.12 + 6%|████████▋ | 5749/89905 [1:48:03<20:46:10, 1.13it/s][2025-04-28 12:13:06] (step=0005750) Train Loss: 6.4182, Train Steps/Sec: 1.12 + 6%|████████▋ | 5774/89905 [1:48:25<20:37:43, 1.13it/s][2025-04-28 12:13:28] (step=0005775) Train Loss: 6.4550, Train Steps/Sec: 1.12 + 6%|████████▊ | 5799/89905 [1:48:47<20:40:06, 1.13it/s][2025-04-28 12:13:51] (step=0005800) Train Loss: 6.3405, Train Steps/Sec: 1.12 + 6%|████████▊ | 5824/89905 [1:49:10<21:23:30, 1.09it/s][2025-04-28 12:14:13] (step=0005825) Train Loss: 6.4379, Train Steps/Sec: 1.12 + 7%|████████▊ | 5849/89905 [1:49:32<21:03:39, 1.11it/s][2025-04-28 12:14:35] (step=0005850) Train Loss: 6.3994, Train Steps/Sec: 1.12 + 7%|████████▉ | 5874/89905 [1:49:54<20:55:34, 1.12it/s][2025-04-28 12:14:58] (step=0005875) Train Loss: 6.3801, Train Steps/Sec: 1.12 + 7%|████████▉ | 5899/89905 [1:50:17<20:43:25, 1.13it/s][2025-04-28 12:15:20] (step=0005900) Train Loss: 6.4163, Train Steps/Sec: 1.12 + 7%|████████▉ | 5924/89905 [1:50:39<20:46:32, 1.12it/s][2025-04-28 12:15:42] (step=0005925) Train Loss: 6.3760, Train Steps/Sec: 1.12 + 7%|████████▉ | 5949/89905 [1:51:01<20:42:11, 1.13it/s][2025-04-28 12:16:05] (step=0005950) Train Loss: 6.3828, Train Steps/Sec: 1.12 + 7%|█████████ | 5974/89905 [1:51:24<20:42:05, 1.13it/s][2025-04-28 12:16:27] (step=0005975) Train Loss: 6.3960, Train Steps/Sec: 1.12 + 7%|█████████ | 5999/89905 [1:51:46<20:43:29, 1.12it/s][2025-04-28 12:16:50] (step=0006000) Train Loss: 6.3470, Train Steps/Sec: 1.10 +You are using a model of type blip-2 to instantiate a model of type instructblip. This is not supported for all configurations of models and can yield errors. +[2025-04-28 12:16:50] text_config is None. Initializing the text config with default values (`OPTConfig`). +model_name_or_path:/tmp/haozhezhao/model/blip2-flan-t5-xl +image_place_holder: +tokenizer length after expend 32101 +tokenizer length before expend 32100 +100%|███████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████| 6/6 [06:00<00:00, 60.04s/it] +[2025-04-28 12:24:09] Finish Eval in 6000 steps...█████████████████████████████████████████████████████████████████████| 6/6 [05:59<00:00, 59.89s/it] +processing left-padding... +processing left-padding... +processing left-padding... +processing left-padding... +processing left-padding... +processing left-padding... +[2025-04-28 12:24:28] Saved checkpoint to checkpoint/X2I_UltraEdiit_fourmask_LLava_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_filtered_objects_4mask_1280_context_after_stage1_5e4/000-GPT-XL/checkpoints/0006000.pt +[2025-04-28 12:24:30] Removed old checkpoint: checkpoint/X2I_UltraEdiit_fourmask_LLava_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_filtered_objects_4mask_1280_context_after_stage1_5e4/000-GPT-XL/checkpoints/0004000.pt + 7%|█████████ | 6024/89905 [1:59:50<21:51:20, 1.07it/s][2025-04-28 12:24:53] (step=0006025) Train Loss: 6.3688, Train Steps/Sec: 0.05 + 7%|█████████▏ | 6049/89905 [2:00:12<21:07:40, 1.10it/s][2025-04-28 12:25:16] (step=0006050) Train Loss: 6.3762, Train Steps/Sec: 1.11 + 7%|█████████▏ | 6074/89905 [2:00:35<20:51:57, 1.12it/s][2025-04-28 12:25:38] (step=0006075) Train Loss: 6.3538, Train Steps/Sec: 1.12 + 7%|█████████▏ | 6099/89905 [2:00:57<20:47:12, 1.12it/s][2025-04-28 12:26:01] (step=0006100) Train Loss: 6.3220, Train Steps/Sec: 1.12 + 7%|█████████▎ | 6124/89905 [2:01:20<20:43:58, 1.12it/s][2025-04-28 12:26:23] (step=0006125) Train Loss: 6.3212, Train Steps/Sec: 1.12 + 7%|█████████▎ | 6149/89905 [2:01:42<20:42:22, 1.12it/s][2025-04-28 12:26:45] (step=0006150) Train Loss: 6.3841, Train Steps/Sec: 1.12 + 7%|█████████▎ | 6174/89905 [2:02:04<20:41:32, 1.12it/s][2025-04-28 12:27:08] (step=0006175) Train Loss: 6.4498, Train Steps/Sec: 1.12 + 7%|█████████▍ | 6199/89905 [2:02:27<20:32:00, 1.13it/s][2025-04-28 12:27:30] (step=0006200) Train Loss: 6.4097, Train Steps/Sec: 1.11 + 7%|█████████▍ | 6224/89905 [2:02:49<21:15:45, 1.09it/s][2025-04-28 12:27:52] (step=0006225) Train Loss: 6.4438, Train Steps/Sec: 1.12 + 7%|█████████▍ | 6249/89905 [2:03:11<20:58:58, 1.11it/s][2025-04-28 12:28:15] (step=0006250) Train Loss: 6.3409, Train Steps/Sec: 1.11 + 7%|█████████▍ | 6274/89905 [2:03:34<20:42:56, 1.12it/s][2025-04-28 12:28:37] (step=0006275) Train Loss: 6.3623, Train Steps/Sec: 1.11 + 7%|█████████▌ | 6299/89905 [2:03:57<20:51:39, 1.11it/s][2025-04-28 12:29:00] (step=0006300) Train Loss: 6.3150, Train Steps/Sec: 1.10 + 7%|█████████▌ | 6324/89905 [2:04:19<20:45:35, 1.12it/s][2025-04-28 12:29:22] (step=0006325) Train Loss: 6.3871, Train Steps/Sec: 1.11 + 7%|█████████▌ | 6349/89905 [2:04:41<20:43:14, 1.12it/s][2025-04-28 12:29:45] (step=0006350) Train Loss: 6.3271, Train Steps/Sec: 1.12 + 7%|█████████▋ | 6374/89905 [2:05:04<20:38:50, 1.12it/s][2025-04-28 12:30:07] (step=0006375) Train Loss: 6.3487, Train Steps/Sec: 1.11 + 7%|█████████▋ | 6399/89905 [2:05:26<20:40:40, 1.12it/s][2025-04-28 12:30:30] (step=0006400) Train Loss: 6.3749, Train Steps/Sec: 1.11 + 7%|█████████▋ | 6424/89905 [2:05:49<21:14:54, 1.09it/s][2025-04-28 12:30:52] (step=0006425) Train Loss: 6.3576, Train Steps/Sec: 1.11 + 7%|█████████▊ | 6449/89905 [2:06:11<20:58:22, 1.11it/s][2025-04-28 12:31:15] (step=0006450) Train Loss: 6.3883, Train Steps/Sec: 1.12 + 7%|█████████▊ | 6474/89905 [2:06:34<20:49:03, 1.11it/s][2025-04-28 12:31:37] (step=0006475) Train Loss: 6.3179, Train Steps/Sec: 1.12 + 7%|█████████▊ | 6499/89905 [2:06:56<20:39:27, 1.12it/s][2025-04-28 12:32:00] (step=0006500) Train Loss: 6.3753, Train Steps/Sec: 1.12 + 7%|█████████▊ | 6524/89905 [2:07:18<20:38:38, 1.12it/s][2025-04-28 12:32:22] (step=0006525) Train Loss: 6.3083, Train Steps/Sec: 1.12 + 7%|█████████▉ | 6549/89905 [2:07:41<20:41:53, 1.12it/s][2025-04-28 12:32:44] (step=0006550) Train Loss: 6.3374, Train Steps/Sec: 1.11 + 7%|█████████▉ | 6574/89905 [2:08:04<20:39:51, 1.12it/s][2025-04-28 12:33:07] (step=0006575) Train Loss: 6.3211, Train Steps/Sec: 1.11 + 7%|█████████▉ | 6599/89905 [2:08:26<20:45:46, 1.11it/s][2025-04-28 12:33:30] (step=0006600) Train Loss: 6.2962, Train Steps/Sec: 1.10 + 7%|██████████ | 6624/89905 [2:08:49<21:13:45, 1.09it/s][2025-04-28 12:33:52] (step=0006625) Train Loss: 6.3448, Train Steps/Sec: 1.11 + 7%|██████████ | 6649/89905 [2:09:11<21:10:12, 1.09it/s][2025-04-28 12:34:15] (step=0006650) Train Loss: 6.3616, Train Steps/Sec: 1.11 + 7%|██████████ | 6674/89905 [2:09:34<20:47:04, 1.11it/s][2025-04-28 12:34:37] (step=0006675) Train Loss: 6.2940, Train Steps/Sec: 1.11 + 7%|██████████▏ | 6699/89905 [2:09:56<20:40:35, 1.12it/s][2025-04-28 12:34:59] (step=0006700) Train Loss: 6.3012, Train Steps/Sec: 1.12 + 7%|██████████▏ | 6724/89905 [2:10:18<20:33:25, 1.12it/s][2025-04-28 12:35:22] (step=0006725) Train Loss: 6.3618, Train Steps/Sec: 1.12 + 8%|██████████▏ | 6749/89905 [2:10:41<20:30:21, 1.13it/s][2025-04-28 12:35:44] (step=0006750) Train Loss: 6.3523, Train Steps/Sec: 1.11 + 8%|██████████▏ | 6774/89905 [2:11:03<20:43:07, 1.11it/s][2025-04-28 12:36:07] (step=0006775) Train Loss: 6.2996, Train Steps/Sec: 1.11 + 8%|██████████▎ | 6799/89905 [2:11:26<20:28:36, 1.13it/s][2025-04-28 12:36:29] (step=0006800) Train Loss: 6.3512, Train Steps/Sec: 1.11 + 8%|██████████▎ | 6824/89905 [2:11:48<21:01:03, 1.10it/s][2025-04-28 12:36:52] (step=0006825) Train Loss: 6.3491, Train Steps/Sec: 1.12 + 8%|██████████▎ | 6849/89905 [2:12:11<20:53:04, 1.10it/s][2025-04-28 12:37:14] (step=0006850) Train Loss: 6.3301, Train Steps/Sec: 1.12 + 8%|██████████▍ | 6874/89905 [2:12:33<21:06:08, 1.09it/s][2025-04-28 12:37:37] (step=0006875) Train Loss: 6.3186, Train Steps/Sec: 1.11 + 8%|██████████▍ | 6899/89905 [2:12:56<20:41:03, 1.11it/s][2025-04-28 12:37:59] (step=0006900) Train Loss: 6.3850, Train Steps/Sec: 1.11 + 8%|██████████▍ | 6924/89905 [2:13:18<20:41:44, 1.11it/s][2025-04-28 12:38:22] (step=0006925) Train Loss: 6.3080, Train Steps/Sec: 1.11 + 8%|██████████▌ | 6949/89905 [2:13:41<20:44:31, 1.11it/s][2025-04-28 12:38:44] (step=0006950) Train Loss: 6.2326, Train Steps/Sec: 1.11 + 8%|██████████▌ | 6974/89905 [2:14:03<20:36:39, 1.12it/s][2025-04-28 12:39:07] (step=0006975) Train Loss: 6.3361, Train Steps/Sec: 1.11 + 8%|██████████▌ | 6999/89905 [2:14:26<20:36:36, 1.12it/s][2025-04-28 12:39:29] (step=0007000) Train Loss: 6.3091, Train Steps/Sec: 1.11 + 8%|██████████▋ | 7024/89905 [2:14:48<21:02:55, 1.09it/s][2025-04-28 12:39:52] (step=0007025) Train Loss: 6.3572, Train Steps/Sec: 1.11 + 8%|██████████▋ | 7049/89905 [2:15:11<20:54:45, 1.10it/s][2025-04-28 12:40:14] (step=0007050) Train Loss: 6.3504, Train Steps/Sec: 1.12 + 8%|██████████▋ | 7074/89905 [2:15:33<20:39:22, 1.11it/s][2025-04-28 12:40:37] (step=0007075) Train Loss: 6.3142, Train Steps/Sec: 1.11 + 8%|██████████▋ | 7099/89905 [2:15:56<20:39:25, 1.11it/s][2025-04-28 12:40:59] (step=0007100) Train Loss: 6.2948, Train Steps/Sec: 1.11 + 8%|██████████▊ | 7124/89905 [2:16:18<20:50:20, 1.10it/s][2025-04-28 12:41:22] (step=0007125) Train Loss: 6.2980, Train Steps/Sec: 1.11 + 8%|██████████▊ | 7149/89905 [2:16:41<20:27:18, 1.12it/s][2025-04-28 12:41:44] (step=0007150) Train Loss: 6.2722, Train Steps/Sec: 1.12 + 8%|██████████▊ | 7174/89905 [2:17:03<20:26:32, 1.12it/s][2025-04-28 12:42:06] (step=0007175) Train Loss: 6.2736, Train Steps/Sec: 1.12 + 8%|██████████▉ | 7199/89905 [2:17:25<20:20:28, 1.13it/s][2025-04-28 12:42:29] (step=0007200) Train Loss: 6.3163, Train Steps/Sec: 1.11 + 8%|██████████▉ | 7224/89905 [2:17:48<21:00:32, 1.09it/s][2025-04-28 12:42:51] (step=0007225) Train Loss: 6.3273, Train Steps/Sec: 1.12 + 8%|██████████▉ | 7249/89905 [2:18:10<20:46:44, 1.10it/s][2025-04-28 12:43:14] (step=0007250) Train Loss: 6.2165, Train Steps/Sec: 1.12 + 8%|███████████ | 7274/89905 [2:18:33<20:36:16, 1.11it/s][2025-04-28 12:43:36] (step=0007275) Train Loss: 6.3048, Train Steps/Sec: 1.12 + 8%|███████████ | 7299/89905 [2:18:55<20:31:07, 1.12it/s][2025-04-28 12:43:58] (step=0007300) Train Loss: 6.3032, Train Steps/Sec: 1.12 + 8%|███████████ | 7324/89905 [2:19:17<20:28:16, 1.12it/s][2025-04-28 12:44:21] (step=0007325) Train Loss: 6.2981, Train Steps/Sec: 1.12 + 8%|███████████ | 7349/89905 [2:19:40<20:24:21, 1.12it/s][2025-04-28 12:44:43] (step=0007350) Train Loss: 6.3170, Train Steps/Sec: 1.12 + 8%|███████████▏ | 7374/89905 [2:20:02<20:22:59, 1.12it/s][2025-04-28 12:45:05] (step=0007375) Train Loss: 6.2532, Train Steps/Sec: 1.12 + 8%|███████████▏ | 7399/89905 [2:20:24<20:17:11, 1.13it/s][2025-04-28 12:45:28] (step=0007400) Train Loss: 6.3362, Train Steps/Sec: 1.11 + 8%|███████████▏ | 7424/89905 [2:20:47<20:57:52, 1.09it/s][2025-04-28 12:45:50] (step=0007425) Train Loss: 6.2670, Train Steps/Sec: 1.12 + 8%|███████████▎ | 7449/89905 [2:21:09<20:38:57, 1.11it/s][2025-04-28 12:46:13] (step=0007450) Train Loss: 6.2893, Train Steps/Sec: 1.12 + 8%|███████████▎ | 7474/89905 [2:21:32<20:30:55, 1.12it/s][2025-04-28 12:46:35] (step=0007475) Train Loss: 6.2692, Train Steps/Sec: 1.12 + 8%|███████████▎ | 7499/89905 [2:21:54<20:24:11, 1.12it/s][2025-04-28 12:46:57] (step=0007500) Train Loss: 6.2634, Train Steps/Sec: 1.12 + 8%|███████████▍ | 7524/89905 [2:22:16<20:20:17, 1.13it/s][2025-04-28 12:47:20] (step=0007525) Train Loss: 6.2611, Train Steps/Sec: 1.12 + 8%|███████████▍ | 7549/89905 [2:22:39<20:19:12, 1.13it/s][2025-04-28 12:47:42] (step=0007550) Train Loss: 6.2244, Train Steps/Sec: 1.12 + 8%|███████████▍ | 7574/89905 [2:23:01<20:15:15, 1.13it/s][2025-04-28 12:48:04] (step=0007575) Train Loss: 6.2689, Train Steps/Sec: 1.12 + 8%|███████████▍ | 7599/89905 [2:23:23<20:12:01, 1.13it/s][2025-04-28 12:48:27] (step=0007600) Train Loss: 6.3749, Train Steps/Sec: 1.11 + 8%|███████████▌ | 7624/89905 [2:23:46<20:50:14, 1.10it/s][2025-04-28 12:48:49] (step=0007625) Train Loss: 6.3153, Train Steps/Sec: 1.12 + 9%|███████████▌ | 7649/89905 [2:24:08<20:39:02, 1.11it/s][2025-04-28 12:49:11] (step=0007650) Train Loss: 6.2967, Train Steps/Sec: 1.12 + 9%|███████████▌ | 7674/89905 [2:24:30<20:29:28, 1.11it/s][2025-04-28 12:49:34] (step=0007675) Train Loss: 6.2646, Train Steps/Sec: 1.12 + 9%|███████████▋ | 7699/89905 [2:24:53<20:22:37, 1.12it/s][2025-04-28 12:49:56] (step=0007700) Train Loss: 6.2918, Train Steps/Sec: 1.12 + 9%|███████████▋ | 7724/89905 [2:25:15<20:18:15, 1.12it/s][2025-04-28 12:50:18] (step=0007725) Train Loss: 6.2810, Train Steps/Sec: 1.12 + 9%|███████████▋ | 7749/89905 [2:25:37<20:18:25, 1.12it/s][2025-04-28 12:50:41] (step=0007750) Train Loss: 6.2429, Train Steps/Sec: 1.12 + 9%|███████████▊ | 7774/89905 [2:26:00<20:17:11, 1.12it/s][2025-04-28 12:51:03] (step=0007775) Train Loss: 6.2446, Train Steps/Sec: 1.12 + 9%|███████████▊ | 7799/89905 [2:26:22<20:11:34, 1.13it/s][2025-04-28 12:51:26] (step=0007800) Train Loss: 6.2847, Train Steps/Sec: 1.11 + 9%|███████████▊ | 7824/89905 [2:26:45<20:52:21, 1.09it/s][2025-04-28 12:51:48] (step=0007825) Train Loss: 6.2613, Train Steps/Sec: 1.12 + 9%|███████████▊ | 7849/89905 [2:27:07<20:29:06, 1.11it/s][2025-04-28 12:52:10] (step=0007850) Train Loss: 6.3335, Train Steps/Sec: 1.12 + 9%|███████████▉ | 7874/89905 [2:27:29<20:28:25, 1.11it/s][2025-04-28 12:52:33] (step=0007875) Train Loss: 6.2170, Train Steps/Sec: 1.12 + 9%|███████████▉ | 7899/89905 [2:27:52<20:20:35, 1.12it/s][2025-04-28 12:52:55] (step=0007900) Train Loss: 6.2997, Train Steps/Sec: 1.12 + 9%|███████████▉ | 7924/89905 [2:28:14<20:17:18, 1.12it/s][2025-04-28 12:53:17] (step=0007925) Train Loss: 6.2252, Train Steps/Sec: 1.12 + 9%|████████████ | 7949/89905 [2:28:36<20:11:20, 1.13it/s][2025-04-28 12:53:40] (step=0007950) Train Loss: 6.3446, Train Steps/Sec: 1.12 + 9%|████████████ | 7974/89905 [2:28:59<20:06:24, 1.13it/s][2025-04-28 12:54:02] (step=0007975) Train Loss: 6.2191, Train Steps/Sec: 1.12 + 9%|████████████ | 7999/89905 [2:29:21<20:12:38, 1.13it/s][2025-04-28 12:54:25] (step=0008000) Train Loss: 6.2446, Train Steps/Sec: 1.11 +You are using a model of type blip-2 to instantiate a model of type instructblip. This is not supported for all configurations of models and can yield errors. +[2025-04-28 12:54:25] text_config is None. Initializing the text config with default values (`OPTConfig`). +model_name_or_path:/tmp/haozhezhao/model/blip2-flan-t5-xl +image_place_holder: +tokenizer length after expend 32101 +tokenizer length before expend 32100 +100%|███████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████| 6/6 [06:02<00:00, 60.39s/it] +[2025-04-28 13:01:45] Finish Eval in 8000 steps...█████████████████████████████████████████████████████████████████████| 6/6 [06:01<00:00, 60.21s/it] +processing left-padding... +processing left-padding... +processing left-padding... +processing left-padding... +processing left-padding... +processing left-padding... +[2025-04-28 13:02:03] Saved checkpoint to checkpoint/X2I_UltraEdiit_fourmask_LLava_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_filtered_objects_4mask_1280_context_after_stage1_5e4/000-GPT-XL/checkpoints/0008000.pt +[2025-04-28 13:02:05] Removed old checkpoint: checkpoint/X2I_UltraEdiit_fourmask_LLava_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_filtered_objects_4mask_1280_context_after_stage1_5e4/000-GPT-XL/checkpoints/0006000.pt + 9%|████████████▏ | 8024/89905 [2:37:26<21:22:05, 1.06it/s][2025-04-28 13:02:29] (step=0008025) Train Loss: 6.3299, Train Steps/Sec: 0.05 + 9%|████████████▏ | 8049/89905 [2:37:48<20:29:47, 1.11it/s][2025-04-28 13:02:52] (step=0008050) Train Loss: 6.3708, Train Steps/Sec: 1.12 + 9%|████████████▏ | 8074/89905 [2:38:11<20:22:27, 1.12it/s][2025-04-28 13:03:14] (step=0008075) Train Loss: 6.2980, Train Steps/Sec: 1.12 + 9%|████████████▎ | 8099/89905 [2:38:33<20:17:39, 1.12it/s][2025-04-28 13:03:36] (step=0008100) Train Loss: 6.2168, Train Steps/Sec: 1.12 + 9%|████████████▎ | 8124/89905 [2:38:55<20:13:15, 1.12it/s][2025-04-28 13:03:59] (step=0008125) Train Loss: 6.2687, Train Steps/Sec: 1.12 + 9%|████████████▎ | 8149/89905 [2:39:18<20:23:38, 1.11it/s][2025-04-28 13:04:21] (step=0008150) Train Loss: 6.2848, Train Steps/Sec: 1.11 + 9%|████████████▎ | 8174/89905 [2:39:40<20:06:24, 1.13it/s][2025-04-28 13:04:43] (step=0008175) Train Loss: 6.3156, Train Steps/Sec: 1.12 + 9%|████████████▍ | 8199/89905 [2:40:02<20:03:28, 1.13it/s][2025-04-28 13:05:06] (step=0008200) Train Loss: 6.2778, Train Steps/Sec: 1.12 + 9%|████████████▍ | 8224/89905 [2:40:25<20:42:24, 1.10it/s][2025-04-28 13:05:28] (step=0008225) Train Loss: 6.2969, Train Steps/Sec: 1.12 + 9%|████████████▍ | 8249/89905 [2:40:47<20:24:26, 1.11it/s][2025-04-28 13:05:51] (step=0008250) Train Loss: 6.2563, Train Steps/Sec: 1.12 + 9%|████████████▌ | 8274/89905 [2:41:09<20:21:01, 1.11it/s][2025-04-28 13:06:13] (step=0008275) Train Loss: 6.2420, Train Steps/Sec: 1.12 + 9%|████████████▌ | 8299/89905 [2:41:32<20:13:47, 1.12it/s][2025-04-28 13:06:35] (step=0008300) Train Loss: 6.2647, Train Steps/Sec: 1.12 + 9%|████████████▌ | 8324/89905 [2:41:54<20:10:50, 1.12it/s][2025-04-28 13:06:58] (step=0008325) Train Loss: 6.2270, Train Steps/Sec: 1.12 + 9%|████████████▋ | 8349/89905 [2:42:17<20:06:04, 1.13it/s][2025-04-28 13:07:20] (step=0008350) Train Loss: 6.2998, Train Steps/Sec: 1.12 + 9%|████████████▋ | 8374/89905 [2:42:39<20:11:21, 1.12it/s][2025-04-28 13:07:42] (step=0008375) Train Loss: 6.2033, Train Steps/Sec: 1.12 + 9%|████████████▋ | 8399/89905 [2:43:01<20:01:32, 1.13it/s][2025-04-28 13:08:05] (step=0008400) Train Loss: 6.2364, Train Steps/Sec: 1.11 + 9%|████████████▋ | 8424/89905 [2:43:24<20:41:48, 1.09it/s][2025-04-28 13:08:27] (step=0008425) Train Loss: 6.2595, Train Steps/Sec: 1.12 + 9%|████████████▊ | 8449/89905 [2:43:46<20:24:08, 1.11it/s][2025-04-28 13:08:50] (step=0008450) Train Loss: 6.2640, Train Steps/Sec: 1.12 + 9%|████████████▊ | 8474/89905 [2:44:08<20:20:09, 1.11it/s][2025-04-28 13:09:12] (step=0008475) Train Loss: 6.2303, Train Steps/Sec: 1.12 + 9%|████████████▊ | 8499/89905 [2:45:11<28:03:39, 1.24s/it][2025-04-28 13:10:15] (step=0008500) Train Loss: 6.2357, Train Steps/Sec: 0.40 + 9%|████████████▉ | 8524/89905 [2:45:52<23:45:27, 1.05s/it][2025-04-28 13:10:56] (step=0008525) Train Loss: 6.2744, Train Steps/Sec: 0.61 + 10%|████████████▉ | 8549/89905 [2:46:24<20:34:23, 1.10it/s][2025-04-28 13:11:27] (step=0008550) Train Loss: 6.3118, Train Steps/Sec: 0.79 + 10%|████████████▉ | 8574/89905 [2:46:46<20:07:05, 1.12it/s][2025-04-28 13:11:50] (step=0008575) Train Loss: 6.2234, Train Steps/Sec: 1.11 + 10%|█████████████ | 8599/89905 [2:47:09<19:55:41, 1.13it/s][2025-04-28 13:12:12] (step=0008600) Train Loss: 6.2050, Train Steps/Sec: 1.11 + 10%|█████████████ | 8624/89905 [2:47:31<20:37:53, 1.09it/s][2025-04-28 13:12:35] (step=0008625) Train Loss: 6.3078, Train Steps/Sec: 1.12 + 10%|█████████████ | 8649/89905 [2:47:54<20:20:26, 1.11it/s][2025-04-28 13:12:57] (step=0008650) Train Loss: 6.2587, Train Steps/Sec: 1.12 + 10%|█████████████ | 8674/89905 [2:48:16<20:11:46, 1.12it/s][2025-04-28 13:13:19] (step=0008675) Train Loss: 6.2271, Train Steps/Sec: 1.12 + 10%|█████████████▏ | 8699/89905 [2:48:38<20:09:00, 1.12it/s][2025-04-28 13:13:42] (step=0008700) Train Loss: 6.2190, Train Steps/Sec: 1.12 + 10%|█████████████▏ | 8724/89905 [2:49:01<19:58:15, 1.13it/s][2025-04-28 13:14:04] (step=0008725) Train Loss: 6.2314, Train Steps/Sec: 1.12 + 10%|█████████████▏ | 8749/89905 [2:49:23<20:03:29, 1.12it/s][2025-04-28 13:14:26] (step=0008750) Train Loss: 6.2384, Train Steps/Sec: 1.12 + 10%|█████████████▎ | 8774/89905 [2:49:45<19:54:18, 1.13it/s][2025-04-28 13:14:49] (step=0008775) Train Loss: 6.2243, Train Steps/Sec: 1.12 + 10%|█████████████▎ | 8799/89905 [2:50:08<19:59:40, 1.13it/s][2025-04-28 13:15:11] (step=0008800) Train Loss: 6.1961, Train Steps/Sec: 1.11 + 10%|█████████████▎ | 8824/89905 [2:50:30<20:33:46, 1.10it/s][2025-04-28 13:15:34] (step=0008825) Train Loss: 6.2154, Train Steps/Sec: 1.12 + 10%|█████████████▍ | 8849/89905 [2:50:52<20:20:50, 1.11it/s][2025-04-28 13:15:56] (step=0008850) Train Loss: 6.1745, Train Steps/Sec: 1.12 + 10%|█████████████▍ | 8874/89905 [2:51:15<20:07:58, 1.12it/s][2025-04-28 13:16:18] (step=0008875) Train Loss: 6.1864, Train Steps/Sec: 1.12 + 10%|█████████████▍ | 8899/89905 [2:51:37<20:05:41, 1.12it/s][2025-04-28 13:16:41] (step=0008900) Train Loss: 6.1780, Train Steps/Sec: 1.12 + 10%|█████████████▍ | 8924/89905 [2:52:00<20:00:58, 1.12it/s][2025-04-28 13:17:03] (step=0008925) Train Loss: 6.1923, Train Steps/Sec: 1.12 + 10%|█████████████▌ | 8949/89905 [2:52:22<19:56:42, 1.13it/s][2025-04-28 13:17:25] (step=0008950) Train Loss: 6.2190, Train Steps/Sec: 1.12 + 10%|█████████████▌ | 8974/89905 [2:52:44<19:58:26, 1.13it/s][2025-04-28 13:17:48] (step=0008975) Train Loss: 6.2236, Train Steps/Sec: 1.12 + 10%|█████████████▌ | 8999/89905 [2:53:07<19:58:10, 1.13it/s][2025-04-28 13:18:10] (step=0009000) Train Loss: 6.1870, Train Steps/Sec: 1.11 + 10%|█████████████▋ | 9024/89905 [2:53:29<20:30:22, 1.10it/s][2025-04-28 13:18:33] (step=0009025) Train Loss: 6.2279, Train Steps/Sec: 1.12 + 10%|█████████████▋ | 9049/89905 [2:53:51<20:11:35, 1.11it/s][2025-04-28 13:18:55] (step=0009050) Train Loss: 6.1987, Train Steps/Sec: 1.12 + 10%|█████████████▋ | 9074/89905 [2:54:14<20:07:15, 1.12it/s][2025-04-28 13:19:17] (step=0009075) Train Loss: 6.2127, Train Steps/Sec: 1.12 + 10%|█████████████▊ | 9099/89905 [2:54:36<20:05:34, 1.12it/s][2025-04-28 13:19:40] (step=0009100) Train Loss: 6.2156, Train Steps/Sec: 1.12 + 10%|█████████████▊ | 9124/89905 [2:54:58<19:58:50, 1.12it/s][2025-04-28 13:20:02] (step=0009125) Train Loss: 6.2214, Train Steps/Sec: 1.12 + 10%|█████████████▊ | 9149/89905 [2:55:21<19:52:58, 1.13it/s][2025-04-28 13:20:24] (step=0009150) Train Loss: 6.1848, Train Steps/Sec: 1.12 + 10%|█████████████▉ | 9174/89905 [2:55:43<19:56:42, 1.12it/s][2025-04-28 13:20:47] (step=0009175) Train Loss: 6.2138, Train Steps/Sec: 1.12 + 10%|█████████████▉ | 9199/89905 [2:56:06<19:52:15, 1.13it/s][2025-04-28 13:21:09] (step=0009200) Train Loss: 6.2082, Train Steps/Sec: 1.11 + 10%|█████████████▉ | 9224/89905 [2:56:28<20:30:48, 1.09it/s][2025-04-28 13:21:31] (step=0009225) Train Loss: 6.1742, Train Steps/Sec: 1.12 + 10%|█████████████▉ | 9249/89905 [2:56:50<20:11:18, 1.11it/s][2025-04-28 13:21:54] (step=0009250) Train Loss: 6.1494, Train Steps/Sec: 1.12 + 10%|██████████████ | 9274/89905 [2:57:13<20:03:24, 1.12it/s][2025-04-28 13:22:16] (step=0009275) Train Loss: 6.1587, Train Steps/Sec: 1.12 + 10%|██████████████ | 9299/89905 [2:57:35<20:00:09, 1.12it/s][2025-04-28 13:22:38] (step=0009300) Train Loss: 6.2256, Train Steps/Sec: 1.12 + 10%|██████████████ | 9324/89905 [2:57:57<19:55:20, 1.12it/s][2025-04-28 13:23:01] (step=0009325) Train Loss: 6.2427, Train Steps/Sec: 1.12 + 10%|██████████████▏ | 9349/89905 [2:58:20<19:55:34, 1.12it/s][2025-04-28 13:23:23] (step=0009350) Train Loss: 6.1845, Train Steps/Sec: 1.12 + 10%|██████████████▏ | 9374/89905 [2:58:42<19:53:26, 1.12it/s][2025-04-28 13:23:45] (step=0009375) Train Loss: 6.2201, Train Steps/Sec: 1.12 + 10%|██████████████▏ | 9399/89905 [2:59:04<19:49:00, 1.13it/s][2025-04-28 13:24:08] (step=0009400) Train Loss: 6.2459, Train Steps/Sec: 1.11 + 10%|██████████████▎ | 9424/89905 [2:59:27<20:24:42, 1.10it/s][2025-04-28 13:24:30] (step=0009425) Train Loss: 6.2020, Train Steps/Sec: 1.12 + 11%|██████████████▎ | 9449/89905 [2:59:49<20:05:48, 1.11it/s][2025-04-28 13:24:53] (step=0009450) Train Loss: 6.1587, Train Steps/Sec: 1.12 + 11%|██████████████▎ | 9474/89905 [3:00:12<20:07:44, 1.11it/s][2025-04-28 13:25:15] (step=0009475) Train Loss: 6.1994, Train Steps/Sec: 1.11 + 11%|██████████████▎ | 9499/89905 [3:00:34<19:57:02, 1.12it/s][2025-04-28 13:25:37] (step=0009500) Train Loss: 6.1808, Train Steps/Sec: 1.12 + 11%|██████████████▍ | 9524/89905 [3:00:56<19:51:36, 1.12it/s][2025-04-28 13:26:00] (step=0009525) Train Loss: 6.2306, Train Steps/Sec: 1.12 + 11%|██████████████▍ | 9549/89905 [3:01:19<19:50:00, 1.13it/s][2025-04-28 13:26:22] (step=0009550) Train Loss: 6.2154, Train Steps/Sec: 1.12 + 11%|██████████████▍ | 9574/89905 [3:01:41<19:45:50, 1.13it/s][2025-04-28 13:26:44] (step=0009575) Train Loss: 6.1647, Train Steps/Sec: 1.12 + 11%|██████████████▌ | 9599/89905 [3:02:03<19:45:07, 1.13it/s][2025-04-28 13:27:07] (step=0009600) Train Loss: 6.1963, Train Steps/Sec: 1.11 + 11%|██████████████▌ | 9624/89905 [3:02:26<20:22:30, 1.09it/s][2025-04-28 13:27:29] (step=0009625) Train Loss: 6.1774, Train Steps/Sec: 1.12 + 11%|██████████████▌ | 9649/89905 [3:02:48<20:04:01, 1.11it/s][2025-04-28 13:27:52] (step=0009650) Train Loss: 6.2172, Train Steps/Sec: 1.12 + 11%|██████████████▋ | 9674/89905 [3:03:10<20:00:16, 1.11it/s][2025-04-28 13:28:14] (step=0009675) Train Loss: 6.2165, Train Steps/Sec: 1.12 + 11%|██████████████▋ | 9699/89905 [3:03:33<19:54:10, 1.12it/s][2025-04-28 13:28:36] (step=0009700) Train Loss: 6.1574, Train Steps/Sec: 1.12 + 11%|██████████████▋ | 9724/89905 [3:03:55<19:47:11, 1.13it/s][2025-04-28 13:28:59] (step=0009725) Train Loss: 6.1364, Train Steps/Sec: 1.12 + 11%|██████████████▋ | 9749/89905 [3:04:18<19:46:33, 1.13it/s][2025-04-28 13:29:21] (step=0009750) Train Loss: 6.1662, Train Steps/Sec: 1.12 + 11%|██████████████▊ | 9774/89905 [3:04:40<19:43:05, 1.13it/s][2025-04-28 13:29:43] (step=0009775) Train Loss: 6.0988, Train Steps/Sec: 1.12 + 11%|██████████████▊ | 9799/89905 [3:05:02<19:38:48, 1.13it/s][2025-04-28 13:30:06] (step=0009800) Train Loss: 6.1786, Train Steps/Sec: 1.11 + 11%|██████████████▊ | 9824/89905 [3:05:25<20:13:09, 1.10it/s][2025-04-28 13:30:28] (step=0009825) Train Loss: 6.2315, Train Steps/Sec: 1.12 + 11%|██████████████▉ | 9849/89905 [3:05:47<20:05:36, 1.11it/s][2025-04-28 13:30:50] (step=0009850) Train Loss: 6.1760, Train Steps/Sec: 1.12 + 11%|██████████████▉ | 9874/89905 [3:06:09<20:00:35, 1.11it/s][2025-04-28 13:31:13] (step=0009875) Train Loss: 6.2055, Train Steps/Sec: 1.12 + 11%|██████████████▉ | 9899/89905 [3:06:32<19:47:11, 1.12it/s][2025-04-28 13:31:35] (step=0009900) Train Loss: 6.1653, Train Steps/Sec: 1.12 + 11%|███████████████ | 9924/89905 [3:06:54<19:50:14, 1.12it/s][2025-04-28 13:31:57] (step=0009925) Train Loss: 6.1622, Train Steps/Sec: 1.12 + 11%|███████████████ | 9949/89905 [3:07:16<19:43:46, 1.13it/s][2025-04-28 13:32:20] (step=0009950) Train Loss: 6.2154, Train Steps/Sec: 1.12 + 11%|███████████████ | 9974/89905 [3:07:39<19:41:12, 1.13it/s][2025-04-28 13:32:42] (step=0009975) Train Loss: 6.1415, Train Steps/Sec: 1.12 + 11%|███████████████▏ | 9999/89905 [3:08:01<19:42:06, 1.13it/s][2025-04-28 13:33:05] (step=0010000) Train Loss: 6.1778, Train Steps/Sec: 1.11 +You are using a model of type blip-2 to instantiate a model of type instructblip. This is not supported for all configurations of models and can yield errors. +[2025-04-28 13:33:05] text_config is None. Initializing the text config with default values (`OPTConfig`). +model_name_or_path:/tmp/haozhezhao/model/blip2-flan-t5-xl +image_place_holder: +tokenizer length after expend 32101 +tokenizer length before expend 32100 +100%|███████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████| 6/6 [06:00<00:00, 60.03s/it] +[2025-04-28 13:40:23] Finish Eval in 10000 steps...████████████████████████████████████████████████████████████████████| 6/6 [05:59<00:00, 59.88s/it] +processing left-padding... +processing left-padding... +processing left-padding... +processing left-padding... +processing left-padding... +processing left-padding... +[2025-04-28 13:40:43] Saved checkpoint to checkpoint/X2I_UltraEdiit_fourmask_LLava_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_filtered_objects_4mask_1280_context_after_stage1_5e4/000-GPT-XL/checkpoints/0010000.pt +[2025-04-28 13:40:45] Removed old checkpoint: checkpoint/X2I_UltraEdiit_fourmask_LLava_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_filtered_objects_4mask_1280_context_after_stage1_5e4/000-GPT-XL/checkpoints/0008000.pt + 11%|███████████████ | 10024/89905 [3:16:04<20:49:29, 1.07it/s][2025-04-28 13:41:08] (step=0010025) Train Loss: 6.2247, Train Steps/Sec: 0.05 + 11%|███████████████ | 10049/89905 [3:16:27<20:02:18, 1.11it/s][2025-04-28 13:41:30] (step=0010050) Train Loss: 6.1984, Train Steps/Sec: 1.11 + 11%|███████████████▏ | 10074/89905 [3:16:49<19:46:59, 1.12it/s][2025-04-28 13:41:53] (step=0010075) Train Loss: 6.1763, Train Steps/Sec: 1.12 + 11%|███████████████▏ | 10099/89905 [3:17:12<19:48:09, 1.12it/s][2025-04-28 13:42:15] (step=0010100) Train Loss: 6.1886, Train Steps/Sec: 1.12 + 11%|███████████████▏ | 10124/89905 [3:17:34<19:39:58, 1.13it/s][2025-04-28 13:42:37] (step=0010125) Train Loss: 6.1658, Train Steps/Sec: 1.12 + 11%|███████████████▏ | 10149/89905 [3:17:56<19:42:31, 1.12it/s][2025-04-28 13:43:00] (step=0010150) Train Loss: 6.1557, Train Steps/Sec: 1.12 + 11%|███████████████▎ | 10174/89905 [3:18:19<19:36:05, 1.13it/s][2025-04-28 13:43:22] (step=0010175) Train Loss: 6.1529, Train Steps/Sec: 1.12 + 11%|███████████████▎ | 10199/89905 [3:18:41<19:36:28, 1.13it/s][2025-04-28 13:43:44] (step=0010200) Train Loss: 6.1729, Train Steps/Sec: 1.11 + 11%|███████████████▎ | 10224/89905 [3:19:03<20:11:02, 1.10it/s][2025-04-28 13:44:07] (step=0010225) Train Loss: 6.1500, Train Steps/Sec: 1.12 + 11%|███████████████▍ | 10249/89905 [3:19:26<19:56:08, 1.11it/s][2025-04-28 13:44:29] (step=0010250) Train Loss: 6.1658, Train Steps/Sec: 1.12 + 11%|███████████████▍ | 10274/89905 [3:19:48<19:55:15, 1.11it/s][2025-04-28 13:44:51] (step=0010275) Train Loss: 6.1547, Train Steps/Sec: 1.12 + 11%|███████████████▍ | 10299/89905 [3:20:10<19:43:04, 1.12it/s][2025-04-28 13:45:14] (step=0010300) Train Loss: 6.1202, Train Steps/Sec: 1.12 + 11%|███████████████▌ | 10324/89905 [3:20:33<19:40:25, 1.12it/s][2025-04-28 13:45:36] (step=0010325) Train Loss: 6.1266, Train Steps/Sec: 1.12 + 12%|███████████████▌ | 10349/89905 [3:20:55<19:36:24, 1.13it/s][2025-04-28 13:45:58] (step=0010350) Train Loss: 6.1741, Train Steps/Sec: 1.12 + 12%|███████████████▌ | 10374/89905 [3:21:17<19:34:13, 1.13it/s][2025-04-28 13:46:21] (step=0010375) Train Loss: 6.2101, Train Steps/Sec: 1.12 + 12%|███████████████▌ | 10399/89905 [3:21:40<19:34:47, 1.13it/s][2025-04-28 13:46:43] (step=0010400) Train Loss: 6.2194, Train Steps/Sec: 1.11 + 12%|███████████████▋ | 10424/89905 [3:22:02<20:06:41, 1.10it/s][2025-04-28 13:47:06] (step=0010425) Train Loss: 6.2016, Train Steps/Sec: 1.12 + 12%|███████████████▋ | 10449/89905 [3:22:25<19:56:27, 1.11it/s][2025-04-28 13:47:28] (step=0010450) Train Loss: 6.1883, Train Steps/Sec: 1.12 + 12%|███████████████▋ | 10474/89905 [3:22:47<19:44:27, 1.12it/s][2025-04-28 13:47:50] (step=0010475) Train Loss: 6.1930, Train Steps/Sec: 1.12 + 12%|███████████████▊ | 10499/89905 [3:23:09<19:41:14, 1.12it/s][2025-04-28 13:48:13] (step=0010500) Train Loss: 6.1420, Train Steps/Sec: 1.12 + 12%|███████████████▊ | 10524/89905 [3:23:32<19:39:10, 1.12it/s][2025-04-28 13:48:35] (step=0010525) Train Loss: 6.1227, Train Steps/Sec: 1.12 + 12%|███████████████▊ | 10549/89905 [3:23:54<19:37:47, 1.12it/s][2025-04-28 13:48:57] (step=0010550) Train Loss: 6.1811, Train Steps/Sec: 1.12 + 12%|███████████████▉ | 10574/89905 [3:24:16<19:38:16, 1.12it/s][2025-04-28 13:49:20] (step=0010575) Train Loss: 6.1873, Train Steps/Sec: 1.12 + 12%|███████████████▉ | 10599/89905 [3:24:39<19:30:02, 1.13it/s][2025-04-28 13:49:42] (step=0010600) Train Loss: 6.1637, Train Steps/Sec: 1.11 + 12%|███████████████▉ | 10624/89905 [3:25:01<20:12:24, 1.09it/s][2025-04-28 13:50:05] (step=0010625) Train Loss: 6.1720, Train Steps/Sec: 1.12 + 12%|███████████████▉ | 10649/89905 [3:25:23<19:50:06, 1.11it/s][2025-04-28 13:50:27] (step=0010650) Train Loss: 6.1231, Train Steps/Sec: 1.12 + 12%|████████████████ | 10674/89905 [3:25:46<19:42:57, 1.12it/s][2025-04-28 13:50:49] (step=0010675) Train Loss: 6.1595, Train Steps/Sec: 1.12 + 12%|████████████████ | 10699/89905 [3:26:08<19:43:43, 1.12it/s][2025-04-28 13:51:12] (step=0010700) Train Loss: 6.1113, Train Steps/Sec: 1.12 + 12%|████████████████ | 10724/89905 [3:26:31<19:34:48, 1.12it/s][2025-04-28 13:51:34] (step=0010725) Train Loss: 6.1167, Train Steps/Sec: 1.12 + 12%|████████████████▏ | 10749/89905 [3:26:53<19:33:34, 1.12it/s][2025-04-28 13:51:56] (step=0010750) Train Loss: 6.1147, Train Steps/Sec: 1.12 + 12%|████████████████▏ | 10774/89905 [3:27:15<19:32:57, 1.12it/s][2025-04-28 13:52:19] (step=0010775) Train Loss: 6.1617, Train Steps/Sec: 1.12 + 12%|████████████████▏ | 10799/89905 [3:27:38<19:22:56, 1.13it/s][2025-04-28 13:52:41] (step=0010800) Train Loss: 6.1974, Train Steps/Sec: 1.12 + 12%|████████████████▎ | 10824/89905 [3:28:00<20:03:44, 1.09it/s][2025-04-28 13:53:03] (step=0010825) Train Loss: 6.1710, Train Steps/Sec: 1.12 + 12%|████████████████▎ | 10849/89905 [3:28:22<19:47:01, 1.11it/s][2025-04-28 13:53:26] (step=0010850) Train Loss: 6.1261, Train Steps/Sec: 1.12 + 12%|████████████████▎ | 10874/89905 [3:28:45<19:40:43, 1.12it/s][2025-04-28 13:53:48] (step=0010875) Train Loss: 6.1458, Train Steps/Sec: 1.12 + 12%|████████████████▎ | 10899/89905 [3:29:07<19:36:02, 1.12it/s][2025-04-28 13:54:11] (step=0010900) Train Loss: 6.1283, Train Steps/Sec: 1.12 + 12%|████████████████▍ | 10924/89905 [3:29:30<19:30:32, 1.12it/s][2025-04-28 13:54:33] (step=0010925) Train Loss: 6.0329, Train Steps/Sec: 1.12 + 12%|████████████████▍ | 10949/89905 [3:29:52<19:26:37, 1.13it/s][2025-04-28 13:54:55] (step=0010950) Train Loss: 6.1676, Train Steps/Sec: 1.12 + 12%|████████████████▍ | 10974/89905 [3:30:14<19:25:30, 1.13it/s][2025-04-28 13:55:18] (step=0010975) Train Loss: 6.1697, Train Steps/Sec: 1.12 + 12%|████████████████▌ | 10999/89905 [3:30:36<19:23:25, 1.13it/s][2025-04-28 13:55:40] (step=0011000) Train Loss: 6.0695, Train Steps/Sec: 1.12 + 12%|████████████████▌ | 11024/89905 [3:30:59<19:55:41, 1.10it/s][2025-04-28 13:56:02] (step=0011025) Train Loss: 6.1667, Train Steps/Sec: 1.12 + 12%|████████████████▌ | 11049/89905 [3:31:21<19:52:25, 1.10it/s][2025-04-28 13:56:25] (step=0011050) Train Loss: 6.2007, Train Steps/Sec: 1.12 + 12%|████████████████▋ | 11074/89905 [3:31:44<19:35:29, 1.12it/s][2025-04-28 13:56:47] (step=0011075) Train Loss: 6.1430, Train Steps/Sec: 1.12 + 12%|████████████████▋ | 11099/89905 [3:32:06<19:36:10, 1.12it/s][2025-04-28 13:57:09] (step=0011100) Train Loss: 6.1191, Train Steps/Sec: 1.12 + 12%|████████████████▋ | 11124/89905 [3:32:28<19:27:49, 1.12it/s][2025-04-28 13:57:32] (step=0011125) Train Loss: 6.1769, Train Steps/Sec: 1.12 + 12%|████████████████▋ | 11149/89905 [3:32:51<19:26:08, 1.13it/s][2025-04-28 13:57:54] (step=0011150) Train Loss: 6.1164, Train Steps/Sec: 1.12 + 12%|████████████████▊ | 11174/89905 [3:33:13<19:26:28, 1.12it/s][2025-04-28 13:58:16] (step=0011175) Train Loss: 6.1840, Train Steps/Sec: 1.12 + 12%|████████████████▊ | 11199/89905 [3:33:35<19:20:26, 1.13it/s][2025-04-28 13:58:39] (step=0011200) Train Loss: 6.0763, Train Steps/Sec: 1.11 + 12%|████████████████▊ | 11224/89905 [3:33:58<19:54:09, 1.10it/s][2025-04-28 13:59:01] (step=0011225) Train Loss: 6.1249, Train Steps/Sec: 1.12 + 13%|████████████████▉ | 11249/89905 [3:34:20<19:45:41, 1.11it/s][2025-04-28 13:59:23] (step=0011250) Train Loss: 6.1757, Train Steps/Sec: 1.12 + 13%|████████████████▉ | 11274/89905 [3:34:42<19:38:49, 1.11it/s][2025-04-28 13:59:46] (step=0011275) Train Loss: 6.0938, Train Steps/Sec: 1.12 + 13%|████████████████▉ | 11299/89905 [3:35:05<19:29:37, 1.12it/s][2025-04-28 14:00:08] (step=0011300) Train Loss: 6.1085, Train Steps/Sec: 1.12 + 13%|█████████████████ | 11324/89905 [3:35:27<19:29:21, 1.12it/s][2025-04-28 14:00:30] (step=0011325) Train Loss: 6.1990, Train Steps/Sec: 1.12 + 13%|█████████████████ | 11349/89905 [3:35:49<19:22:21, 1.13it/s][2025-04-28 14:00:53] (step=0011350) Train Loss: 6.1306, Train Steps/Sec: 1.12 + 13%|█████████████████ | 11374/89905 [3:36:12<19:24:36, 1.12it/s][2025-04-28 14:01:15] (step=0011375) Train Loss: 6.1198, Train Steps/Sec: 1.12 + 13%|█████████████████ | 11399/89905 [3:36:34<19:13:50, 1.13it/s][2025-04-28 14:01:38] (step=0011400) Train Loss: 6.0427, Train Steps/Sec: 1.12 + 13%|█████████████████▏ | 11424/89905 [3:36:57<19:52:04, 1.10it/s][2025-04-28 14:02:00] (step=0011425) Train Loss: 6.0730, Train Steps/Sec: 1.12 + 13%|█████████████████▏ | 11449/89905 [3:37:19<19:39:14, 1.11it/s][2025-04-28 14:02:22] (step=0011450) Train Loss: 6.0705, Train Steps/Sec: 1.12 + 13%|█████████████████▏ | 11474/89905 [3:37:41<19:34:18, 1.11it/s][2025-04-28 14:02:45] (step=0011475) Train Loss: 6.1082, Train Steps/Sec: 1.12 + 13%|█████████████████▎ | 11499/89905 [3:38:04<19:31:01, 1.12it/s][2025-04-28 14:03:07] (step=0011500) Train Loss: 6.0998, Train Steps/Sec: 1.12 + 13%|█████████████████▎ | 11524/89905 [3:38:26<19:19:23, 1.13it/s][2025-04-28 14:03:29] (step=0011525) Train Loss: 6.0717, Train Steps/Sec: 1.12 + 13%|█████████████████▎ | 11549/89905 [3:38:48<19:21:25, 1.12it/s][2025-04-28 14:03:52] (step=0011550) Train Loss: 6.0723, Train Steps/Sec: 1.12 + 13%|█████████████████▍ | 11574/89905 [3:39:11<19:15:46, 1.13it/s][2025-04-28 14:04:14] (step=0011575) Train Loss: 6.0871, Train Steps/Sec: 1.12 + 13%|█████████████████▍ | 11599/89905 [3:39:33<19:16:03, 1.13it/s][2025-04-28 14:04:36] (step=0011600) Train Loss: 6.0766, Train Steps/Sec: 1.12 + 13%|█████████████████▍ | 11624/89905 [3:39:55<19:49:26, 1.10it/s][2025-04-28 14:04:59] (step=0011625) Train Loss: 6.0883, Train Steps/Sec: 1.12 + 13%|█████████████████▍ | 11649/89905 [3:40:18<19:35:00, 1.11it/s][2025-04-28 14:05:21] (step=0011650) Train Loss: 6.0654, Train Steps/Sec: 1.12 + 13%|█████████████████▌ | 11674/89905 [3:40:40<19:25:41, 1.12it/s][2025-04-28 14:05:44] (step=0011675) Train Loss: 6.0538, Train Steps/Sec: 1.12 + 13%|█████████████████▌ | 11699/89905 [3:41:02<19:22:03, 1.12it/s][2025-04-28 14:06:06] (step=0011700) Train Loss: 6.1222, Train Steps/Sec: 1.12 + 13%|█████████████████▌ | 11724/89905 [3:41:25<19:16:02, 1.13it/s][2025-04-28 14:06:28] (step=0011725) Train Loss: 6.0667, Train Steps/Sec: 1.12 + 13%|█████████████████▋ | 11749/89905 [3:41:47<19:18:30, 1.12it/s][2025-04-28 14:06:51] (step=0011750) Train Loss: 6.1275, Train Steps/Sec: 1.12 + 13%|█████████████████▋ | 11774/89905 [3:42:09<19:13:13, 1.13it/s][2025-04-28 14:07:13] (step=0011775) Train Loss: 6.1009, Train Steps/Sec: 1.12 + 13%|█████████████████▋ | 11799/89905 [3:42:32<19:15:16, 1.13it/s][2025-04-28 14:07:35] (step=0011800) Train Loss: 6.1053, Train Steps/Sec: 1.11 + 13%|█████████████████▊ | 11824/89905 [3:42:54<19:49:15, 1.09it/s][2025-04-28 14:07:58] (step=0011825) Train Loss: 6.1412, Train Steps/Sec: 1.12 + 13%|█████████████████▊ | 11849/89905 [3:43:17<19:33:55, 1.11it/s][2025-04-28 14:08:20] (step=0011850) Train Loss: 6.0658, Train Steps/Sec: 1.12 + 13%|█████████████████▊ | 11874/89905 [3:43:39<19:26:59, 1.11it/s][2025-04-28 14:08:42] (step=0011875) Train Loss: 6.0757, Train Steps/Sec: 1.12 + 13%|█████████████████▊ | 11899/89905 [3:44:01<19:20:40, 1.12it/s][2025-04-28 14:09:05] (step=0011900) Train Loss: 6.1077, Train Steps/Sec: 1.12 + 13%|█████████████████▉ | 11924/89905 [3:44:24<19:20:22, 1.12it/s][2025-04-28 14:09:27] (step=0011925) Train Loss: 6.0672, Train Steps/Sec: 1.12 + 13%|█████████████████▉ | 11949/89905 [3:44:46<19:11:46, 1.13it/s][2025-04-28 14:09:49] (step=0011950) Train Loss: 6.1049, Train Steps/Sec: 1.12 + 13%|█████████████████▉ | 11974/89905 [3:45:08<19:11:12, 1.13it/s][2025-04-28 14:10:12] (step=0011975) Train Loss: 6.0695, Train Steps/Sec: 1.12 + 13%|██████████████████ | 11999/89905 [3:45:31<19:07:52, 1.13it/s][2025-04-28 14:10:34] (step=0012000) Train Loss: 6.0708, Train Steps/Sec: 1.11 +You are using a model of type blip-2 to instantiate a model of type instructblip. This is not supported for all configurations of models and can yield errors. +[2025-04-28 14:10:34] text_config is None. Initializing the text config with default values (`OPTConfig`). +model_name_or_path:/tmp/haozhezhao/model/blip2-flan-t5-xl +image_place_holder: +tokenizer length after expend 32101 +tokenizer length before expend 32100 +100%|███████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████| 6/6 [06:00<00:00, 60.04s/it] +[2025-04-28 14:17:53] Finish Eval in 12000 steps...████████████████████████████████████████████████████████████████████| 6/6 [05:59<00:00, 59.84s/it] +processing left-padding... +processing left-padding... +processing left-padding... +processing left-padding... +processing left-padding... +processing left-padding... +[2025-04-28 14:18:13] Saved checkpoint to checkpoint/X2I_UltraEdiit_fourmask_LLava_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_filtered_objects_4mask_1280_context_after_stage1_5e4/000-GPT-XL/checkpoints/0012000.pt +[2025-04-28 14:18:14] Removed old checkpoint: checkpoint/X2I_UltraEdiit_fourmask_LLava_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_filtered_objects_4mask_1280_context_after_stage1_5e4/000-GPT-XL/checkpoints/0010000.pt + 13%|██████████████████ | 12024/89905 [3:53:34<20:21:01, 1.06it/s][2025-04-28 14:18:37] (step=0012025) Train Loss: 6.1536, Train Steps/Sec: 0.05 + 13%|██████████████████ | 12049/89905 [3:54:14<55:34:04, 2.57s/it][2025-04-28 14:19:27] (step=0012050) Train Loss: 6.0633, Train Steps/Sec: 0.50 + 13%|██████████████████▏ | 12074/89905 [3:55:06<19:44:33, 1.10it/s][2025-04-28 14:20:09] (step=0012075) Train Loss: 6.1331, Train Steps/Sec: 0.60 + 13%|██████████████████▏ | 12099/89905 [3:55:37<19:27:12, 1.11it/s][2025-04-28 14:20:41] (step=0012100) Train Loss: 6.0836, Train Steps/Sec: 0.79 + 13%|██████████████████▏ | 12124/89905 [3:56:09<19:18:35, 1.12it/s][2025-04-28 14:21:12] (step=0012125) Train Loss: 6.1123, Train Steps/Sec: 0.79 + 14%|██████████████████▏ | 12149/89905 [3:56:31<19:12:52, 1.12it/s][2025-04-28 14:21:35] (step=0012150) Train Loss: 6.0963, Train Steps/Sec: 1.12 + 14%|██████████████████▎ | 12174/89905 [3:56:54<19:06:50, 1.13it/s][2025-04-28 14:21:57] (step=0012175) Train Loss: 6.0990, Train Steps/Sec: 1.12 + 14%|██████████████████▎ | 12199/89905 [3:57:16<19:12:20, 1.12it/s][2025-04-28 14:22:19] (step=0012200) Train Loss: 6.0688, Train Steps/Sec: 1.12 + 14%|██████████████████▎ | 12224/89905 [3:57:38<19:42:17, 1.10it/s][2025-04-28 14:22:42] (step=0012225) Train Loss: 6.0953, Train Steps/Sec: 1.12 + 14%|██████████████████▍ | 12249/89905 [3:58:01<19:27:29, 1.11it/s][2025-04-28 14:23:04] (step=0012250) Train Loss: 6.0852, Train Steps/Sec: 1.12 + 14%|██████████████████▍ | 12274/89905 [3:58:23<19:18:54, 1.12it/s][2025-04-28 14:23:26] (step=0012275) Train Loss: 6.1081, Train Steps/Sec: 1.12 + 14%|██████████████████▍ | 12299/89905 [3:58:45<19:13:41, 1.12it/s][2025-04-28 14:23:49] (step=0012300) Train Loss: 6.0392, Train Steps/Sec: 1.12 + 14%|██████████████████▌ | 12324/89905 [3:59:08<19:11:15, 1.12it/s][2025-04-28 14:24:11] (step=0012325) Train Loss: 6.1210, Train Steps/Sec: 1.12 + 14%|██████████████████▌ | 12349/89905 [3:59:30<19:10:07, 1.12it/s][2025-04-28 14:24:33] (step=0012350) Train Loss: 6.0219, Train Steps/Sec: 1.12 + 14%|██████████████████▌ | 12374/89905 [3:59:52<19:06:10, 1.13it/s][2025-04-28 14:24:56] (step=0012375) Train Loss: 6.0390, Train Steps/Sec: 1.12 + 14%|██████████████████▌ | 12399/89905 [4:00:15<19:02:45, 1.13it/s][2025-04-28 14:25:18] (step=0012400) Train Loss: 6.0404, Train Steps/Sec: 1.11 + 14%|██████████████████▋ | 12424/89905 [4:00:37<19:38:58, 1.10it/s][2025-04-28 14:25:41] (step=0012425) Train Loss: 6.0278, Train Steps/Sec: 1.12 + 14%|██████████████████▋ | 12449/89905 [4:01:00<19:27:15, 1.11it/s][2025-04-28 14:26:03] (step=0012450) Train Loss: 6.1133, Train Steps/Sec: 1.12 + 14%|██████████████████▋ | 12474/89905 [4:01:22<19:15:04, 1.12it/s][2025-04-28 14:26:25] (step=0012475) Train Loss: 6.1029, Train Steps/Sec: 1.12 + 14%|██████████████████▊ | 12499/89905 [4:01:44<19:10:16, 1.12it/s][2025-04-28 14:26:48] (step=0012500) Train Loss: 6.1089, Train Steps/Sec: 1.12 + 14%|██████████████████▊ | 12524/89905 [4:02:07<19:08:57, 1.12it/s][2025-04-28 14:27:10] (step=0012525) Train Loss: 6.1052, Train Steps/Sec: 1.12 + 14%|██████████████████▊ | 12549/89905 [4:02:29<19:03:43, 1.13it/s][2025-04-28 14:27:32] (step=0012550) Train Loss: 6.1075, Train Steps/Sec: 1.12 + 14%|██████████████████▉ | 12574/89905 [4:02:51<19:07:26, 1.12it/s][2025-04-28 14:27:55] (step=0012575) Train Loss: 6.0938, Train Steps/Sec: 1.12 + 14%|██████████████████▉ | 12599/89905 [4:03:14<19:02:08, 1.13it/s][2025-04-28 14:28:17] (step=0012600) Train Loss: 6.0822, Train Steps/Sec: 1.11 + 14%|██████████████████▉ | 12624/89905 [4:03:36<19:36:18, 1.09it/s][2025-04-28 14:28:40] (step=0012625) Train Loss: 6.1434, Train Steps/Sec: 1.12 + 14%|██████████████████▉ | 12649/89905 [4:03:58<19:18:41, 1.11it/s][2025-04-28 14:29:02] (step=0012650) Train Loss: 6.0677, Train Steps/Sec: 1.12 + 14%|███████████████████ | 12674/89905 [4:04:21<19:17:13, 1.11it/s][2025-04-28 14:29:24] (step=0012675) Train Loss: 6.1159, Train Steps/Sec: 1.12 + 14%|███████████████████ | 12699/89905 [4:04:43<19:06:30, 1.12it/s][2025-04-28 14:29:47] (step=0012700) Train Loss: 5.9995, Train Steps/Sec: 1.12 + 14%|███████████████████ | 12724/89905 [4:05:06<19:06:30, 1.12it/s][2025-04-28 14:30:09] (step=0012725) Train Loss: 6.0708, Train Steps/Sec: 1.12 + 14%|███████████████████▏ | 12749/89905 [4:05:28<18:57:06, 1.13it/s][2025-04-28 14:30:31] (step=0012750) Train Loss: 6.0237, Train Steps/Sec: 1.12 + 14%|███████████████████▏ | 12774/89905 [4:05:50<18:59:06, 1.13it/s][2025-04-28 14:30:54] (step=0012775) Train Loss: 6.0792, Train Steps/Sec: 1.12 + 14%|███████████████████▏ | 12799/89905 [4:06:13<18:59:39, 1.13it/s][2025-04-28 14:31:16] (step=0012800) Train Loss: 6.0647, Train Steps/Sec: 1.11 + 14%|███████████████████▎ | 12824/89905 [4:06:35<19:33:30, 1.09it/s][2025-04-28 14:31:38] (step=0012825) Train Loss: 6.0566, Train Steps/Sec: 1.12 + 14%|███████████████████▎ | 12849/89905 [4:06:57<19:18:27, 1.11it/s][2025-04-28 14:32:01] (step=0012850) Train Loss: 6.0930, Train Steps/Sec: 1.12 + 14%|███████████████████▎ | 12874/89905 [4:07:20<19:11:42, 1.11it/s][2025-04-28 14:32:23] (step=0012875) Train Loss: 6.0327, Train Steps/Sec: 1.12 + 14%|███████████████████▎ | 12899/89905 [4:07:42<19:04:37, 1.12it/s][2025-04-28 14:32:45] (step=0012900) Train Loss: 6.0725, Train Steps/Sec: 1.12 + 14%|███████████████████▍ | 12924/89905 [4:08:04<19:01:13, 1.12it/s][2025-04-28 14:33:08] (step=0012925) Train Loss: 6.0380, Train Steps/Sec: 1.12 + 14%|███████████████████▍ | 12949/89905 [4:08:27<18:54:55, 1.13it/s][2025-04-28 14:33:30] (step=0012950) Train Loss: 6.0675, Train Steps/Sec: 1.12 + 14%|███████████████████▍ | 12974/89905 [4:08:49<18:54:20, 1.13it/s][2025-04-28 14:33:52] (step=0012975) Train Loss: 6.0337, Train Steps/Sec: 1.12 + 14%|███████████████████▌ | 12999/89905 [4:09:11<18:51:21, 1.13it/s][2025-04-28 14:34:15] (step=0013000) Train Loss: 6.0589, Train Steps/Sec: 1.11 + 14%|███████████████████▌ | 13024/89905 [4:09:34<19:34:10, 1.09it/s][2025-04-28 14:34:37] (step=0013025) Train Loss: 6.0369, Train Steps/Sec: 1.12 + 15%|███████████████████▌ | 13049/89905 [4:09:56<19:15:02, 1.11it/s][2025-04-28 14:35:00] (step=0013050) Train Loss: 5.9883, Train Steps/Sec: 1.12 + 15%|███████████████████▋ | 13074/89905 [4:10:18<19:08:18, 1.12it/s][2025-04-28 14:35:22] (step=0013075) Train Loss: 6.1005, Train Steps/Sec: 1.12 + 15%|███████████████████▋ | 13099/89905 [4:10:41<19:05:11, 1.12it/s][2025-04-28 14:35:44] (step=0013100) Train Loss: 5.9906, Train Steps/Sec: 1.12 + 15%|███████████████████▋ | 13124/89905 [4:11:03<18:57:31, 1.12it/s][2025-04-28 14:36:07] (step=0013125) Train Loss: 6.0224, Train Steps/Sec: 1.12 + 15%|███████████████████▋ | 13149/89905 [4:11:26<18:56:36, 1.13it/s][2025-04-28 14:36:29] (step=0013150) Train Loss: 6.0482, Train Steps/Sec: 1.12 + 15%|███████████████████▊ | 13174/89905 [4:11:48<18:50:07, 1.13it/s][2025-04-28 14:36:51] (step=0013175) Train Loss: 6.0040, Train Steps/Sec: 1.12 + 15%|███████████████████▊ | 13199/89905 [4:12:10<18:53:36, 1.13it/s][2025-04-28 14:37:14] (step=0013200) Train Loss: 6.0621, Train Steps/Sec: 1.12 + 15%|███████████████████▊ | 13224/89905 [4:12:33<19:27:42, 1.09it/s][2025-04-28 14:37:36] (step=0013225) Train Loss: 6.0146, Train Steps/Sec: 1.12 + 15%|███████████████████▉ | 13249/89905 [4:12:55<19:14:38, 1.11it/s][2025-04-28 14:37:58] (step=0013250) Train Loss: 6.0281, Train Steps/Sec: 1.12 + 15%|███████████████████▉ | 13274/89905 [4:13:17<19:04:35, 1.12it/s][2025-04-28 14:38:21] (step=0013275) Train Loss: 6.0044, Train Steps/Sec: 1.12 + 15%|███████████████████▉ | 13299/89905 [4:13:40<19:01:31, 1.12it/s][2025-04-28 14:38:43] (step=0013300) Train Loss: 6.0963, Train Steps/Sec: 1.12 + 15%|████████████████████ | 13324/89905 [4:14:02<18:54:22, 1.13it/s][2025-04-28 14:39:05] (step=0013325) Train Loss: 6.0286, Train Steps/Sec: 1.12 + 15%|████████████████████ | 13349/89905 [4:14:24<18:53:45, 1.13it/s][2025-04-28 14:39:28] (step=0013350) Train Loss: 6.0859, Train Steps/Sec: 1.12 + 15%|████████████████████ | 13374/89905 [4:14:47<18:47:42, 1.13it/s][2025-04-28 14:39:50] (step=0013375) Train Loss: 6.0255, Train Steps/Sec: 1.12 + 15%|████████████████████ | 13399/89905 [4:15:09<18:47:32, 1.13it/s][2025-04-28 14:40:12] (step=0013400) Train Loss: 6.0236, Train Steps/Sec: 1.12 + 15%|████████████████████▏ | 13424/89905 [4:15:31<19:21:35, 1.10it/s][2025-04-28 14:40:35] (step=0013425) Train Loss: 6.0473, Train Steps/Sec: 1.12 + 15%|████████████████████▏ | 13449/89905 [4:15:54<19:06:53, 1.11it/s][2025-04-28 14:40:57] (step=0013450) Train Loss: 6.0152, Train Steps/Sec: 1.12 + 15%|████████████████████▏ | 13474/89905 [4:16:16<18:59:32, 1.12it/s][2025-04-28 14:41:19] (step=0013475) Train Loss: 6.0877, Train Steps/Sec: 1.12 + 15%|████████████████████▎ | 13499/89905 [4:16:38<18:58:42, 1.12it/s][2025-04-28 14:41:42] (step=0013500) Train Loss: 6.0035, Train Steps/Sec: 1.12 + 15%|████████████████████▎ | 13524/89905 [4:17:01<18:56:37, 1.12it/s][2025-04-28 14:42:04] (step=0013525) Train Loss: 6.0206, Train Steps/Sec: 1.12 + 15%|████████████████████▎ | 13549/89905 [4:17:23<18:50:15, 1.13it/s][2025-04-28 14:42:26] (step=0013550) Train Loss: 6.0226, Train Steps/Sec: 1.12 + 15%|████████████████████▍ | 13574/89905 [4:17:46<19:01:52, 1.11it/s][2025-04-28 14:42:49] (step=0013575) Train Loss: 5.9984, Train Steps/Sec: 1.11 + 15%|████████████████████▍ | 13599/89905 [4:18:08<18:48:44, 1.13it/s][2025-04-28 14:43:12] (step=0013600) Train Loss: 5.9311, Train Steps/Sec: 1.11 + 15%|████████████████████▍ | 13624/89905 [4:18:30<19:21:19, 1.09it/s][2025-04-28 14:43:34] (step=0013625) Train Loss: 6.0040, Train Steps/Sec: 1.12 + 15%|████████████████████▍ | 13649/89905 [4:18:53<19:04:46, 1.11it/s][2025-04-28 14:43:56] (step=0013650) Train Loss: 6.0289, Train Steps/Sec: 1.12 + 15%|████████████████████▌ | 13674/89905 [4:19:15<18:56:39, 1.12it/s][2025-04-28 14:44:19] (step=0013675) Train Loss: 6.0451, Train Steps/Sec: 1.12 + 15%|████████████████████▌ | 13699/89905 [4:19:38<18:52:33, 1.12it/s][2025-04-28 14:44:41] (step=0013700) Train Loss: 6.0177, Train Steps/Sec: 1.12 + 15%|████████████████████▌ | 13724/89905 [4:20:00<18:45:41, 1.13it/s][2025-04-28 14:45:03] (step=0013725) Train Loss: 6.0121, Train Steps/Sec: 1.12 + 15%|████████████████████▋ | 13749/89905 [4:20:22<18:45:43, 1.13it/s][2025-04-28 14:45:26] (step=0013750) Train Loss: 6.0214, Train Steps/Sec: 1.12 + 15%|████████████████████▋ | 13774/89905 [4:20:45<18:39:56, 1.13it/s][2025-04-28 14:45:48] (step=0013775) Train Loss: 6.0418, Train Steps/Sec: 1.12 + 15%|████████████████████▋ | 13799/89905 [4:21:07<18:51:14, 1.12it/s][2025-04-28 14:46:10] (step=0013800) Train Loss: 6.0957, Train Steps/Sec: 1.11 + 15%|████████████████████▊ | 13824/89905 [4:21:29<19:15:22, 1.10it/s][2025-04-28 14:46:33] (step=0013825) Train Loss: 6.0199, Train Steps/Sec: 1.12 + 15%|████████████████████▊ | 13849/89905 [4:21:52<19:09:24, 1.10it/s][2025-04-28 14:46:55] (step=0013850) Train Loss: 6.0093, Train Steps/Sec: 1.12 + 15%|████████████████████▊ | 13874/89905 [4:22:14<18:56:27, 1.12it/s][2025-04-28 14:47:17] (step=0013875) Train Loss: 6.0680, Train Steps/Sec: 1.12 + 15%|████████████████████▊ | 13899/89905 [4:22:36<18:51:01, 1.12it/s][2025-04-28 14:47:40] (step=0013900) Train Loss: 5.9879, Train Steps/Sec: 1.12 + 15%|████████████████████▉ | 13924/89905 [4:22:59<18:47:14, 1.12it/s][2025-04-28 14:48:02] (step=0013925) Train Loss: 6.0420, Train Steps/Sec: 1.12 + 16%|████████████████████▉ | 13949/89905 [4:23:21<18:46:42, 1.12it/s][2025-04-28 14:48:24] (step=0013950) Train Loss: 6.0141, Train Steps/Sec: 1.12 + 16%|████████████████████▉ | 13974/89905 [4:23:43<18:39:45, 1.13it/s][2025-04-28 14:48:47] (step=0013975) Train Loss: 6.0472, Train Steps/Sec: 1.12 + 16%|█████████████████████ | 13999/89905 [4:24:06<18:38:40, 1.13it/s][2025-04-28 14:49:09] (step=0014000) Train Loss: 6.0177, Train Steps/Sec: 1.11 +You are using a model of type blip-2 to instantiate a model of type instructblip. This is not supported for all configurations of models and can yield errors. +[2025-04-28 14:49:09] text_config is None. Initializing the text config with default values (`OPTConfig`). +model_name_or_path:/tmp/haozhezhao/model/blip2-flan-t5-xl +image_place_holder: +tokenizer length after expend 32101 +tokenizer length before expend 32100 +100%|███████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████| 6/6 [06:00<00:00, 60.08s/it] +[2025-04-28 14:56:27] Finish Eval in 14000 steps...████████████████████████████████████████████████████████████████████| 6/6 [05:59<00:00, 59.89s/it] +processing left-padding... +processing left-padding... +processing left-padding... +processing left-padding... +processing left-padding... +processing left-padding... +[2025-04-28 14:56:48] Saved checkpoint to checkpoint/X2I_UltraEdiit_fourmask_LLava_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_filtered_objects_4mask_1280_context_after_stage1_5e4/000-GPT-XL/checkpoints/0014000.pt +[2025-04-28 14:56:50] Removed old checkpoint: checkpoint/X2I_UltraEdiit_fourmask_LLava_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_filtered_objects_4mask_1280_context_after_stage1_5e4/000-GPT-XL/checkpoints/0012000.pt + 16%|█████████████████████ | 14024/89905 [4:32:09<19:43:24, 1.07it/s][2025-04-28 14:57:12] (step=0014025) Train Loss: 6.0807, Train Steps/Sec: 0.05 + 16%|█████████████████████ | 14049/89905 [4:32:31<18:59:55, 1.11it/s][2025-04-28 14:57:35] (step=0014050) Train Loss: 6.0478, Train Steps/Sec: 1.12 + 16%|█████████████████████▏ | 14074/89905 [4:32:54<18:48:25, 1.12it/s][2025-04-28 14:57:57] (step=0014075) Train Loss: 5.9515, Train Steps/Sec: 1.12 + 16%|█████████████████████▏ | 14099/89905 [4:33:16<18:48:07, 1.12it/s][2025-04-28 14:58:19] (step=0014100) Train Loss: 5.9942, Train Steps/Sec: 1.12 + 16%|█████████████████████▏ | 14124/89905 [4:33:38<18:42:18, 1.13it/s][2025-04-28 14:58:42] (step=0014125) Train Loss: 6.0392, Train Steps/Sec: 1.12 + 16%|█████████████████████▏ | 14149/89905 [4:34:00<18:43:22, 1.12it/s][2025-04-28 14:59:04] (step=0014150) Train Loss: 5.9694, Train Steps/Sec: 1.12 + 16%|█████████████████████▎ | 14174/89905 [4:34:23<18:38:12, 1.13it/s][2025-04-28 14:59:26] (step=0014175) Train Loss: 6.0606, Train Steps/Sec: 1.12 + 16%|█████████████████████▎ | 14199/89905 [4:34:45<18:35:40, 1.13it/s][2025-04-28 14:59:49] (step=0014200) Train Loss: 6.0793, Train Steps/Sec: 1.12 + 16%|█████████████████████▎ | 14224/89905 [4:35:07<19:08:34, 1.10it/s][2025-04-28 15:00:11] (step=0014225) Train Loss: 6.0607, Train Steps/Sec: 1.12 + 16%|█████████████████████▍ | 14249/89905 [4:35:30<18:53:31, 1.11it/s][2025-04-28 15:00:33] (step=0014250) Train Loss: 5.9762, Train Steps/Sec: 1.12 + 16%|█████████████████████▍ | 14274/89905 [4:35:52<18:52:21, 1.11it/s][2025-04-28 15:00:55] (step=0014275) Train Loss: 6.0151, Train Steps/Sec: 1.12 + 16%|█████████████████████▍ | 14299/89905 [4:36:14<18:43:41, 1.12it/s][2025-04-28 15:01:18] (step=0014300) Train Loss: 5.9556, Train Steps/Sec: 1.12 + 16%|█████████████████████▌ | 14324/89905 [4:36:37<18:38:46, 1.13it/s][2025-04-28 15:01:40] (step=0014325) Train Loss: 6.0588, Train Steps/Sec: 1.12 + 16%|█████████████████████▌ | 14349/89905 [4:36:59<18:32:16, 1.13it/s][2025-04-28 15:02:02] (step=0014350) Train Loss: 6.0310, Train Steps/Sec: 1.12 + 16%|█████████████████████▌ | 14374/89905 [4:37:21<18:33:16, 1.13it/s][2025-04-28 15:02:25] (step=0014375) Train Loss: 6.0249, Train Steps/Sec: 1.12 + 16%|█████████████████████▌ | 14399/89905 [4:37:44<18:31:46, 1.13it/s][2025-04-28 15:02:47] (step=0014400) Train Loss: 5.9545, Train Steps/Sec: 1.12 + 16%|█████████████████████▋ | 14424/89905 [4:38:06<19:04:24, 1.10it/s][2025-04-28 15:03:09] (step=0014425) Train Loss: 5.9751, Train Steps/Sec: 1.12 + 16%|█████████████████████▋ | 14449/89905 [4:38:28<18:52:04, 1.11it/s][2025-04-28 15:03:32] (step=0014450) Train Loss: 6.0320, Train Steps/Sec: 1.12 + 16%|█████████████████████▋ | 14474/89905 [4:38:51<18:51:45, 1.11it/s][2025-04-28 15:03:54] (step=0014475) Train Loss: 6.0591, Train Steps/Sec: 1.11 + 16%|█████████████████████▊ | 14499/89905 [4:39:13<18:37:08, 1.12it/s][2025-04-28 15:04:17] (step=0014500) Train Loss: 5.9171, Train Steps/Sec: 1.12 + 16%|█████████████████████▊ | 14524/89905 [4:39:35<18:36:37, 1.13it/s][2025-04-28 15:04:39] (step=0014525) Train Loss: 6.0492, Train Steps/Sec: 1.12 + 16%|█████████████████████▊ | 14549/89905 [4:39:58<18:31:37, 1.13it/s][2025-04-28 15:05:01] (step=0014550) Train Loss: 6.0193, Train Steps/Sec: 1.12 + 16%|█████████████████████▉ | 14574/89905 [4:40:20<18:34:25, 1.13it/s][2025-04-28 15:05:23] (step=0014575) Train Loss: 5.9699, Train Steps/Sec: 1.12 + 16%|█████████████████████▉ | 14599/89905 [4:40:42<18:29:24, 1.13it/s][2025-04-28 15:05:46] (step=0014600) Train Loss: 5.9690, Train Steps/Sec: 1.12 + 16%|█████████████████████▉ | 14624/89905 [4:41:05<19:07:00, 1.09it/s][2025-04-28 15:06:08] (step=0014625) Train Loss: 6.0200, Train Steps/Sec: 1.12 + 16%|█████████████████████▉ | 14649/89905 [4:41:27<18:52:14, 1.11it/s][2025-04-28 15:06:31] (step=0014650) Train Loss: 6.0558, Train Steps/Sec: 1.12 + 16%|██████████████████████ | 14674/89905 [4:41:49<18:41:22, 1.12it/s][2025-04-28 15:06:53] (step=0014675) Train Loss: 6.0372, Train Steps/Sec: 1.12 + 16%|██████████████████████ | 14699/89905 [4:42:12<18:37:31, 1.12it/s][2025-04-28 15:07:15] (step=0014700) Train Loss: 5.9751, Train Steps/Sec: 1.12 + 16%|██████████████████████ | 14724/89905 [4:42:34<18:32:15, 1.13it/s][2025-04-28 15:07:37] (step=0014725) Train Loss: 5.9759, Train Steps/Sec: 1.12 + 16%|██████████████████████▏ | 14749/89905 [4:42:56<18:36:46, 1.12it/s][2025-04-28 15:08:00] (step=0014750) Train Loss: 5.9466, Train Steps/Sec: 1.12 + 16%|██████████████████████▏ | 14774/89905 [4:43:19<18:29:04, 1.13it/s][2025-04-28 15:08:22] (step=0014775) Train Loss: 6.0743, Train Steps/Sec: 1.12 + 16%|██████████████████████▏ | 14799/89905 [4:43:41<18:28:15, 1.13it/s][2025-04-28 15:08:45] (step=0014800) Train Loss: 5.9694, Train Steps/Sec: 1.11 + 16%|██████████████████████▎ | 14824/89905 [4:44:03<19:05:04, 1.09it/s][2025-04-28 15:09:07] (step=0014825) Train Loss: 5.9509, Train Steps/Sec: 1.12 + 17%|██████████████████████▎ | 14849/89905 [4:44:26<18:48:32, 1.11it/s][2025-04-28 15:09:29] (step=0014850) Train Loss: 5.9954, Train Steps/Sec: 1.12 + 17%|██████████████████████▎ | 14874/89905 [4:44:48<18:41:26, 1.12it/s][2025-04-28 15:09:52] (step=0014875) Train Loss: 6.0090, Train Steps/Sec: 1.12 + 17%|██████████████████████▎ | 14899/89905 [4:45:10<18:35:04, 1.12it/s][2025-04-28 15:10:14] (step=0014900) Train Loss: 6.0149, Train Steps/Sec: 1.12 + 17%|██████████████████████▍ | 14924/89905 [4:45:33<18:30:03, 1.13it/s][2025-04-28 15:10:36] (step=0014925) Train Loss: 5.9759, Train Steps/Sec: 1.12 + 17%|██████████████████████▍ | 14949/89905 [4:45:55<18:27:50, 1.13it/s][2025-04-28 15:10:58] (step=0014950) Train Loss: 6.0020, Train Steps/Sec: 1.12 + 17%|██████████████████████▍ | 14974/89905 [4:46:17<18:25:39, 1.13it/s][2025-04-28 15:11:21] (step=0014975) Train Loss: 5.9754, Train Steps/Sec: 1.12 + 17%|██████████████████████▌ | 14999/89905 [4:46:40<18:24:07, 1.13it/s][2025-04-28 15:11:43] (step=0015000) Train Loss: 5.9494, Train Steps/Sec: 1.12 + 17%|██████████████████████▌ | 15024/89905 [4:47:02<18:58:22, 1.10it/s][2025-04-28 15:12:06] (step=0015025) Train Loss: 5.9044, Train Steps/Sec: 1.12 + 17%|██████████████████████▌ | 15049/89905 [4:47:24<18:43:12, 1.11it/s][2025-04-28 15:12:28] (step=0015050) Train Loss: 5.9769, Train Steps/Sec: 1.12 + 17%|██████████████████████▋ | 15074/89905 [4:47:47<18:36:40, 1.12it/s][2025-04-28 15:12:50] (step=0015075) Train Loss: 6.0237, Train Steps/Sec: 1.12 + 17%|██████████████████████▋ | 15099/89905 [4:48:09<18:32:01, 1.12it/s][2025-04-28 15:13:12] (step=0015100) Train Loss: 6.0295, Train Steps/Sec: 1.12 + 17%|██████████████████████▋ | 15124/89905 [4:48:31<18:28:45, 1.12it/s][2025-04-28 15:13:35] (step=0015125) Train Loss: 6.0049, Train Steps/Sec: 1.12 + 17%|██████████████████████▋ | 15149/89905 [4:48:54<18:28:39, 1.12it/s][2025-04-28 15:13:57] (step=0015150) Train Loss: 6.0243, Train Steps/Sec: 1.12 + 17%|██████████████████████▊ | 15174/89905 [4:49:16<18:21:59, 1.13it/s][2025-04-28 15:14:19] (step=0015175) Train Loss: 5.9977, Train Steps/Sec: 1.12 + 17%|██████████████████████▊ | 15199/89905 [4:49:38<18:21:44, 1.13it/s][2025-04-28 15:14:42] (step=0015200) Train Loss: 5.9976, Train Steps/Sec: 1.12 + 17%|██████████████████████▊ | 15224/89905 [4:50:01<18:51:22, 1.10it/s][2025-04-28 15:15:04] (step=0015225) Train Loss: 5.9914, Train Steps/Sec: 1.12 + 17%|██████████████████████▉ | 15249/89905 [4:50:23<18:38:04, 1.11it/s][2025-04-28 15:15:26] (step=0015250) Train Loss: 6.0081, Train Steps/Sec: 1.12 + 17%|██████████████████████▉ | 15274/89905 [4:50:45<18:31:37, 1.12it/s][2025-04-28 15:15:49] (step=0015275) Train Loss: 5.9754, Train Steps/Sec: 1.12 + 17%|██████████████████████▉ | 15299/89905 [4:51:08<18:30:49, 1.12it/s][2025-04-28 15:16:11] (step=0015300) Train Loss: 5.9914, Train Steps/Sec: 1.12 + 17%|███████████████████████ | 15324/89905 [4:51:30<18:26:48, 1.12it/s][2025-04-28 15:16:33] (step=0015325) Train Loss: 6.0026, Train Steps/Sec: 1.12 + 17%|███████████████████████ | 15349/89905 [4:51:52<18:26:07, 1.12it/s][2025-04-28 15:16:56] (step=0015350) Train Loss: 5.9776, Train Steps/Sec: 1.12 + 17%|███████████████████████ | 15374/89905 [4:52:15<18:21:04, 1.13it/s][2025-04-28 15:17:18] (step=0015375) Train Loss: 6.0162, Train Steps/Sec: 1.12 + 17%|███████████████████████ | 15399/89905 [4:52:37<18:19:45, 1.13it/s][2025-04-28 15:17:41] (step=0015400) Train Loss: 6.0034, Train Steps/Sec: 1.11 + 17%|███████████████████████▏ | 15424/89905 [4:52:59<18:52:18, 1.10it/s][2025-04-28 15:18:03] (step=0015425) Train Loss: 5.9350, Train Steps/Sec: 1.12 + 17%|███████████████████████▏ | 15449/89905 [4:53:22<18:36:59, 1.11it/s][2025-04-28 15:18:25] (step=0015450) Train Loss: 5.9396, Train Steps/Sec: 1.12 + 17%|███████████████████████▏ | 15474/89905 [4:53:44<18:29:47, 1.12it/s][2025-04-28 15:18:47] (step=0015475) Train Loss: 5.9610, Train Steps/Sec: 1.12 + 17%|███████████████████████▎ | 15499/89905 [4:54:06<18:22:30, 1.12it/s][2025-04-28 15:19:10] (step=0015500) Train Loss: 6.0061, Train Steps/Sec: 1.12 + 17%|███████████████████████▎ | 15524/89905 [4:54:29<18:20:53, 1.13it/s][2025-04-28 15:19:32] (step=0015525) Train Loss: 5.9912, Train Steps/Sec: 1.12 + 17%|███████████████████████▎ | 15549/89905 [4:54:51<18:19:50, 1.13it/s][2025-04-28 15:19:54] (step=0015550) Train Loss: 5.9695, Train Steps/Sec: 1.12 + 17%|███████████████████████▍ | 15574/89905 [4:55:13<18:18:20, 1.13it/s][2025-04-28 15:20:17] (step=0015575) Train Loss: 5.9883, Train Steps/Sec: 1.12 + 17%|███████████████████████▍ | 15599/89905 [4:55:36<18:16:11, 1.13it/s][2025-04-28 15:20:39] (step=0015600) Train Loss: 5.9789, Train Steps/Sec: 1.12 + 17%|███████████████████████▍ | 15624/89905 [4:56:45<84:06:51, 4.08s/it][2025-04-28 15:21:49] (step=0015625) Train Loss: 5.9784, Train Steps/Sec: 0.36 + 17%|███████████████████████▍ | 15649/89905 [4:57:16<18:57:15, 1.09it/s][2025-04-28 15:22:20] (step=0015650) Train Loss: 6.0302, Train Steps/Sec: 0.80 + 17%|███████████████████████▌ | 15674/89905 [4:57:48<19:35:55, 1.05it/s][2025-04-28 15:22:51] (step=0015675) Train Loss: 5.9327, Train Steps/Sec: 0.79 + 17%|███████████████████████▌ | 15699/89905 [4:58:10<18:22:06, 1.12it/s][2025-04-28 15:23:14] (step=0015700) Train Loss: 5.9652, Train Steps/Sec: 1.12 + 17%|███████████████████████▌ | 15724/89905 [4:58:33<18:18:37, 1.13it/s][2025-04-28 15:23:36] (step=0015725) Train Loss: 5.9430, Train Steps/Sec: 1.12 + 18%|███████████████████████▋ | 15749/89905 [4:58:55<18:14:21, 1.13it/s][2025-04-28 15:23:58] (step=0015750) Train Loss: 5.9395, Train Steps/Sec: 1.12 + 18%|███████████████████████▋ | 15774/89905 [4:59:17<18:10:13, 1.13it/s][2025-04-28 15:24:21] (step=0015775) Train Loss: 5.9757, Train Steps/Sec: 1.12 + 18%|███████████████████████▋ | 15799/89905 [4:59:40<18:12:24, 1.13it/s][2025-04-28 15:24:43] (step=0015800) Train Loss: 5.9704, Train Steps/Sec: 1.11 + 18%|███████████████████████▊ | 15824/89905 [5:00:02<18:45:05, 1.10it/s][2025-04-28 15:25:05] (step=0015825) Train Loss: 5.8617, Train Steps/Sec: 1.12 + 18%|███████████████████████▊ | 15849/89905 [5:00:24<18:30:54, 1.11it/s][2025-04-28 15:25:28] (step=0015850) Train Loss: 6.0001, Train Steps/Sec: 1.12 + 18%|███████████████████████▊ | 15874/89905 [5:00:47<18:24:06, 1.12it/s][2025-04-28 15:25:50] (step=0015875) Train Loss: 5.9245, Train Steps/Sec: 1.12 + 18%|███████████████████████▊ | 15899/89905 [5:01:09<18:20:32, 1.12it/s][2025-04-28 15:26:12] (step=0015900) Train Loss: 5.9441, Train Steps/Sec: 1.12 + 18%|███████████████████████▉ | 15924/89905 [5:01:31<18:16:32, 1.12it/s][2025-04-28 15:26:35] (step=0015925) Train Loss: 5.9410, Train Steps/Sec: 1.12 + 18%|███████████████████████▉ | 15949/89905 [5:01:54<18:16:30, 1.12it/s][2025-04-28 15:26:57] (step=0015950) Train Loss: 5.9531, Train Steps/Sec: 1.12 + 18%|███████████████████████▉ | 15974/89905 [5:02:16<18:08:38, 1.13it/s][2025-04-28 15:27:19] (step=0015975) Train Loss: 5.9442, Train Steps/Sec: 1.12 + 18%|████████████████████████ | 15999/89905 [5:02:38<18:13:15, 1.13it/s][2025-04-28 15:27:42] (step=0016000) Train Loss: 5.9726, Train Steps/Sec: 1.11 +You are using a model of type blip-2 to instantiate a model of type instructblip. This is not supported for all configurations of models and can yield errors. +[2025-04-28 15:27:42] text_config is None. Initializing the text config with default values (`OPTConfig`). +model_name_or_path:/tmp/haozhezhao/model/blip2-flan-t5-xl +image_place_holder: +tokenizer length after expend 32101 +tokenizer length before expend 32100 +100%|███████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████| 6/6 [06:00<00:00, 60.02s/it] +[2025-04-28 15:35:00] Finish Eval in 16000 steps...████████████████████████████████████████████████████████████████████| 6/6 [05:59<00:00, 59.82s/it] +processing left-padding... +processing left-padding... +processing left-padding... +processing left-padding... +processing left-padding... +processing left-padding... +[2025-04-28 15:35:20] Saved checkpoint to checkpoint/X2I_UltraEdiit_fourmask_LLava_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_filtered_objects_4mask_1280_context_after_stage1_5e4/000-GPT-XL/checkpoints/0016000.pt +[2025-04-28 15:35:22] Removed old checkpoint: checkpoint/X2I_UltraEdiit_fourmask_LLava_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_filtered_objects_4mask_1280_context_after_stage1_5e4/000-GPT-XL/checkpoints/0014000.pt + 18%|████████████████████████ | 16024/89905 [5:10:41<19:13:33, 1.07it/s][2025-04-28 15:35:45] (step=0016025) Train Loss: 5.8962, Train Steps/Sec: 0.05 + 18%|████████████████████████ | 16049/89905 [5:11:03<18:23:42, 1.12it/s][2025-04-28 15:36:07] (step=0016050) Train Loss: 5.9402, Train Steps/Sec: 1.12 + 18%|████████████████████████▏ | 16074/89905 [5:11:26<18:19:05, 1.12it/s][2025-04-28 15:36:29] (step=0016075) Train Loss: 5.9906, Train Steps/Sec: 1.12 + 18%|████████████████████████▏ | 16099/89905 [5:11:48<18:15:34, 1.12it/s][2025-04-28 15:36:51] (step=0016100) Train Loss: 6.0432, Train Steps/Sec: 1.12 + 18%|████████████████████████▏ | 16124/89905 [5:12:10<18:14:48, 1.12it/s][2025-04-28 15:37:14] (step=0016125) Train Loss: 5.9399, Train Steps/Sec: 1.12 + 18%|████████████████████████▏ | 16149/89905 [5:12:33<18:12:57, 1.12it/s][2025-04-28 15:37:36] (step=0016150) Train Loss: 5.9174, Train Steps/Sec: 1.12 + 18%|████████████████████████▎ | 16174/89905 [5:12:55<18:06:20, 1.13it/s][2025-04-28 15:37:58] (step=0016175) Train Loss: 5.9304, Train Steps/Sec: 1.12 + 18%|████████████████████████▎ | 16199/89905 [5:13:17<18:06:48, 1.13it/s][2025-04-28 15:38:21] (step=0016200) Train Loss: 5.9640, Train Steps/Sec: 1.12 + 18%|████████████████████████▎ | 16224/89905 [5:13:40<18:46:30, 1.09it/s][2025-04-28 15:38:43] (step=0016225) Train Loss: 5.9457, Train Steps/Sec: 1.12 + 18%|████████████████████████▍ | 16249/89905 [5:14:02<18:28:22, 1.11it/s][2025-04-28 15:39:06] (step=0016250) Train Loss: 5.9469, Train Steps/Sec: 1.12 + 18%|████████████████████████▍ | 16274/89905 [5:14:24<18:18:34, 1.12it/s][2025-04-28 15:39:28] (step=0016275) Train Loss: 5.8930, Train Steps/Sec: 1.12 + 18%|████████████████████████▍ | 16299/89905 [5:14:47<18:16:17, 1.12it/s][2025-04-28 15:39:50] (step=0016300) Train Loss: 5.9471, Train Steps/Sec: 1.12 + 18%|████████████████████████▌ | 16324/89905 [5:15:09<18:05:47, 1.13it/s][2025-04-28 15:40:12] (step=0016325) Train Loss: 5.9980, Train Steps/Sec: 1.12 + 18%|████████████████████████▌ | 16349/89905 [5:15:31<18:03:56, 1.13it/s][2025-04-28 15:40:35] (step=0016350) Train Loss: 5.9456, Train Steps/Sec: 1.12 + 18%|████████████████████████▌ | 16374/89905 [5:15:54<18:04:06, 1.13it/s][2025-04-28 15:40:57] (step=0016375) Train Loss: 5.8609, Train Steps/Sec: 1.12 + 18%|████████████████████████▌ | 16399/89905 [5:16:16<18:03:07, 1.13it/s][2025-04-28 15:41:20] (step=0016400) Train Loss: 5.9608, Train Steps/Sec: 1.12 + 18%|████████████████████████▋ | 16424/89905 [5:16:39<18:37:47, 1.10it/s][2025-04-28 15:41:42] (step=0016425) Train Loss: 5.9370, Train Steps/Sec: 1.12 + 18%|████████████████████████▋ | 16449/89905 [5:17:01<18:21:42, 1.11it/s][2025-04-28 15:42:04] (step=0016450) Train Loss: 5.8545, Train Steps/Sec: 1.12 + 18%|████████████████████████▋ | 16474/89905 [5:17:23<18:17:59, 1.11it/s][2025-04-28 15:42:27] (step=0016475) Train Loss: 5.8663, Train Steps/Sec: 1.12 + 18%|████████████████████████▊ | 16499/89905 [5:17:45<18:06:41, 1.13it/s][2025-04-28 15:42:49] (step=0016500) Train Loss: 5.9601, Train Steps/Sec: 1.12 + 18%|████████████████████████▊ | 16524/89905 [5:18:08<18:10:33, 1.12it/s][2025-04-28 15:43:11] (step=0016525) Train Loss: 5.9354, Train Steps/Sec: 1.12 + 18%|████████████████████████▊ | 16549/89905 [5:18:30<18:06:58, 1.12it/s][2025-04-28 15:43:34] (step=0016550) Train Loss: 5.9224, Train Steps/Sec: 1.12 + 18%|████████████████████████▉ | 16574/89905 [5:18:52<18:01:50, 1.13it/s][2025-04-28 15:43:56] (step=0016575) Train Loss: 5.9486, Train Steps/Sec: 1.12 + 18%|████████████████████████▉ | 16599/89905 [5:19:15<17:57:52, 1.13it/s][2025-04-28 15:44:18] (step=0016600) Train Loss: 5.9610, Train Steps/Sec: 1.12 + 18%|████████████████████████▉ | 16624/89905 [5:19:37<18:37:02, 1.09it/s][2025-04-28 15:44:41] (step=0016625) Train Loss: 5.9015, Train Steps/Sec: 1.12 + 19%|████████████████████████▉ | 16649/89905 [5:20:00<18:21:00, 1.11it/s][2025-04-28 15:45:03] (step=0016650) Train Loss: 5.9494, Train Steps/Sec: 1.12 + 19%|█████████████████████████ | 16674/89905 [5:20:22<18:07:29, 1.12it/s][2025-04-28 15:45:25] (step=0016675) Train Loss: 5.9957, Train Steps/Sec: 1.12 + 19%|█████████████████████████ | 16699/89905 [5:20:44<18:03:14, 1.13it/s][2025-04-28 15:45:48] (step=0016700) Train Loss: 5.9662, Train Steps/Sec: 1.12 + 19%|█████████████████████████ | 16724/89905 [5:21:07<18:07:44, 1.12it/s][2025-04-28 15:46:10] (step=0016725) Train Loss: 5.9848, Train Steps/Sec: 1.12 + 19%|█████████████████████████▏ | 16749/89905 [5:21:29<17:59:34, 1.13it/s][2025-04-28 15:46:32] (step=0016750) Train Loss: 5.9237, Train Steps/Sec: 1.12 + 19%|█████████████████████████▏ | 16774/89905 [5:21:51<17:58:15, 1.13it/s][2025-04-28 15:46:55] (step=0016775) Train Loss: 5.8634, Train Steps/Sec: 1.12 + 19%|█████████████████████████▏ | 16799/89905 [5:22:14<17:59:14, 1.13it/s][2025-04-28 15:47:17] (step=0016800) Train Loss: 5.9497, Train Steps/Sec: 1.11 + 19%|█████████████████████████▎ | 16824/89905 [5:22:36<18:28:55, 1.10it/s][2025-04-28 15:47:39] (step=0016825) Train Loss: 5.9778, Train Steps/Sec: 1.12 + 19%|█████████████████████████▎ | 16849/89905 [5:22:58<18:17:24, 1.11it/s][2025-04-28 15:48:02] (step=0016850) Train Loss: 5.9171, Train Steps/Sec: 1.12 + 19%|█████████████████████████▎ | 16874/89905 [5:23:21<18:07:28, 1.12it/s][2025-04-28 15:48:24] (step=0016875) Train Loss: 5.9030, Train Steps/Sec: 1.12 + 19%|█████████████████████████▍ | 16899/89905 [5:23:43<18:05:01, 1.12it/s][2025-04-28 15:48:46] (step=0016900) Train Loss: 5.9116, Train Steps/Sec: 1.12 + 19%|█████████████████████████▍ | 16924/89905 [5:24:05<18:03:11, 1.12it/s][2025-04-28 15:49:09] (step=0016925) Train Loss: 5.9239, Train Steps/Sec: 1.12 + 19%|█████████████████████████▍ | 16949/89905 [5:24:28<18:00:23, 1.13it/s][2025-04-28 15:49:31] (step=0016950) Train Loss: 5.9444, Train Steps/Sec: 1.12 + 19%|█████████████████████████▍ | 16974/89905 [5:24:50<17:54:29, 1.13it/s][2025-04-28 15:49:53] (step=0016975) Train Loss: 5.9355, Train Steps/Sec: 1.12 + 19%|█████████████████████████▌ | 16999/89905 [5:25:12<17:54:14, 1.13it/s][2025-04-28 15:50:16] (step=0017000) Train Loss: 5.9297, Train Steps/Sec: 1.11 + 19%|█████████████████████████▌ | 17024/89905 [5:25:35<18:26:56, 1.10it/s][2025-04-28 15:50:38] (step=0017025) Train Loss: 5.9212, Train Steps/Sec: 1.12 + 19%|█████████████████████████▌ | 17049/89905 [5:25:57<18:11:19, 1.11it/s][2025-04-28 15:51:01] (step=0017050) Train Loss: 5.9798, Train Steps/Sec: 1.12 + 19%|█████████████████████████▋ | 17074/89905 [5:26:19<18:08:59, 1.11it/s][2025-04-28 15:51:23] (step=0017075) Train Loss: 5.9200, Train Steps/Sec: 1.12 + 19%|█████████████████████████▋ | 17099/89905 [5:26:42<18:05:56, 1.12it/s][2025-04-28 15:51:45] (step=0017100) Train Loss: 5.9390, Train Steps/Sec: 1.12 + 19%|█████████████████████████▋ | 17124/89905 [5:27:04<18:01:00, 1.12it/s][2025-04-28 15:52:08] (step=0017125) Train Loss: 5.9526, Train Steps/Sec: 1.12 + 19%|█████████████████████████▊ | 17149/89905 [5:27:27<17:57:54, 1.12it/s][2025-04-28 15:52:30] (step=0017150) Train Loss: 5.9380, Train Steps/Sec: 1.12 + 19%|█████████████████████████▊ | 17174/89905 [5:27:49<17:52:19, 1.13it/s][2025-04-28 15:52:52] (step=0017175) Train Loss: 5.9884, Train Steps/Sec: 1.12 + 19%|█████████████████████████▊ | 17199/89905 [5:28:11<17:49:23, 1.13it/s][2025-04-28 15:53:15] (step=0017200) Train Loss: 5.9680, Train Steps/Sec: 1.12 + 19%|█████████████████████████▊ | 17224/89905 [5:28:34<18:24:16, 1.10it/s][2025-04-28 15:53:37] (step=0017225) Train Loss: 5.8722, Train Steps/Sec: 1.12 + 19%|█████████████████████████▉ | 17249/89905 [5:28:56<18:10:21, 1.11it/s][2025-04-28 15:53:59] (step=0017250) Train Loss: 5.9967, Train Steps/Sec: 1.12 + 19%|█████████████████████████▉ | 17274/89905 [5:29:18<18:00:18, 1.12it/s][2025-04-28 15:54:22] (step=0017275) Train Loss: 5.8936, Train Steps/Sec: 1.12 + 19%|█████████████████████████▉ | 17299/89905 [5:29:40<17:59:59, 1.12it/s][2025-04-28 15:54:44] (step=0017300) Train Loss: 5.9390, Train Steps/Sec: 1.12 + 19%|██████████████████████████ | 17324/89905 [5:30:03<17:53:50, 1.13it/s][2025-04-28 15:55:06] (step=0017325) Train Loss: 5.8953, Train Steps/Sec: 1.12 + 19%|██████████████████████████ | 17349/89905 [5:30:25<17:55:18, 1.12it/s][2025-04-28 15:55:29] (step=0017350) Train Loss: 5.9512, Train Steps/Sec: 1.12 + 19%|██████████████████████████ | 17374/89905 [5:30:48<17:47:04, 1.13it/s][2025-04-28 15:55:51] (step=0017375) Train Loss: 5.9250, Train Steps/Sec: 1.12 + 19%|██████████████████████████▏ | 17399/89905 [5:31:10<17:50:11, 1.13it/s][2025-04-28 15:56:13] (step=0017400) Train Loss: 5.9069, Train Steps/Sec: 1.11 + 19%|██████████████████████████▏ | 17424/89905 [5:31:32<18:21:09, 1.10it/s][2025-04-28 15:56:36] (step=0017425) Train Loss: 5.8674, Train Steps/Sec: 1.12 + 19%|██████████████████████████▏ | 17449/89905 [5:31:55<18:09:07, 1.11it/s][2025-04-28 15:56:58] (step=0017450) Train Loss: 5.9503, Train Steps/Sec: 1.12 + 19%|██████████████████████████▏ | 17474/89905 [5:32:17<18:02:02, 1.12it/s][2025-04-28 15:57:20] (step=0017475) Train Loss: 5.9811, Train Steps/Sec: 1.12 + 19%|██████████████████████████▎ | 17499/89905 [5:32:39<17:55:56, 1.12it/s][2025-04-28 15:57:43] (step=0017500) Train Loss: 5.9118, Train Steps/Sec: 1.12 + 19%|██████████████████████████▎ | 17524/89905 [5:33:02<17:54:30, 1.12it/s][2025-04-28 15:58:05] (step=0017525) Train Loss: 5.9279, Train Steps/Sec: 1.12 + 20%|██████████████████████████▎ | 17549/89905 [5:33:24<17:48:51, 1.13it/s][2025-04-28 15:58:27] (step=0017550) Train Loss: 5.9098, Train Steps/Sec: 1.12 + 20%|██████████████████████████▍ | 17574/89905 [5:33:46<17:46:28, 1.13it/s][2025-04-28 15:58:50] (step=0017575) Train Loss: 5.9085, Train Steps/Sec: 1.12 + 20%|██████████████████████████▍ | 17599/89905 [5:34:09<17:44:44, 1.13it/s][2025-04-28 15:59:12] (step=0017600) Train Loss: 5.9378, Train Steps/Sec: 1.11 + 20%|██████████████████████████▍ | 17624/89905 [5:34:31<18:17:47, 1.10it/s][2025-04-28 15:59:34] (step=0017625) Train Loss: 5.8908, Train Steps/Sec: 1.12 + 20%|██████████████████████████▌ | 17649/89905 [5:34:53<18:07:44, 1.11it/s][2025-04-28 15:59:57] (step=0017650) Train Loss: 5.9165, Train Steps/Sec: 1.12 + 20%|██████████████████████████▌ | 17674/89905 [5:35:16<17:57:25, 1.12it/s][2025-04-28 16:00:19] (step=0017675) Train Loss: 5.9625, Train Steps/Sec: 1.12 + 20%|██████████████████████████▌ | 17699/89905 [5:35:38<17:53:58, 1.12it/s][2025-04-28 16:00:41] (step=0017700) Train Loss: 5.9401, Train Steps/Sec: 1.11 + 20%|██████████████████████████▌ | 17724/89905 [5:36:00<17:47:56, 1.13it/s][2025-04-28 16:01:04] (step=0017725) Train Loss: 5.9363, Train Steps/Sec: 1.12 + 20%|██████████████████████████▋ | 17749/89905 [5:36:23<17:48:10, 1.13it/s][2025-04-28 16:01:26] (step=0017750) Train Loss: 5.9035, Train Steps/Sec: 1.12 + 20%|██████████████████████████▋ | 17774/89905 [5:36:45<17:43:19, 1.13it/s][2025-04-28 16:01:48] (step=0017775) Train Loss: 5.9369, Train Steps/Sec: 1.12 + 20%|██████████████████████████▋ | 17799/89905 [5:37:07<17:42:36, 1.13it/s][2025-04-28 16:02:11] (step=0017800) Train Loss: 5.9033, Train Steps/Sec: 1.11 + 20%|██████████████████████████▊ | 17824/89905 [5:37:30<18:14:32, 1.10it/s][2025-04-28 16:02:33] (step=0017825) Train Loss: 5.9144, Train Steps/Sec: 1.12 + 20%|██████████████████████████▊ | 17849/89905 [5:37:52<18:02:06, 1.11it/s][2025-04-28 16:02:56] (step=0017850) Train Loss: 5.9025, Train Steps/Sec: 1.12 + 20%|██████████████████████████▊ | 17874/89905 [5:38:15<17:56:16, 1.12it/s][2025-04-28 16:03:18] (step=0017875) Train Loss: 5.9290, Train Steps/Sec: 1.12 + 20%|██████████████████████████▉ | 17899/89905 [5:38:37<17:49:05, 1.12it/s][2025-04-28 16:03:40] (step=0017900) Train Loss: 5.9009, Train Steps/Sec: 1.12 + 20%|██████████████████████████▉ | 17924/89905 [5:38:59<17:49:46, 1.12it/s][2025-04-28 16:04:03] (step=0017925) Train Loss: 5.9162, Train Steps/Sec: 1.12 + 20%|██████████████████████████▉ | 17949/89905 [5:39:22<17:44:42, 1.13it/s][2025-04-28 16:04:25] (step=0017950) Train Loss: 5.8941, Train Steps/Sec: 1.12 + 20%|██████████████████████████▉ | 17974/89905 [5:39:44<17:44:53, 1.13it/s][2025-04-28 16:04:47] (step=0017975) Train Loss: 5.9289, Train Steps/Sec: 1.12 + 20%|███████████████████████████ | 17999/89905 [5:40:06<17:39:42, 1.13it/s][2025-04-28 16:05:10] (step=0018000) Train Loss: 5.8731, Train Steps/Sec: 1.12 +You are using a model of type blip-2 to instantiate a model of type instructblip. This is not supported for all configurations of models and can yield errors. +[2025-04-28 16:05:10] text_config is None. Initializing the text config with default values (`OPTConfig`). +model_name_or_path:/tmp/haozhezhao/model/blip2-flan-t5-xl +image_place_holder: +tokenizer length after expend 32101 +tokenizer length before expend 32100 +100%|███████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████| 6/6 [06:00<00:00, 60.11s/it] +[2025-04-28 16:12:28] Finish Eval in 18000 steps...████████████████████████████████████████████████████████████████████| 6/6 [06:00<00:00, 59.89s/it] +processing left-padding... +processing left-padding... +processing left-padding... +processing left-padding... +processing left-padding... +processing left-padding... +[2025-04-28 16:12:48] Saved checkpoint to checkpoint/X2I_UltraEdiit_fourmask_LLava_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_filtered_objects_4mask_1280_context_after_stage1_5e4/000-GPT-XL/checkpoints/0018000.pt +[2025-04-28 16:12:50] Removed old checkpoint: checkpoint/X2I_UltraEdiit_fourmask_LLava_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_filtered_objects_4mask_1280_context_after_stage1_5e4/000-GPT-XL/checkpoints/0016000.pt + 20%|███████████████████████████ | 18024/89905 [5:48:09<18:41:58, 1.07it/s][2025-04-28 16:13:12] (step=0018025) Train Loss: 5.9053, Train Steps/Sec: 0.05 + 20%|███████████████████████████ | 18049/89905 [5:48:31<17:57:21, 1.11it/s][2025-04-28 16:13:34] (step=0018050) Train Loss: 5.8165, Train Steps/Sec: 1.12 + 20%|███████████████████████████▏ | 18074/89905 [5:48:53<17:50:50, 1.12it/s][2025-04-28 16:13:57] (step=0018075) Train Loss: 5.9039, Train Steps/Sec: 1.12 + 20%|███████████████████████████▏ | 18099/89905 [5:49:16<17:47:04, 1.12it/s][2025-04-28 16:14:19] (step=0018100) Train Loss: 5.8945, Train Steps/Sec: 1.12 + 20%|███████████████████████████▏ | 18124/89905 [5:49:38<17:45:08, 1.12it/s][2025-04-28 16:14:41] (step=0018125) Train Loss: 5.9189, Train Steps/Sec: 1.12 + 20%|███████████████████████████▎ | 18149/89905 [5:50:00<17:37:55, 1.13it/s][2025-04-28 16:15:04] (step=0018150) Train Loss: 5.9610, Train Steps/Sec: 1.12 + 20%|███████████████████████████▎ | 18174/89905 [5:50:23<17:35:03, 1.13it/s][2025-04-28 16:15:26] (step=0018175) Train Loss: 5.8812, Train Steps/Sec: 1.12 + 20%|███████████████████████████▎ | 18199/89905 [5:50:45<17:37:14, 1.13it/s][2025-04-28 16:15:49] (step=0018200) Train Loss: 5.8938, Train Steps/Sec: 1.11 + 20%|███████████████████████████▎ | 18224/89905 [5:51:08<18:10:56, 1.10it/s][2025-04-28 16:16:11] (step=0018225) Train Loss: 5.9171, Train Steps/Sec: 1.12 + 20%|███████████████████████████▍ | 18249/89905 [5:51:30<18:00:38, 1.11it/s][2025-04-28 16:16:33] (step=0018250) Train Loss: 5.9023, Train Steps/Sec: 1.12 + 20%|███████████████████████████▍ | 18274/89905 [5:51:52<17:45:49, 1.12it/s][2025-04-28 16:16:56] (step=0018275) Train Loss: 5.9244, Train Steps/Sec: 1.12 + 20%|███████████████████████████▍ | 18299/89905 [5:52:15<17:47:49, 1.12it/s][2025-04-28 16:17:18] (step=0018300) Train Loss: 5.8111, Train Steps/Sec: 1.12 + 20%|███████████████████████████▌ | 18324/89905 [5:52:37<17:42:42, 1.12it/s][2025-04-28 16:17:40] (step=0018325) Train Loss: 5.9025, Train Steps/Sec: 1.12 + 20%|███████████████████████████▌ | 18349/89905 [5:52:59<17:41:15, 1.12it/s][2025-04-28 16:18:03] (step=0018350) Train Loss: 5.9115, Train Steps/Sec: 1.12 + 20%|███████████████████████████▌ | 18374/89905 [5:53:22<17:36:38, 1.13it/s][2025-04-28 16:18:25] (step=0018375) Train Loss: 5.8695, Train Steps/Sec: 1.12 + 20%|███████████████████████████▋ | 18399/89905 [5:53:44<17:33:54, 1.13it/s][2025-04-28 16:18:47] (step=0018400) Train Loss: 5.9346, Train Steps/Sec: 1.11 + 20%|███████████████████████████▋ | 18424/89905 [5:54:06<18:05:22, 1.10it/s][2025-04-28 16:19:10] (step=0018425) Train Loss: 5.8625, Train Steps/Sec: 1.12 + 21%|███████████████████████████▋ | 18449/89905 [5:54:29<17:51:44, 1.11it/s][2025-04-28 16:19:32] (step=0018450) Train Loss: 5.8998, Train Steps/Sec: 1.12 + 21%|███████████████████████████▋ | 18474/89905 [5:54:51<17:49:00, 1.11it/s][2025-04-28 16:19:54] (step=0018475) Train Loss: 5.9787, Train Steps/Sec: 1.12 + 21%|███████████████████████████▊ | 18499/89905 [5:55:13<17:39:58, 1.12it/s][2025-04-28 16:20:17] (step=0018500) Train Loss: 5.8726, Train Steps/Sec: 1.12 + 21%|███████████████████████████▊ | 18524/89905 [5:55:36<17:36:55, 1.13it/s][2025-04-28 16:20:39] (step=0018525) Train Loss: 5.8758, Train Steps/Sec: 1.12 + 21%|███████████████████████████▊ | 18549/89905 [5:55:58<17:30:53, 1.13it/s][2025-04-28 16:21:01] (step=0018550) Train Loss: 5.9138, Train Steps/Sec: 1.12 + 21%|███████████████████████████▉ | 18574/89905 [5:56:20<17:29:52, 1.13it/s][2025-04-28 16:21:24] (step=0018575) Train Loss: 5.9201, Train Steps/Sec: 1.12 + 21%|███████████████████████████▉ | 18599/89905 [5:56:42<17:29:28, 1.13it/s][2025-04-28 16:21:46] (step=0018600) Train Loss: 5.9273, Train Steps/Sec: 1.12 + 21%|███████████████████████████▉ | 18624/89905 [5:57:05<18:08:22, 1.09it/s][2025-04-28 16:22:08] (step=0018625) Train Loss: 5.8874, Train Steps/Sec: 1.12 + 21%|████████████████████████████ | 18649/89905 [5:57:27<17:50:40, 1.11it/s][2025-04-28 16:22:31] (step=0018650) Train Loss: 5.8914, Train Steps/Sec: 1.12 + 21%|████████████████████████████ | 18674/89905 [5:57:50<17:39:02, 1.12it/s][2025-04-28 16:22:53] (step=0018675) Train Loss: 5.8707, Train Steps/Sec: 1.12 + 21%|████████████████████████████ | 18699/89905 [5:58:12<17:33:09, 1.13it/s][2025-04-28 16:23:15] (step=0018700) Train Loss: 5.9067, Train Steps/Sec: 1.12 + 21%|████████████████████████████ | 18724/89905 [5:58:34<17:33:46, 1.13it/s][2025-04-28 16:23:38] (step=0018725) Train Loss: 5.8876, Train Steps/Sec: 1.12 + 21%|████████████████████████████▏ | 18749/89905 [5:58:56<17:30:03, 1.13it/s][2025-04-28 16:24:00] (step=0018750) Train Loss: 5.9384, Train Steps/Sec: 1.12 + 21%|████████████████████████████▏ | 18774/89905 [5:59:19<17:27:39, 1.13it/s][2025-04-28 16:24:22] (step=0018775) Train Loss: 5.9170, Train Steps/Sec: 1.12 + 21%|████████████████████████████▏ | 18799/89905 [5:59:41<17:28:52, 1.13it/s][2025-04-28 16:24:45] (step=0018800) Train Loss: 5.9049, Train Steps/Sec: 1.12 + 21%|████████████████████████████▎ | 18824/89905 [6:00:03<18:05:18, 1.09it/s][2025-04-28 16:25:07] (step=0018825) Train Loss: 5.9064, Train Steps/Sec: 1.12 + 21%|████████████████████████████▎ | 18849/89905 [6:00:26<17:44:36, 1.11it/s][2025-04-28 16:25:29] (step=0018850) Train Loss: 5.9130, Train Steps/Sec: 1.12 + 21%|████████████████████████████▎ | 18874/89905 [6:00:48<17:40:19, 1.12it/s][2025-04-28 16:25:51] (step=0018875) Train Loss: 5.8333, Train Steps/Sec: 1.12 + 21%|████████████████████████████▍ | 18899/89905 [6:01:10<17:35:22, 1.12it/s][2025-04-28 16:26:14] (step=0018900) Train Loss: 5.9168, Train Steps/Sec: 1.12 + 21%|████████████████████████████▍ | 18924/89905 [6:01:33<17:30:44, 1.13it/s][2025-04-28 16:26:36] (step=0018925) Train Loss: 5.9400, Train Steps/Sec: 1.12 + 21%|████████████████████████████▍ | 18949/89905 [6:01:55<17:27:38, 1.13it/s][2025-04-28 16:26:58] (step=0018950) Train Loss: 5.9034, Train Steps/Sec: 1.12 + 21%|████████████████████████████▍ | 18974/89905 [6:02:17<17:23:15, 1.13it/s][2025-04-28 16:27:21] (step=0018975) Train Loss: 5.8981, Train Steps/Sec: 1.12 + 21%|████████████████████████████▌ | 18999/89905 [6:02:40<17:29:01, 1.13it/s][2025-04-28 16:27:43] (step=0019000) Train Loss: 5.8830, Train Steps/Sec: 1.11 + 21%|████████████████████████████▌ | 19024/89905 [6:03:02<17:55:19, 1.10it/s][2025-04-28 16:28:05] (step=0019025) Train Loss: 5.8705, Train Steps/Sec: 1.12 + 21%|████████████████████████████▌ | 19049/89905 [6:03:24<17:43:05, 1.11it/s][2025-04-28 16:28:28] (step=0019050) Train Loss: 5.9037, Train Steps/Sec: 1.12 + 21%|████████████████████████████▋ | 19074/89905 [6:03:47<17:33:55, 1.12it/s][2025-04-28 16:28:50] (step=0019075) Train Loss: 5.8579, Train Steps/Sec: 1.12 + 21%|████████████████████████████▋ | 19099/89905 [6:04:09<17:31:17, 1.12it/s][2025-04-28 16:29:12] (step=0019100) Train Loss: 5.9198, Train Steps/Sec: 1.12 + 21%|████████████████████████████▋ | 19124/89905 [6:04:31<17:30:14, 1.12it/s][2025-04-28 16:29:35] (step=0019125) Train Loss: 5.8713, Train Steps/Sec: 1.12 + 21%|████████████████████████████▊ | 19149/89905 [6:04:54<17:24:44, 1.13it/s][2025-04-28 16:29:57] (step=0019150) Train Loss: 5.8841, Train Steps/Sec: 1.12 + 21%|████████████████████████████▊ | 19174/89905 [6:05:43<30:07:41, 1.53s/it][2025-04-28 16:30:47] (step=0019175) Train Loss: 5.9350, Train Steps/Sec: 0.51 + 21%|████████████████████████████▊ | 19199/89905 [6:06:35<19:03:13, 1.03it/s][2025-04-28 16:31:38] (step=0019200) Train Loss: 5.8646, Train Steps/Sec: 0.49 + 21%|████████████████████████████▊ | 19224/89905 [6:07:06<36:21:36, 1.85s/it][2025-04-28 16:32:09] (step=0019225) Train Loss: 5.8698, Train Steps/Sec: 0.80 + 21%|████████████████████████████▉ | 19249/89905 [6:07:28<17:37:50, 1.11it/s][2025-04-28 16:32:32] (step=0019250) Train Loss: 5.9725, Train Steps/Sec: 1.12 + 21%|████████████████████████████▉ | 19274/89905 [6:07:51<17:32:21, 1.12it/s][2025-04-28 16:32:54] (step=0019275) Train Loss: 5.8840, Train Steps/Sec: 1.12 + 21%|████████████████████████████▉ | 19299/89905 [6:08:13<17:28:20, 1.12it/s][2025-04-28 16:33:16] (step=0019300) Train Loss: 5.8247, Train Steps/Sec: 1.12 + 21%|█████████████████████████████ | 19324/89905 [6:08:35<17:23:35, 1.13it/s][2025-04-28 16:33:39] (step=0019325) Train Loss: 5.8060, Train Steps/Sec: 1.12 + 22%|█████████████████████████████ | 19349/89905 [6:08:58<17:18:23, 1.13it/s][2025-04-28 16:34:01] (step=0019350) Train Loss: 5.9428, Train Steps/Sec: 1.12 + 22%|█████████████████████████████ | 19374/89905 [6:09:20<17:17:21, 1.13it/s][2025-04-28 16:34:23] (step=0019375) Train Loss: 5.8445, Train Steps/Sec: 1.12 + 22%|█████████████████████████████▏ | 19399/89905 [6:09:42<17:14:13, 1.14it/s][2025-04-28 16:34:46] (step=0019400) Train Loss: 5.8688, Train Steps/Sec: 1.12 + 22%|█████████████████████████████▏ | 19424/89905 [6:10:05<17:44:26, 1.10it/s][2025-04-28 16:35:08] (step=0019425) Train Loss: 5.8530, Train Steps/Sec: 1.12 + 22%|█████████████████████████████▏ | 19449/89905 [6:10:27<17:39:41, 1.11it/s][2025-04-28 16:35:30] (step=0019450) Train Loss: 5.9125, Train Steps/Sec: 1.12 + 22%|█████████████████████████████▏ | 19474/89905 [6:10:49<17:28:05, 1.12it/s][2025-04-28 16:35:53] (step=0019475) Train Loss: 5.8960, Train Steps/Sec: 1.12 + 22%|█████████████████████████████▎ | 19499/89905 [6:11:11<17:26:30, 1.12it/s][2025-04-28 16:36:15] (step=0019500) Train Loss: 5.8947, Train Steps/Sec: 1.12 + 22%|█████████████████████████████▎ | 19524/89905 [6:11:34<17:24:57, 1.12it/s][2025-04-28 16:36:37] (step=0019525) Train Loss: 5.8772, Train Steps/Sec: 1.12 + 22%|█████████████████████████████▎ | 19549/89905 [6:11:56<17:16:57, 1.13it/s][2025-04-28 16:36:59] (step=0019550) Train Loss: 5.9241, Train Steps/Sec: 1.12 + 22%|█████████████████████████████▍ | 19574/89905 [6:12:18<17:17:07, 1.13it/s][2025-04-28 16:37:22] (step=0019575) Train Loss: 5.9176, Train Steps/Sec: 1.12 + 22%|█████████████████████████████▍ | 19599/89905 [6:12:41<17:14:31, 1.13it/s][2025-04-28 16:37:44] (step=0019600) Train Loss: 5.9039, Train Steps/Sec: 1.12 + 22%|█████████████████████████████▍ | 19624/89905 [6:13:03<17:45:46, 1.10it/s][2025-04-28 16:38:06] (step=0019625) Train Loss: 5.8247, Train Steps/Sec: 1.12 + 22%|█████████████████████████████▌ | 19649/89905 [6:13:25<17:35:47, 1.11it/s][2025-04-28 16:38:29] (step=0019650) Train Loss: 5.8421, Train Steps/Sec: 1.12 + 22%|█████████████████████████████▌ | 19674/89905 [6:13:47<17:26:13, 1.12it/s][2025-04-28 16:38:51] (step=0019675) Train Loss: 5.9632, Train Steps/Sec: 1.12 + 22%|█████████████████████████████▌ | 19699/89905 [6:14:10<17:24:28, 1.12it/s][2025-04-28 16:39:13] (step=0019700) Train Loss: 5.8522, Train Steps/Sec: 1.12 + 22%|█████████████████████████████▌ | 19724/89905 [6:14:32<17:16:28, 1.13it/s][2025-04-28 16:39:35] (step=0019725) Train Loss: 5.8771, Train Steps/Sec: 1.12 + 22%|█████████████████████████████▋ | 19749/89905 [6:14:54<17:22:58, 1.12it/s][2025-04-28 16:39:58] (step=0019750) Train Loss: 5.8460, Train Steps/Sec: 1.12 + 22%|█████████████████████████████▋ | 19774/89905 [6:15:17<17:17:22, 1.13it/s][2025-04-28 16:40:20] (step=0019775) Train Loss: 5.8805, Train Steps/Sec: 1.12 + 22%|█████████████████████████████▋ | 19799/89905 [6:15:39<17:11:02, 1.13it/s][2025-04-28 16:40:42] (step=0019800) Train Loss: 5.9148, Train Steps/Sec: 1.12 + 22%|█████████████████████████████▊ | 19824/89905 [6:16:01<17:44:32, 1.10it/s][2025-04-28 16:41:05] (step=0019825) Train Loss: 5.8743, Train Steps/Sec: 1.12 + 22%|█████████████████████████████▊ | 19849/89905 [6:16:24<17:30:06, 1.11it/s][2025-04-28 16:41:27] (step=0019850) Train Loss: 5.7941, Train Steps/Sec: 1.12 + 22%|█████████████████████████████▊ | 19874/89905 [6:16:46<17:23:27, 1.12it/s][2025-04-28 16:41:49] (step=0019875) Train Loss: 5.9963, Train Steps/Sec: 1.12 + 22%|█████████████████████████████▉ | 19899/89905 [6:17:08<17:17:08, 1.12it/s][2025-04-28 16:42:12] (step=0019900) Train Loss: 5.8847, Train Steps/Sec: 1.12 + 22%|█████████████████████████████▉ | 19924/89905 [6:17:30<17:16:45, 1.12it/s][2025-04-28 16:42:34] (step=0019925) Train Loss: 5.8934, Train Steps/Sec: 1.12 + 22%|█████████████████████████████▉ | 19949/89905 [6:17:53<17:11:44, 1.13it/s][2025-04-28 16:42:56] (step=0019950) Train Loss: 5.8758, Train Steps/Sec: 1.12 + 22%|█████████████████████████████▉ | 19974/89905 [6:18:15<17:07:50, 1.13it/s][2025-04-28 16:43:18] (step=0019975) Train Loss: 5.8940, Train Steps/Sec: 1.12 + 22%|██████████████████████████████ | 19999/89905 [6:18:37<17:14:09, 1.13it/s][2025-04-28 16:43:41] (step=0020000) Train Loss: 5.9034, Train Steps/Sec: 1.12 +You are using a model of type blip-2 to instantiate a model of type instructblip. This is not supported for all configurations of models and can yield errors. +[2025-04-28 16:43:41] text_config is None. Initializing the text config with default values (`OPTConfig`). +model_name_or_path:/tmp/haozhezhao/model/blip2-flan-t5-xl +image_place_holder: +tokenizer length after expend 32101 +tokenizer length before expend 32100 +100%|███████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████| 6/6 [06:00<00:00, 60.06s/it] +[2025-04-28 16:50:59] Finish Eval in 20000 steps...████████████████████████████████████████████████████████████████████| 6/6 [05:59<00:00, 59.91s/it] +processing left-padding... +processing left-padding... +processing left-padding... +processing left-padding... +processing left-padding... +processing left-padding... +[2025-04-28 16:51:19] Saved checkpoint to checkpoint/X2I_UltraEdiit_fourmask_LLava_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_filtered_objects_4mask_1280_context_after_stage1_5e4/000-GPT-XL/checkpoints/0020000.pt +[2025-04-28 16:51:21] Removed old checkpoint: checkpoint/X2I_UltraEdiit_fourmask_LLava_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_filtered_objects_4mask_1280_context_after_stage1_5e4/000-GPT-XL/checkpoints/0018000.pt + 22%|██████████████████████████████ | 20024/89905 [6:26:40<18:14:02, 1.06it/s][2025-04-28 16:51:43] (step=0020025) Train Loss: 5.8259, Train Steps/Sec: 0.05 + 22%|██████████████████████████████ | 20049/89905 [6:27:02<17:28:09, 1.11it/s][2025-04-28 16:52:06] (step=0020050) Train Loss: 5.8313, Train Steps/Sec: 1.12 + 22%|██████████████████████████████▏ | 20074/89905 [6:27:25<17:22:44, 1.12it/s][2025-04-28 16:52:28] (step=0020075) Train Loss: 5.8557, Train Steps/Sec: 1.12 + 22%|██████████████████████████████▏ | 20099/89905 [6:27:47<17:18:53, 1.12it/s][2025-04-28 16:52:50] (step=0020100) Train Loss: 5.8188, Train Steps/Sec: 1.12 + 22%|██████████████████████████████▏ | 20124/89905 [6:28:09<17:15:22, 1.12it/s][2025-04-28 16:53:13] (step=0020125) Train Loss: 5.9005, Train Steps/Sec: 1.12 + 22%|██████████████████████████████▎ | 20149/89905 [6:28:32<17:12:37, 1.13it/s][2025-04-28 16:53:35] (step=0020150) Train Loss: 5.8278, Train Steps/Sec: 1.11 + 22%|██████████████████████████████▎ | 20174/89905 [6:28:54<17:14:02, 1.12it/s][2025-04-28 16:53:58] (step=0020175) Train Loss: 5.8725, Train Steps/Sec: 1.12 + 22%|██████████████████████████████▎ | 20199/89905 [6:29:17<17:09:44, 1.13it/s][2025-04-28 16:54:20] (step=0020200) Train Loss: 5.9271, Train Steps/Sec: 1.11 + 22%|██████████████████████████████▎ | 20224/89905 [6:29:39<17:46:43, 1.09it/s][2025-04-28 16:54:43] (step=0020225) Train Loss: 5.8309, Train Steps/Sec: 1.11 + 23%|██████████████████████████████▍ | 20249/89905 [6:30:01<17:28:03, 1.11it/s][2025-04-28 16:55:05] (step=0020250) Train Loss: 5.8869, Train Steps/Sec: 1.12 + 23%|██████████████████████████████▍ | 20274/89905 [6:30:24<17:21:42, 1.11it/s][2025-04-28 16:55:27] (step=0020275) Train Loss: 5.9066, Train Steps/Sec: 1.12 + 23%|██████████████████████████████▍ | 20299/89905 [6:30:46<17:12:57, 1.12it/s][2025-04-28 16:55:50] (step=0020300) Train Loss: 5.9052, Train Steps/Sec: 1.12 + 23%|██████████████████████████████▌ | 20324/89905 [6:31:09<17:15:05, 1.12it/s][2025-04-28 16:56:12] (step=0020325) Train Loss: 5.8845, Train Steps/Sec: 1.12 + 23%|██████████████████████████████▌ | 20349/89905 [6:31:31<17:11:10, 1.12it/s][2025-04-28 16:56:34] (step=0020350) Train Loss: 5.8953, Train Steps/Sec: 1.12 + 23%|██████████████████████████████▌ | 20374/89905 [6:31:53<17:06:49, 1.13it/s][2025-04-28 16:56:57] (step=0020375) Train Loss: 5.8841, Train Steps/Sec: 1.12 + 23%|██████████████████████████████▋ | 20399/89905 [6:32:16<17:03:21, 1.13it/s][2025-04-28 16:57:19] (step=0020400) Train Loss: 5.8900, Train Steps/Sec: 1.11 + 23%|██████████████████████████████▋ | 20424/89905 [6:32:38<17:44:00, 1.09it/s][2025-04-28 16:57:42] (step=0020425) Train Loss: 5.8234, Train Steps/Sec: 1.12 + 23%|██████████████████████████████▋ | 20449/89905 [6:33:00<17:22:16, 1.11it/s][2025-04-28 16:58:04] (step=0020450) Train Loss: 5.8128, Train Steps/Sec: 1.12 + 23%|██████████████████████████████▋ | 20474/89905 [6:33:23<17:21:50, 1.11it/s][2025-04-28 16:58:26] (step=0020475) Train Loss: 5.9233, Train Steps/Sec: 1.12 + 23%|██████████████████████████████▊ | 20499/89905 [6:33:45<17:11:23, 1.12it/s][2025-04-28 16:58:49] (step=0020500) Train Loss: 5.8407, Train Steps/Sec: 1.12 + 23%|██████████████████████████████▊ | 20524/89905 [6:34:08<17:11:00, 1.12it/s][2025-04-28 16:59:11] (step=0020525) Train Loss: 5.8937, Train Steps/Sec: 1.11 + 23%|██████████████████████████████▊ | 20549/89905 [6:34:30<17:15:11, 1.12it/s][2025-04-28 16:59:33] (step=0020550) Train Loss: 5.8181, Train Steps/Sec: 1.12 + 23%|██████████████████████████████▉ | 20574/89905 [6:34:52<17:03:51, 1.13it/s][2025-04-28 16:59:56] (step=0020575) Train Loss: 5.9061, Train Steps/Sec: 1.12 + 23%|██████████████████████████████▉ | 20599/89905 [6:35:15<17:06:03, 1.13it/s][2025-04-28 17:00:18] (step=0020600) Train Loss: 5.9097, Train Steps/Sec: 1.11 + 23%|██████████████████████████████▉ | 20624/89905 [6:35:37<17:31:43, 1.10it/s][2025-04-28 17:00:40] (step=0020625) Train Loss: 5.8740, Train Steps/Sec: 1.12 + 23%|███████████████████████████████ | 20649/89905 [6:35:59<17:22:39, 1.11it/s][2025-04-28 17:01:03] (step=0020650) Train Loss: 5.7996, Train Steps/Sec: 1.12 + 23%|███████████████████████████████ | 20674/89905 [6:36:22<17:08:12, 1.12it/s][2025-04-28 17:01:25] (step=0020675) Train Loss: 5.8285, Train Steps/Sec: 1.12 + 23%|███████████████████████████████ | 20699/89905 [6:36:44<17:07:23, 1.12it/s][2025-04-28 17:01:47] (step=0020700) Train Loss: 5.9078, Train Steps/Sec: 1.12 + 23%|███████████████████████████████ | 20724/89905 [6:37:06<17:01:51, 1.13it/s][2025-04-28 17:02:10] (step=0020725) Train Loss: 5.8164, Train Steps/Sec: 1.12 + 23%|███████████████████████████████▏ | 20749/89905 [6:37:29<17:01:45, 1.13it/s][2025-04-28 17:02:32] (step=0020750) Train Loss: 5.8457, Train Steps/Sec: 1.12 + 23%|███████████████████████████████▏ | 20774/89905 [6:37:51<16:58:10, 1.13it/s][2025-04-28 17:02:54] (step=0020775) Train Loss: 5.8140, Train Steps/Sec: 1.12 + 23%|███████████████████████████████▏ | 20799/89905 [6:38:13<16:59:42, 1.13it/s][2025-04-28 17:03:17] (step=0020800) Train Loss: 5.9246, Train Steps/Sec: 1.11 + 23%|███████████████████████████████▎ | 20824/89905 [6:38:36<17:25:44, 1.10it/s][2025-04-28 17:03:39] (step=0020825) Train Loss: 5.8289, Train Steps/Sec: 1.12 + 23%|███████████████████████████████▎ | 20849/89905 [6:38:58<17:19:39, 1.11it/s][2025-04-28 17:04:01] (step=0020850) Train Loss: 5.8003, Train Steps/Sec: 1.12 + 23%|███████████████████████████████▎ | 20874/89905 [6:39:20<17:06:14, 1.12it/s][2025-04-28 17:04:24] (step=0020875) Train Loss: 5.8321, Train Steps/Sec: 1.12 + 23%|███████████████████████████████▍ | 20899/89905 [6:39:43<17:08:28, 1.12it/s][2025-04-28 17:04:46] (step=0020900) Train Loss: 5.9103, Train Steps/Sec: 1.12 + 23%|███████████████████████████████▍ | 20924/89905 [6:40:05<16:58:56, 1.13it/s][2025-04-28 17:05:08] (step=0020925) Train Loss: 5.8175, Train Steps/Sec: 1.12 + 23%|███████████████████████████████▍ | 20949/89905 [6:40:27<16:59:08, 1.13it/s][2025-04-28 17:05:31] (step=0020950) Train Loss: 5.8762, Train Steps/Sec: 1.12 + 23%|███████████████████████████████▍ | 20974/89905 [6:40:50<17:01:18, 1.12it/s][2025-04-28 17:05:53] (step=0020975) Train Loss: 5.8033, Train Steps/Sec: 1.12 + 23%|███████████████████████████████▌ | 20999/89905 [6:41:12<16:51:43, 1.14it/s][2025-04-28 17:06:15] (step=0021000) Train Loss: 5.8838, Train Steps/Sec: 1.12 + 23%|███████████████████████████████▌ | 21024/89905 [6:41:34<17:29:42, 1.09it/s][2025-04-28 17:06:38] (step=0021025) Train Loss: 5.9170, Train Steps/Sec: 1.12 + 23%|███████████████████████████████▌ | 21049/89905 [6:41:56<17:08:43, 1.12it/s][2025-04-28 17:07:00] (step=0021050) Train Loss: 5.7975, Train Steps/Sec: 1.12 + 23%|███████████████████████████████▋ | 21074/89905 [6:42:19<17:06:06, 1.12it/s][2025-04-28 17:07:22] (step=0021075) Train Loss: 5.8656, Train Steps/Sec: 1.12 + 23%|███████████████████████████████▋ | 21099/89905 [6:42:41<17:02:23, 1.12it/s][2025-04-28 17:07:44] (step=0021100) Train Loss: 5.8788, Train Steps/Sec: 1.12 + 23%|███████████████████████████████▋ | 21124/89905 [6:43:03<17:01:59, 1.12it/s][2025-04-28 17:08:07] (step=0021125) Train Loss: 5.9253, Train Steps/Sec: 1.12 + 24%|███████████████████████████████▊ | 21149/89905 [6:43:26<16:53:52, 1.13it/s][2025-04-28 17:08:29] (step=0021150) Train Loss: 5.8185, Train Steps/Sec: 1.12 + 24%|███████████████████████████████▊ | 21174/89905 [6:43:48<16:54:40, 1.13it/s][2025-04-28 17:08:51] (step=0021175) Train Loss: 5.8546, Train Steps/Sec: 1.12 + 24%|███████████████████████████████▊ | 21199/89905 [6:44:10<16:51:16, 1.13it/s][2025-04-28 17:09:14] (step=0021200) Train Loss: 5.8222, Train Steps/Sec: 1.11 + 24%|███████████████████████████████▊ | 21224/89905 [6:44:33<17:22:15, 1.10it/s][2025-04-28 17:09:36] (step=0021225) Train Loss: 5.8791, Train Steps/Sec: 1.12 + 24%|███████████████████████████████▉ | 21249/89905 [6:44:55<17:13:18, 1.11it/s][2025-04-28 17:09:58] (step=0021250) Train Loss: 5.8633, Train Steps/Sec: 1.12 + 24%|███████████████████████████████▉ | 21274/89905 [6:45:17<17:03:30, 1.12it/s][2025-04-28 17:10:21] (step=0021275) Train Loss: 5.8676, Train Steps/Sec: 1.12 + 24%|███████████████████████████████▉ | 21299/89905 [6:45:40<16:59:24, 1.12it/s][2025-04-28 17:10:43] (step=0021300) Train Loss: 5.8639, Train Steps/Sec: 1.12 + 24%|████████████████████████████████ | 21324/89905 [6:46:02<16:55:30, 1.13it/s][2025-04-28 17:11:05] (step=0021325) Train Loss: 5.7685, Train Steps/Sec: 1.12 + 24%|████████████████████████████████ | 21349/89905 [6:46:24<16:54:57, 1.13it/s][2025-04-28 17:11:28] (step=0021350) Train Loss: 5.8246, Train Steps/Sec: 1.12 + 24%|████████████████████████████████ | 21374/89905 [6:46:47<16:47:19, 1.13it/s][2025-04-28 17:11:50] (step=0021375) Train Loss: 5.7889, Train Steps/Sec: 1.12 + 24%|████████████████████████████████▏ | 21399/89905 [6:47:09<16:48:24, 1.13it/s][2025-04-28 17:12:12] (step=0021400) Train Loss: 5.8123, Train Steps/Sec: 1.11 + 24%|████████████████████████████████▏ | 21424/89905 [6:47:31<17:17:41, 1.10it/s][2025-04-28 17:12:35] (step=0021425) Train Loss: 5.7993, Train Steps/Sec: 1.12 + 24%|████████████████████████████████▏ | 21449/89905 [6:47:54<17:06:02, 1.11it/s][2025-04-28 17:12:57] (step=0021450) Train Loss: 5.8750, Train Steps/Sec: 1.12 + 24%|████████████████████████████████▏ | 21474/89905 [6:48:16<16:56:28, 1.12it/s][2025-04-28 17:13:19] (step=0021475) Train Loss: 5.8134, Train Steps/Sec: 1.12 + 24%|████████████████████████████████▎ | 21499/89905 [6:48:38<16:52:03, 1.13it/s][2025-04-28 17:13:42] (step=0021500) Train Loss: 5.8814, Train Steps/Sec: 1.12 + 24%|████████████████████████████████▎ | 21524/89905 [6:49:00<16:52:47, 1.13it/s][2025-04-28 17:14:04] (step=0021525) Train Loss: 5.8299, Train Steps/Sec: 1.12 + 24%|████████████████████████████████▎ | 21549/89905 [6:49:23<16:50:16, 1.13it/s][2025-04-28 17:14:26] (step=0021550) Train Loss: 5.7884, Train Steps/Sec: 1.12 + 24%|████████████████████████████████▍ | 21574/89905 [6:49:45<16:48:32, 1.13it/s][2025-04-28 17:14:49] (step=0021575) Train Loss: 5.8921, Train Steps/Sec: 1.12 + 24%|████████████████████████████████▍ | 21599/89905 [6:50:07<16:44:08, 1.13it/s][2025-04-28 17:15:11] (step=0021600) Train Loss: 5.9266, Train Steps/Sec: 1.12 + 24%|████████████████████████████████▍ | 21624/89905 [6:50:30<17:19:00, 1.10it/s][2025-04-28 17:15:33] (step=0021625) Train Loss: 5.8653, Train Steps/Sec: 1.12 + 24%|████████████████████████████████▌ | 21649/89905 [6:50:52<17:03:19, 1.11it/s][2025-04-28 17:15:56] (step=0021650) Train Loss: 5.8211, Train Steps/Sec: 1.12 + 24%|████████████████████████████████▌ | 21674/89905 [6:51:14<16:59:24, 1.12it/s][2025-04-28 17:16:18] (step=0021675) Train Loss: 5.8338, Train Steps/Sec: 1.12 + 24%|████████████████████████████████▌ | 21699/89905 [6:51:37<16:50:53, 1.12it/s][2025-04-28 17:16:40] (step=0021700) Train Loss: 5.7888, Train Steps/Sec: 1.12 + 24%|████████████████████████████████▌ | 21724/89905 [6:51:59<16:50:08, 1.12it/s][2025-04-28 17:17:02] (step=0021725) Train Loss: 5.8767, Train Steps/Sec: 1.12 + 24%|████████████████████████████████▋ | 21749/89905 [6:52:21<16:46:18, 1.13it/s][2025-04-28 17:17:25] (step=0021750) Train Loss: 5.9026, Train Steps/Sec: 1.12 + 24%|████████████████████████████████▋ | 21774/89905 [6:52:44<16:48:55, 1.13it/s][2025-04-28 17:17:47] (step=0021775) Train Loss: 5.8991, Train Steps/Sec: 1.12 + 24%|████████████████████████████████▋ | 21799/89905 [6:53:06<16:42:17, 1.13it/s][2025-04-28 17:18:09] (step=0021800) Train Loss: 5.8847, Train Steps/Sec: 1.12 + 24%|████████████████████████████████▊ | 21824/89905 [6:53:28<17:13:26, 1.10it/s][2025-04-28 17:18:32] (step=0021825) Train Loss: 5.7940, Train Steps/Sec: 1.12 + 24%|████████████████████████████████▊ | 21849/89905 [6:53:51<16:57:05, 1.12it/s][2025-04-28 17:18:54] (step=0021850) Train Loss: 5.8380, Train Steps/Sec: 1.12 + 24%|████████████████████████████████▊ | 21874/89905 [6:54:13<16:51:12, 1.12it/s][2025-04-28 17:19:16] (step=0021875) Train Loss: 5.7468, Train Steps/Sec: 1.12 + 24%|████████████████████████████████▉ | 21899/89905 [6:54:35<16:48:28, 1.12it/s][2025-04-28 17:19:39] (step=0021900) Train Loss: 5.7818, Train Steps/Sec: 1.12 + 24%|████████████████████████████████▉ | 21924/89905 [6:54:57<16:45:07, 1.13it/s][2025-04-28 17:20:01] (step=0021925) Train Loss: 5.8480, Train Steps/Sec: 1.12 + 24%|████████████████████████████████▉ | 21949/89905 [6:55:20<16:40:55, 1.13it/s][2025-04-28 17:20:23] (step=0021950) Train Loss: 5.7643, Train Steps/Sec: 1.12 + 24%|████████████████████████████████▉ | 21974/89905 [6:55:42<16:41:10, 1.13it/s][2025-04-28 17:20:45] (step=0021975) Train Loss: 5.8577, Train Steps/Sec: 1.12 + 24%|█████████████████████████████████ | 21999/89905 [6:56:04<16:40:04, 1.13it/s][2025-04-28 17:21:08] (step=0022000) Train Loss: 5.8168, Train Steps/Sec: 1.12 +You are using a model of type blip-2 to instantiate a model of type instructblip. This is not supported for all configurations of models and can yield errors. +[2025-04-28 17:21:08] text_config is None. Initializing the text config with default values (`OPTConfig`). +model_name_or_path:/tmp/haozhezhao/model/blip2-flan-t5-xl +image_place_holder: +tokenizer length after expend 32101 +tokenizer length before expend 32100 +100%|███████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████| 6/6 [06:00<00:00, 60.07s/it] +[2025-04-28 17:28:26] Finish Eval in 22000 steps...████████████████████████████████████████████████████████████████████| 6/6 [05:59<00:00, 59.87s/it] +processing left-padding... +processing left-padding... +processing left-padding... +processing left-padding... +processing left-padding... +processing left-padding... +[2025-04-28 17:28:46] Saved checkpoint to checkpoint/X2I_UltraEdiit_fourmask_LLava_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_filtered_objects_4mask_1280_context_after_stage1_5e4/000-GPT-XL/checkpoints/0022000.pt +[2025-04-28 17:28:48] Removed old checkpoint: checkpoint/X2I_UltraEdiit_fourmask_LLava_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_filtered_objects_4mask_1280_context_after_stage1_5e4/000-GPT-XL/checkpoints/0020000.pt + 24%|█████████████████████████████████ | 22024/89905 [7:04:07<17:44:31, 1.06it/s][2025-04-28 17:29:10] (step=0022025) Train Loss: 5.9128, Train Steps/Sec: 0.05 + 25%|█████████████████████████████████ | 22049/89905 [7:04:29<16:56:05, 1.11it/s][2025-04-28 17:29:33] (step=0022050) Train Loss: 5.7904, Train Steps/Sec: 1.12 + 25%|█████████████████████████████████▏ | 22074/89905 [7:04:52<16:50:44, 1.12it/s][2025-04-28 17:29:55] (step=0022075) Train Loss: 5.8722, Train Steps/Sec: 1.12 + 25%|█████████████████████████████████▏ | 22099/89905 [7:05:14<16:46:37, 1.12it/s][2025-04-28 17:30:17] (step=0022100) Train Loss: 5.8021, Train Steps/Sec: 1.12 + 25%|█████████████████████████████████▏ | 22124/89905 [7:05:36<16:41:41, 1.13it/s][2025-04-28 17:30:40] (step=0022125) Train Loss: 5.8196, Train Steps/Sec: 1.12 + 25%|█████████████████████████████████▎ | 22149/89905 [7:05:59<16:42:16, 1.13it/s][2025-04-28 17:31:02] (step=0022150) Train Loss: 5.8287, Train Steps/Sec: 1.12 + 25%|█████████████████████████████████▎ | 22174/89905 [7:06:21<16:35:52, 1.13it/s][2025-04-28 17:31:24] (step=0022175) Train Loss: 5.8001, Train Steps/Sec: 1.12 + 25%|█████████████████████████████████▎ | 22199/89905 [7:06:43<16:38:08, 1.13it/s][2025-04-28 17:31:47] (step=0022200) Train Loss: 5.8713, Train Steps/Sec: 1.12 + 25%|█████████████████████████████████▎ | 22224/89905 [7:07:06<17:09:39, 1.10it/s][2025-04-28 17:32:09] (step=0022225) Train Loss: 5.8454, Train Steps/Sec: 1.12 + 25%|█████████████████████████████████▍ | 22249/89905 [7:07:28<16:52:20, 1.11it/s][2025-04-28 17:32:31] (step=0022250) Train Loss: 5.8816, Train Steps/Sec: 1.12 + 25%|█████████████████████████████████▍ | 22274/89905 [7:07:50<16:47:27, 1.12it/s][2025-04-28 17:32:54] (step=0022275) Train Loss: 5.8939, Train Steps/Sec: 1.12 + 25%|█████████████████████████████████▍ | 22299/89905 [7:08:12<16:43:09, 1.12it/s][2025-04-28 17:33:16] (step=0022300) Train Loss: 5.8071, Train Steps/Sec: 1.12 + 25%|█████████████████████████████████▌ | 22324/89905 [7:08:35<16:38:42, 1.13it/s][2025-04-28 17:33:38] (step=0022325) Train Loss: 5.7634, Train Steps/Sec: 1.12 + 25%|█████████████████████████████████▌ | 22349/89905 [7:08:57<16:36:21, 1.13it/s][2025-04-28 17:34:00] (step=0022350) Train Loss: 5.8143, Train Steps/Sec: 1.12 + 25%|█████████████████████████████████▌ | 22374/89905 [7:09:19<16:33:13, 1.13it/s][2025-04-28 17:34:23] (step=0022375) Train Loss: 5.8216, Train Steps/Sec: 1.12 + 25%|█████████████████████████████████▋ | 22399/89905 [7:09:42<16:36:19, 1.13it/s][2025-04-28 17:34:45] (step=0022400) Train Loss: 5.8641, Train Steps/Sec: 1.11 + 25%|█████████████████████████████████▋ | 22424/89905 [7:10:04<17:01:40, 1.10it/s][2025-04-28 17:35:08] (step=0022425) Train Loss: 5.7800, Train Steps/Sec: 1.12 + 25%|█████████████████████████████████▋ | 22449/89905 [7:10:26<16:53:14, 1.11it/s][2025-04-28 17:35:30] (step=0022450) Train Loss: 5.8380, Train Steps/Sec: 1.12 + 25%|█████████████████████████████████▋ | 22474/89905 [7:10:49<16:45:15, 1.12it/s][2025-04-28 17:35:52] (step=0022475) Train Loss: 5.8105, Train Steps/Sec: 1.12 + 25%|█████████████████████████████████▊ | 22499/89905 [7:11:11<16:41:55, 1.12it/s][2025-04-28 17:36:14] (step=0022500) Train Loss: 5.8361, Train Steps/Sec: 1.12 + 25%|█████████████████████████████████▊ | 22524/89905 [7:11:33<16:39:34, 1.12it/s][2025-04-28 17:36:37] (step=0022525) Train Loss: 5.8642, Train Steps/Sec: 1.12 + 25%|█████████████████████████████████▊ | 22549/89905 [7:11:56<16:31:33, 1.13it/s][2025-04-28 17:36:59] (step=0022550) Train Loss: 5.8234, Train Steps/Sec: 1.12 + 25%|█████████████████████████████████▉ | 22574/89905 [7:12:18<16:31:43, 1.13it/s][2025-04-28 17:37:21] (step=0022575) Train Loss: 5.8226, Train Steps/Sec: 1.12 + 25%|█████████████████████████████████▉ | 22599/89905 [7:12:40<16:27:19, 1.14it/s][2025-04-28 17:37:44] (step=0022600) Train Loss: 5.8856, Train Steps/Sec: 1.12 + 25%|█████████████████████████████████▉ | 22624/89905 [7:13:03<17:00:27, 1.10it/s][2025-04-28 17:38:06] (step=0022625) Train Loss: 5.7675, Train Steps/Sec: 1.12 + 25%|██████████████████████████████████ | 22649/89905 [7:13:25<16:45:39, 1.11it/s][2025-04-28 17:38:28] (step=0022650) Train Loss: 5.7974, Train Steps/Sec: 1.12 + 25%|██████████████████████████████████ | 22674/89905 [7:13:47<16:42:07, 1.12it/s][2025-04-28 17:38:51] (step=0022675) Train Loss: 5.8495, Train Steps/Sec: 1.12 + 25%|██████████████████████████████████ | 22699/89905 [7:14:09<16:41:15, 1.12it/s][2025-04-28 17:39:13] (step=0022700) Train Loss: 5.8789, Train Steps/Sec: 1.12 + 25%|██████████████████████████████████ | 22724/89905 [7:14:32<16:33:32, 1.13it/s][2025-04-28 17:39:35] (step=0022725) Train Loss: 5.8057, Train Steps/Sec: 1.12 + 25%|██████████████████████████████████▏ | 22749/89905 [7:15:32<43:27:02, 2.33s/it][2025-04-28 17:40:36] (step=0022750) Train Loss: 5.8202, Train Steps/Sec: 0.41 + 25%|██████████████████████████████████▏ | 22774/89905 [7:15:55<16:33:11, 1.13it/s][2025-04-28 17:40:58] (step=0022775) Train Loss: 5.8007, Train Steps/Sec: 1.12 + 25%|██████████████████████████████████▏ | 22799/89905 [7:16:26<17:09:58, 1.09it/s][2025-04-28 17:41:29] (step=0022800) Train Loss: 5.8185, Train Steps/Sec: 0.79 + 25%|██████████████████████████████████▎ | 22824/89905 [7:16:48<17:02:58, 1.09it/s][2025-04-28 17:41:52] (step=0022825) Train Loss: 5.8626, Train Steps/Sec: 1.12 + 25%|██████████████████████████████████▎ | 22849/89905 [7:17:11<16:41:43, 1.12it/s][2025-04-28 17:42:14] (step=0022850) Train Loss: 5.7999, Train Steps/Sec: 1.12 + 25%|██████████████████████████████████▎ | 22874/89905 [7:17:33<16:40:06, 1.12it/s][2025-04-28 17:42:36] (step=0022875) Train Loss: 5.8938, Train Steps/Sec: 1.12 + 25%|██████████████████████████████████▍ | 22899/89905 [7:17:55<16:36:43, 1.12it/s][2025-04-28 17:42:59] (step=0022900) Train Loss: 5.9044, Train Steps/Sec: 1.12 + 25%|██████████████████████████████████▍ | 22924/89905 [7:18:18<16:33:30, 1.12it/s][2025-04-28 17:43:21] (step=0022925) Train Loss: 5.8231, Train Steps/Sec: 1.12 + 26%|██████████████████████████████████▍ | 22949/89905 [7:18:40<16:27:09, 1.13it/s][2025-04-28 17:43:43] (step=0022950) Train Loss: 5.8349, Train Steps/Sec: 1.12 + 26%|██████████████████████████████████▍ | 22974/89905 [7:19:02<16:29:07, 1.13it/s][2025-04-28 17:44:06] (step=0022975) Train Loss: 5.7947, Train Steps/Sec: 1.12 + 26%|██████████████████████████████████▌ | 22999/89905 [7:19:24<16:24:09, 1.13it/s][2025-04-28 17:44:28] (step=0023000) Train Loss: 5.7859, Train Steps/Sec: 1.12 + 26%|██████████████████████████████████▌ | 23024/89905 [7:19:47<16:55:50, 1.10it/s][2025-04-28 17:44:50] (step=0023025) Train Loss: 5.8042, Train Steps/Sec: 1.12 + 26%|██████████████████████████████████▌ | 23049/89905 [7:20:09<16:40:33, 1.11it/s][2025-04-28 17:45:13] (step=0023050) Train Loss: 5.8531, Train Steps/Sec: 1.12 + 26%|██████████████████████████████████▋ | 23074/89905 [7:20:31<16:38:05, 1.12it/s][2025-04-28 17:45:35] (step=0023075) Train Loss: 5.8250, Train Steps/Sec: 1.12 + 26%|██████████████████████████████████▋ | 23099/89905 [7:20:54<16:30:55, 1.12it/s][2025-04-28 17:45:57] (step=0023100) Train Loss: 5.8150, Train Steps/Sec: 1.12 + 26%|██████████████████████████████████▋ | 23124/89905 [7:21:16<16:27:52, 1.13it/s][2025-04-28 17:46:19] (step=0023125) Train Loss: 5.8239, Train Steps/Sec: 1.12 + 26%|██████████████████████████████████▊ | 23149/89905 [7:21:38<16:22:50, 1.13it/s][2025-04-28 17:46:42] (step=0023150) Train Loss: 5.8663, Train Steps/Sec: 1.12 + 26%|██████████████████████████████████▊ | 23174/89905 [7:22:01<16:19:50, 1.14it/s][2025-04-28 17:47:04] (step=0023175) Train Loss: 5.8841, Train Steps/Sec: 1.12 + 26%|██████████████████████████████████▊ | 23199/89905 [7:22:23<16:23:22, 1.13it/s][2025-04-28 17:47:26] (step=0023200) Train Loss: 5.8761, Train Steps/Sec: 1.12 + 26%|██████████████████████████████████▊ | 23224/89905 [7:22:45<16:51:36, 1.10it/s][2025-04-28 17:47:49] (step=0023225) Train Loss: 5.8085, Train Steps/Sec: 1.12 + 26%|██████████████████████████████████▉ | 23249/89905 [7:23:08<16:39:34, 1.11it/s][2025-04-28 17:48:11] (step=0023250) Train Loss: 5.8137, Train Steps/Sec: 1.12 + 26%|██████████████████████████████████▉ | 23274/89905 [7:23:30<16:31:40, 1.12it/s][2025-04-28 17:48:33] (step=0023275) Train Loss: 5.7950, Train Steps/Sec: 1.12 + 26%|██████████████████████████████████▉ | 23299/89905 [7:23:52<16:24:29, 1.13it/s][2025-04-28 17:48:56] (step=0023300) Train Loss: 5.7640, Train Steps/Sec: 1.12 + 26%|███████████████████████████████████ | 23324/89905 [7:24:14<16:20:25, 1.13it/s][2025-04-28 17:49:18] (step=0023325) Train Loss: 5.7652, Train Steps/Sec: 1.12 + 26%|███████████████████████████████████ | 23349/89905 [7:24:37<16:22:14, 1.13it/s][2025-04-28 17:49:40] (step=0023350) Train Loss: 5.8488, Train Steps/Sec: 1.12 + 26%|███████████████████████████████████ | 23374/89905 [7:24:59<16:22:06, 1.13it/s][2025-04-28 17:50:02] (step=0023375) Train Loss: 5.7958, Train Steps/Sec: 1.12 + 26%|███████████████████████████████████▏ | 23399/89905 [7:25:21<16:19:42, 1.13it/s][2025-04-28 17:50:25] (step=0023400) Train Loss: 5.7392, Train Steps/Sec: 1.12 + 26%|███████████████████████████████████▏ | 23424/89905 [7:25:44<16:49:01, 1.10it/s][2025-04-28 17:50:47] (step=0023425) Train Loss: 5.7877, Train Steps/Sec: 1.12 + 26%|███████████████████████████████████▏ | 23449/89905 [7:26:06<16:36:35, 1.11it/s][2025-04-28 17:51:09] (step=0023450) Train Loss: 5.8280, Train Steps/Sec: 1.12 + 26%|███████████████████████████████████▏ | 23474/89905 [7:26:28<16:30:02, 1.12it/s][2025-04-28 17:51:32] (step=0023475) Train Loss: 5.8364, Train Steps/Sec: 1.12 + 26%|███████████████████████████████████▎ | 23499/89905 [7:26:51<16:24:21, 1.12it/s][2025-04-28 17:51:54] (step=0023500) Train Loss: 5.7993, Train Steps/Sec: 1.12 + 26%|███████████████████████████████████▎ | 23524/89905 [7:27:13<16:22:05, 1.13it/s][2025-04-28 17:52:16] (step=0023525) Train Loss: 5.8390, Train Steps/Sec: 1.12 + 26%|███████████████████████████████████▎ | 23549/89905 [7:27:35<16:19:24, 1.13it/s][2025-04-28 17:52:39] (step=0023550) Train Loss: 5.9050, Train Steps/Sec: 1.12 + 26%|███████████████████████████████████▍ | 23574/89905 [7:27:57<16:14:58, 1.13it/s][2025-04-28 17:53:01] (step=0023575) Train Loss: 5.8076, Train Steps/Sec: 1.12 + 26%|███████████████████████████████████▍ | 23599/89905 [7:28:20<16:17:29, 1.13it/s][2025-04-28 17:53:23] (step=0023600) Train Loss: 5.7562, Train Steps/Sec: 1.12 + 26%|███████████████████████████████████▍ | 23624/89905 [7:28:42<16:46:00, 1.10it/s][2025-04-28 17:53:46] (step=0023625) Train Loss: 5.8553, Train Steps/Sec: 1.12 + 26%|███████████████████████████████████▌ | 23649/89905 [7:29:04<16:33:01, 1.11it/s][2025-04-28 17:54:08] (step=0023650) Train Loss: 5.7517, Train Steps/Sec: 1.12 + 26%|███████████████████████████████████▌ | 23674/89905 [7:29:27<16:28:59, 1.12it/s][2025-04-28 17:54:30] (step=0023675) Train Loss: 5.8674, Train Steps/Sec: 1.12 + 26%|███████████████████████████████████▌ | 23699/89905 [7:29:49<16:25:09, 1.12it/s][2025-04-28 17:54:52] (step=0023700) Train Loss: 5.8613, Train Steps/Sec: 1.12 + 26%|███████████████████████████████████▌ | 23724/89905 [7:30:11<16:19:55, 1.13it/s][2025-04-28 17:55:15] (step=0023725) Train Loss: 5.8403, Train Steps/Sec: 1.12 + 26%|███████████████████████████████████▋ | 23749/89905 [7:30:34<16:16:13, 1.13it/s][2025-04-28 17:55:37] (step=0023750) Train Loss: 5.8107, Train Steps/Sec: 1.12 + 26%|███████████████████████████████████▋ | 23774/89905 [7:30:56<16:13:41, 1.13it/s][2025-04-28 17:55:59] (step=0023775) Train Loss: 5.8560, Train Steps/Sec: 1.12 + 26%|███████████████████████████████████▋ | 23799/89905 [7:31:18<16:14:35, 1.13it/s][2025-04-28 17:56:22] (step=0023800) Train Loss: 5.7976, Train Steps/Sec: 1.12 + 26%|███████████████████████████████████▊ | 23824/89905 [7:31:41<16:41:21, 1.10it/s][2025-04-28 17:56:44] (step=0023825) Train Loss: 5.7902, Train Steps/Sec: 1.12 + 27%|███████████████████████████████████▊ | 23849/89905 [7:32:03<16:28:37, 1.11it/s][2025-04-28 17:57:06] (step=0023850) Train Loss: 5.8058, Train Steps/Sec: 1.12 + 27%|███████████████████████████████████▊ | 23874/89905 [7:32:25<16:22:50, 1.12it/s][2025-04-28 17:57:29] (step=0023875) Train Loss: 5.8382, Train Steps/Sec: 1.12 + 27%|███████████████████████████████████▉ | 23899/89905 [7:32:47<16:18:56, 1.12it/s][2025-04-28 17:57:51] (step=0023900) Train Loss: 5.7586, Train Steps/Sec: 1.12 + 27%|███████████████████████████████████▉ | 23924/89905 [7:33:10<16:14:33, 1.13it/s][2025-04-28 17:58:13] (step=0023925) Train Loss: 5.7929, Train Steps/Sec: 1.12 + 27%|███████████████████████████████████▉ | 23949/89905 [7:33:32<16:13:35, 1.13it/s][2025-04-28 17:58:35] (step=0023950) Train Loss: 5.7765, Train Steps/Sec: 1.12 + 27%|███████████████████████████████████▉ | 23974/89905 [7:33:54<16:12:45, 1.13it/s][2025-04-28 17:58:58] (step=0023975) Train Loss: 5.8548, Train Steps/Sec: 1.12 + 27%|████████████████████████████████████ | 23999/89905 [7:34:17<16:08:43, 1.13it/s][2025-04-28 17:59:20] (step=0024000) Train Loss: 5.8276, Train Steps/Sec: 1.12 +You are using a model of type blip-2 to instantiate a model of type instructblip. This is not supported for all configurations of models and can yield errors. +[2025-04-28 17:59:20] text_config is None. Initializing the text config with default values (`OPTConfig`). +model_name_or_path:/tmp/haozhezhao/model/blip2-flan-t5-xl +image_place_holder: +tokenizer length after expend 32101 +tokenizer length before expend 32100 +100%|███████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████| 6/6 [05:59<00:00, 59.93s/it] +[2025-04-28 18:06:38] Finish Eval in 24000 steps...████████████████████████████████████████████████████████████████████| 6/6 [05:59<00:00, 59.76s/it] +processing left-padding... +processing left-padding... +processing left-padding... +processing left-padding... +processing left-padding... +processing left-padding... +[2025-04-28 18:06:58] Saved checkpoint to checkpoint/X2I_UltraEdiit_fourmask_LLava_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_filtered_objects_4mask_1280_context_after_stage1_5e4/000-GPT-XL/checkpoints/0024000.pt +[2025-04-28 18:06:59] Removed old checkpoint: checkpoint/X2I_UltraEdiit_fourmask_LLava_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_filtered_objects_4mask_1280_context_after_stage1_5e4/000-GPT-XL/checkpoints/0022000.pt + 27%|████████████████████████████████████ | 24024/89905 [7:42:19<17:11:03, 1.06it/s][2025-04-28 18:07:22] (step=0024025) Train Loss: 5.7891, Train Steps/Sec: 0.05 + 27%|████████████████████████████████████ | 24049/89905 [7:42:41<16:24:32, 1.11it/s][2025-04-28 18:07:44] (step=0024050) Train Loss: 5.8163, Train Steps/Sec: 1.12 + 27%|████████████████████████████████████▏ | 24074/89905 [7:43:03<16:20:12, 1.12it/s][2025-04-28 18:08:07] (step=0024075) Train Loss: 5.8346, Train Steps/Sec: 1.12 + 27%|████████████████████████████████████▏ | 24099/89905 [7:43:25<16:14:06, 1.13it/s][2025-04-28 18:08:29] (step=0024100) Train Loss: 5.8374, Train Steps/Sec: 1.12 + 27%|████████████████████████████████████▏ | 24124/89905 [7:43:48<16:11:43, 1.13it/s][2025-04-28 18:08:51] (step=0024125) Train Loss: 5.7830, Train Steps/Sec: 1.12 + 27%|████████████████████████████████████▎ | 24149/89905 [7:44:10<16:07:53, 1.13it/s][2025-04-28 18:09:13] (step=0024150) Train Loss: 5.8201, Train Steps/Sec: 1.12 + 27%|████████████████████████████████████▎ | 24174/89905 [7:44:32<16:10:28, 1.13it/s][2025-04-28 18:09:36] (step=0024175) Train Loss: 5.8447, Train Steps/Sec: 1.12 + 27%|████████████████████████████████████▎ | 24199/89905 [7:44:55<16:06:45, 1.13it/s][2025-04-28 18:09:58] (step=0024200) Train Loss: 5.7398, Train Steps/Sec: 1.12 + 27%|████████████████████████████████████▎ | 24224/89905 [7:45:17<16:39:26, 1.10it/s][2025-04-28 18:10:20] (step=0024225) Train Loss: 5.7928, Train Steps/Sec: 1.12 + 27%|████████████████████████████████████▍ | 24249/89905 [7:45:39<16:22:35, 1.11it/s][2025-04-28 18:10:43] (step=0024250) Train Loss: 5.7996, Train Steps/Sec: 1.12 + 27%|████████████████████████████████████▍ | 24274/89905 [7:46:02<16:15:34, 1.12it/s][2025-04-28 18:11:05] (step=0024275) Train Loss: 5.7839, Train Steps/Sec: 1.12 + 27%|████████████████████████████████████▍ | 24299/89905 [7:46:24<16:13:22, 1.12it/s][2025-04-28 18:11:27] (step=0024300) Train Loss: 5.7365, Train Steps/Sec: 1.12 + 27%|████████████████████████████████████▌ | 24324/89905 [7:46:46<16:09:11, 1.13it/s][2025-04-28 18:11:50] (step=0024325) Train Loss: 5.8292, Train Steps/Sec: 1.12 + 27%|████████████████████████████████████▌ | 24349/89905 [7:47:09<16:06:46, 1.13it/s][2025-04-28 18:12:12] (step=0024350) Train Loss: 5.7675, Train Steps/Sec: 1.12 + 27%|████████████████████████████████████▌ | 24374/89905 [7:47:31<16:07:06, 1.13it/s][2025-04-28 18:12:34] (step=0024375) Train Loss: 5.9459, Train Steps/Sec: 1.12 + 27%|████████████████████████████████████▋ | 24399/89905 [7:47:53<16:05:04, 1.13it/s][2025-04-28 18:12:57] (step=0024400) Train Loss: 5.7731, Train Steps/Sec: 1.12 + 27%|████████████████████████████████████▋ | 24424/89905 [7:48:16<16:33:19, 1.10it/s][2025-04-28 18:13:19] (step=0024425) Train Loss: 5.7576, Train Steps/Sec: 1.12 + 27%|████████████████████████████████████▋ | 24449/89905 [7:48:38<16:23:41, 1.11it/s][2025-04-28 18:13:41] (step=0024450) Train Loss: 5.8063, Train Steps/Sec: 1.12 + 27%|████████████████████████████████████▋ | 24474/89905 [7:49:00<16:14:47, 1.12it/s][2025-04-28 18:14:03] (step=0024475) Train Loss: 5.8363, Train Steps/Sec: 1.12 + 27%|████████████████████████████████████▊ | 24499/89905 [7:49:22<16:10:47, 1.12it/s][2025-04-28 18:14:26] (step=0024500) Train Loss: 5.7661, Train Steps/Sec: 1.12 + 27%|████████████████████████████████████▊ | 24524/89905 [7:49:45<16:07:16, 1.13it/s][2025-04-28 18:14:48] (step=0024525) Train Loss: 5.7798, Train Steps/Sec: 1.12 + 27%|████████████████████████████████████▊ | 24549/89905 [7:50:07<16:06:48, 1.13it/s][2025-04-28 18:15:10] (step=0024550) Train Loss: 5.7840, Train Steps/Sec: 1.12 + 27%|████████████████████████████████████▉ | 24574/89905 [7:50:29<16:14:05, 1.12it/s][2025-04-28 18:15:33] (step=0024575) Train Loss: 5.7691, Train Steps/Sec: 1.12 + 27%|████████████████████████████████████▉ | 24599/89905 [7:50:52<16:01:03, 1.13it/s][2025-04-28 18:15:55] (step=0024600) Train Loss: 5.7841, Train Steps/Sec: 1.12 + 27%|████████████████████████████████████▉ | 24624/89905 [7:51:14<16:35:56, 1.09it/s][2025-04-28 18:16:17] (step=0024625) Train Loss: 5.8028, Train Steps/Sec: 1.12 + 27%|█████████████████████████████████████ | 24649/89905 [7:51:36<16:18:58, 1.11it/s][2025-04-28 18:16:40] (step=0024650) Train Loss: 5.8290, Train Steps/Sec: 1.12 + 27%|█████████████████████████████████████ | 24674/89905 [7:51:59<16:11:29, 1.12it/s][2025-04-28 18:17:02] (step=0024675) Train Loss: 5.8274, Train Steps/Sec: 1.12 + 27%|█████████████████████████████████████ | 24699/89905 [7:52:21<16:07:58, 1.12it/s][2025-04-28 18:17:24] (step=0024700) Train Loss: 5.8626, Train Steps/Sec: 1.12 + 28%|█████████████████████████████████████▏ | 24724/89905 [7:52:43<16:02:31, 1.13it/s][2025-04-28 18:17:47] (step=0024725) Train Loss: 5.8492, Train Steps/Sec: 1.12 + 28%|█████████████████████████████████████▏ | 24749/89905 [7:53:06<16:02:44, 1.13it/s][2025-04-28 18:18:09] (step=0024750) Train Loss: 5.8436, Train Steps/Sec: 1.12 + 28%|█████████████████████████████████████▏ | 24774/89905 [7:53:28<16:01:48, 1.13it/s][2025-04-28 18:18:31] (step=0024775) Train Loss: 5.8311, Train Steps/Sec: 1.12 + 28%|█████████████████████████████████████▏ | 24799/89905 [7:53:50<16:00:02, 1.13it/s][2025-04-28 18:18:54] (step=0024800) Train Loss: 5.8001, Train Steps/Sec: 1.12 + 28%|█████████████████████████████████████▎ | 24824/89905 [7:54:13<16:25:03, 1.10it/s][2025-04-28 18:19:16] (step=0024825) Train Loss: 5.8418, Train Steps/Sec: 1.12 + 28%|█████████████████████████████████████▎ | 24849/89905 [7:54:35<16:16:06, 1.11it/s][2025-04-28 18:19:38] (step=0024850) Train Loss: 5.8153, Train Steps/Sec: 1.12 + 28%|█████████████████████████████████████▎ | 24874/89905 [7:54:57<16:11:14, 1.12it/s][2025-04-28 18:20:01] (step=0024875) Train Loss: 5.8516, Train Steps/Sec: 1.12 + 28%|█████████████████████████████████████▍ | 24899/89905 [7:55:19<16:06:31, 1.12it/s][2025-04-28 18:20:23] (step=0024900) Train Loss: 5.8115, Train Steps/Sec: 1.12 + 28%|█████████████████████████████████████▍ | 24924/89905 [7:55:42<16:00:21, 1.13it/s][2025-04-28 18:20:45] (step=0024925) Train Loss: 5.7978, Train Steps/Sec: 1.12 + 28%|█████████████████████████████████████▍ | 24949/89905 [7:56:04<15:55:25, 1.13it/s][2025-04-28 18:21:07] (step=0024950) Train Loss: 5.7316, Train Steps/Sec: 1.12 + 28%|█████████████████████████████████████▌ | 24974/89905 [7:56:26<15:57:30, 1.13it/s][2025-04-28 18:21:30] (step=0024975) Train Loss: 5.8020, Train Steps/Sec: 1.12 + 28%|█████████████████████████████████████▌ | 24999/89905 [7:56:49<15:55:00, 1.13it/s][2025-04-28 18:21:52] (step=0025000) Train Loss: 5.7831, Train Steps/Sec: 1.12 + 28%|█████████████████████████████████████▌ | 25024/89905 [7:57:11<16:25:12, 1.10it/s][2025-04-28 18:22:15] (step=0025025) Train Loss: 5.7303, Train Steps/Sec: 1.12 + 28%|█████████████████████████████████████▌ | 25049/89905 [7:57:33<16:13:41, 1.11it/s][2025-04-28 18:22:37] (step=0025050) Train Loss: 5.7515, Train Steps/Sec: 1.12 + 28%|█████████████████████████████████████▋ | 25074/89905 [7:57:56<16:08:07, 1.12it/s][2025-04-28 18:22:59] (step=0025075) Train Loss: 5.7632, Train Steps/Sec: 1.12 + 28%|█████████████████████████████████████▋ | 25099/89905 [7:58:18<16:01:34, 1.12it/s][2025-04-28 18:23:21] (step=0025100) Train Loss: 5.7978, Train Steps/Sec: 1.12 + 28%|█████████████████████████████████████▋ | 25124/89905 [7:58:40<15:54:51, 1.13it/s][2025-04-28 18:23:44] (step=0025125) Train Loss: 5.7717, Train Steps/Sec: 1.12 + 28%|█████████████████████████████████████▊ | 25149/89905 [7:59:03<15:54:16, 1.13it/s][2025-04-28 18:24:06] (step=0025150) Train Loss: 5.8025, Train Steps/Sec: 1.12 + 28%|█████████████████████████████████████▊ | 25174/89905 [7:59:25<15:52:19, 1.13it/s][2025-04-28 18:24:28] (step=0025175) Train Loss: 5.8365, Train Steps/Sec: 1.12 + 28%|█████████████████████████████████████▊ | 25199/89905 [7:59:47<15:49:15, 1.14it/s][2025-04-28 18:24:51] (step=0025200) Train Loss: 5.7700, Train Steps/Sec: 1.12 + 28%|█████████████████████████████████████▉ | 25224/89905 [8:00:10<16:18:50, 1.10it/s][2025-04-28 18:25:13] (step=0025225) Train Loss: 5.8032, Train Steps/Sec: 1.12 + 28%|█████████████████████████████████████▉ | 25249/89905 [8:00:32<16:10:55, 1.11it/s][2025-04-28 18:25:35] (step=0025250) Train Loss: 5.7752, Train Steps/Sec: 1.12 + 28%|█████████████████████████████████████▉ | 25274/89905 [8:00:54<16:02:33, 1.12it/s][2025-04-28 18:25:58] (step=0025275) Train Loss: 5.7929, Train Steps/Sec: 1.12 + 28%|█████████████████████████████████████▉ | 25299/89905 [8:01:16<16:00:02, 1.12it/s][2025-04-28 18:26:20] (step=0025300) Train Loss: 5.7339, Train Steps/Sec: 1.12 + 28%|██████████████████████████████████████ | 25324/89905 [8:01:39<15:52:43, 1.13it/s][2025-04-28 18:26:42] (step=0025325) Train Loss: 5.7483, Train Steps/Sec: 1.12 + 28%|██████████████████████████████████████ | 25349/89905 [8:02:01<15:50:42, 1.13it/s][2025-04-28 18:27:04] (step=0025350) Train Loss: 5.7573, Train Steps/Sec: 1.12 + 28%|██████████████████████████████████████ | 25374/89905 [8:02:23<15:48:23, 1.13it/s][2025-04-28 18:27:27] (step=0025375) Train Loss: 5.8258, Train Steps/Sec: 1.12 + 28%|██████████████████████████████████████▏ | 25399/89905 [8:02:46<15:49:20, 1.13it/s][2025-04-28 18:27:49] (step=0025400) Train Loss: 5.7418, Train Steps/Sec: 1.12 + 28%|██████████████████████████████████████▏ | 25424/89905 [8:03:08<16:20:33, 1.10it/s][2025-04-28 18:28:11] (step=0025425) Train Loss: 5.7913, Train Steps/Sec: 1.12 + 28%|██████████████████████████████████████▏ | 25449/89905 [8:03:30<16:05:10, 1.11it/s][2025-04-28 18:28:34] (step=0025450) Train Loss: 5.7338, Train Steps/Sec: 1.12 + 28%|██████████████████████████████████████▎ | 25474/89905 [8:03:53<15:58:42, 1.12it/s][2025-04-28 18:28:56] (step=0025475) Train Loss: 5.8132, Train Steps/Sec: 1.12 + 28%|██████████████████████████████████████▎ | 25499/89905 [8:04:15<15:56:58, 1.12it/s][2025-04-28 18:29:18] (step=0025500) Train Loss: 5.8019, Train Steps/Sec: 1.12 + 28%|██████████████████████████████████████▎ | 25524/89905 [8:04:37<15:57:19, 1.12it/s][2025-04-28 18:29:41] (step=0025525) Train Loss: 5.7642, Train Steps/Sec: 1.12 + 28%|██████████████████████████████████████▎ | 25549/89905 [8:05:00<15:52:50, 1.13it/s][2025-04-28 18:30:03] (step=0025550) Train Loss: 5.7844, Train Steps/Sec: 1.12 + 28%|██████████████████████████████████████▍ | 25574/89905 [8:05:22<15:47:37, 1.13it/s][2025-04-28 18:30:25] (step=0025575) Train Loss: 5.7659, Train Steps/Sec: 1.12 + 28%|██████████████████████████████████████▍ | 25599/89905 [8:05:44<15:43:42, 1.14it/s][2025-04-28 18:30:48] (step=0025600) Train Loss: 5.7834, Train Steps/Sec: 1.12 + 29%|██████████████████████████████████████▍ | 25624/89905 [8:06:07<16:16:45, 1.10it/s][2025-04-28 18:31:10] (step=0025625) Train Loss: 5.7418, Train Steps/Sec: 1.12 + 29%|██████████████████████████████████████▌ | 25649/89905 [8:06:29<16:07:15, 1.11it/s][2025-04-28 18:31:32] (step=0025650) Train Loss: 5.7490, Train Steps/Sec: 1.12 + 29%|██████████████████████████████████████▌ | 25674/89905 [8:06:51<15:58:18, 1.12it/s][2025-04-28 18:31:55] (step=0025675) Train Loss: 5.7936, Train Steps/Sec: 1.12 + 29%|██████████████████████████████████████▌ | 25699/89905 [8:07:13<15:51:49, 1.12it/s][2025-04-28 18:32:17] (step=0025700) Train Loss: 5.8064, Train Steps/Sec: 1.12 + 29%|██████████████████████████████████████▋ | 25724/89905 [8:07:36<15:49:22, 1.13it/s][2025-04-28 18:32:39] (step=0025725) Train Loss: 5.8796, Train Steps/Sec: 1.12 + 29%|██████████████████████████████████████▋ | 25749/89905 [8:07:58<15:48:30, 1.13it/s][2025-04-28 18:33:01] (step=0025750) Train Loss: 5.8135, Train Steps/Sec: 1.12 + 29%|██████████████████████████████████████▋ | 25774/89905 [8:08:20<15:46:18, 1.13it/s][2025-04-28 18:33:24] (step=0025775) Train Loss: 5.7404, Train Steps/Sec: 1.12 + 29%|██████████████████████████████████████▋ | 25799/89905 [8:08:43<15:41:21, 1.13it/s][2025-04-28 18:33:46] (step=0025800) Train Loss: 5.7782, Train Steps/Sec: 1.12 + 29%|██████████████████████████████████████▊ | 25824/89905 [8:09:05<16:14:48, 1.10it/s][2025-04-28 18:34:08] (step=0025825) Train Loss: 5.7650, Train Steps/Sec: 1.12 + 29%|██████████████████████████████████████▊ | 25849/89905 [8:09:27<15:59:49, 1.11it/s][2025-04-28 18:34:31] (step=0025850) Train Loss: 5.7790, Train Steps/Sec: 1.12 + 29%|██████████████████████████████████████▊ | 25874/89905 [8:09:49<15:52:15, 1.12it/s][2025-04-28 18:34:53] (step=0025875) Train Loss: 5.8129, Train Steps/Sec: 1.12 + 29%|██████████████████████████████████████▉ | 25899/89905 [8:10:12<15:53:28, 1.12it/s][2025-04-28 18:35:15] (step=0025900) Train Loss: 5.7789, Train Steps/Sec: 1.12 + 29%|██████████████████████████████████████▉ | 25924/89905 [8:10:34<15:43:46, 1.13it/s][2025-04-28 18:35:37] (step=0025925) Train Loss: 5.7799, Train Steps/Sec: 1.12 + 29%|██████████████████████████████████████▉ | 25949/89905 [8:10:56<15:45:14, 1.13it/s][2025-04-28 18:36:00] (step=0025950) Train Loss: 5.8439, Train Steps/Sec: 1.12 + 29%|███████████████████████████████████████ | 25974/89905 [8:11:19<15:40:50, 1.13it/s][2025-04-28 18:36:22] (step=0025975) Train Loss: 5.7907, Train Steps/Sec: 1.12 + 29%|███████████████████████████████████████ | 25999/89905 [8:11:41<15:43:10, 1.13it/s][2025-04-28 18:36:45] (step=0026000) Train Loss: 5.7854, Train Steps/Sec: 1.11 +You are using a model of type blip-2 to instantiate a model of type instructblip. This is not supported for all configurations of models and can yield errors. +[2025-04-28 18:36:45] text_config is None. Initializing the text config with default values (`OPTConfig`). +model_name_or_path:/tmp/haozhezhao/model/blip2-flan-t5-xl +image_place_holder: +tokenizer length after expend 32101 +tokenizer length before expend 32100 +100%|███████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████| 6/6 [05:59<00:00, 59.98s/it] +[2025-04-28 18:44:02] Finish Eval in 26000 steps...████████████████████████████████████████████████████████████████████| 6/6 [05:59<00:00, 59.83s/it] +processing left-padding... +processing left-padding... +processing left-padding... +processing left-padding... +processing left-padding... +processing left-padding... +[2025-04-28 18:44:22] Saved checkpoint to checkpoint/X2I_UltraEdiit_fourmask_LLava_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_filtered_objects_4mask_1280_context_after_stage1_5e4/000-GPT-XL/checkpoints/0026000.pt +[2025-04-28 18:44:24] Removed old checkpoint: checkpoint/X2I_UltraEdiit_fourmask_LLava_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_filtered_objects_4mask_1280_context_after_stage1_5e4/000-GPT-XL/checkpoints/0024000.pt + 29%|███████████████████████████████████████ | 26024/89905 [8:19:43<16:43:31, 1.06it/s][2025-04-28 18:44:46] (step=0026025) Train Loss: 5.8157, Train Steps/Sec: 0.05 + 29%|███████████████████████████████████████ | 26049/89905 [8:20:05<16:02:05, 1.11it/s][2025-04-28 18:45:09] (step=0026050) Train Loss: 5.7660, Train Steps/Sec: 1.12 + 29%|███████████████████████████████████████▏ | 26074/89905 [8:20:28<15:51:01, 1.12it/s][2025-04-28 18:45:31] (step=0026075) Train Loss: 5.7568, Train Steps/Sec: 1.12 + 29%|███████████████████████████████████████▏ | 26099/89905 [8:20:50<15:44:31, 1.13it/s][2025-04-28 18:45:53] (step=0026100) Train Loss: 5.7773, Train Steps/Sec: 1.12 + 29%|███████████████████████████████████████▏ | 26124/89905 [8:21:12<15:49:11, 1.12it/s][2025-04-28 18:46:16] (step=0026125) Train Loss: 5.7023, Train Steps/Sec: 1.12 + 29%|███████████████████████████████████████▎ | 26149/89905 [8:21:35<15:42:58, 1.13it/s][2025-04-28 18:46:38] (step=0026150) Train Loss: 5.8370, Train Steps/Sec: 1.12 + 29%|███████████████████████████████████████▎ | 26174/89905 [8:21:57<15:36:00, 1.13it/s][2025-04-28 18:47:00] (step=0026175) Train Loss: 5.8062, Train Steps/Sec: 1.12 + 29%|███████████████████████████████████████▎ | 26199/89905 [8:22:19<15:40:27, 1.13it/s][2025-04-28 18:47:23] (step=0026200) Train Loss: 5.7596, Train Steps/Sec: 1.12 + 29%|███████████████████████████████████████▍ | 26224/89905 [8:22:42<16:05:17, 1.10it/s][2025-04-28 18:47:45] (step=0026225) Train Loss: 5.7724, Train Steps/Sec: 1.12 + 29%|███████████████████████████████████████▍ | 26249/89905 [8:23:04<15:54:15, 1.11it/s][2025-04-28 18:48:07] (step=0026250) Train Loss: 5.7880, Train Steps/Sec: 1.12 + 29%|███████████████████████████████████████▍ | 26274/89905 [8:23:26<15:48:34, 1.12it/s][2025-04-28 18:48:30] (step=0026275) Train Loss: 5.7499, Train Steps/Sec: 1.12 + 29%|███████████████████████████████████████▍ | 26299/89905 [8:24:15<22:11:40, 1.26s/it][2025-04-28 18:49:19] (step=0026300) Train Loss: 5.7684, Train Steps/Sec: 0.51 + 29%|███████████████████████████████████████▌ | 26324/89905 [8:25:06<37:13:11, 2.11s/it][2025-04-28 18:50:10] (step=0026325) Train Loss: 5.7743, Train Steps/Sec: 0.49 + 29%|███████████████████████████████████████▌ | 26349/89905 [8:25:29<15:39:33, 1.13it/s][2025-04-28 18:50:32] (step=0026350) Train Loss: 5.7724, Train Steps/Sec: 1.12 + 29%|███████████████████████████████████████▌ | 26374/89905 [8:26:00<15:42:26, 1.12it/s][2025-04-28 18:51:03] (step=0026375) Train Loss: 5.7189, Train Steps/Sec: 0.80 + 29%|███████████████████████████████████████▋ | 26399/89905 [8:26:22<15:39:32, 1.13it/s][2025-04-28 18:51:26] (step=0026400) Train Loss: 5.7312, Train Steps/Sec: 1.11 + 29%|███████████████████████████████████████▋ | 26424/89905 [8:26:45<16:06:11, 1.10it/s][2025-04-28 18:51:48] (step=0026425) Train Loss: 5.7747, Train Steps/Sec: 1.12 + 29%|███████████████████████████████████████▋ | 26449/89905 [8:27:07<15:52:35, 1.11it/s][2025-04-28 18:52:11] (step=0026450) Train Loss: 5.7679, Train Steps/Sec: 1.12 + 29%|███████████████████████████████████████▊ | 26474/89905 [8:27:29<15:44:05, 1.12it/s][2025-04-28 18:52:33] (step=0026475) Train Loss: 5.8458, Train Steps/Sec: 1.12 + 29%|███████████████████████████████████████▊ | 26499/89905 [8:27:52<15:40:44, 1.12it/s][2025-04-28 18:52:55] (step=0026500) Train Loss: 5.8095, Train Steps/Sec: 1.12 + 30%|███████████████████████████████████████▊ | 26524/89905 [8:28:14<15:40:12, 1.12it/s][2025-04-28 18:53:18] (step=0026525) Train Loss: 5.8064, Train Steps/Sec: 1.12 + 30%|███████████████████████████████████████▊ | 26549/89905 [8:28:37<15:39:57, 1.12it/s][2025-04-28 18:53:40] (step=0026550) Train Loss: 5.8509, Train Steps/Sec: 1.12 + 30%|███████████████████████████████████████▉ | 26574/89905 [8:28:59<15:32:45, 1.13it/s][2025-04-28 18:54:02] (step=0026575) Train Loss: 5.8336, Train Steps/Sec: 1.12 + 30%|███████████████████████████████████████▉ | 26599/89905 [8:29:21<15:32:01, 1.13it/s][2025-04-28 18:54:25] (step=0026600) Train Loss: 5.7799, Train Steps/Sec: 1.12 + 30%|███████████████████████████████████████▉ | 26624/89905 [8:29:44<16:05:46, 1.09it/s][2025-04-28 18:54:47] (step=0026625) Train Loss: 5.8367, Train Steps/Sec: 1.12 + 30%|████████████████████████████████████████ | 26649/89905 [8:30:06<15:45:44, 1.11it/s][2025-04-28 18:55:09] (step=0026650) Train Loss: 5.7712, Train Steps/Sec: 1.12 + 30%|████████████████████████████████████████ | 26674/89905 [8:30:28<15:43:50, 1.12it/s][2025-04-28 18:55:32] (step=0026675) Train Loss: 5.7993, Train Steps/Sec: 1.12 + 30%|████████████████████████████████████████ | 26699/89905 [8:30:51<15:36:54, 1.12it/s][2025-04-28 18:55:54] (step=0026700) Train Loss: 5.7314, Train Steps/Sec: 1.12 + 30%|████████████████████████████████████████▏ | 26724/89905 [8:31:13<15:32:04, 1.13it/s][2025-04-28 18:56:16] (step=0026725) Train Loss: 5.7131, Train Steps/Sec: 1.12 + 30%|████████████████████████████████████████▏ | 26749/89905 [8:31:35<15:33:01, 1.13it/s][2025-04-28 18:56:39] (step=0026750) Train Loss: 5.7987, Train Steps/Sec: 1.12 + 30%|████████████████████████████████████████▏ | 26774/89905 [8:31:57<15:32:07, 1.13it/s][2025-04-28 18:57:01] (step=0026775) Train Loss: 5.8135, Train Steps/Sec: 1.12 + 30%|████████████████████████████████████████▏ | 26799/89905 [8:32:20<15:30:34, 1.13it/s][2025-04-28 18:57:23] (step=0026800) Train Loss: 5.7490, Train Steps/Sec: 1.12 + 30%|████████████████████████████████████████▎ | 26824/89905 [8:32:42<16:05:21, 1.09it/s][2025-04-28 18:57:46] (step=0026825) Train Loss: 5.7403, Train Steps/Sec: 1.12 + 30%|████████████████████████████████████████▎ | 26849/89905 [8:33:04<15:41:00, 1.12it/s][2025-04-28 18:58:08] (step=0026850) Train Loss: 5.7657, Train Steps/Sec: 1.12 + 30%|████████████████████████████████████████▎ | 26874/89905 [8:33:27<15:40:44, 1.12it/s][2025-04-28 18:58:30] (step=0026875) Train Loss: 5.7431, Train Steps/Sec: 1.12 + 30%|████████████████████████████████████████▍ | 26899/89905 [8:33:49<15:31:45, 1.13it/s][2025-04-28 18:58:52] (step=0026900) Train Loss: 5.6861, Train Steps/Sec: 1.12 + 30%|████████████████████████████████████████▍ | 26924/89905 [8:34:11<15:29:23, 1.13it/s][2025-04-28 18:59:15] (step=0026925) Train Loss: 5.8510, Train Steps/Sec: 1.12 + 30%|████████████████████████████████████████▍ | 26949/89905 [8:34:34<15:31:12, 1.13it/s][2025-04-28 18:59:37] (step=0026950) Train Loss: 5.7466, Train Steps/Sec: 1.12 + 30%|████████████████████████████████████████▌ | 26974/89905 [8:34:56<15:26:04, 1.13it/s][2025-04-28 18:59:59] (step=0026975) Train Loss: 5.8489, Train Steps/Sec: 1.12 + 30%|████████████████████████████████████████▌ | 26999/89905 [8:35:18<15:25:00, 1.13it/s][2025-04-28 19:00:22] (step=0027000) Train Loss: 5.7876, Train Steps/Sec: 1.12 + 30%|████████████████████████████████████████▌ | 27024/89905 [8:35:41<15:59:10, 1.09it/s][2025-04-28 19:00:44] (step=0027025) Train Loss: 5.7910, Train Steps/Sec: 1.12 + 30%|████████████████████████████████████████▌ | 27049/89905 [8:36:03<15:44:07, 1.11it/s][2025-04-28 19:01:06] (step=0027050) Train Loss: 5.7989, Train Steps/Sec: 1.12 + 30%|████████████████████████████████████████▋ | 27074/89905 [8:36:25<15:36:05, 1.12it/s][2025-04-28 19:01:29] (step=0027075) Train Loss: 5.7440, Train Steps/Sec: 1.12 + 30%|████████████████████████████████████████▋ | 27099/89905 [8:36:48<15:34:12, 1.12it/s][2025-04-28 19:01:51] (step=0027100) Train Loss: 5.7788, Train Steps/Sec: 1.12 + 30%|████████████████████████████████████████▋ | 27124/89905 [8:37:10<15:35:33, 1.12it/s][2025-04-28 19:02:13] (step=0027125) Train Loss: 5.7670, Train Steps/Sec: 1.12 + 30%|████████████████████████████████████████▊ | 27149/89905 [8:37:32<15:28:56, 1.13it/s][2025-04-28 19:02:36] (step=0027150) Train Loss: 5.8336, Train Steps/Sec: 1.12 + 30%|████████████████████████████████████████▊ | 27174/89905 [8:37:55<15:27:15, 1.13it/s][2025-04-28 19:02:58] (step=0027175) Train Loss: 5.6953, Train Steps/Sec: 1.12 + 30%|████████████████████████████████████████▊ | 27199/89905 [8:38:17<15:25:46, 1.13it/s][2025-04-28 19:03:20] (step=0027200) Train Loss: 5.7796, Train Steps/Sec: 1.12 + 30%|████████████████████████████████████████▉ | 27224/89905 [8:38:39<15:56:46, 1.09it/s][2025-04-28 19:03:43] (step=0027225) Train Loss: 5.7483, Train Steps/Sec: 1.12 + 30%|████████████████████████████████████████▉ | 27249/89905 [8:39:02<15:37:22, 1.11it/s][2025-04-28 19:04:05] (step=0027250) Train Loss: 5.7692, Train Steps/Sec: 1.12 + 30%|████████████████████████████████████████▉ | 27274/89905 [8:39:24<15:33:56, 1.12it/s][2025-04-28 19:04:27] (step=0027275) Train Loss: 5.7820, Train Steps/Sec: 1.12 + 30%|████████████████████████████████████████▉ | 27299/89905 [8:39:46<15:29:43, 1.12it/s][2025-04-28 19:04:50] (step=0027300) Train Loss: 5.8047, Train Steps/Sec: 1.12 + 30%|█████████████████████████████████████████ | 27324/89905 [8:40:09<15:25:56, 1.13it/s][2025-04-28 19:05:12] (step=0027325) Train Loss: 5.8264, Train Steps/Sec: 1.12 + 30%|█████████████████████████████████████████ | 27349/89905 [8:40:31<15:24:12, 1.13it/s][2025-04-28 19:05:34] (step=0027350) Train Loss: 5.7881, Train Steps/Sec: 1.12 + 30%|█████████████████████████████████████████ | 27374/89905 [8:40:53<15:22:23, 1.13it/s][2025-04-28 19:05:57] (step=0027375) Train Loss: 5.7213, Train Steps/Sec: 1.12 + 30%|█████████████████████████████████████████▏ | 27399/89905 [8:41:15<15:19:05, 1.13it/s][2025-04-28 19:06:19] (step=0027400) Train Loss: 5.7100, Train Steps/Sec: 1.12 + 31%|█████████████████████████████████████████▏ | 27424/89905 [8:41:38<15:53:14, 1.09it/s][2025-04-28 19:06:41] (step=0027425) Train Loss: 5.7808, Train Steps/Sec: 1.12 + 31%|█████████████████████████████████████████▏ | 27449/89905 [8:42:00<15:38:46, 1.11it/s][2025-04-28 19:07:04] (step=0027450) Train Loss: 5.7842, Train Steps/Sec: 1.12 + 31%|█████████████████████████████████████████▎ | 27474/89905 [8:42:23<15:31:34, 1.12it/s][2025-04-28 19:07:26] (step=0027475) Train Loss: 5.7789, Train Steps/Sec: 1.12 + 31%|█████████████████████████████████████████▎ | 27499/89905 [8:42:45<15:27:42, 1.12it/s][2025-04-28 19:07:48] (step=0027500) Train Loss: 5.7548, Train Steps/Sec: 1.12 + 31%|█████████████████████████████████████████▎ | 27524/89905 [8:43:07<15:27:54, 1.12it/s][2025-04-28 19:08:11] (step=0027525) Train Loss: 5.8317, Train Steps/Sec: 1.12 + 31%|█████████████████████████████████████████▎ | 27549/89905 [8:43:30<15:21:48, 1.13it/s][2025-04-28 19:08:33] (step=0027550) Train Loss: 5.7657, Train Steps/Sec: 1.12 + 31%|█████████████████████████████████████████▍ | 27574/89905 [8:43:52<15:21:26, 1.13it/s][2025-04-28 19:08:55] (step=0027575) Train Loss: 5.7479, Train Steps/Sec: 1.12 + 31%|█████████████████████████████████████████▍ | 27599/89905 [8:44:14<15:18:49, 1.13it/s][2025-04-28 19:09:18] (step=0027600) Train Loss: 5.8683, Train Steps/Sec: 1.12 + 31%|█████████████████████████████████████████▍ | 27624/89905 [8:44:37<15:48:07, 1.09it/s][2025-04-28 19:09:40] (step=0027625) Train Loss: 5.7908, Train Steps/Sec: 1.12 + 31%|█████████████████████████████████████████▌ | 27649/89905 [8:44:59<15:33:15, 1.11it/s][2025-04-28 19:10:02] (step=0027650) Train Loss: 5.7558, Train Steps/Sec: 1.12 + 31%|█████████████████████████████████████████▌ | 27674/89905 [8:45:21<15:29:27, 1.12it/s][2025-04-28 19:10:25] (step=0027675) Train Loss: 5.7499, Train Steps/Sec: 1.12 + 31%|█████████████████████████████████████████▌ | 27699/89905 [8:45:44<15:22:37, 1.12it/s][2025-04-28 19:10:47] (step=0027700) Train Loss: 5.7164, Train Steps/Sec: 1.12 + 31%|█████████████████████████████████████████▋ | 27724/89905 [8:46:06<15:19:46, 1.13it/s][2025-04-28 19:11:09] (step=0027725) Train Loss: 5.7763, Train Steps/Sec: 1.12 + 31%|█████████████████████████████████████████▋ | 27749/89905 [8:46:28<15:17:52, 1.13it/s][2025-04-28 19:11:32] (step=0027750) Train Loss: 5.7482, Train Steps/Sec: 1.12 + 31%|█████████████████████████████████████████▋ | 27774/89905 [8:46:50<15:15:24, 1.13it/s][2025-04-28 19:11:54] (step=0027775) Train Loss: 5.6903, Train Steps/Sec: 1.12 + 31%|█████████████████████████████████████████▋ | 27799/89905 [8:47:13<15:18:21, 1.13it/s][2025-04-28 19:12:16] (step=0027800) Train Loss: 5.7504, Train Steps/Sec: 1.11 + 31%|█████████████████████████████████████████▊ | 27824/89905 [8:47:35<15:42:15, 1.10it/s][2025-04-28 19:12:39] (step=0027825) Train Loss: 5.7791, Train Steps/Sec: 1.12 + 31%|█████████████████████████████████████████▊ | 27849/89905 [8:47:57<15:31:09, 1.11it/s][2025-04-28 19:13:01] (step=0027850) Train Loss: 5.6717, Train Steps/Sec: 1.12 + 31%|█████████████████████████████████████████▊ | 27874/89905 [8:48:20<15:21:30, 1.12it/s][2025-04-28 19:13:23] (step=0027875) Train Loss: 5.8017, Train Steps/Sec: 1.12 + 31%|█████████████████████████████████████████▉ | 27899/89905 [8:48:42<15:18:17, 1.13it/s][2025-04-28 19:13:45] (step=0027900) Train Loss: 5.7312, Train Steps/Sec: 1.12 + 31%|█████████████████████████████████████████▉ | 27924/89905 [8:49:04<15:14:57, 1.13it/s][2025-04-28 19:14:08] (step=0027925) Train Loss: 5.8101, Train Steps/Sec: 1.12 + 31%|█████████████████████████████████████████▉ | 27949/89905 [8:49:27<15:12:58, 1.13it/s][2025-04-28 19:14:30] (step=0027950) Train Loss: 5.8122, Train Steps/Sec: 1.12 + 31%|██████████████████████████████████████████ | 27974/89905 [8:49:49<15:08:06, 1.14it/s][2025-04-28 19:14:52] (step=0027975) Train Loss: 5.7779, Train Steps/Sec: 1.12 + 31%|██████████████████████████████████████████ | 27999/89905 [8:50:11<15:10:18, 1.13it/s][2025-04-28 19:15:15] (step=0028000) Train Loss: 5.7598, Train Steps/Sec: 1.12 +You are using a model of type blip-2 to instantiate a model of type instructblip. This is not supported for all configurations of models and can yield errors. +[2025-04-28 19:15:15] text_config is None. Initializing the text config with default values (`OPTConfig`). +model_name_or_path:/tmp/haozhezhao/model/blip2-flan-t5-xl +image_place_holder: +tokenizer length after expend 32101 +tokenizer length before expend 32100 +100%|███████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████| 6/6 [06:00<00:00, 60.05s/it] +[2025-04-28 19:22:34] Finish Eval in 28000 steps...████████████████████████████████████████████████████████████████████| 6/6 [05:59<00:00, 59.88s/it] +processing left-padding... +processing left-padding... +processing left-padding... +processing left-padding... +processing left-padding... +processing left-padding... +[2025-04-28 19:22:54] Saved checkpoint to checkpoint/X2I_UltraEdiit_fourmask_LLava_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_filtered_objects_4mask_1280_context_after_stage1_5e4/000-GPT-XL/checkpoints/0028000.pt +[2025-04-28 19:22:56] Removed old checkpoint: checkpoint/X2I_UltraEdiit_fourmask_LLava_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_filtered_objects_4mask_1280_context_after_stage1_5e4/000-GPT-XL/checkpoints/0026000.pt + 31%|██████████████████████████████████████████ | 28024/89905 [8:58:15<16:17:21, 1.06it/s][2025-04-28 19:23:18] (step=0028025) Train Loss: 5.7491, Train Steps/Sec: 0.05 + 31%|██████████████████████████████████████████ | 28049/89905 [8:58:37<15:24:37, 1.11it/s][2025-04-28 19:23:41] (step=0028050) Train Loss: 5.7993, Train Steps/Sec: 1.12 + 31%|██████████████████████████████████████████▏ | 28074/89905 [8:59:00<15:23:54, 1.12it/s][2025-04-28 19:24:03] (step=0028075) Train Loss: 5.7361, Train Steps/Sec: 1.12 + 31%|██████████████████████████████████████████▏ | 28099/89905 [8:59:22<15:20:21, 1.12it/s][2025-04-28 19:24:25] (step=0028100) Train Loss: 5.7470, Train Steps/Sec: 1.12 + 31%|██████████████████████████████████████████▏ | 28124/89905 [8:59:44<15:12:50, 1.13it/s][2025-04-28 19:24:48] (step=0028125) Train Loss: 5.7701, Train Steps/Sec: 1.12 + 31%|██████████████████████████████████████████▎ | 28149/89905 [9:00:07<15:10:22, 1.13it/s][2025-04-28 19:25:10] (step=0028150) Train Loss: 5.7818, Train Steps/Sec: 1.12 + 31%|██████████████████████████████████████████▎ | 28174/89905 [9:00:29<15:10:19, 1.13it/s][2025-04-28 19:25:32] (step=0028175) Train Loss: 5.7242, Train Steps/Sec: 1.12 + 31%|██████████████████████████████████████████▎ | 28199/89905 [9:00:51<15:08:13, 1.13it/s][2025-04-28 19:25:55] (step=0028200) Train Loss: 5.7440, Train Steps/Sec: 1.12 + 31%|██████████████████████████████████████████▍ | 28224/89905 [9:01:14<15:39:17, 1.09it/s][2025-04-28 19:26:17] (step=0028225) Train Loss: 5.7221, Train Steps/Sec: 1.12 + 31%|██████████████████████████████████████████▍ | 28249/89905 [9:01:36<15:25:20, 1.11it/s][2025-04-28 19:26:39] (step=0028250) Train Loss: 5.7925, Train Steps/Sec: 1.12 + 31%|██████████████████████████████████████████▍ | 28274/89905 [9:01:58<15:15:15, 1.12it/s][2025-04-28 19:27:01] (step=0028275) Train Loss: 5.8085, Train Steps/Sec: 1.12 + 31%|██████████████████████████████████████████▍ | 28299/89905 [9:02:20<15:16:42, 1.12it/s][2025-04-28 19:27:24] (step=0028300) Train Loss: 5.7734, Train Steps/Sec: 1.12 + 32%|██████████████████████████████████████████▌ | 28324/89905 [9:02:43<15:11:39, 1.13it/s][2025-04-28 19:27:46] (step=0028325) Train Loss: 5.8093, Train Steps/Sec: 1.12 + 32%|██████████████████████████████████████████▌ | 28349/89905 [9:03:05<15:23:22, 1.11it/s][2025-04-28 19:28:09] (step=0028350) Train Loss: 5.7456, Train Steps/Sec: 1.11 + 32%|██████████████████████████████████████████▌ | 28374/89905 [9:03:27<15:05:53, 1.13it/s][2025-04-28 19:28:31] (step=0028375) Train Loss: 5.6855, Train Steps/Sec: 1.12 + 32%|██████████████████████████████████████████▋ | 28399/89905 [9:03:50<15:05:58, 1.13it/s][2025-04-28 19:28:53] (step=0028400) Train Loss: 5.7576, Train Steps/Sec: 1.12 + 32%|██████████████████████████████████████████▋ | 28424/89905 [9:04:12<15:34:56, 1.10it/s][2025-04-28 19:29:16] (step=0028425) Train Loss: 5.7682, Train Steps/Sec: 1.12 + 32%|██████████████████████████████████████████▋ | 28449/89905 [9:04:35<15:23:36, 1.11it/s][2025-04-28 19:29:38] (step=0028450) Train Loss: 5.7377, Train Steps/Sec: 1.12 + 32%|██████████████████████████████████████████▊ | 28474/89905 [9:04:57<15:16:17, 1.12it/s][2025-04-28 19:30:00] (step=0028475) Train Loss: 5.7951, Train Steps/Sec: 1.12 + 32%|██████████████████████████████████████████▊ | 28499/89905 [9:05:19<15:17:20, 1.12it/s][2025-04-28 19:30:23] (step=0028500) Train Loss: 5.7360, Train Steps/Sec: 1.12 + 32%|██████████████████████████████████████████▊ | 28524/89905 [9:05:41<15:07:25, 1.13it/s][2025-04-28 19:30:45] (step=0028525) Train Loss: 5.7532, Train Steps/Sec: 1.12 + 32%|██████████████████████████████████████████▊ | 28549/89905 [9:06:04<15:06:47, 1.13it/s][2025-04-28 19:31:07] (step=0028550) Train Loss: 5.8186, Train Steps/Sec: 1.12 + 32%|██████████████████████████████████████████▉ | 28574/89905 [9:06:26<15:04:24, 1.13it/s][2025-04-28 19:31:29] (step=0028575) Train Loss: 5.7906, Train Steps/Sec: 1.12 + 32%|██████████████████████████████████████████▉ | 28599/89905 [9:06:48<15:03:09, 1.13it/s][2025-04-28 19:31:52] (step=0028600) Train Loss: 5.7042, Train Steps/Sec: 1.12 + 32%|██████████████████████████████████████████▉ | 28624/89905 [9:07:11<15:30:22, 1.10it/s][2025-04-28 19:32:14] (step=0028625) Train Loss: 5.7526, Train Steps/Sec: 1.12 + 32%|███████████████████████████████████████████ | 28649/89905 [9:07:33<15:15:55, 1.11it/s][2025-04-28 19:32:36] (step=0028650) Train Loss: 5.7657, Train Steps/Sec: 1.12 + 32%|███████████████████████████████████████████ | 28674/89905 [9:07:55<15:13:16, 1.12it/s][2025-04-28 19:32:59] (step=0028675) Train Loss: 5.7706, Train Steps/Sec: 1.12 + 32%|███████████████████████████████████████████ | 28699/89905 [9:08:18<15:09:40, 1.12it/s][2025-04-28 19:33:21] (step=0028700) Train Loss: 5.7701, Train Steps/Sec: 1.12 + 32%|███████████████████████████████████████████▏ | 28724/89905 [9:08:40<15:07:10, 1.12it/s][2025-04-28 19:33:43] (step=0028725) Train Loss: 5.6989, Train Steps/Sec: 1.12 + 32%|███████████████████████████████████████████▏ | 28749/89905 [9:09:02<15:03:29, 1.13it/s][2025-04-28 19:34:06] (step=0028750) Train Loss: 5.7890, Train Steps/Sec: 1.12 + 32%|███████████████████████████████████████████▏ | 28774/89905 [9:09:24<15:00:24, 1.13it/s][2025-04-28 19:34:28] (step=0028775) Train Loss: 5.7170, Train Steps/Sec: 1.12 + 32%|███████████████████████████████████████████▏ | 28799/89905 [9:09:47<14:59:50, 1.13it/s][2025-04-28 19:34:50] (step=0028800) Train Loss: 5.7648, Train Steps/Sec: 1.12 + 32%|███████████████████████████████████████████▎ | 28824/89905 [9:10:09<15:26:35, 1.10it/s][2025-04-28 19:35:13] (step=0028825) Train Loss: 5.7906, Train Steps/Sec: 1.12 + 32%|███████████████████████████████████████████▎ | 28849/89905 [9:10:32<15:15:49, 1.11it/s][2025-04-28 19:35:35] (step=0028850) Train Loss: 5.7798, Train Steps/Sec: 1.12 + 32%|███████████████████████████████████████████▎ | 28874/89905 [9:10:54<15:08:53, 1.12it/s][2025-04-28 19:35:57] (step=0028875) Train Loss: 5.7935, Train Steps/Sec: 1.12 + 32%|███████████████████████████████████████████▍ | 28899/89905 [9:11:16<15:03:23, 1.13it/s][2025-04-28 19:36:20] (step=0028900) Train Loss: 5.7783, Train Steps/Sec: 1.12 + 32%|███████████████████████████████████████████▍ | 28924/89905 [9:11:38<15:00:20, 1.13it/s][2025-04-28 19:36:42] (step=0028925) Train Loss: 5.8133, Train Steps/Sec: 1.12 + 32%|███████████████████████████████████████████▍ | 28949/89905 [9:12:01<15:01:15, 1.13it/s][2025-04-28 19:37:04] (step=0028950) Train Loss: 5.7790, Train Steps/Sec: 1.12 + 32%|███████████████████████████████████████████▌ | 28974/89905 [9:12:23<14:56:26, 1.13it/s][2025-04-28 19:37:26] (step=0028975) Train Loss: 5.7062, Train Steps/Sec: 1.12 + 32%|███████████████████████████████████████████▌ | 28999/89905 [9:12:45<14:58:06, 1.13it/s][2025-04-28 19:37:49] (step=0029000) Train Loss: 5.7164, Train Steps/Sec: 1.12 + 32%|███████████████████████████████████████████▌ | 29024/89905 [9:13:08<15:27:09, 1.09it/s][2025-04-28 19:38:11] (step=0029025) Train Loss: 5.7563, Train Steps/Sec: 1.12 + 32%|███████████████████████████████████████████▌ | 29049/89905 [9:13:30<15:12:51, 1.11it/s][2025-04-28 19:38:33] (step=0029050) Train Loss: 5.7928, Train Steps/Sec: 1.12 + 32%|███████████████████████████████████████████▋ | 29074/89905 [9:13:52<15:05:44, 1.12it/s][2025-04-28 19:38:56] (step=0029075) Train Loss: 5.7335, Train Steps/Sec: 1.12 + 32%|███████████████████████████████████████████▋ | 29099/89905 [9:14:15<15:03:06, 1.12it/s][2025-04-28 19:39:18] (step=0029100) Train Loss: 5.7275, Train Steps/Sec: 1.12 + 32%|███████████████████████████████████████████▋ | 29124/89905 [9:14:37<14:58:45, 1.13it/s][2025-04-28 19:39:40] (step=0029125) Train Loss: 5.7662, Train Steps/Sec: 1.12 + 32%|███████████████████████████████████████████▊ | 29149/89905 [9:14:59<14:58:31, 1.13it/s][2025-04-28 19:40:03] (step=0029150) Train Loss: 5.6929, Train Steps/Sec: 1.12 + 32%|███████████████████████████████████████████▊ | 29174/89905 [9:15:22<14:53:08, 1.13it/s][2025-04-28 19:40:25] (step=0029175) Train Loss: 5.7550, Train Steps/Sec: 1.12 + 32%|███████████████████████████████████████████▊ | 29199/89905 [9:15:44<14:53:12, 1.13it/s][2025-04-28 19:40:47] (step=0029200) Train Loss: 5.7757, Train Steps/Sec: 1.12 + 33%|███████████████████████████████████████████▉ | 29224/89905 [9:16:06<15:18:26, 1.10it/s][2025-04-28 19:41:10] (step=0029225) Train Loss: 5.7420, Train Steps/Sec: 1.12 + 33%|███████████████████████████████████████████▉ | 29249/89905 [9:16:28<15:07:03, 1.11it/s][2025-04-28 19:41:32] (step=0029250) Train Loss: 5.8289, Train Steps/Sec: 1.12 + 33%|███████████████████████████████████████████▉ | 29274/89905 [9:16:51<15:04:08, 1.12it/s][2025-04-28 19:41:54] (step=0029275) Train Loss: 5.7030, Train Steps/Sec: 1.12 + 33%|███████████████████████████████████████████▉ | 29299/89905 [9:17:13<14:59:00, 1.12it/s][2025-04-28 19:42:17] (step=0029300) Train Loss: 5.7142, Train Steps/Sec: 1.12 + 33%|████████████████████████████████████████████ | 29324/89905 [9:17:35<14:56:38, 1.13it/s][2025-04-28 19:42:39] (step=0029325) Train Loss: 5.7175, Train Steps/Sec: 1.12 + 33%|████████████████████████████████████████████ | 29349/89905 [9:17:58<14:53:07, 1.13it/s][2025-04-28 19:43:01] (step=0029350) Train Loss: 5.7881, Train Steps/Sec: 1.12 + 33%|████████████████████████████████████████████ | 29374/89905 [9:18:20<14:54:16, 1.13it/s][2025-04-28 19:43:23] (step=0029375) Train Loss: 5.7740, Train Steps/Sec: 1.12 + 33%|████████████████████████████████████████████▏ | 29399/89905 [9:18:42<14:49:34, 1.13it/s][2025-04-28 19:43:46] (step=0029400) Train Loss: 5.8085, Train Steps/Sec: 1.12 + 33%|████████████████████████████████████████████▏ | 29424/89905 [9:19:05<15:18:46, 1.10it/s][2025-04-28 19:44:08] (step=0029425) Train Loss: 5.7718, Train Steps/Sec: 1.12 + 33%|████████████████████████████████████████████▏ | 29449/89905 [9:19:27<15:05:15, 1.11it/s][2025-04-28 19:44:30] (step=0029450) Train Loss: 5.7364, Train Steps/Sec: 1.12 + 33%|████████████████████████████████████████████▎ | 29474/89905 [9:19:49<15:01:33, 1.12it/s][2025-04-28 19:44:53] (step=0029475) Train Loss: 5.7540, Train Steps/Sec: 1.12 + 33%|████████████████████████████████████████████▎ | 29499/89905 [9:20:12<14:57:51, 1.12it/s][2025-04-28 19:45:15] (step=0029500) Train Loss: 5.7341, Train Steps/Sec: 1.12 + 33%|████████████████████████████████████████████▎ | 29524/89905 [9:20:34<14:55:44, 1.12it/s][2025-04-28 19:45:37] (step=0029525) Train Loss: 5.8173, Train Steps/Sec: 1.12 + 33%|████████████████████████████████████████████▎ | 29549/89905 [9:20:56<14:51:38, 1.13it/s][2025-04-28 19:46:00] (step=0029550) Train Loss: 5.7843, Train Steps/Sec: 1.12 + 33%|████████████████████████████████████████████▍ | 29574/89905 [9:21:18<14:50:54, 1.13it/s][2025-04-28 19:46:22] (step=0029575) Train Loss: 5.6721, Train Steps/Sec: 1.12 + 33%|████████████████████████████████████████████▍ | 29599/89905 [9:21:41<14:49:03, 1.13it/s][2025-04-28 19:46:44] (step=0029600) Train Loss: 5.7916, Train Steps/Sec: 1.11 + 33%|████████████████████████████████████████████▍ | 29624/89905 [9:22:03<15:17:10, 1.10it/s][2025-04-28 19:47:07] (step=0029625) Train Loss: 5.7758, Train Steps/Sec: 1.12 + 33%|████████████████████████████████████████████▌ | 29649/89905 [9:22:26<15:01:38, 1.11it/s][2025-04-28 19:47:29] (step=0029650) Train Loss: 5.7513, Train Steps/Sec: 1.12 + 33%|████████████████████████████████████████████▌ | 29674/89905 [9:22:48<15:06:43, 1.11it/s][2025-04-28 19:47:51] (step=0029675) Train Loss: 5.8102, Train Steps/Sec: 1.12 + 33%|████████████████████████████████████████████▌ | 29699/89905 [9:23:10<14:50:56, 1.13it/s][2025-04-28 19:48:14] (step=0029700) Train Loss: 5.7195, Train Steps/Sec: 1.12 + 33%|████████████████████████████████████████████▋ | 29724/89905 [9:23:32<14:53:58, 1.12it/s][2025-04-28 19:48:36] (step=0029725) Train Loss: 5.7518, Train Steps/Sec: 1.12 + 33%|████████████████████████████████████████████▋ | 29749/89905 [9:23:55<14:51:52, 1.12it/s][2025-04-28 19:48:58] (step=0029750) Train Loss: 5.7434, Train Steps/Sec: 1.12 + 33%|████████████████████████████████████████████▋ | 29774/89905 [9:24:17<14:47:42, 1.13it/s][2025-04-28 19:49:21] (step=0029775) Train Loss: 5.7848, Train Steps/Sec: 1.12 + 33%|████████████████████████████████████████████▋ | 29799/89905 [9:24:39<14:46:50, 1.13it/s][2025-04-28 19:49:43] (step=0029800) Train Loss: 5.7478, Train Steps/Sec: 1.12 + 33%|████████████████████████████████████████████▊ | 29824/89905 [9:25:02<15:11:44, 1.10it/s][2025-04-28 19:50:05] (step=0029825) Train Loss: 5.7437, Train Steps/Sec: 1.12 + 33%|████████████████████████████████████████████▊ | 29849/89905 [9:25:41<61:16:39, 3.67s/it][2025-04-28 19:50:45] (step=0029850) Train Loss: 5.7620, Train Steps/Sec: 0.63 + 33%|████████████████████████████████████████████▊ | 29874/89905 [9:26:23<39:57:52, 2.40s/it][2025-04-28 19:51:27] (step=0029875) Train Loss: 5.7785, Train Steps/Sec: 0.60 + 33%|████████████████████████████████████████████▉ | 29899/89905 [9:26:55<15:20:03, 1.09it/s][2025-04-28 19:51:58] (step=0029900) Train Loss: 5.7832, Train Steps/Sec: 0.79 + 33%|████████████████████████████████████████████▉ | 29924/89905 [9:27:26<16:04:12, 1.04it/s][2025-04-28 19:52:30] (step=0029925) Train Loss: 5.7612, Train Steps/Sec: 0.79 + 33%|████████████████████████████████████████████▉ | 29949/89905 [9:27:48<14:46:06, 1.13it/s][2025-04-28 19:52:52] (step=0029950) Train Loss: 5.7649, Train Steps/Sec: 1.12 + 33%|█████████████████████████████████████████████ | 29974/89905 [9:28:11<14:43:31, 1.13it/s][2025-04-28 19:53:14] (step=0029975) Train Loss: 5.7484, Train Steps/Sec: 1.12 + 33%|█████████████████████████████████████████████ | 29999/89905 [9:28:33<14:45:48, 1.13it/s][2025-04-28 19:53:37] (step=0030000) Train Loss: 5.7199, Train Steps/Sec: 1.12 +You are using a model of type blip-2 to instantiate a model of type instructblip. This is not supported for all configurations of models and can yield errors. +[2025-04-28 19:53:37] text_config is None. Initializing the text config with default values (`OPTConfig`). +model_name_or_path:/tmp/haozhezhao/model/blip2-flan-t5-xl +image_place_holder: +tokenizer length after expend 32101 +tokenizer length before expend 32100 +100%|███████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████| 6/6 [06:01<00:00, 60.24s/it] +[2025-04-28 20:00:57] Finish Eval in 30000 steps...████████████████████████████████████████████████████████████████████| 6/6 [06:00<00:00, 60.05s/it] +processing left-padding... +processing left-padding... +processing left-padding... +processing left-padding... +processing left-padding... +processing left-padding... +[2025-04-28 20:01:17] Saved checkpoint to checkpoint/X2I_UltraEdiit_fourmask_LLava_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_filtered_objects_4mask_1280_context_after_stage1_5e4/000-GPT-XL/checkpoints/0030000.pt +[2025-04-28 20:01:19] Removed old checkpoint: checkpoint/X2I_UltraEdiit_fourmask_LLava_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_filtered_objects_4mask_1280_context_after_stage1_5e4/000-GPT-XL/checkpoints/0028000.pt + 33%|█████████████████████████████████████████████ | 30024/89905 [9:36:38<15:37:07, 1.06it/s][2025-04-28 20:01:41] (step=0030025) Train Loss: 5.8017, Train Steps/Sec: 0.05 + 33%|█████████████████████████████████████████████ | 30049/89905 [9:37:00<14:56:24, 1.11it/s][2025-04-28 20:02:04] (step=0030050) Train Loss: 5.7140, Train Steps/Sec: 1.12 + 33%|█████████████████████████████████████████████▏ | 30074/89905 [9:37:23<14:51:42, 1.12it/s][2025-04-28 20:02:26] (step=0030075) Train Loss: 5.7819, Train Steps/Sec: 1.12 + 33%|█████████████████████████████████████████████▏ | 30099/89905 [9:37:45<14:48:22, 1.12it/s][2025-04-28 20:02:48] (step=0030100) Train Loss: 5.7306, Train Steps/Sec: 1.12 + 34%|█████████████████████████████████████████████▏ | 30124/89905 [9:38:07<14:44:37, 1.13it/s][2025-04-28 20:03:11] (step=0030125) Train Loss: 5.6795, Train Steps/Sec: 1.12 + 34%|█████████████████████████████████████████████▎ | 30149/89905 [9:38:30<14:39:47, 1.13it/s][2025-04-28 20:03:33] (step=0030150) Train Loss: 5.7199, Train Steps/Sec: 1.12 + 34%|█████████████████████████████████████████████▎ | 30174/89905 [9:38:52<14:42:05, 1.13it/s][2025-04-28 20:03:55] (step=0030175) Train Loss: 5.7943, Train Steps/Sec: 1.12 + 34%|█████████████████████████████████████████████▎ | 30199/89905 [9:39:14<14:40:46, 1.13it/s][2025-04-28 20:04:18] (step=0030200) Train Loss: 5.7593, Train Steps/Sec: 1.11 + 34%|█████████████████████████████████████████████▍ | 30224/89905 [9:39:37<15:03:21, 1.10it/s][2025-04-28 20:04:40] (step=0030225) Train Loss: 5.7964, Train Steps/Sec: 1.12 + 34%|█████████████████████████████████████████████▍ | 30249/89905 [9:39:59<14:54:47, 1.11it/s][2025-04-28 20:05:03] (step=0030250) Train Loss: 5.7190, Train Steps/Sec: 1.11 + 34%|█████████████████████████████████████████████▍ | 30274/89905 [9:40:21<14:46:09, 1.12it/s][2025-04-28 20:05:25] (step=0030275) Train Loss: 5.7197, Train Steps/Sec: 1.12 + 34%|█████████████████████████████████████████████▍ | 30299/89905 [9:40:44<14:42:53, 1.13it/s][2025-04-28 20:05:47] (step=0030300) Train Loss: 5.8365, Train Steps/Sec: 1.12 + 34%|█████████████████████████████████████████████▌ | 30324/89905 [9:41:06<14:40:42, 1.13it/s][2025-04-28 20:06:09] (step=0030325) Train Loss: 5.7860, Train Steps/Sec: 1.12 + 34%|█████████████████████████████████████████████▌ | 30349/89905 [9:41:28<14:36:18, 1.13it/s][2025-04-28 20:06:32] (step=0030350) Train Loss: 5.7517, Train Steps/Sec: 1.12 + 34%|█████████████████████████████████████████████▌ | 30374/89905 [9:41:51<14:38:33, 1.13it/s][2025-04-28 20:06:54] (step=0030375) Train Loss: 5.6922, Train Steps/Sec: 1.12 + 34%|█████████████████████████████████████████████▋ | 30399/89905 [9:42:13<14:36:19, 1.13it/s][2025-04-28 20:07:16] (step=0030400) Train Loss: 5.7484, Train Steps/Sec: 1.11 + 34%|█████████████████████████████████████████████▋ | 30424/89905 [9:42:35<15:02:11, 1.10it/s][2025-04-28 20:07:39] (step=0030425) Train Loss: 5.6814, Train Steps/Sec: 1.12 + 34%|█████████████████████████████████████████████▋ | 30449/89905 [9:42:58<14:49:29, 1.11it/s][2025-04-28 20:08:01] (step=0030450) Train Loss: 5.7319, Train Steps/Sec: 1.12 + 34%|█████████████████████████████████████████████▊ | 30474/89905 [9:43:20<14:43:39, 1.12it/s][2025-04-28 20:08:23] (step=0030475) Train Loss: 5.8237, Train Steps/Sec: 1.12 + 34%|█████████████████████████████████████████████▊ | 30499/89905 [9:43:42<14:45:34, 1.12it/s][2025-04-28 20:08:46] (step=0030500) Train Loss: 5.7452, Train Steps/Sec: 1.12 + 34%|█████████████████████████████████████████████▊ | 30524/89905 [9:44:05<14:37:16, 1.13it/s][2025-04-28 20:09:08] (step=0030525) Train Loss: 5.7061, Train Steps/Sec: 1.12 + 34%|█████████████████████████████████████████████▊ | 30549/89905 [9:44:27<14:36:18, 1.13it/s][2025-04-28 20:09:30] (step=0030550) Train Loss: 5.7481, Train Steps/Sec: 1.12 + 34%|█████████████████████████████████████████████▉ | 30574/89905 [9:44:49<14:32:03, 1.13it/s][2025-04-28 20:09:53] (step=0030575) Train Loss: 5.7504, Train Steps/Sec: 1.12 + 34%|█████████████████████████████████████████████▉ | 30599/89905 [9:45:11<14:34:16, 1.13it/s][2025-04-28 20:10:15] (step=0030600) Train Loss: 5.7360, Train Steps/Sec: 1.12 + 34%|█████████████████████████████████████████████▉ | 30624/89905 [9:45:34<15:00:23, 1.10it/s][2025-04-28 20:10:37] (step=0030625) Train Loss: 5.6676, Train Steps/Sec: 1.12 + 34%|██████████████████████████████████████████████ | 30649/89905 [9:45:56<14:46:48, 1.11it/s][2025-04-28 20:11:00] (step=0030650) Train Loss: 5.7608, Train Steps/Sec: 1.12 + 34%|██████████████████████████████████████████████ | 30674/89905 [9:46:18<14:46:37, 1.11it/s][2025-04-28 20:11:22] (step=0030675) Train Loss: 5.6693, Train Steps/Sec: 1.12 + 34%|██████████████████████████████████████████████ | 30699/89905 [9:46:41<14:40:47, 1.12it/s][2025-04-28 20:11:44] (step=0030700) Train Loss: 5.7223, Train Steps/Sec: 1.12 + 34%|██████████████████████████████████████████████▏ | 30724/89905 [9:47:03<14:36:13, 1.13it/s][2025-04-28 20:12:07] (step=0030725) Train Loss: 5.7608, Train Steps/Sec: 1.12 + 34%|██████████████████████████████████████████████▏ | 30749/89905 [9:47:25<14:31:34, 1.13it/s][2025-04-28 20:12:29] (step=0030750) Train Loss: 5.7139, Train Steps/Sec: 1.12 + 34%|██████████████████████████████████████████████▏ | 30774/89905 [9:47:48<14:36:23, 1.12it/s][2025-04-28 20:12:51] (step=0030775) Train Loss: 5.7359, Train Steps/Sec: 1.12 + 34%|██████████████████████████████████████████████▏ | 30799/89905 [9:48:10<14:32:55, 1.13it/s][2025-04-28 20:13:14] (step=0030800) Train Loss: 5.7503, Train Steps/Sec: 1.12 + 34%|██████████████████████████████████████████████▎ | 30824/89905 [9:48:33<14:57:52, 1.10it/s][2025-04-28 20:13:36] (step=0030825) Train Loss: 5.7402, Train Steps/Sec: 1.12 + 34%|██████████████████████████████████████████████▎ | 30849/89905 [9:48:55<14:45:02, 1.11it/s][2025-04-28 20:13:58] (step=0030850) Train Loss: 5.7258, Train Steps/Sec: 1.12 + 34%|██████████████████████████████████████████████▎ | 30874/89905 [9:49:17<14:43:17, 1.11it/s][2025-04-28 20:14:21] (step=0030875) Train Loss: 5.7663, Train Steps/Sec: 1.12 + 34%|██████████████████████████████████████████████▍ | 30899/89905 [9:49:39<14:39:47, 1.12it/s][2025-04-28 20:14:43] (step=0030900) Train Loss: 5.6854, Train Steps/Sec: 1.12 + 34%|██████████████████████████████████████████████▍ | 30924/89905 [9:50:02<14:33:49, 1.12it/s][2025-04-28 20:15:05] (step=0030925) Train Loss: 5.7526, Train Steps/Sec: 1.12 + 34%|██████████████████████████████████████████████▍ | 30949/89905 [9:50:24<14:29:36, 1.13it/s][2025-04-28 20:15:28] (step=0030950) Train Loss: 5.7828, Train Steps/Sec: 1.12 + 34%|██████████████████████████████████████████████▌ | 30974/89905 [9:50:46<14:30:02, 1.13it/s][2025-04-28 20:15:50] (step=0030975) Train Loss: 5.7292, Train Steps/Sec: 1.12 + 34%|██████████████████████████████████████████████▌ | 30999/89905 [9:51:09<14:27:15, 1.13it/s][2025-04-28 20:16:12] (step=0031000) Train Loss: 5.6985, Train Steps/Sec: 1.12 + 35%|██████████████████████████████████████████████▌ | 31024/89905 [9:51:31<14:53:19, 1.10it/s][2025-04-28 20:16:34] (step=0031025) Train Loss: 5.8217, Train Steps/Sec: 1.12 + 35%|██████████████████████████████████████████████▌ | 31049/89905 [9:51:53<14:47:27, 1.11it/s][2025-04-28 20:16:57] (step=0031050) Train Loss: 5.7229, Train Steps/Sec: 1.12 + 35%|██████████████████████████████████████████████▋ | 31074/89905 [9:52:16<14:36:40, 1.12it/s][2025-04-28 20:17:19] (step=0031075) Train Loss: 5.7492, Train Steps/Sec: 1.12 + 35%|██████████████████████████████████████████████▋ | 31099/89905 [9:52:38<14:33:15, 1.12it/s][2025-04-28 20:17:41] (step=0031100) Train Loss: 5.7321, Train Steps/Sec: 1.12 + 35%|██████████████████████████████████████████████▋ | 31124/89905 [9:53:00<14:32:32, 1.12it/s][2025-04-28 20:18:04] (step=0031125) Train Loss: 5.7653, Train Steps/Sec: 1.12 + 35%|██████████████████████████████████████████████▊ | 31149/89905 [9:53:23<14:29:13, 1.13it/s][2025-04-28 20:18:26] (step=0031150) Train Loss: 5.7325, Train Steps/Sec: 1.12 + 35%|██████████████████████████████████████████████▊ | 31174/89905 [9:53:45<14:25:51, 1.13it/s][2025-04-28 20:18:48] (step=0031175) Train Loss: 5.7317, Train Steps/Sec: 1.12 + 35%|██████████████████████████████████████████████▊ | 31199/89905 [9:54:07<14:27:24, 1.13it/s][2025-04-28 20:19:11] (step=0031200) Train Loss: 5.7444, Train Steps/Sec: 1.12 + 35%|██████████████████████████████████████████████▉ | 31224/89905 [9:54:30<14:50:22, 1.10it/s][2025-04-28 20:19:33] (step=0031225) Train Loss: 5.7337, Train Steps/Sec: 1.12 + 35%|██████████████████████████████████████████████▉ | 31249/89905 [9:54:52<14:38:42, 1.11it/s][2025-04-28 20:19:55] (step=0031250) Train Loss: 5.7173, Train Steps/Sec: 1.12 + 35%|██████████████████████████████████████████████▉ | 31274/89905 [9:55:14<14:34:04, 1.12it/s][2025-04-28 20:20:18] (step=0031275) Train Loss: 5.7721, Train Steps/Sec: 1.12 + 35%|██████████████████████████████████████████████▉ | 31299/89905 [9:55:37<14:28:16, 1.12it/s][2025-04-28 20:20:40] (step=0031300) Train Loss: 5.7619, Train Steps/Sec: 1.12 + 35%|███████████████████████████████████████████████ | 31324/89905 [9:55:59<14:26:51, 1.13it/s][2025-04-28 20:21:02] (step=0031325) Train Loss: 5.7400, Train Steps/Sec: 1.12 + 35%|███████████████████████████████████████████████ | 31349/89905 [9:56:21<14:25:48, 1.13it/s][2025-04-28 20:21:25] (step=0031350) Train Loss: 5.7561, Train Steps/Sec: 1.12 + 35%|███████████████████████████████████████████████ | 31374/89905 [9:56:43<14:20:53, 1.13it/s][2025-04-28 20:21:47] (step=0031375) Train Loss: 5.7455, Train Steps/Sec: 1.12 + 35%|███████████████████████████████████████████████▏ | 31399/89905 [9:57:06<14:20:36, 1.13it/s][2025-04-28 20:22:09] (step=0031400) Train Loss: 5.7293, Train Steps/Sec: 1.12 + 35%|███████████████████████████████████████████████▏ | 31424/89905 [9:57:28<14:49:30, 1.10it/s][2025-04-28 20:22:32] (step=0031425) Train Loss: 5.7178, Train Steps/Sec: 1.12 + 35%|███████████████████████████████████████████████▏ | 31449/89905 [9:57:50<14:32:19, 1.12it/s][2025-04-28 20:22:54] (step=0031450) Train Loss: 5.6944, Train Steps/Sec: 1.12 + 35%|███████████████████████████████████████████████▎ | 31474/89905 [9:58:13<14:30:31, 1.12it/s][2025-04-28 20:23:16] (step=0031475) Train Loss: 5.6916, Train Steps/Sec: 1.12 + 35%|███████████████████████████████████████████████▎ | 31499/89905 [9:58:35<14:26:28, 1.12it/s][2025-04-28 20:23:38] (step=0031500) Train Loss: 5.6788, Train Steps/Sec: 1.12 + 35%|███████████████████████████████████████████████▎ | 31524/89905 [9:58:57<14:23:48, 1.13it/s][2025-04-28 20:24:01] (step=0031525) Train Loss: 5.7317, Train Steps/Sec: 1.12 + 35%|███████████████████████████████████████████████▎ | 31549/89905 [9:59:20<14:21:49, 1.13it/s][2025-04-28 20:24:23] (step=0031550) Train Loss: 5.7233, Train Steps/Sec: 1.12 + 35%|███████████████████████████████████████████████▍ | 31574/89905 [9:59:42<14:19:29, 1.13it/s][2025-04-28 20:24:45] (step=0031575) Train Loss: 5.8137, Train Steps/Sec: 1.12 + 35%|███████████████████████████████████████████████ | 31599/89905 [10:00:04<14:19:19, 1.13it/s][2025-04-28 20:25:08] (step=0031600) Train Loss: 5.7100, Train Steps/Sec: 1.12 + 35%|███████████████████████████████████████████████▏ | 31624/89905 [10:00:27<14:44:48, 1.10it/s][2025-04-28 20:25:30] (step=0031625) Train Loss: 5.7066, Train Steps/Sec: 1.12 + 35%|███████████████████████████████████████████████▏ | 31649/89905 [10:00:49<14:32:24, 1.11it/s][2025-04-28 20:25:52] (step=0031650) Train Loss: 5.6537, Train Steps/Sec: 1.12 + 35%|███████████████████████████████████████████████▏ | 31674/89905 [10:01:11<14:26:41, 1.12it/s][2025-04-28 20:26:15] (step=0031675) Train Loss: 5.7534, Train Steps/Sec: 1.12 + 35%|███████████████████████████████████████████████▏ | 31699/89905 [10:01:34<14:26:30, 1.12it/s][2025-04-28 20:26:37] (step=0031700) Train Loss: 5.7189, Train Steps/Sec: 1.12 + 35%|███████████████████████████████████████████████▎ | 31724/89905 [10:01:56<14:17:17, 1.13it/s][2025-04-28 20:26:59] (step=0031725) Train Loss: 5.7719, Train Steps/Sec: 1.12 + 35%|███████████████████████████████████████████████▎ | 31749/89905 [10:02:18<14:20:49, 1.13it/s][2025-04-28 20:27:22] (step=0031750) Train Loss: 5.6816, Train Steps/Sec: 1.12 + 35%|███████████████████████████████████████████████▎ | 31774/89905 [10:02:40<14:19:40, 1.13it/s][2025-04-28 20:27:44] (step=0031775) Train Loss: 5.8111, Train Steps/Sec: 1.12 + 35%|███████████████████████████████████████████████▍ | 31799/89905 [10:03:03<14:16:38, 1.13it/s][2025-04-28 20:28:06] (step=0031800) Train Loss: 5.7296, Train Steps/Sec: 1.12 + 35%|███████████████████████████████████████████████▍ | 31824/89905 [10:03:25<14:42:25, 1.10it/s][2025-04-28 20:28:29] (step=0031825) Train Loss: 5.7066, Train Steps/Sec: 1.12 + 35%|███████████████████████████████████████████████▍ | 31849/89905 [10:03:47<14:31:59, 1.11it/s][2025-04-28 20:28:51] (step=0031850) Train Loss: 5.7197, Train Steps/Sec: 1.12 + 35%|███████████████████████████████████████████████▌ | 31874/89905 [10:04:10<14:25:18, 1.12it/s][2025-04-28 20:29:13] (step=0031875) Train Loss: 5.7455, Train Steps/Sec: 1.12 + 35%|███████████████████████████████████████████████▌ | 31899/89905 [10:04:32<14:22:54, 1.12it/s][2025-04-28 20:29:35] (step=0031900) Train Loss: 5.7541, Train Steps/Sec: 1.12 + 36%|███████████████████████████████████████████████▌ | 31924/89905 [10:04:54<14:17:07, 1.13it/s][2025-04-28 20:29:58] (step=0031925) Train Loss: 5.7562, Train Steps/Sec: 1.12 + 36%|███████████████████████████████████████████████▌ | 31949/89905 [10:05:17<14:16:26, 1.13it/s][2025-04-28 20:30:20] (step=0031950) Train Loss: 5.7626, Train Steps/Sec: 1.12 + 36%|███████████████████████████████████████████████▋ | 31974/89905 [10:05:39<14:09:28, 1.14it/s][2025-04-28 20:30:42] (step=0031975) Train Loss: 5.7301, Train Steps/Sec: 1.12 + 36%|███████████████████████████████████████████████▋ | 31999/89905 [10:06:01<14:11:15, 1.13it/s][2025-04-28 20:31:05] (step=0032000) Train Loss: 5.7126, Train Steps/Sec: 1.12 +You are using a model of type blip-2 to instantiate a model of type instructblip. This is not supported for all configurations of models and can yield errors. +[2025-04-28 20:31:05] text_config is None. Initializing the text config with default values (`OPTConfig`). +model_name_or_path:/tmp/haozhezhao/model/blip2-flan-t5-xl +image_place_holder: +tokenizer length after expend 32101 +tokenizer length before expend 32100 +100%|███████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████| 6/6 [05:59<00:00, 59.94s/it] +[2025-04-28 20:38:22] Finish Eval in 32000 steps...████████████████████████████████████████████████████████████████████| 6/6 [05:59<00:00, 59.79s/it] +processing left-padding... +processing left-padding... +processing left-padding... +processing left-padding... +processing left-padding... +processing left-padding... +[2025-04-28 20:38:41] Saved checkpoint to checkpoint/X2I_UltraEdiit_fourmask_LLava_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_filtered_objects_4mask_1280_context_after_stage1_5e4/000-GPT-XL/checkpoints/0032000.pt +[2025-04-28 20:38:43] Removed old checkpoint: checkpoint/X2I_UltraEdiit_fourmask_LLava_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_filtered_objects_4mask_1280_context_after_stage1_5e4/000-GPT-XL/checkpoints/0030000.pt + 36%|███████████████████████████████████████████████▋ | 32024/89905 [10:14:02<15:03:40, 1.07it/s][2025-04-28 20:39:06] (step=0032025) Train Loss: 5.6773, Train Steps/Sec: 0.05 + 36%|███████████████████████████████████████████████▊ | 32049/89905 [10:14:25<14:26:25, 1.11it/s][2025-04-28 20:39:28] (step=0032050) Train Loss: 5.8275, Train Steps/Sec: 1.12 + 36%|███████████████████████████████████████████████▊ | 32074/89905 [10:14:47<14:20:46, 1.12it/s][2025-04-28 20:39:50] (step=0032075) Train Loss: 5.7273, Train Steps/Sec: 1.12 + 36%|███████████████████████████████████████████████▊ | 32099/89905 [10:15:09<14:19:29, 1.12it/s][2025-04-28 20:40:13] (step=0032100) Train Loss: 5.8341, Train Steps/Sec: 1.12 + 36%|███████████████████████████████████████████████▉ | 32124/89905 [10:15:31<14:17:30, 1.12it/s][2025-04-28 20:40:35] (step=0032125) Train Loss: 5.7492, Train Steps/Sec: 1.11 + 36%|███████████████████████████████████████████████▉ | 32149/89905 [10:15:54<14:11:12, 1.13it/s][2025-04-28 20:40:57] (step=0032150) Train Loss: 5.7623, Train Steps/Sec: 1.12 + 36%|███████████████████████████████████████████████▉ | 32174/89905 [10:16:16<14:09:21, 1.13it/s][2025-04-28 20:41:20] (step=0032175) Train Loss: 5.7361, Train Steps/Sec: 1.12 + 36%|███████████████████████████████████████████████▉ | 32199/89905 [10:16:38<14:07:45, 1.13it/s][2025-04-28 20:41:42] (step=0032200) Train Loss: 5.7357, Train Steps/Sec: 1.12 + 36%|████████████████████████████████████████████████ | 32224/89905 [10:17:01<14:33:30, 1.10it/s][2025-04-28 20:42:04] (step=0032225) Train Loss: 5.6968, Train Steps/Sec: 1.12 + 36%|████████████████████████████████████████████████ | 32249/89905 [10:17:23<14:26:03, 1.11it/s][2025-04-28 20:42:27] (step=0032250) Train Loss: 5.6980, Train Steps/Sec: 1.12 + 36%|████████████████████████████████████████████████ | 32274/89905 [10:17:45<14:19:27, 1.12it/s][2025-04-28 20:42:49] (step=0032275) Train Loss: 5.7434, Train Steps/Sec: 1.12 + 36%|████████████████████████████████████████████████▏ | 32299/89905 [10:18:08<14:16:40, 1.12it/s][2025-04-28 20:43:11] (step=0032300) Train Loss: 5.7372, Train Steps/Sec: 1.12 + 36%|████████████████████████████████████████████████▏ | 32324/89905 [10:18:30<14:10:28, 1.13it/s][2025-04-28 20:43:33] (step=0032325) Train Loss: 5.7245, Train Steps/Sec: 1.12 + 36%|████████████████████████████████████████████████▏ | 32349/89905 [10:18:52<14:10:37, 1.13it/s][2025-04-28 20:43:56] (step=0032350) Train Loss: 5.7384, Train Steps/Sec: 1.12 + 36%|████████████████████████████████████████████████▎ | 32374/89905 [10:19:14<14:06:58, 1.13it/s][2025-04-28 20:44:18] (step=0032375) Train Loss: 5.7279, Train Steps/Sec: 1.12 + 36%|████████████████████████████████████████████████▎ | 32399/89905 [10:19:37<14:04:41, 1.13it/s][2025-04-28 20:44:40] (step=0032400) Train Loss: 5.6804, Train Steps/Sec: 1.12 + 36%|████████████████████████████████████████████████▎ | 32424/89905 [10:19:59<14:32:03, 1.10it/s][2025-04-28 20:45:03] (step=0032425) Train Loss: 5.7007, Train Steps/Sec: 1.12 + 36%|████████████████████████████████████████████████▎ | 32449/89905 [10:20:21<14:19:53, 1.11it/s][2025-04-28 20:45:25] (step=0032450) Train Loss: 5.6885, Train Steps/Sec: 1.12 + 36%|████████████████████████████████████████████████▍ | 32474/89905 [10:20:44<14:14:46, 1.12it/s][2025-04-28 20:45:47] (step=0032475) Train Loss: 5.7891, Train Steps/Sec: 1.12 + 36%|████████████████████████████████████████████████▍ | 32499/89905 [10:21:06<14:09:29, 1.13it/s][2025-04-28 20:46:09] (step=0032500) Train Loss: 5.7389, Train Steps/Sec: 1.12 + 36%|████████████████████████████████████████████████▍ | 32524/89905 [10:21:28<14:06:39, 1.13it/s][2025-04-28 20:46:32] (step=0032525) Train Loss: 5.7544, Train Steps/Sec: 1.12 + 36%|████████████████████████████████████████████████▌ | 32549/89905 [10:21:50<14:06:09, 1.13it/s][2025-04-28 20:46:54] (step=0032550) Train Loss: 5.7177, Train Steps/Sec: 1.12 + 36%|████████████████████████████████████████████████▌ | 32574/89905 [10:22:13<14:07:46, 1.13it/s][2025-04-28 20:47:16] (step=0032575) Train Loss: 5.6903, Train Steps/Sec: 1.12 + 36%|████████████████████████████████████████████████▌ | 32599/89905 [10:22:35<14:02:41, 1.13it/s][2025-04-28 20:47:39] (step=0032600) Train Loss: 5.7532, Train Steps/Sec: 1.12 + 36%|████████████████████████████████████████████████▌ | 32624/89905 [10:22:57<14:28:28, 1.10it/s][2025-04-28 20:48:01] (step=0032625) Train Loss: 5.7229, Train Steps/Sec: 1.12 + 36%|████████████████████████████████████████████████▋ | 32649/89905 [10:23:20<14:18:09, 1.11it/s][2025-04-28 20:48:23] (step=0032650) Train Loss: 5.7315, Train Steps/Sec: 1.12 + 36%|████████████████████████████████████████████████▋ | 32674/89905 [10:23:42<14:12:09, 1.12it/s][2025-04-28 20:48:45] (step=0032675) Train Loss: 5.6599, Train Steps/Sec: 1.12 + 36%|████████████████████████████████████████████████▋ | 32699/89905 [10:24:04<14:07:46, 1.12it/s][2025-04-28 20:49:08] (step=0032700) Train Loss: 5.7817, Train Steps/Sec: 1.12 + 36%|████████████████████████████████████████████████▊ | 32724/89905 [10:24:27<14:05:38, 1.13it/s][2025-04-28 20:49:30] (step=0032725) Train Loss: 5.6754, Train Steps/Sec: 1.12 + 36%|████████████████████████████████████████████████▊ | 32749/89905 [10:24:49<14:06:38, 1.13it/s][2025-04-28 20:49:52] (step=0032750) Train Loss: 5.7331, Train Steps/Sec: 1.12 + 36%|████████████████████████████████████████████████▊ | 32774/89905 [10:25:11<14:01:25, 1.13it/s][2025-04-28 20:50:15] (step=0032775) Train Loss: 5.7383, Train Steps/Sec: 1.12 + 36%|████████████████████████████████████████████████▉ | 32799/89905 [10:25:33<14:01:33, 1.13it/s][2025-04-28 20:50:37] (step=0032800) Train Loss: 5.6960, Train Steps/Sec: 1.12 + 37%|████████████████████████████████████████████████▉ | 32824/89905 [10:25:56<14:24:22, 1.10it/s][2025-04-28 20:50:59] (step=0032825) Train Loss: 5.7684, Train Steps/Sec: 1.12 + 37%|████████████████████████████████████████████████▉ | 32849/89905 [10:26:18<14:16:50, 1.11it/s][2025-04-28 20:51:22] (step=0032850) Train Loss: 5.6890, Train Steps/Sec: 1.12 + 37%|████████████████████████████████████████████████▉ | 32874/89905 [10:26:40<14:12:12, 1.12it/s][2025-04-28 20:51:44] (step=0032875) Train Loss: 5.7248, Train Steps/Sec: 1.12 + 37%|█████████████████████████████████████████████████ | 32899/89905 [10:27:03<14:03:34, 1.13it/s][2025-04-28 20:52:06] (step=0032900) Train Loss: 5.6911, Train Steps/Sec: 1.12 + 37%|█████████████████████████████████████████████████ | 32924/89905 [10:27:25<14:02:03, 1.13it/s][2025-04-28 20:52:28] (step=0032925) Train Loss: 5.7395, Train Steps/Sec: 1.12 + 37%|█████████████████████████████████████████████████ | 32949/89905 [10:27:47<14:04:20, 1.12it/s][2025-04-28 20:52:51] (step=0032950) Train Loss: 5.7640, Train Steps/Sec: 1.12 + 37%|█████████████████████████████████████████████████▏ | 32974/89905 [10:28:10<13:58:57, 1.13it/s][2025-04-28 20:53:13] (step=0032975) Train Loss: 5.7362, Train Steps/Sec: 1.12 + 37%|█████████████████████████████████████████████████▏ | 32999/89905 [10:28:32<13:56:57, 1.13it/s][2025-04-28 20:53:35] (step=0033000) Train Loss: 5.8173, Train Steps/Sec: 1.12 + 37%|█████████████████████████████████████████████████▏ | 33024/89905 [10:28:54<14:23:21, 1.10it/s][2025-04-28 20:53:58] (step=0033025) Train Loss: 5.7319, Train Steps/Sec: 1.12 + 37%|█████████████████████████████████████████████████▎ | 33049/89905 [10:29:17<14:11:30, 1.11it/s][2025-04-28 20:54:20] (step=0033050) Train Loss: 5.7222, Train Steps/Sec: 1.12 + 37%|█████████████████████████████████████████████████▎ | 33074/89905 [10:29:39<14:08:33, 1.12it/s][2025-04-28 20:54:42] (step=0033075) Train Loss: 5.7329, Train Steps/Sec: 1.12 + 37%|█████████████████████████████████████████████████▎ | 33099/89905 [10:30:01<14:00:43, 1.13it/s][2025-04-28 20:55:05] (step=0033100) Train Loss: 5.7163, Train Steps/Sec: 1.12 + 37%|█████████████████████████████████████████████████▎ | 33124/89905 [10:30:23<14:00:14, 1.13it/s][2025-04-28 20:55:27] (step=0033125) Train Loss: 5.7226, Train Steps/Sec: 1.12 + 37%|█████████████████████████████████████████████████▍ | 33149/89905 [10:30:46<13:55:37, 1.13it/s][2025-04-28 20:55:49] (step=0033150) Train Loss: 5.7942, Train Steps/Sec: 1.12 + 37%|█████████████████████████████████████████████████▍ | 33174/89905 [10:31:08<13:58:25, 1.13it/s][2025-04-28 20:56:11] (step=0033175) Train Loss: 5.7732, Train Steps/Sec: 1.12 + 37%|█████████████████████████████████████████████████▍ | 33199/89905 [10:31:30<13:53:44, 1.13it/s][2025-04-28 20:56:34] (step=0033200) Train Loss: 5.7491, Train Steps/Sec: 1.12 + 37%|█████████████████████████████████████████████████▌ | 33224/89905 [10:31:53<14:18:24, 1.10it/s][2025-04-28 20:56:56] (step=0033225) Train Loss: 5.6619, Train Steps/Sec: 1.12 + 37%|█████████████████████████████████████████████████▌ | 33249/89905 [10:32:15<14:08:43, 1.11it/s][2025-04-28 20:57:18] (step=0033250) Train Loss: 5.6865, Train Steps/Sec: 1.12 + 37%|█████████████████████████████████████████████████▌ | 33274/89905 [10:32:37<14:01:33, 1.12it/s][2025-04-28 20:57:41] (step=0033275) Train Loss: 5.7316, Train Steps/Sec: 1.12 + 37%|█████████████████████████████████████████████████▋ | 33299/89905 [10:32:59<13:57:26, 1.13it/s][2025-04-28 20:58:03] (step=0033300) Train Loss: 5.7431, Train Steps/Sec: 1.12 + 37%|█████████████████████████████████████████████████▋ | 33324/89905 [10:33:22<13:54:38, 1.13it/s][2025-04-28 20:58:25] (step=0033325) Train Loss: 5.7587, Train Steps/Sec: 1.12 + 37%|█████████████████████████████████████████████████▋ | 33349/89905 [10:33:44<13:56:28, 1.13it/s][2025-04-28 20:58:47] (step=0033350) Train Loss: 5.7638, Train Steps/Sec: 1.12 + 37%|█████████████████████████████████████████████████▋ | 33374/89905 [10:34:06<13:51:38, 1.13it/s][2025-04-28 20:59:10] (step=0033375) Train Loss: 5.7484, Train Steps/Sec: 1.12 + 37%|█████████████████████████████████████████████████▊ | 33399/89905 [10:34:29<13:54:43, 1.13it/s][2025-04-28 20:59:32] (step=0033400) Train Loss: 5.7428, Train Steps/Sec: 1.12 + 37%|█████████████████████████████████████████████████▊ | 33424/89905 [10:35:09<29:13:47, 1.86s/it][2025-04-28 21:00:12] (step=0033425) Train Loss: 5.7329, Train Steps/Sec: 0.62 + 37%|█████████████████████████████████████████████████▊ | 33449/89905 [10:35:51<17:55:54, 1.14s/it][2025-04-28 21:00:54] (step=0033450) Train Loss: 5.7259, Train Steps/Sec: 0.60 + 37%|█████████████████████████████████████████████████▉ | 33474/89905 [10:36:22<14:06:20, 1.11it/s][2025-04-28 21:01:25] (step=0033475) Train Loss: 5.6997, Train Steps/Sec: 0.80 + 37%|█████████████████████████████████████████████████▉ | 33499/89905 [10:36:53<13:54:43, 1.13it/s][2025-04-28 21:01:57] (step=0033500) Train Loss: 5.8014, Train Steps/Sec: 0.80 + 37%|█████████████████████████████████████████████████▉ | 33524/89905 [10:37:15<13:54:11, 1.13it/s][2025-04-28 21:02:19] (step=0033525) Train Loss: 5.7161, Train Steps/Sec: 1.12 + 37%|██████████████████████████████████████████████████ | 33549/89905 [10:37:38<13:53:05, 1.13it/s][2025-04-28 21:02:41] (step=0033550) Train Loss: 5.7739, Train Steps/Sec: 1.12 + 37%|██████████████████████████████████████████████████ | 33574/89905 [10:38:00<13:52:10, 1.13it/s][2025-04-28 21:03:03] (step=0033575) Train Loss: 5.7089, Train Steps/Sec: 1.12 + 37%|██████████████████████████████████████████████████ | 33599/89905 [10:38:22<13:48:10, 1.13it/s][2025-04-28 21:03:26] (step=0033600) Train Loss: 5.7108, Train Steps/Sec: 1.12 + 37%|██████████████████████████████████████████████████ | 33624/89905 [10:38:45<14:12:52, 1.10it/s][2025-04-28 21:03:48] (step=0033625) Train Loss: 5.7595, Train Steps/Sec: 1.13 + 37%|██████████████████████████████████████████████████▏ | 33649/89905 [10:39:07<14:04:07, 1.11it/s][2025-04-28 21:04:10] (step=0033650) Train Loss: 5.7198, Train Steps/Sec: 1.12 + 37%|██████████████████████████████████████████████████▏ | 33674/89905 [10:39:29<13:55:38, 1.12it/s][2025-04-28 21:04:33] (step=0033675) Train Loss: 5.6692, Train Steps/Sec: 1.12 + 37%|██████████████████████████████████████████████████▏ | 33699/89905 [10:39:52<13:56:07, 1.12it/s][2025-04-28 21:04:55] (step=0033700) Train Loss: 5.7307, Train Steps/Sec: 1.12 + 38%|██████████████████████████████████████████████████▎ | 33724/89905 [10:40:14<13:48:43, 1.13it/s][2025-04-28 21:05:17] (step=0033725) Train Loss: 5.7001, Train Steps/Sec: 1.12 + 38%|██████████████████████████████████████████████████▎ | 33749/89905 [10:40:36<13:48:58, 1.13it/s][2025-04-28 21:05:40] (step=0033750) Train Loss: 5.7337, Train Steps/Sec: 1.12 + 38%|██████████████████████████████████████████████████▎ | 33774/89905 [10:40:58<13:44:36, 1.13it/s][2025-04-28 21:06:02] (step=0033775) Train Loss: 5.7112, Train Steps/Sec: 1.12 + 38%|██████████████████████████████████████████████████▍ | 33799/89905 [10:41:21<13:51:15, 1.12it/s][2025-04-28 21:06:24] (step=0033800) Train Loss: 5.7075, Train Steps/Sec: 1.11 + 38%|██████████████████████████████████████████████████▍ | 33824/89905 [10:41:43<14:10:28, 1.10it/s][2025-04-28 21:06:47] (step=0033825) Train Loss: 5.7779, Train Steps/Sec: 1.12 + 38%|██████████████████████████████████████████████████▍ | 33849/89905 [10:42:05<14:02:04, 1.11it/s][2025-04-28 21:07:09] (step=0033850) Train Loss: 5.7571, Train Steps/Sec: 1.12 + 38%|██████████████████████████████████████████████████▍ | 33874/89905 [10:42:28<13:57:38, 1.11it/s][2025-04-28 21:07:31] (step=0033875) Train Loss: 5.7889, Train Steps/Sec: 1.12 + 38%|██████████████████████████████████████████████████▌ | 33899/89905 [10:42:50<13:50:27, 1.12it/s][2025-04-28 21:07:53] (step=0033900) Train Loss: 5.7674, Train Steps/Sec: 1.12 + 38%|██████████████████████████████████████████████████▌ | 33924/89905 [10:43:12<13:45:55, 1.13it/s][2025-04-28 21:08:16] (step=0033925) Train Loss: 5.7743, Train Steps/Sec: 1.12 + 38%|██████████████████████████████████████████████████▌ | 33949/89905 [10:43:35<13:46:40, 1.13it/s][2025-04-28 21:08:38] (step=0033950) Train Loss: 5.7238, Train Steps/Sec: 1.12 + 38%|██████████████████████████████████████████████████▋ | 33974/89905 [10:43:57<13:45:40, 1.13it/s][2025-04-28 21:09:00] (step=0033975) Train Loss: 5.7147, Train Steps/Sec: 1.12 + 38%|██████████████████████████████████████████████████▋ | 33999/89905 [10:44:19<13:42:39, 1.13it/s][2025-04-28 21:09:23] (step=0034000) Train Loss: 5.7533, Train Steps/Sec: 1.12 +You are using a model of type blip-2 to instantiate a model of type instructblip. This is not supported for all configurations of models and can yield errors. +[2025-04-28 21:09:23] text_config is None. Initializing the text config with default values (`OPTConfig`). +model_name_or_path:/tmp/haozhezhao/model/blip2-flan-t5-xl +image_place_holder: +tokenizer length after expend 32101 +tokenizer length before expend 32100 +100%|███████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████| 6/6 [06:00<00:00, 60.09s/it] +[2025-04-28 21:16:42] Finish Eval in 34000 steps...████████████████████████████████████████████████████████████████████| 6/6 [06:00<00:00, 59.89s/it] +processing left-padding... +processing left-padding... +processing left-padding... +processing left-padding... +processing left-padding... +processing left-padding... +[2025-04-28 21:17:02] Saved checkpoint to checkpoint/X2I_UltraEdiit_fourmask_LLava_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_filtered_objects_4mask_1280_context_after_stage1_5e4/000-GPT-XL/checkpoints/0034000.pt +[2025-04-28 21:17:04] Removed old checkpoint: checkpoint/X2I_UltraEdiit_fourmask_LLava_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_filtered_objects_4mask_1280_context_after_stage1_5e4/000-GPT-XL/checkpoints/0032000.pt + 38%|██████████████████████████████████████████████████▋ | 34024/89905 [10:52:23<14:36:49, 1.06it/s][2025-04-28 21:17:27] (step=0034025) Train Loss: 5.7266, Train Steps/Sec: 0.05 + 38%|██████████████████████████████████████████████████▋ | 34049/89905 [10:52:46<13:57:35, 1.11it/s][2025-04-28 21:17:49] (step=0034050) Train Loss: 5.7174, Train Steps/Sec: 1.12 + 38%|██████████████████████████████████████████████████▊ | 34074/89905 [10:53:08<13:51:40, 1.12it/s][2025-04-28 21:18:11] (step=0034075) Train Loss: 5.7087, Train Steps/Sec: 1.12 + 38%|██████████████████████████████████████████████████▊ | 34099/89905 [10:53:30<13:45:48, 1.13it/s][2025-04-28 21:18:33] (step=0034100) Train Loss: 5.7833, Train Steps/Sec: 1.12 + 38%|██████████████████████████████████████████████████▊ | 34124/89905 [10:53:52<13:46:24, 1.12it/s][2025-04-28 21:18:56] (step=0034125) Train Loss: 5.7219, Train Steps/Sec: 1.12 + 38%|██████████████████████████████████████████████████▉ | 34149/89905 [10:54:15<13:44:24, 1.13it/s][2025-04-28 21:19:18] (step=0034150) Train Loss: 5.7024, Train Steps/Sec: 1.12 + 38%|██████████████████████████████████████████████████▉ | 34174/89905 [10:54:37<13:41:35, 1.13it/s][2025-04-28 21:19:40] (step=0034175) Train Loss: 5.6929, Train Steps/Sec: 1.12 + 38%|██████████████████████████████████████████████████▉ | 34199/89905 [10:54:59<13:40:37, 1.13it/s][2025-04-28 21:20:03] (step=0034200) Train Loss: 5.7879, Train Steps/Sec: 1.12 + 38%|███████████████████████████████████████████████████ | 34224/89905 [10:55:22<14:05:01, 1.10it/s][2025-04-28 21:20:25] (step=0034225) Train Loss: 5.7549, Train Steps/Sec: 1.12 + 38%|███████████████████████████████████████████████████ | 34249/89905 [10:55:44<13:55:14, 1.11it/s][2025-04-28 21:20:47] (step=0034250) Train Loss: 5.6834, Train Steps/Sec: 1.12 + 38%|███████████████████████████████████████████████████ | 34274/89905 [10:56:06<13:47:46, 1.12it/s][2025-04-28 21:21:10] (step=0034275) Train Loss: 5.7592, Train Steps/Sec: 1.12 + 38%|███████████████████████████████████████████████████ | 34299/89905 [10:56:29<13:45:40, 1.12it/s][2025-04-28 21:21:32] (step=0034300) Train Loss: 5.7056, Train Steps/Sec: 1.12 + 38%|███████████████████████████████████████████████████▏ | 34324/89905 [10:56:51<13:41:16, 1.13it/s][2025-04-28 21:21:54] (step=0034325) Train Loss: 5.7661, Train Steps/Sec: 1.12 + 38%|███████████████████████████████████████████████████▏ | 34349/89905 [10:57:13<13:38:14, 1.13it/s][2025-04-28 21:22:17] (step=0034350) Train Loss: 5.7982, Train Steps/Sec: 1.12 + 38%|███████████████████████████████████████████████████▏ | 34374/89905 [10:57:35<13:42:15, 1.13it/s][2025-04-28 21:22:39] (step=0034375) Train Loss: 5.7112, Train Steps/Sec: 1.12 + 38%|███████████████████████████████████████████████████▎ | 34399/89905 [10:57:58<13:38:09, 1.13it/s][2025-04-28 21:23:01] (step=0034400) Train Loss: 5.7866, Train Steps/Sec: 1.12 + 38%|███████████████████████████████████████████████████▎ | 34424/89905 [10:58:20<14:02:00, 1.10it/s][2025-04-28 21:23:24] (step=0034425) Train Loss: 5.7331, Train Steps/Sec: 1.12 + 38%|███████████████████████████████████████████████████▎ | 34449/89905 [10:58:42<13:51:03, 1.11it/s][2025-04-28 21:23:46] (step=0034450) Train Loss: 5.7497, Train Steps/Sec: 1.12 + 38%|███████████████████████████████████████████████████▍ | 34474/89905 [10:59:05<13:47:28, 1.12it/s][2025-04-28 21:24:08] (step=0034475) Train Loss: 5.7143, Train Steps/Sec: 1.12 + 38%|███████████████████████████████████████████████████▍ | 34499/89905 [10:59:27<13:42:14, 1.12it/s][2025-04-28 21:24:30] (step=0034500) Train Loss: 5.6959, Train Steps/Sec: 1.12 + 38%|███████████████████████████████████████████████████▍ | 34524/89905 [10:59:49<13:39:12, 1.13it/s][2025-04-28 21:24:53] (step=0034525) Train Loss: 5.6838, Train Steps/Sec: 1.12 + 38%|███████████████████████████████████████████████████▍ | 34549/89905 [11:00:12<13:38:25, 1.13it/s][2025-04-28 21:25:15] (step=0034550) Train Loss: 5.6606, Train Steps/Sec: 1.12 + 38%|███████████████████████████████████████████████████▌ | 34574/89905 [11:00:34<13:39:45, 1.12it/s][2025-04-28 21:25:37] (step=0034575) Train Loss: 5.7517, Train Steps/Sec: 1.12 + 38%|███████████████████████████████████████████████████▌ | 34599/89905 [11:00:56<13:34:24, 1.13it/s][2025-04-28 21:26:00] (step=0034600) Train Loss: 5.7099, Train Steps/Sec: 1.12 + 39%|███████████████████████████████████████████████████▌ | 34624/89905 [11:01:19<14:00:27, 1.10it/s][2025-04-28 21:26:22] (step=0034625) Train Loss: 5.6893, Train Steps/Sec: 1.12 + 39%|███████████████████████████████████████████████████▋ | 34649/89905 [11:01:41<13:46:07, 1.11it/s][2025-04-28 21:26:44] (step=0034650) Train Loss: 5.6958, Train Steps/Sec: 1.12 + 39%|███████████████████████████████████████████████████▋ | 34674/89905 [11:02:03<13:50:25, 1.11it/s][2025-04-28 21:27:07] (step=0034675) Train Loss: 5.7545, Train Steps/Sec: 1.12 + 39%|███████████████████████████████████████████████████▋ | 34699/89905 [11:02:26<13:42:20, 1.12it/s][2025-04-28 21:27:29] (step=0034700) Train Loss: 5.6539, Train Steps/Sec: 1.12 + 39%|███████████████████████████████████████████████████▊ | 34724/89905 [11:02:48<13:33:39, 1.13it/s][2025-04-28 21:27:51] (step=0034725) Train Loss: 5.7361, Train Steps/Sec: 1.12 + 39%|███████████████████████████████████████████████████▊ | 34749/89905 [11:03:10<13:33:25, 1.13it/s][2025-04-28 21:28:14] (step=0034750) Train Loss: 5.6570, Train Steps/Sec: 1.12 + 39%|███████████████████████████████████████████████████▊ | 34774/89905 [11:03:32<13:30:54, 1.13it/s][2025-04-28 21:28:36] (step=0034775) Train Loss: 5.6457, Train Steps/Sec: 1.12 + 39%|███████████████████████████████████████████████████▊ | 34799/89905 [11:03:55<13:30:33, 1.13it/s][2025-04-28 21:28:58] (step=0034800) Train Loss: 5.7418, Train Steps/Sec: 1.11 + 39%|███████████████████████████████████████████████████▉ | 34824/89905 [11:04:17<13:56:08, 1.10it/s][2025-04-28 21:29:21] (step=0034825) Train Loss: 5.6786, Train Steps/Sec: 1.12 + 39%|███████████████████████████████████████████████████▉ | 34849/89905 [11:04:39<13:45:32, 1.11it/s][2025-04-28 21:29:43] (step=0034850) Train Loss: 5.6909, Train Steps/Sec: 1.12 + 39%|███████████████████████████████████████████████████▉ | 34874/89905 [11:05:02<13:38:31, 1.12it/s][2025-04-28 21:30:05] (step=0034875) Train Loss: 5.7391, Train Steps/Sec: 1.12 + 39%|████████████████████████████████████████████████████ | 34899/89905 [11:05:24<13:36:35, 1.12it/s][2025-04-28 21:30:27] (step=0034900) Train Loss: 5.7712, Train Steps/Sec: 1.12 + 39%|████████████████████████████████████████████████████ | 34924/89905 [11:05:46<13:31:12, 1.13it/s][2025-04-28 21:30:50] (step=0034925) Train Loss: 5.6927, Train Steps/Sec: 1.12 + 39%|████████████████████████████████████████████████████ | 34949/89905 [11:06:09<13:29:36, 1.13it/s][2025-04-28 21:31:12] (step=0034950) Train Loss: 5.7717, Train Steps/Sec: 1.12 + 39%|████████████████████████████████████████████████████▏ | 34974/89905 [11:06:31<13:31:25, 1.13it/s][2025-04-28 21:31:34] (step=0034975) Train Loss: 5.8055, Train Steps/Sec: 1.12 + 39%|████████████████████████████████████████████████████▏ | 34999/89905 [11:06:53<13:31:44, 1.13it/s][2025-04-28 21:31:57] (step=0035000) Train Loss: 5.7163, Train Steps/Sec: 1.12 + 39%|████████████████████████████████████████████████████▏ | 35024/89905 [11:07:16<13:52:14, 1.10it/s][2025-04-28 21:32:19] (step=0035025) Train Loss: 5.7175, Train Steps/Sec: 1.12 + 39%|████████████████████████████████████████████████████▏ | 35049/89905 [11:07:38<13:43:28, 1.11it/s][2025-04-28 21:32:41] (step=0035050) Train Loss: 5.7261, Train Steps/Sec: 1.12 + 39%|████████████████████████████████████████████████████▎ | 35074/89905 [11:08:00<13:36:14, 1.12it/s][2025-04-28 21:33:04] (step=0035075) Train Loss: 5.7208, Train Steps/Sec: 1.12 + 39%|████████████████████████████████████████████████████▎ | 35099/89905 [11:08:22<13:32:38, 1.12it/s][2025-04-28 21:33:26] (step=0035100) Train Loss: 5.7082, Train Steps/Sec: 1.12 + 39%|████████████████████████████████████████████████████▎ | 35124/89905 [11:08:45<13:33:12, 1.12it/s][2025-04-28 21:33:48] (step=0035125) Train Loss: 5.7542, Train Steps/Sec: 1.12 + 39%|████████████████████████████████████████████████████▍ | 35149/89905 [11:09:07<13:29:06, 1.13it/s][2025-04-28 21:34:10] (step=0035150) Train Loss: 5.7635, Train Steps/Sec: 1.12 + 39%|████████████████████████████████████████████████████▍ | 35174/89905 [11:09:29<13:25:38, 1.13it/s][2025-04-28 21:34:33] (step=0035175) Train Loss: 5.7260, Train Steps/Sec: 1.12 + 39%|████████████████████████████████████████████████████▍ | 35199/89905 [11:09:52<13:25:02, 1.13it/s][2025-04-28 21:34:55] (step=0035200) Train Loss: 5.7031, Train Steps/Sec: 1.12 + 39%|████████████████████████████████████████████████████▌ | 35224/89905 [11:10:14<13:50:43, 1.10it/s][2025-04-28 21:35:17] (step=0035225) Train Loss: 5.7705, Train Steps/Sec: 1.12 + 39%|████████████████████████████████████████████████████▌ | 35249/89905 [11:10:36<13:38:06, 1.11it/s][2025-04-28 21:35:40] (step=0035250) Train Loss: 5.7131, Train Steps/Sec: 1.12 + 39%|████████████████████████████████████████████████████▌ | 35274/89905 [11:10:59<13:32:33, 1.12it/s][2025-04-28 21:36:02] (step=0035275) Train Loss: 5.6865, Train Steps/Sec: 1.12 + 39%|████████████████████████████████████████████████████▌ | 35299/89905 [11:11:21<13:30:35, 1.12it/s][2025-04-28 21:36:24] (step=0035300) Train Loss: 5.7278, Train Steps/Sec: 1.12 + 39%|████████████████████████████████████████████████████▋ | 35324/89905 [11:11:43<13:28:48, 1.12it/s][2025-04-28 21:36:47] (step=0035325) Train Loss: 5.7047, Train Steps/Sec: 1.12 + 39%|████████████████████████████████████████████████████▋ | 35349/89905 [11:12:05<13:25:28, 1.13it/s][2025-04-28 21:37:09] (step=0035350) Train Loss: 5.7503, Train Steps/Sec: 1.12 + 39%|████████████████████████████████████████████████████▋ | 35374/89905 [11:12:28<13:23:48, 1.13it/s][2025-04-28 21:37:31] (step=0035375) Train Loss: 5.8148, Train Steps/Sec: 1.12 + 39%|████████████████████████████████████████████████████▊ | 35399/89905 [11:12:50<13:24:03, 1.13it/s][2025-04-28 21:37:54] (step=0035400) Train Loss: 5.7671, Train Steps/Sec: 1.12 + 39%|████████████████████████████████████████████████████▊ | 35424/89905 [11:13:12<13:47:52, 1.10it/s][2025-04-28 21:38:16] (step=0035425) Train Loss: 5.7650, Train Steps/Sec: 1.12 + 39%|████████████████████████████████████████████████████▊ | 35449/89905 [11:13:35<13:35:34, 1.11it/s][2025-04-28 21:38:38] (step=0035450) Train Loss: 5.7829, Train Steps/Sec: 1.12 + 39%|████████████████████████████████████████████████████▊ | 35474/89905 [11:13:57<13:34:20, 1.11it/s][2025-04-28 21:39:00] (step=0035475) Train Loss: 5.7103, Train Steps/Sec: 1.12 + 39%|████████████████████████████████████████████████████▉ | 35499/89905 [11:14:19<13:29:02, 1.12it/s][2025-04-28 21:39:23] (step=0035500) Train Loss: 5.7325, Train Steps/Sec: 1.12 + 40%|████████████████████████████████████████████████████▉ | 35524/89905 [11:14:42<13:24:48, 1.13it/s][2025-04-28 21:39:45] (step=0035525) Train Loss: 5.7215, Train Steps/Sec: 1.12 + 40%|████████████████████████████████████████████████████▉ | 35549/89905 [11:15:04<13:21:44, 1.13it/s][2025-04-28 21:40:07] (step=0035550) Train Loss: 5.6786, Train Steps/Sec: 1.12 + 40%|█████████████████████████████████████████████████████ | 35574/89905 [11:15:26<13:19:55, 1.13it/s][2025-04-28 21:40:30] (step=0035575) Train Loss: 5.7145, Train Steps/Sec: 1.12 + 40%|█████████████████████████████████████████████████████ | 35599/89905 [11:15:49<13:22:03, 1.13it/s][2025-04-28 21:40:52] (step=0035600) Train Loss: 5.7148, Train Steps/Sec: 1.12 + 40%|█████████████████████████████████████████████████████ | 35624/89905 [11:16:11<13:42:46, 1.10it/s][2025-04-28 21:41:14] (step=0035625) Train Loss: 5.7105, Train Steps/Sec: 1.12 + 40%|█████████████████████████████████████████████████████▏ | 35649/89905 [11:16:33<13:33:37, 1.11it/s][2025-04-28 21:41:37] (step=0035650) Train Loss: 5.6635, Train Steps/Sec: 1.12 + 40%|█████████████████████████████████████████████████████▏ | 35674/89905 [11:16:55<13:25:48, 1.12it/s][2025-04-28 21:41:59] (step=0035675) Train Loss: 5.6668, Train Steps/Sec: 1.12 + 40%|█████████████████████████████████████████████████████▏ | 35699/89905 [11:17:18<13:25:41, 1.12it/s][2025-04-28 21:42:21] (step=0035700) Train Loss: 5.7109, Train Steps/Sec: 1.12 + 40%|█████████████████████████████████████████████████████▏ | 35724/89905 [11:17:40<13:24:10, 1.12it/s][2025-04-28 21:42:44] (step=0035725) Train Loss: 5.7723, Train Steps/Sec: 1.12 + 40%|█████████████████████████████████████████████████████▎ | 35749/89905 [11:18:02<13:19:40, 1.13it/s][2025-04-28 21:43:06] (step=0035750) Train Loss: 5.7345, Train Steps/Sec: 1.12 + 40%|█████████████████████████████████████████████████████▎ | 35774/89905 [11:18:25<13:17:33, 1.13it/s][2025-04-28 21:43:28] (step=0035775) Train Loss: 5.6923, Train Steps/Sec: 1.12 + 40%|█████████████████████████████████████████████████████▎ | 35799/89905 [11:18:47<13:14:49, 1.13it/s][2025-04-28 21:43:50] (step=0035800) Train Loss: 5.6771, Train Steps/Sec: 1.12 + 40%|█████████████████████████████████████████████████████▍ | 35824/89905 [11:19:09<13:39:00, 1.10it/s][2025-04-28 21:44:13] (step=0035825) Train Loss: 5.7000, Train Steps/Sec: 1.12 + 40%|█████████████████████████████████████████████████████▍ | 35849/89905 [11:19:32<13:29:03, 1.11it/s][2025-04-28 21:44:35] (step=0035850) Train Loss: 5.6787, Train Steps/Sec: 1.12 + 40%|█████████████████████████████████████████████████████▍ | 35874/89905 [11:19:54<13:25:12, 1.12it/s][2025-04-28 21:44:57] (step=0035875) Train Loss: 5.7167, Train Steps/Sec: 1.12 + 40%|█████████████████████████████████████████████████████▌ | 35899/89905 [11:20:16<13:21:05, 1.12it/s][2025-04-28 21:45:20] (step=0035900) Train Loss: 5.7969, Train Steps/Sec: 1.12 + 40%|█████████████████████████████████████████████████████▌ | 35924/89905 [11:20:38<13:17:45, 1.13it/s][2025-04-28 21:45:42] (step=0035925) Train Loss: 5.7548, Train Steps/Sec: 1.12 + 40%|█████████████████████████████████████████████████████▌ | 35949/89905 [11:21:01<13:18:12, 1.13it/s][2025-04-28 21:46:04] (step=0035950) Train Loss: 5.7564, Train Steps/Sec: 1.12 + 40%|█████████████████████████████████████████████████████▌ | 35974/89905 [11:21:23<13:14:10, 1.13it/s][2025-04-28 21:46:27] (step=0035975) Train Loss: 5.6578, Train Steps/Sec: 1.12 + 40%|█████████████████████████████████████████████████████▋ | 35999/89905 [11:21:45<13:12:18, 1.13it/s][2025-04-28 21:46:49] (step=0036000) Train Loss: 5.6801, Train Steps/Sec: 1.11 +You are using a model of type blip-2 to instantiate a model of type instructblip. This is not supported for all configurations of models and can yield errors. +[2025-04-28 21:46:49] text_config is None. Initializing the text config with default values (`OPTConfig`). +model_name_or_path:/tmp/haozhezhao/model/blip2-flan-t5-xl +image_place_holder: +tokenizer length after expend 32101 +tokenizer length before expend 32100 +100%|███████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████| 6/6 [05:59<00:00, 59.94s/it] +[2025-04-28 21:54:06] Finish Eval in 36000 steps...████████████████████████████████████████████████████████████████████| 6/6 [05:59<00:00, 59.77s/it] +processing left-padding... +processing left-padding... +processing left-padding... +processing left-padding... +processing left-padding... +processing left-padding... +[2025-04-28 21:54:26] Saved checkpoint to checkpoint/X2I_UltraEdiit_fourmask_LLava_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_filtered_objects_4mask_1280_context_after_stage1_5e4/000-GPT-XL/checkpoints/0036000.pt +[2025-04-28 21:54:28] Removed old checkpoint: checkpoint/X2I_UltraEdiit_fourmask_LLava_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_filtered_objects_4mask_1280_context_after_stage1_5e4/000-GPT-XL/checkpoints/0034000.pt + 40%|█████████████████████████████████████████████████████▋ | 36024/89905 [11:29:47<14:00:55, 1.07it/s][2025-04-28 21:54:51] (step=0036025) Train Loss: 5.7398, Train Steps/Sec: 0.05 + 40%|█████████████████████████████████████████████████████▋ | 36049/89905 [11:30:09<13:27:03, 1.11it/s][2025-04-28 21:55:13] (step=0036050) Train Loss: 5.7708, Train Steps/Sec: 1.12 + 40%|█████████████████████████████████████████████████████▊ | 36074/89905 [11:30:32<13:26:25, 1.11it/s][2025-04-28 21:55:35] (step=0036075) Train Loss: 5.7037, Train Steps/Sec: 1.12 + 40%|█████████████████████████████████████████████████████▊ | 36099/89905 [11:30:54<13:15:22, 1.13it/s][2025-04-28 21:55:57] (step=0036100) Train Loss: 5.6726, Train Steps/Sec: 1.12 + 40%|█████████████████████████████████████████████████████▊ | 36124/89905 [11:31:16<13:14:08, 1.13it/s][2025-04-28 21:56:20] (step=0036125) Train Loss: 5.7392, Train Steps/Sec: 1.12 + 40%|█████████████████████████████████████████████████████▉ | 36149/89905 [11:31:39<13:13:43, 1.13it/s][2025-04-28 21:56:42] (step=0036150) Train Loss: 5.6931, Train Steps/Sec: 1.12 + 40%|█████████████████████████████████████████████████████▉ | 36174/89905 [11:32:01<13:12:19, 1.13it/s][2025-04-28 21:57:04] (step=0036175) Train Loss: 5.6925, Train Steps/Sec: 1.12 + 40%|█████████████████████████████████████████████████████▉ | 36199/89905 [11:32:23<13:09:08, 1.13it/s][2025-04-28 21:57:27] (step=0036200) Train Loss: 5.6923, Train Steps/Sec: 1.12 + 40%|█████████████████████████████████████████████████████▉ | 36224/89905 [11:32:45<13:34:17, 1.10it/s][2025-04-28 21:57:49] (step=0036225) Train Loss: 5.8041, Train Steps/Sec: 1.12 + 40%|██████████████████████████████████████████████████████ | 36249/89905 [11:33:08<13:23:10, 1.11it/s][2025-04-28 21:58:11] (step=0036250) Train Loss: 5.7962, Train Steps/Sec: 1.12 + 40%|██████████████████████████████████████████████████████ | 36274/89905 [11:33:30<13:18:25, 1.12it/s][2025-04-28 21:58:33] (step=0036275) Train Loss: 5.7530, Train Steps/Sec: 1.12 + 40%|██████████████████████████████████████████████████████ | 36299/89905 [11:33:52<13:14:53, 1.12it/s][2025-04-28 21:58:56] (step=0036300) Train Loss: 5.6788, Train Steps/Sec: 1.12 + 40%|██████████████████████████████████████████████████████▏ | 36324/89905 [11:34:15<13:14:23, 1.12it/s][2025-04-28 21:59:18] (step=0036325) Train Loss: 5.6854, Train Steps/Sec: 1.12 + 40%|██████████████████████████████████████████████████████▏ | 36349/89905 [11:34:37<13:12:02, 1.13it/s][2025-04-28 21:59:40] (step=0036350) Train Loss: 5.7182, Train Steps/Sec: 1.12 + 40%|██████████████████████████████████████████████████████▏ | 36374/89905 [11:34:59<13:08:16, 1.13it/s][2025-04-28 22:00:03] (step=0036375) Train Loss: 5.7081, Train Steps/Sec: 1.12 + 40%|██████████████████████████████████████████████████████▎ | 36399/89905 [11:35:21<13:06:25, 1.13it/s][2025-04-28 22:00:25] (step=0036400) Train Loss: 5.7376, Train Steps/Sec: 1.12 + 41%|██████████████████████████████████████████████████████▎ | 36424/89905 [11:35:44<13:32:02, 1.10it/s][2025-04-28 22:00:47] (step=0036425) Train Loss: 5.8012, Train Steps/Sec: 1.12 + 41%|██████████████████████████████████████████████████████▎ | 36449/89905 [11:36:06<13:21:04, 1.11it/s][2025-04-28 22:01:09] (step=0036450) Train Loss: 5.7303, Train Steps/Sec: 1.12 + 41%|██████████████████████████████████████████████████████▎ | 36474/89905 [11:36:28<13:16:19, 1.12it/s][2025-04-28 22:01:32] (step=0036475) Train Loss: 5.7495, Train Steps/Sec: 1.12 + 41%|██████████████████████████████████████████████████████▍ | 36499/89905 [11:36:51<13:09:37, 1.13it/s][2025-04-28 22:01:54] (step=0036500) Train Loss: 5.7064, Train Steps/Sec: 1.12 + 41%|██████████████████████████████████████████████████████▍ | 36524/89905 [11:37:13<13:09:01, 1.13it/s][2025-04-28 22:02:16] (step=0036525) Train Loss: 5.6864, Train Steps/Sec: 1.12 + 41%|██████████████████████████████████████████████████████▍ | 36549/89905 [11:37:35<13:07:02, 1.13it/s][2025-04-28 22:02:39] (step=0036550) Train Loss: 5.6577, Train Steps/Sec: 1.12 + 41%|██████████████████████████████████████████████████████▌ | 36574/89905 [11:37:58<13:09:59, 1.13it/s][2025-04-28 22:03:01] (step=0036575) Train Loss: 5.7237, Train Steps/Sec: 1.12 + 41%|██████████████████████████████████████████████████████▌ | 36599/89905 [11:38:20<13:03:54, 1.13it/s][2025-04-28 22:03:23] (step=0036600) Train Loss: 5.6721, Train Steps/Sec: 1.12 + 41%|██████████████████████████████████████████████████████▌ | 36624/89905 [11:38:42<13:32:52, 1.09it/s][2025-04-28 22:03:46] (step=0036625) Train Loss: 5.6229, Train Steps/Sec: 1.12 + 41%|██████████████████████████████████████████████████████▌ | 36649/89905 [11:39:05<13:16:23, 1.11it/s][2025-04-28 22:04:08] (step=0036650) Train Loss: 5.7044, Train Steps/Sec: 1.12 + 41%|██████████████████████████████████████████████████████▋ | 36674/89905 [11:39:27<13:12:37, 1.12it/s][2025-04-28 22:04:30] (step=0036675) Train Loss: 5.6728, Train Steps/Sec: 1.12 + 41%|██████████████████████████████████████████████████████▋ | 36699/89905 [11:39:49<13:07:43, 1.13it/s][2025-04-28 22:04:53] (step=0036700) Train Loss: 5.7544, Train Steps/Sec: 1.12 + 41%|██████████████████████████████████████████████████████▋ | 36724/89905 [11:40:11<13:08:08, 1.12it/s][2025-04-28 22:05:15] (step=0036725) Train Loss: 5.7528, Train Steps/Sec: 1.12 + 41%|██████████████████████████████████████████████████████▊ | 36749/89905 [11:40:34<13:07:09, 1.13it/s][2025-04-28 22:05:37] (step=0036750) Train Loss: 5.6199, Train Steps/Sec: 1.12 + 41%|██████████████████████████████████████████████████████▊ | 36774/89905 [11:40:56<13:00:27, 1.13it/s][2025-04-28 22:05:59] (step=0036775) Train Loss: 5.7201, Train Steps/Sec: 1.12 + 41%|██████████████████████████████████████████████████████▊ | 36799/89905 [11:41:18<13:01:00, 1.13it/s][2025-04-28 22:06:22] (step=0036800) Train Loss: 5.7652, Train Steps/Sec: 1.12 + 41%|██████████████████████████████████████████████████████▉ | 36824/89905 [11:41:41<13:27:00, 1.10it/s][2025-04-28 22:06:44] (step=0036825) Train Loss: 5.6457, Train Steps/Sec: 1.12 + 41%|██████████████████████████████████████████████████████▉ | 36849/89905 [11:42:03<13:14:52, 1.11it/s][2025-04-28 22:07:06] (step=0036850) Train Loss: 5.7227, Train Steps/Sec: 1.12 + 41%|██████████████████████████████████████████████████████▉ | 36874/89905 [11:42:25<13:07:57, 1.12it/s][2025-04-28 22:07:29] (step=0036875) Train Loss: 5.6403, Train Steps/Sec: 1.12 + 41%|██████████████████████████████████████████████████████▉ | 36899/89905 [11:42:48<13:07:52, 1.12it/s][2025-04-28 22:07:51] (step=0036900) Train Loss: 5.6457, Train Steps/Sec: 1.12 + 41%|███████████████████████████████████████████████████████ | 36924/89905 [11:43:10<13:05:57, 1.12it/s][2025-04-28 22:08:13] (step=0036925) Train Loss: 5.7065, Train Steps/Sec: 1.12 + 41%|███████████████████████████████████████████████████████ | 36949/89905 [11:43:32<13:02:02, 1.13it/s][2025-04-28 22:08:36] (step=0036950) Train Loss: 5.6639, Train Steps/Sec: 1.12 + 41%|███████████████████████████████████████████████████████ | 36974/89905 [11:44:03<22:09:53, 1.51s/it][2025-04-28 22:09:07] (step=0036975) Train Loss: 5.7081, Train Steps/Sec: 0.81 + 41%|███████████████████████████████████████████████████████▏ | 36999/89905 [11:44:45<60:47:22, 4.14s/it][2025-04-28 22:09:49] (step=0037000) Train Loss: 5.7103, Train Steps/Sec: 0.59 + 41%|███████████████████████████████████████████████████████▏ | 37024/89905 [11:45:26<18:01:57, 1.23s/it][2025-04-28 22:10:29] (step=0037025) Train Loss: 5.7567, Train Steps/Sec: 0.62 + 41%|███████████████████████████████████████████████████████▏ | 37049/89905 [11:45:57<14:03:50, 1.04it/s][2025-04-28 22:11:00] (step=0037050) Train Loss: 5.7605, Train Steps/Sec: 0.79 + 41%|███████████████████████████████████████████████████████▎ | 37074/89905 [11:46:19<13:06:50, 1.12it/s][2025-04-28 22:11:23] (step=0037075) Train Loss: 5.7059, Train Steps/Sec: 1.12 + 41%|███████████████████████████████████████████████████████▎ | 37099/89905 [11:46:42<13:02:07, 1.13it/s][2025-04-28 22:11:45] (step=0037100) Train Loss: 5.7503, Train Steps/Sec: 1.12 + 41%|███████████████████████████████████████████████████████▎ | 37124/89905 [11:47:04<13:01:16, 1.13it/s][2025-04-28 22:12:07] (step=0037125) Train Loss: 5.6719, Train Steps/Sec: 1.12 + 41%|███████████████████████████████████████████████████████▎ | 37149/89905 [11:47:26<12:59:37, 1.13it/s][2025-04-28 22:12:30] (step=0037150) Train Loss: 5.6837, Train Steps/Sec: 1.12 + 41%|███████████████████████████████████████████████████████▍ | 37174/89905 [11:47:48<12:57:25, 1.13it/s][2025-04-28 22:12:52] (step=0037175) Train Loss: 5.6936, Train Steps/Sec: 1.12 + 41%|███████████████████████████████████████████████████████▍ | 37199/89905 [11:48:11<12:54:48, 1.13it/s][2025-04-28 22:13:14] (step=0037200) Train Loss: 5.6971, Train Steps/Sec: 1.12 + 41%|███████████████████████████████████████████████████████▍ | 37224/89905 [11:48:33<13:28:01, 1.09it/s][2025-04-28 22:13:37] (step=0037225) Train Loss: 5.6633, Train Steps/Sec: 1.12 + 41%|███████████████████████████████████████████████████████▌ | 37249/89905 [11:48:55<13:08:37, 1.11it/s][2025-04-28 22:13:59] (step=0037250) Train Loss: 5.6747, Train Steps/Sec: 1.12 + 41%|███████████████████████████████████████████████████████▌ | 37274/89905 [11:49:18<13:03:30, 1.12it/s][2025-04-28 22:14:21] (step=0037275) Train Loss: 5.6409, Train Steps/Sec: 1.12 + 41%|███████████████████████████████████████████████████████▌ | 37299/89905 [11:49:40<13:04:35, 1.12it/s][2025-04-28 22:14:43] (step=0037300) Train Loss: 5.6878, Train Steps/Sec: 1.12 + 42%|███████████████████████████████████████████████████████▋ | 37324/89905 [11:50:02<13:00:49, 1.12it/s][2025-04-28 22:15:06] (step=0037325) Train Loss: 5.6884, Train Steps/Sec: 1.12 + 42%|███████████████████████████████████████████████████████▋ | 37349/89905 [11:50:25<12:55:26, 1.13it/s][2025-04-28 22:15:28] (step=0037350) Train Loss: 5.7060, Train Steps/Sec: 1.12 + 42%|███████████████████████████████████████████████████████▋ | 37374/89905 [11:50:47<12:55:39, 1.13it/s][2025-04-28 22:15:50] (step=0037375) Train Loss: 5.7784, Train Steps/Sec: 1.12 + 42%|███████████████████████████████████████████████████████▋ | 37399/89905 [11:51:09<12:51:42, 1.13it/s][2025-04-28 22:16:13] (step=0037400) Train Loss: 5.6881, Train Steps/Sec: 1.12 + 42%|███████████████████████████████████████████████████████▊ | 37424/89905 [11:51:32<13:15:52, 1.10it/s][2025-04-28 22:16:35] (step=0037425) Train Loss: 5.6917, Train Steps/Sec: 1.12 + 42%|███████████████████████████████████████████████████████▊ | 37449/89905 [11:51:54<13:04:11, 1.11it/s][2025-04-28 22:16:57] (step=0037450) Train Loss: 5.6653, Train Steps/Sec: 1.12 + 42%|███████████████████████████████████████████████████████▊ | 37474/89905 [11:52:16<13:03:57, 1.11it/s][2025-04-28 22:17:20] (step=0037475) Train Loss: 5.6357, Train Steps/Sec: 1.12 + 42%|███████████████████████████████████████████████████████▉ | 37499/89905 [11:52:39<12:58:30, 1.12it/s][2025-04-28 22:17:42] (step=0037500) Train Loss: 5.7615, Train Steps/Sec: 1.12 + 42%|███████████████████████████████████████████████████████▉ | 37524/89905 [11:53:01<12:53:04, 1.13it/s][2025-04-28 22:18:04] (step=0037525) Train Loss: 5.7529, Train Steps/Sec: 1.12 + 42%|███████████████████████████████████████████████████████▉ | 37549/89905 [11:53:23<12:50:51, 1.13it/s][2025-04-28 22:18:26] (step=0037550) Train Loss: 5.7341, Train Steps/Sec: 1.12 + 42%|████████████████████████████████████████████████████████ | 37574/89905 [11:53:45<12:54:18, 1.13it/s][2025-04-28 22:18:49] (step=0037575) Train Loss: 5.6826, Train Steps/Sec: 1.12 + 42%|████████████████████████████████████████████████████████ | 37599/89905 [11:54:08<12:51:51, 1.13it/s][2025-04-28 22:19:11] (step=0037600) Train Loss: 5.6852, Train Steps/Sec: 1.11 + 42%|████████████████████████████████████████████████████████ | 37624/89905 [11:54:30<13:14:05, 1.10it/s][2025-04-28 22:19:34] (step=0037625) Train Loss: 5.7020, Train Steps/Sec: 1.12 + 42%|████████████████████████████████████████████████████████ | 37649/89905 [11:54:52<13:04:26, 1.11it/s][2025-04-28 22:19:56] (step=0037650) Train Loss: 5.7029, Train Steps/Sec: 1.12 + 42%|████████████████████████████████████████████████████████▏ | 37674/89905 [11:55:15<12:58:57, 1.12it/s][2025-04-28 22:20:18] (step=0037675) Train Loss: 5.6959, Train Steps/Sec: 1.12 + 42%|████████████████████████████████████████████████████████▏ | 37699/89905 [11:55:37<12:53:06, 1.13it/s][2025-04-28 22:20:41] (step=0037700) Train Loss: 5.6998, Train Steps/Sec: 1.12 + 42%|████████████████████████████████████████████████████████▏ | 37724/89905 [11:55:59<12:52:23, 1.13it/s][2025-04-28 22:21:03] (step=0037725) Train Loss: 5.7070, Train Steps/Sec: 1.12 + 42%|████████████████████████████████████████████████████████▎ | 37749/89905 [11:56:22<12:49:15, 1.13it/s][2025-04-28 22:21:25] (step=0037750) Train Loss: 5.7652, Train Steps/Sec: 1.12 + 42%|████████████████████████████████████████████████████████▎ | 37774/89905 [11:56:44<12:48:28, 1.13it/s][2025-04-28 22:21:47] (step=0037775) Train Loss: 5.6868, Train Steps/Sec: 1.12 + 42%|████████████████████████████████████████████████████████▎ | 37799/89905 [11:57:06<12:46:16, 1.13it/s][2025-04-28 22:22:10] (step=0037800) Train Loss: 5.6366, Train Steps/Sec: 1.12 + 42%|████████████████████████████████████████████████████████▍ | 37824/89905 [11:57:29<13:12:15, 1.10it/s][2025-04-28 22:22:32] (step=0037825) Train Loss: 5.7096, Train Steps/Sec: 1.12 + 42%|████████████████████████████████████████████████████████▍ | 37849/89905 [11:57:51<13:01:30, 1.11it/s][2025-04-28 22:22:54] (step=0037850) Train Loss: 5.7493, Train Steps/Sec: 1.12 + 42%|████████████████████████████████████████████████████████▍ | 37874/89905 [11:58:13<12:55:07, 1.12it/s][2025-04-28 22:23:17] (step=0037875) Train Loss: 5.7320, Train Steps/Sec: 1.12 + 42%|████████████████████████████████████████████████████████▍ | 37899/89905 [11:58:36<12:56:49, 1.12it/s][2025-04-28 22:23:39] (step=0037900) Train Loss: 5.6506, Train Steps/Sec: 1.11 + 42%|████████████████████████████████████████████████████████▌ | 37924/89905 [11:58:58<12:49:15, 1.13it/s][2025-04-28 22:24:01] (step=0037925) Train Loss: 5.7309, Train Steps/Sec: 1.12 + 42%|████████████████████████████████████████████████████████▌ | 37949/89905 [11:59:20<12:44:15, 1.13it/s][2025-04-28 22:24:24] (step=0037950) Train Loss: 5.7203, Train Steps/Sec: 1.12 + 42%|████████████████████████████████████████████████████████▌ | 37974/89905 [11:59:43<12:42:09, 1.14it/s][2025-04-28 22:24:46] (step=0037975) Train Loss: 5.7073, Train Steps/Sec: 1.12 + 42%|████████████████████████████████████████████████████████▋ | 37999/89905 [12:00:05<12:42:15, 1.13it/s][2025-04-28 22:25:08] (step=0038000) Train Loss: 5.7432, Train Steps/Sec: 1.12 +You are using a model of type blip-2 to instantiate a model of type instructblip. This is not supported for all configurations of models and can yield errors. +[2025-04-28 22:25:08] text_config is None. Initializing the text config with default values (`OPTConfig`). +model_name_or_path:/tmp/haozhezhao/model/blip2-flan-t5-xl +image_place_holder: +tokenizer length after expend 32101 +tokenizer length before expend 32100 +100%|███████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████| 6/6 [05:59<00:00, 59.93s/it] +[2025-04-28 22:32:28] Finish Eval in 38000 steps...████████████████████████████████████████████████████████████████████| 6/6 [05:59<00:00, 59.75s/it] +processing left-padding... +processing left-padding... +processing left-padding... +processing left-padding... +processing left-padding... +processing left-padding... +[2025-04-28 22:32:48] Saved checkpoint to checkpoint/X2I_UltraEdiit_fourmask_LLava_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_filtered_objects_4mask_1280_context_after_stage1_5e4/000-GPT-XL/checkpoints/0038000.pt +[2025-04-28 22:32:50] Removed old checkpoint: checkpoint/X2I_UltraEdiit_fourmask_LLava_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_filtered_objects_4mask_1280_context_after_stage1_5e4/000-GPT-XL/checkpoints/0036000.pt + 42%|████████████████████████████████████████████████████████▋ | 38024/89905 [12:08:09<13:32:27, 1.06it/s][2025-04-28 22:33:12] (step=0038025) Train Loss: 5.7008, Train Steps/Sec: 0.05 + 42%|████████████████████████████████████████████████████████▋ | 38049/89905 [12:08:31<12:55:05, 1.12it/s][2025-04-28 22:33:34] (step=0038050) Train Loss: 5.6067, Train Steps/Sec: 1.12 + 42%|████████████████████████████████████████████████████████▋ | 38074/89905 [12:08:53<12:55:18, 1.11it/s][2025-04-28 22:33:57] (step=0038075) Train Loss: 5.6629, Train Steps/Sec: 1.12 + 42%|████████████████████████████████████████████████████████▊ | 38099/89905 [12:09:16<12:48:24, 1.12it/s][2025-04-28 22:34:19] (step=0038100) Train Loss: 5.6873, Train Steps/Sec: 1.12 + 42%|████████████████████████████████████████████████████████▊ | 38124/89905 [12:09:38<12:48:07, 1.12it/s][2025-04-28 22:34:41] (step=0038125) Train Loss: 5.7104, Train Steps/Sec: 1.12 + 42%|████████████████████████████████████████████████████████▊ | 38149/89905 [12:10:00<12:43:03, 1.13it/s][2025-04-28 22:35:04] (step=0038150) Train Loss: 5.7586, Train Steps/Sec: 1.12 + 42%|████████████████████████████████████████████████████████▉ | 38174/89905 [12:10:23<12:41:37, 1.13it/s][2025-04-28 22:35:26] (step=0038175) Train Loss: 5.6545, Train Steps/Sec: 1.12 + 42%|████████████████████████████████████████████████████████▉ | 38199/89905 [12:10:45<12:40:58, 1.13it/s][2025-04-28 22:35:48] (step=0038200) Train Loss: 5.6861, Train Steps/Sec: 1.12 + 43%|████████████████████████████████████████████████████████▉ | 38224/89905 [12:11:07<13:05:17, 1.10it/s][2025-04-28 22:36:11] (step=0038225) Train Loss: 5.6485, Train Steps/Sec: 1.12 + 43%|█████████████████████████████████████████████████████████ | 38249/89905 [12:11:30<12:55:02, 1.11it/s][2025-04-28 22:36:33] (step=0038250) Train Loss: 5.7123, Train Steps/Sec: 1.12 + 43%|█████████████████████████████████████████████████████████ | 38274/89905 [12:11:52<12:47:54, 1.12it/s][2025-04-28 22:36:55] (step=0038275) Train Loss: 5.7171, Train Steps/Sec: 1.12 + 43%|█████████████████████████████████████████████████████████ | 38299/89905 [12:12:14<12:45:37, 1.12it/s][2025-04-28 22:37:18] (step=0038300) Train Loss: 5.6631, Train Steps/Sec: 1.12 + 43%|█████████████████████████████████████████████████████████ | 38324/89905 [12:12:36<12:42:32, 1.13it/s][2025-04-28 22:37:40] (step=0038325) Train Loss: 5.6364, Train Steps/Sec: 1.12 + 43%|█████████████████████████████████████████████████████████▏ | 38349/89905 [12:12:59<12:41:19, 1.13it/s][2025-04-28 22:38:02] (step=0038350) Train Loss: 5.6929, Train Steps/Sec: 1.12 + 43%|█████████████████████████████████████████████████████████▏ | 38374/89905 [12:13:21<12:38:03, 1.13it/s][2025-04-28 22:38:24] (step=0038375) Train Loss: 5.7316, Train Steps/Sec: 1.12 + 43%|█████████████████████████████████████████████████████████▏ | 38399/89905 [12:13:43<12:38:45, 1.13it/s][2025-04-28 22:38:47] (step=0038400) Train Loss: 5.7773, Train Steps/Sec: 1.12 + 43%|█████████████████████████████████████████████████████████▎ | 38424/89905 [12:14:06<13:01:38, 1.10it/s][2025-04-28 22:39:09] (step=0038425) Train Loss: 5.6918, Train Steps/Sec: 1.12 + 43%|█████████████████████████████████████████████████████████▎ | 38449/89905 [12:14:28<12:51:03, 1.11it/s][2025-04-28 22:39:31] (step=0038450) Train Loss: 5.6255, Train Steps/Sec: 1.12 + 43%|█████████████████████████████████████████████████████████▎ | 38474/89905 [12:14:50<12:47:57, 1.12it/s][2025-04-28 22:39:54] (step=0038475) Train Loss: 5.7548, Train Steps/Sec: 1.12 + 43%|█████████████████████████████████████████████████████████▍ | 38499/89905 [12:15:13<12:45:13, 1.12it/s][2025-04-28 22:40:16] (step=0038500) Train Loss: 5.6324, Train Steps/Sec: 1.12 + 43%|█████████████████████████████████████████████████████████▍ | 38524/89905 [12:15:35<12:38:38, 1.13it/s][2025-04-28 22:40:38] (step=0038525) Train Loss: 5.7120, Train Steps/Sec: 1.12 + 43%|█████████████████████████████████████████████████████████▍ | 38549/89905 [12:15:57<12:37:33, 1.13it/s][2025-04-28 22:41:01] (step=0038550) Train Loss: 5.6975, Train Steps/Sec: 1.12 + 43%|█████████████████████████████████████████████████████████▍ | 38574/89905 [12:16:20<12:37:50, 1.13it/s][2025-04-28 22:41:23] (step=0038575) Train Loss: 5.6434, Train Steps/Sec: 1.12 + 43%|█████████████████████████████████████████████████████████▌ | 38599/89905 [12:16:42<12:36:31, 1.13it/s][2025-04-28 22:41:45] (step=0038600) Train Loss: 5.7460, Train Steps/Sec: 1.12 + 43%|█████████████████████████████████████████████████████████▌ | 38624/89905 [12:17:04<13:05:25, 1.09it/s][2025-04-28 22:42:08] (step=0038625) Train Loss: 5.6444, Train Steps/Sec: 1.12 + 43%|█████████████████████████████████████████████████████████▌ | 38649/89905 [12:17:27<12:49:07, 1.11it/s][2025-04-28 22:42:30] (step=0038650) Train Loss: 5.7648, Train Steps/Sec: 1.12 + 43%|█████████████████████████████████████████████████████████▋ | 38674/89905 [12:17:49<12:44:44, 1.12it/s][2025-04-28 22:42:52] (step=0038675) Train Loss: 5.6974, Train Steps/Sec: 1.12 + 43%|█████████████████████████████████████████████████████████▋ | 38699/89905 [12:18:11<12:38:15, 1.13it/s][2025-04-28 22:43:15] (step=0038700) Train Loss: 5.7563, Train Steps/Sec: 1.12 + 43%|█████████████████████████████████████████████████████████▋ | 38724/89905 [12:18:34<12:34:30, 1.13it/s][2025-04-28 22:43:37] (step=0038725) Train Loss: 5.6754, Train Steps/Sec: 1.12 + 43%|█████████████████████████████████████████████████████████▊ | 38749/89905 [12:18:56<12:38:57, 1.12it/s][2025-04-28 22:43:59] (step=0038750) Train Loss: 5.7098, Train Steps/Sec: 1.12 + 43%|█████████████████████████████████████████████████████████▊ | 38774/89905 [12:19:18<12:34:25, 1.13it/s][2025-04-28 22:44:22] (step=0038775) Train Loss: 5.7961, Train Steps/Sec: 1.12 + 43%|█████████████████████████████████████████████████████████▊ | 38799/89905 [12:19:41<12:36:37, 1.13it/s][2025-04-28 22:44:44] (step=0038800) Train Loss: 5.7561, Train Steps/Sec: 1.12 + 43%|█████████████████████████████████████████████████████████▊ | 38824/89905 [12:20:03<12:57:08, 1.10it/s][2025-04-28 22:45:06] (step=0038825) Train Loss: 5.7198, Train Steps/Sec: 1.12 + 43%|█████████████████████████████████████████████████████████▉ | 38849/89905 [12:20:25<12:46:49, 1.11it/s][2025-04-28 22:45:29] (step=0038850) Train Loss: 5.6574, Train Steps/Sec: 1.12 + 43%|█████████████████████████████████████████████████████████▉ | 38874/89905 [12:20:48<12:40:42, 1.12it/s][2025-04-28 22:45:51] (step=0038875) Train Loss: 5.6790, Train Steps/Sec: 1.12 + 43%|█████████████████████████████████████████████████████████▉ | 38899/89905 [12:21:10<12:41:25, 1.12it/s][2025-04-28 22:46:13] (step=0038900) Train Loss: 5.6921, Train Steps/Sec: 1.12 + 43%|██████████████████████████████████████████████████████████ | 38924/89905 [12:21:32<12:33:55, 1.13it/s][2025-04-28 22:46:36] (step=0038925) Train Loss: 5.6348, Train Steps/Sec: 1.12 + 43%|██████████████████████████████████████████████████████████ | 38949/89905 [12:21:55<12:32:32, 1.13it/s][2025-04-28 22:46:58] (step=0038950) Train Loss: 5.7746, Train Steps/Sec: 1.12 + 43%|██████████████████████████████████████████████████████████ | 38974/89905 [12:22:17<12:29:32, 1.13it/s][2025-04-28 22:47:20] (step=0038975) Train Loss: 5.6976, Train Steps/Sec: 1.12 + 43%|██████████████████████████████████████████████████████████▏ | 38999/89905 [12:22:39<12:31:46, 1.13it/s][2025-04-28 22:47:43] (step=0039000) Train Loss: 5.6305, Train Steps/Sec: 1.12 + 43%|██████████████████████████████████████████████████████████▏ | 39024/89905 [12:23:02<12:52:04, 1.10it/s][2025-04-28 22:48:05] (step=0039025) Train Loss: 5.7870, Train Steps/Sec: 1.12 + 43%|██████████████████████████████████████████████████████████▏ | 39049/89905 [12:23:24<12:42:13, 1.11it/s][2025-04-28 22:48:27] (step=0039050) Train Loss: 5.6255, Train Steps/Sec: 1.12 + 43%|██████████████████████████████████████████████████████████▏ | 39074/89905 [12:23:46<12:39:45, 1.12it/s][2025-04-28 22:48:50] (step=0039075) Train Loss: 5.6349, Train Steps/Sec: 1.12 + 43%|██████████████████████████████████████████████████████████▎ | 39099/89905 [12:24:08<12:36:09, 1.12it/s][2025-04-28 22:49:12] (step=0039100) Train Loss: 5.7117, Train Steps/Sec: 1.12 + 44%|██████████████████████████████████████████████████████████▎ | 39124/89905 [12:24:31<12:33:47, 1.12it/s][2025-04-28 22:49:34] (step=0039125) Train Loss: 5.6522, Train Steps/Sec: 1.12 + 44%|██████████████████████████████████████████████████████████▎ | 39149/89905 [12:24:53<12:29:20, 1.13it/s][2025-04-28 22:49:56] (step=0039150) Train Loss: 5.6704, Train Steps/Sec: 1.12 + 44%|██████████████████████████████████████████████████████████▍ | 39174/89905 [12:25:15<12:28:55, 1.13it/s][2025-04-28 22:50:19] (step=0039175) Train Loss: 5.6421, Train Steps/Sec: 1.12 + 44%|██████████████████████████████████████████████████████████▍ | 39199/89905 [12:25:38<12:29:32, 1.13it/s][2025-04-28 22:50:41] (step=0039200) Train Loss: 5.6788, Train Steps/Sec: 1.12 + 44%|██████████████████████████████████████████████████████████▍ | 39224/89905 [12:26:00<12:46:14, 1.10it/s][2025-04-28 22:51:03] (step=0039225) Train Loss: 5.7322, Train Steps/Sec: 1.12 + 44%|██████████████████████████████████████████████████████████▍ | 39249/89905 [12:26:22<12:38:21, 1.11it/s][2025-04-28 22:51:26] (step=0039250) Train Loss: 5.6956, Train Steps/Sec: 1.12 + 44%|██████████████████████████████████████████████████████████▌ | 39274/89905 [12:26:45<12:34:24, 1.12it/s][2025-04-28 22:51:48] (step=0039275) Train Loss: 5.7243, Train Steps/Sec: 1.12 + 44%|██████████████████████████████████████████████████████████▌ | 39299/89905 [12:27:07<12:31:13, 1.12it/s][2025-04-28 22:52:10] (step=0039300) Train Loss: 5.7246, Train Steps/Sec: 1.12 + 44%|██████████████████████████████████████████████████████████▌ | 39324/89905 [12:27:29<12:29:34, 1.12it/s][2025-04-28 22:52:33] (step=0039325) Train Loss: 5.7039, Train Steps/Sec: 1.12 + 44%|██████████████████████████████████████████████████████████▋ | 39349/89905 [12:27:51<12:26:08, 1.13it/s][2025-04-28 22:52:55] (step=0039350) Train Loss: 5.6298, Train Steps/Sec: 1.12 + 44%|██████████████████████████████████████████████████████████▋ | 39374/89905 [12:28:14<12:25:17, 1.13it/s][2025-04-28 22:53:17] (step=0039375) Train Loss: 5.6635, Train Steps/Sec: 1.12 + 44%|██████████████████████████████████████████████████████████▋ | 39399/89905 [12:28:36<12:26:55, 1.13it/s][2025-04-28 22:53:40] (step=0039400) Train Loss: 5.6847, Train Steps/Sec: 1.11 + 44%|██████████████████████████████████████████████████████████▊ | 39424/89905 [12:28:59<12:46:11, 1.10it/s][2025-04-28 22:54:02] (step=0039425) Train Loss: 5.6837, Train Steps/Sec: 1.12 + 44%|██████████████████████████████████████████████████████████▊ | 39449/89905 [12:29:21<12:33:52, 1.12it/s][2025-04-28 22:54:24] (step=0039450) Train Loss: 5.7182, Train Steps/Sec: 1.12 + 44%|██████████████████████████████████████████████████████████▊ | 39474/89905 [12:29:43<12:33:09, 1.12it/s][2025-04-28 22:54:47] (step=0039475) Train Loss: 5.6954, Train Steps/Sec: 1.12 + 44%|██████████████████████████████████████████████████████████▊ | 39499/89905 [12:30:05<12:27:55, 1.12it/s][2025-04-28 22:55:09] (step=0039500) Train Loss: 5.6937, Train Steps/Sec: 1.12 + 44%|██████████████████████████████████████████████████████████▉ | 39524/89905 [12:30:28<12:25:39, 1.13it/s][2025-04-28 22:55:31] (step=0039525) Train Loss: 5.7296, Train Steps/Sec: 1.12 + 44%|██████████████████████████████████████████████████████████▉ | 39549/89905 [12:30:50<12:24:07, 1.13it/s][2025-04-28 22:55:53] (step=0039550) Train Loss: 5.6709, Train Steps/Sec: 1.12 + 44%|██████████████████████████████████████████████████████████▉ | 39574/89905 [12:31:12<12:21:46, 1.13it/s][2025-04-28 22:56:16] (step=0039575) Train Loss: 5.7184, Train Steps/Sec: 1.12 + 44%|███████████████████████████████████████████████████████████ | 39599/89905 [12:31:35<12:19:50, 1.13it/s][2025-04-28 22:56:38] (step=0039600) Train Loss: 5.7036, Train Steps/Sec: 1.12 + 44%|███████████████████████████████████████████████████████████ | 39624/89905 [12:31:57<12:41:52, 1.10it/s][2025-04-28 22:57:00] (step=0039625) Train Loss: 5.7011, Train Steps/Sec: 1.12 + 44%|███████████████████████████████████████████████████████████ | 39649/89905 [12:32:19<12:31:46, 1.11it/s][2025-04-28 22:57:23] (step=0039650) Train Loss: 5.6754, Train Steps/Sec: 1.12 + 44%|███████████████████████████████████████████████████████████▏ | 39674/89905 [12:32:42<12:29:20, 1.12it/s][2025-04-28 22:57:45] (step=0039675) Train Loss: 5.7097, Train Steps/Sec: 1.12 + 44%|███████████████████████████████████████████████████████████▏ | 39699/89905 [12:33:04<12:24:20, 1.12it/s][2025-04-28 22:58:07] (step=0039700) Train Loss: 5.6544, Train Steps/Sec: 1.12 + 44%|███████████████████████████████████████████████████████████▏ | 39724/89905 [12:33:26<12:22:30, 1.13it/s][2025-04-28 22:58:30] (step=0039725) Train Loss: 5.6836, Train Steps/Sec: 1.12 + 44%|███████████████████████████████████████████████████████████▏ | 39749/89905 [12:33:48<12:22:40, 1.13it/s][2025-04-28 22:58:52] (step=0039750) Train Loss: 5.7264, Train Steps/Sec: 1.12 + 44%|███████████████████████████████████████████████████████████▎ | 39774/89905 [12:34:11<12:16:46, 1.13it/s][2025-04-28 22:59:14] (step=0039775) Train Loss: 5.6839, Train Steps/Sec: 1.12 + 44%|███████████████████████████████████████████████████████████▎ | 39799/89905 [12:34:33<12:16:56, 1.13it/s][2025-04-28 22:59:37] (step=0039800) Train Loss: 5.6228, Train Steps/Sec: 1.12 + 44%|███████████████████████████████████████████████████████████▎ | 39824/89905 [12:34:55<12:41:49, 1.10it/s][2025-04-28 22:59:59] (step=0039825) Train Loss: 5.7261, Train Steps/Sec: 1.12 + 44%|███████████████████████████████████████████████████████████▍ | 39849/89905 [12:35:18<12:32:40, 1.11it/s][2025-04-28 23:00:21] (step=0039850) Train Loss: 5.7001, Train Steps/Sec: 1.12 + 44%|███████████████████████████████████████████████████████████▍ | 39874/89905 [12:35:40<12:26:31, 1.12it/s][2025-04-28 23:00:43] (step=0039875) Train Loss: 5.6941, Train Steps/Sec: 1.12 + 44%|███████████████████████████████████████████████████████████▍ | 39899/89905 [12:36:02<12:20:49, 1.13it/s][2025-04-28 23:01:06] (step=0039900) Train Loss: 5.6182, Train Steps/Sec: 1.12 + 44%|███████████████████████████████████████████████████████████▌ | 39924/89905 [12:36:25<12:15:33, 1.13it/s][2025-04-28 23:01:28] (step=0039925) Train Loss: 5.6652, Train Steps/Sec: 1.12 + 44%|███████████████████████████████████████████████████████████▌ | 39949/89905 [12:36:47<12:17:30, 1.13it/s][2025-04-28 23:01:50] (step=0039950) Train Loss: 5.6224, Train Steps/Sec: 1.12 + 44%|███████████████████████████████████████████████████████████▌ | 39974/89905 [12:37:09<12:16:56, 1.13it/s][2025-04-28 23:02:13] (step=0039975) Train Loss: 5.7015, Train Steps/Sec: 1.12 + 44%|███████████████████████████████████████████████████████████▌ | 39999/89905 [12:37:32<12:16:33, 1.13it/s][2025-04-28 23:02:35] (step=0040000) Train Loss: 5.7176, Train Steps/Sec: 1.12 +You are using a model of type blip-2 to instantiate a model of type instructblip. This is not supported for all configurations of models and can yield errors. +[2025-04-28 23:02:35] text_config is None. Initializing the text config with default values (`OPTConfig`). +model_name_or_path:/tmp/haozhezhao/model/blip2-flan-t5-xl +image_place_holder: +tokenizer length after expend 32101 +tokenizer length before expend 32100 +100%|███████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████| 6/6 [06:02<00:00, 60.34s/it] +[2025-04-28 23:09:56] Finish Eval in 40000 steps...████████████████████████████████████████████████████████████████████| 6/6 [06:01<00:00, 60.18s/it] +processing left-padding... +processing left-padding... +processing left-padding... +processing left-padding... +processing left-padding... +processing left-padding... +[2025-04-28 23:10:16] Saved checkpoint to checkpoint/X2I_UltraEdiit_fourmask_LLava_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_filtered_objects_4mask_1280_context_after_stage1_5e4/000-GPT-XL/checkpoints/0040000.pt +[2025-04-28 23:10:18] Removed old checkpoint: checkpoint/X2I_UltraEdiit_fourmask_LLava_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_filtered_objects_4mask_1280_context_after_stage1_5e4/000-GPT-XL/checkpoints/0038000.pt + 45%|███████████████████████████████████████████████████████████▋ | 40024/89905 [12:45:37<12:59:51, 1.07it/s][2025-04-28 23:10:40] (step=0040025) Train Loss: 5.7022, Train Steps/Sec: 0.05 + 45%|███████████████████████████████████████████████████████████▋ | 40049/89905 [12:45:59<12:25:26, 1.11it/s][2025-04-28 23:11:03] (step=0040050) Train Loss: 5.6942, Train Steps/Sec: 1.12 + 45%|███████████████████████████████████████████████████████████▋ | 40074/89905 [12:46:22<12:22:10, 1.12it/s][2025-04-28 23:11:25] (step=0040075) Train Loss: 5.7171, Train Steps/Sec: 1.12 + 45%|███████████████████████████████████████████████████████████▊ | 40099/89905 [12:46:44<12:17:49, 1.13it/s][2025-04-28 23:11:47] (step=0040100) Train Loss: 5.6620, Train Steps/Sec: 1.12 + 45%|███████████████████████████████████████████████████████████▊ | 40124/89905 [12:47:06<12:21:09, 1.12it/s][2025-04-28 23:12:10] (step=0040125) Train Loss: 5.7108, Train Steps/Sec: 1.12 + 45%|███████████████████████████████████████████████████████████▊ | 40149/89905 [12:47:28<12:13:33, 1.13it/s][2025-04-28 23:12:32] (step=0040150) Train Loss: 5.7042, Train Steps/Sec: 1.12 + 45%|███████████████████████████████████████████████████████████▉ | 40174/89905 [12:47:51<12:12:47, 1.13it/s][2025-04-28 23:12:54] (step=0040175) Train Loss: 5.6852, Train Steps/Sec: 1.12 + 45%|███████████████████████████████████████████████████████████▉ | 40199/89905 [12:48:13<12:12:10, 1.13it/s][2025-04-28 23:13:17] (step=0040200) Train Loss: 5.7164, Train Steps/Sec: 1.12 + 45%|███████████████████████████████████████████████████████████▉ | 40224/89905 [12:48:35<12:32:39, 1.10it/s][2025-04-28 23:13:39] (step=0040225) Train Loss: 5.6808, Train Steps/Sec: 1.12 + 45%|███████████████████████████████████████████████████████████▉ | 40249/89905 [12:48:58<12:24:20, 1.11it/s][2025-04-28 23:14:01] (step=0040250) Train Loss: 5.7013, Train Steps/Sec: 1.12 + 45%|████████████████████████████████████████████████████████████ | 40274/89905 [12:49:20<12:19:26, 1.12it/s][2025-04-28 23:14:23] (step=0040275) Train Loss: 5.6397, Train Steps/Sec: 1.12 + 45%|████████████████████████████████████████████████████████████ | 40299/89905 [12:49:42<12:15:34, 1.12it/s][2025-04-28 23:14:46] (step=0040300) Train Loss: 5.6560, Train Steps/Sec: 1.12 + 45%|████████████████████████████████████████████████████████████ | 40324/89905 [12:50:05<12:13:01, 1.13it/s][2025-04-28 23:15:08] (step=0040325) Train Loss: 5.7312, Train Steps/Sec: 1.12 + 45%|████████████████████████████████████████████████████████████▏ | 40349/89905 [12:50:27<12:20:24, 1.12it/s][2025-04-28 23:15:30] (step=0040350) Train Loss: 5.7267, Train Steps/Sec: 1.12 + 45%|████████████████████████████████████████████████████████████▏ | 40374/89905 [12:50:49<12:10:19, 1.13it/s][2025-04-28 23:15:53] (step=0040375) Train Loss: 5.7302, Train Steps/Sec: 1.12 + 45%|████████████████████████████████████████████████████████████▏ | 40399/89905 [12:51:12<12:11:21, 1.13it/s][2025-04-28 23:16:15] (step=0040400) Train Loss: 5.6905, Train Steps/Sec: 1.11 + 45%|████████████████████████████████████████████████████████████▎ | 40424/89905 [12:51:34<12:32:20, 1.10it/s][2025-04-28 23:16:38] (step=0040425) Train Loss: 5.6178, Train Steps/Sec: 1.12 + 45%|████████████████████████████████████████████████████████████▎ | 40449/89905 [12:51:56<12:23:28, 1.11it/s][2025-04-28 23:17:00] (step=0040450) Train Loss: 5.6696, Train Steps/Sec: 1.12 + 45%|████████████████████████████████████████████████████████████▎ | 40474/89905 [12:52:19<12:16:55, 1.12it/s][2025-04-28 23:17:22] (step=0040475) Train Loss: 5.6624, Train Steps/Sec: 1.12 + 45%|████████████████████████████████████████████████████████████▎ | 40499/89905 [12:52:41<12:12:30, 1.12it/s][2025-04-28 23:17:45] (step=0040500) Train Loss: 5.6979, Train Steps/Sec: 1.12 + 45%|████████████████████████████████████████████████████████████▍ | 40524/89905 [12:53:03<12:11:59, 1.12it/s][2025-04-28 23:18:07] (step=0040525) Train Loss: 5.6687, Train Steps/Sec: 1.12 + 45%|████████████████████████████████████████████████████████████▍ | 40549/89905 [12:53:43<12:46:56, 1.07it/s][2025-04-28 23:18:46] (step=0040550) Train Loss: 5.6051, Train Steps/Sec: 0.63 + 45%|████████████████████████████████████████████████████████████▍ | 40574/89905 [12:54:34<25:53:19, 1.89s/it][2025-04-28 23:19:38] (step=0040575) Train Loss: 5.6749, Train Steps/Sec: 0.49 + 45%|████████████████████████████████████████████████████████████▌ | 40599/89905 [12:55:06<12:11:36, 1.12it/s][2025-04-28 23:20:09] (step=0040600) Train Loss: 5.7421, Train Steps/Sec: 0.80 + 45%|████████████████████████████████████████████████████████████▌ | 40624/89905 [12:55:38<12:28:57, 1.10it/s][2025-04-28 23:20:41] (step=0040625) Train Loss: 5.6511, Train Steps/Sec: 0.78 + 45%|████████████████████████████████████████████████████████████▌ | 40649/89905 [12:56:00<12:17:31, 1.11it/s][2025-04-28 23:21:03] (step=0040650) Train Loss: 5.6884, Train Steps/Sec: 1.12 + 45%|████████████████████████████████████████████████████████████▌ | 40674/89905 [12:56:22<12:15:44, 1.12it/s][2025-04-28 23:21:26] (step=0040675) Train Loss: 5.6409, Train Steps/Sec: 1.12 + 45%|████████████████████████████████████████████████████████████▋ | 40699/89905 [12:56:45<12:10:55, 1.12it/s][2025-04-28 23:21:48] (step=0040700) Train Loss: 5.6610, Train Steps/Sec: 1.12 + 45%|████████████████████████████████████████████████████████████▋ | 40724/89905 [12:57:07<12:06:46, 1.13it/s][2025-04-28 23:22:10] (step=0040725) Train Loss: 5.6914, Train Steps/Sec: 1.12 + 45%|████████████████████████████████████████████████████████████▋ | 40749/89905 [12:57:29<12:06:03, 1.13it/s][2025-04-28 23:22:33] (step=0040750) Train Loss: 5.7263, Train Steps/Sec: 1.12 + 45%|████████████████████████████████████████████████████████████▊ | 40774/89905 [12:57:51<12:07:54, 1.12it/s][2025-04-28 23:22:55] (step=0040775) Train Loss: 5.6511, Train Steps/Sec: 1.12 + 45%|████████████████████████████████████████████████████████████▊ | 40799/89905 [12:58:14<12:04:27, 1.13it/s][2025-04-28 23:23:17] (step=0040800) Train Loss: 5.6446, Train Steps/Sec: 1.12 + 45%|████████████████████████████████████████████████████████████▊ | 40824/89905 [12:58:36<12:26:41, 1.10it/s][2025-04-28 23:23:40] (step=0040825) Train Loss: 5.6876, Train Steps/Sec: 1.12 + 45%|████████████████████████████████████████████████████████████▉ | 40849/89905 [12:58:59<12:14:48, 1.11it/s][2025-04-28 23:24:02] (step=0040850) Train Loss: 5.6487, Train Steps/Sec: 1.12 + 45%|████████████████████████████████████████████████████████████▉ | 40874/89905 [12:59:21<12:09:35, 1.12it/s][2025-04-28 23:24:24] (step=0040875) Train Loss: 5.7187, Train Steps/Sec: 1.12 + 45%|████████████████████████████████████████████████████████████▉ | 40899/89905 [12:59:43<12:07:16, 1.12it/s][2025-04-28 23:24:47] (step=0040900) Train Loss: 5.6826, Train Steps/Sec: 1.12 + 46%|████████████████████████████████████████████████████████████▉ | 40924/89905 [13:00:05<12:02:41, 1.13it/s][2025-04-28 23:25:09] (step=0040925) Train Loss: 5.7394, Train Steps/Sec: 1.12 + 46%|█████████████████████████████████████████████████████████████ | 40949/89905 [13:00:28<12:03:38, 1.13it/s][2025-04-28 23:25:31] (step=0040950) Train Loss: 5.7489, Train Steps/Sec: 1.12 + 46%|█████████████████████████████████████████████████████████████ | 40974/89905 [13:00:50<11:58:59, 1.13it/s][2025-04-28 23:25:53] (step=0040975) Train Loss: 5.7062, Train Steps/Sec: 1.12 + 46%|█████████████████████████████████████████████████████████████ | 40999/89905 [13:01:12<12:08:06, 1.12it/s][2025-04-28 23:26:16] (step=0041000) Train Loss: 5.6439, Train Steps/Sec: 1.12 + 46%|█████████████████████████████████████████████████████████████▏ | 41024/89905 [13:01:35<12:26:03, 1.09it/s][2025-04-28 23:26:38] (step=0041025) Train Loss: 5.6978, Train Steps/Sec: 1.12 + 46%|█████████████████████████████████████████████████████████████▏ | 41049/89905 [13:01:57<12:12:07, 1.11it/s][2025-04-28 23:27:00] (step=0041050) Train Loss: 5.5955, Train Steps/Sec: 1.12 + 46%|█████████████████████████████████████████████████████████████▏ | 41074/89905 [13:02:19<12:08:08, 1.12it/s][2025-04-28 23:27:23] (step=0041075) Train Loss: 5.7583, Train Steps/Sec: 1.12 + 46%|█████████████████████████████████████████████████████████████▎ | 41099/89905 [13:02:42<12:03:24, 1.12it/s][2025-04-28 23:27:45] (step=0041100) Train Loss: 5.6113, Train Steps/Sec: 1.12 + 46%|█████████████████████████████████████████████████████████████▎ | 41124/89905 [13:03:04<12:03:43, 1.12it/s][2025-04-28 23:28:07] (step=0041125) Train Loss: 5.6859, Train Steps/Sec: 1.12 + 46%|█████████████████████████████████████████████████████████████▎ | 41149/89905 [13:03:26<11:59:25, 1.13it/s][2025-04-28 23:28:30] (step=0041150) Train Loss: 5.6775, Train Steps/Sec: 1.12 + 46%|█████████████████████████████████████████████████████████████▎ | 41174/89905 [13:03:49<11:58:21, 1.13it/s][2025-04-28 23:28:52] (step=0041175) Train Loss: 5.7132, Train Steps/Sec: 1.12 + 46%|█████████████████████████████████████████████████████████████▍ | 41199/89905 [13:04:11<11:55:39, 1.13it/s][2025-04-28 23:29:14] (step=0041200) Train Loss: 5.6937, Train Steps/Sec: 1.12 + 46%|█████████████████████████████████████████████████████████████▍ | 41224/89905 [13:04:33<12:17:58, 1.10it/s][2025-04-28 23:29:37] (step=0041225) Train Loss: 5.6944, Train Steps/Sec: 1.12 + 46%|█████████████████████████████████████████████████████████████▍ | 41249/89905 [13:04:56<12:10:37, 1.11it/s][2025-04-28 23:29:59] (step=0041250) Train Loss: 5.6879, Train Steps/Sec: 1.12 + 46%|█████████████████████████████████████████████████████████████▌ | 41274/89905 [13:05:18<12:07:32, 1.11it/s][2025-04-28 23:30:21] (step=0041275) Train Loss: 5.7038, Train Steps/Sec: 1.12 + 46%|█████████████████████████████████████████████████████████████▌ | 41299/89905 [13:05:40<12:01:29, 1.12it/s][2025-04-28 23:30:44] (step=0041300) Train Loss: 5.7164, Train Steps/Sec: 1.12 + 46%|█████████████████████████████████████████████████████████████▌ | 41324/89905 [13:06:02<11:58:45, 1.13it/s][2025-04-28 23:31:06] (step=0041325) Train Loss: 5.6224, Train Steps/Sec: 1.12 + 46%|█████████████████████████████████████████████████████████████▋ | 41349/89905 [13:06:25<11:59:27, 1.12it/s][2025-04-28 23:31:28] (step=0041350) Train Loss: 5.6068, Train Steps/Sec: 1.12 + 46%|█████████████████████████████████████████████████████████████▋ | 41374/89905 [13:06:47<11:58:10, 1.13it/s][2025-04-28 23:31:51] (step=0041375) Train Loss: 5.6587, Train Steps/Sec: 1.12 + 46%|█████████████████████████████████████████████████████████████▋ | 41399/89905 [13:07:09<11:52:58, 1.13it/s][2025-04-28 23:32:13] (step=0041400) Train Loss: 5.6697, Train Steps/Sec: 1.12 + 46%|█████████████████████████████████████████████████████████████▋ | 41424/89905 [13:07:32<12:20:15, 1.09it/s][2025-04-28 23:32:35] (step=0041425) Train Loss: 5.7030, Train Steps/Sec: 1.12 + 46%|█████████████████████████████████████████████████████████████▊ | 41449/89905 [13:07:54<12:04:49, 1.11it/s][2025-04-28 23:32:57] (step=0041450) Train Loss: 5.6953, Train Steps/Sec: 1.12 + 46%|█████████████████████████████████████████████████████████████▊ | 41474/89905 [13:08:16<12:03:17, 1.12it/s][2025-04-28 23:33:20] (step=0041475) Train Loss: 5.6086, Train Steps/Sec: 1.12 + 46%|█████████████████████████████████████████████████████████████▊ | 41499/89905 [13:08:39<11:59:25, 1.12it/s][2025-04-28 23:33:42] (step=0041500) Train Loss: 5.7182, Train Steps/Sec: 1.12 + 46%|█████████████████████████████████████████████████████████████▉ | 41524/89905 [13:09:01<11:56:22, 1.13it/s][2025-04-28 23:34:04] (step=0041525) Train Loss: 5.6865, Train Steps/Sec: 1.12 + 46%|█████████████████████████████████████████████████████████████▉ | 41549/89905 [13:09:23<11:57:35, 1.12it/s][2025-04-28 23:34:27] (step=0041550) Train Loss: 5.6465, Train Steps/Sec: 1.12 + 46%|█████████████████████████████████████████████████████████████▉ | 41574/89905 [13:09:46<11:55:13, 1.13it/s][2025-04-28 23:34:49] (step=0041575) Train Loss: 5.6332, Train Steps/Sec: 1.12 + 46%|██████████████████████████████████████████████████████████████ | 41599/89905 [13:10:08<11:53:35, 1.13it/s][2025-04-28 23:35:11] (step=0041600) Train Loss: 5.6448, Train Steps/Sec: 1.11 + 46%|██████████████████████████████████████████████████████████████ | 41624/89905 [13:10:30<12:15:58, 1.09it/s][2025-04-28 23:35:34] (step=0041625) Train Loss: 5.6693, Train Steps/Sec: 1.12 + 46%|██████████████████████████████████████████████████████████████ | 41649/89905 [13:10:53<12:04:01, 1.11it/s][2025-04-28 23:35:56] (step=0041650) Train Loss: 5.6649, Train Steps/Sec: 1.12 + 46%|██████████████████████████████████████████████████████████████ | 41674/89905 [13:11:15<11:58:29, 1.12it/s][2025-04-28 23:36:18] (step=0041675) Train Loss: 5.6704, Train Steps/Sec: 1.12 + 46%|██████████████████████████████████████████████████████████████▏ | 41699/89905 [13:11:37<11:56:28, 1.12it/s][2025-04-28 23:36:41] (step=0041700) Train Loss: 5.6305, Train Steps/Sec: 1.12 + 46%|██████████████████████████████████████████████████████████████▏ | 41724/89905 [13:12:00<11:52:01, 1.13it/s][2025-04-28 23:37:03] (step=0041725) Train Loss: 5.6705, Train Steps/Sec: 1.12 + 46%|██████████████████████████████████████████████████████████████▏ | 41749/89905 [13:12:22<11:52:07, 1.13it/s][2025-04-28 23:37:25] (step=0041750) Train Loss: 5.7412, Train Steps/Sec: 1.12 + 46%|██████████████████████████████████████████████████████████████▎ | 41774/89905 [13:12:44<11:51:49, 1.13it/s][2025-04-28 23:37:48] (step=0041775) Train Loss: 5.6773, Train Steps/Sec: 1.12 + 46%|██████████████████████████████████████████████████████████████▎ | 41799/89905 [13:13:07<11:50:29, 1.13it/s][2025-04-28 23:38:10] (step=0041800) Train Loss: 5.6996, Train Steps/Sec: 1.11 + 47%|██████████████████████████████████████████████████████████████▎ | 41824/89905 [13:13:29<12:08:05, 1.10it/s][2025-04-28 23:38:33] (step=0041825) Train Loss: 5.7208, Train Steps/Sec: 1.12 + 47%|██████████████████████████████████████████████████████████████▎ | 41849/89905 [13:13:51<12:02:10, 1.11it/s][2025-04-28 23:38:55] (step=0041850) Train Loss: 5.6683, Train Steps/Sec: 1.12 + 47%|██████████████████████████████████████████████████████████████▍ | 41874/89905 [13:14:14<11:55:48, 1.12it/s][2025-04-28 23:39:17] (step=0041875) Train Loss: 5.6298, Train Steps/Sec: 1.12 + 47%|██████████████████████████████████████████████████████████████▍ | 41899/89905 [13:14:36<11:57:24, 1.12it/s][2025-04-28 23:39:39] (step=0041900) Train Loss: 5.6729, Train Steps/Sec: 1.12 + 47%|██████████████████████████████████████████████████████████████▍ | 41924/89905 [13:14:58<11:47:16, 1.13it/s][2025-04-28 23:40:02] (step=0041925) Train Loss: 5.6558, Train Steps/Sec: 1.12 + 47%|██████████████████████████████████████████████████████████████▌ | 41949/89905 [13:15:21<11:49:20, 1.13it/s][2025-04-28 23:40:24] (step=0041950) Train Loss: 5.7145, Train Steps/Sec: 1.12 + 47%|██████████████████████████████████████████████████████████████▌ | 41974/89905 [13:15:43<11:46:34, 1.13it/s][2025-04-28 23:40:46] (step=0041975) Train Loss: 5.6427, Train Steps/Sec: 1.12 + 47%|██████████████████████████████████████████████████████████████▌ | 41999/89905 [13:16:05<11:45:24, 1.13it/s][2025-04-28 23:41:09] (step=0042000) Train Loss: 5.6402, Train Steps/Sec: 1.12 +You are using a model of type blip-2 to instantiate a model of type instructblip. This is not supported for all configurations of models and can yield errors. +[2025-04-28 23:41:09] text_config is None. Initializing the text config with default values (`OPTConfig`). +model_name_or_path:/tmp/haozhezhao/model/blip2-flan-t5-xl +image_place_holder: +tokenizer length after expend 32101 +tokenizer length before expend 32100 +100%|███████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████| 6/6 [06:02<00:00, 60.42s/it] +[2025-04-28 23:48:29] Finish Eval in 42000 steps...████████████████████████████████████████████████████████████████████| 6/6 [06:01<00:00, 60.11s/it] +processing left-padding... +processing left-padding... +processing left-padding... +processing left-padding... +processing left-padding... +processing left-padding... +[2025-04-28 23:48:49] Saved checkpoint to checkpoint/X2I_UltraEdiit_fourmask_LLava_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_filtered_objects_4mask_1280_context_after_stage1_5e4/000-GPT-XL/checkpoints/0042000.pt +[2025-04-28 23:48:51] Removed old checkpoint: checkpoint/X2I_UltraEdiit_fourmask_LLava_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_filtered_objects_4mask_1280_context_after_stage1_5e4/000-GPT-XL/checkpoints/0040000.pt + 47%|██████████████████████████████████████████████████████████████▋ | 42024/89905 [13:24:10<12:25:43, 1.07it/s][2025-04-28 23:49:13] (step=0042025) Train Loss: 5.6136, Train Steps/Sec: 0.05 + 47%|██████████████████████████████████████████████████████████████▋ | 42049/89905 [13:24:32<11:54:37, 1.12it/s][2025-04-28 23:49:36] (step=0042050) Train Loss: 5.6493, Train Steps/Sec: 1.12 + 47%|██████████████████████████████████████████████████████████████▋ | 42074/89905 [13:24:54<11:52:26, 1.12it/s][2025-04-28 23:49:58] (step=0042075) Train Loss: 5.6209, Train Steps/Sec: 1.12 + 47%|██████████████████████████████████████████████████████████████▋ | 42099/89905 [13:25:17<11:49:58, 1.12it/s][2025-04-28 23:50:20] (step=0042100) Train Loss: 5.6950, Train Steps/Sec: 1.12 + 47%|██████████████████████████████████████████████████████████████▊ | 42124/89905 [13:25:39<11:46:05, 1.13it/s][2025-04-28 23:50:42] (step=0042125) Train Loss: 5.6629, Train Steps/Sec: 1.12 + 47%|██████████████████████████████████████████████████████████████▊ | 42149/89905 [13:26:01<11:44:36, 1.13it/s][2025-04-28 23:51:05] (step=0042150) Train Loss: 5.6806, Train Steps/Sec: 1.12 + 47%|██████████████████████████████████████████████████████████████▊ | 42174/89905 [13:26:24<11:45:41, 1.13it/s][2025-04-28 23:51:27] (step=0042175) Train Loss: 5.5678, Train Steps/Sec: 1.12 + 47%|██████████████████████████████████████████████████████████████▉ | 42199/89905 [13:26:46<11:42:21, 1.13it/s][2025-04-28 23:51:49] (step=0042200) Train Loss: 5.6823, Train Steps/Sec: 1.12 + 47%|██████████████████████████████████████████████████████████████▉ | 42224/89905 [13:27:08<12:04:02, 1.10it/s][2025-04-28 23:52:12] (step=0042225) Train Loss: 5.7068, Train Steps/Sec: 1.12 + 47%|██████████████████████████████████████████████████████████████▉ | 42249/89905 [13:27:31<11:56:35, 1.11it/s][2025-04-28 23:52:34] (step=0042250) Train Loss: 5.6857, Train Steps/Sec: 1.11 + 47%|███████████████████████████████████████████████████████████████ | 42274/89905 [13:27:53<11:48:31, 1.12it/s][2025-04-28 23:52:57] (step=0042275) Train Loss: 5.6382, Train Steps/Sec: 1.12 + 47%|███████████████████████████████████████████████████████████████ | 42299/89905 [13:28:15<11:47:03, 1.12it/s][2025-04-28 23:53:19] (step=0042300) Train Loss: 5.6656, Train Steps/Sec: 1.12 + 47%|███████████████████████████████████████████████████████████████ | 42324/89905 [13:28:38<11:46:39, 1.12it/s][2025-04-28 23:53:41] (step=0042325) Train Loss: 5.6743, Train Steps/Sec: 1.12 + 47%|███████████████████████████████████████████████████████████████ | 42349/89905 [13:29:00<11:41:12, 1.13it/s][2025-04-28 23:54:03] (step=0042350) Train Loss: 5.6660, Train Steps/Sec: 1.12 + 47%|███████████████████████████████████████████████████████████████▏ | 42374/89905 [13:29:22<11:39:58, 1.13it/s][2025-04-28 23:54:26] (step=0042375) Train Loss: 5.6581, Train Steps/Sec: 1.12 + 47%|███████████████████████████████████████████████████████████████▏ | 42399/89905 [13:29:45<11:39:57, 1.13it/s][2025-04-28 23:54:48] (step=0042400) Train Loss: 5.6170, Train Steps/Sec: 1.11 + 47%|███████████████████████████████████████████████████████████████▏ | 42424/89905 [13:30:07<12:01:06, 1.10it/s][2025-04-28 23:55:10] (step=0042425) Train Loss: 5.5935, Train Steps/Sec: 1.12 + 47%|███████████████████████████████████████████████████████████████▎ | 42449/89905 [13:30:29<11:49:36, 1.11it/s][2025-04-28 23:55:33] (step=0042450) Train Loss: 5.6772, Train Steps/Sec: 1.12 + 47%|███████████████████████████████████████████████████████████████▎ | 42474/89905 [13:30:52<11:46:15, 1.12it/s][2025-04-28 23:55:55] (step=0042475) Train Loss: 5.6675, Train Steps/Sec: 1.12 + 47%|███████████████████████████████████████████████████████████████▎ | 42499/89905 [13:31:14<11:43:10, 1.12it/s][2025-04-28 23:56:17] (step=0042500) Train Loss: 5.6686, Train Steps/Sec: 1.12 + 47%|███████████████████████████████████████████████████████████████▍ | 42524/89905 [13:31:36<11:40:58, 1.13it/s][2025-04-28 23:56:40] (step=0042525) Train Loss: 5.6684, Train Steps/Sec: 1.12 + 47%|███████████████████████████████████████████████████████████████▍ | 42549/89905 [13:31:59<11:39:28, 1.13it/s][2025-04-28 23:57:02] (step=0042550) Train Loss: 5.6257, Train Steps/Sec: 1.12 + 47%|███████████████████████████████████████████████████████████████▍ | 42574/89905 [13:32:21<11:35:23, 1.13it/s][2025-04-28 23:57:24] (step=0042575) Train Loss: 5.6646, Train Steps/Sec: 1.12 + 47%|███████████████████████████████████████████████████████████████▍ | 42599/89905 [13:32:43<11:37:49, 1.13it/s][2025-04-28 23:57:47] (step=0042600) Train Loss: 5.6683, Train Steps/Sec: 1.12 + 47%|███████████████████████████████████████████████████████████████▌ | 42624/89905 [13:33:05<11:58:05, 1.10it/s][2025-04-28 23:58:09] (step=0042625) Train Loss: 5.7004, Train Steps/Sec: 1.12 + 47%|███████████████████████████████████████████████████████████████▌ | 42649/89905 [13:33:28<11:47:54, 1.11it/s][2025-04-28 23:58:31] (step=0042650) Train Loss: 5.6882, Train Steps/Sec: 1.12 + 47%|███████████████████████████████████████████████████████████████▌ | 42674/89905 [13:33:50<11:43:25, 1.12it/s][2025-04-28 23:58:53] (step=0042675) Train Loss: 5.6686, Train Steps/Sec: 1.12 + 47%|███████████████████████████████████████████████████████████████▋ | 42699/89905 [13:34:12<11:39:12, 1.13it/s][2025-04-28 23:59:16] (step=0042700) Train Loss: 5.6066, Train Steps/Sec: 1.12 + 48%|███████████████████████████████████████████████████████████████▋ | 42724/89905 [13:34:35<11:39:55, 1.12it/s][2025-04-28 23:59:38] (step=0042725) Train Loss: 5.6940, Train Steps/Sec: 1.12 + 48%|███████████████████████████████████████████████████████████████▋ | 42749/89905 [13:34:57<11:35:27, 1.13it/s][2025-04-29 00:00:00] (step=0042750) Train Loss: 5.6306, Train Steps/Sec: 1.12 + 48%|███████████████████████████████████████████████████████████████▊ | 42774/89905 [13:35:19<11:37:30, 1.13it/s][2025-04-29 00:00:23] (step=0042775) Train Loss: 5.7247, Train Steps/Sec: 1.12 + 48%|███████████████████████████████████████████████████████████████▊ | 42799/89905 [13:35:42<11:33:25, 1.13it/s][2025-04-29 00:00:45] (step=0042800) Train Loss: 5.6675, Train Steps/Sec: 1.12 + 48%|███████████████████████████████████████████████████████████████▊ | 42824/89905 [13:36:04<11:58:13, 1.09it/s][2025-04-29 00:01:07] (step=0042825) Train Loss: 5.5962, Train Steps/Sec: 1.12 + 48%|███████████████████████████████████████████████████████████████▊ | 42849/89905 [13:36:26<11:45:15, 1.11it/s][2025-04-29 00:01:30] (step=0042850) Train Loss: 5.7044, Train Steps/Sec: 1.12 + 48%|███████████████████████████████████████████████████████████████▉ | 42874/89905 [13:36:49<11:40:51, 1.12it/s][2025-04-29 00:01:52] (step=0042875) Train Loss: 5.6512, Train Steps/Sec: 1.12 + 48%|███████████████████████████████████████████████████████████████▉ | 42899/89905 [13:37:11<11:35:19, 1.13it/s][2025-04-29 00:02:14] (step=0042900) Train Loss: 5.5813, Train Steps/Sec: 1.12 + 48%|███████████████████████████████████████████████████████████████▉ | 42924/89905 [13:37:33<11:35:06, 1.13it/s][2025-04-29 00:02:37] (step=0042925) Train Loss: 5.6691, Train Steps/Sec: 1.12 + 48%|████████████████████████████████████████████████████████████████ | 42949/89905 [13:37:56<11:33:12, 1.13it/s][2025-04-29 00:02:59] (step=0042950) Train Loss: 5.6384, Train Steps/Sec: 1.12 + 48%|████████████████████████████████████████████████████████████████ | 42974/89905 [13:38:18<11:29:04, 1.14it/s][2025-04-29 00:03:21] (step=0042975) Train Loss: 5.7155, Train Steps/Sec: 1.12 + 48%|████████████████████████████████████████████████████████████████ | 42999/89905 [13:38:40<11:32:18, 1.13it/s][2025-04-29 00:03:44] (step=0043000) Train Loss: 5.6430, Train Steps/Sec: 1.12 + 48%|████████████████████████████████████████████████████████████████▏ | 43024/89905 [13:39:03<11:51:41, 1.10it/s][2025-04-29 00:04:06] (step=0043025) Train Loss: 5.6643, Train Steps/Sec: 1.12 + 48%|████████████████████████████████████████████████████████████████▏ | 43049/89905 [13:39:25<11:41:23, 1.11it/s][2025-04-29 00:04:28] (step=0043050) Train Loss: 5.6765, Train Steps/Sec: 1.12 + 48%|████████████████████████████████████████████████████████████████▏ | 43074/89905 [13:39:47<11:42:06, 1.11it/s][2025-04-29 00:04:51] (step=0043075) Train Loss: 5.6856, Train Steps/Sec: 1.12 + 48%|████████████████████████████████████████████████████████████████▏ | 43099/89905 [13:40:09<11:34:46, 1.12it/s][2025-04-29 00:05:13] (step=0043100) Train Loss: 5.6523, Train Steps/Sec: 1.12 + 48%|████████████████████████████████████████████████████████████████▎ | 43124/89905 [13:40:32<11:32:01, 1.13it/s][2025-04-29 00:05:35] (step=0043125) Train Loss: 5.6304, Train Steps/Sec: 1.12 + 48%|████████████████████████████████████████████████████████████████▎ | 43149/89905 [13:40:54<11:29:14, 1.13it/s][2025-04-29 00:05:57] (step=0043150) Train Loss: 5.6913, Train Steps/Sec: 1.12 + 48%|████████████████████████████████████████████████████████████████▎ | 43174/89905 [13:41:16<11:31:21, 1.13it/s][2025-04-29 00:06:20] (step=0043175) Train Loss: 5.6324, Train Steps/Sec: 1.12 + 48%|████████████████████████████████████████████████████████████████▍ | 43199/89905 [13:41:39<11:28:00, 1.13it/s][2025-04-29 00:06:42] (step=0043200) Train Loss: 5.7218, Train Steps/Sec: 1.12 + 48%|████████████████████████████████████████████████████████████████▍ | 43224/89905 [13:42:01<11:48:14, 1.10it/s][2025-04-29 00:07:04] (step=0043225) Train Loss: 5.6937, Train Steps/Sec: 1.12 + 48%|████████████████████████████████████████████████████████████████▍ | 43249/89905 [13:42:23<11:37:32, 1.11it/s][2025-04-29 00:07:27] (step=0043250) Train Loss: 5.6725, Train Steps/Sec: 1.12 + 48%|████████████████████████████████████████████████████████████████▍ | 43274/89905 [13:42:46<11:37:09, 1.11it/s][2025-04-29 00:07:49] (step=0043275) Train Loss: 5.5983, Train Steps/Sec: 1.12 + 48%|████████████████████████████████████████████████████████████████▌ | 43299/89905 [13:43:08<11:31:36, 1.12it/s][2025-04-29 00:08:11] (step=0043300) Train Loss: 5.6985, Train Steps/Sec: 1.12 + 48%|████████████████████████████████████████████████████████████████▌ | 43324/89905 [13:43:30<11:30:19, 1.12it/s][2025-04-29 00:08:34] (step=0043325) Train Loss: 5.5995, Train Steps/Sec: 1.12 + 48%|████████████████████████████████████████████████████████████████▌ | 43349/89905 [13:43:53<11:27:59, 1.13it/s][2025-04-29 00:08:56] (step=0043350) Train Loss: 5.6473, Train Steps/Sec: 1.12 + 48%|████████████████████████████████████████████████████████████████▋ | 43374/89905 [13:44:15<11:25:56, 1.13it/s][2025-04-29 00:09:18] (step=0043375) Train Loss: 5.6176, Train Steps/Sec: 1.12 + 48%|████████████████████████████████████████████████████████████████▋ | 43399/89905 [13:44:37<11:30:39, 1.12it/s][2025-04-29 00:09:41] (step=0043400) Train Loss: 5.6200, Train Steps/Sec: 1.11 + 48%|████████████████████████████████████████████████████████████████▋ | 43424/89905 [13:45:00<11:43:59, 1.10it/s][2025-04-29 00:10:03] (step=0043425) Train Loss: 5.6522, Train Steps/Sec: 1.12 + 48%|████████████████████████████████████████████████████████████████▊ | 43449/89905 [13:45:22<11:35:20, 1.11it/s][2025-04-29 00:10:25] (step=0043450) Train Loss: 5.6896, Train Steps/Sec: 1.12 + 48%|████████████████████████████████████████████████████████████████▊ | 43474/89905 [13:45:44<11:30:40, 1.12it/s][2025-04-29 00:10:48] (step=0043475) Train Loss: 5.6889, Train Steps/Sec: 1.12 + 48%|████████████████████████████████████████████████████████████████▊ | 43499/89905 [13:46:07<11:29:35, 1.12it/s][2025-04-29 00:11:10] (step=0043500) Train Loss: 5.7090, Train Steps/Sec: 1.12 + 48%|████████████████████████████████████████████████████████████████▊ | 43524/89905 [13:46:29<11:26:19, 1.13it/s][2025-04-29 00:11:32] (step=0043525) Train Loss: 5.6773, Train Steps/Sec: 1.12 + 48%|████████████████████████████████████████████████████████████████▉ | 43549/89905 [13:46:51<11:26:38, 1.13it/s][2025-04-29 00:11:55] (step=0043550) Train Loss: 5.7125, Train Steps/Sec: 1.12 + 48%|████████████████████████████████████████████████████████████████▉ | 43574/89905 [13:47:14<11:24:30, 1.13it/s][2025-04-29 00:12:17] (step=0043575) Train Loss: 5.6334, Train Steps/Sec: 1.12 + 48%|████████████████████████████████████████████████████████████████▉ | 43599/89905 [13:47:36<11:23:18, 1.13it/s][2025-04-29 00:12:39] (step=0043600) Train Loss: 5.6817, Train Steps/Sec: 1.12 + 49%|█████████████████████████████████████████████████████████████████ | 43624/89905 [13:47:58<11:46:47, 1.09it/s][2025-04-29 00:13:02] (step=0043625) Train Loss: 5.7197, Train Steps/Sec: 1.12 + 49%|█████████████████████████████████████████████████████████████████ | 43649/89905 [13:48:21<11:30:38, 1.12it/s][2025-04-29 00:13:24] (step=0043650) Train Loss: 5.6980, Train Steps/Sec: 1.12 + 49%|█████████████████████████████████████████████████████████████████ | 43674/89905 [13:48:43<11:29:39, 1.12it/s][2025-04-29 00:13:46] (step=0043675) Train Loss: 5.6495, Train Steps/Sec: 1.12 + 49%|█████████████████████████████████████████████████████████████████▏ | 43699/89905 [13:49:05<11:31:42, 1.11it/s][2025-04-29 00:14:09] (step=0043700) Train Loss: 5.7045, Train Steps/Sec: 1.12 + 49%|█████████████████████████████████████████████████████████████████▏ | 43724/89905 [13:49:28<11:23:56, 1.13it/s][2025-04-29 00:14:31] (step=0043725) Train Loss: 5.7056, Train Steps/Sec: 1.11 + 49%|█████████████████████████████████████████████████████████████████▏ | 43749/89905 [13:49:50<11:23:16, 1.13it/s][2025-04-29 00:14:54] (step=0043750) Train Loss: 5.6773, Train Steps/Sec: 1.11 + 49%|█████████████████████████████████████████████████████████████████▏ | 43774/89905 [13:50:13<11:24:55, 1.12it/s][2025-04-29 00:15:16] (step=0043775) Train Loss: 5.6808, Train Steps/Sec: 1.12 + 49%|█████████████████████████████████████████████████████████████████▎ | 43799/89905 [13:50:35<11:17:56, 1.13it/s][2025-04-29 00:15:39] (step=0043800) Train Loss: 5.6732, Train Steps/Sec: 1.11 + 49%|█████████████████████████████████████████████████████████████████▎ | 43824/89905 [13:50:57<11:39:42, 1.10it/s][2025-04-29 00:16:01] (step=0043825) Train Loss: 5.7029, Train Steps/Sec: 1.12 + 49%|█████████████████████████████████████████████████████████████████▎ | 43849/89905 [13:51:20<11:29:44, 1.11it/s][2025-04-29 00:16:23] (step=0043850) Train Loss: 5.6863, Train Steps/Sec: 1.12 + 49%|█████████████████████████████████████████████████████████████████▍ | 43874/89905 [13:51:42<11:25:55, 1.12it/s][2025-04-29 00:16:45] (step=0043875) Train Loss: 5.6860, Train Steps/Sec: 1.12 + 49%|█████████████████████████████████████████████████████████████████▍ | 43899/89905 [13:52:04<11:22:30, 1.12it/s][2025-04-29 00:17:08] (step=0043900) Train Loss: 5.6361, Train Steps/Sec: 1.12 + 49%|█████████████████████████████████████████████████████████████████▍ | 43924/89905 [13:52:27<11:20:22, 1.13it/s][2025-04-29 00:17:30] (step=0043925) Train Loss: 5.6031, Train Steps/Sec: 1.12 + 49%|█████████████████████████████████████████████████████████████████▌ | 43949/89905 [13:52:49<11:20:18, 1.13it/s][2025-04-29 00:17:52] (step=0043950) Train Loss: 5.5885, Train Steps/Sec: 1.12 + 49%|█████████████████████████████████████████████████████████████████▌ | 43974/89905 [13:53:11<11:18:17, 1.13it/s][2025-04-29 00:18:15] (step=0043975) Train Loss: 5.6607, Train Steps/Sec: 1.12 + 49%|█████████████████████████████████████████████████████████████████▌ | 43999/89905 [13:53:33<11:14:52, 1.13it/s][2025-04-29 00:18:37] (step=0044000) Train Loss: 5.5520, Train Steps/Sec: 1.12 +You are using a model of type blip-2 to instantiate a model of type instructblip. This is not supported for all configurations of models and can yield errors. +[2025-04-29 00:18:37] text_config is None. Initializing the text config with default values (`OPTConfig`). +model_name_or_path:/tmp/haozhezhao/model/blip2-flan-t5-xl +image_place_holder: +tokenizer length after expend 32101 +tokenizer length before expend 32100 +100%|███████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████| 6/6 [06:01<00:00, 60.17s/it] +[2025-04-29 00:25:57] Finish Eval in 44000 steps...████████████████████████████████████████████████████████████████████| 6/6 [06:00<00:00, 60.02s/it] +processing left-padding... +processing left-padding... +processing left-padding... +processing left-padding... +processing left-padding... +processing left-padding... +[2025-04-29 00:26:17] Saved checkpoint to checkpoint/X2I_UltraEdiit_fourmask_LLava_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_filtered_objects_4mask_1280_context_after_stage1_5e4/000-GPT-XL/checkpoints/0044000.pt +[2025-04-29 00:26:19] Removed old checkpoint: checkpoint/X2I_UltraEdiit_fourmask_LLava_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_filtered_objects_4mask_1280_context_after_stage1_5e4/000-GPT-XL/checkpoints/0042000.pt + 49%|█████████████████████████████████████████████████████████████████▌ | 44024/89905 [14:01:38<11:53:59, 1.07it/s][2025-04-29 00:26:41] (step=0044025) Train Loss: 5.6225, Train Steps/Sec: 0.05 + 49%|█████████████████████████████████████████████████████████████████▋ | 44049/89905 [14:02:00<11:26:53, 1.11it/s][2025-04-29 00:27:03] (step=0044050) Train Loss: 5.6339, Train Steps/Sec: 1.12 + 49%|█████████████████████████████████████████████████████████████████▋ | 44074/89905 [14:02:22<11:22:26, 1.12it/s][2025-04-29 00:27:26] (step=0044075) Train Loss: 5.6957, Train Steps/Sec: 1.12 + 49%|█████████████████████████████████████████████████████████████████▋ | 44099/89905 [14:02:53<19:12:51, 1.51s/it][2025-04-29 00:27:57] (step=0044100) Train Loss: 5.6669, Train Steps/Sec: 0.81 + 49%|█████████████████████████████████████████████████████████████████▊ | 44124/89905 [14:03:35<24:35:31, 1.93s/it][2025-04-29 00:28:38] (step=0044125) Train Loss: 5.6462, Train Steps/Sec: 0.60 + 49%|█████████████████████████████████████████████████████████████████▊ | 44149/89905 [14:04:15<21:05:41, 1.66s/it][2025-04-29 00:29:19] (step=0044150) Train Loss: 5.6389, Train Steps/Sec: 0.62 + 49%|█████████████████████████████████████████████████████████████████▊ | 44174/89905 [14:04:47<28:40:05, 2.26s/it][2025-04-29 00:29:50] (step=0044175) Train Loss: 5.7078, Train Steps/Sec: 0.79 + 49%|█████████████████████████████████████████████████████████████████▉ | 44199/89905 [14:05:09<11:11:32, 1.13it/s][2025-04-29 00:30:13] (step=0044200) Train Loss: 5.6398, Train Steps/Sec: 1.12 + 49%|█████████████████████████████████████████████████████████████████▉ | 44224/89905 [14:05:32<11:32:45, 1.10it/s][2025-04-29 00:30:35] (step=0044225) Train Loss: 5.6437, Train Steps/Sec: 1.12 + 49%|█████████████████████████████████████████████████████████████████▉ | 44249/89905 [14:05:54<11:24:23, 1.11it/s][2025-04-29 00:30:57] (step=0044250) Train Loss: 5.5646, Train Steps/Sec: 1.12 + 49%|█████████████████████████████████████████████████████████████████▉ | 44274/89905 [14:06:16<11:24:27, 1.11it/s][2025-04-29 00:31:20] (step=0044275) Train Loss: 5.6698, Train Steps/Sec: 1.12 + 49%|██████████████████████████████████████████████████████████████████ | 44299/89905 [14:06:39<11:16:08, 1.12it/s][2025-04-29 00:31:42] (step=0044300) Train Loss: 5.6446, Train Steps/Sec: 1.12 + 49%|██████████████████████████████████████████████████████████████████ | 44324/89905 [14:07:01<11:13:01, 1.13it/s][2025-04-29 00:32:04] (step=0044325) Train Loss: 5.6691, Train Steps/Sec: 1.12 + 49%|██████████████████████████████████████████████████████████████████ | 44349/89905 [14:07:23<11:12:04, 1.13it/s][2025-04-29 00:32:27] (step=0044350) Train Loss: 5.6494, Train Steps/Sec: 1.12 + 49%|██████████████████████████████████████████████████████████████████▏ | 44374/89905 [14:07:45<11:10:08, 1.13it/s][2025-04-29 00:32:49] (step=0044375) Train Loss: 5.6703, Train Steps/Sec: 1.12 + 49%|██████████████████████████████████████████████████████████████████▏ | 44399/89905 [14:08:08<11:08:19, 1.13it/s][2025-04-29 00:33:11] (step=0044400) Train Loss: 5.6893, Train Steps/Sec: 1.12 + 49%|██████████████████████████████████████████████████████████████████▏ | 44424/89905 [14:08:30<11:28:49, 1.10it/s][2025-04-29 00:33:33] (step=0044425) Train Loss: 5.7007, Train Steps/Sec: 1.12 + 49%|██████████████████████████████████████████████████████████████████▏ | 44449/89905 [14:08:52<11:22:54, 1.11it/s][2025-04-29 00:33:56] (step=0044450) Train Loss: 5.7023, Train Steps/Sec: 1.12 + 49%|██████████████████████████████████████████████████████████████████▎ | 44474/89905 [14:09:15<11:17:36, 1.12it/s][2025-04-29 00:34:18] (step=0044475) Train Loss: 5.6614, Train Steps/Sec: 1.12 + 49%|██████████████████████████████████████████████████████████████████▎ | 44499/89905 [14:09:37<11:13:10, 1.12it/s][2025-04-29 00:34:40] (step=0044500) Train Loss: 5.6989, Train Steps/Sec: 1.12 + 50%|██████████████████████████████████████████████████████████████████▎ | 44524/89905 [14:09:59<11:10:59, 1.13it/s][2025-04-29 00:35:03] (step=0044525) Train Loss: 5.7172, Train Steps/Sec: 1.12 + 50%|██████████████████████████████████████████████████████████████████▍ | 44549/89905 [14:10:22<11:10:50, 1.13it/s][2025-04-29 00:35:25] (step=0044550) Train Loss: 5.6632, Train Steps/Sec: 1.12 + 50%|██████████████████████████████████████████████████████████████████▍ | 44574/89905 [14:10:44<11:08:59, 1.13it/s][2025-04-29 00:35:47] (step=0044575) Train Loss: 5.6868, Train Steps/Sec: 1.12 + 50%|██████████████████████████████████████████████████████████████████▍ | 44599/89905 [14:11:06<11:06:49, 1.13it/s][2025-04-29 00:36:10] (step=0044600) Train Loss: 5.7103, Train Steps/Sec: 1.12 + 50%|██████████████████████████████████████████████████████████████████▌ | 44624/89905 [14:11:29<11:29:41, 1.09it/s][2025-04-29 00:36:32] (step=0044625) Train Loss: 5.6649, Train Steps/Sec: 1.12 + 50%|██████████████████████████████████████████████████████████████████▌ | 44649/89905 [14:11:51<11:16:50, 1.11it/s][2025-04-29 00:36:54] (step=0044650) Train Loss: 5.6666, Train Steps/Sec: 1.12 + 50%|██████████████████████████████████████████████████████████████████▌ | 44674/89905 [14:12:13<11:13:28, 1.12it/s][2025-04-29 00:37:16] (step=0044675) Train Loss: 5.6089, Train Steps/Sec: 1.12 + 50%|██████████████████████████████████████████████████████████████████▌ | 44699/89905 [14:12:35<11:13:43, 1.12it/s][2025-04-29 00:37:39] (step=0044700) Train Loss: 5.6965, Train Steps/Sec: 1.12 + 50%|██████████████████████████████████████████████████████████████████▋ | 44724/89905 [14:12:58<11:12:32, 1.12it/s][2025-04-29 00:38:01] (step=0044725) Train Loss: 5.7018, Train Steps/Sec: 1.12 + 50%|██████████████████████████████████████████████████████████████████▋ | 44749/89905 [14:13:20<11:07:15, 1.13it/s][2025-04-29 00:38:23] (step=0044750) Train Loss: 5.6461, Train Steps/Sec: 1.12 + 50%|██████████████████████████████████████████████████████████████████▋ | 44774/89905 [14:13:42<11:05:00, 1.13it/s][2025-04-29 00:38:46] (step=0044775) Train Loss: 5.5549, Train Steps/Sec: 1.12 + 50%|██████████████████████████████████████████████████████████████████▊ | 44799/89905 [14:14:05<11:04:52, 1.13it/s][2025-04-29 00:39:08] (step=0044800) Train Loss: 5.6778, Train Steps/Sec: 1.12 + 50%|██████████████████████████████████████████████████████████████████▊ | 44824/89905 [14:14:27<11:27:13, 1.09it/s][2025-04-29 00:39:30] (step=0044825) Train Loss: 5.7400, Train Steps/Sec: 1.12 + 50%|██████████████████████████████████████████████████████████████████▊ | 44849/89905 [14:14:49<11:15:13, 1.11it/s][2025-04-29 00:39:53] (step=0044850) Train Loss: 5.6828, Train Steps/Sec: 1.12 + 50%|██████████████████████████████████████████████████████████████████▉ | 44874/89905 [14:15:12<11:09:03, 1.12it/s][2025-04-29 00:40:15] (step=0044875) Train Loss: 5.7268, Train Steps/Sec: 1.12 + 50%|██████████████████████████████████████████████████████████████████▉ | 44899/89905 [14:15:34<11:06:33, 1.13it/s][2025-04-29 00:40:37] (step=0044900) Train Loss: 5.7104, Train Steps/Sec: 1.12 + 50%|██████████████████████████████████████████████████████████████████▉ | 44924/89905 [14:15:56<11:03:45, 1.13it/s][2025-04-29 00:41:00] (step=0044925) Train Loss: 5.7082, Train Steps/Sec: 1.12 + 50%|██████████████████████████████████████████████████████████████████▉ | 44949/89905 [14:16:18<11:07:01, 1.12it/s][2025-04-29 00:41:22] (step=0044950) Train Loss: 5.6950, Train Steps/Sec: 1.12 + 50%|███████████████████████████████████████████████████████████████████ | 44974/89905 [14:16:41<11:02:50, 1.13it/s][2025-04-29 00:41:44] (step=0044975) Train Loss: 5.6528, Train Steps/Sec: 1.12 + 50%|███████████████████████████████████████████████████████████████████ | 44999/89905 [14:17:03<11:00:08, 1.13it/s][2025-04-29 00:42:06] (step=0045000) Train Loss: 5.6408, Train Steps/Sec: 1.12 + 50%|███████████████████████████████████████████████████████████████████ | 45024/89905 [14:17:25<11:22:04, 1.10it/s][2025-04-29 00:42:29] (step=0045025) Train Loss: 5.6076, Train Steps/Sec: 1.12 + 50%|███████████████████████████████████████████████████████████████████▏ | 45049/89905 [14:17:48<11:13:16, 1.11it/s][2025-04-29 00:42:51] (step=0045050) Train Loss: 5.6475, Train Steps/Sec: 1.12 + 50%|███████████████████████████████████████████████████████████████████▏ | 45074/89905 [14:18:10<11:09:44, 1.12it/s][2025-04-29 00:43:13] (step=0045075) Train Loss: 5.6710, Train Steps/Sec: 1.12 + 50%|███████████████████████████████████████████████████████████████████▏ | 45099/89905 [14:18:32<11:04:35, 1.12it/s][2025-04-29 00:43:36] (step=0045100) Train Loss: 5.6068, Train Steps/Sec: 1.12 + 50%|███████████████████████████████████████████████████████████████████▎ | 45124/89905 [14:18:55<11:02:11, 1.13it/s][2025-04-29 00:43:58] (step=0045125) Train Loss: 5.6347, Train Steps/Sec: 1.12 + 50%|███████████████████████████████████████████████████████████████████▎ | 45149/89905 [14:19:17<10:59:24, 1.13it/s][2025-04-29 00:44:20] (step=0045150) Train Loss: 5.6640, Train Steps/Sec: 1.12 + 50%|███████████████████████████████████████████████████████████████████▎ | 45174/89905 [14:19:39<11:00:42, 1.13it/s][2025-04-29 00:44:43] (step=0045175) Train Loss: 5.6386, Train Steps/Sec: 1.12 + 50%|███████████████████████████████████████████████████████████████████▎ | 45199/89905 [14:20:01<10:58:45, 1.13it/s][2025-04-29 00:45:05] (step=0045200) Train Loss: 5.6848, Train Steps/Sec: 1.12 + 50%|███████████████████████████████████████████████████████████████████▍ | 45224/89905 [14:20:24<11:20:51, 1.09it/s][2025-04-29 00:45:27] (step=0045225) Train Loss: 5.7438, Train Steps/Sec: 1.12 + 50%|███████████████████████████████████████████████████████████████████▍ | 45249/89905 [14:20:46<11:08:42, 1.11it/s][2025-04-29 00:45:50] (step=0045250) Train Loss: 5.6714, Train Steps/Sec: 1.12 + 50%|███████████████████████████████████████████████████████████████████▍ | 45274/89905 [14:21:08<11:04:47, 1.12it/s][2025-04-29 00:46:12] (step=0045275) Train Loss: 5.6269, Train Steps/Sec: 1.12 + 50%|███████████████████████████████████████████████████████████████████▌ | 45299/89905 [14:21:31<11:00:34, 1.13it/s][2025-04-29 00:46:34] (step=0045300) Train Loss: 5.6300, Train Steps/Sec: 1.12 + 50%|███████████████████████████████████████████████████████████████████▌ | 45324/89905 [14:21:53<10:58:14, 1.13it/s][2025-04-29 00:46:56] (step=0045325) Train Loss: 5.6631, Train Steps/Sec: 1.12 + 50%|███████████████████████████████████████████████████████████████████▌ | 45349/89905 [14:22:15<10:59:01, 1.13it/s][2025-04-29 00:47:19] (step=0045350) Train Loss: 5.6476, Train Steps/Sec: 1.12 + 50%|███████████████████████████████████████████████████████████████████▋ | 45374/89905 [14:22:37<10:55:06, 1.13it/s][2025-04-29 00:47:41] (step=0045375) Train Loss: 5.6634, Train Steps/Sec: 1.12 + 50%|███████████████████████████████████████████████████████████████████▋ | 45399/89905 [14:23:00<10:55:37, 1.13it/s][2025-04-29 00:48:03] (step=0045400) Train Loss: 5.6954, Train Steps/Sec: 1.12 + 51%|███████████████████████████████████████████████████████████████████▋ | 45424/89905 [14:23:22<11:16:12, 1.10it/s][2025-04-29 00:48:26] (step=0045425) Train Loss: 5.6846, Train Steps/Sec: 1.12 + 51%|███████████████████████████████████████████████████████████████████▋ | 45449/89905 [14:23:44<11:07:22, 1.11it/s][2025-04-29 00:48:48] (step=0045450) Train Loss: 5.6599, Train Steps/Sec: 1.12 + 51%|███████████████████████████████████████████████████████████████████▊ | 45474/89905 [14:24:07<11:03:01, 1.12it/s][2025-04-29 00:49:10] (step=0045475) Train Loss: 5.6389, Train Steps/Sec: 1.12 + 51%|███████████████████████████████████████████████████████████████████▊ | 45499/89905 [14:24:29<10:57:28, 1.13it/s][2025-04-29 00:49:32] (step=0045500) Train Loss: 5.6133, Train Steps/Sec: 1.12 + 51%|███████████████████████████████████████████████████████████████████▊ | 45524/89905 [14:24:51<10:54:30, 1.13it/s][2025-04-29 00:49:55] (step=0045525) Train Loss: 5.6518, Train Steps/Sec: 1.12 + 51%|███████████████████████████████████████████████████████████████████▉ | 45549/89905 [14:25:14<10:55:54, 1.13it/s][2025-04-29 00:50:17] (step=0045550) Train Loss: 5.7374, Train Steps/Sec: 1.12 + 51%|███████████████████████████████████████████████████████████████████▉ | 45574/89905 [14:25:36<10:55:44, 1.13it/s][2025-04-29 00:50:39] (step=0045575) Train Loss: 5.7107, Train Steps/Sec: 1.12 + 51%|███████████████████████████████████████████████████████████████████▉ | 45599/89905 [14:25:58<10:51:46, 1.13it/s][2025-04-29 00:51:02] (step=0045600) Train Loss: 5.6553, Train Steps/Sec: 1.12 + 51%|████████████████████████████████████████████████████████████████████ | 45624/89905 [14:26:21<11:12:12, 1.10it/s][2025-04-29 00:51:24] (step=0045625) Train Loss: 5.6526, Train Steps/Sec: 1.12 + 51%|████████████████████████████████████████████████████████████████████ | 45649/89905 [14:26:43<11:01:15, 1.12it/s][2025-04-29 00:51:46] (step=0045650) Train Loss: 5.6825, Train Steps/Sec: 1.12 + 51%|████████████████████████████████████████████████████████████████████ | 45674/89905 [14:27:05<10:56:17, 1.12it/s][2025-04-29 00:52:08] (step=0045675) Train Loss: 5.6364, Train Steps/Sec: 1.12 + 51%|████████████████████████████████████████████████████████████████████ | 45699/89905 [14:27:27<10:55:47, 1.12it/s][2025-04-29 00:52:31] (step=0045700) Train Loss: 5.5499, Train Steps/Sec: 1.12 + 51%|████████████████████████████████████████████████████████████████████▏ | 45724/89905 [14:27:50<10:50:56, 1.13it/s][2025-04-29 00:52:53] (step=0045725) Train Loss: 5.6614, Train Steps/Sec: 1.12 + 51%|████████████████████████████████████████████████████████████████████▏ | 45749/89905 [14:28:12<10:51:47, 1.13it/s][2025-04-29 00:53:15] (step=0045750) Train Loss: 5.6531, Train Steps/Sec: 1.12 + 51%|████████████████████████████████████████████████████████████████████▏ | 45774/89905 [14:28:34<10:52:00, 1.13it/s][2025-04-29 00:53:38] (step=0045775) Train Loss: 5.6939, Train Steps/Sec: 1.12 + 51%|████████████████████████████████████████████████████████████████████▎ | 45799/89905 [14:28:56<10:49:31, 1.13it/s][2025-04-29 00:54:00] (step=0045800) Train Loss: 5.6931, Train Steps/Sec: 1.12 + 51%|████████████████████████████████████████████████████████████████████▎ | 45824/89905 [14:29:19<11:06:46, 1.10it/s][2025-04-29 00:54:22] (step=0045825) Train Loss: 5.6661, Train Steps/Sec: 1.12 + 51%|████████████████████████████████████████████████████████████████████▎ | 45849/89905 [14:29:41<10:58:30, 1.12it/s][2025-04-29 00:54:45] (step=0045850) Train Loss: 5.6094, Train Steps/Sec: 1.12 + 51%|████████████████████████████████████████████████████████████████████▎ | 45874/89905 [14:30:03<10:54:15, 1.12it/s][2025-04-29 00:55:07] (step=0045875) Train Loss: 5.6916, Train Steps/Sec: 1.12 + 51%|████████████████████████████████████████████████████████████████████▍ | 45899/89905 [14:30:26<10:52:01, 1.12it/s][2025-04-29 00:55:29] (step=0045900) Train Loss: 5.5855, Train Steps/Sec: 1.12 + 51%|████████████████████████████████████████████████████████████████████▍ | 45924/89905 [14:30:48<10:51:07, 1.13it/s][2025-04-29 00:55:51] (step=0045925) Train Loss: 5.6305, Train Steps/Sec: 1.12 + 51%|████████████████████████████████████████████████████████████████████▍ | 45949/89905 [14:31:10<10:45:58, 1.13it/s][2025-04-29 00:56:14] (step=0045950) Train Loss: 5.6352, Train Steps/Sec: 1.12 + 51%|████████████████████████████████████████████████████████████████████▌ | 45974/89905 [14:31:33<10:47:39, 1.13it/s][2025-04-29 00:56:36] (step=0045975) Train Loss: 5.6476, Train Steps/Sec: 1.12 + 51%|████████████████████████████████████████████████████████████████████▌ | 45999/89905 [14:31:55<10:43:37, 1.14it/s][2025-04-29 00:56:58] (step=0046000) Train Loss: 5.6467, Train Steps/Sec: 1.12 +You are using a model of type blip-2 to instantiate a model of type instructblip. This is not supported for all configurations of models and can yield errors. +[2025-04-29 00:56:58] text_config is None. Initializing the text config with default values (`OPTConfig`). +model_name_or_path:/tmp/haozhezhao/model/blip2-flan-t5-xl +image_place_holder: +tokenizer length after expend 32101 +tokenizer length before expend 32100 +100%|███████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████| 6/6 [06:01<00:00, 60.33s/it] +[2025-04-29 01:04:19] Finish Eval in 46000 steps...████████████████████████████████████████████████████████████████████| 6/6 [06:01<00:00, 60.10s/it] +processing left-padding... +processing left-padding... +processing left-padding... +processing left-padding... +processing left-padding... +processing left-padding... +[2025-04-29 01:04:39] Saved checkpoint to checkpoint/X2I_UltraEdiit_fourmask_LLava_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_filtered_objects_4mask_1280_context_after_stage1_5e4/000-GPT-XL/checkpoints/0046000.pt +[2025-04-29 01:04:41] Removed old checkpoint: checkpoint/X2I_UltraEdiit_fourmask_LLava_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_filtered_objects_4mask_1280_context_after_stage1_5e4/000-GPT-XL/checkpoints/0044000.pt + 51%|████████████████████████████████████████████████████████████████████▌ | 46024/89905 [14:40:00<11:26:36, 1.07it/s][2025-04-29 01:05:03] (step=0046025) Train Loss: 5.6026, Train Steps/Sec: 0.05 + 51%|████████████████████████████████████████████████████████████████████▋ | 46049/89905 [14:40:22<10:57:59, 1.11it/s][2025-04-29 01:05:26] (step=0046050) Train Loss: 5.6951, Train Steps/Sec: 1.12 + 51%|████████████████████████████████████████████████████████████████████▋ | 46074/89905 [14:40:45<10:51:51, 1.12it/s][2025-04-29 01:05:48] (step=0046075) Train Loss: 5.6402, Train Steps/Sec: 1.12 + 51%|████████████████████████████████████████████████████████████████████▋ | 46099/89905 [14:41:07<10:52:23, 1.12it/s][2025-04-29 01:06:10] (step=0046100) Train Loss: 5.6464, Train Steps/Sec: 1.12 + 51%|████████████████████████████████████████████████████████████████████▋ | 46124/89905 [14:41:29<10:46:29, 1.13it/s][2025-04-29 01:06:32] (step=0046125) Train Loss: 5.5826, Train Steps/Sec: 1.12 + 51%|████████████████████████████████████████████████████████████████████▊ | 46149/89905 [14:41:51<10:47:54, 1.13it/s][2025-04-29 01:06:55] (step=0046150) Train Loss: 5.5786, Train Steps/Sec: 1.12 + 51%|████████████████████████████████████████████████████████████████████▊ | 46174/89905 [14:42:14<10:44:51, 1.13it/s][2025-04-29 01:07:17] (step=0046175) Train Loss: 5.6548, Train Steps/Sec: 1.12 + 51%|████████████████████████████████████████████████████████████████████▊ | 46199/89905 [14:42:36<10:45:53, 1.13it/s][2025-04-29 01:07:40] (step=0046200) Train Loss: 5.6580, Train Steps/Sec: 1.11 + 51%|████████████████████████████████████████████████████████████████████▉ | 46224/89905 [14:42:58<11:03:48, 1.10it/s][2025-04-29 01:08:02] (step=0046225) Train Loss: 5.6407, Train Steps/Sec: 1.12 + 51%|████████████████████████████████████████████████████████████████████▉ | 46249/89905 [14:43:21<10:53:54, 1.11it/s][2025-04-29 01:08:24] (step=0046250) Train Loss: 5.6010, Train Steps/Sec: 1.12 + 51%|████████████████████████████████████████████████████████████████████▉ | 46274/89905 [14:43:43<10:49:51, 1.12it/s][2025-04-29 01:08:46] (step=0046275) Train Loss: 5.6310, Train Steps/Sec: 1.12 + 51%|█████████████████████████████████████████████████████████████████████ | 46299/89905 [14:44:05<10:47:06, 1.12it/s][2025-04-29 01:09:09] (step=0046300) Train Loss: 5.6470, Train Steps/Sec: 1.12 + 52%|█████████████████████████████████████████████████████████████████████ | 46324/89905 [14:44:28<10:42:10, 1.13it/s][2025-04-29 01:09:31] (step=0046325) Train Loss: 5.6593, Train Steps/Sec: 1.12 + 52%|█████████████████████████████████████████████████████████████████████ | 46349/89905 [14:44:50<10:44:04, 1.13it/s][2025-04-29 01:09:53] (step=0046350) Train Loss: 5.6668, Train Steps/Sec: 1.12 + 52%|█████████████████████████████████████████████████████████████████████ | 46374/89905 [14:45:12<10:40:38, 1.13it/s][2025-04-29 01:10:16] (step=0046375) Train Loss: 5.7013, Train Steps/Sec: 1.12 + 52%|█████████████████████████████████████████████████████████████████████▏ | 46399/89905 [14:45:35<10:40:59, 1.13it/s][2025-04-29 01:10:38] (step=0046400) Train Loss: 5.5764, Train Steps/Sec: 1.12 + 52%|█████████████████████████████████████████████████████████████████████▏ | 46424/89905 [14:45:57<11:01:26, 1.10it/s][2025-04-29 01:11:00] (step=0046425) Train Loss: 5.6383, Train Steps/Sec: 1.12 + 52%|█████████████████████████████████████████████████████████████████████▏ | 46449/89905 [14:46:19<10:50:46, 1.11it/s][2025-04-29 01:11:23] (step=0046450) Train Loss: 5.6294, Train Steps/Sec: 1.12 + 52%|█████████████████████████████████████████████████████████████████████▎ | 46474/89905 [14:46:42<10:46:24, 1.12it/s][2025-04-29 01:11:45] (step=0046475) Train Loss: 5.5900, Train Steps/Sec: 1.12 + 52%|█████████████████████████████████████████████████████████████████████▎ | 46499/89905 [14:47:04<10:44:53, 1.12it/s][2025-04-29 01:12:07] (step=0046500) Train Loss: 5.6494, Train Steps/Sec: 1.12 + 52%|█████████████████████████████████████████████████████████████████████▎ | 46524/89905 [14:47:26<10:43:03, 1.12it/s][2025-04-29 01:12:30] (step=0046525) Train Loss: 5.6869, Train Steps/Sec: 1.12 + 52%|█████████████████████████████████████████████████████████████████████▍ | 46549/89905 [14:47:49<10:40:35, 1.13it/s][2025-04-29 01:12:52] (step=0046550) Train Loss: 5.6843, Train Steps/Sec: 1.12 + 52%|█████████████████████████████████████████████████████████████████████▍ | 46574/89905 [14:48:11<10:40:51, 1.13it/s][2025-04-29 01:13:14] (step=0046575) Train Loss: 5.5866, Train Steps/Sec: 1.12 + 52%|█████████████████████████████████████████████████████████████████████▍ | 46599/89905 [14:48:33<10:38:29, 1.13it/s][2025-04-29 01:13:37] (step=0046600) Train Loss: 5.6570, Train Steps/Sec: 1.11 + 52%|█████████████████████████████████████████████████████████████████████▍ | 46624/89905 [14:48:56<10:59:36, 1.09it/s][2025-04-29 01:13:59] (step=0046625) Train Loss: 5.7337, Train Steps/Sec: 1.12 + 52%|█████████████████████████████████████████████████████████████████████▌ | 46649/89905 [14:49:18<10:51:27, 1.11it/s][2025-04-29 01:14:21] (step=0046650) Train Loss: 5.7141, Train Steps/Sec: 1.12 + 52%|█████████████████████████████████████████████████████████████████████▌ | 46674/89905 [14:49:40<10:46:52, 1.11it/s][2025-04-29 01:14:44] (step=0046675) Train Loss: 5.6506, Train Steps/Sec: 1.12 + 52%|█████████████████████████████████████████████████████████████████████▌ | 46699/89905 [14:50:03<10:39:30, 1.13it/s][2025-04-29 01:15:06] (step=0046700) Train Loss: 5.5793, Train Steps/Sec: 1.12 + 52%|█████████████████████████████████████████████████████████████████████▋ | 46724/89905 [14:50:25<10:39:30, 1.13it/s][2025-04-29 01:15:28] (step=0046725) Train Loss: 5.6166, Train Steps/Sec: 1.12 + 52%|█████████████████████████████████████████████████████████████████████▋ | 46749/89905 [14:50:47<10:40:08, 1.12it/s][2025-04-29 01:15:51] (step=0046750) Train Loss: 5.6955, Train Steps/Sec: 1.12 + 52%|█████████████████████████████████████████████████████████████████████▋ | 46774/89905 [14:51:10<10:38:11, 1.13it/s][2025-04-29 01:16:13] (step=0046775) Train Loss: 5.6770, Train Steps/Sec: 1.12 + 52%|█████████████████████████████████████████████████████████████████████▊ | 46799/89905 [14:51:32<10:33:09, 1.13it/s][2025-04-29 01:16:35] (step=0046800) Train Loss: 5.6471, Train Steps/Sec: 1.12 + 52%|█████████████████████████████████████████████████████████████████████▊ | 46824/89905 [14:51:54<10:57:24, 1.09it/s][2025-04-29 01:16:58] (step=0046825) Train Loss: 5.6445, Train Steps/Sec: 1.12 + 52%|█████████████████████████████████████████████████████████████████████▊ | 46849/89905 [14:52:17<10:46:44, 1.11it/s][2025-04-29 01:17:20] (step=0046850) Train Loss: 5.6422, Train Steps/Sec: 1.12 + 52%|█████████████████████████████████████████████████████████████████████▊ | 46874/89905 [14:52:39<10:42:28, 1.12it/s][2025-04-29 01:17:42] (step=0046875) Train Loss: 5.5732, Train Steps/Sec: 1.12 + 52%|█████████████████████████████████████████████████████████████████████▉ | 46899/89905 [14:53:01<10:37:31, 1.12it/s][2025-04-29 01:18:05] (step=0046900) Train Loss: 5.6204, Train Steps/Sec: 1.12 + 52%|█████████████████████████████████████████████████████████████████████▉ | 46924/89905 [14:53:24<10:34:30, 1.13it/s][2025-04-29 01:18:27] (step=0046925) Train Loss: 5.6200, Train Steps/Sec: 1.12 + 52%|█████████████████████████████████████████████████████████████████████▉ | 46949/89905 [14:53:46<10:35:30, 1.13it/s][2025-04-29 01:18:49] (step=0046950) Train Loss: 5.6061, Train Steps/Sec: 1.12 + 52%|██████████████████████████████████████████████████████████████████████ | 46974/89905 [14:54:08<10:31:50, 1.13it/s][2025-04-29 01:19:12] (step=0046975) Train Loss: 5.6811, Train Steps/Sec: 1.12 + 52%|██████████████████████████████████████████████████████████████████████ | 46999/89905 [14:54:31<10:34:30, 1.13it/s][2025-04-29 01:19:34] (step=0047000) Train Loss: 5.7023, Train Steps/Sec: 1.11 + 52%|██████████████████████████████████████████████████████████████████████ | 47024/89905 [14:54:53<10:49:45, 1.10it/s][2025-04-29 01:19:56] (step=0047025) Train Loss: 5.6133, Train Steps/Sec: 1.12 + 52%|██████████████████████████████████████████████████████████████████████ | 47049/89905 [14:55:15<10:43:38, 1.11it/s][2025-04-29 01:20:19] (step=0047050) Train Loss: 5.6947, Train Steps/Sec: 1.12 + 52%|██████████████████████████████████████████████████████████████████████▏ | 47074/89905 [14:55:38<10:38:12, 1.12it/s][2025-04-29 01:20:41] (step=0047075) Train Loss: 5.6534, Train Steps/Sec: 1.12 + 52%|██████████████████████████████████████████████████████████████████████▏ | 47099/89905 [14:56:00<10:37:27, 1.12it/s][2025-04-29 01:21:03] (step=0047100) Train Loss: 5.5772, Train Steps/Sec: 1.12 + 52%|██████████████████████████████████████████████████████████████████████▏ | 47124/89905 [14:56:22<10:32:52, 1.13it/s][2025-04-29 01:21:26] (step=0047125) Train Loss: 5.6863, Train Steps/Sec: 1.12 + 52%|██████████████████████████████████████████████████████████████████████▎ | 47149/89905 [14:56:45<10:29:57, 1.13it/s][2025-04-29 01:21:48] (step=0047150) Train Loss: 5.6075, Train Steps/Sec: 1.12 + 52%|██████████████████████████████████████████████████████████████████████▎ | 47174/89905 [14:57:07<10:32:08, 1.13it/s][2025-04-29 01:22:10] (step=0047175) Train Loss: 5.6845, Train Steps/Sec: 1.12 + 52%|██████████████████████████████████████████████████████████████████████▎ | 47199/89905 [14:57:29<10:29:19, 1.13it/s][2025-04-29 01:22:33] (step=0047200) Train Loss: 5.6236, Train Steps/Sec: 1.12 + 53%|██████████████████████████████████████████████████████████████████████▍ | 47224/89905 [14:57:52<10:48:10, 1.10it/s][2025-04-29 01:22:55] (step=0047225) Train Loss: 5.6184, Train Steps/Sec: 1.12 + 53%|██████████████████████████████████████████████████████████████████████▍ | 47249/89905 [14:58:14<10:39:16, 1.11it/s][2025-04-29 01:23:17] (step=0047250) Train Loss: 5.6793, Train Steps/Sec: 1.12 + 53%|██████████████████████████████████████████████████████████████████████▍ | 47274/89905 [14:58:36<10:36:04, 1.12it/s][2025-04-29 01:23:40] (step=0047275) Train Loss: 5.6733, Train Steps/Sec: 1.12 + 53%|██████████████████████████████████████████████████████████████████████▍ | 47299/89905 [14:58:58<10:34:07, 1.12it/s][2025-04-29 01:24:02] (step=0047300) Train Loss: 5.6135, Train Steps/Sec: 1.12 + 53%|██████████████████████████████████████████████████████████████████████▌ | 47324/89905 [14:59:21<10:31:53, 1.12it/s][2025-04-29 01:24:24] (step=0047325) Train Loss: 5.6454, Train Steps/Sec: 1.12 + 53%|██████████████████████████████████████████████████████████████████████▌ | 47349/89905 [14:59:43<10:31:09, 1.12it/s][2025-04-29 01:24:47] (step=0047350) Train Loss: 5.6420, Train Steps/Sec: 1.12 + 53%|██████████████████████████████████████████████████████████████████████▌ | 47374/89905 [15:00:05<10:26:16, 1.13it/s][2025-04-29 01:25:09] (step=0047375) Train Loss: 5.6583, Train Steps/Sec: 1.12 + 53%|██████████████████████████████████████████████████████████████████████▋ | 47399/89905 [15:00:28<10:27:26, 1.13it/s][2025-04-29 01:25:31] (step=0047400) Train Loss: 5.6078, Train Steps/Sec: 1.11 + 53%|██████████████████████████████████████████████████████████████████████▋ | 47424/89905 [15:00:50<10:44:23, 1.10it/s][2025-04-29 01:25:54] (step=0047425) Train Loss: 5.6280, Train Steps/Sec: 1.12 + 53%|██████████████████████████████████████████████████████████████████████▋ | 47449/89905 [15:01:13<10:36:39, 1.11it/s][2025-04-29 01:26:16] (step=0047450) Train Loss: 5.6332, Train Steps/Sec: 1.12 + 53%|██████████████████████████████████████████████████████████████████████▊ | 47474/89905 [15:01:35<10:31:59, 1.12it/s][2025-04-29 01:26:38] (step=0047475) Train Loss: 5.6841, Train Steps/Sec: 1.12 + 53%|██████████████████████████████████████████████████████████████████████▊ | 47499/89905 [15:01:57<10:29:44, 1.12it/s][2025-04-29 01:27:01] (step=0047500) Train Loss: 5.6310, Train Steps/Sec: 1.12 + 53%|██████████████████████████████████████████████████████████████████████▊ | 47524/89905 [15:02:19<10:27:30, 1.13it/s][2025-04-29 01:27:23] (step=0047525) Train Loss: 5.6379, Train Steps/Sec: 1.12 + 53%|██████████████████████████████████████████████████████████████████████▊ | 47549/89905 [15:02:42<10:25:43, 1.13it/s][2025-04-29 01:27:45] (step=0047550) Train Loss: 5.5910, Train Steps/Sec: 1.12 + 53%|██████████████████████████████████████████████████████████████████████▉ | 47574/89905 [15:03:04<10:24:08, 1.13it/s][2025-04-29 01:28:07] (step=0047575) Train Loss: 5.6121, Train Steps/Sec: 1.12 + 53%|██████████████████████████████████████████████████████████████████████▉ | 47599/89905 [15:03:26<10:22:07, 1.13it/s][2025-04-29 01:28:30] (step=0047600) Train Loss: 5.6309, Train Steps/Sec: 1.12 + 53%|██████████████████████████████████████████████████████████████████████▉ | 47624/89905 [15:03:49<10:45:55, 1.09it/s][2025-04-29 01:28:52] (step=0047625) Train Loss: 5.6770, Train Steps/Sec: 1.12 + 53%|███████████████████████████████████████████████████████████████████████ | 47649/89905 [15:04:11<10:34:55, 1.11it/s][2025-04-29 01:29:14] (step=0047650) Train Loss: 5.6189, Train Steps/Sec: 1.12 + 53%|███████████████████████████████████████████████████████████████████████ | 47674/89905 [15:04:42<10:42:32, 1.10it/s][2025-04-29 01:29:56] (step=0047675) Train Loss: 5.6766, Train Steps/Sec: 0.61 + 53%|███████████████████████████████████████████████████████████████████████ | 47699/89905 [15:05:24<10:28:21, 1.12it/s][2025-04-29 01:30:27] (step=0047700) Train Loss: 5.6503, Train Steps/Sec: 0.79 + 53%|███████████████████████████████████████████████████████████████████████▏ | 47724/89905 [15:06:04<10:39:01, 1.10it/s][2025-04-29 01:31:08] (step=0047725) Train Loss: 5.6335, Train Steps/Sec: 0.62 + 53%|███████████████████████████████████████████████████████████████████████▏ | 47749/89905 [15:06:36<10:31:25, 1.11it/s][2025-04-29 01:31:39] (step=0047750) Train Loss: 5.6307, Train Steps/Sec: 0.79 + 53%|███████████████████████████████████████████████████████████████████████▏ | 47774/89905 [15:06:58<10:23:10, 1.13it/s][2025-04-29 01:32:02] (step=0047775) Train Loss: 5.6656, Train Steps/Sec: 1.12 + 53%|███████████████████████████████████████████████████████████████████████▏ | 47799/89905 [15:07:20<10:20:23, 1.13it/s][2025-04-29 01:32:24] (step=0047800) Train Loss: 5.6320, Train Steps/Sec: 1.12 + 53%|███████████████████████████████████████████████████████████████████████▎ | 47824/89905 [15:07:43<10:40:13, 1.10it/s][2025-04-29 01:32:46] (step=0047825) Train Loss: 5.6139, Train Steps/Sec: 1.12 + 53%|███████████████████████████████████████████████████████████████████████▎ | 47849/89905 [15:08:05<10:29:13, 1.11it/s][2025-04-29 01:33:09] (step=0047850) Train Loss: 5.6424, Train Steps/Sec: 1.12 + 53%|███████████████████████████████████████████████████████████████████████▎ | 47874/89905 [15:08:27<10:25:40, 1.12it/s][2025-04-29 01:33:31] (step=0047875) Train Loss: 5.6570, Train Steps/Sec: 1.12 + 53%|███████████████████████████████████████████████████████████████████████▍ | 47899/89905 [15:08:50<10:22:44, 1.12it/s][2025-04-29 01:33:53] (step=0047900) Train Loss: 5.6031, Train Steps/Sec: 1.12 + 53%|███████████████████████████████████████████████████████████████████████▍ | 47924/89905 [15:09:12<10:22:34, 1.12it/s][2025-04-29 01:34:15] (step=0047925) Train Loss: 5.6467, Train Steps/Sec: 1.12 + 53%|███████████████████████████████████████████████████████████████████████▍ | 47949/89905 [15:09:34<10:20:19, 1.13it/s][2025-04-29 01:34:38] (step=0047950) Train Loss: 5.7381, Train Steps/Sec: 1.12 + 53%|███████████████████████████████████████████████████████████████████████▌ | 47974/89905 [15:09:57<10:19:07, 1.13it/s][2025-04-29 01:35:00] (step=0047975) Train Loss: 5.6946, Train Steps/Sec: 1.12 + 53%|███████████████████████████████████████████████████████████████████████▌ | 47999/89905 [15:10:19<10:18:11, 1.13it/s][2025-04-29 01:35:23] (step=0048000) Train Loss: 5.7035, Train Steps/Sec: 1.12 +You are using a model of type blip-2 to instantiate a model of type instructblip. This is not supported for all configurations of models and can yield errors. +[2025-04-29 01:35:23] text_config is None. Initializing the text config with default values (`OPTConfig`). +model_name_or_path:/tmp/haozhezhao/model/blip2-flan-t5-xl +image_place_holder: +tokenizer length after expend 32101 +tokenizer length before expend 32100 +100%|███████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████| 6/6 [06:00<00:00, 60.16s/it] +[2025-04-29 01:42:41] Finish Eval in 48000 steps...████████████████████████████████████████████████████████████████████| 6/6 [06:00<00:00, 60.00s/it] +processing left-padding... +processing left-padding... +processing left-padding... +processing left-padding... +processing left-padding... +processing left-padding... +[2025-04-29 01:43:01] Saved checkpoint to checkpoint/X2I_UltraEdiit_fourmask_LLava_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_filtered_objects_4mask_1280_context_after_stage1_5e4/000-GPT-XL/checkpoints/0048000.pt +[2025-04-29 01:43:03] Removed old checkpoint: checkpoint/X2I_UltraEdiit_fourmask_LLava_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_filtered_objects_4mask_1280_context_after_stage1_5e4/000-GPT-XL/checkpoints/0046000.pt + 53%|███████████████████████████████████████████████████████████████████████▌ | 48024/89905 [15:18:23<10:54:46, 1.07it/s][2025-04-29 01:43:26] (step=0048025) Train Loss: 5.6746, Train Steps/Sec: 0.05 + 53%|███████████████████████████████████████████████████████████████████████▌ | 48049/89905 [15:18:45<10:25:38, 1.12it/s][2025-04-29 01:43:48] (step=0048050) Train Loss: 5.6302, Train Steps/Sec: 1.12 + 53%|███████████████████████████████████████████████████████████████████████▋ | 48074/89905 [15:19:07<10:24:48, 1.12it/s][2025-04-29 01:44:11] (step=0048075) Train Loss: 5.5830, Train Steps/Sec: 1.12 + 53%|███████████████████████████████████████████████████████████████████████▋ | 48099/89905 [15:19:29<10:20:53, 1.12it/s][2025-04-29 01:44:33] (step=0048100) Train Loss: 5.6308, Train Steps/Sec: 1.12 + 54%|███████████████████████████████████████████████████████████████████████▋ | 48124/89905 [15:19:52<10:19:32, 1.12it/s][2025-04-29 01:44:55] (step=0048125) Train Loss: 5.6776, Train Steps/Sec: 1.12 + 54%|███████████████████████████████████████████████████████████████████████▊ | 48149/89905 [15:20:14<10:15:17, 1.13it/s][2025-04-29 01:45:17] (step=0048150) Train Loss: 5.6290, Train Steps/Sec: 1.12 + 54%|███████████████████████████████████████████████████████████████████████▊ | 48174/89905 [15:20:36<10:18:24, 1.12it/s][2025-04-29 01:45:40] (step=0048175) Train Loss: 5.5863, Train Steps/Sec: 1.12 + 54%|███████████████████████████████████████████████████████████████████████▊ | 48199/89905 [15:20:59<10:13:21, 1.13it/s][2025-04-29 01:46:02] (step=0048200) Train Loss: 5.6674, Train Steps/Sec: 1.12 + 54%|███████████████████████████████████████████████████████████████████████▉ | 48224/89905 [15:21:21<10:33:39, 1.10it/s][2025-04-29 01:46:24] (step=0048225) Train Loss: 5.6372, Train Steps/Sec: 1.12 + 54%|███████████████████████████████████████████████████████████████████████▉ | 48249/89905 [15:21:43<10:22:49, 1.11it/s][2025-04-29 01:46:47] (step=0048250) Train Loss: 5.6469, Train Steps/Sec: 1.12 + 54%|███████████████████████████████████████████████████████████████████████▉ | 48274/89905 [15:22:06<10:21:56, 1.12it/s][2025-04-29 01:47:09] (step=0048275) Train Loss: 5.6135, Train Steps/Sec: 1.12 + 54%|███████████████████████████████████████████████████████████████████████▉ | 48299/89905 [15:22:28<10:16:22, 1.13it/s][2025-04-29 01:47:31] (step=0048300) Train Loss: 5.6578, Train Steps/Sec: 1.12 + 54%|████████████████████████████████████████████████████████████████████████ | 48324/89905 [15:22:50<10:15:59, 1.13it/s][2025-04-29 01:47:53] (step=0048325) Train Loss: 5.6603, Train Steps/Sec: 1.12 + 54%|████████████████████████████████████████████████████████████████████████ | 48349/89905 [15:23:12<10:12:00, 1.13it/s][2025-04-29 01:48:16] (step=0048350) Train Loss: 5.6501, Train Steps/Sec: 1.12 + 54%|████████████████████████████████████████████████████████████████████████ | 48374/89905 [15:23:35<10:12:30, 1.13it/s][2025-04-29 01:48:38] (step=0048375) Train Loss: 5.6077, Train Steps/Sec: 1.12 + 54%|████████████████████████████████████████████████████████████████████████▏ | 48399/89905 [15:23:57<10:09:43, 1.13it/s][2025-04-29 01:49:00] (step=0048400) Train Loss: 5.6114, Train Steps/Sec: 1.12 + 54%|████████████████████████████████████████████████████████████████████████▏ | 48424/89905 [15:24:19<10:31:04, 1.10it/s][2025-04-29 01:49:23] (step=0048425) Train Loss: 5.6512, Train Steps/Sec: 1.12 + 54%|████████████████████████████████████████████████████████████████████████▏ | 48449/89905 [15:24:42<10:19:27, 1.12it/s][2025-04-29 01:49:45] (step=0048450) Train Loss: 5.5629, Train Steps/Sec: 1.12 + 54%|████████████████████████████████████████████████████████████████████████▏ | 48474/89905 [15:25:04<10:16:56, 1.12it/s][2025-04-29 01:50:07] (step=0048475) Train Loss: 5.6476, Train Steps/Sec: 1.12 + 54%|████████████████████████████████████████████████████████████████████████▎ | 48499/89905 [15:25:26<10:12:37, 1.13it/s][2025-04-29 01:50:29] (step=0048500) Train Loss: 5.7002, Train Steps/Sec: 1.12 + 54%|████████████████████████████████████████████████████████████████████████▎ | 48524/89905 [15:25:48<10:12:10, 1.13it/s][2025-04-29 01:50:52] (step=0048525) Train Loss: 5.6317, Train Steps/Sec: 1.12 + 54%|████████████████████████████████████████████████████████████████████████▎ | 48549/89905 [15:26:11<10:08:42, 1.13it/s][2025-04-29 01:51:14] (step=0048550) Train Loss: 5.6838, Train Steps/Sec: 1.12 + 54%|████████████████████████████████████████████████████████████████████████▍ | 48574/89905 [15:26:33<10:08:10, 1.13it/s][2025-04-29 01:51:36] (step=0048575) Train Loss: 5.6232, Train Steps/Sec: 1.12 + 54%|████████████████████████████████████████████████████████████████████████▍ | 48599/89905 [15:26:55<10:06:07, 1.14it/s][2025-04-29 01:51:59] (step=0048600) Train Loss: 5.5992, Train Steps/Sec: 1.12 + 54%|████████████████████████████████████████████████████████████████████████▍ | 48624/89905 [15:27:18<10:27:36, 1.10it/s][2025-04-29 01:52:21] (step=0048625) Train Loss: 5.5585, Train Steps/Sec: 1.12 + 54%|████████████████████████████████████████████████████████████████████████▌ | 48649/89905 [15:27:40<10:16:42, 1.11it/s][2025-04-29 01:52:43] (step=0048650) Train Loss: 5.5860, Train Steps/Sec: 1.12 + 54%|████████████████████████████████████████████████████████████████████████▌ | 48674/89905 [15:28:02<10:15:02, 1.12it/s][2025-04-29 01:53:06] (step=0048675) Train Loss: 5.6252, Train Steps/Sec: 1.12 + 54%|████████████████████████████████████████████████████████████████████████▌ | 48699/89905 [15:28:24<10:10:08, 1.13it/s][2025-04-29 01:53:28] (step=0048700) Train Loss: 5.6658, Train Steps/Sec: 1.12 + 54%|████████████████████████████████████████████████████████████████████████▌ | 48724/89905 [15:28:47<10:10:27, 1.12it/s][2025-04-29 01:53:50] (step=0048725) Train Loss: 5.6747, Train Steps/Sec: 1.12 + 54%|████████████████████████████████████████████████████████████████████████▋ | 48749/89905 [15:29:09<10:06:54, 1.13it/s][2025-04-29 01:54:12] (step=0048750) Train Loss: 5.6583, Train Steps/Sec: 1.12 + 54%|████████████████████████████████████████████████████████████████████████▋ | 48774/89905 [15:29:31<10:06:07, 1.13it/s][2025-04-29 01:54:35] (step=0048775) Train Loss: 5.6206, Train Steps/Sec: 1.12 + 54%|████████████████████████████████████████████████████████████████████████▋ | 48799/89905 [15:29:54<10:04:58, 1.13it/s][2025-04-29 01:54:57] (step=0048800) Train Loss: 5.5280, Train Steps/Sec: 1.12 + 54%|████████████████████████████████████████████████████████████████████████▊ | 48824/89905 [15:30:16<10:22:53, 1.10it/s][2025-04-29 01:55:19] (step=0048825) Train Loss: 5.6271, Train Steps/Sec: 1.12 + 54%|████████████████████████████████████████████████████████████████████████▊ | 48849/89905 [15:30:38<10:16:18, 1.11it/s][2025-04-29 01:55:42] (step=0048850) Train Loss: 5.5945, Train Steps/Sec: 1.12 + 54%|████████████████████████████████████████████████████████████████████████▊ | 48874/89905 [15:31:00<10:12:02, 1.12it/s][2025-04-29 01:56:04] (step=0048875) Train Loss: 5.7063, Train Steps/Sec: 1.12 + 54%|████████████████████████████████████████████████████████████████████████▉ | 48899/89905 [15:31:23<10:05:18, 1.13it/s][2025-04-29 01:56:26] (step=0048900) Train Loss: 5.6527, Train Steps/Sec: 1.12 + 54%|████████████████████████████████████████████████████████████████████████▉ | 48924/89905 [15:31:45<10:06:54, 1.13it/s][2025-04-29 01:56:48] (step=0048925) Train Loss: 5.6405, Train Steps/Sec: 1.12 + 54%|████████████████████████████████████████████████████████████████████████▉ | 48949/89905 [15:32:07<10:03:03, 1.13it/s][2025-04-29 01:57:11] (step=0048950) Train Loss: 5.6762, Train Steps/Sec: 1.12 + 54%|████████████████████████████████████████████████████████████████████████▉ | 48974/89905 [15:32:30<10:01:58, 1.13it/s][2025-04-29 01:57:33] (step=0048975) Train Loss: 5.5684, Train Steps/Sec: 1.12 + 55%|█████████████████████████████████████████████████████████████████████████ | 48999/89905 [15:32:52<10:00:20, 1.14it/s][2025-04-29 01:57:55] (step=0049000) Train Loss: 5.5945, Train Steps/Sec: 1.12 + 55%|█████████████████████████████████████████████████████████████████████████ | 49024/89905 [15:33:14<10:18:56, 1.10it/s][2025-04-29 01:58:18] (step=0049025) Train Loss: 5.6057, Train Steps/Sec: 1.12 + 55%|█████████████████████████████████████████████████████████████████████████ | 49049/89905 [15:33:36<10:13:55, 1.11it/s][2025-04-29 01:58:40] (step=0049050) Train Loss: 5.6087, Train Steps/Sec: 1.12 + 55%|█████████████████████████████████████████████████████████████████████████▏ | 49074/89905 [15:33:59<10:08:24, 1.12it/s][2025-04-29 01:59:02] (step=0049075) Train Loss: 5.5803, Train Steps/Sec: 1.12 + 55%|█████████████████████████████████████████████████████████████████████████▏ | 49099/89905 [15:34:21<10:06:15, 1.12it/s][2025-04-29 01:59:25] (step=0049100) Train Loss: 5.6757, Train Steps/Sec: 1.12 + 55%|█████████████████████████████████████████████████████████████████████████▏ | 49124/89905 [15:34:43<10:02:45, 1.13it/s][2025-04-29 01:59:47] (step=0049125) Train Loss: 5.6529, Train Steps/Sec: 1.12 + 55%|█████████████████████████████████████████████████████████████████████████▎ | 49149/89905 [15:35:06<10:03:18, 1.13it/s][2025-04-29 02:00:09] (step=0049150) Train Loss: 5.6560, Train Steps/Sec: 1.12 + 55%|█████████████████████████████████████████████████████████████████████████▎ | 49174/89905 [15:35:28<10:00:24, 1.13it/s][2025-04-29 02:00:31] (step=0049175) Train Loss: 5.6609, Train Steps/Sec: 1.12 + 55%|█████████████████████████████████████████████████████████████████████████▉ | 49199/89905 [15:35:50<9:58:18, 1.13it/s][2025-04-29 02:00:54] (step=0049200) Train Loss: 5.5957, Train Steps/Sec: 1.12 + 55%|█████████████████████████████████████████████████████████████████████████▎ | 49224/89905 [15:36:13<10:20:20, 1.09it/s][2025-04-29 02:01:16] (step=0049225) Train Loss: 5.5572, Train Steps/Sec: 1.12 + 55%|█████████████████████████████████████████████████████████████████████████▍ | 49249/89905 [15:36:35<10:09:14, 1.11it/s][2025-04-29 02:01:38] (step=0049250) Train Loss: 5.6461, Train Steps/Sec: 1.12 + 55%|█████████████████████████████████████████████████████████████████████████▍ | 49274/89905 [15:36:57<10:04:59, 1.12it/s][2025-04-29 02:02:01] (step=0049275) Train Loss: 5.5932, Train Steps/Sec: 1.12 + 55%|██████████████████████████████████████████████████████████████████████████ | 49299/89905 [15:37:19<9:59:40, 1.13it/s][2025-04-29 02:02:23] (step=0049300) Train Loss: 5.6418, Train Steps/Sec: 1.12 + 55%|██████████████████████████████████████████████████████████████████████████ | 49324/89905 [15:37:42<9:58:40, 1.13it/s][2025-04-29 02:02:45] (step=0049325) Train Loss: 5.7004, Train Steps/Sec: 1.12 + 55%|██████████████████████████████████████████████████████████████████████████ | 49349/89905 [15:38:04<9:58:33, 1.13it/s][2025-04-29 02:03:07] (step=0049350) Train Loss: 5.6505, Train Steps/Sec: 1.12 + 55%|██████████████████████████████████████████████████████████████████████████▏ | 49374/89905 [15:38:26<9:59:18, 1.13it/s][2025-04-29 02:03:30] (step=0049375) Train Loss: 5.7060, Train Steps/Sec: 1.12 + 55%|██████████████████████████████████████████████████████████████████████████▏ | 49399/89905 [15:38:49<9:56:18, 1.13it/s][2025-04-29 02:03:52] (step=0049400) Train Loss: 5.6231, Train Steps/Sec: 1.12 + 55%|█████████████████████████████████████████████████████████████████████████▋ | 49424/89905 [15:39:11<10:13:51, 1.10it/s][2025-04-29 02:04:14] (step=0049425) Train Loss: 5.6134, Train Steps/Sec: 1.12 + 55%|█████████████████████████████████████████████████████████████████████████▋ | 49449/89905 [15:39:33<10:07:52, 1.11it/s][2025-04-29 02:04:37] (step=0049450) Train Loss: 5.6258, Train Steps/Sec: 1.12 + 55%|█████████████████████████████████████████████████████████████████████████▋ | 49474/89905 [15:39:56<10:01:45, 1.12it/s][2025-04-29 02:04:59] (step=0049475) Train Loss: 5.6864, Train Steps/Sec: 1.12 + 55%|█████████████████████████████████████████████████████████████████████████▊ | 49499/89905 [15:40:18<10:02:16, 1.12it/s][2025-04-29 02:05:21] (step=0049500) Train Loss: 5.6474, Train Steps/Sec: 1.12 + 55%|██████████████████████████████████████████████████████████████████████████▎ | 49524/89905 [15:40:40<9:55:55, 1.13it/s][2025-04-29 02:05:44] (step=0049525) Train Loss: 5.6010, Train Steps/Sec: 1.12 + 55%|██████████████████████████████████████████████████████████████████████████▍ | 49549/89905 [15:41:03<9:55:14, 1.13it/s][2025-04-29 02:06:06] (step=0049550) Train Loss: 5.6772, Train Steps/Sec: 1.12 + 55%|██████████████████████████████████████████████████████████████████████████▍ | 49574/89905 [15:41:25<9:52:57, 1.13it/s][2025-04-29 02:06:28] (step=0049575) Train Loss: 5.5847, Train Steps/Sec: 1.12 + 55%|██████████████████████████████████████████████████████████████████████████▍ | 49599/89905 [15:41:47<9:52:12, 1.13it/s][2025-04-29 02:06:51] (step=0049600) Train Loss: 5.6657, Train Steps/Sec: 1.12 + 55%|█████████████████████████████████████████████████████████████████████████▉ | 49624/89905 [15:42:09<10:13:17, 1.09it/s][2025-04-29 02:07:13] (step=0049625) Train Loss: 5.6329, Train Steps/Sec: 1.12 + 55%|█████████████████████████████████████████████████████████████████████████▉ | 49649/89905 [15:42:32<10:03:52, 1.11it/s][2025-04-29 02:07:35] (step=0049650) Train Loss: 5.6197, Train Steps/Sec: 1.12 + 55%|██████████████████████████████████████████████████████████████████████████▌ | 49674/89905 [15:42:54<9:59:42, 1.12it/s][2025-04-29 02:07:57] (step=0049675) Train Loss: 5.6223, Train Steps/Sec: 1.12 + 55%|██████████████████████████████████████████████████████████████████████████▋ | 49699/89905 [15:43:16<9:56:23, 1.12it/s][2025-04-29 02:08:20] (step=0049700) Train Loss: 5.7153, Train Steps/Sec: 1.12 + 55%|██████████████████████████████████████████████████████████████████████████▋ | 49724/89905 [15:43:39<9:54:16, 1.13it/s][2025-04-29 02:08:42] (step=0049725) Train Loss: 5.6035, Train Steps/Sec: 1.12 + 55%|██████████████████████████████████████████████████████████████████████████▋ | 49749/89905 [15:44:01<9:52:40, 1.13it/s][2025-04-29 02:09:04] (step=0049750) Train Loss: 5.5965, Train Steps/Sec: 1.12 + 55%|██████████████████████████████████████████████████████████████████████████▋ | 49774/89905 [15:44:23<9:50:40, 1.13it/s][2025-04-29 02:09:27] (step=0049775) Train Loss: 5.6668, Train Steps/Sec: 1.12 + 55%|██████████████████████████████████████████████████████████████████████████▊ | 49799/89905 [15:44:46<9:51:33, 1.13it/s][2025-04-29 02:09:49] (step=0049800) Train Loss: 5.6839, Train Steps/Sec: 1.12 + 55%|██████████████████████████████████████████████████████████████████████████▎ | 49824/89905 [15:45:08<10:10:10, 1.09it/s][2025-04-29 02:10:11] (step=0049825) Train Loss: 5.6108, Train Steps/Sec: 1.12 + 55%|██████████████████████████████████████████████████████████████████████████▊ | 49849/89905 [15:45:30<9:59:47, 1.11it/s][2025-04-29 02:10:34] (step=0049850) Train Loss: 5.5880, Train Steps/Sec: 1.12 + 55%|██████████████████████████████████████████████████████████████████████████▉ | 49874/89905 [15:45:52<9:54:48, 1.12it/s][2025-04-29 02:10:56] (step=0049875) Train Loss: 5.5870, Train Steps/Sec: 1.12 + 56%|██████████████████████████████████████████████████████████████████████████▉ | 49899/89905 [15:46:15<9:53:17, 1.12it/s][2025-04-29 02:11:18] (step=0049900) Train Loss: 5.6143, Train Steps/Sec: 1.12 + 56%|██████████████████████████████████████████████████████████████████████████▉ | 49924/89905 [15:46:37<9:51:57, 1.13it/s][2025-04-29 02:11:40] (step=0049925) Train Loss: 5.6079, Train Steps/Sec: 1.12 + 56%|███████████████████████████████████████████████████████████████████████████ | 49949/89905 [15:46:59<9:49:24, 1.13it/s][2025-04-29 02:12:03] (step=0049950) Train Loss: 5.5630, Train Steps/Sec: 1.12 + 56%|███████████████████████████████████████████████████████████████████████████ | 49974/89905 [15:47:22<9:48:52, 1.13it/s][2025-04-29 02:12:25] (step=0049975) Train Loss: 5.6036, Train Steps/Sec: 1.12 + 56%|███████████████████████████████████████████████████████████████████████████ | 49999/89905 [15:47:44<9:48:20, 1.13it/s][2025-04-29 02:12:47] (step=0050000) Train Loss: 5.7343, Train Steps/Sec: 1.12 +You are using a model of type blip-2 to instantiate a model of type instructblip. This is not supported for all configurations of models and can yield errors. +[2025-04-29 02:12:47] text_config is None. Initializing the text config with default values (`OPTConfig`). +model_name_or_path:/tmp/haozhezhao/model/blip2-flan-t5-xl +image_place_holder: +tokenizer length after expend 32101 +tokenizer length before expend 32100 +100%|███████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████| 6/6 [06:01<00:00, 60.27s/it] +[2025-04-29 02:20:08] Finish Eval in 50000 steps...████████████████████████████████████████████████████████████████████| 6/6 [06:01<00:00, 60.04s/it] +processing left-padding... +processing left-padding... +processing left-padding... +processing left-padding... +processing left-padding... +processing left-padding... +[2025-04-29 02:20:27] Saved checkpoint to checkpoint/X2I_UltraEdiit_fourmask_LLava_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_filtered_objects_4mask_1280_context_after_stage1_5e4/000-GPT-XL/checkpoints/0050000.pt +[2025-04-29 02:20:29] Removed old checkpoint: checkpoint/X2I_UltraEdiit_fourmask_LLava_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_filtered_objects_4mask_1280_context_after_stage1_5e4/000-GPT-XL/checkpoints/0048000.pt + 56%|██████████████████████████████████████████████████████████████████████████▌ | 50024/89905 [15:55:48<10:24:23, 1.06it/s][2025-04-29 02:20:51] (step=0050025) Train Loss: 5.5493, Train Steps/Sec: 0.05 + 56%|███████████████████████████████████████████████████████████████████████████▏ | 50049/89905 [15:56:10<9:56:17, 1.11it/s][2025-04-29 02:21:14] (step=0050050) Train Loss: 5.6900, Train Steps/Sec: 1.12 + 56%|███████████████████████████████████████████████████████████████████████████▏ | 50074/89905 [15:56:33<9:54:55, 1.12it/s][2025-04-29 02:21:36] (step=0050075) Train Loss: 5.6614, Train Steps/Sec: 1.12 + 56%|███████████████████████████████████████████████████████████████████████████▏ | 50099/89905 [15:56:55<9:48:58, 1.13it/s][2025-04-29 02:21:58] (step=0050100) Train Loss: 5.6324, Train Steps/Sec: 1.12 + 56%|███████████████████████████████████████████████████████████████████████████▎ | 50124/89905 [15:57:17<9:49:21, 1.12it/s][2025-04-29 02:22:21] (step=0050125) Train Loss: 5.6211, Train Steps/Sec: 1.12 + 56%|███████████████████████████████████████████████████████████████████████████▎ | 50149/89905 [15:57:40<9:45:11, 1.13it/s][2025-04-29 02:22:43] (step=0050150) Train Loss: 5.6057, Train Steps/Sec: 1.12 + 56%|███████████████████████████████████████████████████████████████████████████▎ | 50174/89905 [15:58:02<9:46:51, 1.13it/s][2025-04-29 02:23:05] (step=0050175) Train Loss: 5.6238, Train Steps/Sec: 1.12 + 56%|███████████████████████████████████████████████████████████████████████████▍ | 50199/89905 [15:58:24<9:43:04, 1.13it/s][2025-04-29 02:23:28] (step=0050200) Train Loss: 5.7026, Train Steps/Sec: 1.12 + 56%|██████████████████████████████████████████████████████████████████████████▊ | 50224/89905 [15:58:46<10:01:43, 1.10it/s][2025-04-29 02:23:50] (step=0050225) Train Loss: 5.6001, Train Steps/Sec: 1.12 + 56%|███████████████████████████████████████████████████████████████████████████▍ | 50249/89905 [15:59:09<9:54:37, 1.11it/s][2025-04-29 02:24:12] (step=0050250) Train Loss: 5.6326, Train Steps/Sec: 1.12 + 56%|███████████████████████████████████████████████████████████████████████████▍ | 50274/89905 [15:59:31<9:48:14, 1.12it/s][2025-04-29 02:24:34] (step=0050275) Train Loss: 5.6424, Train Steps/Sec: 1.12 + 56%|███████████████████████████████████████████████████████████████████████████▌ | 50299/89905 [15:59:53<9:47:33, 1.12it/s][2025-04-29 02:24:57] (step=0050300) Train Loss: 5.6366, Train Steps/Sec: 1.12 + 56%|███████████████████████████████████████████████████████████████████████████▌ | 50324/89905 [16:00:16<9:45:12, 1.13it/s][2025-04-29 02:25:19] (step=0050325) Train Loss: 5.5785, Train Steps/Sec: 1.12 + 56%|███████████████████████████████████████████████████████████████████████████▌ | 50349/89905 [16:00:38<9:44:50, 1.13it/s][2025-04-29 02:25:41] (step=0050350) Train Loss: 5.5692, Train Steps/Sec: 1.12 + 56%|███████████████████████████████████████████████████████████████████████████▋ | 50374/89905 [16:01:00<9:42:22, 1.13it/s][2025-04-29 02:26:04] (step=0050375) Train Loss: 5.6382, Train Steps/Sec: 1.12 + 56%|███████████████████████████████████████████████████████████████████████████▋ | 50399/89905 [16:01:22<9:42:28, 1.13it/s][2025-04-29 02:26:26] (step=0050400) Train Loss: 5.5711, Train Steps/Sec: 1.12 + 56%|███████████████████████████████████████████████████████████████████████████▏ | 50424/89905 [16:01:45<10:00:30, 1.10it/s][2025-04-29 02:26:48] (step=0050425) Train Loss: 5.6069, Train Steps/Sec: 1.12 + 56%|███████████████████████████████████████████████████████████████████████████▏ | 50449/89905 [16:02:07<10:00:25, 1.10it/s][2025-04-29 02:27:11] (step=0050450) Train Loss: 5.6342, Train Steps/Sec: 1.12 + 56%|███████████████████████████████████████████████████████████████████████████▊ | 50474/89905 [16:02:30<9:49:50, 1.11it/s][2025-04-29 02:27:33] (step=0050475) Train Loss: 5.6478, Train Steps/Sec: 1.12 + 56%|███████████████████████████████████████████████████████████████████████████▊ | 50499/89905 [16:02:52<9:46:38, 1.12it/s][2025-04-29 02:27:55] (step=0050500) Train Loss: 5.6652, Train Steps/Sec: 1.12 + 56%|███████████████████████████████████████████████████████████████████████████▊ | 50524/89905 [16:03:14<9:43:54, 1.12it/s][2025-04-29 02:28:18] (step=0050525) Train Loss: 5.5626, Train Steps/Sec: 1.12 + 56%|███████████████████████████████████████████████████████████████████████████▉ | 50549/89905 [16:03:37<9:41:41, 1.13it/s][2025-04-29 02:28:40] (step=0050550) Train Loss: 5.6514, Train Steps/Sec: 1.12 + 56%|███████████████████████████████████████████████████████████████████████████▉ | 50574/89905 [16:03:59<9:41:42, 1.13it/s][2025-04-29 02:29:02] (step=0050575) Train Loss: 5.6636, Train Steps/Sec: 1.12 + 56%|███████████████████████████████████████████████████████████████████████████▉ | 50599/89905 [16:04:21<9:38:31, 1.13it/s][2025-04-29 02:29:25] (step=0050600) Train Loss: 5.5701, Train Steps/Sec: 1.12 + 56%|████████████████████████████████████████████████████████████████████████████ | 50624/89905 [16:04:43<9:55:19, 1.10it/s][2025-04-29 02:29:47] (step=0050625) Train Loss: 5.5989, Train Steps/Sec: 1.12 + 56%|████████████████████████████████████████████████████████████████████████████ | 50649/89905 [16:05:06<9:48:10, 1.11it/s][2025-04-29 02:30:09] (step=0050650) Train Loss: 5.6398, Train Steps/Sec: 1.12 + 56%|████████████████████████████████████████████████████████████████████████████ | 50674/89905 [16:05:28<9:45:43, 1.12it/s][2025-04-29 02:30:31] (step=0050675) Train Loss: 5.5757, Train Steps/Sec: 1.12 + 56%|████████████████████████████████████████████████████████████████████████████▏ | 50699/89905 [16:05:50<9:41:16, 1.12it/s][2025-04-29 02:30:54] (step=0050700) Train Loss: 5.5719, Train Steps/Sec: 1.12 + 56%|████████████████████████████████████████████████████████████████████████████▏ | 50724/89905 [16:06:13<9:39:40, 1.13it/s][2025-04-29 02:31:16] (step=0050725) Train Loss: 5.6483, Train Steps/Sec: 1.12 + 56%|████████████████████████████████████████████████████████████████████████████▏ | 50749/89905 [16:06:35<9:37:02, 1.13it/s][2025-04-29 02:31:38] (step=0050750) Train Loss: 5.6335, Train Steps/Sec: 1.12 + 56%|████████████████████████████████████████████████████████████████████████████▏ | 50774/89905 [16:06:57<9:37:43, 1.13it/s][2025-04-29 02:32:01] (step=0050775) Train Loss: 5.5834, Train Steps/Sec: 1.12 + 57%|████████████████████████████████████████████████████████████████████████████▎ | 50799/89905 [16:07:20<9:35:20, 1.13it/s][2025-04-29 02:32:23] (step=0050800) Train Loss: 5.6430, Train Steps/Sec: 1.12 + 57%|████████████████████████████████████████████████████████████████████████████▎ | 50824/89905 [16:07:42<9:51:39, 1.10it/s][2025-04-29 02:32:45] (step=0050825) Train Loss: 5.6490, Train Steps/Sec: 1.12 + 57%|████████████████████████████████████████████████████████████████████████████▎ | 50849/89905 [16:08:04<9:48:12, 1.11it/s][2025-04-29 02:33:08] (step=0050850) Train Loss: 5.6493, Train Steps/Sec: 1.12 + 57%|████████████████████████████████████████████████████████████████████████████▍ | 50874/89905 [16:08:27<9:40:29, 1.12it/s][2025-04-29 02:33:30] (step=0050875) Train Loss: 5.6415, Train Steps/Sec: 1.12 + 57%|████████████████████████████████████████████████████████████████████████████▍ | 50899/89905 [16:08:49<9:39:32, 1.12it/s][2025-04-29 02:33:52] (step=0050900) Train Loss: 5.6429, Train Steps/Sec: 1.12 + 57%|████████████████████████████████████████████████████████████████████████████▍ | 50924/89905 [16:09:11<9:36:48, 1.13it/s][2025-04-29 02:34:15] (step=0050925) Train Loss: 5.6457, Train Steps/Sec: 1.12 + 57%|████████████████████████████████████████████████████████████████████████████▌ | 50949/89905 [16:09:34<9:35:56, 1.13it/s][2025-04-29 02:34:37] (step=0050950) Train Loss: 5.5692, Train Steps/Sec: 1.12 + 57%|████████████████████████████████████████████████████████████████████████████▌ | 50974/89905 [16:09:56<9:33:20, 1.13it/s][2025-04-29 02:34:59] (step=0050975) Train Loss: 5.5841, Train Steps/Sec: 1.12 + 57%|████████████████████████████████████████████████████████████████████████████▌ | 50999/89905 [16:10:18<9:34:41, 1.13it/s][2025-04-29 02:35:22] (step=0051000) Train Loss: 5.6499, Train Steps/Sec: 1.12 + 57%|████████████████████████████████████████████████████████████████████████████▌ | 51024/89905 [16:10:41<9:48:49, 1.10it/s][2025-04-29 02:35:44] (step=0051025) Train Loss: 5.5649, Train Steps/Sec: 1.12 + 57%|████████████████████████████████████████████████████████████████████████████▋ | 51049/89905 [16:11:03<9:43:40, 1.11it/s][2025-04-29 02:36:06] (step=0051050) Train Loss: 5.5786, Train Steps/Sec: 1.12 + 57%|████████████████████████████████████████████████████████████████████████████▋ | 51074/89905 [16:11:25<9:39:53, 1.12it/s][2025-04-29 02:36:29] (step=0051075) Train Loss: 5.6848, Train Steps/Sec: 1.12 + 57%|████████████████████████████████████████████████████████████████████████████▋ | 51099/89905 [16:11:47<9:36:07, 1.12it/s][2025-04-29 02:36:51] (step=0051100) Train Loss: 5.5620, Train Steps/Sec: 1.12 + 57%|████████████████████████████████████████████████████████████████████████████▊ | 51124/89905 [16:12:10<9:35:09, 1.12it/s][2025-04-29 02:37:13] (step=0051125) Train Loss: 5.5884, Train Steps/Sec: 1.12 + 57%|████████████████████████████████████████████████████████████████████████████▊ | 51149/89905 [16:12:32<9:33:13, 1.13it/s][2025-04-29 02:37:36] (step=0051150) Train Loss: 5.6708, Train Steps/Sec: 1.12 + 57%|████████████████████████████████████████████████████████████████████████████▊ | 51174/89905 [16:12:54<9:30:21, 1.13it/s][2025-04-29 02:37:58] (step=0051175) Train Loss: 5.6038, Train Steps/Sec: 1.12 + 57%|████████████████████████████████████████████████████████████████████████████▉ | 51199/89905 [16:13:17<9:29:11, 1.13it/s][2025-04-29 02:38:20] (step=0051200) Train Loss: 5.6930, Train Steps/Sec: 1.12 + 57%|████████████████████████████████████████████████████████████████████████████▎ | 51224/89905 [16:13:48<13:04:15, 1.22s/it][2025-04-29 02:38:51] (step=0051225) Train Loss: 5.6314, Train Steps/Sec: 0.81 + 57%|████████████████████████████████████████████████████████████████████████████▍ | 51249/89905 [16:14:38<10:53:33, 1.01s/it][2025-04-29 02:39:41] (step=0051250) Train Loss: 5.5874, Train Steps/Sec: 0.50 + 57%|████████████████████████████████████████████████████████████████████████████▉ | 51274/89905 [16:15:10<9:44:57, 1.10it/s][2025-04-29 02:40:22] (step=0051275) Train Loss: 5.5342, Train Steps/Sec: 0.61 + 57%|█████████████████████████████████████████████████████████████████████████████ | 51299/89905 [16:15:41<9:32:39, 1.12it/s][2025-04-29 02:40:54] (step=0051300) Train Loss: 5.6334, Train Steps/Sec: 0.79 + 57%|█████████████████████████████████████████████████████████████████████████████ | 51324/89905 [16:16:13<9:32:47, 1.12it/s][2025-04-29 02:41:16] (step=0051325) Train Loss: 5.6428, Train Steps/Sec: 1.12 + 57%|█████████████████████████████████████████████████████████████████████████████ | 51349/89905 [16:16:35<9:29:28, 1.13it/s][2025-04-29 02:41:38] (step=0051350) Train Loss: 5.5513, Train Steps/Sec: 1.12 + 57%|█████████████████████████████████████████████████████████████████████████████▏ | 51374/89905 [16:16:57<9:26:17, 1.13it/s][2025-04-29 02:42:01] (step=0051375) Train Loss: 5.5896, Train Steps/Sec: 1.12 + 57%|█████████████████████████████████████████████████████████████████████████████▏ | 51399/89905 [16:17:20<9:27:18, 1.13it/s][2025-04-29 02:42:23] (step=0051400) Train Loss: 5.6367, Train Steps/Sec: 1.12 + 57%|█████████████████████████████████████████████████████████████████████████████▏ | 51424/89905 [16:17:42<9:44:26, 1.10it/s][2025-04-29 02:42:45] (step=0051425) Train Loss: 5.6231, Train Steps/Sec: 1.12 + 57%|█████████████████████████████████████████████████████████████████████████████▎ | 51449/89905 [16:18:04<9:37:25, 1.11it/s][2025-04-29 02:43:08] (step=0051450) Train Loss: 5.6133, Train Steps/Sec: 1.12 + 57%|█████████████████████████████████████████████████████████████████████████████▎ | 51474/89905 [16:18:27<9:32:24, 1.12it/s][2025-04-29 02:43:30] (step=0051475) Train Loss: 5.5458, Train Steps/Sec: 1.12 + 57%|█████████████████████████████████████████████████████████████████████████████▎ | 51499/89905 [16:18:49<9:31:28, 1.12it/s][2025-04-29 02:43:52] (step=0051500) Train Loss: 5.6182, Train Steps/Sec: 1.12 + 57%|█████████████████████████████████████████████████████████████████████████████▎ | 51510/89905 [16:18:59<9:24:43, 1.13it/s] diff --git a/CKPTS/X2I_UltraEdiit_fourmask_LLava_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_filtered_objects_4mask_1280_context_after_stage1_5e4/wandb/run-20250428_102501-vl9wvenp/files/requirements.txt b/CKPTS/X2I_UltraEdiit_fourmask_LLava_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_filtered_objects_4mask_1280_context_after_stage1_5e4/wandb/run-20250428_102501-vl9wvenp/files/requirements.txt new file mode 100644 index 0000000000000000000000000000000000000000..06dc78369ffff807b210006a0e79d705ffe2a7d7 --- /dev/null +++ b/CKPTS/X2I_UltraEdiit_fourmask_LLava_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_filtered_objects_4mask_1280_context_after_stage1_5e4/wandb/run-20250428_102501-vl9wvenp/files/requirements.txt @@ -0,0 +1,131 @@ +typing_extensions==4.12.2 +pyzmq==26.3.0 +nvidia-cufft-cu12==11.0.2.54 +triton==3.1.0 +nvidia-cublas-cu12==12.1.3.1 +psutil==7.0.0 +nvidia-cuda-cupti-cu12==12.1.105 +smmap==5.0.2 +nvidia-cuda-runtime-cu12==12.1.105 +aiohappyeyeballs==2.6.1 +asttokens==3.0.0 +huggingface-hub==0.29.3 +pyarrow==19.0.1 +fonttools==4.56.0 +python-dateutil==2.9.0.post0 +GitPython==3.1.44 +aiohttp==3.11.14 +wandb==0.19.8 +setproctitle==1.3.5 +PyYAML==6.0.2 +pydantic_core==2.27.2 +safetensors==0.5.3 +nvidia-nvjitlink-cu12==12.1.105 +aiosignal==1.3.2 +dill==0.3.8 +nvidia-cuda-nvrtc-cu12==12.1.105 +multiprocess==0.70.16 +pure_eval==0.2.3 +stack_data==0.6.3 +pydantic==2.10.6 +MarkupSafe==2.1.5 +tornado==6.4.2 +executing==2.1.0 +executing==2.2.0 +opencv-python==4.11.0.86 +nvitop==1.4.2 +multidict==6.2.0 +Jinja2==3.1.4 +torch==2.5.1+cu121 +nvidia-curand-cu12==10.3.2.106 +platformdirs==4.3.6 +six==1.17.0 +mpmath==1.3.0 +zipp==3.21.0 +packaging==24.2 +requests==2.32.3 +certifi==2025.1.31 +docker-pycreds==0.4.0 +torchvision==0.20.1+cu121 +pandas==2.2.3 +networkx==3.3 +exceptiongroup==1.2.2 +pickleshare==0.7.5 +tokenizers==0.21.1 +charset-normalizer==3.4.1 +jupyter_core==5.7.2 +wcwidth==0.2.13 +nvidia-nvtx-cu12==12.1.105 +prompt_toolkit==3.0.50 +fsspec==2024.12.0 +pillow==11.1.0 +propcache==0.3.0 +regex==2024.11.6 +ptyprocess==0.7.0 +contourpy==1.3.1 +importlib_metadata==8.6.1 +idna==3.10 +comm==0.2.2 +protobuf==5.29.3 +yarl==1.18.3 +ipython_pygments_lexers==1.1.1 +pip==25.0 +parso==0.8.4 +joblib==1.4.2 +nvidia-nccl-cu12==2.21.5 +hf_transfer==0.1.9 +Pygments==2.19.1 +decorator==5.2.1 +filelock==3.18.0 +nvidia-cusparse-cu12==12.1.0.106 +debugpy==1.8.13 +urllib3==2.3.0 +traitlets==5.14.3 +tzdata==2025.1 +matplotlib-inline==0.1.7 +matplotlib==3.10.1 +kiwisolver==1.4.8 +nest_asyncio==1.6.0 +frozenlist==1.5.0 +nvidia-ml-py==12.570.86 +transformers==4.49.0 +nltk==3.9.1 +ipykernel==6.29.5 +click==8.1.8 +gitdb==4.0.12 +pyparsing==3.2.1 +attrs==25.3.0 +jedi==0.19.2 +ipython==9.0.2 +nvidia-cudnn-cu12==9.1.0.70 +pexpect==4.9.0 +nvidia-cusolver-cu12==11.4.5.107 +numpy==2.2.4 +tqdm==4.67.1 +pytz==2025.1 +wheel==0.45.1 +sentry-sdk==2.23.1 +torchaudio==2.5.1+cu121 +jupyter_client==8.6.3 +cycler==0.12.1 +annotated-types==0.7.0 +sympy==1.13.1 +xxhash==3.5.0 +datasets==3.4.1 +setuptools==75.8.0 +typing_extensions==4.12.2 +wheel==0.43.0 +importlib_metadata==8.0.0 +backports.tarfile==1.2.0 +autocommand==2.2.2 +packaging==24.2 +tomli==2.0.1 +typeguard==4.3.0 +zipp==3.19.2 +jaraco.context==5.3.0 +jaraco.functools==4.0.1 +more-itertools==10.3.0 +platformdirs==4.2.2 +jaraco.text==3.12.1 +jaraco.collections==5.1.0 +inflect==7.3.1 diff --git a/CKPTS/X2I_UltraEdiit_fourmask_LLava_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_filtered_objects_4mask_1280_context_after_stage1_5e4/wandb/run-20250428_102501-vl9wvenp/files/wandb-metadata.json b/CKPTS/X2I_UltraEdiit_fourmask_LLava_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_filtered_objects_4mask_1280_context_after_stage1_5e4/wandb/run-20250428_102501-vl9wvenp/files/wandb-metadata.json new file mode 100644 index 0000000000000000000000000000000000000000..cb67fd0e1acba419ed6ef713f75f94cd9c1cfc2b --- /dev/null +++ b/CKPTS/X2I_UltraEdiit_fourmask_LLava_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_filtered_objects_4mask_1280_context_after_stage1_5e4/wandb/run-20250428_102501-vl9wvenp/files/wandb-metadata.json @@ -0,0 +1,149 @@ +{ + "os": "Linux-5.15.0-1064-azure-x86_64-with-glibc2.31", + "python": "CPython 3.11.11", + "startedAt": "2025-04-28T10:25:01.669262Z", + "args": [ + "--vq-ckpt", + "/tmp/haozhezhao/model/llamagen_t2i/vq_ds16_t2i.pt", + "--data-path", + "/tmp/haozhezhao/MLLMG/jsonl_data/multiimage_training_for_llava_X2I_fourmask.jsonl", + "--dataset", + "ti2i", + "--image-size", + "512", + "--results-dir", + "checkpoint/X2I_UltraEdiit_fourmask_LLava_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_filtered_objects_4mask_1280_context_after_stage1_5e4", + "--cloud-save-path", + "/tmp/haozhezhao/MLLMG/checkpoint", + "--lr", + "5e-4", + "--val_data_path", + "/tmp/haozhezhao/MLLMG/jsonl_data/multiimage_val_for_llava_X2I_fourmask_dreambenplus.jsonl", + "--use_vision_tower", + "--model_name_or_path", + "/tmp/haozhezhao/model/blip2-flan-t5-xl", + "--image_place_holder", + "", + "--do_eval", + "--eval_steps", + "2000", + "--max_eval_samples", + "128", + "--cfg-scale", + "7.5", + "--top-k", + "16384", + "--load_from_checkpoint", + "/tmp/haozhezhao/MLLMG/checkpoint/FIXed_3M_1epoch_step32000.pt", + "--global-batch-size", + "24", + "--num-workers", + "4", + "--warmup", + "0.05", + "--gradient-accumulation-steps", + "8", + "--train_text_encoder", + "--ckpt-every", + "2000", + "--epochs", + "1", + "--subject_driven", + "--reference_data_path", + "/tmp/haozhezhao/MLLMG/cc12m_reference_tunnel.jsonl", + "--multimodal_encoder", + "llava", + "--do_recovery", + "--find_unused_parameters", + "--cls-token-num", + "1280", + "--train_all", + "--load_fixed_llamagen", + "--save_total_limit", + "1", + "--fix", + "gpt-empty-fix", + "--load_language_projection", + "/tmp/haozhezhao/MLLMG/llava-v1.5-flant5_fixed-pretrain/mm_projector.bin", + "--mm_vision_tower", + "openai/clip-vit-large-patch14", + "--gpt-ckpt", + "/tmp/haozhezhao/MLLMG/MLLMG_ckpts/checkpoints/AfterI2I_llavat5_subject_train_all_T5_subject_t2i_ti2i_120_w_flux_segment_noimageonly_empty_fix_train_all/0100000.pt", + "--dreambench_eval" + ], + "program": "/tmp/haozhezhao/MLLMG/autoregressive/train/train_t2i.py", + "codePath": "autoregressive/train/train_t2i.py", + "email": "mimazhe55360@gmail.com", + "root": "checkpoint/X2I_UltraEdiit_fourmask_LLava_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_filtered_objects_4mask_1280_context_after_stage1_5e4", + "host": "447cc403a8794092814259713c51c1df00001X", + "executable": "/tmp/haozhezhao/anaconda3/envs/nlp/bin/python", + "codePathLocal": "autoregressive/train/train_t2i.py", + "cpu_count": 96, + "cpu_count_logical": 96, + "gpu": "NVIDIA A100-SXM4-80GB", + "gpu_count": 8, + "disk": { + "/": { + "total": "133003395072", + "used": "67483713536" + } + }, + "memory": { + "total": "1902387884032" + }, + "cpu": { + "count": 96, + "countLogical": 96 + }, + "gpu_nvidia": [ + { + "name": "NVIDIA A100-SXM4-80GB", + "memoryTotal": "85899345920", + "cudaCores": 6912, + "architecture": "Ampere" + }, + { + "name": "NVIDIA A100-SXM4-80GB", + "memoryTotal": "85899345920", + "cudaCores": 6912, + "architecture": "Ampere" + }, + { + "name": "NVIDIA A100-SXM4-80GB", + "memoryTotal": "85899345920", + "cudaCores": 6912, + "architecture": "Ampere" + }, + { + "name": "NVIDIA A100-SXM4-80GB", + "memoryTotal": "85899345920", + "cudaCores": 6912, + "architecture": "Ampere" + }, + { + "name": "NVIDIA A100-SXM4-80GB", + "memoryTotal": "85899345920", + "cudaCores": 6912, + "architecture": "Ampere" + }, + { + "name": "NVIDIA A100-SXM4-80GB", + "memoryTotal": "85899345920", + "cudaCores": 6912, + "architecture": "Ampere" + }, + { + "name": "NVIDIA A100-SXM4-80GB", + "memoryTotal": "85899345920", + "cudaCores": 6912, + "architecture": "Ampere" + }, + { + "name": "NVIDIA A100-SXM4-80GB", + "memoryTotal": "85899345920", + "cudaCores": 6912, + "architecture": "Ampere" + } + ], + "cudaVersion": "12.2" +} \ No newline at end of file diff --git a/CKPTS/X2I_UltraEdiit_fourmask_LLava_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_filtered_objects_4mask_1280_context_after_stage1_5e4/wandb/run-20250428_102501-vl9wvenp/logs/debug-core.log b/CKPTS/X2I_UltraEdiit_fourmask_LLava_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_filtered_objects_4mask_1280_context_after_stage1_5e4/wandb/run-20250428_102501-vl9wvenp/logs/debug-core.log new file mode 100644 index 0000000000000000000000000000000000000000..e40bf7171d74abd7c430413c4ac342d39063a0ec --- /dev/null +++ b/CKPTS/X2I_UltraEdiit_fourmask_LLava_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_filtered_objects_4mask_1280_context_after_stage1_5e4/wandb/run-20250428_102501-vl9wvenp/logs/debug-core.log @@ -0,0 +1,7 @@ +{"time":"2025-04-28T10:25:01.142101029Z","level":"INFO","msg":"main: starting server","port-filename":"/tmp/tmpq76bzuob/port-3827947.txt","pid":3827947,"log-level":0,"disable-analytics":false,"shutdown-on-parent-exit":false} +{"time":"2025-04-28T10:25:01.14329236Z","level":"INFO","msg":"Will exit if parent process dies.","ppid":3827947} +{"time":"2025-04-28T10:25:01.143270449Z","level":"INFO","msg":"server is running","addr":{"IP":"127.0.0.1","Port":40727,"Zone":""}} +{"time":"2025-04-28T10:25:01.329957684Z","level":"INFO","msg":"connection: ManageConnectionData: new connection created","id":"127.0.0.1:53202"} +{"time":"2025-04-28T10:25:01.670560761Z","level":"INFO","msg":"handleInformInit: received","streamId":"vl9wvenp","id":"127.0.0.1:53202"} +{"time":"2025-04-28T10:25:01.902138601Z","level":"INFO","msg":"handleInformInit: stream started","streamId":"vl9wvenp","id":"127.0.0.1:53202"} +{"time":"2025-04-29T02:44:06.808032554Z","level":"INFO","msg":"Parent process exited, terminating service process."} diff --git a/CKPTS/X2I_UltraEdiit_fourmask_LLava_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_filtered_objects_4mask_1280_context_after_stage1_5e4/wandb/run-20250428_102501-vl9wvenp/logs/debug-internal.log b/CKPTS/X2I_UltraEdiit_fourmask_LLava_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_filtered_objects_4mask_1280_context_after_stage1_5e4/wandb/run-20250428_102501-vl9wvenp/logs/debug-internal.log new file mode 100644 index 0000000000000000000000000000000000000000..abddb029fc34b637ef4cd2a655c3bc311d194bb6 --- /dev/null +++ b/CKPTS/X2I_UltraEdiit_fourmask_LLava_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_filtered_objects_4mask_1280_context_after_stage1_5e4/wandb/run-20250428_102501-vl9wvenp/logs/debug-internal.log @@ -0,0 +1,10 @@ +{"time":"2025-04-28T10:25:01.670761577Z","level":"INFO","msg":"stream: starting","core version":"0.19.8","symlink path":"checkpoint/X2I_UltraEdiit_fourmask_LLava_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_filtered_objects_4mask_1280_context_after_stage1_5e4/wandb/run-20250428_102501-vl9wvenp/logs/debug-core.log"} +{"time":"2025-04-28T10:25:01.90209546Z","level":"INFO","msg":"created new stream","id":"vl9wvenp"} +{"time":"2025-04-28T10:25:01.902133521Z","level":"INFO","msg":"stream: started","id":"vl9wvenp"} +{"time":"2025-04-28T10:25:01.9021675Z","level":"INFO","msg":"handler: started","stream_id":"vl9wvenp"} +{"time":"2025-04-28T10:25:01.902172604Z","level":"INFO","msg":"writer: Do: started","stream_id":"vl9wvenp"} +{"time":"2025-04-28T10:25:01.902460333Z","level":"INFO","msg":"sender: started","stream_id":"vl9wvenp"} +{"time":"2025-04-28T10:25:02.279006691Z","level":"INFO","msg":"Starting system monitor"} +{"time":"2025-04-28T16:35:20.050683354Z","level":"INFO","msg":"api: retrying HTTP error","status":429,"url":"https://api.wandb.ai/files/haozhezhao/llamagen_ti2i/vl9wvenp/file_stream","body":"{\"error\":\"rate limit exceeded\"}"} +{"time":"2025-04-28T16:35:22.70871011Z","level":"INFO","msg":"api: retrying HTTP error","status":429,"url":"https://api.wandb.ai/files/haozhezhao/llamagen_ti2i/vl9wvenp/file_stream","body":"{\"error\":\"rate limit exceeded\"}"} +{"time":"2025-04-28T16:35:27.831923692Z","level":"INFO","msg":"api: retrying HTTP error","status":429,"url":"https://api.wandb.ai/files/haozhezhao/llamagen_ti2i/vl9wvenp/file_stream","body":"{\"error\":\"rate limit exceeded\"}"} diff --git a/CKPTS/X2I_UltraEdiit_fourmask_LLava_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_filtered_objects_4mask_1280_context_after_stage1_5e4/wandb/run-20250428_102501-vl9wvenp/logs/debug.log b/CKPTS/X2I_UltraEdiit_fourmask_LLava_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_filtered_objects_4mask_1280_context_after_stage1_5e4/wandb/run-20250428_102501-vl9wvenp/logs/debug.log new file mode 100644 index 0000000000000000000000000000000000000000..ca6b02c3535942361246b77338437cf4c8706dd5 --- /dev/null +++ b/CKPTS/X2I_UltraEdiit_fourmask_LLava_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_filtered_objects_4mask_1280_context_after_stage1_5e4/wandb/run-20250428_102501-vl9wvenp/logs/debug.log @@ -0,0 +1,22 @@ +2025-04-28 10:25:01,638 INFO MainThread:3827947 [wandb_setup.py:_flush():67] Current SDK version is 0.19.8 +2025-04-28 10:25:01,638 INFO MainThread:3827947 [wandb_setup.py:_flush():67] Configure stats pid to 3827947 +2025-04-28 10:25:01,638 INFO MainThread:3827947 [wandb_setup.py:_flush():67] Loading settings from /tmp/haozhezhao/.config/wandb/settings +2025-04-28 10:25:01,638 INFO MainThread:3827947 [wandb_setup.py:_flush():67] Loading settings from /tmp/haozhezhao/MLLMG/wandb/settings +2025-04-28 10:25:01,638 INFO MainThread:3827947 [wandb_setup.py:_flush():67] Loading settings from environment variables +2025-04-28 10:25:01,638 INFO MainThread:3827947 [wandb_init.py:setup_run_log_directory():647] Logging user logs to checkpoint/X2I_UltraEdiit_fourmask_LLava_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_filtered_objects_4mask_1280_context_after_stage1_5e4/wandb/run-20250428_102501-vl9wvenp/logs/debug.log +2025-04-28 10:25:01,638 INFO MainThread:3827947 [wandb_init.py:setup_run_log_directory():648] Logging internal logs to checkpoint/X2I_UltraEdiit_fourmask_LLava_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_filtered_objects_4mask_1280_context_after_stage1_5e4/wandb/run-20250428_102501-vl9wvenp/logs/debug-internal.log +2025-04-28 10:25:01,638 INFO MainThread:3827947 [wandb_init.py:init():761] calling init triggers +2025-04-28 10:25:01,638 INFO MainThread:3827947 [wandb_init.py:init():766] wandb.init called with sweep_config: {} +config: {'data_path': '/tmp/haozhezhao/MLLMG/jsonl_data/multiimage_training_for_llava_X2I_fourmask.jsonl', 'cloud_save_path': '/tmp/haozhezhao/MLLMG/checkpoint', 'no_local_save': False, 'vq_model': 'VQ-16', 'vq_ckpt': '/tmp/haozhezhao/model/llamagen_t2i/vq_ds16_t2i.pt', 'codebook_size': 16384, 'codebook_embed_dim': 8, 'gpt_model': 'GPT-XL', 'gpt_ckpt': '/tmp/haozhezhao/MLLMG/MLLMG_ckpts/checkpoints/AfterI2I_llavat5_subject_train_all_T5_subject_t2i_ti2i_120_w_flux_segment_noimageonly_empty_fix_train_all/0100000.pt', 'gpt_type': 't2i', 'vocab_size': 16384, 'cls_token_num': 1280, 'dropout_p': 0.1, 'token_dropout_p': 0.1, 'drop_path': 0.0, 'no_compile': False, 'results_dir': 'checkpoint/X2I_UltraEdiit_fourmask_LLava_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_filtered_objects_4mask_1280_context_after_stage1_5e4', 'dataset': 'ti2i', 'image_size': 512, 'downsample_size': 16, 'num_classes': 1000, 'epochs': 1, 'lr': 0.0005, 'weight_decay': 0.05, 'beta1': 0.9, 'beta2': 0.95, 'max_grad_norm': 1.0, 'global_batch_size': 24, 'global_seed': 0, 'num_workers': 4, 'log_every': 25, 'ckpt_every': 2000, 'gradient_accumulation_steps': 8, 'mixed_precision': 'bf16', 'val_data_path': '/tmp/haozhezhao/MLLMG/jsonl_data/multiimage_val_for_llava_X2I_fourmask_dreambenplus.jsonl', 'use_vision_tower': True, 'model_name_or_path': '/tmp/haozhezhao/model/blip2-flan-t5-xl', 'image_place_holder': '', 'processor_path': None, 'do_eval': True, 'max_eval_samples': 128, 'train_text_encoder': True, 'no_left_padding': False, 'cfg_scale': 7.5, 'top_k': 16384, 'temperature': 0.9, 'top_p': 1.0, 'eval_steps': 2000, 'project_name': 'llamagen_ti2i', 'load_from_checkpoint': '/tmp/haozhezhao/MLLMG/checkpoint/FIXed_3M_1epoch_step32000.pt', 'warmup': 0.05, 'lr_decay_style': 'cosine', 'lr_decay_ratio': 0.1, 'train_iters': 500000, 'class_dropout_prob': 0.1, 'with_image_only': False, 'image_only_rate': 0.1, 'stage2': False, 'subject_driven': True, 'load_subject_embedding': None, 'reference_data_path': '/tmp/haozhezhao/MLLMG/cc12m_reference_tunnel.jsonl', 'multimodal_encoder': 'llava', 'do_recovery': True, 'no_replace': False, 'resume': False, 'dreambench_eval': True, 'find_unused_parameters': True, 'load_visual_encoder': False, 'continue_stage1': False, 'replace_subject': False, 'train_all': True, 'save_total_limit': 1, 'load_language_projection': '/tmp/haozhezhao/MLLMG/llava-v1.5-flant5_fixed-pretrain/mm_projector.bin', 'mm_vision_tower': 'openai/clip-vit-large-patch14', 'load_fixed_llamagen': True, 'unfreeze_output': False, 'fix': 'gpt-empty-fix', 'rank': 0, 'world_size': 8, 'gpu': 0, 'dist_url': 'env://', 'distributed': True, 'dist_backend': 'nccl', '_wandb': {}} +2025-04-28 10:25:01,638 INFO MainThread:3827947 [wandb_init.py:init():784] starting backend +2025-04-28 10:25:01,638 INFO MainThread:3827947 [wandb_init.py:init():788] sending inform_init request +2025-04-28 10:25:01,668 INFO MainThread:3827947 [backend.py:_multiprocessing_setup():101] multiprocessing start_methods=fork,spawn,forkserver, using: spawn +2025-04-28 10:25:01,669 INFO MainThread:3827947 [wandb_init.py:init():798] backend started and connected +2025-04-28 10:25:01,672 INFO MainThread:3827947 [wandb_init.py:init():891] updated telemetry +2025-04-28 10:25:01,672 INFO MainThread:3827947 [wandb_init.py:init():915] communicating run to backend with 90.0 second timeout +2025-04-28 10:25:02,276 INFO MainThread:3827947 [wandb_init.py:init():990] starting run threads in backend +2025-04-28 10:25:02,358 INFO MainThread:3827947 [wandb_run.py:_console_start():2375] atexit reg +2025-04-28 10:25:02,359 INFO MainThread:3827947 [wandb_run.py:_redirect():2227] redirect: wrap_raw +2025-04-28 10:25:02,359 INFO MainThread:3827947 [wandb_run.py:_redirect():2292] Wrapping output streams. +2025-04-28 10:25:02,359 INFO MainThread:3827947 [wandb_run.py:_redirect():2315] Redirects installed. +2025-04-28 10:25:02,361 INFO MainThread:3827947 [wandb_init.py:init():1032] run started, returning control to user process diff --git a/CKPTS/X2I_UltraEdiit_fourmask_LLava_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_filtered_objects_4mask_1280_context_after_stage1_5e4/wandb/run-20250428_102501-vl9wvenp/run-vl9wvenp.wandb b/CKPTS/X2I_UltraEdiit_fourmask_LLava_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_filtered_objects_4mask_1280_context_after_stage1_5e4/wandb/run-20250428_102501-vl9wvenp/run-vl9wvenp.wandb new file mode 100644 index 0000000000000000000000000000000000000000..a460cf3f88f8e18e5fe2204411e555814d457047 --- /dev/null +++ b/CKPTS/X2I_UltraEdiit_fourmask_LLava_just_segment_pretrain_stage3_multiobjects_many_objects_segmentation_bbox_filtered_objects_4mask_1280_context_after_stage1_5e4/wandb/run-20250428_102501-vl9wvenp/run-vl9wvenp.wandb @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:1008e13b681369c7e5979f4cf87d42dc824a8eb53a3cb09db6fc7e92bfdb4899 +size 31752192 diff --git a/CKPTS/query_gpt.py b/CKPTS/query_gpt.py new file mode 100644 index 0000000000000000000000000000000000000000..789c7d3b78b40c2fe2300fa6423387c64b4a757d --- /dev/null +++ b/CKPTS/query_gpt.py @@ -0,0 +1,134 @@ +import argparse +import json +import logging +import os +import random +import time +from concurrent.futures import ThreadPoolExecutor, as_completed + +import jsonlines +from openai import AzureOpenAI +from tqdm import tqdm + +def parse_arguments() -> argparse.Namespace: + p = argparse.ArgumentParser(description="Query GPT in parallel to refine image-caption data.") + p.add_argument("--workers", type=int, default=4, help="Number of concurrent threads.") + p.add_argument("--input_file", type=str, required=True, help="Path to input JSONL with key 'input_text'.") + p.add_argument("--output_file", type=str, required=True, help="Destination JSONL file with GPT results.") + p.add_argument("--model", type=str, default="gpt-4o", help="Deployed Azure OpenAI model name.") + p.add_argument("--temperature", type=float, default=0.7) + p.add_argument("--random_seed", type=int, default=42) + return p.parse_args() + +def load_jsonl(path: str) -> list[dict]: + with jsonlines.open(path, mode="r") as reader: + return [record for record in reader] + +def build_prompt(caption: str) -> str: + return f""" +You will receive one raw **image-caption** that contains one or more occurrences of the exact substring ``. +Rewrite the caption and report the objects associated with each placeholder, then return the result **as a JSON object**. + +Steps: +1. Scan the caption from left to right and note every position where `` appears. +2. For each occurrence, determine the **most specific concrete noun phrase** that refers to that placeholder. + • It may be immediately before the placeholder. + • It may be earlier in the sentence or a previous sentence. + • If no clear reference exists, fall back to **"image"**. +3. Create one line per placeholder: `The {{object}} in .` +4. After the last line, insert **one blank line**. +5. **Append the caption again, but REMOVE all phrases associated with ``.** +6. Build a JSON object with two keys: + • **"revised_caption"** (with newlines as `\n`) + • **"objects"** (list of noun phrases) +7. Return only the JSON object. + +Example: +Input: Three chairs are arranged side by side. The right chair is from . The middle is . The left is . +Output: +{{ + "revised_caption": "The right chair in .\nThe middle chair in .\nThe left chair in .\n\nThree chairs are arranged side by side.", + "objects": ["right chair", "middle chair", "left chair"] +}} + +Now process the following input caption: +{caption} +""" + +def fetch_openai_response(line: dict, idx: int, client: AzureOpenAI, model_name: str, temperature: float) -> dict: + caption = line.get("input_text", "") + prompt = build_prompt(caption) + + retries = 5 + for attempt in range(1, retries + 1): + try: + resp = client.chat.completions.create( + model=model_name, + messages=[ + {"role": "system", "content": "You are a helpful assistant that strictly follows the given task instructions."}, + {"role": "user", "content": prompt}, + ], + temperature=temperature, + extra_body={"chat_template_kwargs": {"enable_thinking": False}}, + ) + gpt_json = resp.choices[0].message.content.strip() + out_record = dict(line) + out_record["model output"] = gpt_json + return {"status": "success", "record": out_record} + except Exception as exc: + logging.warning(f"[{idx}] attempt {attempt}/{retries} failed: {exc}. Retrying after 2s.") + time.sleep(2) + + out_record = dict(line) + out_record["model output"] = "ERROR" + return {"status": "failed", "record": out_record} + +def main() -> None: + args = parse_arguments() + random.seed(args.random_seed) + logging.basicConfig(level=logging.WARNING + , format="%(asctime)s | %(levelname)-8s | %(message)s") + + input_records = load_jsonl(args.input_file) + logging.info(f"Loaded {len(input_records)} records from {args.input_file}.") + + completed_global_idxs = set() + if os.path.exists(args.output_file): + logging.info(f"Existing output file {args.output_file} found. Loading completed indices...") + output_records = load_jsonl(args.output_file) + for rec in output_records: + if "global_idx" in rec: + completed_global_idxs.add(rec["global_idx"]) + logging.info(f"Found {len(completed_global_idxs)} completed records.") + else: + output_records = [] + + records_to_process = [rec for rec in input_records if rec.get("global_idx") not in completed_global_idxs] + logging.info(f"{len(records_to_process)} records remaining to process.") + + if not records_to_process: + logging.info("Nothing left to process. Exiting.") + return + + client = AzureOpenAI( + azure_endpoint=os.getenv("AZURE_OPENAI_ENDPOINT", ""), + api_key=os.getenv("AZURE_OPENAI_API_KEY", ""), + api_version="2024-02-01", + ) + + with jsonlines.open(args.output_file, mode="a") as writer, ThreadPoolExecutor(max_workers=args.workers) as pool: + futures = { + pool.submit(fetch_openai_response, rec, i, client, args.model, args.temperature): i + for i, rec in enumerate(records_to_process) + } + for f in tqdm(as_completed(futures), total=len(futures), desc="Querying GPT", unit="req"): + try: + result = f.result() + writer.write(result["record"]) + except Exception as exc: + logging.error(f"Unhandled exception in worker: {exc}") + + logging.info(f"All done. Final data written to {args.output_file}.") + +if __name__ == "__main__": + main() \ No newline at end of file diff --git a/CKPTS/query_gpt.sh b/CKPTS/query_gpt.sh new file mode 100644 index 0000000000000000000000000000000000000000..3c5256458995ce58a9477b39ee8a725253e14faf --- /dev/null +++ b/CKPTS/query_gpt.sh @@ -0,0 +1,37 @@ +#!/bin/bash +export AZURE_OPENAI_ENDPOINT="https://chatgpt-simulation.openai.azure.com/" +export AZURE_OPENAI_API_KEY="56e72039185246b6ad7ceb6977bf1237" + +# 设置一些默认参数（可以按需修改） +INPUT_FILE="/tmp/haozhezhao/MLLMG/jsonl_data/X2I_Filtered_data_train.jsonl" +OUTPUT_FILE="/tmp/haozhezhao/MLLMG/jsonl_data/X2I_Filtered_data_train_recap.jsonl" +# OUTPUT_FILE="/tmp/haozhezhao/MLLMG/jsonl_data/X2I_Filtered_data_train_test.jsonl" +# rm -f "$OUTPUT_FILE" # 删除旧的输出文件 +MODEL_NAME="gpt-4o" +WORKERS=96 +TEMPERATURE=0.7 +RANDOM_SEED=42 + +# 确保你已经设置了 Azure OpenAI 的环境变量 +# AZURE_OPENAI_ENDPOINT 和 AZURE_OPENAI_API_KEY +# vllm serve /tmp/haozhezhao/model/Qwen3-30B-A3B --port 8000 --host 0.0.0.0 --dtype bfloat16 --tensor-parallel-size 8 --max_num_seqs 2048 --gpu_memory_utilization 0.9 --max_num_batched_tokens 163840 +# vllm serve /tmp/haozhezhao/model/Qwen2.5-3B-Instruct --port 8000 --host 0.0.0.0 --dtype bfloat16 --tensor-parallel-size 8 --max_num_seqs 2048 --gpu_memory_utilization 0.95 --max_num_batched_tokens 163840 + +# python query_gpt.py \ +# --input_file "$INPUT_FILE" \ +# --output_file "$OUTPUT_FILE" \ +# --model "$MODEL_NAME" \ +# --workers "$WORKERS" \ +# --temperature "$TEMPERATURE" \ +# --random_seed "$RANDOM_SEED" + + +python query_vllm.py \ + --input_file "$INPUT_FILE" \ + --output_file "$OUTPUT_FILE" \ + --model "/tmp/haozhezhao/model/Qwen2.5-3B-Instruct" \ + --workers "$WORKERS" \ + --temperature "$TEMPERATURE" \ + --random_seed "$RANDOM_SEED" \ + --vllm_url http://localhost:8000/v1/chat/completions \ + diff --git a/CKPTS/query_vllm.py b/CKPTS/query_vllm.py new file mode 100644 index 0000000000000000000000000000000000000000..410226757d5854b8db62a0848b649e2ef1fa8a33 --- /dev/null +++ b/CKPTS/query_vllm.py @@ -0,0 +1,133 @@ +import argparse +import json +import logging +import os +import random +import time +from concurrent.futures import ThreadPoolExecutor, as_completed + +import jsonlines +import requests +from tqdm import tqdm + +def parse_arguments() -> argparse.Namespace: + p = argparse.ArgumentParser(description="Query Qwen-2 via VLLM Server in parallel.") + p.add_argument("--workers", type=int, default=4, help="Number of concurrent threads.") + p.add_argument("--input_file", type=str, required=True, help="Path to input JSONL with key 'input_text'.") + p.add_argument("--output_file", type=str, required=True, help="Destination JSONL file with model results.") + p.add_argument("--temperature", type=float, default=0.7) + p.add_argument("--vllm_url", type=str, default="http://localhost:8000/v1/chat/completions", help="VLLM server URL.") + p.add_argument("--model", type=str, default="qwen-2", help="Model name (optional for some setups).") + p.add_argument("--random_seed", type=int, default=42) + return p.parse_args() + +def load_jsonl(path: str) -> list[dict]: + with jsonlines.open(path, mode="r") as reader: + return [record for record in reader] + +def build_prompt(caption: str) -> str: + return f""" +You will receive one raw **image-caption** that contains one or more occurrences of the exact substring ``. +Rewrite the caption and report the objects associated with each placeholder, then return the result **as a JSON object**. + +Steps: +1. Scan the caption from left to right and note every position where `` appears. +2. For each occurrence, determine the **most specific concrete noun phrase** that refers to that placeholder. + • It may be immediately before the placeholder. + • It may be earlier in the sentence or a previous sentence. + • If no clear reference exists, fall back to **"image"**. +3. Create one line per placeholder: `The {{object}} in .` +4. After the last line, insert **one blank line**. +5. **Append the caption again, but REMOVE all phrases associated with ``.** +6. Build a JSON object with two keys: + • **"revised_caption"** (with newlines as `\n`) + • **"objects"** (list of noun phrases) +7. Return only the JSON object. + +Example: +Input: Three chairs are arranged side by side. The right chair is from . The middle is . The left is . +Output: +{{ + "revised_caption": "The right chair in .\nThe middle chair in .\nThe left chair in .\n\nThree chairs are arranged side by side.", + "objects": ["right chair", "middle chair", "left chair"] +}} + +Now process the following input caption: +{caption} +""" + +def fetch_vllm_response(line: dict, idx: int, vllm_url: str, temperature: float, model_name: str) -> dict: + caption = line.get("input_text", "") + prompt = build_prompt(caption) + + headers = {"Content-Type": "application/json"} + payload = { + "model": model_name, + "messages": [ + {"role": "system", "content": "You are a helpful assistant that strictly follows the given task instructions."}, + {"role": "user", "content": prompt}, + ], + "temperature": temperature, + "stream": False, + } + + retries = 3 + for attempt in range(1, retries + 1): + try: + response = requests.post(vllm_url, headers=headers, json=payload, timeout=30) + response.raise_for_status() + result = response.json() + gpt_json = result["choices"][0]["message"]["content"].strip() + out_record = dict(line) + out_record["model output"] = gpt_json + return {"status": "success", "record": out_record} + except Exception as exc: + logging.warning(f"[{idx}] attempt {attempt}/{retries} failed: {exc}. Retrying after 2s.") + time.sleep(2) + + out_record = dict(line) + out_record["model output"] = "ERROR" + return {"status": "failed", "record": out_record} + +def main() -> None: + args = parse_arguments() + random.seed(args.random_seed) + logging.basicConfig(level=logging.INFO, format="%(asctime)s | %(levelname)-8s | %(message)s") + + input_records = load_jsonl(args.input_file) + logging.info(f"Loaded {len(input_records)} records from {args.input_file}.") + + completed_global_idxs = set() + if os.path.exists(args.output_file): + logging.info(f"Existing output file {args.output_file} found. Loading completed indices...") + output_records = load_jsonl(args.output_file) + for rec in output_records: + if "global_idx" in rec: + completed_global_idxs.add(rec["global_idx"]) + logging.info(f"Found {len(completed_global_idxs)} completed records.") + else: + output_records = [] + + records_to_process = [rec for rec in input_records if rec.get("global_idx") not in completed_global_idxs] + logging.info(f"{len(records_to_process)} records remaining to process.") + + if not records_to_process: + logging.info("Nothing left to process. Exiting.") + return + + with jsonlines.open(args.output_file, mode="a") as writer, ThreadPoolExecutor(max_workers=args.workers) as pool: + futures = { + pool.submit(fetch_vllm_response, rec, i, args.vllm_url, args.temperature, args.model): i + for i, rec in enumerate(records_to_process) + } + for f in tqdm(as_completed(futures), total=len(futures), desc="Querying Qwen-2", unit="req"): + try: + result = f.result() + writer.write(result["record"]) + except Exception as exc: + logging.error(f"Unhandled exception in worker: {exc}") + + logging.info(f"All done. Final data written to {args.output_file}.") + +if __name__ == "__main__": + main()