Upload folder using huggingface_hub

experiment/thermal_run1/checkpoints/checkpoint.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:a08cacd09cd24a7afa1704b1b728229b0c2d26f85b8b8bc71c4c938816f3cddd
+size 910662494

experiment/thermal_run1/checkpoints/checkpoint_1.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:8be8e0e9be5d1bd7f16e7bdd5dde033a23e380a7f8f35b605db755ee6fe6c935
+size 910662430

experiment/thermal_run1/checkpoints/checkpoint_10.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:bf4aed853f9a17594becbb8e5be9083aede7dee0f1a8a925bdaf5fea34556113
+size 910662494

experiment/thermal_run1/checkpoints/checkpoint_11.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:e988c851be054f9c9358b0e7898962705ec7dbf66b9c6b6ecd17eee53d2e7ef7
+size 910662494

experiment/thermal_run1/checkpoints/checkpoint_12.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:741638750ae87f08f11310a62b6629b0621742e260e2379b39568f2d450c26c6
+size 910662494

experiment/thermal_run1/checkpoints/checkpoint_13.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:7127b6697fdc6af359133b1f581f8e02bc991bdc208de84dffc1ef5fb71a43f0
+size 910662494

experiment/thermal_run1/checkpoints/checkpoint_14.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:65557d72abed4de1571980f2a8a1f209479bead51c8586e347ff5414c2983689
+size 910662494

experiment/thermal_run1/checkpoints/checkpoint_15.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:154609c3c910e5328c62ddb7a7a69cffb8ac3c808558296192852c1abcce1211
+size 910662494

experiment/thermal_run1/checkpoints/checkpoint_16.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:28a5f59da4849fa333917f55c52b1b6a25d7b4732d6714fb5c8c066d93a03ab1
+size 910662494

experiment/thermal_run1/checkpoints/checkpoint_17.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:97b9f700b06a797f06049f86294bffb405fc2bdba2249bf2ab34e5f836a6ab4a
+size 910662494

experiment/thermal_run1/checkpoints/checkpoint_18.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:e90e450f66a8181bead198544fd60de5f37e40d474a402b6cafa3c10dd923725
+size 910662494

experiment/thermal_run1/checkpoints/checkpoint_19.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:18141f1ba0926e7e3f013313f04d7ea6d7e925a46c0e1b9b83f5dd2aa11dfaaa
+size 910662494

experiment/thermal_run1/checkpoints/checkpoint_2.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:8ce4f830aea93cf1923ba745483ea278063876bc4427650d095a8bf2e397d035
+size 910662494

experiment/thermal_run1/checkpoints/checkpoint_20.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:5723da2d8804c7bc5e6c52ae85faa90fc81e6eaa5c8aa7ccf82fc15dba07581b
+size 910662494

experiment/thermal_run1/checkpoints/checkpoint_21.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:a16f8b06bdc8d228f1b465aede71197ad4a3e33eba292550bc94b3025c47f96d
+size 910662494

experiment/thermal_run1/checkpoints/checkpoint_22.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:8913122ef939840271327177ddbee90bec6ccd1d4598030d479117dd4676139b
+size 910662494

experiment/thermal_run1/checkpoints/checkpoint_23.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:c897512fbc9533c6ae0171af2f735cc6618e9f2af5c69d761c26123efe82f158
+size 910662494

experiment/thermal_run1/checkpoints/checkpoint_24.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:5c4e754eee30de9285b3e59c6b953eae6423d0d0938139b3ba29f1a76aee04a5
+size 910662494

experiment/thermal_run1/checkpoints/checkpoint_25.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:a58e92dd643fc840ac89e729580cb3caa78f6747725bae72a602934c07d64af6
+size 910662494

experiment/thermal_run1/checkpoints/checkpoint_26.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:94d51b6043964ee7ff657ecb0d3aa3fc811ce72806141469982ec46eefcbcd54
+size 910662494

experiment/thermal_run1/checkpoints/checkpoint_27.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:deb0f245104633753716ceebdd899ef41b62dea134e50815b3d2fdc2bd83a30f
+size 910662494

experiment/thermal_run1/checkpoints/checkpoint_28.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:5615b19809b17430a340ea91456ea928010b3fe40c0ae599d51d09e3d4ed7e7e
+size 910662494

experiment/thermal_run1/checkpoints/checkpoint_29.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:f9c1198b55bb8a39cb9d5adf80306a9a79508b541e48252a2ed1a3e6263d0566
+size 910662494

experiment/thermal_run1/checkpoints/checkpoint_3.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:ab49e18e7cf1dbb6bcda0c8769cb1f7c29be04e1db72552a9ac12cb1eb502fe6
+size 910662494

experiment/thermal_run1/checkpoints/checkpoint_30.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:121f13583903cc30bc200fa77eacb0c0d0cec5302d9612055b046e0b1f787fe8
+size 910662494

experiment/thermal_run1/checkpoints/checkpoint_31.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:9b87ed7ead47522fe96481d9f140aa56365b5633a00cd4ff28473d03aa366a3e
+size 910662494

experiment/thermal_run1/checkpoints/checkpoint_32.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:3c972da36a8eb74204a8c2aaa24dbf108ba470e94e1996e2925d2ed78fd83115
+size 910662494

experiment/thermal_run1/checkpoints/checkpoint_33.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:559dcdb27c480d5a437fc1495ff01756822150051ea82364a6b3ad9b2bd4194a
+size 910662494

experiment/thermal_run1/checkpoints/checkpoint_34.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:1fe3db97948547e415a25243ba68263b26a51ac0266cf8ba5cbbbcbdcf78cc88
+size 910662494

experiment/thermal_run1/checkpoints/checkpoint_35.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:535c531bfefc6ca4cd50d387f961b303232a3c2fa218399ab5e77ac7328355ae
+size 910662494

experiment/thermal_run1/checkpoints/checkpoint_36.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:6296c062a7b607dd86dc000b9857bee0eb04cc3362922652818f16b45e2d866d
+size 910662494

experiment/thermal_run1/checkpoints/checkpoint_37.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:c0e196e40a20f07a7356a4cde873430988e4131e587a65ad7883977e611302fa
+size 910662494

experiment/thermal_run1/checkpoints/checkpoint_38.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:5370354b5e917526695cb893303d3b2ff55ed11ff4c7a395c8c5c49da4bfc1ee
+size 910662494

experiment/thermal_run1/checkpoints/checkpoint_39.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:2ab630fa6bd6535ee263f5aa584281f268b9ece7d5501e8564bba7696fac07f1
+size 910662494

experiment/thermal_run1/checkpoints/checkpoint_4.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:4173e21af1808616ece284564e2d8ba44ade31ccec5ecb457b46d172bc2e6d73
+size 910662494

experiment/thermal_run1/checkpoints/checkpoint_40.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:a08cacd09cd24a7afa1704b1b728229b0c2d26f85b8b8bc71c4c938816f3cddd
+size 910662494

experiment/thermal_run1/checkpoints/checkpoint_5.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:2cda5c769cdb24325396bf8fa9601a1adc277ffdeb9a4bc2ae4ca2c8a92e879d
+size 910662494

experiment/thermal_run1/checkpoints/checkpoint_6.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:a772cd7044f36d52d9a96ea314f7a09590f85f42d2c3534d86ae0f8ddc2ffa3d
+size 910662494

experiment/thermal_run1/checkpoints/checkpoint_7.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:53ebacbd40cf8a0855fbdedb7f9795d80f169f18ae02aa2b146a065c521d8a97
+size 910662494

experiment/thermal_run1/checkpoints/checkpoint_8.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:c24089637368598c6ae5302fbf61c1ef46994a950cbd4c5f3c114a82e925f6a1
+size 910662494

experiment/thermal_run1/checkpoints/checkpoint_9.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:e2f521f3db0b6438a03e61df33d1339ca910a8c89f9a8fe222b1481bb5a8a384
+size 910662494

experiment/thermal_run1/config.yaml

@@ -0,0 +1,313 @@
+scratch:
+  resolution: 1024
+  train_batch_size: 1
+  num_train_workers: 10
+  num_frames: 1
+  max_num_objects: 3
+  base_lr: 5.0e-06
+  vision_lr: 3.0e-06
+  phases_per_epoch: 1
+  num_epochs: 40
+dataset:
+  img_folder: /work/nvme/bfgd/cmaduabuchi/DATA/thermal_video/thriveseg/train/images
+  gt_folder: /work/nvme/bfgd/cmaduabuchi/DATA/thermal_video/thriveseg/train/masks
+  file_list_txt: null
+  multiplier: 2
+vos:
+  train_transforms:
+  - _target_: training.dataset.transforms.ComposeAPI
+    transforms:
+    - _target_: training.dataset.transforms.RandomHorizontalFlip
+      consistent_transform: true
+    - _target_: training.dataset.transforms.RandomAffine
+      degrees: 25
+      shear: 20
+      image_interpolation: bilinear
+      consistent_transform: true
+    - _target_: training.dataset.transforms.RandomResizeAPI
+      sizes: ${scratch.resolution}
+      square: true
+      consistent_transform: true
+    - _target_: training.dataset.transforms.ColorJitter
+      consistent_transform: true
+      brightness: 0.1
+      contrast: 0.03
+      saturation: 0.03
+      hue: null
+    - _target_: training.dataset.transforms.RandomGrayscale
+      p: 0.05
+      consistent_transform: true
+    - _target_: training.dataset.transforms.ColorJitter
+      consistent_transform: false
+      brightness: 0.1
+      contrast: 0.05
+      saturation: 0.05
+      hue: null
+    - _target_: training.dataset.transforms.ToTensorAPI
+    - _target_: training.dataset.transforms.NormalizeAPI
+      mean:
+      - 0.485
+      - 0.456
+      - 0.406
+      std:
+      - 0.229
+      - 0.224
+      - 0.225
+trainer:
+  _target_: training.trainer.Trainer
+  mode: train_only
+  max_epochs: ${times:${scratch.num_epochs},${scratch.phases_per_epoch}}
+  accelerator: cuda
+  seed_value: 123
+  model:
+    _target_: training.model.sam2.SAM2Train
+    image_encoder:
+      _target_: sam2.modeling.backbones.image_encoder.ImageEncoder
+      scalp: 1
+      trunk:
+        _target_: sam2.modeling.backbones.hieradet.Hiera
+        embed_dim: 112
+        num_heads: 2
+        drop_path_rate: 0.1
+      neck:
+        _target_: sam2.modeling.backbones.image_encoder.FpnNeck
+        position_encoding:
+          _target_: sam2.modeling.position_encoding.PositionEmbeddingSine
+          num_pos_feats: 256
+          normalize: true
+          scale: null
+          temperature: 10000
+        d_model: 256
+        backbone_channel_list:
+        - 896
+        - 448
+        - 224
+        - 112
+        fpn_top_down_levels:
+        - 2
+        - 3
+        fpn_interp_model: nearest
+    memory_attention:
+      _target_: sam2.modeling.memory_attention.MemoryAttention
+      d_model: 256
+      pos_enc_at_input: true
+      layer:
+        _target_: sam2.modeling.memory_attention.MemoryAttentionLayer
+        activation: relu
+        dim_feedforward: 2048
+        dropout: 0.1
+        pos_enc_at_attn: false
+        self_attention:
+          _target_: sam2.modeling.sam.transformer.RoPEAttention
+          rope_theta: 10000.0
+          feat_sizes:
+          - 64
+          - 64
+          embedding_dim: 256
+          num_heads: 1
+          downsample_rate: 1
+          dropout: 0.1
+        d_model: 256
+        pos_enc_at_cross_attn_keys: true
+        pos_enc_at_cross_attn_queries: false
+        cross_attention:
+          _target_: sam2.modeling.sam.transformer.RoPEAttention
+          rope_theta: 10000.0
+          feat_sizes:
+          - 64
+          - 64
+          rope_k_repeat: true
+          embedding_dim: 256
+          num_heads: 1
+          downsample_rate: 1
+          dropout: 0.1
+          kv_in_dim: 64
+      num_layers: 4
+    memory_encoder:
+      _target_: sam2.modeling.memory_encoder.MemoryEncoder
+      out_dim: 64
+      position_encoding:
+        _target_: sam2.modeling.position_encoding.PositionEmbeddingSine
+        num_pos_feats: 64
+        normalize: true
+        scale: null
+        temperature: 10000
+      mask_downsampler:
+        _target_: sam2.modeling.memory_encoder.MaskDownSampler
+        kernel_size: 3
+        stride: 2
+        padding: 1
+      fuser:
+        _target_: sam2.modeling.memory_encoder.Fuser
+        layer:
+          _target_: sam2.modeling.memory_encoder.CXBlock
+          dim: 256
+          kernel_size: 7
+          padding: 3
+          layer_scale_init_value: 1.0e-06
+          use_dwconv: true
+        num_layers: 2
+    num_maskmem: 7
+    image_size: ${scratch.resolution}
+    sigmoid_scale_for_mem_enc: 20.0
+    sigmoid_bias_for_mem_enc: -10.0
+    use_mask_input_as_output_without_sam: true
+    directly_add_no_mem_embed: true
+    no_obj_embed_spatial: true
+    use_high_res_features_in_sam: true
+    multimask_output_in_sam: true
+    iou_prediction_use_sigmoid: true
+    use_obj_ptrs_in_encoder: true
+    add_tpos_enc_to_obj_ptrs: true
+    proj_tpos_enc_in_obj_ptrs: true
+    use_signed_tpos_enc_to_obj_ptrs: true
+    only_obj_ptrs_in_the_past_for_eval: true
+    pred_obj_scores: true
+    pred_obj_scores_mlp: true
+    fixed_no_obj_ptr: true
+    multimask_output_for_tracking: true
+    use_multimask_token_for_obj_ptr: true
+    multimask_min_pt_num: 0
+    multimask_max_pt_num: 1
+    use_mlp_for_obj_ptr_proj: true
+    prob_to_use_pt_input_for_train: 0.5
+    prob_to_use_pt_input_for_eval: 0.0
+    prob_to_use_box_input_for_train: 0.5
+    prob_to_use_box_input_for_eval: 0.0
+    prob_to_sample_from_gt_for_train: 0.1
+    num_frames_to_correct_for_train: 2
+    num_frames_to_correct_for_eval: 1
+    rand_frames_to_correct_for_train: true
+    add_all_frames_to_correct_as_cond: true
+    num_init_cond_frames_for_train: 2
+    rand_init_cond_frames_for_train: true
+    num_correction_pt_per_frame: 7
+    use_act_ckpt_iterative_pt_sampling: false
+    num_init_cond_frames_for_eval: 1
+    forward_backbone_per_frame_for_eval: true
+  data:
+    train:
+      _target_: training.dataset.sam2_datasets.TorchTrainMixedDataset
+      phases_per_epoch: ${scratch.phases_per_epoch}
+      batch_sizes:
+      - ${scratch.train_batch_size}
+      datasets:
+      - _target_: training.dataset.utils.RepeatFactorWrapper
+        dataset:
+          _target_: training.dataset.utils.ConcatDataset
+          datasets:
+          - _target_: training.dataset.vos_dataset.VOSDataset
+            transforms: ${vos.train_transforms}
+            training: true
+            video_dataset:
+              _target_: training.dataset.binary_png_raw_dataset.BinaryPNGRawDataset
+              img_folder: ${dataset.img_folder}
+              gt_folder: ${dataset.gt_folder}
+              file_list_txt: ${dataset.file_list_txt}
+            sampler:
+              _target_: training.dataset.vos_sampler.RandomUniformSampler
+              num_frames: ${scratch.num_frames}
+              max_num_objects: ${scratch.max_num_objects}
+            multiplier: ${dataset.multiplier}
+      shuffle: true
+      num_workers: ${scratch.num_train_workers}
+      pin_memory: true
+      drop_last: true
+      collate_fn:
+        _target_: training.utils.data_utils.collate_fn
+        _partial_: true
+        dict_key: all
+  optim:
+    amp:
+      enabled: true
+      amp_dtype: bfloat16
+    optimizer:
+      _target_: torch.optim.AdamW
+    gradient_clip:
+      _target_: training.optimizer.GradientClipper
+      max_norm: 0.1
+      norm_type: 2
+    param_group_modifiers:
+    - _target_: training.optimizer.layer_decay_param_modifier
+      _partial_: true
+      layer_decay_value: 0.9
+      apply_to: image_encoder.trunk
+      overrides:
+      - pattern: '*pos_embed*'
+        value: 1.0
+    options:
+      lr:
+      - scheduler:
+          _target_: fvcore.common.param_scheduler.CosineParamScheduler
+          start_value: ${scratch.base_lr}
+          end_value: ${divide:${scratch.base_lr},10}
+      - scheduler:
+          _target_: fvcore.common.param_scheduler.CosineParamScheduler
+          start_value: ${scratch.vision_lr}
+          end_value: ${divide:${scratch.vision_lr},10}
+        param_names:
+        - image_encoder.*
+      weight_decay:
+      - scheduler:
+          _target_: fvcore.common.param_scheduler.ConstantParamScheduler
+          value: 0.1
+      - scheduler:
+          _target_: fvcore.common.param_scheduler.ConstantParamScheduler
+          value: 0.0
+        param_names:
+        - '*bias*'
+        module_cls_names:
+        - torch.nn.LayerNorm
+  loss:
+    all:
+      _target_: training.loss_fns.MultiStepMultiMasksAndIous
+      weight_dict:
+        loss_mask: 20
+        loss_dice: 1
+        loss_iou: 1
+        loss_class: 1
+      supervise_all_iou: true
+      iou_use_l1_loss: true
+      pred_obj_scores: true
+      focal_gamma_obj_score: 0.0
+      focal_alpha_obj_score: -1.0
+  distributed:
+    backend: nccl
+    find_unused_parameters: true
+  logging:
+    tensorboard_writer:
+      _target_: training.utils.logger.make_tensorboard_logger
+      log_dir: ${launcher.experiment_log_dir}/tensorboard
+      flush_secs: 120
+      should_log: true
+    log_dir: ${launcher.experiment_log_dir}/logs
+    log_freq: 10
+  checkpoint:
+    save_dir: ${launcher.experiment_log_dir}/checkpoints
+    save_freq: 1
+    model_weight_initializer:
+      _partial_: true
+      _target_: training.utils.checkpoint_utils.load_state_dict_into_model
+      strict: true
+      ignore_unexpected_keys: null
+      ignore_missing_keys: null
+      state_dict:
+        _target_: training.utils.checkpoint_utils.load_checkpoint_and_apply_kernels
+        checkpoint_path: /work/nvme/bfgd/cmaduabuchi/MODELS/THRIVESEG/sam2.1_hiera_base_plus.pt
+        ckpt_state_dict_keys:
+        - model
+launcher:
+  num_nodes: 1
+  gpus_per_node: 8
+  experiment_log_dir: /work/nvme/bfgd/cmaduabuchi/MODELS/THRIVESEG/experiment/thermal_run1
+submitit:
+  partition: null
+  account: null
+  qos: null
+  cpus_per_task: 10
+  use_cluster: false
+  timeout_hour: 24
+  name: null
+  port_range:
+  - 10000
+  - 65000

experiment/thermal_run1/config_resolved.yaml

@@ -0,0 +1,351 @@
+scratch:
+  resolution: 1024
+  train_batch_size: 1
+  num_train_workers: 10
+  num_frames: 1
+  max_num_objects: 3
+  base_lr: 5.0e-06
+  vision_lr: 3.0e-06
+  phases_per_epoch: 1
+  num_epochs: 40
+dataset:
+  img_folder: /work/nvme/bfgd/cmaduabuchi/DATA/thermal_video/thriveseg/train/images
+  gt_folder: /work/nvme/bfgd/cmaduabuchi/DATA/thermal_video/thriveseg/train/masks
+  file_list_txt: null
+  multiplier: 2
+vos:
+  train_transforms:
+  - _target_: training.dataset.transforms.ComposeAPI
+    transforms:
+    - _target_: training.dataset.transforms.RandomHorizontalFlip
+      consistent_transform: true
+    - _target_: training.dataset.transforms.RandomAffine
+      degrees: 25
+      shear: 20
+      image_interpolation: bilinear
+      consistent_transform: true
+    - _target_: training.dataset.transforms.RandomResizeAPI
+      sizes: 1024
+      square: true
+      consistent_transform: true
+    - _target_: training.dataset.transforms.ColorJitter
+      consistent_transform: true
+      brightness: 0.1
+      contrast: 0.03
+      saturation: 0.03
+      hue: null
+    - _target_: training.dataset.transforms.RandomGrayscale
+      p: 0.05
+      consistent_transform: true
+    - _target_: training.dataset.transforms.ColorJitter
+      consistent_transform: false
+      brightness: 0.1
+      contrast: 0.05
+      saturation: 0.05
+      hue: null
+    - _target_: training.dataset.transforms.ToTensorAPI
+    - _target_: training.dataset.transforms.NormalizeAPI
+      mean:
+      - 0.485
+      - 0.456
+      - 0.406
+      std:
+      - 0.229
+      - 0.224
+      - 0.225
+trainer:
+  _target_: training.trainer.Trainer
+  mode: train_only
+  max_epochs: 40
+  accelerator: cuda
+  seed_value: 123
+  model:
+    _target_: training.model.sam2.SAM2Train
+    image_encoder:
+      _target_: sam2.modeling.backbones.image_encoder.ImageEncoder
+      scalp: 1
+      trunk:
+        _target_: sam2.modeling.backbones.hieradet.Hiera
+        embed_dim: 112
+        num_heads: 2
+        drop_path_rate: 0.1
+      neck:
+        _target_: sam2.modeling.backbones.image_encoder.FpnNeck
+        position_encoding:
+          _target_: sam2.modeling.position_encoding.PositionEmbeddingSine
+          num_pos_feats: 256
+          normalize: true
+          scale: null
+          temperature: 10000
+        d_model: 256
+        backbone_channel_list:
+        - 896
+        - 448
+        - 224
+        - 112
+        fpn_top_down_levels:
+        - 2
+        - 3
+        fpn_interp_model: nearest
+    memory_attention:
+      _target_: sam2.modeling.memory_attention.MemoryAttention
+      d_model: 256
+      pos_enc_at_input: true
+      layer:
+        _target_: sam2.modeling.memory_attention.MemoryAttentionLayer
+        activation: relu
+        dim_feedforward: 2048
+        dropout: 0.1
+        pos_enc_at_attn: false
+        self_attention:
+          _target_: sam2.modeling.sam.transformer.RoPEAttention
+          rope_theta: 10000.0
+          feat_sizes:
+          - 64
+          - 64
+          embedding_dim: 256
+          num_heads: 1
+          downsample_rate: 1
+          dropout: 0.1
+        d_model: 256
+        pos_enc_at_cross_attn_keys: true
+        pos_enc_at_cross_attn_queries: false
+        cross_attention:
+          _target_: sam2.modeling.sam.transformer.RoPEAttention
+          rope_theta: 10000.0
+          feat_sizes:
+          - 64
+          - 64
+          rope_k_repeat: true
+          embedding_dim: 256
+          num_heads: 1
+          downsample_rate: 1
+          dropout: 0.1
+          kv_in_dim: 64
+      num_layers: 4
+    memory_encoder:
+      _target_: sam2.modeling.memory_encoder.MemoryEncoder
+      out_dim: 64
+      position_encoding:
+        _target_: sam2.modeling.position_encoding.PositionEmbeddingSine
+        num_pos_feats: 64
+        normalize: true
+        scale: null
+        temperature: 10000
+      mask_downsampler:
+        _target_: sam2.modeling.memory_encoder.MaskDownSampler
+        kernel_size: 3
+        stride: 2
+        padding: 1
+      fuser:
+        _target_: sam2.modeling.memory_encoder.Fuser
+        layer:
+          _target_: sam2.modeling.memory_encoder.CXBlock
+          dim: 256
+          kernel_size: 7
+          padding: 3
+          layer_scale_init_value: 1.0e-06
+          use_dwconv: true
+        num_layers: 2
+    num_maskmem: 7
+    image_size: 1024
+    sigmoid_scale_for_mem_enc: 20.0
+    sigmoid_bias_for_mem_enc: -10.0
+    use_mask_input_as_output_without_sam: true
+    directly_add_no_mem_embed: true
+    no_obj_embed_spatial: true
+    use_high_res_features_in_sam: true
+    multimask_output_in_sam: true
+    iou_prediction_use_sigmoid: true
+    use_obj_ptrs_in_encoder: true
+    add_tpos_enc_to_obj_ptrs: true
+    proj_tpos_enc_in_obj_ptrs: true
+    use_signed_tpos_enc_to_obj_ptrs: true
+    only_obj_ptrs_in_the_past_for_eval: true
+    pred_obj_scores: true
+    pred_obj_scores_mlp: true
+    fixed_no_obj_ptr: true
+    multimask_output_for_tracking: true
+    use_multimask_token_for_obj_ptr: true
+    multimask_min_pt_num: 0
+    multimask_max_pt_num: 1
+    use_mlp_for_obj_ptr_proj: true
+    prob_to_use_pt_input_for_train: 0.5
+    prob_to_use_pt_input_for_eval: 0.0
+    prob_to_use_box_input_for_train: 0.5
+    prob_to_use_box_input_for_eval: 0.0
+    prob_to_sample_from_gt_for_train: 0.1
+    num_frames_to_correct_for_train: 2
+    num_frames_to_correct_for_eval: 1
+    rand_frames_to_correct_for_train: true
+    add_all_frames_to_correct_as_cond: true
+    num_init_cond_frames_for_train: 2
+    rand_init_cond_frames_for_train: true
+    num_correction_pt_per_frame: 7
+    use_act_ckpt_iterative_pt_sampling: false
+    num_init_cond_frames_for_eval: 1
+    forward_backbone_per_frame_for_eval: true
+  data:
+    train:
+      _target_: training.dataset.sam2_datasets.TorchTrainMixedDataset
+      phases_per_epoch: 1
+      batch_sizes:
+      - 1
+      datasets:
+      - _target_: training.dataset.utils.RepeatFactorWrapper
+        dataset:
+          _target_: training.dataset.utils.ConcatDataset
+          datasets:
+          - _target_: training.dataset.vos_dataset.VOSDataset
+            transforms:
+            - _target_: training.dataset.transforms.ComposeAPI
+              transforms:
+              - _target_: training.dataset.transforms.RandomHorizontalFlip
+                consistent_transform: true
+              - _target_: training.dataset.transforms.RandomAffine
+                degrees: 25
+                shear: 20
+                image_interpolation: bilinear
+                consistent_transform: true
+              - _target_: training.dataset.transforms.RandomResizeAPI
+                sizes: 1024
+                square: true
+                consistent_transform: true
+              - _target_: training.dataset.transforms.ColorJitter
+                consistent_transform: true
+                brightness: 0.1
+                contrast: 0.03
+                saturation: 0.03
+                hue: null
+              - _target_: training.dataset.transforms.RandomGrayscale
+                p: 0.05
+                consistent_transform: true
+              - _target_: training.dataset.transforms.ColorJitter
+                consistent_transform: false
+                brightness: 0.1
+                contrast: 0.05
+                saturation: 0.05
+                hue: null
+              - _target_: training.dataset.transforms.ToTensorAPI
+              - _target_: training.dataset.transforms.NormalizeAPI
+                mean:
+                - 0.485
+                - 0.456
+                - 0.406
+                std:
+                - 0.229
+                - 0.224
+                - 0.225
+            training: true
+            video_dataset:
+              _target_: training.dataset.binary_png_raw_dataset.BinaryPNGRawDataset
+              img_folder: /work/nvme/bfgd/cmaduabuchi/DATA/thermal_video/thriveseg/train/images
+              gt_folder: /work/nvme/bfgd/cmaduabuchi/DATA/thermal_video/thriveseg/train/masks
+              file_list_txt: null
+            sampler:
+              _target_: training.dataset.vos_sampler.RandomUniformSampler
+              num_frames: 1
+              max_num_objects: 3
+            multiplier: 2
+      shuffle: true
+      num_workers: 10
+      pin_memory: true
+      drop_last: true
+      collate_fn:
+        _target_: training.utils.data_utils.collate_fn
+        _partial_: true
+        dict_key: all
+  optim:
+    amp:
+      enabled: true
+      amp_dtype: bfloat16
+    optimizer:
+      _target_: torch.optim.AdamW
+    gradient_clip:
+      _target_: training.optimizer.GradientClipper
+      max_norm: 0.1
+      norm_type: 2
+    param_group_modifiers:
+    - _target_: training.optimizer.layer_decay_param_modifier
+      _partial_: true
+      layer_decay_value: 0.9
+      apply_to: image_encoder.trunk
+      overrides:
+      - pattern: '*pos_embed*'
+        value: 1.0
+    options:
+      lr:
+      - scheduler:
+          _target_: fvcore.common.param_scheduler.CosineParamScheduler
+          start_value: 5.0e-06
+          end_value: 5.000000000000001e-07
+      - scheduler:
+          _target_: fvcore.common.param_scheduler.CosineParamScheduler
+          start_value: 3.0e-06
+          end_value: 3.0e-07
+        param_names:
+        - image_encoder.*
+      weight_decay:
+      - scheduler:
+          _target_: fvcore.common.param_scheduler.ConstantParamScheduler
+          value: 0.1
+      - scheduler:
+          _target_: fvcore.common.param_scheduler.ConstantParamScheduler
+          value: 0.0
+        param_names:
+        - '*bias*'
+        module_cls_names:
+        - torch.nn.LayerNorm
+  loss:
+    all:
+      _target_: training.loss_fns.MultiStepMultiMasksAndIous
+      weight_dict:
+        loss_mask: 20
+        loss_dice: 1
+        loss_iou: 1
+        loss_class: 1
+      supervise_all_iou: true
+      iou_use_l1_loss: true
+      pred_obj_scores: true
+      focal_gamma_obj_score: 0.0
+      focal_alpha_obj_score: -1.0
+  distributed:
+    backend: nccl
+    find_unused_parameters: true
+  logging:
+    tensorboard_writer:
+      _target_: training.utils.logger.make_tensorboard_logger
+      log_dir: /work/nvme/bfgd/cmaduabuchi/MODELS/THRIVESEG/experiment/thermal_run1/tensorboard
+      flush_secs: 120
+      should_log: true
+    log_dir: /work/nvme/bfgd/cmaduabuchi/MODELS/THRIVESEG/experiment/thermal_run1/logs
+    log_freq: 10
+  checkpoint:
+    save_dir: /work/nvme/bfgd/cmaduabuchi/MODELS/THRIVESEG/experiment/thermal_run1/checkpoints
+    save_freq: 1
+    model_weight_initializer:
+      _partial_: true
+      _target_: training.utils.checkpoint_utils.load_state_dict_into_model
+      strict: true
+      ignore_unexpected_keys: null
+      ignore_missing_keys: null
+      state_dict:
+        _target_: training.utils.checkpoint_utils.load_checkpoint_and_apply_kernels
+        checkpoint_path: /work/nvme/bfgd/cmaduabuchi/MODELS/THRIVESEG/sam2.1_hiera_base_plus.pt
+        ckpt_state_dict_keys:
+        - model
+launcher:
+  num_nodes: 1
+  gpus_per_node: 8
+  experiment_log_dir: /work/nvme/bfgd/cmaduabuchi/MODELS/THRIVESEG/experiment/thermal_run1
+submitit:
+  partition: null
+  account: null
+  qos: null
+  cpus_per_task: 10
+  use_cluster: false
+  timeout_hour: 24
+  name: null
+  port_range:
+  - 10000
+  - 65000

experiment/thermal_run1/logs/best_stats.json

@@ -0,0 +1,40 @@
+{"Trainer/where": 0.0249937374749499, "Trainer/epoch": 0, "Trainer/steps_train": 3992}
+{"Trainer/where": 0.0499937374749499, "Trainer/epoch": 1, "Trainer/steps_train": 7984}
+{"Trainer/where": 0.0749937374749499, "Trainer/epoch": 2, "Trainer/steps_train": 11976}
+{"Trainer/where": 0.09999373747494991, "Trainer/epoch": 3, "Trainer/steps_train": 15968}
+{"Trainer/where": 0.1249937374749499, "Trainer/epoch": 4, "Trainer/steps_train": 19960}
+{"Trainer/where": 0.1499937374749499, "Trainer/epoch": 5, "Trainer/steps_train": 23952}
+{"Trainer/where": 0.1749937374749499, "Trainer/epoch": 6, "Trainer/steps_train": 27944}
+{"Trainer/where": 0.19999373747494992, "Trainer/epoch": 7, "Trainer/steps_train": 31936}
+{"Trainer/where": 0.2249937374749499, "Trainer/epoch": 8, "Trainer/steps_train": 35928}
+{"Trainer/where": 0.2499937374749499, "Trainer/epoch": 9, "Trainer/steps_train": 39920}
+{"Trainer/where": 0.27499373747494993, "Trainer/epoch": 10, "Trainer/steps_train": 43912}
+{"Trainer/where": 0.2999937374749499, "Trainer/epoch": 11, "Trainer/steps_train": 47904}
+{"Trainer/where": 0.3249937374749499, "Trainer/epoch": 12, "Trainer/steps_train": 51896}
+{"Trainer/where": 0.3499937374749499, "Trainer/epoch": 13, "Trainer/steps_train": 55888}
+{"Trainer/where": 0.3749937374749499, "Trainer/epoch": 14, "Trainer/steps_train": 59880}
+{"Trainer/where": 0.39999373747494993, "Trainer/epoch": 15, "Trainer/steps_train": 63872}
+{"Trainer/where": 0.4249937374749499, "Trainer/epoch": 16, "Trainer/steps_train": 67864}
+{"Trainer/where": 0.4499937374749499, "Trainer/epoch": 17, "Trainer/steps_train": 71856}
+{"Trainer/where": 0.4749937374749499, "Trainer/epoch": 18, "Trainer/steps_train": 75848}
+{"Trainer/where": 0.4999937374749499, "Trainer/epoch": 19, "Trainer/steps_train": 79840}
+{"Trainer/where": 0.5249937374749499, "Trainer/epoch": 20, "Trainer/steps_train": 83832}
+{"Trainer/where": 0.54999373747495, "Trainer/epoch": 21, "Trainer/steps_train": 87824}
+{"Trainer/where": 0.5749937374749499, "Trainer/epoch": 22, "Trainer/steps_train": 91816}
+{"Trainer/where": 0.5999937374749499, "Trainer/epoch": 23, "Trainer/steps_train": 95808}
+{"Trainer/where": 0.6249937374749499, "Trainer/epoch": 24, "Trainer/steps_train": 99800}
+{"Trainer/where": 0.6499937374749499, "Trainer/epoch": 25, "Trainer/steps_train": 103792}
+{"Trainer/where": 0.67499373747495, "Trainer/epoch": 26, "Trainer/steps_train": 107784}
+{"Trainer/where": 0.6999937374749499, "Trainer/epoch": 27, "Trainer/steps_train": 111776}
+{"Trainer/where": 0.7249937374749499, "Trainer/epoch": 28, "Trainer/steps_train": 115768}
+{"Trainer/where": 0.7499937374749499, "Trainer/epoch": 29, "Trainer/steps_train": 119760}
+{"Trainer/where": 0.7749937374749499, "Trainer/epoch": 30, "Trainer/steps_train": 123752}
+{"Trainer/where": 0.79999373747495, "Trainer/epoch": 31, "Trainer/steps_train": 127744}
+{"Trainer/where": 0.8249937374749499, "Trainer/epoch": 32, "Trainer/steps_train": 131736}
+{"Trainer/where": 0.8499937374749498, "Trainer/epoch": 33, "Trainer/steps_train": 135728}
+{"Trainer/where": 0.8749937374749498, "Trainer/epoch": 34, "Trainer/steps_train": 139720}
+{"Trainer/where": 0.8999937374749498, "Trainer/epoch": 35, "Trainer/steps_train": 143712}
+{"Trainer/where": 0.9249937374749498, "Trainer/epoch": 36, "Trainer/steps_train": 147704}
+{"Trainer/where": 0.9499937374749499, "Trainer/epoch": 37, "Trainer/steps_train": 151696}
+{"Trainer/where": 0.9749937374749498, "Trainer/epoch": 38, "Trainer/steps_train": 155688}
+{"Trainer/where": 0.9999937374749498, "Trainer/epoch": 39, "Trainer/steps_train": 159680}

experiment/thermal_run1/logs/log.txt

@@ -0,0 +1,732 @@
+INFO 2025-10-06 06:35:10,376 train_utils.py: 109: MACHINE SEED: 4920
+INFO 2025-10-06 06:35:10,389 train_utils.py: 155: Logging ENV_VARIABLES
+INFO 2025-10-06 06:35:10,389 train_utils.py: 156: BASH_ENV=/usr/share/lmod/lmod/init/bash
+BASH_FUNC_ml%%=() {  eval "$($LMOD_DIR/ml_cmd "$@")"
+}
+BASH_FUNC_module%%=() {  if [ -z "${LMOD_SH_DBG_ON+x}" ]; then
+ case "$-" in 
+ *v*x*)
+ __lmod_sh_dbg='vx'
+ ;;
+ *v*)
+ __lmod_sh_dbg='v'
+ ;;
+ *x*)
+ __lmod_sh_dbg='x'
+ ;;
+ esac;
+ fi;
+ if [ -n "${__lmod_sh_dbg:-}" ]; then
+ set +$__lmod_sh_dbg;
+ echo "Shell debugging temporarily silenced: export LMOD_SH_DBG_ON=1 for Lmod's output" 1>&2;
+ fi;
+ eval "$($LMOD_CMD shell "$@")" && eval "$(${LMOD_SETTARG_CMD:-:} -s sh)";
+ __lmod_my_status=$?;
+ if [ -n "${__lmod_sh_dbg:-}" ]; then
+ echo "Shell debugging restarted" 1>&2;
+ set -$__lmod_sh_dbg;
+ fi;
+ unset __lmod_sh_dbg;
+ return $__lmod_my_status
+}
+BASH_FUNC_which%%=() {  ( alias;
+ eval ${which_declare} ) | /usr/bin/which --tty-only --read-alias --read-functions --show-tilde --show-dot $@
+}
+BROWSER=/u/cmaduabuchi/.vscode-server/cli/servers/Stable-0f0d87fa9e96c856c5212fc86db137ac0d783365/server/bin/helpers/browser.sh
+BUNDLED_DEBUGPY_PATH=/u/cmaduabuchi/.vscode-server/extensions/ms-python.debugpy-2025.10.0-linux-x64/bundled/libs/debugpy
+CC=/sw/spack/deltas11-2023-03/apps/linux-rhel8-x86_64/gcc-8.5.0/gcc-11.4.0-yycklku/bin/gcc
+CMAKE_PREFIX_PATH=/sw/spack/deltas11-2023-03/apps/linux-rhel8-zen3/gcc-11.4.0/cuda-11.8.0-vfixfmc:/sw/spack/deltas11-2023-03/apps/linux-rhel8-zen3/gcc-11.4.0/openmpi-4.1.6-lranp74:/sw/spack/deltas11-2023-03/apps/linux-rhel8-x86_64/gcc-8.5.0/gcc-11.4.0-yycklku
+COLORTERM=truecolor
+CONDA_DEFAULT_ENV=sam2
+CONDA_EXE=/sw/external/python/anaconda3/bin/conda
+CONDA_PREFIX=/u/cmaduabuchi/.conda/envs/sam2
+CONDA_PREFIX_1=/sw/external/python/anaconda3
+CONDA_PREFIX_2=/u/cmaduabuchi/.conda/envs/sam2
+CONDA_PREFIX_3=/sw/external/python/anaconda3
+CONDA_PROMPT_MODIFIER=(sam2) 
+CONDA_PYTHON_EXE=/sw/external/python/anaconda3/bin/python
+CONDA_SHLVL=4
+CPLUS_INCLUDE_PATH=/sw/spack/deltas11-2023-03/apps/linux-rhel8-zen3/gcc-11.4.0/cuda-11.8.0-vfixfmc/include:/sw/spack/deltas11-2023-03/apps/linux-rhel8-zen3/gcc-11.4.0/openmpi-4.1.6-lranp74/include:/sw/spack/deltas11-2023-03/apps/linux-rhel8-x86_64/gcc-8.5.0/gcc-11.4.0-yycklku/include
+CRAY_ENABLE_PE=/etc/cray-pe.d/enable-pe.sh
+CUDA_HOME=/sw/spack/deltas11-2023-03/apps/linux-rhel8-zen3/gcc-11.4.0/cuda-11.8.0-vfixfmc
+CUDA_MODULE_LOADING=LAZY
+CUDA_VISIBLE_DEVICES=0
+CUE_CLUSTER_HOME=/u/cmaduabuchi
+CUE_DOCS=https://portal.xsede.org/ncsa-delta
+CUE_HOME=/u/cmaduabuchi
+CUE_HOME_TYPE=lustre-2.14
+CUE_HOSTNAME=dt-login03.delta.ncsa.illinois.edu
+CUE_PROMPT_HOST=dt-login03.delta
+CUE_RESOURCE_ID=delta-cpu.ncsa.xsede.org
+CXX=/sw/spack/deltas11-2023-03/apps/linux-rhel8-x86_64/gcc-8.5.0/gcc-11.4.0-yycklku/bin/g++
+C_INCLUDE_PATH=/sw/spack/deltas11-2023-03/apps/linux-rhel8-zen3/gcc-11.4.0/cuda-11.8.0-vfixfmc/include:/sw/spack/deltas11-2023-03/apps/linux-rhel8-zen3/gcc-11.4.0/openmpi-4.1.6-lranp74/include:/sw/spack/deltas11-2023-03/apps/linux-rhel8-x86_64/gcc-8.5.0/gcc-11.4.0-yycklku/include
+DBUS_SESSION_BUS_ADDRESS=unix:path=/run/user/77886/bus
+F77=/sw/spack/deltas11-2023-03/apps/linux-rhel8-x86_64/gcc-8.5.0/gcc-11.4.0-yycklku/bin/gfortran
+FC=/sw/spack/deltas11-2023-03/apps/linux-rhel8-x86_64/gcc-8.5.0/gcc-11.4.0-yycklku/bin/gfortran
+FPATH=/usr/share/lmod/lmod/init/ksh_funcs
+GCC_HOME=/sw/spack/deltas11-2023-03/apps/linux-rhel8-x86_64/gcc-8.5.0/gcc-11.4.0-yycklku
+GIT_ASKPASS=/u/cmaduabuchi/.vscode-server/cli/servers/Stable-0f0d87fa9e96c856c5212fc86db137ac0d783365/server/extensions/git/dist/askpass.sh
+GPU_DEVICE_ORDINAL=0
+HISTCONTROL=ignoredups
+HISTFILE=/u/cmaduabuchi/.bash_commands.log
+HISTSIZE=50000
+HISTTIMEFORMAT=%h %d %H:%M:%S 
+HOME=/u/cmaduabuchi
+HOSTNAME=dt-login03.delta.ncsa.illinois.edu
+HYDRA_BOOTSTRAP=slurm
+HYDRA_FULL_ERROR=1
+HYDRA_LAUNCHER_EXTRA_ARGS=--external-launcher
+INCLUDE=/sw/spack/deltas11-2023-03/apps/linux-rhel8-zen3/gcc-11.4.0/cuda-11.8.0-vfixfmc/include:/sw/spack/deltas11-2023-03/apps/linux-rhel8-zen3/gcc-11.4.0/openmpi-4.1.6-lranp74/include:/sw/spack/deltas11-2023-03/apps/linux-rhel8-x86_64/gcc-8.5.0/gcc-11.4.0-yycklku/include
+I_MPI_HYDRA_BOOTSTRAP=slurm
+I_MPI_HYDRA_BOOTSTRAP_EXEC_EXTRA_ARGS=--external-launcher
+LANG=en_US.UTF-8
+LD_LIBRARY_PATH=/sw/spack/delta-2022-03/apps/gcc/11.2.0-gcc-8.4.1-fxgnsyr/lib64:/sw/spack/deltas11-2023-03/apps/linux-rhel8-zen3/gcc-11.4.0/cuda-11.8.0-vfixfmc/lib64:/opt/cray/libfabric/1.15.2.0/lib64:/opt/cray/libfabric/1.15.2.0/lib
+LESSOPEN=||/usr/bin/lesspipe.sh %s
+LIBRARY_PATH=/sw/spack/deltas11-2023-03/apps/linux-rhel8-zen3/gcc-11.4.0/cuda-11.8.0-vfixfmc/lib64:/sw/spack/deltas11-2023-03/apps/linux-rhel8-zen3/gcc-11.4.0/openmpi-4.1.6-lranp74/lib:/sw/spack/deltas11-2023-03/apps/linux-rhel8-x86_64/gcc-8.5.0/gcc-11.4.0-yycklku/lib64:/sw/spack/deltas11-2023-03/apps/linux-rhel8-x86_64/gcc-8.5.0/gcc-11.4.0-yycklku/lib
+LMOD_AVAIL_STYLE=system
+LMOD_CMD=/usr/share/lmod/lmod/libexec/lmod
+LMOD_COLORIZE=yes
+LMOD_DIR=/usr/share/lmod/lmod/libexec
+LMOD_FAMILY_COMPILER=gcc
+LMOD_FAMILY_COMPILER_VERSION=11.4.0
+LMOD_FAMILY_MPI=openmpi
+LMOD_FAMILY_MPI_VERSION=4.1.6
+LMOD_FULL_SETTARG_SUPPORT=no
+LMOD_MPI_NAME=openmpi
+LMOD_MPI_VERSION=4.1.6-lranp74
+LMOD_PKG=/usr/share/lmod/lmod
+LMOD_PREPEND_BLOCK=normal
+LMOD_ROOT=/usr/share/lmod
+LMOD_SETTARG_CMD=:
+LMOD_SETTARG_FULL_SUPPORT=no
+LMOD_SITE_NAME=NCSA
+LMOD_SYSTEM_DEFAULT_MODULES=default-s11
+LMOD_SYSTEM_NAME=Delta
+LMOD_VERSION=8.7.55
+LMOD_arch=x86_64
+LMOD_sys=Linux
+LOADEDMODULES=gcc/11.4.0:openmpi/4.1.6:cuda/11.8.0:cue-login-env/1.1:slurm-env/0.1:default-s11:anaconda3_cpu/23.7.4
+LOCAL_RANK=0
+LOGNAME=cmaduabuchi
+MAIL=/var/spool/mail/cmaduabuchi
+MANPATH=/sw/spack/deltas11-2023-03/apps/linux-rhel8-zen3/gcc-11.4.0/openmpi-4.1.6-lranp74/share/man:/sw/spack/deltas11-2023-03/apps/linux-rhel8-x86_64/gcc-8.5.0/gcc-11.4.0-yycklku/share/man:/usr/share/lmod/lmod/share/man:/usr/local/share/man:/usr/share/man:/opt/ddn/ime/share/man:/opt/puppetlabs/puppet/share/man:/sw/external/python/anaconda3_cpu/man:/sw/external/python/anaconda3/man:/u/cmaduabuchi/.local/share/man:/sw/spack/deltas11-2023-03/apps/linux-rhel8-zen3/gcc-11.4.0/openmpi-4.1.6-lranp74/share/man:/sw/spack/deltas11-2023-03/apps/linux-rhel8-x86_64/gcc-8.5.0/gcc-11.4.0-yycklku/share/man:/usr/local/share/man:/usr/share/man:/opt/ddn/ime/share/man:/sw/external/python/anaconda3/man:/sw/external/python/anaconda3_cpu/man:/u/cmaduabuchi/.local/share/man:/sw/spack/deltas11-2023-03/apps/linux-rhel8-zen3/gcc-11.4.0/openmpi-4.1.6-lranp74/share/man:/sw/spack/deltas11-2023-03/apps/linux-rhel8-x86_64/gcc-8.5.0/gcc-11.4.0-yycklku/share/man:/usr/local/share/man:/usr/share/man:/opt/ddn/ime/share/man:/opt/ddn/ime/share/man:/opt/ddn/ime/share/man:/opt/ddn/ime/share/man:/opt/ddn/ime/share/man
+MASTER_ADDR=localhost
+MASTER_PORT=48488
+MODULEPATH=/sw/spack/deltas11-2023-03/modules/lmod/openmpi/4.1.6-lo6xae6/gcc/11.4.0:/sw/spack/deltas11-2023-03/modules/lmod/openmpi/4.1.6-lranp74/gcc/11.4.0:/sw/spack/deltas11-2023-03/modules/lmod/gcc/11.4.0:/sw/user/modules:/sw/spack/deltas11-2023-03/modules/lmod/Core:/opt/cray/pe/lmod/modulefiles/craype-targets/default:/opt/cray/pe/lmod/modulefiles/core:/opt/cray/modulefiles:/sw/external/lmod/modulefiles/core:/usr/share/lmod/lmod/modulefiles/Core
+MODULEPATH_ROOT=/
+MODULESHOME=/usr/share/lmod/lmod
+MPICC=/sw/spack/deltas11-2023-03/apps/linux-rhel8-zen3/gcc-11.4.0/openmpi-4.1.6-lranp74/bin/mpicc
+MPICXX=/sw/spack/deltas11-2023-03/apps/linux-rhel8-zen3/gcc-11.4.0/openmpi-4.1.6-lranp74/bin/mpic++
+MPIF77=/sw/spack/deltas11-2023-03/apps/linux-rhel8-zen3/gcc-11.4.0/openmpi-4.1.6-lranp74/bin/mpif77
+MPIF90=/sw/spack/deltas11-2023-03/apps/linux-rhel8-zen3/gcc-11.4.0/openmpi-4.1.6-lranp74/bin/mpif90
+NCSA_FAMILY_COMPILER=gcc
+NCSA_FAMILY_COMPILER_VERSION=11.4.0
+NCSA_FAMILY_MPI=openmpi
+NCSA_FAMILY_MPI_VERSION=4.1.6
+NVHPC_CUDA_HOME=/sw/spack/deltas11-2023-03/apps/linux-rhel8-zen3/gcc-11.4.0/cuda-11.8.0-vfixfmc
+OLDPWD=/work/nvme/bfgd/cmaduabuchi/THRIVESEG/sam2
+OMPI_MCA_btl_tcp_endpoint_cache=67108864
+OMPI_MCA_btl_tcp_latency_hsn0=2
+OMPI_MCA_btl_tcp_rdma_pipeline_send_length=1048576
+OMPI_MCA_mtl_ofi_provider_include=cxi
+OMPI_MCA_opal_common_ucx_opal_mem_hooks=1
+OMPI_MCA_opal_warn_on_missing_libcuda=0
+OMPI_MCA_plm_slurm_args=--external-launcher
+OMPI_MCA_ras_base_verbose=10
+OPENMPI_HOME=/sw/spack/deltas11-2023-03/apps/linux-rhel8-zen3/gcc-11.4.0/openmpi-4.1.6-lranp74
+PATH=/u/cmaduabuchi/.vscode-server/cli/servers/Stable-0f0d87fa9e96c856c5212fc86db137ac0d783365/server/bin/remote-cli:/u/cmaduabuchi/.conda/envs/sam2/bin:/sw/external/python/anaconda3/condabin:/sw/external/python/anaconda3_cpu/bin:/u/cmaduabuchi/.local/bin:/u/cmaduabuchi/bin:/sw/spack/deltas11-2023-03/apps/linux-rhel8-zen3/gcc-11.4.0/cuda-11.8.0-vfixfmc/bin:/sw/spack/deltas11-2023-03/apps/linux-rhel8-zen3/gcc-11.4.0/openmpi-4.1.6-lranp74/bin:/sw/spack/deltas11-2023-03/apps/linux-rhel8-x86_64/gcc-8.5.0/gcc-11.4.0-yycklku/bin:/sw/user/scripts:/usr/local/bin:/usr/bin:/usr/local/sbin:/usr/sbin:/opt/ddn/ime/bin:/opt/puppetlabs/bin:/opt/ddn/ime/bin:/u/cmaduabuchi/.vscode-server/extensions/ms-python.debugpy-2025.10.0-linux-x64/bundled/scripts/noConfigScripts:/u/cmaduabuchi/.vscode-server/data/User/globalStorage/github.copilot-chat/debugCommand:/opt/ddn/ime/bin:/opt/ddn/ime/bin:/opt/ddn/ime/bin
+PKG_CONFIG_PATH=/sw/spack/deltas11-2023-03/apps/linux-rhel8-zen3/gcc-11.4.0/openmpi-4.1.6-lranp74/lib/pkgconfig
+PMI_FD=10
+PMI_JOBID=12652122.0
+PMI_RANK=0
+PMI_SIZE=1
+PRTE_MCA_plm_slurm_args=--external-launcher
+PWD=/work/nvme/bfgd/cmaduabuchi/THRIVESEG/sam2
+PYDEVD_DISABLE_FILE_VALIDATION=1
+PYTHONSTARTUP=/u/cmaduabuchi/.vscode-server/data/User/workspaceStorage/e99aa778c8fbcf5603c74656a7713b05/ms-python.python/pythonrc.py
+PYTHON_BASIC_REPL=1
+RANK=0
+ROCR_VISIBLE_DEVICES=0
+SALLOC_KILL_CMD=1
+SHELL=/bin/bash
+SHLVL=8
+SLURMD_DEBUG=2
+SLURMD_NODENAME=gpub060
+SLURM_CLUSTER_NAME=delta
+SLURM_CONF=/var/spool/slurmd/conf-cache/slurm.conf
+SLURM_CPUS_ON_NODE=16
+SLURM_CPUS_PER_GPU=16
+SLURM_CPU_BIND=quiet,mask_cpu:0x0000000000FFFF00
+SLURM_CPU_BIND_LIST=0x0000000000FFFF00
+SLURM_CPU_BIND_TYPE=mask_cpu:
+SLURM_CPU_BIND_VERBOSE=quiet
+SLURM_GPUS_ON_NODE=1
+SLURM_GPUS_PER_NODE=1
+SLURM_GTIDS=0
+SLURM_JOBID=12652122
+SLURM_JOB_ACCOUNT=becs-delta-gpu
+SLURM_JOB_CPUS_PER_NODE=16
+SLURM_JOB_END_TIME=1759922844
+SLURM_JOB_GID=202
+SLURM_JOB_GROUP=grp_202
+SLURM_JOB_ID=12652122
+SLURM_JOB_NAME=demo
+SLURM_JOB_NODELIST=gpub060
+SLURM_JOB_NUM_NODES=1
+SLURM_JOB_PARTITION=gpuA40x4
+SLURM_JOB_QOS=becs-delta-gpu
+SLURM_JOB_START_TIME=1759750044
+SLURM_JOB_UID=77886
+SLURM_JOB_USER=cmaduabuchi
+SLURM_LAUNCH_NODE_IPADDR=141.142.253.42
+SLURM_LOCALID=0
+SLURM_MEM_PER_GPU=81920
+SLURM_MPI_TYPE=pmi2
+SLURM_NNODES=1
+SLURM_NODEID=0
+SLURM_NODELIST=gpub060
+SLURM_NPROCS=1
+SLURM_NTASKS=1
+SLURM_PRIO_PROCESS=0
+SLURM_PROCID=0
+SLURM_PTY_PORT=39771
+SLURM_PTY_WIN_COL=91
+SLURM_PTY_WIN_ROW=69
+SLURM_SRUN_COMM_HOST=141.142.253.42
+SLURM_SRUN_COMM_PORT=43569
+SLURM_STEPID=0
+SLURM_STEP_GPUS=1
+SLURM_STEP_ID=0
+SLURM_STEP_LAUNCHER_PORT=43569
+SLURM_STEP_NODELIST=gpub060
+SLURM_STEP_NUM_NODES=1
+SLURM_STEP_NUM_TASKS=1
+SLURM_STEP_TASKS_PER_NODE=1
+SLURM_SUBMIT_DIR=/work/nvme/bfgd/cmaduabuchi
+SLURM_SUBMIT_HOST=dt-login03.delta.ncsa.illinois.edu
+SLURM_TASKS_PER_NODE=1
+SLURM_TASK_PID=916888
+SLURM_TOPOLOGY_ADDR=ss00.ss11.gpub060
+SLURM_TOPOLOGY_ADDR_PATTERN=switch.switch.node
+SLURM_UMASK=0077
+SQUEUE_FORMAT=%.12i %.12P %.12j %.14u %.2t %.10M %.6D %.18R %f
+SRUN_DEBUG=3
+SSH_CLIENT=128.195.95.47 32161 22
+SSH_CONNECTION=169.234.226.25 58665 141.142.140.196 22
+SSL_CERT_DIR=/etc/pki/tls/certs
+SSL_CERT_FILE=/etc/pki/ca-trust/extracted/pem/tls-ca-bundle.pem
+TERM=screen
+TERM_PROGRAM=vscode
+TERM_PROGRAM_VERSION=1.104.1
+TMOUT=86400
+TMPDIR=/tmp
+TMUX=/tmp/tmux-77886/default,3561957,1
+TMUX_PANE=%1
+TORCH_NCCL_ASYNC_ERROR_HANDLING=1
+USER=cmaduabuchi
+VSCODE_DEBUGPY_ADAPTER_ENDPOINTS=/u/cmaduabuchi/.vscode-server/extensions/ms-python.debugpy-2025.10.0-linux-x64/.noConfigDebugAdapterEndpoints/endpoint-ad8f8f00eace4009.txt
+VSCODE_GIT_ASKPASS_EXTRA_ARGS=
+VSCODE_GIT_ASKPASS_MAIN=/u/cmaduabuchi/.vscode-server/cli/servers/Stable-0f0d87fa9e96c856c5212fc86db137ac0d783365/server/extensions/git/dist/askpass-main.js
+VSCODE_GIT_ASKPASS_NODE=/u/cmaduabuchi/.vscode-server/cli/servers/Stable-0f0d87fa9e96c856c5212fc86db137ac0d783365/server/node
+VSCODE_GIT_IPC_HANDLE=/run/user/77886/vscode-git-922d1ac20a.sock
+VSCODE_IPC_HOOK_CLI=/run/user/77886/vscode-ipc-4d775fc7-d69f-439f-a012-2b869d146d9e.sock
+VSCODE_PYTHON_AUTOACTIVATE_GUARD=1
+WORLD_SIZE=1
+XDG_RUNTIME_DIR=/run/user/77886
+XDG_SESSION_ID=114322
+ZE_AFFINITY_MASK=0
+_=/u/cmaduabuchi/.conda/envs/sam2/bin/python
+_CE_CONDA=
+_CE_M=
+_LMFILES_=/sw/spack/deltas11-2023-03/modules/lmod/Core/gcc/11.4.0.lua:/sw/spack/deltas11-2023-03/modules/lmod/gcc/11.4.0/openmpi/4.1.6.lua:/sw/spack/deltas11-2023-03/modules/lmod/gcc/11.4.0/cuda/11.8.0.lua:/sw/user/modules/cue-login-env/1.1.lua:/sw/user/modules/slurm-env/0.1.lua:/sw/user/modules/default-s11.lua:/sw/user/modules/anaconda3_cpu/23.7.4.lua
+_ModuleTable001_=X01vZHVsZVRhYmxlXyA9IHsKTVR2ZXJzaW9uID0gMywKY19yZWJ1aWxkVGltZSA9IGZhbHNlLApjX3Nob3J0VGltZSA9IGZhbHNlLApkZXB0aFQgPSB7fSwKZmFtaWx5ID0gewpjb21waWxlciA9ICJnY2MiLAptcGkgPSAib3Blbm1waSIsCn0sCm1UID0gewphbmFjb25kYTNfY3B1ID0gewpmbiA9ICIvc3cvdXNlci9tb2R1bGVzL2FuYWNvbmRhM19jcHUvMjMuNy40Lmx1YSIsCmZ1bGxOYW1lID0gImFuYWNvbmRhM19jcHUvMjMuNy40IiwKbG9hZE9yZGVyID0gNywKcHJvcFQgPSB7fSwKc3RhY2tEZXB0aCA9IDAsCnN0YXR1cyA9ICJhY3RpdmUiLAp1c2VyTmFtZSA9ICJhbmFjb25kYTNfY3B1LzIzLjcuNCIsCndWID0gIl4wMDAwMDAyMy4wMDAwMDAwMDcuMDAwMDAwMDA0Lip6
+_ModuleTable002_=ZmluYWwiLAp9LApjdWRhID0gewpmbiA9ICIvc3cvc3BhY2svZGVsdGFzMTEtMjAyMy0wMy9tb2R1bGVzL2xtb2QvZ2NjLzExLjQuMC9jdWRhLzExLjguMC5sdWEiLApmdWxsTmFtZSA9ICJjdWRhLzExLjguMCIsCmxvYWRPcmRlciA9IDMsCnByb3BUID0ge30sCnN0YWNrRGVwdGggPSAxLApzdGF0dXMgPSAiYWN0aXZlIiwKdXNlck5hbWUgPSAiY3VkYSIsCndWID0gIl4wMDAwMDAxMS4wMDAwMDAwMDguKnpmaW5hbCIsCn0sClsiY3VlLWxvZ2luLWVudiJdID0gewpmbiA9ICIvc3cvdXNlci9tb2R1bGVzL2N1ZS1sb2dpbi1lbnYvMS4xLmx1YSIsCmZ1bGxOYW1lID0gImN1ZS1sb2dpbi1lbnYvMS4xIiwKbG9hZE9yZGVyID0gNCwKcHJvcFQgPSB7fSwKc3RhY2tEZXB0aCA9IDEs
+_ModuleTable003_=CnN0YXR1cyA9ICJhY3RpdmUiLAp1c2VyTmFtZSA9ICJjdWUtbG9naW4tZW52IiwKd1YgPSAiMDAwMDAwMDAxLjAwMDAwMDAwMS4qemZpbmFsIiwKfSwKWyJkZWZhdWx0LXMxMSJdID0gewpmbiA9ICIvc3cvdXNlci9tb2R1bGVzL2RlZmF1bHQtczExLmx1YSIsCmZ1bGxOYW1lID0gImRlZmF1bHQtczExIiwKbG9hZE9yZGVyID0gNiwKcHJvcFQgPSB7fSwKc3RhY2tEZXB0aCA9IDAsCnN0YXR1cyA9ICJhY3RpdmUiLAp1c2VyTmFtZSA9ICJkZWZhdWx0LXMxMSIsCndWID0gIk0uKnpmaW5hbCIsCn0sCmdjYyA9IHsKYWN0aW9uQSA9IHsKInByZXBlbmRfcGF0aChcIk1PRFVMRVBBVEhcIixcIi9zdy9zcGFjay9kZWx0YXMxMS0yMDIzLTAzL21vZHVsZXMvbG1vZC9nY2MvMTEuNC4w
+_ModuleTable004_=XCIpIiwKfSwKZm4gPSAiL3N3L3NwYWNrL2RlbHRhczExLTIwMjMtMDMvbW9kdWxlcy9sbW9kL0NvcmUvZ2NjLzExLjQuMC5sdWEiLApmdWxsTmFtZSA9ICJnY2MvMTEuNC4wIiwKbG9hZE9yZGVyID0gMSwKcHJvcFQgPSB7fSwKc3RhY2tEZXB0aCA9IDEsCnN0YXR1cyA9ICJhY3RpdmUiLAp1c2VyTmFtZSA9ICJnY2MiLAp3ViA9ICJeMDAwMDAwMTEuMDAwMDAwMDA0Lip6ZmluYWwiLAp9LApvcGVubXBpID0gewphY3Rpb25BID0gewoKInByZXBlbmRfcGF0aChcIk1PRFVMRVBBVEhcIixcIi9zdy9zcGFjay9kZWx0YXMxMS0yMDIzLTAzL21vZHVsZXMvbG1vZC9vcGVubXBpLzQuMS42LWxyYW5wNzQvZ2NjLzExLjQuMFwiKSIsICJwcmVwZW5kX3BhdGgoXCJNT0RVTEVQQVRIXCIs
+_ModuleTable005_=XCIvc3cvc3BhY2svZGVsdGFzMTEtMjAyMy0wMy9tb2R1bGVzL2xtb2Qvb3Blbm1waS80LjEuNi1sbzZ4YWU2L2djYy8xMS40LjBcIikiLAp9LApmbiA9ICIvc3cvc3BhY2svZGVsdGFzMTEtMjAyMy0wMy9tb2R1bGVzL2xtb2QvZ2NjLzExLjQuMC9vcGVubXBpLzQuMS42Lmx1YSIsCmZ1bGxOYW1lID0gIm9wZW5tcGkvNC4xLjYiLApsb2FkT3JkZXIgPSAyLApwcm9wVCA9IHt9LApzdGFja0RlcHRoID0gMSwKc3RhdHVzID0gImFjdGl2ZSIsCnVzZXJOYW1lID0gIm9wZW5tcGkiLAp3ViA9ICJeMDAwMDAwMDQuMDAwMDAwMDAxLjAwMDAwMDAwNi4qemZpbmFsIiwKfSwKWyJzbHVybS1lbnYiXSA9IHsKZm4gPSAiL3N3L3VzZXIvbW9kdWxlcy9zbHVybS1lbnYvMC4xLmx1YSIsCmZ1
+_ModuleTable006_=bGxOYW1lID0gInNsdXJtLWVudi8wLjEiLApsb2FkT3JkZXIgPSA1LApwcm9wVCA9IHt9LApzdGFja0RlcHRoID0gMSwKc3RhdHVzID0gImFjdGl2ZSIsCnVzZXJOYW1lID0gInNsdXJtLWVudiIsCndWID0gIjAwMDAwMDAwMC4wMDAwMDAwMDEuKnpmaW5hbCIsCn0sCn0sCm1wYXRoQSA9IHsKIi9zdy9zcGFjay9kZWx0YXMxMS0yMDIzLTAzL21vZHVsZXMvbG1vZC9vcGVubXBpLzQuMS42LWxvNnhhZTYvZ2NjLzExLjQuMCIKLCAiL3N3L3NwYWNrL2RlbHRhczExLTIwMjMtMDMvbW9kdWxlcy9sbW9kL29wZW5tcGkvNC4xLjYtbHJhbnA3NC9nY2MvMTEuNC4wIgosICIvc3cvc3BhY2svZGVsdGFzMTEtMjAyMy0wMy9tb2R1bGVzL2xtb2QvZ2NjLzExLjQuMCIsICIvc3cvdXNlci9t
+_ModuleTable007_=b2R1bGVzIgosICIvc3cvc3BhY2svZGVsdGFzMTEtMjAyMy0wMy9tb2R1bGVzL2xtb2QvQ29yZSIKLCAiL29wdC9jcmF5L3BlL2xtb2QvbW9kdWxlZmlsZXMvY3JheXBlLXRhcmdldHMvZGVmYXVsdCIKLCAiL29wdC9jcmF5L3BlL2xtb2QvbW9kdWxlZmlsZXMvY29yZSIsICIvb3B0L2NyYXkvbW9kdWxlZmlsZXMiCiwgIi9zdy9leHRlcm5hbC9sbW9kL21vZHVsZWZpbGVzL2NvcmUiLCAiL3Vzci9zaGFyZS9sbW9kL2xtb2QvbW9kdWxlZmlsZXMvQ29yZSIsCn0sCnN5c3RlbUJhc2VNUEFUSCA9ICIvc3cvdXNlci9tb2R1bGVzOi9zdy9zcGFjay9kZWx0YXMxMS0yMDIzLTAzL21vZHVsZXMvbG1vZC9Db3JlOi9vcHQvY3JheS9wZS9sbW9kL21vZHVsZWZpbGVzL2NyYXlwZS10YXJn
+_ModuleTable008_=ZXRzL2RlZmF1bHQ6L29wdC9jcmF5L3BlL2xtb2QvbW9kdWxlZmlsZXMvY29yZTovb3B0L2NyYXkvbW9kdWxlZmlsZXM6L3N3L2V4dGVybmFsL2xtb2QvbW9kdWxlZmlsZXMvY29yZTovdXNyL3NoYXJlL2xtb2QvbG1vZC9tb2R1bGVmaWxlcy9Db3JlIiwKfQo=
+_ModuleTable_Sz_=8
+__Init_Default_Modules=1
+__LMOD_REF_COUNT_CMAKE_PREFIX_PATH=/sw/spack/deltas11-2023-03/apps/linux-rhel8-zen3/gcc-11.4.0/cuda-11.8.0-vfixfmc:2;/sw/spack/deltas11-2023-03/apps/linux-rhel8-zen3/gcc-11.4.0/openmpi-4.1.6-lranp74:2;/sw/spack/deltas11-2023-03/apps/linux-rhel8-x86_64/gcc-8.5.0/gcc-11.4.0-yycklku:2
+__LMOD_REF_COUNT_CPLUS_INCLUDE_PATH=/sw/spack/deltas11-2023-03/apps/linux-rhel8-zen3/gcc-11.4.0/cuda-11.8.0-vfixfmc/include:1;/sw/spack/deltas11-2023-03/apps/linux-rhel8-zen3/gcc-11.4.0/openmpi-4.1.6-lranp74/include:1;/sw/spack/deltas11-2023-03/apps/linux-rhel8-x86_64/gcc-8.5.0/gcc-11.4.0-yycklku/include:1
+__LMOD_REF_COUNT_C_INCLUDE_PATH=/sw/spack/deltas11-2023-03/apps/linux-rhel8-zen3/gcc-11.4.0/cuda-11.8.0-vfixfmc/include:1;/sw/spack/deltas11-2023-03/apps/linux-rhel8-zen3/gcc-11.4.0/openmpi-4.1.6-lranp74/include:1;/sw/spack/deltas11-2023-03/apps/linux-rhel8-x86_64/gcc-8.5.0/gcc-11.4.0-yycklku/include:1
+__LMOD_REF_COUNT_INCLUDE=/sw/spack/deltas11-2023-03/apps/linux-rhel8-zen3/gcc-11.4.0/cuda-11.8.0-vfixfmc/include:1;/sw/spack/deltas11-2023-03/apps/linux-rhel8-zen3/gcc-11.4.0/openmpi-4.1.6-lranp74/include:1;/sw/spack/deltas11-2023-03/apps/linux-rhel8-x86_64/gcc-8.5.0/gcc-11.4.0-yycklku/include:1
+__LMOD_REF_COUNT_LD_LIBRARY_PATH=/sw/spack/delta-2022-03/apps/gcc/11.2.0-gcc-8.4.1-fxgnsyr/lib64:1;/sw/spack/deltas11-2023-03/apps/linux-rhel8-zen3/gcc-11.4.0/cuda-11.8.0-vfixfmc/lib64:1;/opt/cray/libfabric/1.15.2.0/lib64:1;/opt/cray/libfabric/1.15.2.0/lib:1
+__LMOD_REF_COUNT_LIBRARY_PATH=/sw/spack/deltas11-2023-03/apps/linux-rhel8-zen3/gcc-11.4.0/cuda-11.8.0-vfixfmc/lib64:1;/sw/spack/deltas11-2023-03/apps/linux-rhel8-zen3/gcc-11.4.0/openmpi-4.1.6-lranp74/lib:1;/sw/spack/deltas11-2023-03/apps/linux-rhel8-x86_64/gcc-8.5.0/gcc-11.4.0-yycklku/lib64:1;/sw/spack/deltas11-2023-03/apps/linux-rhel8-x86_64/gcc-8.5.0/gcc-11.4.0-yycklku/lib:1
+__LMOD_REF_COUNT_MANPATH=/sw/spack/deltas11-2023-03/apps/linux-rhel8-zen3/gcc-11.4.0/openmpi-4.1.6-lranp74/share/man:2;/sw/spack/deltas11-2023-03/apps/linux-rhel8-x86_64/gcc-8.5.0/gcc-11.4.0-yycklku/share/man:2;/usr/share/lmod/lmod/share/man:1;/usr/local/share/man:1;/usr/share/man:1;/opt/ddn/ime/share/man:1;/opt/puppetlabs/puppet/share/man:1;:2
+__LMOD_REF_COUNT_MODULEPATH=/sw/spack/deltas11-2023-03/modules/lmod/openmpi/4.1.6-lo6xae6/gcc/11.4.0:1;/sw/spack/deltas11-2023-03/modules/lmod/openmpi/4.1.6-lranp74/gcc/11.4.0:1;/sw/spack/deltas11-2023-03/modules/lmod/gcc/11.4.0:1;/sw/user/modules:1;/sw/spack/deltas11-2023-03/modules/lmod/Core:1;/opt/cray/pe/lmod/modulefiles/craype-targets/default:1;/opt/cray/pe/lmod/modulefiles/core:1;/opt/cray/modulefiles:1;/sw/external/lmod/modulefiles/core:1;/usr/share/lmod/lmod/modulefiles/Core:1
+__LMOD_REF_COUNT_PATH=/sw/external/python/anaconda3_cpu/bin:1;/u/cmaduabuchi/.vscode-server/cli/servers/Stable-0f0d87fa9e96c856c5212fc86db137ac0d783365/server/bin/remote-cli:1;/sw/external/python/anaconda3/bin:1;/sw/external/python/anaconda3/condabin:1;/u/cmaduabuchi/.local/bin:1;/u/cmaduabuchi/bin:1;/sw/spack/deltas11-2023-03/apps/linux-rhel8-zen3/gcc-11.4.0/cuda-11.8.0-vfixfmc/bin:2;/sw/spack/deltas11-2023-03/apps/linux-rhel8-zen3/gcc-11.4.0/openmpi-4.1.6-lranp74/bin:2;/sw/spack/deltas11-2023-03/apps/linux-rhel8-x86_64/gcc-8.5.0/gcc-11.4.0-yycklku/bin:2;/sw/user/scripts:1;/usr/local/bin:1;/usr/bin:1;/usr/local/sbin:1;/usr/sbin:1;/opt/ddn/ime/bin:3;/opt/puppetlabs/bin:1;/u/cmaduabuchi/.vscode-server/extensions/ms-python.debugpy-2025.10.0-linux-x64/bundled/scripts/noConfigScripts:1;/u/cmaduabuchi/.vscode-server/data/User/globalStorage/github.copilot-chat/debugCommand:1
+__LMOD_REF_COUNT_PKG_CONFIG_PATH=/sw/spack/deltas11-2023-03/apps/linux-rhel8-zen3/gcc-11.4.0/openmpi-4.1.6-lranp74/lib/pkgconfig:2
+which_declare=declare -f
+
+INFO 2025-10-06 06:35:10,389 trainer.py: 989: Setting up components: Model, loss, optim, meters etc.
+INFO 2025-10-06 06:35:10,398 logger.py:  66: TensorBoard SummaryWriter instantiated. Files will be stored in: /work/nvme/bfgd/cmaduabuchi/MODELS/THRIVESEG/experiment/thermal_run1/tensorboard
+INFO 2025-10-06 06:35:11,529 sam2.py:  81: Training with points (sampled from masks) as inputs with p=0.5
+INFO 2025-10-06 06:35:11,532 trainer.py:1059: ====================
+INFO 2025-10-06 06:35:11,532 trainer.py:1060: Summary for model <class 'training.model.sam2.SAM2Train'>
+INFO 2025-10-06 06:35:11,534 trainer.py:1061: Model is SAM2Train(
+  (image_encoder): ImageEncoder(
+    (trunk): Hiera(
+      (patch_embed): PatchEmbed(
+        (proj): Conv2d(3, 112, kernel_size=(7, 7), stride=(4, 4), padding=(3, 3))
+      )
+      (blocks): ModuleList(
+        (0): MultiScaleBlock(
+          (norm1): LayerNorm((112,), eps=1e-06, elementwise_affine=True)
+          (attn): MultiScaleAttention(
+            (qkv): Linear(in_features=112, out_features=336, bias=True)
+            (proj): Linear(in_features=112, out_features=112, bias=True)
+          )
+          (drop_path): Identity()
+          (norm2): LayerNorm((112,), eps=1e-06, elementwise_affine=True)
+          (mlp): MLP(
+            (layers): ModuleList(
+              (0): Linear(in_features=112, out_features=448, bias=True)
+              (1): Linear(in_features=448, out_features=112, bias=True)
+            )
+            (act): GELU(approximate='none')
+          )
+        )
+        (1): MultiScaleBlock(
+          (norm1): LayerNorm((112,), eps=1e-06, elementwise_affine=True)
+          (attn): MultiScaleAttention(
+            (qkv): Linear(in_features=112, out_features=336, bias=True)
+            (proj): Linear(in_features=112, out_features=112, bias=True)
+          )
+          (drop_path): DropPath()
+          (norm2): LayerNorm((112,), eps=1e-06, elementwise_affine=True)
+          (mlp): MLP(
+            (layers): ModuleList(
+              (0): Linear(in_features=112, out_features=448, bias=True)
+              (1): Linear(in_features=448, out_features=112, bias=True)
+            )
+            (act): GELU(approximate='none')
+          )
+        )
+        (2): MultiScaleBlock(
+          (norm1): LayerNorm((112,), eps=1e-06, elementwise_affine=True)
+          (pool): MaxPool2d(kernel_size=(2, 2), stride=(2, 2), padding=0, dilation=1, ceil_mode=False)
+          (attn): MultiScaleAttention(
+            (q_pool): MaxPool2d(kernel_size=(2, 2), stride=(2, 2), padding=0, dilation=1, ceil_mode=False)
+            (qkv): Linear(in_features=112, out_features=672, bias=True)
+            (proj): Linear(in_features=224, out_features=224, bias=True)
+          )
+          (drop_path): DropPath()
+          (norm2): LayerNorm((224,), eps=1e-06, elementwise_affine=True)
+          (mlp): MLP(
+            (layers): ModuleList(
+              (0): Linear(in_features=224, out_features=896, bias=True)
+              (1): Linear(in_features=896, out_features=224, bias=True)
+            )
+            (act): GELU(approximate='none')
+          )
+          (proj): Linear(in_features=112, out_features=224, bias=True)
+        )
+        (3-4): 2 x MultiScaleBlock(
+          (norm1): LayerNorm((224,), eps=1e-06, elementwise_affine=True)
+          (attn): MultiScaleAttention(
+            (qkv): Linear(in_features=224, out_features=672, bias=True)
+            (proj): Linear(in_features=224, out_features=224, bias=True)
+          )
+          (drop_path): DropPath()
+          (norm2): LayerNorm((224,), eps=1e-06, elementwise_affine=True)
+          (mlp): MLP(
+            (layers): ModuleList(
+              (0): Linear(in_features=224, out_features=896, bias=True)
+              (1): Linear(in_features=896, out_features=224, bias=True)
+            )
+            (act): GELU(approximate='none')
+          )
+        )
+        (5): MultiScaleBlock(
+          (norm1): LayerNorm((224,), eps=1e-06, elementwise_affine=True)
+          (pool): MaxPool2d(kernel_size=(2, 2), stride=(2, 2), padding=0, dilation=1, ceil_mode=False)
+          (attn): MultiScaleAttention(
+            (q_pool): MaxPool2d(kernel_size=(2, 2), stride=(2, 2), padding=0, dilation=1, ceil_mode=False)
+            (qkv): Linear(in_features=224, out_features=1344, bias=True)
+            (proj): Linear(in_features=448, out_features=448, bias=True)
+          )
+          (drop_path): DropPath()
+          (norm2): LayerNorm((448,), eps=1e-06, elementwise_affine=True)
+          (mlp): MLP(
+            (layers): ModuleList(
+              (0): Linear(in_features=448, out_features=1792, bias=True)
+              (1): Linear(in_features=1792, out_features=448, bias=True)
+            )
+            (act): GELU(approximate='none')
+          )
+          (proj): Linear(in_features=224, out_features=448, bias=True)
+        )
+        (6-20): 15 x MultiScaleBlock(
+          (norm1): LayerNorm((448,), eps=1e-06, elementwise_affine=True)
+          (attn): MultiScaleAttention(
+            (qkv): Linear(in_features=448, out_features=1344, bias=True)
+            (proj): Linear(in_features=448, out_features=448, bias=True)
+          )
+          (drop_path): DropPath()
+          (norm2): LayerNorm((448,), eps=1e-06, elementwise_affine=True)
+          (mlp): MLP(
+            (layers): ModuleList(
+              (0): Linear(in_features=448, out_features=1792, bias=True)
+              (1): Linear(in_features=1792, out_features=448, bias=True)
+            )
+            (act): GELU(approximate='none')
+          )
+        )
+        (21): MultiScaleBlock(
+          (norm1): LayerNorm((448,), eps=1e-06, elementwise_affine=True)
+          (pool): MaxPool2d(kernel_size=(2, 2), stride=(2, 2), padding=0, dilation=1, ceil_mode=False)
+          (attn): MultiScaleAttention(
+            (q_pool): MaxPool2d(kernel_size=(2, 2), stride=(2, 2), padding=0, dilation=1, ceil_mode=False)
+            (qkv): Linear(in_features=448, out_features=2688, bias=True)
+            (proj): Linear(in_features=896, out_features=896, bias=True)
+          )
+          (drop_path): DropPath()
+          (norm2): LayerNorm((896,), eps=1e-06, elementwise_affine=True)
+          (mlp): MLP(
+            (layers): ModuleList(
+              (0): Linear(in_features=896, out_features=3584, bias=True)
+              (1): Linear(in_features=3584, out_features=896, bias=True)
+            )
+            (act): GELU(approximate='none')
+          )
+          (proj): Linear(in_features=448, out_features=896, bias=True)
+        )
+        (22-23): 2 x MultiScaleBlock(
+          (norm1): LayerNorm((896,), eps=1e-06, elementwise_affine=True)
+          (attn): MultiScaleAttention(
+            (qkv): Linear(in_features=896, out_features=2688, bias=True)
+            (proj): Linear(in_features=896, out_features=896, bias=True)
+          )
+          (drop_path): DropPath()
+          (norm2): LayerNorm((896,), eps=1e-06, elementwise_affine=True)
+          (mlp): MLP(
+            (layers): ModuleList(
+              (0): Linear(in_features=896, out_features=3584, bias=True)
+              (1): Linear(in_features=3584, out_features=896, bias=True)
+            )
+            (act): GELU(approximate='none')
+          )
+        )
+      )
+    )
+    (neck): FpnNeck(
+      (position_encoding): PositionEmbeddingSine()
+      (convs): ModuleList(
+        (0): Sequential(
+          (conv): Conv2d(896, 256, kernel_size=(1, 1), stride=(1, 1))
+        )
+        (1): Sequential(
+          (conv): Conv2d(448, 256, kernel_size=(1, 1), stride=(1, 1))
+        )
+        (2): Sequential(
+          (conv): Conv2d(224, 256, kernel_size=(1, 1), stride=(1, 1))
+        )
+        (3): Sequential(
+          (conv): Conv2d(112, 256, kernel_size=(1, 1), stride=(1, 1))
+        )
+      )
+    )
+  )
+  (mask_downsample): Conv2d(1, 1, kernel_size=(4, 4), stride=(4, 4))
+  (memory_attention): MemoryAttention(
+    (layers): ModuleList(
+      (0-3): 4 x MemoryAttentionLayer(
+        (self_attn): RoPEAttention(
+          (q_proj): Linear(in_features=256, out_features=256, bias=True)
+          (k_proj): Linear(in_features=256, out_features=256, bias=True)
+          (v_proj): Linear(in_features=256, out_features=256, bias=True)
+          (out_proj): Linear(in_features=256, out_features=256, bias=True)
+        )
+        (cross_attn_image): RoPEAttention(
+          (q_proj): Linear(in_features=256, out_features=256, bias=True)
+          (k_proj): Linear(in_features=64, out_features=256, bias=True)
+          (v_proj): Linear(in_features=64, out_features=256, bias=True)
+          (out_proj): Linear(in_features=256, out_features=256, bias=True)
+        )
+        (linear1): Linear(in_features=256, out_features=2048, bias=True)
+        (dropout): Dropout(p=0.1, inplace=False)
+        (linear2): Linear(in_features=2048, out_features=256, bias=True)
+        (norm1): LayerNorm((256,), eps=1e-05, elementwise_affine=True)
+        (norm2): LayerNorm((256,), eps=1e-05, elementwise_affine=True)
+        (norm3): LayerNorm((256,), eps=1e-05, elementwise_affine=True)
+        (dropout1): Dropout(p=0.1, inplace=False)
+        (dropout2): Dropout(p=0.1, inplace=False)
+        (dropout3): Dropout(p=0.1, inplace=False)
+      )
+    )
+    (norm): LayerNorm((256,), eps=1e-05, elementwise_affine=True)
+  )
+  (memory_encoder): MemoryEncoder(
+    (mask_downsampler): MaskDownSampler(
+      (encoder): Sequential(
+        (0): Conv2d(1, 4, kernel_size=(3, 3), stride=(2, 2), padding=(1, 1))
+        (1): LayerNorm2d()
+        (2): GELU(approximate='none')
+        (3): Conv2d(4, 16, kernel_size=(3, 3), stride=(2, 2), padding=(1, 1))
+        (4): LayerNorm2d()
+        (5): GELU(approximate='none')
+        (6): Conv2d(16, 64, kernel_size=(3, 3), stride=(2, 2), padding=(1, 1))
+        (7): LayerNorm2d()
+        (8): GELU(approximate='none')
+        (9): Conv2d(64, 256, kernel_size=(3, 3), stride=(2, 2), padding=(1, 1))
+        (10): LayerNorm2d()
+        (11): GELU(approximate='none')
+        (12): Conv2d(256, 256, kernel_size=(1, 1), stride=(1, 1))
+      )
+    )
+    (pix_feat_proj): Conv2d(256, 256, kernel_size=(1, 1), stride=(1, 1))
+    (fuser): Fuser(
+      (proj): Identity()
+      (layers): ModuleList(
+        (0-1): 2 x CXBlock(
+          (dwconv): Conv2d(256, 256, kernel_size=(7, 7), stride=(1, 1), padding=(3, 3), groups=256)
+          (norm): LayerNorm2d()
+          (pwconv1): Linear(in_features=256, out_features=1024, bias=True)
+          (act): GELU(approximate='none')
+          (pwconv2): Linear(in_features=1024, out_features=256, bias=True)
+          (drop_path): Identity()
+        )
+      )
+    )
+    (position_encoding): PositionEmbeddingSine()
+    (out_proj): Conv2d(256, 64, kernel_size=(1, 1), stride=(1, 1))
+  )
+  (sam_prompt_encoder): PromptEncoder(
+    (pe_layer): PositionEmbeddingRandom()
+    (point_embeddings): ModuleList(
+      (0-3): 4 x Embedding(1, 256)
+    )
+    (not_a_point_embed): Embedding(1, 256)
+    (mask_downscaling): Sequential(
+      (0): Conv2d(1, 4, kernel_size=(2, 2), stride=(2, 2))
+      (1): LayerNorm2d()
+      (2): GELU(approximate='none')
+      (3): Conv2d(4, 16, kernel_size=(2, 2), stride=(2, 2))
+      (4): LayerNorm2d()
+      (5): GELU(approximate='none')
+      (6): Conv2d(16, 256, kernel_size=(1, 1), stride=(1, 1))
+    )
+    (no_mask_embed): Embedding(1, 256)
+  )
+  (sam_mask_decoder): MaskDecoder(
+    (transformer): TwoWayTransformer(
+      (layers): ModuleList(
+        (0-1): 2 x TwoWayAttentionBlock(
+          (self_attn): Attention(
+            (q_proj): Linear(in_features=256, out_features=256, bias=True)
+            (k_proj): Linear(in_features=256, out_features=256, bias=True)
+            (v_proj): Linear(in_features=256, out_features=256, bias=True)
+            (out_proj): Linear(in_features=256, out_features=256, bias=True)
+          )
+          (norm1): LayerNorm((256,), eps=1e-05, elementwise_affine=True)
+          (cross_attn_token_to_image): Attention(
+            (q_proj): Linear(in_features=256, out_features=128, bias=True)
+            (k_proj): Linear(in_features=256, out_features=128, bias=True)
+            (v_proj): Linear(in_features=256, out_features=128, bias=True)
+            (out_proj): Linear(in_features=128, out_features=256, bias=True)
+          )
+          (norm2): LayerNorm((256,), eps=1e-05, elementwise_affine=True)
+          (mlp): MLP(
+            (layers): ModuleList(
+              (0): Linear(in_features=256, out_features=2048, bias=True)
+              (1): Linear(in_features=2048, out_features=256, bias=True)
+            )
+            (act): ReLU()
+          )
+          (norm3): LayerNorm((256,), eps=1e-05, elementwise_affine=True)
+          (norm4): LayerNorm((256,), eps=1e-05, elementwise_affine=True)
+          (cross_attn_image_to_token): Attention(
+            (q_proj): Linear(in_features=256, out_features=128, bias=True)
+            (k_proj): Linear(in_features=256, out_features=128, bias=True)
+            (v_proj): Linear(in_features=256, out_features=128, bias=True)
+            (out_proj): Linear(in_features=128, out_features=256, bias=True)
+          )
+        )
+      )
+      (final_attn_token_to_image): Attention(
+        (q_proj): Linear(in_features=256, out_features=128, bias=True)
+        (k_proj): Linear(in_features=256, out_features=128, bias=True)
+        (v_proj): Linear(in_features=256, out_features=128, bias=True)
+        (out_proj): Linear(in_features=128, out_features=256, bias=True)
+      )
+      (norm_final_attn): LayerNorm((256,), eps=1e-05, elementwise_affine=True)
+    )
+    (iou_token): Embedding(1, 256)
+    (mask_tokens): Embedding(4, 256)
+    (obj_score_token): Embedding(1, 256)
+    (output_upscaling): Sequential(
+      (0): ConvTranspose2d(256, 64, kernel_size=(2, 2), stride=(2, 2))
+      (1): LayerNorm2d()
+      (2): GELU(approximate='none')
+      (3): ConvTranspose2d(64, 32, kernel_size=(2, 2), stride=(2, 2))
+      (4): GELU(approximate='none')
+    )
+    (conv_s0): Conv2d(256, 32, kernel_size=(1, 1), stride=(1, 1))
+    (conv_s1): Conv2d(256, 64, kernel_size=(1, 1), stride=(1, 1))
+    (output_hypernetworks_mlps): ModuleList(
+      (0-3): 4 x MLP(
+        (layers): ModuleList(
+          (0-1): 2 x Linear(in_features=256, out_features=256, bias=True)
+          (2): Linear(in_features=256, out_features=32, bias=True)
+        )
+        (act): ReLU()
+      )
+    )
+    (iou_prediction_head): MLP(
+      (layers): ModuleList(
+        (0-1): 2 x Linear(in_features=256, out_features=256, bias=True)
+        (2): Linear(in_features=256, out_features=4, bias=True)
+      )
+      (act): ReLU()
+    )
+    (pred_obj_score_head): MLP(
+      (layers): ModuleList(
+        (0-1): 2 x Linear(in_features=256, out_features=256, bias=True)
+        (2): Linear(in_features=256, out_features=1, bias=True)
+      )
+      (act): ReLU()
+    )
+  )
+  (obj_ptr_proj): MLP(
+    (layers): ModuleList(
+      (0-2): 3 x Linear(in_features=256, out_features=256, bias=True)
+    )
+    (act): ReLU()
+  )
+  (obj_ptr_tpos_proj): Linear(in_features=256, out_features=64, bias=True)
+)
+INFO 2025-10-06 06:35:11,535 trainer.py:1062: 	Total parameters 80.9 M
+INFO 2025-10-06 06:35:11,535 trainer.py:1063: 	Trainable parameters 80.9 M
+INFO 2025-10-06 06:35:11,535 trainer.py:1066: 	Non-Trainable parameters 0  
+INFO 2025-10-06 06:35:11,535 trainer.py:1069: ====================
+INFO 2025-10-06 06:35:11,540 trainer.py:1023: Finished setting up components: Model, loss, optim, meters etc.
+INFO 2025-10-06 06:35:11,540 trainer.py: 314: Moving components to device cuda:0 and local rank 0.
+INFO 2025-10-06 06:35:11,614 trainer.py: 320: Done moving components to device cuda:0 and local rank 0.
+INFO 2025-10-06 06:35:11,631 optimizer.py: 248: Matches for param_name [image_encoder.*]: {'image_encoder.trunk.blocks.3.mlp.layers.0.bias', 'image_encoder.neck.convs.0.conv.weight', 'image_encoder.trunk.blocks.19.mlp.layers.1.bias', 'image_encoder.trunk.blocks.18.mlp.layers.0.weight', 'image_encoder.trunk.blocks.21.attn.proj.weight', 'image_encoder.trunk.blocks.15.attn.qkv.bias', 'image_encoder.trunk.blocks.21.mlp.layers.0.bias', 'image_encoder.trunk.blocks.9.norm1.bias', 'image_encoder.neck.convs.2.conv.weight', 'image_encoder.trunk.blocks.2.norm1.bias', 'image_encoder.trunk.blocks.5.attn.proj.weight', 'image_encoder.trunk.blocks.16.norm2.weight', 'image_encoder.trunk.blocks.0.mlp.layers.1.weight', 'image_encoder.trunk.blocks.4.norm1.weight', 'image_encoder.trunk.blocks.13.mlp.layers.1.weight', 'image_encoder.trunk.blocks.8.norm1.bias', 'image_encoder.trunk.blocks.19.attn.proj.bias', 'image_encoder.trunk.blocks.2.norm2.bias', 'image_encoder.trunk.blocks.19.norm1.weight', 'image_encoder.trunk.blocks.5.attn.qkv.bias', 'image_encoder.trunk.blocks.22.attn.qkv.bias', 'image_encoder.trunk.blocks.13.attn.proj.weight', 'image_encoder.trunk.blocks.19.mlp.layers.1.weight', 'image_encoder.trunk.blocks.1.norm1.weight', 'image_encoder.trunk.blocks.10.mlp.layers.0.bias', 'image_encoder.trunk.blocks.8.mlp.layers.1.bias', 'image_encoder.trunk.blocks.21.proj.bias', 'image_encoder.trunk.blocks.2.norm2.weight', 'image_encoder.trunk.blocks.5.norm2.weight', 'image_encoder.trunk.blocks.2.attn.proj.weight', 'image_encoder.trunk.blocks.11.mlp.layers.0.weight', 'image_encoder.trunk.blocks.1.attn.qkv.bias', 'image_encoder.trunk.blocks.12.mlp.layers.0.weight', 'image_encoder.trunk.blocks.16.norm1.bias', 'image_encoder.trunk.blocks.0.norm2.bias', 'image_encoder.trunk.blocks.10.norm1.bias', 'image_encoder.trunk.blocks.1.mlp.layers.0.weight', 'image_encoder.trunk.blocks.14.mlp.layers.1.bias', 'image_encoder.trunk.blocks.19.norm2.bias', 'image_encoder.trunk.blocks.14.norm1.bias', 'image_encoder.trunk.blocks.20.attn.qkv.bias', 'image_encoder.trunk.blocks.6.attn.qkv.weight', 'image_encoder.trunk.blocks.12.norm2.weight', 'image_encoder.trunk.blocks.21.mlp.layers.0.weight', 'image_encoder.trunk.blocks.18.mlp.layers.0.bias', 'image_encoder.trunk.blocks.1.mlp.layers.0.bias', 'image_encoder.trunk.blocks.23.norm2.bias', 'image_encoder.trunk.blocks.4.norm2.weight', 'image_encoder.trunk.blocks.9.attn.proj.weight', 'image_encoder.trunk.blocks.23.attn.qkv.weight', 'image_encoder.trunk.blocks.17.norm1.weight', 'image_encoder.trunk.blocks.17.mlp.layers.0.bias', 'image_encoder.trunk.blocks.6.mlp.layers.0.bias', 'image_encoder.trunk.blocks.1.attn.proj.weight', 'image_encoder.neck.convs.0.conv.bias', 'image_encoder.trunk.blocks.3.attn.proj.bias', 'image_encoder.trunk.blocks.15.norm1.bias', 'image_encoder.trunk.blocks.7.attn.proj.weight', 'image_encoder.trunk.blocks.14.attn.proj.weight', 'image_encoder.trunk.blocks.23.norm1.bias', 'image_encoder.trunk.blocks.3.attn.qkv.bias', 'image_encoder.trunk.blocks.0.attn.proj.weight', 'image_encoder.trunk.blocks.22.norm2.weight', 'image_encoder.trunk.blocks.7.norm1.bias', 'image_encoder.trunk.blocks.12.mlp.layers.0.bias', 'image_encoder.trunk.blocks.22.norm1.weight', 'image_encoder.trunk.blocks.15.mlp.layers.0.bias', 'image_encoder.trunk.blocks.6.mlp.layers.1.bias', 'image_encoder.trunk.blocks.11.mlp.layers.0.bias', 'image_encoder.trunk.blocks.12.mlp.layers.1.weight', 'image_encoder.trunk.blocks.6.attn.qkv.bias', 'image_encoder.trunk.blocks.6.norm2.weight', 'image_encoder.trunk.blocks.14.norm2.weight', 'image_encoder.trunk.blocks.0.norm2.weight', 'image_encoder.trunk.blocks.3.norm1.bias', 'image_encoder.trunk.blocks.18.attn.proj.weight', 'image_encoder.trunk.blocks.23.norm2.weight', 'image_encoder.trunk.blocks.16.attn.proj.weight', 'image_encoder.trunk.blocks.22.attn.proj.weight', 'image_encoder.trunk.blocks.10.mlp.layers.1.bias', 'image_encoder.trunk.blocks.2.norm1.weight', 'image_encoder.trunk.blocks.13.attn.proj.bias', 'image_encoder.trunk.blocks.22.mlp.layers.0.weight', 'image_encoder.trunk.blocks.4.mlp.layers.1.weight', 'image_encoder.trunk.blocks.14.norm1.weight', 'image_encoder.trunk.blocks.1.attn.qkv.weight', 'image_encoder.trunk.blocks.1.attn.proj.bias', 'image_encoder.trunk.blocks.0.attn.proj.bias', 'image_encoder.trunk.blocks.13.norm1.bias', 'image_encoder.trunk.blocks.12.norm2.bias', 'image_encoder.neck.convs.1.conv.weight', 'image_encoder.neck.convs.2.conv.bias', 'image_encoder.trunk.blocks.4.norm2.bias', 'image_encoder.trunk.blocks.22.attn.qkv.weight', 'image_encoder.trunk.blocks.21.proj.weight', 'image_encoder.trunk.blocks.18.mlp.layers.1.bias', 'image_encoder.trunk.blocks.10.norm2.weight', 'image_encoder.trunk.blocks.19.attn.qkv.weight', 'image_encoder.trunk.blocks.9.norm2.weight', 'image_encoder.trunk.blocks.10.norm2.bias', 'image_encoder.trunk.blocks.5.proj.weight', 'image_encoder.trunk.blocks.17.norm2.bias', 'image_encoder.trunk.blocks.17.attn.proj.bias', 'image_encoder.trunk.blocks.5.mlp.layers.0.bias', 'image_encoder.trunk.blocks.15.norm2.bias', 'image_encoder.trunk.blocks.21.norm2.bias', 'image_encoder.trunk.blocks.6.norm1.bias', 'image_encoder.trunk.blocks.20.mlp.layers.0.weight', 'image_encoder.trunk.blocks.0.mlp.layers.0.bias', 'image_encoder.trunk.blocks.20.mlp.layers.1.bias', 'image_encoder.trunk.blocks.4.attn.proj.bias', 'image_encoder.trunk.blocks.18.attn.qkv.weight', 'image_encoder.trunk.blocks.21.mlp.layers.1.weight', 'image_encoder.trunk.blocks.11.attn.proj.weight', 'image_encoder.trunk.blocks.8.attn.qkv.bias', 'image_encoder.trunk.blocks.21.norm1.bias', 'image_encoder.trunk.blocks.16.mlp.layers.0.weight', 'image_encoder.trunk.blocks.2.mlp.layers.1.weight', 'image_encoder.neck.convs.3.conv.bias', 'image_encoder.trunk.blocks.1.mlp.layers.1.bias', 'image_encoder.trunk.blocks.5.norm2.bias', 'image_encoder.trunk.blocks.11.norm1.weight', 'image_encoder.trunk.blocks.3.mlp.layers.1.weight', 'image_encoder.trunk.blocks.18.norm1.bias', 'image_encoder.trunk.blocks.8.norm1.weight', 'image_encoder.trunk.blocks.15.attn.proj.weight', 'image_encoder.trunk.blocks.3.attn.proj.weight', 'image_encoder.trunk.blocks.23.attn.qkv.bias', 'image_encoder.trunk.blocks.2.mlp.layers.1.bias', 'image_encoder.trunk.blocks.23.mlp.layers.0.weight', 'image_encoder.trunk.blocks.0.attn.qkv.weight', 'image_encoder.trunk.blocks.23.attn.proj.bias', 'image_encoder.trunk.blocks.4.norm1.bias', 'image_encoder.trunk.blocks.22.norm1.bias', 'image_encoder.trunk.blocks.14.norm2.bias', 'image_encoder.trunk.blocks.9.mlp.layers.1.weight', 'image_encoder.trunk.blocks.9.norm2.bias', 'image_encoder.trunk.blocks.12.norm1.bias', 'image_encoder.trunk.blocks.10.norm1.weight', 'image_encoder.trunk.blocks.18.mlp.layers.1.weight', 'image_encoder.trunk.blocks.6.mlp.layers.1.weight', 'image_encoder.trunk.blocks.2.attn.qkv.bias', 'image_encoder.trunk.blocks.21.attn.qkv.bias', 'image_encoder.trunk.blocks.2.proj.weight', 'image_encoder.trunk.blocks.12.attn.qkv.bias', 'image_encoder.trunk.blocks.7.attn.qkv.bias', 'image_encoder.trunk.blocks.10.attn.qkv.bias', 'image_encoder.trunk.blocks.9.mlp.layers.0.bias', 'image_encoder.trunk.blocks.9.norm1.weight', 'image_encoder.trunk.blocks.17.attn.qkv.bias', 'image_encoder.trunk.blocks.14.mlp.layers.0.weight', 'image_encoder.trunk.blocks.16.mlp.layers.1.weight', 'image_encoder.trunk.blocks.3.attn.qkv.weight', 'image_encoder.trunk.blocks.15.norm1.weight', 'image_encoder.trunk.blocks.16.attn.proj.bias', 'image_encoder.trunk.blocks.22.attn.proj.bias', 'image_encoder.trunk.blocks.0.norm1.bias', 'image_encoder.trunk.blocks.8.attn.qkv.weight', 'image_encoder.trunk.blocks.7.mlp.layers.1.weight', 'image_encoder.trunk.blocks.20.norm2.bias', 'image_encoder.trunk.blocks.5.attn.qkv.weight', 'image_encoder.trunk.blocks.20.mlp.layers.1.weight', 'image_encoder.trunk.blocks.5.mlp.layers.1.bias', 'image_encoder.trunk.blocks.17.norm1.bias', 'image_encoder.trunk.blocks.15.mlp.layers.0.weight', 'image_encoder.trunk.blocks.8.mlp.layers.0.bias', 'image_encoder.trunk.blocks.8.mlp.layers.0.weight', 'image_encoder.trunk.blocks.21.attn.qkv.weight', 'image_encoder.trunk.blocks.15.attn.proj.bias', 'image_encoder.trunk.blocks.19.norm1.bias', 'image_encoder.trunk.blocks.17.mlp.layers.1.weight', 'image_encoder.trunk.blocks.9.attn.qkv.bias', 'image_encoder.trunk.blocks.13.attn.qkv.weight', 'image_encoder.trunk.blocks.6.norm1.weight', 'image_encoder.trunk.blocks.16.attn.qkv.weight', 'image_encoder.trunk.blocks.11.mlp.layers.1.weight', 'image_encoder.trunk.pos_embed_window', 'image_encoder.trunk.blocks.7.mlp.layers.1.bias', 'image_encoder.trunk.blocks.13.mlp.layers.0.bias', 'image_encoder.trunk.blocks.23.attn.proj.weight', 'image_encoder.trunk.blocks.14.attn.proj.bias', 'image_encoder.trunk.blocks.20.attn.proj.weight', 'image_encoder.trunk.blocks.2.mlp.layers.0.bias', 'image_encoder.trunk.blocks.23.mlp.layers.0.bias', 'image_encoder.trunk.blocks.5.mlp.layers.1.weight', 'image_encoder.trunk.blocks.0.norm1.weight', 'image_encoder.trunk.blocks.16.norm2.bias', 'image_encoder.trunk.blocks.10.attn.proj.bias', 'image_encoder.trunk.blocks.3.norm1.weight', 'image_encoder.neck.convs.1.conv.bias', 'image_encoder.trunk.blocks.18.norm2.bias', 'image_encoder.trunk.blocks.19.attn.qkv.bias', 'image_encoder.trunk.blocks.13.norm1.weight', 'image_encoder.trunk.blocks.8.attn.proj.weight', 'image_encoder.trunk.blocks.21.norm2.weight', 'image_encoder.trunk.blocks.12.norm1.weight', 'image_encoder.trunk.blocks.7.norm1.weight', 'image_encoder.trunk.blocks.17.attn.proj.weight', 'image_encoder.trunk.blocks.6.norm2.bias', 'image_encoder.trunk.blocks.18.norm1.weight', 'image_encoder.trunk.blocks.4.mlp.layers.0.bias', 'image_encoder.trunk.blocks.5.proj.bias', 'image_encoder.trunk.blocks.19.mlp.layers.0.bias', 'image_encoder.trunk.blocks.2.attn.qkv.weight', 'image_encoder.trunk.blocks.8.attn.proj.bias', 'image_encoder.trunk.blocks.1.norm2.weight', 'image_encoder.trunk.patch_embed.proj.weight', 'image_encoder.trunk.blocks.23.mlp.layers.1.weight', 'image_encoder.trunk.blocks.11.attn.qkv.weight', 'image_encoder.trunk.blocks.14.attn.qkv.bias', 'image_encoder.trunk.blocks.9.mlp.layers.1.bias', 'image_encoder.trunk.blocks.13.norm2.bias', 'image_encoder.trunk.blocks.4.attn.qkv.weight', 'image_encoder.trunk.blocks.6.attn.proj.weight', 'image_encoder.trunk.blocks.9.attn.proj.bias', 'image_encoder.trunk.blocks.11.attn.qkv.bias', 'image_encoder.trunk.blocks.22.mlp.layers.1.bias', 'image_encoder.trunk.blocks.15.mlp.layers.1.bias', 'image_encoder.trunk.blocks.17.mlp.layers.1.bias', 'image_encoder.trunk.blocks.20.norm2.weight', 'image_encoder.trunk.blocks.12.attn.proj.bias', 'image_encoder.trunk.blocks.1.norm1.bias', 'image_encoder.trunk.blocks.23.norm1.weight', 'image_encoder.trunk.blocks.16.mlp.layers.1.bias', 'image_encoder.trunk.blocks.16.norm1.weight', 'image_encoder.trunk.blocks.18.attn.qkv.bias', 'image_encoder.trunk.blocks.11.norm1.bias', 'image_encoder.trunk.blocks.7.mlp.layers.0.bias', 'image_encoder.trunk.blocks.21.mlp.layers.1.bias', 'image_encoder.trunk.blocks.14.attn.qkv.weight', 'image_encoder.trunk.blocks.5.mlp.layers.0.weight', 'image_encoder.trunk.blocks.21.attn.proj.bias', 'image_encoder.trunk.blocks.11.norm2.weight', 'image_encoder.trunk.blocks.13.mlp.layers.0.weight', 'image_encoder.trunk.blocks.17.mlp.layers.0.weight', 'image_encoder.trunk.blocks.3.mlp.layers.1.bias', 'image_encoder.trunk.blocks.22.mlp.layers.1.weight', 'image_encoder.trunk.blocks.7.attn.proj.bias', 'image_encoder.trunk.blocks.16.attn.qkv.bias', 'image_encoder.trunk.blocks.2.proj.bias', 'image_encoder.trunk.blocks.5.norm1.weight', 'image_encoder.trunk.blocks.4.mlp.layers.1.bias', 'image_encoder.trunk.blocks.23.mlp.layers.1.bias', 'image_encoder.trunk.blocks.0.attn.qkv.bias', 'image_encoder.trunk.blocks.10.attn.proj.weight', 'image_encoder.trunk.blocks.1.norm2.bias', 'image_encoder.trunk.blocks.14.mlp.layers.0.bias', 'image_encoder.trunk.blocks.8.norm2.weight', 'image_encoder.trunk.blocks.8.mlp.layers.1.weight', 'image_encoder.trunk.blocks.11.norm2.bias', 'image_encoder.trunk.blocks.18.attn.proj.bias', 'image_encoder.trunk.blocks.13.mlp.layers.1.bias', 'image_encoder.trunk.blocks.2.mlp.layers.0.weight', 'image_encoder.trunk.blocks.15.mlp.layers.1.weight', 'image_encoder.trunk.blocks.20.norm1.bias', 'image_encoder.trunk.blocks.20.mlp.layers.0.bias', 'image_encoder.trunk.blocks.2.attn.proj.bias', 'image_encoder.trunk.blocks.20.attn.proj.bias', 'image_encoder.trunk.blocks.13.norm2.weight', 'image_encoder.trunk.blocks.19.mlp.layers.0.weight', 'image_encoder.trunk.blocks.7.attn.qkv.weight', 'image_encoder.trunk.blocks.3.norm2.bias', 'image_encoder.trunk.blocks.12.mlp.layers.1.bias', 'image_encoder.trunk.blocks.22.norm2.bias', 'image_encoder.trunk.blocks.17.attn.qkv.weight', 'image_encoder.trunk.blocks.7.norm2.weight', 'image_encoder.trunk.blocks.1.mlp.layers.1.weight', 'image_encoder.trunk.blocks.7.norm2.bias', 'image_encoder.trunk.blocks.15.norm2.weight', 'image_encoder.trunk.blocks.9.mlp.layers.0.weight', 'image_encoder.trunk.patch_embed.proj.bias', 'image_encoder.trunk.blocks.5.attn.proj.bias', 'image_encoder.trunk.blocks.20.attn.qkv.weight', 'image_encoder.trunk.blocks.6.attn.proj.bias', 'image_encoder.trunk.blocks.4.attn.qkv.bias', 'image_encoder.trunk.pos_embed', 'image_encoder.neck.convs.3.conv.weight', 'image_encoder.trunk.blocks.0.mlp.layers.0.weight', 'image_encoder.trunk.blocks.7.mlp.layers.0.weight', 'image_encoder.trunk.blocks.12.attn.qkv.weight', 'image_encoder.trunk.blocks.17.norm2.weight', 'image_encoder.trunk.blocks.3.mlp.layers.0.weight', 'image_encoder.trunk.blocks.4.attn.proj.weight', 'image_encoder.trunk.blocks.6.mlp.layers.0.weight', 'image_encoder.trunk.blocks.22.mlp.layers.0.bias', 'image_encoder.trunk.blocks.13.attn.qkv.bias', 'image_encoder.trunk.blocks.14.mlp.layers.1.weight', 'image_encoder.trunk.blocks.4.mlp.layers.0.weight', 'image_encoder.trunk.blocks.11.mlp.layers.1.bias', 'image_encoder.trunk.blocks.16.mlp.layers.0.bias', 'image_encoder.trunk.blocks.3.norm2.weight', 'image_encoder.trunk.blocks.19.norm2.weight', 'image_encoder.trunk.blocks.5.norm1.bias', 'image_encoder.trunk.blocks.20.norm1.weight', 'image_encoder.trunk.blocks.8.norm2.bias', 'image_encoder.trunk.blocks.19.attn.proj.weight', 'image_encoder.trunk.blocks.9.attn.qkv.weight', 'image_encoder.trunk.blocks.11.attn.proj.bias', 'image_encoder.trunk.blocks.0.mlp.layers.1.bias', 'image_encoder.trunk.blocks.21.norm1.weight', 'image_encoder.trunk.blocks.10.mlp.layers.1.weight', 'image_encoder.trunk.blocks.15.attn.qkv.weight', 'image_encoder.trunk.blocks.18.norm2.weight', 'image_encoder.trunk.blocks.10.attn.qkv.weight', 'image_encoder.trunk.blocks.10.mlp.layers.0.weight', 'image_encoder.trunk.blocks.12.attn.proj.weight'}
+INFO 2025-10-06 06:35:11,632 optimizer.py: 248: Matches for param_name [*bias*]: {'image_encoder.trunk.blocks.3.mlp.layers.0.bias', 'memory_attention.layers.1.linear1.bias', 'image_encoder.trunk.blocks.19.mlp.layers.1.bias', 'memory_attention.layers.3.self_attn.out_proj.bias', 'memory_attention.layers.0.cross_attn_image.out_proj.bias', 'obj_ptr_proj.layers.0.bias', 'sam_mask_decoder.transformer.final_attn_token_to_image.out_proj.bias', 'sam_mask_decoder.pred_obj_score_head.layers.1.bias', 'image_encoder.trunk.blocks.15.attn.qkv.bias', 'image_encoder.trunk.blocks.21.mlp.layers.0.bias', 'sam_mask_decoder.transformer.layers.1.cross_attn_image_to_token.out_proj.bias', 'memory_attention.layers.3.norm3.bias', 'image_encoder.trunk.blocks.9.norm1.bias', 'image_encoder.trunk.blocks.2.norm1.bias', 'image_encoder.trunk.blocks.19.attn.proj.bias', 'image_encoder.trunk.blocks.2.norm2.bias', 'sam_mask_decoder.transformer.layers.0.mlp.layers.0.bias', 'image_encoder.trunk.blocks.8.norm1.bias', 'memory_attention.layers.2.linear1.bias', 'image_encoder.trunk.blocks.5.attn.qkv.bias', 'image_encoder.trunk.blocks.22.attn.qkv.bias', 'sam_mask_decoder.iou_prediction_head.layers.2.bias', 'sam_mask_decoder.transformer.layers.0.self_attn.k_proj.bias', 'image_encoder.trunk.blocks.10.mlp.layers.0.bias', 'memory_attention.layers.3.cross_attn_image.q_proj.bias', 'image_encoder.trunk.blocks.8.mlp.layers.1.bias', 'image_encoder.trunk.blocks.21.proj.bias', 'memory_attention.layers.0.linear2.bias', 'sam_mask_decoder.transformer.layers.1.norm3.bias', 'sam_mask_decoder.transformer.layers.0.mlp.layers.1.bias', 'image_encoder.trunk.blocks.1.attn.qkv.bias', 'sam_mask_decoder.transformer.final_attn_token_to_image.q_proj.bias', 'sam_mask_decoder.transformer.layers.1.mlp.layers.1.bias', 'image_encoder.trunk.blocks.10.norm1.bias', 'image_encoder.trunk.blocks.16.norm1.bias', 'image_encoder.trunk.blocks.0.norm2.bias', 'image_encoder.trunk.blocks.14.mlp.layers.1.bias', 'memory_attention.layers.3.self_attn.q_proj.bias', 'image_encoder.trunk.blocks.19.norm2.bias', 'sam_mask_decoder.transformer.layers.0.norm3.bias', 'image_encoder.trunk.blocks.14.norm1.bias', 'image_encoder.trunk.blocks.20.attn.qkv.bias', 'sam_mask_decoder.transformer.norm_final_attn.bias', 'image_encoder.trunk.blocks.18.mlp.layers.0.bias', 'memory_attention.layers.1.cross_attn_image.q_proj.bias', 'image_encoder.trunk.blocks.1.mlp.layers.0.bias', 'image_encoder.trunk.blocks.23.norm2.bias', 'memory_attention.layers.1.self_attn.q_proj.bias', 'memory_attention.layers.0.self_attn.k_proj.bias', 'memory_encoder.mask_downsampler.encoder.10.bias', 'memory_attention.layers.2.cross_attn_image.q_proj.bias', 'image_encoder.trunk.blocks.17.mlp.layers.0.bias', 'memory_attention.layers.1.self_attn.k_proj.bias', 'image_encoder.trunk.blocks.6.mlp.layers.0.bias', 'memory_attention.layers.3.cross_attn_image.out_proj.bias', 'image_encoder.neck.convs.0.conv.bias', 'sam_mask_decoder.output_upscaling.0.bias', 'image_encoder.trunk.blocks.3.attn.proj.bias', 'image_encoder.trunk.blocks.15.norm1.bias', 'image_encoder.trunk.blocks.23.norm1.bias', 'image_encoder.trunk.blocks.3.attn.qkv.bias', 'image_encoder.trunk.blocks.7.norm1.bias', 'memory_attention.layers.3.self_attn.v_proj.bias', 'sam_mask_decoder.transformer.layers.1.self_attn.q_proj.bias', 'sam_mask_decoder.transformer.layers.1.self_attn.k_proj.bias', 'image_encoder.trunk.blocks.12.mlp.layers.0.bias', 'image_encoder.trunk.blocks.15.mlp.layers.0.bias', 'image_encoder.trunk.blocks.6.mlp.layers.1.bias', 'memory_attention.layers.1.norm3.bias', 'memory_attention.layers.0.linear1.bias', 'memory_attention.layers.1.cross_attn_image.out_proj.bias', 'image_encoder.trunk.blocks.6.attn.qkv.bias', 'sam_mask_decoder.transformer.layers.0.norm2.bias', 'image_encoder.trunk.blocks.11.mlp.layers.0.bias', 'memory_attention.norm.bias', 'memory_attention.layers.0.self_attn.q_proj.bias', 'memory_encoder.mask_downsampler.encoder.7.bias', 'image_encoder.trunk.blocks.3.norm1.bias', 'memory_attention.layers.1.norm2.bias', 'memory_attention.layers.1.norm1.bias', 'image_encoder.trunk.blocks.10.mlp.layers.1.bias', 'sam_mask_decoder.transformer.layers.1.cross_attn_token_to_image.q_proj.bias', 'obj_ptr_proj.layers.1.bias', 'sam_mask_decoder.output_hypernetworks_mlps.1.layers.2.bias', 'sam_prompt_encoder.mask_downscaling.6.bias', 'image_encoder.trunk.blocks.13.attn.proj.bias', 'image_encoder.trunk.blocks.1.attn.proj.bias', 'image_encoder.trunk.blocks.13.norm1.bias', 'image_encoder.trunk.blocks.12.norm2.bias', 'image_encoder.trunk.blocks.0.attn.proj.bias', 'sam_prompt_encoder.mask_downscaling.3.bias', 'sam_mask_decoder.transformer.layers.0.cross_attn_token_to_image.out_proj.bias', 'sam_mask_decoder.transformer.layers.0.cross_attn_token_to_image.k_proj.bias', 'image_encoder.neck.convs.2.conv.bias', 'image_encoder.trunk.blocks.4.norm2.bias', 'image_encoder.trunk.blocks.18.mlp.layers.1.bias', 'sam_mask_decoder.output_hypernetworks_mlps.0.layers.1.bias', 'sam_prompt_encoder.mask_downscaling.1.bias', 'image_encoder.trunk.blocks.10.norm2.bias', 'sam_prompt_encoder.mask_downscaling.0.bias', 'sam_mask_decoder.transformer.final_attn_token_to_image.v_proj.bias', 'image_encoder.trunk.blocks.17.norm2.bias', 'sam_mask_decoder.transformer.layers.0.cross_attn_token_to_image.v_proj.bias', 'image_encoder.trunk.blocks.17.attn.proj.bias', 'sam_mask_decoder.transformer.layers.0.cross_attn_image_to_token.out_proj.bias', 'image_encoder.trunk.blocks.15.norm2.bias', 'image_encoder.trunk.blocks.21.norm2.bias', 'memory_attention.layers.1.self_attn.out_proj.bias', 'image_encoder.trunk.blocks.6.norm1.bias', 'image_encoder.trunk.blocks.5.mlp.layers.0.bias', 'memory_encoder.mask_downsampler.encoder.6.bias', 'image_encoder.trunk.blocks.0.mlp.layers.0.bias', 'memory_encoder.fuser.layers.1.norm.bias', 'image_encoder.trunk.blocks.20.mlp.layers.1.bias', 'image_encoder.trunk.blocks.4.attn.proj.bias', 'memory_encoder.fuser.layers.0.norm.bias', 'sam_mask_decoder.transformer.layers.1.self_attn.v_proj.bias', 'sam_mask_decoder.transformer.layers.1.cross_attn_image_to_token.q_proj.bias', 'memory_attention.layers.0.self_attn.v_proj.bias', 'image_encoder.trunk.blocks.8.attn.qkv.bias', 'sam_mask_decoder.transformer.layers.0.norm1.bias', 'image_encoder.trunk.blocks.21.norm1.bias', 'sam_mask_decoder.output_hypernetworks_mlps.3.layers.0.bias', 'memory_attention.layers.2.norm3.bias', 'image_encoder.trunk.blocks.1.mlp.layers.1.bias', 'image_encoder.neck.convs.3.conv.bias', 'image_encoder.trunk.blocks.5.norm2.bias', 'sam_mask_decoder.output_hypernetworks_mlps.1.layers.0.bias', 'image_encoder.trunk.blocks.18.norm1.bias', 'memory_attention.layers.3.cross_attn_image.k_proj.bias', 'sam_mask_decoder.transformer.layers.1.mlp.layers.0.bias', 'sam_mask_decoder.transformer.layers.1.cross_attn_token_to_image.out_proj.bias', 'sam_mask_decoder.transformer.layers.1.cross_attn_image_to_token.k_proj.bias', 'memory_attention.layers.3.linear1.bias', 'image_encoder.trunk.blocks.2.mlp.layers.1.bias', 'image_encoder.trunk.blocks.23.attn.qkv.bias', 'sam_mask_decoder.output_hypernetworks_mlps.2.layers.2.bias', 'image_encoder.trunk.blocks.23.attn.proj.bias', 'image_encoder.trunk.blocks.4.norm1.bias', 'image_encoder.trunk.blocks.22.norm1.bias', 'image_encoder.trunk.blocks.14.norm2.bias', 'image_encoder.trunk.blocks.9.norm2.bias', 'memory_attention.layers.1.cross_attn_image.k_proj.bias', 'image_encoder.trunk.blocks.12.norm1.bias', 'memory_attention.layers.0.norm2.bias', 'image_encoder.trunk.blocks.2.attn.qkv.bias', 'image_encoder.trunk.blocks.21.attn.qkv.bias', 'sam_mask_decoder.conv_s0.bias', 'image_encoder.trunk.blocks.12.attn.qkv.bias', 'sam_mask_decoder.transformer.layers.1.self_attn.out_proj.bias', 'image_encoder.trunk.blocks.7.attn.qkv.bias', 'image_encoder.trunk.blocks.10.attn.qkv.bias', 'image_encoder.trunk.blocks.9.mlp.layers.0.bias', 'image_encoder.trunk.blocks.17.attn.qkv.bias', 'image_encoder.trunk.blocks.16.attn.proj.bias', 'image_encoder.trunk.blocks.22.attn.proj.bias', 'image_encoder.trunk.blocks.0.norm1.bias', 'sam_mask_decoder.pred_obj_score_head.layers.2.bias', 'memory_attention.layers.1.linear2.bias', 'image_encoder.trunk.blocks.20.norm2.bias', 'sam_mask_decoder.transformer.layers.1.cross_attn_token_to_image.v_proj.bias', 'sam_mask_decoder.output_hypernetworks_mlps.0.layers.2.bias', 'memory_attention.layers.2.cross_attn_image.out_proj.bias', 'image_encoder.trunk.blocks.5.mlp.layers.1.bias', 'image_encoder.trunk.blocks.17.norm1.bias', 'sam_mask_decoder.transformer.layers.1.norm1.bias', 'sam_mask_decoder.conv_s1.bias', 'image_encoder.trunk.blocks.8.mlp.layers.0.bias', 'memory_attention.layers.2.self_attn.k_proj.bias', 'memory_encoder.fuser.layers.0.pwconv2.bias', 'image_encoder.trunk.blocks.15.attn.proj.bias', 'image_encoder.trunk.blocks.19.norm1.bias', 'memory_attention.layers.0.cross_attn_image.k_proj.bias', 'memory_encoder.mask_downsampler.encoder.12.bias', 'image_encoder.trunk.blocks.9.attn.qkv.bias', 'memory_attention.layers.3.norm2.bias', 'memory_encoder.fuser.layers.1.dwconv.bias', 'image_encoder.trunk.blocks.13.mlp.layers.0.bias', 'image_encoder.trunk.blocks.7.mlp.layers.1.bias', 'image_encoder.trunk.blocks.14.attn.proj.bias', 'image_encoder.trunk.blocks.2.mlp.layers.0.bias', 'image_encoder.trunk.blocks.23.mlp.layers.0.bias', 'memory_attention.layers.2.norm1.bias', 'image_encoder.trunk.blocks.16.norm2.bias', 'image_encoder.trunk.blocks.10.attn.proj.bias', 'image_encoder.neck.convs.1.conv.bias', 'image_encoder.trunk.blocks.18.norm2.bias', 'memory_attention.layers.1.cross_attn_image.v_proj.bias', 'sam_mask_decoder.transformer.layers.1.cross_attn_image_to_token.v_proj.bias', 'image_encoder.trunk.blocks.19.attn.qkv.bias', 'sam_mask_decoder.output_hypernetworks_mlps.2.layers.0.bias', 'memory_attention.layers.0.cross_attn_image.v_proj.bias', 'image_encoder.trunk.blocks.6.norm2.bias', 'image_encoder.trunk.blocks.4.mlp.layers.0.bias', 'image_encoder.trunk.blocks.5.proj.bias', 'obj_ptr_tpos_proj.bias', 'image_encoder.trunk.blocks.19.mlp.layers.0.bias', 'image_encoder.trunk.blocks.8.attn.proj.bias', 'image_encoder.trunk.blocks.14.attn.qkv.bias', 'image_encoder.trunk.blocks.9.mlp.layers.1.bias', 'memory_attention.layers.2.self_attn.out_proj.bias', 'memory_encoder.pix_feat_proj.bias', 'image_encoder.trunk.blocks.13.norm2.bias', 'memory_attention.layers.2.linear2.bias', 'sam_mask_decoder.transformer.layers.0.cross_attn_token_to_image.q_proj.bias', 'sam_mask_decoder.transformer.layers.0.self_attn.out_proj.bias', 'image_encoder.trunk.blocks.9.attn.proj.bias', 'image_encoder.trunk.blocks.11.attn.qkv.bias', 'image_encoder.trunk.blocks.22.mlp.layers.1.bias', 'image_encoder.trunk.blocks.15.mlp.layers.1.bias', 'sam_mask_decoder.transformer.layers.0.norm4.bias', 'image_encoder.trunk.blocks.17.mlp.layers.1.bias', 'sam_mask_decoder.transformer.layers.1.cross_attn_token_to_image.k_proj.bias', 'memory_encoder.fuser.layers.0.dwconv.bias', 'image_encoder.trunk.blocks.12.attn.proj.bias', 'image_encoder.trunk.blocks.1.norm1.bias', 'image_encoder.trunk.blocks.16.mlp.layers.1.bias', 'sam_mask_decoder.transformer.layers.1.norm2.bias', 'image_encoder.trunk.blocks.18.attn.qkv.bias', 'image_encoder.trunk.blocks.11.norm1.bias', 'memory_encoder.fuser.layers.1.pwconv2.bias', 'memory_encoder.mask_downsampler.encoder.0.bias', 'image_encoder.trunk.blocks.7.mlp.layers.0.bias', 'image_encoder.trunk.blocks.21.mlp.layers.1.bias', 'image_encoder.trunk.blocks.21.attn.proj.bias', 'mask_downsample.bias', 'image_encoder.trunk.blocks.3.mlp.layers.1.bias', 'image_encoder.trunk.blocks.7.attn.proj.bias', 'image_encoder.trunk.blocks.16.attn.qkv.bias', 'image_encoder.trunk.blocks.2.proj.bias', 'sam_prompt_encoder.mask_downscaling.4.bias', 'memory_attention.layers.2.cross_attn_image.v_proj.bias', 'image_encoder.trunk.blocks.4.mlp.layers.1.bias', 'image_encoder.trunk.blocks.23.mlp.layers.1.bias', 'image_encoder.trunk.blocks.0.attn.qkv.bias', 'image_encoder.trunk.blocks.1.norm2.bias', 'image_encoder.trunk.blocks.14.mlp.layers.0.bias', 'memory_encoder.mask_downsampler.encoder.3.bias', 'memory_encoder.mask_downsampler.encoder.4.bias', 'sam_mask_decoder.output_hypernetworks_mlps.2.layers.1.bias', 'image_encoder.trunk.blocks.11.norm2.bias', 'image_encoder.trunk.blocks.18.attn.proj.bias', 'memory_attention.layers.0.self_attn.out_proj.bias', 'image_encoder.trunk.blocks.13.mlp.layers.1.bias', 'sam_mask_decoder.transformer.layers.0.self_attn.q_proj.bias', 'sam_mask_decoder.transformer.layers.1.norm4.bias', 'sam_mask_decoder.output_hypernetworks_mlps.3.layers.1.bias', 'image_encoder.trunk.blocks.20.norm1.bias', 'image_encoder.trunk.blocks.20.mlp.layers.0.bias', 'memory_attention.layers.2.cross_attn_image.k_proj.bias', 'image_encoder.trunk.blocks.2.attn.proj.bias', 'sam_mask_decoder.output_hypernetworks_mlps.0.layers.0.bias', 'image_encoder.trunk.blocks.20.attn.proj.bias', 'image_encoder.trunk.blocks.3.norm2.bias', 'image_encoder.trunk.blocks.12.mlp.layers.1.bias', 'image_encoder.trunk.blocks.22.norm2.bias', 'memory_attention.layers.0.norm3.bias', 'memory_attention.layers.2.self_attn.q_proj.bias', 'memory_attention.layers.3.norm1.bias', 'image_encoder.trunk.blocks.7.norm2.bias', 'memory_encoder.mask_downsampler.encoder.1.bias', 'image_encoder.trunk.patch_embed.proj.bias', 'image_encoder.trunk.blocks.5.attn.proj.bias', 'image_encoder.trunk.blocks.6.attn.proj.bias', 'sam_mask_decoder.output_hypernetworks_mlps.1.layers.1.bias', 'image_encoder.trunk.blocks.4.attn.qkv.bias', 'memory_attention.layers.3.cross_attn_image.v_proj.bias', 'sam_mask_decoder.transformer.layers.0.cross_attn_image_to_token.q_proj.bias', 'memory_encoder.fuser.layers.0.pwconv1.bias', 'image_encoder.trunk.blocks.22.mlp.layers.0.bias', 'sam_mask_decoder.transformer.final_attn_token_to_image.k_proj.bias', 'sam_mask_decoder.output_upscaling.3.bias', 'image_encoder.trunk.blocks.13.attn.qkv.bias', 'memory_attention.layers.3.linear2.bias', 'memory_attention.layers.0.cross_attn_image.q_proj.bias', 'memory_attention.layers.0.norm1.bias', 'memory_encoder.out_proj.bias', 'sam_mask_decoder.pred_obj_score_head.layers.0.bias', 'image_encoder.trunk.blocks.11.mlp.layers.1.bias', 'memory_attention.layers.2.self_attn.v_proj.bias', 'sam_mask_decoder.transformer.layers.0.self_attn.v_proj.bias', 'obj_ptr_proj.layers.2.bias', 'image_encoder.trunk.blocks.5.norm1.bias', 'memory_attention.layers.2.norm2.bias', 'image_encoder.trunk.blocks.16.mlp.layers.0.bias', 'memory_encoder.fuser.layers.1.pwconv1.bias', 'memory_attention.layers.1.self_attn.v_proj.bias', 'sam_mask_decoder.iou_prediction_head.layers.1.bias', 'sam_mask_decoder.transformer.layers.0.cross_attn_image_to_token.v_proj.bias', 'sam_mask_decoder.output_hypernetworks_mlps.3.layers.2.bias', 'image_encoder.trunk.blocks.8.norm2.bias', 'image_encoder.trunk.blocks.11.attn.proj.bias', 'image_encoder.trunk.blocks.0.mlp.layers.1.bias', 'sam_mask_decoder.transformer.layers.0.cross_attn_image_to_token.k_proj.bias', 'memory_encoder.mask_downsampler.encoder.9.bias', 'sam_mask_decoder.iou_prediction_head.layers.0.bias', 'sam_mask_decoder.output_upscaling.1.bias', 'memory_attention.layers.3.self_attn.k_proj.bias'}
+INFO 2025-10-06 06:35:11,633 optimizer.py: 220: Matches for module_cls_name [torch.nn.LayerNorm]: {'memory_attention.layers.3.norm3.bias', 'image_encoder.trunk.blocks.9.norm1.bias', 'image_encoder.trunk.blocks.2.norm1.bias', 'image_encoder.trunk.blocks.4.norm1.weight', 'image_encoder.trunk.blocks.2.norm2.bias', 'image_encoder.trunk.blocks.16.norm2.weight', 'sam_mask_decoder.transformer.layers.1.norm3.weight', 'image_encoder.trunk.blocks.8.norm1.bias', 'image_encoder.trunk.blocks.19.norm1.weight', 'image_encoder.trunk.blocks.1.norm1.weight', 'image_encoder.trunk.blocks.2.norm2.weight', 'image_encoder.trunk.blocks.5.norm2.weight', 'sam_mask_decoder.transformer.layers.1.norm3.bias', 'image_encoder.trunk.blocks.16.norm1.bias', 'image_encoder.trunk.blocks.0.norm2.bias', 'image_encoder.trunk.blocks.10.norm1.bias', 'memory_attention.layers.2.norm3.weight', 'image_encoder.trunk.blocks.19.norm2.bias', 'image_encoder.trunk.blocks.14.norm1.bias', 'sam_mask_decoder.transformer.layers.0.norm3.bias', 'image_encoder.trunk.blocks.12.norm2.weight', 'memory_attention.layers.1.norm1.weight', 'sam_mask_decoder.transformer.norm_final_attn.bias', 'image_encoder.trunk.blocks.4.norm2.weight', 'image_encoder.trunk.blocks.23.norm2.bias', 'image_encoder.trunk.blocks.17.norm1.weight', 'sam_mask_decoder.transformer.layers.0.norm4.weight', 'memory_attention.layers.2.norm2.weight', 'memory_attention.layers.3.norm2.weight', 'image_encoder.trunk.blocks.15.norm1.bias', 'image_encoder.trunk.blocks.23.norm1.bias', 'image_encoder.trunk.blocks.7.norm1.bias', 'image_encoder.trunk.blocks.22.norm2.weight', 'image_encoder.trunk.blocks.22.norm1.weight', 'memory_attention.layers.1.norm3.bias', 'sam_mask_decoder.transformer.layers.0.norm2.bias', 'memory_attention.norm.bias', 'image_encoder.trunk.blocks.0.norm2.weight', 'image_encoder.trunk.blocks.6.norm2.weight', 'image_encoder.trunk.blocks.14.norm2.weight', 'image_encoder.trunk.blocks.3.norm1.bias', 'image_encoder.trunk.blocks.23.norm2.weight', 'memory_attention.layers.1.norm2.bias', 'memory_attention.layers.0.norm2.weight', 'image_encoder.trunk.blocks.2.norm1.weight', 'memory_attention.layers.1.norm1.bias', 'image_encoder.trunk.blocks.14.norm1.weight', 'image_encoder.trunk.blocks.13.norm1.bias', 'image_encoder.trunk.blocks.12.norm2.bias', 'sam_mask_decoder.transformer.layers.0.norm2.weight', 'image_encoder.trunk.blocks.4.norm2.bias', 'image_encoder.trunk.blocks.10.norm2.weight', 'image_encoder.trunk.blocks.9.norm2.weight', 'image_encoder.trunk.blocks.10.norm2.bias', 'image_encoder.trunk.blocks.17.norm2.bias', 'image_encoder.trunk.blocks.15.norm2.bias', 'image_encoder.trunk.blocks.21.norm2.bias', 'image_encoder.trunk.blocks.6.norm1.bias', 'sam_mask_decoder.transformer.layers.1.norm2.weight', 'image_encoder.trunk.blocks.21.norm1.bias', 'sam_mask_decoder.transformer.layers.0.norm1.bias', 'memory_attention.layers.3.norm3.weight', 'memory_attention.layers.3.norm1.weight', 'memory_attention.layers.2.norm3.bias', 'image_encoder.trunk.blocks.5.norm2.bias', 'image_encoder.trunk.blocks.11.norm1.weight', 'image_encoder.trunk.blocks.18.norm1.bias', 'image_encoder.trunk.blocks.8.norm1.weight', 'image_encoder.trunk.blocks.14.norm2.bias', 'image_encoder.trunk.blocks.22.norm1.bias', 'image_encoder.trunk.blocks.9.norm2.bias', 'image_encoder.trunk.blocks.4.norm1.bias', 'image_encoder.trunk.blocks.12.norm1.bias', 'memory_attention.layers.0.norm2.bias', 'image_encoder.trunk.blocks.10.norm1.weight', 'sam_mask_decoder.transformer.norm_final_attn.weight', 'image_encoder.trunk.blocks.9.norm1.weight', 'memory_attention.layers.0.norm1.weight', 'image_encoder.trunk.blocks.15.norm1.weight', 'image_encoder.trunk.blocks.0.norm1.bias', 'memory_attention.norm.weight', 'image_encoder.trunk.blocks.20.norm2.bias', 'sam_mask_decoder.transformer.layers.1.norm4.weight', 'sam_mask_decoder.transformer.layers.1.norm1.bias', 'image_encoder.trunk.blocks.17.norm1.bias', 'image_encoder.trunk.blocks.19.norm1.bias', 'sam_mask_decoder.transformer.layers.0.norm1.weight', 'image_encoder.trunk.blocks.6.norm1.weight', 'memory_attention.layers.3.norm2.bias', 'image_encoder.trunk.blocks.0.norm1.weight', 'memory_attention.layers.2.norm1.bias', 'image_encoder.trunk.blocks.16.norm2.bias', 'image_encoder.trunk.blocks.3.norm1.weight', 'image_encoder.trunk.blocks.18.norm2.bias', 'image_encoder.trunk.blocks.13.norm1.weight', 'image_encoder.trunk.blocks.21.norm2.weight', 'image_encoder.trunk.blocks.7.norm1.weight', 'image_encoder.trunk.blocks.12.norm1.weight', 'image_encoder.trunk.blocks.6.norm2.bias', 'image_encoder.trunk.blocks.18.norm1.weight', 'image_encoder.trunk.blocks.1.norm2.weight', 'memory_attention.layers.0.norm3.weight', 'image_encoder.trunk.blocks.13.norm2.bias', 'sam_mask_decoder.transformer.layers.0.norm4.bias', 'memory_attention.layers.1.norm2.weight', 'image_encoder.trunk.blocks.20.norm2.weight', 'image_encoder.trunk.blocks.1.norm1.bias', 'image_encoder.trunk.blocks.23.norm1.weight', 'image_encoder.trunk.blocks.16.norm1.weight', 'sam_mask_decoder.transformer.layers.1.norm2.bias', 'image_encoder.trunk.blocks.11.norm1.bias', 'sam_mask_decoder.transformer.layers.1.norm1.weight', 'image_encoder.trunk.blocks.11.norm2.weight', 'image_encoder.trunk.blocks.5.norm1.weight', 'image_encoder.trunk.blocks.1.norm2.bias', 'image_encoder.trunk.blocks.8.norm2.weight', 'image_encoder.trunk.blocks.11.norm2.bias', 'memory_attention.layers.1.norm3.weight', 'sam_mask_decoder.transformer.layers.0.norm3.weight', 'sam_mask_decoder.transformer.layers.1.norm4.bias', 'image_encoder.trunk.blocks.20.norm1.bias', 'image_encoder.trunk.blocks.13.norm2.weight', 'image_encoder.trunk.blocks.3.norm2.bias', 'memory_attention.layers.2.norm1.weight', 'image_encoder.trunk.blocks.22.norm2.bias', 'memory_attention.layers.0.norm3.bias', 'image_encoder.trunk.blocks.7.norm2.weight', 'image_encoder.trunk.blocks.7.norm2.bias', 'memory_attention.layers.3.norm1.bias', 'image_encoder.trunk.blocks.15.norm2.weight', 'image_encoder.trunk.blocks.17.norm2.weight', 'memory_attention.layers.0.norm1.bias', 'image_encoder.trunk.blocks.3.norm2.weight', 'image_encoder.trunk.blocks.19.norm2.weight', 'image_encoder.trunk.blocks.5.norm1.bias', 'memory_attention.layers.2.norm2.bias', 'image_encoder.trunk.blocks.20.norm1.weight', 'image_encoder.trunk.blocks.8.norm2.bias', 'image_encoder.trunk.blocks.18.norm2.weight', 'image_encoder.trunk.blocks.21.norm1.weight'} 
+INFO 2025-10-06 06:35:12,861 sam2_datasets.py: 125: Dataset mixing probabilities: [1.0]
+INFO 2025-10-06 06:35:13,341 trainer.py: 417: Loading pretrained checkpoint from {'_partial_': True, '_target_': 'training.utils.checkpoint_utils.load_state_dict_into_model', 'strict': True, 'ignore_unexpected_keys': None, 'ignore_missing_keys': None, 'state_dict': {'_target_': 'training.utils.checkpoint_utils.load_checkpoint_and_apply_kernels', 'checkpoint_path': '/work/nvme/bfgd/cmaduabuchi/MODELS/THRIVESEG/sam2.1_hiera_base_plus.pt', 'ckpt_state_dict_keys': ['model']}}
+INFO 2025-10-06 06:53:51,461 trainer.py: 950: Estimated time remaining: 00d 12h 05m
+INFO 2025-10-06 06:53:51,482 trainer.py: 892: Synchronizing meters
+INFO 2025-10-06 06:53:51,482 trainer.py: 830: Losses and meters: {'Losses/train_all_loss': 0.6424086557137584, 'Losses/train_all_loss_mask': 0.016180070180966514, 'Losses/train_all_loss_dice': 0.2685501508280366, 'Losses/train_all_loss_iou': 0.05025709560852524, 'Losses/train_all_loss_class': 1.2424740680067156e-08, 'Losses/train_all_core_loss': 0.6424086557137584, 'Trainer/where': 0.0249937374749499, 'Trainer/epoch': 0, 'Trainer/steps_train': 3992}
+INFO 2025-10-06 07:12:22,839 trainer.py: 950: Estimated time remaining: 00d 11h 41m
+INFO 2025-10-06 07:12:22,861 trainer.py: 892: Synchronizing meters
+INFO 2025-10-06 07:12:22,861 trainer.py: 830: Losses and meters: {'Losses/train_all_loss': 0.5279103564334108, 'Losses/train_all_loss_mask': 0.012948783188376493, 'Losses/train_all_loss_dice': 0.23409812990016116, 'Losses/train_all_loss_iou': 0.034836555814920164, 'Losses/train_all_loss_class': 1.1137212414612728e-08, 'Losses/train_all_core_loss': 0.5279103564334108, 'Trainer/where': 0.0499937374749499, 'Trainer/epoch': 1, 'Trainer/steps_train': 7984}
+INFO 2025-10-06 07:30:58,749 trainer.py: 950: Estimated time remaining: 00d 11h 25m
+INFO 2025-10-06 07:30:58,771 trainer.py: 892: Synchronizing meters
+INFO 2025-10-06 07:30:58,771 trainer.py: 830: Losses and meters: {'Losses/train_all_loss': 0.46176709332815996, 'Losses/train_all_loss_mask': 0.010168827943158898, 'Losses/train_all_loss_dice': 0.22589266421142704, 'Losses/train_all_loss_iou': 0.03249785962655957, 'Losses/train_all_loss_class': 1.3400916166038667e-08, 'Losses/train_all_core_loss': 0.46176709332815996, 'Trainer/where': 0.0749937374749499, 'Trainer/epoch': 2, 'Trainer/steps_train': 11976}
+INFO 2025-10-06 07:49:42,950 trainer.py: 950: Estimated time remaining: 00d 11h 12m
+INFO 2025-10-06 07:49:42,972 trainer.py: 892: Synchronizing meters
+INFO 2025-10-06 07:49:42,972 trainer.py: 830: Losses and meters: {'Losses/train_all_loss': 0.434850245800041, 'Losses/train_all_loss_mask': 0.008949943296251449, 'Losses/train_all_loss_dice': 0.22050112636688954, 'Losses/train_all_loss_iou': 0.03535024233930884, 'Losses/train_all_loss_class': 1.4512113502645153e-08, 'Losses/train_all_core_loss': 0.434850245800041, 'Trainer/where': 0.09999373747494991, 'Trainer/epoch': 3, 'Trainer/steps_train': 15968}
+INFO 2025-10-06 08:08:34,804 trainer.py: 950: Estimated time remaining: 00d 10h 57m
+INFO 2025-10-06 08:08:34,826 trainer.py: 892: Synchronizing meters
+INFO 2025-10-06 08:08:34,826 trainer.py: 830: Losses and meters: {'Losses/train_all_loss': 0.4899576190065402, 'Losses/train_all_loss_mask': 0.012139334249143563, 'Losses/train_all_loss_dice': 0.2170970643271425, 'Losses/train_all_loss_iou': 0.030073830085804915, 'Losses/train_all_loss_class': 3.6043506326426754e-08, 'Losses/train_all_core_loss': 0.4899576190065402, 'Trainer/where': 0.1249937374749499, 'Trainer/epoch': 4, 'Trainer/steps_train': 19960}
+INFO 2025-10-06 08:27:35,126 trainer.py: 950: Estimated time remaining: 00d 10h 43m
+INFO 2025-10-06 08:27:35,148 trainer.py: 892: Synchronizing meters
+INFO 2025-10-06 08:27:35,148 trainer.py: 830: Losses and meters: {'Losses/train_all_loss': 0.410329947133319, 'Losses/train_all_loss_mask': 0.008565576918526002, 'Losses/train_all_loss_dice': 0.21291188210368395, 'Losses/train_all_loss_iou': 0.026106498928851203, 'Losses/train_all_loss_class': 2.8962244711128784e-08, 'Losses/train_all_core_loss': 0.410329947133319, 'Trainer/where': 0.1499937374749499, 'Trainer/epoch': 5, 'Trainer/steps_train': 23952}
+INFO 2025-10-06 08:46:35,674 trainer.py: 950: Estimated time remaining: 00d 10h 24m
+INFO 2025-10-06 08:46:35,695 trainer.py: 892: Synchronizing meters
+INFO 2025-10-06 08:46:35,695 trainer.py: 830: Losses and meters: {'Losses/train_all_loss': 0.40882908666967405, 'Losses/train_all_loss_mask': 0.008480613211250676, 'Losses/train_all_loss_dice': 0.21131395521706234, 'Losses/train_all_loss_iou': 0.027902849250237918, 'Losses/train_all_loss_class': 2.056003091207329e-08, 'Losses/train_all_core_loss': 0.40882908666967405, 'Trainer/where': 0.1749937374749499, 'Trainer/epoch': 6, 'Trainer/steps_train': 27944}
+INFO 2025-10-06 09:05:27,802 trainer.py: 950: Estimated time remaining: 00d 10h 01m
+INFO 2025-10-06 09:05:27,823 trainer.py: 892: Synchronizing meters
+INFO 2025-10-06 09:05:27,823 trainer.py: 830: Losses and meters: {'Losses/train_all_loss': 0.4277842160713726, 'Losses/train_all_loss_mask': 0.009505709921017888, 'Losses/train_all_loss_dice': 0.2092911386209165, 'Losses/train_all_loss_iou': 0.02837886042178577, 'Losses/train_all_loss_class': 1.8972794202037397e-08, 'Losses/train_all_core_loss': 0.4277842160713726, 'Trainer/where': 0.19999373747494992, 'Trainer/epoch': 7, 'Trainer/steps_train': 31936}
+INFO 2025-10-06 09:24:23,726 trainer.py: 950: Estimated time remaining: 00d 09h 44m
+INFO 2025-10-06 09:24:23,748 trainer.py: 892: Synchronizing meters
+INFO 2025-10-06 09:24:23,749 trainer.py: 830: Losses and meters: {'Losses/train_all_loss': 0.3915867800609711, 'Losses/train_all_loss_mask': 0.007933716841337687, 'Losses/train_all_loss_dice': 0.20667340398909811, 'Losses/train_all_loss_iou': 0.02623902521838522, 'Losses/train_all_loss_class': 1.5299105419432164e-08, 'Losses/train_all_core_loss': 0.3915867800609711, 'Trainer/where': 0.2249937374749499, 'Trainer/epoch': 8, 'Trainer/steps_train': 35928}
+INFO 2025-10-06 09:43:05,171 trainer.py: 950: Estimated time remaining: 00d 09h 18m
+INFO 2025-10-06 09:43:05,193 trainer.py: 892: Synchronizing meters
+INFO 2025-10-06 09:43:05,193 trainer.py: 830: Losses and meters: {'Losses/train_all_loss': 0.3884903532788829, 'Losses/train_all_loss_mask': 0.00786731781312654, 'Losses/train_all_loss_dice': 0.2054668479936873, 'Losses/train_all_loss_iou': 0.025677137474684822, 'Losses/train_all_loss_class': 1.3447321963829661e-08, 'Losses/train_all_core_loss': 0.3884903532788829, 'Trainer/where': 0.2499937374749499, 'Trainer/epoch': 9, 'Trainer/steps_train': 39920}
+INFO 2025-10-06 10:01:40,978 trainer.py: 950: Estimated time remaining: 00d 08h 57m
+INFO 2025-10-06 10:01:40,999 trainer.py: 892: Synchronizing meters
+INFO 2025-10-06 10:01:41,000 trainer.py: 830: Losses and meters: {'Losses/train_all_loss': 0.3757703946919679, 'Losses/train_all_loss_mask': 0.007310912669020113, 'Losses/train_all_loss_dice': 0.20403454606542606, 'Losses/train_all_loss_iou': 0.025517578823297127, 'Losses/train_all_loss_class': 1.685980004621154e-08, 'Losses/train_all_core_loss': 0.3757703946919679, 'Trainer/where': 0.27499373747494993, 'Trainer/epoch': 10, 'Trainer/steps_train': 43912}
+INFO 2025-10-06 10:20:27,893 trainer.py: 950: Estimated time remaining: 00d 08h 43m
+INFO 2025-10-06 10:20:27,913 trainer.py: 892: Synchronizing meters
+INFO 2025-10-06 10:20:27,913 trainer.py: 830: Losses and meters: {'Losses/train_all_loss': 0.3717774872969974, 'Losses/train_all_loss_mask': 0.0072885528313850366, 'Losses/train_all_loss_dice': 0.20273731466763484, 'Losses/train_all_loss_iou': 0.023269104299064116, 'Losses/train_all_loss_class': 1.295583962462502e-08, 'Losses/train_all_core_loss': 0.3717774872969974, 'Trainer/where': 0.2999937374749499, 'Trainer/epoch': 11, 'Trainer/steps_train': 47904}
+INFO 2025-10-06 10:39:13,604 trainer.py: 950: Estimated time remaining: 00d 08h 24m
+INFO 2025-10-06 10:39:13,625 trainer.py: 892: Synchronizing meters
+INFO 2025-10-06 10:39:13,625 trainer.py: 830: Losses and meters: {'Losses/train_all_loss': 0.3672938159928715, 'Losses/train_all_loss_mask': 0.0071393110150955305, 'Losses/train_all_loss_dice': 0.20154863357066152, 'Losses/train_all_loss_iou': 0.022958954503873592, 'Losses/train_all_loss_class': 9.158463877552852e-09, 'Losses/train_all_core_loss': 0.3672938159928715, 'Trainer/where': 0.3249937374749499, 'Trainer/epoch': 12, 'Trainer/steps_train': 51896}
+INFO 2025-10-06 10:58:10,614 trainer.py: 950: Estimated time remaining: 00d 08h 10m
+INFO 2025-10-06 10:58:10,636 trainer.py: 892: Synchronizing meters
+INFO 2025-10-06 10:58:10,636 trainer.py: 830: Losses and meters: {'Losses/train_all_loss': 0.39827264390161676, 'Losses/train_all_loss_mask': 0.00852792210578642, 'Losses/train_all_loss_dice': 0.2016383660042692, 'Losses/train_all_loss_iou': 0.026075829454423077, 'Losses/train_all_loss_class': 6.9018100357850494e-09, 'Losses/train_all_core_loss': 0.39827264390161676, 'Trainer/where': 0.3499937374749499, 'Trainer/epoch': 13, 'Trainer/steps_train': 55888}
+INFO 2025-10-06 11:17:07,125 trainer.py: 950: Estimated time remaining: 00d 07h 51m
+INFO 2025-10-06 11:17:07,147 trainer.py: 892: Synchronizing meters
+INFO 2025-10-06 11:17:07,147 trainer.py: 830: Losses and meters: {'Losses/train_all_loss': 0.3666049185414744, 'Losses/train_all_loss_mask': 0.007134012589840914, 'Losses/train_all_loss_dice': 0.2001152827081556, 'Losses/train_all_loss_iou': 0.023809378810292475, 'Losses/train_all_loss_class': 7.340426579333355e-09, 'Losses/train_all_core_loss': 0.3666049185414744, 'Trainer/where': 0.3749937374749499, 'Trainer/epoch': 14, 'Trainer/steps_train': 59880}
+INFO 2025-10-06 11:36:09,314 trainer.py: 950: Estimated time remaining: 00d 07h 35m
+INFO 2025-10-06 11:36:09,335 trainer.py: 892: Synchronizing meters
+INFO 2025-10-06 11:36:09,335 trainer.py: 830: Losses and meters: {'Losses/train_all_loss': 0.36204657752245556, 'Losses/train_all_loss_mask': 0.0069741076941301034, 'Losses/train_all_loss_dice': 0.1988546567444572, 'Losses/train_all_loss_iou': 0.02370976204282832, 'Losses/train_all_loss_class': 6.914729194342508e-09, 'Losses/train_all_core_loss': 0.36204657752245556, 'Trainer/where': 0.39999373747494993, 'Trainer/epoch': 15, 'Trainer/steps_train': 63872}
+INFO 2025-10-06 11:55:05,493 trainer.py: 950: Estimated time remaining: 00d 07h 13m
+INFO 2025-10-06 11:55:05,515 trainer.py: 892: Synchronizing meters
+INFO 2025-10-06 11:55:05,515 trainer.py: 830: Losses and meters: {'Losses/train_all_loss': 0.36432591524420266, 'Losses/train_all_loss_mask': 0.00711237450751671, 'Losses/train_all_loss_dice': 0.19882314240108273, 'Losses/train_all_loss_iou': 0.023255279706872696, 'Losses/train_all_loss_class': 5.094582148262241e-09, 'Losses/train_all_core_loss': 0.36432591524420266, 'Trainer/where': 0.4249937374749499, 'Trainer/epoch': 16, 'Trainer/steps_train': 67864}
+INFO 2025-10-06 12:14:08,255 trainer.py: 950: Estimated time remaining: 00d 06h 57m
+INFO 2025-10-06 12:14:08,277 trainer.py: 892: Synchronizing meters
+INFO 2025-10-06 12:14:08,277 trainer.py: 830: Losses and meters: {'Losses/train_all_loss': 0.3739850448014286, 'Losses/train_all_loss_mask': 0.007661915094332017, 'Losses/train_all_loss_dice': 0.19808480110698806, 'Losses/train_all_loss_iou': 0.02266193838917906, 'Losses/train_all_loss_class': 5.422859437734412e-09, 'Losses/train_all_core_loss': 0.3739850448014286, 'Trainer/where': 0.4499937374749499, 'Trainer/epoch': 17, 'Trainer/steps_train': 71856}
+INFO 2025-10-06 12:33:11,519 trainer.py: 950: Estimated time remaining: 00d 06h 38m
+INFO 2025-10-06 12:33:11,541 trainer.py: 892: Synchronizing meters
+INFO 2025-10-06 12:33:11,541 trainer.py: 830: Losses and meters: {'Losses/train_all_loss': 0.3802372602399911, 'Losses/train_all_loss_mask': 0.008000960067011056, 'Losses/train_all_loss_dice': 0.19741905017761524, 'Losses/train_all_loss_iou': 0.02279900476346511, 'Losses/train_all_loss_class': 5.678085751427958e-09, 'Losses/train_all_core_loss': 0.3802372602399911, 'Trainer/where': 0.4749937374749499, 'Trainer/epoch': 18, 'Trainer/steps_train': 75848}
+INFO 2025-10-06 12:52:16,750 trainer.py: 950: Estimated time remaining: 00d 06h 20m
+INFO 2025-10-06 12:52:16,771 trainer.py: 892: Synchronizing meters
+INFO 2025-10-06 12:52:16,772 trainer.py: 830: Losses and meters: {'Losses/train_all_loss': 0.376298183201997, 'Losses/train_all_loss_mask': 0.007830308981010056, 'Losses/train_all_loss_dice': 0.1965342678711983, 'Losses/train_all_loss_iou': 0.023157731618651492, 'Losses/train_all_loss_class': 5.28514186526349e-09, 'Losses/train_all_core_loss': 0.376298183201997, 'Trainer/where': 0.4999937374749499, 'Trainer/epoch': 19, 'Trainer/steps_train': 79840}
+INFO 2025-10-06 13:11:24,458 trainer.py: 950: Estimated time remaining: 00d 06h 02m
+INFO 2025-10-06 13:11:24,479 trainer.py: 892: Synchronizing meters
+INFO 2025-10-06 13:11:24,480 trainer.py: 830: Losses and meters: {'Losses/train_all_loss': 0.3772340752292874, 'Losses/train_all_loss_mask': 0.007914954837213077, 'Losses/train_all_loss_dice': 0.19617746839064634, 'Losses/train_all_loss_iou': 0.022757507024552415, 'Losses/train_all_loss_class': 5.4267739055031245e-09, 'Losses/train_all_core_loss': 0.3772340752292874, 'Trainer/where': 0.5249937374749499, 'Trainer/epoch': 20, 'Trainer/steps_train': 83832}
+INFO 2025-10-06 13:30:31,353 trainer.py: 950: Estimated time remaining: 00d 05h 42m
+INFO 2025-10-06 13:30:31,379 trainer.py: 892: Synchronizing meters
+INFO 2025-10-06 13:30:31,379 trainer.py: 830: Losses and meters: {'Losses/train_all_loss': 0.37047547260336655, 'Losses/train_all_loss_mask': 0.007639010061916952, 'Losses/train_all_loss_dice': 0.1958161407786763, 'Losses/train_all_loss_iou': 0.02187912743158927, 'Losses/train_all_loss_class': 5.4363083103324426e-09, 'Losses/train_all_core_loss': 0.37047547260336655, 'Trainer/where': 0.54999373747495, 'Trainer/epoch': 21, 'Trainer/steps_train': 87824}
+INFO 2025-10-06 13:49:40,277 trainer.py: 950: Estimated time remaining: 00d 05h 24m
+INFO 2025-10-06 13:49:40,299 trainer.py: 892: Synchronizing meters
+INFO 2025-10-06 13:49:40,299 trainer.py: 830: Losses and meters: {'Losses/train_all_loss': 0.35376920676995477, 'Losses/train_all_loss_mask': 0.006838303287161567, 'Losses/train_all_loss_dice': 0.19547302056410032, 'Losses/train_all_loss_iou': 0.021530116622054854, 'Losses/train_all_loss_class': 6.322159236068461e-09, 'Losses/train_all_core_loss': 0.35376920676995477, 'Trainer/where': 0.5749937374749499, 'Trainer/epoch': 22, 'Trainer/steps_train': 91816}
+INFO 2025-10-06 14:08:52,182 trainer.py: 950: Estimated time remaining: 00d 05h 06m
+INFO 2025-10-06 14:08:52,203 trainer.py: 892: Synchronizing meters
+INFO 2025-10-06 14:08:52,203 trainer.py: 830: Losses and meters: {'Losses/train_all_loss': 0.3551223263956834, 'Losses/train_all_loss_mask': 0.006957760564867117, 'Losses/train_all_loss_dice': 0.1945757962957413, 'Losses/train_all_loss_iou': 0.02139131658811324, 'Losses/train_all_loss_class': 4.68340592482518e-09, 'Losses/train_all_core_loss': 0.3551223263956834, 'Trainer/where': 0.5999937374749499, 'Trainer/epoch': 23, 'Trainer/steps_train': 95808}
+INFO 2025-10-06 14:28:02,935 trainer.py: 950: Estimated time remaining: 00d 04h 46m
+INFO 2025-10-06 14:28:02,956 trainer.py: 892: Synchronizing meters
+INFO 2025-10-06 14:28:02,956 trainer.py: 830: Losses and meters: {'Losses/train_all_loss': 0.35436655999975974, 'Losses/train_all_loss_mask': 0.006915995280616119, 'Losses/train_all_loss_dice': 0.19451504708351736, 'Losses/train_all_loss_iou': 0.0215316050213107, 'Losses/train_all_loss_class': 4.694827900397807e-09, 'Losses/train_all_core_loss': 0.35436655999975974, 'Trainer/where': 0.6249937374749499, 'Trainer/epoch': 24, 'Trainer/steps_train': 99800}
+INFO 2025-10-06 14:47:14,335 trainer.py: 950: Estimated time remaining: 00d 04h 27m
+INFO 2025-10-06 14:47:14,363 trainer.py: 892: Synchronizing meters
+INFO 2025-10-06 14:47:14,363 trainer.py: 830: Losses and meters: {'Losses/train_all_loss': 0.3539243154792313, 'Losses/train_all_loss_mask': 0.0069243552351725595, 'Losses/train_all_loss_dice': 0.19411168701364426, 'Losses/train_all_loss_iou': 0.021325522290423973, 'Losses/train_all_loss_class': 4.124210543847477e-09, 'Losses/train_all_core_loss': 0.3539243154792313, 'Trainer/where': 0.6499937374749499, 'Trainer/epoch': 25, 'Trainer/steps_train': 103792}
+INFO 2025-10-06 15:06:27,350 trainer.py: 950: Estimated time remaining: 00d 04h 08m
+INFO 2025-10-06 15:06:27,371 trainer.py: 892: Synchronizing meters
+INFO 2025-10-06 15:06:27,371 trainer.py: 830: Losses and meters: {'Losses/train_all_loss': 0.3885851017036615, 'Losses/train_all_loss_mask': 0.008587317138184215, 'Losses/train_all_loss_dice': 0.1942436489498687, 'Losses/train_all_loss_iou': 0.02259510975809444, 'Losses/train_all_loss_class': 3.7102346460758284e-09, 'Losses/train_all_core_loss': 0.3885851017036615, 'Trainer/where': 0.67499373747495, 'Trainer/epoch': 26, 'Trainer/steps_train': 107784}
+INFO 2025-10-06 15:25:39,425 trainer.py: 950: Estimated time remaining: 00d 03h 49m
+INFO 2025-10-06 15:25:39,447 trainer.py: 892: Synchronizing meters
+INFO 2025-10-06 15:25:39,447 trainer.py: 830: Losses and meters: {'Losses/train_all_loss': 0.7125891149229509, 'Losses/train_all_loss_mask': 0.024751994295421284, 'Losses/train_all_loss_dice': 0.194532726758826, 'Losses/train_all_loss_iou': 0.023016485271324848, 'Losses/train_all_loss_class': 3.9748640794428046e-09, 'Losses/train_all_core_loss': 0.7125891149229509, 'Trainer/where': 0.6999937374749499, 'Trainer/epoch': 27, 'Trainer/steps_train': 111776}
+INFO 2025-10-06 15:44:53,252 trainer.py: 950: Estimated time remaining: 00d 03h 30m
+INFO 2025-10-06 15:44:53,273 trainer.py: 892: Synchronizing meters
+INFO 2025-10-06 15:44:53,273 trainer.py: 830: Losses and meters: {'Losses/train_all_loss': 0.8676063124507335, 'Losses/train_all_loss_mask': 0.032492553048528834, 'Losses/train_all_loss_dice': 0.1948043356587987, 'Losses/train_all_loss_iou': 0.022950956187875175, 'Losses/train_all_loss_class': 4.138141090095644e-09, 'Losses/train_all_core_loss': 0.8676063124507335, 'Trainer/where': 0.7249937374749499, 'Trainer/epoch': 28, 'Trainer/steps_train': 115768}
+INFO 2025-10-06 16:04:06,937 trainer.py: 950: Estimated time remaining: 00d 03h 11m
+INFO 2025-10-06 16:04:06,959 trainer.py: 892: Synchronizing meters
+INFO 2025-10-06 16:04:06,959 trainer.py: 830: Losses and meters: {'Losses/train_all_loss': 0.3487479994164496, 'Losses/train_all_loss_mask': 0.0067503434569706235, 'Losses/train_all_loss_dice': 0.19311891421526373, 'Losses/train_all_loss_iou': 0.020622213926316305, 'Losses/train_all_loss_class': 4.467678567490371e-09, 'Losses/train_all_core_loss': 0.3487479994164496, 'Trainer/where': 0.7499937374749499, 'Trainer/epoch': 29, 'Trainer/steps_train': 119760}
+INFO 2025-10-06 16:23:19,709 trainer.py: 950: Estimated time remaining: 00d 02h 52m
+INFO 2025-10-06 16:23:19,730 trainer.py: 892: Synchronizing meters
+INFO 2025-10-06 16:23:19,730 trainer.py: 830: Losses and meters: {'Losses/train_all_loss': 0.36419604813683787, 'Losses/train_all_loss_mask': 0.00751875088206941, 'Losses/train_all_loss_dice': 0.19322081335740005, 'Losses/train_all_loss_iou': 0.0206002136427904, 'Losses/train_all_loss_class': 5.008077296045301e-09, 'Losses/train_all_core_loss': 0.36419604813683787, 'Trainer/where': 0.7749937374749499, 'Trainer/epoch': 30, 'Trainer/steps_train': 123752}
+INFO 2025-10-06 16:42:36,647 trainer.py: 950: Estimated time remaining: 00d 02h 33m
+INFO 2025-10-06 16:42:36,669 trainer.py: 892: Synchronizing meters
+INFO 2025-10-06 16:42:36,669 trainer.py: 830: Losses and meters: {'Losses/train_all_loss': 0.34595307377449336, 'Losses/train_all_loss_mask': 0.006689208541691566, 'Losses/train_all_loss_dice': 0.19237655501267714, 'Losses/train_all_loss_iou': 0.01979234506802322, 'Losses/train_all_loss_class': 5.1453944205931e-09, 'Losses/train_all_core_loss': 0.34595307377449336, 'Trainer/where': 0.79999373747495, 'Trainer/epoch': 31, 'Trainer/steps_train': 127744}
+INFO 2025-10-06 17:01:54,652 trainer.py: 950: Estimated time remaining: 00d 02h 14m
+INFO 2025-10-06 17:01:54,674 trainer.py: 892: Synchronizing meters
+INFO 2025-10-06 17:01:54,674 trainer.py: 830: Losses and meters: {'Losses/train_all_loss': 0.34565046294508334, 'Losses/train_all_loss_mask': 0.006636826759476352, 'Losses/train_all_loss_dice': 0.19229123223281813, 'Losses/train_all_loss_iou': 0.020622693491710736, 'Losses/train_all_loss_class': 4.764466950168121e-09, 'Losses/train_all_core_loss': 0.34565046294508334, 'Trainer/where': 0.8249937374749499, 'Trainer/epoch': 32, 'Trainer/steps_train': 131736}
+INFO 2025-10-06 17:21:09,500 trainer.py: 950: Estimated time remaining: 00d 01h 55m
+INFO 2025-10-06 17:21:09,521 trainer.py: 892: Synchronizing meters
+INFO 2025-10-06 17:21:09,521 trainer.py: 830: Losses and meters: {'Losses/train_all_loss': 0.9008659057842683, 'Losses/train_all_loss_mask': 0.03423682985725872, 'Losses/train_all_loss_dice': 0.1938983456405227, 'Losses/train_all_loss_iou': 0.02223101315849819, 'Losses/train_all_loss_class': 4.9622400122766974e-09, 'Losses/train_all_core_loss': 0.9008659057842683, 'Trainer/where': 0.8499937374749498, 'Trainer/epoch': 33, 'Trainer/steps_train': 135728}
+INFO 2025-10-06 17:40:24,086 trainer.py: 950: Estimated time remaining: 00d 01h 35m
+INFO 2025-10-06 17:40:24,108 trainer.py: 892: Synchronizing meters
+INFO 2025-10-06 17:40:24,108 trainer.py: 830: Losses and meters: {'Losses/train_all_loss': 0.3465600523536368, 'Losses/train_all_loss_mask': 0.0067609497240599245, 'Losses/train_all_loss_dice': 0.19183460045195294, 'Losses/train_all_loss_iou': 0.019506455022350135, 'Losses/train_all_loss_class': 4.679632632014728e-09, 'Losses/train_all_core_loss': 0.3465600523536368, 'Trainer/where': 0.8749937374749498, 'Trainer/epoch': 34, 'Trainer/steps_train': 139720}
+INFO 2025-10-06 17:59:43,886 trainer.py: 950: Estimated time remaining: 00d 01h 17m
+INFO 2025-10-06 17:59:43,908 trainer.py: 892: Synchronizing meters
+INFO 2025-10-06 17:59:43,908 trainer.py: 830: Losses and meters: {'Losses/train_all_loss': 0.34736052319718447, 'Losses/train_all_loss_mask': 0.0067391459940458545, 'Losses/train_all_loss_dice': 0.1922520945270458, 'Losses/train_all_loss_iou': 0.020325507099748605, 'Losses/train_all_loss_class': 4.45559715646195e-09, 'Losses/train_all_core_loss': 0.34736052319718447, 'Trainer/where': 0.8999937374749498, 'Trainer/epoch': 35, 'Trainer/steps_train': 143712}
+INFO 2025-10-06 18:18:47,675 trainer.py: 950: Estimated time remaining: 00d 00h 56m
+INFO 2025-10-06 18:18:47,697 trainer.py: 892: Synchronizing meters
+INFO 2025-10-06 18:18:47,697 trainer.py: 830: Losses and meters: {'Losses/train_all_loss': 0.3672967086860374, 'Losses/train_all_loss_mask': 0.00773209927745972, 'Losses/train_all_loss_dice': 0.19209734350442886, 'Losses/train_all_loss_iou': 0.02055737860401078, 'Losses/train_all_loss_class': 4.592152288894342e-09, 'Losses/train_all_core_loss': 0.3672967086860374, 'Trainer/where': 0.9249937374749498, 'Trainer/epoch': 36, 'Trainer/steps_train': 147704}
+INFO 2025-10-06 18:37:46,625 trainer.py: 950: Estimated time remaining: 00d 00h 37m
+INFO 2025-10-06 18:37:46,646 trainer.py: 892: Synchronizing meters
+INFO 2025-10-06 18:37:46,646 trainer.py: 830: Losses and meters: {'Losses/train_all_loss': 0.8421824378987265, 'Losses/train_all_loss_mask': 0.03133104354486921, 'Losses/train_all_loss_dice': 0.19316817402003522, 'Losses/train_all_loss_iou': 0.02239339469971622, 'Losses/train_all_loss_class': 4.4197077939473966e-09, 'Losses/train_all_core_loss': 0.8421824378987265, 'Trainer/where': 0.9499937374749499, 'Trainer/epoch': 37, 'Trainer/steps_train': 151696}
+INFO 2025-10-06 18:56:48,973 trainer.py: 950: Estimated time remaining: 00d 00h 18m
+INFO 2025-10-06 18:56:48,995 trainer.py: 892: Synchronizing meters
+INFO 2025-10-06 18:56:48,995 trainer.py: 830: Losses and meters: {'Losses/train_all_loss': 0.3468215012218792, 'Losses/train_all_loss_mask': 0.006743870759146193, 'Losses/train_all_loss_dice': 0.19145133795742997, 'Losses/train_all_loss_iou': 0.020492745865770254, 'Losses/train_all_loss_class': 4.498966078856853e-09, 'Losses/train_all_core_loss': 0.3468215012218792, 'Trainer/where': 0.9749937374749498, 'Trainer/epoch': 38, 'Trainer/steps_train': 155688}
+INFO 2025-10-06 19:15:48,999 trainer.py: 950: Estimated time remaining: 00d 00h 00m
+INFO 2025-10-06 19:15:49,026 trainer.py: 892: Synchronizing meters
+INFO 2025-10-06 19:15:49,026 trainer.py: 830: Losses and meters: {'Losses/train_all_loss': 0.361319512792094, 'Losses/train_all_loss_mask': 0.007438624991261454, 'Losses/train_all_loss_dice': 0.19195612370669243, 'Losses/train_all_loss_iou': 0.0205908868190231, 'Losses/train_all_loss_class': 4.3955577136424175e-09, 'Losses/train_all_core_loss': 0.361319512792094, 'Trainer/where': 0.9999937374749498, 'Trainer/epoch': 39, 'Trainer/steps_train': 159680}

experiment/thermal_run1/logs/train_stats.json

@@ -0,0 +1,40 @@
+{"Losses/train_all_loss": 0.6424086557137584, "Losses/train_all_loss_mask": 0.016180070180966514, "Losses/train_all_loss_dice": 0.2685501508280366, "Losses/train_all_loss_iou": 0.05025709560852524, "Losses/train_all_loss_class": 1.2424740680067156e-08, "Losses/train_all_core_loss": 0.6424086557137584, "Trainer/where": 0.0249937374749499, "Trainer/epoch": 0, "Trainer/steps_train": 3992}
+{"Losses/train_all_loss": 0.5279103564334108, "Losses/train_all_loss_mask": 0.012948783188376493, "Losses/train_all_loss_dice": 0.23409812990016116, "Losses/train_all_loss_iou": 0.034836555814920164, "Losses/train_all_loss_class": 1.1137212414612728e-08, "Losses/train_all_core_loss": 0.5279103564334108, "Trainer/where": 0.0499937374749499, "Trainer/epoch": 1, "Trainer/steps_train": 7984}
+{"Losses/train_all_loss": 0.46176709332815996, "Losses/train_all_loss_mask": 0.010168827943158898, "Losses/train_all_loss_dice": 0.22589266421142704, "Losses/train_all_loss_iou": 0.03249785962655957, "Losses/train_all_loss_class": 1.3400916166038667e-08, "Losses/train_all_core_loss": 0.46176709332815996, "Trainer/where": 0.0749937374749499, "Trainer/epoch": 2, "Trainer/steps_train": 11976}
+{"Losses/train_all_loss": 0.434850245800041, "Losses/train_all_loss_mask": 0.008949943296251449, "Losses/train_all_loss_dice": 0.22050112636688954, "Losses/train_all_loss_iou": 0.03535024233930884, "Losses/train_all_loss_class": 1.4512113502645153e-08, "Losses/train_all_core_loss": 0.434850245800041, "Trainer/where": 0.09999373747494991, "Trainer/epoch": 3, "Trainer/steps_train": 15968}
+{"Losses/train_all_loss": 0.4899576190065402, "Losses/train_all_loss_mask": 0.012139334249143563, "Losses/train_all_loss_dice": 0.2170970643271425, "Losses/train_all_loss_iou": 0.030073830085804915, "Losses/train_all_loss_class": 3.6043506326426754e-08, "Losses/train_all_core_loss": 0.4899576190065402, "Trainer/where": 0.1249937374749499, "Trainer/epoch": 4, "Trainer/steps_train": 19960}
+{"Losses/train_all_loss": 0.410329947133319, "Losses/train_all_loss_mask": 0.008565576918526002, "Losses/train_all_loss_dice": 0.21291188210368395, "Losses/train_all_loss_iou": 0.026106498928851203, "Losses/train_all_loss_class": 2.8962244711128784e-08, "Losses/train_all_core_loss": 0.410329947133319, "Trainer/where": 0.1499937374749499, "Trainer/epoch": 5, "Trainer/steps_train": 23952}
+{"Losses/train_all_loss": 0.40882908666967405, "Losses/train_all_loss_mask": 0.008480613211250676, "Losses/train_all_loss_dice": 0.21131395521706234, "Losses/train_all_loss_iou": 0.027902849250237918, "Losses/train_all_loss_class": 2.056003091207329e-08, "Losses/train_all_core_loss": 0.40882908666967405, "Trainer/where": 0.1749937374749499, "Trainer/epoch": 6, "Trainer/steps_train": 27944}
+{"Losses/train_all_loss": 0.4277842160713726, "Losses/train_all_loss_mask": 0.009505709921017888, "Losses/train_all_loss_dice": 0.2092911386209165, "Losses/train_all_loss_iou": 0.02837886042178577, "Losses/train_all_loss_class": 1.8972794202037397e-08, "Losses/train_all_core_loss": 0.4277842160713726, "Trainer/where": 0.19999373747494992, "Trainer/epoch": 7, "Trainer/steps_train": 31936}
+{"Losses/train_all_loss": 0.3915867800609711, "Losses/train_all_loss_mask": 0.007933716841337687, "Losses/train_all_loss_dice": 0.20667340398909811, "Losses/train_all_loss_iou": 0.02623902521838522, "Losses/train_all_loss_class": 1.5299105419432164e-08, "Losses/train_all_core_loss": 0.3915867800609711, "Trainer/where": 0.2249937374749499, "Trainer/epoch": 8, "Trainer/steps_train": 35928}
+{"Losses/train_all_loss": 0.3884903532788829, "Losses/train_all_loss_mask": 0.00786731781312654, "Losses/train_all_loss_dice": 0.2054668479936873, "Losses/train_all_loss_iou": 0.025677137474684822, "Losses/train_all_loss_class": 1.3447321963829661e-08, "Losses/train_all_core_loss": 0.3884903532788829, "Trainer/where": 0.2499937374749499, "Trainer/epoch": 9, "Trainer/steps_train": 39920}
+{"Losses/train_all_loss": 0.3757703946919679, "Losses/train_all_loss_mask": 0.007310912669020113, "Losses/train_all_loss_dice": 0.20403454606542606, "Losses/train_all_loss_iou": 0.025517578823297127, "Losses/train_all_loss_class": 1.685980004621154e-08, "Losses/train_all_core_loss": 0.3757703946919679, "Trainer/where": 0.27499373747494993, "Trainer/epoch": 10, "Trainer/steps_train": 43912}
+{"Losses/train_all_loss": 0.3717774872969974, "Losses/train_all_loss_mask": 0.0072885528313850366, "Losses/train_all_loss_dice": 0.20273731466763484, "Losses/train_all_loss_iou": 0.023269104299064116, "Losses/train_all_loss_class": 1.295583962462502e-08, "Losses/train_all_core_loss": 0.3717774872969974, "Trainer/where": 0.2999937374749499, "Trainer/epoch": 11, "Trainer/steps_train": 47904}
+{"Losses/train_all_loss": 0.3672938159928715, "Losses/train_all_loss_mask": 0.0071393110150955305, "Losses/train_all_loss_dice": 0.20154863357066152, "Losses/train_all_loss_iou": 0.022958954503873592, "Losses/train_all_loss_class": 9.158463877552852e-09, "Losses/train_all_core_loss": 0.3672938159928715, "Trainer/where": 0.3249937374749499, "Trainer/epoch": 12, "Trainer/steps_train": 51896}
+{"Losses/train_all_loss": 0.39827264390161676, "Losses/train_all_loss_mask": 0.00852792210578642, "Losses/train_all_loss_dice": 0.2016383660042692, "Losses/train_all_loss_iou": 0.026075829454423077, "Losses/train_all_loss_class": 6.9018100357850494e-09, "Losses/train_all_core_loss": 0.39827264390161676, "Trainer/where": 0.3499937374749499, "Trainer/epoch": 13, "Trainer/steps_train": 55888}
+{"Losses/train_all_loss": 0.3666049185414744, "Losses/train_all_loss_mask": 0.007134012589840914, "Losses/train_all_loss_dice": 0.2001152827081556, "Losses/train_all_loss_iou": 0.023809378810292475, "Losses/train_all_loss_class": 7.340426579333355e-09, "Losses/train_all_core_loss": 0.3666049185414744, "Trainer/where": 0.3749937374749499, "Trainer/epoch": 14, "Trainer/steps_train": 59880}
+{"Losses/train_all_loss": 0.36204657752245556, "Losses/train_all_loss_mask": 0.0069741076941301034, "Losses/train_all_loss_dice": 0.1988546567444572, "Losses/train_all_loss_iou": 0.02370976204282832, "Losses/train_all_loss_class": 6.914729194342508e-09, "Losses/train_all_core_loss": 0.36204657752245556, "Trainer/where": 0.39999373747494993, "Trainer/epoch": 15, "Trainer/steps_train": 63872}
+{"Losses/train_all_loss": 0.36432591524420266, "Losses/train_all_loss_mask": 0.00711237450751671, "Losses/train_all_loss_dice": 0.19882314240108273, "Losses/train_all_loss_iou": 0.023255279706872696, "Losses/train_all_loss_class": 5.094582148262241e-09, "Losses/train_all_core_loss": 0.36432591524420266, "Trainer/where": 0.4249937374749499, "Trainer/epoch": 16, "Trainer/steps_train": 67864}
+{"Losses/train_all_loss": 0.3739850448014286, "Losses/train_all_loss_mask": 0.007661915094332017, "Losses/train_all_loss_dice": 0.19808480110698806, "Losses/train_all_loss_iou": 0.02266193838917906, "Losses/train_all_loss_class": 5.422859437734412e-09, "Losses/train_all_core_loss": 0.3739850448014286, "Trainer/where": 0.4499937374749499, "Trainer/epoch": 17, "Trainer/steps_train": 71856}
+{"Losses/train_all_loss": 0.3802372602399911, "Losses/train_all_loss_mask": 0.008000960067011056, "Losses/train_all_loss_dice": 0.19741905017761524, "Losses/train_all_loss_iou": 0.02279900476346511, "Losses/train_all_loss_class": 5.678085751427958e-09, "Losses/train_all_core_loss": 0.3802372602399911, "Trainer/where": 0.4749937374749499, "Trainer/epoch": 18, "Trainer/steps_train": 75848}
+{"Losses/train_all_loss": 0.376298183201997, "Losses/train_all_loss_mask": 0.007830308981010056, "Losses/train_all_loss_dice": 0.1965342678711983, "Losses/train_all_loss_iou": 0.023157731618651492, "Losses/train_all_loss_class": 5.28514186526349e-09, "Losses/train_all_core_loss": 0.376298183201997, "Trainer/where": 0.4999937374749499, "Trainer/epoch": 19, "Trainer/steps_train": 79840}
+{"Losses/train_all_loss": 0.3772340752292874, "Losses/train_all_loss_mask": 0.007914954837213077, "Losses/train_all_loss_dice": 0.19617746839064634, "Losses/train_all_loss_iou": 0.022757507024552415, "Losses/train_all_loss_class": 5.4267739055031245e-09, "Losses/train_all_core_loss": 0.3772340752292874, "Trainer/where": 0.5249937374749499, "Trainer/epoch": 20, "Trainer/steps_train": 83832}
+{"Losses/train_all_loss": 0.37047547260336655, "Losses/train_all_loss_mask": 0.007639010061916952, "Losses/train_all_loss_dice": 0.1958161407786763, "Losses/train_all_loss_iou": 0.02187912743158927, "Losses/train_all_loss_class": 5.4363083103324426e-09, "Losses/train_all_core_loss": 0.37047547260336655, "Trainer/where": 0.54999373747495, "Trainer/epoch": 21, "Trainer/steps_train": 87824}
+{"Losses/train_all_loss": 0.35376920676995477, "Losses/train_all_loss_mask": 0.006838303287161567, "Losses/train_all_loss_dice": 0.19547302056410032, "Losses/train_all_loss_iou": 0.021530116622054854, "Losses/train_all_loss_class": 6.322159236068461e-09, "Losses/train_all_core_loss": 0.35376920676995477, "Trainer/where": 0.5749937374749499, "Trainer/epoch": 22, "Trainer/steps_train": 91816}
+{"Losses/train_all_loss": 0.3551223263956834, "Losses/train_all_loss_mask": 0.006957760564867117, "Losses/train_all_loss_dice": 0.1945757962957413, "Losses/train_all_loss_iou": 0.02139131658811324, "Losses/train_all_loss_class": 4.68340592482518e-09, "Losses/train_all_core_loss": 0.3551223263956834, "Trainer/where": 0.5999937374749499, "Trainer/epoch": 23, "Trainer/steps_train": 95808}
+{"Losses/train_all_loss": 0.35436655999975974, "Losses/train_all_loss_mask": 0.006915995280616119, "Losses/train_all_loss_dice": 0.19451504708351736, "Losses/train_all_loss_iou": 0.0215316050213107, "Losses/train_all_loss_class": 4.694827900397807e-09, "Losses/train_all_core_loss": 0.35436655999975974, "Trainer/where": 0.6249937374749499, "Trainer/epoch": 24, "Trainer/steps_train": 99800}
+{"Losses/train_all_loss": 0.3539243154792313, "Losses/train_all_loss_mask": 0.0069243552351725595, "Losses/train_all_loss_dice": 0.19411168701364426, "Losses/train_all_loss_iou": 0.021325522290423973, "Losses/train_all_loss_class": 4.124210543847477e-09, "Losses/train_all_core_loss": 0.3539243154792313, "Trainer/where": 0.6499937374749499, "Trainer/epoch": 25, "Trainer/steps_train": 103792}
+{"Losses/train_all_loss": 0.3885851017036615, "Losses/train_all_loss_mask": 0.008587317138184215, "Losses/train_all_loss_dice": 0.1942436489498687, "Losses/train_all_loss_iou": 0.02259510975809444, "Losses/train_all_loss_class": 3.7102346460758284e-09, "Losses/train_all_core_loss": 0.3885851017036615, "Trainer/where": 0.67499373747495, "Trainer/epoch": 26, "Trainer/steps_train": 107784}
+{"Losses/train_all_loss": 0.7125891149229509, "Losses/train_all_loss_mask": 0.024751994295421284, "Losses/train_all_loss_dice": 0.194532726758826, "Losses/train_all_loss_iou": 0.023016485271324848, "Losses/train_all_loss_class": 3.9748640794428046e-09, "Losses/train_all_core_loss": 0.7125891149229509, "Trainer/where": 0.6999937374749499, "Trainer/epoch": 27, "Trainer/steps_train": 111776}
+{"Losses/train_all_loss": 0.8676063124507335, "Losses/train_all_loss_mask": 0.032492553048528834, "Losses/train_all_loss_dice": 0.1948043356587987, "Losses/train_all_loss_iou": 0.022950956187875175, "Losses/train_all_loss_class": 4.138141090095644e-09, "Losses/train_all_core_loss": 0.8676063124507335, "Trainer/where": 0.7249937374749499, "Trainer/epoch": 28, "Trainer/steps_train": 115768}
+{"Losses/train_all_loss": 0.3487479994164496, "Losses/train_all_loss_mask": 0.0067503434569706235, "Losses/train_all_loss_dice": 0.19311891421526373, "Losses/train_all_loss_iou": 0.020622213926316305, "Losses/train_all_loss_class": 4.467678567490371e-09, "Losses/train_all_core_loss": 0.3487479994164496, "Trainer/where": 0.7499937374749499, "Trainer/epoch": 29, "Trainer/steps_train": 119760}
+{"Losses/train_all_loss": 0.36419604813683787, "Losses/train_all_loss_mask": 0.00751875088206941, "Losses/train_all_loss_dice": 0.19322081335740005, "Losses/train_all_loss_iou": 0.0206002136427904, "Losses/train_all_loss_class": 5.008077296045301e-09, "Losses/train_all_core_loss": 0.36419604813683787, "Trainer/where": 0.7749937374749499, "Trainer/epoch": 30, "Trainer/steps_train": 123752}
+{"Losses/train_all_loss": 0.34595307377449336, "Losses/train_all_loss_mask": 0.006689208541691566, "Losses/train_all_loss_dice": 0.19237655501267714, "Losses/train_all_loss_iou": 0.01979234506802322, "Losses/train_all_loss_class": 5.1453944205931e-09, "Losses/train_all_core_loss": 0.34595307377449336, "Trainer/where": 0.79999373747495, "Trainer/epoch": 31, "Trainer/steps_train": 127744}
+{"Losses/train_all_loss": 0.34565046294508334, "Losses/train_all_loss_mask": 0.006636826759476352, "Losses/train_all_loss_dice": 0.19229123223281813, "Losses/train_all_loss_iou": 0.020622693491710736, "Losses/train_all_loss_class": 4.764466950168121e-09, "Losses/train_all_core_loss": 0.34565046294508334, "Trainer/where": 0.8249937374749499, "Trainer/epoch": 32, "Trainer/steps_train": 131736}
+{"Losses/train_all_loss": 0.9008659057842683, "Losses/train_all_loss_mask": 0.03423682985725872, "Losses/train_all_loss_dice": 0.1938983456405227, "Losses/train_all_loss_iou": 0.02223101315849819, "Losses/train_all_loss_class": 4.9622400122766974e-09, "Losses/train_all_core_loss": 0.9008659057842683, "Trainer/where": 0.8499937374749498, "Trainer/epoch": 33, "Trainer/steps_train": 135728}
+{"Losses/train_all_loss": 0.3465600523536368, "Losses/train_all_loss_mask": 0.0067609497240599245, "Losses/train_all_loss_dice": 0.19183460045195294, "Losses/train_all_loss_iou": 0.019506455022350135, "Losses/train_all_loss_class": 4.679632632014728e-09, "Losses/train_all_core_loss": 0.3465600523536368, "Trainer/where": 0.8749937374749498, "Trainer/epoch": 34, "Trainer/steps_train": 139720}
+{"Losses/train_all_loss": 0.34736052319718447, "Losses/train_all_loss_mask": 0.0067391459940458545, "Losses/train_all_loss_dice": 0.1922520945270458, "Losses/train_all_loss_iou": 0.020325507099748605, "Losses/train_all_loss_class": 4.45559715646195e-09, "Losses/train_all_core_loss": 0.34736052319718447, "Trainer/where": 0.8999937374749498, "Trainer/epoch": 35, "Trainer/steps_train": 143712}
+{"Losses/train_all_loss": 0.3672967086860374, "Losses/train_all_loss_mask": 0.00773209927745972, "Losses/train_all_loss_dice": 0.19209734350442886, "Losses/train_all_loss_iou": 0.02055737860401078, "Losses/train_all_loss_class": 4.592152288894342e-09, "Losses/train_all_core_loss": 0.3672967086860374, "Trainer/where": 0.9249937374749498, "Trainer/epoch": 36, "Trainer/steps_train": 147704}
+{"Losses/train_all_loss": 0.8421824378987265, "Losses/train_all_loss_mask": 0.03133104354486921, "Losses/train_all_loss_dice": 0.19316817402003522, "Losses/train_all_loss_iou": 0.02239339469971622, "Losses/train_all_loss_class": 4.4197077939473966e-09, "Losses/train_all_core_loss": 0.8421824378987265, "Trainer/where": 0.9499937374749499, "Trainer/epoch": 37, "Trainer/steps_train": 151696}
+{"Losses/train_all_loss": 0.3468215012218792, "Losses/train_all_loss_mask": 0.006743870759146193, "Losses/train_all_loss_dice": 0.19145133795742997, "Losses/train_all_loss_iou": 0.020492745865770254, "Losses/train_all_loss_class": 4.498966078856853e-09, "Losses/train_all_core_loss": 0.3468215012218792, "Trainer/where": 0.9749937374749498, "Trainer/epoch": 38, "Trainer/steps_train": 155688}
+{"Losses/train_all_loss": 0.361319512792094, "Losses/train_all_loss_mask": 0.007438624991261454, "Losses/train_all_loss_dice": 0.19195612370669243, "Losses/train_all_loss_iou": 0.0205908868190231, "Losses/train_all_loss_class": 4.3955577136424175e-09, "Losses/train_all_core_loss": 0.361319512792094, "Trainer/where": 0.9999937374749498, "Trainer/epoch": 39, "Trainer/steps_train": 159680}

experiment/thermal_run1/tensorboard/events.out.tfevents.1759750510.gpub060.delta.ncsa.illinois.edu.921762.0c94822e4-1523-4465-a797-8a6fd58b2af0

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:872d92f23e17cdc70e7c071350d5ac8cd50f1b9f4fb30f6477f5c6def1e2256d
+size 14811175

sam2.1_hiera_base_plus.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:a2345aede8715ab1d5d31b4a509fb160c5a4af1970f199d9054ccfb746c004c5
+size 323606802

sam2.1_hiera_large.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:2647878d5dfa5098f2f8649825738a9345572bae2d4350a2468587ece47dd318
+size 898083611

sam2.1_hiera_small.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:6d1aa6f30de5c92224f8172114de081d104bbd23dd9dc5c58996f0cad5dc4d38
+size 184416285