diff --git a/experiment/thermal_run1/checkpoints/checkpoint.pt b/experiment/thermal_run1/checkpoints/checkpoint.pt new file mode 100644 index 0000000000000000000000000000000000000000..db3ab6d9e367e3d11a7db5e97dd003d7c744c0cf --- /dev/null +++ b/experiment/thermal_run1/checkpoints/checkpoint.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:a08cacd09cd24a7afa1704b1b728229b0c2d26f85b8b8bc71c4c938816f3cddd +size 910662494 diff --git a/experiment/thermal_run1/checkpoints/checkpoint_1.pt b/experiment/thermal_run1/checkpoints/checkpoint_1.pt new file mode 100644 index 0000000000000000000000000000000000000000..429737fd0ef0a29acb0ec5c2138e5bdb0a4a356a --- /dev/null +++ b/experiment/thermal_run1/checkpoints/checkpoint_1.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:8be8e0e9be5d1bd7f16e7bdd5dde033a23e380a7f8f35b605db755ee6fe6c935 +size 910662430 diff --git a/experiment/thermal_run1/checkpoints/checkpoint_10.pt b/experiment/thermal_run1/checkpoints/checkpoint_10.pt new file mode 100644 index 0000000000000000000000000000000000000000..b3b7508e02b6ee9646b194562369ce8950ed1049 --- /dev/null +++ b/experiment/thermal_run1/checkpoints/checkpoint_10.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:bf4aed853f9a17594becbb8e5be9083aede7dee0f1a8a925bdaf5fea34556113 +size 910662494 diff --git a/experiment/thermal_run1/checkpoints/checkpoint_11.pt b/experiment/thermal_run1/checkpoints/checkpoint_11.pt new file mode 100644 index 0000000000000000000000000000000000000000..87d2ed6d1a7118c1d5cb4dd67700337bc7e658cd --- /dev/null +++ b/experiment/thermal_run1/checkpoints/checkpoint_11.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:e988c851be054f9c9358b0e7898962705ec7dbf66b9c6b6ecd17eee53d2e7ef7 +size 910662494 diff --git a/experiment/thermal_run1/checkpoints/checkpoint_12.pt b/experiment/thermal_run1/checkpoints/checkpoint_12.pt new file mode 100644 index 0000000000000000000000000000000000000000..f0b163e2b7d0a1512296d28a20b06424209ecf2d --- /dev/null +++ b/experiment/thermal_run1/checkpoints/checkpoint_12.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:741638750ae87f08f11310a62b6629b0621742e260e2379b39568f2d450c26c6 +size 910662494 diff --git a/experiment/thermal_run1/checkpoints/checkpoint_13.pt b/experiment/thermal_run1/checkpoints/checkpoint_13.pt new file mode 100644 index 0000000000000000000000000000000000000000..8aa43bcbeca2ed99592c90859f9b411c0d50a48d --- /dev/null +++ b/experiment/thermal_run1/checkpoints/checkpoint_13.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:7127b6697fdc6af359133b1f581f8e02bc991bdc208de84dffc1ef5fb71a43f0 +size 910662494 diff --git a/experiment/thermal_run1/checkpoints/checkpoint_14.pt b/experiment/thermal_run1/checkpoints/checkpoint_14.pt new file mode 100644 index 0000000000000000000000000000000000000000..2e5f5e19bf042635b922368596b311d57bc80419 --- /dev/null +++ b/experiment/thermal_run1/checkpoints/checkpoint_14.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:65557d72abed4de1571980f2a8a1f209479bead51c8586e347ff5414c2983689 +size 910662494 diff --git a/experiment/thermal_run1/checkpoints/checkpoint_15.pt b/experiment/thermal_run1/checkpoints/checkpoint_15.pt new file mode 100644 index 0000000000000000000000000000000000000000..205b34fe40d343590f9be6e0bf09175733bf8ec1 --- /dev/null +++ b/experiment/thermal_run1/checkpoints/checkpoint_15.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:154609c3c910e5328c62ddb7a7a69cffb8ac3c808558296192852c1abcce1211 +size 910662494 diff --git a/experiment/thermal_run1/checkpoints/checkpoint_16.pt b/experiment/thermal_run1/checkpoints/checkpoint_16.pt new file mode 100644 index 0000000000000000000000000000000000000000..1bff0bd5cfe4e4546fb0f8c047fee6992ab11e30 --- /dev/null +++ b/experiment/thermal_run1/checkpoints/checkpoint_16.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:28a5f59da4849fa333917f55c52b1b6a25d7b4732d6714fb5c8c066d93a03ab1 +size 910662494 diff --git a/experiment/thermal_run1/checkpoints/checkpoint_17.pt b/experiment/thermal_run1/checkpoints/checkpoint_17.pt new file mode 100644 index 0000000000000000000000000000000000000000..f7d9a3e9c66f3f9ffc8615c22bda947697038bf3 --- /dev/null +++ b/experiment/thermal_run1/checkpoints/checkpoint_17.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:97b9f700b06a797f06049f86294bffb405fc2bdba2249bf2ab34e5f836a6ab4a +size 910662494 diff --git a/experiment/thermal_run1/checkpoints/checkpoint_18.pt b/experiment/thermal_run1/checkpoints/checkpoint_18.pt new file mode 100644 index 0000000000000000000000000000000000000000..c16900e6141d49e947ea7efa5a1089ec81af7f61 --- /dev/null +++ b/experiment/thermal_run1/checkpoints/checkpoint_18.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:e90e450f66a8181bead198544fd60de5f37e40d474a402b6cafa3c10dd923725 +size 910662494 diff --git a/experiment/thermal_run1/checkpoints/checkpoint_19.pt b/experiment/thermal_run1/checkpoints/checkpoint_19.pt new file mode 100644 index 0000000000000000000000000000000000000000..fc73a0ff1c4451a3c1335ab18d018b662c6be9e2 --- /dev/null +++ b/experiment/thermal_run1/checkpoints/checkpoint_19.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:18141f1ba0926e7e3f013313f04d7ea6d7e925a46c0e1b9b83f5dd2aa11dfaaa +size 910662494 diff --git a/experiment/thermal_run1/checkpoints/checkpoint_2.pt b/experiment/thermal_run1/checkpoints/checkpoint_2.pt new file mode 100644 index 0000000000000000000000000000000000000000..5a73a99b16c79f052baed565ad42ea86e806ef5f --- /dev/null +++ b/experiment/thermal_run1/checkpoints/checkpoint_2.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:8ce4f830aea93cf1923ba745483ea278063876bc4427650d095a8bf2e397d035 +size 910662494 diff --git a/experiment/thermal_run1/checkpoints/checkpoint_20.pt b/experiment/thermal_run1/checkpoints/checkpoint_20.pt new file mode 100644 index 0000000000000000000000000000000000000000..899af8ed3665a8b4dd4e554c34af32f974e180db --- /dev/null +++ b/experiment/thermal_run1/checkpoints/checkpoint_20.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:5723da2d8804c7bc5e6c52ae85faa90fc81e6eaa5c8aa7ccf82fc15dba07581b +size 910662494 diff --git a/experiment/thermal_run1/checkpoints/checkpoint_21.pt b/experiment/thermal_run1/checkpoints/checkpoint_21.pt new file mode 100644 index 0000000000000000000000000000000000000000..df844e9253eca60a0e8921f52458c4b516dc4927 --- /dev/null +++ b/experiment/thermal_run1/checkpoints/checkpoint_21.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:a16f8b06bdc8d228f1b465aede71197ad4a3e33eba292550bc94b3025c47f96d +size 910662494 diff --git a/experiment/thermal_run1/checkpoints/checkpoint_22.pt b/experiment/thermal_run1/checkpoints/checkpoint_22.pt new file mode 100644 index 0000000000000000000000000000000000000000..a15a8a8bebe90d521d733c1d2d3df24c654b0162 --- /dev/null +++ b/experiment/thermal_run1/checkpoints/checkpoint_22.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:8913122ef939840271327177ddbee90bec6ccd1d4598030d479117dd4676139b +size 910662494 diff --git a/experiment/thermal_run1/checkpoints/checkpoint_23.pt b/experiment/thermal_run1/checkpoints/checkpoint_23.pt new file mode 100644 index 0000000000000000000000000000000000000000..122a871797ce68b0b13ff2d7302707272b6a6f13 --- /dev/null +++ b/experiment/thermal_run1/checkpoints/checkpoint_23.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:c897512fbc9533c6ae0171af2f735cc6618e9f2af5c69d761c26123efe82f158 +size 910662494 diff --git a/experiment/thermal_run1/checkpoints/checkpoint_24.pt b/experiment/thermal_run1/checkpoints/checkpoint_24.pt new file mode 100644 index 0000000000000000000000000000000000000000..a58d140c317b11782c0791ff9e82ffb1a6c0f7bb --- /dev/null +++ b/experiment/thermal_run1/checkpoints/checkpoint_24.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:5c4e754eee30de9285b3e59c6b953eae6423d0d0938139b3ba29f1a76aee04a5 +size 910662494 diff --git a/experiment/thermal_run1/checkpoints/checkpoint_25.pt b/experiment/thermal_run1/checkpoints/checkpoint_25.pt new file mode 100644 index 0000000000000000000000000000000000000000..ec1a84e4dbf155e1aacf0bfa0708e531c4c3d1b0 --- /dev/null +++ b/experiment/thermal_run1/checkpoints/checkpoint_25.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:a58e92dd643fc840ac89e729580cb3caa78f6747725bae72a602934c07d64af6 +size 910662494 diff --git a/experiment/thermal_run1/checkpoints/checkpoint_26.pt b/experiment/thermal_run1/checkpoints/checkpoint_26.pt new file mode 100644 index 0000000000000000000000000000000000000000..56ab3d0f193ca23c4d44d416e09c5277badffc70 --- /dev/null +++ b/experiment/thermal_run1/checkpoints/checkpoint_26.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:94d51b6043964ee7ff657ecb0d3aa3fc811ce72806141469982ec46eefcbcd54 +size 910662494 diff --git a/experiment/thermal_run1/checkpoints/checkpoint_27.pt b/experiment/thermal_run1/checkpoints/checkpoint_27.pt new file mode 100644 index 0000000000000000000000000000000000000000..64e6d4275c919b98454e9312ca9c251c12170988 --- /dev/null +++ b/experiment/thermal_run1/checkpoints/checkpoint_27.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:deb0f245104633753716ceebdd899ef41b62dea134e50815b3d2fdc2bd83a30f +size 910662494 diff --git a/experiment/thermal_run1/checkpoints/checkpoint_28.pt b/experiment/thermal_run1/checkpoints/checkpoint_28.pt new file mode 100644 index 0000000000000000000000000000000000000000..e0ed3ea549b62e94b27afa091bb78e7092c5bd09 --- /dev/null +++ b/experiment/thermal_run1/checkpoints/checkpoint_28.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:5615b19809b17430a340ea91456ea928010b3fe40c0ae599d51d09e3d4ed7e7e +size 910662494 diff --git a/experiment/thermal_run1/checkpoints/checkpoint_29.pt b/experiment/thermal_run1/checkpoints/checkpoint_29.pt new file mode 100644 index 0000000000000000000000000000000000000000..8b1c167251daca6c8a641a8741ad1b7058112790 --- /dev/null +++ b/experiment/thermal_run1/checkpoints/checkpoint_29.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:f9c1198b55bb8a39cb9d5adf80306a9a79508b541e48252a2ed1a3e6263d0566 +size 910662494 diff --git a/experiment/thermal_run1/checkpoints/checkpoint_3.pt b/experiment/thermal_run1/checkpoints/checkpoint_3.pt new file mode 100644 index 0000000000000000000000000000000000000000..56c3b32a08db0db2d6ca2353fec097b5fd014e48 --- /dev/null +++ b/experiment/thermal_run1/checkpoints/checkpoint_3.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:ab49e18e7cf1dbb6bcda0c8769cb1f7c29be04e1db72552a9ac12cb1eb502fe6 +size 910662494 diff --git a/experiment/thermal_run1/checkpoints/checkpoint_30.pt b/experiment/thermal_run1/checkpoints/checkpoint_30.pt new file mode 100644 index 0000000000000000000000000000000000000000..9ed5db26b280b9411a040c802700a35562571dc4 --- /dev/null +++ b/experiment/thermal_run1/checkpoints/checkpoint_30.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:121f13583903cc30bc200fa77eacb0c0d0cec5302d9612055b046e0b1f787fe8 +size 910662494 diff --git a/experiment/thermal_run1/checkpoints/checkpoint_31.pt b/experiment/thermal_run1/checkpoints/checkpoint_31.pt new file mode 100644 index 0000000000000000000000000000000000000000..4f60aa31e0b9f0420d6268eefe2dbfa22cc65427 --- /dev/null +++ b/experiment/thermal_run1/checkpoints/checkpoint_31.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:9b87ed7ead47522fe96481d9f140aa56365b5633a00cd4ff28473d03aa366a3e +size 910662494 diff --git a/experiment/thermal_run1/checkpoints/checkpoint_32.pt b/experiment/thermal_run1/checkpoints/checkpoint_32.pt new file mode 100644 index 0000000000000000000000000000000000000000..434d18e47527806ee2bd22bb7cf2cbf5d68430d2 --- /dev/null +++ b/experiment/thermal_run1/checkpoints/checkpoint_32.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:3c972da36a8eb74204a8c2aaa24dbf108ba470e94e1996e2925d2ed78fd83115 +size 910662494 diff --git a/experiment/thermal_run1/checkpoints/checkpoint_33.pt b/experiment/thermal_run1/checkpoints/checkpoint_33.pt new file mode 100644 index 0000000000000000000000000000000000000000..2374b8f6a5f8383f1e8aaa5e8e83ff9e99a85856 --- /dev/null +++ b/experiment/thermal_run1/checkpoints/checkpoint_33.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:559dcdb27c480d5a437fc1495ff01756822150051ea82364a6b3ad9b2bd4194a +size 910662494 diff --git a/experiment/thermal_run1/checkpoints/checkpoint_34.pt b/experiment/thermal_run1/checkpoints/checkpoint_34.pt new file mode 100644 index 0000000000000000000000000000000000000000..d12ca16169d54cd2cba6bb1c07088016022e3791 --- /dev/null +++ b/experiment/thermal_run1/checkpoints/checkpoint_34.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:1fe3db97948547e415a25243ba68263b26a51ac0266cf8ba5cbbbcbdcf78cc88 +size 910662494 diff --git a/experiment/thermal_run1/checkpoints/checkpoint_35.pt b/experiment/thermal_run1/checkpoints/checkpoint_35.pt new file mode 100644 index 0000000000000000000000000000000000000000..4a429853db3a00d1d8484767e9b6552914a6e751 --- /dev/null +++ b/experiment/thermal_run1/checkpoints/checkpoint_35.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:535c531bfefc6ca4cd50d387f961b303232a3c2fa218399ab5e77ac7328355ae +size 910662494 diff --git a/experiment/thermal_run1/checkpoints/checkpoint_36.pt b/experiment/thermal_run1/checkpoints/checkpoint_36.pt new file mode 100644 index 0000000000000000000000000000000000000000..6fdcdf74a0706fc2ba3eb81965a4514bb9a591e4 --- /dev/null +++ b/experiment/thermal_run1/checkpoints/checkpoint_36.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:6296c062a7b607dd86dc000b9857bee0eb04cc3362922652818f16b45e2d866d +size 910662494 diff --git a/experiment/thermal_run1/checkpoints/checkpoint_37.pt b/experiment/thermal_run1/checkpoints/checkpoint_37.pt new file mode 100644 index 0000000000000000000000000000000000000000..b1c976e6167803482a9f1c7608a7e0d0cea52b96 --- /dev/null +++ b/experiment/thermal_run1/checkpoints/checkpoint_37.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:c0e196e40a20f07a7356a4cde873430988e4131e587a65ad7883977e611302fa +size 910662494 diff --git a/experiment/thermal_run1/checkpoints/checkpoint_38.pt b/experiment/thermal_run1/checkpoints/checkpoint_38.pt new file mode 100644 index 0000000000000000000000000000000000000000..cf4e23caef5f69ac942d1c32372d5275cc1b9c58 --- /dev/null +++ b/experiment/thermal_run1/checkpoints/checkpoint_38.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:5370354b5e917526695cb893303d3b2ff55ed11ff4c7a395c8c5c49da4bfc1ee +size 910662494 diff --git a/experiment/thermal_run1/checkpoints/checkpoint_39.pt b/experiment/thermal_run1/checkpoints/checkpoint_39.pt new file mode 100644 index 0000000000000000000000000000000000000000..0680367e8f6304503b3d36fcbe7fcef3330625ea --- /dev/null +++ b/experiment/thermal_run1/checkpoints/checkpoint_39.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:2ab630fa6bd6535ee263f5aa584281f268b9ece7d5501e8564bba7696fac07f1 +size 910662494 diff --git a/experiment/thermal_run1/checkpoints/checkpoint_4.pt b/experiment/thermal_run1/checkpoints/checkpoint_4.pt new file mode 100644 index 0000000000000000000000000000000000000000..09c9e41a4e75cfd8d5b4d2c49b6abf91984f2bf1 --- /dev/null +++ b/experiment/thermal_run1/checkpoints/checkpoint_4.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:4173e21af1808616ece284564e2d8ba44ade31ccec5ecb457b46d172bc2e6d73 +size 910662494 diff --git a/experiment/thermal_run1/checkpoints/checkpoint_40.pt b/experiment/thermal_run1/checkpoints/checkpoint_40.pt new file mode 100644 index 0000000000000000000000000000000000000000..db3ab6d9e367e3d11a7db5e97dd003d7c744c0cf --- /dev/null +++ b/experiment/thermal_run1/checkpoints/checkpoint_40.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:a08cacd09cd24a7afa1704b1b728229b0c2d26f85b8b8bc71c4c938816f3cddd +size 910662494 diff --git a/experiment/thermal_run1/checkpoints/checkpoint_5.pt b/experiment/thermal_run1/checkpoints/checkpoint_5.pt new file mode 100644 index 0000000000000000000000000000000000000000..eab6d1cffa8edc44f26bd615e958d0b3d24ff143 --- /dev/null +++ b/experiment/thermal_run1/checkpoints/checkpoint_5.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:2cda5c769cdb24325396bf8fa9601a1adc277ffdeb9a4bc2ae4ca2c8a92e879d +size 910662494 diff --git a/experiment/thermal_run1/checkpoints/checkpoint_6.pt b/experiment/thermal_run1/checkpoints/checkpoint_6.pt new file mode 100644 index 0000000000000000000000000000000000000000..d40159736b687247e6b3dad59109f4a78f195b61 --- /dev/null +++ b/experiment/thermal_run1/checkpoints/checkpoint_6.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:a772cd7044f36d52d9a96ea314f7a09590f85f42d2c3534d86ae0f8ddc2ffa3d +size 910662494 diff --git a/experiment/thermal_run1/checkpoints/checkpoint_7.pt b/experiment/thermal_run1/checkpoints/checkpoint_7.pt new file mode 100644 index 0000000000000000000000000000000000000000..d4b144bdca572edf89fab9c2072ddc689b71dad8 --- /dev/null +++ b/experiment/thermal_run1/checkpoints/checkpoint_7.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:53ebacbd40cf8a0855fbdedb7f9795d80f169f18ae02aa2b146a065c521d8a97 +size 910662494 diff --git a/experiment/thermal_run1/checkpoints/checkpoint_8.pt b/experiment/thermal_run1/checkpoints/checkpoint_8.pt new file mode 100644 index 0000000000000000000000000000000000000000..a88f6a0c0ff711b175b6ab7efa854b2595a4b37c --- /dev/null +++ b/experiment/thermal_run1/checkpoints/checkpoint_8.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:c24089637368598c6ae5302fbf61c1ef46994a950cbd4c5f3c114a82e925f6a1 +size 910662494 diff --git a/experiment/thermal_run1/checkpoints/checkpoint_9.pt b/experiment/thermal_run1/checkpoints/checkpoint_9.pt new file mode 100644 index 0000000000000000000000000000000000000000..7164835f4cafda48462a4f41fa4285816c4f3197 --- /dev/null +++ b/experiment/thermal_run1/checkpoints/checkpoint_9.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:e2f521f3db0b6438a03e61df33d1339ca910a8c89f9a8fe222b1481bb5a8a384 +size 910662494 diff --git a/experiment/thermal_run1/config.yaml b/experiment/thermal_run1/config.yaml new file mode 100644 index 0000000000000000000000000000000000000000..24d01e69d26b6d479dc2f3dc8183ddf69bd25ae0 --- /dev/null +++ b/experiment/thermal_run1/config.yaml @@ -0,0 +1,313 @@ +scratch: + resolution: 1024 + train_batch_size: 1 + num_train_workers: 10 + num_frames: 1 + max_num_objects: 3 + base_lr: 5.0e-06 + vision_lr: 3.0e-06 + phases_per_epoch: 1 + num_epochs: 40 +dataset: + img_folder: /work/nvme/bfgd/cmaduabuchi/DATA/thermal_video/thriveseg/train/images + gt_folder: /work/nvme/bfgd/cmaduabuchi/DATA/thermal_video/thriveseg/train/masks + file_list_txt: null + multiplier: 2 +vos: + train_transforms: + - _target_: training.dataset.transforms.ComposeAPI + transforms: + - _target_: training.dataset.transforms.RandomHorizontalFlip + consistent_transform: true + - _target_: training.dataset.transforms.RandomAffine + degrees: 25 + shear: 20 + image_interpolation: bilinear + consistent_transform: true + - _target_: training.dataset.transforms.RandomResizeAPI + sizes: ${scratch.resolution} + square: true + consistent_transform: true + - _target_: training.dataset.transforms.ColorJitter + consistent_transform: true + brightness: 0.1 + contrast: 0.03 + saturation: 0.03 + hue: null + - _target_: training.dataset.transforms.RandomGrayscale + p: 0.05 + consistent_transform: true + - _target_: training.dataset.transforms.ColorJitter + consistent_transform: false + brightness: 0.1 + contrast: 0.05 + saturation: 0.05 + hue: null + - _target_: training.dataset.transforms.ToTensorAPI + - _target_: training.dataset.transforms.NormalizeAPI + mean: + - 0.485 + - 0.456 + - 0.406 + std: + - 0.229 + - 0.224 + - 0.225 +trainer: + _target_: training.trainer.Trainer + mode: train_only + max_epochs: ${times:${scratch.num_epochs},${scratch.phases_per_epoch}} + accelerator: cuda + seed_value: 123 + model: + _target_: training.model.sam2.SAM2Train + image_encoder: + _target_: sam2.modeling.backbones.image_encoder.ImageEncoder + scalp: 1 + trunk: + _target_: sam2.modeling.backbones.hieradet.Hiera + embed_dim: 112 + num_heads: 2 + drop_path_rate: 0.1 + neck: + _target_: sam2.modeling.backbones.image_encoder.FpnNeck + position_encoding: + _target_: sam2.modeling.position_encoding.PositionEmbeddingSine + num_pos_feats: 256 + normalize: true + scale: null + temperature: 10000 + d_model: 256 + backbone_channel_list: + - 896 + - 448 + - 224 + - 112 + fpn_top_down_levels: + - 2 + - 3 + fpn_interp_model: nearest + memory_attention: + _target_: sam2.modeling.memory_attention.MemoryAttention + d_model: 256 + pos_enc_at_input: true + layer: + _target_: sam2.modeling.memory_attention.MemoryAttentionLayer + activation: relu + dim_feedforward: 2048 + dropout: 0.1 + pos_enc_at_attn: false + self_attention: + _target_: sam2.modeling.sam.transformer.RoPEAttention + rope_theta: 10000.0 + feat_sizes: + - 64 + - 64 + embedding_dim: 256 + num_heads: 1 + downsample_rate: 1 + dropout: 0.1 + d_model: 256 + pos_enc_at_cross_attn_keys: true + pos_enc_at_cross_attn_queries: false + cross_attention: + _target_: sam2.modeling.sam.transformer.RoPEAttention + rope_theta: 10000.0 + feat_sizes: + - 64 + - 64 + rope_k_repeat: true + embedding_dim: 256 + num_heads: 1 + downsample_rate: 1 + dropout: 0.1 + kv_in_dim: 64 + num_layers: 4 + memory_encoder: + _target_: sam2.modeling.memory_encoder.MemoryEncoder + out_dim: 64 + position_encoding: + _target_: sam2.modeling.position_encoding.PositionEmbeddingSine + num_pos_feats: 64 + normalize: true + scale: null + temperature: 10000 + mask_downsampler: + _target_: sam2.modeling.memory_encoder.MaskDownSampler + kernel_size: 3 + stride: 2 + padding: 1 + fuser: + _target_: sam2.modeling.memory_encoder.Fuser + layer: + _target_: sam2.modeling.memory_encoder.CXBlock + dim: 256 + kernel_size: 7 + padding: 3 + layer_scale_init_value: 1.0e-06 + use_dwconv: true + num_layers: 2 + num_maskmem: 7 + image_size: ${scratch.resolution} + sigmoid_scale_for_mem_enc: 20.0 + sigmoid_bias_for_mem_enc: -10.0 + use_mask_input_as_output_without_sam: true + directly_add_no_mem_embed: true + no_obj_embed_spatial: true + use_high_res_features_in_sam: true + multimask_output_in_sam: true + iou_prediction_use_sigmoid: true + use_obj_ptrs_in_encoder: true + add_tpos_enc_to_obj_ptrs: true + proj_tpos_enc_in_obj_ptrs: true + use_signed_tpos_enc_to_obj_ptrs: true + only_obj_ptrs_in_the_past_for_eval: true + pred_obj_scores: true + pred_obj_scores_mlp: true + fixed_no_obj_ptr: true + multimask_output_for_tracking: true + use_multimask_token_for_obj_ptr: true + multimask_min_pt_num: 0 + multimask_max_pt_num: 1 + use_mlp_for_obj_ptr_proj: true + prob_to_use_pt_input_for_train: 0.5 + prob_to_use_pt_input_for_eval: 0.0 + prob_to_use_box_input_for_train: 0.5 + prob_to_use_box_input_for_eval: 0.0 + prob_to_sample_from_gt_for_train: 0.1 + num_frames_to_correct_for_train: 2 + num_frames_to_correct_for_eval: 1 + rand_frames_to_correct_for_train: true + add_all_frames_to_correct_as_cond: true + num_init_cond_frames_for_train: 2 + rand_init_cond_frames_for_train: true + num_correction_pt_per_frame: 7 + use_act_ckpt_iterative_pt_sampling: false + num_init_cond_frames_for_eval: 1 + forward_backbone_per_frame_for_eval: true + data: + train: + _target_: training.dataset.sam2_datasets.TorchTrainMixedDataset + phases_per_epoch: ${scratch.phases_per_epoch} + batch_sizes: + - ${scratch.train_batch_size} + datasets: + - _target_: training.dataset.utils.RepeatFactorWrapper + dataset: + _target_: training.dataset.utils.ConcatDataset + datasets: + - _target_: training.dataset.vos_dataset.VOSDataset + transforms: ${vos.train_transforms} + training: true + video_dataset: + _target_: training.dataset.binary_png_raw_dataset.BinaryPNGRawDataset + img_folder: ${dataset.img_folder} + gt_folder: ${dataset.gt_folder} + file_list_txt: ${dataset.file_list_txt} + sampler: + _target_: training.dataset.vos_sampler.RandomUniformSampler + num_frames: ${scratch.num_frames} + max_num_objects: ${scratch.max_num_objects} + multiplier: ${dataset.multiplier} + shuffle: true + num_workers: ${scratch.num_train_workers} + pin_memory: true + drop_last: true + collate_fn: + _target_: training.utils.data_utils.collate_fn + _partial_: true + dict_key: all + optim: + amp: + enabled: true + amp_dtype: bfloat16 + optimizer: + _target_: torch.optim.AdamW + gradient_clip: + _target_: training.optimizer.GradientClipper + max_norm: 0.1 + norm_type: 2 + param_group_modifiers: + - _target_: training.optimizer.layer_decay_param_modifier + _partial_: true + layer_decay_value: 0.9 + apply_to: image_encoder.trunk + overrides: + - pattern: '*pos_embed*' + value: 1.0 + options: + lr: + - scheduler: + _target_: fvcore.common.param_scheduler.CosineParamScheduler + start_value: ${scratch.base_lr} + end_value: ${divide:${scratch.base_lr},10} + - scheduler: + _target_: fvcore.common.param_scheduler.CosineParamScheduler + start_value: ${scratch.vision_lr} + end_value: ${divide:${scratch.vision_lr},10} + param_names: + - image_encoder.* + weight_decay: + - scheduler: + _target_: fvcore.common.param_scheduler.ConstantParamScheduler + value: 0.1 + - scheduler: + _target_: fvcore.common.param_scheduler.ConstantParamScheduler + value: 0.0 + param_names: + - '*bias*' + module_cls_names: + - torch.nn.LayerNorm + loss: + all: + _target_: training.loss_fns.MultiStepMultiMasksAndIous + weight_dict: + loss_mask: 20 + loss_dice: 1 + loss_iou: 1 + loss_class: 1 + supervise_all_iou: true + iou_use_l1_loss: true + pred_obj_scores: true + focal_gamma_obj_score: 0.0 + focal_alpha_obj_score: -1.0 + distributed: + backend: nccl + find_unused_parameters: true + logging: + tensorboard_writer: + _target_: training.utils.logger.make_tensorboard_logger + log_dir: ${launcher.experiment_log_dir}/tensorboard + flush_secs: 120 + should_log: true + log_dir: ${launcher.experiment_log_dir}/logs + log_freq: 10 + checkpoint: + save_dir: ${launcher.experiment_log_dir}/checkpoints + save_freq: 1 + model_weight_initializer: + _partial_: true + _target_: training.utils.checkpoint_utils.load_state_dict_into_model + strict: true + ignore_unexpected_keys: null + ignore_missing_keys: null + state_dict: + _target_: training.utils.checkpoint_utils.load_checkpoint_and_apply_kernels + checkpoint_path: /work/nvme/bfgd/cmaduabuchi/MODELS/THRIVESEG/sam2.1_hiera_base_plus.pt + ckpt_state_dict_keys: + - model +launcher: + num_nodes: 1 + gpus_per_node: 8 + experiment_log_dir: /work/nvme/bfgd/cmaduabuchi/MODELS/THRIVESEG/experiment/thermal_run1 +submitit: + partition: null + account: null + qos: null + cpus_per_task: 10 + use_cluster: false + timeout_hour: 24 + name: null + port_range: + - 10000 + - 65000 diff --git a/experiment/thermal_run1/config_resolved.yaml b/experiment/thermal_run1/config_resolved.yaml new file mode 100644 index 0000000000000000000000000000000000000000..32c82b85dddec9c6474d76a9f27362fbff346f04 --- /dev/null +++ b/experiment/thermal_run1/config_resolved.yaml @@ -0,0 +1,351 @@ +scratch: + resolution: 1024 + train_batch_size: 1 + num_train_workers: 10 + num_frames: 1 + max_num_objects: 3 + base_lr: 5.0e-06 + vision_lr: 3.0e-06 + phases_per_epoch: 1 + num_epochs: 40 +dataset: + img_folder: /work/nvme/bfgd/cmaduabuchi/DATA/thermal_video/thriveseg/train/images + gt_folder: /work/nvme/bfgd/cmaduabuchi/DATA/thermal_video/thriveseg/train/masks + file_list_txt: null + multiplier: 2 +vos: + train_transforms: + - _target_: training.dataset.transforms.ComposeAPI + transforms: + - _target_: training.dataset.transforms.RandomHorizontalFlip + consistent_transform: true + - _target_: training.dataset.transforms.RandomAffine + degrees: 25 + shear: 20 + image_interpolation: bilinear + consistent_transform: true + - _target_: training.dataset.transforms.RandomResizeAPI + sizes: 1024 + square: true + consistent_transform: true + - _target_: training.dataset.transforms.ColorJitter + consistent_transform: true + brightness: 0.1 + contrast: 0.03 + saturation: 0.03 + hue: null + - _target_: training.dataset.transforms.RandomGrayscale + p: 0.05 + consistent_transform: true + - _target_: training.dataset.transforms.ColorJitter + consistent_transform: false + brightness: 0.1 + contrast: 0.05 + saturation: 0.05 + hue: null + - _target_: training.dataset.transforms.ToTensorAPI + - _target_: training.dataset.transforms.NormalizeAPI + mean: + - 0.485 + - 0.456 + - 0.406 + std: + - 0.229 + - 0.224 + - 0.225 +trainer: + _target_: training.trainer.Trainer + mode: train_only + max_epochs: 40 + accelerator: cuda + seed_value: 123 + model: + _target_: training.model.sam2.SAM2Train + image_encoder: + _target_: sam2.modeling.backbones.image_encoder.ImageEncoder + scalp: 1 + trunk: + _target_: sam2.modeling.backbones.hieradet.Hiera + embed_dim: 112 + num_heads: 2 + drop_path_rate: 0.1 + neck: + _target_: sam2.modeling.backbones.image_encoder.FpnNeck + position_encoding: + _target_: sam2.modeling.position_encoding.PositionEmbeddingSine + num_pos_feats: 256 + normalize: true + scale: null + temperature: 10000 + d_model: 256 + backbone_channel_list: + - 896 + - 448 + - 224 + - 112 + fpn_top_down_levels: + - 2 + - 3 + fpn_interp_model: nearest + memory_attention: + _target_: sam2.modeling.memory_attention.MemoryAttention + d_model: 256 + pos_enc_at_input: true + layer: + _target_: sam2.modeling.memory_attention.MemoryAttentionLayer + activation: relu + dim_feedforward: 2048 + dropout: 0.1 + pos_enc_at_attn: false + self_attention: + _target_: sam2.modeling.sam.transformer.RoPEAttention + rope_theta: 10000.0 + feat_sizes: + - 64 + - 64 + embedding_dim: 256 + num_heads: 1 + downsample_rate: 1 + dropout: 0.1 + d_model: 256 + pos_enc_at_cross_attn_keys: true + pos_enc_at_cross_attn_queries: false + cross_attention: + _target_: sam2.modeling.sam.transformer.RoPEAttention + rope_theta: 10000.0 + feat_sizes: + - 64 + - 64 + rope_k_repeat: true + embedding_dim: 256 + num_heads: 1 + downsample_rate: 1 + dropout: 0.1 + kv_in_dim: 64 + num_layers: 4 + memory_encoder: + _target_: sam2.modeling.memory_encoder.MemoryEncoder + out_dim: 64 + position_encoding: + _target_: sam2.modeling.position_encoding.PositionEmbeddingSine + num_pos_feats: 64 + normalize: true + scale: null + temperature: 10000 + mask_downsampler: + _target_: sam2.modeling.memory_encoder.MaskDownSampler + kernel_size: 3 + stride: 2 + padding: 1 + fuser: + _target_: sam2.modeling.memory_encoder.Fuser + layer: + _target_: sam2.modeling.memory_encoder.CXBlock + dim: 256 + kernel_size: 7 + padding: 3 + layer_scale_init_value: 1.0e-06 + use_dwconv: true + num_layers: 2 + num_maskmem: 7 + image_size: 1024 + sigmoid_scale_for_mem_enc: 20.0 + sigmoid_bias_for_mem_enc: -10.0 + use_mask_input_as_output_without_sam: true + directly_add_no_mem_embed: true + no_obj_embed_spatial: true + use_high_res_features_in_sam: true + multimask_output_in_sam: true + iou_prediction_use_sigmoid: true + use_obj_ptrs_in_encoder: true + add_tpos_enc_to_obj_ptrs: true + proj_tpos_enc_in_obj_ptrs: true + use_signed_tpos_enc_to_obj_ptrs: true + only_obj_ptrs_in_the_past_for_eval: true + pred_obj_scores: true + pred_obj_scores_mlp: true + fixed_no_obj_ptr: true + multimask_output_for_tracking: true + use_multimask_token_for_obj_ptr: true + multimask_min_pt_num: 0 + multimask_max_pt_num: 1 + use_mlp_for_obj_ptr_proj: true + prob_to_use_pt_input_for_train: 0.5 + prob_to_use_pt_input_for_eval: 0.0 + prob_to_use_box_input_for_train: 0.5 + prob_to_use_box_input_for_eval: 0.0 + prob_to_sample_from_gt_for_train: 0.1 + num_frames_to_correct_for_train: 2 + num_frames_to_correct_for_eval: 1 + rand_frames_to_correct_for_train: true + add_all_frames_to_correct_as_cond: true + num_init_cond_frames_for_train: 2 + rand_init_cond_frames_for_train: true + num_correction_pt_per_frame: 7 + use_act_ckpt_iterative_pt_sampling: false + num_init_cond_frames_for_eval: 1 + forward_backbone_per_frame_for_eval: true + data: + train: + _target_: training.dataset.sam2_datasets.TorchTrainMixedDataset + phases_per_epoch: 1 + batch_sizes: + - 1 + datasets: + - _target_: training.dataset.utils.RepeatFactorWrapper + dataset: + _target_: training.dataset.utils.ConcatDataset + datasets: + - _target_: training.dataset.vos_dataset.VOSDataset + transforms: + - _target_: training.dataset.transforms.ComposeAPI + transforms: + - _target_: training.dataset.transforms.RandomHorizontalFlip + consistent_transform: true + - _target_: training.dataset.transforms.RandomAffine + degrees: 25 + shear: 20 + image_interpolation: bilinear + consistent_transform: true + - _target_: training.dataset.transforms.RandomResizeAPI + sizes: 1024 + square: true + consistent_transform: true + - _target_: training.dataset.transforms.ColorJitter + consistent_transform: true + brightness: 0.1 + contrast: 0.03 + saturation: 0.03 + hue: null + - _target_: training.dataset.transforms.RandomGrayscale + p: 0.05 + consistent_transform: true + - _target_: training.dataset.transforms.ColorJitter + consistent_transform: false + brightness: 0.1 + contrast: 0.05 + saturation: 0.05 + hue: null + - _target_: training.dataset.transforms.ToTensorAPI + - _target_: training.dataset.transforms.NormalizeAPI + mean: + - 0.485 + - 0.456 + - 0.406 + std: + - 0.229 + - 0.224 + - 0.225 + training: true + video_dataset: + _target_: training.dataset.binary_png_raw_dataset.BinaryPNGRawDataset + img_folder: /work/nvme/bfgd/cmaduabuchi/DATA/thermal_video/thriveseg/train/images + gt_folder: /work/nvme/bfgd/cmaduabuchi/DATA/thermal_video/thriveseg/train/masks + file_list_txt: null + sampler: + _target_: training.dataset.vos_sampler.RandomUniformSampler + num_frames: 1 + max_num_objects: 3 + multiplier: 2 + shuffle: true + num_workers: 10 + pin_memory: true + drop_last: true + collate_fn: + _target_: training.utils.data_utils.collate_fn + _partial_: true + dict_key: all + optim: + amp: + enabled: true + amp_dtype: bfloat16 + optimizer: + _target_: torch.optim.AdamW + gradient_clip: + _target_: training.optimizer.GradientClipper + max_norm: 0.1 + norm_type: 2 + param_group_modifiers: + - _target_: training.optimizer.layer_decay_param_modifier + _partial_: true + layer_decay_value: 0.9 + apply_to: image_encoder.trunk + overrides: + - pattern: '*pos_embed*' + value: 1.0 + options: + lr: + - scheduler: + _target_: fvcore.common.param_scheduler.CosineParamScheduler + start_value: 5.0e-06 + end_value: 5.000000000000001e-07 + - scheduler: + _target_: fvcore.common.param_scheduler.CosineParamScheduler + start_value: 3.0e-06 + end_value: 3.0e-07 + param_names: + - image_encoder.* + weight_decay: + - scheduler: + _target_: fvcore.common.param_scheduler.ConstantParamScheduler + value: 0.1 + - scheduler: + _target_: fvcore.common.param_scheduler.ConstantParamScheduler + value: 0.0 + param_names: + - '*bias*' + module_cls_names: + - torch.nn.LayerNorm + loss: + all: + _target_: training.loss_fns.MultiStepMultiMasksAndIous + weight_dict: + loss_mask: 20 + loss_dice: 1 + loss_iou: 1 + loss_class: 1 + supervise_all_iou: true + iou_use_l1_loss: true + pred_obj_scores: true + focal_gamma_obj_score: 0.0 + focal_alpha_obj_score: -1.0 + distributed: + backend: nccl + find_unused_parameters: true + logging: + tensorboard_writer: + _target_: training.utils.logger.make_tensorboard_logger + log_dir: /work/nvme/bfgd/cmaduabuchi/MODELS/THRIVESEG/experiment/thermal_run1/tensorboard + flush_secs: 120 + should_log: true + log_dir: /work/nvme/bfgd/cmaduabuchi/MODELS/THRIVESEG/experiment/thermal_run1/logs + log_freq: 10 + checkpoint: + save_dir: /work/nvme/bfgd/cmaduabuchi/MODELS/THRIVESEG/experiment/thermal_run1/checkpoints + save_freq: 1 + model_weight_initializer: + _partial_: true + _target_: training.utils.checkpoint_utils.load_state_dict_into_model + strict: true + ignore_unexpected_keys: null + ignore_missing_keys: null + state_dict: + _target_: training.utils.checkpoint_utils.load_checkpoint_and_apply_kernels + checkpoint_path: /work/nvme/bfgd/cmaduabuchi/MODELS/THRIVESEG/sam2.1_hiera_base_plus.pt + ckpt_state_dict_keys: + - model +launcher: + num_nodes: 1 + gpus_per_node: 8 + experiment_log_dir: /work/nvme/bfgd/cmaduabuchi/MODELS/THRIVESEG/experiment/thermal_run1 +submitit: + partition: null + account: null + qos: null + cpus_per_task: 10 + use_cluster: false + timeout_hour: 24 + name: null + port_range: + - 10000 + - 65000 diff --git a/experiment/thermal_run1/logs/best_stats.json b/experiment/thermal_run1/logs/best_stats.json new file mode 100644 index 0000000000000000000000000000000000000000..c23c4e0b8ae501b1fd368a6770a164f631db5de3 --- /dev/null +++ b/experiment/thermal_run1/logs/best_stats.json @@ -0,0 +1,40 @@ +{"Trainer/where": 0.0249937374749499, "Trainer/epoch": 0, "Trainer/steps_train": 3992} +{"Trainer/where": 0.0499937374749499, "Trainer/epoch": 1, "Trainer/steps_train": 7984} +{"Trainer/where": 0.0749937374749499, "Trainer/epoch": 2, "Trainer/steps_train": 11976} +{"Trainer/where": 0.09999373747494991, "Trainer/epoch": 3, "Trainer/steps_train": 15968} +{"Trainer/where": 0.1249937374749499, "Trainer/epoch": 4, "Trainer/steps_train": 19960} +{"Trainer/where": 0.1499937374749499, "Trainer/epoch": 5, "Trainer/steps_train": 23952} +{"Trainer/where": 0.1749937374749499, "Trainer/epoch": 6, "Trainer/steps_train": 27944} +{"Trainer/where": 0.19999373747494992, "Trainer/epoch": 7, "Trainer/steps_train": 31936} +{"Trainer/where": 0.2249937374749499, "Trainer/epoch": 8, "Trainer/steps_train": 35928} +{"Trainer/where": 0.2499937374749499, "Trainer/epoch": 9, "Trainer/steps_train": 39920} +{"Trainer/where": 0.27499373747494993, "Trainer/epoch": 10, "Trainer/steps_train": 43912} +{"Trainer/where": 0.2999937374749499, "Trainer/epoch": 11, "Trainer/steps_train": 47904} +{"Trainer/where": 0.3249937374749499, "Trainer/epoch": 12, "Trainer/steps_train": 51896} +{"Trainer/where": 0.3499937374749499, "Trainer/epoch": 13, "Trainer/steps_train": 55888} +{"Trainer/where": 0.3749937374749499, "Trainer/epoch": 14, "Trainer/steps_train": 59880} +{"Trainer/where": 0.39999373747494993, "Trainer/epoch": 15, "Trainer/steps_train": 63872} +{"Trainer/where": 0.4249937374749499, "Trainer/epoch": 16, "Trainer/steps_train": 67864} +{"Trainer/where": 0.4499937374749499, "Trainer/epoch": 17, "Trainer/steps_train": 71856} +{"Trainer/where": 0.4749937374749499, "Trainer/epoch": 18, "Trainer/steps_train": 75848} +{"Trainer/where": 0.4999937374749499, "Trainer/epoch": 19, "Trainer/steps_train": 79840} +{"Trainer/where": 0.5249937374749499, "Trainer/epoch": 20, "Trainer/steps_train": 83832} +{"Trainer/where": 0.54999373747495, "Trainer/epoch": 21, "Trainer/steps_train": 87824} +{"Trainer/where": 0.5749937374749499, "Trainer/epoch": 22, "Trainer/steps_train": 91816} +{"Trainer/where": 0.5999937374749499, "Trainer/epoch": 23, "Trainer/steps_train": 95808} +{"Trainer/where": 0.6249937374749499, "Trainer/epoch": 24, "Trainer/steps_train": 99800} +{"Trainer/where": 0.6499937374749499, "Trainer/epoch": 25, "Trainer/steps_train": 103792} +{"Trainer/where": 0.67499373747495, "Trainer/epoch": 26, "Trainer/steps_train": 107784} +{"Trainer/where": 0.6999937374749499, "Trainer/epoch": 27, "Trainer/steps_train": 111776} +{"Trainer/where": 0.7249937374749499, "Trainer/epoch": 28, "Trainer/steps_train": 115768} +{"Trainer/where": 0.7499937374749499, "Trainer/epoch": 29, "Trainer/steps_train": 119760} +{"Trainer/where": 0.7749937374749499, "Trainer/epoch": 30, "Trainer/steps_train": 123752} +{"Trainer/where": 0.79999373747495, "Trainer/epoch": 31, "Trainer/steps_train": 127744} +{"Trainer/where": 0.8249937374749499, "Trainer/epoch": 32, "Trainer/steps_train": 131736} +{"Trainer/where": 0.8499937374749498, "Trainer/epoch": 33, "Trainer/steps_train": 135728} +{"Trainer/where": 0.8749937374749498, "Trainer/epoch": 34, "Trainer/steps_train": 139720} +{"Trainer/where": 0.8999937374749498, "Trainer/epoch": 35, "Trainer/steps_train": 143712} +{"Trainer/where": 0.9249937374749498, "Trainer/epoch": 36, "Trainer/steps_train": 147704} +{"Trainer/where": 0.9499937374749499, "Trainer/epoch": 37, "Trainer/steps_train": 151696} +{"Trainer/where": 0.9749937374749498, "Trainer/epoch": 38, "Trainer/steps_train": 155688} +{"Trainer/where": 0.9999937374749498, "Trainer/epoch": 39, "Trainer/steps_train": 159680} diff --git a/experiment/thermal_run1/logs/log.txt b/experiment/thermal_run1/logs/log.txt new file mode 100644 index 0000000000000000000000000000000000000000..6ae2f092e4ff121be7a2df980a29907ec87c715b --- /dev/null +++ b/experiment/thermal_run1/logs/log.txt @@ -0,0 +1,732 @@ +INFO 2025-10-06 06:35:10,376 train_utils.py: 109: MACHINE SEED: 4920 +INFO 2025-10-06 06:35:10,389 train_utils.py: 155: Logging ENV_VARIABLES +INFO 2025-10-06 06:35:10,389 train_utils.py: 156: BASH_ENV=/usr/share/lmod/lmod/init/bash +BASH_FUNC_ml%%=() { eval "$($LMOD_DIR/ml_cmd "$@")" +} +BASH_FUNC_module%%=() { if [ -z "${LMOD_SH_DBG_ON+x}" ]; then + case "$-" in + *v*x*) + __lmod_sh_dbg='vx' + ;; + *v*) + __lmod_sh_dbg='v' + ;; + *x*) + __lmod_sh_dbg='x' + ;; + esac; + fi; + if [ -n "${__lmod_sh_dbg:-}" ]; then + set +$__lmod_sh_dbg; + echo "Shell debugging temporarily silenced: export LMOD_SH_DBG_ON=1 for Lmod's output" 1>&2; + fi; + eval "$($LMOD_CMD shell "$@")" && eval "$(${LMOD_SETTARG_CMD:-:} -s sh)"; + __lmod_my_status=$?; + if [ -n "${__lmod_sh_dbg:-}" ]; then + echo "Shell debugging restarted" 1>&2; + set -$__lmod_sh_dbg; + fi; + unset __lmod_sh_dbg; + return $__lmod_my_status +} +BASH_FUNC_which%%=() { ( alias; + eval ${which_declare} ) | /usr/bin/which --tty-only --read-alias --read-functions --show-tilde --show-dot $@ +} +BROWSER=/u/cmaduabuchi/.vscode-server/cli/servers/Stable-0f0d87fa9e96c856c5212fc86db137ac0d783365/server/bin/helpers/browser.sh +BUNDLED_DEBUGPY_PATH=/u/cmaduabuchi/.vscode-server/extensions/ms-python.debugpy-2025.10.0-linux-x64/bundled/libs/debugpy +CC=/sw/spack/deltas11-2023-03/apps/linux-rhel8-x86_64/gcc-8.5.0/gcc-11.4.0-yycklku/bin/gcc +CMAKE_PREFIX_PATH=/sw/spack/deltas11-2023-03/apps/linux-rhel8-zen3/gcc-11.4.0/cuda-11.8.0-vfixfmc:/sw/spack/deltas11-2023-03/apps/linux-rhel8-zen3/gcc-11.4.0/openmpi-4.1.6-lranp74:/sw/spack/deltas11-2023-03/apps/linux-rhel8-x86_64/gcc-8.5.0/gcc-11.4.0-yycklku +COLORTERM=truecolor +CONDA_DEFAULT_ENV=sam2 +CONDA_EXE=/sw/external/python/anaconda3/bin/conda +CONDA_PREFIX=/u/cmaduabuchi/.conda/envs/sam2 +CONDA_PREFIX_1=/sw/external/python/anaconda3 +CONDA_PREFIX_2=/u/cmaduabuchi/.conda/envs/sam2 +CONDA_PREFIX_3=/sw/external/python/anaconda3 +CONDA_PROMPT_MODIFIER=(sam2) +CONDA_PYTHON_EXE=/sw/external/python/anaconda3/bin/python +CONDA_SHLVL=4 +CPLUS_INCLUDE_PATH=/sw/spack/deltas11-2023-03/apps/linux-rhel8-zen3/gcc-11.4.0/cuda-11.8.0-vfixfmc/include:/sw/spack/deltas11-2023-03/apps/linux-rhel8-zen3/gcc-11.4.0/openmpi-4.1.6-lranp74/include:/sw/spack/deltas11-2023-03/apps/linux-rhel8-x86_64/gcc-8.5.0/gcc-11.4.0-yycklku/include +CRAY_ENABLE_PE=/etc/cray-pe.d/enable-pe.sh +CUDA_HOME=/sw/spack/deltas11-2023-03/apps/linux-rhel8-zen3/gcc-11.4.0/cuda-11.8.0-vfixfmc +CUDA_MODULE_LOADING=LAZY +CUDA_VISIBLE_DEVICES=0 +CUE_CLUSTER_HOME=/u/cmaduabuchi +CUE_DOCS=https://portal.xsede.org/ncsa-delta +CUE_HOME=/u/cmaduabuchi +CUE_HOME_TYPE=lustre-2.14 +CUE_HOSTNAME=dt-login03.delta.ncsa.illinois.edu +CUE_PROMPT_HOST=dt-login03.delta +CUE_RESOURCE_ID=delta-cpu.ncsa.xsede.org +CXX=/sw/spack/deltas11-2023-03/apps/linux-rhel8-x86_64/gcc-8.5.0/gcc-11.4.0-yycklku/bin/g++ +C_INCLUDE_PATH=/sw/spack/deltas11-2023-03/apps/linux-rhel8-zen3/gcc-11.4.0/cuda-11.8.0-vfixfmc/include:/sw/spack/deltas11-2023-03/apps/linux-rhel8-zen3/gcc-11.4.0/openmpi-4.1.6-lranp74/include:/sw/spack/deltas11-2023-03/apps/linux-rhel8-x86_64/gcc-8.5.0/gcc-11.4.0-yycklku/include +DBUS_SESSION_BUS_ADDRESS=unix:path=/run/user/77886/bus +F77=/sw/spack/deltas11-2023-03/apps/linux-rhel8-x86_64/gcc-8.5.0/gcc-11.4.0-yycklku/bin/gfortran +FC=/sw/spack/deltas11-2023-03/apps/linux-rhel8-x86_64/gcc-8.5.0/gcc-11.4.0-yycklku/bin/gfortran +FPATH=/usr/share/lmod/lmod/init/ksh_funcs +GCC_HOME=/sw/spack/deltas11-2023-03/apps/linux-rhel8-x86_64/gcc-8.5.0/gcc-11.4.0-yycklku +GIT_ASKPASS=/u/cmaduabuchi/.vscode-server/cli/servers/Stable-0f0d87fa9e96c856c5212fc86db137ac0d783365/server/extensions/git/dist/askpass.sh +GPU_DEVICE_ORDINAL=0 +HISTCONTROL=ignoredups +HISTFILE=/u/cmaduabuchi/.bash_commands.log +HISTSIZE=50000 +HISTTIMEFORMAT=%h %d %H:%M:%S +HOME=/u/cmaduabuchi +HOSTNAME=dt-login03.delta.ncsa.illinois.edu +HYDRA_BOOTSTRAP=slurm +HYDRA_FULL_ERROR=1 +HYDRA_LAUNCHER_EXTRA_ARGS=--external-launcher +INCLUDE=/sw/spack/deltas11-2023-03/apps/linux-rhel8-zen3/gcc-11.4.0/cuda-11.8.0-vfixfmc/include:/sw/spack/deltas11-2023-03/apps/linux-rhel8-zen3/gcc-11.4.0/openmpi-4.1.6-lranp74/include:/sw/spack/deltas11-2023-03/apps/linux-rhel8-x86_64/gcc-8.5.0/gcc-11.4.0-yycklku/include +I_MPI_HYDRA_BOOTSTRAP=slurm +I_MPI_HYDRA_BOOTSTRAP_EXEC_EXTRA_ARGS=--external-launcher +LANG=en_US.UTF-8 +LD_LIBRARY_PATH=/sw/spack/delta-2022-03/apps/gcc/11.2.0-gcc-8.4.1-fxgnsyr/lib64:/sw/spack/deltas11-2023-03/apps/linux-rhel8-zen3/gcc-11.4.0/cuda-11.8.0-vfixfmc/lib64:/opt/cray/libfabric/1.15.2.0/lib64:/opt/cray/libfabric/1.15.2.0/lib +LESSOPEN=||/usr/bin/lesspipe.sh %s +LIBRARY_PATH=/sw/spack/deltas11-2023-03/apps/linux-rhel8-zen3/gcc-11.4.0/cuda-11.8.0-vfixfmc/lib64:/sw/spack/deltas11-2023-03/apps/linux-rhel8-zen3/gcc-11.4.0/openmpi-4.1.6-lranp74/lib:/sw/spack/deltas11-2023-03/apps/linux-rhel8-x86_64/gcc-8.5.0/gcc-11.4.0-yycklku/lib64:/sw/spack/deltas11-2023-03/apps/linux-rhel8-x86_64/gcc-8.5.0/gcc-11.4.0-yycklku/lib +LMOD_AVAIL_STYLE=system +LMOD_CMD=/usr/share/lmod/lmod/libexec/lmod +LMOD_COLORIZE=yes +LMOD_DIR=/usr/share/lmod/lmod/libexec +LMOD_FAMILY_COMPILER=gcc +LMOD_FAMILY_COMPILER_VERSION=11.4.0 +LMOD_FAMILY_MPI=openmpi +LMOD_FAMILY_MPI_VERSION=4.1.6 +LMOD_FULL_SETTARG_SUPPORT=no +LMOD_MPI_NAME=openmpi +LMOD_MPI_VERSION=4.1.6-lranp74 +LMOD_PKG=/usr/share/lmod/lmod +LMOD_PREPEND_BLOCK=normal +LMOD_ROOT=/usr/share/lmod +LMOD_SETTARG_CMD=: +LMOD_SETTARG_FULL_SUPPORT=no +LMOD_SITE_NAME=NCSA +LMOD_SYSTEM_DEFAULT_MODULES=default-s11 +LMOD_SYSTEM_NAME=Delta +LMOD_VERSION=8.7.55 +LMOD_arch=x86_64 +LMOD_sys=Linux +LOADEDMODULES=gcc/11.4.0:openmpi/4.1.6:cuda/11.8.0:cue-login-env/1.1:slurm-env/0.1:default-s11:anaconda3_cpu/23.7.4 +LOCAL_RANK=0 +LOGNAME=cmaduabuchi +MAIL=/var/spool/mail/cmaduabuchi +MANPATH=/sw/spack/deltas11-2023-03/apps/linux-rhel8-zen3/gcc-11.4.0/openmpi-4.1.6-lranp74/share/man:/sw/spack/deltas11-2023-03/apps/linux-rhel8-x86_64/gcc-8.5.0/gcc-11.4.0-yycklku/share/man:/usr/share/lmod/lmod/share/man:/usr/local/share/man:/usr/share/man:/opt/ddn/ime/share/man:/opt/puppetlabs/puppet/share/man:/sw/external/python/anaconda3_cpu/man:/sw/external/python/anaconda3/man:/u/cmaduabuchi/.local/share/man:/sw/spack/deltas11-2023-03/apps/linux-rhel8-zen3/gcc-11.4.0/openmpi-4.1.6-lranp74/share/man:/sw/spack/deltas11-2023-03/apps/linux-rhel8-x86_64/gcc-8.5.0/gcc-11.4.0-yycklku/share/man:/usr/local/share/man:/usr/share/man:/opt/ddn/ime/share/man:/sw/external/python/anaconda3/man:/sw/external/python/anaconda3_cpu/man:/u/cmaduabuchi/.local/share/man:/sw/spack/deltas11-2023-03/apps/linux-rhel8-zen3/gcc-11.4.0/openmpi-4.1.6-lranp74/share/man:/sw/spack/deltas11-2023-03/apps/linux-rhel8-x86_64/gcc-8.5.0/gcc-11.4.0-yycklku/share/man:/usr/local/share/man:/usr/share/man:/opt/ddn/ime/share/man:/opt/ddn/ime/share/man:/opt/ddn/ime/share/man:/opt/ddn/ime/share/man:/opt/ddn/ime/share/man +MASTER_ADDR=localhost +MASTER_PORT=48488 +MODULEPATH=/sw/spack/deltas11-2023-03/modules/lmod/openmpi/4.1.6-lo6xae6/gcc/11.4.0:/sw/spack/deltas11-2023-03/modules/lmod/openmpi/4.1.6-lranp74/gcc/11.4.0:/sw/spack/deltas11-2023-03/modules/lmod/gcc/11.4.0:/sw/user/modules:/sw/spack/deltas11-2023-03/modules/lmod/Core:/opt/cray/pe/lmod/modulefiles/craype-targets/default:/opt/cray/pe/lmod/modulefiles/core:/opt/cray/modulefiles:/sw/external/lmod/modulefiles/core:/usr/share/lmod/lmod/modulefiles/Core +MODULEPATH_ROOT=/ +MODULESHOME=/usr/share/lmod/lmod +MPICC=/sw/spack/deltas11-2023-03/apps/linux-rhel8-zen3/gcc-11.4.0/openmpi-4.1.6-lranp74/bin/mpicc +MPICXX=/sw/spack/deltas11-2023-03/apps/linux-rhel8-zen3/gcc-11.4.0/openmpi-4.1.6-lranp74/bin/mpic++ +MPIF77=/sw/spack/deltas11-2023-03/apps/linux-rhel8-zen3/gcc-11.4.0/openmpi-4.1.6-lranp74/bin/mpif77 +MPIF90=/sw/spack/deltas11-2023-03/apps/linux-rhel8-zen3/gcc-11.4.0/openmpi-4.1.6-lranp74/bin/mpif90 +NCSA_FAMILY_COMPILER=gcc +NCSA_FAMILY_COMPILER_VERSION=11.4.0 +NCSA_FAMILY_MPI=openmpi +NCSA_FAMILY_MPI_VERSION=4.1.6 +NVHPC_CUDA_HOME=/sw/spack/deltas11-2023-03/apps/linux-rhel8-zen3/gcc-11.4.0/cuda-11.8.0-vfixfmc +OLDPWD=/work/nvme/bfgd/cmaduabuchi/THRIVESEG/sam2 +OMPI_MCA_btl_tcp_endpoint_cache=67108864 +OMPI_MCA_btl_tcp_latency_hsn0=2 +OMPI_MCA_btl_tcp_rdma_pipeline_send_length=1048576 +OMPI_MCA_mtl_ofi_provider_include=cxi +OMPI_MCA_opal_common_ucx_opal_mem_hooks=1 +OMPI_MCA_opal_warn_on_missing_libcuda=0 +OMPI_MCA_plm_slurm_args=--external-launcher +OMPI_MCA_ras_base_verbose=10 +OPENMPI_HOME=/sw/spack/deltas11-2023-03/apps/linux-rhel8-zen3/gcc-11.4.0/openmpi-4.1.6-lranp74 +PATH=/u/cmaduabuchi/.vscode-server/cli/servers/Stable-0f0d87fa9e96c856c5212fc86db137ac0d783365/server/bin/remote-cli:/u/cmaduabuchi/.conda/envs/sam2/bin:/sw/external/python/anaconda3/condabin:/sw/external/python/anaconda3_cpu/bin:/u/cmaduabuchi/.local/bin:/u/cmaduabuchi/bin:/sw/spack/deltas11-2023-03/apps/linux-rhel8-zen3/gcc-11.4.0/cuda-11.8.0-vfixfmc/bin:/sw/spack/deltas11-2023-03/apps/linux-rhel8-zen3/gcc-11.4.0/openmpi-4.1.6-lranp74/bin:/sw/spack/deltas11-2023-03/apps/linux-rhel8-x86_64/gcc-8.5.0/gcc-11.4.0-yycklku/bin:/sw/user/scripts:/usr/local/bin:/usr/bin:/usr/local/sbin:/usr/sbin:/opt/ddn/ime/bin:/opt/puppetlabs/bin:/opt/ddn/ime/bin:/u/cmaduabuchi/.vscode-server/extensions/ms-python.debugpy-2025.10.0-linux-x64/bundled/scripts/noConfigScripts:/u/cmaduabuchi/.vscode-server/data/User/globalStorage/github.copilot-chat/debugCommand:/opt/ddn/ime/bin:/opt/ddn/ime/bin:/opt/ddn/ime/bin +PKG_CONFIG_PATH=/sw/spack/deltas11-2023-03/apps/linux-rhel8-zen3/gcc-11.4.0/openmpi-4.1.6-lranp74/lib/pkgconfig +PMI_FD=10 +PMI_JOBID=12652122.0 +PMI_RANK=0 +PMI_SIZE=1 +PRTE_MCA_plm_slurm_args=--external-launcher +PWD=/work/nvme/bfgd/cmaduabuchi/THRIVESEG/sam2 +PYDEVD_DISABLE_FILE_VALIDATION=1 +PYTHONSTARTUP=/u/cmaduabuchi/.vscode-server/data/User/workspaceStorage/e99aa778c8fbcf5603c74656a7713b05/ms-python.python/pythonrc.py +PYTHON_BASIC_REPL=1 +RANK=0 +ROCR_VISIBLE_DEVICES=0 +SALLOC_KILL_CMD=1 +SHELL=/bin/bash +SHLVL=8 +SLURMD_DEBUG=2 +SLURMD_NODENAME=gpub060 +SLURM_CLUSTER_NAME=delta +SLURM_CONF=/var/spool/slurmd/conf-cache/slurm.conf +SLURM_CPUS_ON_NODE=16 +SLURM_CPUS_PER_GPU=16 +SLURM_CPU_BIND=quiet,mask_cpu:0x0000000000FFFF00 +SLURM_CPU_BIND_LIST=0x0000000000FFFF00 +SLURM_CPU_BIND_TYPE=mask_cpu: +SLURM_CPU_BIND_VERBOSE=quiet +SLURM_GPUS_ON_NODE=1 +SLURM_GPUS_PER_NODE=1 +SLURM_GTIDS=0 +SLURM_JOBID=12652122 +SLURM_JOB_ACCOUNT=becs-delta-gpu +SLURM_JOB_CPUS_PER_NODE=16 +SLURM_JOB_END_TIME=1759922844 +SLURM_JOB_GID=202 +SLURM_JOB_GROUP=grp_202 +SLURM_JOB_ID=12652122 +SLURM_JOB_NAME=demo +SLURM_JOB_NODELIST=gpub060 +SLURM_JOB_NUM_NODES=1 +SLURM_JOB_PARTITION=gpuA40x4 +SLURM_JOB_QOS=becs-delta-gpu +SLURM_JOB_START_TIME=1759750044 +SLURM_JOB_UID=77886 +SLURM_JOB_USER=cmaduabuchi +SLURM_LAUNCH_NODE_IPADDR=141.142.253.42 +SLURM_LOCALID=0 +SLURM_MEM_PER_GPU=81920 +SLURM_MPI_TYPE=pmi2 +SLURM_NNODES=1 +SLURM_NODEID=0 +SLURM_NODELIST=gpub060 +SLURM_NPROCS=1 +SLURM_NTASKS=1 +SLURM_PRIO_PROCESS=0 +SLURM_PROCID=0 +SLURM_PTY_PORT=39771 +SLURM_PTY_WIN_COL=91 +SLURM_PTY_WIN_ROW=69 +SLURM_SRUN_COMM_HOST=141.142.253.42 +SLURM_SRUN_COMM_PORT=43569 +SLURM_STEPID=0 +SLURM_STEP_GPUS=1 +SLURM_STEP_ID=0 +SLURM_STEP_LAUNCHER_PORT=43569 +SLURM_STEP_NODELIST=gpub060 +SLURM_STEP_NUM_NODES=1 +SLURM_STEP_NUM_TASKS=1 +SLURM_STEP_TASKS_PER_NODE=1 +SLURM_SUBMIT_DIR=/work/nvme/bfgd/cmaduabuchi +SLURM_SUBMIT_HOST=dt-login03.delta.ncsa.illinois.edu +SLURM_TASKS_PER_NODE=1 +SLURM_TASK_PID=916888 +SLURM_TOPOLOGY_ADDR=ss00.ss11.gpub060 +SLURM_TOPOLOGY_ADDR_PATTERN=switch.switch.node +SLURM_UMASK=0077 +SQUEUE_FORMAT=%.12i %.12P %.12j %.14u %.2t %.10M %.6D %.18R %f +SRUN_DEBUG=3 +SSH_CLIENT=128.195.95.47 32161 22 +SSH_CONNECTION=169.234.226.25 58665 141.142.140.196 22 +SSL_CERT_DIR=/etc/pki/tls/certs +SSL_CERT_FILE=/etc/pki/ca-trust/extracted/pem/tls-ca-bundle.pem +TERM=screen +TERM_PROGRAM=vscode +TERM_PROGRAM_VERSION=1.104.1 +TMOUT=86400 +TMPDIR=/tmp +TMUX=/tmp/tmux-77886/default,3561957,1 +TMUX_PANE=%1 +TORCH_NCCL_ASYNC_ERROR_HANDLING=1 +USER=cmaduabuchi +VSCODE_DEBUGPY_ADAPTER_ENDPOINTS=/u/cmaduabuchi/.vscode-server/extensions/ms-python.debugpy-2025.10.0-linux-x64/.noConfigDebugAdapterEndpoints/endpoint-ad8f8f00eace4009.txt +VSCODE_GIT_ASKPASS_EXTRA_ARGS= +VSCODE_GIT_ASKPASS_MAIN=/u/cmaduabuchi/.vscode-server/cli/servers/Stable-0f0d87fa9e96c856c5212fc86db137ac0d783365/server/extensions/git/dist/askpass-main.js +VSCODE_GIT_ASKPASS_NODE=/u/cmaduabuchi/.vscode-server/cli/servers/Stable-0f0d87fa9e96c856c5212fc86db137ac0d783365/server/node +VSCODE_GIT_IPC_HANDLE=/run/user/77886/vscode-git-922d1ac20a.sock +VSCODE_IPC_HOOK_CLI=/run/user/77886/vscode-ipc-4d775fc7-d69f-439f-a012-2b869d146d9e.sock +VSCODE_PYTHON_AUTOACTIVATE_GUARD=1 +WORLD_SIZE=1 +XDG_RUNTIME_DIR=/run/user/77886 +XDG_SESSION_ID=114322 +ZE_AFFINITY_MASK=0 +_=/u/cmaduabuchi/.conda/envs/sam2/bin/python +_CE_CONDA= +_CE_M= +_LMFILES_=/sw/spack/deltas11-2023-03/modules/lmod/Core/gcc/11.4.0.lua:/sw/spack/deltas11-2023-03/modules/lmod/gcc/11.4.0/openmpi/4.1.6.lua:/sw/spack/deltas11-2023-03/modules/lmod/gcc/11.4.0/cuda/11.8.0.lua:/sw/user/modules/cue-login-env/1.1.lua:/sw/user/modules/slurm-env/0.1.lua:/sw/user/modules/default-s11.lua:/sw/user/modules/anaconda3_cpu/23.7.4.lua +_ModuleTable001_=X01vZHVsZVRhYmxlXyA9IHsKTVR2ZXJzaW9uID0gMywKY19yZWJ1aWxkVGltZSA9IGZhbHNlLApjX3Nob3J0VGltZSA9IGZhbHNlLApkZXB0aFQgPSB7fSwKZmFtaWx5ID0gewpjb21waWxlciA9ICJnY2MiLAptcGkgPSAib3Blbm1waSIsCn0sCm1UID0gewphbmFjb25kYTNfY3B1ID0gewpmbiA9ICIvc3cvdXNlci9tb2R1bGVzL2FuYWNvbmRhM19jcHUvMjMuNy40Lmx1YSIsCmZ1bGxOYW1lID0gImFuYWNvbmRhM19jcHUvMjMuNy40IiwKbG9hZE9yZGVyID0gNywKcHJvcFQgPSB7fSwKc3RhY2tEZXB0aCA9IDAsCnN0YXR1cyA9ICJhY3RpdmUiLAp1c2VyTmFtZSA9ICJhbmFjb25kYTNfY3B1LzIzLjcuNCIsCndWID0gIl4wMDAwMDAyMy4wMDAwMDAwMDcuMDAwMDAwMDA0Lip6 +_ModuleTable002_=ZmluYWwiLAp9LApjdWRhID0gewpmbiA9ICIvc3cvc3BhY2svZGVsdGFzMTEtMjAyMy0wMy9tb2R1bGVzL2xtb2QvZ2NjLzExLjQuMC9jdWRhLzExLjguMC5sdWEiLApmdWxsTmFtZSA9ICJjdWRhLzExLjguMCIsCmxvYWRPcmRlciA9IDMsCnByb3BUID0ge30sCnN0YWNrRGVwdGggPSAxLApzdGF0dXMgPSAiYWN0aXZlIiwKdXNlck5hbWUgPSAiY3VkYSIsCndWID0gIl4wMDAwMDAxMS4wMDAwMDAwMDguKnpmaW5hbCIsCn0sClsiY3VlLWxvZ2luLWVudiJdID0gewpmbiA9ICIvc3cvdXNlci9tb2R1bGVzL2N1ZS1sb2dpbi1lbnYvMS4xLmx1YSIsCmZ1bGxOYW1lID0gImN1ZS1sb2dpbi1lbnYvMS4xIiwKbG9hZE9yZGVyID0gNCwKcHJvcFQgPSB7fSwKc3RhY2tEZXB0aCA9IDEs +_ModuleTable003_=CnN0YXR1cyA9ICJhY3RpdmUiLAp1c2VyTmFtZSA9ICJjdWUtbG9naW4tZW52IiwKd1YgPSAiMDAwMDAwMDAxLjAwMDAwMDAwMS4qemZpbmFsIiwKfSwKWyJkZWZhdWx0LXMxMSJdID0gewpmbiA9ICIvc3cvdXNlci9tb2R1bGVzL2RlZmF1bHQtczExLmx1YSIsCmZ1bGxOYW1lID0gImRlZmF1bHQtczExIiwKbG9hZE9yZGVyID0gNiwKcHJvcFQgPSB7fSwKc3RhY2tEZXB0aCA9IDAsCnN0YXR1cyA9ICJhY3RpdmUiLAp1c2VyTmFtZSA9ICJkZWZhdWx0LXMxMSIsCndWID0gIk0uKnpmaW5hbCIsCn0sCmdjYyA9IHsKYWN0aW9uQSA9IHsKInByZXBlbmRfcGF0aChcIk1PRFVMRVBBVEhcIixcIi9zdy9zcGFjay9kZWx0YXMxMS0yMDIzLTAzL21vZHVsZXMvbG1vZC9nY2MvMTEuNC4w +_ModuleTable004_=XCIpIiwKfSwKZm4gPSAiL3N3L3NwYWNrL2RlbHRhczExLTIwMjMtMDMvbW9kdWxlcy9sbW9kL0NvcmUvZ2NjLzExLjQuMC5sdWEiLApmdWxsTmFtZSA9ICJnY2MvMTEuNC4wIiwKbG9hZE9yZGVyID0gMSwKcHJvcFQgPSB7fSwKc3RhY2tEZXB0aCA9IDEsCnN0YXR1cyA9ICJhY3RpdmUiLAp1c2VyTmFtZSA9ICJnY2MiLAp3ViA9ICJeMDAwMDAwMTEuMDAwMDAwMDA0Lip6ZmluYWwiLAp9LApvcGVubXBpID0gewphY3Rpb25BID0gewoKInByZXBlbmRfcGF0aChcIk1PRFVMRVBBVEhcIixcIi9zdy9zcGFjay9kZWx0YXMxMS0yMDIzLTAzL21vZHVsZXMvbG1vZC9vcGVubXBpLzQuMS42LWxyYW5wNzQvZ2NjLzExLjQuMFwiKSIsICJwcmVwZW5kX3BhdGgoXCJNT0RVTEVQQVRIXCIs +_ModuleTable005_=XCIvc3cvc3BhY2svZGVsdGFzMTEtMjAyMy0wMy9tb2R1bGVzL2xtb2Qvb3Blbm1waS80LjEuNi1sbzZ4YWU2L2djYy8xMS40LjBcIikiLAp9LApmbiA9ICIvc3cvc3BhY2svZGVsdGFzMTEtMjAyMy0wMy9tb2R1bGVzL2xtb2QvZ2NjLzExLjQuMC9vcGVubXBpLzQuMS42Lmx1YSIsCmZ1bGxOYW1lID0gIm9wZW5tcGkvNC4xLjYiLApsb2FkT3JkZXIgPSAyLApwcm9wVCA9IHt9LApzdGFja0RlcHRoID0gMSwKc3RhdHVzID0gImFjdGl2ZSIsCnVzZXJOYW1lID0gIm9wZW5tcGkiLAp3ViA9ICJeMDAwMDAwMDQuMDAwMDAwMDAxLjAwMDAwMDAwNi4qemZpbmFsIiwKfSwKWyJzbHVybS1lbnYiXSA9IHsKZm4gPSAiL3N3L3VzZXIvbW9kdWxlcy9zbHVybS1lbnYvMC4xLmx1YSIsCmZ1 +_ModuleTable006_=bGxOYW1lID0gInNsdXJtLWVudi8wLjEiLApsb2FkT3JkZXIgPSA1LApwcm9wVCA9IHt9LApzdGFja0RlcHRoID0gMSwKc3RhdHVzID0gImFjdGl2ZSIsCnVzZXJOYW1lID0gInNsdXJtLWVudiIsCndWID0gIjAwMDAwMDAwMC4wMDAwMDAwMDEuKnpmaW5hbCIsCn0sCn0sCm1wYXRoQSA9IHsKIi9zdy9zcGFjay9kZWx0YXMxMS0yMDIzLTAzL21vZHVsZXMvbG1vZC9vcGVubXBpLzQuMS42LWxvNnhhZTYvZ2NjLzExLjQuMCIKLCAiL3N3L3NwYWNrL2RlbHRhczExLTIwMjMtMDMvbW9kdWxlcy9sbW9kL29wZW5tcGkvNC4xLjYtbHJhbnA3NC9nY2MvMTEuNC4wIgosICIvc3cvc3BhY2svZGVsdGFzMTEtMjAyMy0wMy9tb2R1bGVzL2xtb2QvZ2NjLzExLjQuMCIsICIvc3cvdXNlci9t +_ModuleTable007_=b2R1bGVzIgosICIvc3cvc3BhY2svZGVsdGFzMTEtMjAyMy0wMy9tb2R1bGVzL2xtb2QvQ29yZSIKLCAiL29wdC9jcmF5L3BlL2xtb2QvbW9kdWxlZmlsZXMvY3JheXBlLXRhcmdldHMvZGVmYXVsdCIKLCAiL29wdC9jcmF5L3BlL2xtb2QvbW9kdWxlZmlsZXMvY29yZSIsICIvb3B0L2NyYXkvbW9kdWxlZmlsZXMiCiwgIi9zdy9leHRlcm5hbC9sbW9kL21vZHVsZWZpbGVzL2NvcmUiLCAiL3Vzci9zaGFyZS9sbW9kL2xtb2QvbW9kdWxlZmlsZXMvQ29yZSIsCn0sCnN5c3RlbUJhc2VNUEFUSCA9ICIvc3cvdXNlci9tb2R1bGVzOi9zdy9zcGFjay9kZWx0YXMxMS0yMDIzLTAzL21vZHVsZXMvbG1vZC9Db3JlOi9vcHQvY3JheS9wZS9sbW9kL21vZHVsZWZpbGVzL2NyYXlwZS10YXJn +_ModuleTable008_=ZXRzL2RlZmF1bHQ6L29wdC9jcmF5L3BlL2xtb2QvbW9kdWxlZmlsZXMvY29yZTovb3B0L2NyYXkvbW9kdWxlZmlsZXM6L3N3L2V4dGVybmFsL2xtb2QvbW9kdWxlZmlsZXMvY29yZTovdXNyL3NoYXJlL2xtb2QvbG1vZC9tb2R1bGVmaWxlcy9Db3JlIiwKfQo= +_ModuleTable_Sz_=8 +__Init_Default_Modules=1 +__LMOD_REF_COUNT_CMAKE_PREFIX_PATH=/sw/spack/deltas11-2023-03/apps/linux-rhel8-zen3/gcc-11.4.0/cuda-11.8.0-vfixfmc:2;/sw/spack/deltas11-2023-03/apps/linux-rhel8-zen3/gcc-11.4.0/openmpi-4.1.6-lranp74:2;/sw/spack/deltas11-2023-03/apps/linux-rhel8-x86_64/gcc-8.5.0/gcc-11.4.0-yycklku:2 +__LMOD_REF_COUNT_CPLUS_INCLUDE_PATH=/sw/spack/deltas11-2023-03/apps/linux-rhel8-zen3/gcc-11.4.0/cuda-11.8.0-vfixfmc/include:1;/sw/spack/deltas11-2023-03/apps/linux-rhel8-zen3/gcc-11.4.0/openmpi-4.1.6-lranp74/include:1;/sw/spack/deltas11-2023-03/apps/linux-rhel8-x86_64/gcc-8.5.0/gcc-11.4.0-yycklku/include:1 +__LMOD_REF_COUNT_C_INCLUDE_PATH=/sw/spack/deltas11-2023-03/apps/linux-rhel8-zen3/gcc-11.4.0/cuda-11.8.0-vfixfmc/include:1;/sw/spack/deltas11-2023-03/apps/linux-rhel8-zen3/gcc-11.4.0/openmpi-4.1.6-lranp74/include:1;/sw/spack/deltas11-2023-03/apps/linux-rhel8-x86_64/gcc-8.5.0/gcc-11.4.0-yycklku/include:1 +__LMOD_REF_COUNT_INCLUDE=/sw/spack/deltas11-2023-03/apps/linux-rhel8-zen3/gcc-11.4.0/cuda-11.8.0-vfixfmc/include:1;/sw/spack/deltas11-2023-03/apps/linux-rhel8-zen3/gcc-11.4.0/openmpi-4.1.6-lranp74/include:1;/sw/spack/deltas11-2023-03/apps/linux-rhel8-x86_64/gcc-8.5.0/gcc-11.4.0-yycklku/include:1 +__LMOD_REF_COUNT_LD_LIBRARY_PATH=/sw/spack/delta-2022-03/apps/gcc/11.2.0-gcc-8.4.1-fxgnsyr/lib64:1;/sw/spack/deltas11-2023-03/apps/linux-rhel8-zen3/gcc-11.4.0/cuda-11.8.0-vfixfmc/lib64:1;/opt/cray/libfabric/1.15.2.0/lib64:1;/opt/cray/libfabric/1.15.2.0/lib:1 +__LMOD_REF_COUNT_LIBRARY_PATH=/sw/spack/deltas11-2023-03/apps/linux-rhel8-zen3/gcc-11.4.0/cuda-11.8.0-vfixfmc/lib64:1;/sw/spack/deltas11-2023-03/apps/linux-rhel8-zen3/gcc-11.4.0/openmpi-4.1.6-lranp74/lib:1;/sw/spack/deltas11-2023-03/apps/linux-rhel8-x86_64/gcc-8.5.0/gcc-11.4.0-yycklku/lib64:1;/sw/spack/deltas11-2023-03/apps/linux-rhel8-x86_64/gcc-8.5.0/gcc-11.4.0-yycklku/lib:1 +__LMOD_REF_COUNT_MANPATH=/sw/spack/deltas11-2023-03/apps/linux-rhel8-zen3/gcc-11.4.0/openmpi-4.1.6-lranp74/share/man:2;/sw/spack/deltas11-2023-03/apps/linux-rhel8-x86_64/gcc-8.5.0/gcc-11.4.0-yycklku/share/man:2;/usr/share/lmod/lmod/share/man:1;/usr/local/share/man:1;/usr/share/man:1;/opt/ddn/ime/share/man:1;/opt/puppetlabs/puppet/share/man:1;:2 +__LMOD_REF_COUNT_MODULEPATH=/sw/spack/deltas11-2023-03/modules/lmod/openmpi/4.1.6-lo6xae6/gcc/11.4.0:1;/sw/spack/deltas11-2023-03/modules/lmod/openmpi/4.1.6-lranp74/gcc/11.4.0:1;/sw/spack/deltas11-2023-03/modules/lmod/gcc/11.4.0:1;/sw/user/modules:1;/sw/spack/deltas11-2023-03/modules/lmod/Core:1;/opt/cray/pe/lmod/modulefiles/craype-targets/default:1;/opt/cray/pe/lmod/modulefiles/core:1;/opt/cray/modulefiles:1;/sw/external/lmod/modulefiles/core:1;/usr/share/lmod/lmod/modulefiles/Core:1 +__LMOD_REF_COUNT_PATH=/sw/external/python/anaconda3_cpu/bin:1;/u/cmaduabuchi/.vscode-server/cli/servers/Stable-0f0d87fa9e96c856c5212fc86db137ac0d783365/server/bin/remote-cli:1;/sw/external/python/anaconda3/bin:1;/sw/external/python/anaconda3/condabin:1;/u/cmaduabuchi/.local/bin:1;/u/cmaduabuchi/bin:1;/sw/spack/deltas11-2023-03/apps/linux-rhel8-zen3/gcc-11.4.0/cuda-11.8.0-vfixfmc/bin:2;/sw/spack/deltas11-2023-03/apps/linux-rhel8-zen3/gcc-11.4.0/openmpi-4.1.6-lranp74/bin:2;/sw/spack/deltas11-2023-03/apps/linux-rhel8-x86_64/gcc-8.5.0/gcc-11.4.0-yycklku/bin:2;/sw/user/scripts:1;/usr/local/bin:1;/usr/bin:1;/usr/local/sbin:1;/usr/sbin:1;/opt/ddn/ime/bin:3;/opt/puppetlabs/bin:1;/u/cmaduabuchi/.vscode-server/extensions/ms-python.debugpy-2025.10.0-linux-x64/bundled/scripts/noConfigScripts:1;/u/cmaduabuchi/.vscode-server/data/User/globalStorage/github.copilot-chat/debugCommand:1 +__LMOD_REF_COUNT_PKG_CONFIG_PATH=/sw/spack/deltas11-2023-03/apps/linux-rhel8-zen3/gcc-11.4.0/openmpi-4.1.6-lranp74/lib/pkgconfig:2 +which_declare=declare -f + +INFO 2025-10-06 06:35:10,389 trainer.py: 989: Setting up components: Model, loss, optim, meters etc. +INFO 2025-10-06 06:35:10,398 logger.py: 66: TensorBoard SummaryWriter instantiated. Files will be stored in: /work/nvme/bfgd/cmaduabuchi/MODELS/THRIVESEG/experiment/thermal_run1/tensorboard +INFO 2025-10-06 06:35:11,529 sam2.py: 81: Training with points (sampled from masks) as inputs with p=0.5 +INFO 2025-10-06 06:35:11,532 trainer.py:1059: ==================== +INFO 2025-10-06 06:35:11,532 trainer.py:1060: Summary for model +INFO 2025-10-06 06:35:11,534 trainer.py:1061: Model is SAM2Train( + (image_encoder): ImageEncoder( + (trunk): Hiera( + (patch_embed): PatchEmbed( + (proj): Conv2d(3, 112, kernel_size=(7, 7), stride=(4, 4), padding=(3, 3)) + ) + (blocks): ModuleList( + (0): MultiScaleBlock( + (norm1): LayerNorm((112,), eps=1e-06, elementwise_affine=True) + (attn): MultiScaleAttention( + (qkv): Linear(in_features=112, out_features=336, bias=True) + (proj): Linear(in_features=112, out_features=112, bias=True) + ) + (drop_path): Identity() + (norm2): LayerNorm((112,), eps=1e-06, elementwise_affine=True) + (mlp): MLP( + (layers): ModuleList( + (0): Linear(in_features=112, out_features=448, bias=True) + (1): Linear(in_features=448, out_features=112, bias=True) + ) + (act): GELU(approximate='none') + ) + ) + (1): MultiScaleBlock( + (norm1): LayerNorm((112,), eps=1e-06, elementwise_affine=True) + (attn): MultiScaleAttention( + (qkv): Linear(in_features=112, out_features=336, bias=True) + (proj): Linear(in_features=112, out_features=112, bias=True) + ) + (drop_path): DropPath() + (norm2): LayerNorm((112,), eps=1e-06, elementwise_affine=True) + (mlp): MLP( + (layers): ModuleList( + (0): Linear(in_features=112, out_features=448, bias=True) + (1): Linear(in_features=448, out_features=112, bias=True) + ) + (act): GELU(approximate='none') + ) + ) + (2): MultiScaleBlock( + (norm1): LayerNorm((112,), eps=1e-06, elementwise_affine=True) + (pool): MaxPool2d(kernel_size=(2, 2), stride=(2, 2), padding=0, dilation=1, ceil_mode=False) + (attn): MultiScaleAttention( + (q_pool): MaxPool2d(kernel_size=(2, 2), stride=(2, 2), padding=0, dilation=1, ceil_mode=False) + (qkv): Linear(in_features=112, out_features=672, bias=True) + (proj): Linear(in_features=224, out_features=224, bias=True) + ) + (drop_path): DropPath() + (norm2): LayerNorm((224,), eps=1e-06, elementwise_affine=True) + (mlp): MLP( + (layers): ModuleList( + (0): Linear(in_features=224, out_features=896, bias=True) + (1): Linear(in_features=896, out_features=224, bias=True) + ) + (act): GELU(approximate='none') + ) + (proj): Linear(in_features=112, out_features=224, bias=True) + ) + (3-4): 2 x MultiScaleBlock( + (norm1): LayerNorm((224,), eps=1e-06, elementwise_affine=True) + (attn): MultiScaleAttention( + (qkv): Linear(in_features=224, out_features=672, bias=True) + (proj): Linear(in_features=224, out_features=224, bias=True) + ) + (drop_path): DropPath() + (norm2): LayerNorm((224,), eps=1e-06, elementwise_affine=True) + (mlp): MLP( + (layers): ModuleList( + (0): Linear(in_features=224, out_features=896, bias=True) + (1): Linear(in_features=896, out_features=224, bias=True) + ) + (act): GELU(approximate='none') + ) + ) + (5): MultiScaleBlock( + (norm1): LayerNorm((224,), eps=1e-06, elementwise_affine=True) + (pool): MaxPool2d(kernel_size=(2, 2), stride=(2, 2), padding=0, dilation=1, ceil_mode=False) + (attn): MultiScaleAttention( + (q_pool): MaxPool2d(kernel_size=(2, 2), stride=(2, 2), padding=0, dilation=1, ceil_mode=False) + (qkv): Linear(in_features=224, out_features=1344, bias=True) + (proj): Linear(in_features=448, out_features=448, bias=True) + ) + (drop_path): DropPath() + (norm2): LayerNorm((448,), eps=1e-06, elementwise_affine=True) + (mlp): MLP( + (layers): ModuleList( + (0): Linear(in_features=448, out_features=1792, bias=True) + (1): Linear(in_features=1792, out_features=448, bias=True) + ) + (act): GELU(approximate='none') + ) + (proj): Linear(in_features=224, out_features=448, bias=True) + ) + (6-20): 15 x MultiScaleBlock( + (norm1): LayerNorm((448,), eps=1e-06, elementwise_affine=True) + (attn): MultiScaleAttention( + (qkv): Linear(in_features=448, out_features=1344, bias=True) + (proj): Linear(in_features=448, out_features=448, bias=True) + ) + (drop_path): DropPath() + (norm2): LayerNorm((448,), eps=1e-06, elementwise_affine=True) + (mlp): MLP( + (layers): ModuleList( + (0): Linear(in_features=448, out_features=1792, bias=True) + (1): Linear(in_features=1792, out_features=448, bias=True) + ) + (act): GELU(approximate='none') + ) + ) + (21): MultiScaleBlock( + (norm1): LayerNorm((448,), eps=1e-06, elementwise_affine=True) + (pool): MaxPool2d(kernel_size=(2, 2), stride=(2, 2), padding=0, dilation=1, ceil_mode=False) + (attn): MultiScaleAttention( + (q_pool): MaxPool2d(kernel_size=(2, 2), stride=(2, 2), padding=0, dilation=1, ceil_mode=False) + (qkv): Linear(in_features=448, out_features=2688, bias=True) + (proj): Linear(in_features=896, out_features=896, bias=True) + ) + (drop_path): DropPath() + (norm2): LayerNorm((896,), eps=1e-06, elementwise_affine=True) + (mlp): MLP( + (layers): ModuleList( + (0): Linear(in_features=896, out_features=3584, bias=True) + (1): Linear(in_features=3584, out_features=896, bias=True) + ) + (act): GELU(approximate='none') + ) + (proj): Linear(in_features=448, out_features=896, bias=True) + ) + (22-23): 2 x MultiScaleBlock( + (norm1): LayerNorm((896,), eps=1e-06, elementwise_affine=True) + (attn): MultiScaleAttention( + (qkv): Linear(in_features=896, out_features=2688, bias=True) + (proj): Linear(in_features=896, out_features=896, bias=True) + ) + (drop_path): DropPath() + (norm2): LayerNorm((896,), eps=1e-06, elementwise_affine=True) + (mlp): MLP( + (layers): ModuleList( + (0): Linear(in_features=896, out_features=3584, bias=True) + (1): Linear(in_features=3584, out_features=896, bias=True) + ) + (act): GELU(approximate='none') + ) + ) + ) + ) + (neck): FpnNeck( + (position_encoding): PositionEmbeddingSine() + (convs): ModuleList( + (0): Sequential( + (conv): Conv2d(896, 256, kernel_size=(1, 1), stride=(1, 1)) + ) + (1): Sequential( + (conv): Conv2d(448, 256, kernel_size=(1, 1), stride=(1, 1)) + ) + (2): Sequential( + (conv): Conv2d(224, 256, kernel_size=(1, 1), stride=(1, 1)) + ) + (3): Sequential( + (conv): Conv2d(112, 256, kernel_size=(1, 1), stride=(1, 1)) + ) + ) + ) + ) + (mask_downsample): Conv2d(1, 1, kernel_size=(4, 4), stride=(4, 4)) + (memory_attention): MemoryAttention( + (layers): ModuleList( + (0-3): 4 x MemoryAttentionLayer( + (self_attn): RoPEAttention( + (q_proj): Linear(in_features=256, out_features=256, bias=True) + (k_proj): Linear(in_features=256, out_features=256, bias=True) + (v_proj): Linear(in_features=256, out_features=256, bias=True) + (out_proj): Linear(in_features=256, out_features=256, bias=True) + ) + (cross_attn_image): RoPEAttention( + (q_proj): Linear(in_features=256, out_features=256, bias=True) + (k_proj): Linear(in_features=64, out_features=256, bias=True) + (v_proj): Linear(in_features=64, out_features=256, bias=True) + (out_proj): Linear(in_features=256, out_features=256, bias=True) + ) + (linear1): Linear(in_features=256, out_features=2048, bias=True) + (dropout): Dropout(p=0.1, inplace=False) + (linear2): Linear(in_features=2048, out_features=256, bias=True) + (norm1): LayerNorm((256,), eps=1e-05, elementwise_affine=True) + (norm2): LayerNorm((256,), eps=1e-05, elementwise_affine=True) + (norm3): LayerNorm((256,), eps=1e-05, elementwise_affine=True) + (dropout1): Dropout(p=0.1, inplace=False) + (dropout2): Dropout(p=0.1, inplace=False) + (dropout3): Dropout(p=0.1, inplace=False) + ) + ) + (norm): LayerNorm((256,), eps=1e-05, elementwise_affine=True) + ) + (memory_encoder): MemoryEncoder( + (mask_downsampler): MaskDownSampler( + (encoder): Sequential( + (0): Conv2d(1, 4, kernel_size=(3, 3), stride=(2, 2), padding=(1, 1)) + (1): LayerNorm2d() + (2): GELU(approximate='none') + (3): Conv2d(4, 16, kernel_size=(3, 3), stride=(2, 2), padding=(1, 1)) + (4): LayerNorm2d() + (5): GELU(approximate='none') + (6): Conv2d(16, 64, kernel_size=(3, 3), stride=(2, 2), padding=(1, 1)) + (7): LayerNorm2d() + (8): GELU(approximate='none') + (9): Conv2d(64, 256, kernel_size=(3, 3), stride=(2, 2), padding=(1, 1)) + (10): LayerNorm2d() + (11): GELU(approximate='none') + (12): Conv2d(256, 256, kernel_size=(1, 1), stride=(1, 1)) + ) + ) + (pix_feat_proj): Conv2d(256, 256, kernel_size=(1, 1), stride=(1, 1)) + (fuser): Fuser( + (proj): Identity() + (layers): ModuleList( + (0-1): 2 x CXBlock( + (dwconv): Conv2d(256, 256, kernel_size=(7, 7), stride=(1, 1), padding=(3, 3), groups=256) + (norm): LayerNorm2d() + (pwconv1): Linear(in_features=256, out_features=1024, bias=True) + (act): GELU(approximate='none') + (pwconv2): Linear(in_features=1024, out_features=256, bias=True) + (drop_path): Identity() + ) + ) + ) + (position_encoding): PositionEmbeddingSine() + (out_proj): Conv2d(256, 64, kernel_size=(1, 1), stride=(1, 1)) + ) + (sam_prompt_encoder): PromptEncoder( + (pe_layer): PositionEmbeddingRandom() + (point_embeddings): ModuleList( + (0-3): 4 x Embedding(1, 256) + ) + (not_a_point_embed): Embedding(1, 256) + (mask_downscaling): Sequential( + (0): Conv2d(1, 4, kernel_size=(2, 2), stride=(2, 2)) + (1): LayerNorm2d() + (2): GELU(approximate='none') + (3): Conv2d(4, 16, kernel_size=(2, 2), stride=(2, 2)) + (4): LayerNorm2d() + (5): GELU(approximate='none') + (6): Conv2d(16, 256, kernel_size=(1, 1), stride=(1, 1)) + ) + (no_mask_embed): Embedding(1, 256) + ) + (sam_mask_decoder): MaskDecoder( + (transformer): TwoWayTransformer( + (layers): ModuleList( + (0-1): 2 x TwoWayAttentionBlock( + (self_attn): Attention( + (q_proj): Linear(in_features=256, out_features=256, bias=True) + (k_proj): Linear(in_features=256, out_features=256, bias=True) + (v_proj): Linear(in_features=256, out_features=256, bias=True) + (out_proj): Linear(in_features=256, out_features=256, bias=True) + ) + (norm1): LayerNorm((256,), eps=1e-05, elementwise_affine=True) + (cross_attn_token_to_image): Attention( + (q_proj): Linear(in_features=256, out_features=128, bias=True) + (k_proj): Linear(in_features=256, out_features=128, bias=True) + (v_proj): Linear(in_features=256, out_features=128, bias=True) + (out_proj): Linear(in_features=128, out_features=256, bias=True) + ) + (norm2): LayerNorm((256,), eps=1e-05, elementwise_affine=True) + (mlp): MLP( + (layers): ModuleList( + (0): Linear(in_features=256, out_features=2048, bias=True) + (1): Linear(in_features=2048, out_features=256, bias=True) + ) + (act): ReLU() + ) + (norm3): LayerNorm((256,), eps=1e-05, elementwise_affine=True) + (norm4): LayerNorm((256,), eps=1e-05, elementwise_affine=True) + (cross_attn_image_to_token): Attention( + (q_proj): Linear(in_features=256, out_features=128, bias=True) + (k_proj): Linear(in_features=256, out_features=128, bias=True) + (v_proj): Linear(in_features=256, out_features=128, bias=True) + (out_proj): Linear(in_features=128, out_features=256, bias=True) + ) + ) + ) + (final_attn_token_to_image): Attention( + (q_proj): Linear(in_features=256, out_features=128, bias=True) + (k_proj): Linear(in_features=256, out_features=128, bias=True) + (v_proj): Linear(in_features=256, out_features=128, bias=True) + (out_proj): Linear(in_features=128, out_features=256, bias=True) + ) + (norm_final_attn): LayerNorm((256,), eps=1e-05, elementwise_affine=True) + ) + (iou_token): Embedding(1, 256) + (mask_tokens): Embedding(4, 256) + (obj_score_token): Embedding(1, 256) + (output_upscaling): Sequential( + (0): ConvTranspose2d(256, 64, kernel_size=(2, 2), stride=(2, 2)) + (1): LayerNorm2d() + (2): GELU(approximate='none') + (3): ConvTranspose2d(64, 32, kernel_size=(2, 2), stride=(2, 2)) + (4): GELU(approximate='none') + ) + (conv_s0): Conv2d(256, 32, kernel_size=(1, 1), stride=(1, 1)) + (conv_s1): Conv2d(256, 64, kernel_size=(1, 1), stride=(1, 1)) + (output_hypernetworks_mlps): ModuleList( + (0-3): 4 x MLP( + (layers): ModuleList( + (0-1): 2 x Linear(in_features=256, out_features=256, bias=True) + (2): Linear(in_features=256, out_features=32, bias=True) + ) + (act): ReLU() + ) + ) + (iou_prediction_head): MLP( + (layers): ModuleList( + (0-1): 2 x Linear(in_features=256, out_features=256, bias=True) + (2): Linear(in_features=256, out_features=4, bias=True) + ) + (act): ReLU() + ) + (pred_obj_score_head): MLP( + (layers): ModuleList( + (0-1): 2 x Linear(in_features=256, out_features=256, bias=True) + (2): Linear(in_features=256, out_features=1, bias=True) + ) + (act): ReLU() + ) + ) + (obj_ptr_proj): MLP( + (layers): ModuleList( + (0-2): 3 x Linear(in_features=256, out_features=256, bias=True) + ) + (act): ReLU() + ) + (obj_ptr_tpos_proj): Linear(in_features=256, out_features=64, bias=True) +) +INFO 2025-10-06 06:35:11,535 trainer.py:1062: Total parameters 80.9 M +INFO 2025-10-06 06:35:11,535 trainer.py:1063: Trainable parameters 80.9 M +INFO 2025-10-06 06:35:11,535 trainer.py:1066: Non-Trainable parameters 0 +INFO 2025-10-06 06:35:11,535 trainer.py:1069: ==================== +INFO 2025-10-06 06:35:11,540 trainer.py:1023: Finished setting up components: Model, loss, optim, meters etc. +INFO 2025-10-06 06:35:11,540 trainer.py: 314: Moving components to device cuda:0 and local rank 0. +INFO 2025-10-06 06:35:11,614 trainer.py: 320: Done moving components to device cuda:0 and local rank 0. +INFO 2025-10-06 06:35:11,631 optimizer.py: 248: Matches for param_name [image_encoder.*]: {'image_encoder.trunk.blocks.3.mlp.layers.0.bias', 'image_encoder.neck.convs.0.conv.weight', 'image_encoder.trunk.blocks.19.mlp.layers.1.bias', 'image_encoder.trunk.blocks.18.mlp.layers.0.weight', 'image_encoder.trunk.blocks.21.attn.proj.weight', 'image_encoder.trunk.blocks.15.attn.qkv.bias', 'image_encoder.trunk.blocks.21.mlp.layers.0.bias', 'image_encoder.trunk.blocks.9.norm1.bias', 'image_encoder.neck.convs.2.conv.weight', 'image_encoder.trunk.blocks.2.norm1.bias', 'image_encoder.trunk.blocks.5.attn.proj.weight', 'image_encoder.trunk.blocks.16.norm2.weight', 'image_encoder.trunk.blocks.0.mlp.layers.1.weight', 'image_encoder.trunk.blocks.4.norm1.weight', 'image_encoder.trunk.blocks.13.mlp.layers.1.weight', 'image_encoder.trunk.blocks.8.norm1.bias', 'image_encoder.trunk.blocks.19.attn.proj.bias', 'image_encoder.trunk.blocks.2.norm2.bias', 'image_encoder.trunk.blocks.19.norm1.weight', 'image_encoder.trunk.blocks.5.attn.qkv.bias', 'image_encoder.trunk.blocks.22.attn.qkv.bias', 'image_encoder.trunk.blocks.13.attn.proj.weight', 'image_encoder.trunk.blocks.19.mlp.layers.1.weight', 'image_encoder.trunk.blocks.1.norm1.weight', 'image_encoder.trunk.blocks.10.mlp.layers.0.bias', 'image_encoder.trunk.blocks.8.mlp.layers.1.bias', 'image_encoder.trunk.blocks.21.proj.bias', 'image_encoder.trunk.blocks.2.norm2.weight', 'image_encoder.trunk.blocks.5.norm2.weight', 'image_encoder.trunk.blocks.2.attn.proj.weight', 'image_encoder.trunk.blocks.11.mlp.layers.0.weight', 'image_encoder.trunk.blocks.1.attn.qkv.bias', 'image_encoder.trunk.blocks.12.mlp.layers.0.weight', 'image_encoder.trunk.blocks.16.norm1.bias', 'image_encoder.trunk.blocks.0.norm2.bias', 'image_encoder.trunk.blocks.10.norm1.bias', 'image_encoder.trunk.blocks.1.mlp.layers.0.weight', 'image_encoder.trunk.blocks.14.mlp.layers.1.bias', 'image_encoder.trunk.blocks.19.norm2.bias', 'image_encoder.trunk.blocks.14.norm1.bias', 'image_encoder.trunk.blocks.20.attn.qkv.bias', 'image_encoder.trunk.blocks.6.attn.qkv.weight', 'image_encoder.trunk.blocks.12.norm2.weight', 'image_encoder.trunk.blocks.21.mlp.layers.0.weight', 'image_encoder.trunk.blocks.18.mlp.layers.0.bias', 'image_encoder.trunk.blocks.1.mlp.layers.0.bias', 'image_encoder.trunk.blocks.23.norm2.bias', 'image_encoder.trunk.blocks.4.norm2.weight', 'image_encoder.trunk.blocks.9.attn.proj.weight', 'image_encoder.trunk.blocks.23.attn.qkv.weight', 'image_encoder.trunk.blocks.17.norm1.weight', 'image_encoder.trunk.blocks.17.mlp.layers.0.bias', 'image_encoder.trunk.blocks.6.mlp.layers.0.bias', 'image_encoder.trunk.blocks.1.attn.proj.weight', 'image_encoder.neck.convs.0.conv.bias', 'image_encoder.trunk.blocks.3.attn.proj.bias', 'image_encoder.trunk.blocks.15.norm1.bias', 'image_encoder.trunk.blocks.7.attn.proj.weight', 'image_encoder.trunk.blocks.14.attn.proj.weight', 'image_encoder.trunk.blocks.23.norm1.bias', 'image_encoder.trunk.blocks.3.attn.qkv.bias', 'image_encoder.trunk.blocks.0.attn.proj.weight', 'image_encoder.trunk.blocks.22.norm2.weight', 'image_encoder.trunk.blocks.7.norm1.bias', 'image_encoder.trunk.blocks.12.mlp.layers.0.bias', 'image_encoder.trunk.blocks.22.norm1.weight', 'image_encoder.trunk.blocks.15.mlp.layers.0.bias', 'image_encoder.trunk.blocks.6.mlp.layers.1.bias', 'image_encoder.trunk.blocks.11.mlp.layers.0.bias', 'image_encoder.trunk.blocks.12.mlp.layers.1.weight', 'image_encoder.trunk.blocks.6.attn.qkv.bias', 'image_encoder.trunk.blocks.6.norm2.weight', 'image_encoder.trunk.blocks.14.norm2.weight', 'image_encoder.trunk.blocks.0.norm2.weight', 'image_encoder.trunk.blocks.3.norm1.bias', 'image_encoder.trunk.blocks.18.attn.proj.weight', 'image_encoder.trunk.blocks.23.norm2.weight', 'image_encoder.trunk.blocks.16.attn.proj.weight', 'image_encoder.trunk.blocks.22.attn.proj.weight', 'image_encoder.trunk.blocks.10.mlp.layers.1.bias', 'image_encoder.trunk.blocks.2.norm1.weight', 'image_encoder.trunk.blocks.13.attn.proj.bias', 'image_encoder.trunk.blocks.22.mlp.layers.0.weight', 'image_encoder.trunk.blocks.4.mlp.layers.1.weight', 'image_encoder.trunk.blocks.14.norm1.weight', 'image_encoder.trunk.blocks.1.attn.qkv.weight', 'image_encoder.trunk.blocks.1.attn.proj.bias', 'image_encoder.trunk.blocks.0.attn.proj.bias', 'image_encoder.trunk.blocks.13.norm1.bias', 'image_encoder.trunk.blocks.12.norm2.bias', 'image_encoder.neck.convs.1.conv.weight', 'image_encoder.neck.convs.2.conv.bias', 'image_encoder.trunk.blocks.4.norm2.bias', 'image_encoder.trunk.blocks.22.attn.qkv.weight', 'image_encoder.trunk.blocks.21.proj.weight', 'image_encoder.trunk.blocks.18.mlp.layers.1.bias', 'image_encoder.trunk.blocks.10.norm2.weight', 'image_encoder.trunk.blocks.19.attn.qkv.weight', 'image_encoder.trunk.blocks.9.norm2.weight', 'image_encoder.trunk.blocks.10.norm2.bias', 'image_encoder.trunk.blocks.5.proj.weight', 'image_encoder.trunk.blocks.17.norm2.bias', 'image_encoder.trunk.blocks.17.attn.proj.bias', 'image_encoder.trunk.blocks.5.mlp.layers.0.bias', 'image_encoder.trunk.blocks.15.norm2.bias', 'image_encoder.trunk.blocks.21.norm2.bias', 'image_encoder.trunk.blocks.6.norm1.bias', 'image_encoder.trunk.blocks.20.mlp.layers.0.weight', 'image_encoder.trunk.blocks.0.mlp.layers.0.bias', 'image_encoder.trunk.blocks.20.mlp.layers.1.bias', 'image_encoder.trunk.blocks.4.attn.proj.bias', 'image_encoder.trunk.blocks.18.attn.qkv.weight', 'image_encoder.trunk.blocks.21.mlp.layers.1.weight', 'image_encoder.trunk.blocks.11.attn.proj.weight', 'image_encoder.trunk.blocks.8.attn.qkv.bias', 'image_encoder.trunk.blocks.21.norm1.bias', 'image_encoder.trunk.blocks.16.mlp.layers.0.weight', 'image_encoder.trunk.blocks.2.mlp.layers.1.weight', 'image_encoder.neck.convs.3.conv.bias', 'image_encoder.trunk.blocks.1.mlp.layers.1.bias', 'image_encoder.trunk.blocks.5.norm2.bias', 'image_encoder.trunk.blocks.11.norm1.weight', 'image_encoder.trunk.blocks.3.mlp.layers.1.weight', 'image_encoder.trunk.blocks.18.norm1.bias', 'image_encoder.trunk.blocks.8.norm1.weight', 'image_encoder.trunk.blocks.15.attn.proj.weight', 'image_encoder.trunk.blocks.3.attn.proj.weight', 'image_encoder.trunk.blocks.23.attn.qkv.bias', 'image_encoder.trunk.blocks.2.mlp.layers.1.bias', 'image_encoder.trunk.blocks.23.mlp.layers.0.weight', 'image_encoder.trunk.blocks.0.attn.qkv.weight', 'image_encoder.trunk.blocks.23.attn.proj.bias', 'image_encoder.trunk.blocks.4.norm1.bias', 'image_encoder.trunk.blocks.22.norm1.bias', 'image_encoder.trunk.blocks.14.norm2.bias', 'image_encoder.trunk.blocks.9.mlp.layers.1.weight', 'image_encoder.trunk.blocks.9.norm2.bias', 'image_encoder.trunk.blocks.12.norm1.bias', 'image_encoder.trunk.blocks.10.norm1.weight', 'image_encoder.trunk.blocks.18.mlp.layers.1.weight', 'image_encoder.trunk.blocks.6.mlp.layers.1.weight', 'image_encoder.trunk.blocks.2.attn.qkv.bias', 'image_encoder.trunk.blocks.21.attn.qkv.bias', 'image_encoder.trunk.blocks.2.proj.weight', 'image_encoder.trunk.blocks.12.attn.qkv.bias', 'image_encoder.trunk.blocks.7.attn.qkv.bias', 'image_encoder.trunk.blocks.10.attn.qkv.bias', 'image_encoder.trunk.blocks.9.mlp.layers.0.bias', 'image_encoder.trunk.blocks.9.norm1.weight', 'image_encoder.trunk.blocks.17.attn.qkv.bias', 'image_encoder.trunk.blocks.14.mlp.layers.0.weight', 'image_encoder.trunk.blocks.16.mlp.layers.1.weight', 'image_encoder.trunk.blocks.3.attn.qkv.weight', 'image_encoder.trunk.blocks.15.norm1.weight', 'image_encoder.trunk.blocks.16.attn.proj.bias', 'image_encoder.trunk.blocks.22.attn.proj.bias', 'image_encoder.trunk.blocks.0.norm1.bias', 'image_encoder.trunk.blocks.8.attn.qkv.weight', 'image_encoder.trunk.blocks.7.mlp.layers.1.weight', 'image_encoder.trunk.blocks.20.norm2.bias', 'image_encoder.trunk.blocks.5.attn.qkv.weight', 'image_encoder.trunk.blocks.20.mlp.layers.1.weight', 'image_encoder.trunk.blocks.5.mlp.layers.1.bias', 'image_encoder.trunk.blocks.17.norm1.bias', 'image_encoder.trunk.blocks.15.mlp.layers.0.weight', 'image_encoder.trunk.blocks.8.mlp.layers.0.bias', 'image_encoder.trunk.blocks.8.mlp.layers.0.weight', 'image_encoder.trunk.blocks.21.attn.qkv.weight', 'image_encoder.trunk.blocks.15.attn.proj.bias', 'image_encoder.trunk.blocks.19.norm1.bias', 'image_encoder.trunk.blocks.17.mlp.layers.1.weight', 'image_encoder.trunk.blocks.9.attn.qkv.bias', 'image_encoder.trunk.blocks.13.attn.qkv.weight', 'image_encoder.trunk.blocks.6.norm1.weight', 'image_encoder.trunk.blocks.16.attn.qkv.weight', 'image_encoder.trunk.blocks.11.mlp.layers.1.weight', 'image_encoder.trunk.pos_embed_window', 'image_encoder.trunk.blocks.7.mlp.layers.1.bias', 'image_encoder.trunk.blocks.13.mlp.layers.0.bias', 'image_encoder.trunk.blocks.23.attn.proj.weight', 'image_encoder.trunk.blocks.14.attn.proj.bias', 'image_encoder.trunk.blocks.20.attn.proj.weight', 'image_encoder.trunk.blocks.2.mlp.layers.0.bias', 'image_encoder.trunk.blocks.23.mlp.layers.0.bias', 'image_encoder.trunk.blocks.5.mlp.layers.1.weight', 'image_encoder.trunk.blocks.0.norm1.weight', 'image_encoder.trunk.blocks.16.norm2.bias', 'image_encoder.trunk.blocks.10.attn.proj.bias', 'image_encoder.trunk.blocks.3.norm1.weight', 'image_encoder.neck.convs.1.conv.bias', 'image_encoder.trunk.blocks.18.norm2.bias', 'image_encoder.trunk.blocks.19.attn.qkv.bias', 'image_encoder.trunk.blocks.13.norm1.weight', 'image_encoder.trunk.blocks.8.attn.proj.weight', 'image_encoder.trunk.blocks.21.norm2.weight', 'image_encoder.trunk.blocks.12.norm1.weight', 'image_encoder.trunk.blocks.7.norm1.weight', 'image_encoder.trunk.blocks.17.attn.proj.weight', 'image_encoder.trunk.blocks.6.norm2.bias', 'image_encoder.trunk.blocks.18.norm1.weight', 'image_encoder.trunk.blocks.4.mlp.layers.0.bias', 'image_encoder.trunk.blocks.5.proj.bias', 'image_encoder.trunk.blocks.19.mlp.layers.0.bias', 'image_encoder.trunk.blocks.2.attn.qkv.weight', 'image_encoder.trunk.blocks.8.attn.proj.bias', 'image_encoder.trunk.blocks.1.norm2.weight', 'image_encoder.trunk.patch_embed.proj.weight', 'image_encoder.trunk.blocks.23.mlp.layers.1.weight', 'image_encoder.trunk.blocks.11.attn.qkv.weight', 'image_encoder.trunk.blocks.14.attn.qkv.bias', 'image_encoder.trunk.blocks.9.mlp.layers.1.bias', 'image_encoder.trunk.blocks.13.norm2.bias', 'image_encoder.trunk.blocks.4.attn.qkv.weight', 'image_encoder.trunk.blocks.6.attn.proj.weight', 'image_encoder.trunk.blocks.9.attn.proj.bias', 'image_encoder.trunk.blocks.11.attn.qkv.bias', 'image_encoder.trunk.blocks.22.mlp.layers.1.bias', 'image_encoder.trunk.blocks.15.mlp.layers.1.bias', 'image_encoder.trunk.blocks.17.mlp.layers.1.bias', 'image_encoder.trunk.blocks.20.norm2.weight', 'image_encoder.trunk.blocks.12.attn.proj.bias', 'image_encoder.trunk.blocks.1.norm1.bias', 'image_encoder.trunk.blocks.23.norm1.weight', 'image_encoder.trunk.blocks.16.mlp.layers.1.bias', 'image_encoder.trunk.blocks.16.norm1.weight', 'image_encoder.trunk.blocks.18.attn.qkv.bias', 'image_encoder.trunk.blocks.11.norm1.bias', 'image_encoder.trunk.blocks.7.mlp.layers.0.bias', 'image_encoder.trunk.blocks.21.mlp.layers.1.bias', 'image_encoder.trunk.blocks.14.attn.qkv.weight', 'image_encoder.trunk.blocks.5.mlp.layers.0.weight', 'image_encoder.trunk.blocks.21.attn.proj.bias', 'image_encoder.trunk.blocks.11.norm2.weight', 'image_encoder.trunk.blocks.13.mlp.layers.0.weight', 'image_encoder.trunk.blocks.17.mlp.layers.0.weight', 'image_encoder.trunk.blocks.3.mlp.layers.1.bias', 'image_encoder.trunk.blocks.22.mlp.layers.1.weight', 'image_encoder.trunk.blocks.7.attn.proj.bias', 'image_encoder.trunk.blocks.16.attn.qkv.bias', 'image_encoder.trunk.blocks.2.proj.bias', 'image_encoder.trunk.blocks.5.norm1.weight', 'image_encoder.trunk.blocks.4.mlp.layers.1.bias', 'image_encoder.trunk.blocks.23.mlp.layers.1.bias', 'image_encoder.trunk.blocks.0.attn.qkv.bias', 'image_encoder.trunk.blocks.10.attn.proj.weight', 'image_encoder.trunk.blocks.1.norm2.bias', 'image_encoder.trunk.blocks.14.mlp.layers.0.bias', 'image_encoder.trunk.blocks.8.norm2.weight', 'image_encoder.trunk.blocks.8.mlp.layers.1.weight', 'image_encoder.trunk.blocks.11.norm2.bias', 'image_encoder.trunk.blocks.18.attn.proj.bias', 'image_encoder.trunk.blocks.13.mlp.layers.1.bias', 'image_encoder.trunk.blocks.2.mlp.layers.0.weight', 'image_encoder.trunk.blocks.15.mlp.layers.1.weight', 'image_encoder.trunk.blocks.20.norm1.bias', 'image_encoder.trunk.blocks.20.mlp.layers.0.bias', 'image_encoder.trunk.blocks.2.attn.proj.bias', 'image_encoder.trunk.blocks.20.attn.proj.bias', 'image_encoder.trunk.blocks.13.norm2.weight', 'image_encoder.trunk.blocks.19.mlp.layers.0.weight', 'image_encoder.trunk.blocks.7.attn.qkv.weight', 'image_encoder.trunk.blocks.3.norm2.bias', 'image_encoder.trunk.blocks.12.mlp.layers.1.bias', 'image_encoder.trunk.blocks.22.norm2.bias', 'image_encoder.trunk.blocks.17.attn.qkv.weight', 'image_encoder.trunk.blocks.7.norm2.weight', 'image_encoder.trunk.blocks.1.mlp.layers.1.weight', 'image_encoder.trunk.blocks.7.norm2.bias', 'image_encoder.trunk.blocks.15.norm2.weight', 'image_encoder.trunk.blocks.9.mlp.layers.0.weight', 'image_encoder.trunk.patch_embed.proj.bias', 'image_encoder.trunk.blocks.5.attn.proj.bias', 'image_encoder.trunk.blocks.20.attn.qkv.weight', 'image_encoder.trunk.blocks.6.attn.proj.bias', 'image_encoder.trunk.blocks.4.attn.qkv.bias', 'image_encoder.trunk.pos_embed', 'image_encoder.neck.convs.3.conv.weight', 'image_encoder.trunk.blocks.0.mlp.layers.0.weight', 'image_encoder.trunk.blocks.7.mlp.layers.0.weight', 'image_encoder.trunk.blocks.12.attn.qkv.weight', 'image_encoder.trunk.blocks.17.norm2.weight', 'image_encoder.trunk.blocks.3.mlp.layers.0.weight', 'image_encoder.trunk.blocks.4.attn.proj.weight', 'image_encoder.trunk.blocks.6.mlp.layers.0.weight', 'image_encoder.trunk.blocks.22.mlp.layers.0.bias', 'image_encoder.trunk.blocks.13.attn.qkv.bias', 'image_encoder.trunk.blocks.14.mlp.layers.1.weight', 'image_encoder.trunk.blocks.4.mlp.layers.0.weight', 'image_encoder.trunk.blocks.11.mlp.layers.1.bias', 'image_encoder.trunk.blocks.16.mlp.layers.0.bias', 'image_encoder.trunk.blocks.3.norm2.weight', 'image_encoder.trunk.blocks.19.norm2.weight', 'image_encoder.trunk.blocks.5.norm1.bias', 'image_encoder.trunk.blocks.20.norm1.weight', 'image_encoder.trunk.blocks.8.norm2.bias', 'image_encoder.trunk.blocks.19.attn.proj.weight', 'image_encoder.trunk.blocks.9.attn.qkv.weight', 'image_encoder.trunk.blocks.11.attn.proj.bias', 'image_encoder.trunk.blocks.0.mlp.layers.1.bias', 'image_encoder.trunk.blocks.21.norm1.weight', 'image_encoder.trunk.blocks.10.mlp.layers.1.weight', 'image_encoder.trunk.blocks.15.attn.qkv.weight', 'image_encoder.trunk.blocks.18.norm2.weight', 'image_encoder.trunk.blocks.10.attn.qkv.weight', 'image_encoder.trunk.blocks.10.mlp.layers.0.weight', 'image_encoder.trunk.blocks.12.attn.proj.weight'} +INFO 2025-10-06 06:35:11,632 optimizer.py: 248: Matches for param_name [*bias*]: {'image_encoder.trunk.blocks.3.mlp.layers.0.bias', 'memory_attention.layers.1.linear1.bias', 'image_encoder.trunk.blocks.19.mlp.layers.1.bias', 'memory_attention.layers.3.self_attn.out_proj.bias', 'memory_attention.layers.0.cross_attn_image.out_proj.bias', 'obj_ptr_proj.layers.0.bias', 'sam_mask_decoder.transformer.final_attn_token_to_image.out_proj.bias', 'sam_mask_decoder.pred_obj_score_head.layers.1.bias', 'image_encoder.trunk.blocks.15.attn.qkv.bias', 'image_encoder.trunk.blocks.21.mlp.layers.0.bias', 'sam_mask_decoder.transformer.layers.1.cross_attn_image_to_token.out_proj.bias', 'memory_attention.layers.3.norm3.bias', 'image_encoder.trunk.blocks.9.norm1.bias', 'image_encoder.trunk.blocks.2.norm1.bias', 'image_encoder.trunk.blocks.19.attn.proj.bias', 'image_encoder.trunk.blocks.2.norm2.bias', 'sam_mask_decoder.transformer.layers.0.mlp.layers.0.bias', 'image_encoder.trunk.blocks.8.norm1.bias', 'memory_attention.layers.2.linear1.bias', 'image_encoder.trunk.blocks.5.attn.qkv.bias', 'image_encoder.trunk.blocks.22.attn.qkv.bias', 'sam_mask_decoder.iou_prediction_head.layers.2.bias', 'sam_mask_decoder.transformer.layers.0.self_attn.k_proj.bias', 'image_encoder.trunk.blocks.10.mlp.layers.0.bias', 'memory_attention.layers.3.cross_attn_image.q_proj.bias', 'image_encoder.trunk.blocks.8.mlp.layers.1.bias', 'image_encoder.trunk.blocks.21.proj.bias', 'memory_attention.layers.0.linear2.bias', 'sam_mask_decoder.transformer.layers.1.norm3.bias', 'sam_mask_decoder.transformer.layers.0.mlp.layers.1.bias', 'image_encoder.trunk.blocks.1.attn.qkv.bias', 'sam_mask_decoder.transformer.final_attn_token_to_image.q_proj.bias', 'sam_mask_decoder.transformer.layers.1.mlp.layers.1.bias', 'image_encoder.trunk.blocks.10.norm1.bias', 'image_encoder.trunk.blocks.16.norm1.bias', 'image_encoder.trunk.blocks.0.norm2.bias', 'image_encoder.trunk.blocks.14.mlp.layers.1.bias', 'memory_attention.layers.3.self_attn.q_proj.bias', 'image_encoder.trunk.blocks.19.norm2.bias', 'sam_mask_decoder.transformer.layers.0.norm3.bias', 'image_encoder.trunk.blocks.14.norm1.bias', 'image_encoder.trunk.blocks.20.attn.qkv.bias', 'sam_mask_decoder.transformer.norm_final_attn.bias', 'image_encoder.trunk.blocks.18.mlp.layers.0.bias', 'memory_attention.layers.1.cross_attn_image.q_proj.bias', 'image_encoder.trunk.blocks.1.mlp.layers.0.bias', 'image_encoder.trunk.blocks.23.norm2.bias', 'memory_attention.layers.1.self_attn.q_proj.bias', 'memory_attention.layers.0.self_attn.k_proj.bias', 'memory_encoder.mask_downsampler.encoder.10.bias', 'memory_attention.layers.2.cross_attn_image.q_proj.bias', 'image_encoder.trunk.blocks.17.mlp.layers.0.bias', 'memory_attention.layers.1.self_attn.k_proj.bias', 'image_encoder.trunk.blocks.6.mlp.layers.0.bias', 'memory_attention.layers.3.cross_attn_image.out_proj.bias', 'image_encoder.neck.convs.0.conv.bias', 'sam_mask_decoder.output_upscaling.0.bias', 'image_encoder.trunk.blocks.3.attn.proj.bias', 'image_encoder.trunk.blocks.15.norm1.bias', 'image_encoder.trunk.blocks.23.norm1.bias', 'image_encoder.trunk.blocks.3.attn.qkv.bias', 'image_encoder.trunk.blocks.7.norm1.bias', 'memory_attention.layers.3.self_attn.v_proj.bias', 'sam_mask_decoder.transformer.layers.1.self_attn.q_proj.bias', 'sam_mask_decoder.transformer.layers.1.self_attn.k_proj.bias', 'image_encoder.trunk.blocks.12.mlp.layers.0.bias', 'image_encoder.trunk.blocks.15.mlp.layers.0.bias', 'image_encoder.trunk.blocks.6.mlp.layers.1.bias', 'memory_attention.layers.1.norm3.bias', 'memory_attention.layers.0.linear1.bias', 'memory_attention.layers.1.cross_attn_image.out_proj.bias', 'image_encoder.trunk.blocks.6.attn.qkv.bias', 'sam_mask_decoder.transformer.layers.0.norm2.bias', 'image_encoder.trunk.blocks.11.mlp.layers.0.bias', 'memory_attention.norm.bias', 'memory_attention.layers.0.self_attn.q_proj.bias', 'memory_encoder.mask_downsampler.encoder.7.bias', 'image_encoder.trunk.blocks.3.norm1.bias', 'memory_attention.layers.1.norm2.bias', 'memory_attention.layers.1.norm1.bias', 'image_encoder.trunk.blocks.10.mlp.layers.1.bias', 'sam_mask_decoder.transformer.layers.1.cross_attn_token_to_image.q_proj.bias', 'obj_ptr_proj.layers.1.bias', 'sam_mask_decoder.output_hypernetworks_mlps.1.layers.2.bias', 'sam_prompt_encoder.mask_downscaling.6.bias', 'image_encoder.trunk.blocks.13.attn.proj.bias', 'image_encoder.trunk.blocks.1.attn.proj.bias', 'image_encoder.trunk.blocks.13.norm1.bias', 'image_encoder.trunk.blocks.12.norm2.bias', 'image_encoder.trunk.blocks.0.attn.proj.bias', 'sam_prompt_encoder.mask_downscaling.3.bias', 'sam_mask_decoder.transformer.layers.0.cross_attn_token_to_image.out_proj.bias', 'sam_mask_decoder.transformer.layers.0.cross_attn_token_to_image.k_proj.bias', 'image_encoder.neck.convs.2.conv.bias', 'image_encoder.trunk.blocks.4.norm2.bias', 'image_encoder.trunk.blocks.18.mlp.layers.1.bias', 'sam_mask_decoder.output_hypernetworks_mlps.0.layers.1.bias', 'sam_prompt_encoder.mask_downscaling.1.bias', 'image_encoder.trunk.blocks.10.norm2.bias', 'sam_prompt_encoder.mask_downscaling.0.bias', 'sam_mask_decoder.transformer.final_attn_token_to_image.v_proj.bias', 'image_encoder.trunk.blocks.17.norm2.bias', 'sam_mask_decoder.transformer.layers.0.cross_attn_token_to_image.v_proj.bias', 'image_encoder.trunk.blocks.17.attn.proj.bias', 'sam_mask_decoder.transformer.layers.0.cross_attn_image_to_token.out_proj.bias', 'image_encoder.trunk.blocks.15.norm2.bias', 'image_encoder.trunk.blocks.21.norm2.bias', 'memory_attention.layers.1.self_attn.out_proj.bias', 'image_encoder.trunk.blocks.6.norm1.bias', 'image_encoder.trunk.blocks.5.mlp.layers.0.bias', 'memory_encoder.mask_downsampler.encoder.6.bias', 'image_encoder.trunk.blocks.0.mlp.layers.0.bias', 'memory_encoder.fuser.layers.1.norm.bias', 'image_encoder.trunk.blocks.20.mlp.layers.1.bias', 'image_encoder.trunk.blocks.4.attn.proj.bias', 'memory_encoder.fuser.layers.0.norm.bias', 'sam_mask_decoder.transformer.layers.1.self_attn.v_proj.bias', 'sam_mask_decoder.transformer.layers.1.cross_attn_image_to_token.q_proj.bias', 'memory_attention.layers.0.self_attn.v_proj.bias', 'image_encoder.trunk.blocks.8.attn.qkv.bias', 'sam_mask_decoder.transformer.layers.0.norm1.bias', 'image_encoder.trunk.blocks.21.norm1.bias', 'sam_mask_decoder.output_hypernetworks_mlps.3.layers.0.bias', 'memory_attention.layers.2.norm3.bias', 'image_encoder.trunk.blocks.1.mlp.layers.1.bias', 'image_encoder.neck.convs.3.conv.bias', 'image_encoder.trunk.blocks.5.norm2.bias', 'sam_mask_decoder.output_hypernetworks_mlps.1.layers.0.bias', 'image_encoder.trunk.blocks.18.norm1.bias', 'memory_attention.layers.3.cross_attn_image.k_proj.bias', 'sam_mask_decoder.transformer.layers.1.mlp.layers.0.bias', 'sam_mask_decoder.transformer.layers.1.cross_attn_token_to_image.out_proj.bias', 'sam_mask_decoder.transformer.layers.1.cross_attn_image_to_token.k_proj.bias', 'memory_attention.layers.3.linear1.bias', 'image_encoder.trunk.blocks.2.mlp.layers.1.bias', 'image_encoder.trunk.blocks.23.attn.qkv.bias', 'sam_mask_decoder.output_hypernetworks_mlps.2.layers.2.bias', 'image_encoder.trunk.blocks.23.attn.proj.bias', 'image_encoder.trunk.blocks.4.norm1.bias', 'image_encoder.trunk.blocks.22.norm1.bias', 'image_encoder.trunk.blocks.14.norm2.bias', 'image_encoder.trunk.blocks.9.norm2.bias', 'memory_attention.layers.1.cross_attn_image.k_proj.bias', 'image_encoder.trunk.blocks.12.norm1.bias', 'memory_attention.layers.0.norm2.bias', 'image_encoder.trunk.blocks.2.attn.qkv.bias', 'image_encoder.trunk.blocks.21.attn.qkv.bias', 'sam_mask_decoder.conv_s0.bias', 'image_encoder.trunk.blocks.12.attn.qkv.bias', 'sam_mask_decoder.transformer.layers.1.self_attn.out_proj.bias', 'image_encoder.trunk.blocks.7.attn.qkv.bias', 'image_encoder.trunk.blocks.10.attn.qkv.bias', 'image_encoder.trunk.blocks.9.mlp.layers.0.bias', 'image_encoder.trunk.blocks.17.attn.qkv.bias', 'image_encoder.trunk.blocks.16.attn.proj.bias', 'image_encoder.trunk.blocks.22.attn.proj.bias', 'image_encoder.trunk.blocks.0.norm1.bias', 'sam_mask_decoder.pred_obj_score_head.layers.2.bias', 'memory_attention.layers.1.linear2.bias', 'image_encoder.trunk.blocks.20.norm2.bias', 'sam_mask_decoder.transformer.layers.1.cross_attn_token_to_image.v_proj.bias', 'sam_mask_decoder.output_hypernetworks_mlps.0.layers.2.bias', 'memory_attention.layers.2.cross_attn_image.out_proj.bias', 'image_encoder.trunk.blocks.5.mlp.layers.1.bias', 'image_encoder.trunk.blocks.17.norm1.bias', 'sam_mask_decoder.transformer.layers.1.norm1.bias', 'sam_mask_decoder.conv_s1.bias', 'image_encoder.trunk.blocks.8.mlp.layers.0.bias', 'memory_attention.layers.2.self_attn.k_proj.bias', 'memory_encoder.fuser.layers.0.pwconv2.bias', 'image_encoder.trunk.blocks.15.attn.proj.bias', 'image_encoder.trunk.blocks.19.norm1.bias', 'memory_attention.layers.0.cross_attn_image.k_proj.bias', 'memory_encoder.mask_downsampler.encoder.12.bias', 'image_encoder.trunk.blocks.9.attn.qkv.bias', 'memory_attention.layers.3.norm2.bias', 'memory_encoder.fuser.layers.1.dwconv.bias', 'image_encoder.trunk.blocks.13.mlp.layers.0.bias', 'image_encoder.trunk.blocks.7.mlp.layers.1.bias', 'image_encoder.trunk.blocks.14.attn.proj.bias', 'image_encoder.trunk.blocks.2.mlp.layers.0.bias', 'image_encoder.trunk.blocks.23.mlp.layers.0.bias', 'memory_attention.layers.2.norm1.bias', 'image_encoder.trunk.blocks.16.norm2.bias', 'image_encoder.trunk.blocks.10.attn.proj.bias', 'image_encoder.neck.convs.1.conv.bias', 'image_encoder.trunk.blocks.18.norm2.bias', 'memory_attention.layers.1.cross_attn_image.v_proj.bias', 'sam_mask_decoder.transformer.layers.1.cross_attn_image_to_token.v_proj.bias', 'image_encoder.trunk.blocks.19.attn.qkv.bias', 'sam_mask_decoder.output_hypernetworks_mlps.2.layers.0.bias', 'memory_attention.layers.0.cross_attn_image.v_proj.bias', 'image_encoder.trunk.blocks.6.norm2.bias', 'image_encoder.trunk.blocks.4.mlp.layers.0.bias', 'image_encoder.trunk.blocks.5.proj.bias', 'obj_ptr_tpos_proj.bias', 'image_encoder.trunk.blocks.19.mlp.layers.0.bias', 'image_encoder.trunk.blocks.8.attn.proj.bias', 'image_encoder.trunk.blocks.14.attn.qkv.bias', 'image_encoder.trunk.blocks.9.mlp.layers.1.bias', 'memory_attention.layers.2.self_attn.out_proj.bias', 'memory_encoder.pix_feat_proj.bias', 'image_encoder.trunk.blocks.13.norm2.bias', 'memory_attention.layers.2.linear2.bias', 'sam_mask_decoder.transformer.layers.0.cross_attn_token_to_image.q_proj.bias', 'sam_mask_decoder.transformer.layers.0.self_attn.out_proj.bias', 'image_encoder.trunk.blocks.9.attn.proj.bias', 'image_encoder.trunk.blocks.11.attn.qkv.bias', 'image_encoder.trunk.blocks.22.mlp.layers.1.bias', 'image_encoder.trunk.blocks.15.mlp.layers.1.bias', 'sam_mask_decoder.transformer.layers.0.norm4.bias', 'image_encoder.trunk.blocks.17.mlp.layers.1.bias', 'sam_mask_decoder.transformer.layers.1.cross_attn_token_to_image.k_proj.bias', 'memory_encoder.fuser.layers.0.dwconv.bias', 'image_encoder.trunk.blocks.12.attn.proj.bias', 'image_encoder.trunk.blocks.1.norm1.bias', 'image_encoder.trunk.blocks.16.mlp.layers.1.bias', 'sam_mask_decoder.transformer.layers.1.norm2.bias', 'image_encoder.trunk.blocks.18.attn.qkv.bias', 'image_encoder.trunk.blocks.11.norm1.bias', 'memory_encoder.fuser.layers.1.pwconv2.bias', 'memory_encoder.mask_downsampler.encoder.0.bias', 'image_encoder.trunk.blocks.7.mlp.layers.0.bias', 'image_encoder.trunk.blocks.21.mlp.layers.1.bias', 'image_encoder.trunk.blocks.21.attn.proj.bias', 'mask_downsample.bias', 'image_encoder.trunk.blocks.3.mlp.layers.1.bias', 'image_encoder.trunk.blocks.7.attn.proj.bias', 'image_encoder.trunk.blocks.16.attn.qkv.bias', 'image_encoder.trunk.blocks.2.proj.bias', 'sam_prompt_encoder.mask_downscaling.4.bias', 'memory_attention.layers.2.cross_attn_image.v_proj.bias', 'image_encoder.trunk.blocks.4.mlp.layers.1.bias', 'image_encoder.trunk.blocks.23.mlp.layers.1.bias', 'image_encoder.trunk.blocks.0.attn.qkv.bias', 'image_encoder.trunk.blocks.1.norm2.bias', 'image_encoder.trunk.blocks.14.mlp.layers.0.bias', 'memory_encoder.mask_downsampler.encoder.3.bias', 'memory_encoder.mask_downsampler.encoder.4.bias', 'sam_mask_decoder.output_hypernetworks_mlps.2.layers.1.bias', 'image_encoder.trunk.blocks.11.norm2.bias', 'image_encoder.trunk.blocks.18.attn.proj.bias', 'memory_attention.layers.0.self_attn.out_proj.bias', 'image_encoder.trunk.blocks.13.mlp.layers.1.bias', 'sam_mask_decoder.transformer.layers.0.self_attn.q_proj.bias', 'sam_mask_decoder.transformer.layers.1.norm4.bias', 'sam_mask_decoder.output_hypernetworks_mlps.3.layers.1.bias', 'image_encoder.trunk.blocks.20.norm1.bias', 'image_encoder.trunk.blocks.20.mlp.layers.0.bias', 'memory_attention.layers.2.cross_attn_image.k_proj.bias', 'image_encoder.trunk.blocks.2.attn.proj.bias', 'sam_mask_decoder.output_hypernetworks_mlps.0.layers.0.bias', 'image_encoder.trunk.blocks.20.attn.proj.bias', 'image_encoder.trunk.blocks.3.norm2.bias', 'image_encoder.trunk.blocks.12.mlp.layers.1.bias', 'image_encoder.trunk.blocks.22.norm2.bias', 'memory_attention.layers.0.norm3.bias', 'memory_attention.layers.2.self_attn.q_proj.bias', 'memory_attention.layers.3.norm1.bias', 'image_encoder.trunk.blocks.7.norm2.bias', 'memory_encoder.mask_downsampler.encoder.1.bias', 'image_encoder.trunk.patch_embed.proj.bias', 'image_encoder.trunk.blocks.5.attn.proj.bias', 'image_encoder.trunk.blocks.6.attn.proj.bias', 'sam_mask_decoder.output_hypernetworks_mlps.1.layers.1.bias', 'image_encoder.trunk.blocks.4.attn.qkv.bias', 'memory_attention.layers.3.cross_attn_image.v_proj.bias', 'sam_mask_decoder.transformer.layers.0.cross_attn_image_to_token.q_proj.bias', 'memory_encoder.fuser.layers.0.pwconv1.bias', 'image_encoder.trunk.blocks.22.mlp.layers.0.bias', 'sam_mask_decoder.transformer.final_attn_token_to_image.k_proj.bias', 'sam_mask_decoder.output_upscaling.3.bias', 'image_encoder.trunk.blocks.13.attn.qkv.bias', 'memory_attention.layers.3.linear2.bias', 'memory_attention.layers.0.cross_attn_image.q_proj.bias', 'memory_attention.layers.0.norm1.bias', 'memory_encoder.out_proj.bias', 'sam_mask_decoder.pred_obj_score_head.layers.0.bias', 'image_encoder.trunk.blocks.11.mlp.layers.1.bias', 'memory_attention.layers.2.self_attn.v_proj.bias', 'sam_mask_decoder.transformer.layers.0.self_attn.v_proj.bias', 'obj_ptr_proj.layers.2.bias', 'image_encoder.trunk.blocks.5.norm1.bias', 'memory_attention.layers.2.norm2.bias', 'image_encoder.trunk.blocks.16.mlp.layers.0.bias', 'memory_encoder.fuser.layers.1.pwconv1.bias', 'memory_attention.layers.1.self_attn.v_proj.bias', 'sam_mask_decoder.iou_prediction_head.layers.1.bias', 'sam_mask_decoder.transformer.layers.0.cross_attn_image_to_token.v_proj.bias', 'sam_mask_decoder.output_hypernetworks_mlps.3.layers.2.bias', 'image_encoder.trunk.blocks.8.norm2.bias', 'image_encoder.trunk.blocks.11.attn.proj.bias', 'image_encoder.trunk.blocks.0.mlp.layers.1.bias', 'sam_mask_decoder.transformer.layers.0.cross_attn_image_to_token.k_proj.bias', 'memory_encoder.mask_downsampler.encoder.9.bias', 'sam_mask_decoder.iou_prediction_head.layers.0.bias', 'sam_mask_decoder.output_upscaling.1.bias', 'memory_attention.layers.3.self_attn.k_proj.bias'} +INFO 2025-10-06 06:35:11,633 optimizer.py: 220: Matches for module_cls_name [torch.nn.LayerNorm]: {'memory_attention.layers.3.norm3.bias', 'image_encoder.trunk.blocks.9.norm1.bias', 'image_encoder.trunk.blocks.2.norm1.bias', 'image_encoder.trunk.blocks.4.norm1.weight', 'image_encoder.trunk.blocks.2.norm2.bias', 'image_encoder.trunk.blocks.16.norm2.weight', 'sam_mask_decoder.transformer.layers.1.norm3.weight', 'image_encoder.trunk.blocks.8.norm1.bias', 'image_encoder.trunk.blocks.19.norm1.weight', 'image_encoder.trunk.blocks.1.norm1.weight', 'image_encoder.trunk.blocks.2.norm2.weight', 'image_encoder.trunk.blocks.5.norm2.weight', 'sam_mask_decoder.transformer.layers.1.norm3.bias', 'image_encoder.trunk.blocks.16.norm1.bias', 'image_encoder.trunk.blocks.0.norm2.bias', 'image_encoder.trunk.blocks.10.norm1.bias', 'memory_attention.layers.2.norm3.weight', 'image_encoder.trunk.blocks.19.norm2.bias', 'image_encoder.trunk.blocks.14.norm1.bias', 'sam_mask_decoder.transformer.layers.0.norm3.bias', 'image_encoder.trunk.blocks.12.norm2.weight', 'memory_attention.layers.1.norm1.weight', 'sam_mask_decoder.transformer.norm_final_attn.bias', 'image_encoder.trunk.blocks.4.norm2.weight', 'image_encoder.trunk.blocks.23.norm2.bias', 'image_encoder.trunk.blocks.17.norm1.weight', 'sam_mask_decoder.transformer.layers.0.norm4.weight', 'memory_attention.layers.2.norm2.weight', 'memory_attention.layers.3.norm2.weight', 'image_encoder.trunk.blocks.15.norm1.bias', 'image_encoder.trunk.blocks.23.norm1.bias', 'image_encoder.trunk.blocks.7.norm1.bias', 'image_encoder.trunk.blocks.22.norm2.weight', 'image_encoder.trunk.blocks.22.norm1.weight', 'memory_attention.layers.1.norm3.bias', 'sam_mask_decoder.transformer.layers.0.norm2.bias', 'memory_attention.norm.bias', 'image_encoder.trunk.blocks.0.norm2.weight', 'image_encoder.trunk.blocks.6.norm2.weight', 'image_encoder.trunk.blocks.14.norm2.weight', 'image_encoder.trunk.blocks.3.norm1.bias', 'image_encoder.trunk.blocks.23.norm2.weight', 'memory_attention.layers.1.norm2.bias', 'memory_attention.layers.0.norm2.weight', 'image_encoder.trunk.blocks.2.norm1.weight', 'memory_attention.layers.1.norm1.bias', 'image_encoder.trunk.blocks.14.norm1.weight', 'image_encoder.trunk.blocks.13.norm1.bias', 'image_encoder.trunk.blocks.12.norm2.bias', 'sam_mask_decoder.transformer.layers.0.norm2.weight', 'image_encoder.trunk.blocks.4.norm2.bias', 'image_encoder.trunk.blocks.10.norm2.weight', 'image_encoder.trunk.blocks.9.norm2.weight', 'image_encoder.trunk.blocks.10.norm2.bias', 'image_encoder.trunk.blocks.17.norm2.bias', 'image_encoder.trunk.blocks.15.norm2.bias', 'image_encoder.trunk.blocks.21.norm2.bias', 'image_encoder.trunk.blocks.6.norm1.bias', 'sam_mask_decoder.transformer.layers.1.norm2.weight', 'image_encoder.trunk.blocks.21.norm1.bias', 'sam_mask_decoder.transformer.layers.0.norm1.bias', 'memory_attention.layers.3.norm3.weight', 'memory_attention.layers.3.norm1.weight', 'memory_attention.layers.2.norm3.bias', 'image_encoder.trunk.blocks.5.norm2.bias', 'image_encoder.trunk.blocks.11.norm1.weight', 'image_encoder.trunk.blocks.18.norm1.bias', 'image_encoder.trunk.blocks.8.norm1.weight', 'image_encoder.trunk.blocks.14.norm2.bias', 'image_encoder.trunk.blocks.22.norm1.bias', 'image_encoder.trunk.blocks.9.norm2.bias', 'image_encoder.trunk.blocks.4.norm1.bias', 'image_encoder.trunk.blocks.12.norm1.bias', 'memory_attention.layers.0.norm2.bias', 'image_encoder.trunk.blocks.10.norm1.weight', 'sam_mask_decoder.transformer.norm_final_attn.weight', 'image_encoder.trunk.blocks.9.norm1.weight', 'memory_attention.layers.0.norm1.weight', 'image_encoder.trunk.blocks.15.norm1.weight', 'image_encoder.trunk.blocks.0.norm1.bias', 'memory_attention.norm.weight', 'image_encoder.trunk.blocks.20.norm2.bias', 'sam_mask_decoder.transformer.layers.1.norm4.weight', 'sam_mask_decoder.transformer.layers.1.norm1.bias', 'image_encoder.trunk.blocks.17.norm1.bias', 'image_encoder.trunk.blocks.19.norm1.bias', 'sam_mask_decoder.transformer.layers.0.norm1.weight', 'image_encoder.trunk.blocks.6.norm1.weight', 'memory_attention.layers.3.norm2.bias', 'image_encoder.trunk.blocks.0.norm1.weight', 'memory_attention.layers.2.norm1.bias', 'image_encoder.trunk.blocks.16.norm2.bias', 'image_encoder.trunk.blocks.3.norm1.weight', 'image_encoder.trunk.blocks.18.norm2.bias', 'image_encoder.trunk.blocks.13.norm1.weight', 'image_encoder.trunk.blocks.21.norm2.weight', 'image_encoder.trunk.blocks.7.norm1.weight', 'image_encoder.trunk.blocks.12.norm1.weight', 'image_encoder.trunk.blocks.6.norm2.bias', 'image_encoder.trunk.blocks.18.norm1.weight', 'image_encoder.trunk.blocks.1.norm2.weight', 'memory_attention.layers.0.norm3.weight', 'image_encoder.trunk.blocks.13.norm2.bias', 'sam_mask_decoder.transformer.layers.0.norm4.bias', 'memory_attention.layers.1.norm2.weight', 'image_encoder.trunk.blocks.20.norm2.weight', 'image_encoder.trunk.blocks.1.norm1.bias', 'image_encoder.trunk.blocks.23.norm1.weight', 'image_encoder.trunk.blocks.16.norm1.weight', 'sam_mask_decoder.transformer.layers.1.norm2.bias', 'image_encoder.trunk.blocks.11.norm1.bias', 'sam_mask_decoder.transformer.layers.1.norm1.weight', 'image_encoder.trunk.blocks.11.norm2.weight', 'image_encoder.trunk.blocks.5.norm1.weight', 'image_encoder.trunk.blocks.1.norm2.bias', 'image_encoder.trunk.blocks.8.norm2.weight', 'image_encoder.trunk.blocks.11.norm2.bias', 'memory_attention.layers.1.norm3.weight', 'sam_mask_decoder.transformer.layers.0.norm3.weight', 'sam_mask_decoder.transformer.layers.1.norm4.bias', 'image_encoder.trunk.blocks.20.norm1.bias', 'image_encoder.trunk.blocks.13.norm2.weight', 'image_encoder.trunk.blocks.3.norm2.bias', 'memory_attention.layers.2.norm1.weight', 'image_encoder.trunk.blocks.22.norm2.bias', 'memory_attention.layers.0.norm3.bias', 'image_encoder.trunk.blocks.7.norm2.weight', 'image_encoder.trunk.blocks.7.norm2.bias', 'memory_attention.layers.3.norm1.bias', 'image_encoder.trunk.blocks.15.norm2.weight', 'image_encoder.trunk.blocks.17.norm2.weight', 'memory_attention.layers.0.norm1.bias', 'image_encoder.trunk.blocks.3.norm2.weight', 'image_encoder.trunk.blocks.19.norm2.weight', 'image_encoder.trunk.blocks.5.norm1.bias', 'memory_attention.layers.2.norm2.bias', 'image_encoder.trunk.blocks.20.norm1.weight', 'image_encoder.trunk.blocks.8.norm2.bias', 'image_encoder.trunk.blocks.18.norm2.weight', 'image_encoder.trunk.blocks.21.norm1.weight'} +INFO 2025-10-06 06:35:12,861 sam2_datasets.py: 125: Dataset mixing probabilities: [1.0] +INFO 2025-10-06 06:35:13,341 trainer.py: 417: Loading pretrained checkpoint from {'_partial_': True, '_target_': 'training.utils.checkpoint_utils.load_state_dict_into_model', 'strict': True, 'ignore_unexpected_keys': None, 'ignore_missing_keys': None, 'state_dict': {'_target_': 'training.utils.checkpoint_utils.load_checkpoint_and_apply_kernels', 'checkpoint_path': '/work/nvme/bfgd/cmaduabuchi/MODELS/THRIVESEG/sam2.1_hiera_base_plus.pt', 'ckpt_state_dict_keys': ['model']}} +INFO 2025-10-06 06:53:51,461 trainer.py: 950: Estimated time remaining: 00d 12h 05m +INFO 2025-10-06 06:53:51,482 trainer.py: 892: Synchronizing meters +INFO 2025-10-06 06:53:51,482 trainer.py: 830: Losses and meters: {'Losses/train_all_loss': 0.6424086557137584, 'Losses/train_all_loss_mask': 0.016180070180966514, 'Losses/train_all_loss_dice': 0.2685501508280366, 'Losses/train_all_loss_iou': 0.05025709560852524, 'Losses/train_all_loss_class': 1.2424740680067156e-08, 'Losses/train_all_core_loss': 0.6424086557137584, 'Trainer/where': 0.0249937374749499, 'Trainer/epoch': 0, 'Trainer/steps_train': 3992} +INFO 2025-10-06 07:12:22,839 trainer.py: 950: Estimated time remaining: 00d 11h 41m +INFO 2025-10-06 07:12:22,861 trainer.py: 892: Synchronizing meters +INFO 2025-10-06 07:12:22,861 trainer.py: 830: Losses and meters: {'Losses/train_all_loss': 0.5279103564334108, 'Losses/train_all_loss_mask': 0.012948783188376493, 'Losses/train_all_loss_dice': 0.23409812990016116, 'Losses/train_all_loss_iou': 0.034836555814920164, 'Losses/train_all_loss_class': 1.1137212414612728e-08, 'Losses/train_all_core_loss': 0.5279103564334108, 'Trainer/where': 0.0499937374749499, 'Trainer/epoch': 1, 'Trainer/steps_train': 7984} +INFO 2025-10-06 07:30:58,749 trainer.py: 950: Estimated time remaining: 00d 11h 25m +INFO 2025-10-06 07:30:58,771 trainer.py: 892: Synchronizing meters +INFO 2025-10-06 07:30:58,771 trainer.py: 830: Losses and meters: {'Losses/train_all_loss': 0.46176709332815996, 'Losses/train_all_loss_mask': 0.010168827943158898, 'Losses/train_all_loss_dice': 0.22589266421142704, 'Losses/train_all_loss_iou': 0.03249785962655957, 'Losses/train_all_loss_class': 1.3400916166038667e-08, 'Losses/train_all_core_loss': 0.46176709332815996, 'Trainer/where': 0.0749937374749499, 'Trainer/epoch': 2, 'Trainer/steps_train': 11976} +INFO 2025-10-06 07:49:42,950 trainer.py: 950: Estimated time remaining: 00d 11h 12m +INFO 2025-10-06 07:49:42,972 trainer.py: 892: Synchronizing meters +INFO 2025-10-06 07:49:42,972 trainer.py: 830: Losses and meters: {'Losses/train_all_loss': 0.434850245800041, 'Losses/train_all_loss_mask': 0.008949943296251449, 'Losses/train_all_loss_dice': 0.22050112636688954, 'Losses/train_all_loss_iou': 0.03535024233930884, 'Losses/train_all_loss_class': 1.4512113502645153e-08, 'Losses/train_all_core_loss': 0.434850245800041, 'Trainer/where': 0.09999373747494991, 'Trainer/epoch': 3, 'Trainer/steps_train': 15968} +INFO 2025-10-06 08:08:34,804 trainer.py: 950: Estimated time remaining: 00d 10h 57m +INFO 2025-10-06 08:08:34,826 trainer.py: 892: Synchronizing meters +INFO 2025-10-06 08:08:34,826 trainer.py: 830: Losses and meters: {'Losses/train_all_loss': 0.4899576190065402, 'Losses/train_all_loss_mask': 0.012139334249143563, 'Losses/train_all_loss_dice': 0.2170970643271425, 'Losses/train_all_loss_iou': 0.030073830085804915, 'Losses/train_all_loss_class': 3.6043506326426754e-08, 'Losses/train_all_core_loss': 0.4899576190065402, 'Trainer/where': 0.1249937374749499, 'Trainer/epoch': 4, 'Trainer/steps_train': 19960} +INFO 2025-10-06 08:27:35,126 trainer.py: 950: Estimated time remaining: 00d 10h 43m +INFO 2025-10-06 08:27:35,148 trainer.py: 892: Synchronizing meters +INFO 2025-10-06 08:27:35,148 trainer.py: 830: Losses and meters: {'Losses/train_all_loss': 0.410329947133319, 'Losses/train_all_loss_mask': 0.008565576918526002, 'Losses/train_all_loss_dice': 0.21291188210368395, 'Losses/train_all_loss_iou': 0.026106498928851203, 'Losses/train_all_loss_class': 2.8962244711128784e-08, 'Losses/train_all_core_loss': 0.410329947133319, 'Trainer/where': 0.1499937374749499, 'Trainer/epoch': 5, 'Trainer/steps_train': 23952} +INFO 2025-10-06 08:46:35,674 trainer.py: 950: Estimated time remaining: 00d 10h 24m +INFO 2025-10-06 08:46:35,695 trainer.py: 892: Synchronizing meters +INFO 2025-10-06 08:46:35,695 trainer.py: 830: Losses and meters: {'Losses/train_all_loss': 0.40882908666967405, 'Losses/train_all_loss_mask': 0.008480613211250676, 'Losses/train_all_loss_dice': 0.21131395521706234, 'Losses/train_all_loss_iou': 0.027902849250237918, 'Losses/train_all_loss_class': 2.056003091207329e-08, 'Losses/train_all_core_loss': 0.40882908666967405, 'Trainer/where': 0.1749937374749499, 'Trainer/epoch': 6, 'Trainer/steps_train': 27944} +INFO 2025-10-06 09:05:27,802 trainer.py: 950: Estimated time remaining: 00d 10h 01m +INFO 2025-10-06 09:05:27,823 trainer.py: 892: Synchronizing meters +INFO 2025-10-06 09:05:27,823 trainer.py: 830: Losses and meters: {'Losses/train_all_loss': 0.4277842160713726, 'Losses/train_all_loss_mask': 0.009505709921017888, 'Losses/train_all_loss_dice': 0.2092911386209165, 'Losses/train_all_loss_iou': 0.02837886042178577, 'Losses/train_all_loss_class': 1.8972794202037397e-08, 'Losses/train_all_core_loss': 0.4277842160713726, 'Trainer/where': 0.19999373747494992, 'Trainer/epoch': 7, 'Trainer/steps_train': 31936} +INFO 2025-10-06 09:24:23,726 trainer.py: 950: Estimated time remaining: 00d 09h 44m +INFO 2025-10-06 09:24:23,748 trainer.py: 892: Synchronizing meters +INFO 2025-10-06 09:24:23,749 trainer.py: 830: Losses and meters: {'Losses/train_all_loss': 0.3915867800609711, 'Losses/train_all_loss_mask': 0.007933716841337687, 'Losses/train_all_loss_dice': 0.20667340398909811, 'Losses/train_all_loss_iou': 0.02623902521838522, 'Losses/train_all_loss_class': 1.5299105419432164e-08, 'Losses/train_all_core_loss': 0.3915867800609711, 'Trainer/where': 0.2249937374749499, 'Trainer/epoch': 8, 'Trainer/steps_train': 35928} +INFO 2025-10-06 09:43:05,171 trainer.py: 950: Estimated time remaining: 00d 09h 18m +INFO 2025-10-06 09:43:05,193 trainer.py: 892: Synchronizing meters +INFO 2025-10-06 09:43:05,193 trainer.py: 830: Losses and meters: {'Losses/train_all_loss': 0.3884903532788829, 'Losses/train_all_loss_mask': 0.00786731781312654, 'Losses/train_all_loss_dice': 0.2054668479936873, 'Losses/train_all_loss_iou': 0.025677137474684822, 'Losses/train_all_loss_class': 1.3447321963829661e-08, 'Losses/train_all_core_loss': 0.3884903532788829, 'Trainer/where': 0.2499937374749499, 'Trainer/epoch': 9, 'Trainer/steps_train': 39920} +INFO 2025-10-06 10:01:40,978 trainer.py: 950: Estimated time remaining: 00d 08h 57m +INFO 2025-10-06 10:01:40,999 trainer.py: 892: Synchronizing meters +INFO 2025-10-06 10:01:41,000 trainer.py: 830: Losses and meters: {'Losses/train_all_loss': 0.3757703946919679, 'Losses/train_all_loss_mask': 0.007310912669020113, 'Losses/train_all_loss_dice': 0.20403454606542606, 'Losses/train_all_loss_iou': 0.025517578823297127, 'Losses/train_all_loss_class': 1.685980004621154e-08, 'Losses/train_all_core_loss': 0.3757703946919679, 'Trainer/where': 0.27499373747494993, 'Trainer/epoch': 10, 'Trainer/steps_train': 43912} +INFO 2025-10-06 10:20:27,893 trainer.py: 950: Estimated time remaining: 00d 08h 43m +INFO 2025-10-06 10:20:27,913 trainer.py: 892: Synchronizing meters +INFO 2025-10-06 10:20:27,913 trainer.py: 830: Losses and meters: {'Losses/train_all_loss': 0.3717774872969974, 'Losses/train_all_loss_mask': 0.0072885528313850366, 'Losses/train_all_loss_dice': 0.20273731466763484, 'Losses/train_all_loss_iou': 0.023269104299064116, 'Losses/train_all_loss_class': 1.295583962462502e-08, 'Losses/train_all_core_loss': 0.3717774872969974, 'Trainer/where': 0.2999937374749499, 'Trainer/epoch': 11, 'Trainer/steps_train': 47904} +INFO 2025-10-06 10:39:13,604 trainer.py: 950: Estimated time remaining: 00d 08h 24m +INFO 2025-10-06 10:39:13,625 trainer.py: 892: Synchronizing meters +INFO 2025-10-06 10:39:13,625 trainer.py: 830: Losses and meters: {'Losses/train_all_loss': 0.3672938159928715, 'Losses/train_all_loss_mask': 0.0071393110150955305, 'Losses/train_all_loss_dice': 0.20154863357066152, 'Losses/train_all_loss_iou': 0.022958954503873592, 'Losses/train_all_loss_class': 9.158463877552852e-09, 'Losses/train_all_core_loss': 0.3672938159928715, 'Trainer/where': 0.3249937374749499, 'Trainer/epoch': 12, 'Trainer/steps_train': 51896} +INFO 2025-10-06 10:58:10,614 trainer.py: 950: Estimated time remaining: 00d 08h 10m +INFO 2025-10-06 10:58:10,636 trainer.py: 892: Synchronizing meters +INFO 2025-10-06 10:58:10,636 trainer.py: 830: Losses and meters: {'Losses/train_all_loss': 0.39827264390161676, 'Losses/train_all_loss_mask': 0.00852792210578642, 'Losses/train_all_loss_dice': 0.2016383660042692, 'Losses/train_all_loss_iou': 0.026075829454423077, 'Losses/train_all_loss_class': 6.9018100357850494e-09, 'Losses/train_all_core_loss': 0.39827264390161676, 'Trainer/where': 0.3499937374749499, 'Trainer/epoch': 13, 'Trainer/steps_train': 55888} +INFO 2025-10-06 11:17:07,125 trainer.py: 950: Estimated time remaining: 00d 07h 51m +INFO 2025-10-06 11:17:07,147 trainer.py: 892: Synchronizing meters +INFO 2025-10-06 11:17:07,147 trainer.py: 830: Losses and meters: {'Losses/train_all_loss': 0.3666049185414744, 'Losses/train_all_loss_mask': 0.007134012589840914, 'Losses/train_all_loss_dice': 0.2001152827081556, 'Losses/train_all_loss_iou': 0.023809378810292475, 'Losses/train_all_loss_class': 7.340426579333355e-09, 'Losses/train_all_core_loss': 0.3666049185414744, 'Trainer/where': 0.3749937374749499, 'Trainer/epoch': 14, 'Trainer/steps_train': 59880} +INFO 2025-10-06 11:36:09,314 trainer.py: 950: Estimated time remaining: 00d 07h 35m +INFO 2025-10-06 11:36:09,335 trainer.py: 892: Synchronizing meters +INFO 2025-10-06 11:36:09,335 trainer.py: 830: Losses and meters: {'Losses/train_all_loss': 0.36204657752245556, 'Losses/train_all_loss_mask': 0.0069741076941301034, 'Losses/train_all_loss_dice': 0.1988546567444572, 'Losses/train_all_loss_iou': 0.02370976204282832, 'Losses/train_all_loss_class': 6.914729194342508e-09, 'Losses/train_all_core_loss': 0.36204657752245556, 'Trainer/where': 0.39999373747494993, 'Trainer/epoch': 15, 'Trainer/steps_train': 63872} +INFO 2025-10-06 11:55:05,493 trainer.py: 950: Estimated time remaining: 00d 07h 13m +INFO 2025-10-06 11:55:05,515 trainer.py: 892: Synchronizing meters +INFO 2025-10-06 11:55:05,515 trainer.py: 830: Losses and meters: {'Losses/train_all_loss': 0.36432591524420266, 'Losses/train_all_loss_mask': 0.00711237450751671, 'Losses/train_all_loss_dice': 0.19882314240108273, 'Losses/train_all_loss_iou': 0.023255279706872696, 'Losses/train_all_loss_class': 5.094582148262241e-09, 'Losses/train_all_core_loss': 0.36432591524420266, 'Trainer/where': 0.4249937374749499, 'Trainer/epoch': 16, 'Trainer/steps_train': 67864} +INFO 2025-10-06 12:14:08,255 trainer.py: 950: Estimated time remaining: 00d 06h 57m +INFO 2025-10-06 12:14:08,277 trainer.py: 892: Synchronizing meters +INFO 2025-10-06 12:14:08,277 trainer.py: 830: Losses and meters: {'Losses/train_all_loss': 0.3739850448014286, 'Losses/train_all_loss_mask': 0.007661915094332017, 'Losses/train_all_loss_dice': 0.19808480110698806, 'Losses/train_all_loss_iou': 0.02266193838917906, 'Losses/train_all_loss_class': 5.422859437734412e-09, 'Losses/train_all_core_loss': 0.3739850448014286, 'Trainer/where': 0.4499937374749499, 'Trainer/epoch': 17, 'Trainer/steps_train': 71856} +INFO 2025-10-06 12:33:11,519 trainer.py: 950: Estimated time remaining: 00d 06h 38m +INFO 2025-10-06 12:33:11,541 trainer.py: 892: Synchronizing meters +INFO 2025-10-06 12:33:11,541 trainer.py: 830: Losses and meters: {'Losses/train_all_loss': 0.3802372602399911, 'Losses/train_all_loss_mask': 0.008000960067011056, 'Losses/train_all_loss_dice': 0.19741905017761524, 'Losses/train_all_loss_iou': 0.02279900476346511, 'Losses/train_all_loss_class': 5.678085751427958e-09, 'Losses/train_all_core_loss': 0.3802372602399911, 'Trainer/where': 0.4749937374749499, 'Trainer/epoch': 18, 'Trainer/steps_train': 75848} +INFO 2025-10-06 12:52:16,750 trainer.py: 950: Estimated time remaining: 00d 06h 20m +INFO 2025-10-06 12:52:16,771 trainer.py: 892: Synchronizing meters +INFO 2025-10-06 12:52:16,772 trainer.py: 830: Losses and meters: {'Losses/train_all_loss': 0.376298183201997, 'Losses/train_all_loss_mask': 0.007830308981010056, 'Losses/train_all_loss_dice': 0.1965342678711983, 'Losses/train_all_loss_iou': 0.023157731618651492, 'Losses/train_all_loss_class': 5.28514186526349e-09, 'Losses/train_all_core_loss': 0.376298183201997, 'Trainer/where': 0.4999937374749499, 'Trainer/epoch': 19, 'Trainer/steps_train': 79840} +INFO 2025-10-06 13:11:24,458 trainer.py: 950: Estimated time remaining: 00d 06h 02m +INFO 2025-10-06 13:11:24,479 trainer.py: 892: Synchronizing meters +INFO 2025-10-06 13:11:24,480 trainer.py: 830: Losses and meters: {'Losses/train_all_loss': 0.3772340752292874, 'Losses/train_all_loss_mask': 0.007914954837213077, 'Losses/train_all_loss_dice': 0.19617746839064634, 'Losses/train_all_loss_iou': 0.022757507024552415, 'Losses/train_all_loss_class': 5.4267739055031245e-09, 'Losses/train_all_core_loss': 0.3772340752292874, 'Trainer/where': 0.5249937374749499, 'Trainer/epoch': 20, 'Trainer/steps_train': 83832} +INFO 2025-10-06 13:30:31,353 trainer.py: 950: Estimated time remaining: 00d 05h 42m +INFO 2025-10-06 13:30:31,379 trainer.py: 892: Synchronizing meters +INFO 2025-10-06 13:30:31,379 trainer.py: 830: Losses and meters: {'Losses/train_all_loss': 0.37047547260336655, 'Losses/train_all_loss_mask': 0.007639010061916952, 'Losses/train_all_loss_dice': 0.1958161407786763, 'Losses/train_all_loss_iou': 0.02187912743158927, 'Losses/train_all_loss_class': 5.4363083103324426e-09, 'Losses/train_all_core_loss': 0.37047547260336655, 'Trainer/where': 0.54999373747495, 'Trainer/epoch': 21, 'Trainer/steps_train': 87824} +INFO 2025-10-06 13:49:40,277 trainer.py: 950: Estimated time remaining: 00d 05h 24m +INFO 2025-10-06 13:49:40,299 trainer.py: 892: Synchronizing meters +INFO 2025-10-06 13:49:40,299 trainer.py: 830: Losses and meters: {'Losses/train_all_loss': 0.35376920676995477, 'Losses/train_all_loss_mask': 0.006838303287161567, 'Losses/train_all_loss_dice': 0.19547302056410032, 'Losses/train_all_loss_iou': 0.021530116622054854, 'Losses/train_all_loss_class': 6.322159236068461e-09, 'Losses/train_all_core_loss': 0.35376920676995477, 'Trainer/where': 0.5749937374749499, 'Trainer/epoch': 22, 'Trainer/steps_train': 91816} +INFO 2025-10-06 14:08:52,182 trainer.py: 950: Estimated time remaining: 00d 05h 06m +INFO 2025-10-06 14:08:52,203 trainer.py: 892: Synchronizing meters +INFO 2025-10-06 14:08:52,203 trainer.py: 830: Losses and meters: {'Losses/train_all_loss': 0.3551223263956834, 'Losses/train_all_loss_mask': 0.006957760564867117, 'Losses/train_all_loss_dice': 0.1945757962957413, 'Losses/train_all_loss_iou': 0.02139131658811324, 'Losses/train_all_loss_class': 4.68340592482518e-09, 'Losses/train_all_core_loss': 0.3551223263956834, 'Trainer/where': 0.5999937374749499, 'Trainer/epoch': 23, 'Trainer/steps_train': 95808} +INFO 2025-10-06 14:28:02,935 trainer.py: 950: Estimated time remaining: 00d 04h 46m +INFO 2025-10-06 14:28:02,956 trainer.py: 892: Synchronizing meters +INFO 2025-10-06 14:28:02,956 trainer.py: 830: Losses and meters: {'Losses/train_all_loss': 0.35436655999975974, 'Losses/train_all_loss_mask': 0.006915995280616119, 'Losses/train_all_loss_dice': 0.19451504708351736, 'Losses/train_all_loss_iou': 0.0215316050213107, 'Losses/train_all_loss_class': 4.694827900397807e-09, 'Losses/train_all_core_loss': 0.35436655999975974, 'Trainer/where': 0.6249937374749499, 'Trainer/epoch': 24, 'Trainer/steps_train': 99800} +INFO 2025-10-06 14:47:14,335 trainer.py: 950: Estimated time remaining: 00d 04h 27m +INFO 2025-10-06 14:47:14,363 trainer.py: 892: Synchronizing meters +INFO 2025-10-06 14:47:14,363 trainer.py: 830: Losses and meters: {'Losses/train_all_loss': 0.3539243154792313, 'Losses/train_all_loss_mask': 0.0069243552351725595, 'Losses/train_all_loss_dice': 0.19411168701364426, 'Losses/train_all_loss_iou': 0.021325522290423973, 'Losses/train_all_loss_class': 4.124210543847477e-09, 'Losses/train_all_core_loss': 0.3539243154792313, 'Trainer/where': 0.6499937374749499, 'Trainer/epoch': 25, 'Trainer/steps_train': 103792} +INFO 2025-10-06 15:06:27,350 trainer.py: 950: Estimated time remaining: 00d 04h 08m +INFO 2025-10-06 15:06:27,371 trainer.py: 892: Synchronizing meters +INFO 2025-10-06 15:06:27,371 trainer.py: 830: Losses and meters: {'Losses/train_all_loss': 0.3885851017036615, 'Losses/train_all_loss_mask': 0.008587317138184215, 'Losses/train_all_loss_dice': 0.1942436489498687, 'Losses/train_all_loss_iou': 0.02259510975809444, 'Losses/train_all_loss_class': 3.7102346460758284e-09, 'Losses/train_all_core_loss': 0.3885851017036615, 'Trainer/where': 0.67499373747495, 'Trainer/epoch': 26, 'Trainer/steps_train': 107784} +INFO 2025-10-06 15:25:39,425 trainer.py: 950: Estimated time remaining: 00d 03h 49m +INFO 2025-10-06 15:25:39,447 trainer.py: 892: Synchronizing meters +INFO 2025-10-06 15:25:39,447 trainer.py: 830: Losses and meters: {'Losses/train_all_loss': 0.7125891149229509, 'Losses/train_all_loss_mask': 0.024751994295421284, 'Losses/train_all_loss_dice': 0.194532726758826, 'Losses/train_all_loss_iou': 0.023016485271324848, 'Losses/train_all_loss_class': 3.9748640794428046e-09, 'Losses/train_all_core_loss': 0.7125891149229509, 'Trainer/where': 0.6999937374749499, 'Trainer/epoch': 27, 'Trainer/steps_train': 111776} +INFO 2025-10-06 15:44:53,252 trainer.py: 950: Estimated time remaining: 00d 03h 30m +INFO 2025-10-06 15:44:53,273 trainer.py: 892: Synchronizing meters +INFO 2025-10-06 15:44:53,273 trainer.py: 830: Losses and meters: {'Losses/train_all_loss': 0.8676063124507335, 'Losses/train_all_loss_mask': 0.032492553048528834, 'Losses/train_all_loss_dice': 0.1948043356587987, 'Losses/train_all_loss_iou': 0.022950956187875175, 'Losses/train_all_loss_class': 4.138141090095644e-09, 'Losses/train_all_core_loss': 0.8676063124507335, 'Trainer/where': 0.7249937374749499, 'Trainer/epoch': 28, 'Trainer/steps_train': 115768} +INFO 2025-10-06 16:04:06,937 trainer.py: 950: Estimated time remaining: 00d 03h 11m +INFO 2025-10-06 16:04:06,959 trainer.py: 892: Synchronizing meters +INFO 2025-10-06 16:04:06,959 trainer.py: 830: Losses and meters: {'Losses/train_all_loss': 0.3487479994164496, 'Losses/train_all_loss_mask': 0.0067503434569706235, 'Losses/train_all_loss_dice': 0.19311891421526373, 'Losses/train_all_loss_iou': 0.020622213926316305, 'Losses/train_all_loss_class': 4.467678567490371e-09, 'Losses/train_all_core_loss': 0.3487479994164496, 'Trainer/where': 0.7499937374749499, 'Trainer/epoch': 29, 'Trainer/steps_train': 119760} +INFO 2025-10-06 16:23:19,709 trainer.py: 950: Estimated time remaining: 00d 02h 52m +INFO 2025-10-06 16:23:19,730 trainer.py: 892: Synchronizing meters +INFO 2025-10-06 16:23:19,730 trainer.py: 830: Losses and meters: {'Losses/train_all_loss': 0.36419604813683787, 'Losses/train_all_loss_mask': 0.00751875088206941, 'Losses/train_all_loss_dice': 0.19322081335740005, 'Losses/train_all_loss_iou': 0.0206002136427904, 'Losses/train_all_loss_class': 5.008077296045301e-09, 'Losses/train_all_core_loss': 0.36419604813683787, 'Trainer/where': 0.7749937374749499, 'Trainer/epoch': 30, 'Trainer/steps_train': 123752} +INFO 2025-10-06 16:42:36,647 trainer.py: 950: Estimated time remaining: 00d 02h 33m +INFO 2025-10-06 16:42:36,669 trainer.py: 892: Synchronizing meters +INFO 2025-10-06 16:42:36,669 trainer.py: 830: Losses and meters: {'Losses/train_all_loss': 0.34595307377449336, 'Losses/train_all_loss_mask': 0.006689208541691566, 'Losses/train_all_loss_dice': 0.19237655501267714, 'Losses/train_all_loss_iou': 0.01979234506802322, 'Losses/train_all_loss_class': 5.1453944205931e-09, 'Losses/train_all_core_loss': 0.34595307377449336, 'Trainer/where': 0.79999373747495, 'Trainer/epoch': 31, 'Trainer/steps_train': 127744} +INFO 2025-10-06 17:01:54,652 trainer.py: 950: Estimated time remaining: 00d 02h 14m +INFO 2025-10-06 17:01:54,674 trainer.py: 892: Synchronizing meters +INFO 2025-10-06 17:01:54,674 trainer.py: 830: Losses and meters: {'Losses/train_all_loss': 0.34565046294508334, 'Losses/train_all_loss_mask': 0.006636826759476352, 'Losses/train_all_loss_dice': 0.19229123223281813, 'Losses/train_all_loss_iou': 0.020622693491710736, 'Losses/train_all_loss_class': 4.764466950168121e-09, 'Losses/train_all_core_loss': 0.34565046294508334, 'Trainer/where': 0.8249937374749499, 'Trainer/epoch': 32, 'Trainer/steps_train': 131736} +INFO 2025-10-06 17:21:09,500 trainer.py: 950: Estimated time remaining: 00d 01h 55m +INFO 2025-10-06 17:21:09,521 trainer.py: 892: Synchronizing meters +INFO 2025-10-06 17:21:09,521 trainer.py: 830: Losses and meters: {'Losses/train_all_loss': 0.9008659057842683, 'Losses/train_all_loss_mask': 0.03423682985725872, 'Losses/train_all_loss_dice': 0.1938983456405227, 'Losses/train_all_loss_iou': 0.02223101315849819, 'Losses/train_all_loss_class': 4.9622400122766974e-09, 'Losses/train_all_core_loss': 0.9008659057842683, 'Trainer/where': 0.8499937374749498, 'Trainer/epoch': 33, 'Trainer/steps_train': 135728} +INFO 2025-10-06 17:40:24,086 trainer.py: 950: Estimated time remaining: 00d 01h 35m +INFO 2025-10-06 17:40:24,108 trainer.py: 892: Synchronizing meters +INFO 2025-10-06 17:40:24,108 trainer.py: 830: Losses and meters: {'Losses/train_all_loss': 0.3465600523536368, 'Losses/train_all_loss_mask': 0.0067609497240599245, 'Losses/train_all_loss_dice': 0.19183460045195294, 'Losses/train_all_loss_iou': 0.019506455022350135, 'Losses/train_all_loss_class': 4.679632632014728e-09, 'Losses/train_all_core_loss': 0.3465600523536368, 'Trainer/where': 0.8749937374749498, 'Trainer/epoch': 34, 'Trainer/steps_train': 139720} +INFO 2025-10-06 17:59:43,886 trainer.py: 950: Estimated time remaining: 00d 01h 17m +INFO 2025-10-06 17:59:43,908 trainer.py: 892: Synchronizing meters +INFO 2025-10-06 17:59:43,908 trainer.py: 830: Losses and meters: {'Losses/train_all_loss': 0.34736052319718447, 'Losses/train_all_loss_mask': 0.0067391459940458545, 'Losses/train_all_loss_dice': 0.1922520945270458, 'Losses/train_all_loss_iou': 0.020325507099748605, 'Losses/train_all_loss_class': 4.45559715646195e-09, 'Losses/train_all_core_loss': 0.34736052319718447, 'Trainer/where': 0.8999937374749498, 'Trainer/epoch': 35, 'Trainer/steps_train': 143712} +INFO 2025-10-06 18:18:47,675 trainer.py: 950: Estimated time remaining: 00d 00h 56m +INFO 2025-10-06 18:18:47,697 trainer.py: 892: Synchronizing meters +INFO 2025-10-06 18:18:47,697 trainer.py: 830: Losses and meters: {'Losses/train_all_loss': 0.3672967086860374, 'Losses/train_all_loss_mask': 0.00773209927745972, 'Losses/train_all_loss_dice': 0.19209734350442886, 'Losses/train_all_loss_iou': 0.02055737860401078, 'Losses/train_all_loss_class': 4.592152288894342e-09, 'Losses/train_all_core_loss': 0.3672967086860374, 'Trainer/where': 0.9249937374749498, 'Trainer/epoch': 36, 'Trainer/steps_train': 147704} +INFO 2025-10-06 18:37:46,625 trainer.py: 950: Estimated time remaining: 00d 00h 37m +INFO 2025-10-06 18:37:46,646 trainer.py: 892: Synchronizing meters +INFO 2025-10-06 18:37:46,646 trainer.py: 830: Losses and meters: {'Losses/train_all_loss': 0.8421824378987265, 'Losses/train_all_loss_mask': 0.03133104354486921, 'Losses/train_all_loss_dice': 0.19316817402003522, 'Losses/train_all_loss_iou': 0.02239339469971622, 'Losses/train_all_loss_class': 4.4197077939473966e-09, 'Losses/train_all_core_loss': 0.8421824378987265, 'Trainer/where': 0.9499937374749499, 'Trainer/epoch': 37, 'Trainer/steps_train': 151696} +INFO 2025-10-06 18:56:48,973 trainer.py: 950: Estimated time remaining: 00d 00h 18m +INFO 2025-10-06 18:56:48,995 trainer.py: 892: Synchronizing meters +INFO 2025-10-06 18:56:48,995 trainer.py: 830: Losses and meters: {'Losses/train_all_loss': 0.3468215012218792, 'Losses/train_all_loss_mask': 0.006743870759146193, 'Losses/train_all_loss_dice': 0.19145133795742997, 'Losses/train_all_loss_iou': 0.020492745865770254, 'Losses/train_all_loss_class': 4.498966078856853e-09, 'Losses/train_all_core_loss': 0.3468215012218792, 'Trainer/where': 0.9749937374749498, 'Trainer/epoch': 38, 'Trainer/steps_train': 155688} +INFO 2025-10-06 19:15:48,999 trainer.py: 950: Estimated time remaining: 00d 00h 00m +INFO 2025-10-06 19:15:49,026 trainer.py: 892: Synchronizing meters +INFO 2025-10-06 19:15:49,026 trainer.py: 830: Losses and meters: {'Losses/train_all_loss': 0.361319512792094, 'Losses/train_all_loss_mask': 0.007438624991261454, 'Losses/train_all_loss_dice': 0.19195612370669243, 'Losses/train_all_loss_iou': 0.0205908868190231, 'Losses/train_all_loss_class': 4.3955577136424175e-09, 'Losses/train_all_core_loss': 0.361319512792094, 'Trainer/where': 0.9999937374749498, 'Trainer/epoch': 39, 'Trainer/steps_train': 159680} diff --git a/experiment/thermal_run1/logs/train_stats.json b/experiment/thermal_run1/logs/train_stats.json new file mode 100644 index 0000000000000000000000000000000000000000..5110e4af2be36b51982fea8f5d4b7e1fa564d42a --- /dev/null +++ b/experiment/thermal_run1/logs/train_stats.json @@ -0,0 +1,40 @@ +{"Losses/train_all_loss": 0.6424086557137584, "Losses/train_all_loss_mask": 0.016180070180966514, "Losses/train_all_loss_dice": 0.2685501508280366, "Losses/train_all_loss_iou": 0.05025709560852524, "Losses/train_all_loss_class": 1.2424740680067156e-08, "Losses/train_all_core_loss": 0.6424086557137584, "Trainer/where": 0.0249937374749499, "Trainer/epoch": 0, "Trainer/steps_train": 3992} +{"Losses/train_all_loss": 0.5279103564334108, "Losses/train_all_loss_mask": 0.012948783188376493, "Losses/train_all_loss_dice": 0.23409812990016116, "Losses/train_all_loss_iou": 0.034836555814920164, "Losses/train_all_loss_class": 1.1137212414612728e-08, "Losses/train_all_core_loss": 0.5279103564334108, "Trainer/where": 0.0499937374749499, "Trainer/epoch": 1, "Trainer/steps_train": 7984} +{"Losses/train_all_loss": 0.46176709332815996, "Losses/train_all_loss_mask": 0.010168827943158898, "Losses/train_all_loss_dice": 0.22589266421142704, "Losses/train_all_loss_iou": 0.03249785962655957, "Losses/train_all_loss_class": 1.3400916166038667e-08, "Losses/train_all_core_loss": 0.46176709332815996, "Trainer/where": 0.0749937374749499, "Trainer/epoch": 2, "Trainer/steps_train": 11976} +{"Losses/train_all_loss": 0.434850245800041, "Losses/train_all_loss_mask": 0.008949943296251449, "Losses/train_all_loss_dice": 0.22050112636688954, "Losses/train_all_loss_iou": 0.03535024233930884, "Losses/train_all_loss_class": 1.4512113502645153e-08, "Losses/train_all_core_loss": 0.434850245800041, "Trainer/where": 0.09999373747494991, "Trainer/epoch": 3, "Trainer/steps_train": 15968} +{"Losses/train_all_loss": 0.4899576190065402, "Losses/train_all_loss_mask": 0.012139334249143563, "Losses/train_all_loss_dice": 0.2170970643271425, "Losses/train_all_loss_iou": 0.030073830085804915, "Losses/train_all_loss_class": 3.6043506326426754e-08, "Losses/train_all_core_loss": 0.4899576190065402, "Trainer/where": 0.1249937374749499, "Trainer/epoch": 4, "Trainer/steps_train": 19960} +{"Losses/train_all_loss": 0.410329947133319, "Losses/train_all_loss_mask": 0.008565576918526002, "Losses/train_all_loss_dice": 0.21291188210368395, "Losses/train_all_loss_iou": 0.026106498928851203, "Losses/train_all_loss_class": 2.8962244711128784e-08, "Losses/train_all_core_loss": 0.410329947133319, "Trainer/where": 0.1499937374749499, "Trainer/epoch": 5, "Trainer/steps_train": 23952} +{"Losses/train_all_loss": 0.40882908666967405, "Losses/train_all_loss_mask": 0.008480613211250676, "Losses/train_all_loss_dice": 0.21131395521706234, "Losses/train_all_loss_iou": 0.027902849250237918, "Losses/train_all_loss_class": 2.056003091207329e-08, "Losses/train_all_core_loss": 0.40882908666967405, "Trainer/where": 0.1749937374749499, "Trainer/epoch": 6, "Trainer/steps_train": 27944} +{"Losses/train_all_loss": 0.4277842160713726, "Losses/train_all_loss_mask": 0.009505709921017888, "Losses/train_all_loss_dice": 0.2092911386209165, "Losses/train_all_loss_iou": 0.02837886042178577, "Losses/train_all_loss_class": 1.8972794202037397e-08, "Losses/train_all_core_loss": 0.4277842160713726, "Trainer/where": 0.19999373747494992, "Trainer/epoch": 7, "Trainer/steps_train": 31936} +{"Losses/train_all_loss": 0.3915867800609711, "Losses/train_all_loss_mask": 0.007933716841337687, "Losses/train_all_loss_dice": 0.20667340398909811, "Losses/train_all_loss_iou": 0.02623902521838522, "Losses/train_all_loss_class": 1.5299105419432164e-08, "Losses/train_all_core_loss": 0.3915867800609711, "Trainer/where": 0.2249937374749499, "Trainer/epoch": 8, "Trainer/steps_train": 35928} +{"Losses/train_all_loss": 0.3884903532788829, "Losses/train_all_loss_mask": 0.00786731781312654, "Losses/train_all_loss_dice": 0.2054668479936873, "Losses/train_all_loss_iou": 0.025677137474684822, "Losses/train_all_loss_class": 1.3447321963829661e-08, "Losses/train_all_core_loss": 0.3884903532788829, "Trainer/where": 0.2499937374749499, "Trainer/epoch": 9, "Trainer/steps_train": 39920} +{"Losses/train_all_loss": 0.3757703946919679, "Losses/train_all_loss_mask": 0.007310912669020113, "Losses/train_all_loss_dice": 0.20403454606542606, "Losses/train_all_loss_iou": 0.025517578823297127, "Losses/train_all_loss_class": 1.685980004621154e-08, "Losses/train_all_core_loss": 0.3757703946919679, "Trainer/where": 0.27499373747494993, "Trainer/epoch": 10, "Trainer/steps_train": 43912} +{"Losses/train_all_loss": 0.3717774872969974, "Losses/train_all_loss_mask": 0.0072885528313850366, "Losses/train_all_loss_dice": 0.20273731466763484, "Losses/train_all_loss_iou": 0.023269104299064116, "Losses/train_all_loss_class": 1.295583962462502e-08, "Losses/train_all_core_loss": 0.3717774872969974, "Trainer/where": 0.2999937374749499, "Trainer/epoch": 11, "Trainer/steps_train": 47904} +{"Losses/train_all_loss": 0.3672938159928715, "Losses/train_all_loss_mask": 0.0071393110150955305, "Losses/train_all_loss_dice": 0.20154863357066152, "Losses/train_all_loss_iou": 0.022958954503873592, "Losses/train_all_loss_class": 9.158463877552852e-09, "Losses/train_all_core_loss": 0.3672938159928715, "Trainer/where": 0.3249937374749499, "Trainer/epoch": 12, "Trainer/steps_train": 51896} +{"Losses/train_all_loss": 0.39827264390161676, "Losses/train_all_loss_mask": 0.00852792210578642, "Losses/train_all_loss_dice": 0.2016383660042692, "Losses/train_all_loss_iou": 0.026075829454423077, "Losses/train_all_loss_class": 6.9018100357850494e-09, "Losses/train_all_core_loss": 0.39827264390161676, "Trainer/where": 0.3499937374749499, "Trainer/epoch": 13, "Trainer/steps_train": 55888} +{"Losses/train_all_loss": 0.3666049185414744, "Losses/train_all_loss_mask": 0.007134012589840914, "Losses/train_all_loss_dice": 0.2001152827081556, "Losses/train_all_loss_iou": 0.023809378810292475, "Losses/train_all_loss_class": 7.340426579333355e-09, "Losses/train_all_core_loss": 0.3666049185414744, "Trainer/where": 0.3749937374749499, "Trainer/epoch": 14, "Trainer/steps_train": 59880} +{"Losses/train_all_loss": 0.36204657752245556, "Losses/train_all_loss_mask": 0.0069741076941301034, "Losses/train_all_loss_dice": 0.1988546567444572, "Losses/train_all_loss_iou": 0.02370976204282832, "Losses/train_all_loss_class": 6.914729194342508e-09, "Losses/train_all_core_loss": 0.36204657752245556, "Trainer/where": 0.39999373747494993, "Trainer/epoch": 15, "Trainer/steps_train": 63872} +{"Losses/train_all_loss": 0.36432591524420266, "Losses/train_all_loss_mask": 0.00711237450751671, "Losses/train_all_loss_dice": 0.19882314240108273, "Losses/train_all_loss_iou": 0.023255279706872696, "Losses/train_all_loss_class": 5.094582148262241e-09, "Losses/train_all_core_loss": 0.36432591524420266, "Trainer/where": 0.4249937374749499, "Trainer/epoch": 16, "Trainer/steps_train": 67864} +{"Losses/train_all_loss": 0.3739850448014286, "Losses/train_all_loss_mask": 0.007661915094332017, "Losses/train_all_loss_dice": 0.19808480110698806, "Losses/train_all_loss_iou": 0.02266193838917906, "Losses/train_all_loss_class": 5.422859437734412e-09, "Losses/train_all_core_loss": 0.3739850448014286, "Trainer/where": 0.4499937374749499, "Trainer/epoch": 17, "Trainer/steps_train": 71856} +{"Losses/train_all_loss": 0.3802372602399911, "Losses/train_all_loss_mask": 0.008000960067011056, "Losses/train_all_loss_dice": 0.19741905017761524, "Losses/train_all_loss_iou": 0.02279900476346511, "Losses/train_all_loss_class": 5.678085751427958e-09, "Losses/train_all_core_loss": 0.3802372602399911, "Trainer/where": 0.4749937374749499, "Trainer/epoch": 18, "Trainer/steps_train": 75848} +{"Losses/train_all_loss": 0.376298183201997, "Losses/train_all_loss_mask": 0.007830308981010056, "Losses/train_all_loss_dice": 0.1965342678711983, "Losses/train_all_loss_iou": 0.023157731618651492, "Losses/train_all_loss_class": 5.28514186526349e-09, "Losses/train_all_core_loss": 0.376298183201997, "Trainer/where": 0.4999937374749499, "Trainer/epoch": 19, "Trainer/steps_train": 79840} +{"Losses/train_all_loss": 0.3772340752292874, "Losses/train_all_loss_mask": 0.007914954837213077, "Losses/train_all_loss_dice": 0.19617746839064634, "Losses/train_all_loss_iou": 0.022757507024552415, "Losses/train_all_loss_class": 5.4267739055031245e-09, "Losses/train_all_core_loss": 0.3772340752292874, "Trainer/where": 0.5249937374749499, "Trainer/epoch": 20, "Trainer/steps_train": 83832} +{"Losses/train_all_loss": 0.37047547260336655, "Losses/train_all_loss_mask": 0.007639010061916952, "Losses/train_all_loss_dice": 0.1958161407786763, "Losses/train_all_loss_iou": 0.02187912743158927, "Losses/train_all_loss_class": 5.4363083103324426e-09, "Losses/train_all_core_loss": 0.37047547260336655, "Trainer/where": 0.54999373747495, "Trainer/epoch": 21, "Trainer/steps_train": 87824} +{"Losses/train_all_loss": 0.35376920676995477, "Losses/train_all_loss_mask": 0.006838303287161567, "Losses/train_all_loss_dice": 0.19547302056410032, "Losses/train_all_loss_iou": 0.021530116622054854, "Losses/train_all_loss_class": 6.322159236068461e-09, "Losses/train_all_core_loss": 0.35376920676995477, "Trainer/where": 0.5749937374749499, "Trainer/epoch": 22, "Trainer/steps_train": 91816} +{"Losses/train_all_loss": 0.3551223263956834, "Losses/train_all_loss_mask": 0.006957760564867117, "Losses/train_all_loss_dice": 0.1945757962957413, "Losses/train_all_loss_iou": 0.02139131658811324, "Losses/train_all_loss_class": 4.68340592482518e-09, "Losses/train_all_core_loss": 0.3551223263956834, "Trainer/where": 0.5999937374749499, "Trainer/epoch": 23, "Trainer/steps_train": 95808} +{"Losses/train_all_loss": 0.35436655999975974, "Losses/train_all_loss_mask": 0.006915995280616119, "Losses/train_all_loss_dice": 0.19451504708351736, "Losses/train_all_loss_iou": 0.0215316050213107, "Losses/train_all_loss_class": 4.694827900397807e-09, "Losses/train_all_core_loss": 0.35436655999975974, "Trainer/where": 0.6249937374749499, "Trainer/epoch": 24, "Trainer/steps_train": 99800} +{"Losses/train_all_loss": 0.3539243154792313, "Losses/train_all_loss_mask": 0.0069243552351725595, "Losses/train_all_loss_dice": 0.19411168701364426, "Losses/train_all_loss_iou": 0.021325522290423973, "Losses/train_all_loss_class": 4.124210543847477e-09, "Losses/train_all_core_loss": 0.3539243154792313, "Trainer/where": 0.6499937374749499, "Trainer/epoch": 25, "Trainer/steps_train": 103792} +{"Losses/train_all_loss": 0.3885851017036615, "Losses/train_all_loss_mask": 0.008587317138184215, "Losses/train_all_loss_dice": 0.1942436489498687, "Losses/train_all_loss_iou": 0.02259510975809444, "Losses/train_all_loss_class": 3.7102346460758284e-09, "Losses/train_all_core_loss": 0.3885851017036615, "Trainer/where": 0.67499373747495, "Trainer/epoch": 26, "Trainer/steps_train": 107784} +{"Losses/train_all_loss": 0.7125891149229509, "Losses/train_all_loss_mask": 0.024751994295421284, "Losses/train_all_loss_dice": 0.194532726758826, "Losses/train_all_loss_iou": 0.023016485271324848, "Losses/train_all_loss_class": 3.9748640794428046e-09, "Losses/train_all_core_loss": 0.7125891149229509, "Trainer/where": 0.6999937374749499, "Trainer/epoch": 27, "Trainer/steps_train": 111776} +{"Losses/train_all_loss": 0.8676063124507335, "Losses/train_all_loss_mask": 0.032492553048528834, "Losses/train_all_loss_dice": 0.1948043356587987, "Losses/train_all_loss_iou": 0.022950956187875175, "Losses/train_all_loss_class": 4.138141090095644e-09, "Losses/train_all_core_loss": 0.8676063124507335, "Trainer/where": 0.7249937374749499, "Trainer/epoch": 28, "Trainer/steps_train": 115768} +{"Losses/train_all_loss": 0.3487479994164496, "Losses/train_all_loss_mask": 0.0067503434569706235, "Losses/train_all_loss_dice": 0.19311891421526373, "Losses/train_all_loss_iou": 0.020622213926316305, "Losses/train_all_loss_class": 4.467678567490371e-09, "Losses/train_all_core_loss": 0.3487479994164496, "Trainer/where": 0.7499937374749499, "Trainer/epoch": 29, "Trainer/steps_train": 119760} +{"Losses/train_all_loss": 0.36419604813683787, "Losses/train_all_loss_mask": 0.00751875088206941, "Losses/train_all_loss_dice": 0.19322081335740005, "Losses/train_all_loss_iou": 0.0206002136427904, "Losses/train_all_loss_class": 5.008077296045301e-09, "Losses/train_all_core_loss": 0.36419604813683787, "Trainer/where": 0.7749937374749499, "Trainer/epoch": 30, "Trainer/steps_train": 123752} +{"Losses/train_all_loss": 0.34595307377449336, "Losses/train_all_loss_mask": 0.006689208541691566, "Losses/train_all_loss_dice": 0.19237655501267714, "Losses/train_all_loss_iou": 0.01979234506802322, "Losses/train_all_loss_class": 5.1453944205931e-09, "Losses/train_all_core_loss": 0.34595307377449336, "Trainer/where": 0.79999373747495, "Trainer/epoch": 31, "Trainer/steps_train": 127744} +{"Losses/train_all_loss": 0.34565046294508334, "Losses/train_all_loss_mask": 0.006636826759476352, "Losses/train_all_loss_dice": 0.19229123223281813, "Losses/train_all_loss_iou": 0.020622693491710736, "Losses/train_all_loss_class": 4.764466950168121e-09, "Losses/train_all_core_loss": 0.34565046294508334, "Trainer/where": 0.8249937374749499, "Trainer/epoch": 32, "Trainer/steps_train": 131736} +{"Losses/train_all_loss": 0.9008659057842683, "Losses/train_all_loss_mask": 0.03423682985725872, "Losses/train_all_loss_dice": 0.1938983456405227, "Losses/train_all_loss_iou": 0.02223101315849819, "Losses/train_all_loss_class": 4.9622400122766974e-09, "Losses/train_all_core_loss": 0.9008659057842683, "Trainer/where": 0.8499937374749498, "Trainer/epoch": 33, "Trainer/steps_train": 135728} +{"Losses/train_all_loss": 0.3465600523536368, "Losses/train_all_loss_mask": 0.0067609497240599245, "Losses/train_all_loss_dice": 0.19183460045195294, "Losses/train_all_loss_iou": 0.019506455022350135, "Losses/train_all_loss_class": 4.679632632014728e-09, "Losses/train_all_core_loss": 0.3465600523536368, "Trainer/where": 0.8749937374749498, "Trainer/epoch": 34, "Trainer/steps_train": 139720} +{"Losses/train_all_loss": 0.34736052319718447, "Losses/train_all_loss_mask": 0.0067391459940458545, "Losses/train_all_loss_dice": 0.1922520945270458, "Losses/train_all_loss_iou": 0.020325507099748605, "Losses/train_all_loss_class": 4.45559715646195e-09, "Losses/train_all_core_loss": 0.34736052319718447, "Trainer/where": 0.8999937374749498, "Trainer/epoch": 35, "Trainer/steps_train": 143712} +{"Losses/train_all_loss": 0.3672967086860374, "Losses/train_all_loss_mask": 0.00773209927745972, "Losses/train_all_loss_dice": 0.19209734350442886, "Losses/train_all_loss_iou": 0.02055737860401078, "Losses/train_all_loss_class": 4.592152288894342e-09, "Losses/train_all_core_loss": 0.3672967086860374, "Trainer/where": 0.9249937374749498, "Trainer/epoch": 36, "Trainer/steps_train": 147704} +{"Losses/train_all_loss": 0.8421824378987265, "Losses/train_all_loss_mask": 0.03133104354486921, "Losses/train_all_loss_dice": 0.19316817402003522, "Losses/train_all_loss_iou": 0.02239339469971622, "Losses/train_all_loss_class": 4.4197077939473966e-09, "Losses/train_all_core_loss": 0.8421824378987265, "Trainer/where": 0.9499937374749499, "Trainer/epoch": 37, "Trainer/steps_train": 151696} +{"Losses/train_all_loss": 0.3468215012218792, "Losses/train_all_loss_mask": 0.006743870759146193, "Losses/train_all_loss_dice": 0.19145133795742997, "Losses/train_all_loss_iou": 0.020492745865770254, "Losses/train_all_loss_class": 4.498966078856853e-09, "Losses/train_all_core_loss": 0.3468215012218792, "Trainer/where": 0.9749937374749498, "Trainer/epoch": 38, "Trainer/steps_train": 155688} +{"Losses/train_all_loss": 0.361319512792094, "Losses/train_all_loss_mask": 0.007438624991261454, "Losses/train_all_loss_dice": 0.19195612370669243, "Losses/train_all_loss_iou": 0.0205908868190231, "Losses/train_all_loss_class": 4.3955577136424175e-09, "Losses/train_all_core_loss": 0.361319512792094, "Trainer/where": 0.9999937374749498, "Trainer/epoch": 39, "Trainer/steps_train": 159680} diff --git a/experiment/thermal_run1/tensorboard/events.out.tfevents.1759750510.gpub060.delta.ncsa.illinois.edu.921762.0c94822e4-1523-4465-a797-8a6fd58b2af0 b/experiment/thermal_run1/tensorboard/events.out.tfevents.1759750510.gpub060.delta.ncsa.illinois.edu.921762.0c94822e4-1523-4465-a797-8a6fd58b2af0 new file mode 100644 index 0000000000000000000000000000000000000000..dd60ce5f9a5bc68f4a72ec9e2044c1df76ed3f3e --- /dev/null +++ b/experiment/thermal_run1/tensorboard/events.out.tfevents.1759750510.gpub060.delta.ncsa.illinois.edu.921762.0c94822e4-1523-4465-a797-8a6fd58b2af0 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:872d92f23e17cdc70e7c071350d5ac8cd50f1b9f4fb30f6477f5c6def1e2256d +size 14811175 diff --git a/sam2.1_hiera_base_plus.pt b/sam2.1_hiera_base_plus.pt new file mode 100644 index 0000000000000000000000000000000000000000..1466bad0cd2125f1be02ffdc1cf98718592636b4 --- /dev/null +++ b/sam2.1_hiera_base_plus.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:a2345aede8715ab1d5d31b4a509fb160c5a4af1970f199d9054ccfb746c004c5 +size 323606802 diff --git a/sam2.1_hiera_large.pt b/sam2.1_hiera_large.pt new file mode 100644 index 0000000000000000000000000000000000000000..a4229617ddc0bd034440718c11588c0bbcf85a70 --- /dev/null +++ b/sam2.1_hiera_large.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:2647878d5dfa5098f2f8649825738a9345572bae2d4350a2468587ece47dd318 +size 898083611 diff --git a/sam2.1_hiera_small.pt b/sam2.1_hiera_small.pt new file mode 100644 index 0000000000000000000000000000000000000000..eb980035dd665a3f45871f79ee945634915bc31e --- /dev/null +++ b/sam2.1_hiera_small.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:6d1aa6f30de5c92224f8172114de081d104bbd23dd9dc5c58996f0cad5dc4d38 +size 184416285 diff --git a/sam2.1_hiera_tiny.pt b/sam2.1_hiera_tiny.pt new file mode 100644 index 0000000000000000000000000000000000000000..56f8d31a0ff55de24d5f30d11ce0cc19a5e5d10b --- /dev/null +++ b/sam2.1_hiera_tiny.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:7402e0d864fa82708a20fbd15bc84245c2f26dff0eb43a4b5b93452deb34be69 +size 156008466